Merge branch 'main' into python-uv

CHANGELOG.md

@@ -1,5 +1,137 @@
 # Changelog
 
+## [1.423.1](https://github.com/windmill-labs/windmill/compare/v1.423.0...v1.423.1) (2024-11-12)
+
+
+### Bug Fixes
+
+* **autoscaling:** autoscaling thresholds to be >= and not > ([de00944](https://github.com/windmill-labs/windmill/commit/de00944f09699ff33a382f5a14f515ccf90b2454))
+
+## [1.423.0](https://github.com/windmill-labs/windmill/compare/v1.422.1...v1.423.0) (2024-11-12)
+
+
+### Features
+
+* s3 input available for public apps ([#4685](https://github.com/windmill-labs/windmill/issues/4685)) ([1671005](https://github.com/windmill-labs/windmill/commit/167100510032ab53cd609fb2c7629e67faceb093))
+
+
+### Bug Fixes
+
+* prevent underflow for autoscaling scalein ([5e9870a](https://github.com/windmill-labs/windmill/commit/5e9870a8a993032ec7d45c62e0023008da42c74a))
+* support multiple pip-extra-index-url with commas ([50861fc](https://github.com/windmill-labs/windmill/commit/50861fccc0cd86d4c76120c3306adf94f375330e))
+
+## [1.422.1](https://github.com/windmill-labs/windmill/compare/v1.422.0...v1.422.1) (2024-11-12)
+
+
+### Bug Fixes
+
+* fix password inputs ([e5e174a](https://github.com/windmill-labs/windmill/commit/e5e174ae9516f4c6b94ceb6e258b467f5c9a1f1a))
+
+## [1.422.0](https://github.com/windmill-labs/windmill/compare/v1.421.2...v1.422.0) (2024-11-11)
+
+
+### Features
+
+* expandable subflows in flows ([#4683](https://github.com/windmill-labs/windmill/issues/4683)) ([d44976f](https://github.com/windmill-labs/windmill/commit/d44976f35e45ade510d1ec220b5a1503e11f3db9))
+* **frontend:** critical alerts UI ([#4653](https://github.com/windmill-labs/windmill/issues/4653)) ([d9148ea](https://github.com/windmill-labs/windmill/commit/d9148eaa78680a93d81d71847a7df67e01f3c110))
+
+## [1.421.2](https://github.com/windmill-labs/windmill/compare/v1.421.1...v1.421.2) (2024-11-08)
+
+
+### Bug Fixes
+
+* **bash:** correctly propagate exit errors ([8bc9a02](https://github.com/windmill-labs/windmill/commit/8bc9a021a88391147c9170f56b5a0edddd55bc7d))
+
+## [1.421.1](https://github.com/windmill-labs/windmill/compare/v1.421.0...v1.421.1) (2024-11-08)
+
+
+### Bug Fixes
+
+* **python-client:** fix small break params of write_s3_file ([3ea12f1](https://github.com/windmill-labs/windmill/commit/3ea12f1821500e8b84549b892e3e1bb56a6ace4b))
+
+## [1.421.0](https://github.com/windmill-labs/windmill/compare/v1.420.1...v1.421.0) (2024-11-07)
+
+
+### Features
+
+* http custom routes for static assets ([#4666](https://github.com/windmill-labs/windmill/issues/4666)) ([dc8bd6d](https://github.com/windmill-labs/windmill/commit/dc8bd6d2b5d6f5cd8521f9034853175ec78d5639))
+
+
+### Bug Fixes
+
+* improve nested schema editor field change ([a3feca7](https://github.com/windmill-labs/windmill/commit/a3feca719799ea2bf08f2b49350e6b732a24abf4))
+
+## [1.420.1](https://github.com/windmill-labs/windmill/compare/v1.420.0...v1.420.1) (2024-11-07)
+
+
+### Bug Fixes
+
+* improve prop filtering on flow prop picker ([0501ad8](https://github.com/windmill-labs/windmill/commit/0501ad8a99f6f7d9fd26c996b754e8afe8f958b1))
+
+## [1.420.0](https://github.com/windmill-labs/windmill/compare/v1.419.0...v1.420.0) (2024-11-07)
+
+
+### Features
+
+* **frontend:** detect expr in flow input transform + filter right panel based on expr ([#4651](https://github.com/windmill-labs/windmill/issues/4651)) ([e9b7dca](https://github.com/windmill-labs/windmill/commit/e9b7dca20387e775fa50aaecd832890251582cf9))
+* **frontend:** nodes from flow can be connected directly in expr input through a plug icon ([#4652](https://github.com/windmill-labs/windmill/issues/4652)) ([ceaf56c](https://github.com/windmill-labs/windmill/commit/ceaf56c21ee2e548bc93859f9e0303e53b25b241))
+
+## [1.419.0](https://github.com/windmill-labs/windmill/compare/v1.418.0...v1.419.0) (2024-11-06)
+
+
+### Features
+
+* Add full-text search on windmill service logs ([#4576](https://github.com/windmill-labs/windmill/issues/4576)) ([77735d8](https://github.com/windmill-labs/windmill/commit/77735d859cfee9204f67a8e4f9885228d657a41d))
+* show path in flow script picker ([#4574](https://github.com/windmill-labs/windmill/issues/4574)) ([8e27392](https://github.com/windmill-labs/windmill/commit/8e27392afacbb725aaf9f9f892ab8a6171b59ce5))
+* websocket authentication ([#4635](https://github.com/windmill-labs/windmill/issues/4635)) ([4dda0fb](https://github.com/windmill-labs/windmill/commit/4dda0fb8cd8262ad3a2ab2b9d27e7043ac3bb891))
+
+
+### Bug Fixes
+
+* clarify error messages when job timeout or cancelled with more details ([771d740](https://github.com/windmill-labs/windmill/commit/771d740701902166f8b4e3f77aa9c5579237cb15))
+* **cli:** improve handling of deleted items on windows ([9a8dcc9](https://github.com/windmill-labs/windmill/commit/9a8dcc9a250caefa0b7c9523e1321599b7471c8b))
+* display logs in native mode when script fails ([#4655](https://github.com/windmill-labs/windmill/issues/4655)) ([7578ceb](https://github.com/windmill-labs/windmill/commit/7578cebaf92e729a26fd665e4b6f8357d34f59eb))
+* **frontend:** arg input json handling when the value is not of the same type as schema ([#4479](https://github.com/windmill-labs/windmill/issues/4479)) ([8d8156b](https://github.com/windmill-labs/windmill/commit/8d8156bd0773da3ddec81c46ad5fda114ecd3dda))
+* **frontend:** improve flow prop picker design ([323912c](https://github.com/windmill-labs/windmill/commit/323912c73c18d3bb6d136f2e6458389270364a0e))
+
+## [1.418.0](https://github.com/windmill-labs/windmill/compare/v1.417.3...v1.418.0) (2024-11-04)
+
+
+### Features
+
+* **frontend:** improve and simplify scheduled poll flows ([#4560](https://github.com/windmill-labs/windmill/issues/4560)) ([22ab51e](https://github.com/windmill-labs/windmill/commit/22ab51e9914376b47827f0be66708deef3a37767))
+* **oauth:** add snowflake oauth support ([#4622](https://github.com/windmill-labs/windmill/issues/4622)) ([693b7a4](https://github.com/windmill-labs/windmill/commit/693b7a4fd4da0e2b14cb6c8ea19537aa600225be))
+
+
+### Bug Fixes
+
+* **prometheus:** improve queue_count when tags have no more jobs ([09156b6](https://github.com/windmill-labs/windmill/commit/09156b65c2a0f0ac59fce1dede2b68b790323c49))
+
+## [1.417.3](https://github.com/windmill-labs/windmill/compare/v1.417.2...v1.417.3) (2024-11-04)
+
+
+### Bug Fixes
+
+* **cli:** improve pulling instance with folderPerInstance ([1e7909e](https://github.com/windmill-labs/windmill/commit/1e7909e95931e1e121cb3767bad093239254935d))
+
+## [1.417.2](https://github.com/windmill-labs/windmill/compare/v1.417.1...v1.417.2) (2024-11-04)
+
+
+### Bug Fixes
+
+* **cli:** improve pulling instance with folderPerInstance ([00748f4](https://github.com/windmill-labs/windmill/commit/00748f43feab4f28f0dd2682e3dc8ccc15a5f49f))
+
+## [1.417.1](https://github.com/windmill-labs/windmill/compare/v1.417.0...v1.417.1) (2024-11-04)
+
+
+### Bug Fixes
+
+* missing opts for pull and push instance configs ([#4630](https://github.com/windmill-labs/windmill/issues/4630)) ([e92a338](https://github.com/windmill-labs/windmill/commit/e92a338432d2732e6be74c3604b9fa95239e1ca3))
+
+## 1.417.0 (2024-11-03)
+
+*  feat(cli): instance yaml files inside the instance prefix folders (#4627)
+
 ## [1.416.2](https://github.com/windmill-labs/windmill/compare/v1.416.1...v1.416.2) (2024-11-02)
 
 

Dockerfile

@@ -175,9 +175,9 @@ RUN /usr/local/bin/python3 -m pip install pip-tools
 COPY --from=builder /frontend/build /static_frontend
 COPY --from=builder /windmill/target/release/windmill ${APP}/windmill
 
-COPY --from=denoland/deno:2.0.2 --chmod=755 /usr/bin/deno /usr/bin/deno
+COPY --from=denoland/deno:2.0.4 --chmod=755 /usr/bin/deno /usr/bin/deno
 
-COPY --from=oven/bun:1.1.32 /usr/local/bin/bun /usr/bin/bun
+COPY --from=oven/bun:1.1.34 /usr/local/bin/bun /usr/bin/bun
 
 COPY --from=php:8.3.7-cli /usr/local/bin/php /usr/bin/php
 COPY --from=composer:2.7.6 /usr/bin/composer /usr/bin/composer

README.md

@@ -355,6 +355,8 @@ you to have it being synced automatically everyday.
 See the [./frontend/README_DEV.md](./frontend/README_DEV.md) file for all
 running options.
 
+Using [Nix](./frontend/README_DEV.md#nix).
+
 ### only Frontend
 This will use the backend of <https://app.windmill.dev> but your own frontend
 with hot-code reloading. Note that you will need to use a username / password login due to CSRF checks using a different auth provider.

backend/.sqlx/query-00ce4ed3ca0eac7cb6283b047353a64b9e78c4beb423f04baef9a53fbf87e9f9.json

@@ -1,14 +0,0 @@
-{
-  "db_name": "PostgreSQL",
-  "query": "INSERT INTO alerts (alert_type, message) VALUES ('recovered_critical_error', $1)",
-  "describe": {
-    "columns": [],
-    "parameters": {
-      "Left": [
-        "Text"
-      ]
-    },
-    "nullable": []
-  },
-  "hash": "00ce4ed3ca0eac7cb6283b047353a64b9e78c4beb423f04baef9a53fbf87e9f9"
-}

backend/.sqlx/query-0b94bd4c98a11ca1b7e5e34dd1ee6fcb0b7a54ed4218fa3cf23cc929d009d50f.json

@@ -1,6 +1,6 @@
 {
   "db_name": "PostgreSQL",
-  "query": "UPDATE websocket_trigger SET url = $1, script_path = $2, path = $3, is_flow = $4, filters = $5, edited_by = $6, email = $7, edited_at = now(), server_id = NULL, last_server_ping = NULL, error = NULL\n            WHERE workspace_id = $8 AND path = $9",
+  "query": "UPDATE websocket_trigger SET url = $1, script_path = $2, path = $3, is_flow = $4, filters = $5, initial_messages = $6, url_runnable_args = $7, edited_by = $8, email = $9, edited_at = now(), server_id = NULL, last_server_ping = NULL, error = NULL\n            WHERE workspace_id = $10 AND path = $11",
   "describe": {
     "columns": [],
     "parameters": {
@@ -10,6 +10,8 @@
         "Varchar",
         "Bool",
         "JsonbArray",
+        "JsonbArray",
+        "Jsonb",
         "Varchar",
         "Varchar",
         "Text",
@@ -18,5 +20,5 @@
     },
     "nullable": []
   },
-  "hash": "acbf74cf3302bfcf7615285070d3f8958932bb8a2dda715f1b9152ab44442780"
+  "hash": "0b94bd4c98a11ca1b7e5e34dd1ee6fcb0b7a54ed4218fa3cf23cc929d009d50f"
 }

backend/.sqlx/query-0b955f2cff82a2d4ba3840588143e08952f029480d4a42503ecc3c5e70437995.json

@@ -0,0 +1,48 @@
+{
+  "db_name": "PostgreSQL",
+  "query": "SELECT id, alert_type, message, created_at, acknowledged \n             FROM alerts \n             WHERE acknowledged = $1\n             ORDER BY created_at DESC \n             LIMIT $2 OFFSET $3",
+  "describe": {
+    "columns": [
+      {
+        "ordinal": 0,
+        "name": "id",
+        "type_info": "Int4"
+      },
+      {
+        "ordinal": 1,
+        "name": "alert_type",
+        "type_info": "Varchar"
+      },
+      {
+        "ordinal": 2,
+        "name": "message",
+        "type_info": "Text"
+      },
+      {
+        "ordinal": 3,
+        "name": "created_at",
+        "type_info": "Timestamptz"
+      },
+      {
+        "ordinal": 4,
+        "name": "acknowledged",
+        "type_info": "Bool"
+      }
+    ],
+    "parameters": {
+      "Left": [
+        "Bool",
+        "Int8",
+        "Int8"
+      ]
+    },
+    "nullable": [
+      false,
+      false,
+      false,
+      false,
+      true
+    ]
+  },
+  "hash": "0b955f2cff82a2d4ba3840588143e08952f029480d4a42503ecc3c5e70437995"
+}

backend/.sqlx/query-11b698f82a54aac68b3617047dfe2b18dd6da7d962118fee276af354218baac2.json

@@ -1,6 +1,6 @@
 {
   "db_name": "PostgreSQL",
-  "query": "SELECT path, script_path, is_flow, route_path, workspace_id, is_async, requires_auth, edited_by, email, http_method as \"http_method: _\" FROM http_trigger",
+  "query": "SELECT path, script_path, is_flow, route_path, workspace_id, is_async, requires_auth, edited_by, email, http_method as \"http_method: _\", static_asset_config as \"static_asset_config: _\" FROM http_trigger",
   "describe": {
     "columns": [
       {
@@ -65,6 +65,11 @@
             }
           }
         }
+      },
+      {
+        "ordinal": 10,
+        "name": "static_asset_config: _",
+        "type_info": "Jsonb"
       }
     ],
     "parameters": {
@@ -80,8 +85,9 @@
       false,
       false,
       false,
-      false
+      false,
+      true
     ]
   },
-  "hash": "fe4f91ca7e179e58041a5c0b1a36015175e9301c01a99e8f2b82cb853349a183"
+  "hash": "11b698f82a54aac68b3617047dfe2b18dd6da7d962118fee276af354218baac2"
 }

backend/.sqlx/query-333b484ffa030dee08e7b1161fcbc48af411377d2d9f58f92fc9d5eacdf0fba1.json

@@ -1,6 +1,6 @@
 {
   "db_name": "PostgreSQL",
-  "query": "INSERT INTO http_trigger (workspace_id, path, route_path, route_path_key, script_path, is_flow, is_async, requires_auth, http_method, edited_by, email, edited_at) VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, now())",
+  "query": "INSERT INTO http_trigger (workspace_id, path, route_path, route_path_key, script_path, is_flow, is_async, requires_auth, http_method, static_asset_config, edited_by, email, edited_at) VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, $12, now())",
   "describe": {
     "columns": [],
     "parameters": {
@@ -27,11 +27,12 @@
             }
           }
         },
+        "Jsonb",
         "Varchar",
         "Varchar"
       ]
     },
     "nullable": []
   },
-  "hash": "c229744534f17f7622c3dee21bb1e7292ff17e6dffe58e80e53bff8baade07c8"
+  "hash": "333b484ffa030dee08e7b1161fcbc48af411377d2d9f58f92fc9d5eacdf0fba1"
 }

backend/.sqlx/query-33cfb00f61dbae09467bd3732ce8e83c9835ccd30c90cb24788cdb54d1ceb575.json

@@ -0,0 +1,20 @@
+{
+  "db_name": "PostgreSQL",
+  "query": "SELECT value::BOOLEAN as value FROM global_settings WHERE name = 'critical_alert_mute_ui'",
+  "describe": {
+    "columns": [
+      {
+        "ordinal": 0,
+        "name": "value",
+        "type_info": "Bool"
+      }
+    ],
+    "parameters": {
+      "Left": []
+    },
+    "nullable": [
+      null
+    ]
+  },
+  "hash": "33cfb00f61dbae09467bd3732ce8e83c9835ccd30c90cb24788cdb54d1ceb575"
+}

backend/.sqlx/query-3a94ad52c6b7cde844fa868167248cd9ff63e5fdfa1d93d8fbec32a257b6b05e.json

@@ -0,0 +1,15 @@
+{
+  "db_name": "PostgreSQL",
+  "query": "INSERT INTO alerts (alert_type, message, acknowledged) VALUES ('recovered_critical_error', $1, $2)",
+  "describe": {
+    "columns": [],
+    "parameters": {
+      "Left": [
+        "Text",
+        "Bool"
+      ]
+    },
+    "nullable": []
+  },
+  "hash": "3a94ad52c6b7cde844fa868167248cd9ff63e5fdfa1d93d8fbec32a257b6b05e"
+}

backend/.sqlx/query-487d377e2df67fc3ea39d183ba9f99d45828d7c8e0ff10c5d74c454472e0493c.json

@@ -1,6 +1,6 @@
 {
   "db_name": "PostgreSQL",
-  "query": "UPDATE http_trigger \n                SET route_path = $1, route_path_key = $2, script_path = $3, path = $4, is_flow = $5, http_method = $6, edited_by = $7, email = $8, is_async = $9, requires_auth = $10, edited_at = now() \n                WHERE workspace_id = $11 AND path = $12",
+  "query": "UPDATE http_trigger \n                SET route_path = $1, route_path_key = $2, script_path = $3, path = $4, is_flow = $5, http_method = $6, static_asset_config = $7, edited_by = $8, email = $9, is_async = $10, requires_auth = $11, edited_at = now() \n                WHERE workspace_id = $12 AND path = $13",
   "describe": {
     "columns": [],
     "parameters": {
@@ -24,6 +24,7 @@
             }
           }
         },
+        "Jsonb",
         "Varchar",
         "Varchar",
         "Bool",
@@ -34,5 +35,5 @@
     },
     "nullable": []
   },
-  "hash": "4d8640e84fccf1a0b799d8396a51e69345137e68d5096c70ba0a4332075d97ea"
+  "hash": "487d377e2df67fc3ea39d183ba9f99d45828d7c8e0ff10c5d74c454472e0493c"
 }

backend/.sqlx/query-4c0067c2135a259aea5cc2db60f7375a9a33671be8ef406427d90f67a98c9c9f.json

@@ -1,14 +0,0 @@
-{
-  "db_name": "PostgreSQL",
-  "query": "INSERT INTO alerts (alert_type, message) VALUES ('critical_error', $1)",
-  "describe": {
-    "columns": [],
-    "parameters": {
-      "Left": [
-        "Text"
-      ]
-    },
-    "nullable": []
-  },
-  "hash": "4c0067c2135a259aea5cc2db60f7375a9a33671be8ef406427d90f67a98c9c9f"
-}

backend/.sqlx/query-4d22084a5d9860832f30e8f08cbfa1848ed3c1336fa4790f45b8189c4ac97d91.json

@@ -0,0 +1,15 @@
+{
+  "db_name": "PostgreSQL",
+  "query": "INSERT INTO alerts (alert_type, message, acknowledged) VALUES ('critical_error', $1, $2)",
+  "describe": {
+    "columns": [],
+    "parameters": {
+      "Left": [
+        "Text",
+        "Bool"
+      ]
+    },
+    "nullable": []
+  },
+  "hash": "4d22084a5d9860832f30e8f08cbfa1848ed3c1336fa4790f45b8189c4ac97d91"
+}

backend/.sqlx/query-4e9668a46bad9e82baa51422946d373b18b6577198df7545c94bd19be3446775.json

@@ -1,108 +0,0 @@
-{
-  "db_name": "PostgreSQL",
-  "query": "INSERT INTO websocket_trigger (workspace_id, path, url, script_path, is_flow, enabled, filters, edited_by, email, edited_at) VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, now()) RETURNING *",
-  "describe": {
-    "columns": [
-      {
-        "ordinal": 0,
-        "name": "path",
-        "type_info": "Varchar"
-      },
-      {
-        "ordinal": 1,
-        "name": "url",
-        "type_info": "Varchar"
-      },
-      {
-        "ordinal": 2,
-        "name": "script_path",
-        "type_info": "Varchar"
-      },
-      {
-        "ordinal": 3,
-        "name": "is_flow",
-        "type_info": "Bool"
-      },
-      {
-        "ordinal": 4,
-        "name": "workspace_id",
-        "type_info": "Varchar"
-      },
-      {
-        "ordinal": 5,
-        "name": "edited_by",
-        "type_info": "Varchar"
-      },
-      {
-        "ordinal": 6,
-        "name": "email",
-        "type_info": "Varchar"
-      },
-      {
-        "ordinal": 7,
-        "name": "edited_at",
-        "type_info": "Timestamptz"
-      },
-      {
-        "ordinal": 8,
-        "name": "extra_perms",
-        "type_info": "Jsonb"
-      },
-      {
-        "ordinal": 9,
-        "name": "server_id",
-        "type_info": "Varchar"
-      },
-      {
-        "ordinal": 10,
-        "name": "last_server_ping",
-        "type_info": "Timestamptz"
-      },
-      {
-        "ordinal": 11,
-        "name": "error",
-        "type_info": "Text"
-      },
-      {
-        "ordinal": 12,
-        "name": "enabled",
-        "type_info": "Bool"
-      },
-      {
-        "ordinal": 13,
-        "name": "filters",
-        "type_info": "JsonbArray"
-      }
-    ],
-    "parameters": {
-      "Left": [
-        "Varchar",
-        "Varchar",
-        "Varchar",
-        "Varchar",
-        "Bool",
-        "Bool",
-        "JsonbArray",
-        "Varchar",
-        "Varchar"
-      ]
-    },
-    "nullable": [
-      false,
-      false,
-      false,
-      false,
-      false,
-      false,
-      false,
-      false,
-      false,
-      true,
-      true,
-      true,
-      false,
-      false
-    ]
-  },
-  "hash": "4e9668a46bad9e82baa51422946d373b18b6577198df7545c94bd19be3446775"
-}

backend/.sqlx/query-5303cb9dd5903aa4791ef8e5e5881a50a832e65c8c9632e2e12cd9c2747f2fc7.json

@@ -1,98 +0,0 @@
-{
-  "db_name": "PostgreSQL",
-  "query": "SELECT *\n            FROM websocket_trigger\n            WHERE enabled IS TRUE AND (server_id IS NULL OR last_server_ping IS NULL OR last_server_ping < now() - interval '15 seconds')",
-  "describe": {
-    "columns": [
-      {
-        "ordinal": 0,
-        "name": "path",
-        "type_info": "Varchar"
-      },
-      {
-        "ordinal": 1,
-        "name": "url",
-        "type_info": "Varchar"
-      },
-      {
-        "ordinal": 2,
-        "name": "script_path",
-        "type_info": "Varchar"
-      },
-      {
-        "ordinal": 3,
-        "name": "is_flow",
-        "type_info": "Bool"
-      },
-      {
-        "ordinal": 4,
-        "name": "workspace_id",
-        "type_info": "Varchar"
-      },
-      {
-        "ordinal": 5,
-        "name": "edited_by",
-        "type_info": "Varchar"
-      },
-      {
-        "ordinal": 6,
-        "name": "email",
-        "type_info": "Varchar"
-      },
-      {
-        "ordinal": 7,
-        "name": "edited_at",
-        "type_info": "Timestamptz"
-      },
-      {
-        "ordinal": 8,
-        "name": "extra_perms",
-        "type_info": "Jsonb"
-      },
-      {
-        "ordinal": 9,
-        "name": "server_id",
-        "type_info": "Varchar"
-      },
-      {
-        "ordinal": 10,
-        "name": "last_server_ping",
-        "type_info": "Timestamptz"
-      },
-      {
-        "ordinal": 11,
-        "name": "error",
-        "type_info": "Text"
-      },
-      {
-        "ordinal": 12,
-        "name": "enabled",
-        "type_info": "Bool"
-      },
-      {
-        "ordinal": 13,
-        "name": "filters",
-        "type_info": "JsonbArray"
-      }
-    ],
-    "parameters": {
-      "Left": []
-    },
-    "nullable": [
-      false,
-      false,
-      false,
-      false,
-      false,
-      false,
-      false,
-      false,
-      false,
-      true,
-      true,
-      true,
-      false,
-      false
-    ]
-  },
-  "hash": "5303cb9dd5903aa4791ef8e5e5881a50a832e65c8c9632e2e12cd9c2747f2fc7"
-}

backend/.sqlx/query-7113d7cc72e44e4b7e01b69cc18cbe7b0399cf8ec0e9e6d2b05ceef589c432df.json

@@ -1,6 +1,6 @@
 {
   "db_name": "PostgreSQL",
-  "query": "UPDATE http_trigger SET script_path = $1, path = $2, is_flow = $3, http_method = $4, edited_by = $5, email = $6, is_async = $7, requires_auth = $8, edited_at = now() \n                WHERE workspace_id = $9 AND path = $10",
+  "query": "UPDATE http_trigger SET script_path = $1, path = $2, is_flow = $3, http_method = $4, static_asset_config = $5, edited_by = $6, email = $7, is_async = $8, requires_auth = $9, edited_at = now() \n                WHERE workspace_id = $10 AND path = $11",
   "describe": {
     "columns": [],
     "parameters": {
@@ -22,6 +22,7 @@
             }
           }
         },
+        "Jsonb",
         "Varchar",
         "Varchar",
         "Bool",
@@ -32,5 +33,5 @@
     },
     "nullable": []
   },
-  "hash": "55d44f569f8ebfccddf975e1a330ef0dc286f4138efe923832371cdbac7157b0"
+  "hash": "7113d7cc72e44e4b7e01b69cc18cbe7b0399cf8ec0e9e6d2b05ceef589c432df"
 }

backend/.sqlx/query-a69ba7471d1a5faa145bebbd17d43e6dbe02e9e92ebbc31a50c99c3a04719284.json

@@ -0,0 +1,87 @@
+{
+  "db_name": "PostgreSQL",
+  "query": "WITH uuid_table as (\n            select gen_random_uuid() as uuid from generate_series(1, $11)\n        )\n        INSERT INTO queue \n            (id, script_hash, script_path, job_kind, language, args, tag, created_by, permissioned_as, email, scheduled_for, workspace_id, concurrent_limit, concurrency_time_window_s, timeout, raw_code, raw_lock, raw_flow, flow_status)\n            (SELECT uuid, $1, $2, $3, $4, ('{ \"uuid\": \"' || uuid || '\" }')::jsonb, $5, $6, $7, $8, $9, $10, $12, $13, $14, $15, $16, $17, $18 FROM uuid_table) \n        RETURNING id",
+  "describe": {
+    "columns": [
+      {
+        "ordinal": 0,
+        "name": "id",
+        "type_info": "Uuid"
+      }
+    ],
+    "parameters": {
+      "Left": [
+        "Int8",
+        "Varchar",
+        {
+          "Custom": {
+            "name": "job_kind",
+            "kind": {
+              "Enum": [
+                "script",
+                "preview",
+                "flow",
+                "dependencies",
+                "flowpreview",
+                "script_hub",
+                "identity",
+                "flowdependencies",
+                "http",
+                "graphql",
+                "postgresql",
+                "noop",
+                "appdependencies",
+                "deploymentcallback",
+                "singlescriptflow"
+              ]
+            }
+          }
+        },
+        {
+          "Custom": {
+            "name": "script_lang",
+            "kind": {
+              "Enum": [
+                "python3",
+                "deno",
+                "go",
+                "bash",
+                "postgresql",
+                "nativets",
+                "bun",
+                "mysql",
+                "bigquery",
+                "snowflake",
+                "graphql",
+                "powershell",
+                "mssql",
+                "php",
+                "bunnative",
+                "rust",
+                "ansible"
+              ]
+            }
+          }
+        },
+        "Varchar",
+        "Varchar",
+        "Varchar",
+        "Varchar",
+        "Timestamptz",
+        "Varchar",
+        "Int4",
+        "Int4",
+        "Int4",
+        "Int4",
+        "Text",
+        "Text",
+        "Jsonb",
+        "Jsonb"
+      ]
+    },
+    "nullable": [
+      false
+    ]
+  },
+  "hash": "a69ba7471d1a5faa145bebbd17d43e6dbe02e9e92ebbc31a50c99c3a04719284"
+}

backend/.sqlx/query-a7c5008aa7ea43d0afac7d9f19846976ed7af2e90270902f001115e023cb947d.json

@@ -0,0 +1,12 @@
+{
+  "db_name": "PostgreSQL",
+  "query": "UPDATE alerts SET acknowledged = true WHERE acknowledged = false",
+  "describe": {
+    "columns": [],
+    "parameters": {
+      "Left": []
+    },
+    "nullable": []
+  },
+  "hash": "a7c5008aa7ea43d0afac7d9f19846976ed7af2e90270902f001115e023cb947d"
+}

backend/.sqlx/query-ae543dfa106fa6ad4e9bf45cda1110d4702a80f455c63af6fcbd7cf45bbc4f7a.json

@@ -0,0 +1,23 @@
+{
+  "db_name": "PostgreSQL",
+  "query": "SELECT flow_version.value AS \"value: sqlx::types::Json<Box<RawValue>>\" FROM flow \n                    LEFT JOIN flow_version\n                        ON flow_version.id = flow.versions[array_upper(flow.versions, 1)]\n                    WHERE flow.path = $1 AND flow.workspace_id = $2",
+  "describe": {
+    "columns": [
+      {
+        "ordinal": 0,
+        "name": "value: sqlx::types::Json<Box<RawValue>>",
+        "type_info": "Jsonb"
+      }
+    ],
+    "parameters": {
+      "Left": [
+        "Text",
+        "Text"
+      ]
+    },
+    "nullable": [
+      false
+    ]
+  },
+  "hash": "ae543dfa106fa6ad4e9bf45cda1110d4702a80f455c63af6fcbd7cf45bbc4f7a"
+}

backend/.sqlx/query-b5c839baab25c4dcdd503d380cf7a886242277cd50555f20b2e22e13942d2a3a.json

@@ -0,0 +1,65 @@
+{
+  "db_name": "PostgreSQL",
+  "query": "SELECT\n                        hostname,\n                        mode::text,\n                        worker_group,\n                        log_ts,\n                        file_path,\n                        ok_lines,\n                        err_lines,\n                        json_fmt\n                    FROM log_file\n                    WHERE log_ts > $1\n                    ORDER BY log_ts ASC LIMIT $2",
+  "describe": {
+    "columns": [
+      {
+        "ordinal": 0,
+        "name": "hostname",
+        "type_info": "Varchar"
+      },
+      {
+        "ordinal": 1,
+        "name": "mode",
+        "type_info": "Text"
+      },
+      {
+        "ordinal": 2,
+        "name": "worker_group",
+        "type_info": "Varchar"
+      },
+      {
+        "ordinal": 3,
+        "name": "log_ts",
+        "type_info": "Timestamp"
+      },
+      {
+        "ordinal": 4,
+        "name": "file_path",
+        "type_info": "Varchar"
+      },
+      {
+        "ordinal": 5,
+        "name": "ok_lines",
+        "type_info": "Int8"
+      },
+      {
+        "ordinal": 6,
+        "name": "err_lines",
+        "type_info": "Int8"
+      },
+      {
+        "ordinal": 7,
+        "name": "json_fmt",
+        "type_info": "Bool"
+      }
+    ],
+    "parameters": {
+      "Left": [
+        "Timestamp",
+        "Int8"
+      ]
+    },
+    "nullable": [
+      false,
+      null,
+      true,
+      false,
+      false,
+      true,
+      true,
+      true
+    ]
+  },
+  "hash": "b5c839baab25c4dcdd503d380cf7a886242277cd50555f20b2e22e13942d2a3a"
+}

backend/.sqlx/query-be3ae231557e794172336bc27d725f862dcf039bbb3d75ced9d54c86f53d2580.json

@@ -0,0 +1,14 @@
+{
+  "db_name": "PostgreSQL",
+  "query": "UPDATE alerts SET acknowledged = true WHERE id = $1",
+  "describe": {
+    "columns": [],
+    "parameters": {
+      "Left": [
+        "Int4"
+      ]
+    },
+    "nullable": []
+  },
+  "hash": "be3ae231557e794172336bc27d725f862dcf039bbb3d75ced9d54c86f53d2580"
+}

backend/.sqlx/query-c9930fcfe79541af570eace58ba7e15a0816a6b4fd036cf7b991a210654b2633.json

@@ -1,6 +1,6 @@
 {
   "db_name": "PostgreSQL",
-  "query": "SELECT path, script_path, is_flow, route_path, workspace_id, is_async, requires_auth, edited_by, email, http_method as \"http_method: _\" FROM http_trigger WHERE workspace_id = $1",
+  "query": "SELECT path, script_path, is_flow, route_path, workspace_id, is_async, requires_auth, edited_by, email, http_method as \"http_method: _\", static_asset_config as \"static_asset_config: _\" FROM http_trigger WHERE workspace_id = $1",
   "describe": {
     "columns": [
       {
@@ -65,6 +65,11 @@
             }
           }
         }
+      },
+      {
+        "ordinal": 10,
+        "name": "static_asset_config: _",
+        "type_info": "Jsonb"
       }
     ],
     "parameters": {
@@ -82,8 +87,9 @@
       false,
       false,
       false,
-      false
+      false,
+      true
     ]
   },
-  "hash": "02f1a6eeb27067dc438459238e7b016f5ccf9e3fe0ffbe88471f15aad8f74441"
+  "hash": "c9930fcfe79541af570eace58ba7e15a0816a6b4fd036cf7b991a210654b2633"
 }

backend/.sqlx/query-cc5ab80241b88c5befea279f16c4ec68cec17b31dcd277b321f652917346496b.json

@@ -0,0 +1,47 @@
+{
+  "db_name": "PostgreSQL",
+  "query": "SELECT id, alert_type, message, created_at, acknowledged \n             FROM alerts \n             ORDER BY created_at DESC \n             LIMIT $1 OFFSET $2",
+  "describe": {
+    "columns": [
+      {
+        "ordinal": 0,
+        "name": "id",
+        "type_info": "Int4"
+      },
+      {
+        "ordinal": 1,
+        "name": "alert_type",
+        "type_info": "Varchar"
+      },
+      {
+        "ordinal": 2,
+        "name": "message",
+        "type_info": "Text"
+      },
+      {
+        "ordinal": 3,
+        "name": "created_at",
+        "type_info": "Timestamptz"
+      },
+      {
+        "ordinal": 4,
+        "name": "acknowledged",
+        "type_info": "Bool"
+      }
+    ],
+    "parameters": {
+      "Left": [
+        "Int8",
+        "Int8"
+      ]
+    },
+    "nullable": [
+      false,
+      false,
+      false,
+      false,
+      true
+    ]
+  },
+  "hash": "cc5ab80241b88c5befea279f16c4ec68cec17b31dcd277b321f652917346496b"
+}

backend/.sqlx/query-cd33a9d63f4706a7e3b1e23cd0a4b2e3ecb30aae6510d1fcd08493b07c8b0952.json

@@ -0,0 +1,16 @@
+{
+  "db_name": "PostgreSQL",
+  "query": "UPDATE websocket_trigger SET enabled = FALSE, error = $1, server_id = NULL, last_server_ping = NULL WHERE workspace_id = $2 AND path = $3",
+  "describe": {
+    "columns": [],
+    "parameters": {
+      "Left": [
+        "Text",
+        "Text",
+        "Text"
+      ]
+    },
+    "nullable": []
+  },
+  "hash": "cd33a9d63f4706a7e3b1e23cd0a4b2e3ecb30aae6510d1fcd08493b07c8b0952"
+}

backend/.sqlx/query-dd967c5983fa0ff05e2b320ad0e0b5a152784826cb8fb4381c1ffe228cb7feb6.json

@@ -0,0 +1,64 @@
+{
+  "db_name": "PostgreSQL",
+  "query": "SELECT\n                        hostname,\n                        mode::text,\n                        worker_group,\n                        log_ts,\n                        file_path,\n                        ok_lines,\n                        err_lines,\n                        json_fmt\n                    FROM log_file\n                    ORDER BY log_ts ASC LIMIT $1",
+  "describe": {
+    "columns": [
+      {
+        "ordinal": 0,
+        "name": "hostname",
+        "type_info": "Varchar"
+      },
+      {
+        "ordinal": 1,
+        "name": "mode",
+        "type_info": "Text"
+      },
+      {
+        "ordinal": 2,
+        "name": "worker_group",
+        "type_info": "Varchar"
+      },
+      {
+        "ordinal": 3,
+        "name": "log_ts",
+        "type_info": "Timestamp"
+      },
+      {
+        "ordinal": 4,
+        "name": "file_path",
+        "type_info": "Varchar"
+      },
+      {
+        "ordinal": 5,
+        "name": "ok_lines",
+        "type_info": "Int8"
+      },
+      {
+        "ordinal": 6,
+        "name": "err_lines",
+        "type_info": "Int8"
+      },
+      {
+        "ordinal": 7,
+        "name": "json_fmt",
+        "type_info": "Bool"
+      }
+    ],
+    "parameters": {
+      "Left": [
+        "Int8"
+      ]
+    },
+    "nullable": [
+      false,
+      null,
+      true,
+      false,
+      false,
+      true,
+      true,
+      true
+    ]
+  },
+  "hash": "dd967c5983fa0ff05e2b320ad0e0b5a152784826cb8fb4381c1ffe228cb7feb6"
+}

backend/.sqlx/query-f2baee15e6d1fecd6d2d7b39fda1b50a15ec8bd349f1081af54da5dc2f5e3021.json

@@ -1,101 +0,0 @@
-{
-  "db_name": "PostgreSQL",
-  "query": "SELECT *\n          FROM websocket_trigger\n          WHERE workspace_id = $1 AND path = $2",
-  "describe": {
-    "columns": [
-      {
-        "ordinal": 0,
-        "name": "path",
-        "type_info": "Varchar"
-      },
-      {
-        "ordinal": 1,
-        "name": "url",
-        "type_info": "Varchar"
-      },
-      {
-        "ordinal": 2,
-        "name": "script_path",
-        "type_info": "Varchar"
-      },
-      {
-        "ordinal": 3,
-        "name": "is_flow",
-        "type_info": "Bool"
-      },
-      {
-        "ordinal": 4,
-        "name": "workspace_id",
-        "type_info": "Varchar"
-      },
-      {
-        "ordinal": 5,
-        "name": "edited_by",
-        "type_info": "Varchar"
-      },
-      {
-        "ordinal": 6,
-        "name": "email",
-        "type_info": "Varchar"
-      },
-      {
-        "ordinal": 7,
-        "name": "edited_at",
-        "type_info": "Timestamptz"
-      },
-      {
-        "ordinal": 8,
-        "name": "extra_perms",
-        "type_info": "Jsonb"
-      },
-      {
-        "ordinal": 9,
-        "name": "server_id",
-        "type_info": "Varchar"
-      },
-      {
-        "ordinal": 10,
-        "name": "last_server_ping",
-        "type_info": "Timestamptz"
-      },
-      {
-        "ordinal": 11,
-        "name": "error",
-        "type_info": "Text"
-      },
-      {
-        "ordinal": 12,
-        "name": "enabled",
-        "type_info": "Bool"
-      },
-      {
-        "ordinal": 13,
-        "name": "filters",
-        "type_info": "JsonbArray"
-      }
-    ],
-    "parameters": {
-      "Left": [
-        "Text",
-        "Text"
-      ]
-    },
-    "nullable": [
-      false,
-      false,
-      false,
-      false,
-      false,
-      false,
-      false,
-      false,
-      false,
-      true,
-      true,
-      true,
-      false,
-      false
-    ]
-  },
-  "hash": "f2baee15e6d1fecd6d2d7b39fda1b50a15ec8bd349f1081af54da5dc2f5e3021"
-}

backend/.sqlx/query-f904702536c106b0e5da8facae119c6af887c49a29ae44b3a95350ff27fb1ccf.json

@@ -1,6 +1,6 @@
 {
   "db_name": "PostgreSQL",
-  "query": "SELECT workspace_id, path, route_path, route_path_key, script_path, is_flow, http_method as \"http_method: _\", edited_by, email, edited_at, extra_perms, is_async, requires_auth\n            FROM http_trigger\n            WHERE workspace_id = $1 AND path = $2",
+  "query": "SELECT workspace_id, path, route_path, route_path_key, script_path, is_flow, http_method as \"http_method: _\", edited_by, email, edited_at, extra_perms, is_async, requires_auth, static_asset_config as \"static_asset_config: _\"\n            FROM http_trigger\n            WHERE workspace_id = $1 AND path = $2",
   "describe": {
     "columns": [
       {
@@ -80,6 +80,11 @@
         "ordinal": 12,
         "name": "requires_auth",
         "type_info": "Bool"
+      },
+      {
+        "ordinal": 13,
+        "name": "static_asset_config: _",
+        "type_info": "Jsonb"
       }
     ],
     "parameters": {
@@ -101,8 +106,9 @@
       false,
       false,
       false,
-      false
+      false,
+      true
     ]
   },
-  "hash": "4fb95eae1c871241efe2ef79615ce03cba0e4a12aad3274e4829d98e38ca1491"
+  "hash": "f904702536c106b0e5da8facae119c6af887c49a29ae44b3a95350ff27fb1ccf"
 }

backend/Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "windmill"
-version = "1.416.2"
+version = "1.423.1"
 authors.workspace = true
 edition.workspace = true
 
@@ -29,7 +29,7 @@ members = [
 ]
 
 [workspace.package]
-version = "1.416.2"
+version = "1.423.1"
 authors = ["Ruben Fiszel <ruben@windmill.dev>"]
 edition = "2021"
 
@@ -154,7 +154,7 @@ tracing-appender = "^0"
 prometheus = { version = "^0", default-features = false }
 cookie = { version = "0.17.0" }
 phf = { version = "0.11", features = ["macros"] }
-rust-embed = "^6"
+rust-embed = { version = "^6", features = ["interpolate-folder-path"] }
 mime_guess = "^2"
 hex = "^0"
 sql-builder = "^3"

backend/ee-repo-ref.txt

@@ -1 +1 @@
-f136a2f499e0fe7c10c54c79488851980d796eb2
+053cf0a539c64efbe8eee0e2a330b72114a43193

backend/migrations/20241030150214_ws_auth.down.sql

@@ -0,0 +1,2 @@
+-- Add down migration script here
+ALTER TABLE websocket_trigger DROP COLUMN initial_messages, DROP COLUMN url_runnable_args;

backend/migrations/20241030150214_ws_auth.up.sql

@@ -0,0 +1,2 @@
+-- Add up migration script here
+ALTER TABLE websocket_trigger ADD COLUMN initial_messages JSONB[] DEFAULT '{}', ADD COLUMN url_runnable_args JSONB DEFAULT '{}';

backend/migrations/20241105225151_alert_add_acknowledged.down.sql

@@ -0,0 +1 @@
+ALTER TABLE alerts DROP COLUMN acknowledged;

backend/migrations/20241105225151_alert_add_acknowledged.up.sql

@@ -0,0 +1,9 @@
+-- Step 1: Add the new column 'acknowledged' to the 'alerts' table
+ALTER TABLE alerts
+ADD COLUMN acknowledged BOOLEAN DEFAULT NULL;
+
+-- Step 2: Update all existing rows to set 'acknowledged' to true
+-- we don't want to pop up notifications to all users after migrations
+-- but only show new alerts from the point of the upgrade
+UPDATE alerts
+SET acknowledged = TRUE;

backend/migrations/20241107163553_http_route_static_asset.down.sql

@@ -0,0 +1,2 @@
+-- Add down migration script here
+ALTER TABLE http_trigger DROP COLUMN static_asset_config;

backend/migrations/20241107163553_http_route_static_asset.up.sql

@@ -0,0 +1,2 @@
+-- Add up migration script here
+ALTER TABLE http_trigger ADD COLUMN static_asset_config JSONB;

backend/oauth_connect.json

@@ -171,5 +171,6 @@
       "user-library-modify",
       "user-library-read"
     ]
-  }
-}
+  },
+  "snowflake_oauth": {}
+}

backend/parsers/windmill-parser-py-imports/src/lib.rs

@@ -62,6 +62,7 @@ static PYTHON_IMPORTS_REPLACEMENT: phf::Map<&'static str, &'static str> = phf_ma
     "lokalise" => "python-lokalise-api",
     "msgraph" => "msgraph-sdk",
     "pythonjsonlogger" => "python-json-logger",
+    "socks" => "PySocks",
 };
 
 fn replace_import(x: String) -> String {

backend/src/main.rs

@@ -36,7 +36,7 @@ use windmill_common::{
         ENV_SETTINGS, EXPOSE_DEBUG_METRICS_SETTING, EXPOSE_METRICS_SETTING,
         EXTRA_PIP_INDEX_URL_SETTING, HUB_BASE_URL_SETTING, JOB_DEFAULT_TIMEOUT_SECS_SETTING,
         JWT_SECRET_SETTING, KEEP_JOB_DIR_SETTING, LICENSE_KEY_SETTING, NPM_CONFIG_REGISTRY_SETTING,
-        OAUTH_SETTING, PIP_INDEX_URL_SETTING, REQUEST_SIZE_LIMIT_SETTING,
+        OAUTH_SETTING, PIP_INDEX_URL_SETTING, REQUEST_SIZE_LIMIT_SETTING, CRITICAL_ALERT_MUTE_UI_SETTING,
         REQUIRE_PREEXISTING_USER_FOR_OAUTH_SETTING, RETENTION_PERIOD_SECS_SETTING,
         SAML_METADATA_SETTING, SCIM_TOKEN_SETTING, SMTP_SETTING, TIMEOUT_WAIT_RESULT_SETTING,
     },
@@ -78,7 +78,7 @@ use crate::monitor::{
     reload_hub_base_url_setting, reload_job_default_timeout_setting, reload_jwt_secret_setting,
     reload_license_key, reload_npm_config_registry_setting, reload_pip_index_url_setting,
     reload_retention_period_setting, reload_scim_token_setting, reload_smtp_config,
-    reload_worker_config,
+    reload_worker_config, reload_critical_alert_mute_ui_setting,
 };
 
 #[cfg(feature = "parquet")]
@@ -531,7 +531,7 @@ Windmill Community Edition {GIT_VERSION}
 
         #[cfg(feature = "tantivy")]
         let (index_reader, index_writer) = if should_index_jobs {
-            let (r, w) = windmill_indexer::indexer_ee::init_index(&db).await?;
+            let (r, w) = windmill_indexer::completed_runs_ee::init_index(&db).await?;
             (Some(r), Some(w))
         } else {
             (None, None)
@@ -543,26 +543,61 @@ Windmill Community Edition {GIT_VERSION}
             let index_writer2 = index_writer.clone();
             async {
                 if let Some(index_writer) = index_writer2 {
-                    windmill_indexer::indexer_ee::run_indexer(db.clone(), index_writer, indexer_rx)
-                        .await;
+                    windmill_indexer::completed_runs_ee::run_indexer(
+                        db.clone(),
+                        index_writer,
+                        indexer_rx,
+                    )
+                    .await;
+                }
+                Ok(())
+            }
+        };
+
+        #[cfg(all(feature = "tantivy", feature = "parquet"))]
+        let (log_index_reader, log_index_writer) = if should_index_jobs {
+            let (r, w) = windmill_indexer::service_logs_ee::init_index(&db).await?;
+            (Some(r), Some(w))
+        } else {
+            (None, None)
+        };
+
+        #[cfg(all(feature = "tantivy", feature = "parquet"))]
+        let log_indexer_f = {
+            let log_indexer_rx = killpill_rx.resubscribe();
+            let log_index_writer2 = log_index_writer.clone();
+            async {
+                if let Some(log_index_writer) = log_index_writer2 {
+                    windmill_indexer::service_logs_ee::run_indexer(
+                        db.clone(),
+                        log_index_writer,
+                        log_indexer_rx,
+                    )
+                    .await;
                 }
                 Ok(())
             }
         };
 
         #[cfg(not(feature = "tantivy"))]
-        let (index_reader, index_writer) = (None, None);
+        let index_reader = None;
 
         #[cfg(not(feature = "tantivy"))]
         let indexer_f = async { Ok(()) as anyhow::Result<()> };
 
+        #[cfg(not(all(feature = "tantivy", feature = "parquet")))]
+        let log_index_reader = None;
+
+        #[cfg(not(all(feature = "tantivy", feature = "parquet")))]
+        let log_indexer_f = async { Ok(()) as anyhow::Result<()> };
+
         let server_f = async {
             if !is_agent {
                 windmill_api::run_server(
                     db.clone(),
                     rsmq2,
                     index_reader,
-                    index_writer,
+                    log_index_reader,
                     addr,
                     server_killpill_rx,
                     base_internal_tx,
@@ -782,6 +817,12 @@ Windmill Community Edition {GIT_VERSION}
                                                         tracing::error!(error = %e, "Could not reload jwt secret setting");
                                                     }
                                                 },
+                                                CRITICAL_ALERT_MUTE_UI_SETTING => {
+                                                    tracing::info!("Critical alert UI setting changed");
+                                                    if let Err(e) = reload_critical_alert_mute_ui_setting(&db).await {
+                                                        tracing::error!(error = %e, "Could not reload critical alert UI setting");
+                                                    }
+                                                },
                                                 a @_ => {
                                                     tracing::info!("Unrecognized Global Setting Change Payload: {:?}", a);
                                                 }
@@ -842,7 +883,8 @@ Windmill Community Edition {GIT_VERSION}
             monitor_f,
             server_f,
             metrics_f,
-            indexer_f
+            indexer_f,
+            log_indexer_f
         )?;
     } else {
         tracing::info!("Nothing to do, exiting.");

backend/src/monitor.rs

@@ -39,7 +39,7 @@ use windmill_common::{
         EXPOSE_DEBUG_METRICS_SETTING, EXPOSE_METRICS_SETTING, EXTRA_PIP_INDEX_URL_SETTING,
         HUB_BASE_URL_SETTING, JOB_DEFAULT_TIMEOUT_SECS_SETTING, JWT_SECRET_SETTING,
         KEEP_JOB_DIR_SETTING, LICENSE_KEY_SETTING, NPM_CONFIG_REGISTRY_SETTING, OAUTH_SETTING,
-        PIP_INDEX_URL_SETTING, REQUEST_SIZE_LIMIT_SETTING,
+        PIP_INDEX_URL_SETTING, REQUEST_SIZE_LIMIT_SETTING, CRITICAL_ALERT_MUTE_UI_SETTING,
         REQUIRE_PREEXISTING_USER_FOR_OAUTH_SETTING, RETENTION_PERIOD_SECS_SETTING,
         SAML_METADATA_SETTING, SCIM_TOKEN_SETTING, TIMEOUT_WAIT_RESULT_SETTING,
     },
@@ -55,7 +55,7 @@ use windmill_common::{
         WORKER_GROUP,
     },
     BASE_URL, CRITICAL_ERROR_CHANNELS, DB, DEFAULT_HUB_BASE_URL, HUB_BASE_URL, JOB_RETENTION_SECS,
-    METRICS_DEBUG_ENABLED, METRICS_ENABLED,
+    METRICS_DEBUG_ENABLED, METRICS_ENABLED, CRITICAL_ALERT_MUTE_UI_ENABLED
 };
 use windmill_queue::cancel_job;
 use windmill_worker::{
@@ -111,6 +111,9 @@ lazy_static::lazy_static! {
         "Number of jobs in the queue",
         &["tag"]
     ).unwrap();
+
+    static ref QUEUE_COUNT_TAGS: Arc<RwLock<Vec<String>>> = Arc::new(RwLock::new(Vec::new()));
+
 }
 
 pub async fn initial_load(
@@ -128,6 +131,10 @@ pub async fn initial_load(
         tracing::error!("Error loading expose debug metrics: {e:#}");
     }
 
+    if let Err(e) = reload_critical_alert_mute_ui_setting(db).await {
+        tracing::error!("Error loading critical alert mute ui setting: {e:#}");
+    }
+
     if let Err(e) = load_tag_per_workspace_enabled(db).await {
         tracing::error!("Error loading default tag per workpsace: {e:#}");
     }
@@ -223,6 +230,17 @@ pub async fn load_tag_per_workspace_workspaces(db: &DB) -> error::Result<()> {
     Ok(())
 }
 
+pub async fn reload_critical_alert_mute_ui_setting(db: &DB) -> error::Result<()> {
+    let mute = load_value_from_global_settings(db, CRITICAL_ALERT_MUTE_UI_SETTING).await;
+    match mute {
+        Ok(Some(serde_json::Value::Bool(t))) => {
+            CRITICAL_ALERT_MUTE_UI_ENABLED.store(t, Ordering::Relaxed);
+        }
+        _ => (),
+    };
+    Ok(())
+}
+
 pub async fn load_metrics_debug_enabled(db: &DB) -> error::Result<()> {
     let metrics_enabled = load_value_from_global_settings(db, EXPOSE_DEBUG_METRICS_SETTING).await;
     match metrics_enabled {
@@ -1089,12 +1107,23 @@ pub async fn expose_queue_metrics(db: &Pool<Postgres>) {
     if metrics_enabled || save_metrics {
         let queue_counts = windmill_common::queue::get_queue_counts(db).await;
 
+        if metrics_enabled {
+            for q in QUEUE_COUNT_TAGS.read().await.iter() {
+                if queue_counts.get(q).is_none() {
+                    (*QUEUE_COUNT).with_label_values(&[q]).set(0);
+                }
+            }
+        }
+
+        let mut tags_to_watch = vec![];
         for q in queue_counts {
             let count = q.1;
             let tag = q.0;
+
             if metrics_enabled {
                 let metric = (*QUEUE_COUNT).with_label_values(&[&tag]);
                 metric.set(count as i64);
+                tags_to_watch.push(tag.to_string());
             }
 
             // save queue_count and delay metrics per tag
@@ -1119,6 +1148,10 @@ pub async fn expose_queue_metrics(db: &Pool<Postgres>) {
                 }
             }
         }
+        if metrics_enabled {
+            let mut w = QUEUE_COUNT_TAGS.write().await;
+            *w = tags_to_watch;
+        }
     }
 
     // clean queue metrics older than 14 days

backend/tests/worker.rs

@@ -1112,6 +1112,7 @@ async fn test_deno_flow(db: Pool<Postgres>) {
                         custom_concurrency_key: None,
                         concurrent_limit: None,
                         concurrency_time_window_s: None,
+                        is_trigger: None,
                     }
                     .into(),
                     stop_after_if: Default::default(),
@@ -1153,6 +1154,7 @@ async fn test_deno_flow(db: Pool<Postgres>) {
                                 custom_concurrency_key: None,
                                 concurrent_limit: None,
                                 concurrency_time_window_s: None,
+                                is_trigger: None,
                             }
                             .into(),
                             stop_after_if: Default::default(),
@@ -1276,6 +1278,8 @@ async fn test_deno_flow_same_worker(db: Pool<Postgres>) {
                         custom_concurrency_key: None,
                         concurrent_limit: None,
                         concurrency_time_window_s: None,
+                        is_trigger: None,
+
                     }.into(),
                     stop_after_if: Default::default(),
                     stop_after_all_iters_if: Default::default(),
@@ -1327,6 +1331,7 @@ async fn test_deno_flow_same_worker(db: Pool<Postgres>) {
                                     custom_concurrency_key: None,
                                     concurrent_limit: None,
                                     concurrency_time_window_s: None,
+                                    is_trigger: None,
                                 }.into(),
                                 stop_after_if: Default::default(),
                                 stop_after_all_iters_if: Default::default(),
@@ -1364,6 +1369,8 @@ async fn test_deno_flow_same_worker(db: Pool<Postgres>) {
                                     custom_concurrency_key: None,
                                     concurrent_limit: None,
                                     concurrency_time_window_s: None,
+                                    is_trigger: None,
+
                                 }.into(),
                                 stop_after_if: Default::default(),
                                 stop_after_all_iters_if: Default::default(),
@@ -1424,6 +1431,7 @@ async fn test_deno_flow_same_worker(db: Pool<Postgres>) {
                         custom_concurrency_key: None,
                         concurrent_limit: None,
                         concurrency_time_window_s: None,
+                        is_trigger: None,
                     }.into(),
                     stop_after_if: Default::default(),
                     stop_after_all_iters_if: Default::default(),

backend/windmill-api/openapi.yaml

@@ -1,7 +1,7 @@
 openapi: "3.0.3"
 
 info:
-  version: 1.416.2
+  version: 1.423.1
   title: Windmill API
 
   contact:
@@ -190,7 +190,7 @@ paths:
               schema:
                 type: string
 
-  /w/{workspace}/users/{username}:
+  /w/{workspace}/users/get/{username}:
     get:
       summary: get user (require admin privilege)
       operationId: getUser
@@ -724,6 +724,8 @@ paths:
                       type: string
                     tls_implicit:
                       type: boolean
+                    disable_tls:
+                      type: boolean
                   required:
                     - host
                     - username
@@ -731,6 +733,7 @@ paths:
                     - port
                     - from
                     - tls_implicit
+                    - disable_tls
               required:
                 - to
                 - smtp
@@ -770,6 +773,78 @@ paths:
               schema:
                 type: string
 
+  /settings/critical_alerts:
+    get:
+      summary: Get all critical alerts
+      operationId: getCriticalAlerts
+      tags:
+        - setting
+      parameters:
+        - in: query
+          name: page
+          schema:
+            type: integer
+            default: 1
+            description: The page number to retrieve (minimum value is 1)
+        - in: query
+          name: page_size
+          schema:
+            type: integer
+            default: 10
+            maximum: 100
+            description: Number of alerts per page (maximum is 100)
+        - in: query
+          name: acknowledged
+          schema:
+            type: boolean
+            nullable: true
+            description: Filter by acknowledgment status; true for acknowledged, false for unacknowledged, and omit for all alerts
+      responses:
+        "200":
+          description: Successfully retrieved all critical alerts
+          content:
+            application/json:
+              schema:
+                type: array
+                items:
+                  $ref: '#/components/schemas/CriticalAlert'
+
+  /settings/critical_alerts/{id}/acknowledge:
+    post:
+      summary: Acknowledge a critical alert
+      operationId: acknowledgeCriticalAlert
+      tags:
+        - setting
+      parameters:
+        - in: path
+          name: id
+          required: true
+          schema:
+            type: integer
+          description: The ID of the critical alert to acknowledge
+      responses:
+        "200":
+          description: Successfully acknowledged the critical alert
+          content:
+            application/json:
+              schema:
+                type: string
+                example: "Critical alert acknowledged"
+
+  /settings/critical_alerts/acknowledge_all:
+    post:
+      summary: Acknowledge all unacknowledged critical alerts
+      operationId: acknowledgeAllCriticalAlerts
+      tags:
+        - setting
+      responses:
+        "200":
+          description: Successfully acknowledged all unacknowledged critical alerts.
+          content:
+            application/json:
+              schema:
+                type: string
+                example: "All unacknowledged critical alerts acknowledged"
 
   /settings/test_license_key:
     post:
@@ -9143,6 +9218,14 @@ paths:
           in: query
           schema:
             type: string
+        - name: content_type
+          in: query
+          schema:
+            type: string
+        - name: content_disposition
+          in: query
+          schema:
+            type: string
       requestBody:
         description: File content
         required: true
@@ -9546,6 +9629,109 @@ paths:
                     items:
                       $ref: "#/components/schemas/JobSearchHit"
 
+  /srch/index/search/service_logs:
+    get:
+      summary: Search through service logs with a string query
+      operationId: searchLogsIndex
+      tags:
+        - indexSearch
+      parameters:
+        - name: search_query
+          in: query
+          required: true
+          schema:
+            type: string
+        - name: mode
+          in: query
+          required: true
+          schema:
+            type: string
+        - name: worker_group
+          in: query
+          required: false
+          schema:
+            type: string
+        - name: hostname
+          in: query
+          required: true
+          schema:
+            type: string
+        - name: min_ts
+          in: query
+          required: false
+          schema:
+            type: string
+            format: date-time
+        - name: max_ts
+          in: query
+          required: false
+          schema:
+            type: string
+            format: date-time
+      responses:
+        "200":
+          description: search results
+          content:
+            application/json:
+              schema:
+                type: object
+                properties:
+                  query_parse_errors:
+                    description: a list of the terms that couldn't be parsed (and thus ignored)
+                    type: array
+                    items:
+                      type: string
+                  hits:
+                    description: log files that matched the query
+                    type: array
+                    items:
+                      $ref: "#/components/schemas/LogSearchHit"
+
+  /srch/index/search/count_service_logs:
+    get:
+      summary: Search and count the log line hits on every provided host
+      operationId: countSearchLogsIndex
+      tags:
+        - indexSearch
+      parameters:
+        - name: search_query
+          in: query
+          required: true
+          schema:
+            type: string
+        - name: hosts
+          in: query
+          required: true
+          schema:
+            type: string
+        - name: min_ts
+          in: query
+          required: false
+          schema:
+            type: string
+            format: date-time
+        - name: max_ts
+          in: query
+          required: false
+          schema:
+            type: string
+            format: date-time
+      responses:
+        "200":
+          description: search results
+          content:
+            application/json:
+              schema:
+                type: object
+                properties:
+                  query_parse_errors:
+                    description: a list of the terms that couldn't be parsed (and thus ignored)
+                    type: array
+                    items:
+                      type: string
+                  count_per_host:
+                    description: count of log lines that matched the query per hostname
+                    type: object
 
 components:
   securitySchemes:
@@ -11357,6 +11543,17 @@ components:
           type: string
         route_path:
           type: string
+        static_asset_config:
+          type: object
+          properties:
+            s3:
+              type: string
+            storage:
+              type: string
+            filename:
+              type: string
+          required:
+            - s3
         is_flow:
           type: boolean
         extra_perms:
@@ -11403,6 +11600,17 @@ components:
           type: string
         route_path:
           type: string
+        static_asset_config:
+          type: object
+          properties:
+            s3:
+              type: string
+            storage:
+              type: string
+            filename:
+              type: string
+          required:
+            - s3
         is_flow:
           type: boolean
         http_method:
@@ -11436,6 +11644,17 @@ components:
           type: string
         route_path:
           type: string
+        static_asset_config:
+          type: object
+          properties:
+            s3:
+              type: string
+            storage:
+              type: string
+            filename:
+              type: string
+          required:
+            - s3
         is_flow:
           type: boolean
         http_method:
@@ -11522,6 +11741,12 @@ components:
             required:
               - key
               - value
+        initial_messages:
+          type: array
+          items:
+            $ref: "#/components/schemas/WebsocketTriggerInitialMessage"
+        url_runnable_args:
+          $ref: "#/components/schemas/ScriptArgs"
         
       required:
         - path
@@ -11535,6 +11760,8 @@ components:
         - workspace_id
         - enabled
         - filters
+        - initial_messages
+        - url_runnable_args
 
     NewWebsocketTrigger:
       type: object
@@ -11560,6 +11787,12 @@ components:
             required:
               - key
               - value
+        initial_messages:
+          type: array
+          items:
+            $ref: "#/components/schemas/WebsocketTriggerInitialMessage"
+        url_runnable_args:
+          $ref: "#/components/schemas/ScriptArgs"
         
       required:
         - path
@@ -11567,6 +11800,8 @@ components:
         - url
         - is_flow
         - filters
+        - initial_messages
+        - url_runnable_args
 
     EditWebsocketTrigger:
       type: object
@@ -11590,6 +11825,12 @@ components:
             required:
               - key
               - value
+        initial_messages:
+          type: array
+          items:
+            $ref: "#/components/schemas/WebsocketTriggerInitialMessage"
+        url_runnable_args:
+          $ref: "#/components/schemas/ScriptArgs"
               
       required:
         - path
@@ -11597,6 +11838,34 @@ components:
         - url
         - is_flow
         - filters
+        - initial_messages
+        - url_runnable_args
+
+    WebsocketTriggerInitialMessage:
+      anyOf:
+        - type: object
+          properties:
+            raw_message: 
+              type: string
+          required:
+            - raw_message
+        - type: object
+          properties:
+            runnable_result:
+              type: object
+              properties:
+                path: 
+                  type: string
+                args:
+                  $ref: "#/components/schemas/ScriptArgs"
+                is_flow:
+                  type: boolean
+              required:
+                - path
+                - args
+                - is_flow
+          required: 
+            - runnable_result
 
     Group:
       type: object
@@ -11917,6 +12186,10 @@ components:
           type: object
           additionalProperties:
             type: object
+        s3_inputs:
+          type: array
+          items:
+            type: object
         execution_mode:
           type: string
           enum: [viewer, publisher, anonymous]
@@ -12500,6 +12773,12 @@ components:
         dancer:
           type: string
 
+    LogSearchHit:
+      type: object
+      properties:
+        dancer:
+          type: string
+
     AutoscalingEvent:
       type: object
       properties:
@@ -12517,3 +12796,24 @@ components:
         applied_at:
           type: string
           format: date-time
+
+    CriticalAlert:
+      type: object
+      properties:
+        id:
+          type: integer
+          description: Unique identifier for the alert
+        alert_type:
+          type: string
+          description: Type of alert (e.g., critical_error)
+        message:
+          type: string
+          description: The message content of the alert
+        created_at:
+          type: string
+          format: date-time
+          description: Time when the alert was created
+        acknowledged:
+          type: boolean
+          nullable: true
+          description: Acknowledgment status of the alert, can be true, false, or null if not set

backend/windmill-api/src/apps.rs

@@ -7,6 +7,12 @@ use std::collections::HashMap;
  * Please see the included NOTICE for copyright information and
  * LICENSE-AGPL for a copy of the license.
  */
+
+#[cfg(feature = "parquet")]
+use crate::{job_helpers_ee::{
+    get_random_file_name, get_s3_resource, get_workspace_s3_resource, upload_file_internal,
+    UploadFileResponse,
+}, users::fetch_api_authed_from_permissioned_as};
 use crate::{
     db::{ApiAuthed, DB},
     resources::get_resource_value_interpolated_internal,
@@ -23,7 +29,13 @@ use axum::{
     Router,
 };
 use hyper::StatusCode;
+#[cfg(feature = "parquet")]
+use itertools::Itertools;
 use magic_crypt::MagicCryptTrait;
+#[cfg(feature = "parquet")]
+use object_store::{Attribute, Attributes};
+#[cfg(feature = "parquet")]
+use regex::Regex;
 use serde::{Deserialize, Serialize};
 use serde_json::{json, value::RawValue};
 use sha2::{Digest, Sha256};
@@ -32,6 +44,8 @@ use sqlx::{types::Uuid, FromRow};
 use std::str;
 use windmill_audit::audit_ee::audit_log;
 use windmill_audit::ActionKind;
+#[cfg(feature = "parquet")]
+use windmill_common::s3_helpers::build_object_store_client;
 use windmill_common::{
     apps::ListAppQuery,
     db::UserDB,
@@ -69,6 +83,7 @@ pub fn workspaced_service() -> Router {
 pub fn unauthed_service() -> Router {
     Router::new()
         .route("/execute_component/*path", post(execute_component))
+        .route("/upload_s3_file/*path", post(upload_s3_file_from_app))
         .route("/public_app/:secret", get(get_public_app_by_secret))
         .route("/public_resource/*path", get(get_public_resource))
 }
@@ -179,6 +194,14 @@ pub struct PolicyTriggerableInputs {
     allow_user_resources: AllowUserResources,
 }
 
+#[derive(Serialize, Deserialize, Debug, Clone)]
+pub struct S3Input {
+    allowed_resources: Vec<String>,
+    allow_user_resources: bool,
+    allow_workspace_resource: bool,
+    file_key_regex: String,
+}
+
 #[derive(Serialize, Deserialize, Debug, Clone)]
 pub struct Policy {
     pub on_behalf_of: Option<String>,
@@ -192,6 +215,7 @@ pub struct Policy {
     #[serde(skip_serializing_if = "Option::is_none")]
     pub triggerables_v2: Option<HashMap<String, PolicyTriggerableInputs>>,
     pub execution_mode: ExecutionMode,
+    pub s3_inputs: Option<Vec<S3Input>>,
 }
 
 #[derive(Deserialize)]
@@ -432,9 +456,7 @@ async fn get_latest_version(
     authed: ApiAuthed,
     Extension(user_db): Extension<UserDB>,
     Path((w_id, path)): Path<(String, StripPath)>,
-
 ) -> JsonResult<Option<AppHistory>> {
-
     let mut tx = user_db.begin(&authed).await?;
     let row = sqlx::query!(
         "SELECT a.id as app_id, av.id as version_id, dm.deployment_msg as deployment_msg
@@ -457,7 +479,6 @@ async fn get_latest_version(
     } else {
         return Ok(Json(None));
     }
-
 }
 
 async fn update_app_history(
@@ -1067,6 +1088,49 @@ fn digest(code: &str) -> String {
     format!("rawscript/{:x}", result)
 }
 
+async fn get_on_behalf_details_from_policy_and_authed(
+    policy: &Policy,
+    opt_authed: &Option<ApiAuthed>,
+) -> Result<(String, String, String)> {
+    let (username, permissioned_as, email) = match policy.execution_mode {
+        ExecutionMode::Anonymous => {
+            let username = opt_authed
+                .as_ref()
+                .map(|a| a.username.clone())
+                .unwrap_or_else(|| "anonymous".to_string());
+            let (permissioned_as, email) = get_on_behalf_of(&policy)?;
+            (username, permissioned_as, email)
+        }
+        ExecutionMode::Publisher => {
+            let username = opt_authed
+                .as_ref()
+                .map(|a| a.username.clone())
+                .ok_or_else(|| {
+                    Error::BadRequest(
+                        "publisher execution mode requires authentication".to_string(),
+                    )
+                })?;
+            let (permissioned_as, email) = get_on_behalf_of(&policy)?;
+            (username, permissioned_as, email)
+        }
+        ExecutionMode::Viewer => {
+            let (username, email) = opt_authed
+                .as_ref()
+                .map(|a| (a.username.clone(), a.email.clone()))
+                .ok_or_else(|| {
+                    Error::BadRequest("Required to be authed in viewer mode".to_string())
+                })?;
+            (
+                username.clone(),
+                username_to_permissioned_as(&username),
+                email,
+            )
+        }
+    };
+
+    Ok((username, permissioned_as, email))
+}
+
 async fn execute_component(
     OptAuthed(opt_authed): OptAuthed,
     Extension(db): Extension<DB>,
@@ -1129,6 +1193,7 @@ async fn execute_component(
                 triggerables_v2: Some(hm),
                 on_behalf_of: None,
                 on_behalf_of_email: None,
+                s3_inputs: None,
             }
         }
         _ => {
@@ -1146,41 +1211,8 @@ async fn execute_component(
         }
     };
 
-    let (username, permissioned_as, email) = match policy.execution_mode {
-        ExecutionMode::Anonymous => {
-            let username = opt_authed
-                .as_ref()
-                .map(|a| a.username.clone())
-                .unwrap_or_else(|| "anonymous".to_string());
-            let (permissioned_as, email) = get_on_behalf_of(&policy)?;
-            (username, permissioned_as, email)
-        }
-        ExecutionMode::Publisher => {
-            let username = opt_authed
-                .as_ref()
-                .map(|a| a.username.clone())
-                .ok_or_else(|| {
-                    Error::BadRequest(
-                        "publisher execution mode requires authentication".to_string(),
-                    )
-                })?;
-            let (permissioned_as, email) = get_on_behalf_of(&policy)?;
-            (username, permissioned_as, email)
-        }
-        ExecutionMode::Viewer => {
-            let (username, email) = opt_authed
-                .as_ref()
-                .map(|a| (a.username.clone(), a.email.clone()))
-                .ok_or_else(|| {
-                    Error::BadRequest("Required to be authed in viewer mode".to_string())
-                })?;
-            (
-                username.clone(),
-                username_to_permissioned_as(&username),
-                email,
-            )
-        }
-    };
+    let (username, permissioned_as, email) =
+        get_on_behalf_details_from_policy_and_authed(&policy, &opt_authed).await?;
 
     let (job_payload, (args, job_id), tag) = match payload {
         ExecuteApp { args, component, raw_code: Some(raw_code), path: None, .. } => {
@@ -1249,6 +1281,248 @@ async fn execute_component(
     Ok(uuid.to_string())
 }
 
+#[cfg(not(feature = "parquet"))]
+async fn upload_s3_file_from_app() -> Result<()> {
+    return Err(Error::BadRequest(
+        "This endpoint requires the parquet feature to be enabled".to_string(),
+    ));
+}
+
+
+#[cfg(feature = "parquet")]
+#[derive(Debug, Deserialize, Clone)]
+struct UploadFileToS3Query {
+    file_key: Option<String>,
+    file_extension: Option<String>,
+    s3_resource_path: Option<String>,
+    content_type: Option<String>,
+    content_disposition: Option<String>,
+    force_viewer_file_key_regex: Option<String>,
+    force_viewer_allow_user_resources: Option<bool>,
+    force_viewer_allow_workspace_resource: Option<bool>,
+    force_viewer_allowed_resources: Option<String>,
+}
+
+#[cfg(feature = "parquet")]
+async fn upload_s3_file_from_app(
+    OptAuthed(opt_authed): OptAuthed,
+    Extension(db): Extension<DB>,
+    Path((w_id, path)): Path<(String, StripPath)>,
+    Query(query): Query<UploadFileToS3Query>,
+    request: axum::extract::Request,
+) -> JsonResult<UploadFileResponse> {
+    let policy = if let Some(file_key_regex) = query.force_viewer_file_key_regex {
+        Some(Policy {
+            execution_mode: ExecutionMode::Viewer,
+            triggerables: None,
+            triggerables_v2: None,
+            on_behalf_of: None,
+            on_behalf_of_email: None,
+            s3_inputs: Some(vec![S3Input {
+                file_key_regex: file_key_regex,
+                allow_user_resources: query.force_viewer_allow_user_resources.unwrap_or(false),
+                allow_workspace_resource: query
+                    .force_viewer_allow_workspace_resource
+                    .unwrap_or(false),
+                allowed_resources: query
+                    .force_viewer_allowed_resources
+                    .map(|s| s.split(',').map(|s| s.to_string()).collect())
+                    .unwrap_or_default(),
+            }]),
+        })
+    } else {
+        let policy_o = sqlx::query_scalar!(
+            "SELECT policy from app WHERE path = $1 AND workspace_id = $2",
+            &path.0,
+            &w_id
+        )
+        .fetch_optional(&db)
+        .await?;
+
+        policy_o
+            .map(|p| serde_json::from_value::<Policy>(p).map_err(to_anyhow))
+            .transpose()?
+    };
+
+    let user_db = UserDB::new(db.clone());
+
+    let (s3_resource_opt, file_key) = if policy.as_ref().is_some_and(|p| p.s3_inputs.is_some()) {
+        let policy = policy.unwrap();
+        let s3_inputs = policy.s3_inputs.as_ref().unwrap();
+
+        let (username, permissioned_as, email) =
+            get_on_behalf_details_from_policy_and_authed(&policy, &opt_authed).await?;
+
+        let on_behalf_authed =
+            fetch_api_authed_from_permissioned_as(permissioned_as, email, &w_id, &db, username)
+                .await?;
+
+        if let Some(file_key) = query.file_key {
+            // file key is provided => requires workspace, user or list policy and must match the regex
+            let matching_s3_inputs = if let Some(ref s3_resource_path) = query.s3_resource_path {
+                s3_inputs
+                    .iter()
+                    .filter(|s3_input| {
+                        s3_input.allowed_resources.contains(s3_resource_path)
+                            || s3_input.allow_user_resources
+                    })
+                    .sorted_by_key(|i| i.allow_user_resources) // consider user resources last
+                    .collect::<Vec<_>>()
+            } else {
+                s3_inputs
+                    .iter()
+                    .filter(|s3_input| s3_input.allow_workspace_resource)
+                    .collect::<Vec<_>>()
+            };
+
+            let matched_input = matching_s3_inputs.iter().find(|s3_input| {
+                match Regex::new(&s3_input.file_key_regex) {
+                    Ok(re) => re.is_match(&file_key),
+                    Err(e) => {
+                        tracing::error!("Error compiling regex: {}", e);
+                        false
+                    }
+                }
+            });
+
+            if let Some(matched_input) = matched_input {
+                if let Some(ref s3_resource_path) = query.s3_resource_path {
+                    if matched_input.allow_user_resources {
+                        if let Some(authed) = opt_authed {
+                            (
+                                Some(
+                                    get_s3_resource(
+                                        &authed,
+                                        &db,
+                                        Some(user_db),
+                                        "",
+                                        &w_id,
+                                        s3_resource_path,
+                                        None,
+                                        None,
+                                    )
+                                    .await?,
+                                ),
+                                file_key,
+                            )
+                        } else {
+                            return Err(Error::BadRequest(
+                                "User resources are not allowed without being logged in"
+                                    .to_string(),
+                            ));
+                        }
+                    } else {
+                        (
+                            Some(
+                                get_s3_resource(
+                                    &on_behalf_authed,
+                                    &db,
+                                    Some(user_db),
+                                    "",
+                                    &w_id,
+                                    s3_resource_path,
+                                    None,
+                                    None,
+                                )
+                                .await?,
+                            ),
+                            file_key,
+                        )
+                    }
+                } else {
+                    let (_, s3_resource_opt) =
+                        get_workspace_s3_resource(&on_behalf_authed, &db, None, "", &w_id, None)
+                            .await?;
+                    (s3_resource_opt, file_key)
+                }
+            } else {
+                return Err(Error::BadRequest(
+                    "No matching s3 resource found for the given file key".to_string(),
+                ));
+            }
+        } else {
+            // no file key => requires unnamed upload policy => allow workspace resource and file_key_regex is empty
+            let has_unnamed_policy = s3_inputs.iter().any(|s3_input| {
+                s3_input.allow_workspace_resource && s3_input.file_key_regex.is_empty()
+            });
+
+            if !has_unnamed_policy {
+                return Err(Error::BadRequest(
+                    "no policy found for unnamed s3 file uplooad".to_string(),
+                ));
+            }
+
+            // for now, we place all files into `windmill_uploads` folder with a random name
+            // TODO: make the folder configurable via the workspace settings
+            let file_key = get_random_file_name(query.file_extension);
+
+            let (_, s3_resource_opt) =
+                get_workspace_s3_resource(&on_behalf_authed, &db, None, "", &w_id, None).await?;
+
+            (s3_resource_opt, file_key)
+        }
+    } else {
+        // backward compatibility (no policy)
+        // if no policy but logged in, use the user's auth to get the s3 resource
+        if let Some(authed) = opt_authed {
+            let file_key = query
+                .file_key
+                .unwrap_or_else(|| get_random_file_name(query.file_extension));
+
+            if let Some(ref s3_resource_path) = query.s3_resource_path {
+                (
+                    Some(
+                        get_s3_resource(
+                            &authed,
+                            &db,
+                            Some(user_db),
+                            "",
+                            &w_id,
+                            s3_resource_path,
+                            None,
+                            None,
+                        )
+                        .await?,
+                    ),
+                    file_key,
+                )
+            } else {
+                let (_, s3_resource) =
+                    get_workspace_s3_resource(&authed, &db, None, "", &w_id, None).await?;
+
+                (s3_resource, file_key)
+            }
+        } else {
+            return Err(Error::BadRequest("Missing s3 policy".to_string()));
+        }
+    };
+
+    let s3_resource = s3_resource_opt.ok_or(Error::InternalErr(
+        "No files storage resource defined at the workspace level".to_string(),
+    ))?;
+    let s3_client = build_object_store_client(&s3_resource).await?;
+
+    let options = Attributes::from_iter(vec![
+        (
+            Attribute::ContentType,
+            query.content_type.unwrap_or_else(|| {
+                mime_guess::from_path(&file_key)
+                    .first_or_octet_stream()
+                    .to_string()
+            }),
+        ),
+        (
+            Attribute::ContentDisposition,
+            query.content_disposition.unwrap_or("inline".to_string()),
+        ),
+    ])
+    .into();
+
+    upload_file_internal(s3_client, &file_key, request, options).await?;
+
+    return Ok(Json(UploadFileResponse { file_key }));
+}
+
 fn get_on_behalf_of(policy: &Policy) -> Result<(String, String)> {
     let permissioned_as = policy
         .on_behalf_of

backend/windmill-api/src/flows.rs

@@ -1209,6 +1209,7 @@ mod tests {
                         .into(),
                         hash: None,
                         tag_override: None,
+                        is_trigger: None,
                     }),
                     stop_after_if: None,
                     stop_after_all_iters_if: None,
@@ -1236,6 +1237,7 @@ mod tests {
                         custom_concurrency_key: None,
                         concurrent_limit: None,
                         concurrency_time_window_s: None,
+                        is_trigger: None,
                     }),
                     stop_after_if: Some(StopAfterIf {
                         expr: "foo = 'bar'".to_string(),
@@ -1290,6 +1292,7 @@ mod tests {
                     input_transforms: HashMap::new(),
                     hash: None,
                     tag_override: None,
+                    is_trigger: None,
                 }
                 .into(),
                 stop_after_if: Some(StopAfterIf {

backend/windmill-api/src/http_triggers.rs

@@ -1,24 +1,5 @@
-use axum::{
-    extract::{Path, Query},
-    response::IntoResponse,
-    routing::{delete, get, post},
-    Extension, Json, Router,
-};
-use http::{HeaderMap, StatusCode};
-use serde::{Deserialize, Serialize};
-use sql_builder::{bind::Bind, SqlBuilder};
-use sqlx::prelude::FromRow;
-use std::collections::HashMap;
-use tower_http::cors::CorsLayer;
-use windmill_audit::{audit_ee::audit_log, ActionKind};
-use windmill_common::{
-    db::UserDB,
-    error::{self, JsonResult},
-    utils::{not_found_if_none, paginate, require_admin, Pagination, StripPath},
-    worker::{to_raw_value, CLOUD_HOSTED},
-};
-use windmill_queue::PushArgsOwned;
-
+#[cfg(feature = "parquet")]
+use crate::job_helpers_ee::get_workspace_s3_resource;
 use crate::{
     db::{ApiAuthed, DB},
     jobs::{
@@ -27,6 +8,31 @@ use crate::{
     },
     users::{fetch_api_authed, OptAuthed},
 };
+use axum::{
+    extract::{Path, Query},
+    response::IntoResponse,
+    routing::{delete, get, post},
+    Extension, Json, Router,
+};
+#[cfg(feature = "parquet")]
+use http::header::IF_NONE_MATCH;
+use http::{HeaderMap, StatusCode};
+use serde::{Deserialize, Serialize};
+use sql_builder::{bind::Bind, SqlBuilder};
+use sqlx::prelude::FromRow;
+use std::collections::HashMap;
+use tower_http::cors::CorsLayer;
+use windmill_audit::{audit_ee::audit_log, ActionKind};
+#[cfg(feature = "parquet")]
+use windmill_common::s3_helpers::build_object_store_client;
+use windmill_common::{
+    db::UserDB,
+    error::{self, JsonResult},
+    s3_helpers::S3Object,
+    utils::{not_found_if_none, paginate, require_admin, Pagination, StripPath},
+    worker::{to_raw_value, CLOUD_HOSTED},
+};
+use windmill_queue::PushArgsOwned;
 
 lazy_static::lazy_static! {
     static ref ROUTE_PATH_KEY_RE: regex::Regex = regex::Regex::new(r"/:\w+").unwrap();
@@ -99,6 +105,7 @@ struct NewTrigger {
     is_async: bool,
     requires_auth: bool,
     http_method: HttpMethod,
+    static_asset_config: Option<sqlx::types::Json<S3Object>>,
 }
 
 #[derive(FromRow, Serialize)]
@@ -116,6 +123,7 @@ struct Trigger {
     is_async: bool,
     requires_auth: bool,
     http_method: HttpMethod,
+    static_asset_config: Option<sqlx::types::Json<S3Object>>,
 }
 
 #[derive(Deserialize)]
@@ -127,6 +135,7 @@ struct EditTrigger {
     is_async: bool,
     requires_auth: bool,
     http_method: HttpMethod,
+    static_asset_config: Option<sqlx::types::Json<S3Object>>,
 }
 
 #[derive(Deserialize)]
@@ -182,7 +191,7 @@ async fn get_trigger(
     let path = path.to_path();
     let trigger = sqlx::query_as!(
         Trigger,
-        r#"SELECT workspace_id, path, route_path, route_path_key, script_path, is_flow, http_method as "http_method: _", edited_by, email, edited_at, extra_perms, is_async, requires_auth
+        r#"SELECT workspace_id, path, route_path, route_path_key, script_path, is_flow, http_method as "http_method: _", edited_by, email, edited_at, extra_perms, is_async, requires_auth, static_asset_config as "static_asset_config: _"
             FROM http_trigger
             WHERE workspace_id = $1 AND path = $2"#,
         w_id,
@@ -209,7 +218,7 @@ async fn create_trigger(
 
     let mut tx = user_db.begin(&authed).await?;
     sqlx::query!(
-        "INSERT INTO http_trigger (workspace_id, path, route_path, route_path_key, script_path, is_flow, is_async, requires_auth, http_method, edited_by, email, edited_at) VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, now())",
+        "INSERT INTO http_trigger (workspace_id, path, route_path, route_path_key, script_path, is_flow, is_async, requires_auth, http_method, static_asset_config, edited_by, email, edited_at) VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, $12, now())",
         w_id,
         ct.path,
         ct.route_path,
@@ -218,7 +227,8 @@ async fn create_trigger(
         ct.is_flow,
         ct.is_async,
         ct.requires_auth,
-        ct.http_method as HttpMethod,
+        ct.http_method as _,
+        ct.static_asset_config as _,
         &authed.username,
         &authed.email
     )
@@ -261,14 +271,15 @@ async fn update_trigger(
 
         sqlx::query!(
             "UPDATE http_trigger 
-                SET route_path = $1, route_path_key = $2, script_path = $3, path = $4, is_flow = $5, http_method = $6, edited_by = $7, email = $8, is_async = $9, requires_auth = $10, edited_at = now() 
-                WHERE workspace_id = $11 AND path = $12",
+                SET route_path = $1, route_path_key = $2, script_path = $3, path = $4, is_flow = $5, http_method = $6, static_asset_config = $7, edited_by = $8, email = $9, is_async = $10, requires_auth = $11, edited_at = now() 
+                WHERE workspace_id = $12 AND path = $13",
             ct.route_path,
             &route_path_key,
             ct.script_path,
             ct.path,
             ct.is_flow,
-            ct.http_method as HttpMethod,
+            ct.http_method as _,
+            ct.static_asset_config as _,
             &authed.username,
             &authed.email,
             ct.is_async,
@@ -279,12 +290,13 @@ async fn update_trigger(
         .execute(&mut *tx).await?;
     } else {
         sqlx::query!(
-            "UPDATE http_trigger SET script_path = $1, path = $2, is_flow = $3, http_method = $4, edited_by = $5, email = $6, is_async = $7, requires_auth = $8, edited_at = now() 
-                WHERE workspace_id = $9 AND path = $10",
+            "UPDATE http_trigger SET script_path = $1, path = $2, is_flow = $3, http_method = $4, static_asset_config = $5, edited_by = $6, email = $7, is_async = $8, requires_auth = $9, edited_at = now() 
+                WHERE workspace_id = $10 AND path = $11",
             ct.script_path,
             ct.path,
             ct.is_flow,
-            ct.http_method as HttpMethod,
+            ct.http_method as _,
+            ct.static_asset_config as _,
             &authed.username,
             &authed.email,
             ct.is_async,
@@ -405,6 +417,7 @@ struct TriggerRoute {
     edited_by: String,
     email: String,
     http_method: HttpMethod,
+    static_asset_config: Option<sqlx::types::Json<S3Object>>,
 }
 
 async fn get_http_route_trigger(
@@ -421,7 +434,7 @@ async fn get_http_route_trigger(
         let route_path = StripPath(splitted.collect::<Vec<_>>().join("/"));
         let triggers = sqlx::query_as!(
             TriggerRoute,
-            r#"SELECT path, script_path, is_flow, route_path, workspace_id, is_async, requires_auth, edited_by, email, http_method as "http_method: _" FROM http_trigger WHERE workspace_id = $1"#,
+            r#"SELECT path, script_path, is_flow, route_path, workspace_id, is_async, requires_auth, edited_by, email, http_method as "http_method: _", static_asset_config as "static_asset_config: _" FROM http_trigger WHERE workspace_id = $1"#,
             w_id
         )
         .fetch_all(db)
@@ -430,7 +443,7 @@ async fn get_http_route_trigger(
     } else {
         let triggers = sqlx::query_as!(
             TriggerRoute,
-            r#"SELECT path, script_path, is_flow, route_path, workspace_id, is_async, requires_auth, edited_by, email, http_method as "http_method: _" FROM http_trigger"#,
+            r#"SELECT path, script_path, is_flow, route_path, workspace_id, is_async, requires_auth, edited_by, email, http_method as "http_method: _", static_asset_config as "static_asset_config: _" FROM http_trigger"#,
         )
         .fetch_all(db)
         .await?;
@@ -518,6 +531,90 @@ async fn route_job(
             Ok(trigger) => trigger,
             Err(e) => return e.into_response(),
         };
+
+    #[cfg(not(feature = "parquet"))]
+    if trigger.static_asset_config.is_some() {
+        return error::Error::InternalErr(
+            "Static asset configuration is not supported in this build".to_string(),
+        )
+        .into_response();
+    }
+
+    #[cfg(feature = "parquet")]
+    if let Some(sqlx::types::Json(config)) = trigger.static_asset_config {
+        let build_static_response_f = async {
+            let (_, s3_resource_opt) = get_workspace_s3_resource(
+                &authed,
+                &db,
+                None,
+                &"NO_TOKEN".to_string(), // no token is provided in this case
+                &trigger.workspace_id,
+                config.storage,
+            )
+            .await?;
+            let s3_resource = s3_resource_opt.ok_or(error::Error::InternalErr(
+                "No files storage resource defined at the workspace level".to_string(),
+            ))?;
+            let s3_client = build_object_store_client(&s3_resource).await?;
+            let path = object_store::path::Path::from(config.s3);
+            let s3_object = s3_client.get(&path).await.map_err(|err| {
+                tracing::warn!("Error retrieving file from S3: {:?}", err);
+                error::Error::InternalErr(format!("Error retrieving file: {}", err.to_string()))
+            })?;
+            let mut response_headers = http::HeaderMap::new();
+            if let Some(ref e_tag) = s3_object.meta.e_tag {
+                if let Some(if_none_match) = headers.get(IF_NONE_MATCH) {
+                    if if_none_match == e_tag {
+                        return Ok::<_, error::Error>((
+                            StatusCode::NOT_MODIFIED,
+                            response_headers,
+                            axum::body::Body::empty(),
+                        ));
+                    }
+                }
+                if let Ok(e_tag) = e_tag.parse() {
+                    response_headers.insert("etag", e_tag);
+                }
+            }
+            response_headers.insert(
+                "content-type",
+                s3_object
+                    .attributes
+                    .get(&object_store::Attribute::ContentType)
+                    .map(|s| s.parse().ok())
+                    .flatten()
+                    .unwrap_or("application/octet-stream".parse().unwrap()),
+            );
+            response_headers.insert(
+                "content-disposition",
+                config.filename.as_ref().map_or_else(
+                    || {
+                        s3_object
+                            .attributes
+                            .get(&object_store::Attribute::ContentDisposition)
+                            .map(|s| s.parse().ok())
+                            .flatten()
+                            .unwrap_or("inline".parse().unwrap())
+                    },
+                    |filename| {
+                        format!("inline; filename=\"{}\"", filename)
+                            .parse()
+                            .unwrap_or("inline".parse().unwrap())
+                    },
+                ),
+            );
+
+            let body_stream = axum::body::Body::from_stream(s3_object.into_stream());
+            Ok::<_, error::Error>((StatusCode::OK, response_headers, body_stream))
+        };
+        match build_static_response_f.await {
+            Ok((status, headers, body_stream)) => {
+                return (status, headers, body_stream).into_response()
+            }
+            Err(e) => return e.into_response(),
+        }
+    }
+
     let headers = headers
         .iter()
         .map(|(k, v)| (k.to_string(), v.to_str().unwrap_or("").to_string()))

backend/windmill-api/src/indexer_ee.rs

@@ -3,3 +3,7 @@ use axum::Router;
 pub fn workspaced_service() -> Router {
     Router::new()
 }
+
+pub fn global_service() -> Router {
+    Router::new()
+}

backend/windmill-api/src/job_helpers_ee.rs

@@ -1,5 +1,61 @@
 use axum::Router;
+use serde::Serialize;
+use uuid::Uuid;
+use windmill_common::s3_helpers::StorageResourceType;
+
+#[cfg(feature = "parquet")]
+use crate::db::{ApiAuthed, DB};
+#[cfg(feature = "parquet")]
+use object_store::{ObjectStore, PutMultipartOpts};
+#[cfg(feature = "parquet")]
+use std::sync::Arc;
+use windmill_common::error;
+#[cfg(feature = "parquet")]
+use windmill_common::{db::UserDB, s3_helpers::ObjectStoreResource};
+#[derive(Serialize)]
+pub struct UploadFileResponse {
+    pub file_key: String,
+}
 
 pub fn workspaced_service() -> Router {
     Router::new()
 }
+
+#[cfg(feature = "parquet")]
+pub async fn get_workspace_s3_resource<'c>(
+    _authed: &ApiAuthed,
+    _db: &DB,
+    _user_db: Option<UserDB>,
+    _token: &str,
+    _w_id: &str,
+    _storage: Option<String>,
+) -> windmill_common::error::Result<(Option<bool>, Option<ObjectStoreResource>)> {
+    // implementation is not open source
+    Ok((None, None))
+}
+
+pub fn get_random_file_name(_file_extension: Option<String>) -> String {
+    todo!()
+}
+
+pub async fn get_s3_resource<'c>(
+    _authed: &ApiAuthed,
+    _db: &DB,
+    _user_db: Option<UserDB>,
+    _token: &str,
+    _w_id: &str,
+    _resource_path: &str,
+    _resource_type: Option<StorageResourceType>,
+    _job_id: Option<Uuid>,
+) -> error::Result<ObjectStoreResource> {
+    todo!()
+}
+
+pub async fn upload_file_internal(
+    _s3_client: Arc<dyn ObjectStore>,
+    _file_key: &str,
+    _request: axum::extract::Request,
+    _options: PutMultipartOpts,
+) -> error::Result<()> {
+    todo!()
+}

backend/windmill-api/src/jobs.rs

@@ -65,7 +65,7 @@ use windmill_common::{
     db::UserDB,
     error::{self, to_anyhow, Error},
     flow_status::{Approval, FlowStatus, FlowStatusModule},
-    flows::FlowValue,
+    flows::{add_virtual_items_if_necessary, FlowValue},
     jobs::{script_path_to_payload, CompletedJob, JobKind, JobPayload, QueuedJob, RawCode},
     oauth2::HmacSha256,
     scripts::{ScriptHash, ScriptLang},
@@ -4275,18 +4275,26 @@ async fn run_flow_dependencies_job(
     wait_result
 }
 
+#[derive(Deserialize)]
+struct BatchRawScript {
+    content: String,
+    language: Option<ScriptLang>,
+    lock: Option<String>,
+}
+
 #[derive(Deserialize)]
 struct BatchInfo {
     kind: String,
     flow_value: Option<FlowValue>,
     path: Option<String>,
+    rawscript: Option<BatchRawScript>,
 }
 
 #[tracing::instrument(level = "trace", skip_all)]
 async fn add_batch_jobs(
     authed: ApiAuthed,
     Extension(db): Extension<DB>,
-    Extension(rsmq): Extension<Option<rsmq_async::MultiplexedRsmq>>,
+    Extension(_rsmq): Extension<Option<rsmq_async::MultiplexedRsmq>>,
     Path((w_id, n)): Path<(String, i32)>,
     Json(batch_info): Json<BatchInfo>,
 ) -> error::JsonResult<Vec<Uuid>> {
@@ -4302,6 +4310,10 @@ async fn add_batch_jobs(
         concurrent_limit,
         concurrent_time_window_s,
         timeout,
+        raw_code,
+        raw_lock,
+        raw_flow,
+        flow_status
     ) = match batch_info.kind.as_str() {
         "script" => {
             if let Some(path) = batch_info.path {
@@ -4329,6 +4341,10 @@ async fn add_batch_jobs(
                     concurrent_limit,
                     concurrency_time_window_s,
                     timeout,
+                    None,
+                    None,
+                    None,
+                    None,
                 )
             } else {
                 Err(anyhow::anyhow!(
@@ -4336,61 +4352,72 @@ async fn add_batch_jobs(
                 ))?
             }
         }
-        "flow" => {
-            let mut uuids: Vec<Uuid> = Vec::new();
-            let payload = if let Some(ref fv) = batch_info.flow_value {
-                JobPayload::RawFlow { value: fv.clone(), path: None, restarted_from: None }
-            } else {
-                if let Some(path) = batch_info.path.as_ref() {
-                    JobPayload::Flow {
-                        path: path.to_string(),
-                        dedicated_worker: None,
-                        apply_preprocessor: false,
-                    }
-                } else {
-                    Err(anyhow::anyhow!(
-                        "Path is required if no value is not provided"
-                    ))?
-                }
-            };
-            let mut tx = PushIsolationLevel::IsolatedRoot(db.clone(), rsmq);
-            for _ in 0..n {
-                let ehm = HashMap::new();
-                let (uuid, ntx) = push(
-                    &db,
-                    tx,
-                    &w_id,
-                    payload.clone(),
-                    PushArgs::from(&ehm),
-                    authed.display_username(),
-                    &authed.email,
-                    username_to_permissioned_as(&authed.username),
+        "rawscript" => {
+            if let Some(rawscript) = batch_info.rawscript {
+                (
+                    None,
+                    None,
+                    JobKind::Preview,
+                    rawscript.language,
                     None,
                     None,
                     None,
                     None,
                     None,
-                    false,
-                    false,
-                    None,
-                    true,
+                    Some(rawscript.content),
+                    rawscript.lock,
                     None,
                     None,
-                    None,
-                    None,
-                    Some(&authed.clone().into()),
                 )
-                .await?;
-                tx = PushIsolationLevel::Transaction(ntx);
-                uuids.push(uuid);
+            } else {
+                Err(anyhow::anyhow!(
+                    "rawscript is required for `rawscript` kind"
+                ))?
             }
-            match tx {
-                PushIsolationLevel::Transaction(tx) => {
-                    tx.commit().await?;
-                }
-                _ => (),
-            }
-            return Ok(Json(uuids));
+        }
+        "flow" => {
+            let (mut value, job_kind, path) = if let Some(value) = batch_info.flow_value {
+                (value, JobKind::FlowPreview, None)
+            } else if let Some(path) = batch_info.path {
+                let value_json = sqlx::query!(
+                    "SELECT flow_version.value AS \"value: sqlx::types::Json<Box<RawValue>>\" FROM flow 
+                    LEFT JOIN flow_version
+                        ON flow_version.id = flow.versions[array_upper(flow.versions, 1)]
+                    WHERE flow.path = $1 AND flow.workspace_id = $2",
+                    &path, &w_id
+                )
+                .fetch_optional(&db)
+                .await?
+                .ok_or_else(|| Error::InternalErr(format!("not found flow at path {:?}", path)))?;
+                let value =
+                    serde_json::from_str::<FlowValue>(value_json.value.get()).map_err(|err| {
+                        Error::InternalErr(format!(
+                            "could not convert json to flow for {path}: {err:?}"
+                        ))
+                    })?;
+                (value, JobKind::Flow, Some(path))
+            } else {
+                Err(anyhow::anyhow!(
+                    "Path is required if no value is not provided"
+                ))?
+            };
+            add_virtual_items_if_necessary(&mut value.modules);
+            let flow_status = FlowStatus::new(&value);
+            (
+                None,                             // script_hash
+                path,                             // script_path
+                job_kind,                         // job_kind
+                None,                             // language
+                None,                             // dedicated_worker
+                value.concurrency_key.clone(),    // custom_concurrency_key
+                value.concurrent_limit.clone(),   // concurrent_limit
+                value.concurrency_time_window_s,  // concurrency_time_window_s
+                None,                             // timeout
+                None,                             // raw_code
+                None,                             // raw_lock
+                Some(value),                      // raw_flow
+                Some(flow_status),                // flow_status
+            )
         }
         "noop" => (
             None,
@@ -4402,6 +4429,10 @@ async fn add_batch_jobs(
             None,
             None,
             None,
+            None,
+            None,
+            None,
+            None,
         ),
         _ => {
             return Err(error::Error::BadRequest(format!(
@@ -4428,8 +4459,8 @@ async fn add_batch_jobs(
             select gen_random_uuid() as uuid from generate_series(1, $11)
         )
         INSERT INTO queue 
-            (id, script_hash, script_path, job_kind, language, args, tag, created_by, permissioned_as, email, scheduled_for, workspace_id, concurrent_limit, concurrency_time_window_s, timeout)
-            (SELECT uuid, $1, $2, $3, $4, ('{ "uuid": "' || uuid || '" }')::jsonb, $5, $6, $7, $8, $9, $10, $12, $13, $14 FROM uuid_table) 
+            (id, script_hash, script_path, job_kind, language, args, tag, created_by, permissioned_as, email, scheduled_for, workspace_id, concurrent_limit, concurrency_time_window_s, timeout, raw_code, raw_lock, raw_flow, flow_status)
+            (SELECT uuid, $1, $2, $3, $4, ('{ "uuid": "' || uuid || '" }')::jsonb, $5, $6, $7, $8, $9, $10, $12, $13, $14, $15, $16, $17, $18 FROM uuid_table) 
         RETURNING id"#,
             hash.map(|h| h.0),
             path,
@@ -4444,7 +4475,11 @@ async fn add_batch_jobs(
             n,
             concurrent_limit,
             concurrent_time_window_s,
-            timeout
+            timeout,
+            raw_code,
+            raw_lock,
+            raw_flow.map(sqlx::types::Json) as Option<sqlx::types::Json<FlowValue>>,
+            flow_status.map(sqlx::types::Json) as Option<sqlx::types::Json<FlowStatus>>
         )
         .fetch_all(&db)
         .await?;

backend/windmill-api/src/lib.rs

@@ -154,18 +154,18 @@ pub async fn add_webhook_allowed_origin(
 type IndexReader = ();
 
 #[cfg(not(feature = "tantivy"))]
-type IndexWriter = ();
+type ServiceLogIndexReader = ();
 
 #[cfg(feature = "tantivy")]
-type IndexReader = windmill_indexer::indexer_ee::IndexReader;
+type IndexReader = windmill_indexer::completed_runs_ee::IndexReader;
 #[cfg(feature = "tantivy")]
-type IndexWriter = windmill_indexer::indexer_ee::IndexWriter;
+type ServiceLogIndexReader = windmill_indexer::service_logs_ee::ServiceLogIndexReader;
 
 pub async fn run_server(
     db: DB,
     rsmq: Option<rsmq_async::MultiplexedRsmq>,
-    index_reader: Option<IndexReader>,
-    index_writer: Option<IndexWriter>,
+    job_index_reader: Option<IndexReader>,
+    log_index_reader: Option<ServiceLogIndexReader>,
     addr: SocketAddr,
     mut rx: tokio::sync::broadcast::Receiver<()>,
     port_tx: tokio::sync::oneshot::Sender<String>,
@@ -205,8 +205,9 @@ pub async fn run_server(
         .layer(Extension(rsmq.clone()))
         .layer(Extension(user_db.clone()))
         .layer(Extension(auth_cache.clone()))
-        .layer(Extension(index_reader))
-        .layer(Extension(index_writer))
+        .layer(Extension(job_index_reader))
+        .layer(Extension(log_index_reader))
+        // .layer(Extension(index_writer))
         .layer(CookieManagerLayer::new())
         .layer(Extension(WebhookShared::new(rx.resubscribe(), db.clone())))
         .layer(DefaultBodyLimit::max(
@@ -322,6 +323,10 @@ pub async fn run_server(
                     "/srch/w/:workspace_id/index",
                     indexer_ee::workspaced_service(),
                 )
+                .nest(
+                    "/srch/index",
+                    indexer_ee::global_service(),
+                )
                 .nest("/oidc", oidc_ee::global_service())
                 .nest(
                     "/saml",

backend/windmill-api/src/settings.rs

@@ -58,7 +58,10 @@ pub fn global_service() -> Router {
         )
         .route("/renew_license_key", post(renew_license_key))
         .route("/customer_portal", post(create_customer_portal_session))
-        .route("/test_critical_channels", post(test_critical_channels));
+        .route("/test_critical_channels", post(test_critical_channels))
+        .route("/critical_alerts", get(get_critical_alerts))
+        .route("/critical_alerts/:id/acknowledge", post(acknowledge_critical_alert))
+        .route("/critical_alerts/acknowledge_all", post(acknowledge_all_critical_alerts));
 
     #[cfg(feature = "parquet")]
     {
@@ -430,3 +433,117 @@ pub async fn test_critical_channels(
 pub async fn test_critical_channels() -> Result<String> {
     Ok("Critical channels require EE".to_string())
 }
+
+use serde::Serialize;
+
+#[derive(Serialize)]
+pub struct CriticalAlert {
+    id: i32,
+    alert_type: String,
+    message: String,
+    created_at: chrono::DateTime<chrono::Utc>,
+    acknowledged: Option<bool>,
+}
+
+#[cfg(feature = "enterprise")]
+#[derive(Deserialize)]
+pub struct AlertQueryParams {
+    pub page: Option<i32>,
+    pub page_size: Option<i32>,
+    pub acknowledged: Option<bool>,
+}
+
+#[cfg(feature = "enterprise")]
+pub async fn get_critical_alerts(
+    Extension(db): Extension<DB>,
+    authed: ApiAuthed,
+    Query(params): Query<AlertQueryParams>,
+) -> JsonResult<Vec<CriticalAlert>> {
+    require_super_admin(&db, &authed.email).await?;
+
+    // Default pagination values if not provided
+    let page = params.page.unwrap_or(1).max(1);
+    let page_size = params.page_size.unwrap_or(10).min(100) as i64;
+    let offset = ((page - 1) * page_size as i32) as i64;
+
+    let alerts = if let Some(acknowledged) = params.acknowledged {
+        sqlx::query_as!(
+            CriticalAlert,
+            "SELECT id, alert_type, message, created_at, acknowledged 
+             FROM alerts 
+             WHERE acknowledged = $1
+             ORDER BY created_at DESC 
+             LIMIT $2 OFFSET $3",
+            acknowledged,
+            page_size,
+            offset
+        )
+        .fetch_all(&db)
+        .await?
+    } else {
+        sqlx::query_as!(
+            CriticalAlert,
+            "SELECT id, alert_type, message, created_at, acknowledged 
+             FROM alerts 
+             ORDER BY created_at DESC 
+             LIMIT $1 OFFSET $2",
+            page_size,
+            offset
+        )
+        .fetch_all(&db)
+        .await?
+    };
+
+    Ok(Json(alerts))
+}
+
+#[cfg(not(feature = "enterprise"))]
+pub async fn get_critical_alerts() -> error::Error {
+    error::Error::NotFound("Critical Alerts require EE".to_string())
+}
+
+#[cfg(feature = "enterprise")]
+pub async fn acknowledge_critical_alert(
+    Extension(db): Extension<DB>,
+    authed: ApiAuthed,
+    Path(id): Path<i32>,
+) -> error::Result<String> {
+    require_super_admin(&db, &authed.email).await?;
+    
+    sqlx::query!(
+        "UPDATE alerts SET acknowledged = true WHERE id = $1",
+        id
+    )
+    .execute(&db)
+    .await?;
+    
+    tracing::info!("Acknowledged critical alert with id: {}", id);
+    Ok("Critical alert acknowledged".to_string())
+}
+
+#[cfg(not(feature = "enterprise"))]
+pub async fn acknowledge_critical_alert() -> error::Error {
+    error::Error::NotFound("Critical Alerts require EE".to_string())
+}
+
+#[cfg(feature = "enterprise")]
+pub async fn acknowledge_all_critical_alerts(
+    Extension(db): Extension<DB>,
+    authed: ApiAuthed,
+) -> error::Result<String> {
+    require_super_admin(&db, &authed.email).await?;
+
+    sqlx::query!(
+        "UPDATE alerts SET acknowledged = true WHERE acknowledged = false"
+    )
+    .execute(&db)
+    .await?;
+
+    tracing::info!("Acknowledged all unacknowledged critical alerts");
+    Ok("All unacknowledged critical alerts acknowledged".to_string())
+}
+
+#[cfg(not(feature = "enterprise"))]
+pub async fn acknowledge_all_critical_alerts() -> error::Error {
+    error::Error::NotFound("Critical Alerts require EE".to_string())
+}

backend/windmill-api/src/static_assets.rs

@@ -23,7 +23,7 @@ pub async fn static_handler(OriginalUri(original_uri): OriginalUri) -> StaticFil
 }
 
 #[derive(RustEmbed)]
-#[folder = "../../frontend/build/"]
+#[folder = "${FRONTEND_BUILD_DIR:-../../frontend/build/}"]
 struct Asset;
 pub struct StaticFile(Uri);
 

backend/windmill-api/src/users.rs

@@ -129,7 +129,8 @@ fn username_override_from_label(label: Option<String>) -> Option<String> {
         Some(label)
             if label.starts_with("webhook-")
                 || label.starts_with("http-")
-                || label.starts_with("email-") =>
+                || label.starts_with("email-")
+                || label.starts_with("ws-") =>
         {
             Some(label)
         }
@@ -745,10 +746,20 @@ pub async fn fetch_api_authed(
     username_override: String,
 ) -> error::Result<ApiAuthed> {
     let permissioned_as = username_to_permissioned_as(username.as_str());
+    fetch_api_authed_from_permissioned_as(permissioned_as, email, w_id, db, username_override).await
+}
+
+pub async fn fetch_api_authed_from_permissioned_as(
+    permissioned_as: String,
+    email: String,
+    w_id: &str,
+    db: &DB,
+    username_override: String,
+) -> error::Result<ApiAuthed> {
     let authed =
         fetch_authed_from_permissioned_as(permissioned_as, email.clone(), w_id, db).await?;
     Ok(ApiAuthed {
-        username: username,
+        username: authed.username,
         email: email,
         is_admin: authed.is_admin,
         is_operator: authed.is_operator,

backend/windmill-api/src/websocket_triggers.rs

@@ -1,9 +1,10 @@
+use anyhow::Context;
 use axum::{
     extract::{Path, Query},
     routing::{delete, get, post},
     Extension, Json, Router,
 };
-use futures::StreamExt;
+use futures::{stream::SplitSink, SinkExt, StreamExt};
 use http::StatusCode;
 use itertools::Itertools;
 use rand::seq::SliceRandom;
@@ -11,16 +12,20 @@ use serde::{
     de::{self, MapAccess, Visitor},
     Deserialize, Deserializer, Serialize,
 };
-use serde_json::Value;
+use serde_json::{value::RawValue, Value};
 use sql_builder::{bind::Bind, SqlBuilder};
 use sqlx::prelude::FromRow;
 use std::{collections::HashMap, fmt};
-use tokio_tungstenite::connect_async;
+use tokio::net::TcpStream;
+use tokio_tungstenite::{connect_async, tungstenite::Message, MaybeTlsStream, WebSocketStream};
+use uuid::Uuid;
 use windmill_audit::{audit_ee::audit_log, ActionKind};
 use windmill_common::{
     db::UserDB,
-    error::{self, JsonResult},
-    utils::{not_found_if_none, paginate, require_admin, Pagination, StripPath},
+    error::{self, to_anyhow, JsonResult},
+    utils::{
+        not_found_if_none, paginate, report_critical_error, require_admin, Pagination, StripPath,
+    },
     worker::{to_raw_value, CLOUD_HOSTED},
     INSTANCE_NAME,
 };
@@ -28,9 +33,7 @@ use windmill_queue::PushArgsOwned;
 
 use crate::{
     db::{ApiAuthed, DB},
-    jobs::{
-        run_wait_result_flow_by_path_internal, run_wait_result_script_by_path_internal, RunJobQuery,
-    },
+    jobs::{run_flow_by_path_inner, run_script_by_path_inner, RunJobQuery},
     users::fetch_api_authed,
 };
 
@@ -52,7 +55,29 @@ struct NewWebsocketTrigger {
     script_path: String,
     is_flow: bool,
     enabled: Option<bool>,
-    filters: Vec<serde_json::Value>,
+    filters: Vec<Box<RawValue>>,
+    initial_messages: Vec<Box<RawValue>>,
+    url_runnable_args: Box<RawValue>,
+}
+
+#[derive(Deserialize)]
+struct JsonFilter {
+    key: String,
+    value: serde_json::Value,
+}
+
+#[derive(Deserialize)]
+#[serde(untagged)]
+enum Filter {
+    JsonFilter(JsonFilter),
+}
+
+#[derive(Deserialize)]
+enum InitialMessage {
+    #[serde(rename = "raw_message")]
+    RawMessage(String),
+    #[serde(rename = "runnable_result")]
+    RunnableResult { path: String, args: Box<RawValue>, is_flow: bool },
 }
 
 #[derive(FromRow, Serialize, Clone)]
@@ -70,7 +95,9 @@ pub struct WebsocketTrigger {
     extra_perms: serde_json::Value,
     error: Option<String>,
     enabled: bool,
-    filters: Vec<serde_json::Value>,
+    filters: Vec<sqlx::types::Json<Box<RawValue>>>,
+    initial_messages: Vec<sqlx::types::Json<Box<RawValue>>>,
+    url_runnable_args: sqlx::types::Json<Box<RawValue>>,
 }
 
 #[derive(Deserialize)]
@@ -79,7 +106,9 @@ struct EditWebsocketTrigger {
     url: String,
     script_path: String,
     is_flow: bool,
-    filters: Vec<serde_json::Value>,
+    filters: Vec<Box<RawValue>>,
+    initial_messages: Vec<Box<RawValue>>,
+    url_runnable_args: Box<RawValue>,
 }
 
 #[derive(Deserialize)]
@@ -133,14 +162,13 @@ async fn get_websocket_trigger(
 ) -> error::JsonResult<WebsocketTrigger> {
     let mut tx = user_db.begin(&authed).await?;
     let path = path.to_path();
-    let trigger = sqlx::query_as!(
-        WebsocketTrigger,
+    let trigger = sqlx::query_as::<_, WebsocketTrigger>(
         r#"SELECT *
           FROM websocket_trigger
           WHERE workspace_id = $1 AND path = $2"#,
-        w_id,
-        path,
     )
+    .bind(w_id)
+    .bind(path)
     .fetch_optional(&mut *tx)
     .await?;
     tx.commit().await?;
@@ -164,19 +192,27 @@ async fn create_websocket_trigger(
     require_admin(authed.is_admin, &authed.username)?;
 
     let mut tx = user_db.begin(&authed).await?;
-    sqlx::query_as!(
-        WebsocketTrigger,
-      "INSERT INTO websocket_trigger (workspace_id, path, url, script_path, is_flow, enabled, filters, edited_by, email, edited_at) VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, now()) RETURNING *",
-      w_id,
-      ct.path,
-      ct.url,
-      ct.script_path,
-      ct.is_flow,
-      ct.enabled.unwrap_or(true),
-      &ct.filters,
-      &authed.username,
-      &authed.email
+
+    let filters = ct.filters.into_iter().map(sqlx::types::Json).collect_vec();
+    let initial_messages = ct
+        .initial_messages
+        .into_iter()
+        .map(sqlx::types::Json)
+        .collect_vec();
+    sqlx::query_as::<_, WebsocketTrigger>(
+      "INSERT INTO websocket_trigger (workspace_id, path, url, script_path, is_flow, enabled, filters, initial_messages, url_runnable_args, edited_by, email, edited_at) VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, now()) RETURNING *",
     )
+    .bind(&w_id)
+    .bind(&ct.path)
+    .bind(ct.url)
+    .bind(ct.script_path)
+    .bind(ct.is_flow)
+    .bind(ct.enabled.unwrap_or(true))
+    .bind(filters.as_slice())
+    .bind(initial_messages.as_slice())
+    .bind(sqlx::types::Json(ct.url_runnable_args))
+    .bind(&authed.username)
+    .bind(&authed.email)
     .fetch_one(&mut *tx).await?;
 
     audit_log(
@@ -204,15 +240,24 @@ async fn update_websocket_trigger(
     let path = path.to_path();
     let mut tx = user_db.begin(&authed).await?;
 
+    let filters = ct.filters.into_iter().map(sqlx::types::Json).collect_vec();
+    let initial_messages = ct
+        .initial_messages
+        .into_iter()
+        .map(sqlx::types::Json)
+        .collect_vec();
+
     // important to update server_id, last_server_ping and error to NULL to stop current websocket listener
     sqlx::query!(
-        "UPDATE websocket_trigger SET url = $1, script_path = $2, path = $3, is_flow = $4, filters = $5, edited_by = $6, email = $7, edited_at = now(), server_id = NULL, last_server_ping = NULL, error = NULL
-            WHERE workspace_id = $8 AND path = $9",
+        "UPDATE websocket_trigger SET url = $1, script_path = $2, path = $3, is_flow = $4, filters = $5, initial_messages = $6, url_runnable_args = $7, edited_by = $8, email = $9, edited_at = now(), server_id = NULL, last_server_ping = NULL, error = NULL
+            WHERE workspace_id = $10 AND path = $11",
         ct.url,
         ct.script_path,
         ct.path,
         ct.is_flow,
-        &ct.filters,
+        filters.as_slice() as &[sqlx::types::Json<Box<RawValue>>],
+        initial_messages.as_slice() as &[sqlx::types::Json<Box<RawValue>>],
+        sqlx::types::Json(ct.url_runnable_args) as sqlx::types::Json<Box<RawValue>>,
         &authed.username,
         &authed.email,
         w_id,
@@ -335,8 +380,7 @@ async fn listen_to_unlistened_websockets(
     rsmq: &Option<rsmq_async::MultiplexedRsmq>,
     killpill_rx: &tokio::sync::broadcast::Receiver<()>,
 ) -> () {
-    match sqlx::query_as!(
-        WebsocketTrigger,
+    match sqlx::query_as::<_, WebsocketTrigger>(
         r#"SELECT *
             FROM websocket_trigger
             WHERE enabled IS TRUE AND (server_id IS NULL OR last_server_ping IS NULL OR last_server_ping < now() - interval '15 seconds')"#
@@ -422,8 +466,6 @@ impl<'de, 'a> Visitor<'de> for SupersetVisitor<'a> {
             if key == self.key {
                 // Deserialize the value for the key and check if it's a superset
                 let json_value: Value = map.next_value()?;
-                tracing::info!("json_value: {:?}", json_value);
-                tracing::info!("value_to_check: {:?}", self.value_to_check);
                 return Ok(is_superset(&json_value, self.value_to_check));
             } else {
                 // Skip the value if it's not the one we're interested in
@@ -470,149 +512,429 @@ where
     deserializer.deserialize_map(SupersetVisitor { key, value_to_check })
 }
 
+async fn wait_runnable_result(
+    path: String,
+    is_flow: bool,
+    args: &Box<RawValue>,
+    ws_trigger: &WebsocketTrigger,
+    username_override: String,
+    db: &DB,
+    rsmq: Option<rsmq_async::MultiplexedRsmq>,
+) -> error::Result<String> {
+    let user_db = UserDB::new(db.clone());
+    let authed = fetch_api_authed(
+        ws_trigger.edited_by.clone(),
+        ws_trigger.email.clone(),
+        &ws_trigger.workspace_id,
+        &db,
+        username_override,
+    )
+    .await?;
+
+    let args = serde_json::from_str::<Option<HashMap<String, Box<RawValue>>>>(args.get())
+        .map_err(|e| error::Error::BadRequest(format!("invalid json: {}", e)))?
+        .unwrap_or_else(HashMap::new);
+
+    let label_prefix = Some(format!("ws-{}-", ws_trigger.path));
+    let (_, job_id) = if is_flow {
+        run_flow_by_path_inner(
+            authed,
+            db.clone(),
+            user_db,
+            rsmq.clone(),
+            ws_trigger.workspace_id.clone(),
+            StripPath(path.clone()),
+            RunJobQuery::default(),
+            PushArgsOwned { args, extra: None },
+            label_prefix,
+        )
+        .await?
+    } else {
+        run_script_by_path_inner(
+            authed,
+            db.clone(),
+            user_db,
+            rsmq.clone(),
+            ws_trigger.workspace_id.clone(),
+            StripPath(path.clone()),
+            RunJobQuery::default(),
+            PushArgsOwned { args, extra: None },
+            label_prefix,
+        )
+        .await?
+    };
+
+    let start_time = tokio::time::Instant::now();
+
+    loop {
+        if start_time.elapsed() > tokio::time::Duration::from_secs(300) {
+            return Err(anyhow::anyhow!(
+                "Timed out after 5m waiting for runnable {path} (is_flow: {is_flow}) to complete",
+            )
+            .into());
+        }
+
+        #[derive(sqlx::FromRow)]
+        struct RawResult {
+            result: Option<sqlx::types::Json<Box<RawValue>>>,
+            success: bool,
+        }
+
+        let result = sqlx::query_as::<_, RawResult>(
+            "SELECT result, success FROM completed_job WHERE id = $1 AND workspace_id = $2",
+        )
+        .bind(Uuid::parse_str(&job_id).unwrap())
+        .bind(&ws_trigger.workspace_id)
+        .fetch_optional(db)
+        .await;
+
+        match result {
+            Ok(Some(r)) => {
+                if !r.success {
+                    return Err(anyhow::anyhow!(
+                        "Runnable {path} (is_flow: {is_flow}) failed: {:?}",
+                        r.result
+                    )
+                    .into());
+                } else {
+                    return Ok(r.result.map(|r| r.get().to_owned()).unwrap_or_default());
+                }
+            }
+            Ok(None) => {
+                // not yet done, wait for 5s and check again
+                tokio::time::sleep(tokio::time::Duration::from_secs(5)).await;
+            }
+            Err(err) => {
+                return Err(anyhow::anyhow!(
+                    "Error fetching job result for runnable {path} (is_flow: {is_flow}): {err}",
+                )
+                .into());
+            }
+        }
+    }
+}
+
+async fn send_initial_messages(
+    ws_trigger: &WebsocketTrigger,
+    mut writer: SplitSink<WebSocketStream<MaybeTlsStream<TcpStream>>, Message>,
+    db: &DB,
+    rsmq: Option<rsmq_async::MultiplexedRsmq>,
+) -> error::Result<()> {
+    let initial_messages: Vec<InitialMessage> = ws_trigger
+        .initial_messages
+        .iter()
+        .filter_map(|m| serde_json::from_str(m.get()).ok())
+        .collect_vec();
+
+    for start_message in initial_messages {
+        match start_message {
+            InitialMessage::RawMessage(msg) => {
+                let msg = if msg.starts_with("\"") && msg.ends_with("\"") {
+                    msg[1..msg.len() - 1].to_string()
+                } else {
+                    msg
+                };
+                tracing::info!(
+                    "Sending raw message initial message to websocket {}: {}",
+                    ws_trigger.url,
+                    msg
+                );
+                writer
+                    .send(tokio_tungstenite::tungstenite::Message::Text(msg))
+                    .await
+                    .map_err(to_anyhow)
+                    .with_context(|| "failed to send raw message")?;
+            }
+            InitialMessage::RunnableResult { path, is_flow, args } => {
+                tracing::info!(
+                    "Running runnable {path} (is_flow: {is_flow}) for initial message to websocket {}",
+                    ws_trigger.url,
+                );
+
+                let result = wait_runnable_result(
+                    path.clone(),
+                    is_flow,
+                    &args,
+                    ws_trigger,
+                    "init".to_string(),
+                    db,
+                    rsmq.clone(),
+                )
+                .await?;
+
+                tracing::info!(
+                    "Sending runnable {path} (is_flow: {is_flow}) result to websocket {}",
+                    ws_trigger.url
+                );
+
+                let result = if result.starts_with("\"") && result.ends_with("\"") {
+                    result[1..result.len() - 1].to_string()
+                } else {
+                    result
+                };
+
+                writer
+                    .send(tokio_tungstenite::tungstenite::Message::Text(result))
+                    .await
+                    .map_err(to_anyhow)
+                    .with_context(|| {
+                        format!("Failed to send runnable {path} (is_flow: {is_flow}) result")
+                    })?;
+            }
+        }
+    }
+
+    Ok(())
+}
+
+async fn get_url_from_runnable(
+    path: &str,
+    is_flow: bool,
+    ws_trigger: &WebsocketTrigger,
+    db: &DB,
+    rsmq: Option<rsmq_async::MultiplexedRsmq>,
+) -> error::Result<String> {
+    tracing::info!("Running runnable {path} (is_flow: {is_flow}) to get websocket URL",);
+
+    let result = wait_runnable_result(
+        path.to_string(),
+        is_flow,
+        &ws_trigger.url_runnable_args.0,
+        ws_trigger,
+        "url".to_string(),
+        db,
+        rsmq,
+    )
+    .await?;
+
+    if result.starts_with("\"") && result.ends_with("\"") {
+        Ok(result[1..result.len() - 1].to_string())
+    } else {
+        Err(anyhow::anyhow!("Runnable {path} (is_flow: {is_flow}) did not return a string").into())
+    }
+}
+
+async fn update_ping(db: &DB, ws_trigger: &WebsocketTrigger, error: Option<&str>) -> Option<()> {
+    match sqlx::query_scalar!(
+        "UPDATE websocket_trigger SET last_server_ping = now(), error = $1 WHERE workspace_id = $2 AND path = $3 AND server_id = $4 AND enabled IS TRUE RETURNING 1",
+        error,
+        ws_trigger.workspace_id,
+        ws_trigger.path,
+        *INSTANCE_NAME
+    ).fetch_optional(db).await {
+        Ok(updated) => {
+            if updated.flatten().is_none() {
+                tracing::info!("Websocket {} changed, disabled, or deleted, stopping...", ws_trigger.url); 
+                return None;
+            }
+        },
+        Err(err) => {
+            tracing::warn!("Error updating ping of websocket {}: {:?}", ws_trigger.url, err);
+        }
+    };
+
+    Some(())
+}
+
+async fn loop_ping(db: &DB, ws_trigger: &WebsocketTrigger, error: Option<&str>) -> () {
+    loop {
+        if let None = update_ping(db, ws_trigger, error).await {
+            return;
+        }
+        tokio::time::sleep(tokio::time::Duration::from_secs(5)).await;
+    }
+}
+
+async fn disable_with_error(db: &DB, ws_trigger: &WebsocketTrigger, error: String) {
+    match sqlx::query!(
+        "UPDATE websocket_trigger SET enabled = FALSE, error = $1, server_id = NULL, last_server_ping = NULL WHERE workspace_id = $2 AND path = $3",
+        error,
+        ws_trigger.workspace_id,
+        ws_trigger.path,
+    )
+    .execute(db).await {
+        Ok(_) => {
+            report_critical_error(format!("Disabling websocket {} because of error: {}", ws_trigger.url, error), db.clone()).await;
+        },
+        Err(disable_err) => {
+            report_critical_error(
+                format!("Could not disable websocket {} with err {}, disabling because of error {}", ws_trigger.path, disable_err, error), 
+                db.clone()
+            ).await;
+        }
+    }
+}
+
 async fn listen_to_websocket(
     ws_trigger: WebsocketTrigger,
     db: DB,
     rsmq: Option<rsmq_async::MultiplexedRsmq>,
     mut killpill_rx: tokio::sync::broadcast::Receiver<()>,
 ) -> () {
-    async fn update_ping(db: DB, ws_trigger: &WebsocketTrigger, error: Option<&str>) -> Option<()> {
-        match sqlx::query_scalar!(
-            "UPDATE websocket_trigger SET last_server_ping = now(), error = $1 WHERE workspace_id = $2 AND path = $3 AND server_id = $4 AND enabled IS TRUE RETURNING 1",
-            error,
-            ws_trigger.workspace_id,
-            ws_trigger.path,
-            *INSTANCE_NAME
-        ).fetch_optional(&db).await {
-            Ok(updated) => {
-                if updated.flatten().is_none() {
-                    tracing::info!("Websocket {} changed, disabled, or deleted, stopping...", ws_trigger.url); 
-                    return None;
-                }
-            },
-            Err(err) => {
-                tracing::warn!("Error updating ping of websocket {}: {:?}", ws_trigger.url, err);
-            }
-        };
-
-        Some(())
-    }
+    update_ping(&db, &ws_trigger, Some("Connecting...")).await;
 
     let url = ws_trigger.url.as_str();
 
-    #[derive(Deserialize)]
-    struct JsonFilter {
-        key: String,
-        value: serde_json::Value,
-    }
-
-    #[derive(Deserialize)]
-    #[serde(untagged)]
-    enum Filter {
-        JsonFilter(JsonFilter),
-    }
     let filters: Vec<Filter> = ws_trigger
         .filters
         .iter()
-        .filter_map(|m| serde_json::from_value(m.clone()).ok())
+        .filter_map(|m| serde_json::from_str(m.get()).ok())
         .collect_vec();
 
     loop {
+        let connect_url = if url.starts_with("$") {
+            if url.starts_with("$flow:") || url.starts_with("$script:") {
+                let path = url.splitn(2, ':').nth(1).unwrap();
+                tokio::select! {
+                    biased;
+                    _ = killpill_rx.recv() => {
+                        return;
+                    },
+                    _ = loop_ping(&db, &ws_trigger, Some(
+                        "Waiting on runnable to return websocket URL..."
+                    )) => {
+                        return;
+                    },
+                    url_result = get_url_from_runnable(path, url.starts_with("$flow:"), &ws_trigger, &db, rsmq.clone()) => match url_result {
+                        Ok(url) => url,
+                        Err(err) => {
+                            disable_with_error(
+                                &db,
+                                &ws_trigger,
+                                format!(
+                                    "Error getting websocket URL from runnable after 5 tries: {:?}",
+                                    err
+                                ),
+                            )
+                            .await;
+                            return;
+                        }
+                    },
+                }
+            } else {
+                disable_with_error(
+                    &db,
+                    &ws_trigger,
+                    format!("Invalid websocket runnable path: {}", url),
+                )
+                .await;
+                return;
+            }
+        } else {
+            url.to_string()
+        };
+
         tokio::select! {
             biased;
             _ = killpill_rx.recv() => {
                 return;
             },
-            connection = connect_async(url) => {
+            _ = loop_ping(&db, &ws_trigger, Some("Connecting...")) => {
+                return;
+            },
+            connection = connect_async(connect_url) => {
                 match connection {
                     Ok((ws_stream, _)) => {
                         tracing::info!("Listening to websocket {}", url);
-                        if let None = update_ping(db.clone(), &ws_trigger, None).await {
+                        if let None = update_ping(&db, &ws_trigger, None).await {
                             return;
                         }
+                        let (writer, mut reader) = ws_stream.split();
                         let mut last_ping = tokio::time::Instant::now();
-                        let (_, mut read) = ws_stream.split();
-                        loop {
-                            tokio::select! {
-                                biased;
-                                _ = killpill_rx.recv() => {
-                                    return;
+
+                        tokio::select! {
+                            biased;
+                            _ = killpill_rx.recv() => {
+                                return;
+                            }
+                            _ = async {
+                                if let Err(err) = send_initial_messages(&ws_trigger, writer, &db, rsmq.clone()).await {
+                                    disable_with_error(&db, &ws_trigger, format!("Error sending initial messages: {:?}", err)).await;
+                                } else {
+                                    // if initial messages sent successfully, wait forever
+                                    futures::future::pending::<()>().await;
                                 }
-                                msg = read.next() => {
-                                    if let Some(msg) = msg {
-                                        if last_ping.elapsed() > tokio::time::Duration::from_secs(5) {
-                                            if let None = update_ping(db.clone(), &ws_trigger, None).await {
+                            } => {
+                                // was disabled => exit
+                                return;
+                            },
+                            _ = async {
+                                loop {
+                                    tokio::select! {
+                                        biased;
+                                        msg = reader.next() => {
+                                            if let Some(msg) = msg {
+                                                if last_ping.elapsed() > tokio::time::Duration::from_secs(5) {
+                                                    if let None = update_ping(&db, &ws_trigger, None).await {
+                                                        return;
+                                                    }
+                                                    last_ping = tokio::time::Instant::now();
+                                                }
+                                                match msg {
+                                                    Ok(msg) => {
+                                                        match msg {
+                                                            tokio_tungstenite::tungstenite::Message::Text(text) => {
+                                                                let mut should_handle = true;
+                                                                for filter in &filters {
+                                                                    match filter {
+                                                                        Filter::JsonFilter(JsonFilter { key, value }) => {
+                                                                            let mut deserializer = serde_json::Deserializer::from_str(text.as_str());
+                                                                            should_handle = match is_value_superset(&mut deserializer, key, &value) {
+                                                                                Ok(filter_match) => {
+                                                                                    filter_match
+                                                                                },
+                                                                                Err(err) => {
+                                                                                    tracing::warn!("Error deserializing filter for websocket {}: {:?}", url, err);
+                                                                                    false
+                                                                                }
+                                                                            };
+                                                                        }
+                                                                    }
+                                                                    if !should_handle {
+                                                                        break;
+                                                                    }
+                                                                }
+                                                                if should_handle {
+                                                                    if let Err(err) = run_job(&db, rsmq.clone(), &ws_trigger, text).await {
+                                                                        report_critical_error(format!("Failed to trigger job from websocket {}: {:?}", ws_trigger.url, err), db.clone()).await;
+                                                                    };
+                                                                }
+                                                            },
+                                                            _ => {}
+                                                        }
+                                                    },
+                                                    Err(err) => {
+                                                        tracing::error!("Error reading from websocket {}: {:?}", url, err);
+                                                    }
+                                                }
+                                            } else {
+                                                tracing::error!("Websocket {} closed", url);
+                                                if let None =
+                                                    update_ping(&db, &ws_trigger, Some("Websocket closed")).await
+                                                {
+                                                    return;
+                                                }
+                                                tokio::time::sleep(tokio::time::Duration::from_secs(5)).await;
+                                                break;
+                                            }
+                                        },
+                                        _ = tokio::time::sleep(tokio::time::Duration::from_secs(5)) => {
+                                            if let None = update_ping(&db, &ws_trigger, None).await {
                                                 return;
                                             }
                                             last_ping = tokio::time::Instant::now();
-                                        }
-                                        match msg {
-                                            Ok(msg) => {
-                                                match msg {
-                                                    tokio_tungstenite::tungstenite::Message::Text(text) => {
-                                                        let mut should_handle = true;
-                                                        for filter in &filters {
-                                                            match filter {
-                                                                Filter::JsonFilter(JsonFilter { key, value }) => {
-                                                                    let mut deserializer = serde_json::Deserializer::from_str(text.as_str());
-                                                                    should_handle = match is_value_superset(&mut deserializer, key, &value) {
-                                                                        Ok(filter_match) => {
-                                                                            filter_match
-                                                                        },
-                                                                        Err(err) => {
-                                                                            tracing::warn!("Error deserializing filter for websocket {}: {:?}", url, err);
-                                                                            false
-                                                                        }
-                                                                    };
-                                                                }
-                                                            }
-                                                            if !should_handle {
-                                                                break;
-                                                            }
-                                                        }
-                                                        if should_handle {
-                                                            let db_ = db.clone();
-                                                            let rsmq_ = rsmq.clone();
-                                                            let ws_trigger_ = ws_trigger.clone();
-                                                            tokio::spawn(async move {
-                                                                let url = ws_trigger_.url.clone();
-                                                                if let Err(err) = run_job(db_, rsmq_, ws_trigger_, text).await {
-                                                                    tracing::error!("Error running job on websocket {}: {:?}", url, err);
-                                                                };
-                                                            });
-                                                        }
-                                                    },
-                                                    _ => {}
-                                                }
-                                            },
-                                            Err(err) => {
-                                                tracing::error!("Error reading from websocket {}: {:?}", url, err);
-                                            }
-                                        }
-                                    } else {
-                                        tracing::error!("Websocket {} closed", url);
-                                        if let None =
-                                            update_ping(db.clone(), &ws_trigger, Some("Websocket closed")).await
-                                        {
-                                            return;
-                                        }
-                                        tokio::time::sleep(tokio::time::Duration::from_secs(5)).await;
-                                        break;
+                                        },
                                     }
-                                },
-                                _ = tokio::time::sleep(tokio::time::Duration::from_secs(5)) => {
-                                    if let None = update_ping(db.clone(), &ws_trigger, None).await {
-                                        return;
-                                    }
-                                    last_ping = tokio::time::Instant::now();
-                                },
+                                }
+                            } => {
+                                return;
                             }
-                        }
+                        };
                     }
                     Err(err) => {
                         tracing::error!("Error connecting to websocket {}: {:?}", url, err);
                         if let None =
-                            update_ping(db.clone(), &ws_trigger, Some(err.to_string().as_str())).await
+                            update_ping(&db, &ws_trigger, Some(err.to_string().as_str())).await
                         {
                             return;
                         }
@@ -625,16 +947,18 @@ async fn listen_to_websocket(
 }
 
 async fn run_job(
-    db: DB,
+    db: &DB,
     rsmq: Option<rsmq_async::MultiplexedRsmq>,
-    trigger: WebsocketTrigger,
+    trigger: &WebsocketTrigger,
     msg: String,
 ) -> anyhow::Result<()> {
     let args = PushArgsOwned {
         args: HashMap::from([("msg".to_string(), to_raw_value(&msg))]),
         extra: Some(HashMap::from([(
             "wm_trigger".to_string(),
-            to_raw_value(&serde_json::json!({"kind": "websocket"})),
+            to_raw_value(
+                &serde_json::json!({"kind": "websocket", "websocket": { "url": trigger.url }}),
+            ),
         )])),
     };
     let label_prefix = Some(format!("ws-{}-", trigger.path));
@@ -643,7 +967,7 @@ async fn run_job(
         trigger.edited_by.clone(),
         trigger.email.clone(),
         &trigger.workspace_id,
-        &db,
+        db,
         "anonymous".to_string(),
     )
     .await?;
@@ -653,27 +977,27 @@ async fn run_job(
     let run_query = RunJobQuery::default();
 
     if trigger.is_flow {
-        run_wait_result_flow_by_path_internal(
-            db,
-            run_query,
-            StripPath(trigger.script_path.to_owned()),
+        run_flow_by_path_inner(
             authed,
-            rsmq,
+            db.clone(),
             user_db,
-            args,
+            rsmq,
             trigger.workspace_id.clone(),
+            StripPath(trigger.script_path.to_owned()),
+            run_query,
+            args,
             label_prefix,
         )
         .await?;
     } else {
-        run_wait_result_script_by_path_internal(
-            db,
-            run_query,
-            StripPath(trigger.script_path.to_owned()),
+        run_script_by_path_inner(
             authed,
-            rsmq,
+            db.clone(),
             user_db,
+            rsmq,
             trigger.workspace_id.clone(),
+            StripPath(trigger.script_path.to_owned()),
+            run_query,
             args,
             label_prefix,
         )

backend/windmill-common/src/flows.rs

@@ -426,6 +426,8 @@ pub enum FlowModuleValue {
         hash: Option<ScriptHash>,
         #[serde(skip_serializing_if = "Option::is_none")]
         tag_override: Option<String>,
+        #[serde(skip_serializing_if = "Option::is_none")]
+        is_trigger: Option<bool>,
     },
     Flow {
         #[serde(default)]
@@ -474,6 +476,8 @@ pub enum FlowModuleValue {
         concurrent_limit: Option<i32>,
         #[serde(skip_serializing_if = "Option::is_none")]
         concurrency_time_window_s: Option<i32>,
+        #[serde(skip_serializing_if = "Option::is_none")]
+        is_trigger: Option<bool>,
     },
     Identity,
 }
@@ -505,6 +509,7 @@ struct UntaggedFlowModuleValue {
     custom_concurrency_key: Option<String>,
     concurrent_limit: Option<i32>,
     concurrency_time_window_s: Option<i32>,
+    is_trigger: Option<bool>,
 }
 
 impl<'de> Deserialize<'de> for FlowModuleValue {
@@ -522,6 +527,7 @@ impl<'de> Deserialize<'de> for FlowModuleValue {
                     .ok_or_else(|| serde::de::Error::missing_field("path"))?,
                 hash: untagged.hash,
                 tag_override: untagged.tag_override,
+                is_trigger: untagged.is_trigger,
             }),
             "flow" => Ok(FlowModuleValue::Flow {
                 input_transforms: untagged.input_transforms.unwrap_or_default(),
@@ -574,6 +580,7 @@ impl<'de> Deserialize<'de> for FlowModuleValue {
                 custom_concurrency_key: untagged.custom_concurrency_key,
                 concurrent_limit: untagged.concurrent_limit,
                 concurrency_time_window_s: untagged.concurrency_time_window_s,
+                is_trigger: untagged.is_trigger,
             }),
             "identity" => Ok(FlowModuleValue::Identity),
             other => Err(serde::de::Error::unknown_variant(

backend/windmill-common/src/global_settings.rs

@@ -29,6 +29,7 @@ pub const AUTOMATE_USERNAME_CREATION_SETTING: &str = "automate_username_creation
 pub const HUB_BASE_URL_SETTING: &str = "hub_base_url";
 pub const HUB_ACCESSIBLE_URL_SETTING: &str = "hub_accessible_url";
 pub const CRITICAL_ERROR_CHANNELS_SETTING: &str = "critical_error_channels";
+pub const CRITICAL_ALERT_MUTE_UI_SETTING: &str = "critical_alert_mute_ui";
 pub const DEV_INSTANCE_SETTING: &str = "dev_instance";
 pub const JWT_SECRET_SETTING: &str = "jwt_secret";
 pub const EMAIL_DOMAIN_SETTING: &str = "email_domain";

backend/windmill-common/src/lib.rs

@@ -86,6 +86,8 @@ lazy_static::lazy_static! {
     pub static ref METRICS_ENABLED: AtomicBool = AtomicBool::new(std::env::var("METRICS_PORT").is_ok() || std::env::var("METRICS_ADDR").is_ok());
     pub static ref METRICS_DEBUG_ENABLED: AtomicBool = AtomicBool::new(false);
 
+    pub static ref CRITICAL_ALERT_MUTE_UI_ENABLED: AtomicBool = AtomicBool::new(false);
+
     pub static ref BASE_URL: Arc<RwLock<String>> = Arc::new(RwLock::new("".to_string()));
     pub static ref IS_READY: std::sync::atomic::AtomicBool = std::sync::atomic::AtomicBool::new(false);
 

backend/windmill-common/src/s3_helpers.rs

@@ -109,10 +109,13 @@ pub struct S3AwsOidcResource {
     pub audience: Option<String>,
 }
 
-#[derive(Deserialize, Clone)]
+#[derive(Serialize, Deserialize, Clone)]
 pub struct S3Object {
     pub s3: String,
+    #[serde(skip_serializing_if = "Option::is_none")]
     pub storage: Option<String>,
+    #[serde(skip_serializing_if = "Option::is_none")]
+    pub filename: Option<String>,
 }
 
 #[cfg(feature = "parquet")]

backend/windmill-common/src/server.rs

@@ -10,6 +10,7 @@ pub struct Smtp {
     pub port: u16,
     pub from: String,
     pub tls_implicit: Option<bool>,
+    pub disable_tls: Option<bool>,
 }
 
 #[derive(Serialize, Deserialize, PartialEq)]
@@ -20,6 +21,7 @@ pub struct SmtpConfigOpt {
     pub smtp_port: Option<u16>,
     pub smtp_from: Option<String>,
     pub smtp_tls_implicit: Option<bool>,
+    pub smtp_disable_tls: Option<bool>,
 }
 
 pub async fn load_smtp_config(db: &DB) -> error::Result<Option<Smtp>> {
@@ -39,6 +41,7 @@ pub async fn load_smtp_config(db: &DB) -> error::Result<Option<Smtp>> {
             username,
             password,
             tls_implicit: config.smtp_tls_implicit,
+            disable_tls: config.smtp_disable_tls,
             port: config.smtp_port.unwrap_or(587),
             from: config
                 .smtp_from
@@ -60,6 +63,9 @@ pub async fn load_smtp_config(db: &DB) -> error::Result<Option<Smtp>> {
                 tls_implicit: std::env::var("SMTP_TLS_IMPLICIT")
                     .ok()
                     .and_then(|p| p.parse().ok()),
+                disable_tls: std::env::var("SMTP_DISABLE_TLS")
+                    .ok()
+                    .and_then(|p| p.parse().ok()),
                 port: std::env::var("SMTP_PORT")
                     .ok()
                     .and_then(|p| p.parse().ok())
@@ -87,6 +93,7 @@ impl Default for SmtpConfigOpt {
             smtp_port: None,
             smtp_tls_implicit: None,
             smtp_username: None,
+            smtp_disable_tls: None,
         }
     }
 }

backend/windmill-common/src/utils.rs

@@ -28,6 +28,9 @@ pub const DEFAULT_PER_PAGE: usize = 1000;
 pub const GIT_VERSION: &str =
     git_version!(args = ["--tag", "--always"], fallback = "unknown-version");
 
+use std::sync::atomic::Ordering;
+use crate::CRITICAL_ALERT_MUTE_UI_ENABLED;
+
 lazy_static::lazy_static! {
     pub static ref HTTP_CLIENT: Client = reqwest::ClientBuilder::new()
         .user_agent("windmill/beta")
@@ -244,9 +247,12 @@ pub fn generate_lock_id(database_name: &str) -> i64 {
 pub async fn report_critical_error(error_message: String, _db: DB) -> () {
     tracing::error!("CRITICAL ERROR: {error_message}");
 
+    let mute = CRITICAL_ALERT_MUTE_UI_ENABLED.load(Ordering::Relaxed);
+
     if let Err(err) = sqlx::query!(
-        "INSERT INTO alerts (alert_type, message) VALUES ('critical_error', $1)",
-        error_message
+        "INSERT INTO alerts (alert_type, message, acknowledged) VALUES ('critical_error', $1, $2)",
+        error_message,
+        mute
     )
     .execute(&_db)
     .await
@@ -261,9 +267,12 @@ pub async fn report_critical_error(error_message: String, _db: DB) -> () {
 pub async fn report_recovered_critical_error(message: String, _db: DB) -> () {
     tracing::info!("RECOVERED CRITICAL ERROR: {message}");
 
+    let mute = CRITICAL_ALERT_MUTE_UI_ENABLED.load(Ordering::Relaxed);
+
     if let Err(err) = sqlx::query!(
-        "INSERT INTO alerts (alert_type, message) VALUES ('recovered_critical_error', $1)",
-        message
+        "INSERT INTO alerts (alert_type, message, acknowledged) VALUES ('recovered_critical_error', $1, $2)",
+        message,
+        mute
     )
     .execute(&_db)
     .await

backend/windmill-indexer/src/completed_runs_ee.rs

@@ -0,0 +1,21 @@
+use sqlx::{Pool, Postgres};
+use windmill_common::error::Error;
+use anyhow::anyhow;
+
+#[derive(Clone)]
+pub struct IndexReader;
+
+#[derive(Clone)]
+pub struct IndexWriter;
+
+pub async fn init_index() -> Result<(IndexReader, IndexWriter), Error> {
+    Err(anyhow!("Cannot initialize index: not in EE").into())
+}
+
+pub async fn run_indexer(
+    _db: Pool<Postgres>,
+    mut _index_writer: IndexWriter,
+    mut _killpill_rx: tokio::sync::broadcast::Receiver<()>,
+) {
+    tracing::error!("Cannot run indexer: not in EE");
+}

backend/windmill-indexer/src/indexer_ee.rs

@@ -1,21 +0,0 @@
-use anyhow::anyhow;
-use sqlx::{Pool, Postgres};
-use windmill_common::error::Error;
-
-#[derive(Clone)]
-pub struct IndexReader;
-
-#[derive(Clone)]
-pub struct IndexWriter;
-
-pub async fn init_index() -> Result<(IndexReader, IndexWriter), Error> {
-    Err(anyhow!("Cannot initialize index: not in EE").into())
-}
-
-pub async fn run_indexer(
-    _db: Pool<Postgres>,
-    mut _index_writer: IndexWriter,
-    mut _killpill_rx: tokio::sync::broadcast::Receiver<()>,
-) {
-    tracing::error!("Cannot run indexer: not in EE");
-}

backend/windmill-indexer/src/lib.rs

@@ -1 +1,3 @@
+pub mod completed_runs_ee;
+pub mod service_logs_ee;
 pub mod indexer_ee;

backend/windmill-indexer/src/service_logs_ee.rs

@@ -0,0 +1,21 @@
+use sqlx::{Pool, Postgres};
+use windmill_common::error::Error;
+use anyhow::anyhow;
+
+#[derive(Clone)]
+pub struct IndexReader;
+
+#[derive(Clone)]
+pub struct IndexWriter;
+
+pub async fn init_index() -> Result<(IndexReader, IndexWriter), Error> {
+    Err(anyhow!("Cannot initialize index: not in EE").into())
+}
+
+pub async fn run_indexer(
+    _db: Pool<Postgres>,
+    mut _index_writer: IndexWriter,
+    mut _killpill_rx: tokio::sync::broadcast::Receiver<()>,
+) {
+    tracing::error!("Cannot run indexer: not in EE");
+}

backend/windmill-queue/src/jobs.rs

@@ -3522,6 +3522,7 @@ pub async fn push<'c, 'd, R: rsmq_async::RsmqConnection + Send + 'c>(
                             path: path.clone(),
                             hash: Some(hash),
                             tag_override: tag_override,
+                            is_trigger: None,
                         },
                     ),
                     stop_after_if: None,

backend/windmill-worker/src/bash_executor.rs

@@ -89,8 +89,18 @@ cat bp | tail -1 >> ./result2.out &
 # Run main.sh in the same process group
 {bash} ./main.sh "$@" 2>&1 | tee bp &
 
-# Wait for all background processes to finish
-wait
+pid=$!
+
+# Wait for main.sh to finish and capture its exit status
+wait $pid
+exit_status=$?
+
+# Clean up the named pipe and background processes
+rm -f bp
+
+
+# Exit with the captured status
+exit $exit_status
 "#,
         bash = BIN_BASH.as_str(),
     );

backend/windmill-worker/src/bun_executor.rs

@@ -1304,6 +1304,8 @@ try {{
                 job.args.as_ref()
             };
 
+            append_logs(&job.id, &job.workspace_id, format!("{init_logs}\n"), db).await;
+
             let result = crate::js_eval::eval_fetch_timeout(
                 env_code,
                 inner_content.clone(),
@@ -1324,14 +1326,7 @@ try {{
                 "Executed native code in {}ms",
                 started_at.elapsed().as_millis()
             );
-            append_logs(
-                &job.id,
-                &job.workspace_id,
-                format!("{}\n{}", init_logs, result.1),
-                db,
-            )
-            .await;
-            return Ok(result.0);
+            return Ok(result);
         }
     }
     append_logs(&job.id, &job.workspace_id, init_logs, db).await;

backend/windmill-worker/src/common.rs

@@ -562,7 +562,7 @@ pub async fn resolve_job_timeout(
     _w_id: &str,
     _job_id: Uuid,
     custom_timeout_secs: Option<i32>,
-) -> (Duration, Option<String>) {
+) -> (Duration, Option<String>, bool) {
     let mut warn_msg: Option<String> = None;
     #[cfg(feature = "cloud")]
     let cloud_premium_workspace = *CLOUD_HOSTED
@@ -587,12 +587,12 @@ pub async fn resolve_job_timeout(
         Some(timeout_secs)
             if Duration::from_secs(timeout_secs as u64) < global_max_timeout_duration =>
         {
-            (Duration::from_secs(timeout_secs as u64), warn_msg)
+            (Duration::from_secs(timeout_secs as u64), warn_msg, true)
         }
         Some(timeout_secs) => {
             warn_msg = Some(format!("WARNING: Custom job timeout of {timeout_secs} seconds was greater than the maximum timeout. It will be ignored and the max timeout will be used instead"));
             tracing::warn!(warn_msg);
-            (global_max_timeout_duration, warn_msg)
+            (global_max_timeout_duration, warn_msg, false)
         }
         None => {
             // fallback to default timeout or max if not set
@@ -610,7 +610,7 @@ pub async fn resolve_job_timeout(
                     global_max_timeout_duration
                 }
             };
-            (default_timeout, warn_msg)
+            (default_timeout, warn_msg, false)
         }
     }
 }
@@ -799,6 +799,7 @@ pub async fn get_cached_resource_value_if_valid(
                         S3Object {
                             s3: s3_file_key.clone(),
                             storage: cached_resource.storage.clone(),
+                            filename: None,
                         },
                     )
                     .await;

backend/windmill-worker/src/handle_child.rs

@@ -143,15 +143,40 @@ pub async fn handle_child(
         occupancy_metrics,
     );
 
-    #[derive(PartialEq, Debug)]
     enum KillReason {
         TooManyLogs,
-        Timeout,
-        Cancelled,
+        Timeout { is_job_specific: bool },
+        Cancelled(Option<CanceledBy>),
         AlreadyCompleted,
     }
 
-    let (timeout_duration, timeout_warn_msg) =
+    impl std::fmt::Debug for KillReason {
+        fn fmt(&self, f: &mut std::fmt::Formatter) -> std::fmt::Result {
+            match self {
+                KillReason::TooManyLogs => f.write_str("too many logs (max size: 2MB)"),
+                KillReason::Timeout { is_job_specific } => f.write_str(if *is_job_specific {
+                    "timeout after exceeding job-specific duration limit"
+                } else {
+                    "timeout after exceeding instance-wide job duration limit"
+                }),
+                KillReason::Cancelled(canceled_by) => {
+                    let mut reason = "cancelled".to_string();
+                    if let Some(canceled_by) = canceled_by {
+                        if let Some(by) = canceled_by.username.as_ref() {
+                            reason.push_str(&format!(" by {}", by));
+                        }
+                        if let Some(rsn) = canceled_by.reason.as_ref() {
+                            reason.push_str(&format!(" (reason: {})", rsn));
+                        }
+                    }
+                    f.write_str(&reason)
+                }
+                KillReason::AlreadyCompleted => f.write_str("already completed"),
+            }
+        }
+    }
+
+    let (timeout_duration, timeout_warn_msg, is_job_specific) =
         resolve_job_timeout(&db, w_id, job_id, custom_timeout).await;
     if let Some(msg) = timeout_warn_msg {
         append_logs(&job_id, w_id, msg.as_str(), db).await;
@@ -165,9 +190,9 @@ pub async fn handle_child(
             biased;
             result = child.wait() => return result.map(Ok),
             Ok(()) = too_many_logs.changed() => KillReason::TooManyLogs,
-            _ = sleep(timeout_duration) => KillReason::Timeout,
+            _ = sleep(timeout_duration) => KillReason::Timeout { is_job_specific },
             ex = update_job, if job_id != Uuid::nil() => match ex {
-                UpdateJobPollingExit::Done => KillReason::Cancelled,
+                UpdateJobPollingExit::Done(canceled_by) => KillReason::Cancelled(canceled_by),
                 UpdateJobPollingExit::AlreadyCompleted => KillReason::AlreadyCompleted,
             },
         };
@@ -175,7 +200,7 @@ pub async fn handle_child(
         drop(tx);
 
         let set_reason = async {
-            if kill_reason == KillReason::Timeout {
+            if matches!(kill_reason, KillReason::Timeout { .. }) {
                 if let Err(err) = sqlx::query(
                     r#"
                        UPDATE queue
@@ -402,7 +427,7 @@ pub async fn handle_child(
                 Err(Error::AlreadyCompleted("Job already completed".to_string()))
             }
             _ => Err(Error::ExecutionErr(format!(
-                "job process killed because {kill_reason:#?}"
+                "job process terminated due to {kill_reason:#?}"
             ))),
         },
         Err(err) => Err(Error::ExecutionErr(format!("job process io error: {err}"))),
@@ -505,7 +530,10 @@ where
         })?,
         ex = update_job, if job_id != Uuid::nil() => {
             match ex {
-                UpdateJobPollingExit::Done => Err(Error::ExecutionErr("Job cancelled".to_string())).map_err(to_anyhow)?,
+                UpdateJobPollingExit::Done(canceled_by) => {
+                    let (by, reason) = canceled_by.as_ref().map_or(("unknown".to_string(), "unknown".to_string()), |x| (x.username.clone().unwrap_or("".to_string()), x.reason.clone().unwrap_or("".to_string())));
+                    Err(Error::ExecutionErr(format!("Job cancelled by {by} (reason: {reason})",))).map_err(to_anyhow)?
+                },
                 UpdateJobPollingExit::AlreadyCompleted => Err(Error::AlreadyCompleted("Job already completed".to_string())).map_err(to_anyhow)?,
             }
         }
@@ -515,7 +543,7 @@ where
 }
 
 pub enum UpdateJobPollingExit {
-    Done,
+    Done(Option<CanceledBy>),
     AlreadyCompleted,
 }
 
@@ -640,7 +668,7 @@ where
     }
     tracing::info!("job {job_id} finished");
 
-    UpdateJobPollingExit::Done
+    UpdateJobPollingExit::Done(canceled_by_ref.clone())
 }
 
 /// takes stdout and stderr from Child, panics if either are not present

backend/windmill-worker/src/job_logger.rs

@@ -1,15 +1,6 @@
-use itertools::Itertools;
-use swc_ecma_parser::lexer::util::CharExt;
-
-#[cfg(all(feature = "enterprise", feature = "parquet"))]
-use object_store::path::Path;
 use regex::Regex;
 
-#[cfg(all(feature = "enterprise", feature = "parquet"))]
-use windmill_common::s3_helpers::OBJECT_STORE_CACHE_SETTINGS;
-
-use windmill_common::error::{self};
-use windmill_common::worker::{CLOUD_HOSTED, TMP_DIR};
+use windmill_common::worker::CLOUD_HOSTED;
 
 use windmill_queue::append_logs;
 
@@ -19,6 +10,12 @@ use std::sync::Arc;
 use uuid::Uuid;
 use windmill_common::DB;
 
+use crate::job_logger_ee::default_disk_log_storage;
+
+#[cfg(all(feature = "enterprise", feature = "parquet"))]
+use crate::job_logger_ee::s3_storage;
+
+
 pub enum CompactLogs {
     #[cfg(not(all(feature = "enterprise", feature = "parquet")))]
     NotEE,
@@ -28,150 +25,7 @@ pub enum CompactLogs {
     S3,
 }
 
-async fn compact_logs(
-    job_id: Uuid,
-    w_id: &str,
-    db: &DB,
-    nlogs: String,
-    total_size: Arc<AtomicU32>,
-    compact_kind: CompactLogs,
-    _worker_name: &str,
-) -> error::Result<(String, String)> {
-    let mut prev_logs = sqlx::query_scalar!(
-        "SELECT logs FROM job_logs WHERE job_id = $1 AND workspace_id = $2",
-        job_id,
-        w_id
-    )
-    .fetch_optional(db)
-    .await?
-    .flatten()
-    .unwrap_or_default();
-    let size = prev_logs.char_indices().count() as i32;
-    let nlogs_len = nlogs.char_indices().count();
-    let to_keep_in_db = usize::max(
-        usize::min(nlogs_len, 3000),
-        nlogs_len % LARGE_LOG_THRESHOLD_SIZE,
-    );
-    let extra_split = to_keep_in_db < nlogs_len;
-    let stored_in_storage_len = if extra_split {
-        nlogs_len - to_keep_in_db
-    } else {
-        0
-    };
-    let extra_to_newline = nlogs
-        .chars()
-        .skip(stored_in_storage_len)
-        .find_position(|x| x.is_line_break())
-        .map(|(i, _)| i)
-        .unwrap_or(to_keep_in_db);
-    let stored_in_storage_to_newline = stored_in_storage_len + extra_to_newline;
 
-    let (append_to_storage, stored_in_db) = if extra_split {
-        if stored_in_storage_to_newline == nlogs.len() {
-            (nlogs.as_ref(), "".to_string())
-        } else {
-            let split_idx = nlogs
-                .char_indices()
-                .nth(stored_in_storage_to_newline)
-                .map(|(i, _)| i)
-                .unwrap_or(0);
-            let (append_to_storage, stored_in_db) = nlogs.split_at(split_idx);
-            // tracing::error!("{append_to_storage} ||||| {stored_in_db}");
-            // tracing::error!(
-            //     "{:?} {:?} {} {}",
-            //     excess_prev_logs.lines().last(),
-            //     current_logs.lines().next(),
-            //     split_idx,
-            //     excess_size_modulo
-            // );
-            (append_to_storage, stored_in_db.to_string())
-        }
-    } else {
-        // tracing::error!("{:?}", nlogs.lines().last());
-        ("", nlogs.to_string())
-    };
-
-    let new_size_with_excess = size + stored_in_storage_to_newline as i32;
-
-    let new_size = total_size.fetch_add(
-        new_size_with_excess as u32,
-        std::sync::atomic::Ordering::SeqCst,
-    ) + new_size_with_excess as u32;
-
-    let path = format!(
-        "logs/{job_id}/{}_{new_size}.txt",
-        chrono::Utc::now().timestamp_millis()
-    );
-
-    let mut new_current_logs = match compact_kind {
-        CompactLogs::NoS3 => format!("\n[windmill] No object storage set in instance settings. Previous logs have been saved to disk at {path}"),
-        CompactLogs::S3 => format!("\n[windmill] Previous logs have been saved to object storage at {path}"),
-        #[cfg(not(all(feature = "enterprise", feature = "parquet")))]
-        CompactLogs::NotEE => format!("\n[windmill] Previous logs have been saved to disk at {path}"),
-    };
-    new_current_logs.push_str(&stored_in_db);
-
-    sqlx::query!(
-        "UPDATE job_logs SET logs = $1, log_offset = $2, 
-        log_file_index = array_append(coalesce(log_file_index, array[]::text[]), $3) 
-        WHERE workspace_id = $4 AND job_id = $5",
-        new_current_logs,
-        new_size as i32,
-        path,
-        w_id,
-        job_id
-    )
-    .execute(db)
-    .await?;
-    prev_logs.push_str(&append_to_storage);
-
-    return Ok((prev_logs, path));
-}
-
-async fn default_disk_log_storage(
-    job_id: Uuid,
-    w_id: &str,
-    db: &DB,
-    nlogs: String,
-    total_size: Arc<AtomicU32>,
-    compact_kind: CompactLogs,
-    worker_name: &str,
-) {
-    match compact_logs(
-        job_id,
-        &w_id,
-        &db,
-        nlogs,
-        total_size,
-        compact_kind,
-        worker_name,
-    )
-    .await
-    {
-        Err(e) => tracing::error!("Could not compact logs for job {job_id}: {e:?}",),
-        Ok((prev_logs, path)) => {
-            let path = format!("{}/{}", TMP_DIR, path);
-            let splitted = &path.split("/").collect_vec();
-            tokio::fs::create_dir_all(splitted.into_iter().take(splitted.len() - 1).join("/"))
-                .await
-                .map_err(|e| {
-                    tracing::error!("Could not create logs directory: {e:?}",);
-                    e
-                })
-                .ok();
-            let created = tokio::fs::File::create(&path).await;
-            if let Err(e) = created {
-                tracing::error!("Could not create logs file {path}: {e:?}",);
-                return;
-            }
-            if let Err(e) = tokio::fs::write(&path, prev_logs).await {
-                tracing::error!("Could not write to logs file {path}: {e:?}");
-            } else {
-                tracing::info!("Logs length of {job_id} has exceeded a threshold. Previous logs have been saved to disk at {path}");
-            }
-        }
-    }
-}
 
 pub(crate) async fn append_job_logs(
     job_id: Uuid,
@@ -184,43 +38,7 @@ pub(crate) async fn append_job_logs(
 ) -> () {
     if must_compact_logs {
         #[cfg(all(feature = "enterprise", feature = "parquet"))]
-        if let Some(os) = OBJECT_STORE_CACHE_SETTINGS.read().await.clone() {
-            match compact_logs(
-                job_id,
-                &w_id,
-                &db,
-                logs,
-                total_size,
-                CompactLogs::S3,
-                &worker_name,
-            )
-            .await
-            {
-                Err(e) => tracing::error!("Could not compact logs for job {job_id}: {e:?}",),
-                Ok((prev_logs, path)) => {
-                    tracing::info!("Logs length of {job_id} has exceeded a threshold. Previous logs have been saved to object storage at {path}");
-                    let path2 = path.clone();
-                    if let Err(e) = os
-                        .put(&Path::from(path), prev_logs.to_string().into_bytes().into())
-                        .await
-                    {
-                        tracing::error!("Could not save logs to s3: {e:?}");
-                    }
-                    tracing::info!("Logs of {job_id} saved to object storage at {path2}");
-                }
-            }
-        } else {
-            default_disk_log_storage(
-                job_id,
-                &w_id,
-                &db,
-                logs,
-                total_size,
-                CompactLogs::NoS3,
-                &worker_name,
-            )
-            .await;
-        }
+        s3_storage(job_id, &w_id, &db, &logs, &total_size, &worker_name).await;
 
         #[cfg(not(all(feature = "enterprise", feature = "parquet")))]
         {

backend/windmill-worker/src/job_logger_ee.rs

@@ -0,0 +1,26 @@
+use std::sync::atomic::AtomicU32;
+use std::sync::Arc;
+
+use uuid::Uuid;
+use windmill_common::DB;
+
+use crate::job_logger::CompactLogs;
+
+#[cfg(all(feature = "enterprise", feature = "parquet"))]
+pub(crate) async fn s3_storage(_job_id: Uuid, _w_id: &String, _db: &sqlx::Pool<sqlx::Postgres>, _logs: &String, _total_size: &Arc<AtomicU32>, _worker_name: &String) {
+    tracing::info!("Logs length of {job_id} has exceeded a threshold. Implementation to store excess on s3 in not OSS");
+}
+
+pub(crate) async fn default_disk_log_storage(
+    job_id: Uuid,
+    _w_id: &str,
+    _db: &DB,
+    _nlogs: String,
+    _total_size: Arc<AtomicU32>,
+    _compact_kind: CompactLogs,
+    _worker_name: &str,
+) {
+    tracing::info!("Logs length of {job_id} has exceeded a threshold. Implementation to store excess on disk in not OSS");
+}
+
+

backend/windmill-worker/src/js_eval.rs

@@ -754,9 +754,9 @@ pub async fn eval_fetch_timeout(
     _w_id: &str,
     _load_client: bool,
     _occupation_metrics: &mut OccupancyMetrics,
-) -> anyhow::Result<(Box<RawValue>, String)> {
+) -> anyhow::Result<Box<RawValue>> {
     use serde_json::value::to_raw_value;
-    Ok((to_raw_value("require deno_core").unwrap(), "".to_string()))
+    Ok(to_raw_value("require deno_core").unwrap())
 }
 
 #[cfg(feature = "deno_core")]
@@ -774,7 +774,9 @@ pub async fn eval_fetch_timeout(
     w_id: &str,
     load_client: bool,
     occupation_metrics: &mut OccupancyMetrics,
-) -> anyhow::Result<(Box<RawValue>, String)> {
+) -> anyhow::Result<Box<RawValue>> {
+    use windmill_queue::append_logs;
+
     let (sender, mut receiver) = oneshot::channel::<IsolateHandle>();
 
     let parsed_args = windmill_parser_ts::parse_deno_signature(&ts_expr, true, None)?.args;
@@ -805,6 +807,8 @@ pub async fn eval_fetch_timeout(
         ));
     }
 
+    let db_ = db.clone();
+    let w_id_ = w_id.to_string();
     let result_f = tokio::task::spawn_blocking(move || {
         let ops = vec![op_get_static_args(), op_log()];
         let ext = Extension { name: "windmill", ops: ops.into(), ..Default::default() };
@@ -892,28 +896,31 @@ pub async fn eval_fetch_timeout(
             .build()?;
 
         let future = async {
-            tokio::select! {
+            let r = tokio::select! {
                 r = eval_fetch(&mut js_runtime, &js_expr, Some(env_code), load_client) => Ok(r),
                 _ = memory_limit_rx.recv() => Err(Error::ExecutionErr("Memory limit reached, killing isolate".to_string()))
-            }
+            };
+
+            append_logs(
+                &job_id,
+                w_id_.as_str(),
+                format!(
+                    "{extra_logs}{}",
+                    js_runtime.op_state().borrow().borrow::<LogString>().s
+                ),
+                db_,
+            )
+            .await;
+
+            r
         };
         let r = runtime.block_on(future)?;
         // tracing::info!("total: {:?}", instant.elapsed());
 
-        (r as anyhow::Result<Box<RawValue>>).map(|x| {
-            (
-                x,
-                js_runtime
-                    .op_state()
-                    .borrow()
-                    .borrow::<LogString>()
-                    .s
-                    .clone(),
-            )
-        })
+        r
     });
 
-    let (res, logs) = run_future_with_polling_update_job_poller(
+    let res = run_future_with_polling_update_job_poller(
         job_id,
         job_timeout,
         db,
@@ -932,7 +939,7 @@ pub async fn eval_fetch_timeout(
         e
     })?;
     *mem_peak = (res.get().len() / 1000) as i32;
-    Ok((res, format!("{extra_logs}{logs}")))
+    Ok(res)
 }
 
 #[cfg(feature = "deno_core")]

backend/windmill-worker/src/lib.rs

@@ -29,7 +29,7 @@ mod rust_executor;
 mod worker;
 mod worker_flow;
 mod worker_lockfiles;
-
+mod job_logger_ee;
 pub use worker::*;
 
 pub use result_processor::handle_job_error;

backend/windmill-worker/src/python_executor.rs

@@ -205,8 +205,11 @@ pub async fn uv_pip_compile(
             .clone()
             .map(handle_ephemeral_token);
         if let Some(url) = pip_extra_index_url.as_ref() {
-            args.extend(["--extra-index-url", url, "--no-emit-index-url"]);
-            pip_args.push(format!("--extra-index-url {}", url));
+            url.split(",").for_each(|url| {
+                args.extend(["--extra-index-url", url]);
+                pip_args.push(format!("--extra-index-url {}", url));
+            });
+            args.push("--no-emit-index-url");
         }
         let pip_index_url = PIP_INDEX_URL
             .read()
@@ -285,7 +288,9 @@ pub async fn uv_pip_compile(
             .clone()
             .map(handle_ephemeral_token);
         if let Some(url) = pip_extra_index_url.as_ref() {
-            args.extend(["--extra-index-url", url]);
+            url.split(",").for_each(|url| {
+                args.extend(["--extra-index-url", url]);
+            });
         }
         let pip_index_url = PIP_INDEX_URL
             .read()
@@ -1230,7 +1235,9 @@ pub async fn handle_python_reqs(
                 .map(handle_ephemeral_token);
 
             if let Some(url) = pip_extra_index_url.as_ref() {
-                command_args.extend(["--extra-index-url", url]);
+                url.split(",").for_each(|url| {
+                    command_args.extend(["--extra-index-url", url]);
+                });
             }
             let pip_index_url = PIP_INDEX_URL
                 .read()

backend/windmill-worker/src/result_processor.rs

@@ -295,6 +295,7 @@ pub async fn process_result(
                     message: format!("error during execution of the script:\n{}", err),
                     name: "ExecutionErr".to_string(),
                     step_id: job.flow_step_id.clone(),
+                    exit_code: None,
                 }),
             };
 
@@ -608,6 +609,8 @@ pub struct SerializedError {
     pub name: String,
     #[serde(skip_serializing_if = "Option::is_none")]
     pub step_id: Option<String>,
+    #[serde(skip_serializing_if = "Option::is_none")]
+    pub exit_code: Option<i32>,
 }
 pub fn extract_error_value(log_lines: &str, i: i32, step_id: Option<String>) -> Box<RawValue> {
     return to_raw_value(&SerializedError {
@@ -617,5 +620,6 @@ pub fn extract_error_value(log_lines: &str, i: i32, step_id: Option<String>) ->
         ),
         name: "ExecutionErr".to_string(),
         step_id,
+        exit_code: Some(i),
     });
 }

backend/windmill-worker/src/snowflake_executor.rs

@@ -32,9 +32,9 @@ struct Claims {
 #[derive(Deserialize)]
 struct SnowflakeDatabase {
     account_identifier: String,
-    public_key: String,
-    private_key: String,
-    username: String,
+    public_key: Option<String>,
+    private_key: Option<String>,
+    username: Option<String>,
     database: Option<String>,
     schema: Option<String>,
     warehouse: Option<String>,
@@ -119,6 +119,7 @@ fn do_snowflake_inner<'a>(
     mut body: serde_json::Map<String, Value>,
     account_identifier: &'a str,
     token: &'a str,
+    token_is_keypair: bool,
     column_order: Option<&'a mut Option<Vec<String>>>,
     skip_collect: bool,
 ) -> windmill_common::error::Result<BoxFuture<'a, windmill_common::error::Result<Box<RawValue>>>> {
@@ -144,16 +145,19 @@ fn do_snowflake_inner<'a>(
     }
 
     let result_f = async move {
-        let result = HTTP_CLIENT
+        let mut request = HTTP_CLIENT
             .post(format!(
                 "https://{}.snowflakecomputing.com/api/v2/statements/",
                 account_identifier.to_uppercase()
             ))
             .bearer_auth(token)
-            .header("X-Snowflake-Authorization-Token-Type", "KEYPAIR_JWT")
-            .json(&body)
-            .send()
-            .await;
+            .json(&body);
+
+        if token_is_keypair {
+            request = request.header("X-Snowflake-Authorization-Token-Type", "KEYPAIR_JWT");
+        }
+
+        let result = request.send().await;
 
         if skip_collect {
             handle_snowflake_result(result).await?;
@@ -189,11 +193,17 @@ fn do_snowflake_inner<'a>(
                         account_identifier.to_uppercase(),
                         response.statementHandle
                     );
-                    let response = HTTP_CLIENT
+                    let mut request = HTTP_CLIENT
                         .get(url)
                         .bearer_auth(token)
-                        .header("X-Snowflake-Authorization-Token-Type", "KEYPAIR_JWT")
-                        .query(&[("partition", idx.to_string())])
+                        .query(&[("partition", idx.to_string())]);
+
+                    if token_is_keypair {
+                        request =
+                            request.header("X-Snowflake-Authorization-Token-Type", "KEYPAIR_JWT");
+                    }
+
+                    let response = request
                         .send()
                         .await
                         .parse_snowflake_response::<SnowflakeDataOnlyResponse>()
@@ -258,7 +268,7 @@ pub async fn do_snowflake(
         snowflake_args.get("database").cloned()
     };
 
-    let database = if let Some(db) = db_arg {
+    let database = if let Some(ref db) = db_arg {
         serde_json::from_value::<SnowflakeDatabase>(db.clone())
             .map_err(|e| Error::ExecutionErr(e.to_string()))?
     } else {
@@ -267,37 +277,59 @@ pub async fn do_snowflake(
 
     let annotations = windmill_common::worker::SqlAnnotations::parse(query);
 
-    let qualified_username = format!(
-        "{}.{}",
-        database.account_identifier.split('.').next().unwrap_or(""), // get first part of account identifier
-        database.username
-    )
-    .to_uppercase();
+    // Check if the token is present in db_arg and use it if available
+    let (token, token_is_keypair) = if let Some(token) = db_arg
+        .as_ref()
+        .and_then(|db| db.get("token"))
+        .and_then(|t| t.as_str())
+        .filter(|t| !t.is_empty())
+    {
+        tracing::debug!("Using oauth token from db_arg");
+        (token.to_string(), false)
+    } else {
+        tracing::debug!("Generating new oauth token");
 
-    let public_key = pem::parse(database.public_key.as_bytes()).map_err(|e| {
-        Error::ExecutionErr(format!("Failed to parse public key: {}", e.to_string()))
-    })?;
-    let mut public_key_hash = Sha256::new();
-    public_key_hash.update(public_key.contents());
+        let qualified_username = format!(
+            "{}.{}",
+            database.account_identifier.split('.').next().unwrap_or(""),
+            database.username.as_deref().unwrap_or("")
+        )
+        .to_uppercase();
 
-    let public_key_fp = engine::general_purpose::STANDARD.encode(public_key_hash.finalize());
+        let public_key = match database.public_key.as_deref() {
+            Some(key) => pem::parse(key.as_bytes()).map_err(|e| {
+                Error::ExecutionErr(format!("Failed to parse public key: {}", e.to_string()))
+            })?,
+            None => return Err(Error::ExecutionErr("Public key is missing".to_string())),
+        };
+        let mut public_key_hash = Sha256::new();
+        public_key_hash.update(public_key.contents());
 
-    let iss = format!("{}.SHA256:{}", qualified_username, public_key_fp);
+        let public_key_fp = engine::general_purpose::STANDARD.encode(public_key_hash.finalize());
 
-    let claims = Claims {
-        iss: iss,
-        sub: qualified_username,
-        iat: chrono::Utc::now().timestamp(),
-        exp: (chrono::Utc::now() + chrono::Duration::try_hours(1).unwrap()).timestamp(),
+        let iss = format!("{}.SHA256:{}", qualified_username, public_key_fp);
+
+        let claims = Claims {
+            iss: iss,
+            sub: qualified_username,
+            iat: chrono::Utc::now().timestamp(),
+            exp: (chrono::Utc::now() + chrono::Duration::try_hours(1).unwrap()).timestamp(),
+        };
+
+        let private_key = match database.private_key.as_deref() {
+            Some(key) => EncodingKey::from_rsa_pem(key.as_bytes()).map_err(|e| {
+                Error::ExecutionErr(format!("Failed to parse private key: {}", e.to_string()))
+            })?,
+            None => return Err(Error::ExecutionErr("Private key is missing".to_string())),
+        };
+
+        (
+            encode(&Header::new(Algorithm::RS256), &claims, &private_key)
+                .map_err(|e| Error::ExecutionErr(e.to_string()))?,
+            true,
+        )
     };
 
-    let private_key = EncodingKey::from_rsa_pem(database.private_key.as_bytes()).map_err(|e| {
-        Error::ExecutionErr(format!("Failed to parse private key: {}", e.to_string()))
-    })?;
-
-    let token = encode(&Header::new(Algorithm::RS256), &claims, &private_key)
-        .map_err(|e| Error::ExecutionErr(e.to_string()))?;
-
     tracing::debug!("Snowflake token: {}", token);
 
     let mut body = serde_json::Map::new();
@@ -344,6 +376,7 @@ pub async fn do_snowflake(
                     body.clone(),
                     &database.account_identifier,
                     &token,
+                    token_is_keypair,
                     None,
                     annotations.return_last_result && i < queries.len() - 1,
                 )
@@ -371,6 +404,7 @@ pub async fn do_snowflake(
             body.clone(),
             &database.account_identifier,
             &token,
+            token_is_keypair,
             Some(column_order),
             false,
         )?

backend/windmill-worker/src/worker.rs

@@ -1719,7 +1719,7 @@ async fn do_nativets(
     canceled_by: &mut Option<CanceledBy>,
     worker_name: &str,
     occupancy_metrics: &mut OccupancyMetrics,
-) -> windmill_common::error::Result<(Box<RawValue>, String)> {
+) -> windmill_common::error::Result<Box<RawValue>> {
     let args = build_args_map(job, client, db).await?.map(Json);
     let job_args = if args.is_some() {
         args.as_ref()
@@ -1727,7 +1727,7 @@ async fn do_nativets(
         job.args.as_ref()
     };
 
-    let result = eval_fetch_timeout(
+    Ok(eval_fetch_timeout(
         env_code,
         code.clone(),
         transpile_ts(code)?,
@@ -1742,8 +1742,7 @@ async fn do_nativets(
         true,
         occupancy_metrics,
     )
-    .await?;
-    Ok((result.0, result.1))
+    .await?)
 }
 
 #[derive(Deserialize, Serialize, Default)]
@@ -2372,7 +2371,8 @@ async fn handle_code_execution_job(
                 .map(|(k, v)| format!("const {} = '{}';\nprocess.env['{}'] = '{}';\n", k, v, k, v))
                 .collect::<Vec<String>>()
                 .join("\n"));
-        let (result, ts_logs) = do_nativets(
+
+        let result = do_nativets(
             job,
             &client,
             env_code,
@@ -2384,7 +2384,6 @@ async fn handle_code_execution_job(
             occupancy_metrics,
         )
         .await?;
-        append_logs(&job.id, &job.workspace_id, ts_logs, db).await;
         return Ok(result);
     }
 

backend/windmill-worker/src/worker_lockfiles.rs

@@ -735,6 +735,7 @@ async fn lock_modules<'c>(
             custom_concurrency_key,
             concurrent_limit,
             concurrency_time_window_s,
+            is_trigger,
         } = e.get_value()?
         else {
             match e.get_value()? {
@@ -970,6 +971,7 @@ async fn lock_modules<'c>(
                     custom_concurrency_key,
                     concurrent_limit,
                     concurrency_time_window_s,
+                    is_trigger,
                 });
                 new_flow_modules.push(e);
                 continue;
@@ -992,6 +994,7 @@ async fn lock_modules<'c>(
                     custom_concurrency_key,
                     concurrent_limit,
                     concurrency_time_window_s,
+                    is_trigger,
                 });
                 new_flow_modules.push(e);
                 continue;

benchmarks/benchmark_oneoff.ts

@@ -8,6 +8,7 @@ import { DenoLandProvider } from "https://deno.land/x/cliffy@v0.25.7/command/upg
 import { sleep } from "https://deno.land/x/sleep@v1.2.1/mod.ts";
 
 import * as windmill from "https://deno.land/x/windmill@v1.174.0/mod.ts";
+import * as api from "https://deno.land/x/windmill@v1.174.0/windmill-api/index.ts";
 
 import { VERSION, createBenchScript, getFlowPayload, login } from "./lib.ts";
 
@@ -172,8 +173,8 @@ export async function main({
       kind: "script",
       path: "f/benchmarks/" + kind,
     });
-  } else if (["2steps"].includes(kind)) {
-    nStepsFlow = 2;
+  } else if (["2steps", "bigscriptinflow"].includes(kind)) {
+    nStepsFlow = kind == "2steps" ? 2 : 1;
     const payload = getFlowPayload(kind);
     body = JSON.stringify({
       kind: "flow",
@@ -194,6 +195,15 @@ export async function main({
       kind: "script",
       path: kind.substr(7),
     });
+  } else if (kind == "bigrawscript") {
+    noVerify = true;
+    body = JSON.stringify({
+      kind: "rawscript",
+      rawscript: {
+        language: api.RawScript.language.BASH,
+        content: "# let's bloat that bash script, 3.. 2.. 1.. BOOM\n".repeat(25000) + "echo \"$WM_FLOW_JOB_ID\"\n",
+      },
+    });
   } else {
     throw new Error("Unknown script pattern " + kind);
   }
@@ -246,7 +256,7 @@ export async function main({
     } else {
       const elapsed = start ? Date.now() - start : 0;
       completedJobs = await getCompletedJobsCount();
-      if (kind === "2steps" || kind.startsWith("flow:")) {
+      if (nStepsFlow > 0) {
         completedJobs = Math.floor(completedJobs / (nStepsFlow + 1));
       }
       const avgThr = ((completedJobs / elapsed) * 1000).toFixed(2);

benchmarks/lib.ts

@@ -2,7 +2,7 @@ import { sleep } from "https://deno.land/x/sleep@v1.2.1/mod.ts";
 import * as windmill from "https://deno.land/x/windmill@v1.174.0/mod.ts";
 import * as api from "https://deno.land/x/windmill@v1.174.0/windmill-api/index.ts";
 
-export const VERSION = "v1.416.2";
+export const VERSION = "v1.423.1";
 
 export async function login(email: string, password: string): Promise<string> {
   return await windmill.UserService.login({
@@ -235,6 +235,23 @@ export const getFlowPayload = (flowPattern: string): api.FlowPreview => {
         ],
       },
     };
+  } else if (flowPattern == "bigscriptinflow") {
+    return {
+      path: "bigscriptinflow",
+      args: {},
+      value: {
+        modules: [
+          {
+            value: {
+              input_transforms: {},
+              language: api.RawScript.language.BASH,
+              type: "rawscript",
+              content: "# let's bloat that bash script, 3.. 2.. 1.. BOOM\n".repeat(25000) + "echo \"$WM_FLOW_JOB_ID\"\n",
+            },
+          }
+        ],
+      }
+    };
   } else {
     return {
       path: "2steps",

benchmarks/suite_config.json

@@ -30,5 +30,13 @@
   {
     "kind": "nativets",
     "jobs": 2000
+  },
+  {
+    "kind": "bigrawscript",
+    "jobs": 10000
+  },
+  {
+    "kind": "bigscriptinflow",
+    "jobs": 10000
   }
 ]

cli/instance.ts

@@ -184,6 +184,7 @@ export type InstanceSyncOptions = {
   folderPerInstance?: boolean;
   yes?: boolean;
   prefix?: string;
+  prefixSettings?: boolean;
 };
 
 export async function pickInstance(
@@ -260,30 +261,33 @@ export async function pickInstance(
 
   return instance;
 }
-async function instancePull(opts: GlobalOptions & InstanceSyncOptions) {
+async function instancePull(opts: InstanceSyncOptions) {
   const instance = await pickInstance(opts, true);
   log.info("Pulling instance-level changes");
   log.info(`remote (${instance.name}) -> local`);
 
+  opts.prefix = opts.prefix ?? instance.prefix;
   let uChanges = 0;
   if (!opts.skipUsers) {
-    uChanges = (await pullInstanceUsers(true)) as number;
+    uChanges = (await pullInstanceUsers(opts, true)) as number;
   }
   let sChanges = 0;
   if (!opts.skipSettings) {
-    sChanges = (await pullInstanceSettings(true)) as number;
+    sChanges = (await pullInstanceSettings(opts, true)) as number;
   }
   let cChanges = 0;
   if (!opts.skipConfigs) {
-    cChanges = (await pullInstanceConfigs(true)) as number;
+    cChanges = (await pullInstanceConfigs(opts, true)) as number;
   }
   let gChanges = 0;
   if (!opts.skipGroups) {
-    gChanges = (await pullInstanceGroups(true)) as number;
+    gChanges = (await pullInstanceGroups(opts, true)) as number;
   }
 
   const totalChanges = uChanges + sChanges + cChanges + gChanges;
 
+  const rootDir = Deno.cwd();
+
   if (totalChanges > 0) {
     let confirm = true;
     if (opts.yes !== true) {
@@ -294,17 +298,24 @@ async function instancePull(opts: GlobalOptions & InstanceSyncOptions) {
     }
 
     if (confirm) {
+      if (uChanges > 0) {
+        if (opts.folderPerInstance && opts.prefixSettings) {
+          await Deno.mkdir(path.join(rootDir, opts.prefix), {
+            recursive: true,
+          });
+        }
+      }
       if (!opts.skipUsers && uChanges > 0) {
-        await pullInstanceUsers();
+        await pullInstanceUsers(opts);
       }
       if (!opts.skipSettings && sChanges > 0) {
-        await pullInstanceSettings();
+        await pullInstanceSettings(opts);
       }
       if (!opts.skipConfigs && cChanges > 0) {
-        await pullInstanceConfigs();
+        await pullInstanceConfigs(opts);
       }
       if (!opts.skipGroups && gChanges > 0) {
-        await pullInstanceGroups();
+        await pullInstanceGroups(opts);
       }
     }
   } else {
@@ -313,7 +324,6 @@ async function instancePull(opts: GlobalOptions & InstanceSyncOptions) {
 
   if (opts.includeWorkspaces) {
     log.info("\nPulling all workspaces");
-    const rootDir = Deno.cwd();
     const localWorkspaces = await getLocalWorkspaces(
       rootDir,
       instance.prefix,
@@ -391,7 +401,7 @@ async function instancePull(opts: GlobalOptions & InstanceSyncOptions) {
   }
 }
 
-async function instancePush(opts: GlobalOptions & InstanceSyncOptions) {
+async function instancePush(opts: InstanceSyncOptions) {
   let instances = await allInstances();
   const instance = await pickInstance(opts, true);
 
@@ -400,19 +410,19 @@ async function instancePush(opts: GlobalOptions & InstanceSyncOptions) {
 
   let uChanges = 0;
   if (!opts.skipUsers) {
-    uChanges = (await pushInstanceUsers(true)) as number;
+    uChanges = (await pushInstanceUsers(opts, true)) as number;
   }
   let sChanges = 0;
   if (!opts.skipSettings) {
-    sChanges = (await pushInstanceSettings(true, opts.baseUrl)) as number;
+    sChanges = (await pushInstanceSettings(opts, true)) as number;
   }
   let cChanges = 0;
   if (!opts.skipConfigs) {
-    cChanges = (await pushInstanceConfigs(true)) as number;
+    cChanges = (await pushInstanceConfigs(opts, true)) as number;
   }
   let gChanges = 0;
   if (!opts.skipGroups) {
-    gChanges = (await pushInstanceGroups(true)) as number;
+    gChanges = (await pushInstanceGroups(opts, true)) as number;
   }
 
   const totalChanges = uChanges + sChanges + cChanges + gChanges;
@@ -428,16 +438,16 @@ async function instancePush(opts: GlobalOptions & InstanceSyncOptions) {
 
     if (confirm) {
       if (!opts.skipUsers && uChanges > 0) {
-        await pushInstanceUsers();
+        await pushInstanceUsers(opts);
       }
       if (!opts.skipSettings && sChanges > 0) {
-        await pushInstanceSettings(false, opts.baseUrl);
+        await pushInstanceSettings(opts);
       }
       if (!opts.skipConfigs && cChanges > 0) {
-        await pushInstanceConfigs();
+        await pushInstanceConfigs(opts);
       }
       if (!opts.skipGroups && gChanges > 0) {
-        await pushInstanceGroups();
+        await pushInstanceGroups(opts);
       }
     }
   } else {
@@ -565,13 +575,14 @@ async function getLocalWorkspaces(
   }
   if (folderPerInstance) {
     for await (const dir of Deno.readDir(rootDir + "/" + localPrefix)) {
-      const dirName = dir.name;
-      localWorkspaces.push({
-        dir: localPrefix + "/" + dirName,
-        id: dirName,
-      });
+      if (dir.isDirectory) {
+        const dirName = dir.name;
+        localWorkspaces.push({
+          dir: localPrefix + "/" + dirName,
+          id: dirName,
+        });
+      }
     }
-    log.info(localWorkspaces);
   } else {
     for await (const dir of Deno.readDir(rootDir)) {
       const dirName = dir.name;
@@ -583,6 +594,9 @@ async function getLocalWorkspaces(
       }
     }
   }
+  log.info(
+    "Local workspaces found: " + localWorkspaces.map((x) => x.id).join(", ")
+  );
   return localWorkspaces;
 }
 
@@ -709,7 +723,10 @@ const command = new Command()
     "--prefix <prefix:string>",
     "Prefix of the local workspaces to pull, used to create the folders when using --include-workspaces"
   )
-
+  .option(
+    "--prefix-settings",
+    "Store instance yamls inside prefixed folders when using --prefix and --folder-per-instance"
+  )
   .action(instancePull as any)
   .command("push")
   .description(
@@ -730,6 +747,10 @@ const command = new Command()
     "--prefix <prefix:string>",
     "Prefix of the local workspaces folders to push"
   )
+  .option(
+    "--prefix-settings",
+    "Store instance yamls inside prefixed folders when using --prefix and --folder-per-instance"
+  )
   .action(instancePush as any)
   .command("whoami")
   .description("Display information about the currently logged-in user")

cli/main.ts

@@ -60,7 +60,7 @@ export {
 //   }
 // });
 
-export const VERSION = "1.416.2";
+export const VERSION = "1.423.1";
 
 const command = new Command()
   .name("wmill")

cli/settings.ts

@@ -3,8 +3,8 @@ import { Confirm } from "./deps.ts";
 import { colors } from "./deps.ts";
 import { yamlParseFile } from "./deps.ts";
 import { log } from "./deps.ts";
-import { compareInstanceObjects } from "./instance.ts";
-import { isSuperset } from "./types.ts";
+import { compareInstanceObjects, InstanceSyncOptions } from "./instance.ts";
+import { isSuperset, GlobalOptions } from "./types.ts";
 import { deepEqual } from "./utils.ts";
 import * as wmill from "./gen/services.gen.ts";
 import { Config, GlobalSetting } from "./gen/types.gen.ts";
@@ -33,6 +33,22 @@ export interface SimplifiedSettings {
   name: string;
 }
 
+const INSTANCE_SETTINGS_PATH = "instance_settings.yaml";
+let instanceSettingsPath = INSTANCE_SETTINGS_PATH;
+async function checkInstanceSettingsPath(opts: InstanceSyncOptions) {
+  if (opts.prefix && opts.folderPerInstance && opts.prefixSettings) {
+    instanceSettingsPath = `${opts.prefix}/${INSTANCE_SETTINGS_PATH}`;
+  }
+}
+
+const INSTANCE_CONFIGS_PATH = "instance_configs.yaml";
+let instanceConfigsPath = INSTANCE_CONFIGS_PATH;
+async function checkInstanceConfigPath(opts: InstanceSyncOptions) {
+  if (opts.prefix && opts.folderPerInstance && opts.prefixSettings) {
+    instanceConfigsPath = `${opts.prefix}/${INSTANCE_CONFIGS_PATH}`;
+  }
+}
+
 export async function pushWorkspaceSettings(
   workspace: string,
   _path: string,
@@ -113,10 +129,10 @@ export async function pushWorkspaceSettings(
         workspace,
         requestBody: localSettings.auto_invite_enabled
           ? {
-            operator: localSettings.auto_invite_as === "operator",
-            invite_all: true,
-            auto_add: localSettings.auto_invite_mode === "add",
-          }
+              operator: localSettings.auto_invite_as === "operator",
+              invite_all: true,
+              auto_add: localSettings.auto_invite_mode === "add",
+            }
           : {},
       });
     } catch (_) {
@@ -128,10 +144,10 @@ export async function pushWorkspaceSettings(
         workspace,
         requestBody: localSettings.auto_invite_enabled
           ? {
-            operator: localSettings.auto_invite_as === "operator",
-            invite_all: false,
-            auto_add: localSettings.auto_invite_mode === "add",
-          }
+              operator: localSettings.auto_invite_as === "operator",
+              invite_all: false,
+              auto_add: localSettings.auto_invite_mode === "add",
+            }
           : {},
       });
     }
@@ -156,7 +172,7 @@ export async function pushWorkspaceSettings(
       settings.error_handler_extra_args
     ) ||
     localSettings.error_handler_muted_on_cancel !==
-    settings.error_handler_muted_on_cancel
+      settings.error_handler_muted_on_cancel
   ) {
     log.debug(`Updating error handler...`);
     await wmill.editErrorHandler({
@@ -260,32 +276,36 @@ export async function pushWorkspaceKey(
   }
 }
 
-const INSTANCE_SETTINGS_PATH = "instance_settings.yaml";
-
-export async function readInstanceSettings() {
+export async function readInstanceSettings(opts: InstanceSyncOptions) {
   let localSettings: GlobalSetting[] = [];
 
+  await checkInstanceSettingsPath(opts);
+
   try {
-    localSettings = (await yamlParseFile(INSTANCE_SETTINGS_PATH)) as GlobalSetting[];
+    localSettings = (await yamlParseFile(instanceSettingsPath)) as GlobalSetting[];
   } catch {
-    log.warn(`No ${INSTANCE_SETTINGS_PATH} found`);
+    log.warn(`No ${instanceSettingsPath} found`);
   }
   return localSettings;
 }
 
-
 import { decrypt, encrypt } from "./local_encryption.ts";
 
-const SENSITIVE_FIELD: string[] = ["license_key", "jwt_secret"]
+const SENSITIVE_FIELD: string[] = ["license_key", "jwt_secret"];
 
-async function processInstanceSettings(settings: GlobalSetting[], mode: "encode" | "decode"): Promise<GlobalSetting[]> {
+async function processInstanceSettings(
+  settings: GlobalSetting[],
+  mode: "encode" | "decode"
+): Promise<GlobalSetting[]> {
   const encKey = process.env.WMILL_INSTANCE_LOCAL_ENCRYPTION_KEY;
   if (encKey) {
-    const res: GlobalSetting[] = []
+    const res: GlobalSetting[] = [];
 
     for (const s of settings) {
       if (SENSITIVE_FIELD.includes(s.name) && typeof s.value === "string") {
-        res.push(await processField(s, "value", encKey, mode) as GlobalSetting);
+        res.push(
+          (await processField(s, "value", encKey, mode)) as GlobalSetting
+        );
       } else if (s.name == "oauths") {
         if (typeof s.value === "object") {
           const oauths = s.value as { [key: string]: any };
@@ -303,24 +323,42 @@ async function processInstanceSettings(settings: GlobalSetting[], mode: "encode"
     }
     return res;
   } else {
-    log.warn("No encryption key found, skipping encryption. Recommend setting WMILL_INSTANCE_LOCAL_ENCRYPTION_KEY");
+    log.warn(
+      "No encryption key found, skipping encryption. Recommend setting WMILL_INSTANCE_LOCAL_ENCRYPTION_KEY"
+    );
   }
   return settings;
 }
 
-async function processField(obj: { [key: string]: any }, field: string, encKey: string, mode: "encode" | "decode"): Promise<{ [key: string]: any }> {
+async function processField(
+  obj: { [key: string]: any },
+  field: string,
+  encKey: string,
+  mode: "encode" | "decode"
+): Promise<{ [key: string]: any }> {
   return {
     ...obj,
-    [field]: mode === "encode" ? await encrypt(obj[field], encKey) : await decrypt(obj[field], encKey) as any,
-  }
+    [field]:
+      mode === "encode"
+        ? await encrypt(obj[field], encKey)
+        : ((await decrypt(obj[field], encKey)) as any),
+  };
 }
 
-export async function pullInstanceSettings(preview = false) {
+export async function pullInstanceSettings(
+  opts: InstanceSyncOptions,
+  preview = false
+) {
   const remoteSettings = await wmill.listGlobalSettings();
 
+  await checkInstanceSettingsPath(opts);
+
   if (preview) {
-    const localSettings: GlobalSetting[] = await readInstanceSettings();
-    const processedSettings = await processInstanceSettings(remoteSettings, "encode");
+    const localSettings: GlobalSetting[] = await readInstanceSettings(opts);
+    const processedSettings = await processInstanceSettings(
+      remoteSettings,
+      "encode"
+    );
     return compareInstanceObjects(
       processedSettings,
       localSettings,
@@ -330,30 +368,33 @@ export async function pullInstanceSettings(preview = false) {
   } else {
     log.info("Pulling settings from instance");
 
-    const processedSettings = await processInstanceSettings(remoteSettings, "encode");
+    const processedSettings = await processInstanceSettings(
+      remoteSettings,
+      "encode"
+    );
     await Deno.writeTextFile(
-      INSTANCE_SETTINGS_PATH,
+      instanceSettingsPath,
       yamlStringify(processedSettings)
     );
 
-    log.info(colors.green(`Settings written to ${INSTANCE_SETTINGS_PATH}`));
+    log.info(colors.green(`Settings written to ${instanceSettingsPath}`));
   }
 }
 
 export async function pushInstanceSettings(
-  preview: boolean = false,
-  baseUrl?: string
+  opts: InstanceSyncOptions,
+  preview: boolean = false
 ) {
   const remoteSettings = await wmill.listGlobalSettings();
-  let localSettings: GlobalSetting[] = await readInstanceSettings();
+  let localSettings: GlobalSetting[] = await readInstanceSettings(opts);
   localSettings = await processInstanceSettings(localSettings, "decode");
 
-  if (baseUrl) {
+  if (opts.baseUrl) {
     localSettings = localSettings.filter((s) => s.name !== "base_url");
     localSettings.push({
       name: "base_url",
       //@ts-ignore
-      value: baseUrl,
+      value: opts.baseUrl,
     });
   }
 
@@ -404,18 +445,25 @@ export async function pushInstanceSettings(
   }
 }
 
-export async function readLocalConfigs() {
+export async function readLocalConfigs(
+  opts: InstanceSyncOptions
+) {
   let localConfigs: Config[] = [];
 
+  await checkInstanceConfigPath(opts);
+
   try {
-    localConfigs = (await yamlParseFile("instance_configs.yaml")) as Config[];
+    localConfigs = (await yamlParseFile(instanceConfigsPath)) as Config[];
   } catch {
-    log.warn("No instance_configs.yaml found");
+    log.warn(`No ${instanceConfigsPath} found`);
   }
   return localConfigs;
 }
 
-export async function pullInstanceConfigs(preview = false) {
+export async function pullInstanceConfigs(
+  opts: InstanceSyncOptions,
+  preview = false
+) {
   const remoteConfigs = (await wmill.listConfigs()).map((x) => {
     return {
       ...x,
@@ -424,7 +472,7 @@ export async function pullInstanceConfigs(preview = false) {
   });
 
   if (preview) {
-    const localConfigs: Config[] = await readLocalConfigs();
+    const localConfigs: Config[] = await readLocalConfigs(opts);
 
     return compareInstanceObjects(
       remoteConfigs,
@@ -436,22 +484,25 @@ export async function pullInstanceConfigs(preview = false) {
     log.info("Pulling configs from instance");
 
     await Deno.writeTextFile(
-      "instance_configs.yaml",
+      instanceConfigsPath,
       yamlStringify(remoteConfigs as any)
     );
 
-    log.info(colors.green("Configs written to instance_configs.yaml"));
+    log.info(colors.green(`Configs written to ${instanceConfigsPath}`));
   }
 }
 
-export async function pushInstanceConfigs(preview: boolean = false) {
+export async function pushInstanceConfigs(
+  opts: InstanceSyncOptions,
+  preview: boolean = false
+) {
   const remoteConfigs = (await wmill.listConfigs()).map((x) => {
     return {
       ...x,
       name: removeWorkerPrefix(x.name),
     };
   });
-  const localConfigs = await readLocalConfigs();
+  const localConfigs = await readLocalConfigs(opts);
 
   if (preview) {
     return compareInstanceObjects(

cli/sync.ts

@@ -1398,11 +1398,12 @@ export async function push(opts: GlobalOptions & SyncOptions) {
           log.info(`Deleting ${typ} ${change.path}`);
         }
         const workspaceId = workspace.workspaceId;
+        const target = change.path.replaceAll(SEP, "/");
         switch (typ) {
           case "script": {
             const script = await wmill.getScriptByPath({
               workspace: workspaceId,
-              path: removeExtensionToPath(change.path),
+              path: removeExtensionToPath(target),
             });
             await wmill.archiveScriptByHash({
               workspace: workspaceId,
@@ -1419,37 +1420,37 @@ export async function push(opts: GlobalOptions & SyncOptions) {
           case "resource":
             await wmill.deleteResource({
               workspace: workspaceId,
-              path: removeSuffix(change.path, ".resource.json"),
+              path: removeSuffix(target, ".resource.json"),
             });
             break;
           case "resource-type":
             await wmill.deleteResourceType({
               workspace: workspaceId,
-              path: removeSuffix(change.path, ".resource-type.json"),
+              path: removeSuffix(target, ".resource-type.json"),
             });
             break;
           case "flow":
             await wmill.deleteFlowByPath({
               workspace: workspaceId,
-              path: removeSuffix(change.path, ".flow/flow.json"),
+              path: removeSuffix(target, ".flow/flow.json"),
             });
             break;
           case "app":
             await wmill.deleteApp({
               workspace: workspaceId,
-              path: removeSuffix(change.path, ".app/app.json"),
+              path: removeSuffix(target, ".app/app.json"),
             });
             break;
           case "schedule":
             await wmill.deleteSchedule({
               workspace: workspaceId,
-              path: removeSuffix(change.path, ".schedule.json"),
+              path: removeSuffix(target, ".schedule.json"),
             });
             break;
           case "variable":
             await wmill.deleteVariable({
               workspace: workspaceId,
-              path: removeSuffix(change.path, ".variable.json"),
+              path: removeSuffix(target, ".variable.json"),
             });
             break;
           case "user": {

cli/user.ts

@@ -6,7 +6,7 @@ import {
   removeType,
   removePathPrefix,
 } from "./types.ts";
-import { compareInstanceObjects } from "./instance.ts";
+import { compareInstanceObjects, InstanceSyncOptions } from "./instance.ts";
 import {
   colors,
   Command,
@@ -23,6 +23,22 @@ import {
   InstanceGroup,
 } from "./gen/types.gen.ts";
 
+const INSTANCE_USERS_PATH = "instance_users.yaml";
+let instanceUsersPath = INSTANCE_USERS_PATH;
+function checkInstanceUsersPath(opts: InstanceSyncOptions) {
+  if (opts.prefix && opts.folderPerInstance && opts.prefixSettings) {
+    instanceUsersPath = `${opts.prefix}/${INSTANCE_USERS_PATH}`;
+  }
+}
+
+const INSTANCE_GROUPS_PATH = "instance_groups.yaml";
+let instanceGroupsPath = INSTANCE_GROUPS_PATH;
+function checkInstanceGroupsPath(opts: InstanceSyncOptions) {
+  if (opts.prefix && opts.folderPerInstance && opts.prefixSettings) {
+    instanceGroupsPath = `${opts.prefix}/${INSTANCE_GROUPS_PATH}`;
+  }
+}
+
 async function list(opts: GlobalOptions) {
   await requireLogin(opts);
 
@@ -386,47 +402,61 @@ export async function pushGroup(
   }
 }
 
-export async function pullInstanceUsers(preview: boolean = false) {
+export async function pullInstanceUsers(
+  opts: InstanceSyncOptions,
+  preview: boolean = false
+) {
   const remoteUsers = await wmill.globalUsersExport();
 
+  checkInstanceUsersPath(opts);
+
   if (preview) {
-    const localUsers: ExportedUser[] = await readInstanceUsers();
+    const localUsers: ExportedUser[] = await readInstanceUsers(opts);
     return compareInstanceObjects(remoteUsers, localUsers, "email", "user");
   } else {
     log.info("Pulling users from instance...");
     await Deno.writeTextFile(
-      "instance_users.yaml",
+      instanceUsersPath,
       yamlStringify(remoteUsers as any)
     );
-    log.info(colors.green("Users written to instance_users.yaml"));
+    log.info(colors.green(`Users written to ${instanceUsersPath}`));
   }
 }
 
-export async function readInstanceUsers() {
+export async function readInstanceUsers(opts: InstanceSyncOptions) {
   let localUsers: ExportedUser[] = [];
+
+  await checkInstanceUsersPath(opts);
+
   try {
-    localUsers = (await yamlParseFile("instance_users.yaml")) as ExportedUser[];
+    localUsers = (await yamlParseFile(instanceUsersPath)) as ExportedUser[];
   } catch {
-    log.warn("No instance_users.yaml file found");
+    log.warn(`No ${instanceUsersPath} file found`);
   }
   return localUsers;
 }
 
-export async function readInstanceGroups() {
+export async function readInstanceGroups(opts: InstanceSyncOptions) {
   let localGroups: InstanceGroup[] = [];
+
+  checkInstanceGroupsPath(opts);
+
   try {
     localGroups = (await yamlParseFile(
-      "instance_groups.yaml"
+      instanceGroupsPath
     )) as ExportedInstanceGroup[];
   } catch {
-    log.warn("No instance_groups.yaml file found");
+    log.warn(`No ${instanceGroupsPath} file found`);
   }
   return localGroups;
 }
 
-export async function pushInstanceUsers(preview: boolean = false) {
+export async function pushInstanceUsers(
+  opts: InstanceSyncOptions,
+  preview: boolean = false
+) {
   const remoteUsers = await wmill.globalUsersExport();
-  const localUsers: ExportedUser[] = await readInstanceUsers();
+  const localUsers: ExportedUser[] = await readInstanceUsers(opts);
 
   if (preview) {
     return compareInstanceObjects(localUsers, remoteUsers, "email", "user");
@@ -440,27 +470,35 @@ export async function pushInstanceUsers(preview: boolean = false) {
   }
 }
 
-export async function pullInstanceGroups(preview = false) {
+export async function pullInstanceGroups(
+  opts: InstanceSyncOptions,
+  preview = false
+) {
   const remoteGroups = await wmill.exportInstanceGroups();
 
+  checkInstanceGroupsPath(opts);
+
   if (preview) {
-    const localGroups = await readInstanceGroups();
+    const localGroups = await readInstanceGroups(opts);
     return compareInstanceObjects(remoteGroups, localGroups, "name", "group");
   } else {
     log.info("Pulling groups from instance...");
 
     await Deno.writeTextFile(
-      "instance_groups.yaml",
+      instanceGroupsPath,
       yamlStringify(remoteGroups as any)
     );
 
-    log.info(colors.green("Groups written to instance_groups.yaml"));
+    log.info(colors.green(`Groups written to ${instanceGroupsPath}`));
   }
 }
 
-export async function pushInstanceGroups(preview: boolean = false) {
+export async function pushInstanceGroups(
+  opts: InstanceSyncOptions,
+  preview: boolean = false
+) {
   const remoteGroups = await wmill.exportInstanceGroups();
-  const localGroups = await readInstanceGroups();
+  const localGroups = await readInstanceGroups(opts);
 
   if (preview) {
     return compareInstanceObjects(localGroups, remoteGroups, "name", "group");

cli/worker_groups.ts

@@ -56,7 +56,7 @@ export async function displayWorkerGroups(opts: void) {
 async function pullWorkerGroups(opts: InstanceSyncOptions) {
   await pickInstance(opts, true);
 
-  const totalChanges =  await pullInstanceConfigs(true) ?? 0;
+  const totalChanges =  await pullInstanceConfigs(opts, true) ?? 0;
 
   if (totalChanges === 0) {
     log.info("No changes to apply");
@@ -72,14 +72,14 @@ async function pullWorkerGroups(opts: InstanceSyncOptions) {
   }
 
   if (confirm) {
-    await pullInstanceConfigs(false);
+    await pullInstanceConfigs(opts, false);
   }
 }
 
 async function pushWorkerGroups(opts: InstanceSyncOptions) {
   await pickInstance(opts, true);
 
-  const totalChanges = await pushInstanceConfigs(true) ?? 0;
+  const totalChanges = await pushInstanceConfigs(opts, true) ?? 0;
 
   if (totalChanges === 0) {
     log.info("No changes to apply");
@@ -95,7 +95,7 @@ async function pushWorkerGroups(opts: InstanceSyncOptions) {
   }
 
   if (confirm) {
-    await pushInstanceConfigs(false);
+    await pushInstanceConfigs(opts, false);
   }
 }
 

cli/workspace.ts

@@ -285,8 +285,8 @@ export async function removeWorkspace(
     }
     return;
   }
-  if (silent) {
-    log.info(colors.yellow(`Replacing existing workspace ${name}`));
+  if (!silent) {
+    log.info(colors.yellow(`Removing existing workspace ${name}`));
   }
 
   await Deno.writeTextFile(

docker/DockerfileSlim

@@ -20,7 +20,7 @@ RUN /usr/local/bin/python3 -m pip install pip-tools
 # Install UV
 RUN curl --proto '=https' --tlsv1.2 -LsSf https://github.com/astral-sh/uv/releases/download/0.4.18/uv-installer.sh | sh && mv /root/.cargo/bin/uv /usr/local/bin/uv
 
-COPY --from=oven/bun:1.1.32 /usr/local/bin/bun /usr/bin/bun
+COPY --from=oven/bun:1.1.34 /usr/local/bin/bun /usr/bin/bun
 
 # add the docker client to call docker from a worker if enabled
 COPY --from=docker:dind /usr/local/bin/docker /usr/local/bin/

docker/DockerfileSlimEe

@@ -19,7 +19,7 @@ RUN /usr/local/bin/python3 -m pip install pip-tools
 # Install UV
 RUN curl --proto '=https' --tlsv1.2 -LsSf https://github.com/astral-sh/uv/releases/download/0.4.18/uv-installer.sh | sh && mv /root/.cargo/bin/uv /usr/local/bin/uv
 
-COPY --from=oven/bun:1.1.32 /usr/local/bin/bun /usr/bin/bun
+COPY --from=oven/bun:1.1.34 /usr/local/bin/bun /usr/bin/bun
 
 # add the docker client to call docker from a worker if enabled
 COPY --from=docker:dind /usr/local/bin/docker /usr/local/bin/

flake.lock

@@ -0,0 +1,95 @@
+{
+  "nodes": {
+    "flake-utils": {
+      "inputs": {
+        "systems": "systems"
+      },
+      "locked": {
+        "lastModified": 1726560853,
+        "narHash": "sha256-X6rJYSESBVr3hBoH0WbKE5KvhPU5bloyZ2L4K60/fPQ=",
+        "owner": "numtide",
+        "repo": "flake-utils",
+        "rev": "c1dfcf08411b08f6b8615f7d8971a2bfa81d5e8a",
+        "type": "github"
+      },
+      "original": {
+        "owner": "numtide",
+        "repo": "flake-utils",
+        "type": "github"
+      }
+    },
+    "nixpkgs": {
+      "locked": {
+        "lastModified": 1730785428,
+        "narHash": "sha256-Zwl8YgTVJTEum+L+0zVAWvXAGbWAuXHax3KzuejaDyo=",
+        "owner": "NixOS",
+        "repo": "nixpkgs",
+        "rev": "4aa36568d413aca0ea84a1684d2d46f55dbabad7",
+        "type": "github"
+      },
+      "original": {
+        "id": "nixpkgs",
+        "ref": "nixos-unstable",
+        "type": "indirect"
+      }
+    },
+    "nixpkgs_2": {
+      "locked": {
+        "lastModified": 1728538411,
+        "narHash": "sha256-f0SBJz1eZ2yOuKUr5CA9BHULGXVSn6miBuUWdTyhUhU=",
+        "owner": "NixOS",
+        "repo": "nixpkgs",
+        "rev": "b69de56fac8c2b6f8fd27f2eca01dcda8e0a4221",
+        "type": "github"
+      },
+      "original": {
+        "owner": "NixOS",
+        "ref": "nixpkgs-unstable",
+        "repo": "nixpkgs",
+        "type": "github"
+      }
+    },
+    "root": {
+      "inputs": {
+        "flake-utils": "flake-utils",
+        "nixpkgs": "nixpkgs",
+        "rust-overlay": "rust-overlay"
+      }
+    },
+    "rust-overlay": {
+      "inputs": {
+        "nixpkgs": "nixpkgs_2"
+      },
+      "locked": {
+        "lastModified": 1730946479,
+        "narHash": "sha256-AxGJ3BRc44o3RBcfXxZqjVYftVtJ2sl+/WEjiLUmXRY=",
+        "owner": "oxalica",
+        "repo": "rust-overlay",
+        "rev": "7fba269fe89ffad47206e0afba233d337c04cf08",
+        "type": "github"
+      },
+      "original": {
+        "owner": "oxalica",
+        "repo": "rust-overlay",
+        "type": "github"
+      }
+    },
+    "systems": {
+      "locked": {
+        "lastModified": 1681028828,
+        "narHash": "sha256-Vy1rq5AaRuLzOxct8nz4T6wlgyUR7zLU309k9mBC768=",
+        "owner": "nix-systems",
+        "repo": "default",
+        "rev": "da67096a3b9bf56a91d16901293e51ba5b49a27e",
+        "type": "github"
+      },
+      "original": {
+        "owner": "nix-systems",
+        "repo": "default",
+        "type": "github"
+      }
+    }
+  },
+  "root": "root",
+  "version": 7
+}

flake.nix

@@ -0,0 +1,141 @@
+{
+  inputs = {
+    nixpkgs.url = "nixpkgs/nixos-unstable";
+    flake-utils.url = "github:numtide/flake-utils";
+    rust-overlay.url = "github:oxalica/rust-overlay";
+  };
+
+  outputs = { self, nixpkgs, flake-utils, rust-overlay }:
+    flake-utils.lib.eachDefaultSystem (system:
+    let
+      pkgs = import nixpkgs {
+        inherit system;
+        overlays = [ (import rust-overlay) ];
+      };
+      buildInputs = with pkgs; [
+        openssl openssl.dev libxml2.dev xmlsec.dev libxslt.dev
+        rust-bin.stable.latest.default nodejs
+        postgresql
+        pkg-config
+      ];
+      PKG_CONFIG_PATH = pkgs.lib.makeSearchPath "lib/pkgconfig" (with pkgs; [
+        openssl.dev
+        libxml2.dev
+        xmlsec.dev
+        libxslt.dev
+      ]);
+    in {
+      devShell = pkgs.mkShell {
+        buildInputs = buildInputs ++ (with pkgs; [ git go xcaddy sqlx-cli ]);
+        packages = [
+          (pkgs.writeScriptBin "wm-caddy" ''
+            cd ./frontend
+            xcaddy build $* \
+              --with github.com/mholt/caddy-l4@145ec36251a44286f05a10d231d8bfb3a8192e09 \
+              --with github.com/RussellLuo/caddy-ext/layer4@ab1e18cfe426012af351a68463937ae2e934a2a1
+          '')
+          (pkgs.writeScriptBin "wm-build" ''
+            cd ./frontend
+            echo $(pwd)
+            npm install
+            npm run ${if pkgs.stdenv.isDarwin then "generate-backend-client-mac" else "generate-backend-client"}
+            npm run build $*
+          '')
+          (pkgs.writeScriptBin "wm-migrate" ''
+            cd ./backend
+            sqlx migrate run
+          '')
+          (pkgs.writeScriptBin "wm-setup" ''
+            sqlx database create
+            wm-build
+            wm-caddy
+            wm-migrate
+          '')
+          (pkgs.writeScriptBin "wm-reset" ''
+            sqlx database drop -f
+            sqlx database create
+            wm-migrate
+          '')
+          (pkgs.writeScriptBin "wm" ''
+            cd ./frontend
+            npm run dev $*
+          '')
+        ];
+
+        inherit PKG_CONFIG_PATH;
+        NODE_ENV = "development";
+        NODE_OPTIONS = "--max-old-space-size=16384";
+        DATABASE_URL = "postgres://postgres:changeme@127.0.0.1:5432/";
+        REMOTE = "http://127.0.0.1:8000";
+        REMOTE_LSP = "http://127.0.0.1:3001";
+      };
+      packages.default = self.packages.${system}.windmill;
+      packages.windmill-client = pkgs.stdenv.mkDerivation {
+        pname = "windmill-client";
+        version = (pkgs.lib.strings.trim (builtins.readFile ./version.txt));
+
+        src = ./.;
+        buildInputs = with pkgs; [ nodejs ];
+
+        buildPhase = ''
+          export HOME=$(pwd)
+          npm config set strict-ssl false
+          cd frontend
+          npm install --verbose
+          npm run generate-backend-client-mac
+          NODE_OPTIONS="--max-old-space-size=8192" npm run build
+        '';
+
+        installPhase = ''
+          mkdir -p $out/build
+          cp -r build $out/build
+        '';
+      };
+      packages.windmill = pkgs.rustPlatform.buildRustPackage {
+        pname = "windmill";
+        version = (pkgs.lib.strings.trim (builtins.readFile ./version.txt));
+
+        src = ./.;
+        nativeBuildInputs = buildInputs ++ [ self.packages.${system}.windmill-client ];
+
+        cargoRoot = "backend";
+        cargoLock = {
+          lockFile = ./backend/Cargo.lock;
+          outputHashes = {
+            "php-parser-rs-0.1.3" = "sha256-ZeI3KgUPmtjlRfq6eAYveqt8Ay35gwj6B9iOQRjQa9A=";
+            "progenitor-0.3.0" = "sha256-F6XRZFVIN6/HfcM8yI/PyNke45FL7jbcznIiqj22eIQ=";
+            "rustpython-ast-0.3.1" = "sha256-q9N+z3F6YICQuUMp3a10OS792tCq0GiSSlkcaLxi3Gs=";
+            "tiberius-0.12.2" = "sha256-s/S0K3hE+JNCrNVxoSCSs4myLHvukBYTwk2A5vZ7Ae8=";
+            "tinyvector-0.1.0" = "sha256-NYGhofU4rh+2IAM+zwe04YQdXY8Aa4gTmn2V2HtzRfI=";
+          };
+        };
+
+        buildFeatures = [ ];
+        doCheck = false;
+        preBuild = ''
+          export HOME=$(pwd)
+          npm config set strict-ssl false
+          cd backend
+        '';
+
+        inherit PKG_CONFIG_PATH;
+        SQLX_OFFLINE = true;
+        FRONTEND_BUILD_DIR = "${self.packages.${system}.windmill-client}/build";
+        RUSTY_V8_ARCHIVE =
+          let
+            version = "130.0.1";
+            target = pkgs.hostPlatform.rust.rustcTarget;
+            sha256 = {
+              x86_64-linux = pkgs.lib.fakeHash;
+              aarch64-linux = pkgs.lib.fakeHash;
+              x86_64-darwin = pkgs.lib.fakeHash;
+              aarch64-darwin = "sha256-d1QTLt8gOUFxACes4oyIYgDF/srLOEk+5p5Oj1ECajQ=";
+            }.${system};
+          in pkgs.fetchurl {
+            name = "librusty_v8-${version}";
+            url = "https://github.com/denoland/rusty_v8/releases/download/v${version}/librusty_v8_release_${target}.a.gz";
+            inherit sha256;
+          };
+      };
+    });
+}

frontend/README_DEV.md

@@ -1,5 +1,7 @@
 # Developing
 
+[Using Nix](#nix), otherwise:
+
 In the `frontend/` directory:
 
 - install the dependencies with `npm install` (or `pnpm install` or `yarn`)
@@ -136,3 +138,45 @@ NOTCATCHALL=true npm run build
 which will generate an index.html and allow you to serve the frontend with any static server.
 
 Env variables used for build are set in .env file. See [https://vitejs.dev/guide/env-and-mode.html#env-files](https://vitejs.dev/guide/env-and-mode.html#env-files) for more details.
+
+## Nix
+
+Windmill use [nix flakes](https://nixos.wiki/wiki/Flakes):
+```bash
+nix run github:windmill-labs/windmill
+```
+
+### Development
+```bash
+nix develop # enter a dev shell containing all necessary packages.
+
+wm-setup # build the frontend and setup the database.
+wm # run the frontend.
+
+# In an other shell:
+nix develop
+cd backend
+cargo run
+```
+
+### Updating [`flake.nix`](../flake.nix)
+
+```bash
+nix flake update # update the lock file.
+```
+
+Some cargo dependencies use fixed git revisions, which are also fixed in `flake.nix`:
+```nix
+outputHashes = {
+  "php-parser-rs-0.1.3" = "sha256-ZeI3KgUPmtjlRfq6eAYveqt8Ay35gwj6B9iOQRjQa9A=";
+  # ...
+};
+```
+
+When updating a revision, replace the incorrect `sha256` with `pkgs.lib.fakeHash`:
+```diff
+-            "php-parser-rs-0.1.3" = "sha256-ZeI3KgUPmtjlRfq6eAYveqt8Ay35gwj6B9iOQRjQa9A=";
++            "php-parser-rs-0.1.3" = pkgs.lib.fakeHash;
+```
+
+Then run `nix build .#windmill` and update with the correct sha.

frontend/package-lock.json

@@ -1,12 +1,12 @@
 {
 	"name": "windmill-components",
-	"version": "1.416.2",
+	"version": "1.423.1",
 	"lockfileVersion": 3,
 	"requires": true,
 	"packages": {
 		"": {
 			"name": "windmill-components",
-			"version": "1.416.2",
+			"version": "1.423.1",
 			"license": "AGPL-3.0",
 			"dependencies": {
 				"@aws-crypto/sha256-js": "^4.0.0",
@@ -43,7 +43,7 @@
 				"graphql": "^16.7.1",
 				"hash-sum": "^2.0.0",
 				"highlight.js": "^11.8.0",
-				"lucide-svelte": "^0.293.0",
+				"lucide-svelte": "^0.399.0",
 				"minimatch": "^10.0.1",
 				"monaco-editor": "npm:@codingame/monaco-vscode-editor-api@~8.0.2",
 				"monaco-editor-wrapper": "^5.5.2",
@@ -8607,11 +8607,11 @@
 			"optional": true
 		},
 		"node_modules/lucide-svelte": {
-			"version": "0.293.0",
-			"resolved": "https://registry.npmjs.org/lucide-svelte/-/lucide-svelte-0.293.0.tgz",
-			"integrity": "sha512-nQ6QO6anjjiVoipw3dNptzI6NWuiArlhysVtRpoAWICSQVMR3ybwVf6piUUqbjGe+RfGufLzCl93C/yhsS+p4w==",
+			"version": "0.399.0",
+			"resolved": "https://registry.npmjs.org/lucide-svelte/-/lucide-svelte-0.399.0.tgz",
+			"integrity": "sha512-NQ8AxNMKbIJsx7HV//gnAsIY1wJfb3rbXSK2S/ZDjIldvAEdzGngpUT8T8Q8zHYUuii0bavAmVARN8giR4vvpA==",
 			"peerDependencies": {
-				"svelte": ">=3 <5"
+				"svelte": "^3 || ^4 || ^5.0.0-next.42"
 			}
 		},
 		"node_modules/magic-string": {

frontend/package.json

@@ -1,6 +1,6 @@
 {
 	"name": "windmill-components",
-	"version": "1.416.2",
+	"version": "1.423.1",
 	"scripts": {
 		"dev": "vite dev",
 		"build": "vite build",
@@ -116,7 +116,7 @@
 		"graphql": "^16.7.1",
 		"hash-sum": "^2.0.0",
 		"highlight.js": "^11.8.0",
-		"lucide-svelte": "^0.293.0",
+		"lucide-svelte": "^0.399.0",
 		"minimatch": "^10.0.1",
 		"monaco-editor": "npm:@codingame/monaco-vscode-editor-api@~8.0.2",
 		"monaco-editor-wrapper": "^5.5.2",
@@ -182,6 +182,11 @@
 			"svelte": "./package/components/icons/WindmillIcon2.svelte",
 			"default": "./package/components/icons/WindmillIcon2.svelte"
 		},
+		"./components/icons/SchedulePollIcon.svelte": {
+			"types": "./package/components/icons/SchedulePollIcon.d.ts",
+			"svelte": "./package/components/icons/SchedulePollIcon.svelte",
+			"default": "./package/components/icons/SchedulePollIcon.svelte"
+		},
 		"./components/IconedResourceType.svelte": {
 			"types": "./package/components/IconedResourceType.svelte.d.ts",
 			"svelte": "./package/components/IconedResourceType.svelte",

frontend/src/lib/components/ApiConnectForm.svelte

@@ -80,9 +80,8 @@
 	let connectionString = ''
 	let validConnectionString = true
 	function parseConnectionString(close: (_: any) => void) {
-		// parse postgres connection string
 		const regex =
-			/postgres:\/\/(?<user>[^:@]+)(?::(?<password>[^@]+))?@(?<host>[^:\/?]+)(?::(?<port>\d+))?\/(?<dbname>[^\?]+)?(?:\?.*sslmode=(?<sslmode>[^&]+))?/
+			/postgres(?:ql)?:\/\/(?<user>[^:@]+)(?::(?<password>[^@]+))?@(?<host>[^:\/?]+)(?::(?<port>\d+))?\/(?<dbname>[^\?]+)?(?:\?.*sslmode=(?<sslmode>[^&]+))?/
 		const match = connectionString.match(regex)
 		if (match) {
 			validConnectionString = true

frontend/src/lib/components/AppConnectInner.svelte

@@ -250,6 +250,14 @@
 				throw Error(`Resource at path ${path} already exists. Delete it or pick another path`)
 			}
 
+
+			if (resourceType == 'snowflake_oauth') {
+				const account_identifier = extra_params.find(([key, _]) => key == 'account_identifier')
+				if (account_identifier) {
+					args['account_identifier'] = account_identifier[1]
+				}
+			}
+			
 			let account: number | undefined = undefined
 			if (valueToken?.expires_in != undefined) {
 				account = Number(

frontend/src/lib/components/ArgInfo.svelte

@@ -98,7 +98,7 @@
 				}}><Expand size={18} /></button
 			>
 		{/if}
-		<div class="max-h-40 overflow-auto">
+		<div class="max-h-60 overflow-auto">
 			<ObjectViewer collapsed={false} topBrackets={true} pureViewer={true} json={value} />
 		</div>
 	</div>

frontend/src/lib/components/ArgInput.svelte

@@ -128,8 +128,10 @@
 	let s3FilePicker: S3FilePicker
 	let s3FileUploadRawMode: false
 	let isListJson = false
+	let hasIsListJsonChanged = false
 
 	let el: HTMLTextAreaElement | undefined = undefined
+
 	let inputCat = computeInputCat(type, format, itemsType?.type, enum_, contentEncoding)
 
 	$: inputCat = computeInputCat(type, format, itemsType?.type, enum_, contentEncoding)
@@ -170,6 +172,43 @@
 
 	$: computeDefaultValue(value, inputCat, defaultValue, nullable)
 
+	let lastValue: any = undefined
+
+	// By setting isListJson to true, we can render inputs even if the value is not an array of the correct type
+	// This avoids the issue of the input being rendered as a string with value: [object Object], or as a number with value: NaN
+	function checkArrayValueType() {
+		try {
+			if (Array.isArray(value) && value.length > 0) {
+				const firstItem = value?.[0]
+				const type = itemsType?.type
+
+				switch (type) {
+					case 'string':
+						if (typeof firstItem !== 'string') {
+							isListJson = true
+						}
+						break
+					case 'number':
+						if (typeof firstItem !== 'number') {
+							isListJson = true
+						}
+						break
+				}
+			}
+		} catch (e) {
+			console.error(e)
+		}
+
+		lastValue = value
+	}
+
+	$: !isListJson &&
+		inputCat === 'list' &&
+		value != lastValue &&
+		itemsType?.type &&
+		!hasIsListJsonChanged &&
+		checkArrayValueType()
+
 	$: defaultValue != undefined && handleDefaultValueChange()
 
 	let oldDefaultValue = defaultValue
@@ -379,6 +418,9 @@
 									bind:selected={value}
 									options={itemsType?.multiselect ?? []}
 									selectedOptionsDraggable={true}
+									on:open={() => {
+										dispatch('focus')
+									}}
 								/>
 							</div>
 						{:else if itemsType?.enum != undefined && Array.isArray(itemsType?.enum) && Array.isArray(value)}
@@ -389,6 +431,9 @@
 									bind:selected={value}
 									options={itemsType?.enum ?? []}
 									selectedOptionsDraggable={true}
+									on:open={() => {
+										dispatch('focus')
+									}}
 								/>
 							</div>
 						{:else}
@@ -518,6 +563,11 @@
 					<div class="mt-2 mr-4">
 						<Toggle
 							on:change={(e) => {
+								// Once the user has changed the input type, we should not change it back automatically
+								if (!hasIsListJsonChanged) {
+									hasIsListJsonChanged = true
+								}
+
 								evalValueToRaw()
 								isListJson = !isListJson
 							}}
@@ -772,6 +822,12 @@
 						bind:value
 						{enum_}
 						{autofocus}
+						on:focus={() => {
+							dispatch('focus')
+						}}
+						on:blur={(e) => {
+							dispatch('blur')
+						}}
 						enumLabels={extra['enumLabels']}
 					/>
 				</div>
@@ -840,14 +896,16 @@
 				<div class="flex flex-col w-full">
 					<div class="flex flex-row w-full items-center justify-between relative">
 						{#if password || extra?.['password'] == true}
-							{#if value && typeof value == 'string' && value?.startsWith('$var:')}
-								<input type="text" bind:value />
-							{:else if onlyMaskPassword}
-								<Password
-									{disabled}
-									bind:password={value}
-									placeholder={placeholder ?? defaultValue ?? ''}
-								/>
+							{#if onlyMaskPassword}
+								{#if value && typeof value == 'string' && value?.startsWith('$var:')}
+									<input type="text" bind:value />
+								{:else}
+									<Password
+										{disabled}
+										bind:password={value}
+										placeholder={placeholder ?? defaultValue ?? ''}
+									/>
+								{/if}
 							{:else}
 								<PasswordArgInput {disabled} bind:value />
 							{/if}