starred/windmill
1
0
Fork 0
mirror of https://github.com/windmill-labs/windmill.git synced 2026-09-05 00:03:08 +00:00
Code Issues Packages Projects Releases Wiki Activity

Merge branch 'main' into feat/batch-jobs-ui

Browse Source
This commit is contained in:
Clement Zhang
2024-11-18 12:37:43 +01:00
parent 9d8a2cd823 e4583e9b23
commit 259617ebe4
142 changed files with 6121 additions and 2503 deletions
Download Patch File Download Diff File Expand all files Collapse all files
.gitignore
+1 -1
View File
@@ -6,4 +6,4 @@ frontend/src/routes/test.svelte
CaddyfileRemoteMalo
*.swp
**/.idea/
.direnv
.direnv
CHANGELOG.md
+32
View File
@@ -1,5 +1,37 @@
# Changelog
## [1.425.1](https://github.com/windmill-labs/windmill/compare/v1.425.0...v1.425.1) (2024-11-16)
### Bug Fixes
* revert bool to text support in pg ([#4727](https://github.com/windmill-labs/windmill/issues/4727)) ([17d8933](https://github.com/windmill-labs/windmill/commit/17d893315bd86942c7895cdb3e9c3ab34977b258))
## [1.425.0](https://github.com/windmill-labs/windmill/compare/v1.424.0...v1.425.0) (2024-11-15)
### Features
* Handle `pip install` by `uv` ([#4517](https://github.com/windmill-labs/windmill/issues/4517)) ([f240d13](https://github.com/windmill-labs/windmill/commit/f240d1322a3c1669dfb69ebda6d05effcca26ae3))
* **monitoring:** workspace critical alerts ([#4684](https://github.com/windmill-labs/windmill/issues/4684)) ([c32038a](https://github.com/windmill-labs/windmill/commit/c32038a76d9d00703d8b865af72f083ac43414e3))
## [1.424.0](https://github.com/windmill-labs/windmill/compare/v1.423.2...v1.424.0) (2024-11-14)
### Features
* allow setting password and login type from superadmin UI ([2a1bff3](https://github.com/windmill-labs/windmill/commit/2a1bff3160b65eadba399229b5f53950ec2c0d48))
* **backend:** monitor minimal version of living workers ([#4704](https://github.com/windmill-labs/windmill/issues/4704)) ([50ff183](https://github.com/windmill-labs/windmill/commit/50ff183baeaa2a39c2b0188ee8dd6172b08ae801))
* Support mistral anthropic for ai ([#4692](https://github.com/windmill-labs/windmill/issues/4692)) ([556b4a4](https://github.com/windmill-labs/windmill/commit/556b4a41a1d010bb8a1796f485d343c1e412d278))
### Bug Fixes
* add countCompletedJobs api ([2a78359](https://github.com/windmill-labs/windmill/commit/2a78359af7b8e4700634c2ad2745c1d14d2da4f7))
* add queue_couts api ([10b6b1d](https://github.com/windmill-labs/windmill/commit/10b6b1dc04cb2b2d4ec5ceb26a5dbd2ac7bd1394))
* autoscaling when count < min worker set to min_workers ([180bb82](https://github.com/windmill-labs/windmill/commit/180bb826436f9960def8c6cf3e9692714ffdf917))
* deployment callbacks have a concurrency limit of 1 on same path ([c44fc4f](https://github.com/windmill-labs/windmill/commit/c44fc4f5ab8bcb5dffec08d5ed4ccd61feb1cb13))
## [1.423.2](https://github.com/windmill-labs/windmill/compare/v1.423.1...v1.423.2) (2024-11-13)
backend/.gitignore
+1 -1
View File
@@ -2,7 +2,7 @@ target/
.env
oauth.json
oauth2.json
windmill-api/openapi-deref.yaml
tracing.folded
heaptrack*
index/
windmill-api/openapi-*.*
backend/.sqlx/query-00588a40dde5189ac1c61505f17acb0f4c244c60477427505bf5bd1b104d3bf9.json
Generated
+14
View File
@@ -0,0 +1,14 @@
{
"db_name": "PostgreSQL",
"query": "UPDATE alerts SET acknowledged_workspace = true, acknowledged = true WHERE workspace_id = $1",
"describe": {
"columns": [],
"parameters": {
"Left": [
"Text"
]
},
"nullable": []
},
"hash": "00588a40dde5189ac1c61505f17acb0f4c244c60477427505bf5bd1b104d3bf9"
}
backend/.sqlx/query-ceb97024ebf1a1c00ea1ed4952f66b229ca4622c537cc252d8ed52b4d24270ee.json → backend/.sqlx/query-0230bd64d2b6719c3536a106e18a68c7b43b3a9a9efa92b5517132e9c0d8a25b.json
Generated
+4 -4
View File
@@ -1,12 +1,12 @@
{
"db_name": "PostgreSQL",
"query": "SELECT openai_resource_path, code_completion_enabled FROM workspace_settings WHERE workspace_id = $1",
"query": "SELECT ai_resource, code_completion_enabled FROM workspace_settings WHERE workspace_id = $1",
"describe": {
"columns": [
{
"ordinal": 0,
"name": "openai_resource_path",
"type_info": "Varchar"
"name": "ai_resource",
"type_info": "Jsonb"
},
{
"ordinal": 1,
@@ -24,5 +24,5 @@
false
]
},
"hash": "ceb97024ebf1a1c00ea1ed4952f66b229ca4622c537cc252d8ed52b4d24270ee"
"hash": "0230bd64d2b6719c3536a106e18a68c7b43b3a9a9efa92b5517132e9c0d8a25b"
}
backend/.sqlx/query-188534f4b29f6461b1a6214d060f183c830b19a403ebb7b8be55a691675010c3.json → backend/.sqlx/query-0331a81262e2d3c1bcfaeb64617b11eb68ab4599d64e5e5af639a9ac5d791fd0.json
Generated
+4 -4
View File
@@ -1,6 +1,6 @@
{
"db_name": "PostgreSQL",
"query": "SELECT\n -- slack_team_id, \n -- slack_name, \n -- slack_command_script, \n -- CASE WHEN slack_email = 'missing@email.xyz' THEN NULL ELSE slack_email END AS slack_email,\n auto_invite_domain IS NOT NULL AS \"auto_invite_enabled!\",\n CASE WHEN auto_invite_operator IS TRUE THEN 'operator' ELSE 'developer' END AS \"auto_invite_as!\", \n CASE WHEN auto_add IS TRUE THEN 'add' ELSE 'invite' END AS \"auto_invite_mode!\", \n webhook, \n deploy_to, \n error_handler, \n openai_resource_path, \n code_completion_enabled, \n error_handler_extra_args, \n error_handler_muted_on_cancel, \n large_file_storage, \n git_sync,\n default_app,\n default_scripts,\n workspace.name\n FROM workspace_settings\n LEFT JOIN workspace ON workspace.id = workspace_settings.workspace_id\n WHERE workspace_id = $1",
"query": "SELECT\n -- slack_team_id, \n -- slack_name, \n -- slack_command_script, \n -- CASE WHEN slack_email = 'missing@email.xyz' THEN NULL ELSE slack_email END AS slack_email,\n auto_invite_domain IS NOT NULL AS \"auto_invite_enabled!\",\n CASE WHEN auto_invite_operator IS TRUE THEN 'operator' ELSE 'developer' END AS \"auto_invite_as!\", \n CASE WHEN auto_add IS TRUE THEN 'add' ELSE 'invite' END AS \"auto_invite_mode!\", \n webhook, \n deploy_to, \n error_handler, \n ai_resource, \n code_completion_enabled, \n error_handler_extra_args, \n error_handler_muted_on_cancel, \n large_file_storage, \n git_sync,\n default_app,\n default_scripts,\n workspace.name\n FROM workspace_settings\n LEFT JOIN workspace ON workspace.id = workspace_settings.workspace_id\n WHERE workspace_id = $1",
"describe": {
"columns": [
{
@@ -35,8 +35,8 @@
},
{
"ordinal": 6,
"name": "openai_resource_path",
"type_info": "Varchar"
"name": "ai_resource",
"type_info": "Jsonb"
},
{
"ordinal": 7,
@@ -102,5 +102,5 @@
false
]
},
"hash": "188534f4b29f6461b1a6214d060f183c830b19a403ebb7b8be55a691675010c3"
"hash": "0331a81262e2d3c1bcfaeb64617b11eb68ab4599d64e5e5af639a9ac5d791fd0"
}
backend/.sqlx/query-034583442e6f8ae38d6c4e4aac26f17c8d9d0e657f28276228fc90d3e22e1304.json
Generated
-16
View File
@@ -1,16 +0,0 @@
{
"db_name": "PostgreSQL",
"query": "UPDATE workspace_settings SET openai_resource_path = $1, code_completion_enabled = $2 WHERE workspace_id = $3",
"describe": {
"columns": [],
"parameters": {
"Left": [
"Varchar",
"Bool",
"Text"
]
},
"nullable": []
},
"hash": "034583442e6f8ae38d6c4e4aac26f17c8d9d0e657f28276228fc90d3e22e1304"
}
backend/.sqlx/query-044e2b428ee6e2dd4543c87ad8835e239cf7567d18b8b3fa6608ea3a9d206ca7.json
Generated
+18
View File
@@ -0,0 +1,18 @@
{
"db_name": "PostgreSQL",
"query": "INSERT INTO alerts (alert_type, message, acknowledged, acknowledged_workspace, workspace_id, resource)\n VALUES ('critical_error', $1, $2, $3, $4, $5)",
"describe": {
"columns": [],
"parameters": {
"Left": [
"Text",
"Bool",
"Bool",
"Text",
"Text"
]
},
"nullable": []
},
"hash": "044e2b428ee6e2dd4543c87ad8835e239cf7567d18b8b3fa6608ea3a9d206ca7"
}
backend/.sqlx/query-0ee63ef2dd5c88edba2a1f56d31f29876724922f148dad7af35b36efbf70207a.json
Generated
-14
View File
@@ -1,14 +0,0 @@
{
"db_name": "PostgreSQL",
"query": "DELETE FROM healthchecks WHERE check_type = $1",
"describe": {
"columns": [],
"parameters": {
"Left": [
"Text"
]
},
"nullable": []
},
"hash": "0ee63ef2dd5c88edba2a1f56d31f29876724922f148dad7af35b36efbf70207a"
}
backend/.sqlx/query-1610c79b8d238e3a35d795f34a2a0100b933b7f199d41cc2f554d57c811d55f3.json
Generated
+16
View File
@@ -0,0 +1,16 @@
{
"db_name": "PostgreSQL",
"query": "UPDATE workspace_settings SET ai_resource = $1, code_completion_enabled = $2 WHERE workspace_id = $3",
"describe": {
"columns": [],
"parameters": {
"Left": [
"Jsonb",
"Bool",
"Text"
]
},
"nullable": []
},
"hash": "1610c79b8d238e3a35d795f34a2a0100b933b7f199d41cc2f554d57c811d55f3"
}
backend/.sqlx/query-1730f39fd1793d45fbb41b21389c61296a3ff7489ae12f52a19f9543173ac597.json
Generated
+8 -2
View File
@@ -65,8 +65,8 @@
},
{
"ordinal": 12,
"name": "openai_resource_path",
"type_info": "Varchar"
"name": "ai_resource",
"type_info": "Jsonb"
},
{
"ordinal": 13,
@@ -117,6 +117,11 @@
"ordinal": 22,
"name": "deploy_ui",
"type_info": "Jsonb"
},
{
"ordinal": 23,
"name": "mute_critical_alerts",
"type_info": "Bool"
}
],
"parameters": {
@@ -147,6 +152,7 @@
true,
false,
true,
true,
true
]
},
backend/.sqlx/query-1e7ce0c140410ae799f9c0c5772e4be4506bfd238c88f1b1c3ddf39b29071446.json
Generated
+29
View File
@@ -0,0 +1,29 @@
{
"db_name": "PostgreSQL",
"query": "SELECT\n flow_status AS \"flow_status!: Json<Box<RawValue>>\",\n raw_flow->'modules'->(flow_status->'step')::int AS \"module: Json<Box<RawValue>>\"\n FROM queue WHERE id = $1 AND workspace_id = $2 LIMIT 1",
"describe": {
"columns": [
{
"ordinal": 0,
"name": "flow_status!: Json<Box<RawValue>>",
"type_info": "Jsonb"
},
{
"ordinal": 1,
"name": "module: Json<Box<RawValue>>",
"type_info": "Jsonb"
}
],
"parameters": {
"Left": [
"Uuid",
"Text"
]
},
"nullable": [
true,
null
]
},
"hash": "1e7ce0c140410ae799f9c0c5772e4be4506bfd238c88f1b1c3ddf39b29071446"
}
backend/.sqlx/query-acdaa5151f8f7f37bb8c8c5a7d146789887e47db9695fc26b1dfaedd735e1e60.json → backend/.sqlx/query-286fa00c088df146c08c1934556f06a4243c66787ab8759a8045e60effd2fb77.json
Generated
+2 -2
View File
@@ -1,6 +1,6 @@
{
"db_name": "PostgreSQL",
"query": "SELECT value\n FROM resource\n WHERE path = $1 AND workspace_id = $2",
"query": "SELECT value\n FROM resource\n WHERE path = $1 AND workspace_id = $2",
"describe": {
"columns": [
{
@@ -19,5 +19,5 @@
true
]
},
"hash": "acdaa5151f8f7f37bb8c8c5a7d146789887e47db9695fc26b1dfaedd735e1e60"
"hash": "286fa00c088df146c08c1934556f06a4243c66787ab8759a8045e60effd2fb77"
}
backend/.sqlx/query-28987898c7e4b172d466bf08f33c2da733e71f8a253b0122c17b9e021919809e.json
Generated
+18
View File
@@ -0,0 +1,18 @@
{
"db_name": "PostgreSQL",
"query": "INSERT INTO alerts (alert_type, message, acknowledged, acknowledged_workspace, workspace_id, resource)\n VALUES ('recovered_critical_error', $1, $2, $3, $4, $5)",
"describe": {
"columns": [],
"parameters": {
"Left": [
"Text",
"Bool",
"Bool",
"Text",
"Text"
]
},
"nullable": []
},
"hash": "28987898c7e4b172d466bf08f33c2da733e71f8a253b0122c17b9e021919809e"
}
backend/.sqlx/query-344b3a5d9683273a956b5156fed5ab9fdff1a7252198e4ae290d8c8f937ff177.json
Generated
+54
View File
@@ -0,0 +1,54 @@
{
"db_name": "PostgreSQL",
"query": "SELECT id, alert_type, message, created_at, COALESCE(acknowledged, false) AS acknowledged, workspace_id\n FROM alerts\n WHERE COALESCE(acknowledged, false) = $1\n ORDER BY created_at DESC\n LIMIT $2 OFFSET $3",
"describe": {
"columns": [
{
"ordinal": 0,
"name": "id",
"type_info": "Int4"
},
{
"ordinal": 1,
"name": "alert_type",
"type_info": "Varchar"
},
{
"ordinal": 2,
"name": "message",
"type_info": "Text"
},
{
"ordinal": 3,
"name": "created_at",
"type_info": "Timestamptz"
},
{
"ordinal": 4,
"name": "acknowledged",
"type_info": "Bool"
},
{
"ordinal": 5,
"name": "workspace_id",
"type_info": "Text"
}
],
"parameters": {
"Left": [
"Bool",
"Int8",
"Int8"
]
},
"nullable": [
false,
false,
false,
false,
null,
true
]
},
"hash": "344b3a5d9683273a956b5156fed5ab9fdff1a7252198e4ae290d8c8f937ff177"
}
backend/.sqlx/query-349396e8fdd96d45875110bc06767e6eb876792ec8f83e9b03c2fb46bb12e0b9.json
Generated
+15
View File
@@ -0,0 +1,15 @@
{
"db_name": "PostgreSQL",
"query": "UPDATE password SET login_type = $1 WHERE email = $2",
"describe": {
"columns": [],
"parameters": {
"Left": [
"Varchar",
"Text"
]
},
"nullable": []
},
"hash": "349396e8fdd96d45875110bc06767e6eb876792ec8f83e9b03c2fb46bb12e0b9"
}
backend/.sqlx/query-3a94ad52c6b7cde844fa868167248cd9ff63e5fdfa1d93d8fbec32a257b6b05e.json
Generated
-15
View File
@@ -1,15 +0,0 @@
{
"db_name": "PostgreSQL",
"query": "INSERT INTO alerts (alert_type, message, acknowledged) VALUES ('recovered_critical_error', $1, $2)",
"describe": {
"columns": [],
"parameters": {
"Left": [
"Text",
"Bool"
]
},
"nullable": []
},
"hash": "3a94ad52c6b7cde844fa868167248cd9ff63e5fdfa1d93d8fbec32a257b6b05e"
}
backend/.sqlx/query-4a3a8207627418ba7b7eabaccd7ec72884f2a1eacf0ab0e0c6ccbf56c654c14a.json
Generated
+14
View File
@@ -0,0 +1,14 @@
{
"db_name": "PostgreSQL",
"query": "UPDATE alerts SET acknowledged = true, acknowledged_workspace = true WHERE resource = $1 AND alert_type = 'critical_error'",
"describe": {
"columns": [],
"parameters": {
"Left": [
"Text"
]
},
"nullable": []
},
"hash": "4a3a8207627418ba7b7eabaccd7ec72884f2a1eacf0ab0e0c6ccbf56c654c14a"
}
backend/.sqlx/query-4d22084a5d9860832f30e8f08cbfa1848ed3c1336fa4790f45b8189c4ac97d91.json
Generated
-15
View File
@@ -1,15 +0,0 @@
{
"db_name": "PostgreSQL",
"query": "INSERT INTO alerts (alert_type, message, acknowledged) VALUES ('critical_error', $1, $2)",
"describe": {
"columns": [],
"parameters": {
"Left": [
"Text",
"Bool"
]
},
"nullable": []
},
"hash": "4d22084a5d9860832f30e8f08cbfa1848ed3c1336fa4790f45b8189c4ac97d91"
}
backend/.sqlx/query-0b955f2cff82a2d4ba3840588143e08952f029480d4a42503ecc3c5e70437995.json → backend/.sqlx/query-4d30c5a2894d655741d167f5589f5816583f0bae78a0dd3270dedb377dfa944a.json
Generated
+8 -3
View File
@@ -1,6 +1,6 @@
{
"db_name": "PostgreSQL",
"query": "SELECT id, alert_type, message, created_at, acknowledged \n FROM alerts \n WHERE acknowledged = $1\n ORDER BY created_at DESC \n LIMIT $2 OFFSET $3",
"query": "SELECT id, alert_type, message, created_at, COALESCE(acknowledged, false) AS acknowledged, workspace_id\n FROM alerts\n ORDER BY created_at DESC\n LIMIT $1 OFFSET $2",
"describe": {
"columns": [
{
@@ -27,11 +27,15 @@
"ordinal": 4,
"name": "acknowledged",
"type_info": "Bool"
},
{
"ordinal": 5,
"name": "workspace_id",
"type_info": "Text"
}
],
"parameters": {
"Left": [
"Bool",
"Int8",
"Int8"
]
@@ -41,8 +45,9 @@
false,
false,
false,
null,
true
]
},
"hash": "0b955f2cff82a2d4ba3840588143e08952f029480d4a42503ecc3c5e70437995"
"hash": "4d30c5a2894d655741d167f5589f5816583f0bae78a0dd3270dedb377dfa944a"
}
backend/.sqlx/query-55cb03040bc2a8c53dd7fbb42bbdcc40f463cbc52d94ed9315cf9a547d4c89f2.json
Generated
+8 -2
View File
@@ -65,8 +65,8 @@
},
{
"ordinal": 12,
"name": "openai_resource_path",
"type_info": "Varchar"
"name": "ai_resource",
"type_info": "Jsonb"
},
{
"ordinal": 13,
@@ -117,6 +117,11 @@
"ordinal": 22,
"name": "deploy_ui",
"type_info": "Jsonb"
},
{
"ordinal": 23,
"name": "mute_critical_alerts",
"type_info": "Bool"
}
],
"parameters": {
@@ -147,6 +152,7 @@
true,
false,
true,
true,
true
]
},
backend/.sqlx/query-6268eabd561502a44e273d6391102278974623f7a7bcad6891d7abadf4d3ea03.json
Generated
-15
View File
@@ -1,15 +0,0 @@
{
"db_name": "PostgreSQL",
"query": "UPDATE workspace_settings SET openai_resource_path = NULL, code_completion_enabled = $1 WHERE workspace_id = $2",
"describe": {
"columns": [],
"parameters": {
"Left": [
"Bool",
"Text"
]
},
"nullable": []
},
"hash": "6268eabd561502a44e273d6391102278974623f7a7bcad6891d7abadf4d3ea03"
}
backend/.sqlx/query-65da41c7ded54cdee8d33211561c068b72294cc99ff44ed0a13179df508ebc6a.json
Generated
+15
View File
@@ -0,0 +1,15 @@
{
"db_name": "PostgreSQL",
"query": "UPDATE alerts\n SET\n acknowledged = true,\n acknowledged_workspace = CASE\n WHEN $2::text IS NOT NULL AND workspace_id = $2 THEN true\n ELSE acknowledged_workspace\n END\n WHERE id = $1",
"describe": {
"columns": [],
"parameters": {
"Left": [
"Int4",
"Text"
]
},
"nullable": []
},
"hash": "65da41c7ded54cdee8d33211561c068b72294cc99ff44ed0a13179df508ebc6a"
}
backend/.sqlx/query-6940ddc5ee8d2a1048213d46f52d964b199604ba66dedbe9f89cea727d726a2d.json
Generated
+15
View File
@@ -0,0 +1,15 @@
{
"db_name": "PostgreSQL",
"query": "UPDATE workspace_settings SET ai_resource = NULL, code_completion_enabled = $1 WHERE workspace_id = $2",
"describe": {
"columns": [],
"parameters": {
"Left": [
"Bool",
"Text"
]
},
"nullable": []
},
"hash": "6940ddc5ee8d2a1048213d46f52d964b199604ba66dedbe9f89cea727d726a2d"
}
backend/.sqlx/query-cc5ab80241b88c5befea279f16c4ec68cec17b31dcd277b321f652917346496b.json → backend/.sqlx/query-777e7084edf9a5d14f299ff479ae43c8ead47a6a531d4b031a1342c9b2f16506.json
Generated
+9 -2
View File
@@ -1,6 +1,6 @@
{
"db_name": "PostgreSQL",
"query": "SELECT id, alert_type, message, created_at, acknowledged \n FROM alerts \n ORDER BY created_at DESC \n LIMIT $1 OFFSET $2",
"query": "SELECT id, alert_type, message, created_at, COALESCE(acknowledged_workspace, false) AS acknowledged, workspace_id\n FROM alerts\n WHERE workspace_id = $1\n ORDER BY created_at DESC\n LIMIT $2 OFFSET $3",
"describe": {
"columns": [
{
@@ -27,10 +27,16 @@
"ordinal": 4,
"name": "acknowledged",
"type_info": "Bool"
},
{
"ordinal": 5,
"name": "workspace_id",
"type_info": "Text"
}
],
"parameters": {
"Left": [
"Text",
"Int8",
"Int8"
]
@@ -40,8 +46,9 @@
false,
false,
false,
null,
true
]
},
"hash": "cc5ab80241b88c5befea279f16c4ec68cec17b31dcd277b321f652917346496b"
"hash": "777e7084edf9a5d14f299ff479ae43c8ead47a6a531d4b031a1342c9b2f16506"
}
backend/.sqlx/query-7c32176755c6ea2b6ae531860d436caae3fa256fc0803749ec5107632669adb3.json
Generated
+14
View File
@@ -0,0 +1,14 @@
{
"db_name": "PostgreSQL",
"query": "UPDATE alerts \n SET\n acknowledged = true,\n acknowledged_workspace = CASE\n WHEN $1::text IS NOT NULL THEN true\n ELSE acknowledged_workspace\n END\n WHERE ($1::text IS NOT NULL AND workspace_id = $1)\n OR ($1::text IS NULL)",
"describe": {
"columns": [],
"parameters": {
"Left": [
"Text"
]
},
"nullable": []
},
"hash": "7c32176755c6ea2b6ae531860d436caae3fa256fc0803749ec5107632669adb3"
}
backend/.sqlx/query-7c5a29de07cbe42326a15d4d7fd9714c20b767508e27368be6de38d7b18a3a4d.json
Generated
+55
View File
@@ -0,0 +1,55 @@
{
"db_name": "PostgreSQL",
"query": "SELECT id, alert_type, message, created_at, COALESCE(acknowledged_workspace, false) AS acknowledged, workspace_id\n FROM alerts\n WHERE workspace_id = $1 AND COALESCE(acknowledged_workspace, false) = $2\n ORDER BY created_at DESC\n LIMIT $3 OFFSET $4",
"describe": {
"columns": [
{
"ordinal": 0,
"name": "id",
"type_info": "Int4"
},
{
"ordinal": 1,
"name": "alert_type",
"type_info": "Varchar"
},
{
"ordinal": 2,
"name": "message",
"type_info": "Text"
},
{
"ordinal": 3,
"name": "created_at",
"type_info": "Timestamptz"
},
{
"ordinal": 4,
"name": "acknowledged",
"type_info": "Bool"
},
{
"ordinal": 5,
"name": "workspace_id",
"type_info": "Text"
}
],
"parameters": {
"Left": [
"Text",
"Bool",
"Int8",
"Int8"
]
},
"nullable": [
false,
false,
false,
false,
null,
true
]
},
"hash": "7c5a29de07cbe42326a15d4d7fd9714c20b767508e27368be6de38d7b18a3a4d"
}
backend/.sqlx/query-a59fae29ebcc9aa53308b777ead3d2b652618ac454f139db738f4499cb4eb079.json
Generated
+12
View File
@@ -0,0 +1,12 @@
{
"db_name": "PostgreSQL",
"query": "UPDATE alerts SET acknowledged = true",
"describe": {
"columns": [],
"parameters": {
"Left": []
},
"nullable": []
},
"hash": "a59fae29ebcc9aa53308b777ead3d2b652618ac454f139db738f4499cb4eb079"
}
backend/.sqlx/query-a7c5008aa7ea43d0afac7d9f19846976ed7af2e90270902f001115e023cb947d.json
Generated
-12
View File
@@ -1,12 +0,0 @@
{
"db_name": "PostgreSQL",
"query": "UPDATE alerts SET acknowledged = true WHERE acknowledged = false",
"describe": {
"columns": [],
"parameters": {
"Left": []
},
"nullable": []
},
"hash": "a7c5008aa7ea43d0afac7d9f19846976ed7af2e90270902f001115e023cb947d"
}
backend/.sqlx/query-ad03e5acf10ef94abc37cb9f56b1775c67f075c8bc83458e8be2e242347218d6.json
Generated
+22
View File
@@ -0,0 +1,22 @@
{
"db_name": "PostgreSQL",
"query": "SELECT wm_version FROM worker_ping WHERE wm_version != $1 AND ping_at > now() - interval '5 minutes'",
"describe": {
"columns": [
{
"ordinal": 0,
"name": "wm_version",
"type_info": "Varchar"
}
],
"parameters": {
"Left": [
"Text"
]
},
"nullable": [
false
]
},
"hash": "ad03e5acf10ef94abc37cb9f56b1775c67f075c8bc83458e8be2e242347218d6"
}
backend/.sqlx/query-57111a97ff906fd79ecd8a571212cc4fea9cb9c16fe950f082ad5469bfeefc82.json → backend/.sqlx/query-b2c30137b3b64f8c3f6aea062e8687e119599641beee0a0d6c89c5d90e82d0db.json
Generated
+3 -3
View File
@@ -1,6 +1,6 @@
{
"db_name": "PostgreSQL",
"query": "SELECT created_by, CONCAT(coalesce(queue.logs, ''), coalesce(job_logs.logs, '')) as logs, job_logs.log_offset, job_logs.log_file_index\n FROM queue \n LEFT JOIN job_logs ON job_logs.job_id = queue.id \n WHERE queue.id = $1 AND queue.workspace_id = $2",
"query": "SELECT created_by, CONCAT(coalesce(queue.logs, ''), coalesce(job_logs.logs, '')) as logs, coalesce(job_logs.log_offset, 0) as log_offset, job_logs.log_file_index\n FROM queue \n LEFT JOIN job_logs ON job_logs.job_id = queue.id \n WHERE queue.id = $1 AND queue.workspace_id = $2",
"describe": {
"columns": [
{
@@ -33,9 +33,9 @@
"nullable": [
false,
null,
false,
null,
true
]
},
"hash": "57111a97ff906fd79ecd8a571212cc4fea9cb9c16fe950f082ad5469bfeefc82"
"hash": "b2c30137b3b64f8c3f6aea062e8687e119599641beee0a0d6c89c5d90e82d0db"
}
backend/.sqlx/query-be3ae231557e794172336bc27d725f862dcf039bbb3d75ced9d54c86f53d2580.json
Generated
-14
View File
@@ -1,14 +0,0 @@
{
"db_name": "PostgreSQL",
"query": "UPDATE alerts SET acknowledged = true WHERE id = $1",
"describe": {
"columns": [],
"parameters": {
"Left": [
"Int4"
]
},
"nullable": []
},
"hash": "be3ae231557e794172336bc27d725f862dcf039bbb3d75ced9d54c86f53d2580"
}
backend/.sqlx/query-ceda377fa534656ac12a7f41db77db1f054ec751855c8db7352b0e9c20b63ef8.json
Generated
+22
View File
@@ -0,0 +1,22 @@
{
"db_name": "PostgreSQL",
"query": "SELECT ai_resource FROM workspace_settings WHERE workspace_id = $1",
"describe": {
"columns": [
{
"ordinal": 0,
"name": "ai_resource",
"type_info": "Jsonb"
}
],
"parameters": {
"Left": [
"Text"
]
},
"nullable": [
true
]
},
"hash": "ceda377fa534656ac12a7f41db77db1f054ec751855c8db7352b0e9c20b63ef8"
}
backend/.sqlx/query-3e539fef054ad31bc1736e27276087775a721a6ee7ae35b03fd4ce3563ea3838.json → backend/.sqlx/query-de1abe57b6aa61155f747a3bcb98359f70eade6654b5a884915f07f3ef3fe15e.json
Generated
+2 -2
View File
@@ -1,6 +1,6 @@
{
"db_name": "PostgreSQL",
"query": "SELECT raw_flow->'modules'->($1)->'value'->>'type' = 'flow' FROM queue WHERE id = $2",
"query": "SELECT raw_flow->'modules'->($1)::text->'value'->>'type' = 'flow' FROM queue WHERE id = $2 LIMIT 1",
"describe": {
"columns": [
{
@@ -19,5 +19,5 @@
null
]
},
"hash": "3e539fef054ad31bc1736e27276087775a721a6ee7ae35b03fd4ce3563ea3838"
"hash": "de1abe57b6aa61155f747a3bcb98359f70eade6654b5a884915f07f3ef3fe15e"
}
backend/.sqlx/query-f8b689ec3e09f14dc02ac9e9f98d252a14d3a2acfadcd9cd6ba27ca9799f4706.json → backend/.sqlx/query-e05dbe046e846c092a96b6b0a9d872c721169d418586be2d15cee6b929049098.json
Generated
+4 -4
View File
@@ -1,12 +1,12 @@
{
"db_name": "PostgreSQL",
"query": "SELECT openai_resource_path FROM workspace_settings WHERE workspace_id = $1",
"query": "SELECT mute_critical_alerts FROM workspace_settings WHERE workspace_id = $1",
"describe": {
"columns": [
{
"ordinal": 0,
"name": "openai_resource_path",
"type_info": "Varchar"
"name": "mute_critical_alerts",
"type_info": "Bool"
}
],
"parameters": {
@@ -18,5 +18,5 @@
true
]
},
"hash": "f8b689ec3e09f14dc02ac9e9f98d252a14d3a2acfadcd9cd6ba27ca9799f4706"
"hash": "e05dbe046e846c092a96b6b0a9d872c721169d418586be2d15cee6b929049098"
}
backend/.sqlx/query-f22168826350797e88153b65a5b1e906a7868f34c062f154b2ee4f24c57aa5c3.json
Generated
+15
View File
@@ -0,0 +1,15 @@
{
"db_name": "PostgreSQL",
"query": "UPDATE workspace_settings SET mute_critical_alerts = $1 WHERE workspace_id = $2",
"describe": {
"columns": [],
"parameters": {
"Left": [
"Bool",
"Text"
]
},
"nullable": []
},
"hash": "f22168826350797e88153b65a5b1e906a7868f34c062f154b2ee4f24c57aa5c3"
}
backend/Cargo.lock
Generated
+37 -36
View File
@@ -843,9 +843,9 @@ dependencies = [
[[package]]
name = "aws-sdk-sts"
version = "1.49.0"
version = "1.50.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "53dcf5e7d9bd1517b8b998e170e650047cea8a2b85fe1835abe3210713e541b7"
checksum = "6ada54e5f26ac246dc79727def52f7f8ed38915cb47781e2a72213957dc3a7d5"
dependencies = [
"aws-credential-types",
"aws-runtime",
@@ -1032,9 +1032,9 @@ dependencies = [
[[package]]
name = "axum"
version = "0.7.7"
version = "0.7.8"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "504e3947307ac8326a5437504c517c4b56716c9d98fac0028c2acc7ca47d70ae"
checksum = "49c41b948da08fb481a94546cd874843adc1142278b0af4badf9b1b78599d68d"
dependencies = [
"async-trait",
"axum-core",
@@ -1598,9 +1598,9 @@ dependencies = [
[[package]]
name = "cc"
version = "1.2.0"
version = "1.2.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "1aeb932158bd710538c73702db6945cb68a8fb08c519e6e12706b94263b36db8"
checksum = "fd9de9f2205d5ef3fd67e685b0df337994ddd4495e2a28d185500d0e1edfea47"
dependencies = [
"jobserver",
"libc",
@@ -1783,9 +1783,9 @@ dependencies = [
[[package]]
name = "comfy-table"
version = "7.1.2"
version = "7.1.3"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "e0d05af1e006a2407bedef5af410552494ce5be9090444dbbcb57258c1af3d56"
checksum = "24f165e7b643266ea80cb858aed492ad9280e3e05ce24d4a99d7d7b889b6a4d9"
dependencies = [
"strum 0.26.3",
"strum_macros 0.26.4",
@@ -3355,9 +3355,9 @@ dependencies = [
[[package]]
name = "flate2"
version = "1.0.34"
version = "1.0.35"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "a1b589b4dc103969ad3cf85c950899926ec64300a1a46d76c03a6072957036f0"
checksum = "c936bfdafb507ebbf50b8074c54fa31c5be9a1e7e5f467dd659697041407d07c"
dependencies = [
"crc32fast",
"libz-sys",
@@ -4831,9 +4831,9 @@ dependencies = [
[[package]]
name = "libc"
version = "0.2.162"
version = "0.2.163"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "18d287de67fe55fd7e1581fe933d965a5a9477b38e949cfa9f8574ef01506398"
checksum = "1fdaeca4cf44ed4ac623e86ef41f056e848dbeab7ec043ecb7326ba300b36fd0"
[[package]]
name = "libgit2-sys"
@@ -10610,7 +10610,7 @@ checksum = "712e227841d057c1ee1cd2fb22fa7e5a5461ae8e48fa2ca79ec42cfc1931183f"
[[package]]
name = "windmill"
version = "1.423.2"
version = "1.425.1"
dependencies = [
"anyhow",
"axum",
@@ -10652,7 +10652,7 @@ dependencies = [
[[package]]
name = "windmill-api"
version = "1.423.2"
version = "1.425.1"
dependencies = [
"anyhow",
"argon2",
@@ -10737,7 +10737,7 @@ dependencies = [
[[package]]
name = "windmill-api-client"
version = "1.423.2"
version = "1.425.1"
dependencies = [
"base64 0.22.1",
"chrono",
@@ -10755,7 +10755,7 @@ dependencies = [
[[package]]
name = "windmill-audit"
version = "1.423.2"
version = "1.425.1"
dependencies = [
"chrono",
"serde",
@@ -10768,7 +10768,7 @@ dependencies = [
[[package]]
name = "windmill-autoscaling"
version = "1.423.2"
version = "1.425.1"
dependencies = [
"anyhow",
"rsmq_async",
@@ -10783,7 +10783,7 @@ dependencies = [
[[package]]
name = "windmill-common"
version = "1.423.2"
version = "1.425.1"
dependencies = [
"anyhow",
"async-stream",
@@ -10811,6 +10811,7 @@ dependencies = [
"rand 0.8.5",
"regex",
"reqwest 0.12.9",
"semver 1.0.23",
"serde",
"serde_json",
"sha2 0.10.8",
@@ -10829,7 +10830,7 @@ dependencies = [
[[package]]
name = "windmill-git-sync"
version = "1.423.2"
version = "1.425.1"
dependencies = [
"regex",
"rsmq_async",
@@ -10844,7 +10845,7 @@ dependencies = [
[[package]]
name = "windmill-indexer"
version = "1.423.2"
version = "1.425.1"
dependencies = [
"anyhow",
"bytes",
@@ -10866,7 +10867,7 @@ dependencies = [
[[package]]
name = "windmill-macros"
version = "1.423.2"
version = "1.425.1"
dependencies = [
"itertools 0.13.0",
"lazy_static",
@@ -10878,7 +10879,7 @@ dependencies = [
[[package]]
name = "windmill-parser"
version = "1.423.2"
version = "1.425.1"
dependencies = [
"convert_case 0.6.0",
"serde",
@@ -10887,7 +10888,7 @@ dependencies = [
[[package]]
name = "windmill-parser-bash"
version = "1.423.2"
version = "1.425.1"
dependencies = [
"anyhow",
"lazy_static",
@@ -10899,7 +10900,7 @@ dependencies = [
[[package]]
name = "windmill-parser-go"
version = "1.423.2"
version = "1.425.1"
dependencies = [
"anyhow",
"gosyn",
@@ -10911,7 +10912,7 @@ dependencies = [
[[package]]
name = "windmill-parser-graphql"
version = "1.423.2"
version = "1.425.1"
dependencies = [
"anyhow",
"lazy_static",
@@ -10923,7 +10924,7 @@ dependencies = [
[[package]]
name = "windmill-parser-php"
version = "1.423.2"
version = "1.425.1"
dependencies = [
"anyhow",
"itertools 0.13.0",
@@ -10934,7 +10935,7 @@ dependencies = [
[[package]]
name = "windmill-parser-py"
version = "1.423.2"
version = "1.425.1"
dependencies = [
"anyhow",
"itertools 0.13.0",
@@ -10945,7 +10946,7 @@ dependencies = [
[[package]]
name = "windmill-parser-py-imports"
version = "1.423.2"
version = "1.425.1"
dependencies = [
"anyhow",
"async-recursion",
@@ -10963,7 +10964,7 @@ dependencies = [
[[package]]
name = "windmill-parser-rust"
version = "1.423.2"
version = "1.425.1"
dependencies = [
"anyhow",
"convert_case 0.6.0",
@@ -10980,7 +10981,7 @@ dependencies = [
[[package]]
name = "windmill-parser-sql"
version = "1.423.2"
version = "1.425.1"
dependencies = [
"anyhow",
"lazy_static",
@@ -10992,7 +10993,7 @@ dependencies = [
[[package]]
name = "windmill-parser-ts"
version = "1.423.2"
version = "1.425.1"
dependencies = [
"anyhow",
"lazy_static",
@@ -11010,7 +11011,7 @@ dependencies = [
[[package]]
name = "windmill-parser-wasm"
version = "1.423.2"
version = "1.425.1"
dependencies = [
"anyhow",
"getrandom 0.2.15",
@@ -11031,7 +11032,7 @@ dependencies = [
[[package]]
name = "windmill-parser-yaml"
version = "1.423.2"
version = "1.425.1"
dependencies = [
"anyhow",
"serde_json",
@@ -11041,7 +11042,7 @@ dependencies = [
[[package]]
name = "windmill-queue"
version = "1.423.2"
version = "1.425.1"
dependencies = [
"anyhow",
"async-recursion",
@@ -11074,7 +11075,7 @@ dependencies = [
[[package]]
name = "windmill-sql-datatype-parser-wasm"
version = "1.423.2"
version = "1.425.1"
dependencies = [
"wasm-bindgen",
"wasm-bindgen-test",
@@ -11084,7 +11085,7 @@ dependencies = [
[[package]]
name = "windmill-worker"
version = "1.423.2"
version = "1.425.1"
dependencies = [
"anyhow",
"async-recursion",
backend/Cargo.toml
+3 -2
View File
@@ -1,6 +1,6 @@
[package]
name = "windmill"
version = "1.423.2"
version = "1.425.1"
authors.workspace = true
edition.workspace = true
@@ -29,7 +29,7 @@ members = [
]
[workspace.package]
version = "1.423.2"
version = "1.425.1"
authors = ["Ruben Fiszel <ruben@windmill.dev>"]
edition = "2021"
@@ -180,6 +180,7 @@ tokio-util = { version = "^0", features = ["io"] }
json-pointer = "^0"
itertools = "^0"
regex = "^1"
semver = "^1"
deno_fetch = "0.195.0"
deno_tls = "0.158.0"
backend/ee-repo-ref.txt
+1 -1
View File
@@ -1 +1 @@
e3df64af864f9d3546a97ff21f719c553e34ff1c
51dcbf93b0d127af9f33fa346cc63fcd2475d4fa
backend/migrations/20241105133531_alter_openai_resource_path_column_in_workspace_settings.down.sql
+7
View File
@@ -0,0 +1,7 @@
-- Add down migration script here
ALTER TABLE workspace_settings
ALTER COLUMN ai_resource TYPE text
USING ai_resource->>'path';
ALTER TABLE workspace_settings
RENAME COLUMN ai_resource to openai_resource_path;
backend/migrations/20241105133531_alter_openai_resource_path_column_in_workspace_settings.up.sql
+7
View File
@@ -0,0 +1,7 @@
-- Add up migration script here
ALTER TABLE workspace_settings
ALTER COLUMN openai_resource_path TYPE jsonb
USING jsonb_build_object('provider', 'openai', 'path', openai_resource_path);
ALTER TABLE workspace_settings
RENAME COLUMN openai_resource_path TO ai_resource;
backend/migrations/20241108201445_alert_add_workspace_id.down.sql
+4
View File
@@ -0,0 +1,4 @@
ALTER TABLE alerts DROP COLUMN workspace_id;
ALTER TABLE alerts DROP COLUMN acknowledged_workspace;
ALTER TABLE alerts DROP COLUMN resource;
ALTER TABLE workspace_settings DROP COLUMN mute_critical_alerts;
backend/migrations/20241108201445_alert_add_workspace_id.up.sql
+4
View File
@@ -0,0 +1,4 @@
ALTER TABLE alerts ADD COLUMN workspace_id TEXT DEFAULT NULL;
ALTER TABLE alerts ADD COLUMN acknowledged_workspace BOOL DEFAULT NULL;
ALTER TABLE alerts ADD COLUMN resource TEXT DEFAULT NULL;
ALTER TABLE workspace_settings ADD COLUMN mute_critical_alerts BOOL DEFAULT NULL;
backend/src/main.rs
+18 -17
View File
@@ -7,7 +7,6 @@
*/
use anyhow::Context;
use git_version::git_version;
use monitor::{
reload_timeout_wait_result_setting, send_current_log_file_to_object_store,
send_logs_to_object_store,
@@ -31,18 +30,19 @@ use windmill_common::ee::{maybe_renew_license_key_on_start, LICENSE_KEY_ID, LICE
use windmill_common::{
global_settings::{
BASE_URL_SETTING, BUNFIG_INSTALL_SCOPES_SETTING, CRITICAL_ERROR_CHANNELS_SETTING,
CUSTOM_TAGS_SETTING, DEFAULT_TAGS_PER_WORKSPACE_SETTING, DEFAULT_TAGS_WORKSPACES_SETTING,
ENV_SETTINGS, EXPOSE_DEBUG_METRICS_SETTING, EXPOSE_METRICS_SETTING,
EXTRA_PIP_INDEX_URL_SETTING, HUB_BASE_URL_SETTING, JOB_DEFAULT_TIMEOUT_SECS_SETTING,
JWT_SECRET_SETTING, KEEP_JOB_DIR_SETTING, LICENSE_KEY_SETTING, NPM_CONFIG_REGISTRY_SETTING,
OAUTH_SETTING, PIP_INDEX_URL_SETTING, REQUEST_SIZE_LIMIT_SETTING, CRITICAL_ALERT_MUTE_UI_SETTING,
REQUIRE_PREEXISTING_USER_FOR_OAUTH_SETTING, RETENTION_PERIOD_SECS_SETTING,
SAML_METADATA_SETTING, SCIM_TOKEN_SETTING, SMTP_SETTING, TIMEOUT_WAIT_RESULT_SETTING,
BASE_URL_SETTING, BUNFIG_INSTALL_SCOPES_SETTING, CRITICAL_ALERT_MUTE_UI_SETTING,
CRITICAL_ERROR_CHANNELS_SETTING, CUSTOM_TAGS_SETTING, DEFAULT_TAGS_PER_WORKSPACE_SETTING,
DEFAULT_TAGS_WORKSPACES_SETTING, ENV_SETTINGS, EXPOSE_DEBUG_METRICS_SETTING,
EXPOSE_METRICS_SETTING, EXTRA_PIP_INDEX_URL_SETTING, HUB_BASE_URL_SETTING,
JOB_DEFAULT_TIMEOUT_SECS_SETTING, JWT_SECRET_SETTING, KEEP_JOB_DIR_SETTING,
LICENSE_KEY_SETTING, NPM_CONFIG_REGISTRY_SETTING, OAUTH_SETTING, PIP_INDEX_URL_SETTING,
REQUEST_SIZE_LIMIT_SETTING, REQUIRE_PREEXISTING_USER_FOR_OAUTH_SETTING,
RETENTION_PERIOD_SECS_SETTING, SAML_METADATA_SETTING, SCIM_TOKEN_SETTING, SMTP_SETTING,
TIMEOUT_WAIT_RESULT_SETTING,
},
scripts::ScriptLang,
stats_ee::schedule_stats,
utils::{hostname, rd_string, Mode},
utils::{hostname, rd_string, Mode, GIT_VERSION},
worker::{reload_custom_tags_setting, HUB_CACHE_DIR, TMP_DIR, WORKER_GROUP},
DB, METRICS_ENABLED,
};
@@ -67,24 +67,24 @@ use windmill_worker::{
get_hub_script_content_and_requirements, BUN_BUNDLE_CACHE_DIR, BUN_CACHE_DIR,
BUN_DEPSTAR_CACHE_DIR, DENO_CACHE_DIR, DENO_CACHE_DIR_DEPS, DENO_CACHE_DIR_NPM,
GO_BIN_CACHE_DIR, GO_CACHE_DIR, LOCK_CACHE_DIR, PIP_CACHE_DIR, POWERSHELL_CACHE_DIR,
RUST_CACHE_DIR, TAR_PIP_CACHE_DIR, TMP_LOGS_DIR, UV_CACHE_DIR,
PY311_CACHE_DIR, RUST_CACHE_DIR, TAR_PIP_CACHE_DIR, TMP_LOGS_DIR, UV_CACHE_DIR,
};
use crate::monitor::{
initial_load, load_keep_job_dir, load_metrics_debug_enabled, load_require_preexisting_user,
load_tag_per_workspace_enabled, load_tag_per_workspace_workspaces, monitor_db, monitor_pool,
reload_base_url_setting, reload_bunfig_install_scopes_setting,
reload_critical_error_channels_setting, reload_extra_pip_index_url_setting,
reload_hub_base_url_setting, reload_job_default_timeout_setting, reload_jwt_secret_setting,
reload_license_key, reload_npm_config_registry_setting, reload_pip_index_url_setting,
reload_critical_alert_mute_ui_setting, reload_critical_error_channels_setting,
reload_extra_pip_index_url_setting, reload_hub_base_url_setting,
reload_job_default_timeout_setting, reload_jwt_secret_setting, reload_license_key,
reload_npm_config_registry_setting, reload_pip_index_url_setting,
reload_retention_period_setting, reload_scim_token_setting, reload_smtp_config,
reload_worker_config, reload_critical_alert_mute_ui_setting,
reload_worker_config,
};
#[cfg(feature = "parquet")]
use crate::monitor::reload_s3_cache_setting;
const GIT_VERSION: &str = git_version!(args = ["--tag", "--always"], fallback = "unknown-version");
const DEFAULT_NUM_WORKERS: usize = 1;
const DEFAULT_PORT: u16 = 8000;
const DEFAULT_SERVER_BIND_ADDR: Ipv4Addr = Ipv4Addr::new(0, 0, 0, 0);
@@ -997,13 +997,14 @@ pub async fn run_workers<R: rsmq_async::RsmqConnection + Send + Sync + Clone + '
for x in [
LOCK_CACHE_DIR,
TMP_LOGS_DIR,
PIP_CACHE_DIR,
UV_CACHE_DIR,
TAR_PIP_CACHE_DIR,
DENO_CACHE_DIR,
DENO_CACHE_DIR_DEPS,
DENO_CACHE_DIR_NPM,
BUN_CACHE_DIR,
PY311_CACHE_DIR,
PIP_CACHE_DIR,
BUN_DEPSTAR_CACHE_DIR,
BUN_BUNDLE_CACHE_DIR,
GO_CACHE_DIR,
backend/src/monitor.rs
+31 -19
View File
@@ -34,12 +34,12 @@ use windmill_common::{
error,
flow_status::FlowStatusModule,
global_settings::{
BASE_URL_SETTING, BUNFIG_INSTALL_SCOPES_SETTING, CRITICAL_ERROR_CHANNELS_SETTING,
DEFAULT_TAGS_PER_WORKSPACE_SETTING, DEFAULT_TAGS_WORKSPACES_SETTING,
EXPOSE_DEBUG_METRICS_SETTING, EXPOSE_METRICS_SETTING, EXTRA_PIP_INDEX_URL_SETTING,
HUB_BASE_URL_SETTING, JOB_DEFAULT_TIMEOUT_SECS_SETTING, JWT_SECRET_SETTING,
KEEP_JOB_DIR_SETTING, LICENSE_KEY_SETTING, NPM_CONFIG_REGISTRY_SETTING, OAUTH_SETTING,
PIP_INDEX_URL_SETTING, REQUEST_SIZE_LIMIT_SETTING, CRITICAL_ALERT_MUTE_UI_SETTING,
BASE_URL_SETTING, BUNFIG_INSTALL_SCOPES_SETTING, CRITICAL_ALERT_MUTE_UI_SETTING,
CRITICAL_ERROR_CHANNELS_SETTING, DEFAULT_TAGS_PER_WORKSPACE_SETTING,
DEFAULT_TAGS_WORKSPACES_SETTING, EXPOSE_DEBUG_METRICS_SETTING, EXPOSE_METRICS_SETTING,
EXTRA_PIP_INDEX_URL_SETTING, HUB_BASE_URL_SETTING, JOB_DEFAULT_TIMEOUT_SECS_SETTING,
JWT_SECRET_SETTING, KEEP_JOB_DIR_SETTING, LICENSE_KEY_SETTING, NPM_CONFIG_REGISTRY_SETTING,
OAUTH_SETTING, PIP_INDEX_URL_SETTING, REQUEST_SIZE_LIMIT_SETTING,
REQUIRE_PREEXISTING_USER_FOR_OAUTH_SETTING, RETENTION_PERIOD_SECS_SETTING,
SAML_METADATA_SETTING, SCIM_TOKEN_SETTING, TIMEOUT_WAIT_RESULT_SETTING,
},
@@ -51,11 +51,11 @@ use windmill_common::{
utils::{now_from_db, rd_string, report_critical_error, Mode},
worker::{
load_worker_config, make_pull_query, make_suspended_pull_query, reload_custom_tags_setting,
DEFAULT_TAGS_PER_WORKSPACE, DEFAULT_TAGS_WORKSPACES, SMTP_CONFIG, WORKER_CONFIG,
WORKER_GROUP,
update_min_version, DEFAULT_TAGS_PER_WORKSPACE, DEFAULT_TAGS_WORKSPACES, SMTP_CONFIG,
WORKER_CONFIG, WORKER_GROUP,
},
BASE_URL, CRITICAL_ERROR_CHANNELS, DB, DEFAULT_HUB_BASE_URL, HUB_BASE_URL, JOB_RETENTION_SECS,
METRICS_DEBUG_ENABLED, METRICS_ENABLED, CRITICAL_ALERT_MUTE_UI_ENABLED
BASE_URL, CRITICAL_ALERT_MUTE_UI_ENABLED, CRITICAL_ERROR_CHANNELS, DB, DEFAULT_HUB_BASE_URL,
HUB_BASE_URL, JOB_RETENTION_SECS, METRICS_DEBUG_ENABLED, METRICS_ENABLED,
};
use windmill_queue::cancel_job;
use windmill_worker::{
@@ -231,13 +231,20 @@ pub async fn load_tag_per_workspace_workspaces(db: &DB) -> error::Result<()> {
}
pub async fn reload_critical_alert_mute_ui_setting(db: &DB) -> error::Result<()> {
let mute = load_value_from_global_settings(db, CRITICAL_ALERT_MUTE_UI_SETTING).await;
match mute {
Ok(Some(serde_json::Value::Bool(t))) => {
CRITICAL_ALERT_MUTE_UI_ENABLED.store(t, Ordering::Relaxed);
if let Ok(Some(serde_json::Value::Bool(t))) =
load_value_from_global_settings(db, CRITICAL_ALERT_MUTE_UI_SETTING).await
{
CRITICAL_ALERT_MUTE_UI_ENABLED.store(t, Ordering::Relaxed);
if t {
if let Err(e) = sqlx::query!("UPDATE alerts SET acknowledged = true")
.execute(db)
.await
{
tracing::error!("Error updating alerts: {}", e.to_string());
}
}
_ => (),
};
}
Ok(())
}
@@ -1080,6 +1087,10 @@ pub async fn monitor_db(
}
};
let update_min_worker_version_f = async {
update_min_version(db).await;
};
join!(
expired_items_f,
zombie_jobs_f,
@@ -1088,6 +1099,7 @@ pub async fn monitor_db(
worker_groups_alerts_f,
jobs_waiting_alerts_f,
apply_autoscaling_f,
update_min_worker_version_f,
);
}
@@ -1318,7 +1330,7 @@ async fn handle_zombie_jobs<R: rsmq_async::RsmqConnection + Send + Sync + Clone>
ON CONFLICT (job_id) DO UPDATE SET logs = job_logs.logs || '\nRestarted job after not receiving job''s ping for too long the ' || now() || '\n\n' WHERE job_logs.job_id = $1", r.id)
.execute(db).await;
tracing::error!(error_message);
report_critical_error(error_message, db.clone()).await;
report_critical_error(error_message, db.clone(), Some(&r.workspace_id), None).await;
}
}
@@ -1432,7 +1444,7 @@ async fn handle_zombie_flows(
flow.id, flow.workspace_id
);
tracing::error!(error_message);
report_critical_error(error_message, db.clone()).await;
report_critical_error(error_message, db.clone(), Some(&flow.workspace_id), None).await;
// if the flow hasn't started and is a zombie, we can simply restart it
sqlx::query!(
"UPDATE queue SET running = false, started_at = null WHERE id = $1 AND canceled = false",
@@ -1452,7 +1464,7 @@ async fn handle_zombie_flows(
format!("Flow {id} was cancelled because it")
}
);
report_critical_error(reason.clone(), db.clone()).await;
report_critical_error(reason.clone(), db.clone(), Some(&flow.workspace_id), None).await;
cancel_zombie_flow_job(db, flow, &rsmq, reason).await?;
}
}
backend/windmill-api/openapi-deref.json
+228 -10
View File
@@ -1,7 +1,7 @@
{
"openapi": "3.0.3",
"info": {
"version": "1.418.0",
"version": "1.423.2",
"title": "Windmill API",
"contact": {
"name": "Windmill Team",
@@ -286,7 +286,7 @@
}
}
},
"/w/{workspace}/users/{username}": {
"/w/{workspace}/users/get/{username}": {
"get": {
"summary": "get user (require admin privilege)",
"operationId": "getUser",
@@ -436,6 +436,106 @@
}
}
},
"/users/set_password_of/{user}": {
"post": {
"summary": "set password for a specific user (require super admin)",
"operationId": "setPasswordForUser",
"tags": [
"user"
],
"parameters": [
{
"name": "user",
"in": "path",
"required": true,
"schema": {
"type": "string"
}
}
],
"requestBody": {
"description": "set password",
"required": true,
"content": {
"application/json": {
"schema": {
"type": "object",
"properties": {
"password": {
"type": "string"
}
},
"required": [
"password"
]
}
}
}
},
"responses": {
"200": {
"description": "password set",
"content": {
"text/plain": {
"schema": {
"type": "string"
}
}
}
}
}
}
},
"/users/set_login_type/{user}": {
"post": {
"summary": "set login type for a specific user (require super admin)",
"operationId": "setLoginTypeForUser",
"tags": [
"user"
],
"parameters": [
{
"name": "user",
"in": "path",
"required": true,
"schema": {
"type": "string"
}
}
],
"requestBody": {
"description": "set login type",
"required": true,
"content": {
"application/json": {
"schema": {
"type": "object",
"properties": {
"login_type": {
"type": "string"
}
},
"required": [
"login_type"
]
}
}
}
},
"responses": {
"200": {
"description": "login type set",
"content": {
"text/plain": {
"schema": {
"type": "string"
}
}
}
}
}
}
},
"/users/create": {
"post": {
"summary": "create user",
@@ -1105,6 +1205,9 @@
},
"tls_implicit": {
"type": "boolean"
},
"disable_tls": {
"type": "boolean"
}
},
"required": [
@@ -1113,7 +1216,8 @@
"password",
"port",
"from",
"tls_implicit"
"tls_implicit",
"disable_tls"
]
}
},
@@ -2460,8 +2564,8 @@
"deploy_to": {
"type": "string"
},
"openai_resource_path": {
"type": "string"
"ai_resource": {
"$ref": "#/components/schemas/AiResource"
},
"code_completion_enabled": {
"type": "boolean"
@@ -2901,8 +3005,8 @@
"code_completion_enabled"
],
"properties": {
"openai_resource_path": {
"type": "string"
"ai_resource": {
"$ref": "#/components/schemas/AiResource"
},
"code_completion_enabled": {
"type": "boolean"
@@ -2946,7 +3050,10 @@
"schema": {
"type": "object",
"properties": {
"exists_openai_resource_path": {
"ai_provider": {
"type": "string"
},
"exists_ai_resource": {
"type": "boolean"
},
"code_completion_enabled": {
@@ -2954,7 +3061,8 @@
}
},
"required": [
"exists_openai_resource_path",
"ai_provider",
"exists_ai_resource",
"code_completion_enabled"
]
}
@@ -13160,6 +13268,30 @@
}
}
},
"/workers/queue_counts": {
"get": {
"summary": "get counts of jobs waiting for an executor per tag",
"operationId": "getCountsOfJobsWaitingPerTag",
"tags": [
"worker"
],
"responses": {
"200": {
"description": "queue counts",
"content": {
"application/json": {
"schema": {
"type": "object",
"additionalProperties": {
"type": "integer"
}
}
}
}
}
}
}
},
"/configs/list_worker_groups": {
"get": {
"summary": "list worker groups",
@@ -14810,6 +14942,20 @@
"schema": {
"type": "string"
}
},
{
"name": "content_type",
"in": "query",
"schema": {
"type": "string"
}
},
{
"name": "content_disposition",
"in": "query",
"schema": {
"type": "string"
}
}
],
"requestBody": {
@@ -16116,6 +16262,21 @@
}
},
"schemas": {
"AiResource": {
"type": "object",
"properties": {
"path": {
"type": "string"
},
"provider": {
"type": "string"
}
},
"required": [
"path",
"provider"
]
},
"Script": {
"type": "object",
"properties": {
@@ -17226,7 +17387,7 @@
"jobs.disapproval",
"jobs.delete",
"account.delete",
"openai.request",
"ai.request",
"resources.create",
"resources.update",
"resources.delete",
@@ -18067,6 +18228,23 @@
"route_path": {
"type": "string"
},
"static_asset_config": {
"type": "object",
"properties": {
"s3": {
"type": "string"
},
"storage": {
"type": "string"
},
"filename": {
"type": "string"
}
},
"required": [
"s3"
]
},
"is_flow": {
"type": "boolean"
},
@@ -18126,6 +18304,23 @@
"route_path": {
"type": "string"
},
"static_asset_config": {
"type": "object",
"properties": {
"s3": {
"type": "string"
},
"storage": {
"type": "string"
},
"filename": {
"type": "string"
}
},
"required": [
"s3"
]
},
"is_flow": {
"type": "boolean"
},
@@ -18168,6 +18363,23 @@
"route_path": {
"type": "string"
},
"static_asset_config": {
"type": "object",
"properties": {
"s3": {
"type": "string"
},
"storage": {
"type": "string"
},
"filename": {
"type": "string"
}
},
"required": [
"s3"
]
},
"is_flow": {
"type": "boolean"
},
@@ -18917,6 +19129,12 @@
"type": "object"
}
},
"s3_inputs": {
"type": "array",
"items": {
"type": "object"
}
},
"execution_mode": {
"type": "string",
"enum": [
backend/windmill-api/openapi-deref.yaml
+1116 -974
View File
File diff suppressed because it is too large Load Diff
backend/windmill-api/openapi.yaml
+248 -8
View File
@@ -1,7 +1,7 @@
openapi: "3.0.3"
info:
version: 1.423.2
version: 1.425.1
title: Windmill API
contact:
@@ -283,6 +283,71 @@ paths:
text/plain:
schema:
type: string
/users/set_password_of/{user}:
post:
summary: set password for a specific user (require super admin)
operationId: setPasswordForUser
tags:
- user
parameters:
- name: user
in: path
required: true
schema:
type: string
requestBody:
description: set password
required: true
content:
application/json:
schema:
type: object
properties:
password:
type: string
required:
- password
responses:
"200":
description: password set
content:
text/plain:
schema:
type: string
/users/set_login_type/{user}:
post:
summary: set login type for a specific user (require super admin)
operationId: setLoginTypeForUser
tags:
- user
parameters:
- name: user
in: path
required: true
schema:
type: string
requestBody:
description: set login type
required: true
content:
application/json:
schema:
type: object
properties:
login_type:
type: string
required:
- login_type
responses:
"200":
description: login type set
content:
text/plain:
schema:
type: string
/users/create:
post:
@@ -1616,8 +1681,8 @@ paths:
type: string
deploy_to:
type: string
openai_resource_path:
type: string
ai_resource:
$ref: "#/components/schemas/AiResource"
code_completion_enabled:
type: boolean
error_handler:
@@ -1636,6 +1701,8 @@ paths:
type: string
default_scripts:
$ref: "#/components/schemas/WorkspaceDefaultScripts"
mute_critical_alerts:
type: boolean
required:
- code_completion_enabled
- automatic_billing
@@ -1901,8 +1968,8 @@ paths:
required:
- code_completion_enabled
properties:
openai_resource_path:
type: string
ai_resource:
$ref: "#/components/schemas/AiResource"
code_completion_enabled:
type: boolean
responses:
@@ -1930,12 +1997,15 @@ paths:
schema:
type: object
properties:
exists_openai_resource_path:
ai_provider:
type: string
exists_ai_resource:
type: boolean
code_completion_enabled:
type: boolean
required:
- exists_openai_resource_path
- ai_provider
- exists_ai_resource
- code_completion_enabled
/w/{workspace}/workspaces/edit_error_handler:
@@ -2649,6 +2719,112 @@ paths:
items:
$ref: "#/components/schemas/ContextualVariable"
/w/{workspace}/workspaces/critical_alerts:
get:
summary: Get all critical alerts for this workspace
operationId: workspaceGetCriticalAlerts
tags:
- setting
parameters:
- $ref: "#/components/parameters/WorkspaceId"
- in: query
name: page
schema:
type: integer
default: 1
description: The page number to retrieve (minimum value is 1)
- in: query
name: page_size
schema:
type: integer
default: 10
maximum: 100
description: Number of alerts per page (maximum is 100)
- in: query
name: acknowledged
schema:
type: boolean
nullable: true
description: Filter by acknowledgment status; true for acknowledged, false for unacknowledged, and omit for all alerts
responses:
"200":
description: Successfully retrieved all critical alerts
content:
application/json:
schema:
type: array
items:
$ref: '#/components/schemas/CriticalAlert'
/w/{workspace}/workspaces/critical_alerts/{id}/acknowledge:
post:
summary: Acknowledge a critical alert for this workspace
operationId: workspaceAcknowledgeCriticalAlert
tags:
- setting
parameters:
- $ref: "#/components/parameters/WorkspaceId"
- in: path
name: id
required: true
schema:
type: integer
description: The ID of the critical alert to acknowledge
responses:
"200":
description: Successfully acknowledged the critical alert
content:
application/json:
schema:
type: string
example: "Critical alert acknowledged"
/w/{workspace}/workspaces/critical_alerts/acknowledge_all:
post:
summary: Acknowledge all unacknowledged critical alerts for this workspace
operationId: workspaceAcknowledgeAllCriticalAlerts
tags:
- setting
parameters:
- $ref: "#/components/parameters/WorkspaceId"
responses:
"200":
description: Successfully acknowledged all unacknowledged critical alerts.
content:
application/json:
schema:
type: string
example: "All unacknowledged critical alerts acknowledged"
/w/{workspace}/workspaces/critical_alerts/mute:
post:
summary: Mute critical alert UI for this workspace
operationId: workspaceMuteCriticalAlertsUI
tags:
- setting
parameters:
- $ref: "#/components/parameters/WorkspaceId"
requestBody:
description: Boolean flag to mute critical alerts.
required: true
content:
application/json:
schema:
type: object
properties:
mute_critical_alerts:
type: boolean
description: Whether critical alerts should be muted.
example: true
responses:
'200':
description: Successfully updated mute critical alert settings.
content:
application/json:
schema:
type: string
example: "Updated mute critical alert UI settings for workspace: workspace_id"
/oauth/login_callback/{client_name}:
post:
security: []
@@ -5986,6 +6162,39 @@ paths:
required:
- database_length
/w/{workspace}/jobs/completed/count_jobs:
get:
summary: count number of completed jobs with filter
operationId: countCompletedJobs
tags:
- job
parameters:
- $ref: "#/components/parameters/WorkspaceId"
- name: completed_after_s_ago
in: query
schema:
type: integer
- name: success
in: query
schema:
type: boolean
- name: tags
in: query
schema:
type: string
- name: all_workspaces
in: query
schema:
type: boolean
responses:
"200":
description: Count of completed jobs
content:
application/json:
schema:
type: integer
/w/{workspace}/jobs/queue/list_filtered_uuids:
get:
summary: get the ids of all jobs matching the given filters
@@ -8188,6 +8397,23 @@ paths:
- id
- values
/workers/queue_counts:
get:
summary: get counts of jobs waiting for an executor per tag
operationId: getCountsOfJobsWaitingPerTag
tags:
- worker
responses:
"200":
description: queue counts
content:
application/json:
schema:
type: object
additionalProperties:
type: integer
/configs/list_worker_groups:
get:
summary: list worker groups
@@ -10112,6 +10338,16 @@ components:
schemas:
$ref: "../../openflow.openapi.yaml#/components/schemas"
AiResource:
type: object
properties:
path:
type: string
provider:
type: string
required:
- path
- provider
Script:
type: object
properties:
@@ -10930,7 +11166,7 @@ components:
- "jobs.disapproval"
- "jobs.delete"
- "account.delete"
- "openai.request"
- "ai.request"
- "resources.create"
- "resources.update"
- "resources.delete"
@@ -12817,3 +13053,7 @@ components:
type: boolean
nullable: true
description: Acknowledgment status of the alert, can be true, false, or null if not set
workspace_id:
type: string
nullable: true
description: Workspace id if the alert is in the scope of a workspace
backend/windmill-api/src/ai.rs
+507
View File
@@ -0,0 +1,507 @@
use crate::{
db::{ApiAuthed, DB},
variables::decrypt,
};
use anthropic::AnthropicCache;
use axum::{
body::Bytes,
extract::{Path, Query},
response::IntoResponse,
routing::post,
Extension, Router,
};
use lazy_static::lazy_static;
use mistral::MistralCache;
use openai::OpenaiCache;
use quick_cache::sync::Cache;
use reqwest::{Client, RequestBuilder};
use serde::{Deserialize, Deserializer};
use windmill_audit::audit_ee::audit_log;
use windmill_audit::ActionKind;
use windmill_common::error::{to_anyhow, Result};
use windmill_common::variables::build_crypt;
use windmill_common::error::Error;
use serde_json::value::{RawValue, Value};
use std::collections::HashMap;
lazy_static::lazy_static! {
static ref HTTP_CLIENT: Client = reqwest::ClientBuilder::new()
.timeout(std::time::Duration::from_secs(60 * 5))
.user_agent("windmill/beta")
.build().unwrap();
}
trait AiRequest {
fn prepare_request(self, path: &str, body: Bytes) -> Result<RequestBuilder>;
}
mod openai {
use super::*;
use super::{get_variable_or_self, KeyCache};
const API_VERSION: &str = "2023-05-15";
#[derive(Deserialize, Debug)]
struct OpenaiResource {
api_key: String,
organization_id: Option<String>,
}
#[derive(Deserialize, Debug)]
struct OpenaiClientCredentialsOauthResource {
client_id: String,
client_secret: String,
token_url: String,
user: Option<String>,
}
#[derive(Deserialize, Debug)]
#[serde(untagged, rename_all = "snake_case")]
enum OpenaiConfig {
Resource(OpenaiResource),
ClientCredentialsOauthResource(OpenaiClientCredentialsOauthResource),
}
lazy_static::lazy_static! {
pub static ref OPENAI_AZURE_BASE_PATH: Option<String> = std::env::var("OPENAI_AZURE_BASE_PATH").ok();
}
#[derive(Deserialize, Debug)]
struct OpenaiCredentials {
access_token: String,
}
#[derive(Clone, Debug, Deserialize)]
pub struct OpenaiCache {
api_key: String,
organization_id: Option<String>,
azure_base_path: Option<String>,
user: Option<String>,
}
impl OpenaiCache {
pub fn new(
api_key: String,
organization_id: Option<String>,
azure_base_path: Option<String>,
user: Option<String>,
) -> Self {
Self { api_key, organization_id, azure_base_path, user }
}
}
const BASE_URL: &str = "https://api.openai.com/v1";
impl AiRequest for OpenaiCache {
fn prepare_request(self, openai_path: &str, mut body: Bytes) -> Result<RequestBuilder> {
let OpenaiCache { api_key, azure_base_path, organization_id, user } = self;
if user.is_some() {
tracing::debug!("Adding user to request body");
let mut json_body: HashMap<String, Box<RawValue>> = serde_json::from_slice(&body)
.map_err(|e| {
Error::InternalErr(format!("Failed to parse request body: {}", e))
})?;
let user_json_string = serde_json::Value::String(user.unwrap()).to_string(); // makes sure to escape characters
json_body.insert(
"user".to_string(),
RawValue::from_string(user_json_string)
.map_err(|e| Error::InternalErr(format!("Failed to parse user: {}", e)))?,
);
body = serde_json::to_vec(&json_body)
.map_err(|e| {
Error::InternalErr(format!("Failed to reserialize request body: {}", e))
})?
.into();
}
let base_url = if let Some(base_url) = azure_base_path {
base_url
} else {
BASE_URL.to_string()
};
let url = format!("{}/{}", base_url, openai_path);
let mut request = HTTP_CLIENT
.post(url)
.header("content-type", "application/json")
.body(body);
if base_url != BASE_URL {
request = request
.header("api-key", api_key)
.query(&[("api-version", API_VERSION)])
} else {
request = request.header("authorization", format!("Bearer {}", api_key))
}
if let Some(org_id) = organization_id {
request = request.header("OpenAI-Organization", org_id);
}
Ok(request)
}
}
async fn get_openai_key_using_credentials_flow(
mut resource: OpenaiClientCredentialsOauthResource,
db: &DB,
w_id: &str,
) -> Result<String> {
resource.client_id = get_variable_or_self(resource.client_id, db, w_id).await?;
resource.client_secret = get_variable_or_self(resource.client_secret, db, w_id).await?;
resource.token_url = get_variable_or_self(resource.token_url, db, w_id).await?;
let mut params = HashMap::new();
params.insert("grant_type", "client_credentials");
let response = HTTP_CLIENT
.post(resource.token_url)
.form(&params)
.basic_auth(resource.client_id, Some(resource.client_secret))
.send()
.await
.map_err(|err| {
Error::InternalErr(format!(
"Failed to get OpenAI credentials using credentials flow: {}",
err
))
})?;
let response = response.json::<OpenaiCredentials>().await.map_err(|err| {
Error::InternalErr(format!(
"Failed to parse OpenAI credentials from credentials flow: {}",
err
))
})?;
Ok(response.access_token)
}
pub async fn get_cached_value(db: &DB, w_id: &str, resource: Value) -> Result<KeyCache> {
let config = serde_json::from_value(resource)
.map_err(|e| Error::InternalErr(format!("validating openai resource {e:#}")))?;
let mut user = None::<String>;
let mut resource = match config {
OpenaiConfig::Resource(resource) => {
tracing::debug!("Getting OpenAI key from static resource");
resource
}
OpenaiConfig::ClientCredentialsOauthResource(resource) => {
tracing::debug!("Getting OpenAI key with client credentials flow");
user = resource.user.clone();
let token = get_openai_key_using_credentials_flow(resource, db, w_id).await?;
OpenaiResource { api_key: token, organization_id: None }
}
};
resource.api_key = get_variable_or_self(resource.api_key, db, w_id).await?;
if let Some(organization_id) = resource.organization_id {
resource.organization_id = Some(get_variable_or_self(organization_id, db, w_id).await?);
}
if user.is_some() {
user = Some(get_variable_or_self(user.unwrap(), db, w_id).await?);
}
let azure_base_path = sqlx::query_scalar!(
"SELECT value
FROM global_settings
WHERE name = 'openai_azure_base_path'",
)
.fetch_optional(db)
.await?;
let azure_base_path = if let Some(azure_base_path) = azure_base_path {
Some(
serde_json::from_value::<String>(azure_base_path).map_err(|e| {
Error::InternalErr(format!("validating openai azure base path {e:#}"))
})?,
)
} else {
OPENAI_AZURE_BASE_PATH.clone()
};
let workspace_cache = OpenaiCache::new(
resource.api_key.clone(),
resource.organization_id.clone(),
azure_base_path.clone(),
user.clone(),
);
Ok(KeyCache::Openai(workspace_cache))
}
}
mod anthropic {
use super::*;
#[derive(Clone, Deserialize, Debug)]
pub struct AnthropicCache {
#[serde(rename = "apiKey")]
pub api_key: String,
}
const API_VERSION: &str = "2023-06-01";
impl AnthropicCache {
pub fn new(api_key: String) -> Self {
Self { api_key }
}
}
const BASE_URL: &str = "https://api.anthropic.com";
impl AiRequest for AnthropicCache {
fn prepare_request(self, anthropic_path: &str, body: Bytes) -> Result<RequestBuilder> {
let AnthropicCache { api_key } = self;
let url = format!("{}/{}", BASE_URL, anthropic_path);
let request = HTTP_CLIENT
.post(url)
.header("x-api-key", api_key)
.header("anthropic-version", API_VERSION)
.header("content-type", "application/json")
.body(body);
Ok(request)
}
}
pub async fn get_cached_value(db: &DB, w_id: &str, resource: Value) -> Result<KeyCache> {
let mut resource: AnthropicCache = serde_json::from_value(resource)
.map_err(|e| Error::InternalErr(format!("validating anthropic resource {e:#}")))?;
resource.api_key = get_variable_or_self(resource.api_key, db, w_id).await?;
let workspace_cache = AnthropicCache::new(resource.api_key);
Ok(KeyCache::Anthropic(workspace_cache))
}
}
mod mistral {
use super::*;
#[derive(Deserialize, Clone, Debug)]
pub struct MistralCache {
#[serde(rename = "apiKey")]
pub api_key: String,
}
impl MistralCache {
pub fn new(api_key: String) -> Self {
Self { api_key }
}
}
const BASE_URL: &str = "https://api.mistral.ai";
impl AiRequest for MistralCache {
fn prepare_request(self, mistral_path: &str, body: Bytes) -> Result<RequestBuilder> {
let MistralCache { api_key } = self;
let url = format!("{}/{}", BASE_URL, mistral_path);
let request = HTTP_CLIENT
.post(url)
.header("content-type", "application/json")
.header("Accept", "application/json")
.header("authorization", format!("Bearer {}", api_key))
.body(body);
Ok(request)
}
}
pub async fn get_cached_value(db: &DB, w_id: &str, resource: Value) -> Result<KeyCache> {
let mut resource: MistralCache = serde_json::from_value(resource)
.map_err(|e| Error::InternalErr(format!("validating mistral resource {e:#}")))?;
resource.api_key = get_variable_or_self(resource.api_key, db, w_id).await?;
let workspace_cache = MistralCache::new(resource.api_key);
Ok(KeyCache::Mistral(workspace_cache))
}
}
#[derive(Clone, Debug)]
pub enum KeyCache {
Openai(OpenaiCache),
Anthropic(AnthropicCache),
Mistral(MistralCache),
}
#[derive(Clone, Debug)]
pub struct AiCache {
pub path: String,
pub cached_key: KeyCache,
pub expires_at: std::time::Instant,
}
impl AiCache {
pub fn new(path: String, cached_key: KeyCache) -> Self {
Self {
path,
cached_key,
expires_at: std::time::Instant::now() + std::time::Duration::from_secs(60),
}
}
fn is_expired(&self) -> bool {
self.expires_at < std::time::Instant::now()
}
}
lazy_static! {
pub static ref AI_KEY_CACHE: Cache<String, AiCache> = Cache::new(500);
}
struct Variable {
value: String,
is_secret: bool,
}
#[derive(Deserialize, Debug)]
struct ProxyQueryParams {
no_cache: Option<bool>,
}
async fn get_variable_or_self(path: String, db: &DB, w_id: &str) -> Result<String> {
if !path.starts_with("$var:") {
return Ok(path);
}
let path = path.strip_prefix("$var:").unwrap().to_string();
let mut variable = sqlx::query_as!(
Variable,
"SELECT value, is_secret
FROM variable
WHERE path = $1 AND workspace_id = $2",
&path,
&w_id
)
.fetch_one(db)
.await?;
if variable.is_secret {
let mc = build_crypt(db, w_id).await?;
variable.value = decrypt(&mc, variable.value)?;
}
Ok(variable.value)
}
#[derive(Deserialize, Debug)]
pub struct AiResource {
pub path: String,
#[serde(deserialize_with = "check_if_valid_ai_provider")]
pub provider: String,
}
fn check_if_valid_ai_provider<'de, D>(provider: D) -> std::result::Result<String, D::Error>
where
D: Deserializer<'de>,
{
let provider = String::deserialize(provider)?;
match provider.as_str() {
"anthropic" | "openai" | "mistral" => Ok(provider),
_ => Err(serde::de::Error::custom(
"Only the following Ai providers are supported: openai, anthropic and mistral"
.to_string(),
)),
}
}
pub fn workspaced_service() -> Router {
let router = Router::new().route("/proxy/*ai", post(proxy));
router
}
async fn proxy(
authed: ApiAuthed,
Extension(db): Extension<DB>,
Path((w_id, ai_path)): Path<(String, String)>,
Query(query_params): Query<ProxyQueryParams>,
body: Bytes,
) -> impl IntoResponse {
let workspace_cache = AI_KEY_CACHE.get(&w_id);
let ai_cache = match workspace_cache {
Some(cache) if !cache.is_expired() && !query_params.no_cache.unwrap_or(false) => {
cache.cached_key
}
_ => {
let ai_resource = sqlx::query_scalar!(
"SELECT ai_resource FROM workspace_settings WHERE workspace_id = $1",
&w_id
)
.fetch_one(&db)
.await?;
if ai_resource.is_none() {
return Err(Error::InternalErr("AI resource not configured".to_string()));
}
let ai_resource = serde_json::from_value::<AiResource>(ai_resource.unwrap())
.map_err(|e| Error::BadRequest(e.to_string()))?;
let ai_resource_path = ai_resource.path;
let resource = sqlx::query_scalar!(
"SELECT value
FROM resource
WHERE path = $1 AND workspace_id = $2",
&ai_resource_path,
&w_id
)
.fetch_optional(&db)
.await?
.ok_or_else(|| {
Error::InternalErr(format!(
"Could not find the {} resource at path {ai_resource_path}, update the resource path in the workspace settings", ai_resource.provider
))
})?;
if resource.is_none() {
return Err(Error::InternalErr(format!(
"{} resource missing value",
ai_resource.provider
)));
}
let resource = resource.unwrap();
let ai_cache = match ai_resource.provider.as_str() {
"openai" => openai::get_cached_value(&db, &w_id, resource).await,
"anthropic" => anthropic::get_cached_value(&db, &w_id, resource).await,
"mistral" => mistral::get_cached_value(&db, &w_id, resource).await,
provider => {
return Err(Error::BadRequest(format!("{} is not supported", provider)))
}
};
let ai_cache = ai_cache?;
AI_KEY_CACHE.insert(
w_id.clone(),
AiCache::new(ai_resource_path, ai_cache.clone()),
);
ai_cache
}
};
let (path, request) = match ai_cache {
KeyCache::Openai(cached) => ("openai_path", cached.prepare_request(&ai_path, body)),
KeyCache::Anthropic(cached) => ("anthropic_path", cached.prepare_request(&ai_path, body)),
KeyCache::Mistral(cached) => ("mistral_path", cached.prepare_request(&ai_path, body)),
};
let response = request?.send().await.map_err(to_anyhow)?;
let mut tx = db.begin().await?;
audit_log(
&mut *tx,
&authed,
"ai.request",
ActionKind::Execute,
&w_id,
Some(&authed.email),
Some([(path, &format!("{:?}", ai_path)[..])].into()),
)
.await?;
tx.commit().await?;
if response.error_for_status_ref().is_err() {
let err_msg = response.text().await.unwrap_or("".to_string());
return Err(Error::AiError(err_msg));
}
let status_code = response.status();
let headers = response.headers().clone();
let stream = response.bytes_stream();
Ok((status_code, headers, axum::body::Body::from_stream(stream)))
}
backend/windmill-api/src/jobs.rs
+236 -192
View File
@@ -12,6 +12,7 @@ use quick_cache::sync::Cache;
use serde_json::value::RawValue;
use sqlx::Pool;
use std::collections::HashMap;
use std::ops::{Deref, DerefMut};
#[cfg(feature = "prometheus")]
use std::sync::atomic::Ordering;
use tokio::io::AsyncReadExt;
@@ -205,6 +206,7 @@ pub fn workspaced_service() -> Router {
.route("/queue/list_filtered_uuids", get(list_filtered_uuids))
.route("/queue/cancel_selection", post(cancel_selection))
.route("/completed/count", get(count_completed_jobs))
.route("/completed/count_jobs", get(count_completed_jobs_detail))
.route(
"/completed/list",
get(list_completed_jobs).layer(cors.clone()),
@@ -600,7 +602,7 @@ async fn get_flow_job_debug_info(
Extension(db): Extension<DB>,
Path((w_id, id)): Path<(String, Uuid)>,
) -> error::Result<Response> {
let job = get_queued_job(&id, w_id.as_str(), &db).await?;
let job = get_queued_job_ex(&db, &w_id, id, false, None).await?;
if let Some(job) = job {
let is_flow = job.is_flow();
if job.is_flow_step || !is_flow {
@@ -734,6 +736,67 @@ fn generate_get_job_query(no_logs: bool, table: &str) -> String {
{join}
WHERE id = $1 AND {table}.workspace_id = $2");
}
pub async fn get_queued_job_ex(
db: &DB,
workspace_id: &str,
job_id: Uuid,
no_logs: bool,
// first optional is if authed need to be checked, second is the opt_authed itself
opt_authed: Option<&Option<ApiAuthed>>,
) -> error::Result<Option<JobExtended<QueuedJob>>> {
let query = if no_logs { &*GET_QUEUED_JOB_QUERY_NO_LOGS } else { &*GET_QUEUED_JOB_QUERY };
let job = sqlx::query_as::<_, JobExtended<QueuedJob>>(query)
.bind(job_id)
.bind(workspace_id)
.fetch_optional(db)
.await?;
if let Some(job) = job.as_ref() {
if opt_authed.is_some_and(|x| x.is_none()) && job.created_by != "anonymous" {
return Err(Error::BadRequest(
"As a non logged in user, you can only see jobs ran by anonymous users".to_string(),
));
}
}
Ok(job)
}
pub async fn get_completed_job_ex(
db: &DB,
workspace_id: &str,
job_id: Uuid,
no_logs: bool,
// first optional is if authed need to be checked, second is the opt_authed itself
opt_authed: Option<&Option<ApiAuthed>>,
) -> error::Result<Option<JobExtended<CompletedJob>>> {
let query = if no_logs { &*GET_COMPLETED_JOB_QUERY_NO_LOGS } else { &*GET_COMPLETED_JOB_QUERY };
let cjob = sqlx::query_as::<_, JobExtended<CompletedJob>>(query)
.bind(job_id)
.bind(workspace_id)
.fetch_optional(db)
.await?;
if let Some(job) = cjob.as_ref() {
if opt_authed.is_some_and(|x| x.is_none()) && job.created_by != "anonymous" {
return Err(Error::BadRequest(
"As a non logged in user, you can only see jobs ran by anonymous users".to_string(),
));
}
}
if let Some(mut cjob) = cjob {
let CompletedJobWithFormattedResult { mut cj, result } = format_completed_job_result(cjob.inner);
cj.result = match result {
Some(FormattedResult::RawValue(rv)) => rv,
Some(FormattedResult::Vec(v)) => Some(to_raw_value(&v)),
None => None,
}.map(sqlx::types::Json);
cjob.inner = cj;
return Ok(Some(cjob));
}
Ok(cjob)
}
pub async fn get_job_internal(
db: &DB,
workspace_id: &str,
@@ -742,48 +805,18 @@ pub async fn get_job_internal(
// first optional is if authed need to be checked, second is the opt_authed itself
opt_authed: Option<&Option<ApiAuthed>>,
) -> error::Result<Job> {
let cjob_maybe = sqlx::query_as::<_, CompletedJob>(if no_logs {
&*GET_COMPLETED_JOB_QUERY_NO_LOGS
} else {
&*GET_COMPLETED_JOB_QUERY
})
.bind(job_id)
.bind(workspace_id)
.fetch_optional(db)
.await?
.map(Job::CompletedJob);
if let Some(cjob) = cjob_maybe {
Ok(match cjob {
Job::CompletedJob(cjob) => {
if opt_authed.is_some_and(|x| x.is_none()) && cjob.created_by != "anonymous" {
return Err(Error::BadRequest(
"As a non logged in user, you can only see jobs ran by anonymous users"
.to_string(),
));
}
Job::CompletedJobWithFormattedResult(format_completed_job_result(cjob))
}
cjob => cjob,
})
} else {
let job_o = sqlx::query_as::<_, QueuedJob>(if no_logs {
&*GET_QUEUED_JOB_QUERY_NO_LOGS
} else {
&*GET_QUEUED_JOB_QUERY
})
.bind(job_id)
.bind(workspace_id)
.fetch_optional(db)
let cjob = get_completed_job_ex(db, workspace_id, job_id, no_logs, opt_authed.clone())
.await?
.map(Job::QueuedJob);
let job: Job = not_found_if_none(job_o, "Job", job_id.to_string())?;
if opt_authed.is_some_and(|x| x.is_none()) && job.created_by() != "anonymous" {
return Err(Error::BadRequest(
"As a non logged in user, you can only see jobs ran by anonymous users".to_string(),
));
.map(Job::CompletedJob);
match cjob {
Some(cjob) => Ok(cjob),
None => {
let job_maybe = get_queued_job_ex(db, workspace_id, job_id, no_logs, opt_authed)
.await?
.map(Job::QueuedJob);
not_found_if_none(job_maybe, "Job", job_id.to_string())
}
Ok(job)
}
}
@@ -917,7 +950,7 @@ async fn get_job_logs(
Ok(content_plain(Body::from(logs)))
} else {
let text = sqlx::query!(
"SELECT created_by, CONCAT(coalesce(queue.logs, ''), coalesce(job_logs.logs, '')) as logs, job_logs.log_offset, job_logs.log_file_index
"SELECT created_by, CONCAT(coalesce(queue.logs, ''), coalesce(job_logs.logs, '')) as logs, coalesce(job_logs.log_offset, 0) as log_offset, job_logs.log_file_index
FROM queue
LEFT JOIN job_logs ON job_logs.job_id = queue.id
WHERE queue.id = $1 AND queue.workspace_id = $2",
@@ -938,10 +971,10 @@ async fn get_job_logs(
log_job_view(&db, opt_authed.as_ref(), &w_id, &id).await?;
#[cfg(all(feature = "enterprise", feature = "parquet"))]
if let Some(r) = get_logs_from_store(text.log_offset, &logs, &text.log_file_index).await {
if let Some(r) = get_logs_from_store(text.log_offset.unwrap_or(0), &logs, &text.log_file_index).await {
return r.map(content_plain);
}
if let Some(r) = get_logs_from_disk(text.log_offset, &logs, &text.log_file_index).await {
if let Some(r) = get_logs_from_disk(text.log_offset.unwrap_or(0), &logs, &text.log_file_index).await {
return r.map(content_plain);
}
@@ -1560,6 +1593,50 @@ async fn count_queue_jobs(
))
}
#[derive(Deserialize)]
pub struct CountCompletedJobsQuery {
completed_after_s_ago: Option<i64>,
success: Option<bool>,
tags: Option<String>,
all_workspaces: Option<bool>,
}
async fn count_completed_jobs_detail(
Extension(db): Extension<DB>,
Path(w_id): Path<String>,
Query(query): Query<CountCompletedJobsQuery>,
) -> error::JsonResult<i64> {
let mut sqlb = SqlBuilder::select_from("completed_job");
sqlb
.field("COUNT(*) as count");
if !query.all_workspaces.unwrap_or(false) {
sqlb.and_where_eq("workspace_id", "?".bind(&w_id));
}
if let Some(after_s_ago) = query.completed_after_s_ago {
let after = Utc::now() - chrono::Duration::seconds(after_s_ago);
sqlb.and_where_gt("started_at + duration_ms / 1000 * interval '1 second'", "?".bind(&after.to_rfc3339()));
}
if let Some(success) = query.success {
sqlb.and_where_eq("success", "?".bind(&success));
}
if let Some(tags) = query.tags {
sqlb.and_where_in("tag", &tags.split(",").map(|t| format!("'{}'", t)).collect::<Vec<_>>());
}
let sql = sqlb.sql()?;
let stats = sqlx::query_scalar::<_, i64>(&sql)
.fetch_one(&db)
.await?;
Ok(Json(stats))
}
async fn count_completed_jobs(
Extension(db): Extension<DB>,
Path(w_id): Path<String>,
@@ -1735,7 +1812,6 @@ async fn resume_suspended_job_internal(
let trigger_email = match &parent_flow {
Job::CompletedJob(job) => &job.email,
Job::QueuedJob(job) => &job.email,
Job::CompletedJobWithFormattedResult(job) => &job.cj.email,
};
conditionally_require_authed_user(authed.clone(), flow_status, trigger_email)?;
@@ -2010,7 +2086,6 @@ pub async fn get_suspended_job_flow(
let trigger_email = match &flow {
Job::CompletedJob(job) => &job.email,
Job::QueuedJob(job) => &job.email,
Job::CompletedJobWithFormattedResult(job) => &job.cj.email,
};
conditionally_require_authed_user(authed.clone(), flow_status.clone(), trigger_email)?;
@@ -2225,13 +2300,45 @@ pub async fn get_resume_urls(
Ok(Json(res))
}
#[derive(sqlx::FromRow, Debug, Serialize)]
pub struct JobExtended<T> {
#[sqlx(flatten)]
#[serde(flatten)]
inner: T,
#[sqlx(skip)]
#[serde(skip_serializing_if = "Option::is_none")]
pub self_wait_time_ms: Option<i64>,
#[sqlx(skip)]
#[serde(skip_serializing_if = "Option::is_none")]
pub aggregate_wait_time_ms: Option<i64>,
}
impl<T> JobExtended<T> {
pub fn new(self_wait_time_ms: Option<i64>, aggregate_wait_time_ms: Option<i64>, inner: T) -> Self {
Self { inner, self_wait_time_ms, aggregate_wait_time_ms }
}
}
impl<T> Deref for JobExtended<T> {
type Target = T;
fn deref(&self) -> &Self::Target {
&self.inner
}
}
impl<T> DerefMut for JobExtended<T> {
fn deref_mut(&mut self) -> &mut Self::Target {
&mut self.inner
}
}
#[derive(Serialize, Debug)]
#[serde(tag = "type")]
pub enum Job {
QueuedJob(QueuedJob),
CompletedJob(CompletedJob),
#[serde(rename = "CompletedJob")]
CompletedJobWithFormattedResult(CompletedJobWithFormattedResult),
QueuedJob(JobExtended<QueuedJob>),
CompletedJob(JobExtended<CompletedJob>),
}
impl Job {
@@ -2239,27 +2346,6 @@ impl Job {
match self {
Job::QueuedJob(job) => &job.created_by,
Job::CompletedJob(job) => &job.created_by,
Job::CompletedJobWithFormattedResult(job) => &job.cj.created_by,
}
}
pub fn raw_flow(&self) -> Option<FlowValue> {
match self {
Job::QueuedJob(job) => job
.raw_flow
.as_ref()
.map(|rf| serde_json::from_str(rf.0.get()).ok())
.flatten(),
Job::CompletedJob(job) => job
.raw_flow
.as_ref()
.map(|rf| serde_json::from_str(rf.0.get()).ok())
.flatten(),
Job::CompletedJobWithFormattedResult(job) => job
.cj
.raw_flow
.as_ref()
.map(|rf| serde_json::from_str(rf.0.get()).ok())
.flatten(),
}
}
@@ -2279,13 +2365,6 @@ impl Job {
job.logs = Some(logs.to_string());
}
}
Job::CompletedJobWithFormattedResult(job) => {
if let Some(ref mut l) = job.cj.logs {
l.push_str(logs);
} else {
job.cj.logs = Some(logs.to_string());
}
}
}
}
@@ -2293,7 +2372,6 @@ impl Job {
match self {
Job::QueuedJob(job) => job.logs.as_ref().map(|l| l.len()),
Job::CompletedJob(job) => job.logs.as_ref().map(|l| l.len()),
Job::CompletedJobWithFormattedResult(job) => job.cj.logs.as_ref().map(|l| l.len()),
}
}
@@ -2301,7 +2379,6 @@ impl Job {
match self {
Job::QueuedJob(job) => job.logs.clone(),
Job::CompletedJob(job) => job.logs.clone(),
Job::CompletedJobWithFormattedResult(job) => job.cj.logs.clone(),
}
}
pub fn flow_status(&self) -> Option<FlowStatus> {
@@ -2316,19 +2393,12 @@ impl Job {
.as_ref()
.map(|rf| serde_json::from_str(rf.0.get()).ok())
.flatten(),
Job::CompletedJobWithFormattedResult(job) => job
.cj
.flow_status
.as_ref()
.map(|rf| serde_json::from_str(rf.0.get()).ok())
.flatten(),
}
}
pub fn is_flow_step(&self) -> bool {
match self {
Job::QueuedJob(job) => job.is_flow_step,
Job::CompletedJob(job) => job.is_flow_step,
Job::CompletedJobWithFormattedResult(job) => job.cj.is_flow_step,
}
}
@@ -2343,7 +2413,6 @@ impl Job {
match self {
Job::QueuedJob(job) => &job.job_kind,
Job::CompletedJob(job) => &job.job_kind,
Job::CompletedJobWithFormattedResult(job) => &job.cj.job_kind,
}
}
@@ -2351,7 +2420,6 @@ impl Job {
match self {
Job::QueuedJob(job) => job.id,
Job::CompletedJob(job) => job.id,
Job::CompletedJobWithFormattedResult(job) => job.cj.id,
}
}
@@ -2359,7 +2427,6 @@ impl Job {
match self {
Job::QueuedJob(job) => &job.workspace_id,
Job::CompletedJob(job) => &job.workspace_id,
Job::CompletedJobWithFormattedResult(job) => &job.cj.workspace_id,
}
}
@@ -2367,7 +2434,6 @@ impl Job {
match self {
Job::QueuedJob(job) => job.script_path.as_ref(),
Job::CompletedJob(job) => job.script_path.as_ref(),
Job::CompletedJobWithFormattedResult(job) => job.cj.script_path.as_ref(),
}
.map(String::as_str)
.unwrap_or("tmp/main")
@@ -2377,7 +2443,6 @@ impl Job {
match self {
Job::QueuedJob(job) => job.args.as_ref(),
Job::CompletedJob(job) => job.args.as_ref(),
Job::CompletedJobWithFormattedResult(job) => job.cj.args.as_ref(),
}
}
@@ -2426,10 +2491,6 @@ impl Job {
job.self_wait_time_ms = self_wait_time;
job.aggregate_wait_time_ms = aggregate_wait_time;
}
Job::CompletedJobWithFormattedResult(job) => {
job.cj.self_wait_time_ms = self_wait_time;
job.cj.aggregate_wait_time_ms = aggregate_wait_time;
}
}
Ok(())
}
@@ -2557,86 +2618,82 @@ impl UnifiedJob {
impl<'a> From<UnifiedJob> for Job {
fn from(uj: UnifiedJob) -> Self {
match uj.typ.as_ref() {
"CompletedJob" => Job::CompletedJob(CompletedJob {
workspace_id: uj.workspace_id,
id: uj.id,
parent_job: uj.parent_job,
created_by: uj.created_by,
created_at: uj.created_at,
started_at: uj.started_at.unwrap_or(uj.created_at),
duration_ms: uj.duration_ms.unwrap(),
success: uj.success.unwrap(),
script_hash: uj.script_hash,
script_path: uj.script_path,
args: None,
result: None,
logs: None,
flow_status: None,
deleted: uj.deleted,
canceled: uj.canceled,
canceled_by: uj.canceled_by,
raw_code: None,
canceled_reason: None,
job_kind: uj.job_kind,
schedule_path: uj.schedule_path,
permissioned_as: uj.permissioned_as,
raw_flow: None,
is_flow_step: uj.is_flow_step,
language: uj.language,
is_skipped: uj.is_skipped,
email: uj.email,
visible_to_owner: uj.visible_to_owner,
mem_peak: uj.mem_peak,
tag: uj.tag,
priority: uj.priority,
labels: uj.labels,
self_wait_time_ms: uj.self_wait_time_ms,
aggregate_wait_time_ms: uj.aggregate_wait_time_ms,
}),
"QueuedJob" => Job::QueuedJob(QueuedJob {
workspace_id: uj.workspace_id,
id: uj.id,
parent_job: uj.parent_job,
created_by: uj.created_by,
created_at: uj.created_at,
started_at: uj.started_at,
script_hash: uj.script_hash,
script_path: uj.script_path,
args: None,
running: uj.running.unwrap(),
scheduled_for: uj.scheduled_for.unwrap(),
logs: None,
flow_status: None,
raw_code: None,
raw_lock: None,
canceled: uj.canceled,
canceled_by: uj.canceled_by,
canceled_reason: None,
last_ping: None,
job_kind: uj.job_kind,
schedule_path: uj.schedule_path,
permissioned_as: uj.permissioned_as,
raw_flow: None,
is_flow_step: uj.is_flow_step,
language: uj.language,
same_worker: false,
pre_run_error: None,
email: uj.email,
visible_to_owner: uj.visible_to_owner,
suspend: uj.suspend,
mem_peak: uj.mem_peak,
root_job: None,
leaf_jobs: None,
tag: uj.tag,
concurrent_limit: uj.concurrent_limit,
concurrency_time_window_s: uj.concurrency_time_window_s,
timeout: None,
flow_step_id: None,
cache_ttl: None,
priority: uj.priority,
self_wait_time_ms: uj.self_wait_time_ms,
aggregate_wait_time_ms: uj.aggregate_wait_time_ms,
}),
"CompletedJob" => Job::CompletedJob(JobExtended::new(uj.self_wait_time_ms, uj.aggregate_wait_time_ms, CompletedJob {
workspace_id: uj.workspace_id,
id: uj.id,
parent_job: uj.parent_job,
created_by: uj.created_by,
created_at: uj.created_at,
started_at: uj.started_at.unwrap_or(uj.created_at),
duration_ms: uj.duration_ms.unwrap(),
success: uj.success.unwrap(),
script_hash: uj.script_hash,
script_path: uj.script_path,
args: None,
result: None,
logs: None,
flow_status: None,
deleted: uj.deleted,
canceled: uj.canceled,
canceled_by: uj.canceled_by,
raw_code: None,
canceled_reason: None,
job_kind: uj.job_kind,
schedule_path: uj.schedule_path,
permissioned_as: uj.permissioned_as,
raw_flow: None,
is_flow_step: uj.is_flow_step,
language: uj.language,
is_skipped: uj.is_skipped,
email: uj.email,
visible_to_owner: uj.visible_to_owner,
mem_peak: uj.mem_peak,
tag: uj.tag,
priority: uj.priority,
labels: uj.labels,
})),
"QueuedJob" => Job::QueuedJob(JobExtended::new(uj.self_wait_time_ms, uj.aggregate_wait_time_ms, QueuedJob {
workspace_id: uj.workspace_id,
id: uj.id,
parent_job: uj.parent_job,
created_by: uj.created_by,
created_at: uj.created_at,
started_at: uj.started_at,
script_hash: uj.script_hash,
script_path: uj.script_path,
args: None,
running: uj.running.unwrap(),
scheduled_for: uj.scheduled_for.unwrap(),
logs: None,
flow_status: None,
raw_code: None,
raw_lock: None,
canceled: uj.canceled,
canceled_by: uj.canceled_by,
canceled_reason: None,
last_ping: None,
job_kind: uj.job_kind,
schedule_path: uj.schedule_path,
permissioned_as: uj.permissioned_as,
raw_flow: None,
is_flow_step: uj.is_flow_step,
language: uj.language,
same_worker: false,
pre_run_error: None,
email: uj.email,
visible_to_owner: uj.visible_to_owner,
suspend: uj.suspend,
mem_peak: uj.mem_peak,
root_job: None,
leaf_jobs: None,
tag: uj.tag,
concurrent_limit: uj.concurrent_limit,
concurrency_time_window_s: uj.concurrency_time_window_s,
timeout: None,
flow_step_id: None,
cache_ttl: None,
priority: uj.priority,
})),
t => panic!("job type {} not valid", t),
}
}
@@ -3079,7 +3136,7 @@ pub async fn run_workflow_as_code(
i += 1;
}
let job = get_queued_job(&job_id, &w_id, &db).await?;
let job = get_queued_job_ex(&db, &w_id, job_id, true, None).await?;
if *CLOUD_HOSTED {
tracing::info!("workflow_as_code_tracing id {i} ");
@@ -3087,6 +3144,7 @@ pub async fn run_workflow_as_code(
}
let job = not_found_if_none(job, "Queued Job", &job_id.to_string())?;
let JobExtended { inner: job, .. } = job;
let (job_payload, tag, _delete_after_use, timeout) = match job.job_kind {
JobKind::Preview => (
JobPayload::Code(RawCode {
@@ -5058,24 +5116,10 @@ async fn get_completed_job<'a>(
Extension(db): Extension<DB>,
Path((w_id, id)): Path<(String, Uuid)>,
) -> error::Result<Response> {
let job_o = sqlx::query_as::<_, CompletedJob>("SELECT id, workspace_id, parent_job, created_by, created_at, duration_ms, success, script_hash, script_path,
CASE WHEN args is null or pg_column_size(args) < 90000 THEN args ELSE '\"WINDMILL_TOO_BIG\"'::jsonb END as args, CASE WHEN result is null or pg_column_size(result) < 90000 THEN result ELSE '\"WINDMILL_TOO_BIG\"'::jsonb END as result, logs, deleted, raw_code, canceled, canceled_by, canceled_reason, job_kind,
schedule_path, permissioned_as, flow_status, raw_flow, is_flow_step, language, started_at, is_skipped,
raw_lock, email, visible_to_owner, mem_peak, tag, priority, result->'wm_labels' as labels FROM completed_job WHERE id = $1 AND workspace_id = $2")
.bind(id)
.bind(&w_id)
.fetch_optional(&db)
let job_o = get_completed_job_ex(&db, &w_id, id, false, Some(&opt_authed))
.await?;
let cj = not_found_if_none(job_o, "Completed Job", id.to_string())?;
if opt_authed.is_none() && cj.created_by != "anonymous" {
return Err(Error::BadRequest(
"As a non logged in user, you can only see jobs ran by anonymous users".to_string(),
));
}
let cj = format_completed_job_result(cj);
let response = Json(cj).into_response();
// let extra_log = query_scalar!(
// "SELECT substr(logs, $1) as logs FROM large_logs WHERE workspace_id = $2 AND job_id = $3",
backend/windmill-api/src/lib.rs
+5 -7
View File
@@ -25,7 +25,6 @@ use argon2::Argon2;
use axum::extract::DefaultBodyLimit;
use axum::{middleware::from_extractor, routing::get, Extension, Router};
use db::DB;
use git_version::git_version;
use http::HeaderValue;
use reqwest::Client;
use std::collections::HashMap;
@@ -40,7 +39,7 @@ use tower_http::{
};
use windmill_common::db::UserDB;
use windmill_common::worker::{ALL_TAGS, CLOUD_HOSTED};
use windmill_common::{BASE_URL, INSTANCE_NAME};
use windmill_common::{BASE_URL, INSTANCE_NAME, utils::GIT_VERSION};
use crate::scim_ee::has_scim_token;
use windmill_common::error::AppError;
@@ -63,13 +62,14 @@ mod http_triggers;
mod indexer_ee;
mod inputs;
mod integration;
mod ai;
#[cfg(feature = "parquet")]
mod job_helpers_ee;
pub mod job_metrics;
pub mod jobs;
pub mod oauth2_ee;
mod oidc_ee;
mod openai;
mod raw_apps;
mod resources;
mod saml_ee;
@@ -93,11 +93,9 @@ mod workers;
mod workspaces;
mod workspaces_ee;
pub const GIT_VERSION: &str =
git_version!(args = ["--tag", "--always"], fallback = "unknown-version");
pub const DEFAULT_BODY_LIMIT: usize = 2097152 * 100; // 200MB
lazy_static::lazy_static! {
pub static ref REQUEST_SIZE_LIMIT: Arc<RwLock<usize>> = Arc::new(RwLock::new(DEFAULT_BODY_LIMIT));
@@ -282,7 +280,7 @@ pub async fn run_server(
.nest("/job_helpers", job_helpers_service)
.nest("/jobs", jobs::workspaced_service())
.nest("/oauth", oauth2_ee::workspaced_service())
.nest("/openai", openai::workspaced_service())
.nest("/ai", ai::workspaced_service())
.nest("/raw_apps", raw_apps::workspaced_service())
.nest("/resources", resources::workspaced_service())
.nest("/schedules", schedule::workspaced_service())
backend/windmill-api/src/openai.rs
-344
View File
@@ -1,344 +0,0 @@
use std::collections::HashMap;
use crate::db::{ApiAuthed, DB};
use axum::{
body::Bytes,
extract::{Extension, Path, Query},
response::IntoResponse,
routing::post,
Router,
};
use quick_cache::sync::Cache;
use reqwest::Client;
use serde_json::value::RawValue;
use windmill_audit::audit_ee::audit_log;
use windmill_audit::ActionKind;
use windmill_common::{
error::{to_anyhow, Error},
variables::build_crypt,
};
use crate::variables::decrypt;
use serde::Deserialize;
lazy_static::lazy_static! {
static ref HTTP_CLIENT: Client = reqwest::ClientBuilder::new()
.timeout(std::time::Duration::from_secs(60 * 5))
.user_agent("windmill/beta")
.build().unwrap();
}
pub fn workspaced_service() -> Router {
let router = Router::new().route("/proxy/*openai_path", post(proxy));
router
}
#[derive(Deserialize)]
struct OpenaiResource {
api_key: String,
organization_id: Option<String>,
}
#[derive(Deserialize)]
struct OpenaiClientCredentialsOauthResource {
client_id: String,
client_secret: String,
token_url: String,
user: Option<String>,
}
#[derive(Deserialize)]
#[serde(untagged)]
enum OpenaiConfig {
Resource(OpenaiResource),
ClientCredentialsOauthResource(OpenaiClientCredentialsOauthResource),
}
struct Variable {
value: String,
is_secret: bool,
}
async fn get_variable_or_self(path: String, db: &DB, w_id: &String) -> Result<String, Error> {
if !path.starts_with("$var:") {
return Ok(path);
}
let path = path.strip_prefix("$var:").unwrap().to_string();
let mut variable = sqlx::query_as!(
Variable,
"SELECT value, is_secret
FROM variable
WHERE path = $1 AND workspace_id = $2",
&path,
&w_id
)
.fetch_one(db)
.await?;
if variable.is_secret {
let mc = build_crypt(&db, &w_id).await?;
variable.value = decrypt(&mc, variable.value)?;
}
Ok(variable.value)
}
lazy_static::lazy_static! {
pub static ref OPENAI_AZURE_BASE_PATH: Option<String> = std::env::var("OPENAI_AZURE_BASE_PATH").ok();
static ref OPENAI_KEY_CACHE: Cache<String, OpenaiKeyCache> = Cache::new(500);
}
#[derive(Deserialize)]
struct OpenaiCredentials {
access_token: String,
}
async fn get_openai_key_using_credentials_flow(
mut resource: OpenaiClientCredentialsOauthResource,
db: &DB,
w_id: &String,
) -> Result<String, Error> {
resource.client_id = get_variable_or_self(resource.client_id, &db, &w_id).await?;
resource.client_secret = get_variable_or_self(resource.client_secret, &db, &w_id).await?;
resource.token_url = get_variable_or_self(resource.token_url, &db, &w_id).await?;
let mut params = HashMap::new();
params.insert("grant_type", "client_credentials");
let response = HTTP_CLIENT
.post(resource.token_url)
.form(&params)
.basic_auth(resource.client_id, Some(resource.client_secret))
.send()
.await
.map_err(|err| {
Error::InternalErr(format!(
"Failed to get OpenAI credentials using credentials flow: {}",
err
))
})?;
let response = response.json::<OpenaiCredentials>().await.map_err(|err| {
Error::InternalErr(format!(
"Failed to parse OpenAI credentials from credentials flow: {}",
err
))
})?;
Ok(response.access_token)
}
#[derive(Clone)]
struct OpenaiKeyCache {
api_key: String,
organization_id: Option<String>,
azure_base_path: Option<String>,
user: Option<String>,
expires_at: std::time::Instant,
}
impl OpenaiKeyCache {
fn new(
api_key: String,
organization_id: Option<String>,
azure_base_path: Option<String>,
expires_at: std::time::Instant,
user: Option<String>,
) -> Self {
Self { api_key, organization_id, azure_base_path, expires_at, user }
}
fn is_expired(&self) -> bool {
self.expires_at < std::time::Instant::now()
}
}
#[derive(Deserialize)]
struct ProxyQueryParams {
no_cache: Option<bool>,
}
async fn proxy(
authed: ApiAuthed,
Extension(db): Extension<DB>,
Path((w_id, openai_path)): Path<(String, String)>,
Query(query_params): Query<ProxyQueryParams>,
mut body: Bytes,
) -> impl IntoResponse {
let workspace_cache = OPENAI_KEY_CACHE.get(&w_id);
let (api_key, organization_id, azure_base_path, user) = if query_params
.no_cache
.unwrap_or(false)
|| workspace_cache.is_none()
|| workspace_cache.clone().unwrap().is_expired()
{
let openai_resource_path = sqlx::query_scalar!(
"SELECT openai_resource_path FROM workspace_settings WHERE workspace_id = $1",
&w_id
)
.fetch_one(&db)
.await?;
if openai_resource_path.is_none() {
return Err(Error::InternalErr(
"OpenAI resource not configured".to_string(),
));
}
let openai_resource_path = openai_resource_path.unwrap();
let resource = sqlx::query_scalar!(
"SELECT value
FROM resource
WHERE path = $1 AND workspace_id = $2",
&openai_resource_path,
&w_id
)
.fetch_optional(&db)
.await?
.ok_or_else(|| {
Error::InternalErr(format!(
"Could not find the OpenAI resource at path {openai_resource_path}, update the resource path in the workspace settings"
))
})?;
if resource.is_none() {
return Err(Error::InternalErr(
"OpenAI resource missing value".to_string(),
));
}
let config: OpenaiConfig = serde_json::from_value(resource.unwrap())
.map_err(|e| Error::InternalErr(format!("validating openai resource {e:#}")))?;
let mut user = None::<String>;
let mut resource = match config {
OpenaiConfig::Resource(resource) => {
tracing::debug!("Getting OpenAI key from static resource");
resource
}
OpenaiConfig::ClientCredentialsOauthResource(resource) => {
tracing::debug!("Getting OpenAI key with client credentials flow");
user = resource.user.clone();
let token = get_openai_key_using_credentials_flow(resource, &db, &w_id).await?;
OpenaiResource { api_key: token, organization_id: None }
}
};
resource.api_key = get_variable_or_self(resource.api_key, &db, &w_id).await?;
if resource.organization_id.is_some() {
resource.organization_id =
Some(get_variable_or_self(resource.organization_id.unwrap(), &db, &w_id).await?);
}
if user.is_some() {
user = Some(get_variable_or_self(user.unwrap(), &db, &w_id).await?);
}
let expires_at = std::time::Instant::now() + std::time::Duration::from_secs(60);
let azure_base_path = sqlx::query_scalar!(
"SELECT value
FROM global_settings
WHERE name = 'openai_azure_base_path'",
)
.fetch_optional(&db)
.await?;
let azure_base_path = if let Some(azure_base_path) = azure_base_path {
Some(
serde_json::from_value::<String>(azure_base_path).map_err(|e| {
Error::InternalErr(format!("validating openai azure base path {e:#}"))
})?,
)
} else {
OPENAI_AZURE_BASE_PATH.clone()
};
let workspace_cache = OpenaiKeyCache::new(
resource.api_key.clone(),
resource.organization_id.clone(),
azure_base_path.clone(),
expires_at,
user.clone(),
);
OPENAI_KEY_CACHE.insert(w_id.clone(), workspace_cache);
(
resource.api_key,
resource.organization_id,
azure_base_path,
user,
)
} else {
tracing::debug!("Using cached OpenAI key");
let workspace_cache = workspace_cache.unwrap();
(
workspace_cache.api_key.clone(),
workspace_cache.organization_id.clone(),
workspace_cache.azure_base_path.clone(),
workspace_cache.user.clone(),
)
};
if user.is_some() {
tracing::debug!("Adding user to request body");
let mut json_body: HashMap<String, Box<RawValue>> = serde_json::from_slice(&body)
.map_err(|e| Error::InternalErr(format!("Failed to parse request body: {}", e)))?;
let user_json_string = serde_json::Value::String(user.unwrap()).to_string(); // makes sure to escape characters
json_body.insert(
"user".to_string(),
RawValue::from_string(user_json_string)
.map_err(|e| Error::InternalErr(format!("Failed to parse user: {}", e)))?,
);
body = serde_json::to_vec(&json_body)
.map_err(|e| Error::InternalErr(format!("Failed to reserialize request body: {}", e)))?
.into();
}
let base_url = if let Some(base_url) = azure_base_path {
base_url
} else {
"https://api.openai.com/v1".to_string()
};
let url = format!("{}/{}", base_url, openai_path);
let mut request = HTTP_CLIENT
.post(url)
.header("content-type", "application/json")
.body(body);
if base_url != "https://api.openai.com/v1" {
request = request
.header("api-key", api_key)
.query(&[("api-version", "2023-05-15")])
} else {
request = request.header("authorization", format!("Bearer {}", api_key))
}
if let Some(org_id) = organization_id {
request = request.header("OpenAI-Organization", org_id);
}
let response = request.send().await.map_err(to_anyhow)?;
let mut tx = db.begin().await?;
audit_log(
&mut *tx,
&authed,
"openai.request",
ActionKind::Execute,
&w_id,
Some(&authed.email),
Some([("openai_path", &format!("{:?}", openai_path)[..])].into()),
)
.await?;
tx.commit().await?;
if response.error_for_status_ref().is_err() {
return Err(Error::OpenAIError(
response.text().await.unwrap_or("".to_string()),
));
}
let status_code = response.status();
let headers = response.headers().clone();
let stream = response.bytes_stream();
Ok((status_code, headers, axum::body::Body::from_stream(stream)))
}
backend/windmill-api/src/resources.rs
+6 -5
View File
@@ -300,7 +300,7 @@ async fn get_resource(
.await?;
tx.commit().await?;
if resource_o.is_none() {
explain_resource_perm_error(&path, &w_id, &db).await?;
explain_resource_perm_error(&path, &w_id, &db, &authed).await?;
}
let resource = not_found_if_none(resource_o, "Resource", path)?;
Ok(Json(resource))
@@ -343,7 +343,7 @@ async fn get_resource_value(
tx.commit().await?;
if value_o.is_none() {
explain_resource_perm_error(&path, &w_id, &db).await?;
explain_resource_perm_error(&path, &w_id, &db, &authed).await?;
}
let value = not_found_if_none(value_o, "Resource", path)?;
@@ -354,6 +354,7 @@ async fn explain_resource_perm_error(
path: &str,
w_id: &str,
db: &sqlx::Pool<Postgres>,
authed: &ApiAuthed,
) -> windmill_common::error::Result<()> {
let extra_perms = sqlx::query_scalar!(
"SELECT extra_perms from resource WHERE path = $1 AND workspace_id = $2",
@@ -378,12 +379,12 @@ async fn explain_resource_perm_error(
.fetch_optional(db)
.await?;
return Err(Error::NotAuthorized(format!(
"Resource exists but you don't have access to it:\nresource perms: {}\nfolder perms: {}",
"Resource exists but you don't have access to it:\nresource perms: {}\nfolder perms: {}\nauthed as: {authed:?}",
serde_json::to_string_pretty(&extra_perms).unwrap_or_default(), serde_json::to_string_pretty(&folder_extra_perms).unwrap_or_default()
)));
} else {
return Err(Error::NotAuthorized(format!(
"Resource exists but you don't have access to it:\nresource perms: {}",
"Resource exists but you don't have access to it:\nresource perms: {}\nauthed as: {authed:?}",
serde_json::to_string_pretty(&extra_perms).unwrap_or_default()
)));
}
@@ -457,7 +458,7 @@ pub async fn get_resource_value_interpolated_internal(
.await?;
tx.commit().await?;
if value_o.is_none() {
explain_resource_perm_error(path, workspace, db).await?;
explain_resource_perm_error(path, workspace, db, &authed).await?;
}
let value = not_found_if_none(value_o, "Resource", path)?;
backend/windmill-api/src/settings.rs
+5 -72
View File
@@ -434,67 +434,15 @@ pub async fn test_critical_channels() -> Result<String> {
Ok("Critical channels require EE".to_string())
}
use serde::Serialize;
#[derive(Serialize)]
pub struct CriticalAlert {
id: i32,
alert_type: String,
message: String,
created_at: chrono::DateTime<chrono::Utc>,
acknowledged: Option<bool>,
}
#[cfg(feature = "enterprise")]
#[derive(Deserialize)]
pub struct AlertQueryParams {
pub page: Option<i32>,
pub page_size: Option<i32>,
pub acknowledged: Option<bool>,
}
#[cfg(feature = "enterprise")]
pub async fn get_critical_alerts(
Extension(db): Extension<DB>,
authed: ApiAuthed,
Query(params): Query<AlertQueryParams>,
) -> JsonResult<Vec<CriticalAlert>> {
Query(params): Query<crate::utils::AlertQueryParams>,
) -> JsonResult<Vec<crate::utils::CriticalAlert>> {
require_super_admin(&db, &authed.email).await?;
// Default pagination values if not provided
let page = params.page.unwrap_or(1).max(1);
let page_size = params.page_size.unwrap_or(10).min(100) as i64;
let offset = ((page - 1) * page_size as i32) as i64;
let alerts = if let Some(acknowledged) = params.acknowledged {
sqlx::query_as!(
CriticalAlert,
"SELECT id, alert_type, message, created_at, acknowledged
FROM alerts
WHERE acknowledged = $1
ORDER BY created_at DESC
LIMIT $2 OFFSET $3",
acknowledged,
page_size,
offset
)
.fetch_all(&db)
.await?
} else {
sqlx::query_as!(
CriticalAlert,
"SELECT id, alert_type, message, created_at, acknowledged
FROM alerts
ORDER BY created_at DESC
LIMIT $1 OFFSET $2",
page_size,
offset
)
.fetch_all(&db)
.await?
};
Ok(Json(alerts))
crate::utils::get_critical_alerts(db, params, None).await
}
#[cfg(not(feature = "enterprise"))]
@@ -510,15 +458,7 @@ pub async fn acknowledge_critical_alert(
) -> error::Result<String> {
require_super_admin(&db, &authed.email).await?;
sqlx::query!(
"UPDATE alerts SET acknowledged = true WHERE id = $1",
id
)
.execute(&db)
.await?;
tracing::info!("Acknowledged critical alert with id: {}", id);
Ok("Critical alert acknowledged".to_string())
crate::utils::acknowledge_critical_alert(db, None, id).await
}
#[cfg(not(feature = "enterprise"))]
@@ -533,14 +473,7 @@ pub async fn acknowledge_all_critical_alerts(
) -> error::Result<String> {
require_super_admin(&db, &authed.email).await?;
sqlx::query!(
"UPDATE alerts SET acknowledged = true WHERE acknowledged = false"
)
.execute(&db)
.await?;
tracing::info!("Acknowledged all unacknowledged critical alerts");
Ok("All unacknowledged critical alerts acknowledged".to_string())
crate::utils::acknowledge_all_critical_alerts(db, None).await
}
#[cfg(not(feature = "enterprise"))]
backend/windmill-api/src/users.rs
+55 -1
View File
@@ -90,6 +90,9 @@ pub fn global_service() -> Router {
.route("/accept_invite", post(accept_invite))
.route("/list_as_super_admin", get(list_users_as_super_admin))
.route("/setpassword", post(set_password))
.route("/set_password_of/:user", post(set_password_of_user))
.route("/set_login_type/:user", post(set_login_type))
.route("/create", post(create_user))
.route("/update/:user", post(update_user))
.route("/delete/:user", delete(delete_user))
@@ -866,6 +869,12 @@ pub struct EditPassword {
pub password: String,
}
#[derive(Deserialize)]
pub struct EditLoginType {
pub login_type: String,
}
#[derive(FromRow, Serialize)]
pub struct TruncatedToken {
pub label: Option<String>,
@@ -2028,7 +2037,52 @@ async fn set_password(
authed: ApiAuthed,
Json(ep): Json<EditPassword>,
) -> Result<String> {
crate::users_ee::set_password(db, argon2, authed, ep).await
let email = authed.email.clone();
crate::users_ee::set_password(db, argon2, authed, &email, ep).await
}
async fn set_password_of_user(
Extension(db): Extension<DB>,
Extension(argon2): Extension<Arc<Argon2<'_>>>,
Path(email): Path<String>,
authed: ApiAuthed,
Json(ep): Json<EditPassword>,
) -> Result<String> {
require_super_admin(&db, &authed.email).await?;
crate::users_ee::set_password(db, argon2, authed, &email, ep).await
}
async fn set_login_type(
Extension(db): Extension<DB>,
Path(email): Path<String>,
authed: ApiAuthed,
Json(et): Json<EditLoginType>,
) -> Result<String> {
require_super_admin(&db, &authed.email).await?;
let mut tx = db.begin().await?;
sqlx::query!(
"UPDATE password SET login_type = $1 WHERE email = $2",
et.login_type,
email
)
.execute(&mut *tx)
.await?;
audit_log(
&mut *tx,
&authed,
"users.set_login_type",
ActionKind::Update,
"global",
Some(&email),
None,
)
.await?;
tx.commit().await?;
Ok(format!("login type of {} updated to {}", email, et.login_type))
}
async fn login(
backend/windmill-api/src/users_ee.rs
+1
View File
@@ -27,6 +27,7 @@ pub async fn set_password(
_db: DB,
_argon2: Arc<Argon2<'_>>,
_authed: ApiAuthed,
_user_email: &str,
_ep: EditPassword,
) -> Result<String> {
Err(Error::InternalErr(
backend/windmill-api/src/utils.rs
+162
View File
@@ -16,6 +16,12 @@ use windmill_common::{
DB,
};
#[cfg(feature = "enterprise")]
use windmill_common::error::JsonResult;
#[cfg(feature = "enterprise")]
use axum::Json;
#[derive(Deserialize)]
pub struct WithStarredInfoQuery {
pub with_starred_info: Option<bool>,
@@ -162,3 +168,159 @@ pub fn content_plain(body: Body) -> Response {
.body(body)
.unwrap()
}
use serde::Serialize;
#[derive(Serialize)]
pub struct CriticalAlert {
id: i32,
alert_type: String,
message: String,
created_at: chrono::DateTime<chrono::Utc>,
acknowledged: Option<bool>,
workspace_id: Option<String>,
}
#[cfg(feature = "enterprise")]
#[derive(Deserialize, Debug)]
pub struct AlertQueryParams {
pub page: Option<i32>,
pub page_size: Option<i32>,
pub acknowledged: Option<bool>,
}
#[cfg(feature = "enterprise")]
pub async fn get_critical_alerts(
db: DB,
params: AlertQueryParams,
workspace_id: Option<String>,
) -> JsonResult<Vec<CriticalAlert>> {
let page = params.page.unwrap_or(1).max(1);
let page_size = params.page_size.unwrap_or(10).min(100) as i64;
let offset = ((page - 1) * page_size as i32) as i64;
let alerts = if let Some(workspace_id) = workspace_id {
// `workspace_id` is provided => workspace admin
if params.acknowledged.is_none() {
// Case: return all rows where `workspace_id` matches
sqlx::query_as!(
CriticalAlert,
"SELECT id, alert_type, message, created_at, COALESCE(acknowledged_workspace, false) AS acknowledged, workspace_id
FROM alerts
WHERE workspace_id = $1
ORDER BY created_at DESC
LIMIT $2 OFFSET $3",
workspace_id,
page_size,
offset
)
.fetch_all(&db)
.await?
} else {
// Case: return rows where `acknowledged_workspace` matches `params.acknowledged`
sqlx::query_as!(
CriticalAlert,
"SELECT id, alert_type, message, created_at, COALESCE(acknowledged_workspace, false) AS acknowledged, workspace_id
FROM alerts
WHERE workspace_id = $1 AND COALESCE(acknowledged_workspace, false) = $2
ORDER BY created_at DESC
LIMIT $3 OFFSET $4",
workspace_id,
params.acknowledged,
page_size,
offset
)
.fetch_all(&db)
.await?
}
} else {
// `workspace_id` is not provided => superadmin
if params.acknowledged.is_none() {
// Case: Return all rows unfiltered with global acknowledged as acknowledged
sqlx::query_as!(
CriticalAlert,
"SELECT id, alert_type, message, created_at, COALESCE(acknowledged, false) AS acknowledged, workspace_id
FROM alerts
ORDER BY created_at DESC
LIMIT $1 OFFSET $2",
page_size,
offset
)
.fetch_all(&db)
.await?
} else {
// Case: Return rows where global acknowledged matches params.acknowledged
sqlx::query_as!(
CriticalAlert,
"SELECT id, alert_type, message, created_at, COALESCE(acknowledged, false) AS acknowledged, workspace_id
FROM alerts
WHERE COALESCE(acknowledged, false) = $1
ORDER BY created_at DESC
LIMIT $2 OFFSET $3",
params.acknowledged,
page_size,
offset
)
.fetch_all(&db)
.await?
}
};
Ok(Json(alerts))
}
#[cfg(feature = "enterprise")]
pub async fn acknowledge_critical_alert(
db: DB,
workspace_id: Option<String>,
id: i32,
) -> error::Result<String> {
sqlx::query!(
"UPDATE alerts
SET
acknowledged = true,
acknowledged_workspace = CASE
WHEN $2::text IS NOT NULL AND workspace_id = $2 THEN true
ELSE acknowledged_workspace
END
WHERE id = $1",
id,
workspace_id
)
.execute(&db)
.await?;
tracing::info!(
"Acknowledged critical alert with id: {}{}",
id,
workspace_id.map_or_else(|| "".to_string(), |w| format!(" for workspace_id: {}", w))
);
Ok("Critical alert acknowledged".to_string())
}
#[cfg(feature = "enterprise")]
pub async fn acknowledge_all_critical_alerts(
db: DB,
workspace_id: Option<String>,
) -> error::Result<String> {
sqlx::query!(
"UPDATE alerts
SET
acknowledged = true,
acknowledged_workspace = CASE
WHEN $1::text IS NOT NULL THEN true
ELSE acknowledged_workspace
END
WHERE ($1::text IS NOT NULL AND workspace_id = $1)
OR ($1::text IS NULL)",
workspace_id
)
.execute(&db)
.await?;
tracing::info!(
"Acknowledged all unacknowledged critical alerts{}",
workspace_id.map_or_else(|| "".to_string(), |w| format!(" for workspace_id: {}", w))
);
Ok("All unacknowledged critical alerts acknowledged".to_string())
}
backend/windmill-api/src/websocket_triggers.rs
+5 -3
View File
@@ -754,12 +754,14 @@ async fn disable_with_error(db: &DB, ws_trigger: &WebsocketTrigger, error: Strin
)
.execute(db).await {
Ok(_) => {
report_critical_error(format!("Disabling websocket {} because of error: {}", ws_trigger.url, error), db.clone()).await;
report_critical_error(format!("Disabling websocket {} because of error: {}", ws_trigger.url, error), db.clone(), Some(&ws_trigger.workspace_id), None).await;
},
Err(disable_err) => {
report_critical_error(
format!("Could not disable websocket {} with err {}, disabling because of error {}", ws_trigger.path, disable_err, error),
db.clone()
db.clone(),
Some(&ws_trigger.workspace_id),
None,
).await;
}
}
@@ -896,7 +898,7 @@ async fn listen_to_websocket(
}
if should_handle {
if let Err(err) = run_job(&db, rsmq.clone(), &ws_trigger, text).await {
report_critical_error(format!("Failed to trigger job from websocket {}: {:?}", ws_trigger.url, err), db.clone()).await;
report_critical_error(format!("Failed to trigger job from websocket {}: {:?}", ws_trigger.url, err), db.clone(), Some(&ws_trigger.workspace_id), None).await;
};
}
},
backend/windmill-api/src/workers.rs
+10
View File
@@ -36,6 +36,7 @@ pub fn global_service() -> Router {
)
.route("/get_default_tags", get(get_default_tags))
.route("/queue_metrics", get(get_queue_metrics))
.route("/queue_counts", get(get_queue_counts))
}
#[derive(FromRow, Serialize, Deserialize)]
@@ -176,3 +177,12 @@ async fn get_queue_metrics(
Ok(Json(queue_metrics))
}
async fn get_queue_counts(
authed: ApiAuthed,
Extension(db): Extension<DB>,
) -> JsonResult<std::collections::HashMap<String, u32>> {
require_super_admin(&db, &authed.email).await?;
let queue_counts = windmill_common::queue::get_queue_counts(&db).await;
Ok(Json(queue_counts))
}
backend/windmill-api/src/workspaces.rs
+137 -17
View File
@@ -8,6 +8,7 @@
use std::collections::HashMap;
use crate::ai::{AiResource, AI_KEY_CACHE};
use crate::db::ApiAuthed;
use crate::users_ee::send_email_if_possible;
use crate::utils::get_instance_username_or_create_pending;
@@ -118,7 +119,11 @@ pub fn workspaced_service() -> Router {
.route("/change_workspace_name", post(change_workspace_name))
.route("/change_workspace_id", post(change_workspace_id))
.route("/usage", get(get_usage))
.route("/used_triggers", get(get_used_triggers));
.route("/used_triggers", get(get_used_triggers))
.route("/critical_alerts", get(get_critical_alerts))
.route("/critical_alerts/:id/acknowledge", post(acknowledge_critical_alert))
.route("/critical_alerts/acknowledge_all", post(acknowledge_all_critical_alerts))
.route("/critical_alerts/mute", post(mute_critical_alerts));
#[cfg(feature = "stripe")]
{
@@ -168,7 +173,7 @@ pub struct WorkspaceSettings {
pub plan: Option<String>,
pub webhook: Option<String>,
pub deploy_to: Option<String>,
pub openai_resource_path: Option<String>,
pub ai_resource: Option<serde_json::Value>,
pub code_completion_enabled: bool,
pub error_handler: Option<String>,
pub error_handler_extra_args: Option<serde_json::Value>,
@@ -179,6 +184,7 @@ pub struct WorkspaceSettings {
pub default_app: Option<String>,
pub automatic_billing: bool,
pub default_scripts: Option<serde_json::Value>,
pub mute_critical_alerts: Option<bool>,
}
#[derive(FromRow, Serialize, Debug)]
@@ -234,7 +240,7 @@ struct EditWebhook {
#[derive(Deserialize)]
struct EditCopilotConfig {
openai_resource_path: Option<String>,
ai_resource: Option<serde_json::Value>,
code_completion_enabled: bool,
}
@@ -645,23 +651,33 @@ async fn edit_copilot_config(
let mut tx = db.begin().await?;
if let Some(openai_resource_path) = &eo.openai_resource_path {
if let Some(ai_resource) = &eo.ai_resource {
let path = serde_json::from_value::<AiResource>(ai_resource.clone())
.map_err(|e| Error::BadRequest(e.to_string()))?
.path;
sqlx::query!(
"UPDATE workspace_settings SET openai_resource_path = $1, code_completion_enabled = $2 WHERE workspace_id = $3",
openai_resource_path,
"UPDATE workspace_settings SET ai_resource = $1, code_completion_enabled = $2 WHERE workspace_id = $3",
ai_resource,
eo.code_completion_enabled,
&w_id
)
.execute(&mut *tx)
.await?;
if let Some(cached) = AI_KEY_CACHE.get(&w_id) {
if cached.path != path {
AI_KEY_CACHE.remove(&w_id);
}
}
} else {
sqlx::query!(
"UPDATE workspace_settings SET openai_resource_path = NULL, code_completion_enabled = $1 WHERE workspace_id = $2",
"UPDATE workspace_settings SET ai_resource = NULL, code_completion_enabled = $1 WHERE workspace_id = $2",
eo.code_completion_enabled,
&w_id,
)
.execute(&mut *tx)
.await?;
AI_KEY_CACHE.remove(&w_id);
}
audit_log(
&mut *tx,
@@ -672,10 +688,7 @@ async fn edit_copilot_config(
Some(&authed.email),
Some(
[
(
"openai_resource_path",
&format!("{:?}", eo.openai_resource_path)[..],
),
("ai_resource", &format!("{:?}", eo.ai_resource)[..]),
(
"code_completion_enabled",
&format!("{:?}", eo.code_completion_enabled)[..],
@@ -692,7 +705,8 @@ async fn edit_copilot_config(
#[derive(Serialize)]
struct CopilotInfo {
pub exists_openai_resource_path: bool,
pub ai_provider: String,
pub exists_ai_resource: bool,
pub code_completion_enabled: bool,
}
async fn get_copilot_info(
@@ -701,16 +715,32 @@ async fn get_copilot_info(
) -> JsonResult<CopilotInfo> {
let mut tx = db.begin().await?;
let record = sqlx::query!(
"SELECT openai_resource_path, code_completion_enabled FROM workspace_settings WHERE workspace_id = $1",
"SELECT ai_resource, code_completion_enabled FROM workspace_settings WHERE workspace_id = $1",
&w_id
)
.fetch_one(&mut *tx)
.await
.map_err(|e| Error::InternalErr(format!("getting openai_resource_path and code_completion_enabled: {e:#}")))?;
.map_err(|e| Error::InternalErr(format!("getting ai_resource and code_completion_enabled: {e:#}")))?;
tx.commit().await?;
let (ai_provider, exists_ai_resource) = if let Some(ai_resource) = record.ai_resource {
let ai_resource = serde_json::from_value::<AiResource>(ai_resource);
let exist = ai_resource.is_ok();
(
if exist {
ai_resource.unwrap().provider
} else {
"".to_string()
},
exist,
)
} else {
("".to_string(), false)
};
Ok(Json(CopilotInfo {
exists_openai_resource_path: record.openai_resource_path.is_some(),
ai_provider,
exists_ai_resource,
code_completion_enabled: record.code_completion_enabled,
}))
}
@@ -2223,7 +2253,7 @@ struct SimplifiedSettings {
error_handler: Option<String>,
error_handler_extra_args: Option<Value>,
error_handler_muted_on_cancel: bool,
openai_resource_path: Option<String>,
ai_resource: Option<serde_json::Value>,
code_completion_enabled: bool,
large_file_storage: Option<Value>,
git_sync: Option<Value>,
@@ -2588,7 +2618,7 @@ async fn tarball_workspace(
webhook,
deploy_to,
error_handler,
openai_resource_path,
ai_resource,
code_completion_enabled,
error_handler_extra_args,
error_handler_muted_on_cancel,
@@ -3050,3 +3080,93 @@ async fn get_usage(Extension(db): Extension<DB>, Path(w_id): Path<String>) -> Re
.unwrap_or(0);
Ok(usage.to_string())
}
#[cfg(feature = "enterprise")]
pub async fn get_critical_alerts(
Extension(db): Extension<DB>,
Path(w_id): Path<String>,
authed: ApiAuthed,
Query(params): Query<crate::utils::AlertQueryParams>,
) -> JsonResult<Vec<crate::utils::CriticalAlert>> {
require_admin(authed.is_admin, &authed.username)?;
crate::utils::get_critical_alerts(db, params, Some(w_id)).await
}
#[cfg(not(feature = "enterprise"))]
pub async fn get_critical_alerts() -> Error {
Error::NotFound("Critical Alerts require EE".to_string())
}
#[cfg(feature = "enterprise")]
pub async fn acknowledge_critical_alert(
Extension(db): Extension<DB>,
Path((w_id, id)): Path<(String, i32)>,
authed: ApiAuthed,
) -> Result<String> {
require_admin(authed.is_admin, &authed.username)?;
crate::utils::acknowledge_critical_alert(db, Some(w_id), id).await
}
#[cfg(not(feature = "enterprise"))]
pub async fn acknowledge_critical_alert() -> Error {
Error::NotFound("Critical Alerts require EE".to_string())
}
#[cfg(feature = "enterprise")]
pub async fn acknowledge_all_critical_alerts(
Extension(db): Extension<DB>,
Path(w_id): Path<String>,
authed: ApiAuthed,
) -> Result<String> {
require_admin(authed.is_admin, &authed.username)?;
crate::utils::acknowledge_all_critical_alerts(db, Some(w_id)).await
}
#[cfg(not(feature = "enterprise"))]
pub async fn acknowledge_all_critical_alerts() -> Error {
Error::NotFound("Critical Alerts require EE".to_string())
}
#[cfg(feature = "enterprise")]
#[derive(Deserialize)]
pub struct MuteCriticalAlertRequest {
pub mute_critical_alerts: Option<bool>,
}
#[cfg(feature = "enterprise")]
async fn mute_critical_alerts(
Extension(db): Extension<DB>,
Path(w_id): Path<String>,
ApiAuthed { is_admin, username, .. }: ApiAuthed,
Json(m_r): Json<MuteCriticalAlertRequest>,
) -> Result<String> {
require_admin(is_admin, &username)?;
let mute_alerts = m_r.mute_critical_alerts.unwrap_or(false);
if mute_alerts {
sqlx::query!(
"UPDATE alerts SET acknowledged_workspace = true, acknowledged = true WHERE workspace_id = $1",
&w_id
)
.execute(&db)
.await?;
}
sqlx::query!(
"UPDATE workspace_settings SET mute_critical_alerts = $1 WHERE workspace_id = $2",
mute_alerts,
&w_id
)
.execute(&db)
.await?;
Ok(format!("Updated mute criticital alert ui settings for workspace: {}", &w_id))
}
#[cfg(not(feature = "enterprise"))]
pub async fn mute_critical_alerts() -> Error {
Error::NotFound("Critical Alerts require EE".to_string())
}
backend/windmill-common/Cargo.toml
+1
View File
@@ -58,6 +58,7 @@ async-stream.workspace = true
const_format.workspace = true
crc.workspace = true
windmill-macros.workspace = true
semver.workspace = true
[target.'cfg(not(target_env = "msvc"))'.dependencies]
tikv-jemalloc-ctl = { optional = true, workspace = true }
backend/windmill-common/src/error.rs
+2 -2
View File
@@ -59,7 +59,7 @@ pub enum Error {
#[error("Error: {0:#?}")]
JsonErr(serde_json::Value),
#[error("{0}")]
OpenAIError(String),
AiError(String),
#[error("{0}")]
AlreadyCompleted(String),
}
@@ -90,7 +90,7 @@ impl IntoResponse for Error {
Self::RequireAdmin(_) => axum::http::StatusCode::FORBIDDEN,
Self::SqlErr(_)
| Self::BadRequest(_)
| Self::OpenAIError(_)
| Self::AiError(_)
| Self::QuotaExceeded(_) => axum::http::StatusCode::BAD_REQUEST,
_ => axum::http::StatusCode::INTERNAL_SERVER_ERROR,
};
backend/windmill-common/src/jobs.rs
-16
View File
@@ -108,13 +108,6 @@ pub struct QueuedJob {
pub cache_ttl: Option<i32>,
#[serde(skip_serializing_if = "Option::is_none")]
pub priority: Option<i16>,
#[serde(skip_serializing_if = "Option::is_none")]
#[sqlx(skip)]
pub self_wait_time_ms: Option<i64>,
#[serde(skip_serializing_if = "Option::is_none")]
#[sqlx(skip)]
pub aggregate_wait_time_ms: Option<i64>,
}
impl QueuedJob {
@@ -198,8 +191,6 @@ impl Default for QueuedJob {
flow_step_id: None,
cache_ttl: None,
priority: None,
self_wait_time_ms: None,
aggregate_wait_time_ms: None,
}
}
}
@@ -253,13 +244,6 @@ pub struct CompletedJob {
pub priority: Option<i16>,
#[serde(skip_serializing_if = "Option::is_none")]
pub labels: Option<serde_json::Value>,
#[serde(skip_serializing_if = "Option::is_none")]
#[sqlx(skip)]
pub self_wait_time_ms: Option<i64>,
#[serde(skip_serializing_if = "Option::is_none")]
#[sqlx(skip)]
pub aggregate_wait_time_ms: Option<i64>,
}
impl CompletedJob {
backend/windmill-common/src/utils.rs
+106 -18
View File
@@ -21,6 +21,7 @@ use reqwest::Client;
use serde::{Deserialize, Serialize};
use sha2::{Digest, Sha256};
use sqlx::{Pool, Postgres};
use semver::Version;
pub const MAX_PER_PAGE: usize = 10000;
pub const DEFAULT_PER_PAGE: usize = 1000;
@@ -28,8 +29,10 @@ pub const DEFAULT_PER_PAGE: usize = 1000;
pub const GIT_VERSION: &str =
git_version!(args = ["--tag", "--always"], fallback = "unknown-version");
use std::sync::atomic::Ordering;
use crate::CRITICAL_ALERT_MUTE_UI_ENABLED;
use std::sync::atomic::Ordering;
use crate::worker::CLOUD_HOSTED;
lazy_static::lazy_static! {
pub static ref HTTP_CLIENT: Client = reqwest::ClientBuilder::new()
@@ -37,6 +40,10 @@ lazy_static::lazy_static! {
.timeout(std::time::Duration::from_secs(20))
.connect_timeout(std::time::Duration::from_secs(10))
.build().unwrap();
pub static ref GIT_SEM_VERSION: Version = Version::parse(
// skip first `v` character.
GIT_VERSION.split_at(1).1
).unwrap_or(Version::new(0, 1, 0));
}
#[derive(Deserialize, Clone)]
@@ -244,15 +251,40 @@ pub fn generate_lock_id(database_name: &str) -> i64 {
0x3d32ad9e * (CRC_IEEE.checksum(database_name.as_bytes()) as i64)
}
pub async fn report_critical_error(error_message: String, _db: DB) -> () {
pub async fn report_critical_error(
error_message: String,
_db: DB,
workspace_id: Option<&str>,
resource: Option<&str>,
) -> () {
tracing::error!("CRITICAL ERROR: {error_message}");
let mute = CRITICAL_ALERT_MUTE_UI_ENABLED.load(Ordering::Relaxed);
let mute_global = CRITICAL_ALERT_MUTE_UI_ENABLED.load(Ordering::Relaxed);
let mute_workspace = if let Some(workspace_id) = workspace_id {
match fetch_mute_workspace(&_db, workspace_id).await {
Ok(flag) => flag,
Err(err) => {
tracing::error!("Error fetching mute_workspace: {}", err);
false
}
}
} else {
false
};
// we ack_global if mute_global is true, or if mute_workspace is true
// but we ignore global mute setting for ack_workspace
let acknowledge_workspace = mute_workspace;
let acknowledge_global = mute_global || mute_workspace || ( workspace_id.is_some() && *CLOUD_HOSTED);
if let Err(err) = sqlx::query!(
"INSERT INTO alerts (alert_type, message, acknowledged) VALUES ('critical_error', $1, $2)",
"INSERT INTO alerts (alert_type, message, acknowledged, acknowledged_workspace, workspace_id, resource)
VALUES ('critical_error', $1, $2, $3, $4, $5)",
error_message,
mute
acknowledge_global,
acknowledge_workspace,
workspace_id,
resource,
)
.execute(&_db)
.await
@@ -261,30 +293,86 @@ pub async fn report_critical_error(error_message: String, _db: DB) -> () {
}
#[cfg(feature = "enterprise")]
send_critical_alert(error_message, &_db, CriticalAlertKind::CriticalError, None).await;
if *CLOUD_HOSTED && workspace_id.is_some() {
tracing::error!(error_message)
} else {
send_critical_alert(error_message, &_db, CriticalAlertKind::CriticalError, None).await;
}
}
pub async fn report_recovered_critical_error(message: String, _db: DB) -> () {
pub async fn report_recovered_critical_error(
message: String,
_db: DB,
workspace_id: Option<&str>,
resource: Option<&str>,
) -> () {
tracing::info!("RECOVERED CRITICAL ERROR: {message}");
let mute = CRITICAL_ALERT_MUTE_UI_ENABLED.load(Ordering::Relaxed);
if let Err(err) = sqlx::query!(
"INSERT INTO alerts (alert_type, message, acknowledged) VALUES ('recovered_critical_error', $1, $2)",
"INSERT INTO alerts (alert_type, message, acknowledged, acknowledged_workspace, workspace_id, resource)
VALUES ('recovered_critical_error', $1, $2, $3, $4, $5)",
message,
mute
true,
true,
workspace_id,
resource,
)
.execute(&_db)
.await
{
tracing::error!("Failed to save critical error to database: {}", err);
tracing::error!("Failed to save recovered critical error to database: {}", err);
}
// acknowledge all alerts with the same resource
if let Some(resource) = resource {
if let Err(err) = sqlx::query!(
"UPDATE alerts SET acknowledged = true, acknowledged_workspace = true WHERE resource = $1 AND alert_type = 'critical_error'",
resource,
)
.execute(&_db)
.await
{
tracing::error!("Failed to acknowledge critical error alerts for resource {}: {}", resource, err);
}
}
#[cfg(feature = "enterprise")]
send_critical_alert(
message,
&_db,
CriticalAlertKind::RecoveredCriticalError,
None,
if *CLOUD_HOSTED && workspace_id.is_some() {
tracing::error!(message);
} else {
send_critical_alert(
message,
&_db,
CriticalAlertKind::RecoveredCriticalError,
None,
)
.await;
}
}
pub async fn fetch_mute_workspace(_db: &DB, workspace_id: &str) -> Result<bool> {
match sqlx::query!(
"SELECT mute_critical_alerts FROM workspace_settings WHERE workspace_id = $1",
workspace_id
)
.await;
.fetch_optional(_db)
.await
{
Ok(Some(record)) => Ok(record.mute_critical_alerts.unwrap_or(false)),
Ok(None) => {
tracing::warn!(
"Workspace ID {} not found in workspace_settings table",
workspace_id
);
Ok(false)
}
Err(err) => {
tracing::error!(
"Error querying workspace_settings for workspace_id {}: {}",
workspace_id,
err
);
Err(Error::SqlErr(err))
}
}
}
backend/windmill-common/src/worker.rs
+29
View File
@@ -1,6 +1,7 @@
use const_format::concatcp;
use itertools::Itertools;
use regex::Regex;
use semver::Version;
use serde::{Deserialize, Serialize};
use serde_json::value::RawValue;
use std::{
@@ -88,6 +89,7 @@ lazy_static::lazy_static! {
.and_then(|x| x.parse::<bool>().ok())
.unwrap_or(false);
pub static ref MIN_VERSION: Arc<RwLock<Version>> = Arc::new(RwLock::new(Version::new(0, 0, 0)));
}
pub async fn make_suspended_pull_query(wc: &WorkerConfig) {
@@ -308,6 +310,8 @@ fn parse_file<T: FromStr>(path: &str) -> Option<T> {
pub struct PythonAnnotations {
pub no_cache: bool,
pub no_uv: bool,
pub no_uv_install: bool,
pub no_uv_compile: bool,
}
#[annotations("//")]
@@ -548,6 +552,31 @@ pub fn get_windmill_memory_usage() -> Option<i64> {
}
}
pub async fn update_min_version<'c, E: sqlx::Executor<'c, Database = sqlx::Postgres>>(executor: E) -> bool {
use crate::utils::{GIT_VERSION, GIT_SEM_VERSION};
// fetch all pings with a different version than self from the last 5 minutes.
let pings = sqlx::query_scalar!(
"SELECT wm_version FROM worker_ping WHERE wm_version != $1 AND ping_at > now() - interval '5 minutes'",
GIT_VERSION
).fetch_all(executor).await.unwrap_or_default();
let cur_version = GIT_SEM_VERSION.clone();
let min_version = pings
.iter()
.filter(|x| !x.is_empty())
.filter_map(|x| semver::Version::parse(x.split_at(1).1).ok())
.min()
.unwrap_or_else(|| cur_version.clone());
if min_version != cur_version {
tracing::info!("Minimal worker version: {min_version}");
}
*MIN_VERSION.write().await = min_version.clone();
min_version >= cur_version
}
pub async fn update_ping(worker_instance: &str, worker_name: &str, ip: &str, db: &DB) {
let (tags, dw) = {
let wc = WORKER_CONFIG.read().await.clone();
backend/windmill-queue/src/jobs.rs
+9 -7
View File
@@ -924,6 +924,8 @@ pub async fn add_completed_job<
.unwrap_or("".to_string()),
),
db.clone(),
Some(&w_id),
None,
)
.await;
} else if queued_job.email == SCHEDULE_ERROR_HANDLER_USER_EMAIL {
@@ -992,7 +994,7 @@ pub async fn add_completed_job<
"Could not push workspace error handler for failed job ({base_url}/run/{}?workspace={w_id}): {}",
queued_job.id,
err
), db.clone())
), db.clone(), Some(&w_id), None)
.await;
}
}
@@ -1175,10 +1177,10 @@ pub async fn report_error_to_workspace_handler_or_critical_side_channel<
queued_job.id,
err
);
report_critical_error(error_message, db.clone()).await;
report_critical_error(error_message, db.clone(), Some(&w_id), None).await;
}
} else {
report_critical_error(error_message, db.clone()).await;
report_critical_error(error_message, db.clone(), Some(&w_id), None).await;
}
}
@@ -3670,15 +3672,15 @@ pub async fn push<'c, 'd, R: rsmq_async::RsmqConnection + Send + 'c>(
}
JobPayload::DeploymentCallback { path } => (
None,
Some(path),
Some(path.clone()),
None,
JobKind::DeploymentCallback,
None,
None,
None,
None,
None,
None,
Some(format!("{workspace_id}:git_sync")),
Some(1),
Some(0),
None,
None,
None,
backend/windmill-worker/nsjail/download.py.pip.config.proto
+93
View File
@@ -0,0 +1,93 @@
name: "python download pip"
mode: ONCE
hostname: "python"
log_level: ERROR
time_limit: 900
rlimit_as: 2048
rlimit_cpu: 1000
rlimit_fsize: 1024
rlimit_nofile: 64
envar: "HOME=/user"
envar: "LD_LIBRARY_PATH=/usr/local/lib:$LD_LIBRARY_PATH"
cwd: "/tmp"
clone_newnet: false
clone_newuser: {CLONE_NEWUSER}
keep_caps: true
keep_env: true
mount {
src: "/bin"
dst: "/bin"
is_bind: true
}
mount {
src: "/lib"
dst: "/lib"
is_bind: true
}
mount {
src: "/lib64"
dst: "/lib64"
is_bind: true
mandatory: false
}
mount {
src: "/usr"
dst: "/usr"
is_bind: true
}
mount {
src: "/etc"
dst: "/etc"
is_bind: true
}
mount {
src: "/dev/null"
dst: "/dev/null"
is_bind: true
rw: true
}
mount {
dst: "/tmp"
fstype: "tmpfs"
rw: true
options: "size=500000000"
}
mount {
src: "{WORKER_DIR}/download_deps.py.pip.sh"
dst: "/download_deps.sh"
is_bind: true
}
mount {
src: "{CACHE_DIR}"
dst: "{CACHE_DIR}"
is_bind: true
rw: true
}
mount {
src: "/dev/urandom"
dst: "/dev/urandom"
is_bind: true
}
exec_bin {
path: "/bin/sh"
arg: "/download_deps.sh"
}
backend/windmill-worker/nsjail/download_deps.py.pip.sh
Executable
+24
View File
@@ -0,0 +1,24 @@
#/bin/sh
INDEX_URL_ARG=$([ -z "$INDEX_URL" ] && echo ""|| echo "--index-url $INDEX_URL" )
EXTRA_INDEX_URL_ARG=$([ -z "$EXTRA_INDEX_URL" ] && echo ""|| echo "--extra-index-url $EXTRA_INDEX_URL" )
TRUSTED_HOST_ARG=$([ -z "$TRUSTED_HOST" ] && echo "" || echo "--trusted-host $TRUSTED_HOST")
if [ ! -z "$INDEX_URL" ]
then
echo "\$INDEX_URL is set to $INDEX_URL"
fi
if [ ! -z "$EXTRA_INDEX_URL" ]
then
echo "\$EXTRA_INDEX_URL is set to $EXTRA_INDEX_URL"
fi
if [ ! -z "$TRUSTED_HOST" ]
then
echo "\$TRUSTED_HOST is set to $TRUSTED_HOST"
fi
CMD="/usr/local/bin/python3 -m pip install -v \"$REQ\" -I -t \"$TARGET\" --no-cache --no-color --no-deps --isolated --no-warn-conflicts --disable-pip-version-check $INDEX_URL_ARG $EXTRA_INDEX_URL_ARG $TRUSTED_HOST_ARG"
echo $CMD
eval $CMD
backend/windmill-worker/nsjail/download_deps.py.sh
+13 -1
View File
@@ -19,6 +19,18 @@ then
echo "\$TRUSTED_HOST is set to $TRUSTED_HOST"
fi
CMD="/usr/local/bin/python3 -m pip install -v \"$REQ\" -I -t \"$TARGET\" --no-cache --no-color --no-deps --isolated --no-warn-conflicts --disable-pip-version-check $INDEX_URL_ARG $EXTRA_INDEX_URL_ARG $TRUSTED_HOST_ARG"
CMD="/usr/local/bin/uv pip install
\"$REQ\"
--target \"$TARGET\"
--no-cache
--no-config
--no-color
--no-deps
--link-mode=copy
$INDEX_URL_ARG $EXTRA_INDEX_URL_ARG $TRUSTED_HOST_ARG
--index-strategy unsafe-best-match
--system
"
echo $CMD
eval $CMD
backend/windmill-worker/src/ansible_executor.rs
+2
View File
@@ -116,6 +116,8 @@ async fn handle_ansible_python_deps(
job_dir,
worker_dir,
&mut Some(occupancy_metrics),
true,
true,
)
.await?;
additional_python_paths.append(&mut venv_path);
backend/windmill-worker/src/common.rs
+9 -3
View File
@@ -41,7 +41,7 @@ use tokio::{io::AsyncWriteExt, process::Child, time::Instant};
use crate::{
AuthedClient, AuthedClientBackgroundTask, JOB_DEFAULT_TIMEOUT, MAX_RESULT_SIZE,
MAX_TIMEOUT_DURATION,
MAX_TIMEOUT_DURATION, PATH_ENV,
};
pub async fn build_args_map<'a>(
@@ -860,11 +860,17 @@ pub async fn save_in_cache(
}
fn tentatively_improve_error(err: Error, executable: &str) -> Error {
#[cfg(unix)]
let err_msg = "No such file or directory (os error 2)";
#[cfg(windows)]
let err_msg = "program not found";
if err
.to_string()
.contains("No such file or directory (os error 2)")
.contains(&err_msg)
{
return Error::InternalErr(format!("Executable {executable} not found on worker"));
return Error::InternalErr(format!("Executable {executable} not found on worker. PATH: {}", *PATH_ENV));
}
return err;
}
backend/windmill-worker/src/global_cache.rs
+22 -4
View File
@@ -20,13 +20,21 @@ use std::sync::Arc;
pub async fn build_tar_and_push(
s3_client: Arc<dyn ObjectStore>,
folder: String,
no_uv: bool,
) -> error::Result<()> {
use object_store::path::Path;
use crate::PY311_CACHE_DIR;
tracing::info!("Started building and pushing piptar {folder}");
let start = Instant::now();
let folder_name = folder.split("/").last().unwrap();
let tar_path = format!("{PIP_CACHE_DIR}/{folder_name}_tar.tar",);
let prefix = if no_uv {
PIP_CACHE_DIR
} else {
PY311_CACHE_DIR
};
let tar_path = format!("{prefix}/{folder_name}_tar.tar",);
let tar_file = std::fs::File::create(&tar_path)?;
let mut tar = tar::Builder::new(tar_file);
@@ -46,7 +54,10 @@ pub async fn build_tar_and_push(
// })?;
if let Err(e) = s3_client
.put(
&Path::from(format!("/tar/pip/{folder_name}.tar")),
&Path::from(format!(
"/tar/{}/{folder_name}.tar",
if no_uv { "pip" } else { "python_311" }
)),
std::fs::read(&tar_path)?.into(),
)
.await
@@ -71,7 +82,11 @@ pub async fn build_tar_and_push(
}
#[cfg(all(feature = "enterprise", feature = "parquet"))]
pub async fn pull_from_tar(client: Arc<dyn ObjectStore>, folder: String) -> error::Result<()> {
pub async fn pull_from_tar(
client: Arc<dyn ObjectStore>,
folder: String,
no_uv: bool,
) -> error::Result<()> {
use windmill_common::s3_helpers::attempt_fetch_bytes;
let folder_name = folder.split("/").last().unwrap();
@@ -79,7 +94,10 @@ pub async fn pull_from_tar(client: Arc<dyn ObjectStore>, folder: String) -> erro
tracing::info!("Attempting to pull piptar {folder_name} from bucket");
let start = Instant::now();
let tar_path = format!("tar/pip/{folder_name}.tar");
let tar_path = format!(
"tar/{}/{folder_name}.tar",
if no_uv { "pip" } else { "python_311" }
);
let bytes = attempt_fetch_bytes(client, &tar_path).await?;
// tracing::info!("B: {target} {folder}");
backend/windmill-worker/src/pg_executor.rs
+54 -17
View File
@@ -416,24 +416,32 @@ pub async fn do_postgresql(
}
fn map_as_single_type<T>(
vec: &Vec<Value>,
vec: Option<&Vec<Value>>,
f: impl Fn(&Value) -> Option<T>,
) -> anyhow::Result<Vec<Option<T>>> {
vec.into_iter()
.map(|v| {
// allow nulls in arrays
if matches!(v, Value::Null) {
Some(None)
} else {
f(v).map(Some)
}
})
.collect::<Option<Vec<Option<T>>>>()
.ok_or_else(|| anyhow::anyhow!("Mixed types in array"))
) -> anyhow::Result<Option<Vec<Option<T>>>> {
if let Some(vec) = vec {
Ok(Some(
vec.into_iter()
.map(|v| {
// first option is if the value is of the right type (if none, will stop the collection and throw error)
// second option is if the value is null
// allow nulls in arrays
if matches!(v, Value::Null) {
Some(None)
} else {
f(v).map(Some)
}
})
.collect::<Option<Vec<Option<T>>>>()
.ok_or_else(|| anyhow::anyhow!("Mixed types in array"))?,
))
} else {
Ok(None)
}
}
fn convert_vec_val(
vec: &Vec<Value>,
vec: Option<&Vec<Value>>,
arg_t: &String,
) -> windmill_common::error::Result<Box<dyn ToSql + Sync + Send>> {
match arg_t.as_str() {
@@ -489,7 +497,9 @@ fn convert_vec_val(
.unwrap_or_default()
})
})?)),
"jsonb" | "json" => Ok(Box::new(vec.clone().into_iter().map(Some).collect_vec())),
"jsonb" | "json" => Ok(Box::new(
vec.map(|v| v.clone().into_iter().map(Some).collect_vec()),
)),
"bytea" => Ok(Box::new(map_as_single_type(vec, |v| {
v.as_str().map(|x| {
engine::general_purpose::STANDARD
@@ -512,9 +522,31 @@ fn convert_val(
match value {
Value::Array(vec) if arg_t.ends_with("[]") => {
let arg_t = arg_t.trim_end_matches("[]").to_string();
convert_vec_val(vec, &arg_t)
convert_vec_val(Some(vec), &arg_t)
}
Value::Null => Ok(Box::new(None::<bool>)),
Value::Null if arg_t.ends_with("[]") => {
let arg_t = arg_t.trim_end_matches("[]").to_string();
convert_vec_val(None, &arg_t)
}
Value::Null => match arg_t.as_str() {
"bool" | "boolean" => Ok(Box::new(None::<bool>)),
"char" | "character" => Ok(Box::new(None::<i8>)),
"smallint" | "smallserial" | "int2" | "serial2" => Ok(Box::new(None::<i16>)),
"int" | "integer" | "int4" | "serial" => Ok(Box::new(None::<i32>)),
"numeric" | "decimal" => Ok(Box::new(None::<Decimal>)),
"oid" => Ok(Box::new(None::<u32>)),
"bigint" | "bigserial" | "int8" | "serial8" => Ok(Box::new(None::<i64>)),
"real" | "float4" => Ok(Box::new(None::<f32>)),
"double" | "float8" => Ok(Box::new(None::<f64>)),
"uuid" => Ok(Box::new(None::<Uuid>)),
"date" => Ok(Box::new(None::<chrono::NaiveDate>)),
"time" | "timetz" => Ok(Box::new(None::<chrono::NaiveTime>)),
"timestamp" | "timestamptz" => Ok(Box::new(None::<chrono::NaiveDateTime>)),
"jsonb" | "json" => Ok(Box::new(None::<Option<Value>>)),
"bytea" => Ok(Box::new(None::<Vec<u8>>)),
"text" | "varchar" => Ok(Box::new(None::<String>)),
_ => Err(anyhow::anyhow!("Unsupported JSON null type"))?,
},
Value::Bool(b) => Ok(Box::new(b.clone())),
Value::Number(n) if matches!(typ, Typ::Str(_)) => Ok(Box::new(n.to_string())),
Value::Number(n) if arg_t == "char" && n.is_i64() => {
@@ -583,6 +615,11 @@ fn convert_val(
.unwrap_or(vec![]);
Ok(Box::new(bytes))
}
Value::Object(_) if arg_t == "text" || arg_t == "varchar" => {
Ok(Box::new(serde_json::to_string(value).map_err(|err| {
Error::ExecutionErr(format!("Failed to convert JSON to text: {}", err))
})?))
}
Value::Object(_) => Ok(Box::new(value.clone())),
Value::String(s) => Ok(Box::new(s.clone())),
_ => Err(Error::ExecutionErr(format!(
backend/windmill-worker/src/python_executor.rs
+125 -37
View File
@@ -1,4 +1,4 @@
use std::{collections::HashMap, process::Stdio};
use std::{collections::HashMap, fs, process::Stdio};
use itertools::Itertools;
use regex::Regex;
@@ -42,6 +42,10 @@ lazy_static::lazy_static! {
static ref USE_PIP_COMPILE: bool = std::env::var("USE_PIP_COMPILE")
.ok().map(|flag| flag == "true").unwrap_or(false);
/// Use pip install
static ref USE_PIP_INSTALL: bool = std::env::var("USE_PIP_INSTALL")
.ok().map(|flag| flag == "true").unwrap_or(false);
static ref RELATIVE_IMPORT_REGEX: Regex = Regex::new(r#"(import|from)\s(((u|f)\.)|\.)"#).unwrap();
@@ -50,6 +54,8 @@ lazy_static::lazy_static! {
}
const NSJAIL_CONFIG_DOWNLOAD_PY_CONTENT: &str = include_str!("../nsjail/download.py.config.proto");
const NSJAIL_CONFIG_DOWNLOAD_PY_CONTENT_FALLBACK: &str =
include_str!("../nsjail/download.py.pip.config.proto");
const NSJAIL_CONFIG_RUN_PYTHON3_CONTENT: &str = include_str!("../nsjail/run.python3.config.proto");
const RELATIVE_PYTHON_LOADER: &str = include_str!("../loader.py");
@@ -66,8 +72,8 @@ use crate::{
},
handle_child::handle_child,
AuthedClientBackgroundTask, DISABLE_NSJAIL, DISABLE_NUSER, HOME_ENV, LOCK_CACHE_DIR,
NSJAIL_PATH, PATH_ENV, PIP_CACHE_DIR, PIP_EXTRA_INDEX_URL, PIP_INDEX_URL, PROXY_ENVS, TZ_ENV,
UV_CACHE_DIR,
NSJAIL_PATH, PATH_ENV, PIP_CACHE_DIR, PIP_EXTRA_INDEX_URL, PIP_INDEX_URL, PROXY_ENVS,
PY311_CACHE_DIR, TZ_ENV, UV_CACHE_DIR,
};
#[cfg(windows)]
@@ -314,7 +320,7 @@ pub async fn uv_pip_compile(
.args(args)
.stdout(Stdio::piped())
.stderr(Stdio::piped());
let child_process = start_child_process(child_cmd, "/usr/local/bin/uv").await?;
let child_process = start_child_process(child_cmd, uv_cmd).await?;
append_logs(&job_id, &w_id, logs, db).await;
handle_child(
job_id,
@@ -901,10 +907,10 @@ async fn handle_python_deps(
.unwrap_or_else(|| vec![])
.clone();
let annotations = windmill_common::worker::PythonAnnotations::parse(inner_content);
let requirements = match requirements_o {
Some(r) => r,
None => {
let annotation = windmill_common::worker::PythonAnnotations::parse(inner_content);
let mut already_visited = vec![];
let requirements = windmill_parser_py_imports::parse_python_imports(
@@ -929,8 +935,8 @@ async fn handle_python_deps(
worker_name,
w_id,
occupancy_metrics,
annotation.no_uv,
annotation.no_cache,
annotations.no_uv || annotations.no_uv_compile,
annotations.no_cache,
)
.await
.map_err(|e| {
@@ -955,6 +961,8 @@ async fn handle_python_deps(
job_dir,
worker_dir,
occupancy_metrics,
annotations.no_uv || annotations.no_uv_install,
false,
)
.await?;
additional_python_paths.append(&mut venv_path);
@@ -966,6 +974,7 @@ lazy_static::lazy_static! {
static ref PIP_SECRET_VARIABLE: Regex = Regex::new(r"\$\{PIP_SECRET:([^\s\}]+)\}").unwrap();
}
/// pip install, include cached or pull from S3
pub async fn handle_python_reqs(
requirements: Vec<&str>,
job_id: &Uuid,
@@ -977,12 +986,22 @@ pub async fn handle_python_reqs(
job_dir: &str,
worker_dir: &str,
occupancy_metrics: &mut Option<&mut OccupancyMetrics>,
// TODO: Remove (Deprecated)
mut no_uv_install: bool,
is_ansible: bool,
) -> error::Result<Vec<String>> {
let mut req_paths: Vec<String> = vec![];
let mut vars = vec![("PATH", PATH_ENV.as_str())];
let pip_extra_index_url;
let pip_index_url;
no_uv_install |= *USE_PIP_INSTALL;
if no_uv_install && !is_ansible {
append_logs(&job_id, w_id, "\nFallback to pip (Deprecated!)\n", db).await;
tracing::warn!("Fallback to pip");
}
if !*DISABLE_NSJAIL {
pip_extra_index_url = PIP_EXTRA_INDEX_URL
.read()
@@ -1013,10 +1032,21 @@ pub async fn handle_python_reqs(
let _ = write_file(
job_dir,
"download.config.proto",
&NSJAIL_CONFIG_DOWNLOAD_PY_CONTENT
.replace("{WORKER_DIR}", &worker_dir)
.replace("{CACHE_DIR}", PIP_CACHE_DIR)
.replace("{CLONE_NEWUSER}", &(!*DISABLE_NUSER).to_string()),
&(if no_uv_install {
NSJAIL_CONFIG_DOWNLOAD_PY_CONTENT_FALLBACK
} else {
NSJAIL_CONFIG_DOWNLOAD_PY_CONTENT
})
.replace("{WORKER_DIR}", &worker_dir)
.replace(
"{CACHE_DIR}",
if no_uv_install {
PIP_CACHE_DIR
} else {
PY311_CACHE_DIR
},
)
.replace("{CLONE_NEWUSER}", &(!*DISABLE_NUSER).to_string()),
)?;
};
@@ -1026,8 +1056,14 @@ pub async fn handle_python_reqs(
if req.starts_with('#') {
continue;
}
let py_prefix = if no_uv_install {
PIP_CACHE_DIR
} else {
PY311_CACHE_DIR
};
let venv_p = format!(
"{PIP_CACHE_DIR}/{}",
"{py_prefix}/{}",
req.replace(' ', "").replace('/', "").replace(':', "")
);
if metadata(&venv_p).await.is_ok() {
@@ -1073,7 +1109,10 @@ pub async fn handle_python_reqs(
.map(|(req, venv_p)| {
let os = os.clone();
async move {
if pull_from_tar(os, venv_p.clone()).await.is_ok() {
if pull_from_tar(os, venv_p.clone(), no_uv_install)
.await
.is_ok()
{
PullFromTar::Pulled(venv_p.to_string())
} else {
PullFromTar::NotPulled(req.to_string(), venv_p.to_string())
@@ -1114,7 +1153,11 @@ pub async fn handle_python_reqs(
for (req, venv_p) in req_with_penv {
let mut logs1 = String::new();
logs1.push_str("\n\n--- PIP INSTALL ---\n");
if no_uv_install {
logs1.push_str("\n\n--- PIP INSTALL ---\n");
} else {
logs1.push_str("\n\n--- UV PIP INSTALL ---\n");
}
logs1.push_str(&format!("\n{req} is being installed for the first time.\n It will be cached for all ulterior uses."));
append_logs(&job_id, w_id, logs1, db).await;
@@ -1150,21 +1193,47 @@ pub async fn handle_python_reqs(
#[cfg(windows)]
let req = format!("{}", req);
let mut command_args = vec![
PYTHON_PATH.as_str(),
"-m",
"pip",
"install",
&req,
"-I",
"--no-deps",
"--no-color",
"--isolated",
"--no-warn-conflicts",
"--disable-pip-version-check",
"-t",
venv_p.as_str(),
];
let mut command_args = if no_uv_install {
vec![
PYTHON_PATH.as_str(),
"-m",
"pip",
"install",
&req,
"-I",
"--no-deps",
"--no-color",
"--isolated",
"--no-warn-conflicts",
"--disable-pip-version-check",
"-t",
venv_p.as_str(),
]
} else {
vec![
UV_PATH.as_str(),
"pip",
"install",
&req,
"--no-deps",
"--no-color",
// "-p",
// "3.11",
// Prevent uv from discovering configuration files.
"--no-config",
"--link-mode=copy",
// TODO: Doublecheck it
"--system",
// Prefer main index over extra
// https://docs.astral.sh/uv/pip/compatibility/#packages-that-exist-on-multiple-indexes
// TODO: Use env variable that can be toggled from UI
"--index-strategy",
"unsafe-best-match",
"--target",
venv_p.as_str(),
"--no-cache",
]
};
let pip_extra_index_url = PIP_EXTRA_INDEX_URL
.read()
.await
@@ -1196,7 +1265,7 @@ pub async fn handle_python_reqs(
envs.push(("HOME", HOME_ENV.as_str()));
tracing::debug!("pip install command: {:?}", command_args);
tracing::debug!("uv pip install command: {:?}", command_args);
#[cfg(unix)]
{
@@ -1207,7 +1276,12 @@ pub async fn handle_python_reqs(
.envs(envs)
.args([
"-x",
&format!("{}/pip-{}.lock", LOCK_CACHE_DIR, fssafe_req),
&format!(
"{}/{}-{}.lock",
LOCK_CACHE_DIR,
if no_uv_install { "pip" } else { "py311" },
fssafe_req
),
"--command",
&command_args.join(" "),
])
@@ -1218,16 +1292,20 @@ pub async fn handle_python_reqs(
#[cfg(windows)]
{
let mut pip_cmd = Command::new(PYTHON_PATH.as_str());
pip_cmd
.env_clear()
let installer_path = if no_uv_install { command_args[0] } else { "uv" };
let mut cmd: Command = Command::new(&installer_path);
cmd.env_clear()
.envs(envs)
.envs(PROXY_ENVS.clone())
.env("SystemRoot", SYSTEM_ROOT.as_str())
.env(
"TMP",
std::env::var("TMP").unwrap_or_else(|_| String::from("/tmp")),
)
.args(&command_args[1..])
.stdout(Stdio::piped())
.stderr(Stdio::piped());
start_child_process(pip_cmd, PYTHON_PATH.as_str()).await?
start_child_process(cmd, installer_path).await?
}
};
@@ -1240,7 +1318,7 @@ pub async fn handle_python_reqs(
false,
worker_name,
&w_id,
&format!("pip install {req}"),
&format!("uv pip install {req}"),
None,
false,
occupancy_metrics,
@@ -1252,6 +1330,16 @@ pub async fn handle_python_reqs(
"finished setting up python dependencies {}",
job_id
);
if child.is_err() {
if let Err(e) = fs::remove_dir_all(&venv_p) {
tracing::warn!(
workspace_id = %w_id,
"failed to remove cache dir: {:?}",
e
);
}
}
child?;
#[cfg(all(feature = "enterprise", feature = "parquet"))]
@@ -1260,7 +1348,7 @@ pub async fn handle_python_reqs(
tracing::warn!("S3 cache not available in the pro plan");
} else {
let venv_p = venv_p.clone();
tokio::spawn(build_tar_and_push(os, venv_p));
tokio::spawn(build_tar_and_push(os, venv_p, no_uv_install));
}
}
req_paths.push(venv_p);
backend/windmill-worker/src/worker.rs
+18
View File
@@ -236,7 +236,14 @@ pub const TMP_LOGS_DIR: &str = concatcp!(TMP_DIR, "/logs");
pub const ROOT_CACHE_NOMOUNT_DIR: &str = concatcp!(TMP_DIR, "/cache_nomount/");
pub const LOCK_CACHE_DIR: &str = concatcp!(ROOT_CACHE_DIR, "lock");
// Used as fallback now
pub const PIP_CACHE_DIR: &str = concatcp!(ROOT_CACHE_DIR, "pip");
// pub const PY310_CACHE_DIR: &str = concatcp!(ROOT_CACHE_DIR, "python_310");
pub const PY311_CACHE_DIR: &str = concatcp!(ROOT_CACHE_DIR, "python_311");
// pub const PY312_CACHE_DIR: &str = concatcp!(ROOT_CACHE_DIR, "python_312");
// pub const PY313_CACHE_DIR: &str = concatcp!(ROOT_CACHE_DIR, "python_313");
pub const UV_CACHE_DIR: &str = concatcp!(ROOT_CACHE_DIR, "uv");
pub const TAR_PIP_CACHE_DIR: &str = concatcp!(ROOT_CACHE_DIR, "tar/pip");
pub const DENO_CACHE_DIR: &str = concatcp!(ROOT_CACHE_DIR, "deno");
@@ -257,6 +264,7 @@ const NUM_SECS_PING: u64 = 5;
const NUM_SECS_READINGS: u64 = 60;
const INCLUDE_DEPS_PY_SH_CONTENT: &str = include_str!("../nsjail/download_deps.py.sh");
const INCLUDE_DEPS_PY_SH_CONTENT_FALLBACK: &str = include_str!("../nsjail/download_deps.py.pip.sh");
pub const DEFAULT_CLOUD_TIMEOUT: u64 = 900;
pub const DEFAULT_SELFHOSTED_TIMEOUT: u64 = 604800; // 7 days
@@ -311,6 +319,7 @@ lazy_static::lazy_static! {
.and_then(|x| x.parse::<bool>().ok())
.unwrap_or(false);
// pub static ref DISABLE_NSJAIL: bool = false;
pub static ref DISABLE_NSJAIL: bool = std::env::var("DISABLE_NSJAIL")
.ok()
.and_then(|x| x.parse::<bool>().ok())
@@ -740,6 +749,13 @@ pub async fn run_worker<R: rsmq_async::RsmqConnection + Send + Sync + Clone + 's
"download_deps.py.sh",
INCLUDE_DEPS_PY_SH_CONTENT,
);
// TODO: Remove (Deprecated)
let _ = write_file(
&worker_dir,
"download_deps.py.pip.sh",
INCLUDE_DEPS_PY_SH_CONTENT_FALLBACK,
);
}
let mut last_ping = Instant::now() - Duration::from_secs(NUM_SECS_PING + 1);
@@ -1903,8 +1919,10 @@ async fn handle_queued_job<R: rsmq_async::RsmqConnection + Send + Sync + Clone>(
}
};
if job.is_flow() {
let flow = job.parse_raw_flow();
handle_flow(
job,
flow,
db,
&client.get_authed().await,
None,
backend/windmill-worker/src/worker_flow.rs
+44 -66
View File
@@ -8,7 +8,6 @@
use std::collections::hash_map::DefaultHasher;
use std::collections::HashMap;
use std::hash::Hash;
use std::sync::atomic::{AtomicUsize, Ordering};
use std::sync::Arc;
use std::time::Duration;
@@ -177,11 +176,6 @@ struct RecoveryObject {
recover: Option<bool>,
}
#[derive(sqlx::FromRow, Deserialize)]
pub struct RowFlowStatus {
pub flow_status: sqlx::types::Json<Box<serde_json::value::RawValue>>,
pub current_module: Option<sqlx::types::Json<Box<serde_json::value::RawValue>>>,
}
// #[instrument(level = "trace", skip_all)]
pub async fn update_flow_status_after_job_completion_internal<
R: rsmq_async::RsmqConnection + Send + Sync + Clone,
@@ -207,6 +201,7 @@ pub async fn update_flow_status_after_job_completion_internal<
let (
should_continue_flow,
flow_job,
flow_value,
stop_early,
skip_if_stop_early,
nresult,
@@ -215,35 +210,28 @@ pub async fn update_flow_status_after_job_completion_internal<
) = {
// tracing::debug!("UPDATE FLOW STATUS: {flow:?} {success} {result:?} {w_id} {depth}");
let old_status_json = sqlx::query_as::<_, RowFlowStatus>(
"SELECT flow_status, raw_flow->'modules'->(flow_status->'step')::int as current_module FROM queue WHERE id = $1 AND workspace_id = $2",
let (old_status, current_module) = sqlx::query!(
"SELECT
flow_status AS \"flow_status!: Json<Box<RawValue>>\",
raw_flow->'modules'->(flow_status->'step')::int AS \"module: Json<Box<RawValue>>\"
FROM queue WHERE id = $1 AND workspace_id = $2 LIMIT 1",
flow, w_id
)
.bind(flow)
.bind(w_id)
.fetch_one(db)
.await
.map_err(|e| {
Error::InternalErr(format!(
"fetching flow status {flow} while reporting {success} {result:?}: {e:#}"
))
})?;
let old_status = serde_json::from_str::<FlowStatus>(old_status_json.flow_status.get())
.or_else(|e| {
Err(Error::InternalErr(format!(
"requiring status to be parsable as FlowStatus: {e:?}"
)))
})?;
let current_module = if let Some(x) = old_status_json.current_module {
Some(serde_json::from_str::<FlowModule>(x.0.get()).or_else(|e| {
Err(Error::InternalErr(format!(
.map_err(|e| Error::InternalErr(
format!("fetching flow status {flow} while reporting {success} {result:?}: {e:#}")
))
.and_then(|record| Ok((
serde_json::from_str::<FlowStatus>(record.flow_status.0.get()).map_err(|e| Error::InternalErr(
format!("requiring current module to be parsable as FlowStatus: {e:?}")
))?,
record.module.map(|json| {
serde_json::from_str::<FlowModule>(json.0.get()).map_err(|e| Error::InternalErr(format!(
"requiring current module to be parsable as FlowModule: {e:?}"
)))
})?)
} else {
None
};
}).transpose()?,
)))?;
let module_step = Step::from_i32_and_len(old_status.step, old_status.modules.len());
@@ -303,16 +291,15 @@ pub async fn update_flow_status_after_job_completion_internal<
let is_flow = if let Some(step) = step {
sqlx::query_scalar!(
"SELECT raw_flow->'modules'->($1)->'value'->>'type' = 'flow' FROM queue WHERE id = $2",
step as i32,
&flow
"SELECT raw_flow->'modules'->($1)::text->'value'->>'type' = 'flow' FROM queue WHERE id = $2 LIMIT 1",
step as i32, flow
)
.fetch_one(db)
.await
.map_err(|e| {
Error::InternalErr(format!("error during retrieval of step's type: {e:#}"))
})?
.unwrap_or(false)
.fetch_one(db)
.await
.map_err(|e| {
Error::InternalErr(format!("error during retrieval of step's type: {e:#}"))
})?
.unwrap_or(false)
} else {
false
};
@@ -938,10 +925,7 @@ pub async fn update_flow_status_after_job_completion_internal<
.unwrap_or_else(|| "none".to_string());
tracing::info!(id = %flow_job.id, root_id = %job_root, "update flow status");
let module = get_module(&flow_job, &module_step);
// tracing::error!(
// "UPDATE FLOW STATUS 3: {module:#?} {unrecoverable} {} {is_last_step} {success} {skip_error_handler} is_failure_step {is_failure_step}", flow_job.canceled
// );
let flow_value = flow_job.parse_raw_flow();
let should_continue_flow = match success {
_ if stop_early => false,
@@ -953,7 +937,14 @@ pub async fn update_flow_status_after_job_completion_internal<
}
false
if next_retry(
&module.and_then(|m| m.retry.clone()).unwrap_or_default(),
flow_value
.as_ref()
.and_then(|value| match module_step {
Step::PreprocessorStep => value.preprocessor_module.as_ref().and_then(|m| m.retry.as_ref()),
Step::Step(i) => value.modules.get(i).as_ref().and_then(|m| m.retry.as_ref()),
Step::FailureStep => value.failure_module.as_ref().and_then(|m| m.retry.as_ref()),
})
.unwrap_or(&Retry::default()),
&old_status.retry,
)
.is_some() =>
@@ -975,6 +966,7 @@ pub async fn update_flow_status_after_job_completion_internal<
(
should_continue_flow,
flow_job,
flow_value,
stop_early,
skip_if_stop_early,
nresult,
@@ -1042,13 +1034,11 @@ pub async fn update_flow_status_after_job_completion_internal<
let args_hash =
hash_args(db, client, w_id, job_id_for_status, &flow_job.args).await;
let flow_path = flow_job.script_path();
let version_hash = if let Some(rc) = flow_job.raw_flow.as_ref() {
use std::hash::Hasher;
let mut s = DefaultHasher::new();
serde_json::to_string(&rc.0)
.unwrap_or_default()
.hash(&mut s);
format!("flow_{}", hex::encode(s.finish().to_be_bytes()))
let version_hash = if let Some(sqlx::types::Json(s)) = flow_job.raw_flow.as_ref() {
use std::hash::{Hash, Hasher};
let mut h = DefaultHasher::new();
s.get().hash(&mut h);
format!("flow_{}", hex::encode(h.finish().to_be_bytes()))
} else {
"flow_unknown".to_string()
};
@@ -1107,6 +1097,7 @@ pub async fn update_flow_status_after_job_completion_internal<
tracing::debug!(id = %flow_job.id, "start handle flow");
match handle_flow(
flow_job.clone(),
flow_value,
db,
client,
Some(nresult.clone()),
@@ -1239,19 +1230,6 @@ async fn retrieve_flow_jobs_results(
Ok(to_raw_value(&results))
}
fn get_module(flow_job: &QueuedJob, module_step: &Step) -> Option<FlowModule> {
let raw_flow = flow_job.parse_raw_flow();
if let Some(raw_flow) = raw_flow {
match module_step {
Step::PreprocessorStep => raw_flow.preprocessor_module.map(|x| *x.clone()),
Step::Step(i) => raw_flow.modules.get(*i).map(|x| x.clone()),
Step::FailureStep => raw_flow.failure_module.map(|x| *x.clone()),
}
} else {
None
}
}
async fn compute_skip_branchall_failure<'c>(
job: &Uuid,
branch: usize,
@@ -1506,6 +1484,7 @@ async fn transform_input(
#[instrument(level = "trace", skip_all)]
pub async fn handle_flow<R: rsmq_async::RsmqConnection + Send + Sync + Clone>(
flow_job: Arc<QueuedJob>,
flow_value: Option<FlowValue>,
db: &sqlx::Pool<sqlx::Postgres>,
client: &AuthedClient,
last_result: Option<Arc<Box<RawValue>>>,
@@ -1514,8 +1493,7 @@ pub async fn handle_flow<R: rsmq_async::RsmqConnection + Send + Sync + Clone>(
rsmq: Option<R>,
job_completed_tx: Sender<SendResult>,
) -> anyhow::Result<()> {
let flow = flow_job
.parse_raw_flow()
let flow = flow_value
.with_context(|| "Unable to parse flow definition")?;
let status = flow_job
.parse_flow_status()
backend/windmill-worker/src/worker_lockfiles.rs
+2
View File
@@ -1312,6 +1312,8 @@ async fn python_dep(
job_dir,
worker_dir,
occupancy_metrics,
false,
false,
)
.await;
benchmarks/lib.ts
+1 -1
View File
@@ -2,7 +2,7 @@ import { sleep } from "https://deno.land/x/sleep@v1.2.1/mod.ts";
import * as windmill from "https://deno.land/x/windmill@v1.174.0/mod.ts";
import * as api from "https://deno.land/x/windmill@v1.174.0/windmill-api/index.ts";
export const VERSION = "v1.423.2";
export const VERSION = "v1.425.1";
export async function login(email: string, password: string): Promise<string> {
return await windmill.UserService.login({
benchmarks/suite_config.json
+2 -2
View File
@@ -33,10 +33,10 @@
},
{
"kind": "bigrawscript",
"jobs": 10000
"jobs": 4000
},
{
"kind": "bigscriptinflow",
"jobs": 10000
"jobs": 4000
}
]
cli/gen/core/OpenAPI.ts
+1 -1
View File
@@ -54,7 +54,7 @@ export const OpenAPI: OpenAPIConfig = {
PASSWORD: undefined,
TOKEN: getEnv("WM_TOKEN"),
USERNAME: undefined,
VERSION: '1.407.2',
VERSION: '1.425.1',
WITH_CREDENTIALS: true,
interceptors: {
request: new Interceptors(),
cli/gen/services.gen.ts
+519 -9
View File
File diff suppressed because one or more lines are too long
cli/gen/types.gen.ts
+434 -11
View File
@@ -1,5 +1,10 @@
// This file is auto-generated by @hey-api/openapi-ts
export type AiResource = {
path: string;
provider: string;
};
export type Script = {
workspace_id?: string;
hash: string;
@@ -257,18 +262,21 @@ export type TruncatedToken = {
created_at: string;
last_used_at: string;
scopes?: Array<(string)>;
email?: string;
};
export type NewToken = {
label?: string;
expiration?: string;
scopes?: Array<(string)>;
workspace_id?: string;
};
export type NewTokenImpersonate = {
label?: string;
expiration?: string;
impersonate_email: string;
workspace_id?: string;
};
export type ListableVariable = {
@@ -317,7 +325,7 @@ export type AuditLog = {
id: number;
timestamp: string;
username: string;
operation: 'jobs.run' | 'jobs.run.script' | 'jobs.run.preview' | 'jobs.run.flow' | 'jobs.run.flow_preview' | 'jobs.run.script_hub' | 'jobs.run.dependencies' | 'jobs.run.identity' | 'jobs.run.noop' | 'jobs.flow_dependencies' | 'jobs' | 'jobs.cancel' | 'jobs.force_cancel' | 'jobs.disapproval' | 'jobs.delete' | 'account.delete' | 'openai.request' | 'resources.create' | 'resources.update' | 'resources.delete' | 'resource_types.create' | 'resource_types.update' | 'resource_types.delete' | 'schedule.create' | 'schedule.setenabled' | 'schedule.edit' | 'schedule.delete' | 'scripts.create' | 'scripts.update' | 'scripts.archive' | 'scripts.delete' | 'users.create' | 'users.delete' | 'users.update' | 'users.login' | 'users.logout' | 'users.accept_invite' | 'users.decline_invite' | 'users.token.create' | 'users.token.delete' | 'users.add_to_workspace' | 'users.add_global' | 'users.setpassword' | 'users.impersonate' | 'users.leave_workspace' | 'oauth.login' | 'oauth.signup' | 'variables.create' | 'variables.delete' | 'variables.update' | 'flows.create' | 'flows.update' | 'flows.delete' | 'flows.archive' | 'apps.create' | 'apps.update' | 'apps.delete' | 'folder.create' | 'folder.update' | 'folder.delete' | 'folder.add_owner' | 'folder.remove_owner' | 'group.create' | 'group.delete' | 'group.edit' | 'group.adduser' | 'group.removeuser' | 'igroup.create' | 'igroup.delete' | 'igroup.adduser' | 'igroup.removeuser' | 'variables.decrypt_secret' | 'workspaces.edit_command_script' | 'workspaces.edit_deploy_to' | 'workspaces.edit_auto_invite_domain' | 'workspaces.edit_webhook' | 'workspaces.edit_copilot_config' | 'workspaces.edit_error_handler' | 'workspaces.create' | 'workspaces.update' | 'workspaces.archive' | 'workspaces.unarchive' | 'workspaces.delete';
operation: 'jobs.run' | 'jobs.run.script' | 'jobs.run.preview' | 'jobs.run.flow' | 'jobs.run.flow_preview' | 'jobs.run.script_hub' | 'jobs.run.dependencies' | 'jobs.run.identity' | 'jobs.run.noop' | 'jobs.flow_dependencies' | 'jobs' | 'jobs.cancel' | 'jobs.force_cancel' | 'jobs.disapproval' | 'jobs.delete' | 'account.delete' | 'ai.request' | 'resources.create' | 'resources.update' | 'resources.delete' | 'resource_types.create' | 'resource_types.update' | 'resource_types.delete' | 'schedule.create' | 'schedule.setenabled' | 'schedule.edit' | 'schedule.delete' | 'scripts.create' | 'scripts.update' | 'scripts.archive' | 'scripts.delete' | 'users.create' | 'users.delete' | 'users.update' | 'users.login' | 'users.login_failure' | 'users.logout' | 'users.accept_invite' | 'users.decline_invite' | 'users.token.create' | 'users.token.delete' | 'users.add_to_workspace' | 'users.add_global' | 'users.setpassword' | 'users.impersonate' | 'users.leave_workspace' | 'oauth.login' | 'oauth.login_failure' | 'oauth.signup' | 'variables.create' | 'variables.delete' | 'variables.update' | 'flows.create' | 'flows.update' | 'flows.delete' | 'flows.archive' | 'apps.create' | 'apps.update' | 'apps.delete' | 'folder.create' | 'folder.update' | 'folder.delete' | 'folder.add_owner' | 'folder.remove_owner' | 'group.create' | 'group.delete' | 'group.edit' | 'group.adduser' | 'group.removeuser' | 'igroup.create' | 'igroup.delete' | 'igroup.adduser' | 'igroup.removeuser' | 'variables.decrypt_secret' | 'workspaces.edit_command_script' | 'workspaces.edit_deploy_to' | 'workspaces.edit_auto_invite_domain' | 'workspaces.edit_webhook' | 'workspaces.edit_copilot_config' | 'workspaces.edit_error_handler' | 'workspaces.create' | 'workspaces.update' | 'workspaces.archive' | 'workspaces.unarchive' | 'workspaces.delete';
action_kind: 'Created' | 'Updated' | 'Delete' | 'Execute';
resource?: string;
parameters?: {
@@ -325,7 +333,7 @@ export type AuditLog = {
};
};
export type operation = 'jobs.run' | 'jobs.run.script' | 'jobs.run.preview' | 'jobs.run.flow' | 'jobs.run.flow_preview' | 'jobs.run.script_hub' | 'jobs.run.dependencies' | 'jobs.run.identity' | 'jobs.run.noop' | 'jobs.flow_dependencies' | 'jobs' | 'jobs.cancel' | 'jobs.force_cancel' | 'jobs.disapproval' | 'jobs.delete' | 'account.delete' | 'openai.request' | 'resources.create' | 'resources.update' | 'resources.delete' | 'resource_types.create' | 'resource_types.update' | 'resource_types.delete' | 'schedule.create' | 'schedule.setenabled' | 'schedule.edit' | 'schedule.delete' | 'scripts.create' | 'scripts.update' | 'scripts.archive' | 'scripts.delete' | 'users.create' | 'users.delete' | 'users.update' | 'users.login' | 'users.logout' | 'users.accept_invite' | 'users.decline_invite' | 'users.token.create' | 'users.token.delete' | 'users.add_to_workspace' | 'users.add_global' | 'users.setpassword' | 'users.impersonate' | 'users.leave_workspace' | 'oauth.login' | 'oauth.signup' | 'variables.create' | 'variables.delete' | 'variables.update' | 'flows.create' | 'flows.update' | 'flows.delete' | 'flows.archive' | 'apps.create' | 'apps.update' | 'apps.delete' | 'folder.create' | 'folder.update' | 'folder.delete' | 'folder.add_owner' | 'folder.remove_owner' | 'group.create' | 'group.delete' | 'group.edit' | 'group.adduser' | 'group.removeuser' | 'igroup.create' | 'igroup.delete' | 'igroup.adduser' | 'igroup.removeuser' | 'variables.decrypt_secret' | 'workspaces.edit_command_script' | 'workspaces.edit_deploy_to' | 'workspaces.edit_auto_invite_domain' | 'workspaces.edit_webhook' | 'workspaces.edit_copilot_config' | 'workspaces.edit_error_handler' | 'workspaces.create' | 'workspaces.update' | 'workspaces.archive' | 'workspaces.unarchive' | 'workspaces.delete';
export type operation = 'jobs.run' | 'jobs.run.script' | 'jobs.run.preview' | 'jobs.run.flow' | 'jobs.run.flow_preview' | 'jobs.run.script_hub' | 'jobs.run.dependencies' | 'jobs.run.identity' | 'jobs.run.noop' | 'jobs.flow_dependencies' | 'jobs' | 'jobs.cancel' | 'jobs.force_cancel' | 'jobs.disapproval' | 'jobs.delete' | 'account.delete' | 'ai.request' | 'resources.create' | 'resources.update' | 'resources.delete' | 'resource_types.create' | 'resource_types.update' | 'resource_types.delete' | 'schedule.create' | 'schedule.setenabled' | 'schedule.edit' | 'schedule.delete' | 'scripts.create' | 'scripts.update' | 'scripts.archive' | 'scripts.delete' | 'users.create' | 'users.delete' | 'users.update' | 'users.login' | 'users.login_failure' | 'users.logout' | 'users.accept_invite' | 'users.decline_invite' | 'users.token.create' | 'users.token.delete' | 'users.add_to_workspace' | 'users.add_global' | 'users.setpassword' | 'users.impersonate' | 'users.leave_workspace' | 'oauth.login' | 'oauth.login_failure' | 'oauth.signup' | 'variables.create' | 'variables.delete' | 'variables.update' | 'flows.create' | 'flows.update' | 'flows.delete' | 'flows.archive' | 'apps.create' | 'apps.update' | 'apps.delete' | 'folder.create' | 'folder.update' | 'folder.delete' | 'folder.add_owner' | 'folder.remove_owner' | 'group.create' | 'group.delete' | 'group.edit' | 'group.adduser' | 'group.removeuser' | 'igroup.create' | 'igroup.delete' | 'igroup.adduser' | 'igroup.removeuser' | 'variables.decrypt_secret' | 'workspaces.edit_command_script' | 'workspaces.edit_deploy_to' | 'workspaces.edit_auto_invite_domain' | 'workspaces.edit_webhook' | 'workspaces.edit_copilot_config' | 'workspaces.edit_error_handler' | 'workspaces.create' | 'workspaces.update' | 'workspaces.archive' | 'workspaces.unarchive' | 'workspaces.delete';
export type action_kind = 'Created' | 'Updated' | 'Delete' | 'Execute';
@@ -542,6 +550,11 @@ export type HttpTrigger = {
edited_at: string;
script_path: string;
route_path: string;
static_asset_config?: {
s3: string;
storage?: string;
filename?: string;
};
is_flow: boolean;
extra_perms: {
[key: string]: (boolean);
@@ -559,6 +572,11 @@ export type NewHttpTrigger = {
path: string;
script_path: string;
route_path: string;
static_asset_config?: {
s3: string;
storage?: string;
filename?: string;
};
is_flow: boolean;
http_method: 'get' | 'post' | 'put' | 'delete' | 'patch';
is_async: boolean;
@@ -569,12 +587,89 @@ export type EditHttpTrigger = {
path: string;
script_path: string;
route_path?: string;
static_asset_config?: {
s3: string;
storage?: string;
filename?: string;
};
is_flow: boolean;
http_method: 'get' | 'post' | 'put' | 'delete' | 'patch';
is_async: boolean;
requires_auth: boolean;
};
export type TriggersCount = {
primary_schedule?: {
schedule?: string;
};
schedule_count?: number;
http_routes_count?: number;
webhook_count?: number;
email_count?: number;
websocket_count?: number;
};
export type WebsocketTrigger = {
path: string;
edited_by: string;
edited_at: string;
script_path: string;
url: string;
is_flow: boolean;
extra_perms: {
[key: string]: (boolean);
};
email: string;
workspace_id: string;
server_id?: string;
last_server_ping?: string;
error?: string;
enabled: boolean;
filters: Array<{
key: string;
value: unknown;
}>;
initial_messages: Array<WebsocketTriggerInitialMessage>;
url_runnable_args: ScriptArgs;
};
export type NewWebsocketTrigger = {
path: string;
script_path: string;
is_flow: boolean;
url: string;
enabled?: boolean;
filters: Array<{
key: string;
value: unknown;
}>;
initial_messages: Array<WebsocketTriggerInitialMessage>;
url_runnable_args: ScriptArgs;
};
export type EditWebsocketTrigger = {
url: string;
path: string;
script_path: string;
is_flow: boolean;
filters: Array<{
key: string;
value: unknown;
}>;
initial_messages: Array<WebsocketTriggerInitialMessage>;
url_runnable_args: ScriptArgs;
};
export type WebsocketTriggerInitialMessage = {
raw_message: string;
} | {
runnable_result: {
path: string;
args: ScriptArgs;
is_flow: boolean;
};
};
export type Group = {
name: string;
summary?: string;
@@ -660,6 +755,7 @@ export type GlobalUserInfo = {
name?: string;
company?: string;
username?: string;
operator_only?: boolean;
};
export type login_type = 'password' | 'github';
@@ -722,6 +818,9 @@ export type Policy = {
[key: string]: unknown;
};
};
s3_inputs?: Array<{
[key: string]: unknown;
}>;
execution_mode?: 'viewer' | 'publisher' | 'anonymous';
on_behalf_of?: string;
on_behalf_of_email?: string;
@@ -968,6 +1067,46 @@ export type JobSearchHit = {
dancer?: string;
};
export type LogSearchHit = {
dancer?: string;
};
export type AutoscalingEvent = {
id?: number;
worker_group?: string;
event_type?: string;
desired_workers?: number;
reason?: string;
applied_at?: string;
};
export type CriticalAlert = {
/**
* Unique identifier for the alert
*/
id?: number;
/**
* Type of alert (e.g., critical_error)
*/
alert_type?: string;
/**
* The message content of the alert
*/
message?: string;
/**
* Time when the alert was created
*/
created_at?: string;
/**
* Acknowledgment status of the alert, can be true, false, or null if not set
*/
acknowledged?: (boolean) | null;
/**
* Workspace id if the alert is in the scope of a workspace
*/
workspace_id?: (string) | null;
};
export type OpenFlow = {
summary: string;
description?: string;
@@ -1073,6 +1212,7 @@ export type RawScript = {
concurrent_limit?: number;
concurrency_time_window_s?: number;
custom_concurrency_key?: string;
is_trigger?: boolean;
};
export type language2 = 'deno' | 'bun' | 'python3' | 'go' | 'bash' | 'powershell' | 'postgresql' | 'mysql' | 'bigquery' | 'snowflake' | 'mssql' | 'graphql' | 'nativets' | 'php';
@@ -1085,6 +1225,7 @@ export type PathScript = {
hash?: string;
type: 'script';
tag_override?: string;
is_trigger?: boolean;
};
export type PathFlow = {
@@ -1518,6 +1659,30 @@ export type SetPasswordData = {
export type SetPasswordResponse = (string);
export type SetPasswordForUserData = {
/**
* set password
*/
requestBody: {
password: string;
};
user: string;
};
export type SetPasswordForUserResponse = (string);
export type SetLoginTypeForUserData = {
/**
* set login type
*/
requestBody: {
login_type: string;
};
user: string;
};
export type SetLoginTypeForUserResponse = (string);
export type CreateUserGloballyData = {
/**
* user info
@@ -1675,6 +1840,7 @@ export type TestSmtpData = {
port: number;
from: string;
tls_implicit: boolean;
disable_tls: boolean;
};
};
};
@@ -1693,6 +1859,25 @@ export type TestCriticalChannelsData = {
export type TestCriticalChannelsResponse = (string);
export type GetCriticalAlertsData = {
acknowledged?: (boolean) | null;
page?: number;
pageSize?: number;
};
export type GetCriticalAlertsResponse = (Array<CriticalAlert>);
export type AcknowledgeCriticalAlertData = {
/**
* The ID of the critical alert to acknowledge
*/
id: number;
};
export type AcknowledgeCriticalAlertResponse = (string);
export type AcknowledgeAllCriticalAlertsResponse = (string);
export type TestLicenseKeyData = {
/**
* test license key
@@ -1917,6 +2102,10 @@ export type ExistsEmailData = {
export type ExistsEmailResponse = (boolean);
export type ListUsersAsSuperAdminData = {
/**
* filter only active users
*/
activeOnly?: boolean;
/**
* which page to return (start at 1, default 1)
*/
@@ -1952,7 +2141,7 @@ export type GetSettingsResponse = ({
customer_id?: string;
webhook?: string;
deploy_to?: string;
openai_resource_path?: string;
ai_resource?: AiResource;
code_completion_enabled: boolean;
error_handler?: string;
error_handler_extra_args?: ScriptArgs;
@@ -1962,6 +2151,7 @@ export type GetSettingsResponse = ({
deploy_ui?: WorkspaceDeployUISettings;
default_app?: string;
default_scripts?: WorkspaceDefaultScripts;
mute_critical_alerts?: boolean;
});
export type GetDeployToData = {
@@ -2070,7 +2260,7 @@ export type EditCopilotConfigData = {
* WorkspaceCopilotConfig
*/
requestBody: {
openai_resource_path?: string;
ai_resource?: AiResource;
code_completion_enabled: boolean;
};
workspace: string;
@@ -2083,7 +2273,8 @@ export type GetCopilotInfoData = {
};
export type GetCopilotInfoResponse = ({
exists_openai_resource_path: boolean;
ai_provider: string;
exists_ai_resource: boolean;
code_completion_enabled: boolean;
});
@@ -2219,6 +2410,15 @@ export type GetWorkspaceUsageData = {
export type GetWorkspaceUsageResponse = (number);
export type GetUsedTriggersData = {
workspace: string;
};
export type GetUsedTriggersResponse = ({
http_routes_used: boolean;
websocket_used: boolean;
});
export type ListUsersData = {
workspace: string;
};
@@ -2382,6 +2582,46 @@ export type ListContextualVariablesData = {
export type ListContextualVariablesResponse = (Array<ContextualVariable>);
export type WorkspaceGetCriticalAlertsData = {
acknowledged?: (boolean) | null;
page?: number;
pageSize?: number;
workspace: string;
};
export type WorkspaceGetCriticalAlertsResponse = (Array<CriticalAlert>);
export type WorkspaceAcknowledgeCriticalAlertData = {
/**
* The ID of the critical alert to acknowledge
*/
id: number;
workspace: string;
};
export type WorkspaceAcknowledgeCriticalAlertResponse = (string);
export type WorkspaceAcknowledgeAllCriticalAlertsData = {
workspace: string;
};
export type WorkspaceAcknowledgeAllCriticalAlertsResponse = (string);
export type WorkspaceMuteCriticalAlertsUiData = {
/**
* Boolean flag to mute critical alerts.
*/
requestBody: {
/**
* Whether critical alerts should be muted.
*/
mute_critical_alerts?: boolean;
};
workspace: string;
};
export type WorkspaceMuteCriticalAlertsUiResponse = (string);
export type LoginWithOauthData = {
clientName: string;
/**
@@ -2474,7 +2714,10 @@ export type DisconnectSlackData = {
export type DisconnectSlackResponse = (string);
export type ListOauthLoginsResponse = ({
oauth: Array<(string)>;
oauth: Array<{
type: string;
display_name?: string;
}>;
saml?: string;
});
@@ -3011,6 +3254,20 @@ export type GetScriptByPathData = {
export type GetScriptByPathResponse = (Script);
export type GetTriggersCountOfScriptData = {
path: string;
workspace: string;
};
export type GetTriggersCountOfScriptResponse = (TriggersCount);
export type ListTokensOfScriptData = {
path: string;
workspace: string;
};
export type ListTokensOfScriptResponse = (Array<TruncatedToken>);
export type GetScriptByPathWithDraftData = {
path: string;
workspace: string;
@@ -3025,6 +3282,13 @@ export type GetScriptHistoryByPathData = {
export type GetScriptHistoryByPathResponse = (Array<ScriptHistory>);
export type GetScriptLatestVersionData = {
path: string;
workspace: string;
};
export type GetScriptLatestVersionResponse = (ScriptHistory);
export type UpdateScriptHistoryData = {
hash: string;
path: string;
@@ -3378,6 +3642,13 @@ export type GetFlowHistoryData = {
export type GetFlowHistoryResponse = (Array<FlowVersion>);
export type GetFlowLatestVersionData = {
path: string;
workspace: string;
};
export type GetFlowLatestVersionResponse = (FlowVersion);
export type GetFlowVersionData = {
path: string;
version: number;
@@ -3408,6 +3679,20 @@ export type GetFlowByPathData = {
export type GetFlowByPathResponse = (Flow);
export type GetTriggersCountOfFlowData = {
path: string;
workspace: string;
};
export type GetTriggersCountOfFlowResponse = (TriggersCount);
export type ListTokensOfFlowData = {
path: string;
workspace: string;
};
export type ListTokensOfFlowResponse = (Array<TruncatedToken>);
export type ToggleWorkspaceErrorHandlerForFlowData = {
path: string;
/**
@@ -3637,6 +3922,13 @@ export type GetAppHistoryByPathData = {
export type GetAppHistoryByPathResponse = (Array<AppHistory>);
export type GetAppLatestVersionData = {
path: string;
workspace: string;
};
export type GetAppLatestVersionResponse = (AppHistory);
export type UpdateAppHistoryData = {
id: number;
/**
@@ -4099,6 +4391,16 @@ export type GetCompletedCountResponse = ({
database_length: number;
});
export type CountCompletedJobsData = {
allWorkspaces?: boolean;
completedAfterSAgo?: number;
success?: boolean;
tags?: string;
workspace: string;
};
export type CountCompletedJobsResponse = (number);
export type ListFilteredUuidsData = {
/**
* get jobs from all workspaces (only valid if request come from the `admins` workspace)
@@ -4888,11 +5190,80 @@ export type ExistsRouteData = {
export type ExistsRouteResponse = (boolean);
export type UsedData = {
export type CreateWebsocketTriggerData = {
/**
* new websocket trigger
*/
requestBody: NewWebsocketTrigger;
workspace: string;
};
export type UsedResponse = (boolean);
export type CreateWebsocketTriggerResponse = (string);
export type UpdateWebsocketTriggerData = {
path: string;
/**
* updated trigger
*/
requestBody: EditWebsocketTrigger;
workspace: string;
};
export type UpdateWebsocketTriggerResponse = (string);
export type DeleteWebsocketTriggerData = {
path: string;
workspace: string;
};
export type DeleteWebsocketTriggerResponse = (string);
export type GetWebsocketTriggerData = {
path: string;
workspace: string;
};
export type GetWebsocketTriggerResponse = (WebsocketTrigger);
export type ListWebsocketTriggersData = {
isFlow?: boolean;
/**
* which page to return (start at 1, default 1)
*/
page?: number;
/**
* filter by path
*/
path?: string;
pathStart?: string;
/**
* number of items to return for a given page (default 30, max 100)
*/
perPage?: number;
workspace: string;
};
export type ListWebsocketTriggersResponse = (Array<WebsocketTrigger>);
export type ExistsWebsocketTriggerData = {
path: string;
workspace: string;
};
export type ExistsWebsocketTriggerResponse = (boolean);
export type SetWebsocketTriggerEnabledData = {
path: string;
/**
* updated websocket trigger enable
*/
requestBody: {
enabled: boolean;
};
workspace: string;
};
export type SetWebsocketTriggerEnabledResponse = (string);
export type ListInstanceGroupsResponse = (Array<InstanceGroup>);
@@ -5201,6 +5572,10 @@ export type GetQueueMetricsResponse = (Array<{
}>;
}>);
export type GetCountsOfJobsWaitingPerTagResponse = ({
[key: string]: (number);
});
export type ListWorkerGroupsResponse = (Array<{
name: string;
config: unknown;
@@ -5230,8 +5605,14 @@ export type DeleteConfigResponse = (string);
export type ListConfigsResponse = (Array<Config>);
export type ListAutoscalingEventsData = {
workerGroup: string;
};
export type ListAutoscalingEventsResponse = (Array<AutoscalingEvent>);
export type GetGranularAclsData = {
kind: 'script' | 'group_' | 'resource' | 'schedule' | 'variable' | 'flow' | 'folder' | 'app' | 'raw_app' | 'http_trigger';
kind: 'script' | 'group_' | 'resource' | 'schedule' | 'variable' | 'flow' | 'folder' | 'app' | 'raw_app' | 'http_trigger' | 'websocket_trigger';
path: string;
workspace: string;
};
@@ -5241,7 +5622,7 @@ export type GetGranularAclsResponse = ({
});
export type AddGranularAclsData = {
kind: 'script' | 'group_' | 'resource' | 'schedule' | 'variable' | 'flow' | 'folder' | 'app' | 'raw_app' | 'http_trigger';
kind: 'script' | 'group_' | 'resource' | 'schedule' | 'variable' | 'flow' | 'folder' | 'app' | 'raw_app' | 'http_trigger' | 'websocket_trigger';
path: string;
/**
* acl to add
@@ -5256,7 +5637,7 @@ export type AddGranularAclsData = {
export type AddGranularAclsResponse = (string);
export type RemoveGranularAclsData = {
kind: 'script' | 'group_' | 'resource' | 'schedule' | 'variable' | 'flow' | 'folder' | 'app' | 'raw_app' | 'http_trigger';
kind: 'script' | 'group_' | 'resource' | 'schedule' | 'variable' | 'flow' | 'folder' | 'app' | 'raw_app' | 'http_trigger' | 'websocket_trigger';
path: string;
/**
* acl to add
@@ -5569,6 +5950,8 @@ export type MoveS3FileData = {
export type MoveS3FileResponse = (unknown);
export type FileUploadData = {
contentDisposition?: string;
contentType?: string;
fileExtension?: string;
fileKey?: string;
/**
@@ -5817,4 +6200,44 @@ export type SearchJobsIndexResponse = ({
* the jobs that matched the query
*/
hits?: Array<JobSearchHit>;
});
export type SearchLogsIndexData = {
hostname: string;
maxTs?: string;
minTs?: string;
mode: string;
searchQuery: string;
workerGroup?: string;
};
export type SearchLogsIndexResponse = ({
/**
* a list of the terms that couldn't be parsed (and thus ignored)
*/
query_parse_errors?: Array<(string)>;
/**
* log files that matched the query
*/
hits?: Array<LogSearchHit>;
});
export type CountSearchLogsIndexData = {
hosts: string;
maxTs?: string;
minTs?: string;
searchQuery: string;
};
export type CountSearchLogsIndexResponse = ({
/**
* a list of the terms that couldn't be parsed (and thus ignored)
*/
query_parse_errors?: Array<(string)>;
/**
* count of log lines that matched the query per hostname
*/
count_per_host?: {
[key: string]: unknown;
};
});
cli/main.ts
+1 -1
View File
@@ -60,7 +60,7 @@ export {
// }
// });
export const VERSION = "1.423.2";
export const VERSION = "1.425.1";
const command = new Command()
.name("wmill")
cli/settings.ts
+6 -6
View File
@@ -4,10 +4,10 @@ import { colors } from "./deps.ts";
import { yamlParseFile } from "./deps.ts";
import { log } from "./deps.ts";
import { compareInstanceObjects, InstanceSyncOptions } from "./instance.ts";
import { isSuperset, GlobalOptions } from "./types.ts";
import { isSuperset } from "./types.ts";
import { deepEqual } from "./utils.ts";
import * as wmill from "./gen/services.gen.ts";
import { Config, GlobalSetting } from "./gen/types.gen.ts";
import { AiResource, Config, GlobalSetting } from "./gen/types.gen.ts";
import { removeWorkerPrefix } from "./worker_groups.ts";
import process from "node:process";
@@ -24,7 +24,7 @@ export interface SimplifiedSettings {
error_handler?: string;
error_handler_extra_args?: any;
error_handler_muted_on_cancel?: boolean;
openai_resource_path?: string;
ai_resource?: AiResource;
code_completion_enabled: boolean;
large_file_storage?: any;
git_sync?: any;
@@ -80,7 +80,7 @@ export async function pushWorkspaceSettings(
error_handler_extra_args: remoteSettings.error_handler_extra_args,
error_handler_muted_on_cancel:
remoteSettings.error_handler_muted_on_cancel,
openai_resource_path: remoteSettings.openai_resource_path,
ai_resource: remoteSettings.ai_resource,
code_completion_enabled: remoteSettings.code_completion_enabled,
large_file_storage: remoteSettings.large_file_storage,
git_sync: remoteSettings.git_sync,
@@ -153,14 +153,14 @@ export async function pushWorkspaceSettings(
}
}
if (
localSettings.openai_resource_path !== settings.openai_resource_path ||
localSettings.ai_resource !== settings.ai_resource ||
localSettings.code_completion_enabled !== settings.code_completion_enabled
) {
log.debug(`Updating openai settings...`);
await wmill.editCopilotConfig({
workspace,
requestBody: {
openai_resource_path: localSettings.openai_resource_path,
ai_resource: localSettings.ai_resource,
code_completion_enabled: localSettings.code_completion_enabled,
},
});
frontend/package-lock.json
Generated
+46 -2
View File
@@ -1,14 +1,15 @@
{
"name": "windmill-components",
"version": "1.423.2",
"version": "1.425.1",
"lockfileVersion": 3,
"requires": true,
"packages": {
"": {
"name": "windmill-components",
"version": "1.423.2",
"version": "1.425.1",
"license": "AGPL-3.0",
"dependencies": {
"@anthropic-ai/sdk": "^0.32.1",
"@aws-crypto/sha256-js": "^4.0.0",
"@codingame/monaco-vscode-configuration-service-override": "~8.0.2",
"@codingame/monaco-vscode-files-service-override": "~8.0.2",
@@ -21,6 +22,7 @@
"@codingame/monaco-vscode-standalone-typescript-language-features": "~8.0.2",
"@json2csv/plainjs": "^7.0.6",
"@leeoniya/ufuzzy": "^1.0.8",
"@mistralai/mistralai": "^1.3.0",
"@popperjs/core": "^2.11.6",
"@redocly/json-to-json-schema": "^0.0.1",
"@tanstack/svelte-table": "^8.9.9",
@@ -175,6 +177,30 @@
"node": ">=6.0.0"
}
},
"node_modules/@anthropic-ai/sdk": {
"version": "0.32.1",
"resolved": "https://registry.npmjs.org/@anthropic-ai/sdk/-/sdk-0.32.1.tgz",
"integrity": "sha512-U9JwTrDvdQ9iWuABVsMLj8nJVwAyQz6QXvgLsVhryhCEPkLsbcP/MXxm+jYcAwLoV8ESbaTTjnD4kuAFa+Hyjg==",
"license": "MIT",
"dependencies": {
"@types/node": "^18.11.18",
"@types/node-fetch": "^2.6.4",
"abort-controller": "^3.0.0",
"agentkeepalive": "^4.2.1",
"form-data-encoder": "1.7.2",
"formdata-node": "^4.3.2",
"node-fetch": "^2.6.7"
}
},
"node_modules/@anthropic-ai/sdk/node_modules/@types/node": {
"version": "18.19.64",
"resolved": "https://registry.npmjs.org/@types/node/-/node-18.19.64.tgz",
"integrity": "sha512-955mDqvO2vFf/oL7V3WiUtiz+BugyX8uVbaT2H8oj3+8dRyH2FLiNdowe7eNqRM7IOIZvzDH76EoAT+gwm6aIQ==",
"license": "MIT",
"dependencies": {
"undici-types": "~5.26.4"
}
},
"node_modules/@apidevtools/json-schema-ref-parser": {
"version": "11.6.1",
"resolved": "https://registry.npmjs.org/@apidevtools/json-schema-ref-parser/-/json-schema-ref-parser-11.6.1.tgz",
@@ -3659,6 +3685,14 @@
"resolved": "https://registry.npmjs.org/@mapbox/unitbezier/-/unitbezier-0.0.0.tgz",
"integrity": "sha512-HPnRdYO0WjFjRTSwO3frz1wKaU649OBFPX3Zo/2WZvuRi6zMiRGui8SnPQiQABgqCf8YikDe5t3HViTVw1WUzA=="
},
"node_modules/@mistralai/mistralai": {
"version": "1.3.0",
"resolved": "https://registry.npmjs.org/@mistralai/mistralai/-/mistralai-1.3.0.tgz",
"integrity": "sha512-G5DPCSC8sEhG3LUEZDYLD7qEZWDuZXgaX3IcoC3a/ydm9jFuh2pRZtknsgMx2sU8d7kxRuxblY3fPH5C38wnhQ==",
"peerDependencies": {
"zod": ">= 3"
}
},
"node_modules/@nodelib/fs.scandir": {
"version": "2.1.5",
"resolved": "https://registry.npmjs.org/@nodelib/fs.scandir/-/fs.scandir-2.1.5.tgz",
@@ -13738,6 +13772,16 @@
"integrity": "sha512-A7AMeJfGefk317I/3tBoUYRcDcNavKEkpiPN/nQsBz/viI2GvT7BtrqdPD6rGqBFN8Ax7v4obf+Cl32JF9DDVw==",
"dev": true
},
"node_modules/zod": {
"version": "3.23.8",
"resolved": "https://registry.npmjs.org/zod/-/zod-3.23.8.tgz",
"integrity": "sha512-XBx9AXhXktjUqnepgTiE5flcKIYWi/rme0Eaj+5Y0lftuGBq+jyRu/md4WnuxqgP1ubdpNCsYEYPxrzVHD8d6g==",
"license": "MIT",
"peer": true,
"funding": {
"url": "https://github.com/sponsors/colinhacks"
}
},
"node_modules/zstddec": {
"version": "0.1.0",
"resolved": "https://registry.npmjs.org/zstddec/-/zstddec-0.1.0.tgz",
frontend/package.json
+3 -1
View File
@@ -1,6 +1,6 @@
{
"name": "windmill-components",
"version": "1.423.2",
"version": "1.425.1",
"scripts": {
"dev": "vite dev",
"build": "vite build",
@@ -82,6 +82,7 @@
},
"type": "module",
"dependencies": {
"@anthropic-ai/sdk": "^0.32.1",
"@aws-crypto/sha256-js": "^4.0.0",
"@codingame/monaco-vscode-configuration-service-override": "~8.0.2",
"@codingame/monaco-vscode-files-service-override": "~8.0.2",
@@ -94,6 +95,7 @@
"@codingame/monaco-vscode-standalone-typescript-language-features": "~8.0.2",
"@json2csv/plainjs": "^7.0.6",
"@leeoniya/ufuzzy": "^1.0.8",
"@mistralai/mistralai": "^1.3.0",
"@popperjs/core": "^2.11.6",
"@redocly/json-to-json-schema": "^0.0.1",
"@tanstack/svelte-table": "^8.9.9",
frontend/src/lib/components/ChangeInstanceUsernameInner.svelte
+4 -2
View File
@@ -72,11 +72,13 @@
</script>
<div class="flex flex-col max-w-2xl p-2">
{#if isConflict}
<span class="text-sm mb-2 leading-6 font-semibold"
>{isConflict ? 'Fix username conflict' : 'Change username'}</span
>Fix username conflict</span
>
{/if}
<span class="text-xs mb-1 leading-6"
<span class="text-sm font-semibold mb-1 leading-6"
>{isConflict ? 'Auto-generated instance username' : 'New username'}</span
>
<input
frontend/src/lib/components/Dev.svelte
+4 -3
View File
@@ -39,7 +39,6 @@
import { CornerDownLeft, Play } from 'lucide-svelte'
import Toggle from './Toggle.svelte'
import { setLicense } from '$lib/enterpriseUtils'
import { workspacedOpenai } from './copilot/lib'
import type { FlowCopilotContext, FlowCopilotModule } from './copilot/flow'
import { pickScript } from './flows/flowStateUtils'
import {
@@ -49,6 +48,7 @@
} from '$lib/relative_imports'
import Tooltip from './Tooltip.svelte'
import type { ScheduleTrigger, TriggerContext } from './triggers'
import { initAllAiWorkspace } from './copilot/lib'
import type { FlowPropPickerConfig, PropPickerContext } from './prop_picker'
import type { PickableProperties } from './flows/previousResults'
$: token = $page.url.searchParams.get('wm_token') ?? undefined
@@ -110,12 +110,13 @@
async function setCopilotInfo() {
if (workspace) {
workspacedOpenai.init(workspace, token)
initAllAiWorkspace(workspace)
try {
copilotInfo.set(await WorkspaceService.getCopilotInfo({ workspace }))
} catch (err) {
copilotInfo.set({
exists_openai_resource_path: false,
ai_provider: '',
exists_ai_resource: false,
code_completion_enabled: false
})
frontend/src/lib/components/DiffDrawer.svelte
+3 -1
View File
@@ -8,6 +8,7 @@
cleanValueProperties,
orderedJsonStringify,
orderedYamlStringify,
replaceFalseWithUndefined,
type Value
} from '$lib/utils'
import type { Script } from '$lib/gen'
@@ -54,8 +55,9 @@
diffViewer.closeDrawer()
}
function prepareDiff(data: Value) {
const metadata = structuredClone(cleanValueProperties(data))
const metadata = structuredClone(cleanValueProperties(replaceFalseWithUndefined(data)))
const content = metadata['content']
if (metadata['content'] !== undefined) {
metadata['content'] = 'check content diff'
frontend/src/lib/components/Editor.svelte
+5 -2
View File
@@ -171,6 +171,7 @@
import { initVim } from './monaco_keybindings'
import { buildWorkerDefinition } from '$lib/monaco_workers/build_workers'
import { parseTypescriptDeps } from '$lib/relative_imports'
import type { AiProviderTypes } from './copilot/lib'
// import EditorTheme from './EditorTheme.svelte'
@@ -593,11 +594,13 @@
token.onCancellationRequested(() => {
abortController?.abort()
})
const aiProvider = $copilotInfo.ai_provider
const insertText = await editorCodeCompletion(
textUntilPosition,
textAfterPosition,
lang,
abortController
abortController,
aiProvider as AiProviderTypes
)
if (insertText) {
items = [
@@ -624,7 +627,7 @@
)
}
$: $copilotInfo.exists_openai_resource_path &&
$: $copilotInfo.exists_ai_resource &&
$copilotInfo.code_completion_enabled &&
$codeCompletionSessionEnabled &&
initialized &&
frontend/src/lib/components/FlowBuilder.svelte
+25 -21
View File
@@ -25,6 +25,7 @@
encodeState,
formatCron,
orderedJsonStringify,
replaceFalseWithUndefined,
sleep,
type Value
} from '$lib/utils'
@@ -85,6 +86,7 @@
import type { FlowBuilderWhitelabelCustomUi } from './custom_ui'
import FlowYamlEditor from './flows/header/FlowYamlEditor.svelte'
import { type TriggerContext, type ScheduleTrigger } from './triggers'
import type { AiProviderTypes } from './copilot/lib'
export let initialPath: string = ''
export let pathStoreInit: string | undefined = undefined
@@ -293,12 +295,21 @@
// We need it for diff
await syncWithDeployed()
// Handle through confirmation modal
confirmCallback = async () => {
if (
deployedValue &&
$flowStore &&
orderedJsonStringify(deployedValue) ===
orderedJsonStringify(replaceFalseWithUndefined({ ...$flowStore, path: $pathStore }))
) {
await saveFlow(deploymentMsg)
} else {
// Handle through confirmation modal
confirmCallback = async () => {
await saveFlow(deploymentMsg)
}
// Open confirmation modal
open = true
}
// Open confirmation modal
open = true
}
}
async function syncWithDeployed() {
@@ -307,18 +318,12 @@
path: initialPath,
withStarredInfo: true
})
deployedValue = {
deployedValue = replaceFalseWithUndefined({
...flow,
starred: undefined,
id: undefined,
edited_at: undefined,
edited_by: undefined,
workspace_id: undefined,
archived: undefined,
same_worker: undefined,
visible_to_runner_only: undefined,
ws_error_handler_muted: undefined
}
workspace_id: undefined
})
deployedBy = flow.edited_by
}
@@ -789,6 +794,7 @@
try {
push(history, $flowStore)
let module = stepOnly ? $copilotModulesStore[0] : $copilotModulesStore[idx]
const aiProvider = $copilotInfo.ai_provider as AiProviderTypes
copilotLoading = true
copilotStatus = "Generating code for step '" + module.id + "'..."
@@ -918,7 +924,8 @@
})
: undefined,
isFirstInLoop,
abortController
abortController,
aiProvider
)
unsubscribe()
}
@@ -934,7 +941,7 @@
pastModule.value.type === 'script')
) {
const stepSchema: Schema = JSON.parse(JSON.stringify($flowStateStore[module.id].schema)) // deep copy
if (isHubStep && pastModule !== undefined && $copilotInfo.exists_openai_resource_path) {
if (isHubStep && pastModule !== undefined && $copilotInfo.exists_ai_resource) {
// ask AI to set step inputs
abortController = new AbortController()
const { inputs, allExprs } = await glueCopilot(
@@ -944,7 +951,8 @@
value: RawScript | PathScript
},
isFirstInLoop,
abortController
abortController,
aiProvider
)
// create flow inputs used by AI for autocompletion
@@ -992,11 +1000,7 @@
$shouldUpdatePropertyType[key] = 'javascript'
})
} else {
if (
isHubStep &&
pastModule !== undefined &&
!$copilotInfo.exists_openai_resource_path
) {
if (isHubStep && pastModule !== undefined && !$copilotInfo.exists_ai_resource) {
sendUserToast(
'For better input generation, enable Windmill AI in the workspace settings',
true
frontend/src/lib/components/InstanceNameEditor.svelte
+89 -4
View File
@@ -5,17 +5,36 @@
import Popup from './common/popup/Popup.svelte'
import { offset, flip, shift } from 'svelte-floating-ui/dom'
import ChangeInstanceUsernameInner from './ChangeInstanceUsernameInner.svelte'
import { UserService } from '$lib/gen'
import { sendUserToast } from '$lib/toast'
export let value: string | undefined
export let email: string
export let username: string | undefined = undefined
export let automateUsernameCreation: boolean = false
export let login_type: string
let password: string = ''
const dispatch = createEventDispatcher()
function save() {
function saveName() {
dispatch('save', value)
}
async function savePassword() {
if (password.length < 5) {
sendUserToast('Password must be at least 5 characters long', true)
return
}
await UserService.setPasswordForUser({user: email, requestBody:{ password }})
sendUserToast(`Password updated for ${email}`)
}
async function saveLoginType() {
await UserService.setLoginTypeForUser({user: email, requestBody:{ login_type }})
sendUserToast(`Login type updated for ${email}`)
dispatch('refresh')
}
</script>
<Popup
@@ -34,7 +53,7 @@
<ChangeInstanceUsernameInner {email} {username} on:renamed />
{/if}
<label class="block text-primary">
<div class="pb-1 text-xs text-secondary">Name</div>
<div class="pb-2 text-sm font-semibold text-primary">Name</div>
<div class="flex w-full">
<input
type="text"
@@ -44,7 +63,7 @@
on:keydown|stopPropagation
on:keypress|stopPropagation={({ key }) => {
if (key === 'Enter') {
save()
saveName()
close(null)
}
}}
@@ -57,12 +76,78 @@
btnClasses="mt-2 "
aria-label="Save ID"
on:click={() => {
save()
saveName()
close(null)
}}
>
Update name
</Button>
</label>
<label class="block text-primary">
<div class="pb-2 text-sm font-semibold text-primary">Password</div>
<div class="flex w-full">
<input
type="password"
bind:value={password}
class="!w-auto grow"
on:click|stopPropagation={() => {}}
on:keydown|stopPropagation
on:keypress|stopPropagation={({ key }) => {
if (key === 'Enter') {
savePassword()
close(null)
}
}}
/>
</div>
<Button
size="xs"
color="blue"
buttonType="button"
btnClasses="mt-2 "
aria-label="Save ID"
on:click={() => {
savePassword()
close(null)
}}
>
Update password
</Button>
</label>
<label class="block text-primary">
<div class="pb-2 text-sm font-semibold text-primary">Login type</div>
<div class="text-xs text-secondary mb-2">
Must match exact SSO name, "password" or "saml". Examples: password, google, saml, microsoft
</div>
<div class="flex w-full">
<input
type="text"
bind:value={login_type}
class="!w-auto grow"
on:click|stopPropagation={() => {}}
on:keydown|stopPropagation
on:keypress|stopPropagation={({ key }) => {
if (key === 'Enter') {
saveLoginType()
close(null)
}
}}
/>
</div>
<Button
size="xs"
color="blue"
buttonType="button"
btnClasses="mt-2 "
aria-label="Save login type"
on:click={() => {
saveLoginType()
close(null)
}}
>
Update login type
</Button>
</label>
</div>
</Popup>
frontend/src/lib/components/InstanceSettings.svelte
+22 -1
View File
@@ -14,7 +14,7 @@
import Alert from './common/alert/Alert.svelte'
import { isCloudHosted } from '$lib/cloud'
import { capitalize, classNames, sleep } from '$lib/utils'
import { enterpriseLicense } from '$lib/stores'
import { enterpriseLicense, isCriticalAlertsUIOpen } from '$lib/stores'
import CustomOauth from './CustomOauth.svelte'
import {
AlertCircle,
@@ -43,6 +43,7 @@
export let tab: string = 'Core'
export let hideTabs: boolean = false
export let hideSave: boolean = false
export let closeDrawer: (() => void) | undefined = () => {}
let values: Record<string, any> = {}
let initialOauths: Record<string, any> = {}
@@ -814,6 +815,26 @@
placeholder={setting.placeholder}
bind:value={values[setting.key]}
/>
{:else if setting.key == 'critical_alert_mute_ui'}
<div class="flex flex-col gap-y-2 my-2 py-2">
<Toggle
disabled={!$enterpriseLicense}
bind:checked={values[setting.key]}
options={{ right: setting.description }}
/>
<div class="flex flex-row">
<Button
disabled={!$enterpriseLicense}
size="sm"
on:click={() => {
isCriticalAlertsUIOpen.set(true)
closeDrawer?.()
}}
>
Show Critical Alerts
</Button>
</div>
</div>
{:else if setting.fieldType == 'critical_error_channels'}
<div class="w-full flex gap-x-16 flex-wrap">
<div class="w-full max-w-lg">
frontend/src/lib/components/ScriptBuilder.svelte
+28 -17
View File
@@ -18,6 +18,7 @@
encodeState,
formatCron,
orderedJsonStringify,
replaceFalseWithUndefined,
type Value
} from '$lib/utils'
import Path from './Path.svelte'
@@ -278,17 +279,28 @@
// Fetch entire script, since we need it to show Diff
await syncWithDeployed()
// Handle through confirmation modal
confirmCallback = async () => {
open = false
if (actual_parent_hash) {
await editScript(stay, actual_parent_hash, deployMsg)
} else {
sendUserToast('Could not fetch latest version of the script', true)
if (
deployedValue &&
script &&
orderedJsonStringify({ ...deployedValue, hash: undefined }) ===
orderedJsonStringify(
replaceFalseWithUndefined({ ...script, hash: undefined, parent_hash: undefined })
)
) {
await editScript(stay, actual_parent_hash, deployMsg)
} else {
// Handle through confirmation modal
confirmCallback = async () => {
open = false
if (actual_parent_hash) {
await editScript(stay, actual_parent_hash, deployMsg)
} else {
sendUserToast('Could not fetch latest version of the script', true)
}
}
// Open confirmation modal
open = true
}
// Open confirmation modal
open = true
}
}
@@ -299,20 +311,16 @@
withStarredInfo: true
})
deployedValue = {
deployedValue = replaceFalseWithUndefined({
...latestScript,
starred: undefined,
workspace_id: undefined,
archived: undefined,
created_at: undefined,
created_by: undefined,
deleted: undefined,
extra_perms: undefined,
is_template: undefined,
lock: undefined,
lock_error_logs: undefined,
parent_hashes: undefined
}
})
deployedBy = latestScript.created_by
}
@@ -691,8 +699,11 @@
</Label>
<Label label="Path">
<svelte:fragment slot="header">
<Tooltip documentationLink="https://www.windmill.dev/docs/core_concepts/roles_and_permissions#path">
The unique identifier of the script in the workspace that defines permissions
<Tooltip
documentationLink="https://www.windmill.dev/docs/core_concepts/roles_and_permissions#path"
>
The unique identifier of the script in the workspace that defines
permissions
</Tooltip>
</svelte:fragment>
<Path
frontend/src/lib/components/SuperadminSettings.svelte
+5 -1
View File
@@ -276,9 +276,13 @@
<td>
<div class="flex flex-row gap-x-1 justify-end">
<InstanceNameEditor
login_type={login_type}
value={name}
{username}
{email}
on:refresh={() => {
listUsers(activeOnly)
}}
on:save={(e) => {
updateName(e.detail, email)
}}
@@ -321,7 +325,7 @@
</div>
</TabContent>
<TabContent value="" values={settingsKeys}>
<InstanceSettings bind:this={instanceSettings} hideTabs hideSave {tab} />
<InstanceSettings bind:this={instanceSettings} hideTabs hideSave {tab} {closeDrawer}/>
</TabContent>
</svelte:fragment>
</Tabs>
frontend/src/lib/components/apps/editor/AppEditorHeader.svelte
+24 -10
View File
@@ -41,7 +41,8 @@
copyToClipboard,
truncateRev,
orderedJsonStringify,
type Value
type Value,
replaceFalseWithUndefined
} from '../../../utils'
import type {
AppInput,
@@ -422,13 +423,27 @@
// We need it to show diff
// Handle through confirmation modal
await syncWithDeployed()
confirmCallback = async () => {
open = false
if (
deployedValue &&
savedApp &&
$app &&
orderedJsonStringify(deployedValue) ===
orderedJsonStringify(replaceFalseWithUndefined({
summary: $summary,
value: $app,
path: newPath || savedApp.draft?.path || savedApp.path,
policy
}))
) {
await updateApp(npath)
} else {
confirmCallback = async () => {
open = false
await updateApp(npath)
}
// Open confirmation modal
open = true
}
// Open confirmation modal
open = true
}
}
@@ -442,15 +457,14 @@
deployedBy = deployedApp.created_by
// Strip off extra information
deployedValue = {
deployedValue = replaceFalseWithUndefined({
...deployedApp,
starred: undefined,
id: undefined,
created_at: undefined,
created_by: undefined,
versions: undefined,
extra_perms: undefined //
}
extra_perms: undefined
})
}
async function updateApp(npath: string) {
frontend/src/lib/components/auditLogs/AuditLogsFilters.svelte
+1 -1
View File
@@ -221,7 +221,7 @@
JOBS_DISAPPROVAL: 'jobs.disapproval',
JOBS_DELETE: 'jobs.delete',
ACCOUNT_DELETE: 'account.delete',
OPENAI_REQUEST: 'openai.request',
AI_REQUEST: 'ai.request',
RESOURCES_CREATE: 'resources.create',
RESOURCES_UPDATE: 'resources.update',
RESOURCES_DELETE: 'resources.delete',
frontend/src/lib/components/common/confirmationModal/UnsavedConfirmationModal.svelte
+10 -2
View File
@@ -4,7 +4,12 @@
import { goto as gotoUrl } from '$app/navigation'
import Button from '../button/Button.svelte'
import type DiffDrawer from '$lib/components/DiffDrawer.svelte'
import { cleanValueProperties, orderedJsonStringify, type Value } from '$lib/utils'
import {
cleanValueProperties,
orderedJsonStringify,
replaceFalseWithUndefined,
type Value
} from '$lib/utils'
import { tick } from 'svelte'
import { page } from '$app/stores'
@@ -36,7 +41,10 @@
path: undefined
})
const current = cleanValueProperties({ ...(modifiedValue ?? {}), path: undefined })
if (orderedJsonStringify(draftOrDeployed) === orderedJsonStringify(current)) {
if (
orderedJsonStringify(replaceFalseWithUndefined(draftOrDeployed)) ===
orderedJsonStringify(replaceFalseWithUndefined(current))
) {
bypassBeforeNavigate = true
gotoUrl(goingTo)
} else {
frontend/src/lib/components/common/modal/Modal.svelte
+1 -1
View File
@@ -44,7 +44,7 @@
<div
on:click={() => (open = false)}
transition:fadeFast|local
class={'absolute top-0 bottom-0 left-0 right-0 z-50'}
class={'absolute top-0 bottom-0 left-0 right-0 z-[9999]'}
role="dialog"
>
<div
frontend/src/lib/components/copilot/CodeCompletionStatus.svelte
+1 -1
View File
@@ -12,7 +12,7 @@
}
</script>
{#if $copilotInfo.exists_openai_resource_path && $copilotInfo.code_completion_enabled}
{#if $copilotInfo.exists_ai_resource && $copilotInfo.code_completion_enabled}
<Toggle
size="xs"
bind:checked={$codeCompletionSessionEnabled}

Some files were not shown because too many files have changed in this diff Show More