starred/windmill
1
0
Fork 0
mirror of https://github.com/windmill-labs/windmill.git synced 2026-10-19 00:03:19 +00:00
Code Issues Packages Projects Releases Wiki Activity

Merge main into feat/cli-branch-override (folder support)

Browse Source
This commit is contained in:
Ruben Fiszel
2026-01-19 17:12:15 +00:00
parent c029ddb1b2 3574269d96
commit e865b2ccb2
122 changed files with 7615 additions and 2186 deletions
Download Patch File Download Diff File Expand all files Collapse all files
CHANGELOG.md
+47
View File
@@ -1,5 +1,52 @@
# Changelog
## [1.611.0](https://github.com/windmill-labs/windmill/compare/v1.610.1...v1.611.0) (2026-01-19)
### Features
* add HashiCorp Vault secret storage integration ([#7599](https://github.com/windmill-labs/windmill/issues/7599)) ([1b9d1c5](https://github.com/windmill-labs/windmill/commit/1b9d1c56c7e49042677326eb397e10d34a3ddcdf))
### Bug Fixes
* **flow-chat:** handle SSE timeout and fix temp message race condition ([4f8110e](https://github.com/windmill-labs/windmill/commit/4f8110eb9852b78b48aabbed114c75cbf0d1a2ef))
## [1.610.1](https://github.com/windmill-labs/windmill/compare/v1.610.0...v1.610.1) (2026-01-17)
### Bug Fixes
* resolve BlobPart type incompatibility between Deno and Node.js ([2eac74c](https://github.com/windmill-labs/windmill/commit/2eac74cef4aa5a987fb16110388f99e912951db8))
* use type cast instead of slice() for BlobPart compatibility ([ff77154](https://github.com/windmill-labs/windmill/commit/ff771546380ef26dfa443f9d459853048bc8029c))
## [1.610.0](https://github.com/windmill-labs/windmill/compare/v1.609.0...v1.610.0) (2026-01-17)
### Features
* add private npm registry proxy support for ATA in webide ([#7597](https://github.com/windmill-labs/windmill/issues/7597)) ([b3cb41e](https://github.com/windmill-labs/windmill/commit/b3cb41efa4520fd3243a6dbab7c985923dd538e3))
* add workspace success handler with 60s TTL caching ([#7598](https://github.com/windmill-labs/windmill/issues/7598)) ([73c4ce3](https://github.com/windmill-labs/windmill/commit/73c4ce30127af587510b16fbc7fa223436f6845c))
### Bug Fixes
* handle missing storage key in S3Object for write_s3_file ([03daa34](https://github.com/windmill-labs/windmill/commit/03daa341eb91118c27e92b1b51731400a0dce30c))
* improve job deletion performance and batching ([8dd5e81](https://github.com/windmill-labs/windmill/commit/8dd5e81a32b97814eab3b72964807bdfe0ea8b49))
## [1.609.0](https://github.com/windmill-labs/windmill/compare/v1.608.0...v1.609.0) (2026-01-16)
### Features
* cli branch override ([#7592](https://github.com/windmill-labs/windmill/issues/7592)) ([dcee9fe](https://github.com/windmill-labs/windmill/commit/dcee9fe7b163993836691988a552a5bc6042b9a2))
### Bug Fixes
* Fix MS SQL S3 Mode ([#7595](https://github.com/windmill-labs/windmill/issues/7595)) ([c7a6a05](https://github.com/windmill-labs/windmill/commit/c7a6a05925681bb1b2cec8d2c11037bc3d339798))
* transparency issue of instance setting save button ([#7594](https://github.com/windmill-labs/windmill/issues/7594)) ([86ebf9e](https://github.com/windmill-labs/windmill/commit/86ebf9e25a03db99453269832bce030438c677c3))
## [1.608.0](https://github.com/windmill-labs/windmill/compare/v1.607.1...v1.608.0) (2026-01-16)
backend/.sqlx/query-020c031c3de6c85577e30421ada9d39a5a47ca1b6cf3dbfd6988aa0694d7364c.json
Generated
+23
View File
@@ -0,0 +1,23 @@
{
"db_name": "PostgreSQL",
"query": "SELECT value FROM variable WHERE path = $1 AND workspace_id = $2 AND is_secret = true",
"describe": {
"columns": [
{
"ordinal": 0,
"name": "value",
"type_info": "Varchar"
}
],
"parameters": {
"Left": [
"Text",
"Text"
]
},
"nullable": [
false
]
},
"hash": "020c031c3de6c85577e30421ada9d39a5a47ca1b6cf3dbfd6988aa0694d7364c"
}
backend/.sqlx/query-052d42b46d5faba6b41f1fdcbf6a012db51b9e5a255ec0da9a8a0999d668d336.json
Generated
+20
View File
@@ -0,0 +1,20 @@
{
"db_name": "PostgreSQL",
"query": "SELECT COUNT(*) as count FROM variable WHERE is_secret = true AND value != 'CLEARED'",
"describe": {
"columns": [
{
"ordinal": 0,
"name": "count",
"type_info": "Int8"
}
],
"parameters": {
"Left": []
},
"nullable": [
null
]
},
"hash": "052d42b46d5faba6b41f1fdcbf6a012db51b9e5a255ec0da9a8a0999d668d336"
}
backend/.sqlx/query-0600f2a9179f83502c6b13e8e4284f85ca82636f274f5dce47da5a8320a60088.json
Generated
+26
View File
@@ -0,0 +1,26 @@
{
"db_name": "PostgreSQL",
"query": "SELECT workspace_id, path FROM variable WHERE is_secret = true",
"describe": {
"columns": [
{
"ordinal": 0,
"name": "workspace_id",
"type_info": "Varchar"
},
{
"ordinal": 1,
"name": "path",
"type_info": "Varchar"
}
],
"parameters": {
"Left": []
},
"nullable": [
false,
false
]
},
"hash": "0600f2a9179f83502c6b13e8e4284f85ca82636f274f5dce47da5a8320a60088"
}
backend/.sqlx/query-085bbcfcebc9e56984295046c48722f3510548eaf525c499ad0e24cdf8332b22.json
Generated
+20
View File
@@ -0,0 +1,20 @@
{
"db_name": "PostgreSQL",
"query": "SELECT success_handler_extra_args FROM workspace_settings WHERE workspace_id = 'test-workspace'",
"describe": {
"columns": [
{
"ordinal": 0,
"name": "success_handler_extra_args",
"type_info": "Json"
}
],
"parameters": {
"Left": []
},
"nullable": [
true
]
},
"hash": "085bbcfcebc9e56984295046c48722f3510548eaf525c499ad0e24cdf8332b22"
}
backend/.sqlx/query-08f288d2781d823e109a9e5b8848234ca7d1efeee9661f3901f298da375e73f7.json
Generated
+12
View File
@@ -177,6 +177,16 @@
"ordinal": 34,
"name": "teams_team_guid",
"type_info": "Text"
},
{
"ordinal": 35,
"name": "success_handler",
"type_info": "Text"
},
{
"ordinal": 36,
"name": "success_handler_extra_args",
"type_info": "Json"
}
],
"parameters": {
@@ -219,6 +229,8 @@
true,
true,
true,
true,
true,
true
]
},
backend/.sqlx/query-0a1c10bd2232b0770a7816e1bd8d758dc393f797890d597e5996146247f512ac.json
Generated
-38
View File
@@ -1,38 +0,0 @@
{
"db_name": "PostgreSQL",
"query": "\nWITH lockable_counters AS (\n SELECT concurrency_id, job_uuids\n FROM concurrency_counter\n WHERE job_uuids != '{}'::jsonb\n FOR UPDATE SKIP LOCKED\n),\nall_job_uuids AS (\n SELECT DISTINCT jsonb_object_keys(job_uuids) AS job_uuid\n FROM lockable_counters\n),\norphaned_job_uuids AS (\n SELECT job_uuid\n FROM all_job_uuids\n WHERE job_uuid NOT IN (\n SELECT id::text \n FROM v2_job_queue \n FOR SHARE SKIP LOCKED\n )\n),\norphaned_array AS (\n SELECT ARRAY(SELECT job_uuid FROM orphaned_job_uuids) AS orphaned_keys\n),\nbefore_update AS (\n SELECT lc.concurrency_id, lc.job_uuids, oa.orphaned_keys\n FROM lockable_counters lc, orphaned_array oa\n WHERE lc.job_uuids ?| oa.orphaned_keys\n),\naffected_rows AS (\n UPDATE concurrency_counter \n SET job_uuids = job_uuids - orphaned_array.orphaned_keys\n FROM orphaned_array\n WHERE concurrency_counter.concurrency_id IN (\n SELECT concurrency_id FROM before_update\n )\n RETURNING concurrency_id, job_uuids AS updated_job_uuids\n),\nexpanded_orphaned AS (\n SELECT bu.concurrency_id, \n bu.job_uuids AS original_job_uuids,\n unnest(bu.orphaned_keys) AS orphaned_key\n FROM before_update bu\n)\nSELECT \n eo.concurrency_id,\n eo.orphaned_key,\n eo.original_job_uuids,\n ar.updated_job_uuids\nFROM expanded_orphaned eo\nJOIN affected_rows ar ON eo.concurrency_id = ar.concurrency_id\nWHERE eo.original_job_uuids ? eo.orphaned_key\nORDER BY eo.concurrency_id, eo.orphaned_key\n",
"describe": {
"columns": [
{
"ordinal": 0,
"name": "concurrency_id",
"type_info": "Varchar"
},
{
"ordinal": 1,
"name": "orphaned_key",
"type_info": "Text"
},
{
"ordinal": 2,
"name": "original_job_uuids",
"type_info": "Jsonb"
},
{
"ordinal": 3,
"name": "updated_job_uuids",
"type_info": "Jsonb"
}
],
"parameters": {
"Left": []
},
"nullable": [
false,
null,
false,
false
]
},
"hash": "0a1c10bd2232b0770a7816e1bd8d758dc393f797890d597e5996146247f512ac"
}
backend/.sqlx/query-0c8a3eb810c96230ba3a5466c55bf24a94eb8a52ceb82cc29dade173ad87569d.json
Generated
+23
View File
@@ -0,0 +1,23 @@
{
"db_name": "PostgreSQL",
"query": "SELECT path FROM variable WHERE path = ANY($1) AND workspace_id = $2 AND is_secret = true",
"describe": {
"columns": [
{
"ordinal": 0,
"name": "path",
"type_info": "Varchar"
}
],
"parameters": {
"Left": [
"TextArray",
"Text"
]
},
"nullable": [
false
]
},
"hash": "0c8a3eb810c96230ba3a5466c55bf24a94eb8a52ceb82cc29dade173ad87569d"
}
backend/.sqlx/query-10a40902f5aba38aea23f2b0776b54752362c9cc7deb179f75c7f1795866d13a.json
Generated
+20
View File
@@ -0,0 +1,20 @@
{
"db_name": "PostgreSQL",
"query": "SELECT success_handler FROM workspace_settings WHERE workspace_id = 'test-workspace'",
"describe": {
"columns": [
{
"ordinal": 0,
"name": "success_handler",
"type_info": "Text"
}
],
"parameters": {
"Left": []
},
"nullable": [
true
]
},
"hash": "10a40902f5aba38aea23f2b0776b54752362c9cc7deb179f75c7f1795866d13a"
}
backend/.sqlx/query-146f0e42ada3068a5cdae0ffdbb54b63f8c06c9143b16ce399170c1b5a6b911e.json
Generated
+29
View File
@@ -0,0 +1,29 @@
{
"db_name": "PostgreSQL",
"query": "SELECT value, is_secret FROM variable WHERE path = $1 AND workspace_id = $2",
"describe": {
"columns": [
{
"ordinal": 0,
"name": "value",
"type_info": "Varchar"
},
{
"ordinal": 1,
"name": "is_secret",
"type_info": "Bool"
}
],
"parameters": {
"Left": [
"Text",
"Text"
]
},
"nullable": [
false,
false
]
},
"hash": "146f0e42ada3068a5cdae0ffdbb54b63f8c06c9143b16ce399170c1b5a6b911e"
}
backend/.sqlx/query-1730f39fd1793d45fbb41b21389c61296a3ff7489ae12f52a19f9543173ac597.json
Generated
+12
View File
@@ -177,6 +177,16 @@
"ordinal": 34,
"name": "teams_team_guid",
"type_info": "Text"
},
{
"ordinal": 35,
"name": "success_handler",
"type_info": "Text"
},
{
"ordinal": 36,
"name": "success_handler_extra_args",
"type_info": "Json"
}
],
"parameters": {
@@ -219,6 +229,8 @@
true,
true,
true,
true,
true,
true
]
},
backend/.sqlx/query-18aad20ed9cb2dde46f9d899dc4aa6f80ecf1628bd2c073d7a237dea9b8e0c65.json
Generated
+23
View File
@@ -0,0 +1,23 @@
{
"db_name": "PostgreSQL",
"query": "SELECT is_secret FROM variable WHERE path = $1 AND workspace_id = $2",
"describe": {
"columns": [
{
"ordinal": 0,
"name": "is_secret",
"type_info": "Bool"
}
],
"parameters": {
"Left": [
"Text",
"Text"
]
},
"nullable": [
false
]
},
"hash": "18aad20ed9cb2dde46f9d899dc4aa6f80ecf1628bd2c073d7a237dea9b8e0c65"
}
backend/.sqlx/query-1ccc9db6c6cb51bebeb90d003e12c8f7922450515d1eaee8b3abee5b31f72527.json
Generated
+14
View File
@@ -0,0 +1,14 @@
{
"db_name": "PostgreSQL",
"query": "DELETE FROM concurrency_key WHERE ended_at <= now() - ($1::bigint::text || ' s')::interval",
"describe": {
"columns": [],
"parameters": {
"Left": [
"Int8"
]
},
"nullable": []
},
"hash": "1ccc9db6c6cb51bebeb90d003e12c8f7922450515d1eaee8b3abee5b31f72527"
}
backend/.sqlx/query-2c2ed5c91a4893966e94305df8b9f8ac904c8d0781a83fdea66c918320fb5601.json
Generated
+12
View File
@@ -0,0 +1,12 @@
{
"db_name": "PostgreSQL",
"query": "\n UPDATE workspace_settings\n SET success_handler = NULL\n WHERE workspace_id = 'test-workspace'\n ",
"describe": {
"columns": [],
"parameters": {
"Left": []
},
"nullable": []
},
"hash": "2c2ed5c91a4893966e94305df8b9f8ac904c8d0781a83fdea66c918320fb5601"
}
backend/.sqlx/query-2d43c1889b492df6f77380e739e41e20fe6bea0553cc39e30b7aa8ae59963f95.json
Generated
+14
View File
@@ -0,0 +1,14 @@
{
"db_name": "PostgreSQL",
"query": "\n INSERT INTO script (workspace_id, hash, path, content, language, kind, created_by, schema, summary, description, lock)\n VALUES ('test-workspace', 1234567890, 'f/test/success_handler', $1, 'deno', 'script', 'test-user', '{}', 'Success handler script', 'Handles successful job completions', '')\n ",
"describe": {
"columns": [],
"parameters": {
"Left": [
"Text"
]
},
"nullable": []
},
"hash": "2d43c1889b492df6f77380e739e41e20fe6bea0553cc39e30b7aa8ae59963f95"
}
backend/.sqlx/query-2fcddda99dd0aacf5007ed459cb27caa754424e062427edf5ddcb95f9d96888e.json
Generated
+29
View File
@@ -0,0 +1,29 @@
{
"db_name": "PostgreSQL",
"query": "SELECT path, value FROM variable\n WHERE path LIKE ('u/' || $1 || '/%')\n AND workspace_id = $2\n AND is_secret = true\n AND value LIKE '$vault:%'",
"describe": {
"columns": [
{
"ordinal": 0,
"name": "path",
"type_info": "Varchar"
},
{
"ordinal": 1,
"name": "value",
"type_info": "Varchar"
}
],
"parameters": {
"Left": [
"Text",
"Text"
]
},
"nullable": [
false,
false
]
},
"hash": "2fcddda99dd0aacf5007ed459cb27caa754424e062427edf5ddcb95f9d96888e"
}
backend/.sqlx/query-306e0156ee1541710c1c6512ecb4f61baeb3ae6f31ba3fd57a3ec485108a7f49.json
Generated
+23
View File
@@ -0,0 +1,23 @@
{
"db_name": "PostgreSQL",
"query": "DELETE FROM v2_job_completed\n WHERE id IN (\n SELECT id FROM v2_job_completed\n WHERE completed_at <= now() - ($1::bigint::text || ' s')::interval\n ORDER BY completed_at ASC\n LIMIT $2\n FOR UPDATE SKIP LOCKED\n )\n RETURNING id",
"describe": {
"columns": [
{
"ordinal": 0,
"name": "id",
"type_info": "Uuid"
}
],
"parameters": {
"Left": [
"Int8",
"Int8"
]
},
"nullable": [
false
]
},
"hash": "306e0156ee1541710c1c6512ecb4f61baeb3ae6f31ba3fd57a3ec485108a7f49"
}
backend/.sqlx/query-37e0c601a463819748078b92d1c87f63348a92908e2ca52f5a092149191e6200.json
Generated
+16
View File
@@ -0,0 +1,16 @@
{
"db_name": "PostgreSQL",
"query": "UPDATE variable SET value = $1 WHERE path = $2 AND workspace_id = $3 AND is_secret = true",
"describe": {
"columns": [],
"parameters": {
"Left": [
"Varchar",
"Text",
"Text"
]
},
"nullable": []
},
"hash": "37e0c601a463819748078b92d1c87f63348a92908e2ca52f5a092149191e6200"
}
backend/.sqlx/query-3a71ab02239d8dd4e2f440a35155459a7d5091c125a0998e79d89f8c560a3fd3.json
Generated
+12
View File
@@ -0,0 +1,12 @@
{
"db_name": "PostgreSQL",
"query": "\n INSERT INTO group_ (workspace_id, name, summary, extra_perms)\n VALUES ('test-workspace', 'success_handler', 'The group the success handler acts on behalf of', '{\"u/test-user\": true}')\n ON CONFLICT DO NOTHING\n ",
"describe": {
"columns": [],
"parameters": {
"Left": []
},
"nullable": []
},
"hash": "3a71ab02239d8dd4e2f440a35155459a7d5091c125a0998e79d89f8c560a3fd3"
}
backend/.sqlx/query-c6ef0acdf20bd71dd26de981fb49f178ba8a1b8c1e01e0fec1dfd6a54ea7a894.json → backend/.sqlx/query-45d0552a169b63241a0ecc0e7ec6a533d7eec520376b99b2fd8000f0f0acba7b.json
Generated
+2 -2
View File
@@ -1,6 +1,6 @@
{
"db_name": "PostgreSQL",
"query": "\nWITH rows_to_delete AS (\n SELECT concurrency_id\n FROM concurrency_counter\n \n WHERE job_uuids = '{}'::jsonb\n FOR UPDATE SKIP LOCKED\n)\nDELETE FROM concurrency_counter\nWHERE concurrency_id IN (SELECT concurrency_id FROM rows_to_delete) RETURNING concurrency_id",
"query": "\nWITH rows_to_delete AS (\n SELECT concurrency_id\n FROM concurrency_counter\n\n WHERE job_uuids = '{}'::jsonb\n FOR UPDATE SKIP LOCKED\n)\nDELETE FROM concurrency_counter\nWHERE concurrency_id IN (SELECT concurrency_id FROM rows_to_delete) RETURNING concurrency_id",
"describe": {
"columns": [
{
@@ -16,5 +16,5 @@
false
]
},
"hash": "c6ef0acdf20bd71dd26de981fb49f178ba8a1b8c1e01e0fec1dfd6a54ea7a894"
"hash": "45d0552a169b63241a0ecc0e7ec6a533d7eec520376b99b2fd8000f0f0acba7b"
}
backend/.sqlx/query-4977b4027861ca234e95559a8b550bc8849f7ed60a471d46f3be3d39664a48ca.json
Generated
+14
View File
@@ -0,0 +1,14 @@
{
"db_name": "PostgreSQL",
"query": "\n UPDATE\n workspace_settings\n SET\n success_handler = NULL,\n success_handler_extra_args = NULL\n WHERE\n workspace_id = $1\n ",
"describe": {
"columns": [],
"parameters": {
"Left": [
"Text"
]
},
"nullable": []
},
"hash": "4977b4027861ca234e95559a8b550bc8849f7ed60a471d46f3be3d39664a48ca"
}
backend/.sqlx/query-70a6880960d17218bc5bf05287e2a6d9a6393c6bb1783ab8903d87dd099e236b.json → backend/.sqlx/query-4d80985dd0794a01a2af18ae7abf4a3ab8ba3d162ed5d04735caea7295da0b20.json
Generated
+2 -2
View File
@@ -1,6 +1,6 @@
{
"db_name": "PostgreSQL",
"query": "\n SELECT\n j.id AS \"id!\", j.workspace_id AS \"workspace_id!\", j.parent_job, j.flow_step_id IS NOT NULL AS \"is_flow_step?\",\n COALESCE(s.flow_status, s.workflow_as_code_status) AS \"flow_status: Box<str>\", r.ping AS last_ping, j.same_worker AS \"same_worker?\"\n FROM v2_job_queue q JOIN v2_job j USING (id) LEFT JOIN v2_job_runtime r USING (id) LEFT JOIN v2_job_status s USING (id)\n WHERE q.running = true AND q.suspend = 0 AND q.suspend_until IS null AND q.scheduled_for <= now()\n AND (j.kind = 'flow' OR j.kind = 'flowpreview' OR j.kind = 'flownode')\n AND r.ping IS NOT NULL AND r.ping < NOW() - ($1 || ' seconds')::interval\n AND q.canceled_by IS NULL\n \n ",
"query": "\n SELECT\n j.id AS \"id!\", j.workspace_id AS \"workspace_id!\", j.parent_job, j.flow_step_id IS NOT NULL AS \"is_flow_step?\",\n COALESCE(s.flow_status, s.workflow_as_code_status) AS \"flow_status: Box<str>\", r.ping AS last_ping, j.same_worker AS \"same_worker?\"\n FROM v2_job_queue q JOIN v2_job j USING (id) LEFT JOIN v2_job_runtime r USING (id) LEFT JOIN v2_job_status s USING (id)\n WHERE q.running = true AND q.suspend = 0 AND q.suspend_until IS null AND q.scheduled_for <= now()\n AND (j.kind = 'flow' OR j.kind = 'flowpreview' OR j.kind = 'flownode')\n AND r.ping IS NOT NULL AND r.ping < NOW() - ($1 || ' seconds')::interval\n AND q.canceled_by IS NULL\n\n ",
"describe": {
"columns": [
{
@@ -54,5 +54,5 @@
false
]
},
"hash": "70a6880960d17218bc5bf05287e2a6d9a6393c6bb1783ab8903d87dd099e236b"
"hash": "4d80985dd0794a01a2af18ae7abf4a3ab8ba3d162ed5d04735caea7295da0b20"
}
backend/.sqlx/query-5820d34be1a7f7b72e656c692f53146f45ad4a6e584e917a0a86280d8f473c10.json
Generated
-22
View File
@@ -1,22 +0,0 @@
{
"db_name": "PostgreSQL",
"query": "DELETE FROM v2_job_completed c\n WHERE completed_at <= now() - ($1::bigint::text || ' s')::interval \n RETURNING c.id",
"describe": {
"columns": [
{
"ordinal": 0,
"name": "id",
"type_info": "Uuid"
}
],
"parameters": {
"Left": [
"Int8"
]
},
"nullable": [
false
]
},
"hash": "5820d34be1a7f7b72e656c692f53146f45ad4a6e584e917a0a86280d8f473c10"
}
backend/.sqlx/query-598805056a3b102c9c2fd69d8330fa78b06106236522e48ab5fe009cac059468.json
Generated
+38
View File
@@ -0,0 +1,38 @@
{
"db_name": "PostgreSQL",
"query": "\nWITH lockable_counters AS (\n SELECT concurrency_id, job_uuids\n FROM concurrency_counter\n WHERE job_uuids != '{}'::jsonb\n FOR UPDATE SKIP LOCKED\n),\nall_job_uuids AS (\n SELECT DISTINCT jsonb_object_keys(job_uuids) AS job_uuid\n FROM lockable_counters\n),\norphaned_job_uuids AS (\n SELECT job_uuid\n FROM all_job_uuids\n WHERE job_uuid NOT IN (\n SELECT id::text\n FROM v2_job_queue\n FOR SHARE SKIP LOCKED\n )\n),\norphaned_array AS (\n SELECT ARRAY(SELECT job_uuid FROM orphaned_job_uuids) AS orphaned_keys\n),\nbefore_update AS (\n SELECT lc.concurrency_id, lc.job_uuids, oa.orphaned_keys\n FROM lockable_counters lc, orphaned_array oa\n WHERE lc.job_uuids ?| oa.orphaned_keys\n),\naffected_rows AS (\n UPDATE concurrency_counter\n SET job_uuids = job_uuids - orphaned_array.orphaned_keys\n FROM orphaned_array\n WHERE concurrency_counter.concurrency_id IN (\n SELECT concurrency_id FROM before_update\n )\n RETURNING concurrency_id, job_uuids AS updated_job_uuids\n),\nexpanded_orphaned AS (\n SELECT bu.concurrency_id,\n bu.job_uuids AS original_job_uuids,\n unnest(bu.orphaned_keys) AS orphaned_key\n FROM before_update bu\n)\nSELECT\n eo.concurrency_id,\n eo.orphaned_key,\n eo.original_job_uuids,\n ar.updated_job_uuids\nFROM expanded_orphaned eo\nJOIN affected_rows ar ON eo.concurrency_id = ar.concurrency_id\nWHERE eo.original_job_uuids ? eo.orphaned_key\nORDER BY eo.concurrency_id, eo.orphaned_key\n",
"describe": {
"columns": [
{
"ordinal": 0,
"name": "concurrency_id",
"type_info": "Varchar"
},
{
"ordinal": 1,
"name": "orphaned_key",
"type_info": "Text"
},
{
"ordinal": 2,
"name": "original_job_uuids",
"type_info": "Jsonb"
},
{
"ordinal": 3,
"name": "updated_job_uuids",
"type_info": "Jsonb"
}
],
"parameters": {
"Left": []
},
"nullable": [
false,
null,
false,
false
]
},
"hash": "598805056a3b102c9c2fd69d8330fa78b06106236522e48ab5fe009cac059468"
}
backend/.sqlx/query-5a219a2532517869578c4504ff3153c43903f929ae5d62fbba12610f89c36d55.json
Generated
+1 -1
View File
@@ -15,7 +15,7 @@
]
},
"nullable": [
null
true
]
},
"hash": "5a219a2532517869578c4504ff3153c43903f929ae5d62fbba12610f89c36d55"
backend/.sqlx/query-6dd7e38c902f6b8d397aa3e9e698fe1541bd3f22634af341849303de84c0034d.json
Generated
+32
View File
@@ -0,0 +1,32 @@
{
"db_name": "PostgreSQL",
"query": "SELECT workspace_id, path, value FROM variable WHERE is_secret = true",
"describe": {
"columns": [
{
"ordinal": 0,
"name": "workspace_id",
"type_info": "Varchar"
},
{
"ordinal": 1,
"name": "path",
"type_info": "Varchar"
},
{
"ordinal": 2,
"name": "value",
"type_info": "Varchar"
}
],
"parameters": {
"Left": []
},
"nullable": [
false,
false,
false
]
},
"hash": "6dd7e38c902f6b8d397aa3e9e698fe1541bd3f22634af341849303de84c0034d"
}
backend/.sqlx/query-6eaa0af9c5e0a8d64c3202753c9c3cef39361b0987a22b7d2b2346652f81177e.json
Generated
+12
View File
@@ -0,0 +1,12 @@
{
"db_name": "PostgreSQL",
"query": "\n UPDATE workspace_settings\n SET success_handler = 'script/f/test/success_handler',\n success_handler_extra_args = '{\"key\": \"value\"}'::json\n WHERE workspace_id = 'test-workspace'\n ",
"describe": {
"columns": [],
"parameters": {
"Left": []
},
"nullable": []
},
"hash": "6eaa0af9c5e0a8d64c3202753c9c3cef39361b0987a22b7d2b2346652f81177e"
}
backend/.sqlx/query-71de038f332593e2177f33284854b6c0bfe9bb4999f2c4e9082a4c5f3b341496.json
Generated
+38
View File
@@ -0,0 +1,38 @@
{
"db_name": "PostgreSQL",
"query": "\n SELECT\n id,\n runnable_path,\n permissioned_as_email,\n parent_job\n FROM v2_job\n WHERE workspace_id = 'test-workspace'\n AND permissioned_as_email = 'success_handler@windmill.dev'\n ORDER BY created_at DESC\n LIMIT 1\n ",
"describe": {
"columns": [
{
"ordinal": 0,
"name": "id",
"type_info": "Uuid"
},
{
"ordinal": 1,
"name": "runnable_path",
"type_info": "Varchar"
},
{
"ordinal": 2,
"name": "permissioned_as_email",
"type_info": "Varchar"
},
{
"ordinal": 3,
"name": "parent_job",
"type_info": "Uuid"
}
],
"parameters": {
"Left": []
},
"nullable": [
false,
true,
false,
true
]
},
"hash": "71de038f332593e2177f33284854b6c0bfe9bb4999f2c4e9082a4c5f3b341496"
}
backend/.sqlx/query-8f5722afde37d22c56851da23895daa1653e14b6d12013f6e3bb8bf042447a6a.json
Generated
+12
View File
@@ -0,0 +1,12 @@
{
"db_name": "PostgreSQL",
"query": "UPDATE variable SET value = 'ROUND_TRIP_CLEARED' WHERE is_secret = true",
"describe": {
"columns": [],
"parameters": {
"Left": []
},
"nullable": []
},
"hash": "8f5722afde37d22c56851da23895daa1653e14b6d12013f6e3bb8bf042447a6a"
}
backend/.sqlx/query-9044c71dc8aab7f5d17bf27c4702e67f3eee46d7dbccb09cadeaafa0534952d6.json
Generated
+16
View File
@@ -0,0 +1,16 @@
{
"db_name": "PostgreSQL",
"query": "\n UPDATE\n workspace_settings\n SET\n success_handler = $1,\n success_handler_extra_args = $2\n WHERE\n workspace_id = $3\n ",
"describe": {
"columns": [],
"parameters": {
"Left": [
"Text",
"Json",
"Text"
]
},
"nullable": []
},
"hash": "9044c71dc8aab7f5d17bf27c4702e67f3eee46d7dbccb09cadeaafa0534952d6"
}
backend/.sqlx/query-93ef241a4f624cb76d680418d313cd956e4c807c3a0031913dbfded80f0e5881.json
Generated
+12
View File
@@ -0,0 +1,12 @@
{
"db_name": "PostgreSQL",
"query": "UPDATE variable SET value = 'CLEARED' WHERE is_secret = true",
"describe": {
"columns": [],
"parameters": {
"Left": []
},
"nullable": []
},
"hash": "93ef241a4f624cb76d680418d313cd956e4c807c3a0031913dbfded80f0e5881"
}
backend/.sqlx/query-9da0cea2a5d0464ca78cfeccf6cedf2b1c0e6e6cb3c9183a937a68465debdb06.json
Generated
-14
View File
@@ -1,14 +0,0 @@
{
"db_name": "PostgreSQL",
"query": "DELETE FROM concurrency_key WHERE ended_at <= now() - ($1::bigint::text || ' s')::interval ",
"describe": {
"columns": [],
"parameters": {
"Left": [
"Int8"
]
},
"nullable": []
},
"hash": "9da0cea2a5d0464ca78cfeccf6cedf2b1c0e6e6cb3c9183a937a68465debdb06"
}
backend/.sqlx/query-a2bc410e1066ddc89ca4557ca0e29339e41cbc7a7ef6ad97b3f0eb93de04c359.json
Generated
+12
View File
@@ -0,0 +1,12 @@
{
"db_name": "PostgreSQL",
"query": "\n UPDATE workspace_settings\n SET success_handler = 'flow/f/test/success_flow'\n WHERE workspace_id = 'test-workspace'\n ",
"describe": {
"columns": [],
"parameters": {
"Left": []
},
"nullable": []
},
"hash": "a2bc410e1066ddc89ca4557ca0e29339e41cbc7a7ef6ad97b3f0eb93de04c359"
}
backend/.sqlx/query-ade0696ec69bec2258d2e3ff86bfba6ed3b1d573c298f9d7fe8056ba7e32ed81.json
Generated
+32
View File
@@ -0,0 +1,32 @@
{
"db_name": "PostgreSQL",
"query": "SELECT workspace_id, path, value FROM variable WHERE is_secret = true AND value IS NOT NULL AND value != ''",
"describe": {
"columns": [
{
"ordinal": 0,
"name": "workspace_id",
"type_info": "Varchar"
},
{
"ordinal": 1,
"name": "path",
"type_info": "Varchar"
},
{
"ordinal": 2,
"name": "value",
"type_info": "Varchar"
}
],
"parameters": {
"Left": []
},
"nullable": [
false,
false,
false
]
},
"hash": "ade0696ec69bec2258d2e3ff86bfba6ed3b1d573c298f9d7fe8056ba7e32ed81"
}
backend/.sqlx/query-b030efec7c7f770bf2ed5331a469aae723925270d9d5124cc4cb2c03db61dea8.json
Generated
+20
View File
@@ -0,0 +1,20 @@
{
"db_name": "PostgreSQL",
"query": "SELECT COUNT(*) as count FROM variable WHERE is_secret = true AND value = 'CLEARED'",
"describe": {
"columns": [
{
"ordinal": 0,
"name": "count",
"type_info": "Int8"
}
],
"parameters": {
"Left": []
},
"nullable": [
null
]
},
"hash": "b030efec7c7f770bf2ed5331a469aae723925270d9d5124cc4cb2c03db61dea8"
}
backend/.sqlx/query-b107d602c60c08f0edcaa99695f8546bf30074bad903959f7df030e0ac70a86f.json
Generated
+32
View File
@@ -0,0 +1,32 @@
{
"db_name": "PostgreSQL",
"query": "SELECT workspace_id, path, value FROM variable WHERE is_secret = true AND value != 'CLEARED' ORDER BY workspace_id, path",
"describe": {
"columns": [
{
"ordinal": 0,
"name": "workspace_id",
"type_info": "Varchar"
},
{
"ordinal": 1,
"name": "path",
"type_info": "Varchar"
},
{
"ordinal": 2,
"name": "value",
"type_info": "Varchar"
}
],
"parameters": {
"Left": []
},
"nullable": [
false,
false,
false
]
},
"hash": "b107d602c60c08f0edcaa99695f8546bf30074bad903959f7df030e0ac70a86f"
}
backend/.sqlx/query-b10d2ab53bd8d24b0bbcede8211de229d507784fbcdf46c309907df123e35018.json
Generated
+32
View File
@@ -0,0 +1,32 @@
{
"db_name": "PostgreSQL",
"query": "SELECT workspace_id, path, value FROM variable WHERE is_secret = true ORDER BY workspace_id, path",
"describe": {
"columns": [
{
"ordinal": 0,
"name": "workspace_id",
"type_info": "Varchar"
},
{
"ordinal": 1,
"name": "path",
"type_info": "Varchar"
},
{
"ordinal": 2,
"name": "value",
"type_info": "Varchar"
}
],
"parameters": {
"Left": []
},
"nullable": [
false,
false,
false
]
},
"hash": "b10d2ab53bd8d24b0bbcede8211de229d507784fbcdf46c309907df123e35018"
}
backend/.sqlx/query-bb289fd24f443f0f8917ec55bc9fc3113e37dd8009425e558b5f5d1e1543b513.json
Generated
+16
View File
@@ -0,0 +1,16 @@
{
"db_name": "PostgreSQL",
"query": "UPDATE variable SET value = $1 WHERE path = $2 AND workspace_id = $3",
"describe": {
"columns": [],
"parameters": {
"Left": [
"Varchar",
"Text",
"Text"
]
},
"nullable": []
},
"hash": "bb289fd24f443f0f8917ec55bc9fc3113e37dd8009425e558b5f5d1e1543b513"
}
backend/.sqlx/query-3c53de373b9f1034b5f43002bf4715e12ddc641f4ca52efe0335719fa9461bb0.json → backend/.sqlx/query-d0dfd217a4f53aaee7b83d5b5267f74e5ad617830dd835a45e17fd378bcbeada.json
Generated
+26 -14
View File
@@ -1,6 +1,6 @@
{
"db_name": "PostgreSQL",
"query": "\n SELECT\n workspace_id,\n slack_team_id,\n teams_team_id,\n teams_team_name,\n teams_team_guid,\n slack_name,\n slack_command_script,\n teams_command_script,\n slack_email,\n slack_oauth_client_id,\n slack_oauth_client_secret,\n auto_invite_domain,\n auto_invite_operator,\n auto_add,\n customer_id,\n plan,\n webhook,\n deploy_to,\n ai_config,\n error_handler,\n error_handler_extra_args,\n error_handler_muted_on_cancel,\n large_file_storage,\n datatable,\n ducklake,\n git_sync,\n deploy_ui,\n default_app,\n default_scripts,\n mute_critical_alerts,\n color,\n operator_settings,\n git_app_installations,\n auto_add_instance_groups,\n auto_add_instance_groups_roles\n FROM\n workspace_settings\n WHERE\n workspace_id = $1\n ",
"query": "\n SELECT\n workspace_id,\n slack_team_id,\n teams_team_id,\n teams_team_name,\n teams_team_guid,\n slack_name,\n slack_command_script,\n teams_command_script,\n slack_email,\n slack_oauth_client_id,\n slack_oauth_client_secret,\n auto_invite_domain,\n auto_invite_operator,\n auto_add,\n customer_id,\n plan,\n webhook,\n deploy_to,\n ai_config,\n error_handler,\n error_handler_extra_args,\n error_handler_muted_on_cancel,\n success_handler,\n success_handler_extra_args,\n large_file_storage,\n datatable,\n ducklake,\n git_sync,\n deploy_ui,\n default_app,\n default_scripts,\n mute_critical_alerts,\n color,\n operator_settings,\n git_app_installations,\n auto_add_instance_groups,\n auto_add_instance_groups_roles\n FROM\n workspace_settings\n WHERE\n workspace_id = $1\n ",
"describe": {
"columns": [
{
@@ -115,66 +115,76 @@
},
{
"ordinal": 22,
"name": "success_handler",
"type_info": "Text"
},
{
"ordinal": 23,
"name": "success_handler_extra_args",
"type_info": "Json"
},
{
"ordinal": 24,
"name": "large_file_storage",
"type_info": "Jsonb"
},
{
"ordinal": 23,
"ordinal": 25,
"name": "datatable",
"type_info": "Jsonb"
},
{
"ordinal": 24,
"ordinal": 26,
"name": "ducklake",
"type_info": "Jsonb"
},
{
"ordinal": 25,
"ordinal": 27,
"name": "git_sync",
"type_info": "Jsonb"
},
{
"ordinal": 26,
"ordinal": 28,
"name": "deploy_ui",
"type_info": "Jsonb"
},
{
"ordinal": 27,
"ordinal": 29,
"name": "default_app",
"type_info": "Varchar"
},
{
"ordinal": 28,
"ordinal": 30,
"name": "default_scripts",
"type_info": "Jsonb"
},
{
"ordinal": 29,
"ordinal": 31,
"name": "mute_critical_alerts",
"type_info": "Bool"
},
{
"ordinal": 30,
"ordinal": 32,
"name": "color",
"type_info": "Varchar"
},
{
"ordinal": 31,
"ordinal": 33,
"name": "operator_settings",
"type_info": "Jsonb"
},
{
"ordinal": 32,
"ordinal": 34,
"name": "git_app_installations",
"type_info": "Jsonb"
},
{
"ordinal": 33,
"ordinal": 35,
"name": "auto_add_instance_groups",
"type_info": "TextArray"
},
{
"ordinal": 34,
"ordinal": 36,
"name": "auto_add_instance_groups_roles",
"type_info": "Jsonb"
}
@@ -217,10 +227,12 @@
true,
true,
true,
true,
true,
false,
true,
true
]
},
"hash": "3c53de373b9f1034b5f43002bf4715e12ddc641f4ca52efe0335719fa9461bb0"
"hash": "d0dfd217a4f53aaee7b83d5b5267f74e5ad617830dd835a45e17fd378bcbeada"
}
backend/.sqlx/query-dbc05cbf6720cc85e01118bcdcbb478724ac0ea8f3e043da2de9d932216cc730.json
Generated
+15
View File
@@ -0,0 +1,15 @@
{
"db_name": "PostgreSQL",
"query": "\n INSERT INTO script (workspace_id, hash, path, content, language, kind, created_by, schema, summary, description, lock)\n VALUES ('test-workspace', $1, 'f/test/simple_script', $2, 'deno', 'script', 'test-user', '{}', 'Simple test script', 'A simple test script', '')\n ",
"describe": {
"columns": [],
"parameters": {
"Left": [
"Int8",
"Text"
]
},
"nullable": []
},
"hash": "dbc05cbf6720cc85e01118bcdcbb478724ac0ea8f3e043da2de9d932216cc730"
}
backend/.sqlx/query-e3f2459d9ac3534531d5724bb2d67be6f33f8ca02ef4768bccd32a33a70a4300.json
Generated
+12
View File
@@ -0,0 +1,12 @@
{
"db_name": "PostgreSQL",
"query": "\n UPDATE workspace_settings\n SET success_handler = 'script/f/test/success_handler'\n WHERE workspace_id = 'test-workspace'\n ",
"describe": {
"columns": [],
"parameters": {
"Left": []
},
"nullable": []
},
"hash": "e3f2459d9ac3534531d5724bb2d67be6f33f8ca02ef4768bccd32a33a70a4300"
}
backend/Cargo.lock
Generated
+125 -118
View File
@@ -1716,7 +1716,7 @@ dependencies = [
"serde_json",
"serde_repr",
"serde_urlencoded",
"thiserror 2.0.17",
"thiserror 2.0.18",
"tokio",
"tokio-util",
"tower-service",
@@ -2082,9 +2082,9 @@ dependencies = [
[[package]]
name = "cc"
version = "1.2.52"
version = "1.2.53"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "cd4932aefd12402b36c60956a4fe0035421f544799057659ff86f923657aada3"
checksum = "755d2fce177175ffca841e9a06afdb2c4ab0f593d53b4dee48147dfaade85932"
dependencies = [
"find-msvc-tools",
"jobserver",
@@ -3557,7 +3557,7 @@ dependencies = [
"swc_visit",
"swc_visit_macros",
"text_lines",
"thiserror 2.0.17",
"thiserror 2.0.18",
"unicode-width 0.1.14",
"url",
]
@@ -3571,7 +3571,7 @@ dependencies = [
"async-trait",
"deno_core",
"deno_error",
"thiserror 2.0.17",
"thiserror 2.0.18",
"tokio",
"uuid",
]
@@ -3588,7 +3588,7 @@ dependencies = [
"rusqlite",
"serde",
"sha2 0.10.9",
"thiserror 2.0.17",
"thiserror 2.0.18",
"tokio",
]
@@ -3632,7 +3632,7 @@ dependencies = [
"deno_webgpu",
"image",
"serde",
"thiserror 2.0.17",
"thiserror 2.0.18",
]
[[package]]
@@ -3658,7 +3658,7 @@ dependencies = [
"serde",
"serde_json",
"sys_traits",
"thiserror 2.0.17",
"thiserror 2.0.18",
"url",
]
@@ -3703,7 +3703,7 @@ dependencies = [
"smallvec",
"sourcemap 8.0.1",
"static_assertions",
"thiserror 2.0.17",
"thiserror 2.0.18",
"tokio",
"url",
"v8",
@@ -3728,7 +3728,7 @@ dependencies = [
"deno_core",
"deno_error",
"saffron",
"thiserror 2.0.17",
"thiserror 2.0.18",
"tokio",
]
@@ -3766,7 +3766,7 @@ dependencies = [
"sha2 0.10.9",
"signature",
"spki",
"thiserror 2.0.17",
"thiserror 2.0.18",
"tokio",
"uuid",
"x25519-dalek",
@@ -3826,7 +3826,7 @@ dependencies = [
"rustls-webpki 0.102.8",
"serde",
"serde_json",
"thiserror 2.0.17",
"thiserror 2.0.18",
"tokio",
"tokio-rustls 0.26.4",
"tokio-socks",
@@ -3854,7 +3854,7 @@ dependencies = [
"serde",
"serde-value",
"serde_json",
"thiserror 2.0.17",
"thiserror 2.0.18",
"tokio",
"winapi",
]
@@ -3880,7 +3880,7 @@ dependencies = [
"rand 0.8.5",
"rayon",
"serde",
"thiserror 2.0.17",
"thiserror 2.0.18",
"winapi",
"windows-sys 0.59.0",
]
@@ -3919,7 +3919,7 @@ dependencies = [
"scopeguard",
"serde",
"smallvec",
"thiserror 2.0.17",
"thiserror 2.0.18",
"tokio",
"tokio-util",
]
@@ -3979,7 +3979,7 @@ dependencies = [
"rand 0.8.5",
"rusqlite",
"serde",
"thiserror 2.0.17",
"thiserror 2.0.18",
"url",
]
@@ -3992,7 +3992,7 @@ dependencies = [
"deno_semver",
"serde",
"serde_json",
"thiserror 2.0.17",
"thiserror 2.0.18",
]
[[package]]
@@ -4019,7 +4019,7 @@ dependencies = [
"libloading 0.7.4",
"log",
"napi_sym",
"thiserror 2.0.17",
"thiserror 2.0.18",
"windows-sys 0.59.0",
]
@@ -4053,7 +4053,7 @@ dependencies = [
"rustls-tokio-stream",
"serde",
"socket2 0.5.10",
"thiserror 2.0.17",
"thiserror 2.0.18",
"tokio",
]
@@ -4140,7 +4140,7 @@ dependencies = [
"spki",
"stable_deref_trait",
"sys_traits",
"thiserror 2.0.17",
"thiserror 2.0.18",
"tokio",
"tokio-eld",
"url",
@@ -4168,7 +4168,7 @@ dependencies = [
"monch",
"serde",
"serde_json",
"thiserror 2.0.17",
"thiserror 2.0.18",
"url",
]
@@ -4186,7 +4186,7 @@ dependencies = [
"strum 0.25.0",
"strum_macros 0.25.3",
"syn 2.0.114",
"thiserror 2.0.17",
"thiserror 2.0.18",
]
[[package]]
@@ -4207,7 +4207,7 @@ dependencies = [
"serde",
"signal-hook",
"signal-hook-registry",
"thiserror 2.0.17",
"thiserror 2.0.18",
"tokio",
"winapi",
]
@@ -4226,7 +4226,7 @@ dependencies = [
"serde",
"serde_json",
"sys_traits",
"thiserror 2.0.17",
"thiserror 2.0.18",
"url",
]
@@ -4239,7 +4239,7 @@ dependencies = [
"deno_error",
"percent-encoding",
"sys_traits",
"thiserror 2.0.17",
"thiserror 2.0.18",
"url",
]
@@ -4260,7 +4260,7 @@ dependencies = [
"once_cell",
"percent-encoding",
"serde",
"thiserror 2.0.17",
"thiserror 2.0.18",
"which 6.0.3",
"winapi",
]
@@ -4287,7 +4287,7 @@ dependencies = [
"serde",
"simd-json",
"tempfile",
"thiserror 2.0.17",
"thiserror 2.0.18",
"tokio",
"which 6.0.3",
"winapi",
@@ -4321,7 +4321,7 @@ dependencies = [
"once_cell",
"parking_lot",
"sys_traits",
"thiserror 2.0.17",
"thiserror 2.0.18",
"url",
]
@@ -4386,7 +4386,7 @@ dependencies = [
"serde",
"sys_traits",
"tempfile",
"thiserror 2.0.17",
"thiserror 2.0.18",
"tokio",
"tokio-metrics",
"twox-hash 1.6.3",
@@ -4409,7 +4409,7 @@ dependencies = [
"monch",
"once_cell",
"serde",
"thiserror 2.0.17",
"thiserror 2.0.18",
"url",
]
@@ -4436,7 +4436,7 @@ dependencies = [
"opentelemetry_sdk",
"pin-project",
"serde",
"thiserror 2.0.17",
"thiserror 2.0.18",
"tokio",
]
@@ -4464,7 +4464,7 @@ dependencies = [
"rustls-tokio-stream",
"rustls-webpki 0.102.8",
"serde",
"thiserror 2.0.17",
"thiserror 2.0.18",
"tokio",
"webpki-roots 0.26.11",
]
@@ -4488,7 +4488,7 @@ checksum = "d79e743ad841f7826d46c6944580f5ba665fe9ab4c31a68c4eed8b5a78225da3"
dependencies = [
"deno_core",
"deno_error",
"thiserror 2.0.17",
"thiserror 2.0.18",
"urlpattern",
]
@@ -4508,7 +4508,7 @@ dependencies = [
"flate2",
"futures",
"serde",
"thiserror 2.0.17",
"thiserror 2.0.18",
"tokio",
"uuid",
]
@@ -4523,7 +4523,7 @@ dependencies = [
"deno_error",
"raw-window-handle",
"serde",
"thiserror 2.0.17",
"thiserror 2.0.18",
"tokio",
"wgpu-core",
"wgpu-types",
@@ -4559,7 +4559,7 @@ dependencies = [
"once_cell",
"rustls-tokio-stream",
"serde",
"thiserror 2.0.17",
"thiserror 2.0.18",
"tokio",
]
@@ -4573,7 +4573,7 @@ dependencies = [
"deno_error",
"deno_web",
"rusqlite",
"thiserror 2.0.17",
"thiserror 2.0.18",
]
[[package]]
@@ -4621,7 +4621,7 @@ dependencies = [
"rand 0.8.5",
"serde",
"serde_json",
"thiserror 2.0.17",
"thiserror 2.0.18",
"tokio",
"tokio-util",
"url",
@@ -4646,7 +4646,7 @@ dependencies = [
"rand 0.8.5",
"rusqlite",
"serde_json",
"thiserror 2.0.17",
"thiserror 2.0.18",
"tokio",
"tokio-stream",
"uuid",
@@ -5474,21 +5474,20 @@ checksum = "28dea519a9695b9977216879a3ebfddf92f1c08c05d984f8996aecd6ecdc811d"
[[package]]
name = "filetime"
version = "0.2.26"
version = "0.2.27"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "bc0505cd1b6fa6580283f6bdf70a73fcf4aba1184038c90902b92b3dd0df63ed"
checksum = "f98844151eee8917efc50bd9e8318cb963ae8b297431495d3f758616ea5c57db"
dependencies = [
"cfg-if",
"libc",
"libredox",
"windows-sys 0.60.2",
]
[[package]]
name = "find-msvc-tools"
version = "0.1.7"
version = "0.1.8"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "f449e6c6c08c865631d4890cfacf252b3d396c9bcc83adb6623cdb02a8336c41"
checksum = "8591b0bcc8a98a64310a2fae1bb3e9b8564dd10e381e6e28010fde8e8e8568db"
[[package]]
name = "fixedbitset"
@@ -6575,7 +6574,7 @@ dependencies = [
"reqwest 0.12.28",
"serde",
"serde_json",
"thiserror 2.0.17",
"thiserror 2.0.18",
"tokio",
"ureq",
"windows-sys 0.60.2",
@@ -6600,7 +6599,7 @@ dependencies = [
"once_cell",
"rand 0.9.0",
"serde",
"thiserror 2.0.17",
"thiserror 2.0.18",
"tinyvec",
"tokio",
"tracing",
@@ -6624,7 +6623,7 @@ dependencies = [
"resolv-conf",
"serde",
"smallvec",
"thiserror 2.0.17",
"thiserror 2.0.18",
"tokio",
"tracing",
]
@@ -7147,7 +7146,7 @@ dependencies = [
"percent-encoding",
"serde",
"serde_json",
"thiserror 2.0.17",
"thiserror 2.0.18",
"url",
]
@@ -7427,7 +7426,7 @@ dependencies = [
"pest_derive",
"regex",
"serde_json",
"thiserror 2.0.17",
"thiserror 2.0.18",
]
[[package]]
@@ -7617,7 +7616,7 @@ dependencies = [
"serde",
"serde_json",
"serde_yaml",
"thiserror 2.0.17",
"thiserror 2.0.18",
"tokio",
"tokio-util",
"tower 0.5.3",
@@ -7641,7 +7640,7 @@ dependencies = [
"serde",
"serde-value",
"serde_json",
"thiserror 2.0.17",
"thiserror 2.0.18",
]
[[package]]
@@ -7679,7 +7678,7 @@ dependencies = [
"pin-project",
"serde",
"serde_json",
"thiserror 2.0.17",
"thiserror 2.0.18",
"tokio",
"tokio-util",
"tracing",
@@ -8470,7 +8469,7 @@ dependencies = [
"quote",
"syn 2.0.114",
"termcolor",
"thiserror 2.0.17",
"thiserror 2.0.18",
]
[[package]]
@@ -8495,7 +8494,7 @@ dependencies = [
"serde",
"serde_json",
"socket2 0.5.10",
"thiserror 2.0.17",
"thiserror 2.0.18",
"tokio",
"tokio-native-tls",
"tokio-util",
@@ -8527,7 +8526,7 @@ dependencies = [
"serde_json",
"sha1",
"sha2 0.10.9",
"thiserror 2.0.17",
"thiserror 2.0.18",
"uuid",
]
@@ -8697,7 +8696,7 @@ dependencies = [
"serde",
"serde_json",
"sys_traits",
"thiserror 2.0.17",
"thiserror 2.0.18",
"url",
]
@@ -8845,7 +8844,7 @@ dependencies = [
"num-format",
"serde",
"serde_json",
"thiserror 2.0.17",
"thiserror 2.0.18",
"typetag",
"windows-sys 0.48.0",
]
@@ -9120,7 +9119,7 @@ dependencies = [
"serde",
"serde_json",
"serde_urlencoded",
"thiserror 2.0.17",
"thiserror 2.0.18",
"tokio",
"tracing",
"url",
@@ -10202,7 +10201,7 @@ dependencies = [
"lazy_static",
"memchr",
"parking_lot",
"thiserror 2.0.17",
"thiserror 2.0.18",
]
[[package]]
@@ -10387,7 +10386,7 @@ dependencies = [
"rustc-hash 2.1.1",
"rustls 0.23.35",
"socket2 0.6.1",
"thiserror 2.0.17",
"thiserror 2.0.18",
"tokio",
"tracing",
"web-time",
@@ -10409,7 +10408,7 @@ dependencies = [
"rustls 0.23.35",
"rustls-pki-types",
"slab",
"thiserror 2.0.17",
"thiserror 2.0.18",
"tinyvec",
"tracing",
"web-time",
@@ -10696,7 +10695,7 @@ checksum = "a4e608c6638b9c18977b00b475ac1f28d14e84b27d8d42f70e0bf1e3dec127ac"
dependencies = [
"getrandom 0.2.17",
"libredox",
"thiserror 2.0.17",
"thiserror 2.0.18",
]
[[package]]
@@ -10873,7 +10872,7 @@ dependencies = [
"http 1.4.0",
"reqwest 0.13.1",
"serde",
"thiserror 2.0.17",
"thiserror 2.0.18",
"tower-service",
]
@@ -10892,7 +10891,7 @@ dependencies = [
"reqwest 0.13.1",
"reqwest-middleware",
"retry-policies",
"thiserror 2.0.17",
"thiserror 2.0.18",
"tokio",
"tracing",
"wasmtimer",
@@ -11015,7 +11014,7 @@ dependencies = [
"serde",
"serde_json",
"sse-stream",
"thiserror 2.0.17",
"thiserror 2.0.18",
"tokio",
"tokio-stream",
"tokio-util",
@@ -11274,7 +11273,7 @@ dependencies = [
"once_cell",
"ring 0.17.14",
"rustls-pki-types",
"rustls-webpki 0.103.8",
"rustls-webpki 0.103.9",
"subtle",
"zeroize",
]
@@ -11336,9 +11335,9 @@ dependencies = [
[[package]]
name = "rustls-pki-types"
version = "1.13.3"
version = "1.14.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "4910321ebe4151be888e35fe062169554e74aad01beafed60410131420ceffbc"
checksum = "be040f8b0a225e40375822a563fa9524378b9d63112f53e19ffff34df5d33fdd"
dependencies = [
"web-time",
"zeroize",
@@ -11358,7 +11357,7 @@ dependencies = [
"rustls 0.23.35",
"rustls-native-certs 0.8.3",
"rustls-platform-verifier-android",
"rustls-webpki 0.103.8",
"rustls-webpki 0.103.9",
"security-framework 3.5.1",
"security-framework-sys",
"webpki-root-certs 1.0.5",
@@ -11406,9 +11405,9 @@ dependencies = [
[[package]]
name = "rustls-webpki"
version = "0.103.8"
version = "0.103.9"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "2ffdfa2f5286e2247234e03f680868ac2815974dc39e00ea15adc445d0aafe52"
checksum = "d7df23109aa6c1567d1c575b9952556388da57401e4ace1d15f79eedad0d8f53"
dependencies = [
"aws-lc-rs",
"ring 0.17.14",
@@ -11938,7 +11937,7 @@ dependencies = [
"num-bigint",
"serde",
"smallvec",
"thiserror 2.0.17",
"thiserror 2.0.18",
"v8",
]
@@ -12167,7 +12166,7 @@ checksum = "297f631f50729c8c99b84667867963997ec0b50f32b2a7dbcab828ef0541e8bb"
dependencies = [
"num-bigint",
"num-traits",
"thiserror 2.0.17",
"thiserror 2.0.18",
"time",
]
@@ -12460,7 +12459,7 @@ dependencies = [
"serde_json",
"sha2 0.10.9",
"smallvec",
"thiserror 2.0.17",
"thiserror 2.0.18",
"tokio",
"tokio-stream",
"tracing",
@@ -12546,7 +12545,7 @@ dependencies = [
"smallvec",
"sqlx-core",
"stringprep",
"thiserror 2.0.17",
"thiserror 2.0.18",
"tracing",
"uuid",
"whoami",
@@ -12587,7 +12586,7 @@ dependencies = [
"smallvec",
"sqlx-core",
"stringprep",
"thiserror 2.0.17",
"thiserror 2.0.18",
"tracing",
"uuid",
"whoami",
@@ -12613,7 +12612,7 @@ dependencies = [
"serde",
"serde_urlencoded",
"sqlx-core",
"thiserror 2.0.17",
"thiserror 2.0.18",
"tracing",
"url",
"uuid",
@@ -13345,7 +13344,7 @@ dependencies = [
"tantivy-stacker",
"tantivy-tokenizer-api",
"tempfile",
"thiserror 2.0.17",
"thiserror 2.0.18",
"time",
"uuid",
"winapi",
@@ -13517,11 +13516,11 @@ dependencies = [
[[package]]
name = "thiserror"
version = "2.0.17"
version = "2.0.18"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "f63587ca0f12b72a0600bcba1d40081f830876000bb46dd2337a3051618f4fc8"
checksum = "4288b5bcbc7920c07a1149a35cf9590a2aa808e0bc1eafaade0b80947865fbc4"
dependencies = [
"thiserror-impl 2.0.17",
"thiserror-impl 2.0.18",
]
[[package]]
@@ -13537,9 +13536,9 @@ dependencies = [
[[package]]
name = "thiserror-impl"
version = "2.0.17"
version = "2.0.18"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "3ff15c8ecd7de3849db632e14d18d2571fa09dfc5ed93479bc4485c7a517c913"
checksum = "ebc4ee7f67670e9b64d05fa4253e753e016c6c95ff35b89b7941d6b856dec1d5"
dependencies = [
"proc-macro2",
"quote",
@@ -14191,7 +14190,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "786d480bce6247ab75f005b14ae1624ad978d3029d9113f0a22fa1ac773faeaf"
dependencies = [
"crossbeam-channel",
"thiserror 2.0.17",
"thiserror 2.0.18",
"time",
"tracing-subscriber",
]
@@ -15013,7 +15012,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "2eeee3bdea6257cc36d756fa745a70f9d393571e47d69e0ed97581676a5369ca"
dependencies = [
"deno_error",
"thiserror 2.0.17",
"thiserror 2.0.18",
]
[[package]]
@@ -15243,7 +15242,7 @@ checksum = "712e227841d057c1ee1cd2fb22fa7e5a5461ae8e48fa2ca79ec42cfc1931183f"
[[package]]
name = "windmill"
version = "1.608.0"
version = "1.611.0"
dependencies = [
"anyhow",
"aws-sdk-config",
@@ -15306,7 +15305,7 @@ dependencies = [
[[package]]
name = "windmill-api"
version = "1.608.0"
version = "1.611.0"
dependencies = [
"anyhow",
"argon2",
@@ -15343,6 +15342,7 @@ dependencies = [
"deno_core",
"deno_error",
"ed25519-dalek",
"flate2",
"futures",
"git-version",
"google-cloud-googleapis",
@@ -15390,8 +15390,9 @@ dependencies = [
"sha2 0.10.9",
"sql-builder",
"sqlx",
"tar",
"tempfile",
"thiserror 2.0.17",
"thiserror 2.0.18",
"time",
"tinyvector",
"tokenizers",
@@ -15429,7 +15430,7 @@ dependencies = [
[[package]]
name = "windmill-api-client"
version = "1.608.0"
version = "1.611.0"
dependencies = [
"reqwest 0.12.28",
"serde",
@@ -15439,7 +15440,7 @@ dependencies = [
[[package]]
name = "windmill-audit"
version = "1.608.0"
version = "1.611.0"
dependencies = [
"chrono",
"lazy_static",
@@ -15453,7 +15454,7 @@ dependencies = [
[[package]]
name = "windmill-autoscaling"
version = "1.608.0"
version = "1.611.0"
dependencies = [
"anyhow",
"axum 0.7.9",
@@ -15462,7 +15463,7 @@ dependencies = [
"serde",
"serde_json",
"sqlx",
"thiserror 2.0.17",
"thiserror 2.0.18",
"tokio",
"tracing",
"uuid",
@@ -15472,7 +15473,7 @@ dependencies = [
[[package]]
name = "windmill-common"
version = "1.608.0"
version = "1.611.0"
dependencies = [
"anyhow",
"async-recursion",
@@ -15543,7 +15544,7 @@ dependencies = [
"systemstat",
"tar",
"tempfile",
"thiserror 2.0.17",
"thiserror 2.0.18",
"tikv-jemalloc-ctl",
"tokio",
"tokio-postgres 0.7.13",
@@ -15566,7 +15567,7 @@ dependencies = [
[[package]]
name = "windmill-git-sync"
version = "1.608.0"
version = "1.611.0"
dependencies = [
"regex",
"serde",
@@ -15581,7 +15582,7 @@ dependencies = [
[[package]]
name = "windmill-indexer"
version = "1.608.0"
version = "1.611.0"
dependencies = [
"anyhow",
"astral-tokio-tar",
@@ -15605,7 +15606,7 @@ dependencies = [
[[package]]
name = "windmill-macros"
version = "1.608.0"
version = "1.611.0"
dependencies = [
"itertools 0.14.0",
"lazy_static",
@@ -15621,21 +15622,27 @@ dependencies = [
[[package]]
name = "windmill-mcp"
version = "1.608.0"
version = "1.611.0"
dependencies = [
"anyhow",
"async-trait",
"futures",
"http 1.4.0",
"oauth2",
"reqwest 0.12.28",
"rmcp",
"serde",
"serde_json",
"sqlx",
"tokio",
"tokio-util",
"tracing",
"windmill-common",
]
[[package]]
name = "windmill-oauth"
version = "1.608.0"
version = "1.611.0"
dependencies = [
"anyhow",
"async-oauth2",
@@ -15659,7 +15666,7 @@ dependencies = [
[[package]]
name = "windmill-parser"
version = "1.608.0"
version = "1.611.0"
dependencies = [
"convert_case 0.6.0",
"serde",
@@ -15668,7 +15675,7 @@ dependencies = [
[[package]]
name = "windmill-parser-bash"
version = "1.608.0"
version = "1.611.0"
dependencies = [
"anyhow",
"lazy_static",
@@ -15680,7 +15687,7 @@ dependencies = [
[[package]]
name = "windmill-parser-csharp"
version = "1.608.0"
version = "1.611.0"
dependencies = [
"anyhow",
"serde_json",
@@ -15692,7 +15699,7 @@ dependencies = [
[[package]]
name = "windmill-parser-go"
version = "1.608.0"
version = "1.611.0"
dependencies = [
"anyhow",
"gosyn",
@@ -15704,7 +15711,7 @@ dependencies = [
[[package]]
name = "windmill-parser-graphql"
version = "1.608.0"
version = "1.611.0"
dependencies = [
"anyhow",
"lazy_static",
@@ -15716,7 +15723,7 @@ dependencies = [
[[package]]
name = "windmill-parser-java"
version = "1.608.0"
version = "1.611.0"
dependencies = [
"anyhow",
"serde_json",
@@ -15728,7 +15735,7 @@ dependencies = [
[[package]]
name = "windmill-parser-nu"
version = "1.608.0"
version = "1.611.0"
dependencies = [
"anyhow",
"nu-parser",
@@ -15739,7 +15746,7 @@ dependencies = [
[[package]]
name = "windmill-parser-php"
version = "1.608.0"
version = "1.611.0"
dependencies = [
"anyhow",
"itertools 0.14.0",
@@ -15750,7 +15757,7 @@ dependencies = [
[[package]]
name = "windmill-parser-py"
version = "1.608.0"
version = "1.611.0"
dependencies = [
"anyhow",
"itertools 0.14.0",
@@ -15763,7 +15770,7 @@ dependencies = [
[[package]]
name = "windmill-parser-py-imports"
version = "1.608.0"
version = "1.611.0"
dependencies = [
"anyhow",
"async-recursion",
@@ -15787,7 +15794,7 @@ dependencies = [
[[package]]
name = "windmill-parser-ruby"
version = "1.608.0"
version = "1.611.0"
dependencies = [
"anyhow",
"lazy_static",
@@ -15801,7 +15808,7 @@ dependencies = [
[[package]]
name = "windmill-parser-rust"
version = "1.608.0"
version = "1.611.0"
dependencies = [
"anyhow",
"convert_case 0.6.0",
@@ -15818,7 +15825,7 @@ dependencies = [
[[package]]
name = "windmill-parser-sql"
version = "1.608.0"
version = "1.611.0"
dependencies = [
"anyhow",
"lazy_static",
@@ -15832,7 +15839,7 @@ dependencies = [
[[package]]
name = "windmill-parser-ts"
version = "1.608.0"
version = "1.611.0"
dependencies = [
"anyhow",
"lazy_static",
@@ -15851,7 +15858,7 @@ dependencies = [
[[package]]
name = "windmill-parser-yaml"
version = "1.608.0"
version = "1.611.0"
dependencies = [
"anyhow",
"serde",
@@ -15862,7 +15869,7 @@ dependencies = [
[[package]]
name = "windmill-queue"
version = "1.608.0"
version = "1.611.0"
dependencies = [
"anyhow",
"async-recursion",
@@ -15888,7 +15895,7 @@ dependencies = [
"serde_urlencoded",
"sql-builder",
"sqlx",
"thiserror 2.0.17",
"thiserror 2.0.18",
"tokio",
"tracing",
"ulid",
@@ -15899,7 +15906,7 @@ dependencies = [
[[package]]
name = "windmill-sql-datatype-parser-wasm"
version = "1.608.0"
version = "1.611.0"
dependencies = [
"wasm-bindgen",
"wasm-bindgen-test",
@@ -15909,7 +15916,7 @@ dependencies = [
[[package]]
name = "windmill-worker"
version = "1.608.0"
version = "1.611.0"
dependencies = [
"anyhow",
"async-once-cell",
backend/Cargo.toml
+3 -2
View File
@@ -1,6 +1,6 @@
[package]
name = "windmill"
version = "1.608.0"
version = "1.611.0"
authors.workspace = true
edition.workspace = true
@@ -35,7 +35,7 @@ members = [
exclude = ["./windmill-duckdb-ffi-internal"]
[workspace.package]
version = "1.608.0"
version = "1.611.0"
authors = ["Ruben Fiszel <ruben@windmill.dev>"]
edition = "2021"
@@ -405,6 +405,7 @@ async-once-cell = "0.5.4"
aws-smithy-types-convert = { version = "^0", features = ["convert-chrono"] }
crc = "^3"
tar = "^0"
flate2 = "^1"
http = "^1"
async-stream = "^0"
backend/ee-repo-ref.txt
+1 -1
View File
@@ -1 +1 @@
da0a3751abd2b0fed46ca5c7a3aef35923403b28
a4790328e8fb78256b6f746a8825ec6ec499c3a8
backend/generate_mcp_endpoints_tools/generate_mcp_tools.py
+5 -23
View File
@@ -13,21 +13,7 @@ from typing import Dict, List, Any, Optional
IMPORTS = """
use std::borrow::Cow;
use serde::{Deserialize, Serialize};
"""
ENDPOINT_STRUCT = """
#[derive(Debug, Clone, Deserialize, Serialize)]
pub struct EndpointTool {
pub name: Cow<'static, str>,
pub description: Cow<'static, str>,
pub instructions: Cow<'static, str>,
pub path: Cow<'static, str>,
pub method: Cow<'static, str>,
pub path_params_schema: Option<serde_json::Value>,
pub query_params_schema: Option<serde_json::Value>,
pub body_schema: Option<serde_json::Value>,
}
use windmill_mcp::server::EndpointTool;
"""
def load_openapi_spec(file_path: str) -> Dict[str, Any]:
@@ -341,13 +327,11 @@ export const mcpEndpointTools: EndpointTool[] = [
def generate_rust_code(tools: List[Dict[str, Any]], spec: Dict[str, Any], base_path: str = "") -> str:
"""Generate the complete Rust code with MCP tools."""
if not tools:
return """// No MCP tools found in the OpenAPI specification
return f"""// No MCP tools found in the OpenAPI specification
{IMPORTS}
{ENDPOINT_STRUCT}
pub fn all_tools() -> Vec<EndpointTool> {
pub fn all_tools() -> Vec<EndpointTool> {{
vec![]
}
}}
"""
tool_definitions = []
@@ -386,9 +370,7 @@ pub fn all_tools() -> Vec<EndpointTool> {
rust_code = f"""// Auto-generated MCP tools from OpenAPI specification
// This file is generated by generate_mcp_tools.py - DO NOT EDIT MANUALLY
{IMPORTS}
{ENDPOINT_STRUCT}
pub fn all_tools() -> Vec<EndpointTool> {{
vec![
{tool_definitions_str}
@@ -405,7 +387,7 @@ def main():
project_dir = backend_dir.parent
openapi_file = backend_dir / "windmill-api" / "openapi.yaml"
rust_output_file = backend_dir / "windmill-api" / "src" / "mcp" / "tools" / "auto_generated_endpoints.rs"
rust_output_file = backend_dir / "windmill-api" / "src" / "mcp" / "auto_generated_endpoints.rs"
ts_output_file = project_dir / "frontend" / "src" / "lib" / "mcpEndpointTools.ts"
if not openapi_file.exists():
backend/migrations/20260117105517_add_workspace_success_handler.down.sql
+4
View File
@@ -0,0 +1,4 @@
-- Remove success_handler columns from workspace_settings
ALTER TABLE workspace_settings
DROP COLUMN IF EXISTS success_handler,
DROP COLUMN IF EXISTS success_handler_extra_args;
backend/migrations/20260117105517_add_workspace_success_handler.up.sql
+4
View File
@@ -0,0 +1,4 @@
-- Add success_handler columns to workspace_settings
ALTER TABLE workspace_settings
ADD COLUMN IF NOT EXISTS success_handler TEXT,
ADD COLUMN IF NOT EXISTS success_handler_extra_args JSON;
backend/src/monitor.rs
+162 -94
View File
@@ -155,6 +155,20 @@ lazy_static::lazy_static! {
.ok()
.and_then(|x| x.parse::<u64>().ok());
/// Batch size for job cleanup deletion queries. Default: 10000.
/// Larger values delete more jobs per batch but hold locks longer.
static ref JOB_CLEANUP_BATCH_SIZE: i64 = std::env::var("JOB_CLEANUP_BATCH_SIZE")
.ok()
.and_then(|x| x.parse::<i64>().ok())
.unwrap_or(20000);
/// Maximum number of batches to process per cleanup iteration. Default: 10.
/// Set to 0 for unlimited (process until no expired jobs remain).
static ref JOB_CLEANUP_MAX_BATCHES: i32 = std::env::var("JOB_CLEANUP_MAX_BATCHES")
.ok()
.and_then(|x| x.parse::<i32>().ok())
.unwrap_or(20);
}
pub async fn initial_load(
@@ -910,99 +924,153 @@ pub async fn delete_expired_items(db: &DB) -> () {
let job_retention_secs = *JOB_RETENTION_SECS.read().await;
if job_retention_secs > 0 {
match db.begin().await {
Ok(mut tx) => {
let deleted_jobs = sqlx::query_scalar!(
"DELETE FROM v2_job_completed c
WHERE completed_at <= now() - ($1::bigint::text || ' s')::interval
RETURNING c.id",
job_retention_secs
)
.fetch_all(&mut *tx)
.await;
let batch_size = *JOB_CLEANUP_BATCH_SIZE;
let max_batches = *JOB_CLEANUP_MAX_BATCHES;
let cleanup_start = Instant::now();
let mut total_deleted = 0u64;
let mut batch_num = 0i32;
match deleted_jobs {
Ok(deleted_jobs) => {
if deleted_jobs.len() > 0 {
tracing::info!(
"deleted {} jobs completed JOB_RETENTION_SECS {} ago: {:?}",
deleted_jobs.len(),
job_retention_secs,
deleted_jobs,
);
if let Err(e) = sqlx::query!(
"DELETE FROM job_stats WHERE job_id = ANY($1)",
&deleted_jobs
)
.execute(&mut *tx)
.await
{
tracing::error!("Error deleting job stats: {:?}", e);
}
match sqlx::query_scalar!(
"DELETE FROM job_logs WHERE job_id = ANY($1) RETURNING log_file_index",
&deleted_jobs
)
.fetch_all(&mut *tx)
.await
{
Ok(log_file_index) => {
let paths = log_file_index
.into_iter()
.filter_map(|opt| opt)
.flat_map(|inner_vec| inner_vec.into_iter())
.collect();
delete_log_files_from_disk_and_store(paths, TMP_DIR, "").await;
}
Err(e) => tracing::error!("Error deleting job stats: {:?}", e),
}
if let Err(e) = sqlx::query!(
"DELETE FROM concurrency_key WHERE ended_at <= now() - ($1::bigint::text || ' s')::interval ",
job_retention_secs
)
.execute(&mut *tx)
.await
{
tracing::error!("Error deleting custom concurrency key: {:?}", e);
}
if let Err(e) =
sqlx::query!("DELETE FROM v2_job WHERE id = ANY($1)", &deleted_jobs)
.execute(&mut *tx)
.await
{
tracing::error!("Error deleting job: {:?}", e);
}
// should already be deleted but just in case
if let Err(e) = sqlx::query!(
"DELETE FROM job_result_stream_v2 WHERE job_id = ANY($1)",
&deleted_jobs
)
.execute(&mut *tx)
.await
{
tracing::error!("Error deleting job result stream: {:?}", e);
}
}
}
Err(e) => {
tracing::error!("Error deleting expired jobs: {:?}", e)
}
}
match tx.commit().await {
Ok(_) => (),
Err(err) => tracing::error!("Error deleting expired jobs: {:?}", err),
}
// Process batches until no more expired jobs or max batches reached
loop {
if max_batches > 0 && batch_num >= max_batches {
tracing::debug!(
"Job cleanup: reached max batches limit ({}), will continue next iteration",
max_batches
);
break;
}
Err(err) => {
tracing::error!("Error deleting expired jobs: {:?}", err)
// Each batch runs in its own transaction to avoid long-running locks
let batch_result = delete_expired_jobs_batch(db, job_retention_secs, batch_size).await;
match batch_result {
Ok(deleted_count) => {
if deleted_count == 0 {
// No more expired jobs to delete
break;
}
total_deleted += deleted_count as u64;
batch_num += 1;
}
Err(e) => {
tracing::error!("Error in job cleanup batch {}: {:?}", batch_num, e);
break;
}
}
}
if total_deleted > 0 {
tracing::info!(
"Job cleanup completed: deleted {} jobs in {} batches, took {:?}",
total_deleted,
batch_num,
cleanup_start.elapsed()
);
}
// Clean up concurrency keys separately (not tied to specific job IDs)
if let Err(e) = sqlx::query!(
"DELETE FROM concurrency_key WHERE ended_at <= now() - ($1::bigint::text || ' s')::interval",
job_retention_secs
)
.execute(db)
.await
{
tracing::error!("Error deleting custom concurrency key: {:?}", e);
}
}
}
/// Delete a batch of expired jobs with LIMIT and SKIP LOCKED for high-scale environments.
/// Uses a single transaction per batch to minimize lock duration.
/// Returns the number of jobs deleted in this batch.
async fn delete_expired_jobs_batch(
db: &DB,
job_retention_secs: i64,
batch_size: i64,
) -> error::Result<usize> {
let mut tx = db.begin().await?;
// Use FOR UPDATE SKIP LOCKED to avoid contention between replicas
// ORDER BY completed_at ensures we delete oldest jobs first
let deleted_jobs: Vec<Uuid> = sqlx::query_scalar!(
"DELETE FROM v2_job_completed
WHERE id IN (
SELECT id FROM v2_job_completed
WHERE completed_at <= now() - ($1::bigint::text || ' s')::interval
ORDER BY completed_at ASC
LIMIT $2
FOR UPDATE SKIP LOCKED
)
RETURNING id",
job_retention_secs,
batch_size
)
.fetch_all(&mut *tx)
.await?;
let deleted_count = deleted_jobs.len();
if deleted_count > 0 {
tracing::debug!(
"Deleting batch of {} expired jobs (retention: {}s)",
deleted_count,
job_retention_secs
);
// Delete related records for this batch
if let Err(e) = sqlx::query!(
"DELETE FROM job_stats WHERE job_id = ANY($1)",
&deleted_jobs
)
.execute(&mut *tx)
.await
{
tracing::error!("Error deleting job stats: {:?}", e);
}
match sqlx::query_scalar!(
"DELETE FROM job_logs WHERE job_id = ANY($1) RETURNING log_file_index",
&deleted_jobs
)
.fetch_all(&mut *tx)
.await
{
Ok(log_file_index) => {
let paths = log_file_index
.into_iter()
.filter_map(|opt| opt)
.flat_map(|inner_vec| inner_vec.into_iter())
.collect();
delete_log_files_from_disk_and_store(paths, TMP_DIR, "").await;
}
Err(e) => tracing::error!("Error deleting job logs: {:?}", e),
}
if let Err(e) = sqlx::query!("DELETE FROM v2_job WHERE id = ANY($1)", &deleted_jobs)
.execute(&mut *tx)
.await
{
tracing::error!("Error deleting job: {:?}", e);
}
// Should already be deleted but just in case
if let Err(e) = sqlx::query!(
"DELETE FROM job_result_stream_v2 WHERE job_id = ANY($1)",
&deleted_jobs
)
.execute(&mut *tx)
.await
{
tracing::error!("Error deleting job result stream: {:?}", e);
}
}
tx.commit().await?;
Ok(deleted_count)
}
async fn delete_log_files_from_disk_and_store(
paths_to_delete: Vec<String>,
tmp_dir: &str,
@@ -2262,7 +2330,7 @@ async fn handle_zombie_jobs(db: &Pool<Postgres>, base_internal_url: &str, node_n
sqlx::query_scalar!("SELECT j.id
FROM v2_job_queue q JOIN v2_job j USING (id) LEFT JOIN v2_job_runtime r USING (id) LEFT JOIN v2_job_status s USING (id)
WHERE r.ping < now() - ($1 || ' seconds')::interval
AND q.running = true AND j.kind NOT IN ('flow', 'flowpreview', 'flownode', 'singlestepflow') AND j.same_worker = false",
AND q.running = true AND j.kind NOT IN ('flow', 'flowpreview', 'flownode', 'singlestepflow') AND j.same_worker = false",
ZOMBIE_JOB_TIMEOUT.as_str())
.fetch_all(db)
.await
@@ -2396,8 +2464,8 @@ orphaned_job_uuids AS (
SELECT job_uuid
FROM all_job_uuids
WHERE job_uuid NOT IN (
SELECT id::text
FROM v2_job_queue
SELECT id::text
FROM v2_job_queue
FOR SHARE SKIP LOCKED
)
),
@@ -2410,7 +2478,7 @@ before_update AS (
WHERE lc.job_uuids ?| oa.orphaned_keys
),
affected_rows AS (
UPDATE concurrency_counter
UPDATE concurrency_counter
SET job_uuids = job_uuids - orphaned_array.orphaned_keys
FROM orphaned_array
WHERE concurrency_counter.concurrency_id IN (
@@ -2419,12 +2487,12 @@ affected_rows AS (
RETURNING concurrency_id, job_uuids AS updated_job_uuids
),
expanded_orphaned AS (
SELECT bu.concurrency_id,
SELECT bu.concurrency_id,
bu.job_uuids AS original_job_uuids,
unnest(bu.orphaned_keys) AS orphaned_key
FROM before_update bu
)
SELECT
SELECT
eo.concurrency_id,
eo.orphaned_key,
eo.original_job_uuids,
@@ -2453,7 +2521,7 @@ async fn cleanup_concurrency_counters_empty_keys(db: &DB) -> error::Result<()> {
WITH rows_to_delete AS (
SELECT concurrency_id
FROM concurrency_counter
WHERE job_uuids = '{}'::jsonb
FOR UPDATE SKIP LOCKED
)
@@ -2489,7 +2557,7 @@ async fn handle_zombie_flows(db: &DB) -> error::Result<()> {
AND (j.kind = 'flow' OR j.kind = 'flowpreview' OR j.kind = 'flownode')
AND r.ping IS NOT NULL AND r.ping < NOW() - ($1 || ' seconds')::interval
AND q.canceled_by IS NULL
"#,
FLOW_ZOMBIE_TRANSITION_TIMEOUT.as_str()
)
backend/summarized_schema.txt
+2
View File
@@ -1065,6 +1065,8 @@ TABLE: workspace_settings
- ai_config (jsonb)
- error_handler_extra_args (json)
- error_handler_muted_on_cancel (boolean)
- success_handler (text)
- success_handler_extra_args (json)
- large_file_storage (jsonb)
- git_sync (jsonb)
- default_app (character)
backend/tests/fixtures/secret_backend.sql
Vendored
+31
View File
@@ -0,0 +1,31 @@
-- Fixture for secret backend migration tests
-- Sets up test secrets in the variable table
-- Create a second workspace for testing workspace isolation
INSERT INTO workspace (id, name, owner)
VALUES ('test-workspace-2', 'test-workspace-2', 'test-user')
ON CONFLICT DO NOTHING;
INSERT INTO workspace_settings (workspace_id)
VALUES ('test-workspace-2')
ON CONFLICT DO NOTHING;
INSERT INTO workspace_key(workspace_id, kind, key)
VALUES ('test-workspace-2', 'cloud', 'test-key-2')
ON CONFLICT DO NOTHING;
-- Insert test secrets for workspace 1
-- Note: The 'value' column stores encrypted values in production,
-- but for tests we'll use plain text that the migration will handle
INSERT INTO variable (workspace_id, path, value, is_secret, description, extra_perms)
VALUES
('test-workspace', 'u/test-user/db_password', 'encrypted-db-pass-123', true, 'Database password', '{}'),
('test-workspace', 'u/test-user/api_key', 'encrypted-api-key-abc', true, 'API key for external service', '{}'),
('test-workspace', 'u/test-user/public_var', 'not-a-secret', false, 'A non-secret variable', '{}')
ON CONFLICT DO NOTHING;
-- Insert test secrets for workspace 2 (to test isolation)
INSERT INTO variable (workspace_id, path, value, is_secret, description, extra_perms)
VALUES
('test-workspace-2', 'u/test-user/other_secret', 'encrypted-other-secret', true, 'Secret in workspace 2', '{}')
ON CONFLICT DO NOTHING;
backend/tests/secret_backend_integration.rs
+557
View File
@@ -0,0 +1,557 @@
//! Integration tests for HashiCorp Vault secret backend.
//!
//! These tests require:
//! 1. A PostgreSQL database (handled by sqlx test framework)
//! 2. A running HashiCorp Vault instance
//! 3. The RUN_VAULT_TESTS=1 environment variable to be set
//!
//! Environment variables:
//! - RUN_VAULT_TESTS=1 - Required to run the tests
//! - VAULT_ADDR - Vault server address (default: http://127.0.0.1:8200)
//! - VAULT_TOKEN - Static token for static token tests (default: test-root-token)
//! - BASE_URL - Windmill instance URL for JWT tests (default: http://localhost:8000)
//!
//! Run tests (static token mode):
//! ```bash
//! RUN_VAULT_TESTS=1 VAULT_TOKEN=your-token cargo test -p windmill \
//! secret_backend_integration --features private,enterprise -- --nocapture
//! ```
//!
//! Run tests (JWT mode - requires Windmill instance running for JWKS endpoint):
//! ```bash
//! RUN_VAULT_TESTS=1 BASE_URL=http://localhost:8000 cargo test -p windmill \
//! secret_backend_integration --features private,enterprise,openidconnect -- --nocapture
//! ```
#[cfg(all(feature = "private", feature = "enterprise"))]
mod tests {
use sqlx::{Pool, Postgres};
use std::collections::HashMap;
use windmill_common::secret_backend::{
migrate_secrets_to_database, migrate_secrets_to_vault, test_vault_connection,
SecretBackend, VaultBackend, VaultSettings,
};
/// Check if vault tests should run (requires RUN_VAULT_TESTS=1 env var)
fn should_run_vault_tests() -> bool {
std::env::var("RUN_VAULT_TESTS")
.map(|v| v == "1" || v.to_lowercase() == "true")
.unwrap_or(false)
}
/// Set up BASE_URL for JWT tests (required for OIDC issuer URL generation)
async fn setup_base_url() {
let base_url = std::env::var("BASE_URL")
.unwrap_or_else(|_| "http://localhost:8000".to_string());
let mut url = windmill_common::BASE_URL.write().await;
*url = base_url;
}
/// Skip test if RUN_VAULT_TESTS is not set
macro_rules! skip_if_no_vault {
() => {
if !should_run_vault_tests() {
println!("Skipping test: RUN_VAULT_TESTS=1 not set");
println!("To run vault tests: RUN_VAULT_TESTS=1 cargo test ...");
return;
}
};
}
fn vault_settings_static_token() -> VaultSettings {
VaultSettings {
address: std::env::var("VAULT_ADDR")
.unwrap_or_else(|_| "http://127.0.0.1:8200".to_string()),
mount_path: "windmill".to_string(),
jwt_role: None, // Static token mode
namespace: None,
token: Some(
std::env::var("VAULT_TOKEN").unwrap_or_else(|_| "test-root-token".to_string()),
),
}
}
fn vault_settings_jwt() -> VaultSettings {
VaultSettings {
address: std::env::var("VAULT_ADDR")
.unwrap_or_else(|_| "http://127.0.0.1:8200".to_string()),
mount_path: "windmill".to_string(),
jwt_role: Some("windmill-secrets".to_string()), // JWT mode
namespace: None,
token: None, // No static token - use JWT
}
}
// ==================== Static Token Tests ====================
/// Test Vault connection with static token
#[sqlx::test(fixtures("base", "secret_backend"))]
async fn test_vault_connection_static_token(db: Pool<Postgres>) {
skip_if_no_vault!();
let settings = vault_settings_static_token();
println!("Testing Vault connection with static token...");
println!(" Address: {}", settings.address);
let result = test_vault_connection(&settings, Some(&db)).await;
assert!(
result.is_ok(),
"Failed to connect to Vault: {:?}",
result.err()
);
println!("✓ Successfully connected to Vault with static token");
}
/// Test basic CRUD operations with static token
#[sqlx::test(fixtures("base", "secret_backend"))]
async fn test_vault_crud_static_token(_db: Pool<Postgres>) {
skip_if_no_vault!();
let settings = vault_settings_static_token();
let backend = VaultBackend::new(settings);
let workspace_id = "test-crud-static";
let path = "test-secret";
let value = "my-super-secret-value-123";
println!("Testing CRUD with static token...");
// Create
println!(" Creating secret...");
backend
.set_secret(workspace_id, path, value)
.await
.expect("Failed to create secret");
println!(" ✓ Created");
// Read
println!(" Reading secret...");
let read_value = backend
.get_secret(workspace_id, path)
.await
.expect("Failed to read secret");
assert_eq!(read_value, value);
println!(" ✓ Read (value matches)");
// Update
println!(" Updating secret...");
let new_value = "updated-secret-value-456";
backend
.set_secret(workspace_id, path, new_value)
.await
.expect("Failed to update secret");
let updated = backend
.get_secret(workspace_id, path)
.await
.expect("Failed to read updated secret");
assert_eq!(updated, new_value);
println!(" ✓ Updated");
// Delete
println!(" Deleting secret...");
backend
.delete_secret(workspace_id, path)
.await
.expect("Failed to delete secret");
let result = backend.get_secret(workspace_id, path).await;
assert!(result.is_err(), "Secret should be deleted");
println!(" ✓ Deleted");
println!("✓ CRUD operations successful with static token");
}
// ==================== JWT Auth Tests ====================
/// Test Vault connection with JWT authentication
#[sqlx::test(fixtures("base", "secret_backend"))]
async fn test_vault_connection_jwt(db: Pool<Postgres>) {
skip_if_no_vault!();
setup_base_url().await;
let settings = vault_settings_jwt();
println!("Testing Vault connection with JWT auth...");
println!(" Address: {}", settings.address);
println!(" JWT Role: {:?}", settings.jwt_role);
println!(" BASE_URL: {}", windmill_common::BASE_URL.read().await.clone());
let result = test_vault_connection(&settings, Some(&db)).await;
assert!(
result.is_ok(),
"Failed to connect to Vault with JWT: {:?}",
result.err()
);
println!("✓ Successfully connected to Vault with JWT auth");
}
/// Test basic CRUD operations with JWT authentication
#[cfg(feature = "openidconnect")]
#[sqlx::test(fixtures("base", "secret_backend"))]
async fn test_vault_crud_jwt(db: Pool<Postgres>) {
skip_if_no_vault!();
setup_base_url().await;
let settings = vault_settings_jwt();
let backend = VaultBackend::new_with_db(settings, db.clone());
let workspace_id = "test-crud-jwt";
let path = "jwt-test-secret";
let value = "jwt-authenticated-secret-value";
println!("Testing CRUD with JWT auth...");
// Create
println!(" Creating secret...");
backend
.set_secret(workspace_id, path, value)
.await
.expect("Failed to create secret with JWT");
println!(" ✓ Created");
// Read
println!(" Reading secret...");
let read_value = backend
.get_secret(workspace_id, path)
.await
.expect("Failed to read secret with JWT");
assert_eq!(read_value, value);
println!(" ✓ Read (value matches)");
// Delete (cleanup)
println!(" Deleting secret...");
backend
.delete_secret(workspace_id, path)
.await
.expect("Failed to delete secret with JWT");
println!(" ✓ Deleted");
println!("✓ CRUD operations successful with JWT auth");
}
// ==================== Migration Tests ====================
/// Test migration from database to Vault
#[sqlx::test(fixtures("base", "secret_backend"))]
async fn test_migrate_db_to_vault(db: Pool<Postgres>) {
skip_if_no_vault!();
let settings = vault_settings_static_token();
// Verify Vault connection
test_vault_connection(&settings, Some(&db))
.await
.expect("Failed to connect to Vault");
// Check initial state
let secrets_before = sqlx::query!(
"SELECT workspace_id, path, value FROM variable WHERE is_secret = true ORDER BY workspace_id, path"
)
.fetch_all(&db)
.await
.expect("Failed to query secrets");
println!(
"Found {} secrets in database before migration:",
secrets_before.len()
);
for s in &secrets_before {
println!(" - {}/{}: {} chars", s.workspace_id, s.path, s.value.len());
}
// Run migration
println!("\nMigrating secrets to Vault...");
let report = migrate_secrets_to_vault(&db, &settings)
.await
.expect("Migration to Vault failed");
println!("Migration report:");
println!(" Total secrets: {}", report.total_secrets);
println!(" Migrated: {}", report.migrated_count);
println!(" Failed: {}", report.failed_count);
if !report.failures.is_empty() {
println!(" Failures:");
for f in &report.failures {
println!(" - {}/{}: {}", f.workspace_id, f.path, f.error);
}
}
assert_eq!(report.failed_count, 0, "Migration had failures");
assert!(report.migrated_count > 0, "No secrets were migrated");
// Verify secrets in Vault
println!("\nVerifying secrets in Vault...");
let vault_backend = VaultBackend::new(settings.clone());
for secret in &secrets_before {
let result = vault_backend
.get_secret(&secret.workspace_id, &secret.path)
.await;
assert!(
result.is_ok(),
"Failed to read secret {}/{} from Vault: {:?}",
secret.workspace_id,
secret.path,
result.err()
);
println!(
" ✓ {}/{} exists in Vault",
secret.workspace_id, secret.path
);
}
println!("\n✓ Migration to Vault completed successfully");
}
/// Test migration from Vault back to database
#[sqlx::test(fixtures("base", "secret_backend"))]
async fn test_migrate_vault_to_db(db: Pool<Postgres>) {
skip_if_no_vault!();
let settings = vault_settings_static_token();
test_vault_connection(&settings, Some(&db))
.await
.expect("Failed to connect to Vault");
// First migrate TO Vault
println!("Setting up: migrating secrets to Vault first...");
let to_vault = migrate_secrets_to_vault(&db, &settings)
.await
.expect("Initial migration to Vault failed");
assert!(to_vault.migrated_count > 0, "No secrets to test with");
println!(" Migrated {} secrets to Vault", to_vault.migrated_count);
// Clear database values
println!("\nClearing database secret values...");
sqlx::query!("UPDATE variable SET value = 'CLEARED' WHERE is_secret = true")
.execute(&db)
.await
.expect("Failed to clear values");
// Migrate back from Vault
println!("\nMigrating secrets from Vault to database...");
let report = migrate_secrets_to_database(&db, &settings)
.await
.expect("Migration to database failed");
println!("Migration report:");
println!(" Total secrets: {}", report.total_secrets);
println!(" Migrated: {}", report.migrated_count);
println!(" Failed: {}", report.failed_count);
assert_eq!(report.failed_count, 0, "Migration had failures");
assert!(report.migrated_count > 0, "No secrets were migrated");
// Verify restored
let restored = sqlx::query!(
"SELECT COUNT(*) as count FROM variable WHERE is_secret = true AND value != 'CLEARED'"
)
.fetch_one(&db)
.await
.expect("Failed to count restored");
assert!(
restored.count.unwrap_or(0) > 0,
"No secrets were restored in database"
);
println!("\n✓ Migration to database completed successfully");
}
// ==================== Variable Rename Tests ====================
/// Test renaming a variable path in Vault
#[sqlx::test(fixtures("base", "secret_backend"))]
async fn test_variable_rename(db: Pool<Postgres>) {
skip_if_no_vault!();
let _ = &db; // suppress unused warning
let settings = vault_settings_static_token();
let backend = VaultBackend::new(settings);
let workspace_id = "test-workspace";
let old_path = "u/test-user/old_secret_name";
let new_path = "u/test-user/new_secret_name";
let value = "secret-value-for-rename-test";
println!("Testing variable rename in Vault...");
// Create secret at old path
println!(" Creating secret at old path: {}", old_path);
backend
.set_secret(workspace_id, old_path, value)
.await
.expect("Failed to create secret");
// Verify it exists
let read_value = backend
.get_secret(workspace_id, old_path)
.await
.expect("Failed to read secret at old path");
assert_eq!(read_value, value);
println!(" ✓ Secret exists at old path");
// Simulate rename: read from old, write to new, delete old
println!(" Renaming: {} -> {}", old_path, new_path);
let secret_value = backend
.get_secret(workspace_id, old_path)
.await
.expect("Failed to read for rename");
backend
.set_secret(workspace_id, new_path, &secret_value)
.await
.expect("Failed to write to new path");
backend
.delete_secret(workspace_id, old_path)
.await
.expect("Failed to delete old path");
// Verify old path is gone
let old_result = backend.get_secret(workspace_id, old_path).await;
assert!(old_result.is_err(), "Old path should not exist");
println!(" ✓ Old path deleted");
// Verify new path exists with correct value
let new_value = backend
.get_secret(workspace_id, new_path)
.await
.expect("Failed to read new path");
assert_eq!(new_value, value);
println!(" ✓ New path exists with correct value");
// Cleanup
backend
.delete_secret(workspace_id, new_path)
.await
.expect("Failed to cleanup");
println!("\n✓ Variable rename completed successfully");
}
// ==================== Full Round Trip Test ====================
/// Test full round-trip: DB -> Vault -> DB with verification
#[sqlx::test(fixtures("base", "secret_backend"))]
async fn test_full_round_trip(db: Pool<Postgres>) {
skip_if_no_vault!();
let settings = vault_settings_static_token();
test_vault_connection(&settings, Some(&db))
.await
.expect("Failed to connect to Vault");
// Get original secrets
let original: HashMap<(String, String), String> = sqlx::query!(
"SELECT workspace_id, path, value FROM variable WHERE is_secret = true"
)
.fetch_all(&db)
.await
.expect("Failed to query")
.into_iter()
.map(|r| ((r.workspace_id, r.path), r.value))
.collect();
println!("Original secrets: {} entries", original.len());
// Step 1: DB -> Vault
println!("\n=== Step 1: Migrate DB -> Vault ===");
let to_vault = migrate_secrets_to_vault(&db, &settings)
.await
.expect("Migration to Vault failed");
println!("Migrated {} secrets to Vault", to_vault.migrated_count);
assert_eq!(to_vault.failed_count, 0);
// Step 2: Clear DB
println!("\n=== Step 2: Clear database values ===");
sqlx::query!("UPDATE variable SET value = 'ROUND_TRIP_CLEARED' WHERE is_secret = true")
.execute(&db)
.await
.expect("Failed to clear");
// Step 3: Vault -> DB
println!("\n=== Step 3: Migrate Vault -> DB ===");
let to_db = migrate_secrets_to_database(&db, &settings)
.await
.expect("Migration to database failed");
println!("Migrated {} secrets to database", to_db.migrated_count);
assert_eq!(to_db.failed_count, 0);
// Step 4: Verify
println!("\n=== Step 4: Verify round-trip integrity ===");
let restored: HashMap<(String, String), String> = sqlx::query!(
"SELECT workspace_id, path, value FROM variable WHERE is_secret = true"
)
.fetch_all(&db)
.await
.expect("Failed to query")
.into_iter()
.map(|r| ((r.workspace_id, r.path), r.value))
.collect();
for ((ws, path), _) in &original {
let restored_value = restored
.get(&(ws.clone(), path.clone()))
.expect(&format!("Secret {}/{} not found after round-trip", ws, path));
assert_ne!(
restored_value, "ROUND_TRIP_CLEARED",
"Secret {}/{} was not restored",
ws, path
);
println!(" ✓ {}/{}: restored", ws, path);
}
println!("\n✓ Full round-trip completed successfully!");
}
// ==================== Workspace Isolation Test ====================
/// Test that workspace isolation is maintained
#[sqlx::test(fixtures("base", "secret_backend"))]
async fn test_workspace_isolation(db: Pool<Postgres>) {
skip_if_no_vault!();
let settings = vault_settings_static_token();
let backend = VaultBackend::new(settings.clone());
// First migrate secrets to Vault
migrate_secrets_to_vault(&db, &settings)
.await
.expect("Migration failed");
println!("Testing workspace isolation...");
// Try to access workspace-2 secret from workspace-1 path (should fail)
let cross_access = backend
.get_secret("test-workspace", "u/test-user/other_secret")
.await;
assert!(
cross_access.is_err(),
"Cross-workspace access should fail!"
);
println!("✓ Cross-workspace access correctly denied");
// Verify own workspace access works
let ws1 = backend
.get_secret("test-workspace", "u/test-user/db_password")
.await;
assert!(ws1.is_ok(), "Same-workspace access should work");
println!("✓ Same-workspace access works");
println!("\n✓ Workspace isolation verified!");
}
}
// OSS version - just a placeholder to avoid compilation errors
#[cfg(not(all(feature = "private", feature = "enterprise")))]
mod tests {
#[test]
fn test_vault_requires_enterprise() {
println!("Vault integration tests require Enterprise Edition features");
println!("Run with: cargo test --features private,enterprise,openidconnect");
}
}
backend/tests/secret_backend_migration.rs
+323
View File
@@ -0,0 +1,323 @@
//! Integration tests for secret backend migration between database and HashiCorp Vault.
//!
//! These tests require:
//! 1. A PostgreSQL database (handled by sqlx test framework)
//! 2. A running HashiCorp Vault instance at http://127.0.0.1:8200
//!
//! To run these tests:
//! ```bash
//! # Start Vault in dev mode
//! podman run -d --name vault-test -p 8200:8200 \
//! -e VAULT_DEV_ROOT_TOKEN_ID=test-root-token \
//! docker.io/hashicorp/vault:latest
//!
//! # Enable KV v2 secrets engine
//! curl -s -H "X-Vault-Token: test-root-token" -X POST \
//! --data '{"type":"kv-v2"}' \
//! http://127.0.0.1:8200/v1/sys/mounts/windmill
//!
//! # Run the tests
//! cargo test -p windmill secret_backend_migration -- --ignored --nocapture
//! ```
use sqlx::{Pool, Postgres};
use windmill_common::error::Result;
use windmill_common::secret_backend::{
vault_oss::{migrate_secrets_to_database, migrate_secrets_to_vault, test_vault_connection, VaultBackend},
SecretBackend, VaultSettings,
};
mod common;
fn test_vault_settings() -> VaultSettings {
VaultSettings {
address: std::env::var("VAULT_ADDR").unwrap_or_else(|_| "http://127.0.0.1:8200".to_string()),
mount_path: "windmill".to_string(),
jwt_role: Some("windmill-secrets".to_string()),
namespace: None,
token: Some(
std::env::var("VAULT_TOKEN").unwrap_or_else(|_| "test-root-token".to_string()),
),
}
}
/// Test that we can connect to Vault
#[sqlx::test(fixtures("base", "secret_backend"))]
#[ignore = "requires running Vault instance"]
async fn test_vault_connection_works(db: Pool<Postgres>) {
let settings = test_vault_settings();
let result = test_vault_connection(&settings, Some(&db)).await;
assert!(result.is_ok(), "Failed to connect to Vault: {:?}", result.err());
println!("✓ Successfully connected to Vault at {}", settings.address);
}
/// Test migration from database to Vault
#[sqlx::test(fixtures("base", "secret_backend"))]
#[ignore = "requires running Vault instance"]
async fn test_migrate_db_to_vault(db: Pool<Postgres>) {
let settings = test_vault_settings();
// First verify we can connect to Vault
test_vault_connection(&settings, Some(&db))
.await
.expect("Failed to connect to Vault");
// Check initial state - secrets should exist in database
let secrets_before = sqlx::query!(
"SELECT workspace_id, path, value FROM variable WHERE is_secret = true ORDER BY workspace_id, path"
)
.fetch_all(&db)
.await
.expect("Failed to query secrets");
println!("Found {} secrets in database before migration:", secrets_before.len());
for s in &secrets_before {
println!(" - {}/{}: {} chars", s.workspace_id, s.path, s.value.len());
}
// Run migration to Vault
println!("\nMigrating secrets to Vault...");
let report = migrate_secrets_to_vault(&db, &settings)
.await
.expect("Migration to Vault failed");
println!("Migration report:");
println!(" Total secrets: {}", report.total_secrets);
println!(" Migrated: {}", report.migrated_count);
println!(" Failed: {}", report.failed_count);
if !report.failures.is_empty() {
println!(" Failures:");
for f in &report.failures {
println!(" - {}/{}: {}", f.workspace_id, f.path, f.error);
}
}
assert_eq!(report.failed_count, 0, "Migration had failures");
assert!(report.migrated_count > 0, "No secrets were migrated");
// Verify secrets are in Vault
println!("\nVerifying secrets in Vault...");
let vault_backend = VaultBackend::new(settings.clone());
for secret in &secrets_before {
let result: Result<String> = vault_backend
.get_secret(&secret.workspace_id, &secret.path)
.await;
assert!(
result.is_ok(),
"Failed to read secret {}/{} from Vault: {:?}",
secret.workspace_id,
secret.path,
result.err()
);
println!(" ✓ {}/{} exists in Vault", secret.workspace_id, secret.path);
}
println!("\n✓ Migration to Vault completed successfully");
}
/// Test migration from Vault to database
#[sqlx::test(fixtures("base", "secret_backend"))]
#[ignore = "requires running Vault instance"]
async fn test_migrate_vault_to_db(db: Pool<Postgres>) {
let settings = test_vault_settings();
// First verify we can connect to Vault
test_vault_connection(&settings, Some(&db))
.await
.expect("Failed to connect to Vault");
// First, migrate secrets TO Vault so we have something to migrate back
println!("Setting up: migrating secrets to Vault first...");
let to_vault_report = migrate_secrets_to_vault(&db, &settings)
.await
.expect("Initial migration to Vault failed");
assert!(to_vault_report.migrated_count > 0, "No secrets to test with");
println!(" Migrated {} secrets to Vault", to_vault_report.migrated_count);
// Clear the database values to simulate fresh migration back
println!("\nClearing database secret values...");
sqlx::query!("UPDATE variable SET value = 'CLEARED' WHERE is_secret = true")
.execute(&db)
.await
.expect("Failed to clear database values");
// Verify they were cleared
let cleared = sqlx::query!(
"SELECT COUNT(*) as count FROM variable WHERE is_secret = true AND value = 'CLEARED'"
)
.fetch_one(&db)
.await
.expect("Failed to count cleared");
println!(" Cleared {} secret values in database", cleared.count.unwrap_or(0));
// Now migrate from Vault back to database
println!("\nMigrating secrets from Vault to database...");
let report = migrate_secrets_to_database(&db, &settings)
.await
.expect("Migration to database failed");
println!("Migration report:");
println!(" Total secrets: {}", report.total_secrets);
println!(" Migrated: {}", report.migrated_count);
println!(" Failed: {}", report.failed_count);
if !report.failures.is_empty() {
println!(" Failures:");
for f in &report.failures {
println!(" - {}/{}: {}", f.workspace_id, f.path, f.error);
}
}
assert_eq!(report.failed_count, 0, "Migration had failures");
assert!(report.migrated_count > 0, "No secrets were migrated");
// Verify secrets are restored in database
println!("\nVerifying secrets in database...");
let secrets_after = sqlx::query!(
"SELECT workspace_id, path, value FROM variable WHERE is_secret = true AND value != 'CLEARED' ORDER BY workspace_id, path"
)
.fetch_all(&db)
.await
.expect("Failed to query restored secrets");
assert!(
!secrets_after.is_empty(),
"No secrets were restored in database"
);
for s in &secrets_after {
println!(" ✓ {}/{}: {} chars", s.workspace_id, s.path, s.value.len());
}
println!("\n✓ Migration to database completed successfully");
}
/// Test full round-trip migration: DB -> Vault -> DB
#[sqlx::test(fixtures("base", "secret_backend"))]
#[ignore = "requires running Vault instance"]
async fn test_full_round_trip_migration(db: Pool<Postgres>) {
let settings = test_vault_settings();
// Verify Vault connection
test_vault_connection(&settings, Some(&db))
.await
.expect("Failed to connect to Vault");
// Get original secrets
let original_secrets: std::collections::HashMap<(String, String), String> = sqlx::query!(
"SELECT workspace_id, path, value FROM variable WHERE is_secret = true"
)
.fetch_all(&db)
.await
.expect("Failed to query original secrets")
.into_iter()
.map(|r| ((r.workspace_id, r.path), r.value))
.collect();
println!("Original secrets: {} entries", original_secrets.len());
// Step 1: Migrate to Vault
println!("\n=== Step 1: Migrate DB -> Vault ===");
let to_vault = migrate_secrets_to_vault(&db, &settings)
.await
.expect("Migration to Vault failed");
println!("Migrated {} secrets to Vault", to_vault.migrated_count);
assert_eq!(to_vault.failed_count, 0);
// Step 2: Clear database values
println!("\n=== Step 2: Clear database values ===");
sqlx::query!("UPDATE variable SET value = 'ROUND_TRIP_CLEARED' WHERE is_secret = true")
.execute(&db)
.await
.expect("Failed to clear values");
// Step 3: Migrate back from Vault
println!("\n=== Step 3: Migrate Vault -> DB ===");
let to_db = migrate_secrets_to_database(&db, &settings)
.await
.expect("Migration to database failed");
println!("Migrated {} secrets to database", to_db.migrated_count);
assert_eq!(to_db.failed_count, 0);
// Step 4: Verify round-trip integrity
println!("\n=== Step 4: Verify round-trip integrity ===");
let restored_secrets: std::collections::HashMap<(String, String), String> = sqlx::query!(
"SELECT workspace_id, path, value FROM variable WHERE is_secret = true"
)
.fetch_all(&db)
.await
.expect("Failed to query restored secrets")
.into_iter()
.map(|r| ((r.workspace_id, r.path), r.value))
.collect();
// Compare original and restored
for ((ws, path), _original_value) in &original_secrets {
let restored_value = restored_secrets
.get(&(ws.clone(), path.clone()))
.expect(&format!("Secret {}/{} not found after round-trip", ws, path));
// Note: Values might differ slightly due to encryption/decryption
// but they should not be the cleared value
assert_ne!(
restored_value, "ROUND_TRIP_CLEARED",
"Secret {}/{} was not restored",
ws, path
);
println!(" ✓ {}/{}: restored ({} chars)", ws, path, restored_value.len());
}
println!("\n✓ Full round-trip migration completed successfully!");
println!(" Original secrets: {}", original_secrets.len());
println!(" Restored secrets: {}", restored_secrets.len());
}
/// Test that workspace isolation is maintained during migration
#[sqlx::test(fixtures("base", "secret_backend"))]
#[ignore = "requires running Vault instance"]
async fn test_workspace_isolation(db: Pool<Postgres>) {
let settings = test_vault_settings();
test_vault_connection(&settings, Some(&db))
.await
.expect("Failed to connect to Vault");
// Migrate all secrets to Vault
let report = migrate_secrets_to_vault(&db, &settings)
.await
.expect("Migration failed");
println!("Migrated {} secrets across workspaces", report.migrated_count);
// Verify workspace isolation in Vault
let vault_backend = VaultBackend::new(settings.clone());
// Try to access test-workspace-2 secret from test-workspace path (should fail)
let cross_workspace_result: Result<String> = vault_backend
.get_secret("test-workspace", "u/test-user/other_secret")
.await;
assert!(
cross_workspace_result.is_err(),
"Cross-workspace access should fail - workspace isolation violated!"
);
println!("✓ Cross-workspace access correctly denied");
// Verify each workspace's secrets are accessible from their own workspace
let ws1_result: Result<String> = vault_backend
.get_secret("test-workspace", "u/test-user/db_password")
.await;
assert!(ws1_result.is_ok(), "test-workspace secret should be accessible");
println!("✓ test-workspace secrets accessible");
let ws2_result: Result<String> = vault_backend
.get_secret("test-workspace-2", "u/test-user/other_secret")
.await;
assert!(ws2_result.is_ok(), "test-workspace-2 secret should be accessible");
println!("✓ test-workspace-2 secrets accessible");
println!("\n✓ Workspace isolation verified!");
}
backend/tests/success_handler.rs
+241
View File
@@ -0,0 +1,241 @@
use sqlx::{Pool, Postgres};
mod common;
use common::*;
/// Test that the workspace success handler cache works correctly with 60s TTL
#[cfg(feature = "deno_core")]
#[sqlx::test(fixtures("base"))]
async fn test_success_handler_cache(db: Pool<Postgres>) -> anyhow::Result<()> {
initialize_tracing().await;
// First, create a success handler script
let _server = ApiServer::start(db.clone()).await?;
// Set up a success handler in workspace_settings
sqlx::query!(
r#"
UPDATE workspace_settings
SET success_handler = 'script/f/test/success_handler',
success_handler_extra_args = '{"key": "value"}'::json
WHERE workspace_id = 'test-workspace'
"#
)
.execute(&db)
.await?;
// Verify the success handler was set
let result = sqlx::query_scalar!(
r#"SELECT success_handler FROM workspace_settings WHERE workspace_id = 'test-workspace'"#
)
.fetch_one(&db)
.await?;
assert_eq!(result, Some("script/f/test/success_handler".to_string()));
// Verify extra args were set
let extra_args = sqlx::query_scalar!(
r#"SELECT success_handler_extra_args FROM workspace_settings WHERE workspace_id = 'test-workspace'"#
)
.fetch_one(&db)
.await?;
assert!(extra_args.is_some());
Ok(())
}
/// Test that success handler can be set and removed via database operations
#[cfg(feature = "deno_core")]
#[sqlx::test(fixtures("base"))]
async fn test_success_handler_settings(db: Pool<Postgres>) -> anyhow::Result<()> {
initialize_tracing().await;
let _server = ApiServer::start(db.clone()).await?;
// Initially success_handler should be NULL
let initial = sqlx::query_scalar!(
r#"SELECT success_handler FROM workspace_settings WHERE workspace_id = 'test-workspace'"#
)
.fetch_one(&db)
.await?;
assert!(initial.is_none());
// Set success handler
sqlx::query!(
r#"
UPDATE workspace_settings
SET success_handler = 'flow/f/test/success_flow'
WHERE workspace_id = 'test-workspace'
"#
)
.execute(&db)
.await?;
let after_set = sqlx::query_scalar!(
r#"SELECT success_handler FROM workspace_settings WHERE workspace_id = 'test-workspace'"#
)
.fetch_one(&db)
.await?;
assert_eq!(after_set, Some("flow/f/test/success_flow".to_string()));
// Remove success handler
sqlx::query!(
r#"
UPDATE workspace_settings
SET success_handler = NULL
WHERE workspace_id = 'test-workspace'
"#
)
.execute(&db)
.await?;
let after_remove = sqlx::query_scalar!(
r#"SELECT success_handler FROM workspace_settings WHERE workspace_id = 'test-workspace'"#
)
.fetch_one(&db)
.await?;
assert!(after_remove.is_none());
Ok(())
}
/// Test that a successful job completion triggers the success handler
#[cfg(all(feature = "deno_core", feature = "enterprise", feature = "private"))]
#[sqlx::test(fixtures("base"))]
async fn test_success_handler_triggered_on_success(db: Pool<Postgres>) -> anyhow::Result<()> {
use serde_json::json;
use windmill_common::jobs::JobPayload;
use windmill_common::runnable_settings::{ConcurrencySettings, DebouncingSettings};
use windmill_common::scripts::{ScriptHash, ScriptLang};
initialize_tracing().await;
let server = ApiServer::start(db.clone()).await?;
// Create a simple success handler script
// Note: lock must be non-null for script to be considered "deployed"
let success_handler_code = r#"
export async function main(path: string, email: string, job_id: string, is_flow: boolean, workspace_id: string, result: any) {
console.log("Success handler called for job:", job_id);
return { handled: true, original_path: path };
}
"#;
sqlx::query!(
r#"
INSERT INTO script (workspace_id, hash, path, content, language, kind, created_by, schema, summary, description, lock)
VALUES ('test-workspace', 1234567890, 'f/test/success_handler', $1, 'deno', 'script', 'test-user', '{}', 'Success handler script', 'Handles successful job completions', '')
"#,
success_handler_code
)
.execute(&db)
.await?;
// Create a simple test script that we'll run (needs to be JobKind::Script, not Preview)
// Note: lock must be non-null for script to be considered "deployed"
let test_script_code = "export function main() { return 'success'; }";
let test_script_hash: i64 = 9876543210;
sqlx::query!(
r#"
INSERT INTO script (workspace_id, hash, path, content, language, kind, created_by, schema, summary, description, lock)
VALUES ('test-workspace', $1, 'f/test/simple_script', $2, 'deno', 'script', 'test-user', '{}', 'Simple test script', 'A simple test script', '')
"#,
test_script_hash,
test_script_code
)
.execute(&db)
.await?;
// Set up the success handler in workspace_settings
sqlx::query!(
r#"
UPDATE workspace_settings
SET success_handler = 'script/f/test/success_handler'
WHERE workspace_id = 'test-workspace'
"#
)
.execute(&db)
.await?;
// Create the success_handler group
sqlx::query!(
r#"
INSERT INTO group_ (workspace_id, name, summary, extra_perms)
VALUES ('test-workspace', 'success_handler', 'The group the success handler acts on behalf of', '{"u/test-user": true}')
ON CONFLICT DO NOTHING
"#
)
.execute(&db)
.await?;
// Run a script using ScriptHash (produces JobKind::Script, not Preview)
let completed_job = RunJob::from(JobPayload::ScriptHash {
hash: ScriptHash(test_script_hash),
path: "f/test/simple_script".to_string(),
cache_ttl: None,
cache_ignore_s3_path: None,
dedicated_worker: None,
language: ScriptLang::Deno,
priority: None,
apply_preprocessor: false,
concurrency_settings: ConcurrencySettings::default(),
debouncing_settings: DebouncingSettings::default(),
})
.run_until_complete(&db, false, server.addr.port())
.await;
let result = completed_job.json_result().unwrap();
assert_eq!(result, json!("success"));
let main_job_id = completed_job.id;
// Wait a short time for the success handler job to be created
tokio::time::sleep(tokio::time::Duration::from_millis(500)).await;
// Verify the success handler job was created (query by email since trigger is not set)
let success_handler_job = sqlx::query!(
r#"
SELECT
id,
runnable_path,
permissioned_as_email,
parent_job
FROM v2_job
WHERE workspace_id = 'test-workspace'
AND permissioned_as_email = 'success_handler@windmill.dev'
ORDER BY created_at DESC
LIMIT 1
"#
)
.fetch_optional(&db)
.await?;
assert!(
success_handler_job.is_some(),
"Success handler job should have been created"
);
let handler_job = success_handler_job.unwrap();
// Verify the success handler job has correct parameters
assert_eq!(
handler_job.runnable_path.as_deref(),
Some("f/test/success_handler"),
"Success handler should run the configured script"
);
assert_eq!(
handler_job.permissioned_as_email.as_str(),
"success_handler@windmill.dev",
"Success handler should run as success_handler user"
);
assert_eq!(
handler_job.parent_job,
Some(main_job_id),
"Success handler should have main job as parent"
);
// Note: root_job may be None when it equals parent_job (optimization in push function)
Ok(())
}
backend/windmill-api/Cargo.toml
+3 -1
View File
@@ -10,7 +10,7 @@ path = "src/lib.rs"
[features]
default = []
private = ["windmill-audit/private"]
private = ["windmill-audit/private", "windmill-common/private"]
enterprise = ["windmill-queue/enterprise", "windmill-audit/enterprise", "windmill-git-sync/enterprise", "windmill-common/enterprise", "windmill-worker/enterprise"]
stripe = []
agent_worker_server = []
@@ -157,6 +157,8 @@ google-cloud-googleapis = { workspace = true , optional = true }
tonic = { workspace = true, optional = true }
deno_error = { workspace = true, optional = true }
deno_core = { workspace = true, optional = true }
tar.workspace = true
flate2.workspace = true
backon = {workspace = true, optional = true}
[build-dependencies]
backend/windmill-api/openapi.yaml
+322 -1
View File
@@ -1,7 +1,7 @@
openapi: "3.0.3"
info:
version: 1.608.0
version: 1.611.0
title: Windmill API
contact:
@@ -1411,6 +1411,83 @@ paths:
items:
$ref: "#/components/schemas/GlobalSetting"
/.well-known/jwks.json:
get:
summary: get JWKS for Vault JWT authentication
operationId: getJwks
tags:
- setting
responses:
"200":
description: JSON Web Key Set
content:
application/json:
schema:
$ref: "#/components/schemas/JwksResponse"
/settings/test_secret_backend:
post:
summary: test secret backend connection (HashiCorp Vault)
operationId: testSecretBackend
tags:
- setting
requestBody:
description: Vault settings to test
required: true
content:
application/json:
schema:
$ref: "#/components/schemas/VaultSettings"
responses:
"200":
description: connection successful
content:
text/plain:
schema:
type: string
/settings/migrate_secrets_to_vault:
post:
summary: migrate secrets from database to HashiCorp Vault
operationId: migrateSecretsToVault
tags:
- setting
requestBody:
description: Vault settings for migration target
required: true
content:
application/json:
schema:
$ref: "#/components/schemas/VaultSettings"
responses:
"200":
description: migration report
content:
application/json:
schema:
$ref: "#/components/schemas/SecretMigrationReport"
/settings/migrate_secrets_to_database:
post:
summary: migrate secrets from HashiCorp Vault to database
operationId: migrateSecretsToDatabase
tags:
- setting
requestBody:
description: Vault settings for migration source
required: true
content:
application/json:
schema:
$ref: "#/components/schemas/VaultSettings"
responses:
"200":
description: migration report
content:
application/json:
schema:
$ref: "#/components/schemas/SecretMigrationReport"
/users/email:
get:
summary: get current user email (if logged in)
@@ -2276,6 +2353,10 @@ paths:
$ref: "#/components/schemas/ScriptArgs"
error_handler_muted_on_cancel:
type: boolean
success_handler:
type: string
success_handler_extra_args:
$ref: "#/components/schemas/ScriptArgs"
large_file_storage:
$ref: "#/components/schemas/LargeFileStorage"
ducklake:
@@ -2992,6 +3073,34 @@ paths:
schema:
type: string
/w/{workspace}/workspaces/edit_success_handler:
post:
summary: edit success handler
operationId: editSuccessHandler
tags:
- workspace
parameters:
- $ref: "#/components/parameters/WorkspaceId"
requestBody:
description: WorkspaceSuccessHandler
required: true
content:
application/json:
schema:
type: object
properties:
success_handler:
type: string
success_handler_extra_args:
$ref: "#/components/schemas/ScriptArgs"
responses:
"200":
description: status
content:
text/plain:
schema:
type: string
/w/{workspace}/workspaces/edit_large_file_storage_config:
post:
summary: edit large file storage settings
@@ -4977,6 +5086,141 @@ paths:
items:
type: string
/w/{workspace}/npm_proxy/metadata/{package}:
get:
summary: get npm package metadata from private registry
operationId: getNpmPackageMetadata
tags:
- npm_proxy
parameters:
- $ref: "#/components/parameters/WorkspaceId"
- name: package
description: npm package name
in: path
required: true
schema:
type: string
responses:
"200":
description: package metadata
content:
application/json:
schema:
type: object
properties:
tags:
type: object
additionalProperties:
type: string
versions:
type: array
items:
type: string
/w/{workspace}/npm_proxy/resolve/{package}:
get:
summary: resolve npm package version from private registry
operationId: resolveNpmPackageVersion
tags:
- npm_proxy
parameters:
- $ref: "#/components/parameters/WorkspaceId"
- name: package
description: npm package name
in: path
required: true
schema:
type: string
- name: tag
description: version tag or reference
in: query
required: false
schema:
type: string
responses:
"200":
description: resolved version
content:
application/json:
schema:
type: object
properties:
version:
type: string
nullable: true
/w/{workspace}/npm_proxy/filetree/{package}/{version}:
get:
summary: get npm package file tree from private registry
operationId: getNpmPackageFiletree
tags:
- npm_proxy
parameters:
- $ref: "#/components/parameters/WorkspaceId"
- name: package
description: npm package name
in: path
required: true
schema:
type: string
- name: version
description: package version
in: path
required: true
schema:
type: string
responses:
"200":
description: package file tree
content:
application/json:
schema:
type: object
properties:
default:
type: string
files:
type: array
items:
type: object
properties:
name:
type: string
/w/{workspace}/npm_proxy/file/{package}/{version}/{filepath}:
get:
summary: get specific file from npm package in private registry
operationId: getNpmPackageFile
tags:
- npm_proxy
parameters:
- $ref: "#/components/parameters/WorkspaceId"
- name: package
description: npm package name
in: path
required: true
schema:
type: string
- name: version
description: package version
in: path
required: true
schema:
type: string
- name: filepath
description: file path within package
in: path
required: true
schema:
type: string
responses:
"200":
description: file content
content:
text/plain:
schema:
type: string
/w/{workspace}/embeddings/query_resource_types:
get:
summary: query resource types by similarity
@@ -16430,6 +16674,83 @@ components:
# -- INLINE END --
# Do not change line above
VaultSettings:
type: object
required:
- address
- mount_path
properties:
address:
type: string
description: HashiCorp Vault server address (e.g., https://vault.company.com:8200)
mount_path:
type: string
description: KV v2 secrets engine mount path (e.g., windmill)
jwt_role:
type: string
description: Vault JWT auth role name for Windmill (optional, if not provided token auth is used)
namespace:
type: string
description: Vault Enterprise namespace (optional)
token:
type: string
description: Static Vault token for testing/development (optional, if provided this is used instead of JWT authentication)
SecretMigrationFailure:
type: object
required:
- workspace_id
- path
- error
properties:
workspace_id:
type: string
description: Workspace ID where the secret is located
path:
type: string
description: Path of the secret that failed to migrate
error:
type: string
description: Error message
SecretMigrationReport:
type: object
required:
- total_secrets
- migrated_count
- failed_count
- failures
properties:
total_secrets:
type: integer
format: int64
description: Total number of secrets found
migrated_count:
type: integer
format: int64
description: Number of secrets successfully migrated
failed_count:
type: integer
format: int64
description: Number of secrets that failed to migrate
failures:
type: array
items:
$ref: "#/components/schemas/SecretMigrationFailure"
description: Details of any failures encountered during migration
JwksResponse:
type: object
required:
- keys
properties:
keys:
type: array
items:
type: object
additionalProperties: true
description: Array of JSON Web Keys for JWT verification
FlowConversation:
type: object
required:
backend/windmill-api/src/lib.rs
+8 -2
View File
@@ -106,6 +106,7 @@ mod inkeep_oss;
mod inputs;
mod integration;
mod live_migrations;
mod npm_proxy;
#[cfg(feature = "http_trigger")]
mod openapi;
#[cfg(all(feature = "private", feature = "parquet"))]
@@ -148,6 +149,7 @@ mod scim_oss;
mod scopes;
mod scripts;
mod service_logs;
mod secret_backend_ext;
mod settings;
mod slack_approvals;
#[cfg(all(feature = "smtp", feature = "private"))]
@@ -355,7 +357,7 @@ pub async fn run_server(
{
let smtp_server = Arc::new(SmtpServer {
db: db.clone(),
user_db: user_db,
user_db: user_db.clone(),
auth_cache: auth_cache.clone(),
base_internal_url: _base_internal_url.clone(),
});
@@ -409,7 +411,8 @@ pub async fn run_server(
let (mcp_router, mcp_cancellation_token) = {
#[cfg(feature = "mcp")]
if server_mode || mcp_mode {
let (mcp_router, mcp_cancellation_token) = setup_mcp_server().await?;
let (mcp_router, mcp_cancellation_token) =
setup_mcp_server(db.clone(), user_db).await?;
let mcp_middleware = axum::middleware::from_fn(extract_and_store_workspace_id);
(
mcp_router.layer(mcp_middleware),
@@ -495,6 +498,7 @@ pub async fn run_server(
Router::new()
})
.nest("/ai", ai::workspaced_service())
.nest("/npm_proxy", npm_proxy::workspaced_service())
.nest("/raw_apps", raw_apps::workspaced_service())
.nest("/resources", resources::workspaced_service())
.nest("/schedules", schedule::workspaced_service())
@@ -716,6 +720,8 @@ pub async fn run_server(
.route("/openapi.yaml", get(openapi))
.route("/openapi.json", get(openapi_json)),
)
// JWKS endpoint for HashiCorp Vault JWT authentication (must be outside /api prefix)
.route("/.well-known/jwks.json", get(settings::get_jwks))
.fallback(static_assets::static_handler)
.layer(middleware_stack);
backend/windmill-api/src/mcp/tools/auto_generated_endpoints.rs → backend/windmill-api/src/mcp/auto_generated_endpoints.rs
+1 -15
View File
@@ -1,22 +1,8 @@
// Auto-generated MCP tools from OpenAPI specification
// This file is generated by generate_mcp_tools.py - DO NOT EDIT MANUALLY
use std::borrow::Cow;
use serde::{Deserialize, Serialize};
#[derive(Debug, Clone, Deserialize, Serialize)]
pub struct EndpointTool {
pub name: Cow<'static, str>,
pub description: Cow<'static, str>,
pub instructions: Cow<'static, str>,
pub path: Cow<'static, str>,
pub method: Cow<'static, str>,
pub path_params_schema: Option<serde_json::Value>,
pub query_params_schema: Option<serde_json::Value>,
pub body_schema: Option<serde_json::Value>,
}
use windmill_mcp::server::EndpointTool;
pub fn all_tools() -> Vec<EndpointTool> {
vec![
backend/windmill-api/src/mcp/core.rs
+448
View File
@@ -0,0 +1,448 @@
//! Windmill MCP Backend implementation
//!
//! This module provides the concrete implementation of the McpBackend trait
//! for the Windmill platform.
use async_trait::async_trait;
use serde_json::Value;
use std::collections::HashMap;
use windmill_common::{db::UserDB, utils::StripPath, DB};
use windmill_mcp::common::transform::apply_key_transformation;
use windmill_mcp::common::types::{
FlowInfo, HubScriptInfo, ResourceInfo, ResourceType, SchemaType, ScriptInfo,
};
use windmill_mcp::server::{BackendResult, EndpointTool, ErrorData, McpAuth, McpBackend};
use crate::db::ApiAuthed;
use crate::jobs::{
run_wait_result_flow_by_path_internal, run_wait_result_script_by_path_internal, RunJobQuery,
};
use super::auto_generated_endpoints::all_tools;
use super::utils::{
build_query_string, build_request_body, create_http_request, get_hub_script_schema,
get_item_schema, get_items, get_resources, get_resources_types, get_scripts_from_hub,
parse_response_body, prepare_push_args, substitute_path_params,
};
use std::sync::Arc;
use std::time::Duration;
use tokio_util::sync::CancellationToken;
use windmill_mcp::server::{
LocalSessionManager, Runner, StreamableHttpServerConfig, StreamableHttpService,
};
use windmill_mcp::WorkspaceId;
use axum::{
extract::Path, http::Request, middleware::Next, response::Response, routing::get, Json, Router,
};
use windmill_common::error::JsonResult;
/// Implement McpAuth for ApiAuthed
impl McpAuth for ApiAuthed {
fn username(&self) -> &str {
&self.username
}
fn email(&self) -> &str {
&self.email
}
fn is_admin(&self) -> bool {
self.is_admin
}
fn is_operator(&self) -> bool {
self.is_operator
}
fn groups(&self) -> &[String] {
&self.groups
}
fn folders(&self) -> &[(String, bool, bool)] {
&self.folders
}
fn scopes(&self) -> Option<&[String]> {
self.scopes.as_deref()
}
}
/// Windmill's MCP backend implementation
#[derive(Clone)]
pub struct WindmillBackend {
pub db: DB,
pub user_db: UserDB,
}
impl WindmillBackend {
pub fn new(db: DB, user_db: UserDB) -> Self {
Self { db, user_db }
}
}
#[async_trait]
impl McpBackend for WindmillBackend {
type Auth = ApiAuthed;
async fn list_scripts(
&self,
auth: &ApiAuthed,
workspace_id: &str,
favorites_only: bool,
) -> BackendResult<Vec<ScriptInfo>> {
let scope_type = if favorites_only { "favorites" } else { "all" };
get_items::<ScriptInfo>(&self.user_db, auth, workspace_id, scope_type, "script")
.await
.map_err(|e| ErrorData::internal_error(e.message, None))
}
async fn list_flows(
&self,
auth: &ApiAuthed,
workspace_id: &str,
favorites_only: bool,
) -> BackendResult<Vec<FlowInfo>> {
let scope_type = if favorites_only { "favorites" } else { "all" };
get_items::<FlowInfo>(&self.user_db, auth, workspace_id, scope_type, "flow")
.await
.map_err(|e| ErrorData::internal_error(e.message, None))
}
async fn list_resource_types(
&self,
auth: &ApiAuthed,
workspace_id: &str,
) -> BackendResult<Vec<ResourceType>> {
get_resources_types(&self.user_db, auth, workspace_id)
.await
.map_err(|e| ErrorData::internal_error(e.message, None))
}
async fn list_resources(
&self,
auth: &ApiAuthed,
workspace_id: &str,
resource_type: &str,
) -> BackendResult<Vec<ResourceInfo>> {
get_resources(&self.user_db, auth, workspace_id, resource_type)
.await
.map_err(|e| ErrorData::internal_error(e.message, None))
}
async fn list_hub_scripts(
&self,
app_filter: Option<&str>,
) -> BackendResult<Vec<HubScriptInfo>> {
get_scripts_from_hub(&self.db, app_filter)
.await
.map_err(|e| ErrorData::internal_error(e.message, None))
}
async fn get_item_schema(
&self,
auth: &ApiAuthed,
workspace_id: &str,
path: &str,
item_type: &str,
) -> BackendResult<Option<SchemaType>> {
let schema = get_item_schema(path, &self.user_db, auth, workspace_id, item_type)
.await
.map_err(|e| ErrorData::internal_error(e.message, None))?;
if let Some(ref s) = schema {
match serde_json::from_str::<SchemaType>(s.0.get()) {
Ok(val) => Ok(Some(val)),
Err(e) => {
tracing::warn!("Failed to parse schema: {}", e);
Ok(None)
}
}
} else {
Ok(None)
}
}
async fn get_hub_script_schema(&self, path: &str) -> BackendResult<Option<SchemaType>> {
let schema = get_hub_script_schema(path, &self.db)
.await
.map_err(|e| ErrorData::internal_error(e.message, None))?;
if let Some(ref s) = schema {
match serde_json::from_str::<SchemaType>(s.0.get()) {
Ok(val) => Ok(Some(val)),
Err(e) => {
tracing::warn!("Failed to parse hub schema: {}", e);
Ok(None)
}
}
} else {
Ok(None)
}
}
fn transform_schema_for_resources(
&self,
schema: &SchemaType,
resources_cache: &HashMap<String, Vec<ResourceInfo>>,
resources_types: &[ResourceType],
) -> SchemaType {
let mut schema_obj = schema.clone();
// Replace invalid char in property key with underscore
let replacements: Vec<(String, String, Value)> = schema_obj
.properties
.iter()
.filter_map(|(key, value)| {
if key.chars().any(|c| !c.is_alphanumeric() && c != '_') {
let new_key = apply_key_transformation(key);
Some((key.clone(), new_key, value.clone()))
} else {
None
}
})
.collect();
for (old_key, new_key, value) in replacements {
schema_obj.properties.remove(&old_key);
schema_obj.properties.insert(new_key, value);
}
for (_key, prop_value) in schema_obj.properties.iter_mut() {
if let Value::Object(prop_map) = prop_value {
if let Some(format_value) = prop_map.get("format") {
if let Value::String(format_str) = format_value {
if format_str.starts_with("resource-") {
let resource_type_key =
format_str.split("-").last().unwrap_or_default().to_string();
let resource_type = resources_types
.iter()
.find(|rt| rt.name == resource_type_key);
let resource_type_obj = resource_type.cloned();
if let Some(resource_cache) = resources_cache.get(&resource_type_key) {
let resources_count = resource_cache.len();
let description = match resource_type_obj {
Some(resource_type_obj) => format!(
"This is a resource named `{}` with the following description: `{}`.\\nThe path of the resource should be used to specify the resource.\\n{}",
resource_type_obj.name,
resource_type_obj.description.as_deref().unwrap_or("No description"),
if resources_count == 0 {
"This resource does not have any available instances, you should create one from your windmill workspace."
} else if resources_count > 1 {
"This resource has multiple available instances, you should precisely select the one you want to use."
} else {
"There is 1 resource available."
}
),
None => "An object parameter.".to_string(),
};
prop_map.insert(
"type".to_string(),
Value::String("string".to_string()),
);
prop_map
.insert("description".to_string(), Value::String(description));
if resources_count > 0 {
let resources_description = resource_cache
.iter()
.map(|resource| {
format!(
"{}: $res:{}",
resource
.description
.as_deref()
.unwrap_or("No title"),
resource.path
)
})
.collect::<Vec<String>>()
.join("\\n");
prop_map.insert(
"description".to_string(),
Value::String(format!(
"{}\\nHere are the available resources, in the format title:path. Title can be empty. Path should be used to specify the resource:\\n{}",
prop_map.get("description").unwrap_or(&Value::String("No description".to_string())),
resources_description
)),
);
}
}
}
}
}
}
}
schema_obj
}
async fn run_script(
&self,
auth: &ApiAuthed,
workspace_id: &str,
path: &str,
args: Value,
) -> BackendResult<Value> {
let push_args = prepare_push_args(args);
let result = run_wait_result_script_by_path_internal(
self.db.clone(),
RunJobQuery::default(),
StripPath(path.to_string()),
auth.clone(),
self.user_db.clone(),
workspace_id.to_string(),
push_args,
)
.await
.map_err(|e| ErrorData::internal_error(e.to_string(), None))?;
parse_response_body(result).await
}
async fn run_flow(
&self,
auth: &ApiAuthed,
workspace_id: &str,
path: &str,
args: Value,
) -> BackendResult<Value> {
let push_args = prepare_push_args(args);
let result = run_wait_result_flow_by_path_internal(
self.db.clone(),
RunJobQuery::default(),
StripPath(path.to_string()),
auth.clone(),
self.user_db.clone(),
push_args,
workspace_id.to_string(),
)
.await
.map_err(|e| ErrorData::internal_error(e.to_string(), None))?;
parse_response_body(result).await
}
async fn call_endpoint(
&self,
auth: &ApiAuthed,
workspace_id: &str,
endpoint_tool: &EndpointTool,
args: Value,
) -> BackendResult<Value> {
let args_map = match &args {
Value::Object(map) => map,
_ => {
return Err(ErrorData::invalid_params(
"Arguments must be an object",
None,
));
}
};
// Build URL with path substitutions
let path_template = substitute_path_params(
&endpoint_tool.path,
workspace_id,
args_map,
&endpoint_tool.path_params_schema,
)?;
let query_string = build_query_string(args_map, &endpoint_tool.query_params_schema);
let full_url = format!(
"{}/api{}{}",
windmill_common::BASE_INTERNAL_URL.as_str(),
path_template,
query_string
);
// Prepare request body
let body_json =
build_request_body(&endpoint_tool.method, args_map, &endpoint_tool.body_schema);
// Create and execute request
let response = create_http_request(
&endpoint_tool.method,
&full_url,
workspace_id,
auth,
body_json,
)
.await?;
let status = response.status();
let response_text = response.text().await.map_err(|e| {
ErrorData::internal_error(format!("Failed to read response text: {}", e), None)
})?;
if status.is_success() {
Ok(serde_json::from_str(&response_text)
.unwrap_or_else(|_| Value::String(response_text)))
} else {
Err(ErrorData::internal_error(
format!(
"HTTP {} {}: {}",
status.as_u16(),
status.canonical_reason().unwrap_or(""),
response_text
),
None,
))
}
}
fn all_endpoint_tools(&self) -> Vec<EndpointTool> {
all_tools()
}
}
/// Extract workspace ID from path and store it in request extensions
pub async fn extract_and_store_workspace_id(
Path(params): Path<String>,
mut request: Request<axum::body::Body>,
next: Next,
) -> Response {
let workspace_id = params;
request.extensions_mut().insert(WorkspaceId(workspace_id));
next.run(request).await
}
/// Setup the MCP server with HTTP transport
pub async fn setup_mcp_server(
db: DB,
user_db: UserDB,
) -> anyhow::Result<(Router, CancellationToken)> {
let cancellation_token = CancellationToken::new();
let session_manager = Arc::new(LocalSessionManager::default());
let backend = WindmillBackend::new(db, user_db);
let runner = Runner::new(backend);
let service_config = StreamableHttpServerConfig {
sse_keep_alive: Some(Duration::from_secs(15)),
stateful_mode: false,
cancellation_token: cancellation_token.clone(),
sse_retry: Some(Duration::from_secs(15)),
};
let service =
StreamableHttpService::new(move || Ok(runner.clone()), session_manager, service_config);
let router = Router::new().nest_service("/", service);
Ok((router, cancellation_token))
}
/// HTTP handler to list MCP tools as JSON
async fn list_mcp_tools_handler() -> JsonResult<Vec<EndpointTool>> {
let endpoint_tools = all_tools();
Ok(Json(endpoint_tools))
}
/// Creates a router service for listing MCP tools
pub fn list_tools_service() -> Router {
Router::new().route("/", get(list_mcp_tools_handler))
}
backend/windmill-api/src/mcp/mod.rs
+5 -5
View File
@@ -3,9 +3,9 @@
//! This module provides the MCP server implementation that exposes Windmill scripts,
//! flows, and API endpoints as MCP tools for AI assistants to interact with.
pub mod server;
pub mod tools;
pub mod utils;
mod auto_generated_endpoints;
mod core;
mod utils;
// Re-export main components
pub use server::{extract_and_store_workspace_id, list_tools_service, setup_mcp_server};
// Re-export only what's needed externally
pub use core::{extract_and_store_workspace_id, list_tools_service, setup_mcp_server};
backend/windmill-api/src/mcp/server.rs
-561
View File
@@ -1,561 +0,0 @@
//! MCP Server implementation
//!
//! Contains the core MCP server handler that implements the Model Context Protocol
//! specification. This is a thin orchestration layer that delegates to the appropriate
//! modules for tool management, database operations, and schema transformation.
use std::collections::HashMap;
use std::sync::Arc;
use std::{borrow::Cow, time::Duration};
use axum::body::to_bytes;
use serde_json::Value;
use tokio::try_join;
use tokio_util::sync::CancellationToken;
use windmill_common::db::UserDB;
use windmill_common::worker::to_raw_value;
use windmill_common::{utils::StripPath, DB};
use windmill_mcp::server::{
Annotated, CallToolRequestParam, CallToolResult, Content, ErrorData, Implementation,
InitializeRequestParam, InitializeResult, ListPromptsResult, ListResourceTemplatesResult,
ListResourcesResult, ListToolsResult, LocalSessionManager, PaginatedRequestParam,
ProtocolVersion, RawContent, RawTextContent, RequestContext, RoleServer, ServerCapabilities,
ServerHandler, ServerInfo, StreamableHttpServerConfig, StreamableHttpService, Tool,
ToolAnnotations,
};
use crate::db::ApiAuthed;
use crate::jobs::{
run_wait_result_flow_by_path_internal, run_wait_result_script_by_path_internal, RunJobQuery,
};
use super::tools::endpoint_tools::{
all_endpoint_tools, call_endpoint_tool, endpoint_tools_to_mcp_tools, EndpointTool,
};
use super::utils::{
database::{
check_scopes, get_hub_script_schema, get_item_schema, get_items, get_resources_types,
get_scripts_from_hub,
},
models::{
FlowInfo, ResourceInfo, ResourceType, SchemaType, ScriptInfo, ToolableItem, WorkspaceId,
},
schema::transform_schema_for_resources,
scope_matcher::{is_resource_allowed, parse_mcp_scopes},
transform::{reverse_transform, reverse_transform_key},
};
use axum::{
extract::Path, http::Request, middleware::Next, response::Response, routing::get, Json, Router,
};
use windmill_common::error::JsonResult;
/// MCP Server Runner - implements the core MCP protocol handlers
#[derive(Clone)]
pub struct Runner {}
impl Runner {
pub fn new() -> Self {
Self {}
}
/// Creates a Tool from a ToolableItem
async fn create_tool_from_item<T: ToolableItem>(
item: &T,
user_db: &UserDB,
authed: &ApiAuthed,
workspace_id: &str,
resources_cache: &mut HashMap<String, Vec<ResourceInfo>>,
resources_types: &Vec<ResourceType>,
) -> Result<Tool, ErrorData> {
let is_hub = item.is_hub();
let path = item.get_path_or_id();
let item_type = item.item_type();
let description = format!(
"This is a {} named `{}` with the following description: `{}`.{}",
item_type,
item.get_summary(),
item.get_description(),
if is_hub {
format!(
" It is a tool used for the following app: {}",
item.get_integration_type()
.unwrap_or("No integration type".to_string())
)
} else {
"".to_string()
}
);
let schema_obj = transform_schema_for_resources(
&item.get_schema(),
user_db,
authed,
&workspace_id,
resources_cache,
&resources_types,
)
.await?;
let input_schema_map = match serde_json::to_value(schema_obj) {
Ok(Value::Object(map)) => map,
Ok(_) => {
tracing::warn!("Schema object for tool '{}' did not serialize to a JSON object, using empty schema.", path);
serde_json::Map::new()
}
Err(e) => {
tracing::error!(
"Failed to serialize schema object for tool '{}': {}. Using empty schema.",
path,
e
);
serde_json::Map::new()
}
};
Ok(Tool {
name: Cow::Owned(path),
description: Some(Cow::Owned(description)),
input_schema: Arc::new(input_schema_map),
title: Some(item.get_summary().to_string()),
output_schema: None,
icons: None,
annotations: Some(ToolAnnotations {
title: Some(item.get_summary().to_string()),
read_only_hint: Some(false), // Can modify environment
destructive_hint: Some(true), // Can potentially be destructive
idempotent_hint: Some(false), // Are not guaranteed to be idempotent
open_world_hint: Some(true), // Can interact with external services
}),
meta: None,
})
}
}
impl ServerHandler for Runner {
/// Handles the `CallTool` request from the MCP client
async fn call_tool(
&self,
request: CallToolRequestParam,
context: RequestContext<RoleServer>,
) -> Result<CallToolResult, ErrorData> {
let http_parts = context
.extensions
.get::<axum::http::request::Parts>()
.ok_or_else(|| {
tracing::error!("http::request::Parts not found");
ErrorData::internal_error("http::request::Parts not found", None)
})?;
let authed = http_parts.extensions.get::<ApiAuthed>().ok_or_else(|| {
tracing::error!("ApiAuthed Axum extension not found");
ErrorData::internal_error("ApiAuthed Axum extension not found", None)
})?;
check_scopes(authed)?;
// Parse MCP scopes for authorization
let scopes = authed.scopes.as_ref().map(|s| s.as_slice()).unwrap_or(&[]);
let scope_config = parse_mcp_scopes(scopes)?;
if request.name.ends_with("_TRUNC") {
return Ok(CallToolResult::error(
vec![
Annotated::new(
RawContent::Text(RawTextContent {
text:
"Tool path is too long. Consider shortening it to make it compatible with MCP."
.to_string(),
meta: None,
}),
None
),
]
));
}
let db = http_parts.extensions.get::<DB>().ok_or_else(|| {
tracing::error!("DB Axum extension not found");
ErrorData::internal_error("DB Axum extension not found", None)
})?;
let user_db = http_parts.extensions.get::<UserDB>().ok_or_else(|| {
tracing::error!("UserDB Axum extension not found");
ErrorData::internal_error("UserDB Axum extension not found", None)
})?;
let args = request.arguments.map(Value::Object).ok_or_else(|| {
ErrorData::invalid_params(
"Missing arguments for tool",
Some(request.name.clone().into()),
)
})?;
let workspace_id = http_parts
.extensions
.get::<WorkspaceId>()
.ok_or_else(|| {
tracing::error!("WorkspaceId not found");
ErrorData::internal_error("WorkspaceId not found", None)
})
.map(|w_id| w_id.0.clone())?;
// Check if this is a generated endpoint tool
let endpoint_tools = all_endpoint_tools();
for endpoint_tool in endpoint_tools {
if endpoint_tool.name.as_ref() == request.name {
// Validate endpoint scope
if scope_config.granular
&& !is_resource_allowed(&endpoint_tool.name, &scope_config.endpoints)
{
return Err(ErrorData::internal_error(
format!(
"Access denied: endpoint '{}' not in token scope",
endpoint_tool.name
),
None,
));
}
// This is an endpoint tool, forward to the actual HTTP endpoint
let result =
call_endpoint_tool(&endpoint_tool, args.clone(), &workspace_id, &authed)
.await?;
return Ok(CallToolResult::success(vec![Content::text(
serde_json::to_string_pretty(&result).unwrap_or_else(|_| "{}".to_string()),
)]));
}
}
// Continue with script/flow logic
let (tool_type, path, is_hub) = reverse_transform(&request.name).map_err(|e| {
ErrorData::internal_error(format!("Failed to reverse transform path: {}", e), None)
})?;
// Validate script/flow scope
if !is_hub && scope_config.granular {
if tool_type == "script" && !is_resource_allowed(&path, &scope_config.scripts) {
return Err(ErrorData::internal_error(
format!("Access denied: script '{}' not in token scope", path),
None,
));
} else if tool_type == "flow" && !is_resource_allowed(&path, &scope_config.flows) {
return Err(ErrorData::internal_error(
format!("Access denied: flow '{}' not in token scope", path),
None,
));
}
}
let item_schema = if is_hub {
get_hub_script_schema(&format!("hub/{}", path), db).await?
} else {
get_item_schema(&path, user_db, authed, &workspace_id, &tool_type).await?
};
let schema_obj = if let Some(ref s) = item_schema {
match serde_json::from_str::<SchemaType>(s.0.get()) {
Ok(val) => Some(val),
Err(e) => {
tracing::warn!("Failed to parse schema: {}", e);
None
}
}
} else {
None
};
let push_args = if let Value::Object(map) = args.clone() {
let mut args_hash = HashMap::new();
for (k, v) in map {
// need to transform back the key without invalid characters to the original key
let original_key = reverse_transform_key(&k, &schema_obj);
args_hash.insert(original_key, to_raw_value(&v));
}
windmill_queue::PushArgsOwned { extra: None, args: args_hash }
} else {
windmill_queue::PushArgsOwned::default()
};
let script_or_flow_path = if is_hub {
StripPath(format!("hub/{}", path))
} else {
StripPath(path)
};
let run_query = RunJobQuery::default();
let result = if tool_type == "script" {
run_wait_result_script_by_path_internal(
db.clone(),
run_query,
script_or_flow_path,
authed.clone(),
user_db.clone(),
workspace_id.clone(),
push_args,
)
.await
} else {
run_wait_result_flow_by_path_internal(
db.clone(),
run_query,
script_or_flow_path,
authed.clone(),
user_db.clone(),
push_args,
workspace_id.clone(),
)
.await
};
match result {
Ok(response) => {
let body_bytes = to_bytes(response.into_body(), usize::MAX)
.await
.map_err(|e| {
ErrorData::internal_error(
format!("Failed to read response body: {}", e),
None,
)
})?;
let body_str = String::from_utf8(body_bytes.to_vec()).map_err(|e| {
ErrorData::internal_error(
format!("Failed to decode response body: {}", e),
None,
)
})?;
Ok(CallToolResult::success(vec![Content::text(body_str)]))
}
Err(e) => Err(ErrorData::internal_error(
format!("Failed to run script: {}", e),
None,
)),
}
}
/// Fetches available tools (scripts, flows, hub scripts) based on the user's scope
async fn list_tools(
&self,
_request: Option<PaginatedRequestParam>,
mut _context: RequestContext<RoleServer>,
) -> Result<ListToolsResult, ErrorData> {
let http_parts = _context
.extensions
.get::<axum::http::request::Parts>()
.ok_or_else(|| {
tracing::error!("http::request::Parts not found");
ErrorData::internal_error("http::request::Parts not found", None)
})?;
let authed = http_parts.extensions.get::<ApiAuthed>().ok_or_else(|| {
tracing::error!("ApiAuthed Axum extension not found");
ErrorData::internal_error("ApiAuthed Axum extension not found", None)
})?;
check_scopes(authed)?;
let db = http_parts.extensions.get::<DB>().ok_or_else(|| {
tracing::error!("DB Axum extension not found");
ErrorData::internal_error("DB Axum extension not found", None)
})?;
let user_db = http_parts.extensions.get::<UserDB>().ok_or_else(|| {
tracing::error!("UserDB Axum extension not found");
ErrorData::internal_error("UserDB Axum extension not found", None)
})?;
let workspace_id = http_parts
.extensions
.get::<WorkspaceId>()
.ok_or_else(|| {
tracing::error!("WorkspaceId not found");
ErrorData::internal_error("WorkspaceId not found", None)
})
.map(|w_id| w_id.0.clone())?;
// Parse MCP scopes to determine what to expose
let scopes = authed.scopes.as_ref().map(|s| s.as_slice()).unwrap_or(&[]);
let scope_config = parse_mcp_scopes(scopes)?;
let scope_type = if scope_config.favorites {
"favorites"
} else {
// Fetch all items if either all or granular scope set (we filter later for granular scopes)
"all"
};
let scripts_fn =
get_items::<ScriptInfo>(user_db, authed, &workspace_id, scope_type, "script");
let flows_fn = get_items::<FlowInfo>(user_db, authed, &workspace_id, scope_type, "flow");
let resources_types_fn = get_resources_types(user_db, authed, &workspace_id);
let hub_scripts_fn = get_scripts_from_hub(db, scope_config.hub_apps.as_deref());
let (scripts, flows, resources_types, hub_scripts) = if scope_config.hub_apps.is_some() {
let (scripts, flows, resources_types, hub_scripts) =
try_join!(scripts_fn, flows_fn, resources_types_fn, hub_scripts_fn)?;
(scripts, flows, resources_types, hub_scripts)
} else {
let (scripts, flows, resources_types) =
try_join!(scripts_fn, flows_fn, resources_types_fn)?;
(scripts, flows, resources_types, vec![])
};
let mut resources_cache: HashMap<String, Vec<ResourceInfo>> = HashMap::new();
let mut tools: Vec<Tool> = Vec::new();
// Filter and add scripts based on scope
for script in scripts {
// For granular scopes, filter by path
if scope_config.granular && !is_resource_allowed(&script.path, &scope_config.scripts) {
continue;
}
tools.push(
Runner::create_tool_from_item(
&script,
user_db,
authed,
&workspace_id,
&mut resources_cache,
&resources_types,
)
.await?,
);
}
// Filter and add flows based on scope
for flow in flows {
// For granular scopes, filter by path
if scope_config.granular && !is_resource_allowed(&flow.path, &scope_config.flows) {
continue;
}
tools.push(
Runner::create_tool_from_item(
&flow,
user_db,
authed,
&workspace_id,
&mut resources_cache,
&resources_types,
)
.await?,
);
}
for hub_script in hub_scripts {
tools.push(
Runner::create_tool_from_item(
&hub_script,
user_db,
authed,
&workspace_id,
&mut resources_cache,
&resources_types,
)
.await?,
);
}
// Add endpoint tools from the generated MCP tools, filtered by scope
let endpoint_tools = all_endpoint_tools();
for endpoint_tool in endpoint_tools {
// For granular scopes, filter by endpoint name
if scope_config.granular
&& !is_resource_allowed(&endpoint_tool.name, &scope_config.endpoints)
{
continue;
}
tools.push(
endpoint_tools_to_mcp_tools(vec![endpoint_tool])
.into_iter()
.next()
.unwrap(),
);
}
Ok(ListToolsResult { tools, next_cursor: None, meta: None })
}
fn get_info(&self) -> ServerInfo {
ServerInfo {
protocol_version: ProtocolVersion::default(),
capabilities: ServerCapabilities::builder()
.enable_tools()
.build(),
server_info: Implementation::from_build_env(),
instructions: Some("This server provides a list of scripts and flows the user can run on Windmill. Each flow and script is a tool callable with their respective arguments.".to_string()),
}
}
async fn initialize(
&self,
_request: InitializeRequestParam,
_context: RequestContext<RoleServer>,
) -> Result<InitializeResult, ErrorData> {
Ok(self.get_info())
}
async fn list_resources(
&self,
_request: Option<PaginatedRequestParam>,
_context: RequestContext<RoleServer>,
) -> Result<ListResourcesResult, ErrorData> {
Ok(ListResourcesResult { resources: vec![], next_cursor: None, meta: None })
}
async fn list_prompts(
&self,
_request: Option<PaginatedRequestParam>,
_context: RequestContext<RoleServer>,
) -> Result<ListPromptsResult, ErrorData> {
Ok(ListPromptsResult::default())
}
async fn list_resource_templates(
&self,
_request: Option<PaginatedRequestParam>,
_context: RequestContext<RoleServer>,
) -> Result<ListResourceTemplatesResult, ErrorData> {
Ok(ListResourceTemplatesResult::default())
}
}
/// Extract workspace ID from path and store it in request extensions
pub async fn extract_and_store_workspace_id(
Path(params): Path<String>,
mut request: Request<axum::body::Body>,
next: Next,
) -> Response {
let workspace_id = params;
request.extensions_mut().insert(WorkspaceId(workspace_id));
next.run(request).await
}
/// Setup the MCP server with HTTP transport
pub async fn setup_mcp_server() -> anyhow::Result<(Router, CancellationToken)> {
let cancellation_token = CancellationToken::new();
let session_manager = Arc::new(LocalSessionManager::default());
let service_config = StreamableHttpServerConfig {
sse_keep_alive: Some(Duration::from_secs(15)),
stateful_mode: false,
cancellation_token: cancellation_token.clone(),
sse_retry: Some(Duration::from_secs(15)),
};
let service = StreamableHttpService::new(
|| Ok(Runner::new()),
session_manager.clone(),
service_config,
);
let router = axum::Router::new().nest_service("/", service);
Ok((router, cancellation_token))
}
/// HTTP handler to list MCP tools as JSON
async fn list_mcp_tools_handler() -> JsonResult<Vec<EndpointTool>> {
let endpoint_tools = all_endpoint_tools();
Ok(Json(endpoint_tools))
}
/// Creates a router service for listing MCP tools
pub fn list_tools_service() -> Router {
Router::new().route("/", get(list_mcp_tools_handler))
}
backend/windmill-api/src/mcp/tools/endpoint_tools.rs
-307
View File
@@ -1,307 +0,0 @@
//! Endpoint tools for MCP server
//!
//! Contains the auto-generated endpoint tools and utilities for converting
//! them to MCP tools and handling HTTP calls to Windmill API endpoints.
use crate::db::ApiAuthed;
use std::sync::Arc;
use windmill_common::db::Authed;
use windmill_common::{auth::create_jwt_token, BASE_INTERNAL_URL};
use windmill_mcp::server::{ErrorData, Tool, ToolAnnotations};
// Import the auto-generated tools
use super::auto_generated_endpoints;
pub use auto_generated_endpoints::{all_tools, EndpointTool};
/// Get all available endpoint tools
pub fn all_endpoint_tools() -> Vec<EndpointTool> {
all_tools()
}
/// Convert endpoint tools to MCP tools
pub fn endpoint_tools_to_mcp_tools(endpoint_tools: Vec<EndpointTool>) -> Vec<Tool> {
endpoint_tools
.into_iter()
.map(|tool| endpoint_tool_to_mcp_tool(&tool))
.collect()
}
/// Convert a single endpoint tool to MCP tool
pub fn endpoint_tool_to_mcp_tool(tool: &EndpointTool) -> Tool {
let mut combined_properties = serde_json::Map::new();
let mut combined_required = Vec::new();
// Combine all parameter schemas
let schemas = [
&tool.path_params_schema,
&tool.query_params_schema,
&tool.body_schema,
];
for schema in schemas.iter().filter_map(|s| s.as_ref()) {
merge_schema_into(&mut combined_properties, &mut combined_required, schema);
}
let combined_schema = serde_json::json!({
"type": "object",
"properties": combined_properties,
"required": combined_required
});
let description = format!("{}. {}", tool.description, tool.instructions);
// Create annotations based on HTTP method and endpoint characteristics
let annotations = create_endpoint_annotations(tool);
Tool {
name: tool.name.clone(),
description: Some(description.into()),
input_schema: Arc::new(combined_schema.as_object().unwrap().clone()),
title: Some(tool.name.to_string()),
output_schema: None,
icons: None,
annotations: Some(annotations),
meta: None,
}
}
/// Create appropriate annotations for endpoint tools based on HTTP method
fn create_endpoint_annotations(tool: &EndpointTool) -> ToolAnnotations {
let method = tool.method.as_ref();
// Determine characteristics based on HTTP method
let (read_only, destructive, idempotent, open_world) = match method {
"GET" => (true, false, true, true), // Read-only, safe, idempotent
"POST" => (false, true, false, true), // Can modify, potentially destructive, not idempotent
"PUT" => (false, false, true, true), // Can modify, typically idempotent updates
"DELETE" => (false, true, true, true), // Destructive but idempotent
"PATCH" => (false, false, false, true), // Partial updates, not guaranteed idempotent
_ => (false, true, false, true), // Default: assume can modify and be destructive
};
ToolAnnotations {
title: Some(format!("{} {}", method, tool.path)),
read_only_hint: Some(read_only),
destructive_hint: Some(destructive),
idempotent_hint: Some(idempotent),
open_world_hint: Some(open_world),
}
}
/// Merge schema into combined properties and required fields
fn merge_schema_into(
combined_properties: &mut serde_json::Map<String, serde_json::Value>,
combined_required: &mut Vec<String>,
schema: &serde_json::Value,
) {
if let Some(props) = schema.get("properties").and_then(|p| p.as_object()) {
for (key, value) in props {
combined_properties.insert(key.clone(), value.clone());
}
}
if let Some(required) = schema.get("required").and_then(|r| r.as_array()) {
for req in required.iter().filter_map(|r| r.as_str()) {
combined_required.push(req.to_string());
}
}
}
/// Call an endpoint tool by making HTTP request to Windmill API
pub async fn call_endpoint_tool(
tool: &EndpointTool,
args: serde_json::Value,
workspace_id: &str,
api_authed: &ApiAuthed,
) -> Result<serde_json::Value, ErrorData> {
let args_map = match &args {
serde_json::Value::Object(map) => map,
_ => {
return Err(ErrorData::invalid_params(
"Arguments must be an object",
Some(tool.name.clone().into()),
))
}
};
// Build URL with path substitutions
let path_template =
substitute_path_params(&tool.path, workspace_id, args_map, &tool.path_params_schema)?;
let query_string = build_query_string(args_map, &tool.query_params_schema);
let full_url = format!(
"{}/api{}{}",
BASE_INTERNAL_URL.as_str(),
path_template,
query_string
);
// Prepare request body
let body_json = build_request_body(&tool.method, args_map, &tool.body_schema);
// Create and execute request
let response =
create_http_request(&tool.method, &full_url, workspace_id, api_authed, body_json).await?;
let status = response.status();
let response_text = response.text().await.map_err(|e| {
ErrorData::internal_error(format!("Failed to read response text: {}", e), None)
})?;
if status.is_success() {
Ok(serde_json::from_str(&response_text)
.unwrap_or_else(|_| serde_json::Value::String(response_text)))
} else {
Err(ErrorData::internal_error(
format!(
"HTTP {} {}: {}",
status.as_u16(),
status.canonical_reason().unwrap_or(""),
response_text
),
None,
))
}
}
/// Substitute path parameters in the URL template
fn substitute_path_params(
path: &str,
workspace_id: &str,
args_map: &serde_json::Map<String, serde_json::Value>,
path_schema: &Option<serde_json::Value>,
) -> Result<String, ErrorData> {
let mut path_template = path.replace("{workspace}", workspace_id);
if let Some(schema) = path_schema {
if let Some(props) = schema.get("properties").and_then(|p| p.as_object()) {
for (param_name, _) in props {
let placeholder = format!("{{{}}}", param_name);
match args_map.get(param_name) {
Some(param_value) => {
if let Some(str_val) = param_value.as_str() {
path_template = path_template.replace(&placeholder, str_val);
}
}
None => {
tracing::warn!("Missing required path parameter: {}", param_name);
return Err(ErrorData::invalid_params(
format!("Missing required path parameter: {}", param_name),
None,
));
}
}
}
}
}
Ok(path_template)
}
/// Build query string from arguments
fn build_query_string(
args_map: &serde_json::Map<String, serde_json::Value>,
query_schema: &Option<serde_json::Value>,
) -> String {
let Some(schema) = query_schema else {
return String::new();
};
let Some(props) = schema.get("properties").and_then(|p| p.as_object()) else {
return String::new();
};
let query_params: Vec<String> = props
.keys()
.filter_map(|param_name| {
args_map
.get(param_name)
.filter(|v| !v.is_null())
.map(|value| {
let value_str = value.to_string();
let str_val = value_str.trim_matches('"');
format!(
"{}={}",
urlencoding::encode(param_name),
urlencoding::encode(str_val)
)
})
})
.collect();
if query_params.is_empty() {
String::new()
} else {
format!("?{}", query_params.join("&"))
}
}
/// Build request body from arguments
fn build_request_body(
method: &str,
args_map: &serde_json::Map<String, serde_json::Value>,
body_schema: &Option<serde_json::Value>,
) -> Option<serde_json::Value> {
if method == "GET" {
return None;
}
let schema = body_schema.as_ref()?;
let props = schema.get("properties")?.as_object()?;
let body_map: serde_json::Map<String, serde_json::Value> = props
.keys()
.filter_map(|param_name| {
args_map
.get(param_name)
.map(|value| (param_name.clone(), value.clone()))
})
.collect();
if body_map.is_empty() {
None
} else {
Some(serde_json::Value::Object(body_map))
}
}
/// Create HTTP request with authentication
async fn create_http_request(
method: &str,
url: &str,
workspace_id: &str,
api_authed: &ApiAuthed,
body_json: Option<serde_json::Value>,
) -> Result<reqwest::Response, ErrorData> {
let client = &crate::HTTP_CLIENT;
let mut request_builder = match method {
"GET" => client.get(url),
"POST" => client.post(url),
"PUT" => client.put(url),
"DELETE" => client.delete(url),
"PATCH" => client.patch(url),
_ => {
return Err(ErrorData::invalid_params(
format!("Unsupported HTTP method: {}", method),
None,
))
}
};
// Add authorization header
let authed = Authed::from(api_authed.clone());
let token = create_jwt_token(authed, workspace_id, 3600, None, None, None, None)
.await
.map_err(|e| ErrorData::internal_error(e.to_string(), None))?;
request_builder = request_builder.header("Authorization", format!("Bearer {}", token));
// Add body if present
if let Some(body) = body_json {
request_builder = request_builder
.header("Content-Type", "application/json")
.json(&body);
}
request_builder
.send()
.await
.map_err(|e| ErrorData::internal_error(format!("Failed to execute request: {}", e), None))
}
backend/windmill-api/src/mcp/tools/flow_tools.rs
-40
View File
@@ -1,40 +0,0 @@
//! Flow tools for MCP server
//!
//! Contains functionality for converting Windmill flows into MCP tools.
use super::super::utils::{
models::{FlowInfo, ToolableItem, SchemaType},
schema::convert_schema_to_schema_type,
transform::transform_path,
};
/// Implementation of ToolableItem for FlowInfo
impl ToolableItem for FlowInfo {
fn get_path_or_id(&self) -> String {
transform_path(&self.path, "flow")
}
fn get_summary(&self) -> &str {
self.summary.as_deref().unwrap_or("No summary")
}
fn get_description(&self) -> &str {
self.description.as_deref().unwrap_or("No description")
}
fn get_schema(&self) -> SchemaType {
convert_schema_to_schema_type(self.schema.clone())
}
fn is_hub(&self) -> bool {
false
}
fn item_type(&self) -> &'static str {
"flow"
}
fn get_integration_type(&self) -> Option<String> {
None
}
}
backend/windmill-api/src/mcp/tools/hub_tools.rs
-43
View File
@@ -1,43 +0,0 @@
//! Hub tools for MCP server
//!
//! Contains functionality for integrating Windmill Hub scripts as MCP tools.
use super::super::utils::{
models::{HubScriptInfo, ToolableItem, SchemaType},
};
/// Implementation of ToolableItem for HubScriptInfo
impl ToolableItem for HubScriptInfo {
fn get_path_or_id(&self) -> String {
let id = self.version_id;
let summary = self.summary.as_deref().unwrap_or("No summary");
format!("hs-{}-{}", id, summary.replace(" ", "_"))
}
fn get_summary(&self) -> &str {
self.summary.as_deref().unwrap_or("No summary")
}
fn get_description(&self) -> &str {
self.description.as_deref().unwrap_or("No description")
}
fn get_schema(&self) -> SchemaType {
match serde_json::from_value::<SchemaType>(self.schema.clone().unwrap_or_default()) {
Ok(schema_type) => schema_type,
Err(_) => SchemaType::default(),
}
}
fn is_hub(&self) -> bool {
true
}
fn item_type(&self) -> &'static str {
"script"
}
fn get_integration_type(&self) -> Option<String> {
self.app.clone()
}
}
backend/windmill-api/src/mcp/tools/mod.rs
-10
View File
@@ -1,10 +0,0 @@
//! Tool management for MCP server
//!
//! This module handles the conversion of Windmill scripts, flows, and endpoints
//! into MCP tools that can be used by AI assistants.
pub mod script_tools;
pub mod flow_tools;
pub mod hub_tools;
pub mod endpoint_tools;
pub mod auto_generated_endpoints;
backend/windmill-api/src/mcp/tools/script_tools.rs
-40
View File
@@ -1,40 +0,0 @@
//! Script tools for MCP server
//!
//! Contains functionality for converting Windmill scripts into MCP tools.
use super::super::utils::{
models::{ScriptInfo, ToolableItem, SchemaType},
schema::convert_schema_to_schema_type,
transform::transform_path,
};
/// Implementation of ToolableItem for ScriptInfo
impl ToolableItem for ScriptInfo {
fn get_path_or_id(&self) -> String {
transform_path(&self.path, "script")
}
fn get_summary(&self) -> &str {
self.summary.as_deref().unwrap_or("No summary")
}
fn get_description(&self) -> &str {
self.description.as_deref().unwrap_or("No description")
}
fn get_schema(&self) -> SchemaType {
convert_schema_to_schema_type(self.schema.clone())
}
fn is_hub(&self) -> bool {
false
}
fn item_type(&self) -> &'static str {
"script"
}
fn get_integration_type(&self) -> Option<String> {
None
}
}
backend/windmill-api/src/mcp/utils.rs
+414
View File
@@ -0,0 +1,414 @@
//! Utility functions for MCP server
//!
//! Contains database query functions and HTTP request helpers
//! used by the MCP server implementation.
use std::collections::HashMap;
use axum::body::{to_bytes, Body};
use axum::response::Response;
use serde_json::Value;
use sql_builder::prelude::*;
use windmill_common::auth::create_jwt_token;
use windmill_common::db::{Authed, UserDB};
use windmill_common::scripts::{get_full_hub_script_by_path, Schema};
use windmill_common::utils::{query_elems_from_hub, StripPath};
use windmill_common::worker::to_raw_value;
use windmill_common::{DB, HUB_BASE_URL};
use windmill_mcp::server::{BackendResult, ErrorData};
use windmill_mcp::{HubResponse, HubScriptInfo, ItemSchema, ResourceInfo, ResourceType};
use crate::db::ApiAuthed;
use crate::HTTP_CLIENT;
// items max limit
const ITEMS_FETCH_MAX_LIMIT: usize = 100;
// ============================================================================
// Database utilities
// ============================================================================
/// Get the schema for a specific item (script or flow)
pub async fn get_item_schema(
path: &str,
user_db: &UserDB,
authed: &ApiAuthed,
workspace_id: &str,
item_type: &str,
) -> Result<Option<Schema>, ErrorData> {
let mut sqlb = SqlBuilder::select_from(&format!("{} as o", item_type));
sqlb.fields(&["o.schema"]);
sqlb.and_where("o.path = ?".bind(&path));
sqlb.and_where("o.workspace_id = ?".bind(&workspace_id));
sqlb.and_where("o.archived = false");
sqlb.and_where("o.draft_only IS NOT TRUE");
let sql = sqlb.sql().map_err(|e| {
tracing::error!("failed to build sql: {}", e);
ErrorData::internal_error(format!("failed to build sql: {}", e), None)
})?;
let mut tx = user_db.clone().begin(authed).await.map_err(|e| {
tracing::error!("failed to begin transaction: {}", e);
ErrorData::internal_error(format!("failed to begin transaction: {}", e), None)
})?;
let item = sqlx::query_as::<_, ItemSchema>(&sql)
.fetch_one(&mut *tx)
.await
.map_err(|e| {
tracing::error!("failed to fetch item schema: {}", e);
ErrorData::internal_error(format!("failed to fetch item schema: {}", e), None)
})?;
tx.commit().await.map_err(|e| {
tracing::error!("failed to commit transaction: {}", e);
ErrorData::internal_error(format!("failed to commit transaction: {}", e), None)
})?;
Ok(item.schema)
}
/// Get all resource types from the database
pub async fn get_resources_types(
user_db: &UserDB,
authed: &ApiAuthed,
workspace_id: &str,
) -> Result<Vec<ResourceType>, ErrorData> {
let mut sqlb = SqlBuilder::select_from("resource_type as o");
sqlb.fields(&["o.name", "o.description"]);
sqlb.and_where("o.workspace_id = ?".bind(&workspace_id));
let sql = sqlb.sql().map_err(|e| {
tracing::error!("failed to build sql: {}", e);
ErrorData::internal_error(format!("failed to build sql: {}", e), None)
})?;
let mut tx = user_db.clone().begin(authed).await.map_err(|e| {
tracing::error!("failed to begin transaction: {}", e);
ErrorData::internal_error(format!("failed to begin transaction: {}", e), None)
})?;
let rows = sqlx::query_as::<_, ResourceType>(&sql)
.fetch_all(&mut *tx)
.await
.map_err(|e| {
tracing::error!("failed to fetch resource types: {}", e);
ErrorData::internal_error(format!("failed to fetch resource types: {}", e), None)
})?;
tx.commit().await.map_err(|e| {
tracing::error!("failed to commit transaction: {}", e);
ErrorData::internal_error(format!("failed to commit transaction: {}", e), None)
})?;
Ok(rows)
}
/// Get resources by type from the database
pub async fn get_resources(
user_db: &UserDB,
authed: &ApiAuthed,
workspace_id: &str,
resource_type: &str,
) -> Result<Vec<ResourceInfo>, ErrorData> {
let mut sqlb = SqlBuilder::select_from("resource as o");
sqlb.fields(&["o.path", "o.description", "o.resource_type"]);
sqlb.and_where("o.workspace_id = ?".bind(&workspace_id));
sqlb.and_where("o.resource_type = ?".bind(&resource_type));
let sql = sqlb.sql().map_err(|e| {
tracing::error!("failed to build sql: {}", e);
ErrorData::internal_error(format!("failed to build sql: {}", e), None)
})?;
let mut tx = user_db.clone().begin(authed).await.map_err(|e| {
tracing::error!("failed to begin transaction: {}", e);
ErrorData::internal_error(format!("failed to begin transaction: {}", e), None)
})?;
let rows = sqlx::query_as::<_, ResourceInfo>(&sql)
.fetch_all(&mut *tx)
.await
.map_err(|e| {
tracing::error!("failed to fetch resources: {}", e);
ErrorData::internal_error(format!("failed to fetch resources: {}", e), None)
})?;
tx.commit().await.map_err(|e| {
tracing::error!("failed to commit transaction: {}", e);
ErrorData::internal_error(format!("failed to commit transaction: {}", e), None)
})?;
Ok(rows)
}
/// Generic function to get items (scripts or flows) from the database
pub async fn get_items<T: for<'a> sqlx::FromRow<'a, sqlx::postgres::PgRow> + Send + Unpin>(
user_db: &UserDB,
authed: &ApiAuthed,
workspace_id: &str,
scope_type: &str,
item_type: &str,
) -> Result<Vec<T>, ErrorData> {
let mut sqlb = SqlBuilder::select_from(&format!("{} as o", item_type));
let fields = vec!["o.path", "o.summary", "o.description", "o.schema"];
sqlb.fields(&fields);
if scope_type == "favorites" {
sqlb.join("favorite")
.on("favorite.favorite_kind = ? AND favorite.workspace_id = o.workspace_id AND favorite.path = o.path AND favorite.usr = ?".bind(&item_type)
.bind(&authed.username));
}
sqlb.and_where("o.workspace_id = ?".bind(&workspace_id))
.and_where("o.archived = false")
.and_where("o.draft_only IS NOT TRUE");
if item_type == "script" {
sqlb.and_where("(o.no_main_func IS NOT TRUE OR o.no_main_func IS NULL)");
}
sqlb.order_by(
if item_type == "flow" {
"o.edited_at"
} else {
"o.created_at"
},
false,
)
.limit(ITEMS_FETCH_MAX_LIMIT);
let sql = sqlb.sql().map_err(|e| {
tracing::error!("failed to build sql: {}", e);
ErrorData::internal_error(format!("failed to build sql: {}", e), None)
})?;
let mut tx = user_db.clone().begin(authed).await.map_err(|e| {
tracing::error!("failed to begin transaction: {}", e);
ErrorData::internal_error(format!("failed to begin transaction: {}", e), None)
})?;
let rows = sqlx::query_as::<_, T>(&sql)
.fetch_all(&mut *tx)
.await
.map_err(|e| {
tracing::error!("failed to fetch {}: {}", item_type, e);
ErrorData::internal_error(format!("failed to fetch {}: {}", item_type, e), None)
})?;
tx.commit().await.map_err(|e| {
tracing::error!("failed to commit transaction: {}", e);
ErrorData::internal_error(format!("failed to commit transaction: {}", e), None)
})?;
Ok(rows)
}
/// Get scripts from the Hub
pub async fn get_scripts_from_hub(
db: &DB,
scope_integrations: Option<&str>,
) -> Result<Vec<HubScriptInfo>, ErrorData> {
let query_params = Some(vec![
("limit", ITEMS_FETCH_MAX_LIMIT.to_string()),
("with_schema", "true".to_string()),
("apps", scope_integrations.unwrap_or("").to_string()),
]);
let url = format!("{}/scripts/top", *HUB_BASE_URL.read().await);
let (_status_code, _headers, response) =
query_elems_from_hub(&HTTP_CLIENT, &url, query_params, &db)
.await
.map_err(|e| {
tracing::error!("Failed to get items from hub: {}", e);
ErrorData::internal_error(format!("Failed to get items from hub: {}", e), None)
})?;
use axum::body::to_bytes;
let body_bytes = to_bytes(response, usize::MAX).await.map_err(|e| {
tracing::error!("Failed to read response body: {}", e);
ErrorData::internal_error(format!("Failed to read response body: {}", e), None)
})?;
let body_str = String::from_utf8(body_bytes.to_vec()).map_err(|e| {
tracing::error!("Failed to decode response body: {}", e);
ErrorData::internal_error(format!("Failed to decode response body: {}", e), None)
})?;
let hub_response: HubResponse = serde_json::from_str(&body_str).map_err(|e| {
tracing::error!("Failed to parse hub response: {}", e);
ErrorData::internal_error(format!("Failed to parse hub response: {}", e), None)
})?;
Ok(hub_response.asks)
}
/// Get the schema for a Hub script
pub async fn get_hub_script_schema(path: &str, db: &DB) -> Result<Option<Schema>, ErrorData> {
let strip_path = StripPath(path.to_string());
let res = get_full_hub_script_by_path(strip_path, &HTTP_CLIENT, Some(db))
.await
.map_err(|e| {
tracing::error!("Failed to get hub script: {}", e);
ErrorData::internal_error(format!("Failed to get hub script: {}", e), None)
})?;
match serde_json::from_str::<Schema>(res.schema.get()) {
Ok(schema) => Ok(Some(schema)),
Err(e) => {
tracing::warn!("Failed to convert schema: {}", e);
Ok(None)
}
}
}
// ============================================================================
// HTTP request utilities for endpoint tools
// ============================================================================
/// Substitute path parameters in the URL template
pub fn substitute_path_params(
path: &str,
workspace_id: &str,
args_map: &serde_json::Map<String, Value>,
path_schema: &Option<Value>,
) -> BackendResult<String> {
let mut path_template = path.replace("{workspace}", workspace_id);
if let Some(schema) = path_schema {
if let Some(props) = schema.get("properties").and_then(|p| p.as_object()) {
for (param_name, _) in props {
let placeholder = format!("{{{}}}", param_name);
match args_map.get(param_name) {
Some(param_value) => {
if let Some(str_val) = param_value.as_str() {
path_template = path_template.replace(&placeholder, str_val);
}
}
None => {
tracing::warn!("Missing required path parameter: {}", param_name);
return Err(ErrorData::invalid_params(
format!("Missing required path parameter: {}", param_name),
None,
));
}
}
}
}
}
Ok(path_template)
}
/// Build query string from arguments
pub fn build_query_string(
args_map: &serde_json::Map<String, Value>,
query_schema: &Option<Value>,
) -> String {
let Some(schema) = query_schema else {
return String::new();
};
let Some(props) = schema.get("properties").and_then(|p| p.as_object()) else {
return String::new();
};
let query_params: Vec<String> = props
.keys()
.filter_map(|param_name| {
args_map
.get(param_name)
.filter(|v| !v.is_null())
.map(|value| {
let value_str = value.to_string();
let str_val = value_str.trim_matches('"');
format!(
"{}={}",
urlencoding::encode(param_name),
urlencoding::encode(str_val)
)
})
})
.collect();
if query_params.is_empty() {
String::new()
} else {
format!("?{}", query_params.join("&"))
}
}
/// Build request body from arguments
pub fn build_request_body(
method: &str,
args_map: &serde_json::Map<String, Value>,
body_schema: &Option<Value>,
) -> Option<Value> {
if method == "GET" {
return None;
}
let schema = body_schema.as_ref()?;
let props = schema.get("properties")?.as_object()?;
let body_map: serde_json::Map<String, Value> = props
.keys()
.filter_map(|param_name| {
args_map
.get(param_name)
.map(|value| (param_name.clone(), value.clone()))
})
.collect();
if body_map.is_empty() {
None
} else {
Some(Value::Object(body_map))
}
}
/// Create HTTP request with authentication
pub async fn create_http_request(
method: &str,
url: &str,
workspace_id: &str,
api_authed: &ApiAuthed,
body_json: Option<Value>,
) -> BackendResult<reqwest::Response> {
let client = &HTTP_CLIENT;
let mut request_builder = match method {
"GET" => client.get(url),
"POST" => client.post(url),
"PUT" => client.put(url),
"DELETE" => client.delete(url),
"PATCH" => client.patch(url),
_ => {
return Err(ErrorData::invalid_params(
format!("Unsupported HTTP method: {}", method),
None,
));
}
};
// Add authorization header
let authed = Authed::from(api_authed.clone());
let token = create_jwt_token(authed, workspace_id, 3600, None, None, None, None)
.await
.map_err(|e| ErrorData::internal_error(e.to_string(), None))?;
request_builder = request_builder.header("Authorization", format!("Bearer {}", token));
// Add body if present
if let Some(body) = body_json {
request_builder = request_builder
.header("Content-Type", "application/json")
.json(&body);
}
request_builder
.send()
.await
.map_err(|e| ErrorData::internal_error(format!("Failed to execute request: {}", e), None))
}
/// Convert a JSON Value into PushArgsOwned for job execution
pub fn prepare_push_args(args: Value) -> windmill_queue::PushArgsOwned {
if let Value::Object(map) = args {
let mut args_hash = HashMap::new();
for (k, v) in map {
args_hash.insert(k, to_raw_value(&v));
}
windmill_queue::PushArgsOwned { extra: None, args: args_hash }
} else {
windmill_queue::PushArgsOwned::default()
}
}
/// Parse an HTTP response body into a JSON Value
pub async fn parse_response_body(response: Response<Body>) -> BackendResult<Value> {
let body_bytes = to_bytes(response.into_body(), usize::MAX)
.await
.map_err(|e| {
ErrorData::internal_error(format!("Failed to read response body: {}", e), None)
})?;
let body_str = String::from_utf8(body_bytes.to_vec()).map_err(|e| {
ErrorData::internal_error(format!("Failed to decode response body: {}", e), None)
})?;
Ok(serde_json::from_str(&body_str).unwrap_or_else(|_| Value::String(body_str)))
}
backend/windmill-api/src/mcp/utils/database.rs
-243
View File
@@ -1,243 +0,0 @@
//! Database operations for MCP server
//!
//! Contains all database query functions and database-related utilities
//! used by the MCP server implementation.
use windmill_mcp::server::ErrorData;
use sql_builder::prelude::*;
use windmill_common::db::UserDB;
use windmill_common::scripts::{get_full_hub_script_by_path, Schema};
use windmill_common::utils::{query_elems_from_hub, StripPath};
use windmill_common::{DB, HUB_BASE_URL};
use super::models::*;
use crate::db::ApiAuthed;
use crate::HTTP_CLIENT;
/// Check if the user has proper MCP scopes
pub fn check_scopes(authed: &ApiAuthed) -> Result<(), ErrorData> {
let scopes = authed.scopes.as_ref();
if scopes.is_none()
|| scopes
.unwrap()
.iter()
.all(|scope| !scope.starts_with("mcp:"))
{
tracing::error!("Unauthorized: missing mcp scope");
return Err(ErrorData::internal_error(
"Unauthorized: missing mcp scope".to_string(),
None,
));
}
Ok(())
}
/// Get the schema for a specific item (script or flow)
pub async fn get_item_schema(
path: &str,
user_db: &UserDB,
authed: &ApiAuthed,
workspace_id: &str,
item_type: &str,
) -> Result<Option<Schema>, ErrorData> {
let mut sqlb = SqlBuilder::select_from(&format!("{} as o", item_type));
sqlb.fields(&["o.schema"]);
sqlb.and_where("o.path = ?".bind(&path));
sqlb.and_where("o.workspace_id = ?".bind(&workspace_id));
sqlb.and_where("o.archived = false");
sqlb.and_where("o.draft_only IS NOT TRUE");
let sql = sqlb.sql().map_err(|_e| {
tracing::error!("failed to build sql: {}", _e);
ErrorData::internal_error("failed to build sql", None)
})?;
let mut tx = user_db
.clone()
.begin(authed)
.await
.map_err(|_e| ErrorData::internal_error("failed to begin transaction", None))?;
let item = sqlx::query_as::<_, ItemSchema>(&sql)
.fetch_one(&mut *tx)
.await
.map_err(|_e| {
tracing::error!("failed to fetch item schema: {}", _e);
ErrorData::internal_error("failed to fetch item schema", None)
})?;
tx.commit()
.await
.map_err(|_e| ErrorData::internal_error("failed to commit transaction", None))?;
Ok(item.schema)
}
/// Get all resource types from the database
pub async fn get_resources_types(
user_db: &UserDB,
authed: &ApiAuthed,
workspace_id: &str,
) -> Result<Vec<ResourceType>, ErrorData> {
let mut sqlb = SqlBuilder::select_from("resource_type as o");
sqlb.fields(&["o.name", "o.description"]);
sqlb.and_where("o.workspace_id = ?".bind(&workspace_id));
let sql = sqlb.sql().map_err(|_e| {
tracing::error!("failed to build sql: {}", _e);
ErrorData::internal_error("failed to build sql", None)
})?;
let mut tx = user_db
.clone()
.begin(authed)
.await
.map_err(|_e| ErrorData::internal_error("failed to begin transaction", None))?;
let rows = sqlx::query_as::<_, ResourceType>(&sql)
.fetch_all(&mut *tx)
.await
.map_err(|_e| {
tracing::error!("Failed to fetch resource types: {}", _e);
ErrorData::internal_error("failed to fetch resource types", None)
})?;
tx.commit()
.await
.map_err(|_e| ErrorData::internal_error("failed to commit transaction", None))?;
Ok(rows)
}
/// Get resources by type from the database
pub async fn get_resources(
user_db: &UserDB,
authed: &ApiAuthed,
workspace_id: &str,
resource_type: &str,
) -> Result<Vec<ResourceInfo>, ErrorData> {
let mut sqlb = SqlBuilder::select_from("resource as o");
sqlb.fields(&["o.path", "o.description", "o.resource_type"]);
sqlb.and_where("o.workspace_id = ?".bind(&workspace_id));
sqlb.and_where("o.resource_type = ?".bind(&resource_type));
let sql = sqlb.sql().map_err(|_e| {
tracing::error!("failed to build sql: {}", _e);
ErrorData::internal_error("failed to build sql", None)
})?;
let mut tx = user_db
.clone()
.begin(authed)
.await
.map_err(|_e| ErrorData::internal_error("failed to begin transaction", None))?;
let rows = sqlx::query_as::<_, ResourceInfo>(&sql)
.fetch_all(&mut *tx)
.await
.map_err(|_e| {
tracing::error!("Failed to fetch resources: {}", _e);
ErrorData::internal_error("failed to fetch resources", None)
})?;
tx.commit()
.await
.map_err(|_e| ErrorData::internal_error("failed to commit transaction", None))?;
Ok(rows)
}
/// Generic function to get items (scripts or flows) from the database
pub async fn get_items<T: for<'a> sqlx::FromRow<'a, sqlx::postgres::PgRow> + Send + Unpin>(
user_db: &UserDB,
authed: &ApiAuthed,
workspace_id: &str,
scope_type: &str,
item_type: &str,
) -> Result<Vec<T>, ErrorData> {
let mut sqlb = SqlBuilder::select_from(&format!("{} as o", item_type));
let fields = vec!["o.path", "o.summary", "o.description", "o.schema"];
sqlb.fields(&fields);
if scope_type == "favorites" {
sqlb.join("favorite")
.on("favorite.favorite_kind = ? AND favorite.workspace_id = o.workspace_id AND favorite.path = o.path AND favorite.usr = ?".bind(&item_type)
.bind(&authed.username));
}
sqlb.and_where("o.workspace_id = ?".bind(&workspace_id))
.and_where("o.archived = false")
.and_where("o.draft_only IS NOT TRUE");
if item_type == "script" {
sqlb.and_where("(o.no_main_func IS NOT TRUE OR o.no_main_func IS NULL)");
}
sqlb.order_by(
if item_type == "flow" {
"o.edited_at"
} else {
"o.created_at"
},
false,
)
.limit(100);
let sql = sqlb.sql().map_err(|_e| {
tracing::error!("failed to build sql: {}", _e);
ErrorData::internal_error("failed to build sql", None)
})?;
let mut tx = user_db
.clone()
.begin(authed)
.await
.map_err(|_e| ErrorData::internal_error("failed to begin transaction", None))?;
let rows = sqlx::query_as::<_, T>(&sql)
.fetch_all(&mut *tx)
.await
.map_err(|_e| {
tracing::error!("Failed to fetch {}: {}", item_type, _e);
ErrorData::internal_error(format!("failed to fetch {}", item_type), None)
})?;
tx.commit()
.await
.map_err(|_e| ErrorData::internal_error("failed to commit transaction", None))?;
Ok(rows)
}
/// Get scripts from the Hub
pub async fn get_scripts_from_hub(
db: &DB,
scope_integrations: Option<&str>,
) -> Result<Vec<HubScriptInfo>, ErrorData> {
let query_params = Some(vec![
("limit", "100".to_string()),
("with_schema", "true".to_string()),
("apps", scope_integrations.unwrap_or("").to_string()),
]);
let url = format!("{}/scripts/top", *HUB_BASE_URL.read().await);
let (_status_code, _headers, response) =
query_elems_from_hub(&HTTP_CLIENT, &url, query_params, &db)
.await
.map_err(|e| {
tracing::error!("Failed to get items from hub: {}", e);
ErrorData::internal_error(format!("Failed to get items from hub: {}", e), None)
})?;
use axum::body::to_bytes;
let body_bytes = to_bytes(response, usize::MAX).await.map_err(|e| {
tracing::error!("Failed to read response body: {}", e);
ErrorData::internal_error(format!("Failed to read response body: {}", e), None)
})?;
let body_str = String::from_utf8(body_bytes.to_vec()).map_err(|e| {
tracing::error!("Failed to decode response body: {}", e);
ErrorData::internal_error(format!("Failed to decode response body: {}", e), None)
})?;
let hub_response: HubResponse = serde_json::from_str(&body_str).map_err(|e| {
tracing::error!("Failed to parse hub response: {}", e);
ErrorData::internal_error(format!("Failed to parse hub response: {}", e), None)
})?;
Ok(hub_response.asks)
}
/// Get the schema for a Hub script
pub async fn get_hub_script_schema(path: &str, db: &DB) -> Result<Option<Schema>, ErrorData> {
let strip_path = StripPath(path.to_string());
let res = get_full_hub_script_by_path(strip_path, &HTTP_CLIENT, Some(db))
.await
.map_err(|e| {
tracing::error!("Failed to get hub script: {}", e);
ErrorData::internal_error(format!("Failed to get hub script: {}", e), None)
})?;
match serde_json::from_str::<Schema>(res.schema.get()) {
Ok(schema) => Ok(Some(schema)),
Err(e) => {
tracing::warn!("Failed to convert schema: {}", e);
Ok(None)
}
}
}
backend/windmill-api/src/mcp/utils/mod.rs
-10
View File
@@ -1,10 +0,0 @@
//! Utility functions and helpers for MCP server
//!
//! This module contains various utility functions for schema transformation,
//! database operations, data models, and path transformations.
pub mod models;
pub mod database;
pub mod schema;
pub mod transform;
pub mod scope_matcher;
backend/windmill-api/src/mcp/utils/schema.rs
-143
View File
@@ -1,143 +0,0 @@
//! Schema transformation utilities for MCP server
//!
//! Contains functions for transforming Windmill schemas into MCP-compatible formats,
//! including resource enrichment and schema conversion utilities.
use windmill_mcp::server::ErrorData;
use serde_json::Value;
use std::collections::HashMap;
use windmill_common::db::UserDB;
use windmill_common::scripts::Schema;
use super::database::get_resources;
use super::models::{ResourceInfo, ResourceType, SchemaType};
use super::transform::apply_key_transformation;
use crate::db::ApiAuthed;
/// Convert a Windmill Schema to a SchemaType
pub fn convert_schema_to_schema_type(schema: Option<Schema>) -> SchemaType {
let schema_obj = if let Some(ref s) = schema {
match serde_json::from_str::<SchemaType>(s.0.get()) {
Ok(val) => val,
Err(_) => SchemaType::default(),
}
} else {
SchemaType::default()
};
schema_obj
}
/// Transform the schema for resources by enriching with resource information
pub async fn transform_schema_for_resources(
schema: &SchemaType,
user_db: &UserDB,
authed: &ApiAuthed,
w_id: &str,
resources_cache: &mut HashMap<String, Vec<ResourceInfo>>,
resources_types: &Vec<ResourceType>,
) -> Result<SchemaType, ErrorData> {
let mut schema_obj: SchemaType = schema.clone();
// replace invalid char in property key with underscore
let replacements: Vec<(String, String, Value)> = schema_obj
.properties
.iter()
.filter_map(|(key, value)| {
if key.chars().any(|c| !c.is_alphanumeric() && c != '_') {
let new_key = apply_key_transformation(key);
Some((key.clone(), new_key, value.clone()))
} else {
None
}
})
.collect();
for (old_key, new_key, value) in replacements {
schema_obj.properties.remove(&old_key);
schema_obj.properties.insert(new_key, value);
}
for (_key, prop_value) in schema_obj.properties.iter_mut() {
if let Value::Object(prop_map) = prop_value {
// if property is a resource, fetch the resource type infos, and add each available resource to the description
if let Some(format_value) = prop_map.get("format") {
if let Value::String(format_str) = format_value {
if format_str.starts_with("resource-") {
let resource_type_key =
format_str.split("-").last().unwrap_or_default().to_string();
let resource_type = resources_types
.iter()
.find(|rt| rt.name == resource_type_key);
let resource_type_obj = resource_type.cloned();
if !resources_cache.contains_key(&resource_type_key) {
let available_resources =
get_resources(user_db, authed, &w_id, &resource_type_key).await;
match available_resources {
Ok(cache_data) => {
resources_cache.insert(resource_type_key.clone(), cache_data);
}
Err(e) => {
tracing::error!("Failed to fetch resource cache data: {}", e);
continue; // Skip this property if fetching failed
}
}
}
if let Some(resource_cache) = resources_cache.get(&resource_type_key) {
let resources_count = resource_cache.len();
let description = match resource_type_obj {
Some(resource_type_obj) => format!(
"This is a resource named `{}` with the following description: `{}`.\\nThe path of the resource should be used to specify the resource.\\n{}",
resource_type_obj.name,
resource_type_obj.description.as_deref().unwrap_or("No description"),
if resources_count == 0 {
"This resource does not have any available instances, you should create one from your windmill workspace."
} else if resources_count > 1 {
"This resource has multiple available instances, you should precisely select the one you want to use."
} else {
"There is 1 resource available."
}
),
None => "An object parameter.".to_string()
};
prop_map
.insert("type".to_string(), Value::String("string".to_string()));
prop_map.insert("description".to_string(), Value::String(description));
if resources_count > 0 {
let resources_description = resource_cache
.iter()
.map(|resource| {
format!(
"{}: $res:{}",
resource.description.as_deref().unwrap_or("No title"),
resource.path
)
})
.collect::<Vec<String>>()
.join("\\n");
prop_map.insert(
"description".to_string(),
Value::String(format!(
"{}\\nHere are the available resources, in the format title:path. Title can be empty. Path should be used to specify the resource:\\n{}",
prop_map.get("description").unwrap_or(&Value::String("No description".to_string())),
resources_description
)),
);
}
}
}
}
}
} else {
tracing::warn!(
"Schema property value is not a JSON object: {:?}",
prop_value
);
}
}
Ok(schema_obj)
}
backend/windmill-api/src/npm_proxy.rs
+502
View File
@@ -0,0 +1,502 @@
/*
* This file provides a proxy endpoint for npm package requests
* to support private registries in the frontend ATA (Automatic Type Acquisition)
*/
use axum::{
extract::{Path, Query},
routing::get,
Extension, Json, Router,
};
use serde::{Deserialize, Serialize};
use sqlx::types::JsonValue;
use std::collections::HashMap;
use tower_http::cors::{Any, CorsLayer};
use windmill_common::{
error::{Error, JsonResult, Result},
global_settings::{load_value_from_global_settings, NPM_CONFIG_REGISTRY_SETTING},
utils::StripPath,
};
use crate::{db::ApiAuthed, HTTP_CLIENT};
#[derive(Deserialize)]
struct ProxyQuery {
tag: Option<String>,
}
/// Parse a scoped package path like "@scope/name" or "name" from a wildcard path
fn parse_package_name(path: &str) -> String {
// Remove leading slash if present
path.trim_start_matches('/').to_string()
}
/// Parse package and version from a path like "@scope/name/1.0.0" or "name/1.0.0"
fn parse_package_and_version(path: &str) -> Result<(String, String)> {
let path = path.trim_start_matches('/');
if path.starts_with('@') {
// Scoped package: @scope/name/version
let parts: Vec<&str> = path.splitn(3, '/').collect();
if parts.len() < 3 {
return Err(Error::BadRequest("Invalid scoped package path, expected @scope/name/version".to_string()));
}
let package = format!("{}/{}", parts[0], parts[1]);
let version = parts[2].to_string();
Ok((package, version))
} else {
// Regular package: name/version
let parts: Vec<&str> = path.splitn(2, '/').collect();
if parts.len() < 2 {
return Err(Error::BadRequest("Invalid package path, expected name/version".to_string()));
}
Ok((parts[0].to_string(), parts[1].to_string()))
}
}
/// Parse package, version, and filepath from a path like "@scope/name/1.0.0/index.d.ts"
fn parse_package_version_and_file(path: &str) -> Result<(String, String, String)> {
let path = path.trim_start_matches('/');
if path.starts_with('@') {
// Scoped package: @scope/name/version/filepath
let parts: Vec<&str> = path.splitn(4, '/').collect();
if parts.len() < 4 {
return Err(Error::BadRequest("Invalid scoped package file path, expected @scope/name/version/filepath".to_string()));
}
let package = format!("{}/{}", parts[0], parts[1]);
let version = parts[2].to_string();
let filepath = parts[3].to_string();
Ok((package, version, filepath))
} else {
// Regular package: name/version/filepath
let parts: Vec<&str> = path.splitn(3, '/').collect();
if parts.len() < 3 {
return Err(Error::BadRequest("Invalid package file path, expected name/version/filepath".to_string()));
}
Ok((parts[0].to_string(), parts[1].to_string(), parts[2].to_string()))
}
}
#[derive(Serialize)]
struct PackageVersions {
tags: HashMap<String, String>,
versions: Vec<String>,
}
#[derive(Serialize)]
struct PackageVersion {
version: Option<String>,
}
#[derive(Serialize)]
struct PackageFiletree {
default: String,
files: Vec<FileEntry>,
}
#[derive(Serialize)]
struct FileEntry {
name: String,
}
pub fn workspaced_service() -> Router {
Router::new()
// Use wildcards for package names to support scoped packages like @scope/package
.route("/metadata/*package", get(get_package_metadata))
.route("/resolve/*package", get(resolve_package_version))
.route("/filetree/*package_version", get(get_package_filetree))
.route("/file/*package_version_filepath", get(get_package_file))
.layer(
CorsLayer::new()
.allow_origin(Any)
.allow_methods(Any)
.allow_headers(Any),
)
}
/// Get package metadata (versions and tags) from the private registry
async fn get_package_metadata(
_authed: ApiAuthed,
Path((_w_id, package_path)): Path<(String, StripPath)>,
Extension(db): Extension<sqlx::Pool<sqlx::Postgres>>,
) -> JsonResult<PackageVersions> {
let package = parse_package_name(package_path.to_path());
let npm_registry = get_npm_registry(&db).await?;
if npm_registry.is_none() {
return Err(Error::BadRequest(
"No private npm registry configured".to_string(),
));
}
let registry_url = npm_registry.unwrap();
let package_url = format_registry_url(&registry_url, &package, None, None);
tracing::info!("Fetching package metadata from: {}", package_url);
let response = HTTP_CLIENT
.get(&package_url)
.send()
.await
.map_err(|e| Error::InternalErr(format!("Failed to fetch package metadata: {}", e)))?;
if !response.status().is_success() {
return Err(Error::NotFound(format!(
"Package {} not found in private registry",
package
)));
}
let package_json: JsonValue = response
.json()
.await
.map_err(|e| Error::InternalErr(format!("Failed to parse package metadata: {}", e)))?;
// Extract versions and dist-tags from the package metadata
let mut versions = Vec::new();
let mut tags = HashMap::new();
if let Some(versions_obj) = package_json.get("versions").and_then(|v| v.as_object()) {
versions = versions_obj.keys().cloned().collect();
}
if let Some(tags_obj) = package_json.get("dist-tags").and_then(|v| v.as_object()) {
for (tag, version) in tags_obj {
if let Some(version_str) = version.as_str() {
tags.insert(tag.clone(), version_str.to_string());
}
}
}
Ok(Json(PackageVersions { tags, versions }))
}
/// Resolve a package tag/version reference to a specific version
async fn resolve_package_version(
_authed: ApiAuthed,
Path((_w_id, package_path)): Path<(String, StripPath)>,
Query(query): Query<ProxyQuery>,
Extension(db): Extension<sqlx::Pool<sqlx::Postgres>>,
) -> JsonResult<PackageVersion> {
let package = parse_package_name(package_path.to_path());
let npm_registry = get_npm_registry(&db).await?;
if npm_registry.is_none() {
return Err(Error::BadRequest(
"No private npm registry configured".to_string(),
));
}
let registry_url = npm_registry.unwrap();
let reference = query.tag.unwrap_or_else(|| "latest".to_string());
let package_url = format_registry_url(&registry_url, &package, None, None);
tracing::info!("Resolving package version from: {}", package_url);
let response = HTTP_CLIENT
.get(&package_url)
.send()
.await
.map_err(|e| Error::InternalErr(format!("Failed to fetch package metadata: {}", e)))?;
if !response.status().is_success() {
return Err(Error::NotFound(format!(
"Package {} not found in private registry",
package
)));
}
let package_json: JsonValue = response
.json()
.await
.map_err(|e| Error::InternalErr(format!("Failed to parse package metadata: {}", e)))?;
// Try to resolve the reference as a tag first
let version = if let Some(tags) = package_json.get("dist-tags").and_then(|v| v.as_object()) {
if let Some(version) = tags.get(&reference).and_then(|v| v.as_str()) {
Some(version.to_string())
} else {
// If not a tag, check if it's a valid version
if let Some(versions) = package_json.get("versions").and_then(|v| v.as_object()) {
if versions.contains_key(&reference) {
Some(reference)
} else {
None
}
} else {
None
}
}
} else {
None
};
Ok(Json(PackageVersion { version }))
}
/// Get the file tree for a specific package version
async fn get_package_filetree(
_authed: ApiAuthed,
Path((_w_id, package_version_path)): Path<(String, StripPath)>,
Extension(db): Extension<sqlx::Pool<sqlx::Postgres>>,
) -> JsonResult<PackageFiletree> {
let (package, version) = parse_package_and_version(package_version_path.to_path())?;
let npm_registry = get_npm_registry(&db).await?;
if npm_registry.is_none() {
return Err(Error::BadRequest(
"No private npm registry configured".to_string(),
));
}
let registry_url = npm_registry.unwrap();
let package_url = format_registry_url(&registry_url, &package, None, None);
tracing::info!("Fetching package filetree from: {}", package_url);
let response = HTTP_CLIENT
.get(&package_url)
.send()
.await
.map_err(|e| Error::InternalErr(format!("Failed to fetch package metadata: {}", e)))?;
if !response.status().is_success() {
return Err(Error::NotFound(format!(
"Package {} not found in private registry",
package
)));
}
let package_json: JsonValue = response
.json()
.await
.map_err(|e| Error::InternalErr(format!("Failed to parse package metadata: {}", e)))?;
// Get the tarball URL for this version
let tarball_url = package_json
.get("versions")
.and_then(|v| v.get(&version))
.and_then(|v| v.get("dist"))
.and_then(|d| d.get("tarball"))
.and_then(|t| t.as_str())
.ok_or_else(|| Error::NotFound(format!("Tarball not found for {}@{}", package, version)))?;
// Download and extract tarball to get file list
let tarball_response = HTTP_CLIENT
.get(tarball_url)
.send()
.await
.map_err(|e| Error::InternalErr(format!("Failed to download tarball: {}", e)))?;
if !tarball_response.status().is_success() {
return Err(Error::NotFound(format!(
"Failed to download tarball for {}@{}",
package, version
)));
}
let tarball_bytes = tarball_response
.bytes()
.await
.map_err(|e| Error::InternalErr(format!("Failed to read tarball: {}", e)))?;
// Extract file list from tarball
let files = extract_tarball_files(&tarball_bytes)?;
// Find the main entry point
let main = package_json
.get("versions")
.and_then(|v| v.get(&version))
.and_then(|v| v.get("main"))
.and_then(|m| m.as_str())
.unwrap_or("index.js")
.to_string();
Ok(Json(PackageFiletree {
default: main,
files,
}))
}
/// Get a specific file from a package version
async fn get_package_file(
_authed: ApiAuthed,
Path((_w_id, full_path)): Path<(String, StripPath)>,
Extension(db): Extension<sqlx::Pool<sqlx::Postgres>>,
) -> Result<String> {
let (package, version, filepath) = parse_package_version_and_file(full_path.to_path())?;
let npm_registry = get_npm_registry(&db).await?;
if npm_registry.is_none() {
return Err(Error::BadRequest(
"No private npm registry configured".to_string(),
));
}
let registry_url = npm_registry.unwrap();
let package_url = format_registry_url(&registry_url, &package, None, None);
tracing::info!("Fetching package file from: {}", package_url);
let response = HTTP_CLIENT
.get(&package_url)
.send()
.await
.map_err(|e| Error::InternalErr(format!("Failed to fetch package metadata: {}", e)))?;
if !response.status().is_success() {
return Err(Error::NotFound(format!(
"Package {} not found in private registry",
package
)));
}
let package_json: JsonValue = response
.json()
.await
.map_err(|e| Error::InternalErr(format!("Failed to parse package metadata: {}", e)))?;
// Get the tarball URL for this version
let tarball_url = package_json
.get("versions")
.and_then(|v| v.get(&version))
.and_then(|v| v.get("dist"))
.and_then(|d| d.get("tarball"))
.and_then(|t| t.as_str())
.ok_or_else(|| Error::NotFound(format!("Tarball not found for {}@{}", package, version)))?;
// Download tarball
let tarball_response = HTTP_CLIENT
.get(tarball_url)
.send()
.await
.map_err(|e| Error::InternalErr(format!("Failed to download tarball: {}", e)))?;
if !tarball_response.status().is_success() {
return Err(Error::NotFound(format!(
"Failed to download tarball for {}@{}",
package, version
)));
}
let tarball_bytes = tarball_response
.bytes()
.await
.map_err(|e| Error::InternalErr(format!("Failed to read tarball: {}", e)))?;
// Extract the specific file from the tarball
let file_content = extract_file_from_tarball(&tarball_bytes, &filepath)?;
Ok(file_content)
}
/// Get the npm registry URL from global settings
async fn get_npm_registry(db: &sqlx::Pool<sqlx::Postgres>) -> Result<Option<String>> {
let registry = load_value_from_global_settings(db, NPM_CONFIG_REGISTRY_SETTING)
.await?
.and_then(|v| v.as_str().map(|s| s.to_string()));
Ok(registry)
}
/// Format a registry URL for a package
fn format_registry_url(
registry_base: &str,
package: &str,
version: Option<&str>,
file: Option<&str>,
) -> String {
let registry_base = registry_base.trim_end_matches('/');
// Handle scoped packages (e.g., @types/node)
let package_path = if package.starts_with('@') {
// Scoped packages need to be URL encoded properly
package.replace('/', "%2F")
} else {
package.to_string()
};
match (version, file) {
(Some(v), Some(f)) => {
format!("{}/{}/{}/-/{}-{}/{}", registry_base, package_path, v, package, v, f)
}
(Some(v), None) => {
format!("{}/{}/{}", registry_base, package_path, v)
}
_ => {
format!("{}/{}", registry_base, package_path)
}
}
}
/// Extract file list from a tarball
fn extract_tarball_files(tarball_bytes: &[u8]) -> Result<Vec<FileEntry>> {
use flate2::read::GzDecoder;
use tar::Archive;
let gz = GzDecoder::new(tarball_bytes);
let mut archive = Archive::new(gz);
let mut files = Vec::new();
for entry in archive
.entries()
.map_err(|e| Error::InternalErr(format!("Failed to read tarball entries: {}", e)))?
{
let entry = entry
.map_err(|e| Error::InternalErr(format!("Failed to read tarball entry: {}", e)))?;
let path = entry
.path()
.map_err(|e| Error::InternalErr(format!("Failed to read entry path: {}", e)))?;
// Remove the package/ prefix that npm tarballs have
let path_str = path.to_string_lossy().to_string();
if let Some(stripped) = path_str.strip_prefix("package/") {
files.push(FileEntry {
name: format!("/{}", stripped),
});
}
}
Ok(files)
}
/// Extract a specific file from a tarball
fn extract_file_from_tarball(tarball_bytes: &[u8], target_file: &str) -> Result<String> {
use flate2::read::GzDecoder;
use std::io::Read;
use tar::Archive;
let gz = GzDecoder::new(tarball_bytes);
let mut archive = Archive::new(gz);
// Normalize the target file path
let target = target_file.trim_start_matches('/');
for entry in archive
.entries()
.map_err(|e| Error::InternalErr(format!("Failed to read tarball entries: {}", e)))?
{
let mut entry = entry
.map_err(|e| Error::InternalErr(format!("Failed to read tarball entry: {}", e)))?;
let path = entry
.path()
.map_err(|e| Error::InternalErr(format!("Failed to read entry path: {}", e)))?;
let path_str = path.to_string_lossy().to_string();
// Check if this is the file we're looking for
if let Some(stripped) = path_str.strip_prefix("package/") {
if stripped == target {
let mut content = String::new();
entry
.read_to_string(&mut content)
.map_err(|e| Error::InternalErr(format!("Failed to read file content: {}", e)))?;
return Ok(content);
}
}
}
Err(Error::NotFound(format!("File {} not found in tarball", target_file)))
}
backend/windmill-api/src/resources.rs
+30 -1
View File
@@ -10,6 +10,7 @@ use std::collections::HashMap;
use crate::{
db::{ApiAuthed, DB},
secret_backend_ext::rename_vault_secret,
users::{maybe_refresh_folders, require_owner_of_path, Tokened},
utils::{check_scopes, require_super_admin, BulkDeleteRequest},
var_resource_cache::{cache_resource, get_cached_resource},
@@ -947,12 +948,40 @@ async fn update_resource(
let mut tx = user_db.begin(&authed).await?;
if let Some(npath) = ns.path {
if let Some(npath) = ns.path.clone() {
if npath != path {
check_path_conflict(&mut tx, &w_id, &npath).await?;
require_owner_of_path(&authed, path)?;
// Handle Vault secret rename if the linked variable is a Vault-stored secret
let linked_var = sqlx::query!(
"SELECT value, is_secret FROM variable WHERE path = $1 AND workspace_id = $2",
path,
w_id
)
.fetch_optional(&mut *tx)
.await?;
if let Some(var) = linked_var {
if var.is_secret {
// Check if this is a Vault-stored secret and rename it
if let Some(new_value) =
rename_vault_secret(&db, &w_id, path, &npath, &var.value).await?
{
// Update the variable's value to point to the new Vault path
sqlx::query!(
"UPDATE variable SET value = $1 WHERE path = $2 AND workspace_id = $3",
new_value,
path,
w_id
)
.execute(&mut *tx)
.await?;
}
}
}
sqlx::query!(
"UPDATE variable SET path = $1 WHERE path = $2 AND workspace_id = $3",
npath,
backend/windmill-api/src/secret_backend_ext.rs
+408
View File
@@ -0,0 +1,408 @@
/*
* Author: Ruben Fiszel
* Copyright: Windmill Labs, Inc 2024
* This file and its contents are licensed under the AGPLv3 License.
* Please see the included NOTICE for copyright information and
* LICENSE-AGPL for a copy of the license.
*/
//! Secret backend extension for the API layer
//!
//! This module provides helper functions for integrating the SecretBackend
//! trait with variable operations in the API.
//!
//! Note: HashiCorp Vault integration requires Enterprise Edition.
//! The OSS version only supports the database backend.
use std::sync::Arc;
use windmill_common::{
db::DB,
error::{Error, Result},
secret_backend::{database::DatabaseBackend, SecretBackend},
variables::{build_crypt, decrypt, encrypt},
};
#[cfg(all(feature = "private", feature = "enterprise"))]
use windmill_common::{
global_settings::{load_value_from_global_settings, SECRET_BACKEND_SETTING},
secret_backend::{SecretBackendConfig, VaultBackend, VaultSettings},
};
#[cfg(all(feature = "private", feature = "enterprise"))]
use tokio::sync::RwLock;
// Cached Vault backend to avoid recreating it for every request
// This enables connection pooling and avoids repeated setup overhead
#[cfg(all(feature = "private", feature = "enterprise"))]
struct CachedVaultBackend {
backend: Arc<dyn SecretBackend>,
settings: VaultSettings,
}
#[cfg(all(feature = "private", feature = "enterprise"))]
lazy_static::lazy_static! {
static ref VAULT_BACKEND_CACHE: RwLock<Option<CachedVaultBackend>> = RwLock::new(None);
}
/// Get the current secret backend based on global settings
///
/// OSS: Always returns DatabaseBackend
/// EE: Returns configured backend (Database or Vault)
#[cfg(not(all(feature = "private", feature = "enterprise")))]
pub async fn get_secret_backend(db: &DB) -> Result<Arc<dyn SecretBackend>> {
Ok(Arc::new(DatabaseBackend::new(db.clone())))
}
#[cfg(all(feature = "private", feature = "enterprise"))]
pub async fn get_secret_backend(db: &DB) -> Result<Arc<dyn SecretBackend>> {
let config = match load_value_from_global_settings(db, SECRET_BACKEND_SETTING).await? {
Some(value) => serde_json::from_value::<SecretBackendConfig>(value).unwrap_or_default(),
None => SecretBackendConfig::default(),
};
match config {
SecretBackendConfig::Database => Ok(Arc::new(DatabaseBackend::new(db.clone()))),
SecretBackendConfig::HashiCorpVault(settings) => {
get_or_create_vault_backend(db, settings).await
}
}
}
/// Get a cached Vault backend or create a new one if settings changed
#[cfg(all(feature = "private", feature = "enterprise"))]
async fn get_or_create_vault_backend(
_db: &DB,
settings: VaultSettings,
) -> Result<Arc<dyn SecretBackend>> {
// Check if we have a cached backend with matching settings (read lock)
{
let cache = VAULT_BACKEND_CACHE.read().await;
if let Some(ref cached) = *cache {
if cached.settings == settings {
return Ok(cached.backend.clone());
}
}
}
// Need to create a new backend - acquire write lock
let mut cache = VAULT_BACKEND_CACHE.write().await;
// Double-check (another task may have created it while we waited)
if let Some(ref cached) = *cache {
if cached.settings == settings {
return Ok(cached.backend.clone());
}
}
// Create new backend
let backend: Arc<dyn SecretBackend> = {
#[cfg(feature = "openidconnect")]
if settings.token.is_none() {
Arc::new(VaultBackend::new_with_db(settings.clone(), _db.clone()))
} else {
Arc::new(VaultBackend::new(settings.clone()))
}
#[cfg(not(feature = "openidconnect"))]
Arc::new(VaultBackend::new(settings.clone()))
};
// Cache it
*cache = Some(CachedVaultBackend {
backend: backend.clone(),
settings,
});
Ok(backend)
}
/// Check if a Vault backend is currently configured
///
/// OSS: Always returns false
/// EE: Checks global settings
#[cfg(not(all(feature = "private", feature = "enterprise")))]
pub async fn is_vault_backend_configured(_db: &DB) -> Result<bool> {
Ok(false)
}
#[cfg(all(feature = "private", feature = "enterprise"))]
pub async fn is_vault_backend_configured(db: &DB) -> Result<bool> {
let config = match load_value_from_global_settings(db, SECRET_BACKEND_SETTING).await? {
Some(value) => serde_json::from_value::<SecretBackendConfig>(value).unwrap_or_default(),
None => SecretBackendConfig::default(),
};
Ok(matches!(config, SecretBackendConfig::HashiCorpVault(_)))
}
/// Get a secret value using the configured backend
///
/// For database backend: decrypts using workspace key
/// For vault backend (EE only): fetches from Vault directly
pub async fn get_secret_value(
db: &DB,
workspace_id: &str,
path: &str,
encrypted_value: &str,
) -> Result<String> {
let backend = get_secret_backend(db).await?;
match backend.backend_name() {
"database" => {
// Use existing database decryption
let mc = build_crypt(db, workspace_id).await?;
decrypt(&mc, encrypted_value.to_string()).map_err(|e| {
Error::internal_err(format!("Error decrypting variable {}: {}", path, e))
})
}
"hashicorp_vault" => {
// Fetch from Vault directly
backend.get_secret(workspace_id, path).await
}
_ => Err(Error::internal_err(format!(
"Unknown backend: {}",
backend.backend_name()
))),
}
}
/// Store a secret value using the configured backend
///
/// For database backend: encrypts using workspace key and returns encrypted value
/// For vault backend (EE only): stores in Vault and returns a placeholder for DB storage
pub async fn store_secret_value(
db: &DB,
workspace_id: &str,
path: &str,
plain_value: &str,
) -> Result<String> {
let backend = get_secret_backend(db).await?;
match backend.backend_name() {
"database" => {
// Use existing database encryption
let mc = build_crypt(db, workspace_id).await?;
Ok(encrypt(&mc, plain_value))
}
"hashicorp_vault" => {
// Store in Vault and return a marker for DB
backend.set_secret(workspace_id, path, plain_value).await?;
// Return a marker indicating the value is stored in Vault
// The actual value in the DB will be this marker
Ok(format!("$vault:{}", path))
}
_ => Err(Error::internal_err(format!(
"Unknown backend: {}",
backend.backend_name()
))),
}
}
/// Delete a secret from the configured backend (if using Vault)
///
/// For database backend: no-op (DB delete is handled separately)
/// For vault backend (EE only): deletes from Vault
pub async fn delete_secret_from_backend(
db: &DB,
workspace_id: &str,
path: &str,
) -> Result<()> {
if is_vault_backend_configured(db).await? {
let backend = get_secret_backend(db).await?;
// Ignore NotFound errors during deletion (secret might not exist in Vault)
match backend.delete_secret(workspace_id, path).await {
Ok(()) => Ok(()),
Err(Error::NotFound(_)) => Ok(()),
Err(e) => Err(e),
}
} else {
Ok(())
}
}
/// Check if a value is stored in Vault (indicated by the $vault: prefix)
pub fn is_vault_stored_value(value: &str) -> bool {
value.starts_with("$vault:")
}
/// Rename a secret in Vault when a variable path changes (EE only)
///
/// This function:
/// 1. Reads the secret value from the old path
/// 2. Writes it to the new path
/// 3. Deletes from the old path
/// 4. Returns the new marker value ($vault:new_path)
///
/// If the value is not a Vault-stored value, returns None (no action needed).
/// If Vault is not configured, returns None.
#[cfg(not(all(feature = "private", feature = "enterprise")))]
pub async fn rename_vault_secret(
_db: &DB,
_workspace_id: &str,
_old_path: &str,
new_path: &str,
current_value: &str,
) -> Result<Option<String>> {
// OSS: If value has $vault: prefix, just update the reference
// (This handles edge case where EE was used before downgrading to OSS)
if is_vault_stored_value(current_value) {
tracing::warn!(
"Variable has $vault: prefix but Vault requires Enterprise Edition. \
Updating DB reference to {}",
new_path
);
return Ok(Some(format!("$vault:{}", new_path)));
}
Ok(None)
}
#[cfg(all(feature = "private", feature = "enterprise"))]
pub async fn rename_vault_secret(
db: &DB,
workspace_id: &str,
old_path: &str,
new_path: &str,
current_value: &str,
) -> Result<Option<String>> {
// Only handle Vault-stored values
if !is_vault_stored_value(current_value) {
return Ok(None);
}
// Check if Vault backend is configured
if !is_vault_backend_configured(db).await? {
// Vault not configured but value has $vault: prefix - this is an inconsistent state
// Log warning and return new marker to at least update the DB reference
tracing::warn!(
"Variable value has $vault: prefix but Vault is not configured. \
Updating DB reference from {} to {}",
old_path,
new_path
);
return Ok(Some(format!("$vault:{}", new_path)));
}
let backend = get_secret_backend(db).await?;
// Read from old path
let secret_value = match backend.get_secret(workspace_id, old_path).await {
Ok(value) => value,
Err(Error::NotFound(_)) => {
// Secret doesn't exist in Vault - just update the DB reference
tracing::warn!(
"Secret not found in Vault at path {} during rename to {}",
old_path,
new_path
);
return Ok(Some(format!("$vault:{}", new_path)));
}
Err(e) => return Err(e),
};
// Write to new path
backend
.set_secret(workspace_id, new_path, &secret_value)
.await?;
// Delete from old path (ignore errors - new path is already written)
if let Err(e) = backend.delete_secret(workspace_id, old_path).await {
tracing::warn!(
"Failed to delete old secret at {} after rename to {}: {}",
old_path,
new_path,
e
);
}
Ok(Some(format!("$vault:{}", new_path)))
}
/// Bulk rename secrets in Vault when a path prefix changes (e.g., user rename)
/// EE only feature.
///
/// This is used when renaming users where many secrets need their paths updated.
/// Returns a list of (old_path, new_value) pairs for updating the database.
#[cfg(not(all(feature = "private", feature = "enterprise")))]
pub async fn rename_vault_secrets_with_prefix(
_db: &DB,
_workspace_id: &str,
_old_prefix: &str,
_new_prefix: &str,
_variables: Vec<(String, String)>,
) -> Result<Vec<(String, String)>> {
// OSS: No Vault support, return empty
Ok(vec![])
}
#[cfg(all(feature = "private", feature = "enterprise"))]
pub async fn rename_vault_secrets_with_prefix(
db: &DB,
workspace_id: &str,
old_prefix: &str,
new_prefix: &str,
variables: Vec<(String, String)>, // (path, value) pairs
) -> Result<Vec<(String, String)>> {
// Only process if Vault is configured
if !is_vault_backend_configured(db).await? {
return Ok(vec![]);
}
let backend = get_secret_backend(db).await?;
let mut updates = Vec::new();
for (old_path, value) in variables {
// Only handle Vault-stored values
if !is_vault_stored_value(&value) {
continue;
}
// Calculate new path by replacing prefix
let new_path = if old_path.starts_with(old_prefix) {
format!("{}{}", new_prefix, &old_path[old_prefix.len()..])
} else {
continue; // Path doesn't match prefix, skip
};
// Read from old path
let secret_value = match backend.get_secret(workspace_id, &old_path).await {
Ok(v) => v,
Err(Error::NotFound(_)) => {
// Just update DB reference
updates.push((old_path, format!("$vault:{}", new_path)));
continue;
}
Err(e) => {
tracing::error!(
"Failed to read secret at {} during bulk rename: {}",
old_path,
e
);
continue;
}
};
// Write to new path
if let Err(e) = backend.set_secret(workspace_id, &new_path, &secret_value).await {
tracing::error!(
"Failed to write secret to {} during bulk rename: {}",
new_path,
e
);
continue;
}
// Delete from old path
if let Err(e) = backend.delete_secret(workspace_id, &old_path).await {
tracing::warn!(
"Failed to delete old secret at {} after rename: {}",
old_path,
e
);
}
updates.push((old_path, format!("$vault:{}", new_path)));
}
Ok(updates)
}
backend/windmill-api/src/settings.rs
+109
View File
@@ -42,6 +42,8 @@ use windmill_common::{
},
server::Smtp,
};
#[cfg(all(feature = "private", feature = "enterprise"))]
use windmill_common::secret_backend::{SecretMigrationReport, VaultSettings};
use windmill_common::{error::to_anyhow, PgDatabase};
pub fn global_service() -> Router {
@@ -85,6 +87,16 @@ pub fn global_service() -> Router {
post(acknowledge_all_critical_alerts),
);
// Vault integration routes (EE only - requires both private and enterprise features)
#[cfg(all(feature = "private", feature = "enterprise"))]
let r = r
.route("/test_secret_backend", post(test_secret_backend))
.route("/migrate_secrets_to_vault", post(migrate_secrets_to_vault))
.route(
"/migrate_secrets_to_database",
post(migrate_secrets_to_database),
);
#[cfg(feature = "parquet")]
{
return r.route("/test_object_storage_config", post(test_s3_bucket));
@@ -798,3 +810,100 @@ async fn setup_custom_instance_pg_database_inner(
Ok(())
}
// ============================================================================
// Secret Backend Settings (HashiCorp Vault Integration) - Enterprise Edition
// ============================================================================
/// Test connection to a secret backend (HashiCorp Vault)
///
/// This endpoint validates that the Vault settings are correct and that
/// Windmill can successfully authenticate and communicate with Vault.
///
/// This is an Enterprise Edition feature.
#[cfg(all(feature = "private", feature = "enterprise"))]
pub async fn test_secret_backend(
Extension(db): Extension<DB>,
authed: ApiAuthed,
Json(settings): Json<VaultSettings>,
) -> Result<String> {
require_super_admin(&db, &authed.email).await?;
windmill_common::secret_backend::test_vault_connection(&settings, Some(&db)).await?;
Ok("Successfully connected to HashiCorp Vault".to_string())
}
/// Migrate existing secrets from database to HashiCorp Vault
///
/// This endpoint reads all encrypted secrets from the database, decrypts them,
/// and stores them in HashiCorp Vault. The database values are NOT deleted
/// automatically to allow for rollback if needed.
///
/// This is an Enterprise Edition feature.
#[cfg(all(feature = "private", feature = "enterprise"))]
pub async fn migrate_secrets_to_vault(
Extension(db): Extension<DB>,
authed: ApiAuthed,
Json(settings): Json<VaultSettings>,
) -> JsonResult<SecretMigrationReport> {
require_super_admin(&db, &authed.email).await?;
let report = windmill_common::secret_backend::migrate_secrets_to_vault(&db, &settings).await?;
Ok(Json(report))
}
/// Migrate secrets from HashiCorp Vault back to database
///
/// This endpoint reads all secrets from HashiCorp Vault, encrypts them using
/// the workspace encryption keys, and stores them in the database. The Vault
/// values are NOT deleted automatically to allow for rollback if needed.
///
/// This is an Enterprise Edition feature.
#[cfg(all(feature = "private", feature = "enterprise"))]
pub async fn migrate_secrets_to_database(
Extension(db): Extension<DB>,
authed: ApiAuthed,
Json(settings): Json<VaultSettings>,
) -> JsonResult<SecretMigrationReport> {
require_super_admin(&db, &authed.email).await?;
let report =
windmill_common::secret_backend::migrate_secrets_to_database(&db, &settings).await?;
Ok(Json(report))
}
// ============================================================================
// JWKS Endpoint for Vault JWT Authentication
// ============================================================================
/// JSON Web Key Set response structure
#[derive(Serialize)]
pub struct JwksResponse {
pub keys: Vec<serde_json::Value>,
}
/// JWKS endpoint for HashiCorp Vault to validate JWTs
///
/// Vault calls this endpoint to fetch the public keys used to verify
/// JWTs generated by Windmill for authentication.
///
/// In the open-source version, this returns an empty JWKS.
/// The Enterprise Edition provides the actual key set.
pub async fn get_jwks() -> JsonResult<JwksResponse> {
// Open source version returns empty JWKS
// Enterprise Edition will override this with actual public keys
#[cfg(not(feature = "enterprise"))]
{
Ok(Json(JwksResponse { keys: vec![] }))
}
#[cfg(feature = "enterprise")]
{
// In enterprise mode, the actual keys would be fetched from global settings
// For now, return empty - the EE implementation would override this
Ok(Json(JwksResponse { keys: vec![] }))
}
}
backend/windmill-api/src/users.rs
+40
View File
@@ -19,6 +19,7 @@ use crate::db::ApiAuthed;
pub use crate::auth::Tokened;
use crate::secret_backend_ext::rename_vault_secrets_with_prefix;
use crate::utils::{
generate_instance_wide_unique_username, get_instance_username_or_create_pending,
};
@@ -2661,6 +2662,7 @@ async fn rename_user(
}
update_username_in_workpsace(
&mut tx,
&db,
&user_email,
&w_u.username,
&ru.new_username,
@@ -2688,6 +2690,7 @@ async fn rename_user(
async fn update_username_in_workpsace<'c>(
tx: &mut sqlx::Transaction<'c, sqlx::Postgres>,
db: &DB,
email: &str,
old_username: &str,
new_username: &str,
@@ -2805,6 +2808,43 @@ async fn update_username_in_workpsace<'c>(
// ---- variables ----
// Handle Vault secret renames before updating paths in DB
let old_prefix = format!("u/{}/", old_username);
let new_prefix = format!("u/{}/", new_username);
// Fetch all Vault-stored secret variables under this user's path
let vault_secrets: Vec<(String, String)> = sqlx::query!(
r#"SELECT path, value FROM variable
WHERE path LIKE ('u/' || $1 || '/%')
AND workspace_id = $2
AND is_secret = true
AND value LIKE '$vault:%'"#,
old_username,
w_id
)
.fetch_all(&mut **tx)
.await?
.into_iter()
.map(|r| (r.path, r.value))
.collect();
// Rename secrets in Vault and get the new values
let vault_updates =
rename_vault_secrets_with_prefix(db, w_id, &old_prefix, &new_prefix, vault_secrets).await?;
// Update the values in the DB for renamed Vault secrets (using OLD path, before path update)
for (old_path, new_value) in vault_updates {
sqlx::query!(
"UPDATE variable SET value = $1 WHERE path = $2 AND workspace_id = $3",
new_value,
old_path,
w_id
)
.execute(&mut **tx)
.await?;
}
// Now update the paths in the database
sqlx::query!(
r#"UPDATE variable SET path = REGEXP_REPLACE(path,'u/' || $2 || '/(.*)','u/' || $1 || '/\1') WHERE path LIKE ('u/' || $2 || '/%') AND workspace_id = $3"#,
new_username,
backend/windmill-api/src/variables.rs
+76 -21
View File
@@ -8,6 +8,10 @@
use crate::{
db::{ApiAuthed, DB},
secret_backend_ext::{
delete_secret_from_backend, get_secret_value, is_vault_stored_value, rename_vault_secret,
store_secret_value,
},
users::{maybe_refresh_folders, require_owner_of_path},
utils::{check_scopes, BulkDeleteRequest},
webhook_util::{WebhookMessage, WebhookShared},
@@ -39,7 +43,7 @@ use crate::var_resource_cache::{cache_variable, get_cached_variable};
use lazy_static::lazy_static;
use serde::Deserialize;
use sqlx::{Acquire, Postgres, Transaction};
use windmill_common::variables::{decrypt, encrypt};
use windmill_common::variables::encrypt;
use windmill_git_sync::{handle_deployment_metadata, DeployedObject};
lazy_static! {
@@ -210,13 +214,8 @@ async fn get_variable(
return Err(Error::internal_err("Require oauth2 feature".to_string()));
} else if !value.is_empty() && decrypt_secret {
let _ = tx.commit().await;
let mc = build_crypt(&db, &w_id).await?;
Some(decrypt(&mc, value).map_err(|e| {
Error::internal_err(format!(
"Error decrypting variable {}: {}",
variable.path, e
))
})?)
// Use secret backend for decryption (supports both DB and Vault)
Some(get_secret_value(&db, &w_id, &variable.path, &value).await?)
} else if q.include_encrypted.unwrap_or(false) {
Some(value)
} else {
@@ -355,8 +354,8 @@ async fn create_variable(
check_path_conflict(&db, &w_id, &variable.path).await?;
let value = if variable.is_secret && !already_encrypted.unwrap_or(false) {
let mc = build_crypt(&db, &w_id).await?;
encrypt(&mc, &variable.value)
// Use secret backend for encryption (supports both DB and Vault)
store_secret_value(&db, &w_id, &variable.path, &variable.value).await?
} else {
variable.value
};
@@ -436,6 +435,16 @@ async fn delete_variable(
check_scopes(&authed, || format!("variables:write:{}", path))?;
// Check if variable is a secret before deleting (for Vault cleanup)
let is_secret = sqlx::query_scalar!(
"SELECT is_secret FROM variable WHERE path = $1 AND workspace_id = $2",
path,
&w_id
)
.fetch_optional(&db)
.await?
.unwrap_or(false);
let mut tx = user_db.begin(&authed).await?;
sqlx::query!(
@@ -465,6 +474,11 @@ async fn delete_variable(
tx.commit().await?;
// If variable was a secret, also delete from Vault backend (if configured)
if is_secret {
delete_secret_from_backend(&db, &w_id, path).await?;
}
handle_deployment_metadata(
&authed.email,
&authed.username,
@@ -496,6 +510,15 @@ async fn delete_variables_bulk(
check_scopes(&authed, || format!("variables:write:{}", path))?;
}
// Query which paths are secrets before deletion (for Vault cleanup)
let secret_paths: Vec<String> = sqlx::query_scalar!(
"SELECT path FROM variable WHERE path = ANY($1) AND workspace_id = $2 AND is_secret = true",
&request.paths,
&w_id
)
.fetch_all(&db)
.await?;
let mut tx = user_db.begin(&authed).await?;
let deleted_paths = sqlx::query_scalar!(
@@ -526,6 +549,13 @@ async fn delete_variables_bulk(
tx.commit().await?;
// Delete secrets from Vault backend (if configured)
for path in &secret_paths {
if deleted_paths.contains(path) {
delete_secret_from_backend(&db, &w_id, path).await?;
}
}
try_join_all(deleted_paths.iter().map(|path| {
handle_deployment_metadata(
&authed.email,
@@ -589,7 +619,9 @@ async fn update_variable(
sqlb.set_str("path", npath);
}
let ns_value_is_none = ns.value.is_none();
if let Some(nvalue) = ns.value {
// Determine the target path for storing secrets (use new path if provided)
let target_path = ns.path.as_deref().unwrap_or(path);
if let Some(nvalue) = ns.value.clone() {
let is_secret = if ns.is_secret.is_some() {
ns.is_secret.unwrap()
} else {
@@ -604,8 +636,9 @@ async fn update_variable(
};
let value = if is_secret && !already_encrypted.unwrap_or(false) {
let mc = build_crypt(&db, &w_id).await?;
encrypt(&mc, &nvalue)
// Use secret backend for encryption (supports both DB and Vault)
// Store at target_path (new path if renaming, otherwise current path)
store_secret_value(&db, &w_id, target_path, &nvalue).await?
} else {
nvalue
};
@@ -654,11 +687,38 @@ async fn update_variable(
let mut tx: Transaction<'_, Postgres> = user_db.begin(&authed).await?;
if let Some(npath) = ns.path {
if let Some(npath) = ns.path.clone() {
if npath != path {
check_path_conflict(&db, &w_id, &npath).await?;
require_owner_of_path(&authed, path)?;
// Handle Vault secret rename if the variable is a secret stored in Vault
let current_var = sqlx::query!(
"SELECT value, is_secret FROM variable WHERE path = $1 AND workspace_id = $2",
path,
w_id
)
.fetch_optional(&mut *tx)
.await?;
if let Some(var) = current_var {
if var.is_secret && is_vault_stored_value(&var.value) {
if ns.value.is_some() {
// New value was provided and already stored at new path
// Just delete the old secret from Vault
delete_secret_from_backend(&db, &w_id, path).await?;
} else {
// No new value - rename the secret in Vault
if let Some(new_value) =
rename_vault_secret(&db, &w_id, path, &npath, &var.value).await?
{
// Update the variable's value to point to the new Vault path
sqlb.set_str("value", &new_value);
}
}
}
}
let mut v = sqlx::query_scalar!(
"SELECT value FROM resource WHERE path = $1 AND workspace_id = $2",
path,
@@ -835,13 +895,8 @@ pub async fn get_value_internal<'a>(
#[cfg(not(feature = "oauth2"))]
return Err(Error::internal_err("Require oauth2 feature".to_string()));
} else if !value.is_empty() {
let mc = build_crypt(db_with_opt_authed.db(), &w_id).await?;
decrypt(&mc, value).map_err(|e| {
Error::internal_err(format!(
"Error decrypting variable {}: {}",
variable.path, e
))
})?
// Use secret backend for decryption (supports both DB and Vault)
get_secret_value(db_with_opt_authed.db(), &w_id, &variable.path, &value).await?
} else {
"".to_string()
}
backend/windmill-api/src/workspaces.rs
+95
View File
@@ -127,6 +127,7 @@ pub fn workspaced_service() -> Router {
.route("/edit_copilot_config", post(edit_copilot_config))
.route("/get_copilot_info", get(get_copilot_info))
.route("/edit_error_handler", post(edit_error_handler))
.route("/edit_success_handler", post(edit_success_handler))
.route(
"/edit_large_file_storage_config",
post(edit_large_file_storage_config),
@@ -270,6 +271,10 @@ pub struct WorkspaceSettings {
#[serde(skip_serializing_if = "Option::is_none")]
pub error_handler_muted_on_cancel: Option<bool>,
#[serde(skip_serializing_if = "Option::is_none")]
pub success_handler: Option<String>,
#[serde(skip_serializing_if = "Option::is_none")]
pub success_handler_extra_args: Option<serde_json::Value>,
#[serde(skip_serializing_if = "Option::is_none")]
pub large_file_storage: Option<serde_json::Value>, // effectively: DatasetsStorage
#[serde(skip_serializing_if = "Option::is_none")]
pub ducklake: Option<serde_json::Value>,
@@ -442,6 +447,12 @@ pub struct EditErrorHandler {
pub error_handler_muted_on_cancel: Option<bool>,
}
#[derive(Deserialize)]
pub struct EditSuccessHandler {
pub success_handler: Option<String>,
pub success_handler_extra_args: Option<serde_json::Value>,
}
lazy_static::lazy_static! {
pub static ref EMAIL_REGEXP: Regex = Regex::new(r"^[a-zA-Z0-9._%+-]+@[a-zA-Z0-9.-]+\.[a-zA-Z]{2,}$").unwrap();
}
@@ -555,6 +566,8 @@ async fn get_settings(
error_handler,
error_handler_extra_args,
error_handler_muted_on_cancel,
success_handler,
success_handler_extra_args,
large_file_storage,
datatable,
ducklake,
@@ -2251,6 +2264,88 @@ async fn edit_error_handler(
Ok(format!("Edit error_handler for workspace {}", &w_id))
}
async fn edit_success_handler(
authed: ApiAuthed,
Extension(db): Extension<DB>,
Path(w_id): Path<String>,
Json(es): Json<EditSuccessHandler>,
) -> Result<String> {
require_admin(authed.is_admin, &authed.username)?;
let mut tx = db.begin().await?;
sqlx::query_as!(
Group,
"INSERT INTO group_ (workspace_id, name, summary, extra_perms) VALUES ($1, $2, $3, $4) ON CONFLICT DO NOTHING",
w_id,
"success_handler",
"The group the success handler acts on behalf of",
serde_json::json!({username_to_permissioned_as(&authed.username): true})
)
.execute(&mut *tx)
.await?;
if let Some(success_handler) = &es.success_handler {
sqlx::query!(
r#"
UPDATE
workspace_settings
SET
success_handler = $1,
success_handler_extra_args = $2
WHERE
workspace_id = $3
"#,
success_handler,
es.success_handler_extra_args,
&w_id
)
.execute(&mut *tx)
.await?;
} else {
sqlx::query!(
r#"
UPDATE
workspace_settings
SET
success_handler = NULL,
success_handler_extra_args = NULL
WHERE
workspace_id = $1
"#,
&w_id
)
.execute(&mut *tx)
.await?;
}
audit_log(
&mut *tx,
&authed,
"workspaces.edit_success_handler",
ActionKind::Update,
&w_id,
Some(&authed.email),
Some([("success_handler", &format!("{:?}", es.success_handler)[..])].into()),
)
.await?;
tx.commit().await?;
// Trigger git sync for success handler changes
handle_deployment_metadata(
&authed.email,
&authed.username,
&db,
&w_id,
windmill_git_sync::DeployedObject::Settings { setting_type: "success_handler".to_string() },
Some("Success handler configuration updated".to_string()),
false,
)
.await?;
Ok(format!("Edit success_handler for workspace {}", &w_id))
}
#[derive(Deserialize)]
struct NewEnvironmentVariable {
name: String,
backend/windmill-common/src/global_settings.rs
+3
View File
@@ -47,6 +47,7 @@ pub const JWT_SECRET_SETTING: &str = "jwt_secret";
pub const EMAIL_DOMAIN_SETTING: &str = "email_domain";
pub const OTEL_SETTING: &str = "otel";
pub const APP_WORKSPACED_ROUTE_SETTING: &str = "app_workspaced_route";
pub const SECRET_BACKEND_SETTING: &str = "secret_backend";
pub const ENV_SETTINGS: &[&str] = &[
"DISABLE_NSJAIL",
@@ -115,6 +116,8 @@ pub const ENV_SETTINGS: &[&str] = &[
"PG_SCHEMA",
"PG_LISTENER_REFRESH_PERIOD_SECS",
"AI_REQUEST_TIMEOUT_SECONDS",
"JOB_CLEANUP_BATCH_SIZE",
"JOB_CLEANUP_MAX_BATCHES",
];
use crate::error;
backend/windmill-common/src/lib.rs
+1
View File
@@ -78,6 +78,7 @@ pub mod result_stream;
pub mod runnable_settings;
pub mod s3_helpers;
pub mod schedule;
pub mod secret_backend;
pub mod schema;
pub mod scripts;
pub mod server;
backend/windmill-common/src/secret_backend/database.rs
+126
View File
@@ -0,0 +1,126 @@
/*
* Author: Ruben Fiszel
* Copyright: Windmill Labs, Inc 2024
* This file and its contents are licensed under the AGPLv3 License.
* Please see the included NOTICE for copyright information and
* LICENSE-AGPL for a copy of the license.
*/
//! Database backend for secret storage
//!
//! This is the default backend that stores secrets encrypted in the PostgreSQL database
//! using the existing magic_crypt encryption with workspace-specific keys.
use async_trait::async_trait;
use crate::db::DB;
use crate::error::{Error, Result};
use crate::variables::{build_crypt, decrypt, encrypt};
use super::SecretBackend;
/// Database-backed secret storage
///
/// This backend stores secrets encrypted in the `variable` table using
/// the workspace's encryption key. This is the default and original
/// behavior of Windmill.
pub struct DatabaseBackend {
db: DB,
}
impl DatabaseBackend {
/// Create a new database backend
pub fn new(db: DB) -> Self {
Self { db }
}
}
#[async_trait]
impl SecretBackend for DatabaseBackend {
async fn get_secret(&self, workspace_id: &str, path: &str) -> Result<String> {
let variable = sqlx::query!(
"SELECT value FROM variable WHERE path = $1 AND workspace_id = $2 AND is_secret = true",
path,
workspace_id
)
.fetch_optional(&self.db)
.await?;
let variable = variable.ok_or_else(|| {
Error::NotFound(format!(
"Secret variable {} not found in workspace {}",
path, workspace_id
))
})?;
let value = variable.value;
if value.is_empty() {
return Ok(String::new());
}
let mc = build_crypt(&self.db, workspace_id).await?;
decrypt(&mc, value).map_err(|e| {
Error::internal_err(format!("Error decrypting variable {}: {}", path, e))
})
}
async fn set_secret(&self, workspace_id: &str, path: &str, value: &str) -> Result<()> {
let mc = build_crypt(&self.db, workspace_id).await?;
let encrypted_value = encrypt(&mc, value);
// Update the value in the database
// Note: This assumes the variable row already exists (created via the normal API)
let result = sqlx::query!(
"UPDATE variable SET value = $1 WHERE path = $2 AND workspace_id = $3 AND is_secret = true",
encrypted_value,
path,
workspace_id
)
.execute(&self.db)
.await?;
if result.rows_affected() == 0 {
return Err(Error::NotFound(format!(
"Secret variable {} not found in workspace {}",
path, workspace_id
)));
}
Ok(())
}
async fn delete_secret(&self, _workspace_id: &str, _path: &str) -> Result<()> {
// For database backend, deletion is handled by the normal variable deletion flow
// The encrypted value is just deleted along with the row
// This method is a no-op for database backend since the caller handles the DELETE
Ok(())
}
fn backend_name(&self) -> &'static str {
"database"
}
}
/// Encrypt a value for storage in the database
///
/// This is a convenience function for use when creating new secrets.
pub async fn encrypt_for_database(db: &DB, workspace_id: &str, value: &str) -> Result<String> {
let mc = build_crypt(db, workspace_id).await?;
Ok(encrypt(&mc, value))
}
/// Decrypt a value from the database
///
/// This is a convenience function for reading secrets.
pub async fn decrypt_from_database(
db: &DB,
workspace_id: &str,
encrypted_value: String,
) -> Result<String> {
if encrypted_value.is_empty() {
return Ok(String::new());
}
let mc = build_crypt(db, workspace_id).await?;
decrypt(&mc, encrypted_value)
.map_err(|e| Error::internal_err(format!("Error decrypting value: {}", e)))
}
backend/windmill-common/src/secret_backend/mod.rs
+133
View File
@@ -0,0 +1,133 @@
/*
* Author: Ruben Fiszel
* Copyright: Windmill Labs, Inc 2024
* This file and its contents are licensed under the AGPLv3 License.
* Please see the included NOTICE for copyright information and
* LICENSE-AGPL for a copy of the license.
*/
//! Secret Backend abstraction for storing secrets in external vaults
//!
//! This module provides a trait-based abstraction for secret storage,
//! allowing secrets to be stored in the database (default) or in external
//! vaults like HashiCorp Vault (Enterprise Edition).
pub mod database;
#[cfg(feature = "private")]
pub mod vault_ee;
pub mod vault_oss;
#[cfg(test)]
mod tests;
#[cfg(feature = "private")]
pub use vault_ee::*;
#[cfg(not(feature = "private"))]
pub use vault_oss::*;
use async_trait::async_trait;
use serde::{Deserialize, Serialize};
use crate::error::Result;
/// Trait for secret storage backends
///
/// Implementations of this trait handle the storage and retrieval of secrets.
/// The default implementation stores secrets encrypted in the database.
/// Enterprise Edition supports HashiCorp Vault as an alternative backend.
#[async_trait]
pub trait SecretBackend: Send + Sync {
/// Retrieve a secret value
///
/// # Arguments
/// * `workspace_id` - The workspace identifier
/// * `path` - The path/name of the secret variable
///
/// # Returns
/// The decrypted secret value
async fn get_secret(&self, workspace_id: &str, path: &str) -> Result<String>;
/// Store a secret value
///
/// # Arguments
/// * `workspace_id` - The workspace identifier
/// * `path` - The path/name of the secret variable
/// * `value` - The plaintext secret value to store
async fn set_secret(&self, workspace_id: &str, path: &str, value: &str) -> Result<()>;
/// Delete a secret
///
/// # Arguments
/// * `workspace_id` - The workspace identifier
/// * `path` - The path/name of the secret variable
async fn delete_secret(&self, workspace_id: &str, path: &str) -> Result<()>;
/// Get the name of this backend for logging/debugging
fn backend_name(&self) -> &'static str;
}
/// Configuration for secret storage backend
///
/// This enum is stored in global_settings and determines which backend
/// is used for secret storage at the instance level.
#[derive(Debug, Clone, Serialize, Deserialize, PartialEq)]
#[serde(tag = "type")]
pub enum SecretBackendConfig {
/// Store secrets encrypted in the database (default behavior)
Database,
/// Store secrets in HashiCorp Vault (Enterprise Edition only)
HashiCorpVault(VaultSettings),
}
impl Default for SecretBackendConfig {
fn default() -> Self {
SecretBackendConfig::Database
}
}
/// Settings for HashiCorp Vault integration
#[derive(Debug, Clone, Serialize, Deserialize, PartialEq)]
pub struct VaultSettings {
/// Vault server address (e.g., "https://vault.company.com:8200")
pub address: String,
/// KV v2 mount path (e.g., "windmill")
pub mount_path: String,
/// JWT auth role name configured in Vault (used for JWT/OIDC auth)
/// Optional - if not provided, token auth is used
#[serde(skip_serializing_if = "Option::is_none")]
pub jwt_role: Option<String>,
/// Vault Enterprise namespace (optional)
#[serde(skip_serializing_if = "Option::is_none")]
pub namespace: Option<String>,
/// Static Vault token for testing/development (optional)
/// If provided, this is used instead of JWT authentication
#[serde(skip_serializing_if = "Option::is_none")]
pub token: Option<String>,
}
/// Result of a secret migration operation
#[derive(Debug, Clone, Serialize, Deserialize)]
pub struct SecretMigrationReport {
/// Total number of secrets found
pub total_secrets: usize,
/// Number of secrets successfully migrated
pub migrated_count: usize,
/// Number of secrets that failed to migrate
pub failed_count: usize,
/// Details of any failures
pub failures: Vec<SecretMigrationFailure>,
}
/// Details of a failed secret migration
#[derive(Debug, Clone, Serialize, Deserialize)]
pub struct SecretMigrationFailure {
/// Workspace ID where the secret is located
pub workspace_id: String,
/// Path of the secret that failed to migrate
pub path: String,
/// Error message
pub error: String,
}
backend/windmill-common/src/secret_backend/tests.rs
+143
View File
@@ -0,0 +1,143 @@
/*
* Integration tests for secret backend
*
* These tests require a running Vault instance at http://127.0.0.1:8200
* with the token "test-root-token" and a KV v2 mount at "windmill".
*
* Run Vault dev server:
* podman run -d --name vault-test --rm -p 8200:8200 \
* -e 'VAULT_DEV_ROOT_TOKEN_ID=test-root-token' \
* -e 'VAULT_DEV_LISTEN_ADDRESS=0.0.0.0:8200' \
* docker.io/hashicorp/vault:latest
*
* Then enable the windmill mount:
* curl --header "X-Vault-Token: test-root-token" \
* --request POST \
* --data '{"type": "kv", "options": {"version": "2"}}' \
* http://127.0.0.1:8200/v1/sys/mounts/windmill
*/
#[cfg(test)]
mod tests {
use crate::secret_backend::{SecretBackend, VaultBackend, VaultSettings};
fn test_settings() -> VaultSettings {
VaultSettings {
address: "http://127.0.0.1:8200".to_string(),
mount_path: "windmill".to_string(),
jwt_role: Some("windmill-secrets".to_string()),
namespace: None,
token: Some("test-root-token".to_string()),
}
}
#[tokio::test]
#[ignore] // Run with --ignored to execute
async fn test_vault_write_and_read() {
let settings = test_settings();
let backend = VaultBackend::new(settings);
let workspace_id = "test-ws-1";
let path = "test-secret";
let value = "my-super-secret-value";
// Write secret
backend
.set_secret(workspace_id, path, value)
.await
.expect("Failed to write secret");
// Read secret
let read_value = backend
.get_secret(workspace_id, path)
.await
.expect("Failed to read secret");
assert_eq!(read_value, value);
// Delete secret
backend
.delete_secret(workspace_id, path)
.await
.expect("Failed to delete secret");
// Verify deleted
let result = backend.get_secret(workspace_id, path).await;
assert!(result.is_err(), "Secret should be deleted");
}
#[tokio::test]
#[ignore]
async fn test_vault_multiple_secrets() {
let settings = test_settings();
let backend = VaultBackend::new(settings);
let workspace_id = "test-ws-2";
// Write multiple secrets
for i in 0..5 {
let path = format!("secret-{}", i);
let value = format!("value-{}", i);
backend
.set_secret(workspace_id, &path, &value)
.await
.expect(&format!("Failed to write secret-{}", i));
}
// Read and verify
for i in 0..5 {
let path = format!("secret-{}", i);
let expected = format!("value-{}", i);
let value = backend
.get_secret(workspace_id, &path)
.await
.expect(&format!("Failed to read secret-{}", i));
assert_eq!(value, expected);
}
// Cleanup
for i in 0..5 {
let path = format!("secret-{}", i);
backend
.delete_secret(workspace_id, &path)
.await
.expect(&format!("Failed to delete secret-{}", i));
}
}
#[tokio::test]
#[ignore]
async fn test_vault_overwrite_secret() {
let settings = test_settings();
let backend = VaultBackend::new(settings);
let workspace_id = "test-ws-3";
let path = "overwrite-test";
// Write initial value
backend
.set_secret(workspace_id, path, "initial-value")
.await
.expect("Failed to write initial value");
// Overwrite
backend
.set_secret(workspace_id, path, "new-value")
.await
.expect("Failed to overwrite");
// Read and verify
let value = backend
.get_secret(workspace_id, path)
.await
.expect("Failed to read");
assert_eq!(value, "new-value");
// Cleanup
backend
.delete_secret(workspace_id, path)
.await
.expect("Failed to delete");
}
}
backend/windmill-common/src/secret_backend/vault_oss.rs
+107
View File
@@ -0,0 +1,107 @@
/*
* Author: Ruben Fiszel
* Copyright: Windmill Labs, Inc 2024
* This file and its contents are licensed under the AGPLv3 License.
* Please see the included NOTICE for copyright information and
* LICENSE-AGPL for a copy of the license.
*/
//! HashiCorp Vault secret backend stubs (Open Source Edition)
//!
//! This module provides stub implementations for Vault integration.
//! The actual Vault integration requires Enterprise Edition.
use std::sync::Arc;
use crate::db::DB;
use crate::error::{Error, Result};
use super::{database::DatabaseBackend, SecretBackend, SecretBackendConfig, SecretMigrationReport, VaultSettings};
/// Stub VaultBackend for OSS - all operations return EE required error
pub struct VaultBackend;
impl VaultBackend {
pub fn new(_settings: VaultSettings) -> Self {
Self
}
}
#[async_trait::async_trait]
impl SecretBackend for VaultBackend {
async fn get_secret(&self, _workspace_id: &str, _path: &str) -> Result<String> {
Err(Error::internal_err(
"HashiCorp Vault integration requires Enterprise Edition".to_string(),
))
}
async fn set_secret(&self, _workspace_id: &str, _path: &str, _value: &str) -> Result<()> {
Err(Error::internal_err(
"HashiCorp Vault integration requires Enterprise Edition".to_string(),
))
}
async fn delete_secret(&self, _workspace_id: &str, _path: &str) -> Result<()> {
Err(Error::internal_err(
"HashiCorp Vault integration requires Enterprise Edition".to_string(),
))
}
fn backend_name(&self) -> &'static str {
"hashicorp_vault"
}
}
/// Create the appropriate secret backend based on configuration
///
/// In OSS, always returns DatabaseBackend regardless of config.
/// Vault configuration is ignored with a warning.
pub async fn create_secret_backend(
db: DB,
config: &SecretBackendConfig,
) -> Result<Arc<dyn SecretBackend>> {
match config {
SecretBackendConfig::Database => Ok(Arc::new(DatabaseBackend::new(db))),
SecretBackendConfig::HashiCorpVault(_) => {
tracing::warn!(
"HashiCorp Vault is configured but requires Enterprise Edition. \
Falling back to database backend."
);
Ok(Arc::new(DatabaseBackend::new(db)))
}
}
}
/// Test connection to Vault (OSS stub)
pub async fn test_vault_connection(_settings: &VaultSettings, _db: Option<&DB>) -> Result<()> {
Err(Error::internal_err(
"HashiCorp Vault integration requires Enterprise Edition".to_string(),
))
}
/// Migrate secrets from database to Vault (OSS stub)
pub async fn migrate_secrets_to_vault(
_db: &DB,
_settings: &VaultSettings,
) -> Result<SecretMigrationReport> {
Err(Error::internal_err(
"HashiCorp Vault integration requires Enterprise Edition".to_string(),
))
}
/// Migrate secrets from Vault back to database (OSS stub)
pub async fn migrate_secrets_to_database(
_db: &DB,
_settings: &VaultSettings,
) -> Result<SecretMigrationReport> {
Err(Error::internal_err(
"HashiCorp Vault integration requires Enterprise Edition".to_string(),
))
}
/// Generate a JWT for Vault authentication (OSS stub)
pub async fn generate_vault_jwt(_db: &DB, _vault_address: &str) -> Result<String> {
Err(Error::internal_err(
"HashiCorp Vault integration requires Enterprise Edition".to_string(),
))
}
backend/windmill-mcp/Cargo.toml
+7 -1
View File
@@ -10,7 +10,7 @@ path = "src/lib.rs"
[features]
default = []
server = ["rmcp/transport-streamable-http-server", "rmcp/transport-streamable-http-server-session", "rmcp/transport-worker"]
server = ["rmcp/transport-streamable-http-server", "rmcp/transport-streamable-http-server-session", "rmcp/transport-worker", "dep:sqlx", "dep:async-trait", "dep:http", "dep:tokio-util", "dep:tokio"]
auth = ["rmcp/auth", "dep:oauth2"]
[dependencies]
@@ -22,3 +22,9 @@ serde.workspace = true
serde_json.workspace = true
tracing.workspace = true
rmcp.workspace = true
sqlx = { workspace = true, optional = true }
async-trait = { workspace = true, optional = true }
http = { workspace = true, optional = true }
tokio-util = { workspace = true, features = ["rt"], optional = true }
tokio = { workspace = true, optional = true }
futures.workspace = true
backend/windmill-mcp/src/client/mod.rs
+209
View File
@@ -0,0 +1,209 @@
//! MCP Client implementation
//!
//! This module provides functionality for connecting to external MCP servers
//! and executing tools on them.
mod types;
pub use types::{McpResource, McpToolSource};
use anyhow::{Context, Result};
use reqwest::header::{HeaderMap, HeaderName, HeaderValue};
use rmcp::{
model::{
CallToolRequestParam, ClientCapabilities, ClientInfo, Implementation,
InitializeRequestParam, Tool as McpTool,
},
service::RunningService,
transport::{
streamable_http_client::StreamableHttpClientTransportConfig, StreamableHttpClientTransport,
},
RoleClient, ServiceExt,
};
use serde_json::{json, Value};
use std::str::FromStr;
use windmill_common::variables::get_secret_value_as_admin;
use windmill_common::DB;
/// MCP client for communicating with external MCP servers
pub struct McpClient {
/// The underlying rmcp client
client: RunningService<RoleClient, InitializeRequestParam>,
/// Cached list of available tools from the server
available_tools: Vec<McpTool>,
}
impl McpClient {
/// Create a new MCP client from a resource configuration
pub async fn from_resource(resource: McpResource, db: &DB, w_id: &str) -> Result<Self> {
// Build custom reqwest client with headers if provided
let mut headers = HeaderMap::new();
if let Some(token_path) = &resource.token {
if !token_path.trim().is_empty() {
let value =
get_secret_value_as_admin(db, w_id, token_path.trim_start_matches("$var:"))
.await?;
headers.insert(
HeaderName::from_static("authorization"),
HeaderValue::from_str(format!("Bearer {}", value).as_str())?,
);
}
}
if let Some(resource_headers) = &resource.headers {
for (key, value) in resource_headers {
match (HeaderName::from_str(key), HeaderValue::from_str(value)) {
(Ok(name), Ok(value)) => {
headers.insert(name, value);
}
_ => {
tracing::warn!("Invalid header: {}={}", key, value);
}
}
}
}
let reqwest_client = reqwest::Client::builder()
.default_headers(headers)
.build()
.context("Failed to build HTTP client")?;
// Create the HTTP transport with custom client
let config = StreamableHttpClientTransportConfig::with_uri(resource.url.as_str());
let transport = StreamableHttpClientTransport::with_client(reqwest_client, config);
// Set up client info
let client_info = ClientInfo {
protocol_version: Default::default(),
capabilities: ClientCapabilities::default(),
client_info: Implementation {
name: "windmill-ai-agent".to_string(),
title: Some("Windmill AI Agent".to_string()),
version: env!("CARGO_PKG_VERSION").to_string(),
website_url: None,
icons: None,
},
};
// Initialize the connection
let client = client_info
.serve(transport)
.await
.context("Failed to connect to MCP server")?;
// Immediately fetch available tools
let available_tools = client
.list_tools(Default::default())
.await
.context("Failed to list tools from MCP server")?
.tools;
Ok(Self { client, available_tools })
}
/// Get the list of available tools from the MCP server
pub fn available_tools(&self) -> &[McpTool] {
&self.available_tools
}
/// Call a tool on the MCP server, with openai-style arguments
pub async fn call_tool(&self, name: &str, arguments: &str) -> Result<serde_json::Value> {
// Convert OpenAI-style arguments to MCP format
let mcp_args =
Self::openai_args_to_mcp_args(arguments).context("Failed to parse tool arguments")?;
let result = self
.client
.call_tool(CallToolRequestParam {
name: name.to_string().into(),
arguments: mcp_args,
task: None,
})
.await
.context(format!("Failed to call MCP tool: {}", name))?;
// Convert the result to a JSON value
// MCP tools return ToolResult which contains content array
let result_json =
serde_json::to_value(&result).context("Failed to serialize MCP tool result")?;
Ok(result_json)
}
/// Close the connection
pub async fn shutdown(self) -> Result<()> {
self.client.cancel().await?;
Ok(())
}
/// Fix array schemas to ensure they have the required 'items' property
/// OpenAI requires all array types to have an 'items' field. MCP servers may
/// return schemas without this field, so we add a default.
pub fn fix_array_schemas(schema: &mut Value) {
if let Value::Object(obj) = schema {
// Check if this is an array type
if let Some(type_val) = obj.get("type") {
let is_array = match type_val {
Value::String(s) => s == "array",
Value::Array(arr) => arr.iter().any(|v| v.as_str() == Some("array")),
_ => false,
};
// If it's an array and missing 'items', add a default
if is_array && !obj.contains_key("items") {
obj.insert("items".to_string(), json!({}));
}
}
// Recursively fix nested schemas
if let Some(Value::Object(props)) = obj.get_mut("properties") {
for value in props.values_mut() {
Self::fix_array_schemas(value);
}
}
// Fix items if present (for nested arrays)
if let Some(items) = obj.get_mut("items") {
Self::fix_array_schemas(items);
}
// Fix oneOf, anyOf, allOf schemas
for key in &["oneOf", "anyOf", "allOf"] {
if let Some(Value::Array(schemas)) = obj.get_mut(*key) {
for schema in schemas {
Self::fix_array_schemas(schema);
}
}
}
// Fix additionalProperties if it's a schema
if let Some(additional) = obj.get_mut("additionalProperties") {
if additional.is_object() {
Self::fix_array_schemas(additional);
}
}
}
}
/// Convert OpenAI-style tool call arguments to MCP format
/// OpenAI sends arguments as a JSON string, MCP expects a Map
fn openai_args_to_mcp_args(
args_str: &str,
) -> Result<Option<serde_json::Map<String, serde_json::Value>>> {
if args_str.trim().is_empty() {
return Ok(None);
}
let args_value: serde_json::Value =
serde_json::from_str(args_str).context("Failed to parse tool call arguments")?;
match args_value {
serde_json::Value::Object(map) => Ok(Some(map)),
serde_json::Value::Null => Ok(None),
_ => Ok(Some(
vec![("value".to_string(), args_value)]
.into_iter()
.collect(),
)),
}
}
}
backend/windmill-mcp/src/client/types.rs
+32
View File
@@ -0,0 +1,32 @@
//! Client-specific types for MCP
//!
//! Contains configuration and metadata types used for MCP client connections.
use serde::{Deserialize, Serialize};
use std::collections::HashMap;
/// MCP server resource configuration
#[derive(Debug, Clone, Serialize, Deserialize)]
pub struct McpResource {
/// Name of the MCP resource (used for prefixing tools)
pub name: String,
/// HTTP URL for the MCP server endpoint
pub url: String,
/// Optional token for authentication
#[serde(skip_serializing_if = "Option::is_none")]
pub token: Option<String>,
/// Optional headers
#[serde(skip_serializing_if = "Option::is_none")]
pub headers: Option<HashMap<String, String>>,
}
/// Metadata for tracking MCP tool sources
#[derive(Debug, Clone, Serialize, Deserialize)]
pub struct McpToolSource {
/// Name of the MCP resource this tool comes from
pub name: String,
/// Original tool name in the MCP server
pub tool_name: String,
/// Path of the MCP resource
pub resource_path: String,
}
backend/windmill-mcp/src/common/mod.rs
+16
View File
@@ -0,0 +1,16 @@
//! Common types and utilities for MCP server and client
//!
//! This module contains shared data structures, transformation utilities,
//! and scope parsing functionality used throughout the MCP implementation.
pub mod schema;
pub mod scope;
pub mod transform;
pub mod types;
pub use schema::convert_schema_to_schema_type;
pub use scope::{is_resource_allowed, parse_mcp_scopes, McpScopeConfig};
pub use transform::{
apply_key_transformation, reverse_transform, reverse_transform_key, transform_path,
};
pub use types::*;
backend/windmill-mcp/src/common/schema.rs
+41
View File
@@ -0,0 +1,41 @@
//! Schema conversion utilities for MCP server
//!
//! Contains functions for converting Windmill schemas into MCP-compatible formats.
use std::collections::HashSet;
use serde_json::Value;
use super::types::SchemaType;
use windmill_common::scripts::Schema;
/// Convert a Windmill Schema to a SchemaType
pub fn convert_schema_to_schema_type(schema: Option<Schema>) -> SchemaType {
let schema_obj = if let Some(ref s) = schema {
match serde_json::from_str::<SchemaType>(s.0.get()) {
Ok(val) => val,
Err(_) => SchemaType::default(),
}
} else {
SchemaType::default()
};
schema_obj
}
/// Extract resource type keys from a schema
///
/// Scans the schema properties for fields with format "resource-{type}"
/// and returns a set of all unique resource type names found.
pub fn extract_resource_types_from_schema(schema: &SchemaType) -> HashSet<String> {
let mut resource_types = HashSet::new();
for (_key, prop_value) in schema.properties.iter() {
if let Value::Object(prop_map) = prop_value {
if let Some(Value::String(format_str)) = prop_map.get("format") {
if let Some(rt) = format_str.strip_prefix("resource-") {
resource_types.insert(rt.to_string());
}
}
}
}
resource_types
}
backend/windmill-api/src/mcp/utils/scope_matcher.rs → backend/windmill-mcp/src/common/scope.rs
+38 -10
View File
@@ -3,8 +3,6 @@
//! Contains utilities for parsing and matching MCP token scopes to determine
//! which scripts, flows, and endpoints a token has access to.
use windmill_mcp::server::ErrorData;
/// Configuration for MCP scopes parsed from token scopes
#[derive(Debug, Clone, Default)]
pub struct McpScopeConfig {
@@ -24,8 +22,26 @@ pub struct McpScopeConfig {
pub hub_apps: Option<String>,
}
impl McpScopeConfig {
/// Check if a resource is allowed based on its type and path
pub fn is_allowed(&self, resource_type: &str, path: &str) -> bool {
if self.all {
return true;
}
let patterns = match resource_type {
"script" => &self.scripts,
"flow" => &self.flows,
"endpoint" => &self.endpoints,
_ => return false,
};
is_resource_allowed(path, patterns)
}
}
/// Parse MCP scopes from token scope strings
pub fn parse_mcp_scopes(scopes: &[String]) -> Result<McpScopeConfig, ErrorData> {
pub fn parse_mcp_scopes(scopes: &[String]) -> Result<McpScopeConfig, String> {
let mut config = McpScopeConfig::default();
for scope in scopes {
@@ -69,19 +85,19 @@ pub fn parse_mcp_scopes(scopes: &[String]) -> Result<McpScopeConfig, ErrorData>
if let Some(resources) = scope.strip_prefix("mcp:scripts:") {
// New granular script scope: mcp:scripts:path1,path2,f/folder/*
config.scripts.extend(parse_resource_list(resources)?);
config.scripts.extend(parse_resource_list(resources));
continue;
}
if let Some(resources) = scope.strip_prefix("mcp:flows:") {
// New granular flow scope: mcp:flows:path1,path2,f/folder/*
config.flows.extend(parse_resource_list(resources)?);
config.flows.extend(parse_resource_list(resources));
continue;
}
if let Some(resources) = scope.strip_prefix("mcp:endpoints:") {
// New granular endpoint scope: mcp:endpoints:name1,name2
config.endpoints.extend(parse_resource_list(resources)?);
config.endpoints.extend(parse_resource_list(resources));
continue;
}
@@ -94,16 +110,16 @@ pub fn parse_mcp_scopes(scopes: &[String]) -> Result<McpScopeConfig, ErrorData>
}
/// Parse comma-separated resource list
fn parse_resource_list(resources: &str) -> Result<Vec<String>, ErrorData> {
fn parse_resource_list(resources: &str) -> Vec<String> {
if resources.is_empty() {
return Ok(vec![]);
return vec![];
}
Ok(resources
resources
.split(',')
.map(|s| s.trim().to_string())
.filter(|s| !s.is_empty())
.collect())
.collect()
}
/// Check if a resource path matches any pattern in the allowed list
@@ -226,4 +242,16 @@ mod tests {
let empty: Vec<String> = vec![];
assert!(!is_resource_allowed("any/path", &empty));
}
#[test]
fn test_scope_config_is_allowed() {
let mut config = McpScopeConfig::default();
config.scripts.push("u/admin/*".to_string());
config.flows.push("f/automation/*".to_string());
assert!(config.is_allowed("script", "u/admin/test"));
assert!(!config.is_allowed("script", "u/other/test"));
assert!(config.is_allowed("flow", "f/automation/test"));
assert!(!config.is_allowed("flow", "f/other/test"));
}
}
backend/windmill-api/src/mcp/utils/transform.rs → backend/windmill-mcp/src/common/transform.rs
+37 -2
View File
@@ -3,9 +3,9 @@
//! Contains functions for transforming paths, keys, and other identifiers
//! to make them compatible with MCP tool naming requirements.
use super::models::SchemaType;
use super::types::SchemaType;
// MCP clients do not allow names longer than 60 characters
/// MCP clients do not allow names longer than 60 characters
const MAX_PATH_LENGTH: usize = 60;
/// Transform the path for workspace scripts/flows
@@ -114,3 +114,38 @@ pub fn reverse_transform_key(transformed_key: &str, schema_obj: &Option<SchemaTy
transformed_key.to_string()
}
#[cfg(test)]
mod tests {
use super::*;
#[test]
fn test_transform_path() {
assert_eq!(
transform_path("u/admin/script", "script"),
"s-u_admin_script"
);
assert_eq!(transform_path("f/folder/flow", "flow"), "f-f_folder_flow");
assert_eq!(transform_path("my_script", "script"), "s-my__script");
}
#[test]
fn test_reverse_transform() {
let (type_str, path, is_hub) = reverse_transform("s-u_admin_script").unwrap();
assert_eq!(type_str, "script");
assert_eq!(path, "u/admin/script");
assert!(!is_hub);
let (type_str, path, is_hub) = reverse_transform("f-f_folder_flow").unwrap();
assert_eq!(type_str, "flow");
assert_eq!(path, "f/folder/flow");
assert!(!is_hub);
}
#[test]
fn test_apply_key_transformation() {
assert_eq!(apply_key_transformation("my key"), "my_key");
assert_eq!(apply_key_transformation("key!@#"), "key");
assert_eq!(apply_key_transformation("key_123"), "key_123");
}
}
backend/windmill-api/src/mcp/utils/models.rs → backend/windmill-mcp/src/common/types.rs
+24 -13
View File
@@ -5,10 +5,12 @@
use serde::{Deserialize, Serialize};
use serde_json::Value;
use sqlx::FromRow;
use std::collections::HashMap;
use windmill_common::scripts::Schema;
#[cfg(feature = "server")]
use sqlx::FromRow;
/// Workspace ID wrapper for Axum extensions
#[derive(Clone, Debug)]
pub struct WorkspaceId(pub String);
@@ -30,7 +32,8 @@ pub struct HubScriptInfo {
}
/// Schema type structure for JSON schemas
#[derive(Serialize, FromRow, Deserialize, Debug, Clone)]
#[derive(Serialize, Deserialize, Debug, Clone)]
#[cfg_attr(feature = "server", derive(FromRow))]
pub struct SchemaType {
pub r#type: String,
pub properties: HashMap<String, Value>,
@@ -39,16 +42,13 @@ pub struct SchemaType {
impl Default for SchemaType {
fn default() -> Self {
Self {
r#type: "object".to_string(),
properties: HashMap::new(),
required: vec![],
}
Self { r#type: "object".to_string(), properties: HashMap::new(), required: vec![] }
}
}
/// Script information from database
#[derive(Serialize, FromRow, Debug)]
#[derive(Serialize, Debug)]
#[cfg_attr(feature = "server", derive(FromRow))]
pub struct ScriptInfo {
pub path: String,
pub summary: Option<String>,
@@ -57,7 +57,8 @@ pub struct ScriptInfo {
}
/// Flow information from database
#[derive(Serialize, FromRow, Debug)]
#[derive(Serialize, Debug)]
#[cfg_attr(feature = "server", derive(FromRow))]
pub struct FlowInfo {
pub path: String,
pub summary: Option<String>,
@@ -66,7 +67,8 @@ pub struct FlowInfo {
}
/// Resource information from database
#[derive(Serialize, FromRow, Debug, Clone)]
#[derive(Serialize, Debug, Clone)]
#[cfg_attr(feature = "server", derive(FromRow))]
pub struct ResourceInfo {
pub path: String,
pub description: Option<String>,
@@ -74,25 +76,34 @@ pub struct ResourceInfo {
}
/// Resource type information from database
#[derive(Serialize, FromRow, Debug, Clone)]
#[derive(Serialize, Debug, Clone)]
#[cfg_attr(feature = "server", derive(FromRow))]
pub struct ResourceType {
pub name: String,
pub description: Option<String>,
}
/// Schema holder for database queries
#[derive(Serialize, FromRow)]
#[derive(Serialize)]
#[cfg_attr(feature = "server", derive(FromRow))]
pub struct ItemSchema {
pub schema: Option<Schema>,
}
/// Trait for objects that can be converted to MCP tools
pub trait ToolableItem {
/// Get the path or identifier for this item (transformed for MCP compatibility)
fn get_path_or_id(&self) -> String;
/// Get the summary/title of this item
fn get_summary(&self) -> &str;
/// Get the description of this item
fn get_description(&self) -> &str;
/// Get the JSON schema for this item's parameters
fn get_schema(&self) -> SchemaType;
/// Whether this item is from the Hub
fn is_hub(&self) -> bool;
/// Get the type of this item ("script" or "flow")
fn item_type(&self) -> &'static str;
/// Get the integration type (for hub scripts)
fn get_integration_type(&self) -> Option<String>;
}
}
backend/windmill-mcp/src/lib.rs
+24 -251
View File
@@ -1,51 +1,33 @@
/*
* Author: Ruben Fiszel
* Copyright: Windmill Labs, Inc 2022
* This file and its contents are licensed under the AGPLv3 License.
* Please see the included NOTICE for copyright information and
* LICENSE-AGPL for a copy of the license.
*/
//! Windmill MCP (Model Context Protocol) implementation
//!
//! This crate provides:
//! - MCP client for connecting to external MCP servers (used by AI agents)
//! - Common types and utilities for MCP implementations
//! - MCP server types (when `server` feature is enabled)
//! - OAuth support (when `auth` feature is enabled)
use anyhow::{Context, Result};
use reqwest::header::{HeaderMap, HeaderName, HeaderValue};
use serde_json::{json, Value};
use std::str::FromStr;
use windmill_common::variables::get_secret_value_as_admin;
use windmill_common::DB;
// Common types and utilities module
pub mod common;
// Client module
pub mod client;
// Re-export common types at crate root for convenience
pub use common::{
convert_schema_to_schema_type, is_resource_allowed, parse_mcp_scopes, transform_path, FlowInfo,
HubResponse, HubScriptInfo, ItemSchema, McpScopeConfig, ResourceInfo, ResourceType, SchemaType,
ScriptInfo, ToolableItem, WorkspaceId,
};
// Re-export client types at crate root for backward compatibility
pub use client::{McpClient, McpResource, McpToolSource};
// Re-export rmcp types for client usage
pub use rmcp::model::Tool as McpTool;
use rmcp::{
model::{
CallToolRequestParam, ClientCapabilities, ClientInfo, Implementation,
InitializeRequestParam,
},
service::RunningService,
transport::{
streamable_http_client::StreamableHttpClientTransportConfig, StreamableHttpClientTransport,
},
RoleClient, ServiceExt,
};
// Re-export rmcp server types when server feature is enabled
// Server module (when server feature is enabled)
#[cfg(feature = "server")]
pub mod server {
//! Re-exports of rmcp server types for MCP server implementations
pub use rmcp::handler::server::ServerHandler;
pub use rmcp::model::{
Annotated, CallToolRequestParam, CallToolResult, Content, Implementation,
InitializeRequestParam, InitializeResult, ListPromptsResult, ListResourceTemplatesResult,
ListResourcesResult, ListToolsResult, PaginatedRequestParam, ProtocolVersion, RawContent,
RawTextContent, ServerCapabilities, ServerInfo, Tool, ToolAnnotations,
};
pub use rmcp::service::{RequestContext, RoleServer};
pub use rmcp::transport::streamable_http_server::{
session::local::LocalSessionManager, StreamableHttpService,
};
pub use rmcp::transport::StreamableHttpServerConfig;
pub use rmcp::ErrorData;
}
pub mod server;
// Re-export rmcp auth types when auth feature is enabled
#[cfg(feature = "auth")]
@@ -60,212 +42,3 @@ pub mod oauth {
RedirectUrl, Scope, TokenUrl,
};
}
use std::collections::HashMap;
use serde::{Deserialize, Serialize};
/// MCP server resource configuration
#[derive(Debug, Clone, Serialize, Deserialize)]
pub struct McpResource {
/// Name of the MCP resource (used for prefixing tools)
pub name: String,
/// HTTP URL for the MCP server endpoint
pub url: String,
/// Optional token for authentication
#[serde(skip_serializing_if = "Option::is_none")]
pub token: Option<String>,
/// Optional headers
#[serde(skip_serializing_if = "Option::is_none")]
pub headers: Option<HashMap<String, String>>,
}
/// Metadata for tracking MCP tool sources
#[derive(Debug, Clone, Serialize, Deserialize)]
pub struct McpToolSource {
/// Name of the MCP resource this tool comes from
pub name: String,
/// Original tool name in the MCP server
pub tool_name: String,
/// Path of the MCP resource
pub resource_path: String,
}
/// MCP client for communicating with external MCP servers
pub struct McpClient {
/// The underlying rmcp client
client: RunningService<RoleClient, InitializeRequestParam>,
/// Cached list of available tools from the server
available_tools: Vec<McpTool>,
}
impl McpClient {
/// Create a new MCP client from a resource configuration
pub async fn from_resource(resource: McpResource, db: &DB, w_id: &str) -> Result<Self> {
// Build custom reqwest client with headers if provided
let mut headers = HeaderMap::new();
if let Some(token_path) = &resource.token {
if !token_path.trim().is_empty() {
let value =
get_secret_value_as_admin(db, w_id, token_path.trim_start_matches("$var:"))
.await?;
headers.insert(
HeaderName::from_static("authorization"),
HeaderValue::from_str(format!("Bearer {}", value).as_str())?,
);
}
}
if let Some(resource_headers) = &resource.headers {
for (key, value) in resource_headers {
match (HeaderName::from_str(key), HeaderValue::from_str(value)) {
(Ok(name), Ok(value)) => {
headers.insert(name, value);
}
_ => {
tracing::warn!("Invalid header: {}={}", key, value);
}
}
}
}
let reqwest_client = reqwest::Client::builder()
.default_headers(headers)
.build()
.context("Failed to build HTTP client")?;
// Create the HTTP transport with custom client
let config = StreamableHttpClientTransportConfig::with_uri(resource.url.as_str());
let transport = StreamableHttpClientTransport::with_client(reqwest_client, config);
// Set up client info
let client_info = ClientInfo {
protocol_version: Default::default(),
capabilities: ClientCapabilities::default(),
client_info: Implementation {
name: "windmill-ai-agent".to_string(),
title: Some("Windmill AI Agent".to_string()),
version: env!("CARGO_PKG_VERSION").to_string(),
website_url: None,
icons: None,
},
};
// Initialize the connection
let client = client_info
.serve(transport)
.await
.context("Failed to connect to MCP server")?;
// Immediately fetch available tools
let available_tools = client
.list_tools(Default::default())
.await
.context("Failed to list tools from MCP server")?
.tools;
Ok(Self { client, available_tools })
}
/// Get the list of available tools from the MCP server
pub fn available_tools(&self) -> &[McpTool] {
&self.available_tools
}
/// Call a tool on the MCP server, with openai-style arguments
pub async fn call_tool(&self, name: &str, arguments: &str) -> Result<serde_json::Value> {
// Convert OpenAI-style arguments to MCP format
let mcp_args =
Self::openai_args_to_mcp_args(arguments).context("Failed to parse tool arguments")?;
let result = self
.client
.call_tool(CallToolRequestParam { name: name.to_string().into(), arguments: mcp_args, task: None })
.await
.context(format!("Failed to call MCP tool: {}", name))?;
// Convert the result to a JSON value
// MCP tools return ToolResult which contains content array
let result_json =
serde_json::to_value(&result).context("Failed to serialize MCP tool result")?;
Ok(result_json)
}
/// Close the connection
pub async fn shutdown(self) -> Result<()> {
self.client.cancel().await?;
Ok(())
}
/// Fix array schemas to ensure they have the required 'items' property
/// OpenAI requires all array types to have an 'items' field. MCP servers may
/// return schemas without this field, so we add a default.
pub fn fix_array_schemas(schema: &mut Value) {
if let Value::Object(obj) = schema {
// Check if this is an array type
if let Some(type_val) = obj.get("type") {
let is_array = match type_val {
Value::String(s) => s == "array",
Value::Array(arr) => arr.iter().any(|v| v.as_str() == Some("array")),
_ => false,
};
// If it's an array and missing 'items', add a default
if is_array && !obj.contains_key("items") {
obj.insert("items".to_string(), json!({}));
}
}
// Recursively fix nested schemas
if let Some(Value::Object(props)) = obj.get_mut("properties") {
for value in props.values_mut() {
Self::fix_array_schemas(value);
}
}
// Fix items if present (for nested arrays)
if let Some(items) = obj.get_mut("items") {
Self::fix_array_schemas(items);
}
// Fix oneOf, anyOf, allOf schemas
for key in &["oneOf", "anyOf", "allOf"] {
if let Some(Value::Array(schemas)) = obj.get_mut(*key) {
for schema in schemas {
Self::fix_array_schemas(schema);
}
}
}
// Fix additionalProperties if it's a schema
if let Some(additional) = obj.get_mut("additionalProperties") {
if additional.is_object() {
Self::fix_array_schemas(additional);
}
}
}
}
/// Convert OpenAI-style tool call arguments to MCP format
/// OpenAI sends arguments as a JSON string, MCP expects a Map
fn openai_args_to_mcp_args(
args_str: &str,
) -> Result<Option<serde_json::Map<String, serde_json::Value>>> {
if args_str.trim().is_empty() {
return Ok(None);
}
let args_value: serde_json::Value =
serde_json::from_str(args_str).context("Failed to parse tool call arguments")?;
match args_value {
serde_json::Value::Object(map) => Ok(Some(map)),
serde_json::Value::Null => Ok(None),
_ => Ok(Some(
vec![("value".to_string(), args_value)]
.into_iter()
.collect(),
)),
}
}
}
backend/windmill-mcp/src/server/backend.rs
+159
View File
@@ -0,0 +1,159 @@
//! MCP Backend trait definitions
//!
//! This module defines the traits that must be implemented by the backend
//! (typically windmill-api) to provide the actual functionality for the MCP server.
use async_trait::async_trait;
use rmcp::ErrorData;
use serde_json::Value;
use std::collections::HashMap;
use crate::common::types::{
FlowInfo, HubScriptInfo, ResourceInfo, ResourceType, SchemaType, ScriptInfo,
};
use crate::server::endpoints::EndpointTool;
/// Result type for backend operations using rmcp's ErrorData directly
pub type BackendResult<T> = Result<T, ErrorData>;
/// Authentication context required by the MCP server
pub trait McpAuth: Send + Sync + Clone + 'static {
/// Get the username
fn username(&self) -> &str;
/// Get the email
fn email(&self) -> &str;
/// Check if user is admin
fn is_admin(&self) -> bool;
/// Check if user is operator
fn is_operator(&self) -> bool;
/// Get user's groups
fn groups(&self) -> &[String];
/// Get user's folders as (name, can_write, is_owner)
fn folders(&self) -> &[(String, bool, bool)];
/// Get token scopes
fn scopes(&self) -> Option<&[String]>;
/// Check if the user has an MCP scope
fn has_mcp_scope(&self) -> bool {
self.scopes()
.map(|s| s.iter().any(|scope| scope.starts_with("mcp:")))
.unwrap_or(false)
}
}
/// The core backend trait that windmill-api implements
///
/// This trait abstracts the windmill-api specific operations needed by the MCP server.
/// By implementing this trait, windmill-api can inject its database access, job execution,
/// and other functionality without windmill-mcp needing to depend on windmill-api directly.
#[async_trait]
pub trait McpBackend: Send + Sync + Clone + 'static {
/// The authentication context type
type Auth: McpAuth;
// ─────────────────────────────────────────────────────────────────
// Listing Operations
// ─────────────────────────────────────────────────────────────────
/// List scripts, optionally filtered to favorites only
async fn list_scripts(
&self,
auth: &Self::Auth,
workspace_id: &str,
favorites_only: bool,
) -> BackendResult<Vec<ScriptInfo>>;
/// List flows, optionally filtered to favorites only
async fn list_flows(
&self,
auth: &Self::Auth,
workspace_id: &str,
favorites_only: bool,
) -> BackendResult<Vec<FlowInfo>>;
/// List resource types in workspace
async fn list_resource_types(
&self,
auth: &Self::Auth,
workspace_id: &str,
) -> BackendResult<Vec<ResourceType>>;
/// List resources of a specific type
async fn list_resources(
&self,
auth: &Self::Auth,
workspace_id: &str,
resource_type: &str,
) -> BackendResult<Vec<ResourceInfo>>;
/// List hub scripts, optionally filtered by app integrations
async fn list_hub_scripts(&self, app_filter: Option<&str>)
-> BackendResult<Vec<HubScriptInfo>>;
// ─────────────────────────────────────────────────────────────────
// Schema Operations
// ─────────────────────────────────────────────────────────────────
/// Get schema for a script or flow
async fn get_item_schema(
&self,
auth: &Self::Auth,
workspace_id: &str,
path: &str,
item_type: &str,
) -> BackendResult<Option<SchemaType>>;
/// Get schema for a hub script
async fn get_hub_script_schema(&self, path: &str) -> BackendResult<Option<SchemaType>>;
// ─────────────────────────────────────────────────────────────────
// Schema Transformation (requires DB access for resources)
// ─────────────────────────────────────────────────────────────────
/// Transform schema for resources by enriching with available resource information.
/// The resources_cache should be pre-populated with all needed resource types.
fn transform_schema_for_resources(
&self,
schema: &SchemaType,
resources_cache: &HashMap<String, Vec<ResourceInfo>>,
resources_types: &[ResourceType],
) -> SchemaType;
// ─────────────────────────────────────────────────────────────────
// Execution Operations
// ─────────────────────────────────────────────────────────────────
/// Run a script and wait for result
async fn run_script(
&self,
auth: &Self::Auth,
workspace_id: &str,
path: &str,
args: Value,
) -> BackendResult<Value>;
/// Run a flow and wait for result
async fn run_flow(
&self,
auth: &Self::Auth,
workspace_id: &str,
path: &str,
args: Value,
) -> BackendResult<Value>;
/// Call an endpoint tool (generated API endpoint)
async fn call_endpoint(
&self,
auth: &Self::Auth,
workspace_id: &str,
endpoint_tool: &EndpointTool,
args: Value,
) -> BackendResult<Value>;
// ─────────────────────────────────────────────────────────────────
// Endpoint Tools
// ─────────────────────────────────────────────────────────────────
/// Get all available endpoint tools
fn all_endpoint_tools(&self) -> Vec<EndpointTool>;
}
backend/windmill-mcp/src/server/endpoints.rs
+103
View File
@@ -0,0 +1,103 @@
//! Endpoint tools for MCP server
//!
//! Contains the EndpointTool structure and utilities for converting
//! them to MCP tools.
use rmcp::model::{Tool, ToolAnnotations};
use serde::{Deserialize, Serialize};
use std::borrow::Cow;
use std::sync::Arc;
/// Represents an auto-generated endpoint tool from OpenAPI specification
#[derive(Debug, Clone, Deserialize, Serialize)]
pub struct EndpointTool {
pub name: Cow<'static, str>,
pub description: Cow<'static, str>,
pub instructions: Cow<'static, str>,
pub path: Cow<'static, str>,
pub method: Cow<'static, str>,
pub path_params_schema: Option<serde_json::Value>,
pub query_params_schema: Option<serde_json::Value>,
pub body_schema: Option<serde_json::Value>,
}
/// Convert a single endpoint tool to MCP tool
pub fn endpoint_tool_to_mcp_tool(tool: &EndpointTool) -> Tool {
let mut combined_properties = serde_json::Map::new();
let mut combined_required = Vec::new();
// Combine all parameter schemas
let schemas = [
&tool.path_params_schema,
&tool.query_params_schema,
&tool.body_schema,
];
for schema in schemas.iter().filter_map(|s| s.as_ref()) {
merge_schema_into(&mut combined_properties, &mut combined_required, schema);
}
let combined_schema = serde_json::json!({
"type": "object",
"properties": combined_properties,
"required": combined_required
});
let description = format!("{}. {}", tool.description, tool.instructions);
// Create annotations based on HTTP method and endpoint characteristics
let annotations = create_endpoint_annotations(tool);
Tool {
name: tool.name.clone(),
description: Some(description.into()),
input_schema: Arc::new(combined_schema.as_object().unwrap().clone()),
title: Some(tool.name.to_string()),
output_schema: None,
icons: None,
annotations: Some(annotations),
meta: None,
}
}
/// Create appropriate annotations for endpoint tools based on HTTP method
fn create_endpoint_annotations(tool: &EndpointTool) -> ToolAnnotations {
let method = tool.method.as_ref();
// Determine characteristics based on HTTP method
let (read_only, destructive, idempotent, open_world) = match method {
"GET" => (true, false, true, true), // Read-only, safe, idempotent
"POST" => (false, true, false, true), // Can modify, potentially destructive, not idempotent
"PUT" => (false, false, true, true), // Can modify, typically idempotent updates
"DELETE" => (false, true, true, true), // Destructive but idempotent
"PATCH" => (false, false, false, true), // Partial updates, not guaranteed idempotent
_ => (false, true, false, true), // Default: assume can modify and be destructive
};
ToolAnnotations {
title: Some(format!("{} {}", method, tool.path)),
read_only_hint: Some(read_only),
destructive_hint: Some(destructive),
idempotent_hint: Some(idempotent),
open_world_hint: Some(open_world),
}
}
/// Merge schema into combined properties and required fields
fn merge_schema_into(
combined_properties: &mut serde_json::Map<String, serde_json::Value>,
combined_required: &mut Vec<String>,
schema: &serde_json::Value,
) {
if let Some(props) = schema.get("properties").and_then(|p| p.as_object()) {
for (key, value) in props {
combined_properties.insert(key.clone(), value.clone());
}
}
if let Some(required) = schema.get("required").and_then(|r| r.as_array()) {
for req in required.iter().filter_map(|r| r.as_str()) {
combined_required.push(req.to_string());
}
}
}
backend/windmill-mcp/src/server/mod.rs
+32
View File
@@ -0,0 +1,32 @@
//! MCP Server module
//!
//! This module provides the MCP server implementation including:
//! - `McpBackend` trait for backend implementations
//! - `Runner` struct that implements the MCP protocol
//! - Re-exports of rmcp types
pub mod backend;
pub mod endpoints;
pub mod runner;
pub mod tools;
// Re-export main types
pub use backend::{BackendResult, McpAuth, McpBackend};
pub use endpoints::{endpoint_tool_to_mcp_tool, EndpointTool};
pub use runner::Runner;
pub use tools::create_tool_from_item;
// Re-export rmcp types for convenience
pub use rmcp::handler::server::ServerHandler;
pub use rmcp::model::{
Annotated, CallToolRequestParam, CallToolResult, Content, Implementation,
InitializeRequestParam, InitializeResult, ListPromptsResult, ListResourceTemplatesResult,
ListResourcesResult, ListToolsResult, PaginatedRequestParam, ProtocolVersion, RawContent,
RawTextContent, ServerCapabilities, ServerInfo, Tool, ToolAnnotations,
};
pub use rmcp::service::{RequestContext, RoleServer};
pub use rmcp::transport::streamable_http_server::{
session::local::LocalSessionManager, StreamableHttpService,
};
pub use rmcp::transport::StreamableHttpServerConfig;
pub use rmcp::ErrorData;
backend/windmill-mcp/src/server/runner.rs
+373
View File
@@ -0,0 +1,373 @@
//! MCP Server Runner implementation
//!
//! Contains the generic Runner that implements the MCP ServerHandler trait
//! and delegates to a McpBackend for actual functionality.
use crate::common::schema::extract_resource_types_from_schema;
use crate::common::scope::parse_mcp_scopes;
use crate::common::transform::{reverse_transform, reverse_transform_key};
use crate::common::types::{ResourceInfo, ToolableItem, WorkspaceId};
use crate::server::backend::{McpAuth, McpBackend};
use crate::server::endpoints::endpoint_tool_to_mcp_tool;
use crate::server::tools::create_tool_from_item;
use rmcp::handler::server::ServerHandler;
use rmcp::model::{
CallToolRequestParam, CallToolResult, Content, Implementation, InitializeRequestParam,
InitializeResult, ListPromptsResult, ListResourceTemplatesResult, ListResourcesResult,
ListToolsResult, PaginatedRequestParam, ProtocolVersion, ServerCapabilities, ServerInfo,
};
use rmcp::service::{RequestContext, RoleServer};
use rmcp::ErrorData;
use serde_json::Value;
use std::collections::{HashMap, HashSet};
use std::sync::Arc;
// Re-export from http crate for extracting request parts
use http::request::Parts as HttpParts;
/// MCP Server Runner - generic over the backend implementation
///
/// This struct implements the MCP ServerHandler trait and uses a McpBackend
/// to perform the actual operations (database queries, job execution, etc.)
pub struct Runner<B: McpBackend> {
backend: Arc<B>,
}
impl<B: McpBackend> Clone for Runner<B> {
fn clone(&self) -> Self {
Self { backend: self.backend.clone() }
}
}
impl<B: McpBackend> Runner<B> {
/// Create a new Runner with the given backend
pub fn new(backend: B) -> Self {
Self { backend: Arc::new(backend) }
}
/// Extract authentication and workspace from request context
fn extract_context(
context: &RequestContext<RoleServer>,
) -> Result<(B::Auth, String), ErrorData> {
let http_parts = context.extensions.get::<HttpParts>().ok_or_else(|| {
tracing::error!("http::request::Parts not found");
ErrorData::internal_error("http::request::Parts not found", None)
})?;
let auth = http_parts.extensions.get::<B::Auth>().ok_or_else(|| {
tracing::error!("Auth extension not found");
ErrorData::internal_error("Auth extension not found", None)
})?;
let workspace_id = http_parts
.extensions
.get::<WorkspaceId>()
.ok_or_else(|| {
tracing::error!("WorkspaceId not found");
ErrorData::internal_error("WorkspaceId not found", None)
})
.map(|w_id| w_id.0.clone())?;
// Validate MCP scope
if !auth.has_mcp_scope() {
tracing::error!("Unauthorized: missing mcp scope");
return Err(ErrorData::internal_error(
"Unauthorized: missing mcp scope",
None,
));
}
Ok((auth.clone(), workspace_id))
}
}
impl<B: McpBackend> ServerHandler for Runner<B> {
fn get_info(&self) -> ServerInfo {
ServerInfo {
protocol_version: ProtocolVersion::default(),
capabilities: ServerCapabilities::builder().enable_tools().build(),
server_info: Implementation::from_build_env(),
instructions: Some(
"This server provides a list of scripts and flows the user can run on Windmill. \
Each flow and script is a tool callable with their respective arguments."
.to_string(),
),
}
}
async fn initialize(
&self,
_request: InitializeRequestParam,
_context: RequestContext<RoleServer>,
) -> Result<InitializeResult, ErrorData> {
Ok(self.get_info())
}
async fn list_tools(
&self,
_request: Option<PaginatedRequestParam>,
context: RequestContext<RoleServer>,
) -> Result<ListToolsResult, ErrorData> {
let (auth, workspace_id) = Self::extract_context(&context)?;
// Parse MCP scopes to determine what to expose
let scopes = auth.scopes().unwrap_or(&[]);
let scope_config =
parse_mcp_scopes(scopes).map_err(|e| ErrorData::internal_error(e, None))?;
let favorites_only = scope_config.favorites;
// Fetch all items concurrently
let (scripts, flows, resource_types, hub_scripts) = tokio::try_join!(
self.backend
.list_scripts(&auth, &workspace_id, favorites_only),
self.backend
.list_flows(&auth, &workspace_id, favorites_only),
self.backend.list_resource_types(&auth, &workspace_id),
async {
if let Some(ref apps) = scope_config.hub_apps {
self.backend.list_hub_scripts(Some(apps)).await
} else {
Ok(vec![])
}
}
)?;
// Filter items based on scope
let filtered_scripts: Vec<_> = scripts
.into_iter()
.filter(|s| !scope_config.granular || scope_config.is_allowed("script", &s.path))
.collect();
let filtered_flows: Vec<_> = flows
.into_iter()
.filter(|f| !scope_config.granular || scope_config.is_allowed("flow", &f.path))
.collect();
// Collect all needed resource types from all schemas
let mut needed_resource_types: HashSet<String> = HashSet::new();
for script in &filtered_scripts {
needed_resource_types.extend(extract_resource_types_from_schema(&script.get_schema()));
}
for flow in &filtered_flows {
needed_resource_types.extend(extract_resource_types_from_schema(&flow.get_schema()));
}
for hub_script in &hub_scripts {
needed_resource_types
.extend(extract_resource_types_from_schema(&hub_script.get_schema()));
}
// Pre-fetch all resources
let resource_futures: Vec<_> = needed_resource_types
.into_iter()
.map(|rt| {
let backend = self.backend.clone();
let auth = auth.clone();
let workspace_id = workspace_id.clone();
async move {
backend
.list_resources(&auth, &workspace_id, &rt)
.await
.map(|resources| (rt, resources))
}
})
.collect();
let resource_results = futures::future::try_join_all(resource_futures).await?;
let resources_cache: HashMap<String, Vec<ResourceInfo>> =
resource_results.into_iter().collect();
let mut tools = Vec::new();
for script in &filtered_scripts {
tools.push(create_tool_from_item(
script,
self.backend.as_ref(),
&resources_cache,
&resource_types,
));
}
for flow in &filtered_flows {
tools.push(create_tool_from_item(
flow,
self.backend.as_ref(),
&resources_cache,
&resource_types,
));
}
for hub_script in &hub_scripts {
tools.push(create_tool_from_item(
hub_script,
self.backend.as_ref(),
&resources_cache,
&resource_types,
));
}
// Add endpoint tools from the generated MCP tools, filtered by scope
let endpoint_tools = self.backend.all_endpoint_tools();
for endpoint_tool in endpoint_tools {
if scope_config.granular && !scope_config.is_allowed("endpoint", &endpoint_tool.name) {
continue;
}
tools.push(endpoint_tool_to_mcp_tool(&endpoint_tool));
}
Ok(ListToolsResult { tools, next_cursor: None, meta: None })
}
async fn call_tool(
&self,
request: CallToolRequestParam,
context: RequestContext<RoleServer>,
) -> Result<CallToolResult, ErrorData> {
let (auth, workspace_id) = Self::extract_context(&context)?;
// Parse MCP scopes for authorization
let scopes = auth.scopes().unwrap_or(&[]);
let scope_config =
parse_mcp_scopes(scopes).map_err(|e| ErrorData::internal_error(e, None))?;
// Handle truncated tool names
if request.name.ends_with("_TRUNC") {
return Ok(CallToolResult::error(vec![rmcp::model::Annotated::new(
rmcp::model::RawContent::Text(rmcp::model::RawTextContent {
text: "Tool path is too long. Consider shortening it to make it compatible with MCP.".to_string(),
meta: None,
}),
None,
)]));
}
let args = request.arguments.map(Value::Object).unwrap_or(Value::Null);
// Check if this is an endpoint tool
let endpoint_tools = self.backend.all_endpoint_tools();
for endpoint_tool in &endpoint_tools {
if endpoint_tool.name.as_ref() == request.name {
// Validate endpoint scope
if scope_config.granular
&& !scope_config.is_allowed("endpoint", &endpoint_tool.name)
{
return Err(ErrorData::internal_error(
format!(
"Access denied: endpoint '{}' not in token scope",
endpoint_tool.name
),
None,
));
}
// This is an endpoint tool, call via backend
let result = self
.backend
.call_endpoint(&auth, &workspace_id, endpoint_tool, args)
.await
.map_err(|e| ErrorData::internal_error(e.message, None))?;
return Ok(CallToolResult::success(vec![Content::text(
serde_json::to_string_pretty(&result).unwrap_or_else(|_| "{}".to_string()),
)]));
}
}
// Not an endpoint tool - parse as script/flow
let (tool_type, path, is_hub) = reverse_transform(&request.name).map_err(|e| {
ErrorData::internal_error(format!("Failed to parse tool name: {}", e), None)
})?;
// Validate script/flow scope
if !is_hub && scope_config.granular {
if tool_type == "script" && !scope_config.is_allowed("script", &path) {
return Err(ErrorData::internal_error(
format!("Access denied: script '{}' not in token scope", path),
None,
));
} else if tool_type == "flow" && !scope_config.is_allowed("flow", &path) {
return Err(ErrorData::internal_error(
format!("Access denied: flow '{}' not in token scope", path),
None,
));
}
}
// Get item schema for argument transformation
let item_schema = if is_hub {
self.backend
.get_hub_script_schema(&format!("hub/{}", path))
.await
.map_err(|e| ErrorData::internal_error(e.message, None))?
} else {
self.backend
.get_item_schema(&auth, &workspace_id, &path, tool_type)
.await
.map_err(|e| ErrorData::internal_error(e.message, None))?
};
// Transform arguments back to original key names
let transformed_args = if let Value::Object(map) = args {
let mut args_hash = HashMap::new();
for (k, v) in map {
let original_key = reverse_transform_key(&k, &item_schema);
args_hash.insert(original_key, v);
}
Value::Object(args_hash.into_iter().collect())
} else {
args
};
let script_or_flow_path = if is_hub {
format!("hub/{}", path)
} else {
path
};
// Execute script or flow
let result = if tool_type == "script" {
self.backend
.run_script(&auth, &workspace_id, &script_or_flow_path, transformed_args)
.await
} else {
self.backend
.run_flow(&auth, &workspace_id, &script_or_flow_path, transformed_args)
.await
};
match result {
Ok(value) => Ok(CallToolResult::success(vec![Content::text(
serde_json::to_string_pretty(&value).unwrap_or_else(|_| "{}".to_string()),
)])),
Err(e) => Err(ErrorData::internal_error(
format!("Failed to run {}: {}", tool_type, e.message),
None,
)),
}
}
async fn list_resources(
&self,
_request: Option<PaginatedRequestParam>,
_context: RequestContext<RoleServer>,
) -> Result<ListResourcesResult, ErrorData> {
Ok(ListResourcesResult { resources: vec![], next_cursor: None, meta: None })
}
async fn list_prompts(
&self,
_request: Option<PaginatedRequestParam>,
_context: RequestContext<RoleServer>,
) -> Result<ListPromptsResult, ErrorData> {
Ok(ListPromptsResult::default())
}
async fn list_resource_templates(
&self,
_request: Option<PaginatedRequestParam>,
_context: RequestContext<RoleServer>,
) -> Result<ListResourceTemplatesResult, ErrorData> {
Ok(ListResourceTemplatesResult::default())
}
}
backend/windmill-mcp/src/server/tools.rs
+184
View File
@@ -0,0 +1,184 @@
//! Tool creation utilities for MCP server
//!
//! Contains functionality for converting Windmill items (scripts, flows, hub scripts)
//! into MCP tools.
use rmcp::model::{Tool, ToolAnnotations};
use std::borrow::Cow;
use std::collections::HashMap;
use std::sync::Arc;
use crate::common::schema::convert_schema_to_schema_type;
use crate::common::transform::transform_path;
use crate::common::types::{
FlowInfo, HubScriptInfo, ResourceInfo, ResourceType, SchemaType, ScriptInfo, ToolableItem,
};
use crate::server::backend::McpBackend;
/// Implementation of ToolableItem for ScriptInfo
impl ToolableItem for ScriptInfo {
fn get_path_or_id(&self) -> String {
transform_path(&self.path, "script")
}
fn get_summary(&self) -> &str {
self.summary.as_deref().unwrap_or("No summary")
}
fn get_description(&self) -> &str {
self.description.as_deref().unwrap_or("No description")
}
fn get_schema(&self) -> SchemaType {
convert_schema_to_schema_type(self.schema.clone())
}
fn is_hub(&self) -> bool {
false
}
fn item_type(&self) -> &'static str {
"script"
}
fn get_integration_type(&self) -> Option<String> {
None
}
}
/// Implementation of ToolableItem for FlowInfo
impl ToolableItem for FlowInfo {
fn get_path_or_id(&self) -> String {
transform_path(&self.path, "flow")
}
fn get_summary(&self) -> &str {
self.summary.as_deref().unwrap_or("No summary")
}
fn get_description(&self) -> &str {
self.description.as_deref().unwrap_or("No description")
}
fn get_schema(&self) -> SchemaType {
convert_schema_to_schema_type(self.schema.clone())
}
fn is_hub(&self) -> bool {
false
}
fn item_type(&self) -> &'static str {
"flow"
}
fn get_integration_type(&self) -> Option<String> {
None
}
}
/// Implementation of ToolableItem for HubScriptInfo
impl ToolableItem for HubScriptInfo {
fn get_path_or_id(&self) -> String {
let id = self.version_id;
let summary = self.summary.as_deref().unwrap_or("No summary");
format!("hs-{}-{}", id, summary.replace(" ", "_"))
}
fn get_summary(&self) -> &str {
self.summary.as_deref().unwrap_or("No summary")
}
fn get_description(&self) -> &str {
self.description.as_deref().unwrap_or("No description")
}
fn get_schema(&self) -> SchemaType {
match serde_json::from_value::<SchemaType>(self.schema.clone().unwrap_or_default()) {
Ok(schema_type) => schema_type,
Err(_) => SchemaType::default(),
}
}
fn is_hub(&self) -> bool {
true
}
fn item_type(&self) -> &'static str {
"script"
}
fn get_integration_type(&self) -> Option<String> {
self.app.clone()
}
}
/// Create an MCP Tool from a ToolableItem
///
/// The resources_cache should be pre-populated with all resource types
/// that may be referenced by the item's schema.
pub fn create_tool_from_item<T: ToolableItem, B: McpBackend>(
item: &T,
backend: &B,
resources_cache: &HashMap<String, Vec<ResourceInfo>>,
resources_types: &[ResourceType],
) -> Tool {
let is_hub = item.is_hub();
let path = item.get_path_or_id();
let item_type = item.item_type();
let description = format!(
"This is a {} named `{}` with the following description: `{}`.{}",
item_type,
item.get_summary(),
item.get_description(),
if is_hub {
format!(
" It is a tool used for the following app: {}",
item.get_integration_type()
.unwrap_or("No integration type".to_string())
)
} else {
"".to_string()
}
);
let schema = item.get_schema();
let schema_obj =
backend.transform_schema_for_resources(&schema, resources_cache, resources_types);
let input_schema_map = match serde_json::to_value(schema_obj) {
Ok(serde_json::Value::Object(map)) => map,
Ok(_) => {
tracing::warn!(
"Schema object for tool '{}' did not serialize to a JSON object, using empty schema.",
path
);
serde_json::Map::new()
}
Err(e) => {
tracing::error!(
"Failed to serialize schema object for tool '{}': {}. Using empty schema.",
path,
e
);
serde_json::Map::new()
}
};
Tool {
name: Cow::Owned(path),
description: Some(Cow::Owned(description)),
input_schema: Arc::new(input_schema_map),
title: Some(item.get_summary().to_string()),
output_schema: None,
icons: None,
annotations: Some(ToolAnnotations {
title: Some(item.get_summary().to_string()),
read_only_hint: Some(false), // Can modify environment
destructive_hint: Some(true), // Can potentially be destructive
idempotent_hint: Some(false), // Are not guaranteed to be idempotent
open_world_hint: Some(true), // Can interact with external services
}),
meta: None,
}
}
backend/windmill-queue/src/jobs.rs
+191 -64
View File
@@ -138,10 +138,13 @@ const MAX_FREE_CONCURRENT_RUNS: i32 = 30;
const ERROR_HANDLER_USERNAME: &str = "error_handler";
const SCHEDULE_ERROR_HANDLER_USERNAME: &str = "schedule_error_handler";
const GLOBAL_ERROR_HANDLER_USERNAME: &str = "global";
const SUCCESS_HANDLER_USERNAME: &str = "success_handler";
pub const ERROR_HANDLER_USER_GROUP: &str = "g/error_handler";
pub const ERROR_HANDLER_USER_EMAIL: &str = "error_handler@windmill.dev";
pub const SCHEDULE_ERROR_HANDLER_USER_EMAIL: &str = "schedule_error_handler@windmill.dev";
pub const SUCCESS_HANDLER_USER_GROUP: &str = "g/success_handler";
pub const SUCCESS_HANDLER_USER_EMAIL: &str = "success_handler@windmill.dev";
#[derive(Clone, Debug, Serialize, Deserialize)]
pub struct CanceledBy {
@@ -775,8 +778,14 @@ lazy_static::lazy_static! {
// Cache for workspace error handler settings with 60s TTL
// Key: workspace_id, Value: (error_handler, error_handler_extra_args, error_handler_muted_on_cancel, expiry_timestamp)
static ref WORKSPACE_ERROR_HANDLER_CACHE: Cache<String, (Option<String>, Option<Json<Box<RawValue>>>, bool, i64)> = Cache::new(1000);
// Cache for workspace success handler settings with 60s TTL
// Key: workspace_id, Value: (success_handler, success_handler_extra_args, expiry_timestamp)
static ref WORKSPACE_SUCCESS_HANDLER_CACHE: Cache<String, (Option<String>, Option<Json<Box<RawValue>>>, i64)> = Cache::new(1000);
}
const WORKSPACE_HANDLER_CACHE_TTL_SECONDS: i64 = 60;
pub async fn add_completed_job<T: Serialize + Send + Sync + ValidableJson>(
db: &Pool<Postgres>,
completed_job: &MiniCompletedJob,
@@ -1565,6 +1574,24 @@ pub async fn report_error_to_workspace_handler_or_critical_side_channel(
}
}
async fn fetch_error_handler_from_db(
db: &Pool<Postgres>,
w_id: &str,
) -> Result<(Option<String>, Option<Json<Box<RawValue>>>, bool), Error> {
sqlx::query_as::<_, (Option<String>, Option<Json<Box<RawValue>>>, bool)>(
r#"
SELECT error_handler, error_handler_extra_args, error_handler_muted_on_cancel
FROM workspace_settings
WHERE workspace_id = $1
"#,
)
.bind(w_id)
.fetch_optional(db)
.await
.context("fetching error handler info from workspace_settings")?
.ok_or_else(|| Error::internal_err(format!("no workspace settings for id {w_id}")))
}
pub async fn send_error_to_workspace_handler<'a, 'c, T: Serialize + Send + Sync>(
queued_job: &MiniCompletedJob,
is_canceled: bool,
@@ -1573,84 +1600,28 @@ pub async fn send_error_to_workspace_handler<'a, 'c, T: Serialize + Send + Sync>
) -> Result<(), Error> {
let w_id = &queued_job.workspace_id;
// Try to get from cache first, checking if entry is still valid (within 60s TTL)
let now = chrono::Utc::now().timestamp();
let (error_handler, error_handler_extra_args, error_handler_muted_on_cancel) =
if let Some(cached) = WORKSPACE_ERROR_HANDLER_CACHE.get(w_id) {
if cached.3 > now {
// Cache hit and not expired
(cached.0.clone(), cached.1.clone(), cached.2)
} else {
// Cache expired, fetch from database
let row_result =
sqlx::query_as::<_, (Option<String>, Option<Json<Box<RawValue>>>, bool)>(
r#"
SELECT
error_handler,
error_handler_extra_args,
error_handler_muted_on_cancel
FROM
workspace_settings
WHERE
workspace_id = $1
"#,
)
.bind(&w_id)
.fetch_optional(db)
.await
.context("fetching error handler info from workspace_settings")?
.ok_or_else(|| {
Error::internal_err(format!("no workspace settings for id {w_id}"))
})?;
// Update cache with 60s TTL
let expiry = now + 60;
let row = fetch_error_handler_from_db(db, w_id).await?;
let expiry = now + WORKSPACE_HANDLER_CACHE_TTL_SECONDS;
WORKSPACE_ERROR_HANDLER_CACHE.insert(
w_id.clone(),
(
row_result.0.clone(),
row_result.1.clone(),
row_result.2,
expiry,
),
(row.0.clone(), row.1.clone(), row.2, expiry),
);
row_result
row
}
} else {
// Cache miss, fetch from database
let row_result =
sqlx::query_as::<_, (Option<String>, Option<Json<Box<RawValue>>>, bool)>(
r#"
SELECT
error_handler,
error_handler_extra_args,
error_handler_muted_on_cancel
FROM
workspace_settings
WHERE
workspace_id = $1
"#,
)
.bind(&w_id)
.fetch_optional(db)
.await
.context("fetching error handler info from workspace_settings")?
.ok_or_else(|| {
Error::internal_err(format!("no workspace settings for id {w_id}"))
})?;
// Store in cache with 60s TTL
let expiry = now + 60;
let row = fetch_error_handler_from_db(db, w_id).await?;
let expiry = now + WORKSPACE_HANDLER_CACHE_TTL_SECONDS;
WORKSPACE_ERROR_HANDLER_CACHE.insert(
w_id.clone(),
(
row_result.0.clone(),
row_result.1.clone(),
row_result.2,
expiry,
),
(row.0.clone(), row.1.clone(), row.2, expiry),
);
row_result
row
};
if is_canceled && error_handler_muted_on_cancel {
@@ -1707,6 +1678,77 @@ pub async fn send_error_to_workspace_handler<'a, 'c, T: Serialize + Send + Sync>
Ok(())
}
async fn fetch_success_handler_from_db(
db: &Pool<Postgres>,
w_id: &str,
) -> Result<(Option<String>, Option<Json<Box<RawValue>>>), Error> {
sqlx::query_as::<_, (Option<String>, Option<Json<Box<RawValue>>>)>(
r#"
SELECT success_handler, success_handler_extra_args
FROM workspace_settings
WHERE workspace_id = $1
"#,
)
.bind(w_id)
.fetch_optional(db)
.await
.context("fetching success handler info from workspace_settings")?
.ok_or_else(|| Error::internal_err(format!("no workspace settings for id {w_id}")))
}
pub async fn send_success_to_workspace_handler<'a, 'c, T: Serialize + Send + Sync>(
queued_job: &MiniCompletedJob,
db: &Pool<Postgres>,
result: Json<&'a T>,
) -> Result<(), Error> {
let w_id = &queued_job.workspace_id;
let now = chrono::Utc::now().timestamp();
let (success_handler, success_handler_extra_args) =
if let Some(cached) = WORKSPACE_SUCCESS_HANDLER_CACHE.get(w_id) {
if cached.2 > now {
(cached.0.clone(), cached.1.clone())
} else {
let row = fetch_success_handler_from_db(db, w_id).await?;
let expiry = now + WORKSPACE_HANDLER_CACHE_TTL_SECONDS;
WORKSPACE_SUCCESS_HANDLER_CACHE.insert(
w_id.clone(),
(row.0.clone(), row.1.clone(), expiry),
);
row
}
} else {
let row = fetch_success_handler_from_db(db, w_id).await?;
let expiry = now + WORKSPACE_HANDLER_CACHE_TTL_SECONDS;
WORKSPACE_SUCCESS_HANDLER_CACHE.insert(
w_id.clone(),
(row.0.clone(), row.1.clone(), expiry),
);
row
};
if let Some(success_handler) = success_handler {
tracing::info!("workspace success handler for job {}", &queued_job.id);
push_success_handler(
db,
queued_job.id,
queued_job.schedule_path(),
queued_job.runnable_path.clone(),
queued_job.is_flow(),
&queued_job.workspace_id,
&success_handler,
result,
queued_job.started_at,
success_handler_extra_args,
&queued_job.permissioned_as_email,
None,
)
.await?;
}
Ok(())
}
pub async fn handle_maybe_scheduled_job<'c>(
db: &Pool<Postgres>,
job: &MiniCompletedJob,
@@ -1966,6 +2008,91 @@ pub async fn push_error_handler<'a, 'c, T: Serialize + Send + Sync>(
return Ok(uuid);
}
pub async fn push_success_handler<'a, 'c, T: Serialize + Send + Sync>(
db: &Pool<Postgres>,
job_id: Uuid,
schedule_path: Option<String>,
script_path: Option<String>,
is_flow: bool,
w_id: &str,
on_success_path: &str,
result: Json<&'a T>,
started_at: Option<DateTime<Utc>>,
extra_args: Option<Json<Box<RawValue>>>,
email: &str,
priority: Option<i16>,
) -> windmill_common::error::Result<Uuid> {
let (payload, tag, on_behalf_of) =
get_payload_tag_from_prefixed_path(on_success_path, db, w_id).await?;
let mut extra = HashMap::new();
if let Some(schedule_path) = schedule_path {
extra.insert("schedule_path".to_string(), to_raw_value(&schedule_path));
}
extra.insert("workspace_id".to_string(), to_raw_value(&w_id));
extra.insert("job_id".to_string(), to_raw_value(&job_id));
extra.insert("path".to_string(), to_raw_value(&script_path));
extra.insert("is_flow".to_string(), to_raw_value(&is_flow));
extra.insert("started_at".to_string(), to_raw_value(&started_at));
extra.insert("email".to_string(), to_raw_value(&email));
if let Some(args_v) = extra_args {
if let Ok(args_m) = serde_json::from_str::<HashMap<String, Box<RawValue>>>(args_v.get()) {
extra.extend(args_m);
} else {
return Err(error::Error::ExecutionErr(
"args of scripts needs to be dict".to_string(),
));
}
}
let result = sanitize_result(result);
let (email, permissioned_as) = if let Some(on_behalf_of) = on_behalf_of.as_ref() {
(
on_behalf_of.email.as_str(),
on_behalf_of.permissioned_as.clone(),
)
} else {
(SUCCESS_HANDLER_USER_EMAIL, SUCCESS_HANDLER_USER_GROUP.to_string())
};
let tx = PushIsolationLevel::IsolatedRoot(db.clone());
let (uuid, tx) = push(
&db,
tx,
w_id,
payload,
PushArgs { extra: Some(extra), args: &result },
SUCCESS_HANDLER_USERNAME,
email,
permissioned_as,
Some(&format!("success.handler.{job_id}")),
None,
None,
Some(job_id), // parent_job
Some(job_id), // root_job
Some(job_id), // flow_innermost_root_job
None,
false,
false,
None,
true,
tag,
None,
None,
priority,
None,
false,
None,
None,
None,
)
.await?;
tx.commit().await?;
return Ok(uuid);
}
fn sanitize_result<T: Serialize + Send + Sync>(result: Json<&T>) -> HashMap<String, Box<RawValue>> {
let as_str = serde_json::to_string(result.0).unwrap_or_else(|_| "{}".to_string());
serde_json::from_str::<HashMap<String, Box<RawValue>>>(&as_str)
backend/windmill-worker/src/handle_child.rs
+19 -28
View File
@@ -36,7 +36,6 @@ use std::{io, panic, time::Duration};
use tracing::{trace_span, Instrument};
use uuid::Uuid;
#[cfg(feature = "enterprise")]
use windmill_common::job_metrics;
#[cfg(target_os = "linux")]
@@ -696,7 +695,6 @@ where
let mut i = 0;
#[cfg(feature = "enterprise")]
let mut memory_metric_id: Result<String, Error> =
Err(Error::NotFound("not yet initialized".to_string()));
@@ -731,32 +729,26 @@ where
let update_job_row = i == 2 || (!*SLOW_LOGS && (i < 20 || (i < 120 && i % 5 == 0) || i % 10 == 0)) || i % 20 == 0;
if update_job_row {
#[cfg(feature = "enterprise")]
{
if job_id != Uuid::nil() {
if let Connection::Sql(ref db) = conn {
// tracking metric starting at i >= 2 b/c first point it useless and we don't want to track metric for super fast jobs
if i == 2 {
memory_metric_id = job_metrics::register_metric_for_job(
&db,
w_id.to_string(),
job_id,
"memory_kb".to_string(),
job_metrics::MetricKind::TimeseriesInt,
Some("Job Memory Footprint (kB)".to_string()),
)
.await;
}
if let Ok(ref metric_id) = memory_metric_id {
if let Err(err) = job_metrics::record_metric(&db, w_id.to_string(), job_id, metric_id.to_owned(), job_metrics::MetricNumericValue::Integer(current_mem)).await {
tracing::error!("Unable to save memory stat for job {} in workspace {}. Error was: {:?}", job_id, w_id, err);
}
if update_job_row && job_id != Uuid::nil() {
if let Connection::Sql(ref db) = conn {
// tracking metric starting at i >= 2 b/c first point it useless and we don't want to track metric for super fast jobs
if i == 2 {
memory_metric_id = job_metrics::register_metric_for_job(
&db,
w_id.to_string(),
job_id,
"memory_kb".to_string(),
job_metrics::MetricKind::TimeseriesInt,
Some("Job Memory Footprint (kB)".to_string()),
)
.await;
}
if let Ok(ref metric_id) = memory_metric_id {
if let Err(err) = job_metrics::record_metric(&db, w_id.to_string(), job_id, metric_id.to_owned(), job_metrics::MetricNumericValue::Integer(current_mem)).await {
tracing::error!("Unable to save memory stat for job {} in workspace {}. Error was: {:?}", job_id, w_id, err);
}
}
}
}
if job_id != Uuid::nil() {
if matches!(conn, Connection::Http(_)) {
if i % 4 != 0 {
// only ping every 4th time (2s) on http agent mode
@@ -764,8 +756,8 @@ where
}
}
let ping_job_status = ping_job_status(&conn, &job_id, Some(*mem_peak), if current_mem > 0 { Some(current_mem) } else { None }).await.unwrap_or_else(|e| {
tracing::error!("Unable to ping job status for job {job_id}. Error was: {:?}", e);
PingJobStatusResponse {
tracing::error!("Unable to ping job status for job {job_id}. Error was: {:?}", e);
PingJobStatusResponse {
canceled_by: None,
canceled_reason: None,
already_completed: false,
@@ -782,7 +774,6 @@ where
break
}
}
}
},
);
}
backend/windmill-worker/src/mssql_executor.rs
+1 -1
View File
@@ -230,7 +230,7 @@ pub async fn do_mssql(
let rows_stream = async_stream::stream! {
let mut stream = prepared_query.query(&mut client).await.map_err(to_anyhow)?.into_row_stream().map(|row| {
let raw_value = row_to_json(row.map_err(to_anyhow)?).map_err(to_anyhow);
let json = raw_value.and_then(|raw_value| serde_json::to_value(raw_value.get()).map_err(to_anyhow));
let json = raw_value.and_then(|raw_value| serde_json::from_str(raw_value.get()).map_err(to_anyhow));
json
});
while let Some(row) = stream.next().await {
benchmarks/lib.ts
+1 -1
View File
@@ -2,7 +2,7 @@ import { sleep } from "https://deno.land/x/sleep@v1.2.1/mod.ts";
import * as windmill from "https://deno.land/x/windmill@v1.174.0/mod.ts";
import * as api from "https://deno.land/x/windmill@v1.174.0/windmill-api/index.ts";
export const VERSION = "v1.608.0";
export const VERSION = "v1.611.0";
export async function login(email: string, password: string): Promise<string> {
return await windmill.UserService.login({

Some files were not shown because too many files have changed in this diff Show More