diff --git a/CHANGELOG.md b/CHANGELOG.md index 2fabf1239c..594fdac819 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,52 @@ # Changelog +## [1.611.0](https://github.com/windmill-labs/windmill/compare/v1.610.1...v1.611.0) (2026-01-19) + + +### Features + +* add HashiCorp Vault secret storage integration ([#7599](https://github.com/windmill-labs/windmill/issues/7599)) ([1b9d1c5](https://github.com/windmill-labs/windmill/commit/1b9d1c56c7e49042677326eb397e10d34a3ddcdf)) + + +### Bug Fixes + +* **flow-chat:** handle SSE timeout and fix temp message race condition ([4f8110e](https://github.com/windmill-labs/windmill/commit/4f8110eb9852b78b48aabbed114c75cbf0d1a2ef)) + +## [1.610.1](https://github.com/windmill-labs/windmill/compare/v1.610.0...v1.610.1) (2026-01-17) + + +### Bug Fixes + +* resolve BlobPart type incompatibility between Deno and Node.js ([2eac74c](https://github.com/windmill-labs/windmill/commit/2eac74cef4aa5a987fb16110388f99e912951db8)) +* use type cast instead of slice() for BlobPart compatibility ([ff77154](https://github.com/windmill-labs/windmill/commit/ff771546380ef26dfa443f9d459853048bc8029c)) + +## [1.610.0](https://github.com/windmill-labs/windmill/compare/v1.609.0...v1.610.0) (2026-01-17) + + +### Features + +* add private npm registry proxy support for ATA in webide ([#7597](https://github.com/windmill-labs/windmill/issues/7597)) ([b3cb41e](https://github.com/windmill-labs/windmill/commit/b3cb41efa4520fd3243a6dbab7c985923dd538e3)) +* add workspace success handler with 60s TTL caching ([#7598](https://github.com/windmill-labs/windmill/issues/7598)) ([73c4ce3](https://github.com/windmill-labs/windmill/commit/73c4ce30127af587510b16fbc7fa223436f6845c)) + + +### Bug Fixes + +* handle missing storage key in S3Object for write_s3_file ([03daa34](https://github.com/windmill-labs/windmill/commit/03daa341eb91118c27e92b1b51731400a0dce30c)) +* improve job deletion performance and batching ([8dd5e81](https://github.com/windmill-labs/windmill/commit/8dd5e81a32b97814eab3b72964807bdfe0ea8b49)) + +## [1.609.0](https://github.com/windmill-labs/windmill/compare/v1.608.0...v1.609.0) (2026-01-16) + + +### Features + +* cli branch override ([#7592](https://github.com/windmill-labs/windmill/issues/7592)) ([dcee9fe](https://github.com/windmill-labs/windmill/commit/dcee9fe7b163993836691988a552a5bc6042b9a2)) + + +### Bug Fixes + +* Fix MS SQL S3 Mode ([#7595](https://github.com/windmill-labs/windmill/issues/7595)) ([c7a6a05](https://github.com/windmill-labs/windmill/commit/c7a6a05925681bb1b2cec8d2c11037bc3d339798)) +* transparency issue of instance setting save button ([#7594](https://github.com/windmill-labs/windmill/issues/7594)) ([86ebf9e](https://github.com/windmill-labs/windmill/commit/86ebf9e25a03db99453269832bce030438c677c3)) + ## [1.608.0](https://github.com/windmill-labs/windmill/compare/v1.607.1...v1.608.0) (2026-01-16) diff --git a/backend/.sqlx/query-020c031c3de6c85577e30421ada9d39a5a47ca1b6cf3dbfd6988aa0694d7364c.json b/backend/.sqlx/query-020c031c3de6c85577e30421ada9d39a5a47ca1b6cf3dbfd6988aa0694d7364c.json new file mode 100644 index 0000000000..e8edd87bda --- /dev/null +++ b/backend/.sqlx/query-020c031c3de6c85577e30421ada9d39a5a47ca1b6cf3dbfd6988aa0694d7364c.json @@ -0,0 +1,23 @@ +{ + "db_name": "PostgreSQL", + "query": "SELECT value FROM variable WHERE path = $1 AND workspace_id = $2 AND is_secret = true", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "value", + "type_info": "Varchar" + } + ], + "parameters": { + "Left": [ + "Text", + "Text" + ] + }, + "nullable": [ + false + ] + }, + "hash": "020c031c3de6c85577e30421ada9d39a5a47ca1b6cf3dbfd6988aa0694d7364c" +} diff --git a/backend/.sqlx/query-052d42b46d5faba6b41f1fdcbf6a012db51b9e5a255ec0da9a8a0999d668d336.json b/backend/.sqlx/query-052d42b46d5faba6b41f1fdcbf6a012db51b9e5a255ec0da9a8a0999d668d336.json new file mode 100644 index 0000000000..7ddd14bb5c --- /dev/null +++ b/backend/.sqlx/query-052d42b46d5faba6b41f1fdcbf6a012db51b9e5a255ec0da9a8a0999d668d336.json @@ -0,0 +1,20 @@ +{ + "db_name": "PostgreSQL", + "query": "SELECT COUNT(*) as count FROM variable WHERE is_secret = true AND value != 'CLEARED'", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "count", + "type_info": "Int8" + } + ], + "parameters": { + "Left": [] + }, + "nullable": [ + null + ] + }, + "hash": "052d42b46d5faba6b41f1fdcbf6a012db51b9e5a255ec0da9a8a0999d668d336" +} diff --git a/backend/.sqlx/query-0600f2a9179f83502c6b13e8e4284f85ca82636f274f5dce47da5a8320a60088.json b/backend/.sqlx/query-0600f2a9179f83502c6b13e8e4284f85ca82636f274f5dce47da5a8320a60088.json new file mode 100644 index 0000000000..e5aec65b91 --- /dev/null +++ b/backend/.sqlx/query-0600f2a9179f83502c6b13e8e4284f85ca82636f274f5dce47da5a8320a60088.json @@ -0,0 +1,26 @@ +{ + "db_name": "PostgreSQL", + "query": "SELECT workspace_id, path FROM variable WHERE is_secret = true", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "workspace_id", + "type_info": "Varchar" + }, + { + "ordinal": 1, + "name": "path", + "type_info": "Varchar" + } + ], + "parameters": { + "Left": [] + }, + "nullable": [ + false, + false + ] + }, + "hash": "0600f2a9179f83502c6b13e8e4284f85ca82636f274f5dce47da5a8320a60088" +} diff --git a/backend/.sqlx/query-085bbcfcebc9e56984295046c48722f3510548eaf525c499ad0e24cdf8332b22.json b/backend/.sqlx/query-085bbcfcebc9e56984295046c48722f3510548eaf525c499ad0e24cdf8332b22.json new file mode 100644 index 0000000000..0a474cb3a9 --- /dev/null +++ b/backend/.sqlx/query-085bbcfcebc9e56984295046c48722f3510548eaf525c499ad0e24cdf8332b22.json @@ -0,0 +1,20 @@ +{ + "db_name": "PostgreSQL", + "query": "SELECT success_handler_extra_args FROM workspace_settings WHERE workspace_id = 'test-workspace'", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "success_handler_extra_args", + "type_info": "Json" + } + ], + "parameters": { + "Left": [] + }, + "nullable": [ + true + ] + }, + "hash": "085bbcfcebc9e56984295046c48722f3510548eaf525c499ad0e24cdf8332b22" +} diff --git a/backend/.sqlx/query-08f288d2781d823e109a9e5b8848234ca7d1efeee9661f3901f298da375e73f7.json b/backend/.sqlx/query-08f288d2781d823e109a9e5b8848234ca7d1efeee9661f3901f298da375e73f7.json index c67c6e89a8..b9956f4ee4 100644 --- a/backend/.sqlx/query-08f288d2781d823e109a9e5b8848234ca7d1efeee9661f3901f298da375e73f7.json +++ b/backend/.sqlx/query-08f288d2781d823e109a9e5b8848234ca7d1efeee9661f3901f298da375e73f7.json @@ -177,6 +177,16 @@ "ordinal": 34, "name": "teams_team_guid", "type_info": "Text" + }, + { + "ordinal": 35, + "name": "success_handler", + "type_info": "Text" + }, + { + "ordinal": 36, + "name": "success_handler_extra_args", + "type_info": "Json" } ], "parameters": { @@ -219,6 +229,8 @@ true, true, true, + true, + true, true ] }, diff --git a/backend/.sqlx/query-0a1c10bd2232b0770a7816e1bd8d758dc393f797890d597e5996146247f512ac.json b/backend/.sqlx/query-0a1c10bd2232b0770a7816e1bd8d758dc393f797890d597e5996146247f512ac.json deleted file mode 100644 index 469bd6f42c..0000000000 --- a/backend/.sqlx/query-0a1c10bd2232b0770a7816e1bd8d758dc393f797890d597e5996146247f512ac.json +++ /dev/null @@ -1,38 +0,0 @@ -{ - "db_name": "PostgreSQL", - "query": "\nWITH lockable_counters AS (\n SELECT concurrency_id, job_uuids\n FROM concurrency_counter\n WHERE job_uuids != '{}'::jsonb\n FOR UPDATE SKIP LOCKED\n),\nall_job_uuids AS (\n SELECT DISTINCT jsonb_object_keys(job_uuids) AS job_uuid\n FROM lockable_counters\n),\norphaned_job_uuids AS (\n SELECT job_uuid\n FROM all_job_uuids\n WHERE job_uuid NOT IN (\n SELECT id::text \n FROM v2_job_queue \n FOR SHARE SKIP LOCKED\n )\n),\norphaned_array AS (\n SELECT ARRAY(SELECT job_uuid FROM orphaned_job_uuids) AS orphaned_keys\n),\nbefore_update AS (\n SELECT lc.concurrency_id, lc.job_uuids, oa.orphaned_keys\n FROM lockable_counters lc, orphaned_array oa\n WHERE lc.job_uuids ?| oa.orphaned_keys\n),\naffected_rows AS (\n UPDATE concurrency_counter \n SET job_uuids = job_uuids - orphaned_array.orphaned_keys\n FROM orphaned_array\n WHERE concurrency_counter.concurrency_id IN (\n SELECT concurrency_id FROM before_update\n )\n RETURNING concurrency_id, job_uuids AS updated_job_uuids\n),\nexpanded_orphaned AS (\n SELECT bu.concurrency_id, \n bu.job_uuids AS original_job_uuids,\n unnest(bu.orphaned_keys) AS orphaned_key\n FROM before_update bu\n)\nSELECT \n eo.concurrency_id,\n eo.orphaned_key,\n eo.original_job_uuids,\n ar.updated_job_uuids\nFROM expanded_orphaned eo\nJOIN affected_rows ar ON eo.concurrency_id = ar.concurrency_id\nWHERE eo.original_job_uuids ? eo.orphaned_key\nORDER BY eo.concurrency_id, eo.orphaned_key\n", - "describe": { - "columns": [ - { - "ordinal": 0, - "name": "concurrency_id", - "type_info": "Varchar" - }, - { - "ordinal": 1, - "name": "orphaned_key", - "type_info": "Text" - }, - { - "ordinal": 2, - "name": "original_job_uuids", - "type_info": "Jsonb" - }, - { - "ordinal": 3, - "name": "updated_job_uuids", - "type_info": "Jsonb" - } - ], - "parameters": { - "Left": [] - }, - "nullable": [ - false, - null, - false, - false - ] - }, - "hash": "0a1c10bd2232b0770a7816e1bd8d758dc393f797890d597e5996146247f512ac" -} diff --git a/backend/.sqlx/query-0c8a3eb810c96230ba3a5466c55bf24a94eb8a52ceb82cc29dade173ad87569d.json b/backend/.sqlx/query-0c8a3eb810c96230ba3a5466c55bf24a94eb8a52ceb82cc29dade173ad87569d.json new file mode 100644 index 0000000000..e68f16713a --- /dev/null +++ b/backend/.sqlx/query-0c8a3eb810c96230ba3a5466c55bf24a94eb8a52ceb82cc29dade173ad87569d.json @@ -0,0 +1,23 @@ +{ + "db_name": "PostgreSQL", + "query": "SELECT path FROM variable WHERE path = ANY($1) AND workspace_id = $2 AND is_secret = true", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "path", + "type_info": "Varchar" + } + ], + "parameters": { + "Left": [ + "TextArray", + "Text" + ] + }, + "nullable": [ + false + ] + }, + "hash": "0c8a3eb810c96230ba3a5466c55bf24a94eb8a52ceb82cc29dade173ad87569d" +} diff --git a/backend/.sqlx/query-10a40902f5aba38aea23f2b0776b54752362c9cc7deb179f75c7f1795866d13a.json b/backend/.sqlx/query-10a40902f5aba38aea23f2b0776b54752362c9cc7deb179f75c7f1795866d13a.json new file mode 100644 index 0000000000..844b522215 --- /dev/null +++ b/backend/.sqlx/query-10a40902f5aba38aea23f2b0776b54752362c9cc7deb179f75c7f1795866d13a.json @@ -0,0 +1,20 @@ +{ + "db_name": "PostgreSQL", + "query": "SELECT success_handler FROM workspace_settings WHERE workspace_id = 'test-workspace'", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "success_handler", + "type_info": "Text" + } + ], + "parameters": { + "Left": [] + }, + "nullable": [ + true + ] + }, + "hash": "10a40902f5aba38aea23f2b0776b54752362c9cc7deb179f75c7f1795866d13a" +} diff --git a/backend/.sqlx/query-146f0e42ada3068a5cdae0ffdbb54b63f8c06c9143b16ce399170c1b5a6b911e.json b/backend/.sqlx/query-146f0e42ada3068a5cdae0ffdbb54b63f8c06c9143b16ce399170c1b5a6b911e.json new file mode 100644 index 0000000000..9fd6daded5 --- /dev/null +++ b/backend/.sqlx/query-146f0e42ada3068a5cdae0ffdbb54b63f8c06c9143b16ce399170c1b5a6b911e.json @@ -0,0 +1,29 @@ +{ + "db_name": "PostgreSQL", + "query": "SELECT value, is_secret FROM variable WHERE path = $1 AND workspace_id = $2", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "value", + "type_info": "Varchar" + }, + { + "ordinal": 1, + "name": "is_secret", + "type_info": "Bool" + } + ], + "parameters": { + "Left": [ + "Text", + "Text" + ] + }, + "nullable": [ + false, + false + ] + }, + "hash": "146f0e42ada3068a5cdae0ffdbb54b63f8c06c9143b16ce399170c1b5a6b911e" +} diff --git a/backend/.sqlx/query-1730f39fd1793d45fbb41b21389c61296a3ff7489ae12f52a19f9543173ac597.json b/backend/.sqlx/query-1730f39fd1793d45fbb41b21389c61296a3ff7489ae12f52a19f9543173ac597.json index a7db9044d8..4661f7c12a 100644 --- a/backend/.sqlx/query-1730f39fd1793d45fbb41b21389c61296a3ff7489ae12f52a19f9543173ac597.json +++ b/backend/.sqlx/query-1730f39fd1793d45fbb41b21389c61296a3ff7489ae12f52a19f9543173ac597.json @@ -177,6 +177,16 @@ "ordinal": 34, "name": "teams_team_guid", "type_info": "Text" + }, + { + "ordinal": 35, + "name": "success_handler", + "type_info": "Text" + }, + { + "ordinal": 36, + "name": "success_handler_extra_args", + "type_info": "Json" } ], "parameters": { @@ -219,6 +229,8 @@ true, true, true, + true, + true, true ] }, diff --git a/backend/.sqlx/query-18aad20ed9cb2dde46f9d899dc4aa6f80ecf1628bd2c073d7a237dea9b8e0c65.json b/backend/.sqlx/query-18aad20ed9cb2dde46f9d899dc4aa6f80ecf1628bd2c073d7a237dea9b8e0c65.json new file mode 100644 index 0000000000..d0c8c1fe6b --- /dev/null +++ b/backend/.sqlx/query-18aad20ed9cb2dde46f9d899dc4aa6f80ecf1628bd2c073d7a237dea9b8e0c65.json @@ -0,0 +1,23 @@ +{ + "db_name": "PostgreSQL", + "query": "SELECT is_secret FROM variable WHERE path = $1 AND workspace_id = $2", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "is_secret", + "type_info": "Bool" + } + ], + "parameters": { + "Left": [ + "Text", + "Text" + ] + }, + "nullable": [ + false + ] + }, + "hash": "18aad20ed9cb2dde46f9d899dc4aa6f80ecf1628bd2c073d7a237dea9b8e0c65" +} diff --git a/backend/.sqlx/query-1ccc9db6c6cb51bebeb90d003e12c8f7922450515d1eaee8b3abee5b31f72527.json b/backend/.sqlx/query-1ccc9db6c6cb51bebeb90d003e12c8f7922450515d1eaee8b3abee5b31f72527.json new file mode 100644 index 0000000000..4dc0e74577 --- /dev/null +++ b/backend/.sqlx/query-1ccc9db6c6cb51bebeb90d003e12c8f7922450515d1eaee8b3abee5b31f72527.json @@ -0,0 +1,14 @@ +{ + "db_name": "PostgreSQL", + "query": "DELETE FROM concurrency_key WHERE ended_at <= now() - ($1::bigint::text || ' s')::interval", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "Int8" + ] + }, + "nullable": [] + }, + "hash": "1ccc9db6c6cb51bebeb90d003e12c8f7922450515d1eaee8b3abee5b31f72527" +} diff --git a/backend/.sqlx/query-2c2ed5c91a4893966e94305df8b9f8ac904c8d0781a83fdea66c918320fb5601.json b/backend/.sqlx/query-2c2ed5c91a4893966e94305df8b9f8ac904c8d0781a83fdea66c918320fb5601.json new file mode 100644 index 0000000000..4b692f11a2 --- /dev/null +++ b/backend/.sqlx/query-2c2ed5c91a4893966e94305df8b9f8ac904c8d0781a83fdea66c918320fb5601.json @@ -0,0 +1,12 @@ +{ + "db_name": "PostgreSQL", + "query": "\n UPDATE workspace_settings\n SET success_handler = NULL\n WHERE workspace_id = 'test-workspace'\n ", + "describe": { + "columns": [], + "parameters": { + "Left": [] + }, + "nullable": [] + }, + "hash": "2c2ed5c91a4893966e94305df8b9f8ac904c8d0781a83fdea66c918320fb5601" +} diff --git a/backend/.sqlx/query-2d43c1889b492df6f77380e739e41e20fe6bea0553cc39e30b7aa8ae59963f95.json b/backend/.sqlx/query-2d43c1889b492df6f77380e739e41e20fe6bea0553cc39e30b7aa8ae59963f95.json new file mode 100644 index 0000000000..532fc430ab --- /dev/null +++ b/backend/.sqlx/query-2d43c1889b492df6f77380e739e41e20fe6bea0553cc39e30b7aa8ae59963f95.json @@ -0,0 +1,14 @@ +{ + "db_name": "PostgreSQL", + "query": "\n INSERT INTO script (workspace_id, hash, path, content, language, kind, created_by, schema, summary, description, lock)\n VALUES ('test-workspace', 1234567890, 'f/test/success_handler', $1, 'deno', 'script', 'test-user', '{}', 'Success handler script', 'Handles successful job completions', '')\n ", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "Text" + ] + }, + "nullable": [] + }, + "hash": "2d43c1889b492df6f77380e739e41e20fe6bea0553cc39e30b7aa8ae59963f95" +} diff --git a/backend/.sqlx/query-2fcddda99dd0aacf5007ed459cb27caa754424e062427edf5ddcb95f9d96888e.json b/backend/.sqlx/query-2fcddda99dd0aacf5007ed459cb27caa754424e062427edf5ddcb95f9d96888e.json new file mode 100644 index 0000000000..5a1fbca834 --- /dev/null +++ b/backend/.sqlx/query-2fcddda99dd0aacf5007ed459cb27caa754424e062427edf5ddcb95f9d96888e.json @@ -0,0 +1,29 @@ +{ + "db_name": "PostgreSQL", + "query": "SELECT path, value FROM variable\n WHERE path LIKE ('u/' || $1 || '/%')\n AND workspace_id = $2\n AND is_secret = true\n AND value LIKE '$vault:%'", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "path", + "type_info": "Varchar" + }, + { + "ordinal": 1, + "name": "value", + "type_info": "Varchar" + } + ], + "parameters": { + "Left": [ + "Text", + "Text" + ] + }, + "nullable": [ + false, + false + ] + }, + "hash": "2fcddda99dd0aacf5007ed459cb27caa754424e062427edf5ddcb95f9d96888e" +} diff --git a/backend/.sqlx/query-306e0156ee1541710c1c6512ecb4f61baeb3ae6f31ba3fd57a3ec485108a7f49.json b/backend/.sqlx/query-306e0156ee1541710c1c6512ecb4f61baeb3ae6f31ba3fd57a3ec485108a7f49.json new file mode 100644 index 0000000000..1e05f032f0 --- /dev/null +++ b/backend/.sqlx/query-306e0156ee1541710c1c6512ecb4f61baeb3ae6f31ba3fd57a3ec485108a7f49.json @@ -0,0 +1,23 @@ +{ + "db_name": "PostgreSQL", + "query": "DELETE FROM v2_job_completed\n WHERE id IN (\n SELECT id FROM v2_job_completed\n WHERE completed_at <= now() - ($1::bigint::text || ' s')::interval\n ORDER BY completed_at ASC\n LIMIT $2\n FOR UPDATE SKIP LOCKED\n )\n RETURNING id", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "id", + "type_info": "Uuid" + } + ], + "parameters": { + "Left": [ + "Int8", + "Int8" + ] + }, + "nullable": [ + false + ] + }, + "hash": "306e0156ee1541710c1c6512ecb4f61baeb3ae6f31ba3fd57a3ec485108a7f49" +} diff --git a/backend/.sqlx/query-37e0c601a463819748078b92d1c87f63348a92908e2ca52f5a092149191e6200.json b/backend/.sqlx/query-37e0c601a463819748078b92d1c87f63348a92908e2ca52f5a092149191e6200.json new file mode 100644 index 0000000000..d1f3f24b82 --- /dev/null +++ b/backend/.sqlx/query-37e0c601a463819748078b92d1c87f63348a92908e2ca52f5a092149191e6200.json @@ -0,0 +1,16 @@ +{ + "db_name": "PostgreSQL", + "query": "UPDATE variable SET value = $1 WHERE path = $2 AND workspace_id = $3 AND is_secret = true", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "Varchar", + "Text", + "Text" + ] + }, + "nullable": [] + }, + "hash": "37e0c601a463819748078b92d1c87f63348a92908e2ca52f5a092149191e6200" +} diff --git a/backend/.sqlx/query-3a71ab02239d8dd4e2f440a35155459a7d5091c125a0998e79d89f8c560a3fd3.json b/backend/.sqlx/query-3a71ab02239d8dd4e2f440a35155459a7d5091c125a0998e79d89f8c560a3fd3.json new file mode 100644 index 0000000000..1c370d3ccf --- /dev/null +++ b/backend/.sqlx/query-3a71ab02239d8dd4e2f440a35155459a7d5091c125a0998e79d89f8c560a3fd3.json @@ -0,0 +1,12 @@ +{ + "db_name": "PostgreSQL", + "query": "\n INSERT INTO group_ (workspace_id, name, summary, extra_perms)\n VALUES ('test-workspace', 'success_handler', 'The group the success handler acts on behalf of', '{\"u/test-user\": true}')\n ON CONFLICT DO NOTHING\n ", + "describe": { + "columns": [], + "parameters": { + "Left": [] + }, + "nullable": [] + }, + "hash": "3a71ab02239d8dd4e2f440a35155459a7d5091c125a0998e79d89f8c560a3fd3" +} diff --git a/backend/.sqlx/query-c6ef0acdf20bd71dd26de981fb49f178ba8a1b8c1e01e0fec1dfd6a54ea7a894.json b/backend/.sqlx/query-45d0552a169b63241a0ecc0e7ec6a533d7eec520376b99b2fd8000f0f0acba7b.json similarity index 50% rename from backend/.sqlx/query-c6ef0acdf20bd71dd26de981fb49f178ba8a1b8c1e01e0fec1dfd6a54ea7a894.json rename to backend/.sqlx/query-45d0552a169b63241a0ecc0e7ec6a533d7eec520376b99b2fd8000f0f0acba7b.json index 0bd82635da..acad0a4a1d 100644 --- a/backend/.sqlx/query-c6ef0acdf20bd71dd26de981fb49f178ba8a1b8c1e01e0fec1dfd6a54ea7a894.json +++ b/backend/.sqlx/query-45d0552a169b63241a0ecc0e7ec6a533d7eec520376b99b2fd8000f0f0acba7b.json @@ -1,6 +1,6 @@ { "db_name": "PostgreSQL", - "query": "\nWITH rows_to_delete AS (\n SELECT concurrency_id\n FROM concurrency_counter\n \n WHERE job_uuids = '{}'::jsonb\n FOR UPDATE SKIP LOCKED\n)\nDELETE FROM concurrency_counter\nWHERE concurrency_id IN (SELECT concurrency_id FROM rows_to_delete) RETURNING concurrency_id", + "query": "\nWITH rows_to_delete AS (\n SELECT concurrency_id\n FROM concurrency_counter\n\n WHERE job_uuids = '{}'::jsonb\n FOR UPDATE SKIP LOCKED\n)\nDELETE FROM concurrency_counter\nWHERE concurrency_id IN (SELECT concurrency_id FROM rows_to_delete) RETURNING concurrency_id", "describe": { "columns": [ { @@ -16,5 +16,5 @@ false ] }, - "hash": "c6ef0acdf20bd71dd26de981fb49f178ba8a1b8c1e01e0fec1dfd6a54ea7a894" + "hash": "45d0552a169b63241a0ecc0e7ec6a533d7eec520376b99b2fd8000f0f0acba7b" } diff --git a/backend/.sqlx/query-4977b4027861ca234e95559a8b550bc8849f7ed60a471d46f3be3d39664a48ca.json b/backend/.sqlx/query-4977b4027861ca234e95559a8b550bc8849f7ed60a471d46f3be3d39664a48ca.json new file mode 100644 index 0000000000..01184690ce --- /dev/null +++ b/backend/.sqlx/query-4977b4027861ca234e95559a8b550bc8849f7ed60a471d46f3be3d39664a48ca.json @@ -0,0 +1,14 @@ +{ + "db_name": "PostgreSQL", + "query": "\n UPDATE\n workspace_settings\n SET\n success_handler = NULL,\n success_handler_extra_args = NULL\n WHERE\n workspace_id = $1\n ", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "Text" + ] + }, + "nullable": [] + }, + "hash": "4977b4027861ca234e95559a8b550bc8849f7ed60a471d46f3be3d39664a48ca" +} diff --git a/backend/.sqlx/query-70a6880960d17218bc5bf05287e2a6d9a6393c6bb1783ab8903d87dd099e236b.json b/backend/.sqlx/query-4d80985dd0794a01a2af18ae7abf4a3ab8ba3d162ed5d04735caea7295da0b20.json similarity index 91% rename from backend/.sqlx/query-70a6880960d17218bc5bf05287e2a6d9a6393c6bb1783ab8903d87dd099e236b.json rename to backend/.sqlx/query-4d80985dd0794a01a2af18ae7abf4a3ab8ba3d162ed5d04735caea7295da0b20.json index 7cfd1e5b64..4847a9a2b1 100644 --- a/backend/.sqlx/query-70a6880960d17218bc5bf05287e2a6d9a6393c6bb1783ab8903d87dd099e236b.json +++ b/backend/.sqlx/query-4d80985dd0794a01a2af18ae7abf4a3ab8ba3d162ed5d04735caea7295da0b20.json @@ -1,6 +1,6 @@ { "db_name": "PostgreSQL", - "query": "\n SELECT\n j.id AS \"id!\", j.workspace_id AS \"workspace_id!\", j.parent_job, j.flow_step_id IS NOT NULL AS \"is_flow_step?\",\n COALESCE(s.flow_status, s.workflow_as_code_status) AS \"flow_status: Box\", r.ping AS last_ping, j.same_worker AS \"same_worker?\"\n FROM v2_job_queue q JOIN v2_job j USING (id) LEFT JOIN v2_job_runtime r USING (id) LEFT JOIN v2_job_status s USING (id)\n WHERE q.running = true AND q.suspend = 0 AND q.suspend_until IS null AND q.scheduled_for <= now()\n AND (j.kind = 'flow' OR j.kind = 'flowpreview' OR j.kind = 'flownode')\n AND r.ping IS NOT NULL AND r.ping < NOW() - ($1 || ' seconds')::interval\n AND q.canceled_by IS NULL\n \n ", + "query": "\n SELECT\n j.id AS \"id!\", j.workspace_id AS \"workspace_id!\", j.parent_job, j.flow_step_id IS NOT NULL AS \"is_flow_step?\",\n COALESCE(s.flow_status, s.workflow_as_code_status) AS \"flow_status: Box\", r.ping AS last_ping, j.same_worker AS \"same_worker?\"\n FROM v2_job_queue q JOIN v2_job j USING (id) LEFT JOIN v2_job_runtime r USING (id) LEFT JOIN v2_job_status s USING (id)\n WHERE q.running = true AND q.suspend = 0 AND q.suspend_until IS null AND q.scheduled_for <= now()\n AND (j.kind = 'flow' OR j.kind = 'flowpreview' OR j.kind = 'flownode')\n AND r.ping IS NOT NULL AND r.ping < NOW() - ($1 || ' seconds')::interval\n AND q.canceled_by IS NULL\n\n ", "describe": { "columns": [ { @@ -54,5 +54,5 @@ false ] }, - "hash": "70a6880960d17218bc5bf05287e2a6d9a6393c6bb1783ab8903d87dd099e236b" + "hash": "4d80985dd0794a01a2af18ae7abf4a3ab8ba3d162ed5d04735caea7295da0b20" } diff --git a/backend/.sqlx/query-5820d34be1a7f7b72e656c692f53146f45ad4a6e584e917a0a86280d8f473c10.json b/backend/.sqlx/query-5820d34be1a7f7b72e656c692f53146f45ad4a6e584e917a0a86280d8f473c10.json deleted file mode 100644 index b83ec1db19..0000000000 --- a/backend/.sqlx/query-5820d34be1a7f7b72e656c692f53146f45ad4a6e584e917a0a86280d8f473c10.json +++ /dev/null @@ -1,22 +0,0 @@ -{ - "db_name": "PostgreSQL", - "query": "DELETE FROM v2_job_completed c\n WHERE completed_at <= now() - ($1::bigint::text || ' s')::interval \n RETURNING c.id", - "describe": { - "columns": [ - { - "ordinal": 0, - "name": "id", - "type_info": "Uuid" - } - ], - "parameters": { - "Left": [ - "Int8" - ] - }, - "nullable": [ - false - ] - }, - "hash": "5820d34be1a7f7b72e656c692f53146f45ad4a6e584e917a0a86280d8f473c10" -} diff --git a/backend/.sqlx/query-598805056a3b102c9c2fd69d8330fa78b06106236522e48ab5fe009cac059468.json b/backend/.sqlx/query-598805056a3b102c9c2fd69d8330fa78b06106236522e48ab5fe009cac059468.json new file mode 100644 index 0000000000..3c0b930a43 --- /dev/null +++ b/backend/.sqlx/query-598805056a3b102c9c2fd69d8330fa78b06106236522e48ab5fe009cac059468.json @@ -0,0 +1,38 @@ +{ + "db_name": "PostgreSQL", + "query": "\nWITH lockable_counters AS (\n SELECT concurrency_id, job_uuids\n FROM concurrency_counter\n WHERE job_uuids != '{}'::jsonb\n FOR UPDATE SKIP LOCKED\n),\nall_job_uuids AS (\n SELECT DISTINCT jsonb_object_keys(job_uuids) AS job_uuid\n FROM lockable_counters\n),\norphaned_job_uuids AS (\n SELECT job_uuid\n FROM all_job_uuids\n WHERE job_uuid NOT IN (\n SELECT id::text\n FROM v2_job_queue\n FOR SHARE SKIP LOCKED\n )\n),\norphaned_array AS (\n SELECT ARRAY(SELECT job_uuid FROM orphaned_job_uuids) AS orphaned_keys\n),\nbefore_update AS (\n SELECT lc.concurrency_id, lc.job_uuids, oa.orphaned_keys\n FROM lockable_counters lc, orphaned_array oa\n WHERE lc.job_uuids ?| oa.orphaned_keys\n),\naffected_rows AS (\n UPDATE concurrency_counter\n SET job_uuids = job_uuids - orphaned_array.orphaned_keys\n FROM orphaned_array\n WHERE concurrency_counter.concurrency_id IN (\n SELECT concurrency_id FROM before_update\n )\n RETURNING concurrency_id, job_uuids AS updated_job_uuids\n),\nexpanded_orphaned AS (\n SELECT bu.concurrency_id,\n bu.job_uuids AS original_job_uuids,\n unnest(bu.orphaned_keys) AS orphaned_key\n FROM before_update bu\n)\nSELECT\n eo.concurrency_id,\n eo.orphaned_key,\n eo.original_job_uuids,\n ar.updated_job_uuids\nFROM expanded_orphaned eo\nJOIN affected_rows ar ON eo.concurrency_id = ar.concurrency_id\nWHERE eo.original_job_uuids ? eo.orphaned_key\nORDER BY eo.concurrency_id, eo.orphaned_key\n", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "concurrency_id", + "type_info": "Varchar" + }, + { + "ordinal": 1, + "name": "orphaned_key", + "type_info": "Text" + }, + { + "ordinal": 2, + "name": "original_job_uuids", + "type_info": "Jsonb" + }, + { + "ordinal": 3, + "name": "updated_job_uuids", + "type_info": "Jsonb" + } + ], + "parameters": { + "Left": [] + }, + "nullable": [ + false, + null, + false, + false + ] + }, + "hash": "598805056a3b102c9c2fd69d8330fa78b06106236522e48ab5fe009cac059468" +} diff --git a/backend/.sqlx/query-5a219a2532517869578c4504ff3153c43903f929ae5d62fbba12610f89c36d55.json b/backend/.sqlx/query-5a219a2532517869578c4504ff3153c43903f929ae5d62fbba12610f89c36d55.json index 713ccb9dd3..36ddb8ab9f 100644 --- a/backend/.sqlx/query-5a219a2532517869578c4504ff3153c43903f929ae5d62fbba12610f89c36d55.json +++ b/backend/.sqlx/query-5a219a2532517869578c4504ff3153c43903f929ae5d62fbba12610f89c36d55.json @@ -15,7 +15,7 @@ ] }, "nullable": [ - null + true ] }, "hash": "5a219a2532517869578c4504ff3153c43903f929ae5d62fbba12610f89c36d55" diff --git a/backend/.sqlx/query-6dd7e38c902f6b8d397aa3e9e698fe1541bd3f22634af341849303de84c0034d.json b/backend/.sqlx/query-6dd7e38c902f6b8d397aa3e9e698fe1541bd3f22634af341849303de84c0034d.json new file mode 100644 index 0000000000..0107528cdd --- /dev/null +++ b/backend/.sqlx/query-6dd7e38c902f6b8d397aa3e9e698fe1541bd3f22634af341849303de84c0034d.json @@ -0,0 +1,32 @@ +{ + "db_name": "PostgreSQL", + "query": "SELECT workspace_id, path, value FROM variable WHERE is_secret = true", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "workspace_id", + "type_info": "Varchar" + }, + { + "ordinal": 1, + "name": "path", + "type_info": "Varchar" + }, + { + "ordinal": 2, + "name": "value", + "type_info": "Varchar" + } + ], + "parameters": { + "Left": [] + }, + "nullable": [ + false, + false, + false + ] + }, + "hash": "6dd7e38c902f6b8d397aa3e9e698fe1541bd3f22634af341849303de84c0034d" +} diff --git a/backend/.sqlx/query-6eaa0af9c5e0a8d64c3202753c9c3cef39361b0987a22b7d2b2346652f81177e.json b/backend/.sqlx/query-6eaa0af9c5e0a8d64c3202753c9c3cef39361b0987a22b7d2b2346652f81177e.json new file mode 100644 index 0000000000..e5a4943731 --- /dev/null +++ b/backend/.sqlx/query-6eaa0af9c5e0a8d64c3202753c9c3cef39361b0987a22b7d2b2346652f81177e.json @@ -0,0 +1,12 @@ +{ + "db_name": "PostgreSQL", + "query": "\n UPDATE workspace_settings\n SET success_handler = 'script/f/test/success_handler',\n success_handler_extra_args = '{\"key\": \"value\"}'::json\n WHERE workspace_id = 'test-workspace'\n ", + "describe": { + "columns": [], + "parameters": { + "Left": [] + }, + "nullable": [] + }, + "hash": "6eaa0af9c5e0a8d64c3202753c9c3cef39361b0987a22b7d2b2346652f81177e" +} diff --git a/backend/.sqlx/query-71de038f332593e2177f33284854b6c0bfe9bb4999f2c4e9082a4c5f3b341496.json b/backend/.sqlx/query-71de038f332593e2177f33284854b6c0bfe9bb4999f2c4e9082a4c5f3b341496.json new file mode 100644 index 0000000000..b647ec29fa --- /dev/null +++ b/backend/.sqlx/query-71de038f332593e2177f33284854b6c0bfe9bb4999f2c4e9082a4c5f3b341496.json @@ -0,0 +1,38 @@ +{ + "db_name": "PostgreSQL", + "query": "\n SELECT\n id,\n runnable_path,\n permissioned_as_email,\n parent_job\n FROM v2_job\n WHERE workspace_id = 'test-workspace'\n AND permissioned_as_email = 'success_handler@windmill.dev'\n ORDER BY created_at DESC\n LIMIT 1\n ", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "id", + "type_info": "Uuid" + }, + { + "ordinal": 1, + "name": "runnable_path", + "type_info": "Varchar" + }, + { + "ordinal": 2, + "name": "permissioned_as_email", + "type_info": "Varchar" + }, + { + "ordinal": 3, + "name": "parent_job", + "type_info": "Uuid" + } + ], + "parameters": { + "Left": [] + }, + "nullable": [ + false, + true, + false, + true + ] + }, + "hash": "71de038f332593e2177f33284854b6c0bfe9bb4999f2c4e9082a4c5f3b341496" +} diff --git a/backend/.sqlx/query-8f5722afde37d22c56851da23895daa1653e14b6d12013f6e3bb8bf042447a6a.json b/backend/.sqlx/query-8f5722afde37d22c56851da23895daa1653e14b6d12013f6e3bb8bf042447a6a.json new file mode 100644 index 0000000000..a298c5fb8c --- /dev/null +++ b/backend/.sqlx/query-8f5722afde37d22c56851da23895daa1653e14b6d12013f6e3bb8bf042447a6a.json @@ -0,0 +1,12 @@ +{ + "db_name": "PostgreSQL", + "query": "UPDATE variable SET value = 'ROUND_TRIP_CLEARED' WHERE is_secret = true", + "describe": { + "columns": [], + "parameters": { + "Left": [] + }, + "nullable": [] + }, + "hash": "8f5722afde37d22c56851da23895daa1653e14b6d12013f6e3bb8bf042447a6a" +} diff --git a/backend/.sqlx/query-9044c71dc8aab7f5d17bf27c4702e67f3eee46d7dbccb09cadeaafa0534952d6.json b/backend/.sqlx/query-9044c71dc8aab7f5d17bf27c4702e67f3eee46d7dbccb09cadeaafa0534952d6.json new file mode 100644 index 0000000000..ed96f00f9c --- /dev/null +++ b/backend/.sqlx/query-9044c71dc8aab7f5d17bf27c4702e67f3eee46d7dbccb09cadeaafa0534952d6.json @@ -0,0 +1,16 @@ +{ + "db_name": "PostgreSQL", + "query": "\n UPDATE\n workspace_settings\n SET\n success_handler = $1,\n success_handler_extra_args = $2\n WHERE\n workspace_id = $3\n ", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "Text", + "Json", + "Text" + ] + }, + "nullable": [] + }, + "hash": "9044c71dc8aab7f5d17bf27c4702e67f3eee46d7dbccb09cadeaafa0534952d6" +} diff --git a/backend/.sqlx/query-93ef241a4f624cb76d680418d313cd956e4c807c3a0031913dbfded80f0e5881.json b/backend/.sqlx/query-93ef241a4f624cb76d680418d313cd956e4c807c3a0031913dbfded80f0e5881.json new file mode 100644 index 0000000000..288f40474f --- /dev/null +++ b/backend/.sqlx/query-93ef241a4f624cb76d680418d313cd956e4c807c3a0031913dbfded80f0e5881.json @@ -0,0 +1,12 @@ +{ + "db_name": "PostgreSQL", + "query": "UPDATE variable SET value = 'CLEARED' WHERE is_secret = true", + "describe": { + "columns": [], + "parameters": { + "Left": [] + }, + "nullable": [] + }, + "hash": "93ef241a4f624cb76d680418d313cd956e4c807c3a0031913dbfded80f0e5881" +} diff --git a/backend/.sqlx/query-9da0cea2a5d0464ca78cfeccf6cedf2b1c0e6e6cb3c9183a937a68465debdb06.json b/backend/.sqlx/query-9da0cea2a5d0464ca78cfeccf6cedf2b1c0e6e6cb3c9183a937a68465debdb06.json deleted file mode 100644 index 9f42f0f54d..0000000000 --- a/backend/.sqlx/query-9da0cea2a5d0464ca78cfeccf6cedf2b1c0e6e6cb3c9183a937a68465debdb06.json +++ /dev/null @@ -1,14 +0,0 @@ -{ - "db_name": "PostgreSQL", - "query": "DELETE FROM concurrency_key WHERE ended_at <= now() - ($1::bigint::text || ' s')::interval ", - "describe": { - "columns": [], - "parameters": { - "Left": [ - "Int8" - ] - }, - "nullable": [] - }, - "hash": "9da0cea2a5d0464ca78cfeccf6cedf2b1c0e6e6cb3c9183a937a68465debdb06" -} diff --git a/backend/.sqlx/query-a2bc410e1066ddc89ca4557ca0e29339e41cbc7a7ef6ad97b3f0eb93de04c359.json b/backend/.sqlx/query-a2bc410e1066ddc89ca4557ca0e29339e41cbc7a7ef6ad97b3f0eb93de04c359.json new file mode 100644 index 0000000000..388a6900d3 --- /dev/null +++ b/backend/.sqlx/query-a2bc410e1066ddc89ca4557ca0e29339e41cbc7a7ef6ad97b3f0eb93de04c359.json @@ -0,0 +1,12 @@ +{ + "db_name": "PostgreSQL", + "query": "\n UPDATE workspace_settings\n SET success_handler = 'flow/f/test/success_flow'\n WHERE workspace_id = 'test-workspace'\n ", + "describe": { + "columns": [], + "parameters": { + "Left": [] + }, + "nullable": [] + }, + "hash": "a2bc410e1066ddc89ca4557ca0e29339e41cbc7a7ef6ad97b3f0eb93de04c359" +} diff --git a/backend/.sqlx/query-ade0696ec69bec2258d2e3ff86bfba6ed3b1d573c298f9d7fe8056ba7e32ed81.json b/backend/.sqlx/query-ade0696ec69bec2258d2e3ff86bfba6ed3b1d573c298f9d7fe8056ba7e32ed81.json new file mode 100644 index 0000000000..3fe61f6f8e --- /dev/null +++ b/backend/.sqlx/query-ade0696ec69bec2258d2e3ff86bfba6ed3b1d573c298f9d7fe8056ba7e32ed81.json @@ -0,0 +1,32 @@ +{ + "db_name": "PostgreSQL", + "query": "SELECT workspace_id, path, value FROM variable WHERE is_secret = true AND value IS NOT NULL AND value != ''", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "workspace_id", + "type_info": "Varchar" + }, + { + "ordinal": 1, + "name": "path", + "type_info": "Varchar" + }, + { + "ordinal": 2, + "name": "value", + "type_info": "Varchar" + } + ], + "parameters": { + "Left": [] + }, + "nullable": [ + false, + false, + false + ] + }, + "hash": "ade0696ec69bec2258d2e3ff86bfba6ed3b1d573c298f9d7fe8056ba7e32ed81" +} diff --git a/backend/.sqlx/query-b030efec7c7f770bf2ed5331a469aae723925270d9d5124cc4cb2c03db61dea8.json b/backend/.sqlx/query-b030efec7c7f770bf2ed5331a469aae723925270d9d5124cc4cb2c03db61dea8.json new file mode 100644 index 0000000000..8a0de9e06e --- /dev/null +++ b/backend/.sqlx/query-b030efec7c7f770bf2ed5331a469aae723925270d9d5124cc4cb2c03db61dea8.json @@ -0,0 +1,20 @@ +{ + "db_name": "PostgreSQL", + "query": "SELECT COUNT(*) as count FROM variable WHERE is_secret = true AND value = 'CLEARED'", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "count", + "type_info": "Int8" + } + ], + "parameters": { + "Left": [] + }, + "nullable": [ + null + ] + }, + "hash": "b030efec7c7f770bf2ed5331a469aae723925270d9d5124cc4cb2c03db61dea8" +} diff --git a/backend/.sqlx/query-b107d602c60c08f0edcaa99695f8546bf30074bad903959f7df030e0ac70a86f.json b/backend/.sqlx/query-b107d602c60c08f0edcaa99695f8546bf30074bad903959f7df030e0ac70a86f.json new file mode 100644 index 0000000000..705e0b4c5b --- /dev/null +++ b/backend/.sqlx/query-b107d602c60c08f0edcaa99695f8546bf30074bad903959f7df030e0ac70a86f.json @@ -0,0 +1,32 @@ +{ + "db_name": "PostgreSQL", + "query": "SELECT workspace_id, path, value FROM variable WHERE is_secret = true AND value != 'CLEARED' ORDER BY workspace_id, path", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "workspace_id", + "type_info": "Varchar" + }, + { + "ordinal": 1, + "name": "path", + "type_info": "Varchar" + }, + { + "ordinal": 2, + "name": "value", + "type_info": "Varchar" + } + ], + "parameters": { + "Left": [] + }, + "nullable": [ + false, + false, + false + ] + }, + "hash": "b107d602c60c08f0edcaa99695f8546bf30074bad903959f7df030e0ac70a86f" +} diff --git a/backend/.sqlx/query-b10d2ab53bd8d24b0bbcede8211de229d507784fbcdf46c309907df123e35018.json b/backend/.sqlx/query-b10d2ab53bd8d24b0bbcede8211de229d507784fbcdf46c309907df123e35018.json new file mode 100644 index 0000000000..534eea5676 --- /dev/null +++ b/backend/.sqlx/query-b10d2ab53bd8d24b0bbcede8211de229d507784fbcdf46c309907df123e35018.json @@ -0,0 +1,32 @@ +{ + "db_name": "PostgreSQL", + "query": "SELECT workspace_id, path, value FROM variable WHERE is_secret = true ORDER BY workspace_id, path", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "workspace_id", + "type_info": "Varchar" + }, + { + "ordinal": 1, + "name": "path", + "type_info": "Varchar" + }, + { + "ordinal": 2, + "name": "value", + "type_info": "Varchar" + } + ], + "parameters": { + "Left": [] + }, + "nullable": [ + false, + false, + false + ] + }, + "hash": "b10d2ab53bd8d24b0bbcede8211de229d507784fbcdf46c309907df123e35018" +} diff --git a/backend/.sqlx/query-bb289fd24f443f0f8917ec55bc9fc3113e37dd8009425e558b5f5d1e1543b513.json b/backend/.sqlx/query-bb289fd24f443f0f8917ec55bc9fc3113e37dd8009425e558b5f5d1e1543b513.json new file mode 100644 index 0000000000..a687e1a7e3 --- /dev/null +++ b/backend/.sqlx/query-bb289fd24f443f0f8917ec55bc9fc3113e37dd8009425e558b5f5d1e1543b513.json @@ -0,0 +1,16 @@ +{ + "db_name": "PostgreSQL", + "query": "UPDATE variable SET value = $1 WHERE path = $2 AND workspace_id = $3", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "Varchar", + "Text", + "Text" + ] + }, + "nullable": [] + }, + "hash": "bb289fd24f443f0f8917ec55bc9fc3113e37dd8009425e558b5f5d1e1543b513" +} diff --git a/backend/.sqlx/query-3c53de373b9f1034b5f43002bf4715e12ddc641f4ca52efe0335719fa9461bb0.json b/backend/.sqlx/query-d0dfd217a4f53aaee7b83d5b5267f74e5ad617830dd835a45e17fd378bcbeada.json similarity index 84% rename from backend/.sqlx/query-3c53de373b9f1034b5f43002bf4715e12ddc641f4ca52efe0335719fa9461bb0.json rename to backend/.sqlx/query-d0dfd217a4f53aaee7b83d5b5267f74e5ad617830dd835a45e17fd378bcbeada.json index 083f74e184..fc7f0e62a8 100644 --- a/backend/.sqlx/query-3c53de373b9f1034b5f43002bf4715e12ddc641f4ca52efe0335719fa9461bb0.json +++ b/backend/.sqlx/query-d0dfd217a4f53aaee7b83d5b5267f74e5ad617830dd835a45e17fd378bcbeada.json @@ -1,6 +1,6 @@ { "db_name": "PostgreSQL", - "query": "\n SELECT\n workspace_id,\n slack_team_id,\n teams_team_id,\n teams_team_name,\n teams_team_guid,\n slack_name,\n slack_command_script,\n teams_command_script,\n slack_email,\n slack_oauth_client_id,\n slack_oauth_client_secret,\n auto_invite_domain,\n auto_invite_operator,\n auto_add,\n customer_id,\n plan,\n webhook,\n deploy_to,\n ai_config,\n error_handler,\n error_handler_extra_args,\n error_handler_muted_on_cancel,\n large_file_storage,\n datatable,\n ducklake,\n git_sync,\n deploy_ui,\n default_app,\n default_scripts,\n mute_critical_alerts,\n color,\n operator_settings,\n git_app_installations,\n auto_add_instance_groups,\n auto_add_instance_groups_roles\n FROM\n workspace_settings\n WHERE\n workspace_id = $1\n ", + "query": "\n SELECT\n workspace_id,\n slack_team_id,\n teams_team_id,\n teams_team_name,\n teams_team_guid,\n slack_name,\n slack_command_script,\n teams_command_script,\n slack_email,\n slack_oauth_client_id,\n slack_oauth_client_secret,\n auto_invite_domain,\n auto_invite_operator,\n auto_add,\n customer_id,\n plan,\n webhook,\n deploy_to,\n ai_config,\n error_handler,\n error_handler_extra_args,\n error_handler_muted_on_cancel,\n success_handler,\n success_handler_extra_args,\n large_file_storage,\n datatable,\n ducklake,\n git_sync,\n deploy_ui,\n default_app,\n default_scripts,\n mute_critical_alerts,\n color,\n operator_settings,\n git_app_installations,\n auto_add_instance_groups,\n auto_add_instance_groups_roles\n FROM\n workspace_settings\n WHERE\n workspace_id = $1\n ", "describe": { "columns": [ { @@ -115,66 +115,76 @@ }, { "ordinal": 22, + "name": "success_handler", + "type_info": "Text" + }, + { + "ordinal": 23, + "name": "success_handler_extra_args", + "type_info": "Json" + }, + { + "ordinal": 24, "name": "large_file_storage", "type_info": "Jsonb" }, { - "ordinal": 23, + "ordinal": 25, "name": "datatable", "type_info": "Jsonb" }, { - "ordinal": 24, + "ordinal": 26, "name": "ducklake", "type_info": "Jsonb" }, { - "ordinal": 25, + "ordinal": 27, "name": "git_sync", "type_info": "Jsonb" }, { - "ordinal": 26, + "ordinal": 28, "name": "deploy_ui", "type_info": "Jsonb" }, { - "ordinal": 27, + "ordinal": 29, "name": "default_app", "type_info": "Varchar" }, { - "ordinal": 28, + "ordinal": 30, "name": "default_scripts", "type_info": "Jsonb" }, { - "ordinal": 29, + "ordinal": 31, "name": "mute_critical_alerts", "type_info": "Bool" }, { - "ordinal": 30, + "ordinal": 32, "name": "color", "type_info": "Varchar" }, { - "ordinal": 31, + "ordinal": 33, "name": "operator_settings", "type_info": "Jsonb" }, { - "ordinal": 32, + "ordinal": 34, "name": "git_app_installations", "type_info": "Jsonb" }, { - "ordinal": 33, + "ordinal": 35, "name": "auto_add_instance_groups", "type_info": "TextArray" }, { - "ordinal": 34, + "ordinal": 36, "name": "auto_add_instance_groups_roles", "type_info": "Jsonb" } @@ -217,10 +227,12 @@ true, true, true, + true, + true, false, true, true ] }, - "hash": "3c53de373b9f1034b5f43002bf4715e12ddc641f4ca52efe0335719fa9461bb0" + "hash": "d0dfd217a4f53aaee7b83d5b5267f74e5ad617830dd835a45e17fd378bcbeada" } diff --git a/backend/.sqlx/query-dbc05cbf6720cc85e01118bcdcbb478724ac0ea8f3e043da2de9d932216cc730.json b/backend/.sqlx/query-dbc05cbf6720cc85e01118bcdcbb478724ac0ea8f3e043da2de9d932216cc730.json new file mode 100644 index 0000000000..6e5f8645bf --- /dev/null +++ b/backend/.sqlx/query-dbc05cbf6720cc85e01118bcdcbb478724ac0ea8f3e043da2de9d932216cc730.json @@ -0,0 +1,15 @@ +{ + "db_name": "PostgreSQL", + "query": "\n INSERT INTO script (workspace_id, hash, path, content, language, kind, created_by, schema, summary, description, lock)\n VALUES ('test-workspace', $1, 'f/test/simple_script', $2, 'deno', 'script', 'test-user', '{}', 'Simple test script', 'A simple test script', '')\n ", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "Int8", + "Text" + ] + }, + "nullable": [] + }, + "hash": "dbc05cbf6720cc85e01118bcdcbb478724ac0ea8f3e043da2de9d932216cc730" +} diff --git a/backend/.sqlx/query-e3f2459d9ac3534531d5724bb2d67be6f33f8ca02ef4768bccd32a33a70a4300.json b/backend/.sqlx/query-e3f2459d9ac3534531d5724bb2d67be6f33f8ca02ef4768bccd32a33a70a4300.json new file mode 100644 index 0000000000..fd0ae9a5ab --- /dev/null +++ b/backend/.sqlx/query-e3f2459d9ac3534531d5724bb2d67be6f33f8ca02ef4768bccd32a33a70a4300.json @@ -0,0 +1,12 @@ +{ + "db_name": "PostgreSQL", + "query": "\n UPDATE workspace_settings\n SET success_handler = 'script/f/test/success_handler'\n WHERE workspace_id = 'test-workspace'\n ", + "describe": { + "columns": [], + "parameters": { + "Left": [] + }, + "nullable": [] + }, + "hash": "e3f2459d9ac3534531d5724bb2d67be6f33f8ca02ef4768bccd32a33a70a4300" +} diff --git a/backend/Cargo.lock b/backend/Cargo.lock index 7455c4e5a6..7981eebf65 100644 --- a/backend/Cargo.lock +++ b/backend/Cargo.lock @@ -1716,7 +1716,7 @@ dependencies = [ "serde_json", "serde_repr", "serde_urlencoded", - "thiserror 2.0.17", + "thiserror 2.0.18", "tokio", "tokio-util", "tower-service", @@ -2082,9 +2082,9 @@ dependencies = [ [[package]] name = "cc" -version = "1.2.52" +version = "1.2.53" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cd4932aefd12402b36c60956a4fe0035421f544799057659ff86f923657aada3" +checksum = "755d2fce177175ffca841e9a06afdb2c4ab0f593d53b4dee48147dfaade85932" dependencies = [ "find-msvc-tools", "jobserver", @@ -3557,7 +3557,7 @@ dependencies = [ "swc_visit", "swc_visit_macros", "text_lines", - "thiserror 2.0.17", + "thiserror 2.0.18", "unicode-width 0.1.14", "url", ] @@ -3571,7 +3571,7 @@ dependencies = [ "async-trait", "deno_core", "deno_error", - "thiserror 2.0.17", + "thiserror 2.0.18", "tokio", "uuid", ] @@ -3588,7 +3588,7 @@ dependencies = [ "rusqlite", "serde", "sha2 0.10.9", - "thiserror 2.0.17", + "thiserror 2.0.18", "tokio", ] @@ -3632,7 +3632,7 @@ dependencies = [ "deno_webgpu", "image", "serde", - "thiserror 2.0.17", + "thiserror 2.0.18", ] [[package]] @@ -3658,7 +3658,7 @@ dependencies = [ "serde", "serde_json", "sys_traits", - "thiserror 2.0.17", + "thiserror 2.0.18", "url", ] @@ -3703,7 +3703,7 @@ dependencies = [ "smallvec", "sourcemap 8.0.1", "static_assertions", - "thiserror 2.0.17", + "thiserror 2.0.18", "tokio", "url", "v8", @@ -3728,7 +3728,7 @@ dependencies = [ "deno_core", "deno_error", "saffron", - "thiserror 2.0.17", + "thiserror 2.0.18", "tokio", ] @@ -3766,7 +3766,7 @@ dependencies = [ "sha2 0.10.9", "signature", "spki", - "thiserror 2.0.17", + "thiserror 2.0.18", "tokio", "uuid", "x25519-dalek", @@ -3826,7 +3826,7 @@ dependencies = [ "rustls-webpki 0.102.8", "serde", "serde_json", - "thiserror 2.0.17", + "thiserror 2.0.18", "tokio", "tokio-rustls 0.26.4", "tokio-socks", @@ -3854,7 +3854,7 @@ dependencies = [ "serde", "serde-value", "serde_json", - "thiserror 2.0.17", + "thiserror 2.0.18", "tokio", "winapi", ] @@ -3880,7 +3880,7 @@ dependencies = [ "rand 0.8.5", "rayon", "serde", - "thiserror 2.0.17", + "thiserror 2.0.18", "winapi", "windows-sys 0.59.0", ] @@ -3919,7 +3919,7 @@ dependencies = [ "scopeguard", "serde", "smallvec", - "thiserror 2.0.17", + "thiserror 2.0.18", "tokio", "tokio-util", ] @@ -3979,7 +3979,7 @@ dependencies = [ "rand 0.8.5", "rusqlite", "serde", - "thiserror 2.0.17", + "thiserror 2.0.18", "url", ] @@ -3992,7 +3992,7 @@ dependencies = [ "deno_semver", "serde", "serde_json", - "thiserror 2.0.17", + "thiserror 2.0.18", ] [[package]] @@ -4019,7 +4019,7 @@ dependencies = [ "libloading 0.7.4", "log", "napi_sym", - "thiserror 2.0.17", + "thiserror 2.0.18", "windows-sys 0.59.0", ] @@ -4053,7 +4053,7 @@ dependencies = [ "rustls-tokio-stream", "serde", "socket2 0.5.10", - "thiserror 2.0.17", + "thiserror 2.0.18", "tokio", ] @@ -4140,7 +4140,7 @@ dependencies = [ "spki", "stable_deref_trait", "sys_traits", - "thiserror 2.0.17", + "thiserror 2.0.18", "tokio", "tokio-eld", "url", @@ -4168,7 +4168,7 @@ dependencies = [ "monch", "serde", "serde_json", - "thiserror 2.0.17", + "thiserror 2.0.18", "url", ] @@ -4186,7 +4186,7 @@ dependencies = [ "strum 0.25.0", "strum_macros 0.25.3", "syn 2.0.114", - "thiserror 2.0.17", + "thiserror 2.0.18", ] [[package]] @@ -4207,7 +4207,7 @@ dependencies = [ "serde", "signal-hook", "signal-hook-registry", - "thiserror 2.0.17", + "thiserror 2.0.18", "tokio", "winapi", ] @@ -4226,7 +4226,7 @@ dependencies = [ "serde", "serde_json", "sys_traits", - "thiserror 2.0.17", + "thiserror 2.0.18", "url", ] @@ -4239,7 +4239,7 @@ dependencies = [ "deno_error", "percent-encoding", "sys_traits", - "thiserror 2.0.17", + "thiserror 2.0.18", "url", ] @@ -4260,7 +4260,7 @@ dependencies = [ "once_cell", "percent-encoding", "serde", - "thiserror 2.0.17", + "thiserror 2.0.18", "which 6.0.3", "winapi", ] @@ -4287,7 +4287,7 @@ dependencies = [ "serde", "simd-json", "tempfile", - "thiserror 2.0.17", + "thiserror 2.0.18", "tokio", "which 6.0.3", "winapi", @@ -4321,7 +4321,7 @@ dependencies = [ "once_cell", "parking_lot", "sys_traits", - "thiserror 2.0.17", + "thiserror 2.0.18", "url", ] @@ -4386,7 +4386,7 @@ dependencies = [ "serde", "sys_traits", "tempfile", - "thiserror 2.0.17", + "thiserror 2.0.18", "tokio", "tokio-metrics", "twox-hash 1.6.3", @@ -4409,7 +4409,7 @@ dependencies = [ "monch", "once_cell", "serde", - "thiserror 2.0.17", + "thiserror 2.0.18", "url", ] @@ -4436,7 +4436,7 @@ dependencies = [ "opentelemetry_sdk", "pin-project", "serde", - "thiserror 2.0.17", + "thiserror 2.0.18", "tokio", ] @@ -4464,7 +4464,7 @@ dependencies = [ "rustls-tokio-stream", "rustls-webpki 0.102.8", "serde", - "thiserror 2.0.17", + "thiserror 2.0.18", "tokio", "webpki-roots 0.26.11", ] @@ -4488,7 +4488,7 @@ checksum = "d79e743ad841f7826d46c6944580f5ba665fe9ab4c31a68c4eed8b5a78225da3" dependencies = [ "deno_core", "deno_error", - "thiserror 2.0.17", + "thiserror 2.0.18", "urlpattern", ] @@ -4508,7 +4508,7 @@ dependencies = [ "flate2", "futures", "serde", - "thiserror 2.0.17", + "thiserror 2.0.18", "tokio", "uuid", ] @@ -4523,7 +4523,7 @@ dependencies = [ "deno_error", "raw-window-handle", "serde", - "thiserror 2.0.17", + "thiserror 2.0.18", "tokio", "wgpu-core", "wgpu-types", @@ -4559,7 +4559,7 @@ dependencies = [ "once_cell", "rustls-tokio-stream", "serde", - "thiserror 2.0.17", + "thiserror 2.0.18", "tokio", ] @@ -4573,7 +4573,7 @@ dependencies = [ "deno_error", "deno_web", "rusqlite", - "thiserror 2.0.17", + "thiserror 2.0.18", ] [[package]] @@ -4621,7 +4621,7 @@ dependencies = [ "rand 0.8.5", "serde", "serde_json", - "thiserror 2.0.17", + "thiserror 2.0.18", "tokio", "tokio-util", "url", @@ -4646,7 +4646,7 @@ dependencies = [ "rand 0.8.5", "rusqlite", "serde_json", - "thiserror 2.0.17", + "thiserror 2.0.18", "tokio", "tokio-stream", "uuid", @@ -5474,21 +5474,20 @@ checksum = "28dea519a9695b9977216879a3ebfddf92f1c08c05d984f8996aecd6ecdc811d" [[package]] name = "filetime" -version = "0.2.26" +version = "0.2.27" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bc0505cd1b6fa6580283f6bdf70a73fcf4aba1184038c90902b92b3dd0df63ed" +checksum = "f98844151eee8917efc50bd9e8318cb963ae8b297431495d3f758616ea5c57db" dependencies = [ "cfg-if", "libc", "libredox", - "windows-sys 0.60.2", ] [[package]] name = "find-msvc-tools" -version = "0.1.7" +version = "0.1.8" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f449e6c6c08c865631d4890cfacf252b3d396c9bcc83adb6623cdb02a8336c41" +checksum = "8591b0bcc8a98a64310a2fae1bb3e9b8564dd10e381e6e28010fde8e8e8568db" [[package]] name = "fixedbitset" @@ -6575,7 +6574,7 @@ dependencies = [ "reqwest 0.12.28", "serde", "serde_json", - "thiserror 2.0.17", + "thiserror 2.0.18", "tokio", "ureq", "windows-sys 0.60.2", @@ -6600,7 +6599,7 @@ dependencies = [ "once_cell", "rand 0.9.0", "serde", - "thiserror 2.0.17", + "thiserror 2.0.18", "tinyvec", "tokio", "tracing", @@ -6624,7 +6623,7 @@ dependencies = [ "resolv-conf", "serde", "smallvec", - "thiserror 2.0.17", + "thiserror 2.0.18", "tokio", "tracing", ] @@ -7147,7 +7146,7 @@ dependencies = [ "percent-encoding", "serde", "serde_json", - "thiserror 2.0.17", + "thiserror 2.0.18", "url", ] @@ -7427,7 +7426,7 @@ dependencies = [ "pest_derive", "regex", "serde_json", - "thiserror 2.0.17", + "thiserror 2.0.18", ] [[package]] @@ -7617,7 +7616,7 @@ dependencies = [ "serde", "serde_json", "serde_yaml", - "thiserror 2.0.17", + "thiserror 2.0.18", "tokio", "tokio-util", "tower 0.5.3", @@ -7641,7 +7640,7 @@ dependencies = [ "serde", "serde-value", "serde_json", - "thiserror 2.0.17", + "thiserror 2.0.18", ] [[package]] @@ -7679,7 +7678,7 @@ dependencies = [ "pin-project", "serde", "serde_json", - "thiserror 2.0.17", + "thiserror 2.0.18", "tokio", "tokio-util", "tracing", @@ -8470,7 +8469,7 @@ dependencies = [ "quote", "syn 2.0.114", "termcolor", - "thiserror 2.0.17", + "thiserror 2.0.18", ] [[package]] @@ -8495,7 +8494,7 @@ dependencies = [ "serde", "serde_json", "socket2 0.5.10", - "thiserror 2.0.17", + "thiserror 2.0.18", "tokio", "tokio-native-tls", "tokio-util", @@ -8527,7 +8526,7 @@ dependencies = [ "serde_json", "sha1", "sha2 0.10.9", - "thiserror 2.0.17", + "thiserror 2.0.18", "uuid", ] @@ -8697,7 +8696,7 @@ dependencies = [ "serde", "serde_json", "sys_traits", - "thiserror 2.0.17", + "thiserror 2.0.18", "url", ] @@ -8845,7 +8844,7 @@ dependencies = [ "num-format", "serde", "serde_json", - "thiserror 2.0.17", + "thiserror 2.0.18", "typetag", "windows-sys 0.48.0", ] @@ -9120,7 +9119,7 @@ dependencies = [ "serde", "serde_json", "serde_urlencoded", - "thiserror 2.0.17", + "thiserror 2.0.18", "tokio", "tracing", "url", @@ -10202,7 +10201,7 @@ dependencies = [ "lazy_static", "memchr", "parking_lot", - "thiserror 2.0.17", + "thiserror 2.0.18", ] [[package]] @@ -10387,7 +10386,7 @@ dependencies = [ "rustc-hash 2.1.1", "rustls 0.23.35", "socket2 0.6.1", - "thiserror 2.0.17", + "thiserror 2.0.18", "tokio", "tracing", "web-time", @@ -10409,7 +10408,7 @@ dependencies = [ "rustls 0.23.35", "rustls-pki-types", "slab", - "thiserror 2.0.17", + "thiserror 2.0.18", "tinyvec", "tracing", "web-time", @@ -10696,7 +10695,7 @@ checksum = "a4e608c6638b9c18977b00b475ac1f28d14e84b27d8d42f70e0bf1e3dec127ac" dependencies = [ "getrandom 0.2.17", "libredox", - "thiserror 2.0.17", + "thiserror 2.0.18", ] [[package]] @@ -10873,7 +10872,7 @@ dependencies = [ "http 1.4.0", "reqwest 0.13.1", "serde", - "thiserror 2.0.17", + "thiserror 2.0.18", "tower-service", ] @@ -10892,7 +10891,7 @@ dependencies = [ "reqwest 0.13.1", "reqwest-middleware", "retry-policies", - "thiserror 2.0.17", + "thiserror 2.0.18", "tokio", "tracing", "wasmtimer", @@ -11015,7 +11014,7 @@ dependencies = [ "serde", "serde_json", "sse-stream", - "thiserror 2.0.17", + "thiserror 2.0.18", "tokio", "tokio-stream", "tokio-util", @@ -11274,7 +11273,7 @@ dependencies = [ "once_cell", "ring 0.17.14", "rustls-pki-types", - "rustls-webpki 0.103.8", + "rustls-webpki 0.103.9", "subtle", "zeroize", ] @@ -11336,9 +11335,9 @@ dependencies = [ [[package]] name = "rustls-pki-types" -version = "1.13.3" +version = "1.14.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4910321ebe4151be888e35fe062169554e74aad01beafed60410131420ceffbc" +checksum = "be040f8b0a225e40375822a563fa9524378b9d63112f53e19ffff34df5d33fdd" dependencies = [ "web-time", "zeroize", @@ -11358,7 +11357,7 @@ dependencies = [ "rustls 0.23.35", "rustls-native-certs 0.8.3", "rustls-platform-verifier-android", - "rustls-webpki 0.103.8", + "rustls-webpki 0.103.9", "security-framework 3.5.1", "security-framework-sys", "webpki-root-certs 1.0.5", @@ -11406,9 +11405,9 @@ dependencies = [ [[package]] name = "rustls-webpki" -version = "0.103.8" +version = "0.103.9" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2ffdfa2f5286e2247234e03f680868ac2815974dc39e00ea15adc445d0aafe52" +checksum = "d7df23109aa6c1567d1c575b9952556388da57401e4ace1d15f79eedad0d8f53" dependencies = [ "aws-lc-rs", "ring 0.17.14", @@ -11938,7 +11937,7 @@ dependencies = [ "num-bigint", "serde", "smallvec", - "thiserror 2.0.17", + "thiserror 2.0.18", "v8", ] @@ -12167,7 +12166,7 @@ checksum = "297f631f50729c8c99b84667867963997ec0b50f32b2a7dbcab828ef0541e8bb" dependencies = [ "num-bigint", "num-traits", - "thiserror 2.0.17", + "thiserror 2.0.18", "time", ] @@ -12460,7 +12459,7 @@ dependencies = [ "serde_json", "sha2 0.10.9", "smallvec", - "thiserror 2.0.17", + "thiserror 2.0.18", "tokio", "tokio-stream", "tracing", @@ -12546,7 +12545,7 @@ dependencies = [ "smallvec", "sqlx-core", "stringprep", - "thiserror 2.0.17", + "thiserror 2.0.18", "tracing", "uuid", "whoami", @@ -12587,7 +12586,7 @@ dependencies = [ "smallvec", "sqlx-core", "stringprep", - "thiserror 2.0.17", + "thiserror 2.0.18", "tracing", "uuid", "whoami", @@ -12613,7 +12612,7 @@ dependencies = [ "serde", "serde_urlencoded", "sqlx-core", - "thiserror 2.0.17", + "thiserror 2.0.18", "tracing", "url", "uuid", @@ -13345,7 +13344,7 @@ dependencies = [ "tantivy-stacker", "tantivy-tokenizer-api", "tempfile", - "thiserror 2.0.17", + "thiserror 2.0.18", "time", "uuid", "winapi", @@ -13517,11 +13516,11 @@ dependencies = [ [[package]] name = "thiserror" -version = "2.0.17" +version = "2.0.18" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f63587ca0f12b72a0600bcba1d40081f830876000bb46dd2337a3051618f4fc8" +checksum = "4288b5bcbc7920c07a1149a35cf9590a2aa808e0bc1eafaade0b80947865fbc4" dependencies = [ - "thiserror-impl 2.0.17", + "thiserror-impl 2.0.18", ] [[package]] @@ -13537,9 +13536,9 @@ dependencies = [ [[package]] name = "thiserror-impl" -version = "2.0.17" +version = "2.0.18" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3ff15c8ecd7de3849db632e14d18d2571fa09dfc5ed93479bc4485c7a517c913" +checksum = "ebc4ee7f67670e9b64d05fa4253e753e016c6c95ff35b89b7941d6b856dec1d5" dependencies = [ "proc-macro2", "quote", @@ -14191,7 +14190,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "786d480bce6247ab75f005b14ae1624ad978d3029d9113f0a22fa1ac773faeaf" dependencies = [ "crossbeam-channel", - "thiserror 2.0.17", + "thiserror 2.0.18", "time", "tracing-subscriber", ] @@ -15013,7 +15012,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "2eeee3bdea6257cc36d756fa745a70f9d393571e47d69e0ed97581676a5369ca" dependencies = [ "deno_error", - "thiserror 2.0.17", + "thiserror 2.0.18", ] [[package]] @@ -15243,7 +15242,7 @@ checksum = "712e227841d057c1ee1cd2fb22fa7e5a5461ae8e48fa2ca79ec42cfc1931183f" [[package]] name = "windmill" -version = "1.608.0" +version = "1.611.0" dependencies = [ "anyhow", "aws-sdk-config", @@ -15306,7 +15305,7 @@ dependencies = [ [[package]] name = "windmill-api" -version = "1.608.0" +version = "1.611.0" dependencies = [ "anyhow", "argon2", @@ -15343,6 +15342,7 @@ dependencies = [ "deno_core", "deno_error", "ed25519-dalek", + "flate2", "futures", "git-version", "google-cloud-googleapis", @@ -15390,8 +15390,9 @@ dependencies = [ "sha2 0.10.9", "sql-builder", "sqlx", + "tar", "tempfile", - "thiserror 2.0.17", + "thiserror 2.0.18", "time", "tinyvector", "tokenizers", @@ -15429,7 +15430,7 @@ dependencies = [ [[package]] name = "windmill-api-client" -version = "1.608.0" +version = "1.611.0" dependencies = [ "reqwest 0.12.28", "serde", @@ -15439,7 +15440,7 @@ dependencies = [ [[package]] name = "windmill-audit" -version = "1.608.0" +version = "1.611.0" dependencies = [ "chrono", "lazy_static", @@ -15453,7 +15454,7 @@ dependencies = [ [[package]] name = "windmill-autoscaling" -version = "1.608.0" +version = "1.611.0" dependencies = [ "anyhow", "axum 0.7.9", @@ -15462,7 +15463,7 @@ dependencies = [ "serde", "serde_json", "sqlx", - "thiserror 2.0.17", + "thiserror 2.0.18", "tokio", "tracing", "uuid", @@ -15472,7 +15473,7 @@ dependencies = [ [[package]] name = "windmill-common" -version = "1.608.0" +version = "1.611.0" dependencies = [ "anyhow", "async-recursion", @@ -15543,7 +15544,7 @@ dependencies = [ "systemstat", "tar", "tempfile", - "thiserror 2.0.17", + "thiserror 2.0.18", "tikv-jemalloc-ctl", "tokio", "tokio-postgres 0.7.13", @@ -15566,7 +15567,7 @@ dependencies = [ [[package]] name = "windmill-git-sync" -version = "1.608.0" +version = "1.611.0" dependencies = [ "regex", "serde", @@ -15581,7 +15582,7 @@ dependencies = [ [[package]] name = "windmill-indexer" -version = "1.608.0" +version = "1.611.0" dependencies = [ "anyhow", "astral-tokio-tar", @@ -15605,7 +15606,7 @@ dependencies = [ [[package]] name = "windmill-macros" -version = "1.608.0" +version = "1.611.0" dependencies = [ "itertools 0.14.0", "lazy_static", @@ -15621,21 +15622,27 @@ dependencies = [ [[package]] name = "windmill-mcp" -version = "1.608.0" +version = "1.611.0" dependencies = [ "anyhow", + "async-trait", + "futures", + "http 1.4.0", "oauth2", "reqwest 0.12.28", "rmcp", "serde", "serde_json", + "sqlx", + "tokio", + "tokio-util", "tracing", "windmill-common", ] [[package]] name = "windmill-oauth" -version = "1.608.0" +version = "1.611.0" dependencies = [ "anyhow", "async-oauth2", @@ -15659,7 +15666,7 @@ dependencies = [ [[package]] name = "windmill-parser" -version = "1.608.0" +version = "1.611.0" dependencies = [ "convert_case 0.6.0", "serde", @@ -15668,7 +15675,7 @@ dependencies = [ [[package]] name = "windmill-parser-bash" -version = "1.608.0" +version = "1.611.0" dependencies = [ "anyhow", "lazy_static", @@ -15680,7 +15687,7 @@ dependencies = [ [[package]] name = "windmill-parser-csharp" -version = "1.608.0" +version = "1.611.0" dependencies = [ "anyhow", "serde_json", @@ -15692,7 +15699,7 @@ dependencies = [ [[package]] name = "windmill-parser-go" -version = "1.608.0" +version = "1.611.0" dependencies = [ "anyhow", "gosyn", @@ -15704,7 +15711,7 @@ dependencies = [ [[package]] name = "windmill-parser-graphql" -version = "1.608.0" +version = "1.611.0" dependencies = [ "anyhow", "lazy_static", @@ -15716,7 +15723,7 @@ dependencies = [ [[package]] name = "windmill-parser-java" -version = "1.608.0" +version = "1.611.0" dependencies = [ "anyhow", "serde_json", @@ -15728,7 +15735,7 @@ dependencies = [ [[package]] name = "windmill-parser-nu" -version = "1.608.0" +version = "1.611.0" dependencies = [ "anyhow", "nu-parser", @@ -15739,7 +15746,7 @@ dependencies = [ [[package]] name = "windmill-parser-php" -version = "1.608.0" +version = "1.611.0" dependencies = [ "anyhow", "itertools 0.14.0", @@ -15750,7 +15757,7 @@ dependencies = [ [[package]] name = "windmill-parser-py" -version = "1.608.0" +version = "1.611.0" dependencies = [ "anyhow", "itertools 0.14.0", @@ -15763,7 +15770,7 @@ dependencies = [ [[package]] name = "windmill-parser-py-imports" -version = "1.608.0" +version = "1.611.0" dependencies = [ "anyhow", "async-recursion", @@ -15787,7 +15794,7 @@ dependencies = [ [[package]] name = "windmill-parser-ruby" -version = "1.608.0" +version = "1.611.0" dependencies = [ "anyhow", "lazy_static", @@ -15801,7 +15808,7 @@ dependencies = [ [[package]] name = "windmill-parser-rust" -version = "1.608.0" +version = "1.611.0" dependencies = [ "anyhow", "convert_case 0.6.0", @@ -15818,7 +15825,7 @@ dependencies = [ [[package]] name = "windmill-parser-sql" -version = "1.608.0" +version = "1.611.0" dependencies = [ "anyhow", "lazy_static", @@ -15832,7 +15839,7 @@ dependencies = [ [[package]] name = "windmill-parser-ts" -version = "1.608.0" +version = "1.611.0" dependencies = [ "anyhow", "lazy_static", @@ -15851,7 +15858,7 @@ dependencies = [ [[package]] name = "windmill-parser-yaml" -version = "1.608.0" +version = "1.611.0" dependencies = [ "anyhow", "serde", @@ -15862,7 +15869,7 @@ dependencies = [ [[package]] name = "windmill-queue" -version = "1.608.0" +version = "1.611.0" dependencies = [ "anyhow", "async-recursion", @@ -15888,7 +15895,7 @@ dependencies = [ "serde_urlencoded", "sql-builder", "sqlx", - "thiserror 2.0.17", + "thiserror 2.0.18", "tokio", "tracing", "ulid", @@ -15899,7 +15906,7 @@ dependencies = [ [[package]] name = "windmill-sql-datatype-parser-wasm" -version = "1.608.0" +version = "1.611.0" dependencies = [ "wasm-bindgen", "wasm-bindgen-test", @@ -15909,7 +15916,7 @@ dependencies = [ [[package]] name = "windmill-worker" -version = "1.608.0" +version = "1.611.0" dependencies = [ "anyhow", "async-once-cell", diff --git a/backend/Cargo.toml b/backend/Cargo.toml index 81ffd2ec26..e940b0b625 100644 --- a/backend/Cargo.toml +++ b/backend/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "windmill" -version = "1.608.0" +version = "1.611.0" authors.workspace = true edition.workspace = true @@ -35,7 +35,7 @@ members = [ exclude = ["./windmill-duckdb-ffi-internal"] [workspace.package] -version = "1.608.0" +version = "1.611.0" authors = ["Ruben Fiszel "] edition = "2021" @@ -405,6 +405,7 @@ async-once-cell = "0.5.4" aws-smithy-types-convert = { version = "^0", features = ["convert-chrono"] } crc = "^3" tar = "^0" +flate2 = "^1" http = "^1" async-stream = "^0" diff --git a/backend/ee-repo-ref.txt b/backend/ee-repo-ref.txt index dac8a90ba9..130e307a08 100644 --- a/backend/ee-repo-ref.txt +++ b/backend/ee-repo-ref.txt @@ -1 +1 @@ -da0a3751abd2b0fed46ca5c7a3aef35923403b28 \ No newline at end of file +a4790328e8fb78256b6f746a8825ec6ec499c3a8 \ No newline at end of file diff --git a/backend/generate_mcp_endpoints_tools/generate_mcp_tools.py b/backend/generate_mcp_endpoints_tools/generate_mcp_tools.py index ae1e27a288..154e3e4376 100644 --- a/backend/generate_mcp_endpoints_tools/generate_mcp_tools.py +++ b/backend/generate_mcp_endpoints_tools/generate_mcp_tools.py @@ -13,21 +13,7 @@ from typing import Dict, List, Any, Optional IMPORTS = """ use std::borrow::Cow; -use serde::{Deserialize, Serialize}; -""" - -ENDPOINT_STRUCT = """ -#[derive(Debug, Clone, Deserialize, Serialize)] -pub struct EndpointTool { - pub name: Cow<'static, str>, - pub description: Cow<'static, str>, - pub instructions: Cow<'static, str>, - pub path: Cow<'static, str>, - pub method: Cow<'static, str>, - pub path_params_schema: Option, - pub query_params_schema: Option, - pub body_schema: Option, -} +use windmill_mcp::server::EndpointTool; """ def load_openapi_spec(file_path: str) -> Dict[str, Any]: @@ -341,13 +327,11 @@ export const mcpEndpointTools: EndpointTool[] = [ def generate_rust_code(tools: List[Dict[str, Any]], spec: Dict[str, Any], base_path: str = "") -> str: """Generate the complete Rust code with MCP tools.""" if not tools: - return """// No MCP tools found in the OpenAPI specification - + return f"""// No MCP tools found in the OpenAPI specification {IMPORTS} -{ENDPOINT_STRUCT} -pub fn all_tools() -> Vec { +pub fn all_tools() -> Vec {{ vec![] -} +}} """ tool_definitions = [] @@ -386,9 +370,7 @@ pub fn all_tools() -> Vec { rust_code = f"""// Auto-generated MCP tools from OpenAPI specification // This file is generated by generate_mcp_tools.py - DO NOT EDIT MANUALLY - {IMPORTS} -{ENDPOINT_STRUCT} pub fn all_tools() -> Vec {{ vec![ {tool_definitions_str} @@ -405,7 +387,7 @@ def main(): project_dir = backend_dir.parent openapi_file = backend_dir / "windmill-api" / "openapi.yaml" - rust_output_file = backend_dir / "windmill-api" / "src" / "mcp" / "tools" / "auto_generated_endpoints.rs" + rust_output_file = backend_dir / "windmill-api" / "src" / "mcp" / "auto_generated_endpoints.rs" ts_output_file = project_dir / "frontend" / "src" / "lib" / "mcpEndpointTools.ts" if not openapi_file.exists(): diff --git a/backend/migrations/20260117105517_add_workspace_success_handler.down.sql b/backend/migrations/20260117105517_add_workspace_success_handler.down.sql new file mode 100644 index 0000000000..fbfa2f0fd9 --- /dev/null +++ b/backend/migrations/20260117105517_add_workspace_success_handler.down.sql @@ -0,0 +1,4 @@ +-- Remove success_handler columns from workspace_settings +ALTER TABLE workspace_settings + DROP COLUMN IF EXISTS success_handler, + DROP COLUMN IF EXISTS success_handler_extra_args; diff --git a/backend/migrations/20260117105517_add_workspace_success_handler.up.sql b/backend/migrations/20260117105517_add_workspace_success_handler.up.sql new file mode 100644 index 0000000000..a48742e210 --- /dev/null +++ b/backend/migrations/20260117105517_add_workspace_success_handler.up.sql @@ -0,0 +1,4 @@ +-- Add success_handler columns to workspace_settings +ALTER TABLE workspace_settings + ADD COLUMN IF NOT EXISTS success_handler TEXT, + ADD COLUMN IF NOT EXISTS success_handler_extra_args JSON; diff --git a/backend/src/monitor.rs b/backend/src/monitor.rs index 51ab25fee9..115dcf6d27 100644 --- a/backend/src/monitor.rs +++ b/backend/src/monitor.rs @@ -155,6 +155,20 @@ lazy_static::lazy_static! { .ok() .and_then(|x| x.parse::().ok()); + + /// Batch size for job cleanup deletion queries. Default: 10000. + /// Larger values delete more jobs per batch but hold locks longer. + static ref JOB_CLEANUP_BATCH_SIZE: i64 = std::env::var("JOB_CLEANUP_BATCH_SIZE") + .ok() + .and_then(|x| x.parse::().ok()) + .unwrap_or(20000); + + /// Maximum number of batches to process per cleanup iteration. Default: 10. + /// Set to 0 for unlimited (process until no expired jobs remain). + static ref JOB_CLEANUP_MAX_BATCHES: i32 = std::env::var("JOB_CLEANUP_MAX_BATCHES") + .ok() + .and_then(|x| x.parse::().ok()) + .unwrap_or(20); } pub async fn initial_load( @@ -910,99 +924,153 @@ pub async fn delete_expired_items(db: &DB) -> () { let job_retention_secs = *JOB_RETENTION_SECS.read().await; if job_retention_secs > 0 { - match db.begin().await { - Ok(mut tx) => { - let deleted_jobs = sqlx::query_scalar!( - "DELETE FROM v2_job_completed c - WHERE completed_at <= now() - ($1::bigint::text || ' s')::interval - RETURNING c.id", - job_retention_secs - ) - .fetch_all(&mut *tx) - .await; + let batch_size = *JOB_CLEANUP_BATCH_SIZE; + let max_batches = *JOB_CLEANUP_MAX_BATCHES; + let cleanup_start = Instant::now(); + let mut total_deleted = 0u64; + let mut batch_num = 0i32; - match deleted_jobs { - Ok(deleted_jobs) => { - if deleted_jobs.len() > 0 { - tracing::info!( - "deleted {} jobs completed JOB_RETENTION_SECS {} ago: {:?}", - deleted_jobs.len(), - job_retention_secs, - deleted_jobs, - ); - if let Err(e) = sqlx::query!( - "DELETE FROM job_stats WHERE job_id = ANY($1)", - &deleted_jobs - ) - .execute(&mut *tx) - .await - { - tracing::error!("Error deleting job stats: {:?}", e); - } - match sqlx::query_scalar!( - "DELETE FROM job_logs WHERE job_id = ANY($1) RETURNING log_file_index", - &deleted_jobs - ) - .fetch_all(&mut *tx) - .await - { - Ok(log_file_index) => { - let paths = log_file_index - .into_iter() - .filter_map(|opt| opt) - .flat_map(|inner_vec| inner_vec.into_iter()) - .collect(); - delete_log_files_from_disk_and_store(paths, TMP_DIR, "").await; - } - Err(e) => tracing::error!("Error deleting job stats: {:?}", e), - } - if let Err(e) = sqlx::query!( - "DELETE FROM concurrency_key WHERE ended_at <= now() - ($1::bigint::text || ' s')::interval ", - job_retention_secs - ) - .execute(&mut *tx) - .await - { - tracing::error!("Error deleting custom concurrency key: {:?}", e); - } - - if let Err(e) = - sqlx::query!("DELETE FROM v2_job WHERE id = ANY($1)", &deleted_jobs) - .execute(&mut *tx) - .await - { - tracing::error!("Error deleting job: {:?}", e); - } - - // should already be deleted but just in case - if let Err(e) = sqlx::query!( - "DELETE FROM job_result_stream_v2 WHERE job_id = ANY($1)", - &deleted_jobs - ) - .execute(&mut *tx) - .await - { - tracing::error!("Error deleting job result stream: {:?}", e); - } - } - } - Err(e) => { - tracing::error!("Error deleting expired jobs: {:?}", e) - } - } - - match tx.commit().await { - Ok(_) => (), - Err(err) => tracing::error!("Error deleting expired jobs: {:?}", err), - } + // Process batches until no more expired jobs or max batches reached + loop { + if max_batches > 0 && batch_num >= max_batches { + tracing::debug!( + "Job cleanup: reached max batches limit ({}), will continue next iteration", + max_batches + ); + break; } - Err(err) => { - tracing::error!("Error deleting expired jobs: {:?}", err) + + // Each batch runs in its own transaction to avoid long-running locks + let batch_result = delete_expired_jobs_batch(db, job_retention_secs, batch_size).await; + + match batch_result { + Ok(deleted_count) => { + if deleted_count == 0 { + // No more expired jobs to delete + break; + } + total_deleted += deleted_count as u64; + batch_num += 1; + } + Err(e) => { + tracing::error!("Error in job cleanup batch {}: {:?}", batch_num, e); + break; + } } } + + if total_deleted > 0 { + tracing::info!( + "Job cleanup completed: deleted {} jobs in {} batches, took {:?}", + total_deleted, + batch_num, + cleanup_start.elapsed() + ); + } + + // Clean up concurrency keys separately (not tied to specific job IDs) + if let Err(e) = sqlx::query!( + "DELETE FROM concurrency_key WHERE ended_at <= now() - ($1::bigint::text || ' s')::interval", + job_retention_secs + ) + .execute(db) + .await + { + tracing::error!("Error deleting custom concurrency key: {:?}", e); + } } } +/// Delete a batch of expired jobs with LIMIT and SKIP LOCKED for high-scale environments. +/// Uses a single transaction per batch to minimize lock duration. +/// Returns the number of jobs deleted in this batch. +async fn delete_expired_jobs_batch( + db: &DB, + job_retention_secs: i64, + batch_size: i64, +) -> error::Result { + let mut tx = db.begin().await?; + + // Use FOR UPDATE SKIP LOCKED to avoid contention between replicas + // ORDER BY completed_at ensures we delete oldest jobs first + let deleted_jobs: Vec = sqlx::query_scalar!( + "DELETE FROM v2_job_completed + WHERE id IN ( + SELECT id FROM v2_job_completed + WHERE completed_at <= now() - ($1::bigint::text || ' s')::interval + ORDER BY completed_at ASC + LIMIT $2 + FOR UPDATE SKIP LOCKED + ) + RETURNING id", + job_retention_secs, + batch_size + ) + .fetch_all(&mut *tx) + .await?; + + let deleted_count = deleted_jobs.len(); + + if deleted_count > 0 { + tracing::debug!( + "Deleting batch of {} expired jobs (retention: {}s)", + deleted_count, + job_retention_secs + ); + + // Delete related records for this batch + if let Err(e) = sqlx::query!( + "DELETE FROM job_stats WHERE job_id = ANY($1)", + &deleted_jobs + ) + .execute(&mut *tx) + .await + { + tracing::error!("Error deleting job stats: {:?}", e); + } + + match sqlx::query_scalar!( + "DELETE FROM job_logs WHERE job_id = ANY($1) RETURNING log_file_index", + &deleted_jobs + ) + .fetch_all(&mut *tx) + .await + { + Ok(log_file_index) => { + let paths = log_file_index + .into_iter() + .filter_map(|opt| opt) + .flat_map(|inner_vec| inner_vec.into_iter()) + .collect(); + delete_log_files_from_disk_and_store(paths, TMP_DIR, "").await; + } + Err(e) => tracing::error!("Error deleting job logs: {:?}", e), + } + + if let Err(e) = sqlx::query!("DELETE FROM v2_job WHERE id = ANY($1)", &deleted_jobs) + .execute(&mut *tx) + .await + { + tracing::error!("Error deleting job: {:?}", e); + } + + // Should already be deleted but just in case + if let Err(e) = sqlx::query!( + "DELETE FROM job_result_stream_v2 WHERE job_id = ANY($1)", + &deleted_jobs + ) + .execute(&mut *tx) + .await + { + tracing::error!("Error deleting job result stream: {:?}", e); + } + } + + tx.commit().await?; + + Ok(deleted_count) +} + async fn delete_log_files_from_disk_and_store( paths_to_delete: Vec, tmp_dir: &str, @@ -2262,7 +2330,7 @@ async fn handle_zombie_jobs(db: &Pool, base_internal_url: &str, node_n sqlx::query_scalar!("SELECT j.id FROM v2_job_queue q JOIN v2_job j USING (id) LEFT JOIN v2_job_runtime r USING (id) LEFT JOIN v2_job_status s USING (id) WHERE r.ping < now() - ($1 || ' seconds')::interval - AND q.running = true AND j.kind NOT IN ('flow', 'flowpreview', 'flownode', 'singlestepflow') AND j.same_worker = false", + AND q.running = true AND j.kind NOT IN ('flow', 'flowpreview', 'flownode', 'singlestepflow') AND j.same_worker = false", ZOMBIE_JOB_TIMEOUT.as_str()) .fetch_all(db) .await @@ -2396,8 +2464,8 @@ orphaned_job_uuids AS ( SELECT job_uuid FROM all_job_uuids WHERE job_uuid NOT IN ( - SELECT id::text - FROM v2_job_queue + SELECT id::text + FROM v2_job_queue FOR SHARE SKIP LOCKED ) ), @@ -2410,7 +2478,7 @@ before_update AS ( WHERE lc.job_uuids ?| oa.orphaned_keys ), affected_rows AS ( - UPDATE concurrency_counter + UPDATE concurrency_counter SET job_uuids = job_uuids - orphaned_array.orphaned_keys FROM orphaned_array WHERE concurrency_counter.concurrency_id IN ( @@ -2419,12 +2487,12 @@ affected_rows AS ( RETURNING concurrency_id, job_uuids AS updated_job_uuids ), expanded_orphaned AS ( - SELECT bu.concurrency_id, + SELECT bu.concurrency_id, bu.job_uuids AS original_job_uuids, unnest(bu.orphaned_keys) AS orphaned_key FROM before_update bu ) -SELECT +SELECT eo.concurrency_id, eo.orphaned_key, eo.original_job_uuids, @@ -2453,7 +2521,7 @@ async fn cleanup_concurrency_counters_empty_keys(db: &DB) -> error::Result<()> { WITH rows_to_delete AS ( SELECT concurrency_id FROM concurrency_counter - + WHERE job_uuids = '{}'::jsonb FOR UPDATE SKIP LOCKED ) @@ -2489,7 +2557,7 @@ async fn handle_zombie_flows(db: &DB) -> error::Result<()> { AND (j.kind = 'flow' OR j.kind = 'flowpreview' OR j.kind = 'flownode') AND r.ping IS NOT NULL AND r.ping < NOW() - ($1 || ' seconds')::interval AND q.canceled_by IS NULL - + "#, FLOW_ZOMBIE_TRANSITION_TIMEOUT.as_str() ) diff --git a/backend/summarized_schema.txt b/backend/summarized_schema.txt index a1ae0c1847..864acd0236 100644 --- a/backend/summarized_schema.txt +++ b/backend/summarized_schema.txt @@ -1065,6 +1065,8 @@ TABLE: workspace_settings - ai_config (jsonb) - error_handler_extra_args (json) - error_handler_muted_on_cancel (boolean) + - success_handler (text) + - success_handler_extra_args (json) - large_file_storage (jsonb) - git_sync (jsonb) - default_app (character) diff --git a/backend/tests/fixtures/secret_backend.sql b/backend/tests/fixtures/secret_backend.sql new file mode 100644 index 0000000000..0bc5a6f52a --- /dev/null +++ b/backend/tests/fixtures/secret_backend.sql @@ -0,0 +1,31 @@ +-- Fixture for secret backend migration tests +-- Sets up test secrets in the variable table + +-- Create a second workspace for testing workspace isolation +INSERT INTO workspace (id, name, owner) +VALUES ('test-workspace-2', 'test-workspace-2', 'test-user') +ON CONFLICT DO NOTHING; + +INSERT INTO workspace_settings (workspace_id) +VALUES ('test-workspace-2') +ON CONFLICT DO NOTHING; + +INSERT INTO workspace_key(workspace_id, kind, key) +VALUES ('test-workspace-2', 'cloud', 'test-key-2') +ON CONFLICT DO NOTHING; + +-- Insert test secrets for workspace 1 +-- Note: The 'value' column stores encrypted values in production, +-- but for tests we'll use plain text that the migration will handle +INSERT INTO variable (workspace_id, path, value, is_secret, description, extra_perms) +VALUES + ('test-workspace', 'u/test-user/db_password', 'encrypted-db-pass-123', true, 'Database password', '{}'), + ('test-workspace', 'u/test-user/api_key', 'encrypted-api-key-abc', true, 'API key for external service', '{}'), + ('test-workspace', 'u/test-user/public_var', 'not-a-secret', false, 'A non-secret variable', '{}') +ON CONFLICT DO NOTHING; + +-- Insert test secrets for workspace 2 (to test isolation) +INSERT INTO variable (workspace_id, path, value, is_secret, description, extra_perms) +VALUES + ('test-workspace-2', 'u/test-user/other_secret', 'encrypted-other-secret', true, 'Secret in workspace 2', '{}') +ON CONFLICT DO NOTHING; diff --git a/backend/tests/secret_backend_integration.rs b/backend/tests/secret_backend_integration.rs new file mode 100644 index 0000000000..e6deb84ff9 --- /dev/null +++ b/backend/tests/secret_backend_integration.rs @@ -0,0 +1,557 @@ +//! Integration tests for HashiCorp Vault secret backend. +//! +//! These tests require: +//! 1. A PostgreSQL database (handled by sqlx test framework) +//! 2. A running HashiCorp Vault instance +//! 3. The RUN_VAULT_TESTS=1 environment variable to be set +//! +//! Environment variables: +//! - RUN_VAULT_TESTS=1 - Required to run the tests +//! - VAULT_ADDR - Vault server address (default: http://127.0.0.1:8200) +//! - VAULT_TOKEN - Static token for static token tests (default: test-root-token) +//! - BASE_URL - Windmill instance URL for JWT tests (default: http://localhost:8000) +//! +//! Run tests (static token mode): +//! ```bash +//! RUN_VAULT_TESTS=1 VAULT_TOKEN=your-token cargo test -p windmill \ +//! secret_backend_integration --features private,enterprise -- --nocapture +//! ``` +//! +//! Run tests (JWT mode - requires Windmill instance running for JWKS endpoint): +//! ```bash +//! RUN_VAULT_TESTS=1 BASE_URL=http://localhost:8000 cargo test -p windmill \ +//! secret_backend_integration --features private,enterprise,openidconnect -- --nocapture +//! ``` + +#[cfg(all(feature = "private", feature = "enterprise"))] +mod tests { + use sqlx::{Pool, Postgres}; + use std::collections::HashMap; + use windmill_common::secret_backend::{ + migrate_secrets_to_database, migrate_secrets_to_vault, test_vault_connection, + SecretBackend, VaultBackend, VaultSettings, + }; + + /// Check if vault tests should run (requires RUN_VAULT_TESTS=1 env var) + fn should_run_vault_tests() -> bool { + std::env::var("RUN_VAULT_TESTS") + .map(|v| v == "1" || v.to_lowercase() == "true") + .unwrap_or(false) + } + + /// Set up BASE_URL for JWT tests (required for OIDC issuer URL generation) + async fn setup_base_url() { + let base_url = std::env::var("BASE_URL") + .unwrap_or_else(|_| "http://localhost:8000".to_string()); + let mut url = windmill_common::BASE_URL.write().await; + *url = base_url; + } + + /// Skip test if RUN_VAULT_TESTS is not set + macro_rules! skip_if_no_vault { + () => { + if !should_run_vault_tests() { + println!("Skipping test: RUN_VAULT_TESTS=1 not set"); + println!("To run vault tests: RUN_VAULT_TESTS=1 cargo test ..."); + return; + } + }; + } + + fn vault_settings_static_token() -> VaultSettings { + VaultSettings { + address: std::env::var("VAULT_ADDR") + .unwrap_or_else(|_| "http://127.0.0.1:8200".to_string()), + mount_path: "windmill".to_string(), + jwt_role: None, // Static token mode + namespace: None, + token: Some( + std::env::var("VAULT_TOKEN").unwrap_or_else(|_| "test-root-token".to_string()), + ), + } + } + + fn vault_settings_jwt() -> VaultSettings { + VaultSettings { + address: std::env::var("VAULT_ADDR") + .unwrap_or_else(|_| "http://127.0.0.1:8200".to_string()), + mount_path: "windmill".to_string(), + jwt_role: Some("windmill-secrets".to_string()), // JWT mode + namespace: None, + token: None, // No static token - use JWT + } + } + + // ==================== Static Token Tests ==================== + + /// Test Vault connection with static token + #[sqlx::test(fixtures("base", "secret_backend"))] + async fn test_vault_connection_static_token(db: Pool) { + skip_if_no_vault!(); + + let settings = vault_settings_static_token(); + println!("Testing Vault connection with static token..."); + println!(" Address: {}", settings.address); + + let result = test_vault_connection(&settings, Some(&db)).await; + assert!( + result.is_ok(), + "Failed to connect to Vault: {:?}", + result.err() + ); + println!("✓ Successfully connected to Vault with static token"); + } + + /// Test basic CRUD operations with static token + #[sqlx::test(fixtures("base", "secret_backend"))] + async fn test_vault_crud_static_token(_db: Pool) { + skip_if_no_vault!(); + + let settings = vault_settings_static_token(); + let backend = VaultBackend::new(settings); + + let workspace_id = "test-crud-static"; + let path = "test-secret"; + let value = "my-super-secret-value-123"; + + println!("Testing CRUD with static token..."); + + // Create + println!(" Creating secret..."); + backend + .set_secret(workspace_id, path, value) + .await + .expect("Failed to create secret"); + println!(" ✓ Created"); + + // Read + println!(" Reading secret..."); + let read_value = backend + .get_secret(workspace_id, path) + .await + .expect("Failed to read secret"); + assert_eq!(read_value, value); + println!(" ✓ Read (value matches)"); + + // Update + println!(" Updating secret..."); + let new_value = "updated-secret-value-456"; + backend + .set_secret(workspace_id, path, new_value) + .await + .expect("Failed to update secret"); + let updated = backend + .get_secret(workspace_id, path) + .await + .expect("Failed to read updated secret"); + assert_eq!(updated, new_value); + println!(" ✓ Updated"); + + // Delete + println!(" Deleting secret..."); + backend + .delete_secret(workspace_id, path) + .await + .expect("Failed to delete secret"); + let result = backend.get_secret(workspace_id, path).await; + assert!(result.is_err(), "Secret should be deleted"); + println!(" ✓ Deleted"); + + println!("✓ CRUD operations successful with static token"); + } + + // ==================== JWT Auth Tests ==================== + + /// Test Vault connection with JWT authentication + #[sqlx::test(fixtures("base", "secret_backend"))] + async fn test_vault_connection_jwt(db: Pool) { + skip_if_no_vault!(); + setup_base_url().await; + + let settings = vault_settings_jwt(); + println!("Testing Vault connection with JWT auth..."); + println!(" Address: {}", settings.address); + println!(" JWT Role: {:?}", settings.jwt_role); + println!(" BASE_URL: {}", windmill_common::BASE_URL.read().await.clone()); + + let result = test_vault_connection(&settings, Some(&db)).await; + assert!( + result.is_ok(), + "Failed to connect to Vault with JWT: {:?}", + result.err() + ); + println!("✓ Successfully connected to Vault with JWT auth"); + } + + /// Test basic CRUD operations with JWT authentication + #[cfg(feature = "openidconnect")] + #[sqlx::test(fixtures("base", "secret_backend"))] + async fn test_vault_crud_jwt(db: Pool) { + skip_if_no_vault!(); + setup_base_url().await; + + let settings = vault_settings_jwt(); + let backend = VaultBackend::new_with_db(settings, db.clone()); + + let workspace_id = "test-crud-jwt"; + let path = "jwt-test-secret"; + let value = "jwt-authenticated-secret-value"; + + println!("Testing CRUD with JWT auth..."); + + // Create + println!(" Creating secret..."); + backend + .set_secret(workspace_id, path, value) + .await + .expect("Failed to create secret with JWT"); + println!(" ✓ Created"); + + // Read + println!(" Reading secret..."); + let read_value = backend + .get_secret(workspace_id, path) + .await + .expect("Failed to read secret with JWT"); + assert_eq!(read_value, value); + println!(" ✓ Read (value matches)"); + + // Delete (cleanup) + println!(" Deleting secret..."); + backend + .delete_secret(workspace_id, path) + .await + .expect("Failed to delete secret with JWT"); + println!(" ✓ Deleted"); + + println!("✓ CRUD operations successful with JWT auth"); + } + + // ==================== Migration Tests ==================== + + /// Test migration from database to Vault + #[sqlx::test(fixtures("base", "secret_backend"))] + async fn test_migrate_db_to_vault(db: Pool) { + skip_if_no_vault!(); + + let settings = vault_settings_static_token(); + + // Verify Vault connection + test_vault_connection(&settings, Some(&db)) + .await + .expect("Failed to connect to Vault"); + + // Check initial state + let secrets_before = sqlx::query!( + "SELECT workspace_id, path, value FROM variable WHERE is_secret = true ORDER BY workspace_id, path" + ) + .fetch_all(&db) + .await + .expect("Failed to query secrets"); + + println!( + "Found {} secrets in database before migration:", + secrets_before.len() + ); + for s in &secrets_before { + println!(" - {}/{}: {} chars", s.workspace_id, s.path, s.value.len()); + } + + // Run migration + println!("\nMigrating secrets to Vault..."); + let report = migrate_secrets_to_vault(&db, &settings) + .await + .expect("Migration to Vault failed"); + + println!("Migration report:"); + println!(" Total secrets: {}", report.total_secrets); + println!(" Migrated: {}", report.migrated_count); + println!(" Failed: {}", report.failed_count); + + if !report.failures.is_empty() { + println!(" Failures:"); + for f in &report.failures { + println!(" - {}/{}: {}", f.workspace_id, f.path, f.error); + } + } + + assert_eq!(report.failed_count, 0, "Migration had failures"); + assert!(report.migrated_count > 0, "No secrets were migrated"); + + // Verify secrets in Vault + println!("\nVerifying secrets in Vault..."); + let vault_backend = VaultBackend::new(settings.clone()); + + for secret in &secrets_before { + let result = vault_backend + .get_secret(&secret.workspace_id, &secret.path) + .await; + assert!( + result.is_ok(), + "Failed to read secret {}/{} from Vault: {:?}", + secret.workspace_id, + secret.path, + result.err() + ); + println!( + " ✓ {}/{} exists in Vault", + secret.workspace_id, secret.path + ); + } + + println!("\n✓ Migration to Vault completed successfully"); + } + + /// Test migration from Vault back to database + #[sqlx::test(fixtures("base", "secret_backend"))] + async fn test_migrate_vault_to_db(db: Pool) { + skip_if_no_vault!(); + + let settings = vault_settings_static_token(); + + test_vault_connection(&settings, Some(&db)) + .await + .expect("Failed to connect to Vault"); + + // First migrate TO Vault + println!("Setting up: migrating secrets to Vault first..."); + let to_vault = migrate_secrets_to_vault(&db, &settings) + .await + .expect("Initial migration to Vault failed"); + assert!(to_vault.migrated_count > 0, "No secrets to test with"); + println!(" Migrated {} secrets to Vault", to_vault.migrated_count); + + // Clear database values + println!("\nClearing database secret values..."); + sqlx::query!("UPDATE variable SET value = 'CLEARED' WHERE is_secret = true") + .execute(&db) + .await + .expect("Failed to clear values"); + + // Migrate back from Vault + println!("\nMigrating secrets from Vault to database..."); + let report = migrate_secrets_to_database(&db, &settings) + .await + .expect("Migration to database failed"); + + println!("Migration report:"); + println!(" Total secrets: {}", report.total_secrets); + println!(" Migrated: {}", report.migrated_count); + println!(" Failed: {}", report.failed_count); + + assert_eq!(report.failed_count, 0, "Migration had failures"); + assert!(report.migrated_count > 0, "No secrets were migrated"); + + // Verify restored + let restored = sqlx::query!( + "SELECT COUNT(*) as count FROM variable WHERE is_secret = true AND value != 'CLEARED'" + ) + .fetch_one(&db) + .await + .expect("Failed to count restored"); + + assert!( + restored.count.unwrap_or(0) > 0, + "No secrets were restored in database" + ); + + println!("\n✓ Migration to database completed successfully"); + } + + // ==================== Variable Rename Tests ==================== + + /// Test renaming a variable path in Vault + #[sqlx::test(fixtures("base", "secret_backend"))] + async fn test_variable_rename(db: Pool) { + skip_if_no_vault!(); + let _ = &db; // suppress unused warning + + let settings = vault_settings_static_token(); + let backend = VaultBackend::new(settings); + + let workspace_id = "test-workspace"; + let old_path = "u/test-user/old_secret_name"; + let new_path = "u/test-user/new_secret_name"; + let value = "secret-value-for-rename-test"; + + println!("Testing variable rename in Vault..."); + + // Create secret at old path + println!(" Creating secret at old path: {}", old_path); + backend + .set_secret(workspace_id, old_path, value) + .await + .expect("Failed to create secret"); + + // Verify it exists + let read_value = backend + .get_secret(workspace_id, old_path) + .await + .expect("Failed to read secret at old path"); + assert_eq!(read_value, value); + println!(" ✓ Secret exists at old path"); + + // Simulate rename: read from old, write to new, delete old + println!(" Renaming: {} -> {}", old_path, new_path); + let secret_value = backend + .get_secret(workspace_id, old_path) + .await + .expect("Failed to read for rename"); + + backend + .set_secret(workspace_id, new_path, &secret_value) + .await + .expect("Failed to write to new path"); + + backend + .delete_secret(workspace_id, old_path) + .await + .expect("Failed to delete old path"); + + // Verify old path is gone + let old_result = backend.get_secret(workspace_id, old_path).await; + assert!(old_result.is_err(), "Old path should not exist"); + println!(" ✓ Old path deleted"); + + // Verify new path exists with correct value + let new_value = backend + .get_secret(workspace_id, new_path) + .await + .expect("Failed to read new path"); + assert_eq!(new_value, value); + println!(" ✓ New path exists with correct value"); + + // Cleanup + backend + .delete_secret(workspace_id, new_path) + .await + .expect("Failed to cleanup"); + + println!("\n✓ Variable rename completed successfully"); + } + + // ==================== Full Round Trip Test ==================== + + /// Test full round-trip: DB -> Vault -> DB with verification + #[sqlx::test(fixtures("base", "secret_backend"))] + async fn test_full_round_trip(db: Pool) { + skip_if_no_vault!(); + + let settings = vault_settings_static_token(); + + test_vault_connection(&settings, Some(&db)) + .await + .expect("Failed to connect to Vault"); + + // Get original secrets + let original: HashMap<(String, String), String> = sqlx::query!( + "SELECT workspace_id, path, value FROM variable WHERE is_secret = true" + ) + .fetch_all(&db) + .await + .expect("Failed to query") + .into_iter() + .map(|r| ((r.workspace_id, r.path), r.value)) + .collect(); + + println!("Original secrets: {} entries", original.len()); + + // Step 1: DB -> Vault + println!("\n=== Step 1: Migrate DB -> Vault ==="); + let to_vault = migrate_secrets_to_vault(&db, &settings) + .await + .expect("Migration to Vault failed"); + println!("Migrated {} secrets to Vault", to_vault.migrated_count); + assert_eq!(to_vault.failed_count, 0); + + // Step 2: Clear DB + println!("\n=== Step 2: Clear database values ==="); + sqlx::query!("UPDATE variable SET value = 'ROUND_TRIP_CLEARED' WHERE is_secret = true") + .execute(&db) + .await + .expect("Failed to clear"); + + // Step 3: Vault -> DB + println!("\n=== Step 3: Migrate Vault -> DB ==="); + let to_db = migrate_secrets_to_database(&db, &settings) + .await + .expect("Migration to database failed"); + println!("Migrated {} secrets to database", to_db.migrated_count); + assert_eq!(to_db.failed_count, 0); + + // Step 4: Verify + println!("\n=== Step 4: Verify round-trip integrity ==="); + let restored: HashMap<(String, String), String> = sqlx::query!( + "SELECT workspace_id, path, value FROM variable WHERE is_secret = true" + ) + .fetch_all(&db) + .await + .expect("Failed to query") + .into_iter() + .map(|r| ((r.workspace_id, r.path), r.value)) + .collect(); + + for ((ws, path), _) in &original { + let restored_value = restored + .get(&(ws.clone(), path.clone())) + .expect(&format!("Secret {}/{} not found after round-trip", ws, path)); + + assert_ne!( + restored_value, "ROUND_TRIP_CLEARED", + "Secret {}/{} was not restored", + ws, path + ); + println!(" ✓ {}/{}: restored", ws, path); + } + + println!("\n✓ Full round-trip completed successfully!"); + } + + // ==================== Workspace Isolation Test ==================== + + /// Test that workspace isolation is maintained + #[sqlx::test(fixtures("base", "secret_backend"))] + async fn test_workspace_isolation(db: Pool) { + skip_if_no_vault!(); + + let settings = vault_settings_static_token(); + let backend = VaultBackend::new(settings.clone()); + + // First migrate secrets to Vault + migrate_secrets_to_vault(&db, &settings) + .await + .expect("Migration failed"); + + println!("Testing workspace isolation..."); + + // Try to access workspace-2 secret from workspace-1 path (should fail) + let cross_access = backend + .get_secret("test-workspace", "u/test-user/other_secret") + .await; + + assert!( + cross_access.is_err(), + "Cross-workspace access should fail!" + ); + println!("✓ Cross-workspace access correctly denied"); + + // Verify own workspace access works + let ws1 = backend + .get_secret("test-workspace", "u/test-user/db_password") + .await; + assert!(ws1.is_ok(), "Same-workspace access should work"); + println!("✓ Same-workspace access works"); + + println!("\n✓ Workspace isolation verified!"); + } +} + +// OSS version - just a placeholder to avoid compilation errors +#[cfg(not(all(feature = "private", feature = "enterprise")))] +mod tests { + #[test] + fn test_vault_requires_enterprise() { + println!("Vault integration tests require Enterprise Edition features"); + println!("Run with: cargo test --features private,enterprise,openidconnect"); + } +} diff --git a/backend/tests/secret_backend_migration.rs b/backend/tests/secret_backend_migration.rs new file mode 100644 index 0000000000..d78da8603c --- /dev/null +++ b/backend/tests/secret_backend_migration.rs @@ -0,0 +1,323 @@ +//! Integration tests for secret backend migration between database and HashiCorp Vault. +//! +//! These tests require: +//! 1. A PostgreSQL database (handled by sqlx test framework) +//! 2. A running HashiCorp Vault instance at http://127.0.0.1:8200 +//! +//! To run these tests: +//! ```bash +//! # Start Vault in dev mode +//! podman run -d --name vault-test -p 8200:8200 \ +//! -e VAULT_DEV_ROOT_TOKEN_ID=test-root-token \ +//! docker.io/hashicorp/vault:latest +//! +//! # Enable KV v2 secrets engine +//! curl -s -H "X-Vault-Token: test-root-token" -X POST \ +//! --data '{"type":"kv-v2"}' \ +//! http://127.0.0.1:8200/v1/sys/mounts/windmill +//! +//! # Run the tests +//! cargo test -p windmill secret_backend_migration -- --ignored --nocapture +//! ``` + +use sqlx::{Pool, Postgres}; +use windmill_common::error::Result; +use windmill_common::secret_backend::{ + vault_oss::{migrate_secrets_to_database, migrate_secrets_to_vault, test_vault_connection, VaultBackend}, + SecretBackend, VaultSettings, +}; + +mod common; + +fn test_vault_settings() -> VaultSettings { + VaultSettings { + address: std::env::var("VAULT_ADDR").unwrap_or_else(|_| "http://127.0.0.1:8200".to_string()), + mount_path: "windmill".to_string(), + jwt_role: Some("windmill-secrets".to_string()), + namespace: None, + token: Some( + std::env::var("VAULT_TOKEN").unwrap_or_else(|_| "test-root-token".to_string()), + ), + } +} + +/// Test that we can connect to Vault +#[sqlx::test(fixtures("base", "secret_backend"))] +#[ignore = "requires running Vault instance"] +async fn test_vault_connection_works(db: Pool) { + let settings = test_vault_settings(); + + let result = test_vault_connection(&settings, Some(&db)).await; + assert!(result.is_ok(), "Failed to connect to Vault: {:?}", result.err()); + println!("✓ Successfully connected to Vault at {}", settings.address); +} + +/// Test migration from database to Vault +#[sqlx::test(fixtures("base", "secret_backend"))] +#[ignore = "requires running Vault instance"] +async fn test_migrate_db_to_vault(db: Pool) { + let settings = test_vault_settings(); + + // First verify we can connect to Vault + test_vault_connection(&settings, Some(&db)) + .await + .expect("Failed to connect to Vault"); + + // Check initial state - secrets should exist in database + let secrets_before = sqlx::query!( + "SELECT workspace_id, path, value FROM variable WHERE is_secret = true ORDER BY workspace_id, path" + ) + .fetch_all(&db) + .await + .expect("Failed to query secrets"); + + println!("Found {} secrets in database before migration:", secrets_before.len()); + for s in &secrets_before { + println!(" - {}/{}: {} chars", s.workspace_id, s.path, s.value.len()); + } + + // Run migration to Vault + println!("\nMigrating secrets to Vault..."); + let report = migrate_secrets_to_vault(&db, &settings) + .await + .expect("Migration to Vault failed"); + + println!("Migration report:"); + println!(" Total secrets: {}", report.total_secrets); + println!(" Migrated: {}", report.migrated_count); + println!(" Failed: {}", report.failed_count); + + if !report.failures.is_empty() { + println!(" Failures:"); + for f in &report.failures { + println!(" - {}/{}: {}", f.workspace_id, f.path, f.error); + } + } + + assert_eq!(report.failed_count, 0, "Migration had failures"); + assert!(report.migrated_count > 0, "No secrets were migrated"); + + // Verify secrets are in Vault + println!("\nVerifying secrets in Vault..."); + let vault_backend = VaultBackend::new(settings.clone()); + + for secret in &secrets_before { + let result: Result = vault_backend + .get_secret(&secret.workspace_id, &secret.path) + .await; + assert!( + result.is_ok(), + "Failed to read secret {}/{} from Vault: {:?}", + secret.workspace_id, + secret.path, + result.err() + ); + println!(" ✓ {}/{} exists in Vault", secret.workspace_id, secret.path); + } + + println!("\n✓ Migration to Vault completed successfully"); +} + +/// Test migration from Vault to database +#[sqlx::test(fixtures("base", "secret_backend"))] +#[ignore = "requires running Vault instance"] +async fn test_migrate_vault_to_db(db: Pool) { + let settings = test_vault_settings(); + + // First verify we can connect to Vault + test_vault_connection(&settings, Some(&db)) + .await + .expect("Failed to connect to Vault"); + + // First, migrate secrets TO Vault so we have something to migrate back + println!("Setting up: migrating secrets to Vault first..."); + let to_vault_report = migrate_secrets_to_vault(&db, &settings) + .await + .expect("Initial migration to Vault failed"); + assert!(to_vault_report.migrated_count > 0, "No secrets to test with"); + println!(" Migrated {} secrets to Vault", to_vault_report.migrated_count); + + // Clear the database values to simulate fresh migration back + println!("\nClearing database secret values..."); + sqlx::query!("UPDATE variable SET value = 'CLEARED' WHERE is_secret = true") + .execute(&db) + .await + .expect("Failed to clear database values"); + + // Verify they were cleared + let cleared = sqlx::query!( + "SELECT COUNT(*) as count FROM variable WHERE is_secret = true AND value = 'CLEARED'" + ) + .fetch_one(&db) + .await + .expect("Failed to count cleared"); + println!(" Cleared {} secret values in database", cleared.count.unwrap_or(0)); + + // Now migrate from Vault back to database + println!("\nMigrating secrets from Vault to database..."); + let report = migrate_secrets_to_database(&db, &settings) + .await + .expect("Migration to database failed"); + + println!("Migration report:"); + println!(" Total secrets: {}", report.total_secrets); + println!(" Migrated: {}", report.migrated_count); + println!(" Failed: {}", report.failed_count); + + if !report.failures.is_empty() { + println!(" Failures:"); + for f in &report.failures { + println!(" - {}/{}: {}", f.workspace_id, f.path, f.error); + } + } + + assert_eq!(report.failed_count, 0, "Migration had failures"); + assert!(report.migrated_count > 0, "No secrets were migrated"); + + // Verify secrets are restored in database + println!("\nVerifying secrets in database..."); + let secrets_after = sqlx::query!( + "SELECT workspace_id, path, value FROM variable WHERE is_secret = true AND value != 'CLEARED' ORDER BY workspace_id, path" + ) + .fetch_all(&db) + .await + .expect("Failed to query restored secrets"); + + assert!( + !secrets_after.is_empty(), + "No secrets were restored in database" + ); + + for s in &secrets_after { + println!(" ✓ {}/{}: {} chars", s.workspace_id, s.path, s.value.len()); + } + + println!("\n✓ Migration to database completed successfully"); +} + +/// Test full round-trip migration: DB -> Vault -> DB +#[sqlx::test(fixtures("base", "secret_backend"))] +#[ignore = "requires running Vault instance"] +async fn test_full_round_trip_migration(db: Pool) { + let settings = test_vault_settings(); + + // Verify Vault connection + test_vault_connection(&settings, Some(&db)) + .await + .expect("Failed to connect to Vault"); + + // Get original secrets + let original_secrets: std::collections::HashMap<(String, String), String> = sqlx::query!( + "SELECT workspace_id, path, value FROM variable WHERE is_secret = true" + ) + .fetch_all(&db) + .await + .expect("Failed to query original secrets") + .into_iter() + .map(|r| ((r.workspace_id, r.path), r.value)) + .collect(); + + println!("Original secrets: {} entries", original_secrets.len()); + + // Step 1: Migrate to Vault + println!("\n=== Step 1: Migrate DB -> Vault ==="); + let to_vault = migrate_secrets_to_vault(&db, &settings) + .await + .expect("Migration to Vault failed"); + println!("Migrated {} secrets to Vault", to_vault.migrated_count); + assert_eq!(to_vault.failed_count, 0); + + // Step 2: Clear database values + println!("\n=== Step 2: Clear database values ==="); + sqlx::query!("UPDATE variable SET value = 'ROUND_TRIP_CLEARED' WHERE is_secret = true") + .execute(&db) + .await + .expect("Failed to clear values"); + + // Step 3: Migrate back from Vault + println!("\n=== Step 3: Migrate Vault -> DB ==="); + let to_db = migrate_secrets_to_database(&db, &settings) + .await + .expect("Migration to database failed"); + println!("Migrated {} secrets to database", to_db.migrated_count); + assert_eq!(to_db.failed_count, 0); + + // Step 4: Verify round-trip integrity + println!("\n=== Step 4: Verify round-trip integrity ==="); + let restored_secrets: std::collections::HashMap<(String, String), String> = sqlx::query!( + "SELECT workspace_id, path, value FROM variable WHERE is_secret = true" + ) + .fetch_all(&db) + .await + .expect("Failed to query restored secrets") + .into_iter() + .map(|r| ((r.workspace_id, r.path), r.value)) + .collect(); + + // Compare original and restored + for ((ws, path), _original_value) in &original_secrets { + let restored_value = restored_secrets + .get(&(ws.clone(), path.clone())) + .expect(&format!("Secret {}/{} not found after round-trip", ws, path)); + + // Note: Values might differ slightly due to encryption/decryption + // but they should not be the cleared value + assert_ne!( + restored_value, "ROUND_TRIP_CLEARED", + "Secret {}/{} was not restored", + ws, path + ); + println!(" ✓ {}/{}: restored ({} chars)", ws, path, restored_value.len()); + } + + println!("\n✓ Full round-trip migration completed successfully!"); + println!(" Original secrets: {}", original_secrets.len()); + println!(" Restored secrets: {}", restored_secrets.len()); +} + +/// Test that workspace isolation is maintained during migration +#[sqlx::test(fixtures("base", "secret_backend"))] +#[ignore = "requires running Vault instance"] +async fn test_workspace_isolation(db: Pool) { + let settings = test_vault_settings(); + + test_vault_connection(&settings, Some(&db)) + .await + .expect("Failed to connect to Vault"); + + // Migrate all secrets to Vault + let report = migrate_secrets_to_vault(&db, &settings) + .await + .expect("Migration failed"); + + println!("Migrated {} secrets across workspaces", report.migrated_count); + + // Verify workspace isolation in Vault + let vault_backend = VaultBackend::new(settings.clone()); + + // Try to access test-workspace-2 secret from test-workspace path (should fail) + let cross_workspace_result: Result = vault_backend + .get_secret("test-workspace", "u/test-user/other_secret") + .await; + + assert!( + cross_workspace_result.is_err(), + "Cross-workspace access should fail - workspace isolation violated!" + ); + println!("✓ Cross-workspace access correctly denied"); + + // Verify each workspace's secrets are accessible from their own workspace + let ws1_result: Result = vault_backend + .get_secret("test-workspace", "u/test-user/db_password") + .await; + assert!(ws1_result.is_ok(), "test-workspace secret should be accessible"); + println!("✓ test-workspace secrets accessible"); + + let ws2_result: Result = vault_backend + .get_secret("test-workspace-2", "u/test-user/other_secret") + .await; + assert!(ws2_result.is_ok(), "test-workspace-2 secret should be accessible"); + println!("✓ test-workspace-2 secrets accessible"); + + println!("\n✓ Workspace isolation verified!"); +} diff --git a/backend/tests/success_handler.rs b/backend/tests/success_handler.rs new file mode 100644 index 0000000000..52416e5271 --- /dev/null +++ b/backend/tests/success_handler.rs @@ -0,0 +1,241 @@ +use sqlx::{Pool, Postgres}; + +mod common; +use common::*; + +/// Test that the workspace success handler cache works correctly with 60s TTL +#[cfg(feature = "deno_core")] +#[sqlx::test(fixtures("base"))] +async fn test_success_handler_cache(db: Pool) -> anyhow::Result<()> { + initialize_tracing().await; + + // First, create a success handler script + let _server = ApiServer::start(db.clone()).await?; + + // Set up a success handler in workspace_settings + sqlx::query!( + r#" + UPDATE workspace_settings + SET success_handler = 'script/f/test/success_handler', + success_handler_extra_args = '{"key": "value"}'::json + WHERE workspace_id = 'test-workspace' + "# + ) + .execute(&db) + .await?; + + // Verify the success handler was set + let result = sqlx::query_scalar!( + r#"SELECT success_handler FROM workspace_settings WHERE workspace_id = 'test-workspace'"# + ) + .fetch_one(&db) + .await?; + + assert_eq!(result, Some("script/f/test/success_handler".to_string())); + + // Verify extra args were set + let extra_args = sqlx::query_scalar!( + r#"SELECT success_handler_extra_args FROM workspace_settings WHERE workspace_id = 'test-workspace'"# + ) + .fetch_one(&db) + .await?; + + assert!(extra_args.is_some()); + + Ok(()) +} + +/// Test that success handler can be set and removed via database operations +#[cfg(feature = "deno_core")] +#[sqlx::test(fixtures("base"))] +async fn test_success_handler_settings(db: Pool) -> anyhow::Result<()> { + initialize_tracing().await; + + let _server = ApiServer::start(db.clone()).await?; + + // Initially success_handler should be NULL + let initial = sqlx::query_scalar!( + r#"SELECT success_handler FROM workspace_settings WHERE workspace_id = 'test-workspace'"# + ) + .fetch_one(&db) + .await?; + assert!(initial.is_none()); + + // Set success handler + sqlx::query!( + r#" + UPDATE workspace_settings + SET success_handler = 'flow/f/test/success_flow' + WHERE workspace_id = 'test-workspace' + "# + ) + .execute(&db) + .await?; + + let after_set = sqlx::query_scalar!( + r#"SELECT success_handler FROM workspace_settings WHERE workspace_id = 'test-workspace'"# + ) + .fetch_one(&db) + .await?; + assert_eq!(after_set, Some("flow/f/test/success_flow".to_string())); + + // Remove success handler + sqlx::query!( + r#" + UPDATE workspace_settings + SET success_handler = NULL + WHERE workspace_id = 'test-workspace' + "# + ) + .execute(&db) + .await?; + + let after_remove = sqlx::query_scalar!( + r#"SELECT success_handler FROM workspace_settings WHERE workspace_id = 'test-workspace'"# + ) + .fetch_one(&db) + .await?; + assert!(after_remove.is_none()); + + Ok(()) +} + +/// Test that a successful job completion triggers the success handler +#[cfg(all(feature = "deno_core", feature = "enterprise", feature = "private"))] +#[sqlx::test(fixtures("base"))] +async fn test_success_handler_triggered_on_success(db: Pool) -> anyhow::Result<()> { + use serde_json::json; + use windmill_common::jobs::JobPayload; + use windmill_common::runnable_settings::{ConcurrencySettings, DebouncingSettings}; + use windmill_common::scripts::{ScriptHash, ScriptLang}; + + initialize_tracing().await; + + let server = ApiServer::start(db.clone()).await?; + + // Create a simple success handler script + // Note: lock must be non-null for script to be considered "deployed" + let success_handler_code = r#" +export async function main(path: string, email: string, job_id: string, is_flow: boolean, workspace_id: string, result: any) { + console.log("Success handler called for job:", job_id); + return { handled: true, original_path: path }; +} +"#; + + sqlx::query!( + r#" + INSERT INTO script (workspace_id, hash, path, content, language, kind, created_by, schema, summary, description, lock) + VALUES ('test-workspace', 1234567890, 'f/test/success_handler', $1, 'deno', 'script', 'test-user', '{}', 'Success handler script', 'Handles successful job completions', '') + "#, + success_handler_code + ) + .execute(&db) + .await?; + + // Create a simple test script that we'll run (needs to be JobKind::Script, not Preview) + // Note: lock must be non-null for script to be considered "deployed" + let test_script_code = "export function main() { return 'success'; }"; + let test_script_hash: i64 = 9876543210; + + sqlx::query!( + r#" + INSERT INTO script (workspace_id, hash, path, content, language, kind, created_by, schema, summary, description, lock) + VALUES ('test-workspace', $1, 'f/test/simple_script', $2, 'deno', 'script', 'test-user', '{}', 'Simple test script', 'A simple test script', '') + "#, + test_script_hash, + test_script_code + ) + .execute(&db) + .await?; + + // Set up the success handler in workspace_settings + sqlx::query!( + r#" + UPDATE workspace_settings + SET success_handler = 'script/f/test/success_handler' + WHERE workspace_id = 'test-workspace' + "# + ) + .execute(&db) + .await?; + + // Create the success_handler group + sqlx::query!( + r#" + INSERT INTO group_ (workspace_id, name, summary, extra_perms) + VALUES ('test-workspace', 'success_handler', 'The group the success handler acts on behalf of', '{"u/test-user": true}') + ON CONFLICT DO NOTHING + "# + ) + .execute(&db) + .await?; + + // Run a script using ScriptHash (produces JobKind::Script, not Preview) + let completed_job = RunJob::from(JobPayload::ScriptHash { + hash: ScriptHash(test_script_hash), + path: "f/test/simple_script".to_string(), + cache_ttl: None, + cache_ignore_s3_path: None, + dedicated_worker: None, + language: ScriptLang::Deno, + priority: None, + apply_preprocessor: false, + concurrency_settings: ConcurrencySettings::default(), + debouncing_settings: DebouncingSettings::default(), + }) + .run_until_complete(&db, false, server.addr.port()) + .await; + + let result = completed_job.json_result().unwrap(); + assert_eq!(result, json!("success")); + + let main_job_id = completed_job.id; + + // Wait a short time for the success handler job to be created + tokio::time::sleep(tokio::time::Duration::from_millis(500)).await; + + // Verify the success handler job was created (query by email since trigger is not set) + let success_handler_job = sqlx::query!( + r#" + SELECT + id, + runnable_path, + permissioned_as_email, + parent_job + FROM v2_job + WHERE workspace_id = 'test-workspace' + AND permissioned_as_email = 'success_handler@windmill.dev' + ORDER BY created_at DESC + LIMIT 1 + "# + ) + .fetch_optional(&db) + .await?; + + assert!( + success_handler_job.is_some(), + "Success handler job should have been created" + ); + + let handler_job = success_handler_job.unwrap(); + + // Verify the success handler job has correct parameters + assert_eq!( + handler_job.runnable_path.as_deref(), + Some("f/test/success_handler"), + "Success handler should run the configured script" + ); + assert_eq!( + handler_job.permissioned_as_email.as_str(), + "success_handler@windmill.dev", + "Success handler should run as success_handler user" + ); + assert_eq!( + handler_job.parent_job, + Some(main_job_id), + "Success handler should have main job as parent" + ); + // Note: root_job may be None when it equals parent_job (optimization in push function) + + Ok(()) +} diff --git a/backend/windmill-api/Cargo.toml b/backend/windmill-api/Cargo.toml index 08845d1e05..ae4372ac24 100644 --- a/backend/windmill-api/Cargo.toml +++ b/backend/windmill-api/Cargo.toml @@ -10,7 +10,7 @@ path = "src/lib.rs" [features] default = [] -private = ["windmill-audit/private"] +private = ["windmill-audit/private", "windmill-common/private"] enterprise = ["windmill-queue/enterprise", "windmill-audit/enterprise", "windmill-git-sync/enterprise", "windmill-common/enterprise", "windmill-worker/enterprise"] stripe = [] agent_worker_server = [] @@ -157,6 +157,8 @@ google-cloud-googleapis = { workspace = true , optional = true } tonic = { workspace = true, optional = true } deno_error = { workspace = true, optional = true } deno_core = { workspace = true, optional = true } +tar.workspace = true +flate2.workspace = true backon = {workspace = true, optional = true} [build-dependencies] diff --git a/backend/windmill-api/openapi.yaml b/backend/windmill-api/openapi.yaml index 639e9c4217..d20e4ff7d2 100644 --- a/backend/windmill-api/openapi.yaml +++ b/backend/windmill-api/openapi.yaml @@ -1,7 +1,7 @@ openapi: "3.0.3" info: - version: 1.608.0 + version: 1.611.0 title: Windmill API contact: @@ -1411,6 +1411,83 @@ paths: items: $ref: "#/components/schemas/GlobalSetting" + /.well-known/jwks.json: + get: + summary: get JWKS for Vault JWT authentication + operationId: getJwks + tags: + - setting + responses: + "200": + description: JSON Web Key Set + content: + application/json: + schema: + $ref: "#/components/schemas/JwksResponse" + + /settings/test_secret_backend: + post: + summary: test secret backend connection (HashiCorp Vault) + operationId: testSecretBackend + tags: + - setting + requestBody: + description: Vault settings to test + required: true + content: + application/json: + schema: + $ref: "#/components/schemas/VaultSettings" + responses: + "200": + description: connection successful + content: + text/plain: + schema: + type: string + + /settings/migrate_secrets_to_vault: + post: + summary: migrate secrets from database to HashiCorp Vault + operationId: migrateSecretsToVault + tags: + - setting + requestBody: + description: Vault settings for migration target + required: true + content: + application/json: + schema: + $ref: "#/components/schemas/VaultSettings" + responses: + "200": + description: migration report + content: + application/json: + schema: + $ref: "#/components/schemas/SecretMigrationReport" + + /settings/migrate_secrets_to_database: + post: + summary: migrate secrets from HashiCorp Vault to database + operationId: migrateSecretsToDatabase + tags: + - setting + requestBody: + description: Vault settings for migration source + required: true + content: + application/json: + schema: + $ref: "#/components/schemas/VaultSettings" + responses: + "200": + description: migration report + content: + application/json: + schema: + $ref: "#/components/schemas/SecretMigrationReport" + /users/email: get: summary: get current user email (if logged in) @@ -2276,6 +2353,10 @@ paths: $ref: "#/components/schemas/ScriptArgs" error_handler_muted_on_cancel: type: boolean + success_handler: + type: string + success_handler_extra_args: + $ref: "#/components/schemas/ScriptArgs" large_file_storage: $ref: "#/components/schemas/LargeFileStorage" ducklake: @@ -2992,6 +3073,34 @@ paths: schema: type: string + /w/{workspace}/workspaces/edit_success_handler: + post: + summary: edit success handler + operationId: editSuccessHandler + tags: + - workspace + parameters: + - $ref: "#/components/parameters/WorkspaceId" + requestBody: + description: WorkspaceSuccessHandler + required: true + content: + application/json: + schema: + type: object + properties: + success_handler: + type: string + success_handler_extra_args: + $ref: "#/components/schemas/ScriptArgs" + responses: + "200": + description: status + content: + text/plain: + schema: + type: string + /w/{workspace}/workspaces/edit_large_file_storage_config: post: summary: edit large file storage settings @@ -4977,6 +5086,141 @@ paths: items: type: string + /w/{workspace}/npm_proxy/metadata/{package}: + get: + summary: get npm package metadata from private registry + operationId: getNpmPackageMetadata + tags: + - npm_proxy + parameters: + - $ref: "#/components/parameters/WorkspaceId" + - name: package + description: npm package name + in: path + required: true + schema: + type: string + responses: + "200": + description: package metadata + content: + application/json: + schema: + type: object + properties: + tags: + type: object + additionalProperties: + type: string + versions: + type: array + items: + type: string + + /w/{workspace}/npm_proxy/resolve/{package}: + get: + summary: resolve npm package version from private registry + operationId: resolveNpmPackageVersion + tags: + - npm_proxy + parameters: + - $ref: "#/components/parameters/WorkspaceId" + - name: package + description: npm package name + in: path + required: true + schema: + type: string + - name: tag + description: version tag or reference + in: query + required: false + schema: + type: string + responses: + "200": + description: resolved version + content: + application/json: + schema: + type: object + properties: + version: + type: string + nullable: true + + /w/{workspace}/npm_proxy/filetree/{package}/{version}: + get: + summary: get npm package file tree from private registry + operationId: getNpmPackageFiletree + tags: + - npm_proxy + parameters: + - $ref: "#/components/parameters/WorkspaceId" + - name: package + description: npm package name + in: path + required: true + schema: + type: string + - name: version + description: package version + in: path + required: true + schema: + type: string + responses: + "200": + description: package file tree + content: + application/json: + schema: + type: object + properties: + default: + type: string + files: + type: array + items: + type: object + properties: + name: + type: string + + /w/{workspace}/npm_proxy/file/{package}/{version}/{filepath}: + get: + summary: get specific file from npm package in private registry + operationId: getNpmPackageFile + tags: + - npm_proxy + parameters: + - $ref: "#/components/parameters/WorkspaceId" + - name: package + description: npm package name + in: path + required: true + schema: + type: string + - name: version + description: package version + in: path + required: true + schema: + type: string + - name: filepath + description: file path within package + in: path + required: true + schema: + type: string + responses: + "200": + description: file content + content: + text/plain: + schema: + type: string + /w/{workspace}/embeddings/query_resource_types: get: summary: query resource types by similarity @@ -16430,6 +16674,83 @@ components: # -- INLINE END -- # Do not change line above + VaultSettings: + type: object + required: + - address + - mount_path + properties: + address: + type: string + description: HashiCorp Vault server address (e.g., https://vault.company.com:8200) + mount_path: + type: string + description: KV v2 secrets engine mount path (e.g., windmill) + jwt_role: + type: string + description: Vault JWT auth role name for Windmill (optional, if not provided token auth is used) + namespace: + type: string + description: Vault Enterprise namespace (optional) + token: + type: string + description: Static Vault token for testing/development (optional, if provided this is used instead of JWT authentication) + + SecretMigrationFailure: + type: object + required: + - workspace_id + - path + - error + properties: + workspace_id: + type: string + description: Workspace ID where the secret is located + path: + type: string + description: Path of the secret that failed to migrate + error: + type: string + description: Error message + + SecretMigrationReport: + type: object + required: + - total_secrets + - migrated_count + - failed_count + - failures + properties: + total_secrets: + type: integer + format: int64 + description: Total number of secrets found + migrated_count: + type: integer + format: int64 + description: Number of secrets successfully migrated + failed_count: + type: integer + format: int64 + description: Number of secrets that failed to migrate + failures: + type: array + items: + $ref: "#/components/schemas/SecretMigrationFailure" + description: Details of any failures encountered during migration + + JwksResponse: + type: object + required: + - keys + properties: + keys: + type: array + items: + type: object + additionalProperties: true + description: Array of JSON Web Keys for JWT verification + FlowConversation: type: object required: diff --git a/backend/windmill-api/src/lib.rs b/backend/windmill-api/src/lib.rs index a46f37a811..7fbfee8c9a 100644 --- a/backend/windmill-api/src/lib.rs +++ b/backend/windmill-api/src/lib.rs @@ -106,6 +106,7 @@ mod inkeep_oss; mod inputs; mod integration; mod live_migrations; +mod npm_proxy; #[cfg(feature = "http_trigger")] mod openapi; #[cfg(all(feature = "private", feature = "parquet"))] @@ -148,6 +149,7 @@ mod scim_oss; mod scopes; mod scripts; mod service_logs; +mod secret_backend_ext; mod settings; mod slack_approvals; #[cfg(all(feature = "smtp", feature = "private"))] @@ -355,7 +357,7 @@ pub async fn run_server( { let smtp_server = Arc::new(SmtpServer { db: db.clone(), - user_db: user_db, + user_db: user_db.clone(), auth_cache: auth_cache.clone(), base_internal_url: _base_internal_url.clone(), }); @@ -409,7 +411,8 @@ pub async fn run_server( let (mcp_router, mcp_cancellation_token) = { #[cfg(feature = "mcp")] if server_mode || mcp_mode { - let (mcp_router, mcp_cancellation_token) = setup_mcp_server().await?; + let (mcp_router, mcp_cancellation_token) = + setup_mcp_server(db.clone(), user_db).await?; let mcp_middleware = axum::middleware::from_fn(extract_and_store_workspace_id); ( mcp_router.layer(mcp_middleware), @@ -495,6 +498,7 @@ pub async fn run_server( Router::new() }) .nest("/ai", ai::workspaced_service()) + .nest("/npm_proxy", npm_proxy::workspaced_service()) .nest("/raw_apps", raw_apps::workspaced_service()) .nest("/resources", resources::workspaced_service()) .nest("/schedules", schedule::workspaced_service()) @@ -716,6 +720,8 @@ pub async fn run_server( .route("/openapi.yaml", get(openapi)) .route("/openapi.json", get(openapi_json)), ) + // JWKS endpoint for HashiCorp Vault JWT authentication (must be outside /api prefix) + .route("/.well-known/jwks.json", get(settings::get_jwks)) .fallback(static_assets::static_handler) .layer(middleware_stack); diff --git a/backend/windmill-api/src/mcp/tools/auto_generated_endpoints.rs b/backend/windmill-api/src/mcp/auto_generated_endpoints.rs similarity index 99% rename from backend/windmill-api/src/mcp/tools/auto_generated_endpoints.rs rename to backend/windmill-api/src/mcp/auto_generated_endpoints.rs index d5425dcd7f..91a07e3f7a 100644 --- a/backend/windmill-api/src/mcp/tools/auto_generated_endpoints.rs +++ b/backend/windmill-api/src/mcp/auto_generated_endpoints.rs @@ -1,22 +1,8 @@ // Auto-generated MCP tools from OpenAPI specification // This file is generated by generate_mcp_tools.py - DO NOT EDIT MANUALLY - use std::borrow::Cow; -use serde::{Deserialize, Serialize}; - - -#[derive(Debug, Clone, Deserialize, Serialize)] -pub struct EndpointTool { - pub name: Cow<'static, str>, - pub description: Cow<'static, str>, - pub instructions: Cow<'static, str>, - pub path: Cow<'static, str>, - pub method: Cow<'static, str>, - pub path_params_schema: Option, - pub query_params_schema: Option, - pub body_schema: Option, -} +use windmill_mcp::server::EndpointTool; pub fn all_tools() -> Vec { vec![ diff --git a/backend/windmill-api/src/mcp/core.rs b/backend/windmill-api/src/mcp/core.rs new file mode 100644 index 0000000000..6b5917398e --- /dev/null +++ b/backend/windmill-api/src/mcp/core.rs @@ -0,0 +1,448 @@ +//! Windmill MCP Backend implementation +//! +//! This module provides the concrete implementation of the McpBackend trait +//! for the Windmill platform. + +use async_trait::async_trait; +use serde_json::Value; +use std::collections::HashMap; +use windmill_common::{db::UserDB, utils::StripPath, DB}; +use windmill_mcp::common::transform::apply_key_transformation; +use windmill_mcp::common::types::{ + FlowInfo, HubScriptInfo, ResourceInfo, ResourceType, SchemaType, ScriptInfo, +}; +use windmill_mcp::server::{BackendResult, EndpointTool, ErrorData, McpAuth, McpBackend}; + +use crate::db::ApiAuthed; +use crate::jobs::{ + run_wait_result_flow_by_path_internal, run_wait_result_script_by_path_internal, RunJobQuery, +}; + +use super::auto_generated_endpoints::all_tools; +use super::utils::{ + build_query_string, build_request_body, create_http_request, get_hub_script_schema, + get_item_schema, get_items, get_resources, get_resources_types, get_scripts_from_hub, + parse_response_body, prepare_push_args, substitute_path_params, +}; + +use std::sync::Arc; +use std::time::Duration; + +use tokio_util::sync::CancellationToken; +use windmill_mcp::server::{ + LocalSessionManager, Runner, StreamableHttpServerConfig, StreamableHttpService, +}; +use windmill_mcp::WorkspaceId; + +use axum::{ + extract::Path, http::Request, middleware::Next, response::Response, routing::get, Json, Router, +}; +use windmill_common::error::JsonResult; + +/// Implement McpAuth for ApiAuthed +impl McpAuth for ApiAuthed { + fn username(&self) -> &str { + &self.username + } + + fn email(&self) -> &str { + &self.email + } + + fn is_admin(&self) -> bool { + self.is_admin + } + + fn is_operator(&self) -> bool { + self.is_operator + } + + fn groups(&self) -> &[String] { + &self.groups + } + + fn folders(&self) -> &[(String, bool, bool)] { + &self.folders + } + + fn scopes(&self) -> Option<&[String]> { + self.scopes.as_deref() + } +} + +/// Windmill's MCP backend implementation +#[derive(Clone)] +pub struct WindmillBackend { + pub db: DB, + pub user_db: UserDB, +} + +impl WindmillBackend { + pub fn new(db: DB, user_db: UserDB) -> Self { + Self { db, user_db } + } +} + +#[async_trait] +impl McpBackend for WindmillBackend { + type Auth = ApiAuthed; + + async fn list_scripts( + &self, + auth: &ApiAuthed, + workspace_id: &str, + favorites_only: bool, + ) -> BackendResult> { + let scope_type = if favorites_only { "favorites" } else { "all" }; + get_items::(&self.user_db, auth, workspace_id, scope_type, "script") + .await + .map_err(|e| ErrorData::internal_error(e.message, None)) + } + + async fn list_flows( + &self, + auth: &ApiAuthed, + workspace_id: &str, + favorites_only: bool, + ) -> BackendResult> { + let scope_type = if favorites_only { "favorites" } else { "all" }; + get_items::(&self.user_db, auth, workspace_id, scope_type, "flow") + .await + .map_err(|e| ErrorData::internal_error(e.message, None)) + } + + async fn list_resource_types( + &self, + auth: &ApiAuthed, + workspace_id: &str, + ) -> BackendResult> { + get_resources_types(&self.user_db, auth, workspace_id) + .await + .map_err(|e| ErrorData::internal_error(e.message, None)) + } + + async fn list_resources( + &self, + auth: &ApiAuthed, + workspace_id: &str, + resource_type: &str, + ) -> BackendResult> { + get_resources(&self.user_db, auth, workspace_id, resource_type) + .await + .map_err(|e| ErrorData::internal_error(e.message, None)) + } + + async fn list_hub_scripts( + &self, + app_filter: Option<&str>, + ) -> BackendResult> { + get_scripts_from_hub(&self.db, app_filter) + .await + .map_err(|e| ErrorData::internal_error(e.message, None)) + } + + async fn get_item_schema( + &self, + auth: &ApiAuthed, + workspace_id: &str, + path: &str, + item_type: &str, + ) -> BackendResult> { + let schema = get_item_schema(path, &self.user_db, auth, workspace_id, item_type) + .await + .map_err(|e| ErrorData::internal_error(e.message, None))?; + + if let Some(ref s) = schema { + match serde_json::from_str::(s.0.get()) { + Ok(val) => Ok(Some(val)), + Err(e) => { + tracing::warn!("Failed to parse schema: {}", e); + Ok(None) + } + } + } else { + Ok(None) + } + } + + async fn get_hub_script_schema(&self, path: &str) -> BackendResult> { + let schema = get_hub_script_schema(path, &self.db) + .await + .map_err(|e| ErrorData::internal_error(e.message, None))?; + + if let Some(ref s) = schema { + match serde_json::from_str::(s.0.get()) { + Ok(val) => Ok(Some(val)), + Err(e) => { + tracing::warn!("Failed to parse hub schema: {}", e); + Ok(None) + } + } + } else { + Ok(None) + } + } + + fn transform_schema_for_resources( + &self, + schema: &SchemaType, + resources_cache: &HashMap>, + resources_types: &[ResourceType], + ) -> SchemaType { + let mut schema_obj = schema.clone(); + + // Replace invalid char in property key with underscore + let replacements: Vec<(String, String, Value)> = schema_obj + .properties + .iter() + .filter_map(|(key, value)| { + if key.chars().any(|c| !c.is_alphanumeric() && c != '_') { + let new_key = apply_key_transformation(key); + Some((key.clone(), new_key, value.clone())) + } else { + None + } + }) + .collect(); + + for (old_key, new_key, value) in replacements { + schema_obj.properties.remove(&old_key); + schema_obj.properties.insert(new_key, value); + } + + for (_key, prop_value) in schema_obj.properties.iter_mut() { + if let Value::Object(prop_map) = prop_value { + if let Some(format_value) = prop_map.get("format") { + if let Value::String(format_str) = format_value { + if format_str.starts_with("resource-") { + let resource_type_key = + format_str.split("-").last().unwrap_or_default().to_string(); + let resource_type = resources_types + .iter() + .find(|rt| rt.name == resource_type_key); + let resource_type_obj = resource_type.cloned(); + + if let Some(resource_cache) = resources_cache.get(&resource_type_key) { + let resources_count = resource_cache.len(); + let description = match resource_type_obj { + Some(resource_type_obj) => format!( + "This is a resource named `{}` with the following description: `{}`.\\nThe path of the resource should be used to specify the resource.\\n{}", + resource_type_obj.name, + resource_type_obj.description.as_deref().unwrap_or("No description"), + if resources_count == 0 { + "This resource does not have any available instances, you should create one from your windmill workspace." + } else if resources_count > 1 { + "This resource has multiple available instances, you should precisely select the one you want to use." + } else { + "There is 1 resource available." + } + ), + None => "An object parameter.".to_string(), + }; + prop_map.insert( + "type".to_string(), + Value::String("string".to_string()), + ); + prop_map + .insert("description".to_string(), Value::String(description)); + if resources_count > 0 { + let resources_description = resource_cache + .iter() + .map(|resource| { + format!( + "{}: $res:{}", + resource + .description + .as_deref() + .unwrap_or("No title"), + resource.path + ) + }) + .collect::>() + .join("\\n"); + + prop_map.insert( + "description".to_string(), + Value::String(format!( + "{}\\nHere are the available resources, in the format title:path. Title can be empty. Path should be used to specify the resource:\\n{}", + prop_map.get("description").unwrap_or(&Value::String("No description".to_string())), + resources_description + )), + ); + } + } + } + } + } + } + } + + schema_obj + } + + async fn run_script( + &self, + auth: &ApiAuthed, + workspace_id: &str, + path: &str, + args: Value, + ) -> BackendResult { + let push_args = prepare_push_args(args); + + let result = run_wait_result_script_by_path_internal( + self.db.clone(), + RunJobQuery::default(), + StripPath(path.to_string()), + auth.clone(), + self.user_db.clone(), + workspace_id.to_string(), + push_args, + ) + .await + .map_err(|e| ErrorData::internal_error(e.to_string(), None))?; + + parse_response_body(result).await + } + + async fn run_flow( + &self, + auth: &ApiAuthed, + workspace_id: &str, + path: &str, + args: Value, + ) -> BackendResult { + let push_args = prepare_push_args(args); + + let result = run_wait_result_flow_by_path_internal( + self.db.clone(), + RunJobQuery::default(), + StripPath(path.to_string()), + auth.clone(), + self.user_db.clone(), + push_args, + workspace_id.to_string(), + ) + .await + .map_err(|e| ErrorData::internal_error(e.to_string(), None))?; + + parse_response_body(result).await + } + + async fn call_endpoint( + &self, + auth: &ApiAuthed, + workspace_id: &str, + endpoint_tool: &EndpointTool, + args: Value, + ) -> BackendResult { + let args_map = match &args { + Value::Object(map) => map, + _ => { + return Err(ErrorData::invalid_params( + "Arguments must be an object", + None, + )); + } + }; + + // Build URL with path substitutions + let path_template = substitute_path_params( + &endpoint_tool.path, + workspace_id, + args_map, + &endpoint_tool.path_params_schema, + )?; + let query_string = build_query_string(args_map, &endpoint_tool.query_params_schema); + let full_url = format!( + "{}/api{}{}", + windmill_common::BASE_INTERNAL_URL.as_str(), + path_template, + query_string + ); + + // Prepare request body + let body_json = + build_request_body(&endpoint_tool.method, args_map, &endpoint_tool.body_schema); + + // Create and execute request + let response = create_http_request( + &endpoint_tool.method, + &full_url, + workspace_id, + auth, + body_json, + ) + .await?; + + let status = response.status(); + let response_text = response.text().await.map_err(|e| { + ErrorData::internal_error(format!("Failed to read response text: {}", e), None) + })?; + + if status.is_success() { + Ok(serde_json::from_str(&response_text) + .unwrap_or_else(|_| Value::String(response_text))) + } else { + Err(ErrorData::internal_error( + format!( + "HTTP {} {}: {}", + status.as_u16(), + status.canonical_reason().unwrap_or(""), + response_text + ), + None, + )) + } + } + + fn all_endpoint_tools(&self) -> Vec { + all_tools() + } +} + +/// Extract workspace ID from path and store it in request extensions +pub async fn extract_and_store_workspace_id( + Path(params): Path, + mut request: Request, + next: Next, +) -> Response { + let workspace_id = params; + request.extensions_mut().insert(WorkspaceId(workspace_id)); + next.run(request).await +} + +/// Setup the MCP server with HTTP transport +pub async fn setup_mcp_server( + db: DB, + user_db: UserDB, +) -> anyhow::Result<(Router, CancellationToken)> { + let cancellation_token = CancellationToken::new(); + let session_manager = Arc::new(LocalSessionManager::default()); + + let backend = WindmillBackend::new(db, user_db); + let runner = Runner::new(backend); + + let service_config = StreamableHttpServerConfig { + sse_keep_alive: Some(Duration::from_secs(15)), + stateful_mode: false, + cancellation_token: cancellation_token.clone(), + sse_retry: Some(Duration::from_secs(15)), + }; + + let service = + StreamableHttpService::new(move || Ok(runner.clone()), session_manager, service_config); + + let router = Router::new().nest_service("/", service); + Ok((router, cancellation_token)) +} + +/// HTTP handler to list MCP tools as JSON +async fn list_mcp_tools_handler() -> JsonResult> { + let endpoint_tools = all_tools(); + Ok(Json(endpoint_tools)) +} + +/// Creates a router service for listing MCP tools +pub fn list_tools_service() -> Router { + Router::new().route("/", get(list_mcp_tools_handler)) +} diff --git a/backend/windmill-api/src/mcp/mod.rs b/backend/windmill-api/src/mcp/mod.rs index e1cd855547..b7853fd7de 100644 --- a/backend/windmill-api/src/mcp/mod.rs +++ b/backend/windmill-api/src/mcp/mod.rs @@ -3,9 +3,9 @@ //! This module provides the MCP server implementation that exposes Windmill scripts, //! flows, and API endpoints as MCP tools for AI assistants to interact with. -pub mod server; -pub mod tools; -pub mod utils; +mod auto_generated_endpoints; +mod core; +mod utils; -// Re-export main components -pub use server::{extract_and_store_workspace_id, list_tools_service, setup_mcp_server}; +// Re-export only what's needed externally +pub use core::{extract_and_store_workspace_id, list_tools_service, setup_mcp_server}; diff --git a/backend/windmill-api/src/mcp/server.rs b/backend/windmill-api/src/mcp/server.rs deleted file mode 100644 index ba38de4b1c..0000000000 --- a/backend/windmill-api/src/mcp/server.rs +++ /dev/null @@ -1,561 +0,0 @@ -//! MCP Server implementation -//! -//! Contains the core MCP server handler that implements the Model Context Protocol -//! specification. This is a thin orchestration layer that delegates to the appropriate -//! modules for tool management, database operations, and schema transformation. - -use std::collections::HashMap; -use std::sync::Arc; -use std::{borrow::Cow, time::Duration}; - -use axum::body::to_bytes; -use serde_json::Value; -use tokio::try_join; -use tokio_util::sync::CancellationToken; -use windmill_common::db::UserDB; -use windmill_common::worker::to_raw_value; -use windmill_common::{utils::StripPath, DB}; -use windmill_mcp::server::{ - Annotated, CallToolRequestParam, CallToolResult, Content, ErrorData, Implementation, - InitializeRequestParam, InitializeResult, ListPromptsResult, ListResourceTemplatesResult, - ListResourcesResult, ListToolsResult, LocalSessionManager, PaginatedRequestParam, - ProtocolVersion, RawContent, RawTextContent, RequestContext, RoleServer, ServerCapabilities, - ServerHandler, ServerInfo, StreamableHttpServerConfig, StreamableHttpService, Tool, - ToolAnnotations, -}; - -use crate::db::ApiAuthed; -use crate::jobs::{ - run_wait_result_flow_by_path_internal, run_wait_result_script_by_path_internal, RunJobQuery, -}; - -use super::tools::endpoint_tools::{ - all_endpoint_tools, call_endpoint_tool, endpoint_tools_to_mcp_tools, EndpointTool, -}; -use super::utils::{ - database::{ - check_scopes, get_hub_script_schema, get_item_schema, get_items, get_resources_types, - get_scripts_from_hub, - }, - models::{ - FlowInfo, ResourceInfo, ResourceType, SchemaType, ScriptInfo, ToolableItem, WorkspaceId, - }, - schema::transform_schema_for_resources, - scope_matcher::{is_resource_allowed, parse_mcp_scopes}, - transform::{reverse_transform, reverse_transform_key}, -}; - -use axum::{ - extract::Path, http::Request, middleware::Next, response::Response, routing::get, Json, Router, -}; -use windmill_common::error::JsonResult; - -/// MCP Server Runner - implements the core MCP protocol handlers -#[derive(Clone)] -pub struct Runner {} - -impl Runner { - pub fn new() -> Self { - Self {} - } - - /// Creates a Tool from a ToolableItem - async fn create_tool_from_item( - item: &T, - user_db: &UserDB, - authed: &ApiAuthed, - workspace_id: &str, - resources_cache: &mut HashMap>, - resources_types: &Vec, - ) -> Result { - let is_hub = item.is_hub(); - let path = item.get_path_or_id(); - let item_type = item.item_type(); - let description = format!( - "This is a {} named `{}` with the following description: `{}`.{}", - item_type, - item.get_summary(), - item.get_description(), - if is_hub { - format!( - " It is a tool used for the following app: {}", - item.get_integration_type() - .unwrap_or("No integration type".to_string()) - ) - } else { - "".to_string() - } - ); - let schema_obj = transform_schema_for_resources( - &item.get_schema(), - user_db, - authed, - &workspace_id, - resources_cache, - &resources_types, - ) - .await?; - let input_schema_map = match serde_json::to_value(schema_obj) { - Ok(Value::Object(map)) => map, - Ok(_) => { - tracing::warn!("Schema object for tool '{}' did not serialize to a JSON object, using empty schema.", path); - serde_json::Map::new() - } - Err(e) => { - tracing::error!( - "Failed to serialize schema object for tool '{}': {}. Using empty schema.", - path, - e - ); - serde_json::Map::new() - } - }; - - Ok(Tool { - name: Cow::Owned(path), - description: Some(Cow::Owned(description)), - input_schema: Arc::new(input_schema_map), - title: Some(item.get_summary().to_string()), - output_schema: None, - icons: None, - annotations: Some(ToolAnnotations { - title: Some(item.get_summary().to_string()), - read_only_hint: Some(false), // Can modify environment - destructive_hint: Some(true), // Can potentially be destructive - idempotent_hint: Some(false), // Are not guaranteed to be idempotent - open_world_hint: Some(true), // Can interact with external services - }), - meta: None, - }) - } -} - -impl ServerHandler for Runner { - /// Handles the `CallTool` request from the MCP client - async fn call_tool( - &self, - request: CallToolRequestParam, - context: RequestContext, - ) -> Result { - let http_parts = context - .extensions - .get::() - .ok_or_else(|| { - tracing::error!("http::request::Parts not found"); - ErrorData::internal_error("http::request::Parts not found", None) - })?; - - let authed = http_parts.extensions.get::().ok_or_else(|| { - tracing::error!("ApiAuthed Axum extension not found"); - ErrorData::internal_error("ApiAuthed Axum extension not found", None) - })?; - - check_scopes(authed)?; - - // Parse MCP scopes for authorization - let scopes = authed.scopes.as_ref().map(|s| s.as_slice()).unwrap_or(&[]); - let scope_config = parse_mcp_scopes(scopes)?; - - if request.name.ends_with("_TRUNC") { - return Ok(CallToolResult::error( - vec![ - Annotated::new( - RawContent::Text(RawTextContent { - text: - "Tool path is too long. Consider shortening it to make it compatible with MCP." - .to_string(), - meta: None, - }), - None - ), - ] - )); - } - - let db = http_parts.extensions.get::().ok_or_else(|| { - tracing::error!("DB Axum extension not found"); - ErrorData::internal_error("DB Axum extension not found", None) - })?; - - let user_db = http_parts.extensions.get::().ok_or_else(|| { - tracing::error!("UserDB Axum extension not found"); - ErrorData::internal_error("UserDB Axum extension not found", None) - })?; - - let args = request.arguments.map(Value::Object).ok_or_else(|| { - ErrorData::invalid_params( - "Missing arguments for tool", - Some(request.name.clone().into()), - ) - })?; - - let workspace_id = http_parts - .extensions - .get::() - .ok_or_else(|| { - tracing::error!("WorkspaceId not found"); - ErrorData::internal_error("WorkspaceId not found", None) - }) - .map(|w_id| w_id.0.clone())?; - - // Check if this is a generated endpoint tool - let endpoint_tools = all_endpoint_tools(); - for endpoint_tool in endpoint_tools { - if endpoint_tool.name.as_ref() == request.name { - // Validate endpoint scope - if scope_config.granular - && !is_resource_allowed(&endpoint_tool.name, &scope_config.endpoints) - { - return Err(ErrorData::internal_error( - format!( - "Access denied: endpoint '{}' not in token scope", - endpoint_tool.name - ), - None, - )); - } - - // This is an endpoint tool, forward to the actual HTTP endpoint - let result = - call_endpoint_tool(&endpoint_tool, args.clone(), &workspace_id, &authed) - .await?; - return Ok(CallToolResult::success(vec![Content::text( - serde_json::to_string_pretty(&result).unwrap_or_else(|_| "{}".to_string()), - )])); - } - } - - // Continue with script/flow logic - let (tool_type, path, is_hub) = reverse_transform(&request.name).map_err(|e| { - ErrorData::internal_error(format!("Failed to reverse transform path: {}", e), None) - })?; - - // Validate script/flow scope - if !is_hub && scope_config.granular { - if tool_type == "script" && !is_resource_allowed(&path, &scope_config.scripts) { - return Err(ErrorData::internal_error( - format!("Access denied: script '{}' not in token scope", path), - None, - )); - } else if tool_type == "flow" && !is_resource_allowed(&path, &scope_config.flows) { - return Err(ErrorData::internal_error( - format!("Access denied: flow '{}' not in token scope", path), - None, - )); - } - } - - let item_schema = if is_hub { - get_hub_script_schema(&format!("hub/{}", path), db).await? - } else { - get_item_schema(&path, user_db, authed, &workspace_id, &tool_type).await? - }; - - let schema_obj = if let Some(ref s) = item_schema { - match serde_json::from_str::(s.0.get()) { - Ok(val) => Some(val), - Err(e) => { - tracing::warn!("Failed to parse schema: {}", e); - None - } - } - } else { - None - }; - - let push_args = if let Value::Object(map) = args.clone() { - let mut args_hash = HashMap::new(); - for (k, v) in map { - // need to transform back the key without invalid characters to the original key - let original_key = reverse_transform_key(&k, &schema_obj); - args_hash.insert(original_key, to_raw_value(&v)); - } - windmill_queue::PushArgsOwned { extra: None, args: args_hash } - } else { - windmill_queue::PushArgsOwned::default() - }; - let script_or_flow_path = if is_hub { - StripPath(format!("hub/{}", path)) - } else { - StripPath(path) - }; - let run_query = RunJobQuery::default(); - - let result = if tool_type == "script" { - run_wait_result_script_by_path_internal( - db.clone(), - run_query, - script_or_flow_path, - authed.clone(), - user_db.clone(), - workspace_id.clone(), - push_args, - ) - .await - } else { - run_wait_result_flow_by_path_internal( - db.clone(), - run_query, - script_or_flow_path, - authed.clone(), - user_db.clone(), - push_args, - workspace_id.clone(), - ) - .await - }; - - match result { - Ok(response) => { - let body_bytes = to_bytes(response.into_body(), usize::MAX) - .await - .map_err(|e| { - ErrorData::internal_error( - format!("Failed to read response body: {}", e), - None, - ) - })?; - let body_str = String::from_utf8(body_bytes.to_vec()).map_err(|e| { - ErrorData::internal_error( - format!("Failed to decode response body: {}", e), - None, - ) - })?; - Ok(CallToolResult::success(vec![Content::text(body_str)])) - } - Err(e) => Err(ErrorData::internal_error( - format!("Failed to run script: {}", e), - None, - )), - } - } - - /// Fetches available tools (scripts, flows, hub scripts) based on the user's scope - async fn list_tools( - &self, - _request: Option, - mut _context: RequestContext, - ) -> Result { - let http_parts = _context - .extensions - .get::() - .ok_or_else(|| { - tracing::error!("http::request::Parts not found"); - ErrorData::internal_error("http::request::Parts not found", None) - })?; - - let authed = http_parts.extensions.get::().ok_or_else(|| { - tracing::error!("ApiAuthed Axum extension not found"); - ErrorData::internal_error("ApiAuthed Axum extension not found", None) - })?; - - check_scopes(authed)?; - - let db = http_parts.extensions.get::().ok_or_else(|| { - tracing::error!("DB Axum extension not found"); - ErrorData::internal_error("DB Axum extension not found", None) - })?; - - let user_db = http_parts.extensions.get::().ok_or_else(|| { - tracing::error!("UserDB Axum extension not found"); - ErrorData::internal_error("UserDB Axum extension not found", None) - })?; - - let workspace_id = http_parts - .extensions - .get::() - .ok_or_else(|| { - tracing::error!("WorkspaceId not found"); - ErrorData::internal_error("WorkspaceId not found", None) - }) - .map(|w_id| w_id.0.clone())?; - - // Parse MCP scopes to determine what to expose - let scopes = authed.scopes.as_ref().map(|s| s.as_slice()).unwrap_or(&[]); - let scope_config = parse_mcp_scopes(scopes)?; - - let scope_type = if scope_config.favorites { - "favorites" - } else { - // Fetch all items if either all or granular scope set (we filter later for granular scopes) - "all" - }; - - let scripts_fn = - get_items::(user_db, authed, &workspace_id, scope_type, "script"); - let flows_fn = get_items::(user_db, authed, &workspace_id, scope_type, "flow"); - let resources_types_fn = get_resources_types(user_db, authed, &workspace_id); - let hub_scripts_fn = get_scripts_from_hub(db, scope_config.hub_apps.as_deref()); - let (scripts, flows, resources_types, hub_scripts) = if scope_config.hub_apps.is_some() { - let (scripts, flows, resources_types, hub_scripts) = - try_join!(scripts_fn, flows_fn, resources_types_fn, hub_scripts_fn)?; - (scripts, flows, resources_types, hub_scripts) - } else { - let (scripts, flows, resources_types) = - try_join!(scripts_fn, flows_fn, resources_types_fn)?; - (scripts, flows, resources_types, vec![]) - }; - - let mut resources_cache: HashMap> = HashMap::new(); - let mut tools: Vec = Vec::new(); - - // Filter and add scripts based on scope - for script in scripts { - // For granular scopes, filter by path - if scope_config.granular && !is_resource_allowed(&script.path, &scope_config.scripts) { - continue; - } - - tools.push( - Runner::create_tool_from_item( - &script, - user_db, - authed, - &workspace_id, - &mut resources_cache, - &resources_types, - ) - .await?, - ); - } - - // Filter and add flows based on scope - for flow in flows { - // For granular scopes, filter by path - if scope_config.granular && !is_resource_allowed(&flow.path, &scope_config.flows) { - continue; - } - - tools.push( - Runner::create_tool_from_item( - &flow, - user_db, - authed, - &workspace_id, - &mut resources_cache, - &resources_types, - ) - .await?, - ); - } - - for hub_script in hub_scripts { - tools.push( - Runner::create_tool_from_item( - &hub_script, - user_db, - authed, - &workspace_id, - &mut resources_cache, - &resources_types, - ) - .await?, - ); - } - - // Add endpoint tools from the generated MCP tools, filtered by scope - let endpoint_tools = all_endpoint_tools(); - for endpoint_tool in endpoint_tools { - // For granular scopes, filter by endpoint name - if scope_config.granular - && !is_resource_allowed(&endpoint_tool.name, &scope_config.endpoints) - { - continue; - } - - tools.push( - endpoint_tools_to_mcp_tools(vec![endpoint_tool]) - .into_iter() - .next() - .unwrap(), - ); - } - - Ok(ListToolsResult { tools, next_cursor: None, meta: None }) - } - - fn get_info(&self) -> ServerInfo { - ServerInfo { - protocol_version: ProtocolVersion::default(), - capabilities: ServerCapabilities::builder() - .enable_tools() - .build(), - server_info: Implementation::from_build_env(), - instructions: Some("This server provides a list of scripts and flows the user can run on Windmill. Each flow and script is a tool callable with their respective arguments.".to_string()), - } - } - - async fn initialize( - &self, - _request: InitializeRequestParam, - _context: RequestContext, - ) -> Result { - Ok(self.get_info()) - } - - async fn list_resources( - &self, - _request: Option, - _context: RequestContext, - ) -> Result { - Ok(ListResourcesResult { resources: vec![], next_cursor: None, meta: None }) - } - - async fn list_prompts( - &self, - _request: Option, - _context: RequestContext, - ) -> Result { - Ok(ListPromptsResult::default()) - } - - async fn list_resource_templates( - &self, - _request: Option, - _context: RequestContext, - ) -> Result { - Ok(ListResourceTemplatesResult::default()) - } -} - -/// Extract workspace ID from path and store it in request extensions -pub async fn extract_and_store_workspace_id( - Path(params): Path, - mut request: Request, - next: Next, -) -> Response { - let workspace_id = params; - request.extensions_mut().insert(WorkspaceId(workspace_id)); - next.run(request).await -} - -/// Setup the MCP server with HTTP transport -pub async fn setup_mcp_server() -> anyhow::Result<(Router, CancellationToken)> { - let cancellation_token = CancellationToken::new(); - let session_manager = Arc::new(LocalSessionManager::default()); - let service_config = StreamableHttpServerConfig { - sse_keep_alive: Some(Duration::from_secs(15)), - stateful_mode: false, - cancellation_token: cancellation_token.clone(), - sse_retry: Some(Duration::from_secs(15)), - }; - let service = StreamableHttpService::new( - || Ok(Runner::new()), - session_manager.clone(), - service_config, - ); - - let router = axum::Router::new().nest_service("/", service); - Ok((router, cancellation_token)) -} - -/// HTTP handler to list MCP tools as JSON -async fn list_mcp_tools_handler() -> JsonResult> { - let endpoint_tools = all_endpoint_tools(); - Ok(Json(endpoint_tools)) -} - -/// Creates a router service for listing MCP tools -pub fn list_tools_service() -> Router { - Router::new().route("/", get(list_mcp_tools_handler)) -} diff --git a/backend/windmill-api/src/mcp/tools/endpoint_tools.rs b/backend/windmill-api/src/mcp/tools/endpoint_tools.rs deleted file mode 100644 index 372310b463..0000000000 --- a/backend/windmill-api/src/mcp/tools/endpoint_tools.rs +++ /dev/null @@ -1,307 +0,0 @@ -//! Endpoint tools for MCP server -//! -//! Contains the auto-generated endpoint tools and utilities for converting -//! them to MCP tools and handling HTTP calls to Windmill API endpoints. - -use crate::db::ApiAuthed; -use std::sync::Arc; -use windmill_common::db::Authed; -use windmill_common::{auth::create_jwt_token, BASE_INTERNAL_URL}; -use windmill_mcp::server::{ErrorData, Tool, ToolAnnotations}; - -// Import the auto-generated tools -use super::auto_generated_endpoints; -pub use auto_generated_endpoints::{all_tools, EndpointTool}; - -/// Get all available endpoint tools -pub fn all_endpoint_tools() -> Vec { - all_tools() -} - -/// Convert endpoint tools to MCP tools -pub fn endpoint_tools_to_mcp_tools(endpoint_tools: Vec) -> Vec { - endpoint_tools - .into_iter() - .map(|tool| endpoint_tool_to_mcp_tool(&tool)) - .collect() -} - -/// Convert a single endpoint tool to MCP tool -pub fn endpoint_tool_to_mcp_tool(tool: &EndpointTool) -> Tool { - let mut combined_properties = serde_json::Map::new(); - let mut combined_required = Vec::new(); - - // Combine all parameter schemas - let schemas = [ - &tool.path_params_schema, - &tool.query_params_schema, - &tool.body_schema, - ]; - - for schema in schemas.iter().filter_map(|s| s.as_ref()) { - merge_schema_into(&mut combined_properties, &mut combined_required, schema); - } - - let combined_schema = serde_json::json!({ - "type": "object", - "properties": combined_properties, - "required": combined_required - }); - - let description = format!("{}. {}", tool.description, tool.instructions); - - // Create annotations based on HTTP method and endpoint characteristics - let annotations = create_endpoint_annotations(tool); - - Tool { - name: tool.name.clone(), - description: Some(description.into()), - input_schema: Arc::new(combined_schema.as_object().unwrap().clone()), - title: Some(tool.name.to_string()), - output_schema: None, - icons: None, - annotations: Some(annotations), - meta: None, - } -} - -/// Create appropriate annotations for endpoint tools based on HTTP method -fn create_endpoint_annotations(tool: &EndpointTool) -> ToolAnnotations { - let method = tool.method.as_ref(); - - // Determine characteristics based on HTTP method - let (read_only, destructive, idempotent, open_world) = match method { - "GET" => (true, false, true, true), // Read-only, safe, idempotent - "POST" => (false, true, false, true), // Can modify, potentially destructive, not idempotent - "PUT" => (false, false, true, true), // Can modify, typically idempotent updates - "DELETE" => (false, true, true, true), // Destructive but idempotent - "PATCH" => (false, false, false, true), // Partial updates, not guaranteed idempotent - _ => (false, true, false, true), // Default: assume can modify and be destructive - }; - - ToolAnnotations { - title: Some(format!("{} {}", method, tool.path)), - read_only_hint: Some(read_only), - destructive_hint: Some(destructive), - idempotent_hint: Some(idempotent), - open_world_hint: Some(open_world), - } -} - -/// Merge schema into combined properties and required fields -fn merge_schema_into( - combined_properties: &mut serde_json::Map, - combined_required: &mut Vec, - schema: &serde_json::Value, -) { - if let Some(props) = schema.get("properties").and_then(|p| p.as_object()) { - for (key, value) in props { - combined_properties.insert(key.clone(), value.clone()); - } - } - - if let Some(required) = schema.get("required").and_then(|r| r.as_array()) { - for req in required.iter().filter_map(|r| r.as_str()) { - combined_required.push(req.to_string()); - } - } -} - -/// Call an endpoint tool by making HTTP request to Windmill API -pub async fn call_endpoint_tool( - tool: &EndpointTool, - args: serde_json::Value, - workspace_id: &str, - api_authed: &ApiAuthed, -) -> Result { - let args_map = match &args { - serde_json::Value::Object(map) => map, - _ => { - return Err(ErrorData::invalid_params( - "Arguments must be an object", - Some(tool.name.clone().into()), - )) - } - }; - - // Build URL with path substitutions - let path_template = - substitute_path_params(&tool.path, workspace_id, args_map, &tool.path_params_schema)?; - let query_string = build_query_string(args_map, &tool.query_params_schema); - let full_url = format!( - "{}/api{}{}", - BASE_INTERNAL_URL.as_str(), - path_template, - query_string - ); - - // Prepare request body - let body_json = build_request_body(&tool.method, args_map, &tool.body_schema); - - // Create and execute request - let response = - create_http_request(&tool.method, &full_url, workspace_id, api_authed, body_json).await?; - - let status = response.status(); - let response_text = response.text().await.map_err(|e| { - ErrorData::internal_error(format!("Failed to read response text: {}", e), None) - })?; - - if status.is_success() { - Ok(serde_json::from_str(&response_text) - .unwrap_or_else(|_| serde_json::Value::String(response_text))) - } else { - Err(ErrorData::internal_error( - format!( - "HTTP {} {}: {}", - status.as_u16(), - status.canonical_reason().unwrap_or(""), - response_text - ), - None, - )) - } -} - -/// Substitute path parameters in the URL template -fn substitute_path_params( - path: &str, - workspace_id: &str, - args_map: &serde_json::Map, - path_schema: &Option, -) -> Result { - let mut path_template = path.replace("{workspace}", workspace_id); - - if let Some(schema) = path_schema { - if let Some(props) = schema.get("properties").and_then(|p| p.as_object()) { - for (param_name, _) in props { - let placeholder = format!("{{{}}}", param_name); - match args_map.get(param_name) { - Some(param_value) => { - if let Some(str_val) = param_value.as_str() { - path_template = path_template.replace(&placeholder, str_val); - } - } - None => { - tracing::warn!("Missing required path parameter: {}", param_name); - return Err(ErrorData::invalid_params( - format!("Missing required path parameter: {}", param_name), - None, - )); - } - } - } - } - } - - Ok(path_template) -} - -/// Build query string from arguments -fn build_query_string( - args_map: &serde_json::Map, - query_schema: &Option, -) -> String { - let Some(schema) = query_schema else { - return String::new(); - }; - let Some(props) = schema.get("properties").and_then(|p| p.as_object()) else { - return String::new(); - }; - - let query_params: Vec = props - .keys() - .filter_map(|param_name| { - args_map - .get(param_name) - .filter(|v| !v.is_null()) - .map(|value| { - let value_str = value.to_string(); - let str_val = value_str.trim_matches('"'); - format!( - "{}={}", - urlencoding::encode(param_name), - urlencoding::encode(str_val) - ) - }) - }) - .collect(); - - if query_params.is_empty() { - String::new() - } else { - format!("?{}", query_params.join("&")) - } -} - -/// Build request body from arguments -fn build_request_body( - method: &str, - args_map: &serde_json::Map, - body_schema: &Option, -) -> Option { - if method == "GET" { - return None; - } - - let schema = body_schema.as_ref()?; - let props = schema.get("properties")?.as_object()?; - - let body_map: serde_json::Map = props - .keys() - .filter_map(|param_name| { - args_map - .get(param_name) - .map(|value| (param_name.clone(), value.clone())) - }) - .collect(); - - if body_map.is_empty() { - None - } else { - Some(serde_json::Value::Object(body_map)) - } -} - -/// Create HTTP request with authentication -async fn create_http_request( - method: &str, - url: &str, - workspace_id: &str, - api_authed: &ApiAuthed, - body_json: Option, -) -> Result { - let client = &crate::HTTP_CLIENT; - let mut request_builder = match method { - "GET" => client.get(url), - "POST" => client.post(url), - "PUT" => client.put(url), - "DELETE" => client.delete(url), - "PATCH" => client.patch(url), - _ => { - return Err(ErrorData::invalid_params( - format!("Unsupported HTTP method: {}", method), - None, - )) - } - }; - - // Add authorization header - let authed = Authed::from(api_authed.clone()); - let token = create_jwt_token(authed, workspace_id, 3600, None, None, None, None) - .await - .map_err(|e| ErrorData::internal_error(e.to_string(), None))?; - request_builder = request_builder.header("Authorization", format!("Bearer {}", token)); - - // Add body if present - if let Some(body) = body_json { - request_builder = request_builder - .header("Content-Type", "application/json") - .json(&body); - } - - request_builder - .send() - .await - .map_err(|e| ErrorData::internal_error(format!("Failed to execute request: {}", e), None)) -} diff --git a/backend/windmill-api/src/mcp/tools/flow_tools.rs b/backend/windmill-api/src/mcp/tools/flow_tools.rs deleted file mode 100644 index 03b47472bf..0000000000 --- a/backend/windmill-api/src/mcp/tools/flow_tools.rs +++ /dev/null @@ -1,40 +0,0 @@ -//! Flow tools for MCP server -//! -//! Contains functionality for converting Windmill flows into MCP tools. - -use super::super::utils::{ - models::{FlowInfo, ToolableItem, SchemaType}, - schema::convert_schema_to_schema_type, - transform::transform_path, -}; - -/// Implementation of ToolableItem for FlowInfo -impl ToolableItem for FlowInfo { - fn get_path_or_id(&self) -> String { - transform_path(&self.path, "flow") - } - - fn get_summary(&self) -> &str { - self.summary.as_deref().unwrap_or("No summary") - } - - fn get_description(&self) -> &str { - self.description.as_deref().unwrap_or("No description") - } - - fn get_schema(&self) -> SchemaType { - convert_schema_to_schema_type(self.schema.clone()) - } - - fn is_hub(&self) -> bool { - false - } - - fn item_type(&self) -> &'static str { - "flow" - } - - fn get_integration_type(&self) -> Option { - None - } -} \ No newline at end of file diff --git a/backend/windmill-api/src/mcp/tools/hub_tools.rs b/backend/windmill-api/src/mcp/tools/hub_tools.rs deleted file mode 100644 index b81b973b30..0000000000 --- a/backend/windmill-api/src/mcp/tools/hub_tools.rs +++ /dev/null @@ -1,43 +0,0 @@ -//! Hub tools for MCP server -//! -//! Contains functionality for integrating Windmill Hub scripts as MCP tools. - -use super::super::utils::{ - models::{HubScriptInfo, ToolableItem, SchemaType}, -}; - -/// Implementation of ToolableItem for HubScriptInfo -impl ToolableItem for HubScriptInfo { - fn get_path_or_id(&self) -> String { - let id = self.version_id; - let summary = self.summary.as_deref().unwrap_or("No summary"); - format!("hs-{}-{}", id, summary.replace(" ", "_")) - } - - fn get_summary(&self) -> &str { - self.summary.as_deref().unwrap_or("No summary") - } - - fn get_description(&self) -> &str { - self.description.as_deref().unwrap_or("No description") - } - - fn get_schema(&self) -> SchemaType { - match serde_json::from_value::(self.schema.clone().unwrap_or_default()) { - Ok(schema_type) => schema_type, - Err(_) => SchemaType::default(), - } - } - - fn is_hub(&self) -> bool { - true - } - - fn item_type(&self) -> &'static str { - "script" - } - - fn get_integration_type(&self) -> Option { - self.app.clone() - } -} \ No newline at end of file diff --git a/backend/windmill-api/src/mcp/tools/mod.rs b/backend/windmill-api/src/mcp/tools/mod.rs deleted file mode 100644 index 1488b22d8a..0000000000 --- a/backend/windmill-api/src/mcp/tools/mod.rs +++ /dev/null @@ -1,10 +0,0 @@ -//! Tool management for MCP server -//! -//! This module handles the conversion of Windmill scripts, flows, and endpoints -//! into MCP tools that can be used by AI assistants. - -pub mod script_tools; -pub mod flow_tools; -pub mod hub_tools; -pub mod endpoint_tools; -pub mod auto_generated_endpoints; \ No newline at end of file diff --git a/backend/windmill-api/src/mcp/tools/script_tools.rs b/backend/windmill-api/src/mcp/tools/script_tools.rs deleted file mode 100644 index 716fae3a87..0000000000 --- a/backend/windmill-api/src/mcp/tools/script_tools.rs +++ /dev/null @@ -1,40 +0,0 @@ -//! Script tools for MCP server -//! -//! Contains functionality for converting Windmill scripts into MCP tools. - -use super::super::utils::{ - models::{ScriptInfo, ToolableItem, SchemaType}, - schema::convert_schema_to_schema_type, - transform::transform_path, -}; - -/// Implementation of ToolableItem for ScriptInfo -impl ToolableItem for ScriptInfo { - fn get_path_or_id(&self) -> String { - transform_path(&self.path, "script") - } - - fn get_summary(&self) -> &str { - self.summary.as_deref().unwrap_or("No summary") - } - - fn get_description(&self) -> &str { - self.description.as_deref().unwrap_or("No description") - } - - fn get_schema(&self) -> SchemaType { - convert_schema_to_schema_type(self.schema.clone()) - } - - fn is_hub(&self) -> bool { - false - } - - fn item_type(&self) -> &'static str { - "script" - } - - fn get_integration_type(&self) -> Option { - None - } -} \ No newline at end of file diff --git a/backend/windmill-api/src/mcp/utils.rs b/backend/windmill-api/src/mcp/utils.rs new file mode 100644 index 0000000000..a2513ef5f7 --- /dev/null +++ b/backend/windmill-api/src/mcp/utils.rs @@ -0,0 +1,414 @@ +//! Utility functions for MCP server +//! +//! Contains database query functions and HTTP request helpers +//! used by the MCP server implementation. + +use std::collections::HashMap; + +use axum::body::{to_bytes, Body}; +use axum::response::Response; +use serde_json::Value; +use sql_builder::prelude::*; +use windmill_common::auth::create_jwt_token; +use windmill_common::db::{Authed, UserDB}; +use windmill_common::scripts::{get_full_hub_script_by_path, Schema}; +use windmill_common::utils::{query_elems_from_hub, StripPath}; +use windmill_common::worker::to_raw_value; +use windmill_common::{DB, HUB_BASE_URL}; +use windmill_mcp::server::{BackendResult, ErrorData}; +use windmill_mcp::{HubResponse, HubScriptInfo, ItemSchema, ResourceInfo, ResourceType}; + +use crate::db::ApiAuthed; +use crate::HTTP_CLIENT; + +// items max limit +const ITEMS_FETCH_MAX_LIMIT: usize = 100; + +// ============================================================================ +// Database utilities +// ============================================================================ + +/// Get the schema for a specific item (script or flow) +pub async fn get_item_schema( + path: &str, + user_db: &UserDB, + authed: &ApiAuthed, + workspace_id: &str, + item_type: &str, +) -> Result, ErrorData> { + let mut sqlb = SqlBuilder::select_from(&format!("{} as o", item_type)); + sqlb.fields(&["o.schema"]); + sqlb.and_where("o.path = ?".bind(&path)); + sqlb.and_where("o.workspace_id = ?".bind(&workspace_id)); + sqlb.and_where("o.archived = false"); + sqlb.and_where("o.draft_only IS NOT TRUE"); + let sql = sqlb.sql().map_err(|e| { + tracing::error!("failed to build sql: {}", e); + ErrorData::internal_error(format!("failed to build sql: {}", e), None) + })?; + let mut tx = user_db.clone().begin(authed).await.map_err(|e| { + tracing::error!("failed to begin transaction: {}", e); + ErrorData::internal_error(format!("failed to begin transaction: {}", e), None) + })?; + let item = sqlx::query_as::<_, ItemSchema>(&sql) + .fetch_one(&mut *tx) + .await + .map_err(|e| { + tracing::error!("failed to fetch item schema: {}", e); + ErrorData::internal_error(format!("failed to fetch item schema: {}", e), None) + })?; + tx.commit().await.map_err(|e| { + tracing::error!("failed to commit transaction: {}", e); + ErrorData::internal_error(format!("failed to commit transaction: {}", e), None) + })?; + Ok(item.schema) +} + +/// Get all resource types from the database +pub async fn get_resources_types( + user_db: &UserDB, + authed: &ApiAuthed, + workspace_id: &str, +) -> Result, ErrorData> { + let mut sqlb = SqlBuilder::select_from("resource_type as o"); + sqlb.fields(&["o.name", "o.description"]); + sqlb.and_where("o.workspace_id = ?".bind(&workspace_id)); + let sql = sqlb.sql().map_err(|e| { + tracing::error!("failed to build sql: {}", e); + ErrorData::internal_error(format!("failed to build sql: {}", e), None) + })?; + let mut tx = user_db.clone().begin(authed).await.map_err(|e| { + tracing::error!("failed to begin transaction: {}", e); + ErrorData::internal_error(format!("failed to begin transaction: {}", e), None) + })?; + let rows = sqlx::query_as::<_, ResourceType>(&sql) + .fetch_all(&mut *tx) + .await + .map_err(|e| { + tracing::error!("failed to fetch resource types: {}", e); + ErrorData::internal_error(format!("failed to fetch resource types: {}", e), None) + })?; + tx.commit().await.map_err(|e| { + tracing::error!("failed to commit transaction: {}", e); + ErrorData::internal_error(format!("failed to commit transaction: {}", e), None) + })?; + Ok(rows) +} + +/// Get resources by type from the database +pub async fn get_resources( + user_db: &UserDB, + authed: &ApiAuthed, + workspace_id: &str, + resource_type: &str, +) -> Result, ErrorData> { + let mut sqlb = SqlBuilder::select_from("resource as o"); + sqlb.fields(&["o.path", "o.description", "o.resource_type"]); + sqlb.and_where("o.workspace_id = ?".bind(&workspace_id)); + sqlb.and_where("o.resource_type = ?".bind(&resource_type)); + let sql = sqlb.sql().map_err(|e| { + tracing::error!("failed to build sql: {}", e); + ErrorData::internal_error(format!("failed to build sql: {}", e), None) + })?; + let mut tx = user_db.clone().begin(authed).await.map_err(|e| { + tracing::error!("failed to begin transaction: {}", e); + ErrorData::internal_error(format!("failed to begin transaction: {}", e), None) + })?; + let rows = sqlx::query_as::<_, ResourceInfo>(&sql) + .fetch_all(&mut *tx) + .await + .map_err(|e| { + tracing::error!("failed to fetch resources: {}", e); + ErrorData::internal_error(format!("failed to fetch resources: {}", e), None) + })?; + tx.commit().await.map_err(|e| { + tracing::error!("failed to commit transaction: {}", e); + ErrorData::internal_error(format!("failed to commit transaction: {}", e), None) + })?; + + Ok(rows) +} + +/// Generic function to get items (scripts or flows) from the database +pub async fn get_items sqlx::FromRow<'a, sqlx::postgres::PgRow> + Send + Unpin>( + user_db: &UserDB, + authed: &ApiAuthed, + workspace_id: &str, + scope_type: &str, + item_type: &str, +) -> Result, ErrorData> { + let mut sqlb = SqlBuilder::select_from(&format!("{} as o", item_type)); + let fields = vec!["o.path", "o.summary", "o.description", "o.schema"]; + sqlb.fields(&fields); + if scope_type == "favorites" { + sqlb.join("favorite") + .on("favorite.favorite_kind = ? AND favorite.workspace_id = o.workspace_id AND favorite.path = o.path AND favorite.usr = ?".bind(&item_type) + .bind(&authed.username)); + } + sqlb.and_where("o.workspace_id = ?".bind(&workspace_id)) + .and_where("o.archived = false") + .and_where("o.draft_only IS NOT TRUE"); + + if item_type == "script" { + sqlb.and_where("(o.no_main_func IS NOT TRUE OR o.no_main_func IS NULL)"); + } + + sqlb.order_by( + if item_type == "flow" { + "o.edited_at" + } else { + "o.created_at" + }, + false, + ) + .limit(ITEMS_FETCH_MAX_LIMIT); + let sql = sqlb.sql().map_err(|e| { + tracing::error!("failed to build sql: {}", e); + ErrorData::internal_error(format!("failed to build sql: {}", e), None) + })?; + let mut tx = user_db.clone().begin(authed).await.map_err(|e| { + tracing::error!("failed to begin transaction: {}", e); + ErrorData::internal_error(format!("failed to begin transaction: {}", e), None) + })?; + let rows = sqlx::query_as::<_, T>(&sql) + .fetch_all(&mut *tx) + .await + .map_err(|e| { + tracing::error!("failed to fetch {}: {}", item_type, e); + ErrorData::internal_error(format!("failed to fetch {}: {}", item_type, e), None) + })?; + tx.commit().await.map_err(|e| { + tracing::error!("failed to commit transaction: {}", e); + ErrorData::internal_error(format!("failed to commit transaction: {}", e), None) + })?; + Ok(rows) +} + +/// Get scripts from the Hub +pub async fn get_scripts_from_hub( + db: &DB, + scope_integrations: Option<&str>, +) -> Result, ErrorData> { + let query_params = Some(vec![ + ("limit", ITEMS_FETCH_MAX_LIMIT.to_string()), + ("with_schema", "true".to_string()), + ("apps", scope_integrations.unwrap_or("").to_string()), + ]); + let url = format!("{}/scripts/top", *HUB_BASE_URL.read().await); + let (_status_code, _headers, response) = + query_elems_from_hub(&HTTP_CLIENT, &url, query_params, &db) + .await + .map_err(|e| { + tracing::error!("Failed to get items from hub: {}", e); + ErrorData::internal_error(format!("Failed to get items from hub: {}", e), None) + })?; + + use axum::body::to_bytes; + let body_bytes = to_bytes(response, usize::MAX).await.map_err(|e| { + tracing::error!("Failed to read response body: {}", e); + ErrorData::internal_error(format!("Failed to read response body: {}", e), None) + })?; + let body_str = String::from_utf8(body_bytes.to_vec()).map_err(|e| { + tracing::error!("Failed to decode response body: {}", e); + ErrorData::internal_error(format!("Failed to decode response body: {}", e), None) + })?; + let hub_response: HubResponse = serde_json::from_str(&body_str).map_err(|e| { + tracing::error!("Failed to parse hub response: {}", e); + ErrorData::internal_error(format!("Failed to parse hub response: {}", e), None) + })?; + + Ok(hub_response.asks) +} + +/// Get the schema for a Hub script +pub async fn get_hub_script_schema(path: &str, db: &DB) -> Result, ErrorData> { + let strip_path = StripPath(path.to_string()); + let res = get_full_hub_script_by_path(strip_path, &HTTP_CLIENT, Some(db)) + .await + .map_err(|e| { + tracing::error!("Failed to get hub script: {}", e); + ErrorData::internal_error(format!("Failed to get hub script: {}", e), None) + })?; + match serde_json::from_str::(res.schema.get()) { + Ok(schema) => Ok(Some(schema)), + Err(e) => { + tracing::warn!("Failed to convert schema: {}", e); + Ok(None) + } + } +} + +// ============================================================================ +// HTTP request utilities for endpoint tools +// ============================================================================ + +/// Substitute path parameters in the URL template +pub fn substitute_path_params( + path: &str, + workspace_id: &str, + args_map: &serde_json::Map, + path_schema: &Option, +) -> BackendResult { + let mut path_template = path.replace("{workspace}", workspace_id); + + if let Some(schema) = path_schema { + if let Some(props) = schema.get("properties").and_then(|p| p.as_object()) { + for (param_name, _) in props { + let placeholder = format!("{{{}}}", param_name); + match args_map.get(param_name) { + Some(param_value) => { + if let Some(str_val) = param_value.as_str() { + path_template = path_template.replace(&placeholder, str_val); + } + } + None => { + tracing::warn!("Missing required path parameter: {}", param_name); + return Err(ErrorData::invalid_params( + format!("Missing required path parameter: {}", param_name), + None, + )); + } + } + } + } + } + + Ok(path_template) +} + +/// Build query string from arguments +pub fn build_query_string( + args_map: &serde_json::Map, + query_schema: &Option, +) -> String { + let Some(schema) = query_schema else { + return String::new(); + }; + let Some(props) = schema.get("properties").and_then(|p| p.as_object()) else { + return String::new(); + }; + + let query_params: Vec = props + .keys() + .filter_map(|param_name| { + args_map + .get(param_name) + .filter(|v| !v.is_null()) + .map(|value| { + let value_str = value.to_string(); + let str_val = value_str.trim_matches('"'); + format!( + "{}={}", + urlencoding::encode(param_name), + urlencoding::encode(str_val) + ) + }) + }) + .collect(); + + if query_params.is_empty() { + String::new() + } else { + format!("?{}", query_params.join("&")) + } +} + +/// Build request body from arguments +pub fn build_request_body( + method: &str, + args_map: &serde_json::Map, + body_schema: &Option, +) -> Option { + if method == "GET" { + return None; + } + + let schema = body_schema.as_ref()?; + let props = schema.get("properties")?.as_object()?; + + let body_map: serde_json::Map = props + .keys() + .filter_map(|param_name| { + args_map + .get(param_name) + .map(|value| (param_name.clone(), value.clone())) + }) + .collect(); + + if body_map.is_empty() { + None + } else { + Some(Value::Object(body_map)) + } +} + +/// Create HTTP request with authentication +pub async fn create_http_request( + method: &str, + url: &str, + workspace_id: &str, + api_authed: &ApiAuthed, + body_json: Option, +) -> BackendResult { + let client = &HTTP_CLIENT; + let mut request_builder = match method { + "GET" => client.get(url), + "POST" => client.post(url), + "PUT" => client.put(url), + "DELETE" => client.delete(url), + "PATCH" => client.patch(url), + _ => { + return Err(ErrorData::invalid_params( + format!("Unsupported HTTP method: {}", method), + None, + )); + } + }; + + // Add authorization header + let authed = Authed::from(api_authed.clone()); + let token = create_jwt_token(authed, workspace_id, 3600, None, None, None, None) + .await + .map_err(|e| ErrorData::internal_error(e.to_string(), None))?; + request_builder = request_builder.header("Authorization", format!("Bearer {}", token)); + + // Add body if present + if let Some(body) = body_json { + request_builder = request_builder + .header("Content-Type", "application/json") + .json(&body); + } + + request_builder + .send() + .await + .map_err(|e| ErrorData::internal_error(format!("Failed to execute request: {}", e), None)) +} + +/// Convert a JSON Value into PushArgsOwned for job execution +pub fn prepare_push_args(args: Value) -> windmill_queue::PushArgsOwned { + if let Value::Object(map) = args { + let mut args_hash = HashMap::new(); + for (k, v) in map { + args_hash.insert(k, to_raw_value(&v)); + } + windmill_queue::PushArgsOwned { extra: None, args: args_hash } + } else { + windmill_queue::PushArgsOwned::default() + } +} + +/// Parse an HTTP response body into a JSON Value +pub async fn parse_response_body(response: Response) -> BackendResult { + let body_bytes = to_bytes(response.into_body(), usize::MAX) + .await + .map_err(|e| { + ErrorData::internal_error(format!("Failed to read response body: {}", e), None) + })?; + + let body_str = String::from_utf8(body_bytes.to_vec()).map_err(|e| { + ErrorData::internal_error(format!("Failed to decode response body: {}", e), None) + })?; + + Ok(serde_json::from_str(&body_str).unwrap_or_else(|_| Value::String(body_str))) +} diff --git a/backend/windmill-api/src/mcp/utils/database.rs b/backend/windmill-api/src/mcp/utils/database.rs deleted file mode 100644 index 8453177ac0..0000000000 --- a/backend/windmill-api/src/mcp/utils/database.rs +++ /dev/null @@ -1,243 +0,0 @@ -//! Database operations for MCP server -//! -//! Contains all database query functions and database-related utilities -//! used by the MCP server implementation. - -use windmill_mcp::server::ErrorData; -use sql_builder::prelude::*; -use windmill_common::db::UserDB; -use windmill_common::scripts::{get_full_hub_script_by_path, Schema}; -use windmill_common::utils::{query_elems_from_hub, StripPath}; -use windmill_common::{DB, HUB_BASE_URL}; - -use super::models::*; -use crate::db::ApiAuthed; -use crate::HTTP_CLIENT; - -/// Check if the user has proper MCP scopes -pub fn check_scopes(authed: &ApiAuthed) -> Result<(), ErrorData> { - let scopes = authed.scopes.as_ref(); - if scopes.is_none() - || scopes - .unwrap() - .iter() - .all(|scope| !scope.starts_with("mcp:")) - { - tracing::error!("Unauthorized: missing mcp scope"); - return Err(ErrorData::internal_error( - "Unauthorized: missing mcp scope".to_string(), - None, - )); - } - Ok(()) -} - -/// Get the schema for a specific item (script or flow) -pub async fn get_item_schema( - path: &str, - user_db: &UserDB, - authed: &ApiAuthed, - workspace_id: &str, - item_type: &str, -) -> Result, ErrorData> { - let mut sqlb = SqlBuilder::select_from(&format!("{} as o", item_type)); - sqlb.fields(&["o.schema"]); - sqlb.and_where("o.path = ?".bind(&path)); - sqlb.and_where("o.workspace_id = ?".bind(&workspace_id)); - sqlb.and_where("o.archived = false"); - sqlb.and_where("o.draft_only IS NOT TRUE"); - let sql = sqlb.sql().map_err(|_e| { - tracing::error!("failed to build sql: {}", _e); - ErrorData::internal_error("failed to build sql", None) - })?; - let mut tx = user_db - .clone() - .begin(authed) - .await - .map_err(|_e| ErrorData::internal_error("failed to begin transaction", None))?; - let item = sqlx::query_as::<_, ItemSchema>(&sql) - .fetch_one(&mut *tx) - .await - .map_err(|_e| { - tracing::error!("failed to fetch item schema: {}", _e); - ErrorData::internal_error("failed to fetch item schema", None) - })?; - tx.commit() - .await - .map_err(|_e| ErrorData::internal_error("failed to commit transaction", None))?; - Ok(item.schema) -} - -/// Get all resource types from the database -pub async fn get_resources_types( - user_db: &UserDB, - authed: &ApiAuthed, - workspace_id: &str, -) -> Result, ErrorData> { - let mut sqlb = SqlBuilder::select_from("resource_type as o"); - sqlb.fields(&["o.name", "o.description"]); - sqlb.and_where("o.workspace_id = ?".bind(&workspace_id)); - let sql = sqlb.sql().map_err(|_e| { - tracing::error!("failed to build sql: {}", _e); - ErrorData::internal_error("failed to build sql", None) - })?; - let mut tx = user_db - .clone() - .begin(authed) - .await - .map_err(|_e| ErrorData::internal_error("failed to begin transaction", None))?; - let rows = sqlx::query_as::<_, ResourceType>(&sql) - .fetch_all(&mut *tx) - .await - .map_err(|_e| { - tracing::error!("Failed to fetch resource types: {}", _e); - ErrorData::internal_error("failed to fetch resource types", None) - })?; - tx.commit() - .await - .map_err(|_e| ErrorData::internal_error("failed to commit transaction", None))?; - Ok(rows) -} - -/// Get resources by type from the database -pub async fn get_resources( - user_db: &UserDB, - authed: &ApiAuthed, - workspace_id: &str, - resource_type: &str, -) -> Result, ErrorData> { - let mut sqlb = SqlBuilder::select_from("resource as o"); - sqlb.fields(&["o.path", "o.description", "o.resource_type"]); - sqlb.and_where("o.workspace_id = ?".bind(&workspace_id)); - sqlb.and_where("o.resource_type = ?".bind(&resource_type)); - let sql = sqlb.sql().map_err(|_e| { - tracing::error!("failed to build sql: {}", _e); - ErrorData::internal_error("failed to build sql", None) - })?; - let mut tx = user_db - .clone() - .begin(authed) - .await - .map_err(|_e| ErrorData::internal_error("failed to begin transaction", None))?; - let rows = sqlx::query_as::<_, ResourceInfo>(&sql) - .fetch_all(&mut *tx) - .await - .map_err(|_e| { - tracing::error!("Failed to fetch resources: {}", _e); - ErrorData::internal_error("failed to fetch resources", None) - })?; - tx.commit() - .await - .map_err(|_e| ErrorData::internal_error("failed to commit transaction", None))?; - - Ok(rows) -} - -/// Generic function to get items (scripts or flows) from the database -pub async fn get_items sqlx::FromRow<'a, sqlx::postgres::PgRow> + Send + Unpin>( - user_db: &UserDB, - authed: &ApiAuthed, - workspace_id: &str, - scope_type: &str, - item_type: &str, -) -> Result, ErrorData> { - let mut sqlb = SqlBuilder::select_from(&format!("{} as o", item_type)); - let fields = vec!["o.path", "o.summary", "o.description", "o.schema"]; - sqlb.fields(&fields); - if scope_type == "favorites" { - sqlb.join("favorite") - .on("favorite.favorite_kind = ? AND favorite.workspace_id = o.workspace_id AND favorite.path = o.path AND favorite.usr = ?".bind(&item_type) - .bind(&authed.username)); - } - sqlb.and_where("o.workspace_id = ?".bind(&workspace_id)) - .and_where("o.archived = false") - .and_where("o.draft_only IS NOT TRUE"); - - if item_type == "script" { - sqlb.and_where("(o.no_main_func IS NOT TRUE OR o.no_main_func IS NULL)"); - } - - sqlb.order_by( - if item_type == "flow" { - "o.edited_at" - } else { - "o.created_at" - }, - false, - ) - .limit(100); - let sql = sqlb.sql().map_err(|_e| { - tracing::error!("failed to build sql: {}", _e); - ErrorData::internal_error("failed to build sql", None) - })?; - let mut tx = user_db - .clone() - .begin(authed) - .await - .map_err(|_e| ErrorData::internal_error("failed to begin transaction", None))?; - let rows = sqlx::query_as::<_, T>(&sql) - .fetch_all(&mut *tx) - .await - .map_err(|_e| { - tracing::error!("Failed to fetch {}: {}", item_type, _e); - ErrorData::internal_error(format!("failed to fetch {}", item_type), None) - })?; - tx.commit() - .await - .map_err(|_e| ErrorData::internal_error("failed to commit transaction", None))?; - Ok(rows) -} - -/// Get scripts from the Hub -pub async fn get_scripts_from_hub( - db: &DB, - scope_integrations: Option<&str>, -) -> Result, ErrorData> { - let query_params = Some(vec![ - ("limit", "100".to_string()), - ("with_schema", "true".to_string()), - ("apps", scope_integrations.unwrap_or("").to_string()), - ]); - let url = format!("{}/scripts/top", *HUB_BASE_URL.read().await); - let (_status_code, _headers, response) = - query_elems_from_hub(&HTTP_CLIENT, &url, query_params, &db) - .await - .map_err(|e| { - tracing::error!("Failed to get items from hub: {}", e); - ErrorData::internal_error(format!("Failed to get items from hub: {}", e), None) - })?; - - use axum::body::to_bytes; - let body_bytes = to_bytes(response, usize::MAX).await.map_err(|e| { - tracing::error!("Failed to read response body: {}", e); - ErrorData::internal_error(format!("Failed to read response body: {}", e), None) - })?; - let body_str = String::from_utf8(body_bytes.to_vec()).map_err(|e| { - tracing::error!("Failed to decode response body: {}", e); - ErrorData::internal_error(format!("Failed to decode response body: {}", e), None) - })?; - let hub_response: HubResponse = serde_json::from_str(&body_str).map_err(|e| { - tracing::error!("Failed to parse hub response: {}", e); - ErrorData::internal_error(format!("Failed to parse hub response: {}", e), None) - })?; - - Ok(hub_response.asks) -} - -/// Get the schema for a Hub script -pub async fn get_hub_script_schema(path: &str, db: &DB) -> Result, ErrorData> { - let strip_path = StripPath(path.to_string()); - let res = get_full_hub_script_by_path(strip_path, &HTTP_CLIENT, Some(db)) - .await - .map_err(|e| { - tracing::error!("Failed to get hub script: {}", e); - ErrorData::internal_error(format!("Failed to get hub script: {}", e), None) - })?; - match serde_json::from_str::(res.schema.get()) { - Ok(schema) => Ok(Some(schema)), - Err(e) => { - tracing::warn!("Failed to convert schema: {}", e); - Ok(None) - } - } -} diff --git a/backend/windmill-api/src/mcp/utils/mod.rs b/backend/windmill-api/src/mcp/utils/mod.rs deleted file mode 100644 index e4072313a0..0000000000 --- a/backend/windmill-api/src/mcp/utils/mod.rs +++ /dev/null @@ -1,10 +0,0 @@ -//! Utility functions and helpers for MCP server -//! -//! This module contains various utility functions for schema transformation, -//! database operations, data models, and path transformations. - -pub mod models; -pub mod database; -pub mod schema; -pub mod transform; -pub mod scope_matcher; \ No newline at end of file diff --git a/backend/windmill-api/src/mcp/utils/schema.rs b/backend/windmill-api/src/mcp/utils/schema.rs deleted file mode 100644 index b441d94a78..0000000000 --- a/backend/windmill-api/src/mcp/utils/schema.rs +++ /dev/null @@ -1,143 +0,0 @@ -//! Schema transformation utilities for MCP server -//! -//! Contains functions for transforming Windmill schemas into MCP-compatible formats, -//! including resource enrichment and schema conversion utilities. - -use windmill_mcp::server::ErrorData; -use serde_json::Value; -use std::collections::HashMap; -use windmill_common::db::UserDB; -use windmill_common::scripts::Schema; - -use super::database::get_resources; -use super::models::{ResourceInfo, ResourceType, SchemaType}; -use super::transform::apply_key_transformation; -use crate::db::ApiAuthed; - -/// Convert a Windmill Schema to a SchemaType -pub fn convert_schema_to_schema_type(schema: Option) -> SchemaType { - let schema_obj = if let Some(ref s) = schema { - match serde_json::from_str::(s.0.get()) { - Ok(val) => val, - Err(_) => SchemaType::default(), - } - } else { - SchemaType::default() - }; - schema_obj -} - -/// Transform the schema for resources by enriching with resource information -pub async fn transform_schema_for_resources( - schema: &SchemaType, - user_db: &UserDB, - authed: &ApiAuthed, - w_id: &str, - resources_cache: &mut HashMap>, - resources_types: &Vec, -) -> Result { - let mut schema_obj: SchemaType = schema.clone(); - - // replace invalid char in property key with underscore - let replacements: Vec<(String, String, Value)> = schema_obj - .properties - .iter() - .filter_map(|(key, value)| { - if key.chars().any(|c| !c.is_alphanumeric() && c != '_') { - let new_key = apply_key_transformation(key); - Some((key.clone(), new_key, value.clone())) - } else { - None - } - }) - .collect(); - - for (old_key, new_key, value) in replacements { - schema_obj.properties.remove(&old_key); - schema_obj.properties.insert(new_key, value); - } - - for (_key, prop_value) in schema_obj.properties.iter_mut() { - if let Value::Object(prop_map) = prop_value { - // if property is a resource, fetch the resource type infos, and add each available resource to the description - if let Some(format_value) = prop_map.get("format") { - if let Value::String(format_str) = format_value { - if format_str.starts_with("resource-") { - let resource_type_key = - format_str.split("-").last().unwrap_or_default().to_string(); - let resource_type = resources_types - .iter() - .find(|rt| rt.name == resource_type_key); - let resource_type_obj = resource_type.cloned(); - - if !resources_cache.contains_key(&resource_type_key) { - let available_resources = - get_resources(user_db, authed, &w_id, &resource_type_key).await; - - match available_resources { - Ok(cache_data) => { - resources_cache.insert(resource_type_key.clone(), cache_data); - } - Err(e) => { - tracing::error!("Failed to fetch resource cache data: {}", e); - continue; // Skip this property if fetching failed - } - } - } - - if let Some(resource_cache) = resources_cache.get(&resource_type_key) { - let resources_count = resource_cache.len(); - let description = match resource_type_obj { - Some(resource_type_obj) => format!( - "This is a resource named `{}` with the following description: `{}`.\\nThe path of the resource should be used to specify the resource.\\n{}", - resource_type_obj.name, - resource_type_obj.description.as_deref().unwrap_or("No description"), - if resources_count == 0 { - "This resource does not have any available instances, you should create one from your windmill workspace." - } else if resources_count > 1 { - "This resource has multiple available instances, you should precisely select the one you want to use." - } else { - "There is 1 resource available." - } - ), - None => "An object parameter.".to_string() - }; - prop_map - .insert("type".to_string(), Value::String("string".to_string())); - prop_map.insert("description".to_string(), Value::String(description)); - if resources_count > 0 { - let resources_description = resource_cache - .iter() - .map(|resource| { - format!( - "{}: $res:{}", - resource.description.as_deref().unwrap_or("No title"), - resource.path - ) - }) - .collect::>() - .join("\\n"); - - prop_map.insert( - "description".to_string(), - Value::String(format!( - "{}\\nHere are the available resources, in the format title:path. Title can be empty. Path should be used to specify the resource:\\n{}", - prop_map.get("description").unwrap_or(&Value::String("No description".to_string())), - resources_description - )), - ); - } - } - } - } - } - } else { - tracing::warn!( - "Schema property value is not a JSON object: {:?}", - prop_value - ); - } - } - - Ok(schema_obj) -} diff --git a/backend/windmill-api/src/npm_proxy.rs b/backend/windmill-api/src/npm_proxy.rs new file mode 100644 index 0000000000..8bd7f851f7 --- /dev/null +++ b/backend/windmill-api/src/npm_proxy.rs @@ -0,0 +1,502 @@ +/* + * This file provides a proxy endpoint for npm package requests + * to support private registries in the frontend ATA (Automatic Type Acquisition) + */ + +use axum::{ + extract::{Path, Query}, + routing::get, + Extension, Json, Router, +}; +use serde::{Deserialize, Serialize}; +use sqlx::types::JsonValue; +use std::collections::HashMap; +use tower_http::cors::{Any, CorsLayer}; +use windmill_common::{ + error::{Error, JsonResult, Result}, + global_settings::{load_value_from_global_settings, NPM_CONFIG_REGISTRY_SETTING}, + utils::StripPath, +}; + +use crate::{db::ApiAuthed, HTTP_CLIENT}; + +#[derive(Deserialize)] +struct ProxyQuery { + tag: Option, +} + +/// Parse a scoped package path like "@scope/name" or "name" from a wildcard path +fn parse_package_name(path: &str) -> String { + // Remove leading slash if present + path.trim_start_matches('/').to_string() +} + +/// Parse package and version from a path like "@scope/name/1.0.0" or "name/1.0.0" +fn parse_package_and_version(path: &str) -> Result<(String, String)> { + let path = path.trim_start_matches('/'); + + if path.starts_with('@') { + // Scoped package: @scope/name/version + let parts: Vec<&str> = path.splitn(3, '/').collect(); + if parts.len() < 3 { + return Err(Error::BadRequest("Invalid scoped package path, expected @scope/name/version".to_string())); + } + let package = format!("{}/{}", parts[0], parts[1]); + let version = parts[2].to_string(); + Ok((package, version)) + } else { + // Regular package: name/version + let parts: Vec<&str> = path.splitn(2, '/').collect(); + if parts.len() < 2 { + return Err(Error::BadRequest("Invalid package path, expected name/version".to_string())); + } + Ok((parts[0].to_string(), parts[1].to_string())) + } +} + +/// Parse package, version, and filepath from a path like "@scope/name/1.0.0/index.d.ts" +fn parse_package_version_and_file(path: &str) -> Result<(String, String, String)> { + let path = path.trim_start_matches('/'); + + if path.starts_with('@') { + // Scoped package: @scope/name/version/filepath + let parts: Vec<&str> = path.splitn(4, '/').collect(); + if parts.len() < 4 { + return Err(Error::BadRequest("Invalid scoped package file path, expected @scope/name/version/filepath".to_string())); + } + let package = format!("{}/{}", parts[0], parts[1]); + let version = parts[2].to_string(); + let filepath = parts[3].to_string(); + Ok((package, version, filepath)) + } else { + // Regular package: name/version/filepath + let parts: Vec<&str> = path.splitn(3, '/').collect(); + if parts.len() < 3 { + return Err(Error::BadRequest("Invalid package file path, expected name/version/filepath".to_string())); + } + Ok((parts[0].to_string(), parts[1].to_string(), parts[2].to_string())) + } +} + +#[derive(Serialize)] +struct PackageVersions { + tags: HashMap, + versions: Vec, +} + +#[derive(Serialize)] +struct PackageVersion { + version: Option, +} + +#[derive(Serialize)] +struct PackageFiletree { + default: String, + files: Vec, +} + +#[derive(Serialize)] +struct FileEntry { + name: String, +} + +pub fn workspaced_service() -> Router { + Router::new() + // Use wildcards for package names to support scoped packages like @scope/package + .route("/metadata/*package", get(get_package_metadata)) + .route("/resolve/*package", get(resolve_package_version)) + .route("/filetree/*package_version", get(get_package_filetree)) + .route("/file/*package_version_filepath", get(get_package_file)) + .layer( + CorsLayer::new() + .allow_origin(Any) + .allow_methods(Any) + .allow_headers(Any), + ) +} + +/// Get package metadata (versions and tags) from the private registry +async fn get_package_metadata( + _authed: ApiAuthed, + Path((_w_id, package_path)): Path<(String, StripPath)>, + Extension(db): Extension>, +) -> JsonResult { + let package = parse_package_name(package_path.to_path()); + let npm_registry = get_npm_registry(&db).await?; + + if npm_registry.is_none() { + return Err(Error::BadRequest( + "No private npm registry configured".to_string(), + )); + } + + let registry_url = npm_registry.unwrap(); + let package_url = format_registry_url(®istry_url, &package, None, None); + + tracing::info!("Fetching package metadata from: {}", package_url); + + let response = HTTP_CLIENT + .get(&package_url) + .send() + .await + .map_err(|e| Error::InternalErr(format!("Failed to fetch package metadata: {}", e)))?; + + if !response.status().is_success() { + return Err(Error::NotFound(format!( + "Package {} not found in private registry", + package + ))); + } + + let package_json: JsonValue = response + .json() + .await + .map_err(|e| Error::InternalErr(format!("Failed to parse package metadata: {}", e)))?; + + // Extract versions and dist-tags from the package metadata + let mut versions = Vec::new(); + let mut tags = HashMap::new(); + + if let Some(versions_obj) = package_json.get("versions").and_then(|v| v.as_object()) { + versions = versions_obj.keys().cloned().collect(); + } + + if let Some(tags_obj) = package_json.get("dist-tags").and_then(|v| v.as_object()) { + for (tag, version) in tags_obj { + if let Some(version_str) = version.as_str() { + tags.insert(tag.clone(), version_str.to_string()); + } + } + } + + Ok(Json(PackageVersions { tags, versions })) +} + +/// Resolve a package tag/version reference to a specific version +async fn resolve_package_version( + _authed: ApiAuthed, + Path((_w_id, package_path)): Path<(String, StripPath)>, + Query(query): Query, + Extension(db): Extension>, +) -> JsonResult { + let package = parse_package_name(package_path.to_path()); + let npm_registry = get_npm_registry(&db).await?; + + if npm_registry.is_none() { + return Err(Error::BadRequest( + "No private npm registry configured".to_string(), + )); + } + + let registry_url = npm_registry.unwrap(); + let reference = query.tag.unwrap_or_else(|| "latest".to_string()); + let package_url = format_registry_url(®istry_url, &package, None, None); + + tracing::info!("Resolving package version from: {}", package_url); + + let response = HTTP_CLIENT + .get(&package_url) + .send() + .await + .map_err(|e| Error::InternalErr(format!("Failed to fetch package metadata: {}", e)))?; + + if !response.status().is_success() { + return Err(Error::NotFound(format!( + "Package {} not found in private registry", + package + ))); + } + + let package_json: JsonValue = response + .json() + .await + .map_err(|e| Error::InternalErr(format!("Failed to parse package metadata: {}", e)))?; + + // Try to resolve the reference as a tag first + let version = if let Some(tags) = package_json.get("dist-tags").and_then(|v| v.as_object()) { + if let Some(version) = tags.get(&reference).and_then(|v| v.as_str()) { + Some(version.to_string()) + } else { + // If not a tag, check if it's a valid version + if let Some(versions) = package_json.get("versions").and_then(|v| v.as_object()) { + if versions.contains_key(&reference) { + Some(reference) + } else { + None + } + } else { + None + } + } + } else { + None + }; + + Ok(Json(PackageVersion { version })) +} + +/// Get the file tree for a specific package version +async fn get_package_filetree( + _authed: ApiAuthed, + Path((_w_id, package_version_path)): Path<(String, StripPath)>, + Extension(db): Extension>, +) -> JsonResult { + let (package, version) = parse_package_and_version(package_version_path.to_path())?; + let npm_registry = get_npm_registry(&db).await?; + + if npm_registry.is_none() { + return Err(Error::BadRequest( + "No private npm registry configured".to_string(), + )); + } + + let registry_url = npm_registry.unwrap(); + let package_url = format_registry_url(®istry_url, &package, None, None); + + tracing::info!("Fetching package filetree from: {}", package_url); + + let response = HTTP_CLIENT + .get(&package_url) + .send() + .await + .map_err(|e| Error::InternalErr(format!("Failed to fetch package metadata: {}", e)))?; + + if !response.status().is_success() { + return Err(Error::NotFound(format!( + "Package {} not found in private registry", + package + ))); + } + + let package_json: JsonValue = response + .json() + .await + .map_err(|e| Error::InternalErr(format!("Failed to parse package metadata: {}", e)))?; + + // Get the tarball URL for this version + let tarball_url = package_json + .get("versions") + .and_then(|v| v.get(&version)) + .and_then(|v| v.get("dist")) + .and_then(|d| d.get("tarball")) + .and_then(|t| t.as_str()) + .ok_or_else(|| Error::NotFound(format!("Tarball not found for {}@{}", package, version)))?; + + // Download and extract tarball to get file list + let tarball_response = HTTP_CLIENT + .get(tarball_url) + .send() + .await + .map_err(|e| Error::InternalErr(format!("Failed to download tarball: {}", e)))?; + + if !tarball_response.status().is_success() { + return Err(Error::NotFound(format!( + "Failed to download tarball for {}@{}", + package, version + ))); + } + + let tarball_bytes = tarball_response + .bytes() + .await + .map_err(|e| Error::InternalErr(format!("Failed to read tarball: {}", e)))?; + + // Extract file list from tarball + let files = extract_tarball_files(&tarball_bytes)?; + + // Find the main entry point + let main = package_json + .get("versions") + .and_then(|v| v.get(&version)) + .and_then(|v| v.get("main")) + .and_then(|m| m.as_str()) + .unwrap_or("index.js") + .to_string(); + + Ok(Json(PackageFiletree { + default: main, + files, + })) +} + +/// Get a specific file from a package version +async fn get_package_file( + _authed: ApiAuthed, + Path((_w_id, full_path)): Path<(String, StripPath)>, + Extension(db): Extension>, +) -> Result { + let (package, version, filepath) = parse_package_version_and_file(full_path.to_path())?; + let npm_registry = get_npm_registry(&db).await?; + + if npm_registry.is_none() { + return Err(Error::BadRequest( + "No private npm registry configured".to_string(), + )); + } + + let registry_url = npm_registry.unwrap(); + let package_url = format_registry_url(®istry_url, &package, None, None); + + tracing::info!("Fetching package file from: {}", package_url); + + let response = HTTP_CLIENT + .get(&package_url) + .send() + .await + .map_err(|e| Error::InternalErr(format!("Failed to fetch package metadata: {}", e)))?; + + if !response.status().is_success() { + return Err(Error::NotFound(format!( + "Package {} not found in private registry", + package + ))); + } + + let package_json: JsonValue = response + .json() + .await + .map_err(|e| Error::InternalErr(format!("Failed to parse package metadata: {}", e)))?; + + // Get the tarball URL for this version + let tarball_url = package_json + .get("versions") + .and_then(|v| v.get(&version)) + .and_then(|v| v.get("dist")) + .and_then(|d| d.get("tarball")) + .and_then(|t| t.as_str()) + .ok_or_else(|| Error::NotFound(format!("Tarball not found for {}@{}", package, version)))?; + + // Download tarball + let tarball_response = HTTP_CLIENT + .get(tarball_url) + .send() + .await + .map_err(|e| Error::InternalErr(format!("Failed to download tarball: {}", e)))?; + + if !tarball_response.status().is_success() { + return Err(Error::NotFound(format!( + "Failed to download tarball for {}@{}", + package, version + ))); + } + + let tarball_bytes = tarball_response + .bytes() + .await + .map_err(|e| Error::InternalErr(format!("Failed to read tarball: {}", e)))?; + + // Extract the specific file from the tarball + let file_content = extract_file_from_tarball(&tarball_bytes, &filepath)?; + + Ok(file_content) +} + +/// Get the npm registry URL from global settings +async fn get_npm_registry(db: &sqlx::Pool) -> Result> { + let registry = load_value_from_global_settings(db, NPM_CONFIG_REGISTRY_SETTING) + .await? + .and_then(|v| v.as_str().map(|s| s.to_string())); + + Ok(registry) +} + +/// Format a registry URL for a package +fn format_registry_url( + registry_base: &str, + package: &str, + version: Option<&str>, + file: Option<&str>, +) -> String { + let registry_base = registry_base.trim_end_matches('/'); + + // Handle scoped packages (e.g., @types/node) + let package_path = if package.starts_with('@') { + // Scoped packages need to be URL encoded properly + package.replace('/', "%2F") + } else { + package.to_string() + }; + + match (version, file) { + (Some(v), Some(f)) => { + format!("{}/{}/{}/-/{}-{}/{}", registry_base, package_path, v, package, v, f) + } + (Some(v), None) => { + format!("{}/{}/{}", registry_base, package_path, v) + } + _ => { + format!("{}/{}", registry_base, package_path) + } + } +} + +/// Extract file list from a tarball +fn extract_tarball_files(tarball_bytes: &[u8]) -> Result> { + use flate2::read::GzDecoder; + use tar::Archive; + + let gz = GzDecoder::new(tarball_bytes); + let mut archive = Archive::new(gz); + + let mut files = Vec::new(); + + for entry in archive + .entries() + .map_err(|e| Error::InternalErr(format!("Failed to read tarball entries: {}", e)))? + { + let entry = entry + .map_err(|e| Error::InternalErr(format!("Failed to read tarball entry: {}", e)))?; + let path = entry + .path() + .map_err(|e| Error::InternalErr(format!("Failed to read entry path: {}", e)))?; + + // Remove the package/ prefix that npm tarballs have + let path_str = path.to_string_lossy().to_string(); + if let Some(stripped) = path_str.strip_prefix("package/") { + files.push(FileEntry { + name: format!("/{}", stripped), + }); + } + } + + Ok(files) +} + +/// Extract a specific file from a tarball +fn extract_file_from_tarball(tarball_bytes: &[u8], target_file: &str) -> Result { + use flate2::read::GzDecoder; + use std::io::Read; + use tar::Archive; + + let gz = GzDecoder::new(tarball_bytes); + let mut archive = Archive::new(gz); + + // Normalize the target file path + let target = target_file.trim_start_matches('/'); + + for entry in archive + .entries() + .map_err(|e| Error::InternalErr(format!("Failed to read tarball entries: {}", e)))? + { + let mut entry = entry + .map_err(|e| Error::InternalErr(format!("Failed to read tarball entry: {}", e)))?; + let path = entry + .path() + .map_err(|e| Error::InternalErr(format!("Failed to read entry path: {}", e)))?; + + let path_str = path.to_string_lossy().to_string(); + + // Check if this is the file we're looking for + if let Some(stripped) = path_str.strip_prefix("package/") { + if stripped == target { + let mut content = String::new(); + entry + .read_to_string(&mut content) + .map_err(|e| Error::InternalErr(format!("Failed to read file content: {}", e)))?; + return Ok(content); + } + } + } + + Err(Error::NotFound(format!("File {} not found in tarball", target_file))) +} diff --git a/backend/windmill-api/src/resources.rs b/backend/windmill-api/src/resources.rs index f1f9c471ec..37c34e8feb 100644 --- a/backend/windmill-api/src/resources.rs +++ b/backend/windmill-api/src/resources.rs @@ -10,6 +10,7 @@ use std::collections::HashMap; use crate::{ db::{ApiAuthed, DB}, + secret_backend_ext::rename_vault_secret, users::{maybe_refresh_folders, require_owner_of_path, Tokened}, utils::{check_scopes, require_super_admin, BulkDeleteRequest}, var_resource_cache::{cache_resource, get_cached_resource}, @@ -947,12 +948,40 @@ async fn update_resource( let mut tx = user_db.begin(&authed).await?; - if let Some(npath) = ns.path { + if let Some(npath) = ns.path.clone() { if npath != path { check_path_conflict(&mut tx, &w_id, &npath).await?; require_owner_of_path(&authed, path)?; + // Handle Vault secret rename if the linked variable is a Vault-stored secret + let linked_var = sqlx::query!( + "SELECT value, is_secret FROM variable WHERE path = $1 AND workspace_id = $2", + path, + w_id + ) + .fetch_optional(&mut *tx) + .await?; + + if let Some(var) = linked_var { + if var.is_secret { + // Check if this is a Vault-stored secret and rename it + if let Some(new_value) = + rename_vault_secret(&db, &w_id, path, &npath, &var.value).await? + { + // Update the variable's value to point to the new Vault path + sqlx::query!( + "UPDATE variable SET value = $1 WHERE path = $2 AND workspace_id = $3", + new_value, + path, + w_id + ) + .execute(&mut *tx) + .await?; + } + } + } + sqlx::query!( "UPDATE variable SET path = $1 WHERE path = $2 AND workspace_id = $3", npath, diff --git a/backend/windmill-api/src/secret_backend_ext.rs b/backend/windmill-api/src/secret_backend_ext.rs new file mode 100644 index 0000000000..41d2267a94 --- /dev/null +++ b/backend/windmill-api/src/secret_backend_ext.rs @@ -0,0 +1,408 @@ +/* + * Author: Ruben Fiszel + * Copyright: Windmill Labs, Inc 2024 + * This file and its contents are licensed under the AGPLv3 License. + * Please see the included NOTICE for copyright information and + * LICENSE-AGPL for a copy of the license. + */ + +//! Secret backend extension for the API layer +//! +//! This module provides helper functions for integrating the SecretBackend +//! trait with variable operations in the API. +//! +//! Note: HashiCorp Vault integration requires Enterprise Edition. +//! The OSS version only supports the database backend. + +use std::sync::Arc; + +use windmill_common::{ + db::DB, + error::{Error, Result}, + secret_backend::{database::DatabaseBackend, SecretBackend}, + variables::{build_crypt, decrypt, encrypt}, +}; + +#[cfg(all(feature = "private", feature = "enterprise"))] +use windmill_common::{ + global_settings::{load_value_from_global_settings, SECRET_BACKEND_SETTING}, + secret_backend::{SecretBackendConfig, VaultBackend, VaultSettings}, +}; + +#[cfg(all(feature = "private", feature = "enterprise"))] +use tokio::sync::RwLock; + +// Cached Vault backend to avoid recreating it for every request +// This enables connection pooling and avoids repeated setup overhead +#[cfg(all(feature = "private", feature = "enterprise"))] +struct CachedVaultBackend { + backend: Arc, + settings: VaultSettings, +} + +#[cfg(all(feature = "private", feature = "enterprise"))] +lazy_static::lazy_static! { + static ref VAULT_BACKEND_CACHE: RwLock> = RwLock::new(None); +} + +/// Get the current secret backend based on global settings +/// +/// OSS: Always returns DatabaseBackend +/// EE: Returns configured backend (Database or Vault) +#[cfg(not(all(feature = "private", feature = "enterprise")))] +pub async fn get_secret_backend(db: &DB) -> Result> { + Ok(Arc::new(DatabaseBackend::new(db.clone()))) +} + +#[cfg(all(feature = "private", feature = "enterprise"))] +pub async fn get_secret_backend(db: &DB) -> Result> { + let config = match load_value_from_global_settings(db, SECRET_BACKEND_SETTING).await? { + Some(value) => serde_json::from_value::(value).unwrap_or_default(), + None => SecretBackendConfig::default(), + }; + + match config { + SecretBackendConfig::Database => Ok(Arc::new(DatabaseBackend::new(db.clone()))), + SecretBackendConfig::HashiCorpVault(settings) => { + get_or_create_vault_backend(db, settings).await + } + } +} + +/// Get a cached Vault backend or create a new one if settings changed +#[cfg(all(feature = "private", feature = "enterprise"))] +async fn get_or_create_vault_backend( + _db: &DB, + settings: VaultSettings, +) -> Result> { + // Check if we have a cached backend with matching settings (read lock) + { + let cache = VAULT_BACKEND_CACHE.read().await; + if let Some(ref cached) = *cache { + if cached.settings == settings { + return Ok(cached.backend.clone()); + } + } + } + + // Need to create a new backend - acquire write lock + let mut cache = VAULT_BACKEND_CACHE.write().await; + + // Double-check (another task may have created it while we waited) + if let Some(ref cached) = *cache { + if cached.settings == settings { + return Ok(cached.backend.clone()); + } + } + + // Create new backend + let backend: Arc = { + #[cfg(feature = "openidconnect")] + if settings.token.is_none() { + Arc::new(VaultBackend::new_with_db(settings.clone(), _db.clone())) + } else { + Arc::new(VaultBackend::new(settings.clone())) + } + + #[cfg(not(feature = "openidconnect"))] + Arc::new(VaultBackend::new(settings.clone())) + }; + + // Cache it + *cache = Some(CachedVaultBackend { + backend: backend.clone(), + settings, + }); + + Ok(backend) +} + +/// Check if a Vault backend is currently configured +/// +/// OSS: Always returns false +/// EE: Checks global settings +#[cfg(not(all(feature = "private", feature = "enterprise")))] +pub async fn is_vault_backend_configured(_db: &DB) -> Result { + Ok(false) +} + +#[cfg(all(feature = "private", feature = "enterprise"))] +pub async fn is_vault_backend_configured(db: &DB) -> Result { + let config = match load_value_from_global_settings(db, SECRET_BACKEND_SETTING).await? { + Some(value) => serde_json::from_value::(value).unwrap_or_default(), + None => SecretBackendConfig::default(), + }; + + Ok(matches!(config, SecretBackendConfig::HashiCorpVault(_))) +} + +/// Get a secret value using the configured backend +/// +/// For database backend: decrypts using workspace key +/// For vault backend (EE only): fetches from Vault directly +pub async fn get_secret_value( + db: &DB, + workspace_id: &str, + path: &str, + encrypted_value: &str, +) -> Result { + let backend = get_secret_backend(db).await?; + + match backend.backend_name() { + "database" => { + // Use existing database decryption + let mc = build_crypt(db, workspace_id).await?; + decrypt(&mc, encrypted_value.to_string()).map_err(|e| { + Error::internal_err(format!("Error decrypting variable {}: {}", path, e)) + }) + } + "hashicorp_vault" => { + // Fetch from Vault directly + backend.get_secret(workspace_id, path).await + } + _ => Err(Error::internal_err(format!( + "Unknown backend: {}", + backend.backend_name() + ))), + } +} + +/// Store a secret value using the configured backend +/// +/// For database backend: encrypts using workspace key and returns encrypted value +/// For vault backend (EE only): stores in Vault and returns a placeholder for DB storage +pub async fn store_secret_value( + db: &DB, + workspace_id: &str, + path: &str, + plain_value: &str, +) -> Result { + let backend = get_secret_backend(db).await?; + + match backend.backend_name() { + "database" => { + // Use existing database encryption + let mc = build_crypt(db, workspace_id).await?; + Ok(encrypt(&mc, plain_value)) + } + "hashicorp_vault" => { + // Store in Vault and return a marker for DB + backend.set_secret(workspace_id, path, plain_value).await?; + // Return a marker indicating the value is stored in Vault + // The actual value in the DB will be this marker + Ok(format!("$vault:{}", path)) + } + _ => Err(Error::internal_err(format!( + "Unknown backend: {}", + backend.backend_name() + ))), + } +} + +/// Delete a secret from the configured backend (if using Vault) +/// +/// For database backend: no-op (DB delete is handled separately) +/// For vault backend (EE only): deletes from Vault +pub async fn delete_secret_from_backend( + db: &DB, + workspace_id: &str, + path: &str, +) -> Result<()> { + if is_vault_backend_configured(db).await? { + let backend = get_secret_backend(db).await?; + // Ignore NotFound errors during deletion (secret might not exist in Vault) + match backend.delete_secret(workspace_id, path).await { + Ok(()) => Ok(()), + Err(Error::NotFound(_)) => Ok(()), + Err(e) => Err(e), + } + } else { + Ok(()) + } +} + +/// Check if a value is stored in Vault (indicated by the $vault: prefix) +pub fn is_vault_stored_value(value: &str) -> bool { + value.starts_with("$vault:") +} + +/// Rename a secret in Vault when a variable path changes (EE only) +/// +/// This function: +/// 1. Reads the secret value from the old path +/// 2. Writes it to the new path +/// 3. Deletes from the old path +/// 4. Returns the new marker value ($vault:new_path) +/// +/// If the value is not a Vault-stored value, returns None (no action needed). +/// If Vault is not configured, returns None. +#[cfg(not(all(feature = "private", feature = "enterprise")))] +pub async fn rename_vault_secret( + _db: &DB, + _workspace_id: &str, + _old_path: &str, + new_path: &str, + current_value: &str, +) -> Result> { + // OSS: If value has $vault: prefix, just update the reference + // (This handles edge case where EE was used before downgrading to OSS) + if is_vault_stored_value(current_value) { + tracing::warn!( + "Variable has $vault: prefix but Vault requires Enterprise Edition. \ + Updating DB reference to {}", + new_path + ); + return Ok(Some(format!("$vault:{}", new_path))); + } + Ok(None) +} + +#[cfg(all(feature = "private", feature = "enterprise"))] +pub async fn rename_vault_secret( + db: &DB, + workspace_id: &str, + old_path: &str, + new_path: &str, + current_value: &str, +) -> Result> { + // Only handle Vault-stored values + if !is_vault_stored_value(current_value) { + return Ok(None); + } + + // Check if Vault backend is configured + if !is_vault_backend_configured(db).await? { + // Vault not configured but value has $vault: prefix - this is an inconsistent state + // Log warning and return new marker to at least update the DB reference + tracing::warn!( + "Variable value has $vault: prefix but Vault is not configured. \ + Updating DB reference from {} to {}", + old_path, + new_path + ); + return Ok(Some(format!("$vault:{}", new_path))); + } + + let backend = get_secret_backend(db).await?; + + // Read from old path + let secret_value = match backend.get_secret(workspace_id, old_path).await { + Ok(value) => value, + Err(Error::NotFound(_)) => { + // Secret doesn't exist in Vault - just update the DB reference + tracing::warn!( + "Secret not found in Vault at path {} during rename to {}", + old_path, + new_path + ); + return Ok(Some(format!("$vault:{}", new_path))); + } + Err(e) => return Err(e), + }; + + // Write to new path + backend + .set_secret(workspace_id, new_path, &secret_value) + .await?; + + // Delete from old path (ignore errors - new path is already written) + if let Err(e) = backend.delete_secret(workspace_id, old_path).await { + tracing::warn!( + "Failed to delete old secret at {} after rename to {}: {}", + old_path, + new_path, + e + ); + } + + Ok(Some(format!("$vault:{}", new_path))) +} + +/// Bulk rename secrets in Vault when a path prefix changes (e.g., user rename) +/// EE only feature. +/// +/// This is used when renaming users where many secrets need their paths updated. +/// Returns a list of (old_path, new_value) pairs for updating the database. +#[cfg(not(all(feature = "private", feature = "enterprise")))] +pub async fn rename_vault_secrets_with_prefix( + _db: &DB, + _workspace_id: &str, + _old_prefix: &str, + _new_prefix: &str, + _variables: Vec<(String, String)>, +) -> Result> { + // OSS: No Vault support, return empty + Ok(vec![]) +} + +#[cfg(all(feature = "private", feature = "enterprise"))] +pub async fn rename_vault_secrets_with_prefix( + db: &DB, + workspace_id: &str, + old_prefix: &str, + new_prefix: &str, + variables: Vec<(String, String)>, // (path, value) pairs +) -> Result> { + // Only process if Vault is configured + if !is_vault_backend_configured(db).await? { + return Ok(vec![]); + } + + let backend = get_secret_backend(db).await?; + let mut updates = Vec::new(); + + for (old_path, value) in variables { + // Only handle Vault-stored values + if !is_vault_stored_value(&value) { + continue; + } + + // Calculate new path by replacing prefix + let new_path = if old_path.starts_with(old_prefix) { + format!("{}{}", new_prefix, &old_path[old_prefix.len()..]) + } else { + continue; // Path doesn't match prefix, skip + }; + + // Read from old path + let secret_value = match backend.get_secret(workspace_id, &old_path).await { + Ok(v) => v, + Err(Error::NotFound(_)) => { + // Just update DB reference + updates.push((old_path, format!("$vault:{}", new_path))); + continue; + } + Err(e) => { + tracing::error!( + "Failed to read secret at {} during bulk rename: {}", + old_path, + e + ); + continue; + } + }; + + // Write to new path + if let Err(e) = backend.set_secret(workspace_id, &new_path, &secret_value).await { + tracing::error!( + "Failed to write secret to {} during bulk rename: {}", + new_path, + e + ); + continue; + } + + // Delete from old path + if let Err(e) = backend.delete_secret(workspace_id, &old_path).await { + tracing::warn!( + "Failed to delete old secret at {} after rename: {}", + old_path, + e + ); + } + + updates.push((old_path, format!("$vault:{}", new_path))); + } + + Ok(updates) +} diff --git a/backend/windmill-api/src/settings.rs b/backend/windmill-api/src/settings.rs index 939ae9fa57..d6d747a142 100644 --- a/backend/windmill-api/src/settings.rs +++ b/backend/windmill-api/src/settings.rs @@ -42,6 +42,8 @@ use windmill_common::{ }, server::Smtp, }; +#[cfg(all(feature = "private", feature = "enterprise"))] +use windmill_common::secret_backend::{SecretMigrationReport, VaultSettings}; use windmill_common::{error::to_anyhow, PgDatabase}; pub fn global_service() -> Router { @@ -85,6 +87,16 @@ pub fn global_service() -> Router { post(acknowledge_all_critical_alerts), ); + // Vault integration routes (EE only - requires both private and enterprise features) + #[cfg(all(feature = "private", feature = "enterprise"))] + let r = r + .route("/test_secret_backend", post(test_secret_backend)) + .route("/migrate_secrets_to_vault", post(migrate_secrets_to_vault)) + .route( + "/migrate_secrets_to_database", + post(migrate_secrets_to_database), + ); + #[cfg(feature = "parquet")] { return r.route("/test_object_storage_config", post(test_s3_bucket)); @@ -798,3 +810,100 @@ async fn setup_custom_instance_pg_database_inner( Ok(()) } + +// ============================================================================ +// Secret Backend Settings (HashiCorp Vault Integration) - Enterprise Edition +// ============================================================================ + +/// Test connection to a secret backend (HashiCorp Vault) +/// +/// This endpoint validates that the Vault settings are correct and that +/// Windmill can successfully authenticate and communicate with Vault. +/// +/// This is an Enterprise Edition feature. +#[cfg(all(feature = "private", feature = "enterprise"))] +pub async fn test_secret_backend( + Extension(db): Extension, + authed: ApiAuthed, + Json(settings): Json, +) -> Result { + require_super_admin(&db, &authed.email).await?; + + windmill_common::secret_backend::test_vault_connection(&settings, Some(&db)).await?; + + Ok("Successfully connected to HashiCorp Vault".to_string()) +} + +/// Migrate existing secrets from database to HashiCorp Vault +/// +/// This endpoint reads all encrypted secrets from the database, decrypts them, +/// and stores them in HashiCorp Vault. The database values are NOT deleted +/// automatically to allow for rollback if needed. +/// +/// This is an Enterprise Edition feature. +#[cfg(all(feature = "private", feature = "enterprise"))] +pub async fn migrate_secrets_to_vault( + Extension(db): Extension, + authed: ApiAuthed, + Json(settings): Json, +) -> JsonResult { + require_super_admin(&db, &authed.email).await?; + + let report = windmill_common::secret_backend::migrate_secrets_to_vault(&db, &settings).await?; + + Ok(Json(report)) +} + +/// Migrate secrets from HashiCorp Vault back to database +/// +/// This endpoint reads all secrets from HashiCorp Vault, encrypts them using +/// the workspace encryption keys, and stores them in the database. The Vault +/// values are NOT deleted automatically to allow for rollback if needed. +/// +/// This is an Enterprise Edition feature. +#[cfg(all(feature = "private", feature = "enterprise"))] +pub async fn migrate_secrets_to_database( + Extension(db): Extension, + authed: ApiAuthed, + Json(settings): Json, +) -> JsonResult { + require_super_admin(&db, &authed.email).await?; + + let report = + windmill_common::secret_backend::migrate_secrets_to_database(&db, &settings).await?; + + Ok(Json(report)) +} + +// ============================================================================ +// JWKS Endpoint for Vault JWT Authentication +// ============================================================================ + +/// JSON Web Key Set response structure +#[derive(Serialize)] +pub struct JwksResponse { + pub keys: Vec, +} + +/// JWKS endpoint for HashiCorp Vault to validate JWTs +/// +/// Vault calls this endpoint to fetch the public keys used to verify +/// JWTs generated by Windmill for authentication. +/// +/// In the open-source version, this returns an empty JWKS. +/// The Enterprise Edition provides the actual key set. +pub async fn get_jwks() -> JsonResult { + // Open source version returns empty JWKS + // Enterprise Edition will override this with actual public keys + #[cfg(not(feature = "enterprise"))] + { + Ok(Json(JwksResponse { keys: vec![] })) + } + + #[cfg(feature = "enterprise")] + { + // In enterprise mode, the actual keys would be fetched from global settings + // For now, return empty - the EE implementation would override this + Ok(Json(JwksResponse { keys: vec![] })) + } +} diff --git a/backend/windmill-api/src/users.rs b/backend/windmill-api/src/users.rs index 5c03d54646..10cf4cfddd 100644 --- a/backend/windmill-api/src/users.rs +++ b/backend/windmill-api/src/users.rs @@ -19,6 +19,7 @@ use crate::db::ApiAuthed; pub use crate::auth::Tokened; +use crate::secret_backend_ext::rename_vault_secrets_with_prefix; use crate::utils::{ generate_instance_wide_unique_username, get_instance_username_or_create_pending, }; @@ -2661,6 +2662,7 @@ async fn rename_user( } update_username_in_workpsace( &mut tx, + &db, &user_email, &w_u.username, &ru.new_username, @@ -2688,6 +2690,7 @@ async fn rename_user( async fn update_username_in_workpsace<'c>( tx: &mut sqlx::Transaction<'c, sqlx::Postgres>, + db: &DB, email: &str, old_username: &str, new_username: &str, @@ -2805,6 +2808,43 @@ async fn update_username_in_workpsace<'c>( // ---- variables ---- + // Handle Vault secret renames before updating paths in DB + let old_prefix = format!("u/{}/", old_username); + let new_prefix = format!("u/{}/", new_username); + + // Fetch all Vault-stored secret variables under this user's path + let vault_secrets: Vec<(String, String)> = sqlx::query!( + r#"SELECT path, value FROM variable + WHERE path LIKE ('u/' || $1 || '/%') + AND workspace_id = $2 + AND is_secret = true + AND value LIKE '$vault:%'"#, + old_username, + w_id + ) + .fetch_all(&mut **tx) + .await? + .into_iter() + .map(|r| (r.path, r.value)) + .collect(); + + // Rename secrets in Vault and get the new values + let vault_updates = + rename_vault_secrets_with_prefix(db, w_id, &old_prefix, &new_prefix, vault_secrets).await?; + + // Update the values in the DB for renamed Vault secrets (using OLD path, before path update) + for (old_path, new_value) in vault_updates { + sqlx::query!( + "UPDATE variable SET value = $1 WHERE path = $2 AND workspace_id = $3", + new_value, + old_path, + w_id + ) + .execute(&mut **tx) + .await?; + } + + // Now update the paths in the database sqlx::query!( r#"UPDATE variable SET path = REGEXP_REPLACE(path,'u/' || $2 || '/(.*)','u/' || $1 || '/\1') WHERE path LIKE ('u/' || $2 || '/%') AND workspace_id = $3"#, new_username, diff --git a/backend/windmill-api/src/variables.rs b/backend/windmill-api/src/variables.rs index a5493cd2e5..54e68f9a9b 100644 --- a/backend/windmill-api/src/variables.rs +++ b/backend/windmill-api/src/variables.rs @@ -8,6 +8,10 @@ use crate::{ db::{ApiAuthed, DB}, + secret_backend_ext::{ + delete_secret_from_backend, get_secret_value, is_vault_stored_value, rename_vault_secret, + store_secret_value, + }, users::{maybe_refresh_folders, require_owner_of_path}, utils::{check_scopes, BulkDeleteRequest}, webhook_util::{WebhookMessage, WebhookShared}, @@ -39,7 +43,7 @@ use crate::var_resource_cache::{cache_variable, get_cached_variable}; use lazy_static::lazy_static; use serde::Deserialize; use sqlx::{Acquire, Postgres, Transaction}; -use windmill_common::variables::{decrypt, encrypt}; +use windmill_common::variables::encrypt; use windmill_git_sync::{handle_deployment_metadata, DeployedObject}; lazy_static! { @@ -210,13 +214,8 @@ async fn get_variable( return Err(Error::internal_err("Require oauth2 feature".to_string())); } else if !value.is_empty() && decrypt_secret { let _ = tx.commit().await; - let mc = build_crypt(&db, &w_id).await?; - Some(decrypt(&mc, value).map_err(|e| { - Error::internal_err(format!( - "Error decrypting variable {}: {}", - variable.path, e - )) - })?) + // Use secret backend for decryption (supports both DB and Vault) + Some(get_secret_value(&db, &w_id, &variable.path, &value).await?) } else if q.include_encrypted.unwrap_or(false) { Some(value) } else { @@ -355,8 +354,8 @@ async fn create_variable( check_path_conflict(&db, &w_id, &variable.path).await?; let value = if variable.is_secret && !already_encrypted.unwrap_or(false) { - let mc = build_crypt(&db, &w_id).await?; - encrypt(&mc, &variable.value) + // Use secret backend for encryption (supports both DB and Vault) + store_secret_value(&db, &w_id, &variable.path, &variable.value).await? } else { variable.value }; @@ -436,6 +435,16 @@ async fn delete_variable( check_scopes(&authed, || format!("variables:write:{}", path))?; + // Check if variable is a secret before deleting (for Vault cleanup) + let is_secret = sqlx::query_scalar!( + "SELECT is_secret FROM variable WHERE path = $1 AND workspace_id = $2", + path, + &w_id + ) + .fetch_optional(&db) + .await? + .unwrap_or(false); + let mut tx = user_db.begin(&authed).await?; sqlx::query!( @@ -465,6 +474,11 @@ async fn delete_variable( tx.commit().await?; + // If variable was a secret, also delete from Vault backend (if configured) + if is_secret { + delete_secret_from_backend(&db, &w_id, path).await?; + } + handle_deployment_metadata( &authed.email, &authed.username, @@ -496,6 +510,15 @@ async fn delete_variables_bulk( check_scopes(&authed, || format!("variables:write:{}", path))?; } + // Query which paths are secrets before deletion (for Vault cleanup) + let secret_paths: Vec = sqlx::query_scalar!( + "SELECT path FROM variable WHERE path = ANY($1) AND workspace_id = $2 AND is_secret = true", + &request.paths, + &w_id + ) + .fetch_all(&db) + .await?; + let mut tx = user_db.begin(&authed).await?; let deleted_paths = sqlx::query_scalar!( @@ -526,6 +549,13 @@ async fn delete_variables_bulk( tx.commit().await?; + // Delete secrets from Vault backend (if configured) + for path in &secret_paths { + if deleted_paths.contains(path) { + delete_secret_from_backend(&db, &w_id, path).await?; + } + } + try_join_all(deleted_paths.iter().map(|path| { handle_deployment_metadata( &authed.email, @@ -589,7 +619,9 @@ async fn update_variable( sqlb.set_str("path", npath); } let ns_value_is_none = ns.value.is_none(); - if let Some(nvalue) = ns.value { + // Determine the target path for storing secrets (use new path if provided) + let target_path = ns.path.as_deref().unwrap_or(path); + if let Some(nvalue) = ns.value.clone() { let is_secret = if ns.is_secret.is_some() { ns.is_secret.unwrap() } else { @@ -604,8 +636,9 @@ async fn update_variable( }; let value = if is_secret && !already_encrypted.unwrap_or(false) { - let mc = build_crypt(&db, &w_id).await?; - encrypt(&mc, &nvalue) + // Use secret backend for encryption (supports both DB and Vault) + // Store at target_path (new path if renaming, otherwise current path) + store_secret_value(&db, &w_id, target_path, &nvalue).await? } else { nvalue }; @@ -654,11 +687,38 @@ async fn update_variable( let mut tx: Transaction<'_, Postgres> = user_db.begin(&authed).await?; - if let Some(npath) = ns.path { + if let Some(npath) = ns.path.clone() { if npath != path { check_path_conflict(&db, &w_id, &npath).await?; require_owner_of_path(&authed, path)?; + // Handle Vault secret rename if the variable is a secret stored in Vault + let current_var = sqlx::query!( + "SELECT value, is_secret FROM variable WHERE path = $1 AND workspace_id = $2", + path, + w_id + ) + .fetch_optional(&mut *tx) + .await?; + + if let Some(var) = current_var { + if var.is_secret && is_vault_stored_value(&var.value) { + if ns.value.is_some() { + // New value was provided and already stored at new path + // Just delete the old secret from Vault + delete_secret_from_backend(&db, &w_id, path).await?; + } else { + // No new value - rename the secret in Vault + if let Some(new_value) = + rename_vault_secret(&db, &w_id, path, &npath, &var.value).await? + { + // Update the variable's value to point to the new Vault path + sqlb.set_str("value", &new_value); + } + } + } + } + let mut v = sqlx::query_scalar!( "SELECT value FROM resource WHERE path = $1 AND workspace_id = $2", path, @@ -835,13 +895,8 @@ pub async fn get_value_internal<'a>( #[cfg(not(feature = "oauth2"))] return Err(Error::internal_err("Require oauth2 feature".to_string())); } else if !value.is_empty() { - let mc = build_crypt(db_with_opt_authed.db(), &w_id).await?; - decrypt(&mc, value).map_err(|e| { - Error::internal_err(format!( - "Error decrypting variable {}: {}", - variable.path, e - )) - })? + // Use secret backend for decryption (supports both DB and Vault) + get_secret_value(db_with_opt_authed.db(), &w_id, &variable.path, &value).await? } else { "".to_string() } diff --git a/backend/windmill-api/src/workspaces.rs b/backend/windmill-api/src/workspaces.rs index bf42f3589f..538e239982 100644 --- a/backend/windmill-api/src/workspaces.rs +++ b/backend/windmill-api/src/workspaces.rs @@ -127,6 +127,7 @@ pub fn workspaced_service() -> Router { .route("/edit_copilot_config", post(edit_copilot_config)) .route("/get_copilot_info", get(get_copilot_info)) .route("/edit_error_handler", post(edit_error_handler)) + .route("/edit_success_handler", post(edit_success_handler)) .route( "/edit_large_file_storage_config", post(edit_large_file_storage_config), @@ -270,6 +271,10 @@ pub struct WorkspaceSettings { #[serde(skip_serializing_if = "Option::is_none")] pub error_handler_muted_on_cancel: Option, #[serde(skip_serializing_if = "Option::is_none")] + pub success_handler: Option, + #[serde(skip_serializing_if = "Option::is_none")] + pub success_handler_extra_args: Option, + #[serde(skip_serializing_if = "Option::is_none")] pub large_file_storage: Option, // effectively: DatasetsStorage #[serde(skip_serializing_if = "Option::is_none")] pub ducklake: Option, @@ -442,6 +447,12 @@ pub struct EditErrorHandler { pub error_handler_muted_on_cancel: Option, } +#[derive(Deserialize)] +pub struct EditSuccessHandler { + pub success_handler: Option, + pub success_handler_extra_args: Option, +} + lazy_static::lazy_static! { pub static ref EMAIL_REGEXP: Regex = Regex::new(r"^[a-zA-Z0-9._%+-]+@[a-zA-Z0-9.-]+\.[a-zA-Z]{2,}$").unwrap(); } @@ -555,6 +566,8 @@ async fn get_settings( error_handler, error_handler_extra_args, error_handler_muted_on_cancel, + success_handler, + success_handler_extra_args, large_file_storage, datatable, ducklake, @@ -2251,6 +2264,88 @@ async fn edit_error_handler( Ok(format!("Edit error_handler for workspace {}", &w_id)) } +async fn edit_success_handler( + authed: ApiAuthed, + Extension(db): Extension, + Path(w_id): Path, + Json(es): Json, +) -> Result { + require_admin(authed.is_admin, &authed.username)?; + + let mut tx = db.begin().await?; + + sqlx::query_as!( + Group, + "INSERT INTO group_ (workspace_id, name, summary, extra_perms) VALUES ($1, $2, $3, $4) ON CONFLICT DO NOTHING", + w_id, + "success_handler", + "The group the success handler acts on behalf of", + serde_json::json!({username_to_permissioned_as(&authed.username): true}) + ) + .execute(&mut *tx) + .await?; + + if let Some(success_handler) = &es.success_handler { + sqlx::query!( + r#" + UPDATE + workspace_settings + SET + success_handler = $1, + success_handler_extra_args = $2 + WHERE + workspace_id = $3 + "#, + success_handler, + es.success_handler_extra_args, + &w_id + ) + .execute(&mut *tx) + .await?; + } else { + sqlx::query!( + r#" + UPDATE + workspace_settings + SET + success_handler = NULL, + success_handler_extra_args = NULL + WHERE + workspace_id = $1 + "#, + &w_id + ) + .execute(&mut *tx) + .await?; + } + + audit_log( + &mut *tx, + &authed, + "workspaces.edit_success_handler", + ActionKind::Update, + &w_id, + Some(&authed.email), + Some([("success_handler", &format!("{:?}", es.success_handler)[..])].into()), + ) + .await?; + tx.commit().await?; + + // Trigger git sync for success handler changes + handle_deployment_metadata( + &authed.email, + &authed.username, + &db, + &w_id, + windmill_git_sync::DeployedObject::Settings { setting_type: "success_handler".to_string() }, + Some("Success handler configuration updated".to_string()), + false, + ) + .await?; + + Ok(format!("Edit success_handler for workspace {}", &w_id)) +} + #[derive(Deserialize)] struct NewEnvironmentVariable { name: String, diff --git a/backend/windmill-common/src/global_settings.rs b/backend/windmill-common/src/global_settings.rs index a73f488aae..c5428e5c5c 100644 --- a/backend/windmill-common/src/global_settings.rs +++ b/backend/windmill-common/src/global_settings.rs @@ -47,6 +47,7 @@ pub const JWT_SECRET_SETTING: &str = "jwt_secret"; pub const EMAIL_DOMAIN_SETTING: &str = "email_domain"; pub const OTEL_SETTING: &str = "otel"; pub const APP_WORKSPACED_ROUTE_SETTING: &str = "app_workspaced_route"; +pub const SECRET_BACKEND_SETTING: &str = "secret_backend"; pub const ENV_SETTINGS: &[&str] = &[ "DISABLE_NSJAIL", @@ -115,6 +116,8 @@ pub const ENV_SETTINGS: &[&str] = &[ "PG_SCHEMA", "PG_LISTENER_REFRESH_PERIOD_SECS", "AI_REQUEST_TIMEOUT_SECONDS", + "JOB_CLEANUP_BATCH_SIZE", + "JOB_CLEANUP_MAX_BATCHES", ]; use crate::error; diff --git a/backend/windmill-common/src/lib.rs b/backend/windmill-common/src/lib.rs index a650f6bf0c..c61f4263cc 100644 --- a/backend/windmill-common/src/lib.rs +++ b/backend/windmill-common/src/lib.rs @@ -78,6 +78,7 @@ pub mod result_stream; pub mod runnable_settings; pub mod s3_helpers; pub mod schedule; +pub mod secret_backend; pub mod schema; pub mod scripts; pub mod server; diff --git a/backend/windmill-common/src/secret_backend/database.rs b/backend/windmill-common/src/secret_backend/database.rs new file mode 100644 index 0000000000..3dee114481 --- /dev/null +++ b/backend/windmill-common/src/secret_backend/database.rs @@ -0,0 +1,126 @@ +/* + * Author: Ruben Fiszel + * Copyright: Windmill Labs, Inc 2024 + * This file and its contents are licensed under the AGPLv3 License. + * Please see the included NOTICE for copyright information and + * LICENSE-AGPL for a copy of the license. + */ + +//! Database backend for secret storage +//! +//! This is the default backend that stores secrets encrypted in the PostgreSQL database +//! using the existing magic_crypt encryption with workspace-specific keys. + +use async_trait::async_trait; + +use crate::db::DB; +use crate::error::{Error, Result}; +use crate::variables::{build_crypt, decrypt, encrypt}; + +use super::SecretBackend; + +/// Database-backed secret storage +/// +/// This backend stores secrets encrypted in the `variable` table using +/// the workspace's encryption key. This is the default and original +/// behavior of Windmill. +pub struct DatabaseBackend { + db: DB, +} + +impl DatabaseBackend { + /// Create a new database backend + pub fn new(db: DB) -> Self { + Self { db } + } +} + +#[async_trait] +impl SecretBackend for DatabaseBackend { + async fn get_secret(&self, workspace_id: &str, path: &str) -> Result { + let variable = sqlx::query!( + "SELECT value FROM variable WHERE path = $1 AND workspace_id = $2 AND is_secret = true", + path, + workspace_id + ) + .fetch_optional(&self.db) + .await?; + + let variable = variable.ok_or_else(|| { + Error::NotFound(format!( + "Secret variable {} not found in workspace {}", + path, workspace_id + )) + })?; + + let value = variable.value; + if value.is_empty() { + return Ok(String::new()); + } + + let mc = build_crypt(&self.db, workspace_id).await?; + decrypt(&mc, value).map_err(|e| { + Error::internal_err(format!("Error decrypting variable {}: {}", path, e)) + }) + } + + async fn set_secret(&self, workspace_id: &str, path: &str, value: &str) -> Result<()> { + let mc = build_crypt(&self.db, workspace_id).await?; + let encrypted_value = encrypt(&mc, value); + + // Update the value in the database + // Note: This assumes the variable row already exists (created via the normal API) + let result = sqlx::query!( + "UPDATE variable SET value = $1 WHERE path = $2 AND workspace_id = $3 AND is_secret = true", + encrypted_value, + path, + workspace_id + ) + .execute(&self.db) + .await?; + + if result.rows_affected() == 0 { + return Err(Error::NotFound(format!( + "Secret variable {} not found in workspace {}", + path, workspace_id + ))); + } + + Ok(()) + } + + async fn delete_secret(&self, _workspace_id: &str, _path: &str) -> Result<()> { + // For database backend, deletion is handled by the normal variable deletion flow + // The encrypted value is just deleted along with the row + // This method is a no-op for database backend since the caller handles the DELETE + Ok(()) + } + + fn backend_name(&self) -> &'static str { + "database" + } +} + +/// Encrypt a value for storage in the database +/// +/// This is a convenience function for use when creating new secrets. +pub async fn encrypt_for_database(db: &DB, workspace_id: &str, value: &str) -> Result { + let mc = build_crypt(db, workspace_id).await?; + Ok(encrypt(&mc, value)) +} + +/// Decrypt a value from the database +/// +/// This is a convenience function for reading secrets. +pub async fn decrypt_from_database( + db: &DB, + workspace_id: &str, + encrypted_value: String, +) -> Result { + if encrypted_value.is_empty() { + return Ok(String::new()); + } + let mc = build_crypt(db, workspace_id).await?; + decrypt(&mc, encrypted_value) + .map_err(|e| Error::internal_err(format!("Error decrypting value: {}", e))) +} diff --git a/backend/windmill-common/src/secret_backend/mod.rs b/backend/windmill-common/src/secret_backend/mod.rs new file mode 100644 index 0000000000..772a5ebe6a --- /dev/null +++ b/backend/windmill-common/src/secret_backend/mod.rs @@ -0,0 +1,133 @@ +/* + * Author: Ruben Fiszel + * Copyright: Windmill Labs, Inc 2024 + * This file and its contents are licensed under the AGPLv3 License. + * Please see the included NOTICE for copyright information and + * LICENSE-AGPL for a copy of the license. + */ + +//! Secret Backend abstraction for storing secrets in external vaults +//! +//! This module provides a trait-based abstraction for secret storage, +//! allowing secrets to be stored in the database (default) or in external +//! vaults like HashiCorp Vault (Enterprise Edition). + +pub mod database; + +#[cfg(feature = "private")] +pub mod vault_ee; + +pub mod vault_oss; + +#[cfg(test)] +mod tests; + +#[cfg(feature = "private")] +pub use vault_ee::*; + +#[cfg(not(feature = "private"))] +pub use vault_oss::*; + +use async_trait::async_trait; +use serde::{Deserialize, Serialize}; + +use crate::error::Result; + +/// Trait for secret storage backends +/// +/// Implementations of this trait handle the storage and retrieval of secrets. +/// The default implementation stores secrets encrypted in the database. +/// Enterprise Edition supports HashiCorp Vault as an alternative backend. +#[async_trait] +pub trait SecretBackend: Send + Sync { + /// Retrieve a secret value + /// + /// # Arguments + /// * `workspace_id` - The workspace identifier + /// * `path` - The path/name of the secret variable + /// + /// # Returns + /// The decrypted secret value + async fn get_secret(&self, workspace_id: &str, path: &str) -> Result; + + /// Store a secret value + /// + /// # Arguments + /// * `workspace_id` - The workspace identifier + /// * `path` - The path/name of the secret variable + /// * `value` - The plaintext secret value to store + async fn set_secret(&self, workspace_id: &str, path: &str, value: &str) -> Result<()>; + + /// Delete a secret + /// + /// # Arguments + /// * `workspace_id` - The workspace identifier + /// * `path` - The path/name of the secret variable + async fn delete_secret(&self, workspace_id: &str, path: &str) -> Result<()>; + + /// Get the name of this backend for logging/debugging + fn backend_name(&self) -> &'static str; +} + +/// Configuration for secret storage backend +/// +/// This enum is stored in global_settings and determines which backend +/// is used for secret storage at the instance level. +#[derive(Debug, Clone, Serialize, Deserialize, PartialEq)] +#[serde(tag = "type")] +pub enum SecretBackendConfig { + /// Store secrets encrypted in the database (default behavior) + Database, + /// Store secrets in HashiCorp Vault (Enterprise Edition only) + HashiCorpVault(VaultSettings), +} + +impl Default for SecretBackendConfig { + fn default() -> Self { + SecretBackendConfig::Database + } +} + +/// Settings for HashiCorp Vault integration +#[derive(Debug, Clone, Serialize, Deserialize, PartialEq)] +pub struct VaultSettings { + /// Vault server address (e.g., "https://vault.company.com:8200") + pub address: String, + /// KV v2 mount path (e.g., "windmill") + pub mount_path: String, + /// JWT auth role name configured in Vault (used for JWT/OIDC auth) + /// Optional - if not provided, token auth is used + #[serde(skip_serializing_if = "Option::is_none")] + pub jwt_role: Option, + /// Vault Enterprise namespace (optional) + #[serde(skip_serializing_if = "Option::is_none")] + pub namespace: Option, + /// Static Vault token for testing/development (optional) + /// If provided, this is used instead of JWT authentication + #[serde(skip_serializing_if = "Option::is_none")] + pub token: Option, +} + +/// Result of a secret migration operation +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct SecretMigrationReport { + /// Total number of secrets found + pub total_secrets: usize, + /// Number of secrets successfully migrated + pub migrated_count: usize, + /// Number of secrets that failed to migrate + pub failed_count: usize, + /// Details of any failures + pub failures: Vec, +} + +/// Details of a failed secret migration +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct SecretMigrationFailure { + /// Workspace ID where the secret is located + pub workspace_id: String, + /// Path of the secret that failed to migrate + pub path: String, + /// Error message + pub error: String, +} diff --git a/backend/windmill-common/src/secret_backend/tests.rs b/backend/windmill-common/src/secret_backend/tests.rs new file mode 100644 index 0000000000..2a6962f8dd --- /dev/null +++ b/backend/windmill-common/src/secret_backend/tests.rs @@ -0,0 +1,143 @@ +/* + * Integration tests for secret backend + * + * These tests require a running Vault instance at http://127.0.0.1:8200 + * with the token "test-root-token" and a KV v2 mount at "windmill". + * + * Run Vault dev server: + * podman run -d --name vault-test --rm -p 8200:8200 \ + * -e 'VAULT_DEV_ROOT_TOKEN_ID=test-root-token' \ + * -e 'VAULT_DEV_LISTEN_ADDRESS=0.0.0.0:8200' \ + * docker.io/hashicorp/vault:latest + * + * Then enable the windmill mount: + * curl --header "X-Vault-Token: test-root-token" \ + * --request POST \ + * --data '{"type": "kv", "options": {"version": "2"}}' \ + * http://127.0.0.1:8200/v1/sys/mounts/windmill + */ + +#[cfg(test)] +mod tests { + use crate::secret_backend::{SecretBackend, VaultBackend, VaultSettings}; + + fn test_settings() -> VaultSettings { + VaultSettings { + address: "http://127.0.0.1:8200".to_string(), + mount_path: "windmill".to_string(), + jwt_role: Some("windmill-secrets".to_string()), + namespace: None, + token: Some("test-root-token".to_string()), + } + } + + #[tokio::test] + #[ignore] // Run with --ignored to execute + async fn test_vault_write_and_read() { + let settings = test_settings(); + let backend = VaultBackend::new(settings); + + let workspace_id = "test-ws-1"; + let path = "test-secret"; + let value = "my-super-secret-value"; + + // Write secret + backend + .set_secret(workspace_id, path, value) + .await + .expect("Failed to write secret"); + + // Read secret + let read_value = backend + .get_secret(workspace_id, path) + .await + .expect("Failed to read secret"); + + assert_eq!(read_value, value); + + // Delete secret + backend + .delete_secret(workspace_id, path) + .await + .expect("Failed to delete secret"); + + // Verify deleted + let result = backend.get_secret(workspace_id, path).await; + assert!(result.is_err(), "Secret should be deleted"); + } + + #[tokio::test] + #[ignore] + async fn test_vault_multiple_secrets() { + let settings = test_settings(); + let backend = VaultBackend::new(settings); + + let workspace_id = "test-ws-2"; + + // Write multiple secrets + for i in 0..5 { + let path = format!("secret-{}", i); + let value = format!("value-{}", i); + backend + .set_secret(workspace_id, &path, &value) + .await + .expect(&format!("Failed to write secret-{}", i)); + } + + // Read and verify + for i in 0..5 { + let path = format!("secret-{}", i); + let expected = format!("value-{}", i); + let value = backend + .get_secret(workspace_id, &path) + .await + .expect(&format!("Failed to read secret-{}", i)); + assert_eq!(value, expected); + } + + // Cleanup + for i in 0..5 { + let path = format!("secret-{}", i); + backend + .delete_secret(workspace_id, &path) + .await + .expect(&format!("Failed to delete secret-{}", i)); + } + } + + #[tokio::test] + #[ignore] + async fn test_vault_overwrite_secret() { + let settings = test_settings(); + let backend = VaultBackend::new(settings); + + let workspace_id = "test-ws-3"; + let path = "overwrite-test"; + + // Write initial value + backend + .set_secret(workspace_id, path, "initial-value") + .await + .expect("Failed to write initial value"); + + // Overwrite + backend + .set_secret(workspace_id, path, "new-value") + .await + .expect("Failed to overwrite"); + + // Read and verify + let value = backend + .get_secret(workspace_id, path) + .await + .expect("Failed to read"); + + assert_eq!(value, "new-value"); + + // Cleanup + backend + .delete_secret(workspace_id, path) + .await + .expect("Failed to delete"); + } +} diff --git a/backend/windmill-common/src/secret_backend/vault_oss.rs b/backend/windmill-common/src/secret_backend/vault_oss.rs new file mode 100644 index 0000000000..b338388312 --- /dev/null +++ b/backend/windmill-common/src/secret_backend/vault_oss.rs @@ -0,0 +1,107 @@ +/* + * Author: Ruben Fiszel + * Copyright: Windmill Labs, Inc 2024 + * This file and its contents are licensed under the AGPLv3 License. + * Please see the included NOTICE for copyright information and + * LICENSE-AGPL for a copy of the license. + */ + +//! HashiCorp Vault secret backend stubs (Open Source Edition) +//! +//! This module provides stub implementations for Vault integration. +//! The actual Vault integration requires Enterprise Edition. + +use std::sync::Arc; + +use crate::db::DB; +use crate::error::{Error, Result}; + +use super::{database::DatabaseBackend, SecretBackend, SecretBackendConfig, SecretMigrationReport, VaultSettings}; + +/// Stub VaultBackend for OSS - all operations return EE required error +pub struct VaultBackend; + +impl VaultBackend { + pub fn new(_settings: VaultSettings) -> Self { + Self + } +} + +#[async_trait::async_trait] +impl SecretBackend for VaultBackend { + async fn get_secret(&self, _workspace_id: &str, _path: &str) -> Result { + Err(Error::internal_err( + "HashiCorp Vault integration requires Enterprise Edition".to_string(), + )) + } + + async fn set_secret(&self, _workspace_id: &str, _path: &str, _value: &str) -> Result<()> { + Err(Error::internal_err( + "HashiCorp Vault integration requires Enterprise Edition".to_string(), + )) + } + + async fn delete_secret(&self, _workspace_id: &str, _path: &str) -> Result<()> { + Err(Error::internal_err( + "HashiCorp Vault integration requires Enterprise Edition".to_string(), + )) + } + + fn backend_name(&self) -> &'static str { + "hashicorp_vault" + } +} + +/// Create the appropriate secret backend based on configuration +/// +/// In OSS, always returns DatabaseBackend regardless of config. +/// Vault configuration is ignored with a warning. +pub async fn create_secret_backend( + db: DB, + config: &SecretBackendConfig, +) -> Result> { + match config { + SecretBackendConfig::Database => Ok(Arc::new(DatabaseBackend::new(db))), + SecretBackendConfig::HashiCorpVault(_) => { + tracing::warn!( + "HashiCorp Vault is configured but requires Enterprise Edition. \ + Falling back to database backend." + ); + Ok(Arc::new(DatabaseBackend::new(db))) + } + } +} + +/// Test connection to Vault (OSS stub) +pub async fn test_vault_connection(_settings: &VaultSettings, _db: Option<&DB>) -> Result<()> { + Err(Error::internal_err( + "HashiCorp Vault integration requires Enterprise Edition".to_string(), + )) +} + +/// Migrate secrets from database to Vault (OSS stub) +pub async fn migrate_secrets_to_vault( + _db: &DB, + _settings: &VaultSettings, +) -> Result { + Err(Error::internal_err( + "HashiCorp Vault integration requires Enterprise Edition".to_string(), + )) +} + +/// Migrate secrets from Vault back to database (OSS stub) +pub async fn migrate_secrets_to_database( + _db: &DB, + _settings: &VaultSettings, +) -> Result { + Err(Error::internal_err( + "HashiCorp Vault integration requires Enterprise Edition".to_string(), + )) +} + +/// Generate a JWT for Vault authentication (OSS stub) +pub async fn generate_vault_jwt(_db: &DB, _vault_address: &str) -> Result { + Err(Error::internal_err( + "HashiCorp Vault integration requires Enterprise Edition".to_string(), + )) +} diff --git a/backend/windmill-mcp/Cargo.toml b/backend/windmill-mcp/Cargo.toml index a3e4bfed5f..afe2547f19 100644 --- a/backend/windmill-mcp/Cargo.toml +++ b/backend/windmill-mcp/Cargo.toml @@ -10,7 +10,7 @@ path = "src/lib.rs" [features] default = [] -server = ["rmcp/transport-streamable-http-server", "rmcp/transport-streamable-http-server-session", "rmcp/transport-worker"] +server = ["rmcp/transport-streamable-http-server", "rmcp/transport-streamable-http-server-session", "rmcp/transport-worker", "dep:sqlx", "dep:async-trait", "dep:http", "dep:tokio-util", "dep:tokio"] auth = ["rmcp/auth", "dep:oauth2"] [dependencies] @@ -22,3 +22,9 @@ serde.workspace = true serde_json.workspace = true tracing.workspace = true rmcp.workspace = true +sqlx = { workspace = true, optional = true } +async-trait = { workspace = true, optional = true } +http = { workspace = true, optional = true } +tokio-util = { workspace = true, features = ["rt"], optional = true } +tokio = { workspace = true, optional = true } +futures.workspace = true diff --git a/backend/windmill-mcp/src/client/mod.rs b/backend/windmill-mcp/src/client/mod.rs new file mode 100644 index 0000000000..2364571486 --- /dev/null +++ b/backend/windmill-mcp/src/client/mod.rs @@ -0,0 +1,209 @@ +//! MCP Client implementation +//! +//! This module provides functionality for connecting to external MCP servers +//! and executing tools on them. + +mod types; + +pub use types::{McpResource, McpToolSource}; + +use anyhow::{Context, Result}; +use reqwest::header::{HeaderMap, HeaderName, HeaderValue}; +use rmcp::{ + model::{ + CallToolRequestParam, ClientCapabilities, ClientInfo, Implementation, + InitializeRequestParam, Tool as McpTool, + }, + service::RunningService, + transport::{ + streamable_http_client::StreamableHttpClientTransportConfig, StreamableHttpClientTransport, + }, + RoleClient, ServiceExt, +}; +use serde_json::{json, Value}; +use std::str::FromStr; +use windmill_common::variables::get_secret_value_as_admin; +use windmill_common::DB; + +/// MCP client for communicating with external MCP servers +pub struct McpClient { + /// The underlying rmcp client + client: RunningService, + /// Cached list of available tools from the server + available_tools: Vec, +} + +impl McpClient { + /// Create a new MCP client from a resource configuration + pub async fn from_resource(resource: McpResource, db: &DB, w_id: &str) -> Result { + // Build custom reqwest client with headers if provided + let mut headers = HeaderMap::new(); + if let Some(token_path) = &resource.token { + if !token_path.trim().is_empty() { + let value = + get_secret_value_as_admin(db, w_id, token_path.trim_start_matches("$var:")) + .await?; + headers.insert( + HeaderName::from_static("authorization"), + HeaderValue::from_str(format!("Bearer {}", value).as_str())?, + ); + } + } + if let Some(resource_headers) = &resource.headers { + for (key, value) in resource_headers { + match (HeaderName::from_str(key), HeaderValue::from_str(value)) { + (Ok(name), Ok(value)) => { + headers.insert(name, value); + } + _ => { + tracing::warn!("Invalid header: {}={}", key, value); + } + } + } + } + + let reqwest_client = reqwest::Client::builder() + .default_headers(headers) + .build() + .context("Failed to build HTTP client")?; + + // Create the HTTP transport with custom client + let config = StreamableHttpClientTransportConfig::with_uri(resource.url.as_str()); + let transport = StreamableHttpClientTransport::with_client(reqwest_client, config); + + // Set up client info + let client_info = ClientInfo { + protocol_version: Default::default(), + capabilities: ClientCapabilities::default(), + client_info: Implementation { + name: "windmill-ai-agent".to_string(), + title: Some("Windmill AI Agent".to_string()), + version: env!("CARGO_PKG_VERSION").to_string(), + website_url: None, + icons: None, + }, + }; + + // Initialize the connection + let client = client_info + .serve(transport) + .await + .context("Failed to connect to MCP server")?; + + // Immediately fetch available tools + let available_tools = client + .list_tools(Default::default()) + .await + .context("Failed to list tools from MCP server")? + .tools; + + Ok(Self { client, available_tools }) + } + + /// Get the list of available tools from the MCP server + pub fn available_tools(&self) -> &[McpTool] { + &self.available_tools + } + + /// Call a tool on the MCP server, with openai-style arguments + pub async fn call_tool(&self, name: &str, arguments: &str) -> Result { + // Convert OpenAI-style arguments to MCP format + let mcp_args = + Self::openai_args_to_mcp_args(arguments).context("Failed to parse tool arguments")?; + + let result = self + .client + .call_tool(CallToolRequestParam { + name: name.to_string().into(), + arguments: mcp_args, + task: None, + }) + .await + .context(format!("Failed to call MCP tool: {}", name))?; + + // Convert the result to a JSON value + // MCP tools return ToolResult which contains content array + let result_json = + serde_json::to_value(&result).context("Failed to serialize MCP tool result")?; + + Ok(result_json) + } + + /// Close the connection + pub async fn shutdown(self) -> Result<()> { + self.client.cancel().await?; + Ok(()) + } + + /// Fix array schemas to ensure they have the required 'items' property + /// OpenAI requires all array types to have an 'items' field. MCP servers may + /// return schemas without this field, so we add a default. + pub fn fix_array_schemas(schema: &mut Value) { + if let Value::Object(obj) = schema { + // Check if this is an array type + if let Some(type_val) = obj.get("type") { + let is_array = match type_val { + Value::String(s) => s == "array", + Value::Array(arr) => arr.iter().any(|v| v.as_str() == Some("array")), + _ => false, + }; + + // If it's an array and missing 'items', add a default + if is_array && !obj.contains_key("items") { + obj.insert("items".to_string(), json!({})); + } + } + + // Recursively fix nested schemas + if let Some(Value::Object(props)) = obj.get_mut("properties") { + for value in props.values_mut() { + Self::fix_array_schemas(value); + } + } + + // Fix items if present (for nested arrays) + if let Some(items) = obj.get_mut("items") { + Self::fix_array_schemas(items); + } + + // Fix oneOf, anyOf, allOf schemas + for key in &["oneOf", "anyOf", "allOf"] { + if let Some(Value::Array(schemas)) = obj.get_mut(*key) { + for schema in schemas { + Self::fix_array_schemas(schema); + } + } + } + + // Fix additionalProperties if it's a schema + if let Some(additional) = obj.get_mut("additionalProperties") { + if additional.is_object() { + Self::fix_array_schemas(additional); + } + } + } + } + + /// Convert OpenAI-style tool call arguments to MCP format + /// OpenAI sends arguments as a JSON string, MCP expects a Map + fn openai_args_to_mcp_args( + args_str: &str, + ) -> Result>> { + if args_str.trim().is_empty() { + return Ok(None); + } + + let args_value: serde_json::Value = + serde_json::from_str(args_str).context("Failed to parse tool call arguments")?; + + match args_value { + serde_json::Value::Object(map) => Ok(Some(map)), + serde_json::Value::Null => Ok(None), + _ => Ok(Some( + vec![("value".to_string(), args_value)] + .into_iter() + .collect(), + )), + } + } +} diff --git a/backend/windmill-mcp/src/client/types.rs b/backend/windmill-mcp/src/client/types.rs new file mode 100644 index 0000000000..232f2a5a9a --- /dev/null +++ b/backend/windmill-mcp/src/client/types.rs @@ -0,0 +1,32 @@ +//! Client-specific types for MCP +//! +//! Contains configuration and metadata types used for MCP client connections. + +use serde::{Deserialize, Serialize}; +use std::collections::HashMap; + +/// MCP server resource configuration +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct McpResource { + /// Name of the MCP resource (used for prefixing tools) + pub name: String, + /// HTTP URL for the MCP server endpoint + pub url: String, + /// Optional token for authentication + #[serde(skip_serializing_if = "Option::is_none")] + pub token: Option, + /// Optional headers + #[serde(skip_serializing_if = "Option::is_none")] + pub headers: Option>, +} + +/// Metadata for tracking MCP tool sources +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct McpToolSource { + /// Name of the MCP resource this tool comes from + pub name: String, + /// Original tool name in the MCP server + pub tool_name: String, + /// Path of the MCP resource + pub resource_path: String, +} diff --git a/backend/windmill-mcp/src/common/mod.rs b/backend/windmill-mcp/src/common/mod.rs new file mode 100644 index 0000000000..d67c1718cc --- /dev/null +++ b/backend/windmill-mcp/src/common/mod.rs @@ -0,0 +1,16 @@ +//! Common types and utilities for MCP server and client +//! +//! This module contains shared data structures, transformation utilities, +//! and scope parsing functionality used throughout the MCP implementation. + +pub mod schema; +pub mod scope; +pub mod transform; +pub mod types; + +pub use schema::convert_schema_to_schema_type; +pub use scope::{is_resource_allowed, parse_mcp_scopes, McpScopeConfig}; +pub use transform::{ + apply_key_transformation, reverse_transform, reverse_transform_key, transform_path, +}; +pub use types::*; diff --git a/backend/windmill-mcp/src/common/schema.rs b/backend/windmill-mcp/src/common/schema.rs new file mode 100644 index 0000000000..c7cec8849e --- /dev/null +++ b/backend/windmill-mcp/src/common/schema.rs @@ -0,0 +1,41 @@ +//! Schema conversion utilities for MCP server +//! +//! Contains functions for converting Windmill schemas into MCP-compatible formats. + +use std::collections::HashSet; + +use serde_json::Value; + +use super::types::SchemaType; +use windmill_common::scripts::Schema; + +/// Convert a Windmill Schema to a SchemaType +pub fn convert_schema_to_schema_type(schema: Option) -> SchemaType { + let schema_obj = if let Some(ref s) = schema { + match serde_json::from_str::(s.0.get()) { + Ok(val) => val, + Err(_) => SchemaType::default(), + } + } else { + SchemaType::default() + }; + schema_obj +} + +/// Extract resource type keys from a schema +/// +/// Scans the schema properties for fields with format "resource-{type}" +/// and returns a set of all unique resource type names found. +pub fn extract_resource_types_from_schema(schema: &SchemaType) -> HashSet { + let mut resource_types = HashSet::new(); + for (_key, prop_value) in schema.properties.iter() { + if let Value::Object(prop_map) = prop_value { + if let Some(Value::String(format_str)) = prop_map.get("format") { + if let Some(rt) = format_str.strip_prefix("resource-") { + resource_types.insert(rt.to_string()); + } + } + } + } + resource_types +} diff --git a/backend/windmill-api/src/mcp/utils/scope_matcher.rs b/backend/windmill-mcp/src/common/scope.rs similarity index 86% rename from backend/windmill-api/src/mcp/utils/scope_matcher.rs rename to backend/windmill-mcp/src/common/scope.rs index 4b50d747ac..7da4e2cebc 100644 --- a/backend/windmill-api/src/mcp/utils/scope_matcher.rs +++ b/backend/windmill-mcp/src/common/scope.rs @@ -3,8 +3,6 @@ //! Contains utilities for parsing and matching MCP token scopes to determine //! which scripts, flows, and endpoints a token has access to. -use windmill_mcp::server::ErrorData; - /// Configuration for MCP scopes parsed from token scopes #[derive(Debug, Clone, Default)] pub struct McpScopeConfig { @@ -24,8 +22,26 @@ pub struct McpScopeConfig { pub hub_apps: Option, } +impl McpScopeConfig { + /// Check if a resource is allowed based on its type and path + pub fn is_allowed(&self, resource_type: &str, path: &str) -> bool { + if self.all { + return true; + } + + let patterns = match resource_type { + "script" => &self.scripts, + "flow" => &self.flows, + "endpoint" => &self.endpoints, + _ => return false, + }; + + is_resource_allowed(path, patterns) + } +} + /// Parse MCP scopes from token scope strings -pub fn parse_mcp_scopes(scopes: &[String]) -> Result { +pub fn parse_mcp_scopes(scopes: &[String]) -> Result { let mut config = McpScopeConfig::default(); for scope in scopes { @@ -69,19 +85,19 @@ pub fn parse_mcp_scopes(scopes: &[String]) -> Result if let Some(resources) = scope.strip_prefix("mcp:scripts:") { // New granular script scope: mcp:scripts:path1,path2,f/folder/* - config.scripts.extend(parse_resource_list(resources)?); + config.scripts.extend(parse_resource_list(resources)); continue; } if let Some(resources) = scope.strip_prefix("mcp:flows:") { // New granular flow scope: mcp:flows:path1,path2,f/folder/* - config.flows.extend(parse_resource_list(resources)?); + config.flows.extend(parse_resource_list(resources)); continue; } if let Some(resources) = scope.strip_prefix("mcp:endpoints:") { // New granular endpoint scope: mcp:endpoints:name1,name2 - config.endpoints.extend(parse_resource_list(resources)?); + config.endpoints.extend(parse_resource_list(resources)); continue; } @@ -94,16 +110,16 @@ pub fn parse_mcp_scopes(scopes: &[String]) -> Result } /// Parse comma-separated resource list -fn parse_resource_list(resources: &str) -> Result, ErrorData> { +fn parse_resource_list(resources: &str) -> Vec { if resources.is_empty() { - return Ok(vec![]); + return vec![]; } - Ok(resources + resources .split(',') .map(|s| s.trim().to_string()) .filter(|s| !s.is_empty()) - .collect()) + .collect() } /// Check if a resource path matches any pattern in the allowed list @@ -226,4 +242,16 @@ mod tests { let empty: Vec = vec![]; assert!(!is_resource_allowed("any/path", &empty)); } + + #[test] + fn test_scope_config_is_allowed() { + let mut config = McpScopeConfig::default(); + config.scripts.push("u/admin/*".to_string()); + config.flows.push("f/automation/*".to_string()); + + assert!(config.is_allowed("script", "u/admin/test")); + assert!(!config.is_allowed("script", "u/other/test")); + assert!(config.is_allowed("flow", "f/automation/test")); + assert!(!config.is_allowed("flow", "f/other/test")); + } } diff --git a/backend/windmill-api/src/mcp/utils/transform.rs b/backend/windmill-mcp/src/common/transform.rs similarity index 78% rename from backend/windmill-api/src/mcp/utils/transform.rs rename to backend/windmill-mcp/src/common/transform.rs index d4ef52fd3f..9a6fa317b1 100644 --- a/backend/windmill-api/src/mcp/utils/transform.rs +++ b/backend/windmill-mcp/src/common/transform.rs @@ -3,9 +3,9 @@ //! Contains functions for transforming paths, keys, and other identifiers //! to make them compatible with MCP tool naming requirements. -use super::models::SchemaType; +use super::types::SchemaType; -// MCP clients do not allow names longer than 60 characters +/// MCP clients do not allow names longer than 60 characters const MAX_PATH_LENGTH: usize = 60; /// Transform the path for workspace scripts/flows @@ -114,3 +114,38 @@ pub fn reverse_transform_key(transformed_key: &str, schema_obj: &Option, @@ -39,16 +42,13 @@ pub struct SchemaType { impl Default for SchemaType { fn default() -> Self { - Self { - r#type: "object".to_string(), - properties: HashMap::new(), - required: vec![], - } + Self { r#type: "object".to_string(), properties: HashMap::new(), required: vec![] } } } /// Script information from database -#[derive(Serialize, FromRow, Debug)] +#[derive(Serialize, Debug)] +#[cfg_attr(feature = "server", derive(FromRow))] pub struct ScriptInfo { pub path: String, pub summary: Option, @@ -57,7 +57,8 @@ pub struct ScriptInfo { } /// Flow information from database -#[derive(Serialize, FromRow, Debug)] +#[derive(Serialize, Debug)] +#[cfg_attr(feature = "server", derive(FromRow))] pub struct FlowInfo { pub path: String, pub summary: Option, @@ -66,7 +67,8 @@ pub struct FlowInfo { } /// Resource information from database -#[derive(Serialize, FromRow, Debug, Clone)] +#[derive(Serialize, Debug, Clone)] +#[cfg_attr(feature = "server", derive(FromRow))] pub struct ResourceInfo { pub path: String, pub description: Option, @@ -74,25 +76,34 @@ pub struct ResourceInfo { } /// Resource type information from database -#[derive(Serialize, FromRow, Debug, Clone)] +#[derive(Serialize, Debug, Clone)] +#[cfg_attr(feature = "server", derive(FromRow))] pub struct ResourceType { pub name: String, pub description: Option, } /// Schema holder for database queries -#[derive(Serialize, FromRow)] +#[derive(Serialize)] +#[cfg_attr(feature = "server", derive(FromRow))] pub struct ItemSchema { pub schema: Option, } /// Trait for objects that can be converted to MCP tools pub trait ToolableItem { + /// Get the path or identifier for this item (transformed for MCP compatibility) fn get_path_or_id(&self) -> String; + /// Get the summary/title of this item fn get_summary(&self) -> &str; + /// Get the description of this item fn get_description(&self) -> &str; + /// Get the JSON schema for this item's parameters fn get_schema(&self) -> SchemaType; + /// Whether this item is from the Hub fn is_hub(&self) -> bool; + /// Get the type of this item ("script" or "flow") fn item_type(&self) -> &'static str; + /// Get the integration type (for hub scripts) fn get_integration_type(&self) -> Option; -} \ No newline at end of file +} diff --git a/backend/windmill-mcp/src/lib.rs b/backend/windmill-mcp/src/lib.rs index bf05dc7c45..9026837493 100644 --- a/backend/windmill-mcp/src/lib.rs +++ b/backend/windmill-mcp/src/lib.rs @@ -1,51 +1,33 @@ -/* - * Author: Ruben Fiszel - * Copyright: Windmill Labs, Inc 2022 - * This file and its contents are licensed under the AGPLv3 License. - * Please see the included NOTICE for copyright information and - * LICENSE-AGPL for a copy of the license. - */ +//! Windmill MCP (Model Context Protocol) implementation +//! +//! This crate provides: +//! - MCP client for connecting to external MCP servers (used by AI agents) +//! - Common types and utilities for MCP implementations +//! - MCP server types (when `server` feature is enabled) +//! - OAuth support (when `auth` feature is enabled) -use anyhow::{Context, Result}; -use reqwest::header::{HeaderMap, HeaderName, HeaderValue}; -use serde_json::{json, Value}; -use std::str::FromStr; -use windmill_common::variables::get_secret_value_as_admin; -use windmill_common::DB; +// Common types and utilities module +pub mod common; + +// Client module +pub mod client; + +// Re-export common types at crate root for convenience +pub use common::{ + convert_schema_to_schema_type, is_resource_allowed, parse_mcp_scopes, transform_path, FlowInfo, + HubResponse, HubScriptInfo, ItemSchema, McpScopeConfig, ResourceInfo, ResourceType, SchemaType, + ScriptInfo, ToolableItem, WorkspaceId, +}; + +// Re-export client types at crate root for backward compatibility +pub use client::{McpClient, McpResource, McpToolSource}; // Re-export rmcp types for client usage pub use rmcp::model::Tool as McpTool; -use rmcp::{ - model::{ - CallToolRequestParam, ClientCapabilities, ClientInfo, Implementation, - InitializeRequestParam, - }, - service::RunningService, - transport::{ - streamable_http_client::StreamableHttpClientTransportConfig, StreamableHttpClientTransport, - }, - RoleClient, ServiceExt, -}; -// Re-export rmcp server types when server feature is enabled +// Server module (when server feature is enabled) #[cfg(feature = "server")] -pub mod server { - //! Re-exports of rmcp server types for MCP server implementations - - pub use rmcp::handler::server::ServerHandler; - pub use rmcp::model::{ - Annotated, CallToolRequestParam, CallToolResult, Content, Implementation, - InitializeRequestParam, InitializeResult, ListPromptsResult, ListResourceTemplatesResult, - ListResourcesResult, ListToolsResult, PaginatedRequestParam, ProtocolVersion, RawContent, - RawTextContent, ServerCapabilities, ServerInfo, Tool, ToolAnnotations, - }; - pub use rmcp::service::{RequestContext, RoleServer}; - pub use rmcp::transport::streamable_http_server::{ - session::local::LocalSessionManager, StreamableHttpService, - }; - pub use rmcp::transport::StreamableHttpServerConfig; - pub use rmcp::ErrorData; -} +pub mod server; // Re-export rmcp auth types when auth feature is enabled #[cfg(feature = "auth")] @@ -60,212 +42,3 @@ pub mod oauth { RedirectUrl, Scope, TokenUrl, }; } - -use std::collections::HashMap; - -use serde::{Deserialize, Serialize}; - -/// MCP server resource configuration -#[derive(Debug, Clone, Serialize, Deserialize)] -pub struct McpResource { - /// Name of the MCP resource (used for prefixing tools) - pub name: String, - /// HTTP URL for the MCP server endpoint - pub url: String, - /// Optional token for authentication - #[serde(skip_serializing_if = "Option::is_none")] - pub token: Option, - /// Optional headers - #[serde(skip_serializing_if = "Option::is_none")] - pub headers: Option>, -} - -/// Metadata for tracking MCP tool sources -#[derive(Debug, Clone, Serialize, Deserialize)] -pub struct McpToolSource { - /// Name of the MCP resource this tool comes from - pub name: String, - /// Original tool name in the MCP server - pub tool_name: String, - /// Path of the MCP resource - pub resource_path: String, -} - -/// MCP client for communicating with external MCP servers -pub struct McpClient { - /// The underlying rmcp client - client: RunningService, - /// Cached list of available tools from the server - available_tools: Vec, -} - -impl McpClient { - /// Create a new MCP client from a resource configuration - pub async fn from_resource(resource: McpResource, db: &DB, w_id: &str) -> Result { - // Build custom reqwest client with headers if provided - let mut headers = HeaderMap::new(); - if let Some(token_path) = &resource.token { - if !token_path.trim().is_empty() { - let value = - get_secret_value_as_admin(db, w_id, token_path.trim_start_matches("$var:")) - .await?; - headers.insert( - HeaderName::from_static("authorization"), - HeaderValue::from_str(format!("Bearer {}", value).as_str())?, - ); - } - } - if let Some(resource_headers) = &resource.headers { - for (key, value) in resource_headers { - match (HeaderName::from_str(key), HeaderValue::from_str(value)) { - (Ok(name), Ok(value)) => { - headers.insert(name, value); - } - _ => { - tracing::warn!("Invalid header: {}={}", key, value); - } - } - } - } - - let reqwest_client = reqwest::Client::builder() - .default_headers(headers) - .build() - .context("Failed to build HTTP client")?; - - // Create the HTTP transport with custom client - let config = StreamableHttpClientTransportConfig::with_uri(resource.url.as_str()); - let transport = StreamableHttpClientTransport::with_client(reqwest_client, config); - - // Set up client info - let client_info = ClientInfo { - protocol_version: Default::default(), - capabilities: ClientCapabilities::default(), - client_info: Implementation { - name: "windmill-ai-agent".to_string(), - title: Some("Windmill AI Agent".to_string()), - version: env!("CARGO_PKG_VERSION").to_string(), - website_url: None, - icons: None, - }, - }; - - // Initialize the connection - let client = client_info - .serve(transport) - .await - .context("Failed to connect to MCP server")?; - - // Immediately fetch available tools - let available_tools = client - .list_tools(Default::default()) - .await - .context("Failed to list tools from MCP server")? - .tools; - - Ok(Self { client, available_tools }) - } - - /// Get the list of available tools from the MCP server - pub fn available_tools(&self) -> &[McpTool] { - &self.available_tools - } - - /// Call a tool on the MCP server, with openai-style arguments - pub async fn call_tool(&self, name: &str, arguments: &str) -> Result { - // Convert OpenAI-style arguments to MCP format - let mcp_args = - Self::openai_args_to_mcp_args(arguments).context("Failed to parse tool arguments")?; - - let result = self - .client - .call_tool(CallToolRequestParam { name: name.to_string().into(), arguments: mcp_args, task: None }) - .await - .context(format!("Failed to call MCP tool: {}", name))?; - - // Convert the result to a JSON value - // MCP tools return ToolResult which contains content array - let result_json = - serde_json::to_value(&result).context("Failed to serialize MCP tool result")?; - - Ok(result_json) - } - - /// Close the connection - pub async fn shutdown(self) -> Result<()> { - self.client.cancel().await?; - Ok(()) - } - - /// Fix array schemas to ensure they have the required 'items' property - /// OpenAI requires all array types to have an 'items' field. MCP servers may - /// return schemas without this field, so we add a default. - pub fn fix_array_schemas(schema: &mut Value) { - if let Value::Object(obj) = schema { - // Check if this is an array type - if let Some(type_val) = obj.get("type") { - let is_array = match type_val { - Value::String(s) => s == "array", - Value::Array(arr) => arr.iter().any(|v| v.as_str() == Some("array")), - _ => false, - }; - - // If it's an array and missing 'items', add a default - if is_array && !obj.contains_key("items") { - obj.insert("items".to_string(), json!({})); - } - } - - // Recursively fix nested schemas - if let Some(Value::Object(props)) = obj.get_mut("properties") { - for value in props.values_mut() { - Self::fix_array_schemas(value); - } - } - - // Fix items if present (for nested arrays) - if let Some(items) = obj.get_mut("items") { - Self::fix_array_schemas(items); - } - - // Fix oneOf, anyOf, allOf schemas - for key in &["oneOf", "anyOf", "allOf"] { - if let Some(Value::Array(schemas)) = obj.get_mut(*key) { - for schema in schemas { - Self::fix_array_schemas(schema); - } - } - } - - // Fix additionalProperties if it's a schema - if let Some(additional) = obj.get_mut("additionalProperties") { - if additional.is_object() { - Self::fix_array_schemas(additional); - } - } - } - } - - /// Convert OpenAI-style tool call arguments to MCP format - /// OpenAI sends arguments as a JSON string, MCP expects a Map - fn openai_args_to_mcp_args( - args_str: &str, - ) -> Result>> { - if args_str.trim().is_empty() { - return Ok(None); - } - - let args_value: serde_json::Value = - serde_json::from_str(args_str).context("Failed to parse tool call arguments")?; - - match args_value { - serde_json::Value::Object(map) => Ok(Some(map)), - serde_json::Value::Null => Ok(None), - _ => Ok(Some( - vec![("value".to_string(), args_value)] - .into_iter() - .collect(), - )), - } - } -} diff --git a/backend/windmill-mcp/src/server/backend.rs b/backend/windmill-mcp/src/server/backend.rs new file mode 100644 index 0000000000..fd01b2b26c --- /dev/null +++ b/backend/windmill-mcp/src/server/backend.rs @@ -0,0 +1,159 @@ +//! MCP Backend trait definitions +//! +//! This module defines the traits that must be implemented by the backend +//! (typically windmill-api) to provide the actual functionality for the MCP server. + +use async_trait::async_trait; +use rmcp::ErrorData; +use serde_json::Value; +use std::collections::HashMap; + +use crate::common::types::{ + FlowInfo, HubScriptInfo, ResourceInfo, ResourceType, SchemaType, ScriptInfo, +}; +use crate::server::endpoints::EndpointTool; + +/// Result type for backend operations using rmcp's ErrorData directly +pub type BackendResult = Result; + +/// Authentication context required by the MCP server +pub trait McpAuth: Send + Sync + Clone + 'static { + /// Get the username + fn username(&self) -> &str; + /// Get the email + fn email(&self) -> &str; + /// Check if user is admin + fn is_admin(&self) -> bool; + /// Check if user is operator + fn is_operator(&self) -> bool; + /// Get user's groups + fn groups(&self) -> &[String]; + /// Get user's folders as (name, can_write, is_owner) + fn folders(&self) -> &[(String, bool, bool)]; + /// Get token scopes + fn scopes(&self) -> Option<&[String]>; + + /// Check if the user has an MCP scope + fn has_mcp_scope(&self) -> bool { + self.scopes() + .map(|s| s.iter().any(|scope| scope.starts_with("mcp:"))) + .unwrap_or(false) + } +} + +/// The core backend trait that windmill-api implements +/// +/// This trait abstracts the windmill-api specific operations needed by the MCP server. +/// By implementing this trait, windmill-api can inject its database access, job execution, +/// and other functionality without windmill-mcp needing to depend on windmill-api directly. +#[async_trait] +pub trait McpBackend: Send + Sync + Clone + 'static { + /// The authentication context type + type Auth: McpAuth; + + // ───────────────────────────────────────────────────────────────── + // Listing Operations + // ───────────────────────────────────────────────────────────────── + + /// List scripts, optionally filtered to favorites only + async fn list_scripts( + &self, + auth: &Self::Auth, + workspace_id: &str, + favorites_only: bool, + ) -> BackendResult>; + + /// List flows, optionally filtered to favorites only + async fn list_flows( + &self, + auth: &Self::Auth, + workspace_id: &str, + favorites_only: bool, + ) -> BackendResult>; + + /// List resource types in workspace + async fn list_resource_types( + &self, + auth: &Self::Auth, + workspace_id: &str, + ) -> BackendResult>; + + /// List resources of a specific type + async fn list_resources( + &self, + auth: &Self::Auth, + workspace_id: &str, + resource_type: &str, + ) -> BackendResult>; + + /// List hub scripts, optionally filtered by app integrations + async fn list_hub_scripts(&self, app_filter: Option<&str>) + -> BackendResult>; + + // ───────────────────────────────────────────────────────────────── + // Schema Operations + // ───────────────────────────────────────────────────────────────── + + /// Get schema for a script or flow + async fn get_item_schema( + &self, + auth: &Self::Auth, + workspace_id: &str, + path: &str, + item_type: &str, + ) -> BackendResult>; + + /// Get schema for a hub script + async fn get_hub_script_schema(&self, path: &str) -> BackendResult>; + + // ───────────────────────────────────────────────────────────────── + // Schema Transformation (requires DB access for resources) + // ───────────────────────────────────────────────────────────────── + + /// Transform schema for resources by enriching with available resource information. + /// The resources_cache should be pre-populated with all needed resource types. + fn transform_schema_for_resources( + &self, + schema: &SchemaType, + resources_cache: &HashMap>, + resources_types: &[ResourceType], + ) -> SchemaType; + + // ───────────────────────────────────────────────────────────────── + // Execution Operations + // ───────────────────────────────────────────────────────────────── + + /// Run a script and wait for result + async fn run_script( + &self, + auth: &Self::Auth, + workspace_id: &str, + path: &str, + args: Value, + ) -> BackendResult; + + /// Run a flow and wait for result + async fn run_flow( + &self, + auth: &Self::Auth, + workspace_id: &str, + path: &str, + args: Value, + ) -> BackendResult; + + /// Call an endpoint tool (generated API endpoint) + async fn call_endpoint( + &self, + auth: &Self::Auth, + workspace_id: &str, + endpoint_tool: &EndpointTool, + args: Value, + ) -> BackendResult; + + // ───────────────────────────────────────────────────────────────── + // Endpoint Tools + // ───────────────────────────────────────────────────────────────── + + /// Get all available endpoint tools + fn all_endpoint_tools(&self) -> Vec; +} diff --git a/backend/windmill-mcp/src/server/endpoints.rs b/backend/windmill-mcp/src/server/endpoints.rs new file mode 100644 index 0000000000..ddf531bd1e --- /dev/null +++ b/backend/windmill-mcp/src/server/endpoints.rs @@ -0,0 +1,103 @@ +//! Endpoint tools for MCP server +//! +//! Contains the EndpointTool structure and utilities for converting +//! them to MCP tools. + +use rmcp::model::{Tool, ToolAnnotations}; +use serde::{Deserialize, Serialize}; +use std::borrow::Cow; +use std::sync::Arc; + +/// Represents an auto-generated endpoint tool from OpenAPI specification +#[derive(Debug, Clone, Deserialize, Serialize)] +pub struct EndpointTool { + pub name: Cow<'static, str>, + pub description: Cow<'static, str>, + pub instructions: Cow<'static, str>, + pub path: Cow<'static, str>, + pub method: Cow<'static, str>, + pub path_params_schema: Option, + pub query_params_schema: Option, + pub body_schema: Option, +} + +/// Convert a single endpoint tool to MCP tool +pub fn endpoint_tool_to_mcp_tool(tool: &EndpointTool) -> Tool { + let mut combined_properties = serde_json::Map::new(); + let mut combined_required = Vec::new(); + + // Combine all parameter schemas + let schemas = [ + &tool.path_params_schema, + &tool.query_params_schema, + &tool.body_schema, + ]; + + for schema in schemas.iter().filter_map(|s| s.as_ref()) { + merge_schema_into(&mut combined_properties, &mut combined_required, schema); + } + + let combined_schema = serde_json::json!({ + "type": "object", + "properties": combined_properties, + "required": combined_required + }); + + let description = format!("{}. {}", tool.description, tool.instructions); + + // Create annotations based on HTTP method and endpoint characteristics + let annotations = create_endpoint_annotations(tool); + + Tool { + name: tool.name.clone(), + description: Some(description.into()), + input_schema: Arc::new(combined_schema.as_object().unwrap().clone()), + title: Some(tool.name.to_string()), + output_schema: None, + icons: None, + annotations: Some(annotations), + meta: None, + } +} + +/// Create appropriate annotations for endpoint tools based on HTTP method +fn create_endpoint_annotations(tool: &EndpointTool) -> ToolAnnotations { + let method = tool.method.as_ref(); + + // Determine characteristics based on HTTP method + let (read_only, destructive, idempotent, open_world) = match method { + "GET" => (true, false, true, true), // Read-only, safe, idempotent + "POST" => (false, true, false, true), // Can modify, potentially destructive, not idempotent + "PUT" => (false, false, true, true), // Can modify, typically idempotent updates + "DELETE" => (false, true, true, true), // Destructive but idempotent + "PATCH" => (false, false, false, true), // Partial updates, not guaranteed idempotent + _ => (false, true, false, true), // Default: assume can modify and be destructive + }; + + ToolAnnotations { + title: Some(format!("{} {}", method, tool.path)), + read_only_hint: Some(read_only), + destructive_hint: Some(destructive), + idempotent_hint: Some(idempotent), + open_world_hint: Some(open_world), + } +} + +/// Merge schema into combined properties and required fields +fn merge_schema_into( + combined_properties: &mut serde_json::Map, + combined_required: &mut Vec, + schema: &serde_json::Value, +) { + if let Some(props) = schema.get("properties").and_then(|p| p.as_object()) { + for (key, value) in props { + combined_properties.insert(key.clone(), value.clone()); + } + } + + if let Some(required) = schema.get("required").and_then(|r| r.as_array()) { + for req in required.iter().filter_map(|r| r.as_str()) { + combined_required.push(req.to_string()); + } + } +} diff --git a/backend/windmill-mcp/src/server/mod.rs b/backend/windmill-mcp/src/server/mod.rs new file mode 100644 index 0000000000..90e6083435 --- /dev/null +++ b/backend/windmill-mcp/src/server/mod.rs @@ -0,0 +1,32 @@ +//! MCP Server module +//! +//! This module provides the MCP server implementation including: +//! - `McpBackend` trait for backend implementations +//! - `Runner` struct that implements the MCP protocol +//! - Re-exports of rmcp types + +pub mod backend; +pub mod endpoints; +pub mod runner; +pub mod tools; + +// Re-export main types +pub use backend::{BackendResult, McpAuth, McpBackend}; +pub use endpoints::{endpoint_tool_to_mcp_tool, EndpointTool}; +pub use runner::Runner; +pub use tools::create_tool_from_item; + +// Re-export rmcp types for convenience +pub use rmcp::handler::server::ServerHandler; +pub use rmcp::model::{ + Annotated, CallToolRequestParam, CallToolResult, Content, Implementation, + InitializeRequestParam, InitializeResult, ListPromptsResult, ListResourceTemplatesResult, + ListResourcesResult, ListToolsResult, PaginatedRequestParam, ProtocolVersion, RawContent, + RawTextContent, ServerCapabilities, ServerInfo, Tool, ToolAnnotations, +}; +pub use rmcp::service::{RequestContext, RoleServer}; +pub use rmcp::transport::streamable_http_server::{ + session::local::LocalSessionManager, StreamableHttpService, +}; +pub use rmcp::transport::StreamableHttpServerConfig; +pub use rmcp::ErrorData; diff --git a/backend/windmill-mcp/src/server/runner.rs b/backend/windmill-mcp/src/server/runner.rs new file mode 100644 index 0000000000..39fbda5c7a --- /dev/null +++ b/backend/windmill-mcp/src/server/runner.rs @@ -0,0 +1,373 @@ +//! MCP Server Runner implementation +//! +//! Contains the generic Runner that implements the MCP ServerHandler trait +//! and delegates to a McpBackend for actual functionality. + +use crate::common::schema::extract_resource_types_from_schema; +use crate::common::scope::parse_mcp_scopes; +use crate::common::transform::{reverse_transform, reverse_transform_key}; +use crate::common::types::{ResourceInfo, ToolableItem, WorkspaceId}; +use crate::server::backend::{McpAuth, McpBackend}; +use crate::server::endpoints::endpoint_tool_to_mcp_tool; +use crate::server::tools::create_tool_from_item; +use rmcp::handler::server::ServerHandler; +use rmcp::model::{ + CallToolRequestParam, CallToolResult, Content, Implementation, InitializeRequestParam, + InitializeResult, ListPromptsResult, ListResourceTemplatesResult, ListResourcesResult, + ListToolsResult, PaginatedRequestParam, ProtocolVersion, ServerCapabilities, ServerInfo, +}; +use rmcp::service::{RequestContext, RoleServer}; +use rmcp::ErrorData; +use serde_json::Value; +use std::collections::{HashMap, HashSet}; +use std::sync::Arc; + +// Re-export from http crate for extracting request parts +use http::request::Parts as HttpParts; + +/// MCP Server Runner - generic over the backend implementation +/// +/// This struct implements the MCP ServerHandler trait and uses a McpBackend +/// to perform the actual operations (database queries, job execution, etc.) +pub struct Runner { + backend: Arc, +} + +impl Clone for Runner { + fn clone(&self) -> Self { + Self { backend: self.backend.clone() } + } +} + +impl Runner { + /// Create a new Runner with the given backend + pub fn new(backend: B) -> Self { + Self { backend: Arc::new(backend) } + } + + /// Extract authentication and workspace from request context + fn extract_context( + context: &RequestContext, + ) -> Result<(B::Auth, String), ErrorData> { + let http_parts = context.extensions.get::().ok_or_else(|| { + tracing::error!("http::request::Parts not found"); + ErrorData::internal_error("http::request::Parts not found", None) + })?; + + let auth = http_parts.extensions.get::().ok_or_else(|| { + tracing::error!("Auth extension not found"); + ErrorData::internal_error("Auth extension not found", None) + })?; + + let workspace_id = http_parts + .extensions + .get::() + .ok_or_else(|| { + tracing::error!("WorkspaceId not found"); + ErrorData::internal_error("WorkspaceId not found", None) + }) + .map(|w_id| w_id.0.clone())?; + + // Validate MCP scope + if !auth.has_mcp_scope() { + tracing::error!("Unauthorized: missing mcp scope"); + return Err(ErrorData::internal_error( + "Unauthorized: missing mcp scope", + None, + )); + } + + Ok((auth.clone(), workspace_id)) + } +} + +impl ServerHandler for Runner { + fn get_info(&self) -> ServerInfo { + ServerInfo { + protocol_version: ProtocolVersion::default(), + capabilities: ServerCapabilities::builder().enable_tools().build(), + server_info: Implementation::from_build_env(), + instructions: Some( + "This server provides a list of scripts and flows the user can run on Windmill. \ + Each flow and script is a tool callable with their respective arguments." + .to_string(), + ), + } + } + + async fn initialize( + &self, + _request: InitializeRequestParam, + _context: RequestContext, + ) -> Result { + Ok(self.get_info()) + } + + async fn list_tools( + &self, + _request: Option, + context: RequestContext, + ) -> Result { + let (auth, workspace_id) = Self::extract_context(&context)?; + + // Parse MCP scopes to determine what to expose + let scopes = auth.scopes().unwrap_or(&[]); + let scope_config = + parse_mcp_scopes(scopes).map_err(|e| ErrorData::internal_error(e, None))?; + + let favorites_only = scope_config.favorites; + + // Fetch all items concurrently + let (scripts, flows, resource_types, hub_scripts) = tokio::try_join!( + self.backend + .list_scripts(&auth, &workspace_id, favorites_only), + self.backend + .list_flows(&auth, &workspace_id, favorites_only), + self.backend.list_resource_types(&auth, &workspace_id), + async { + if let Some(ref apps) = scope_config.hub_apps { + self.backend.list_hub_scripts(Some(apps)).await + } else { + Ok(vec![]) + } + } + )?; + + // Filter items based on scope + let filtered_scripts: Vec<_> = scripts + .into_iter() + .filter(|s| !scope_config.granular || scope_config.is_allowed("script", &s.path)) + .collect(); + + let filtered_flows: Vec<_> = flows + .into_iter() + .filter(|f| !scope_config.granular || scope_config.is_allowed("flow", &f.path)) + .collect(); + + // Collect all needed resource types from all schemas + let mut needed_resource_types: HashSet = HashSet::new(); + for script in &filtered_scripts { + needed_resource_types.extend(extract_resource_types_from_schema(&script.get_schema())); + } + for flow in &filtered_flows { + needed_resource_types.extend(extract_resource_types_from_schema(&flow.get_schema())); + } + for hub_script in &hub_scripts { + needed_resource_types + .extend(extract_resource_types_from_schema(&hub_script.get_schema())); + } + + // Pre-fetch all resources + let resource_futures: Vec<_> = needed_resource_types + .into_iter() + .map(|rt| { + let backend = self.backend.clone(); + let auth = auth.clone(); + let workspace_id = workspace_id.clone(); + async move { + backend + .list_resources(&auth, &workspace_id, &rt) + .await + .map(|resources| (rt, resources)) + } + }) + .collect(); + + let resource_results = futures::future::try_join_all(resource_futures).await?; + let resources_cache: HashMap> = + resource_results.into_iter().collect(); + + let mut tools = Vec::new(); + + for script in &filtered_scripts { + tools.push(create_tool_from_item( + script, + self.backend.as_ref(), + &resources_cache, + &resource_types, + )); + } + + for flow in &filtered_flows { + tools.push(create_tool_from_item( + flow, + self.backend.as_ref(), + &resources_cache, + &resource_types, + )); + } + + for hub_script in &hub_scripts { + tools.push(create_tool_from_item( + hub_script, + self.backend.as_ref(), + &resources_cache, + &resource_types, + )); + } + + // Add endpoint tools from the generated MCP tools, filtered by scope + let endpoint_tools = self.backend.all_endpoint_tools(); + for endpoint_tool in endpoint_tools { + if scope_config.granular && !scope_config.is_allowed("endpoint", &endpoint_tool.name) { + continue; + } + + tools.push(endpoint_tool_to_mcp_tool(&endpoint_tool)); + } + + Ok(ListToolsResult { tools, next_cursor: None, meta: None }) + } + + async fn call_tool( + &self, + request: CallToolRequestParam, + context: RequestContext, + ) -> Result { + let (auth, workspace_id) = Self::extract_context(&context)?; + + // Parse MCP scopes for authorization + let scopes = auth.scopes().unwrap_or(&[]); + let scope_config = + parse_mcp_scopes(scopes).map_err(|e| ErrorData::internal_error(e, None))?; + + // Handle truncated tool names + if request.name.ends_with("_TRUNC") { + return Ok(CallToolResult::error(vec![rmcp::model::Annotated::new( + rmcp::model::RawContent::Text(rmcp::model::RawTextContent { + text: "Tool path is too long. Consider shortening it to make it compatible with MCP.".to_string(), + meta: None, + }), + None, + )])); + } + + let args = request.arguments.map(Value::Object).unwrap_or(Value::Null); + + // Check if this is an endpoint tool + let endpoint_tools = self.backend.all_endpoint_tools(); + for endpoint_tool in &endpoint_tools { + if endpoint_tool.name.as_ref() == request.name { + // Validate endpoint scope + if scope_config.granular + && !scope_config.is_allowed("endpoint", &endpoint_tool.name) + { + return Err(ErrorData::internal_error( + format!( + "Access denied: endpoint '{}' not in token scope", + endpoint_tool.name + ), + None, + )); + } + + // This is an endpoint tool, call via backend + let result = self + .backend + .call_endpoint(&auth, &workspace_id, endpoint_tool, args) + .await + .map_err(|e| ErrorData::internal_error(e.message, None))?; + + return Ok(CallToolResult::success(vec![Content::text( + serde_json::to_string_pretty(&result).unwrap_or_else(|_| "{}".to_string()), + )])); + } + } + + // Not an endpoint tool - parse as script/flow + let (tool_type, path, is_hub) = reverse_transform(&request.name).map_err(|e| { + ErrorData::internal_error(format!("Failed to parse tool name: {}", e), None) + })?; + + // Validate script/flow scope + if !is_hub && scope_config.granular { + if tool_type == "script" && !scope_config.is_allowed("script", &path) { + return Err(ErrorData::internal_error( + format!("Access denied: script '{}' not in token scope", path), + None, + )); + } else if tool_type == "flow" && !scope_config.is_allowed("flow", &path) { + return Err(ErrorData::internal_error( + format!("Access denied: flow '{}' not in token scope", path), + None, + )); + } + } + + // Get item schema for argument transformation + let item_schema = if is_hub { + self.backend + .get_hub_script_schema(&format!("hub/{}", path)) + .await + .map_err(|e| ErrorData::internal_error(e.message, None))? + } else { + self.backend + .get_item_schema(&auth, &workspace_id, &path, tool_type) + .await + .map_err(|e| ErrorData::internal_error(e.message, None))? + }; + + // Transform arguments back to original key names + let transformed_args = if let Value::Object(map) = args { + let mut args_hash = HashMap::new(); + for (k, v) in map { + let original_key = reverse_transform_key(&k, &item_schema); + args_hash.insert(original_key, v); + } + Value::Object(args_hash.into_iter().collect()) + } else { + args + }; + + let script_or_flow_path = if is_hub { + format!("hub/{}", path) + } else { + path + }; + + // Execute script or flow + let result = if tool_type == "script" { + self.backend + .run_script(&auth, &workspace_id, &script_or_flow_path, transformed_args) + .await + } else { + self.backend + .run_flow(&auth, &workspace_id, &script_or_flow_path, transformed_args) + .await + }; + + match result { + Ok(value) => Ok(CallToolResult::success(vec![Content::text( + serde_json::to_string_pretty(&value).unwrap_or_else(|_| "{}".to_string()), + )])), + Err(e) => Err(ErrorData::internal_error( + format!("Failed to run {}: {}", tool_type, e.message), + None, + )), + } + } + + async fn list_resources( + &self, + _request: Option, + _context: RequestContext, + ) -> Result { + Ok(ListResourcesResult { resources: vec![], next_cursor: None, meta: None }) + } + + async fn list_prompts( + &self, + _request: Option, + _context: RequestContext, + ) -> Result { + Ok(ListPromptsResult::default()) + } + + async fn list_resource_templates( + &self, + _request: Option, + _context: RequestContext, + ) -> Result { + Ok(ListResourceTemplatesResult::default()) + } +} diff --git a/backend/windmill-mcp/src/server/tools.rs b/backend/windmill-mcp/src/server/tools.rs new file mode 100644 index 0000000000..fea875e498 --- /dev/null +++ b/backend/windmill-mcp/src/server/tools.rs @@ -0,0 +1,184 @@ +//! Tool creation utilities for MCP server +//! +//! Contains functionality for converting Windmill items (scripts, flows, hub scripts) +//! into MCP tools. + +use rmcp::model::{Tool, ToolAnnotations}; +use std::borrow::Cow; +use std::collections::HashMap; +use std::sync::Arc; + +use crate::common::schema::convert_schema_to_schema_type; +use crate::common::transform::transform_path; +use crate::common::types::{ + FlowInfo, HubScriptInfo, ResourceInfo, ResourceType, SchemaType, ScriptInfo, ToolableItem, +}; +use crate::server::backend::McpBackend; + +/// Implementation of ToolableItem for ScriptInfo +impl ToolableItem for ScriptInfo { + fn get_path_or_id(&self) -> String { + transform_path(&self.path, "script") + } + + fn get_summary(&self) -> &str { + self.summary.as_deref().unwrap_or("No summary") + } + + fn get_description(&self) -> &str { + self.description.as_deref().unwrap_or("No description") + } + + fn get_schema(&self) -> SchemaType { + convert_schema_to_schema_type(self.schema.clone()) + } + + fn is_hub(&self) -> bool { + false + } + + fn item_type(&self) -> &'static str { + "script" + } + + fn get_integration_type(&self) -> Option { + None + } +} + +/// Implementation of ToolableItem for FlowInfo +impl ToolableItem for FlowInfo { + fn get_path_or_id(&self) -> String { + transform_path(&self.path, "flow") + } + + fn get_summary(&self) -> &str { + self.summary.as_deref().unwrap_or("No summary") + } + + fn get_description(&self) -> &str { + self.description.as_deref().unwrap_or("No description") + } + + fn get_schema(&self) -> SchemaType { + convert_schema_to_schema_type(self.schema.clone()) + } + + fn is_hub(&self) -> bool { + false + } + + fn item_type(&self) -> &'static str { + "flow" + } + + fn get_integration_type(&self) -> Option { + None + } +} + +/// Implementation of ToolableItem for HubScriptInfo +impl ToolableItem for HubScriptInfo { + fn get_path_or_id(&self) -> String { + let id = self.version_id; + let summary = self.summary.as_deref().unwrap_or("No summary"); + format!("hs-{}-{}", id, summary.replace(" ", "_")) + } + + fn get_summary(&self) -> &str { + self.summary.as_deref().unwrap_or("No summary") + } + + fn get_description(&self) -> &str { + self.description.as_deref().unwrap_or("No description") + } + + fn get_schema(&self) -> SchemaType { + match serde_json::from_value::(self.schema.clone().unwrap_or_default()) { + Ok(schema_type) => schema_type, + Err(_) => SchemaType::default(), + } + } + + fn is_hub(&self) -> bool { + true + } + + fn item_type(&self) -> &'static str { + "script" + } + + fn get_integration_type(&self) -> Option { + self.app.clone() + } +} + +/// Create an MCP Tool from a ToolableItem +/// +/// The resources_cache should be pre-populated with all resource types +/// that may be referenced by the item's schema. +pub fn create_tool_from_item( + item: &T, + backend: &B, + resources_cache: &HashMap>, + resources_types: &[ResourceType], +) -> Tool { + let is_hub = item.is_hub(); + let path = item.get_path_or_id(); + let item_type = item.item_type(); + let description = format!( + "This is a {} named `{}` with the following description: `{}`.{}", + item_type, + item.get_summary(), + item.get_description(), + if is_hub { + format!( + " It is a tool used for the following app: {}", + item.get_integration_type() + .unwrap_or("No integration type".to_string()) + ) + } else { + "".to_string() + } + ); + + let schema = item.get_schema(); + let schema_obj = + backend.transform_schema_for_resources(&schema, resources_cache, resources_types); + + let input_schema_map = match serde_json::to_value(schema_obj) { + Ok(serde_json::Value::Object(map)) => map, + Ok(_) => { + tracing::warn!( + "Schema object for tool '{}' did not serialize to a JSON object, using empty schema.", + path + ); + serde_json::Map::new() + } + Err(e) => { + tracing::error!( + "Failed to serialize schema object for tool '{}': {}. Using empty schema.", + path, + e + ); + serde_json::Map::new() + } + }; + + Tool { + name: Cow::Owned(path), + description: Some(Cow::Owned(description)), + input_schema: Arc::new(input_schema_map), + title: Some(item.get_summary().to_string()), + output_schema: None, + icons: None, + annotations: Some(ToolAnnotations { + title: Some(item.get_summary().to_string()), + read_only_hint: Some(false), // Can modify environment + destructive_hint: Some(true), // Can potentially be destructive + idempotent_hint: Some(false), // Are not guaranteed to be idempotent + open_world_hint: Some(true), // Can interact with external services + }), + meta: None, + } +} diff --git a/backend/windmill-queue/src/jobs.rs b/backend/windmill-queue/src/jobs.rs index cb3bdbdf92..fc0811277d 100644 --- a/backend/windmill-queue/src/jobs.rs +++ b/backend/windmill-queue/src/jobs.rs @@ -138,10 +138,13 @@ const MAX_FREE_CONCURRENT_RUNS: i32 = 30; const ERROR_HANDLER_USERNAME: &str = "error_handler"; const SCHEDULE_ERROR_HANDLER_USERNAME: &str = "schedule_error_handler"; const GLOBAL_ERROR_HANDLER_USERNAME: &str = "global"; +const SUCCESS_HANDLER_USERNAME: &str = "success_handler"; pub const ERROR_HANDLER_USER_GROUP: &str = "g/error_handler"; pub const ERROR_HANDLER_USER_EMAIL: &str = "error_handler@windmill.dev"; pub const SCHEDULE_ERROR_HANDLER_USER_EMAIL: &str = "schedule_error_handler@windmill.dev"; +pub const SUCCESS_HANDLER_USER_GROUP: &str = "g/success_handler"; +pub const SUCCESS_HANDLER_USER_EMAIL: &str = "success_handler@windmill.dev"; #[derive(Clone, Debug, Serialize, Deserialize)] pub struct CanceledBy { @@ -775,8 +778,14 @@ lazy_static::lazy_static! { // Cache for workspace error handler settings with 60s TTL // Key: workspace_id, Value: (error_handler, error_handler_extra_args, error_handler_muted_on_cancel, expiry_timestamp) static ref WORKSPACE_ERROR_HANDLER_CACHE: Cache, Option>>, bool, i64)> = Cache::new(1000); + + // Cache for workspace success handler settings with 60s TTL + // Key: workspace_id, Value: (success_handler, success_handler_extra_args, expiry_timestamp) + static ref WORKSPACE_SUCCESS_HANDLER_CACHE: Cache, Option>>, i64)> = Cache::new(1000); } +const WORKSPACE_HANDLER_CACHE_TTL_SECONDS: i64 = 60; + pub async fn add_completed_job( db: &Pool, completed_job: &MiniCompletedJob, @@ -1565,6 +1574,24 @@ pub async fn report_error_to_workspace_handler_or_critical_side_channel( } } +async fn fetch_error_handler_from_db( + db: &Pool, + w_id: &str, +) -> Result<(Option, Option>>, bool), Error> { + sqlx::query_as::<_, (Option, Option>>, bool)>( + r#" + SELECT error_handler, error_handler_extra_args, error_handler_muted_on_cancel + FROM workspace_settings + WHERE workspace_id = $1 + "#, + ) + .bind(w_id) + .fetch_optional(db) + .await + .context("fetching error handler info from workspace_settings")? + .ok_or_else(|| Error::internal_err(format!("no workspace settings for id {w_id}"))) +} + pub async fn send_error_to_workspace_handler<'a, 'c, T: Serialize + Send + Sync>( queued_job: &MiniCompletedJob, is_canceled: bool, @@ -1573,84 +1600,28 @@ pub async fn send_error_to_workspace_handler<'a, 'c, T: Serialize + Send + Sync> ) -> Result<(), Error> { let w_id = &queued_job.workspace_id; - // Try to get from cache first, checking if entry is still valid (within 60s TTL) let now = chrono::Utc::now().timestamp(); let (error_handler, error_handler_extra_args, error_handler_muted_on_cancel) = if let Some(cached) = WORKSPACE_ERROR_HANDLER_CACHE.get(w_id) { if cached.3 > now { - // Cache hit and not expired (cached.0.clone(), cached.1.clone(), cached.2) } else { - // Cache expired, fetch from database - let row_result = - sqlx::query_as::<_, (Option, Option>>, bool)>( - r#" - SELECT - error_handler, - error_handler_extra_args, - error_handler_muted_on_cancel - FROM - workspace_settings - WHERE - workspace_id = $1 - "#, - ) - .bind(&w_id) - .fetch_optional(db) - .await - .context("fetching error handler info from workspace_settings")? - .ok_or_else(|| { - Error::internal_err(format!("no workspace settings for id {w_id}")) - })?; - - // Update cache with 60s TTL - let expiry = now + 60; + let row = fetch_error_handler_from_db(db, w_id).await?; + let expiry = now + WORKSPACE_HANDLER_CACHE_TTL_SECONDS; WORKSPACE_ERROR_HANDLER_CACHE.insert( w_id.clone(), - ( - row_result.0.clone(), - row_result.1.clone(), - row_result.2, - expiry, - ), + (row.0.clone(), row.1.clone(), row.2, expiry), ); - row_result + row } } else { - // Cache miss, fetch from database - let row_result = - sqlx::query_as::<_, (Option, Option>>, bool)>( - r#" - SELECT - error_handler, - error_handler_extra_args, - error_handler_muted_on_cancel - FROM - workspace_settings - WHERE - workspace_id = $1 - "#, - ) - .bind(&w_id) - .fetch_optional(db) - .await - .context("fetching error handler info from workspace_settings")? - .ok_or_else(|| { - Error::internal_err(format!("no workspace settings for id {w_id}")) - })?; - - // Store in cache with 60s TTL - let expiry = now + 60; + let row = fetch_error_handler_from_db(db, w_id).await?; + let expiry = now + WORKSPACE_HANDLER_CACHE_TTL_SECONDS; WORKSPACE_ERROR_HANDLER_CACHE.insert( w_id.clone(), - ( - row_result.0.clone(), - row_result.1.clone(), - row_result.2, - expiry, - ), + (row.0.clone(), row.1.clone(), row.2, expiry), ); - row_result + row }; if is_canceled && error_handler_muted_on_cancel { @@ -1707,6 +1678,77 @@ pub async fn send_error_to_workspace_handler<'a, 'c, T: Serialize + Send + Sync> Ok(()) } +async fn fetch_success_handler_from_db( + db: &Pool, + w_id: &str, +) -> Result<(Option, Option>>), Error> { + sqlx::query_as::<_, (Option, Option>>)>( + r#" + SELECT success_handler, success_handler_extra_args + FROM workspace_settings + WHERE workspace_id = $1 + "#, + ) + .bind(w_id) + .fetch_optional(db) + .await + .context("fetching success handler info from workspace_settings")? + .ok_or_else(|| Error::internal_err(format!("no workspace settings for id {w_id}"))) +} + +pub async fn send_success_to_workspace_handler<'a, 'c, T: Serialize + Send + Sync>( + queued_job: &MiniCompletedJob, + db: &Pool, + result: Json<&'a T>, +) -> Result<(), Error> { + let w_id = &queued_job.workspace_id; + + let now = chrono::Utc::now().timestamp(); + let (success_handler, success_handler_extra_args) = + if let Some(cached) = WORKSPACE_SUCCESS_HANDLER_CACHE.get(w_id) { + if cached.2 > now { + (cached.0.clone(), cached.1.clone()) + } else { + let row = fetch_success_handler_from_db(db, w_id).await?; + let expiry = now + WORKSPACE_HANDLER_CACHE_TTL_SECONDS; + WORKSPACE_SUCCESS_HANDLER_CACHE.insert( + w_id.clone(), + (row.0.clone(), row.1.clone(), expiry), + ); + row + } + } else { + let row = fetch_success_handler_from_db(db, w_id).await?; + let expiry = now + WORKSPACE_HANDLER_CACHE_TTL_SECONDS; + WORKSPACE_SUCCESS_HANDLER_CACHE.insert( + w_id.clone(), + (row.0.clone(), row.1.clone(), expiry), + ); + row + }; + + if let Some(success_handler) = success_handler { + tracing::info!("workspace success handler for job {}", &queued_job.id); + + push_success_handler( + db, + queued_job.id, + queued_job.schedule_path(), + queued_job.runnable_path.clone(), + queued_job.is_flow(), + &queued_job.workspace_id, + &success_handler, + result, + queued_job.started_at, + success_handler_extra_args, + &queued_job.permissioned_as_email, + None, + ) + .await?; + } + Ok(()) +} + pub async fn handle_maybe_scheduled_job<'c>( db: &Pool, job: &MiniCompletedJob, @@ -1966,6 +2008,91 @@ pub async fn push_error_handler<'a, 'c, T: Serialize + Send + Sync>( return Ok(uuid); } +pub async fn push_success_handler<'a, 'c, T: Serialize + Send + Sync>( + db: &Pool, + job_id: Uuid, + schedule_path: Option, + script_path: Option, + is_flow: bool, + w_id: &str, + on_success_path: &str, + result: Json<&'a T>, + started_at: Option>, + extra_args: Option>>, + email: &str, + priority: Option, +) -> windmill_common::error::Result { + let (payload, tag, on_behalf_of) = + get_payload_tag_from_prefixed_path(on_success_path, db, w_id).await?; + + let mut extra = HashMap::new(); + if let Some(schedule_path) = schedule_path { + extra.insert("schedule_path".to_string(), to_raw_value(&schedule_path)); + } + extra.insert("workspace_id".to_string(), to_raw_value(&w_id)); + extra.insert("job_id".to_string(), to_raw_value(&job_id)); + extra.insert("path".to_string(), to_raw_value(&script_path)); + extra.insert("is_flow".to_string(), to_raw_value(&is_flow)); + extra.insert("started_at".to_string(), to_raw_value(&started_at)); + extra.insert("email".to_string(), to_raw_value(&email)); + + if let Some(args_v) = extra_args { + if let Ok(args_m) = serde_json::from_str::>>(args_v.get()) { + extra.extend(args_m); + } else { + return Err(error::Error::ExecutionErr( + "args of scripts needs to be dict".to_string(), + )); + } + } + + let result = sanitize_result(result); + + let (email, permissioned_as) = if let Some(on_behalf_of) = on_behalf_of.as_ref() { + ( + on_behalf_of.email.as_str(), + on_behalf_of.permissioned_as.clone(), + ) + } else { + (SUCCESS_HANDLER_USER_EMAIL, SUCCESS_HANDLER_USER_GROUP.to_string()) + }; + + let tx = PushIsolationLevel::IsolatedRoot(db.clone()); + let (uuid, tx) = push( + &db, + tx, + w_id, + payload, + PushArgs { extra: Some(extra), args: &result }, + SUCCESS_HANDLER_USERNAME, + email, + permissioned_as, + Some(&format!("success.handler.{job_id}")), + None, + None, + Some(job_id), // parent_job + Some(job_id), // root_job + Some(job_id), // flow_innermost_root_job + None, + false, + false, + None, + true, + tag, + None, + None, + priority, + None, + false, + None, + None, + None, + ) + .await?; + tx.commit().await?; + return Ok(uuid); +} + fn sanitize_result(result: Json<&T>) -> HashMap> { let as_str = serde_json::to_string(result.0).unwrap_or_else(|_| "{}".to_string()); serde_json::from_str::>>(&as_str) diff --git a/backend/windmill-worker/src/handle_child.rs b/backend/windmill-worker/src/handle_child.rs index 18e053116a..8de0899c04 100644 --- a/backend/windmill-worker/src/handle_child.rs +++ b/backend/windmill-worker/src/handle_child.rs @@ -36,7 +36,6 @@ use std::{io, panic, time::Duration}; use tracing::{trace_span, Instrument}; use uuid::Uuid; -#[cfg(feature = "enterprise")] use windmill_common::job_metrics; #[cfg(target_os = "linux")] @@ -696,7 +695,6 @@ where let mut i = 0; - #[cfg(feature = "enterprise")] let mut memory_metric_id: Result = Err(Error::NotFound("not yet initialized".to_string())); @@ -731,32 +729,26 @@ where let update_job_row = i == 2 || (!*SLOW_LOGS && (i < 20 || (i < 120 && i % 5 == 0) || i % 10 == 0)) || i % 20 == 0; - if update_job_row { - #[cfg(feature = "enterprise")] - { - if job_id != Uuid::nil() { - if let Connection::Sql(ref db) = conn { - // tracking metric starting at i >= 2 b/c first point it useless and we don't want to track metric for super fast jobs - if i == 2 { - memory_metric_id = job_metrics::register_metric_for_job( - &db, - w_id.to_string(), - job_id, - "memory_kb".to_string(), - job_metrics::MetricKind::TimeseriesInt, - Some("Job Memory Footprint (kB)".to_string()), - ) - .await; - } - if let Ok(ref metric_id) = memory_metric_id { - if let Err(err) = job_metrics::record_metric(&db, w_id.to_string(), job_id, metric_id.to_owned(), job_metrics::MetricNumericValue::Integer(current_mem)).await { - tracing::error!("Unable to save memory stat for job {} in workspace {}. Error was: {:?}", job_id, w_id, err); - } + if update_job_row && job_id != Uuid::nil() { + if let Connection::Sql(ref db) = conn { + // tracking metric starting at i >= 2 b/c first point it useless and we don't want to track metric for super fast jobs + if i == 2 { + memory_metric_id = job_metrics::register_metric_for_job( + &db, + w_id.to_string(), + job_id, + "memory_kb".to_string(), + job_metrics::MetricKind::TimeseriesInt, + Some("Job Memory Footprint (kB)".to_string()), + ) + .await; + } + if let Ok(ref metric_id) = memory_metric_id { + if let Err(err) = job_metrics::record_metric(&db, w_id.to_string(), job_id, metric_id.to_owned(), job_metrics::MetricNumericValue::Integer(current_mem)).await { + tracing::error!("Unable to save memory stat for job {} in workspace {}. Error was: {:?}", job_id, w_id, err); } } } - } - if job_id != Uuid::nil() { if matches!(conn, Connection::Http(_)) { if i % 4 != 0 { // only ping every 4th time (2s) on http agent mode @@ -764,8 +756,8 @@ where } } let ping_job_status = ping_job_status(&conn, &job_id, Some(*mem_peak), if current_mem > 0 { Some(current_mem) } else { None }).await.unwrap_or_else(|e| { - tracing::error!("Unable to ping job status for job {job_id}. Error was: {:?}", e); - PingJobStatusResponse { + tracing::error!("Unable to ping job status for job {job_id}. Error was: {:?}", e); + PingJobStatusResponse { canceled_by: None, canceled_reason: None, already_completed: false, @@ -782,7 +774,6 @@ where break } } - } }, ); } diff --git a/backend/windmill-worker/src/mssql_executor.rs b/backend/windmill-worker/src/mssql_executor.rs index 6d809969cb..d9f1dda928 100644 --- a/backend/windmill-worker/src/mssql_executor.rs +++ b/backend/windmill-worker/src/mssql_executor.rs @@ -230,7 +230,7 @@ pub async fn do_mssql( let rows_stream = async_stream::stream! { let mut stream = prepared_query.query(&mut client).await.map_err(to_anyhow)?.into_row_stream().map(|row| { let raw_value = row_to_json(row.map_err(to_anyhow)?).map_err(to_anyhow); - let json = raw_value.and_then(|raw_value| serde_json::to_value(raw_value.get()).map_err(to_anyhow)); + let json = raw_value.and_then(|raw_value| serde_json::from_str(raw_value.get()).map_err(to_anyhow)); json }); while let Some(row) = stream.next().await { diff --git a/benchmarks/lib.ts b/benchmarks/lib.ts index dd456fccd1..07cd3c0364 100644 --- a/benchmarks/lib.ts +++ b/benchmarks/lib.ts @@ -2,7 +2,7 @@ import { sleep } from "https://deno.land/x/sleep@v1.2.1/mod.ts"; import * as windmill from "https://deno.land/x/windmill@v1.174.0/mod.ts"; import * as api from "https://deno.land/x/windmill@v1.174.0/windmill-api/index.ts"; -export const VERSION = "v1.608.0"; +export const VERSION = "v1.611.0"; export async function login(email: string, password: string): Promise { return await windmill.UserService.login({ diff --git a/cli/src/core/specific_items.ts b/cli/src/core/specific_items.ts index bef871b598..a81652c4db 100644 --- a/cli/src/core/specific_items.ts +++ b/cli/src/core/specific_items.ts @@ -8,6 +8,7 @@ export interface SpecificItemsConfig { variables?: string[]; resources?: string[]; triggers?: string[]; + folders?: string[]; } // Define all branch-specific file types (computed lazily) @@ -15,6 +16,7 @@ function getBranchSpecificTypes() { return { variable: '.variable.yaml', resource: '.resource.yaml', + folder: '/folder.meta.yaml', // Generate trigger patterns from the list ...Object.fromEntries( TRIGGER_TYPES.map(t => [`${t}_trigger`, `.${t}_trigger.yaml`]) @@ -142,6 +144,16 @@ export function isSpecificItem(path: string, specificItems: SpecificItemsConfig return specificItems.triggers ? matchesPatterns(path, specificItems.triggers) : false; } + // Check for folder meta files + if (path.endsWith('/folder.meta.yaml')) { + if (specificItems.folders) { + // Match against the folder path (without /folder.meta.yaml) + const folderPath = path.slice(0, -'/folder.meta.yaml'.length); + return matchesPatterns(folderPath, specificItems.folders); + } + return false; + } + // Check for resource files using the standard detection function if (isFileResource(path)) { // Extract the base path without the file extension to match against patterns @@ -197,7 +209,20 @@ export function fromBranchSpecificPath(branchSpecificPath: string, branchName: s const sanitizedBranchName = branchName.replace(/[\/\\:*?"<>|.]/g, '_'); const escapedBranchName = sanitizedBranchName.replace(/[.*+?^${}()|[\]\\]/g, '\\$&'); - // Check for resource file pattern first + // Check for folder pattern: path.branchName/folder.meta.yaml -> path/folder.meta.yaml + const folderPattern = new RegExp(`\\.${escapedBranchName}(/folder\\.meta\\.yaml)$`); + const folderMatch = branchSpecificPath.match(folderPattern); + + if (folderMatch) { + const extension = folderMatch[1]; + const pathWithoutBranchAndExtension = branchSpecificPath.substring( + 0, + branchSpecificPath.length - `.${sanitizedBranchName}${extension}`.length + ); + return `${pathWithoutBranchAndExtension}${extension}`; + } + + // Check for resource file pattern const resourceFilePattern = new RegExp(`\\.${escapedBranchName}(\\.resource\\.file\\..+)$`); const resourceFileMatch = branchSpecificPath.match(resourceFilePattern); @@ -283,7 +308,11 @@ export function isCurrentBranchFile(path: string, branchOverride?: string): bool // Use cached pattern or create and cache new one let pattern = branchPatternCache.get(currentBranch); if (!pattern) { - pattern = new RegExp(`\\.${escapedBranchName}\\.${buildYamlTypePattern()}\\.yaml$|\\.${escapedBranchName}\\.resource\\.file\\..+$`); + pattern = new RegExp( + `\\.${escapedBranchName}\\.${buildYamlTypePattern()}\\.yaml$|` + + `\\.${escapedBranchName}\\.resource\\.file\\..+$|` + + `\\.${escapedBranchName}/folder\\.meta\\.yaml$` + ); branchPatternCache.set(currentBranch, pattern); } @@ -296,5 +325,9 @@ export function isCurrentBranchFile(path: string, branchOverride?: string): bool */ export function isBranchSpecificFile(path: string): boolean { const yamlTypePattern = buildYamlTypePattern(); - return new RegExp(`\\.[^.]+\\.${yamlTypePattern}\\.yaml$|\\.[^.]+\\.resource\\.file\\..+$`).test(path); + return new RegExp( + `\\.[^.]+\\.${yamlTypePattern}\\.yaml$|` + + `\\.[^.]+\\.resource\\.file\\..+$|` + + `\\.[^.]+/folder\\.meta\\.yaml$` + ).test(path); } diff --git a/cli/src/main.ts b/cli/src/main.ts index 849c584297..d1797ecd28 100644 --- a/cli/src/main.ts +++ b/cli/src/main.ts @@ -70,7 +70,7 @@ export { // } // }); -export const VERSION = "1.608.0"; +export const VERSION = "1.611.0"; // Re-exported from constants.ts to maintain backwards compatibility export { WM_FORK_PREFIX } from "./core/constants.ts"; diff --git a/cli/test/specific_items.test.ts b/cli/test/specific_items.test.ts index 4448f32db7..a638540657 100644 --- a/cli/test/specific_items.test.ts +++ b/cli/test/specific_items.test.ts @@ -336,3 +336,56 @@ Deno.test("branchOverride: getSpecificItemsForCurrentBranch merges common and br assertEquals(result?.resources, ["shared/**"]); assertEquals(result?.triggers, ["dev/triggers/**"]); }); + +// ============================================================================= +// FOLDER BRANCH-SPECIFIC TESTS +// ============================================================================= + +Deno.test("toBranchSpecificPath: converts folder path to branch-specific", () => { + const result = toBranchSpecificPath("f/my_folder/folder.meta.yaml", "main"); + assertEquals(result, "f/my_folder.main/folder.meta.yaml"); +}); + +Deno.test("toBranchSpecificPath: converts nested folder path to branch-specific", () => { + const result = toBranchSpecificPath("f/parent/child/folder.meta.yaml", "develop"); + assertEquals(result, "f/parent/child.develop/folder.meta.yaml"); +}); + +Deno.test("fromBranchSpecificPath: converts branch-specific folder back to base", () => { + const result = fromBranchSpecificPath("f/my_folder.main/folder.meta.yaml", "main"); + assertEquals(result, "f/my_folder/folder.meta.yaml"); +}); + +Deno.test("fromBranchSpecificPath: handles sanitized branch names for folders", () => { + const result = fromBranchSpecificPath("f/my_folder.feature_test/folder.meta.yaml", "feature/test"); + assertEquals(result, "f/my_folder/folder.meta.yaml"); +}); + +Deno.test("isSpecificItem: matches folder paths with glob pattern", () => { + const config: SpecificItemsConfig = { + folders: ["f/env_*"], + }; + assertEquals(isSpecificItem("f/env_staging/folder.meta.yaml", config), true); + assertEquals(isSpecificItem("f/env_production/folder.meta.yaml", config), true); + assertEquals(isSpecificItem("f/other/folder.meta.yaml", config), false); +}); + +Deno.test("isBranchSpecificFile: detects branch-specific folder files", () => { + assertEquals(isBranchSpecificFile("f/my_folder.main/folder.meta.yaml"), true); + assertEquals(isBranchSpecificFile("f/my_folder.develop/folder.meta.yaml"), true); + assertEquals(isBranchSpecificFile("f/my_folder/folder.meta.yaml"), false); +}); + +Deno.test("isCurrentBranchFile: detects branch-specific folder for current branch", () => { + assertEquals(isCurrentBranchFile("f/my_folder.staging/folder.meta.yaml", "staging"), true); + assertEquals(isCurrentBranchFile("f/my_folder.staging/folder.meta.yaml", "production"), false); + assertEquals(isCurrentBranchFile("f/my_folder/folder.meta.yaml", "staging"), false); +}); + +Deno.test("round-trip: folder path conversion", () => { + const original = "f/configs/env_folder/folder.meta.yaml"; + const branch = "feature/new-env"; + const branchSpecific = toBranchSpecificPath(original, branch); + const restored = fromBranchSpecificPath(branchSpecific, branch); + assertEquals(restored, original); +}); diff --git a/frontend/package-lock.json b/frontend/package-lock.json index 931b6f0fdd..c60dade046 100644 --- a/frontend/package-lock.json +++ b/frontend/package-lock.json @@ -1,12 +1,12 @@ { "name": "windmill-components", - "version": "1.608.0", + "version": "1.611.0", "lockfileVersion": 3, "requires": true, "packages": { "": { "name": "windmill-components", - "version": "1.608.0", + "version": "1.611.0", "hasInstallScript": true, "license": "AGPL-3.0", "dependencies": { diff --git a/frontend/package.json b/frontend/package.json index 111af50ebe..33d7756634 100644 --- a/frontend/package.json +++ b/frontend/package.json @@ -1,6 +1,6 @@ { "name": "windmill-components", - "version": "1.608.0", + "version": "1.611.0", "scripts": { "dev": "vite dev", "build": "vite build", diff --git a/frontend/src/lib/ata/apis.ts b/frontend/src/lib/ata/apis.ts index b2a18d3fec..d54ca5d671 100644 --- a/frontend/src/lib/ata/apis.ts +++ b/frontend/src/lib/ata/apis.ts @@ -1,21 +1,84 @@ // https://github.com/jsdelivr/data.jsdelivr.com import pLimit from 'p-limit' +import { workspaceStore } from '$lib/stores' +import { get } from 'svelte/store' -export const getNPMVersionsForModule = (moduleName: string, resLimit: ResLimit) => { - const url = `https://data.jsdelivr.com/v1/package/npm/${moduleName}` - return api<{ tags: Record; versions: string[] }>(url, resLimit, { - cache: 'no-store' - }) +// Backend proxy fallback functions +const getBackendProxyUrl = () => { + const workspace = get(workspaceStore) + if (!workspace) { + throw new Error('No workspace available') + } + return `/api/w/${workspace}/npm_proxy` } -export const getNPMVersionForModuleReference = ( +const backendProxyApi = async (endpoint: string, resLimit: ResLimit): Promise => { + if (isOverlimit(resLimit)) { + console.warn( + `Exceeded limit of types downloaded for the needs of the assistant fetching: ${endpoint}, ${resLimit.usage}` + ) + return new Error('Exceeded limit of 100MB of data downloaded.') + } + + try { + const baseUrl = getBackendProxyUrl() + const url = `${baseUrl}${endpoint}` + + return limit(() => + fetch(url, { credentials: 'include' }).then((res) => { + if (res.ok) { + return res.text().then((text) => { + resLimit.usage += text.length + console.log('resLimit (backend proxy)', url, resLimit.usage) + return JSON.parse(text) as T + }) as Promise + } else { + return new Error('Backend proxy request failed') + } + }) + ) + } catch (e) { + return new Error('Backend proxy not available') + } +} + +export const getNPMVersionsForModule = async (moduleName: string, resLimit: ResLimit) => { + const url = `https://data.jsdelivr.com/v1/package/npm/${moduleName}` + const result = await api<{ tags: Record; versions: string[] }>(url, resLimit, { + cache: 'no-store' + }) + + // If jsdelivr fails, try backend proxy + if (result instanceof Error) { + console.log('jsdelivr failed for', moduleName, 'trying backend proxy') + return backendProxyApi<{ tags: Record; versions: string[] }>( + `/metadata/${encodeURIComponent(moduleName)}`, + resLimit + ) + } + + return result +} + +export const getNPMVersionForModuleReference = async ( moduleName: string, reference: string, resLimit: ResLimit ) => { const url = `https://data.jsdelivr.com/v1/package/resolve/npm/${moduleName}@${reference}` - return api<{ version: string | null }>(url, resLimit) + const result = await api<{ version: string | null }>(url, resLimit) + + // If jsdelivr fails, try backend proxy + if (result instanceof Error) { + console.log('jsdelivr failed for', moduleName, reference, 'trying backend proxy') + return backendProxyApi<{ version: string | null }>( + `/resolve/${encodeURIComponent(moduleName)}?tag=${encodeURIComponent(reference)}`, + resLimit + ) + } + + return result } export type NPMTreeMeta = { @@ -35,7 +98,22 @@ export const getFiletreeForModuleWithVersion = async ( const url = `https://data.jsdelivr.com/v1/package/npm/${moduleName}@${version}/flat` const res = await api(url, resLimit) if (res instanceof Error) { - return res + // Try backend proxy + console.log('jsdelivr failed for', moduleName, version, 'trying backend proxy') + const backendRes = await backendProxyApi( + `/filetree/${encodeURIComponent(moduleName)}/${encodeURIComponent(version)}`, + resLimit + ) + if (backendRes instanceof Error) { + return res + } else { + return { + ...backendRes, + moduleName, + version, + raw + } + } } else { return { ...res, @@ -57,6 +135,18 @@ export const getDTSFileForModuleWithVersion = async ( if (res.ok) { return res.text() } else { + // Try backend proxy + console.log('jsdelivr failed for file', moduleName, version, file, 'trying backend proxy') + try { + const baseUrl = getBackendProxyUrl() + const proxyUrl = `${baseUrl}/file/${encodeURIComponent(moduleName)}/${encodeURIComponent(version)}${file}` + const proxyRes = await limit(() => fetch(proxyUrl, { credentials: 'include' })) + if (proxyRes.ok) { + return proxyRes.text() + } + } catch (e) { + console.log('Backend proxy failed for file', e) + } return new Error('OK') } } diff --git a/frontend/src/lib/components/InstanceSetting.svelte b/frontend/src/lib/components/InstanceSetting.svelte index b1920a04ff..42a77b88dd 100644 --- a/frontend/src/lib/components/InstanceSetting.svelte +++ b/frontend/src/lib/components/InstanceSetting.svelte @@ -28,6 +28,7 @@ import EEOnly from './EEOnly.svelte' import CriticalAlertChannels from './instanceSettings/CriticalAlertChannels.svelte' import SmtpSettings from './instanceSettings/SmtpSettings.svelte' + import SecretBackendConfig from './instanceSettings/SecretBackendConfig.svelte' import TextInput from './text_input/TextInput.svelte' import Label from './Label.svelte' @@ -886,6 +887,8 @@ TODO {:else if setting.fieldType == 'smtp_connect'} + {:else if setting.fieldType == 'secret_backend'} + {/if} {#if hasError} diff --git a/frontend/src/lib/components/MemoryFootprintViewer.svelte b/frontend/src/lib/components/MemoryFootprintViewer.svelte index afcdfb82a4..66da99e6e6 100644 --- a/frontend/src/lib/components/MemoryFootprintViewer.svelte +++ b/frontend/src/lib/components/MemoryFootprintViewer.svelte @@ -1,7 +1,7 @@
- {#if !$enterpriseLicense} - - Job metrics are only available on Windmill Enterprise Edition. - - {:else if (jobMemoryStats?.length ?? 0) === 0} + {#if (jobMemoryStats?.length ?? 0) === 0} No data points available for this job. Metrics are recorded only for jobs running for more than 500ms.
{#if tab != 'users'} -
+
+
+ + +
+ + + Migrate secrets between the database and HashiCorp Vault. Original values are NOT + deleted to allow for rollback. + + +
+ +
+
+ + + +
+

Database → Vault

+

+ Decrypt secrets from database and store in Vault +

+ +
+ + +
+
+ + + +
+

Vault → Database

+

+ Read secrets from Vault and encrypt in database +

+ +
+
+
+
+ + {/if} + + + + { + migrateToVaultModalOpen = false + }} + onConfirmed={migrateSecretsToVault} +> + {#snippet children()} +
+

+ This will migrate all existing secrets from the database to HashiCorp Vault. The process + will: +

+
    +
  1. Read all encrypted secrets from the database
    2. +
  2. Decrypt them using the workspace encryption keys
    3. +
  3. Store them in HashiCorp Vault under the configured mount path
    4. +
+

+ Note: Database values are NOT deleted automatically. You can manually clear them after + verifying the migration was successful. +

+

Are you sure you want to proceed?

+
+ {/snippet} + + + + { + migrateToDatabaseModalOpen = false + }} + onConfirmed={migrateSecretsToDatabase} +> + {#snippet children()} +
+

+ This will migrate all secrets from HashiCorp Vault back to the database. The process will: +

+
    +
  1. List all secrets in Vault for each workspace
    2. +
  2. Read each secret value from Vault
    3. +
  3. Encrypt and store them in the database
    4. +
+

+ Note: Vault values are NOT deleted automatically. Only secrets that already exist in the + database will be updated. +

+

Are you sure you want to proceed?

+
+ {/snippet} + diff --git a/frontend/src/routes/(root)/(logged)/workspace_settings/+page.svelte b/frontend/src/routes/(root)/(logged)/workspace_settings/+page.svelte index 1227746277..b99f4f5c76 100644 --- a/frontend/src/routes/(root)/(logged)/workspace_settings/+page.svelte +++ b/frontend/src/routes/(root)/(logged)/workspace_settings/+page.svelte @@ -32,7 +32,7 @@ isCriticalAlertsUIOpen } from '$lib/stores' import { sendUserToast } from '$lib/toast' - import { clone, emptyString } from '$lib/utils' + import { clone, emptyString, encodeState } from '$lib/utils' import { RotateCw, Save, Slack } from 'lucide-svelte' import PremiumInfo from '$lib/components/settings/PremiumInfo.svelte' @@ -98,6 +98,7 @@ let errorHandlerItemKind: 'flow' | 'script' = $state('script') let errorHandlerExtraArgs: Record = $state({}) let errorHandlerMutedOnCancel: boolean | undefined = $state(undefined) + let successHandlerScriptPath: string | undefined = $state(undefined) let criticalAlertUIMuted: boolean | undefined = $state(undefined) let initialCriticalAlertUIMuted: boolean | undefined = $state(undefined) @@ -347,6 +348,7 @@ errorHandlerSelected = getHandlerType(errorHandlerScriptPath) } errorHandlerExtraArgs = settings.error_handler_extra_args ?? {} + successHandlerScriptPath = (settings.success_handler ?? '').split('/').slice(1).join('/') workspaceDefaultAppPath = settings.default_app s3ResourceSettings = convertBackendSettingsToFrontendSettings( @@ -515,6 +517,26 @@ } } + async function editSuccessHandler() { + if (successHandlerScriptPath) { + await WorkspaceService.editSuccessHandler({ + workspace: $workspaceStore!, + requestBody: { + success_handler: `script/${successHandlerScriptPath}` + } + }) + sendUserToast(`workspace success handler set to ${successHandlerScriptPath}`) + } else { + await WorkspaceService.editSuccessHandler({ + workspace: $workspaceStore!, + requestBody: { + success_handler: undefined + } + }) + sendUserToast(`workspace success handler removed`) + } + } + async function editCriticalAlertMuteSetting() { await SettingService.workspaceMuteCriticalAlertsUi({ workspace: $workspaceStore!, @@ -1149,12 +1171,113 @@ +
+
+ {#if !$enterpriseLicense} + + Workspace success handler is a Windmill Enterprise Edition feature that allows you to + run a script whenever any job in the workspace completes successfully. + + {/if} +
+
+ { + successHandlerScriptPath = ev?.detail?.path + }} + /> + +
+
+ +
+ + {#if successHandlerScriptPath} + + {/if} +
+
+
+ {#if !$enterpriseLicense} + + Workspace critical alerts is a Windmill Enterprise Edition feature that sends + notifications to workspace admins when critical events occur. + + {/if} list[S3Object]: """Sign S3 objects for use by anonymous users in public apps. diff --git a/python-client/wmill_pg/pyproject.toml b/python-client/wmill_pg/pyproject.toml index 714ecb4d38..5889857365 100644 --- a/python-client/wmill_pg/pyproject.toml +++ b/python-client/wmill_pg/pyproject.toml @@ -1,6 +1,6 @@ [tool.poetry] name = "wmill-pg" -version = "1.608.0" +version = "1.611.0" description = "An extension client for the wmill client library focused on pg" license = "Apache-2.0" homepage = "https://windmill.dev" diff --git a/typescript-client/jsr.json b/typescript-client/jsr.json index 0142e9905b..12104e55c4 100644 --- a/typescript-client/jsr.json +++ b/typescript-client/jsr.json @@ -1,6 +1,6 @@ { "name": "@windmill/windmill", - "version": "1.608.0", + "version": "1.611.0", "exports": "./src/index.ts", "publish": { "exclude": ["!src", "./s3Types.ts", "./sqlUtils.ts", "./client.ts"] diff --git a/typescript-client/package.json b/typescript-client/package.json index fd0f744636..6043d7a468 100644 --- a/typescript-client/package.json +++ b/typescript-client/package.json @@ -1,7 +1,7 @@ { "name": "windmill-client", "description": "Windmill SDK client for browsers and Node.js", - "version": "1.608.0", + "version": "1.611.0", "author": "Ruben Fiszel", "license": "Apache 2.0", "devDependencies": { diff --git a/version.txt b/version.txt index b1fa1afd25..cc23b145bf 100644 --- a/version.txt +++ b/version.txt @@ -1 +1 @@ -1.608.0 +1.611.0