From d34fe303fd7f04a5b990bbbc233b15bd325679a6 Mon Sep 17 00:00:00 2001 From: Ruben Fiszel Date: Sun, 18 Jan 2026 23:08:29 +0000 Subject: [PATCH] feat: add HashiCorp Vault secret storage integration (#7599) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * feat: add HashiCorp Vault secret storage integration - Create SecretBackend trait abstraction for secret storage - Add VaultBackend implementation with CRUD operations - Integrate secret backend into variable CRUD operations - Add migration functions (DB → Vault and Vault → DB) - Add frontend configuration UI for secret backend - Add test connection and migration endpoints --- ...9d39a5a47ca1b6cf3dbfd6988aa0694d7364c.json | 23 + ...a012db51b9e5a255ec0da9a8a0999d668d336.json | 20 + ...84f85ca82636f274f5dce47da5a8320a60088.json | 26 + ...bf24a94eb8a52ceb82cc29dade173ad87569d.json | 23 + ...54b63f8c06c9143b16ce399170c1b5a6b911e.json | 29 + ...aa6f80ecf1628bd2c073d7a237dea9b8e0c65.json | 23 + ...27caa754424e062427edf5ddcb95f9d96888e.json | 29 + ...87f63348a92908e2ca52f5a092149191e6200.json | 16 + ...153c43903f929ae5d62fbba12610f89c36d55.json | 2 +- ...8fe1541bd3f22634af341849303de84c0034d.json | 32 + ...5daa1653e14b6d12013f6e3bb8bf042447a6a.json | 12 + ...3cd956e4c807c3a0031913dbfded80f0e5881.json | 12 + ...fba6ed3b1d573c298f9d7fe8056ba7e32ed81.json | 32 + ...9aae723925270d9d5124cc4cb2c03db61dea8.json | 20 + ...8546bf30074bad903959f7df030e0ac70a86f.json | 32 + ...de229d507784fbcdf46c309907df123e35018.json | 32 + ...fc3113e37dd8009425e558b5f5d1e1543b513.json | 16 + backend/ee-repo-ref.txt | 2 +- backend/tests/fixtures/secret_backend.sql | 31 + backend/tests/secret_backend_integration.rs | 557 ++++++++++++++++++ backend/tests/secret_backend_migration.rs | 323 ++++++++++ backend/windmill-api/Cargo.toml | 2 +- backend/windmill-api/openapi.yaml | 154 +++++ backend/windmill-api/src/lib.rs | 3 + backend/windmill-api/src/resources.rs | 31 +- .../windmill-api/src/secret_backend_ext.rs | 408 +++++++++++++ backend/windmill-api/src/settings.rs | 109 ++++ backend/windmill-api/src/users.rs | 40 ++ backend/windmill-api/src/variables.rs | 97 ++- .../windmill-common/src/global_settings.rs | 1 + backend/windmill-common/src/lib.rs | 1 + .../src/secret_backend/database.rs | 126 ++++ .../windmill-common/src/secret_backend/mod.rs | 133 +++++ .../src/secret_backend/tests.rs | 143 +++++ .../src/secret_backend/vault_oss.rs | 107 ++++ frontend/package-lock.json | 46 +- .../src/lib/components/InstanceSetting.svelte | 3 + .../src/lib/components/instanceSettings.ts | 12 + .../SecretBackendConfig.svelte | 539 +++++++++++++++++ 39 files changed, 3177 insertions(+), 70 deletions(-) create mode 100644 backend/.sqlx/query-020c031c3de6c85577e30421ada9d39a5a47ca1b6cf3dbfd6988aa0694d7364c.json create mode 100644 backend/.sqlx/query-052d42b46d5faba6b41f1fdcbf6a012db51b9e5a255ec0da9a8a0999d668d336.json create mode 100644 backend/.sqlx/query-0600f2a9179f83502c6b13e8e4284f85ca82636f274f5dce47da5a8320a60088.json create mode 100644 backend/.sqlx/query-0c8a3eb810c96230ba3a5466c55bf24a94eb8a52ceb82cc29dade173ad87569d.json create mode 100644 backend/.sqlx/query-146f0e42ada3068a5cdae0ffdbb54b63f8c06c9143b16ce399170c1b5a6b911e.json create mode 100644 backend/.sqlx/query-18aad20ed9cb2dde46f9d899dc4aa6f80ecf1628bd2c073d7a237dea9b8e0c65.json create mode 100644 backend/.sqlx/query-2fcddda99dd0aacf5007ed459cb27caa754424e062427edf5ddcb95f9d96888e.json create mode 100644 backend/.sqlx/query-37e0c601a463819748078b92d1c87f63348a92908e2ca52f5a092149191e6200.json create mode 100644 backend/.sqlx/query-6dd7e38c902f6b8d397aa3e9e698fe1541bd3f22634af341849303de84c0034d.json create mode 100644 backend/.sqlx/query-8f5722afde37d22c56851da23895daa1653e14b6d12013f6e3bb8bf042447a6a.json create mode 100644 backend/.sqlx/query-93ef241a4f624cb76d680418d313cd956e4c807c3a0031913dbfded80f0e5881.json create mode 100644 backend/.sqlx/query-ade0696ec69bec2258d2e3ff86bfba6ed3b1d573c298f9d7fe8056ba7e32ed81.json create mode 100644 backend/.sqlx/query-b030efec7c7f770bf2ed5331a469aae723925270d9d5124cc4cb2c03db61dea8.json create mode 100644 backend/.sqlx/query-b107d602c60c08f0edcaa99695f8546bf30074bad903959f7df030e0ac70a86f.json create mode 100644 backend/.sqlx/query-b10d2ab53bd8d24b0bbcede8211de229d507784fbcdf46c309907df123e35018.json create mode 100644 backend/.sqlx/query-bb289fd24f443f0f8917ec55bc9fc3113e37dd8009425e558b5f5d1e1543b513.json create mode 100644 backend/tests/fixtures/secret_backend.sql create mode 100644 backend/tests/secret_backend_integration.rs create mode 100644 backend/tests/secret_backend_migration.rs create mode 100644 backend/windmill-api/src/secret_backend_ext.rs create mode 100644 backend/windmill-common/src/secret_backend/database.rs create mode 100644 backend/windmill-common/src/secret_backend/mod.rs create mode 100644 backend/windmill-common/src/secret_backend/tests.rs create mode 100644 backend/windmill-common/src/secret_backend/vault_oss.rs create mode 100644 frontend/src/lib/components/instanceSettings/SecretBackendConfig.svelte diff --git a/backend/.sqlx/query-020c031c3de6c85577e30421ada9d39a5a47ca1b6cf3dbfd6988aa0694d7364c.json b/backend/.sqlx/query-020c031c3de6c85577e30421ada9d39a5a47ca1b6cf3dbfd6988aa0694d7364c.json new file mode 100644 index 0000000000..e8edd87bda --- /dev/null +++ b/backend/.sqlx/query-020c031c3de6c85577e30421ada9d39a5a47ca1b6cf3dbfd6988aa0694d7364c.json @@ -0,0 +1,23 @@ +{ + "db_name": "PostgreSQL", + "query": "SELECT value FROM variable WHERE path = $1 AND workspace_id = $2 AND is_secret = true", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "value", + "type_info": "Varchar" + } + ], + "parameters": { + "Left": [ + "Text", + "Text" + ] + }, + "nullable": [ + false + ] + }, + "hash": "020c031c3de6c85577e30421ada9d39a5a47ca1b6cf3dbfd6988aa0694d7364c" +} diff --git a/backend/.sqlx/query-052d42b46d5faba6b41f1fdcbf6a012db51b9e5a255ec0da9a8a0999d668d336.json b/backend/.sqlx/query-052d42b46d5faba6b41f1fdcbf6a012db51b9e5a255ec0da9a8a0999d668d336.json new file mode 100644 index 0000000000..7ddd14bb5c --- /dev/null +++ b/backend/.sqlx/query-052d42b46d5faba6b41f1fdcbf6a012db51b9e5a255ec0da9a8a0999d668d336.json @@ -0,0 +1,20 @@ +{ + "db_name": "PostgreSQL", + "query": "SELECT COUNT(*) as count FROM variable WHERE is_secret = true AND value != 'CLEARED'", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "count", + "type_info": "Int8" + } + ], + "parameters": { + "Left": [] + }, + "nullable": [ + null + ] + }, + "hash": "052d42b46d5faba6b41f1fdcbf6a012db51b9e5a255ec0da9a8a0999d668d336" +} diff --git a/backend/.sqlx/query-0600f2a9179f83502c6b13e8e4284f85ca82636f274f5dce47da5a8320a60088.json b/backend/.sqlx/query-0600f2a9179f83502c6b13e8e4284f85ca82636f274f5dce47da5a8320a60088.json new file mode 100644 index 0000000000..e5aec65b91 --- /dev/null +++ b/backend/.sqlx/query-0600f2a9179f83502c6b13e8e4284f85ca82636f274f5dce47da5a8320a60088.json @@ -0,0 +1,26 @@ +{ + "db_name": "PostgreSQL", + "query": "SELECT workspace_id, path FROM variable WHERE is_secret = true", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "workspace_id", + "type_info": "Varchar" + }, + { + "ordinal": 1, + "name": "path", + "type_info": "Varchar" + } + ], + "parameters": { + "Left": [] + }, + "nullable": [ + false, + false + ] + }, + "hash": "0600f2a9179f83502c6b13e8e4284f85ca82636f274f5dce47da5a8320a60088" +} diff --git a/backend/.sqlx/query-0c8a3eb810c96230ba3a5466c55bf24a94eb8a52ceb82cc29dade173ad87569d.json b/backend/.sqlx/query-0c8a3eb810c96230ba3a5466c55bf24a94eb8a52ceb82cc29dade173ad87569d.json new file mode 100644 index 0000000000..e68f16713a --- /dev/null +++ b/backend/.sqlx/query-0c8a3eb810c96230ba3a5466c55bf24a94eb8a52ceb82cc29dade173ad87569d.json @@ -0,0 +1,23 @@ +{ + "db_name": "PostgreSQL", + "query": "SELECT path FROM variable WHERE path = ANY($1) AND workspace_id = $2 AND is_secret = true", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "path", + "type_info": "Varchar" + } + ], + "parameters": { + "Left": [ + "TextArray", + "Text" + ] + }, + "nullable": [ + false + ] + }, + "hash": "0c8a3eb810c96230ba3a5466c55bf24a94eb8a52ceb82cc29dade173ad87569d" +} diff --git a/backend/.sqlx/query-146f0e42ada3068a5cdae0ffdbb54b63f8c06c9143b16ce399170c1b5a6b911e.json b/backend/.sqlx/query-146f0e42ada3068a5cdae0ffdbb54b63f8c06c9143b16ce399170c1b5a6b911e.json new file mode 100644 index 0000000000..9fd6daded5 --- /dev/null +++ b/backend/.sqlx/query-146f0e42ada3068a5cdae0ffdbb54b63f8c06c9143b16ce399170c1b5a6b911e.json @@ -0,0 +1,29 @@ +{ + "db_name": "PostgreSQL", + "query": "SELECT value, is_secret FROM variable WHERE path = $1 AND workspace_id = $2", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "value", + "type_info": "Varchar" + }, + { + "ordinal": 1, + "name": "is_secret", + "type_info": "Bool" + } + ], + "parameters": { + "Left": [ + "Text", + "Text" + ] + }, + "nullable": [ + false, + false + ] + }, + "hash": "146f0e42ada3068a5cdae0ffdbb54b63f8c06c9143b16ce399170c1b5a6b911e" +} diff --git a/backend/.sqlx/query-18aad20ed9cb2dde46f9d899dc4aa6f80ecf1628bd2c073d7a237dea9b8e0c65.json b/backend/.sqlx/query-18aad20ed9cb2dde46f9d899dc4aa6f80ecf1628bd2c073d7a237dea9b8e0c65.json new file mode 100644 index 0000000000..d0c8c1fe6b --- /dev/null +++ b/backend/.sqlx/query-18aad20ed9cb2dde46f9d899dc4aa6f80ecf1628bd2c073d7a237dea9b8e0c65.json @@ -0,0 +1,23 @@ +{ + "db_name": "PostgreSQL", + "query": "SELECT is_secret FROM variable WHERE path = $1 AND workspace_id = $2", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "is_secret", + "type_info": "Bool" + } + ], + "parameters": { + "Left": [ + "Text", + "Text" + ] + }, + "nullable": [ + false + ] + }, + "hash": "18aad20ed9cb2dde46f9d899dc4aa6f80ecf1628bd2c073d7a237dea9b8e0c65" +} diff --git a/backend/.sqlx/query-2fcddda99dd0aacf5007ed459cb27caa754424e062427edf5ddcb95f9d96888e.json b/backend/.sqlx/query-2fcddda99dd0aacf5007ed459cb27caa754424e062427edf5ddcb95f9d96888e.json new file mode 100644 index 0000000000..5a1fbca834 --- /dev/null +++ b/backend/.sqlx/query-2fcddda99dd0aacf5007ed459cb27caa754424e062427edf5ddcb95f9d96888e.json @@ -0,0 +1,29 @@ +{ + "db_name": "PostgreSQL", + "query": "SELECT path, value FROM variable\n WHERE path LIKE ('u/' || $1 || '/%')\n AND workspace_id = $2\n AND is_secret = true\n AND value LIKE '$vault:%'", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "path", + "type_info": "Varchar" + }, + { + "ordinal": 1, + "name": "value", + "type_info": "Varchar" + } + ], + "parameters": { + "Left": [ + "Text", + "Text" + ] + }, + "nullable": [ + false, + false + ] + }, + "hash": "2fcddda99dd0aacf5007ed459cb27caa754424e062427edf5ddcb95f9d96888e" +} diff --git a/backend/.sqlx/query-37e0c601a463819748078b92d1c87f63348a92908e2ca52f5a092149191e6200.json b/backend/.sqlx/query-37e0c601a463819748078b92d1c87f63348a92908e2ca52f5a092149191e6200.json new file mode 100644 index 0000000000..d1f3f24b82 --- /dev/null +++ b/backend/.sqlx/query-37e0c601a463819748078b92d1c87f63348a92908e2ca52f5a092149191e6200.json @@ -0,0 +1,16 @@ +{ + "db_name": "PostgreSQL", + "query": "UPDATE variable SET value = $1 WHERE path = $2 AND workspace_id = $3 AND is_secret = true", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "Varchar", + "Text", + "Text" + ] + }, + "nullable": [] + }, + "hash": "37e0c601a463819748078b92d1c87f63348a92908e2ca52f5a092149191e6200" +} diff --git a/backend/.sqlx/query-5a219a2532517869578c4504ff3153c43903f929ae5d62fbba12610f89c36d55.json b/backend/.sqlx/query-5a219a2532517869578c4504ff3153c43903f929ae5d62fbba12610f89c36d55.json index 713ccb9dd3..36ddb8ab9f 100644 --- a/backend/.sqlx/query-5a219a2532517869578c4504ff3153c43903f929ae5d62fbba12610f89c36d55.json +++ b/backend/.sqlx/query-5a219a2532517869578c4504ff3153c43903f929ae5d62fbba12610f89c36d55.json @@ -15,7 +15,7 @@ ] }, "nullable": [ - null + true ] }, "hash": "5a219a2532517869578c4504ff3153c43903f929ae5d62fbba12610f89c36d55" diff --git a/backend/.sqlx/query-6dd7e38c902f6b8d397aa3e9e698fe1541bd3f22634af341849303de84c0034d.json b/backend/.sqlx/query-6dd7e38c902f6b8d397aa3e9e698fe1541bd3f22634af341849303de84c0034d.json new file mode 100644 index 0000000000..0107528cdd --- /dev/null +++ b/backend/.sqlx/query-6dd7e38c902f6b8d397aa3e9e698fe1541bd3f22634af341849303de84c0034d.json @@ -0,0 +1,32 @@ +{ + "db_name": "PostgreSQL", + "query": "SELECT workspace_id, path, value FROM variable WHERE is_secret = true", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "workspace_id", + "type_info": "Varchar" + }, + { + "ordinal": 1, + "name": "path", + "type_info": "Varchar" + }, + { + "ordinal": 2, + "name": "value", + "type_info": "Varchar" + } + ], + "parameters": { + "Left": [] + }, + "nullable": [ + false, + false, + false + ] + }, + "hash": "6dd7e38c902f6b8d397aa3e9e698fe1541bd3f22634af341849303de84c0034d" +} diff --git a/backend/.sqlx/query-8f5722afde37d22c56851da23895daa1653e14b6d12013f6e3bb8bf042447a6a.json b/backend/.sqlx/query-8f5722afde37d22c56851da23895daa1653e14b6d12013f6e3bb8bf042447a6a.json new file mode 100644 index 0000000000..a298c5fb8c --- /dev/null +++ b/backend/.sqlx/query-8f5722afde37d22c56851da23895daa1653e14b6d12013f6e3bb8bf042447a6a.json @@ -0,0 +1,12 @@ +{ + "db_name": "PostgreSQL", + "query": "UPDATE variable SET value = 'ROUND_TRIP_CLEARED' WHERE is_secret = true", + "describe": { + "columns": [], + "parameters": { + "Left": [] + }, + "nullable": [] + }, + "hash": "8f5722afde37d22c56851da23895daa1653e14b6d12013f6e3bb8bf042447a6a" +} diff --git a/backend/.sqlx/query-93ef241a4f624cb76d680418d313cd956e4c807c3a0031913dbfded80f0e5881.json b/backend/.sqlx/query-93ef241a4f624cb76d680418d313cd956e4c807c3a0031913dbfded80f0e5881.json new file mode 100644 index 0000000000..288f40474f --- /dev/null +++ b/backend/.sqlx/query-93ef241a4f624cb76d680418d313cd956e4c807c3a0031913dbfded80f0e5881.json @@ -0,0 +1,12 @@ +{ + "db_name": "PostgreSQL", + "query": "UPDATE variable SET value = 'CLEARED' WHERE is_secret = true", + "describe": { + "columns": [], + "parameters": { + "Left": [] + }, + "nullable": [] + }, + "hash": "93ef241a4f624cb76d680418d313cd956e4c807c3a0031913dbfded80f0e5881" +} diff --git a/backend/.sqlx/query-ade0696ec69bec2258d2e3ff86bfba6ed3b1d573c298f9d7fe8056ba7e32ed81.json b/backend/.sqlx/query-ade0696ec69bec2258d2e3ff86bfba6ed3b1d573c298f9d7fe8056ba7e32ed81.json new file mode 100644 index 0000000000..3fe61f6f8e --- /dev/null +++ b/backend/.sqlx/query-ade0696ec69bec2258d2e3ff86bfba6ed3b1d573c298f9d7fe8056ba7e32ed81.json @@ -0,0 +1,32 @@ +{ + "db_name": "PostgreSQL", + "query": "SELECT workspace_id, path, value FROM variable WHERE is_secret = true AND value IS NOT NULL AND value != ''", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "workspace_id", + "type_info": "Varchar" + }, + { + "ordinal": 1, + "name": "path", + "type_info": "Varchar" + }, + { + "ordinal": 2, + "name": "value", + "type_info": "Varchar" + } + ], + "parameters": { + "Left": [] + }, + "nullable": [ + false, + false, + false + ] + }, + "hash": "ade0696ec69bec2258d2e3ff86bfba6ed3b1d573c298f9d7fe8056ba7e32ed81" +} diff --git a/backend/.sqlx/query-b030efec7c7f770bf2ed5331a469aae723925270d9d5124cc4cb2c03db61dea8.json b/backend/.sqlx/query-b030efec7c7f770bf2ed5331a469aae723925270d9d5124cc4cb2c03db61dea8.json new file mode 100644 index 0000000000..8a0de9e06e --- /dev/null +++ b/backend/.sqlx/query-b030efec7c7f770bf2ed5331a469aae723925270d9d5124cc4cb2c03db61dea8.json @@ -0,0 +1,20 @@ +{ + "db_name": "PostgreSQL", + "query": "SELECT COUNT(*) as count FROM variable WHERE is_secret = true AND value = 'CLEARED'", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "count", + "type_info": "Int8" + } + ], + "parameters": { + "Left": [] + }, + "nullable": [ + null + ] + }, + "hash": "b030efec7c7f770bf2ed5331a469aae723925270d9d5124cc4cb2c03db61dea8" +} diff --git a/backend/.sqlx/query-b107d602c60c08f0edcaa99695f8546bf30074bad903959f7df030e0ac70a86f.json b/backend/.sqlx/query-b107d602c60c08f0edcaa99695f8546bf30074bad903959f7df030e0ac70a86f.json new file mode 100644 index 0000000000..705e0b4c5b --- /dev/null +++ b/backend/.sqlx/query-b107d602c60c08f0edcaa99695f8546bf30074bad903959f7df030e0ac70a86f.json @@ -0,0 +1,32 @@ +{ + "db_name": "PostgreSQL", + "query": "SELECT workspace_id, path, value FROM variable WHERE is_secret = true AND value != 'CLEARED' ORDER BY workspace_id, path", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "workspace_id", + "type_info": "Varchar" + }, + { + "ordinal": 1, + "name": "path", + "type_info": "Varchar" + }, + { + "ordinal": 2, + "name": "value", + "type_info": "Varchar" + } + ], + "parameters": { + "Left": [] + }, + "nullable": [ + false, + false, + false + ] + }, + "hash": "b107d602c60c08f0edcaa99695f8546bf30074bad903959f7df030e0ac70a86f" +} diff --git a/backend/.sqlx/query-b10d2ab53bd8d24b0bbcede8211de229d507784fbcdf46c309907df123e35018.json b/backend/.sqlx/query-b10d2ab53bd8d24b0bbcede8211de229d507784fbcdf46c309907df123e35018.json new file mode 100644 index 0000000000..534eea5676 --- /dev/null +++ b/backend/.sqlx/query-b10d2ab53bd8d24b0bbcede8211de229d507784fbcdf46c309907df123e35018.json @@ -0,0 +1,32 @@ +{ + "db_name": "PostgreSQL", + "query": "SELECT workspace_id, path, value FROM variable WHERE is_secret = true ORDER BY workspace_id, path", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "workspace_id", + "type_info": "Varchar" + }, + { + "ordinal": 1, + "name": "path", + "type_info": "Varchar" + }, + { + "ordinal": 2, + "name": "value", + "type_info": "Varchar" + } + ], + "parameters": { + "Left": [] + }, + "nullable": [ + false, + false, + false + ] + }, + "hash": "b10d2ab53bd8d24b0bbcede8211de229d507784fbcdf46c309907df123e35018" +} diff --git a/backend/.sqlx/query-bb289fd24f443f0f8917ec55bc9fc3113e37dd8009425e558b5f5d1e1543b513.json b/backend/.sqlx/query-bb289fd24f443f0f8917ec55bc9fc3113e37dd8009425e558b5f5d1e1543b513.json new file mode 100644 index 0000000000..a687e1a7e3 --- /dev/null +++ b/backend/.sqlx/query-bb289fd24f443f0f8917ec55bc9fc3113e37dd8009425e558b5f5d1e1543b513.json @@ -0,0 +1,16 @@ +{ + "db_name": "PostgreSQL", + "query": "UPDATE variable SET value = $1 WHERE path = $2 AND workspace_id = $3", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "Varchar", + "Text", + "Text" + ] + }, + "nullable": [] + }, + "hash": "bb289fd24f443f0f8917ec55bc9fc3113e37dd8009425e558b5f5d1e1543b513" +} diff --git a/backend/ee-repo-ref.txt b/backend/ee-repo-ref.txt index 56e9d8a0be..60e3437748 100644 --- a/backend/ee-repo-ref.txt +++ b/backend/ee-repo-ref.txt @@ -1 +1 @@ -da1518ed54410478ca209f1d06298a1407be38a8 \ No newline at end of file +54a01bfeb32cedb779c8d39ae2fbfd26c8be0482 \ No newline at end of file diff --git a/backend/tests/fixtures/secret_backend.sql b/backend/tests/fixtures/secret_backend.sql new file mode 100644 index 0000000000..0bc5a6f52a --- /dev/null +++ b/backend/tests/fixtures/secret_backend.sql @@ -0,0 +1,31 @@ +-- Fixture for secret backend migration tests +-- Sets up test secrets in the variable table + +-- Create a second workspace for testing workspace isolation +INSERT INTO workspace (id, name, owner) +VALUES ('test-workspace-2', 'test-workspace-2', 'test-user') +ON CONFLICT DO NOTHING; + +INSERT INTO workspace_settings (workspace_id) +VALUES ('test-workspace-2') +ON CONFLICT DO NOTHING; + +INSERT INTO workspace_key(workspace_id, kind, key) +VALUES ('test-workspace-2', 'cloud', 'test-key-2') +ON CONFLICT DO NOTHING; + +-- Insert test secrets for workspace 1 +-- Note: The 'value' column stores encrypted values in production, +-- but for tests we'll use plain text that the migration will handle +INSERT INTO variable (workspace_id, path, value, is_secret, description, extra_perms) +VALUES + ('test-workspace', 'u/test-user/db_password', 'encrypted-db-pass-123', true, 'Database password', '{}'), + ('test-workspace', 'u/test-user/api_key', 'encrypted-api-key-abc', true, 'API key for external service', '{}'), + ('test-workspace', 'u/test-user/public_var', 'not-a-secret', false, 'A non-secret variable', '{}') +ON CONFLICT DO NOTHING; + +-- Insert test secrets for workspace 2 (to test isolation) +INSERT INTO variable (workspace_id, path, value, is_secret, description, extra_perms) +VALUES + ('test-workspace-2', 'u/test-user/other_secret', 'encrypted-other-secret', true, 'Secret in workspace 2', '{}') +ON CONFLICT DO NOTHING; diff --git a/backend/tests/secret_backend_integration.rs b/backend/tests/secret_backend_integration.rs new file mode 100644 index 0000000000..e6deb84ff9 --- /dev/null +++ b/backend/tests/secret_backend_integration.rs @@ -0,0 +1,557 @@ +//! Integration tests for HashiCorp Vault secret backend. +//! +//! These tests require: +//! 1. A PostgreSQL database (handled by sqlx test framework) +//! 2. A running HashiCorp Vault instance +//! 3. The RUN_VAULT_TESTS=1 environment variable to be set +//! +//! Environment variables: +//! - RUN_VAULT_TESTS=1 - Required to run the tests +//! - VAULT_ADDR - Vault server address (default: http://127.0.0.1:8200) +//! - VAULT_TOKEN - Static token for static token tests (default: test-root-token) +//! - BASE_URL - Windmill instance URL for JWT tests (default: http://localhost:8000) +//! +//! Run tests (static token mode): +//! ```bash +//! RUN_VAULT_TESTS=1 VAULT_TOKEN=your-token cargo test -p windmill \ +//! secret_backend_integration --features private,enterprise -- --nocapture +//! ``` +//! +//! Run tests (JWT mode - requires Windmill instance running for JWKS endpoint): +//! ```bash +//! RUN_VAULT_TESTS=1 BASE_URL=http://localhost:8000 cargo test -p windmill \ +//! secret_backend_integration --features private,enterprise,openidconnect -- --nocapture +//! ``` + +#[cfg(all(feature = "private", feature = "enterprise"))] +mod tests { + use sqlx::{Pool, Postgres}; + use std::collections::HashMap; + use windmill_common::secret_backend::{ + migrate_secrets_to_database, migrate_secrets_to_vault, test_vault_connection, + SecretBackend, VaultBackend, VaultSettings, + }; + + /// Check if vault tests should run (requires RUN_VAULT_TESTS=1 env var) + fn should_run_vault_tests() -> bool { + std::env::var("RUN_VAULT_TESTS") + .map(|v| v == "1" || v.to_lowercase() == "true") + .unwrap_or(false) + } + + /// Set up BASE_URL for JWT tests (required for OIDC issuer URL generation) + async fn setup_base_url() { + let base_url = std::env::var("BASE_URL") + .unwrap_or_else(|_| "http://localhost:8000".to_string()); + let mut url = windmill_common::BASE_URL.write().await; + *url = base_url; + } + + /// Skip test if RUN_VAULT_TESTS is not set + macro_rules! skip_if_no_vault { + () => { + if !should_run_vault_tests() { + println!("Skipping test: RUN_VAULT_TESTS=1 not set"); + println!("To run vault tests: RUN_VAULT_TESTS=1 cargo test ..."); + return; + } + }; + } + + fn vault_settings_static_token() -> VaultSettings { + VaultSettings { + address: std::env::var("VAULT_ADDR") + .unwrap_or_else(|_| "http://127.0.0.1:8200".to_string()), + mount_path: "windmill".to_string(), + jwt_role: None, // Static token mode + namespace: None, + token: Some( + std::env::var("VAULT_TOKEN").unwrap_or_else(|_| "test-root-token".to_string()), + ), + } + } + + fn vault_settings_jwt() -> VaultSettings { + VaultSettings { + address: std::env::var("VAULT_ADDR") + .unwrap_or_else(|_| "http://127.0.0.1:8200".to_string()), + mount_path: "windmill".to_string(), + jwt_role: Some("windmill-secrets".to_string()), // JWT mode + namespace: None, + token: None, // No static token - use JWT + } + } + + // ==================== Static Token Tests ==================== + + /// Test Vault connection with static token + #[sqlx::test(fixtures("base", "secret_backend"))] + async fn test_vault_connection_static_token(db: Pool) { + skip_if_no_vault!(); + + let settings = vault_settings_static_token(); + println!("Testing Vault connection with static token..."); + println!(" Address: {}", settings.address); + + let result = test_vault_connection(&settings, Some(&db)).await; + assert!( + result.is_ok(), + "Failed to connect to Vault: {:?}", + result.err() + ); + println!("✓ Successfully connected to Vault with static token"); + } + + /// Test basic CRUD operations with static token + #[sqlx::test(fixtures("base", "secret_backend"))] + async fn test_vault_crud_static_token(_db: Pool) { + skip_if_no_vault!(); + + let settings = vault_settings_static_token(); + let backend = VaultBackend::new(settings); + + let workspace_id = "test-crud-static"; + let path = "test-secret"; + let value = "my-super-secret-value-123"; + + println!("Testing CRUD with static token..."); + + // Create + println!(" Creating secret..."); + backend + .set_secret(workspace_id, path, value) + .await + .expect("Failed to create secret"); + println!(" ✓ Created"); + + // Read + println!(" Reading secret..."); + let read_value = backend + .get_secret(workspace_id, path) + .await + .expect("Failed to read secret"); + assert_eq!(read_value, value); + println!(" ✓ Read (value matches)"); + + // Update + println!(" Updating secret..."); + let new_value = "updated-secret-value-456"; + backend + .set_secret(workspace_id, path, new_value) + .await + .expect("Failed to update secret"); + let updated = backend + .get_secret(workspace_id, path) + .await + .expect("Failed to read updated secret"); + assert_eq!(updated, new_value); + println!(" ✓ Updated"); + + // Delete + println!(" Deleting secret..."); + backend + .delete_secret(workspace_id, path) + .await + .expect("Failed to delete secret"); + let result = backend.get_secret(workspace_id, path).await; + assert!(result.is_err(), "Secret should be deleted"); + println!(" ✓ Deleted"); + + println!("✓ CRUD operations successful with static token"); + } + + // ==================== JWT Auth Tests ==================== + + /// Test Vault connection with JWT authentication + #[sqlx::test(fixtures("base", "secret_backend"))] + async fn test_vault_connection_jwt(db: Pool) { + skip_if_no_vault!(); + setup_base_url().await; + + let settings = vault_settings_jwt(); + println!("Testing Vault connection with JWT auth..."); + println!(" Address: {}", settings.address); + println!(" JWT Role: {:?}", settings.jwt_role); + println!(" BASE_URL: {}", windmill_common::BASE_URL.read().await.clone()); + + let result = test_vault_connection(&settings, Some(&db)).await; + assert!( + result.is_ok(), + "Failed to connect to Vault with JWT: {:?}", + result.err() + ); + println!("✓ Successfully connected to Vault with JWT auth"); + } + + /// Test basic CRUD operations with JWT authentication + #[cfg(feature = "openidconnect")] + #[sqlx::test(fixtures("base", "secret_backend"))] + async fn test_vault_crud_jwt(db: Pool) { + skip_if_no_vault!(); + setup_base_url().await; + + let settings = vault_settings_jwt(); + let backend = VaultBackend::new_with_db(settings, db.clone()); + + let workspace_id = "test-crud-jwt"; + let path = "jwt-test-secret"; + let value = "jwt-authenticated-secret-value"; + + println!("Testing CRUD with JWT auth..."); + + // Create + println!(" Creating secret..."); + backend + .set_secret(workspace_id, path, value) + .await + .expect("Failed to create secret with JWT"); + println!(" ✓ Created"); + + // Read + println!(" Reading secret..."); + let read_value = backend + .get_secret(workspace_id, path) + .await + .expect("Failed to read secret with JWT"); + assert_eq!(read_value, value); + println!(" ✓ Read (value matches)"); + + // Delete (cleanup) + println!(" Deleting secret..."); + backend + .delete_secret(workspace_id, path) + .await + .expect("Failed to delete secret with JWT"); + println!(" ✓ Deleted"); + + println!("✓ CRUD operations successful with JWT auth"); + } + + // ==================== Migration Tests ==================== + + /// Test migration from database to Vault + #[sqlx::test(fixtures("base", "secret_backend"))] + async fn test_migrate_db_to_vault(db: Pool) { + skip_if_no_vault!(); + + let settings = vault_settings_static_token(); + + // Verify Vault connection + test_vault_connection(&settings, Some(&db)) + .await + .expect("Failed to connect to Vault"); + + // Check initial state + let secrets_before = sqlx::query!( + "SELECT workspace_id, path, value FROM variable WHERE is_secret = true ORDER BY workspace_id, path" + ) + .fetch_all(&db) + .await + .expect("Failed to query secrets"); + + println!( + "Found {} secrets in database before migration:", + secrets_before.len() + ); + for s in &secrets_before { + println!(" - {}/{}: {} chars", s.workspace_id, s.path, s.value.len()); + } + + // Run migration + println!("\nMigrating secrets to Vault..."); + let report = migrate_secrets_to_vault(&db, &settings) + .await + .expect("Migration to Vault failed"); + + println!("Migration report:"); + println!(" Total secrets: {}", report.total_secrets); + println!(" Migrated: {}", report.migrated_count); + println!(" Failed: {}", report.failed_count); + + if !report.failures.is_empty() { + println!(" Failures:"); + for f in &report.failures { + println!(" - {}/{}: {}", f.workspace_id, f.path, f.error); + } + } + + assert_eq!(report.failed_count, 0, "Migration had failures"); + assert!(report.migrated_count > 0, "No secrets were migrated"); + + // Verify secrets in Vault + println!("\nVerifying secrets in Vault..."); + let vault_backend = VaultBackend::new(settings.clone()); + + for secret in &secrets_before { + let result = vault_backend + .get_secret(&secret.workspace_id, &secret.path) + .await; + assert!( + result.is_ok(), + "Failed to read secret {}/{} from Vault: {:?}", + secret.workspace_id, + secret.path, + result.err() + ); + println!( + " ✓ {}/{} exists in Vault", + secret.workspace_id, secret.path + ); + } + + println!("\n✓ Migration to Vault completed successfully"); + } + + /// Test migration from Vault back to database + #[sqlx::test(fixtures("base", "secret_backend"))] + async fn test_migrate_vault_to_db(db: Pool) { + skip_if_no_vault!(); + + let settings = vault_settings_static_token(); + + test_vault_connection(&settings, Some(&db)) + .await + .expect("Failed to connect to Vault"); + + // First migrate TO Vault + println!("Setting up: migrating secrets to Vault first..."); + let to_vault = migrate_secrets_to_vault(&db, &settings) + .await + .expect("Initial migration to Vault failed"); + assert!(to_vault.migrated_count > 0, "No secrets to test with"); + println!(" Migrated {} secrets to Vault", to_vault.migrated_count); + + // Clear database values + println!("\nClearing database secret values..."); + sqlx::query!("UPDATE variable SET value = 'CLEARED' WHERE is_secret = true") + .execute(&db) + .await + .expect("Failed to clear values"); + + // Migrate back from Vault + println!("\nMigrating secrets from Vault to database..."); + let report = migrate_secrets_to_database(&db, &settings) + .await + .expect("Migration to database failed"); + + println!("Migration report:"); + println!(" Total secrets: {}", report.total_secrets); + println!(" Migrated: {}", report.migrated_count); + println!(" Failed: {}", report.failed_count); + + assert_eq!(report.failed_count, 0, "Migration had failures"); + assert!(report.migrated_count > 0, "No secrets were migrated"); + + // Verify restored + let restored = sqlx::query!( + "SELECT COUNT(*) as count FROM variable WHERE is_secret = true AND value != 'CLEARED'" + ) + .fetch_one(&db) + .await + .expect("Failed to count restored"); + + assert!( + restored.count.unwrap_or(0) > 0, + "No secrets were restored in database" + ); + + println!("\n✓ Migration to database completed successfully"); + } + + // ==================== Variable Rename Tests ==================== + + /// Test renaming a variable path in Vault + #[sqlx::test(fixtures("base", "secret_backend"))] + async fn test_variable_rename(db: Pool) { + skip_if_no_vault!(); + let _ = &db; // suppress unused warning + + let settings = vault_settings_static_token(); + let backend = VaultBackend::new(settings); + + let workspace_id = "test-workspace"; + let old_path = "u/test-user/old_secret_name"; + let new_path = "u/test-user/new_secret_name"; + let value = "secret-value-for-rename-test"; + + println!("Testing variable rename in Vault..."); + + // Create secret at old path + println!(" Creating secret at old path: {}", old_path); + backend + .set_secret(workspace_id, old_path, value) + .await + .expect("Failed to create secret"); + + // Verify it exists + let read_value = backend + .get_secret(workspace_id, old_path) + .await + .expect("Failed to read secret at old path"); + assert_eq!(read_value, value); + println!(" ✓ Secret exists at old path"); + + // Simulate rename: read from old, write to new, delete old + println!(" Renaming: {} -> {}", old_path, new_path); + let secret_value = backend + .get_secret(workspace_id, old_path) + .await + .expect("Failed to read for rename"); + + backend + .set_secret(workspace_id, new_path, &secret_value) + .await + .expect("Failed to write to new path"); + + backend + .delete_secret(workspace_id, old_path) + .await + .expect("Failed to delete old path"); + + // Verify old path is gone + let old_result = backend.get_secret(workspace_id, old_path).await; + assert!(old_result.is_err(), "Old path should not exist"); + println!(" ✓ Old path deleted"); + + // Verify new path exists with correct value + let new_value = backend + .get_secret(workspace_id, new_path) + .await + .expect("Failed to read new path"); + assert_eq!(new_value, value); + println!(" ✓ New path exists with correct value"); + + // Cleanup + backend + .delete_secret(workspace_id, new_path) + .await + .expect("Failed to cleanup"); + + println!("\n✓ Variable rename completed successfully"); + } + + // ==================== Full Round Trip Test ==================== + + /// Test full round-trip: DB -> Vault -> DB with verification + #[sqlx::test(fixtures("base", "secret_backend"))] + async fn test_full_round_trip(db: Pool) { + skip_if_no_vault!(); + + let settings = vault_settings_static_token(); + + test_vault_connection(&settings, Some(&db)) + .await + .expect("Failed to connect to Vault"); + + // Get original secrets + let original: HashMap<(String, String), String> = sqlx::query!( + "SELECT workspace_id, path, value FROM variable WHERE is_secret = true" + ) + .fetch_all(&db) + .await + .expect("Failed to query") + .into_iter() + .map(|r| ((r.workspace_id, r.path), r.value)) + .collect(); + + println!("Original secrets: {} entries", original.len()); + + // Step 1: DB -> Vault + println!("\n=== Step 1: Migrate DB -> Vault ==="); + let to_vault = migrate_secrets_to_vault(&db, &settings) + .await + .expect("Migration to Vault failed"); + println!("Migrated {} secrets to Vault", to_vault.migrated_count); + assert_eq!(to_vault.failed_count, 0); + + // Step 2: Clear DB + println!("\n=== Step 2: Clear database values ==="); + sqlx::query!("UPDATE variable SET value = 'ROUND_TRIP_CLEARED' WHERE is_secret = true") + .execute(&db) + .await + .expect("Failed to clear"); + + // Step 3: Vault -> DB + println!("\n=== Step 3: Migrate Vault -> DB ==="); + let to_db = migrate_secrets_to_database(&db, &settings) + .await + .expect("Migration to database failed"); + println!("Migrated {} secrets to database", to_db.migrated_count); + assert_eq!(to_db.failed_count, 0); + + // Step 4: Verify + println!("\n=== Step 4: Verify round-trip integrity ==="); + let restored: HashMap<(String, String), String> = sqlx::query!( + "SELECT workspace_id, path, value FROM variable WHERE is_secret = true" + ) + .fetch_all(&db) + .await + .expect("Failed to query") + .into_iter() + .map(|r| ((r.workspace_id, r.path), r.value)) + .collect(); + + for ((ws, path), _) in &original { + let restored_value = restored + .get(&(ws.clone(), path.clone())) + .expect(&format!("Secret {}/{} not found after round-trip", ws, path)); + + assert_ne!( + restored_value, "ROUND_TRIP_CLEARED", + "Secret {}/{} was not restored", + ws, path + ); + println!(" ✓ {}/{}: restored", ws, path); + } + + println!("\n✓ Full round-trip completed successfully!"); + } + + // ==================== Workspace Isolation Test ==================== + + /// Test that workspace isolation is maintained + #[sqlx::test(fixtures("base", "secret_backend"))] + async fn test_workspace_isolation(db: Pool) { + skip_if_no_vault!(); + + let settings = vault_settings_static_token(); + let backend = VaultBackend::new(settings.clone()); + + // First migrate secrets to Vault + migrate_secrets_to_vault(&db, &settings) + .await + .expect("Migration failed"); + + println!("Testing workspace isolation..."); + + // Try to access workspace-2 secret from workspace-1 path (should fail) + let cross_access = backend + .get_secret("test-workspace", "u/test-user/other_secret") + .await; + + assert!( + cross_access.is_err(), + "Cross-workspace access should fail!" + ); + println!("✓ Cross-workspace access correctly denied"); + + // Verify own workspace access works + let ws1 = backend + .get_secret("test-workspace", "u/test-user/db_password") + .await; + assert!(ws1.is_ok(), "Same-workspace access should work"); + println!("✓ Same-workspace access works"); + + println!("\n✓ Workspace isolation verified!"); + } +} + +// OSS version - just a placeholder to avoid compilation errors +#[cfg(not(all(feature = "private", feature = "enterprise")))] +mod tests { + #[test] + fn test_vault_requires_enterprise() { + println!("Vault integration tests require Enterprise Edition features"); + println!("Run with: cargo test --features private,enterprise,openidconnect"); + } +} diff --git a/backend/tests/secret_backend_migration.rs b/backend/tests/secret_backend_migration.rs new file mode 100644 index 0000000000..d78da8603c --- /dev/null +++ b/backend/tests/secret_backend_migration.rs @@ -0,0 +1,323 @@ +//! Integration tests for secret backend migration between database and HashiCorp Vault. +//! +//! These tests require: +//! 1. A PostgreSQL database (handled by sqlx test framework) +//! 2. A running HashiCorp Vault instance at http://127.0.0.1:8200 +//! +//! To run these tests: +//! ```bash +//! # Start Vault in dev mode +//! podman run -d --name vault-test -p 8200:8200 \ +//! -e VAULT_DEV_ROOT_TOKEN_ID=test-root-token \ +//! docker.io/hashicorp/vault:latest +//! +//! # Enable KV v2 secrets engine +//! curl -s -H "X-Vault-Token: test-root-token" -X POST \ +//! --data '{"type":"kv-v2"}' \ +//! http://127.0.0.1:8200/v1/sys/mounts/windmill +//! +//! # Run the tests +//! cargo test -p windmill secret_backend_migration -- --ignored --nocapture +//! ``` + +use sqlx::{Pool, Postgres}; +use windmill_common::error::Result; +use windmill_common::secret_backend::{ + vault_oss::{migrate_secrets_to_database, migrate_secrets_to_vault, test_vault_connection, VaultBackend}, + SecretBackend, VaultSettings, +}; + +mod common; + +fn test_vault_settings() -> VaultSettings { + VaultSettings { + address: std::env::var("VAULT_ADDR").unwrap_or_else(|_| "http://127.0.0.1:8200".to_string()), + mount_path: "windmill".to_string(), + jwt_role: Some("windmill-secrets".to_string()), + namespace: None, + token: Some( + std::env::var("VAULT_TOKEN").unwrap_or_else(|_| "test-root-token".to_string()), + ), + } +} + +/// Test that we can connect to Vault +#[sqlx::test(fixtures("base", "secret_backend"))] +#[ignore = "requires running Vault instance"] +async fn test_vault_connection_works(db: Pool) { + let settings = test_vault_settings(); + + let result = test_vault_connection(&settings, Some(&db)).await; + assert!(result.is_ok(), "Failed to connect to Vault: {:?}", result.err()); + println!("✓ Successfully connected to Vault at {}", settings.address); +} + +/// Test migration from database to Vault +#[sqlx::test(fixtures("base", "secret_backend"))] +#[ignore = "requires running Vault instance"] +async fn test_migrate_db_to_vault(db: Pool) { + let settings = test_vault_settings(); + + // First verify we can connect to Vault + test_vault_connection(&settings, Some(&db)) + .await + .expect("Failed to connect to Vault"); + + // Check initial state - secrets should exist in database + let secrets_before = sqlx::query!( + "SELECT workspace_id, path, value FROM variable WHERE is_secret = true ORDER BY workspace_id, path" + ) + .fetch_all(&db) + .await + .expect("Failed to query secrets"); + + println!("Found {} secrets in database before migration:", secrets_before.len()); + for s in &secrets_before { + println!(" - {}/{}: {} chars", s.workspace_id, s.path, s.value.len()); + } + + // Run migration to Vault + println!("\nMigrating secrets to Vault..."); + let report = migrate_secrets_to_vault(&db, &settings) + .await + .expect("Migration to Vault failed"); + + println!("Migration report:"); + println!(" Total secrets: {}", report.total_secrets); + println!(" Migrated: {}", report.migrated_count); + println!(" Failed: {}", report.failed_count); + + if !report.failures.is_empty() { + println!(" Failures:"); + for f in &report.failures { + println!(" - {}/{}: {}", f.workspace_id, f.path, f.error); + } + } + + assert_eq!(report.failed_count, 0, "Migration had failures"); + assert!(report.migrated_count > 0, "No secrets were migrated"); + + // Verify secrets are in Vault + println!("\nVerifying secrets in Vault..."); + let vault_backend = VaultBackend::new(settings.clone()); + + for secret in &secrets_before { + let result: Result = vault_backend + .get_secret(&secret.workspace_id, &secret.path) + .await; + assert!( + result.is_ok(), + "Failed to read secret {}/{} from Vault: {:?}", + secret.workspace_id, + secret.path, + result.err() + ); + println!(" ✓ {}/{} exists in Vault", secret.workspace_id, secret.path); + } + + println!("\n✓ Migration to Vault completed successfully"); +} + +/// Test migration from Vault to database +#[sqlx::test(fixtures("base", "secret_backend"))] +#[ignore = "requires running Vault instance"] +async fn test_migrate_vault_to_db(db: Pool) { + let settings = test_vault_settings(); + + // First verify we can connect to Vault + test_vault_connection(&settings, Some(&db)) + .await + .expect("Failed to connect to Vault"); + + // First, migrate secrets TO Vault so we have something to migrate back + println!("Setting up: migrating secrets to Vault first..."); + let to_vault_report = migrate_secrets_to_vault(&db, &settings) + .await + .expect("Initial migration to Vault failed"); + assert!(to_vault_report.migrated_count > 0, "No secrets to test with"); + println!(" Migrated {} secrets to Vault", to_vault_report.migrated_count); + + // Clear the database values to simulate fresh migration back + println!("\nClearing database secret values..."); + sqlx::query!("UPDATE variable SET value = 'CLEARED' WHERE is_secret = true") + .execute(&db) + .await + .expect("Failed to clear database values"); + + // Verify they were cleared + let cleared = sqlx::query!( + "SELECT COUNT(*) as count FROM variable WHERE is_secret = true AND value = 'CLEARED'" + ) + .fetch_one(&db) + .await + .expect("Failed to count cleared"); + println!(" Cleared {} secret values in database", cleared.count.unwrap_or(0)); + + // Now migrate from Vault back to database + println!("\nMigrating secrets from Vault to database..."); + let report = migrate_secrets_to_database(&db, &settings) + .await + .expect("Migration to database failed"); + + println!("Migration report:"); + println!(" Total secrets: {}", report.total_secrets); + println!(" Migrated: {}", report.migrated_count); + println!(" Failed: {}", report.failed_count); + + if !report.failures.is_empty() { + println!(" Failures:"); + for f in &report.failures { + println!(" - {}/{}: {}", f.workspace_id, f.path, f.error); + } + } + + assert_eq!(report.failed_count, 0, "Migration had failures"); + assert!(report.migrated_count > 0, "No secrets were migrated"); + + // Verify secrets are restored in database + println!("\nVerifying secrets in database..."); + let secrets_after = sqlx::query!( + "SELECT workspace_id, path, value FROM variable WHERE is_secret = true AND value != 'CLEARED' ORDER BY workspace_id, path" + ) + .fetch_all(&db) + .await + .expect("Failed to query restored secrets"); + + assert!( + !secrets_after.is_empty(), + "No secrets were restored in database" + ); + + for s in &secrets_after { + println!(" ✓ {}/{}: {} chars", s.workspace_id, s.path, s.value.len()); + } + + println!("\n✓ Migration to database completed successfully"); +} + +/// Test full round-trip migration: DB -> Vault -> DB +#[sqlx::test(fixtures("base", "secret_backend"))] +#[ignore = "requires running Vault instance"] +async fn test_full_round_trip_migration(db: Pool) { + let settings = test_vault_settings(); + + // Verify Vault connection + test_vault_connection(&settings, Some(&db)) + .await + .expect("Failed to connect to Vault"); + + // Get original secrets + let original_secrets: std::collections::HashMap<(String, String), String> = sqlx::query!( + "SELECT workspace_id, path, value FROM variable WHERE is_secret = true" + ) + .fetch_all(&db) + .await + .expect("Failed to query original secrets") + .into_iter() + .map(|r| ((r.workspace_id, r.path), r.value)) + .collect(); + + println!("Original secrets: {} entries", original_secrets.len()); + + // Step 1: Migrate to Vault + println!("\n=== Step 1: Migrate DB -> Vault ==="); + let to_vault = migrate_secrets_to_vault(&db, &settings) + .await + .expect("Migration to Vault failed"); + println!("Migrated {} secrets to Vault", to_vault.migrated_count); + assert_eq!(to_vault.failed_count, 0); + + // Step 2: Clear database values + println!("\n=== Step 2: Clear database values ==="); + sqlx::query!("UPDATE variable SET value = 'ROUND_TRIP_CLEARED' WHERE is_secret = true") + .execute(&db) + .await + .expect("Failed to clear values"); + + // Step 3: Migrate back from Vault + println!("\n=== Step 3: Migrate Vault -> DB ==="); + let to_db = migrate_secrets_to_database(&db, &settings) + .await + .expect("Migration to database failed"); + println!("Migrated {} secrets to database", to_db.migrated_count); + assert_eq!(to_db.failed_count, 0); + + // Step 4: Verify round-trip integrity + println!("\n=== Step 4: Verify round-trip integrity ==="); + let restored_secrets: std::collections::HashMap<(String, String), String> = sqlx::query!( + "SELECT workspace_id, path, value FROM variable WHERE is_secret = true" + ) + .fetch_all(&db) + .await + .expect("Failed to query restored secrets") + .into_iter() + .map(|r| ((r.workspace_id, r.path), r.value)) + .collect(); + + // Compare original and restored + for ((ws, path), _original_value) in &original_secrets { + let restored_value = restored_secrets + .get(&(ws.clone(), path.clone())) + .expect(&format!("Secret {}/{} not found after round-trip", ws, path)); + + // Note: Values might differ slightly due to encryption/decryption + // but they should not be the cleared value + assert_ne!( + restored_value, "ROUND_TRIP_CLEARED", + "Secret {}/{} was not restored", + ws, path + ); + println!(" ✓ {}/{}: restored ({} chars)", ws, path, restored_value.len()); + } + + println!("\n✓ Full round-trip migration completed successfully!"); + println!(" Original secrets: {}", original_secrets.len()); + println!(" Restored secrets: {}", restored_secrets.len()); +} + +/// Test that workspace isolation is maintained during migration +#[sqlx::test(fixtures("base", "secret_backend"))] +#[ignore = "requires running Vault instance"] +async fn test_workspace_isolation(db: Pool) { + let settings = test_vault_settings(); + + test_vault_connection(&settings, Some(&db)) + .await + .expect("Failed to connect to Vault"); + + // Migrate all secrets to Vault + let report = migrate_secrets_to_vault(&db, &settings) + .await + .expect("Migration failed"); + + println!("Migrated {} secrets across workspaces", report.migrated_count); + + // Verify workspace isolation in Vault + let vault_backend = VaultBackend::new(settings.clone()); + + // Try to access test-workspace-2 secret from test-workspace path (should fail) + let cross_workspace_result: Result = vault_backend + .get_secret("test-workspace", "u/test-user/other_secret") + .await; + + assert!( + cross_workspace_result.is_err(), + "Cross-workspace access should fail - workspace isolation violated!" + ); + println!("✓ Cross-workspace access correctly denied"); + + // Verify each workspace's secrets are accessible from their own workspace + let ws1_result: Result = vault_backend + .get_secret("test-workspace", "u/test-user/db_password") + .await; + assert!(ws1_result.is_ok(), "test-workspace secret should be accessible"); + println!("✓ test-workspace secrets accessible"); + + let ws2_result: Result = vault_backend + .get_secret("test-workspace-2", "u/test-user/other_secret") + .await; + assert!(ws2_result.is_ok(), "test-workspace-2 secret should be accessible"); + println!("✓ test-workspace-2 secrets accessible"); + + println!("\n✓ Workspace isolation verified!"); +} diff --git a/backend/windmill-api/Cargo.toml b/backend/windmill-api/Cargo.toml index 087d904d3f..ae4372ac24 100644 --- a/backend/windmill-api/Cargo.toml +++ b/backend/windmill-api/Cargo.toml @@ -10,7 +10,7 @@ path = "src/lib.rs" [features] default = [] -private = ["windmill-audit/private"] +private = ["windmill-audit/private", "windmill-common/private"] enterprise = ["windmill-queue/enterprise", "windmill-audit/enterprise", "windmill-git-sync/enterprise", "windmill-common/enterprise", "windmill-worker/enterprise"] stripe = [] agent_worker_server = [] diff --git a/backend/windmill-api/openapi.yaml b/backend/windmill-api/openapi.yaml index b334483309..5f635d38a9 100644 --- a/backend/windmill-api/openapi.yaml +++ b/backend/windmill-api/openapi.yaml @@ -1411,6 +1411,83 @@ paths: items: $ref: "#/components/schemas/GlobalSetting" + /.well-known/jwks.json: + get: + summary: get JWKS for Vault JWT authentication + operationId: getJwks + tags: + - setting + responses: + "200": + description: JSON Web Key Set + content: + application/json: + schema: + $ref: "#/components/schemas/JwksResponse" + + /settings/test_secret_backend: + post: + summary: test secret backend connection (HashiCorp Vault) + operationId: testSecretBackend + tags: + - setting + requestBody: + description: Vault settings to test + required: true + content: + application/json: + schema: + $ref: "#/components/schemas/VaultSettings" + responses: + "200": + description: connection successful + content: + text/plain: + schema: + type: string + + /settings/migrate_secrets_to_vault: + post: + summary: migrate secrets from database to HashiCorp Vault + operationId: migrateSecretsToVault + tags: + - setting + requestBody: + description: Vault settings for migration target + required: true + content: + application/json: + schema: + $ref: "#/components/schemas/VaultSettings" + responses: + "200": + description: migration report + content: + application/json: + schema: + $ref: "#/components/schemas/SecretMigrationReport" + + /settings/migrate_secrets_to_database: + post: + summary: migrate secrets from HashiCorp Vault to database + operationId: migrateSecretsToDatabase + tags: + - setting + requestBody: + description: Vault settings for migration source + required: true + content: + application/json: + schema: + $ref: "#/components/schemas/VaultSettings" + responses: + "200": + description: migration report + content: + application/json: + schema: + $ref: "#/components/schemas/SecretMigrationReport" + /users/email: get: summary: get current user email (if logged in) @@ -16597,6 +16674,83 @@ components: # -- INLINE END -- # Do not change line above + VaultSettings: + type: object + required: + - address + - mount_path + properties: + address: + type: string + description: HashiCorp Vault server address (e.g., https://vault.company.com:8200) + mount_path: + type: string + description: KV v2 secrets engine mount path (e.g., windmill) + jwt_role: + type: string + description: Vault JWT auth role name for Windmill (optional, if not provided token auth is used) + namespace: + type: string + description: Vault Enterprise namespace (optional) + token: + type: string + description: Static Vault token for testing/development (optional, if provided this is used instead of JWT authentication) + + SecretMigrationFailure: + type: object + required: + - workspace_id + - path + - error + properties: + workspace_id: + type: string + description: Workspace ID where the secret is located + path: + type: string + description: Path of the secret that failed to migrate + error: + type: string + description: Error message + + SecretMigrationReport: + type: object + required: + - total_secrets + - migrated_count + - failed_count + - failures + properties: + total_secrets: + type: integer + format: int64 + description: Total number of secrets found + migrated_count: + type: integer + format: int64 + description: Number of secrets successfully migrated + failed_count: + type: integer + format: int64 + description: Number of secrets that failed to migrate + failures: + type: array + items: + $ref: "#/components/schemas/SecretMigrationFailure" + description: Details of any failures encountered during migration + + JwksResponse: + type: object + required: + - keys + properties: + keys: + type: array + items: + type: object + additionalProperties: true + description: Array of JSON Web Keys for JWT verification + FlowConversation: type: object required: diff --git a/backend/windmill-api/src/lib.rs b/backend/windmill-api/src/lib.rs index c6f5400f30..7fbfee8c9a 100644 --- a/backend/windmill-api/src/lib.rs +++ b/backend/windmill-api/src/lib.rs @@ -149,6 +149,7 @@ mod scim_oss; mod scopes; mod scripts; mod service_logs; +mod secret_backend_ext; mod settings; mod slack_approvals; #[cfg(all(feature = "smtp", feature = "private"))] @@ -719,6 +720,8 @@ pub async fn run_server( .route("/openapi.yaml", get(openapi)) .route("/openapi.json", get(openapi_json)), ) + // JWKS endpoint for HashiCorp Vault JWT authentication (must be outside /api prefix) + .route("/.well-known/jwks.json", get(settings::get_jwks)) .fallback(static_assets::static_handler) .layer(middleware_stack); diff --git a/backend/windmill-api/src/resources.rs b/backend/windmill-api/src/resources.rs index f1f9c471ec..37c34e8feb 100644 --- a/backend/windmill-api/src/resources.rs +++ b/backend/windmill-api/src/resources.rs @@ -10,6 +10,7 @@ use std::collections::HashMap; use crate::{ db::{ApiAuthed, DB}, + secret_backend_ext::rename_vault_secret, users::{maybe_refresh_folders, require_owner_of_path, Tokened}, utils::{check_scopes, require_super_admin, BulkDeleteRequest}, var_resource_cache::{cache_resource, get_cached_resource}, @@ -947,12 +948,40 @@ async fn update_resource( let mut tx = user_db.begin(&authed).await?; - if let Some(npath) = ns.path { + if let Some(npath) = ns.path.clone() { if npath != path { check_path_conflict(&mut tx, &w_id, &npath).await?; require_owner_of_path(&authed, path)?; + // Handle Vault secret rename if the linked variable is a Vault-stored secret + let linked_var = sqlx::query!( + "SELECT value, is_secret FROM variable WHERE path = $1 AND workspace_id = $2", + path, + w_id + ) + .fetch_optional(&mut *tx) + .await?; + + if let Some(var) = linked_var { + if var.is_secret { + // Check if this is a Vault-stored secret and rename it + if let Some(new_value) = + rename_vault_secret(&db, &w_id, path, &npath, &var.value).await? + { + // Update the variable's value to point to the new Vault path + sqlx::query!( + "UPDATE variable SET value = $1 WHERE path = $2 AND workspace_id = $3", + new_value, + path, + w_id + ) + .execute(&mut *tx) + .await?; + } + } + } + sqlx::query!( "UPDATE variable SET path = $1 WHERE path = $2 AND workspace_id = $3", npath, diff --git a/backend/windmill-api/src/secret_backend_ext.rs b/backend/windmill-api/src/secret_backend_ext.rs new file mode 100644 index 0000000000..41d2267a94 --- /dev/null +++ b/backend/windmill-api/src/secret_backend_ext.rs @@ -0,0 +1,408 @@ +/* + * Author: Ruben Fiszel + * Copyright: Windmill Labs, Inc 2024 + * This file and its contents are licensed under the AGPLv3 License. + * Please see the included NOTICE for copyright information and + * LICENSE-AGPL for a copy of the license. + */ + +//! Secret backend extension for the API layer +//! +//! This module provides helper functions for integrating the SecretBackend +//! trait with variable operations in the API. +//! +//! Note: HashiCorp Vault integration requires Enterprise Edition. +//! The OSS version only supports the database backend. + +use std::sync::Arc; + +use windmill_common::{ + db::DB, + error::{Error, Result}, + secret_backend::{database::DatabaseBackend, SecretBackend}, + variables::{build_crypt, decrypt, encrypt}, +}; + +#[cfg(all(feature = "private", feature = "enterprise"))] +use windmill_common::{ + global_settings::{load_value_from_global_settings, SECRET_BACKEND_SETTING}, + secret_backend::{SecretBackendConfig, VaultBackend, VaultSettings}, +}; + +#[cfg(all(feature = "private", feature = "enterprise"))] +use tokio::sync::RwLock; + +// Cached Vault backend to avoid recreating it for every request +// This enables connection pooling and avoids repeated setup overhead +#[cfg(all(feature = "private", feature = "enterprise"))] +struct CachedVaultBackend { + backend: Arc, + settings: VaultSettings, +} + +#[cfg(all(feature = "private", feature = "enterprise"))] +lazy_static::lazy_static! { + static ref VAULT_BACKEND_CACHE: RwLock> = RwLock::new(None); +} + +/// Get the current secret backend based on global settings +/// +/// OSS: Always returns DatabaseBackend +/// EE: Returns configured backend (Database or Vault) +#[cfg(not(all(feature = "private", feature = "enterprise")))] +pub async fn get_secret_backend(db: &DB) -> Result> { + Ok(Arc::new(DatabaseBackend::new(db.clone()))) +} + +#[cfg(all(feature = "private", feature = "enterprise"))] +pub async fn get_secret_backend(db: &DB) -> Result> { + let config = match load_value_from_global_settings(db, SECRET_BACKEND_SETTING).await? { + Some(value) => serde_json::from_value::(value).unwrap_or_default(), + None => SecretBackendConfig::default(), + }; + + match config { + SecretBackendConfig::Database => Ok(Arc::new(DatabaseBackend::new(db.clone()))), + SecretBackendConfig::HashiCorpVault(settings) => { + get_or_create_vault_backend(db, settings).await + } + } +} + +/// Get a cached Vault backend or create a new one if settings changed +#[cfg(all(feature = "private", feature = "enterprise"))] +async fn get_or_create_vault_backend( + _db: &DB, + settings: VaultSettings, +) -> Result> { + // Check if we have a cached backend with matching settings (read lock) + { + let cache = VAULT_BACKEND_CACHE.read().await; + if let Some(ref cached) = *cache { + if cached.settings == settings { + return Ok(cached.backend.clone()); + } + } + } + + // Need to create a new backend - acquire write lock + let mut cache = VAULT_BACKEND_CACHE.write().await; + + // Double-check (another task may have created it while we waited) + if let Some(ref cached) = *cache { + if cached.settings == settings { + return Ok(cached.backend.clone()); + } + } + + // Create new backend + let backend: Arc = { + #[cfg(feature = "openidconnect")] + if settings.token.is_none() { + Arc::new(VaultBackend::new_with_db(settings.clone(), _db.clone())) + } else { + Arc::new(VaultBackend::new(settings.clone())) + } + + #[cfg(not(feature = "openidconnect"))] + Arc::new(VaultBackend::new(settings.clone())) + }; + + // Cache it + *cache = Some(CachedVaultBackend { + backend: backend.clone(), + settings, + }); + + Ok(backend) +} + +/// Check if a Vault backend is currently configured +/// +/// OSS: Always returns false +/// EE: Checks global settings +#[cfg(not(all(feature = "private", feature = "enterprise")))] +pub async fn is_vault_backend_configured(_db: &DB) -> Result { + Ok(false) +} + +#[cfg(all(feature = "private", feature = "enterprise"))] +pub async fn is_vault_backend_configured(db: &DB) -> Result { + let config = match load_value_from_global_settings(db, SECRET_BACKEND_SETTING).await? { + Some(value) => serde_json::from_value::(value).unwrap_or_default(), + None => SecretBackendConfig::default(), + }; + + Ok(matches!(config, SecretBackendConfig::HashiCorpVault(_))) +} + +/// Get a secret value using the configured backend +/// +/// For database backend: decrypts using workspace key +/// For vault backend (EE only): fetches from Vault directly +pub async fn get_secret_value( + db: &DB, + workspace_id: &str, + path: &str, + encrypted_value: &str, +) -> Result { + let backend = get_secret_backend(db).await?; + + match backend.backend_name() { + "database" => { + // Use existing database decryption + let mc = build_crypt(db, workspace_id).await?; + decrypt(&mc, encrypted_value.to_string()).map_err(|e| { + Error::internal_err(format!("Error decrypting variable {}: {}", path, e)) + }) + } + "hashicorp_vault" => { + // Fetch from Vault directly + backend.get_secret(workspace_id, path).await + } + _ => Err(Error::internal_err(format!( + "Unknown backend: {}", + backend.backend_name() + ))), + } +} + +/// Store a secret value using the configured backend +/// +/// For database backend: encrypts using workspace key and returns encrypted value +/// For vault backend (EE only): stores in Vault and returns a placeholder for DB storage +pub async fn store_secret_value( + db: &DB, + workspace_id: &str, + path: &str, + plain_value: &str, +) -> Result { + let backend = get_secret_backend(db).await?; + + match backend.backend_name() { + "database" => { + // Use existing database encryption + let mc = build_crypt(db, workspace_id).await?; + Ok(encrypt(&mc, plain_value)) + } + "hashicorp_vault" => { + // Store in Vault and return a marker for DB + backend.set_secret(workspace_id, path, plain_value).await?; + // Return a marker indicating the value is stored in Vault + // The actual value in the DB will be this marker + Ok(format!("$vault:{}", path)) + } + _ => Err(Error::internal_err(format!( + "Unknown backend: {}", + backend.backend_name() + ))), + } +} + +/// Delete a secret from the configured backend (if using Vault) +/// +/// For database backend: no-op (DB delete is handled separately) +/// For vault backend (EE only): deletes from Vault +pub async fn delete_secret_from_backend( + db: &DB, + workspace_id: &str, + path: &str, +) -> Result<()> { + if is_vault_backend_configured(db).await? { + let backend = get_secret_backend(db).await?; + // Ignore NotFound errors during deletion (secret might not exist in Vault) + match backend.delete_secret(workspace_id, path).await { + Ok(()) => Ok(()), + Err(Error::NotFound(_)) => Ok(()), + Err(e) => Err(e), + } + } else { + Ok(()) + } +} + +/// Check if a value is stored in Vault (indicated by the $vault: prefix) +pub fn is_vault_stored_value(value: &str) -> bool { + value.starts_with("$vault:") +} + +/// Rename a secret in Vault when a variable path changes (EE only) +/// +/// This function: +/// 1. Reads the secret value from the old path +/// 2. Writes it to the new path +/// 3. Deletes from the old path +/// 4. Returns the new marker value ($vault:new_path) +/// +/// If the value is not a Vault-stored value, returns None (no action needed). +/// If Vault is not configured, returns None. +#[cfg(not(all(feature = "private", feature = "enterprise")))] +pub async fn rename_vault_secret( + _db: &DB, + _workspace_id: &str, + _old_path: &str, + new_path: &str, + current_value: &str, +) -> Result> { + // OSS: If value has $vault: prefix, just update the reference + // (This handles edge case where EE was used before downgrading to OSS) + if is_vault_stored_value(current_value) { + tracing::warn!( + "Variable has $vault: prefix but Vault requires Enterprise Edition. \ + Updating DB reference to {}", + new_path + ); + return Ok(Some(format!("$vault:{}", new_path))); + } + Ok(None) +} + +#[cfg(all(feature = "private", feature = "enterprise"))] +pub async fn rename_vault_secret( + db: &DB, + workspace_id: &str, + old_path: &str, + new_path: &str, + current_value: &str, +) -> Result> { + // Only handle Vault-stored values + if !is_vault_stored_value(current_value) { + return Ok(None); + } + + // Check if Vault backend is configured + if !is_vault_backend_configured(db).await? { + // Vault not configured but value has $vault: prefix - this is an inconsistent state + // Log warning and return new marker to at least update the DB reference + tracing::warn!( + "Variable value has $vault: prefix but Vault is not configured. \ + Updating DB reference from {} to {}", + old_path, + new_path + ); + return Ok(Some(format!("$vault:{}", new_path))); + } + + let backend = get_secret_backend(db).await?; + + // Read from old path + let secret_value = match backend.get_secret(workspace_id, old_path).await { + Ok(value) => value, + Err(Error::NotFound(_)) => { + // Secret doesn't exist in Vault - just update the DB reference + tracing::warn!( + "Secret not found in Vault at path {} during rename to {}", + old_path, + new_path + ); + return Ok(Some(format!("$vault:{}", new_path))); + } + Err(e) => return Err(e), + }; + + // Write to new path + backend + .set_secret(workspace_id, new_path, &secret_value) + .await?; + + // Delete from old path (ignore errors - new path is already written) + if let Err(e) = backend.delete_secret(workspace_id, old_path).await { + tracing::warn!( + "Failed to delete old secret at {} after rename to {}: {}", + old_path, + new_path, + e + ); + } + + Ok(Some(format!("$vault:{}", new_path))) +} + +/// Bulk rename secrets in Vault when a path prefix changes (e.g., user rename) +/// EE only feature. +/// +/// This is used when renaming users where many secrets need their paths updated. +/// Returns a list of (old_path, new_value) pairs for updating the database. +#[cfg(not(all(feature = "private", feature = "enterprise")))] +pub async fn rename_vault_secrets_with_prefix( + _db: &DB, + _workspace_id: &str, + _old_prefix: &str, + _new_prefix: &str, + _variables: Vec<(String, String)>, +) -> Result> { + // OSS: No Vault support, return empty + Ok(vec![]) +} + +#[cfg(all(feature = "private", feature = "enterprise"))] +pub async fn rename_vault_secrets_with_prefix( + db: &DB, + workspace_id: &str, + old_prefix: &str, + new_prefix: &str, + variables: Vec<(String, String)>, // (path, value) pairs +) -> Result> { + // Only process if Vault is configured + if !is_vault_backend_configured(db).await? { + return Ok(vec![]); + } + + let backend = get_secret_backend(db).await?; + let mut updates = Vec::new(); + + for (old_path, value) in variables { + // Only handle Vault-stored values + if !is_vault_stored_value(&value) { + continue; + } + + // Calculate new path by replacing prefix + let new_path = if old_path.starts_with(old_prefix) { + format!("{}{}", new_prefix, &old_path[old_prefix.len()..]) + } else { + continue; // Path doesn't match prefix, skip + }; + + // Read from old path + let secret_value = match backend.get_secret(workspace_id, &old_path).await { + Ok(v) => v, + Err(Error::NotFound(_)) => { + // Just update DB reference + updates.push((old_path, format!("$vault:{}", new_path))); + continue; + } + Err(e) => { + tracing::error!( + "Failed to read secret at {} during bulk rename: {}", + old_path, + e + ); + continue; + } + }; + + // Write to new path + if let Err(e) = backend.set_secret(workspace_id, &new_path, &secret_value).await { + tracing::error!( + "Failed to write secret to {} during bulk rename: {}", + new_path, + e + ); + continue; + } + + // Delete from old path + if let Err(e) = backend.delete_secret(workspace_id, &old_path).await { + tracing::warn!( + "Failed to delete old secret at {} after rename: {}", + old_path, + e + ); + } + + updates.push((old_path, format!("$vault:{}", new_path))); + } + + Ok(updates) +} diff --git a/backend/windmill-api/src/settings.rs b/backend/windmill-api/src/settings.rs index 939ae9fa57..d6d747a142 100644 --- a/backend/windmill-api/src/settings.rs +++ b/backend/windmill-api/src/settings.rs @@ -42,6 +42,8 @@ use windmill_common::{ }, server::Smtp, }; +#[cfg(all(feature = "private", feature = "enterprise"))] +use windmill_common::secret_backend::{SecretMigrationReport, VaultSettings}; use windmill_common::{error::to_anyhow, PgDatabase}; pub fn global_service() -> Router { @@ -85,6 +87,16 @@ pub fn global_service() -> Router { post(acknowledge_all_critical_alerts), ); + // Vault integration routes (EE only - requires both private and enterprise features) + #[cfg(all(feature = "private", feature = "enterprise"))] + let r = r + .route("/test_secret_backend", post(test_secret_backend)) + .route("/migrate_secrets_to_vault", post(migrate_secrets_to_vault)) + .route( + "/migrate_secrets_to_database", + post(migrate_secrets_to_database), + ); + #[cfg(feature = "parquet")] { return r.route("/test_object_storage_config", post(test_s3_bucket)); @@ -798,3 +810,100 @@ async fn setup_custom_instance_pg_database_inner( Ok(()) } + +// ============================================================================ +// Secret Backend Settings (HashiCorp Vault Integration) - Enterprise Edition +// ============================================================================ + +/// Test connection to a secret backend (HashiCorp Vault) +/// +/// This endpoint validates that the Vault settings are correct and that +/// Windmill can successfully authenticate and communicate with Vault. +/// +/// This is an Enterprise Edition feature. +#[cfg(all(feature = "private", feature = "enterprise"))] +pub async fn test_secret_backend( + Extension(db): Extension, + authed: ApiAuthed, + Json(settings): Json, +) -> Result { + require_super_admin(&db, &authed.email).await?; + + windmill_common::secret_backend::test_vault_connection(&settings, Some(&db)).await?; + + Ok("Successfully connected to HashiCorp Vault".to_string()) +} + +/// Migrate existing secrets from database to HashiCorp Vault +/// +/// This endpoint reads all encrypted secrets from the database, decrypts them, +/// and stores them in HashiCorp Vault. The database values are NOT deleted +/// automatically to allow for rollback if needed. +/// +/// This is an Enterprise Edition feature. +#[cfg(all(feature = "private", feature = "enterprise"))] +pub async fn migrate_secrets_to_vault( + Extension(db): Extension, + authed: ApiAuthed, + Json(settings): Json, +) -> JsonResult { + require_super_admin(&db, &authed.email).await?; + + let report = windmill_common::secret_backend::migrate_secrets_to_vault(&db, &settings).await?; + + Ok(Json(report)) +} + +/// Migrate secrets from HashiCorp Vault back to database +/// +/// This endpoint reads all secrets from HashiCorp Vault, encrypts them using +/// the workspace encryption keys, and stores them in the database. The Vault +/// values are NOT deleted automatically to allow for rollback if needed. +/// +/// This is an Enterprise Edition feature. +#[cfg(all(feature = "private", feature = "enterprise"))] +pub async fn migrate_secrets_to_database( + Extension(db): Extension, + authed: ApiAuthed, + Json(settings): Json, +) -> JsonResult { + require_super_admin(&db, &authed.email).await?; + + let report = + windmill_common::secret_backend::migrate_secrets_to_database(&db, &settings).await?; + + Ok(Json(report)) +} + +// ============================================================================ +// JWKS Endpoint for Vault JWT Authentication +// ============================================================================ + +/// JSON Web Key Set response structure +#[derive(Serialize)] +pub struct JwksResponse { + pub keys: Vec, +} + +/// JWKS endpoint for HashiCorp Vault to validate JWTs +/// +/// Vault calls this endpoint to fetch the public keys used to verify +/// JWTs generated by Windmill for authentication. +/// +/// In the open-source version, this returns an empty JWKS. +/// The Enterprise Edition provides the actual key set. +pub async fn get_jwks() -> JsonResult { + // Open source version returns empty JWKS + // Enterprise Edition will override this with actual public keys + #[cfg(not(feature = "enterprise"))] + { + Ok(Json(JwksResponse { keys: vec![] })) + } + + #[cfg(feature = "enterprise")] + { + // In enterprise mode, the actual keys would be fetched from global settings + // For now, return empty - the EE implementation would override this + Ok(Json(JwksResponse { keys: vec![] })) + } +} diff --git a/backend/windmill-api/src/users.rs b/backend/windmill-api/src/users.rs index 5c03d54646..10cf4cfddd 100644 --- a/backend/windmill-api/src/users.rs +++ b/backend/windmill-api/src/users.rs @@ -19,6 +19,7 @@ use crate::db::ApiAuthed; pub use crate::auth::Tokened; +use crate::secret_backend_ext::rename_vault_secrets_with_prefix; use crate::utils::{ generate_instance_wide_unique_username, get_instance_username_or_create_pending, }; @@ -2661,6 +2662,7 @@ async fn rename_user( } update_username_in_workpsace( &mut tx, + &db, &user_email, &w_u.username, &ru.new_username, @@ -2688,6 +2690,7 @@ async fn rename_user( async fn update_username_in_workpsace<'c>( tx: &mut sqlx::Transaction<'c, sqlx::Postgres>, + db: &DB, email: &str, old_username: &str, new_username: &str, @@ -2805,6 +2808,43 @@ async fn update_username_in_workpsace<'c>( // ---- variables ---- + // Handle Vault secret renames before updating paths in DB + let old_prefix = format!("u/{}/", old_username); + let new_prefix = format!("u/{}/", new_username); + + // Fetch all Vault-stored secret variables under this user's path + let vault_secrets: Vec<(String, String)> = sqlx::query!( + r#"SELECT path, value FROM variable + WHERE path LIKE ('u/' || $1 || '/%') + AND workspace_id = $2 + AND is_secret = true + AND value LIKE '$vault:%'"#, + old_username, + w_id + ) + .fetch_all(&mut **tx) + .await? + .into_iter() + .map(|r| (r.path, r.value)) + .collect(); + + // Rename secrets in Vault and get the new values + let vault_updates = + rename_vault_secrets_with_prefix(db, w_id, &old_prefix, &new_prefix, vault_secrets).await?; + + // Update the values in the DB for renamed Vault secrets (using OLD path, before path update) + for (old_path, new_value) in vault_updates { + sqlx::query!( + "UPDATE variable SET value = $1 WHERE path = $2 AND workspace_id = $3", + new_value, + old_path, + w_id + ) + .execute(&mut **tx) + .await?; + } + + // Now update the paths in the database sqlx::query!( r#"UPDATE variable SET path = REGEXP_REPLACE(path,'u/' || $2 || '/(.*)','u/' || $1 || '/\1') WHERE path LIKE ('u/' || $2 || '/%') AND workspace_id = $3"#, new_username, diff --git a/backend/windmill-api/src/variables.rs b/backend/windmill-api/src/variables.rs index a5493cd2e5..54e68f9a9b 100644 --- a/backend/windmill-api/src/variables.rs +++ b/backend/windmill-api/src/variables.rs @@ -8,6 +8,10 @@ use crate::{ db::{ApiAuthed, DB}, + secret_backend_ext::{ + delete_secret_from_backend, get_secret_value, is_vault_stored_value, rename_vault_secret, + store_secret_value, + }, users::{maybe_refresh_folders, require_owner_of_path}, utils::{check_scopes, BulkDeleteRequest}, webhook_util::{WebhookMessage, WebhookShared}, @@ -39,7 +43,7 @@ use crate::var_resource_cache::{cache_variable, get_cached_variable}; use lazy_static::lazy_static; use serde::Deserialize; use sqlx::{Acquire, Postgres, Transaction}; -use windmill_common::variables::{decrypt, encrypt}; +use windmill_common::variables::encrypt; use windmill_git_sync::{handle_deployment_metadata, DeployedObject}; lazy_static! { @@ -210,13 +214,8 @@ async fn get_variable( return Err(Error::internal_err("Require oauth2 feature".to_string())); } else if !value.is_empty() && decrypt_secret { let _ = tx.commit().await; - let mc = build_crypt(&db, &w_id).await?; - Some(decrypt(&mc, value).map_err(|e| { - Error::internal_err(format!( - "Error decrypting variable {}: {}", - variable.path, e - )) - })?) + // Use secret backend for decryption (supports both DB and Vault) + Some(get_secret_value(&db, &w_id, &variable.path, &value).await?) } else if q.include_encrypted.unwrap_or(false) { Some(value) } else { @@ -355,8 +354,8 @@ async fn create_variable( check_path_conflict(&db, &w_id, &variable.path).await?; let value = if variable.is_secret && !already_encrypted.unwrap_or(false) { - let mc = build_crypt(&db, &w_id).await?; - encrypt(&mc, &variable.value) + // Use secret backend for encryption (supports both DB and Vault) + store_secret_value(&db, &w_id, &variable.path, &variable.value).await? } else { variable.value }; @@ -436,6 +435,16 @@ async fn delete_variable( check_scopes(&authed, || format!("variables:write:{}", path))?; + // Check if variable is a secret before deleting (for Vault cleanup) + let is_secret = sqlx::query_scalar!( + "SELECT is_secret FROM variable WHERE path = $1 AND workspace_id = $2", + path, + &w_id + ) + .fetch_optional(&db) + .await? + .unwrap_or(false); + let mut tx = user_db.begin(&authed).await?; sqlx::query!( @@ -465,6 +474,11 @@ async fn delete_variable( tx.commit().await?; + // If variable was a secret, also delete from Vault backend (if configured) + if is_secret { + delete_secret_from_backend(&db, &w_id, path).await?; + } + handle_deployment_metadata( &authed.email, &authed.username, @@ -496,6 +510,15 @@ async fn delete_variables_bulk( check_scopes(&authed, || format!("variables:write:{}", path))?; } + // Query which paths are secrets before deletion (for Vault cleanup) + let secret_paths: Vec = sqlx::query_scalar!( + "SELECT path FROM variable WHERE path = ANY($1) AND workspace_id = $2 AND is_secret = true", + &request.paths, + &w_id + ) + .fetch_all(&db) + .await?; + let mut tx = user_db.begin(&authed).await?; let deleted_paths = sqlx::query_scalar!( @@ -526,6 +549,13 @@ async fn delete_variables_bulk( tx.commit().await?; + // Delete secrets from Vault backend (if configured) + for path in &secret_paths { + if deleted_paths.contains(path) { + delete_secret_from_backend(&db, &w_id, path).await?; + } + } + try_join_all(deleted_paths.iter().map(|path| { handle_deployment_metadata( &authed.email, @@ -589,7 +619,9 @@ async fn update_variable( sqlb.set_str("path", npath); } let ns_value_is_none = ns.value.is_none(); - if let Some(nvalue) = ns.value { + // Determine the target path for storing secrets (use new path if provided) + let target_path = ns.path.as_deref().unwrap_or(path); + if let Some(nvalue) = ns.value.clone() { let is_secret = if ns.is_secret.is_some() { ns.is_secret.unwrap() } else { @@ -604,8 +636,9 @@ async fn update_variable( }; let value = if is_secret && !already_encrypted.unwrap_or(false) { - let mc = build_crypt(&db, &w_id).await?; - encrypt(&mc, &nvalue) + // Use secret backend for encryption (supports both DB and Vault) + // Store at target_path (new path if renaming, otherwise current path) + store_secret_value(&db, &w_id, target_path, &nvalue).await? } else { nvalue }; @@ -654,11 +687,38 @@ async fn update_variable( let mut tx: Transaction<'_, Postgres> = user_db.begin(&authed).await?; - if let Some(npath) = ns.path { + if let Some(npath) = ns.path.clone() { if npath != path { check_path_conflict(&db, &w_id, &npath).await?; require_owner_of_path(&authed, path)?; + // Handle Vault secret rename if the variable is a secret stored in Vault + let current_var = sqlx::query!( + "SELECT value, is_secret FROM variable WHERE path = $1 AND workspace_id = $2", + path, + w_id + ) + .fetch_optional(&mut *tx) + .await?; + + if let Some(var) = current_var { + if var.is_secret && is_vault_stored_value(&var.value) { + if ns.value.is_some() { + // New value was provided and already stored at new path + // Just delete the old secret from Vault + delete_secret_from_backend(&db, &w_id, path).await?; + } else { + // No new value - rename the secret in Vault + if let Some(new_value) = + rename_vault_secret(&db, &w_id, path, &npath, &var.value).await? + { + // Update the variable's value to point to the new Vault path + sqlb.set_str("value", &new_value); + } + } + } + } + let mut v = sqlx::query_scalar!( "SELECT value FROM resource WHERE path = $1 AND workspace_id = $2", path, @@ -835,13 +895,8 @@ pub async fn get_value_internal<'a>( #[cfg(not(feature = "oauth2"))] return Err(Error::internal_err("Require oauth2 feature".to_string())); } else if !value.is_empty() { - let mc = build_crypt(db_with_opt_authed.db(), &w_id).await?; - decrypt(&mc, value).map_err(|e| { - Error::internal_err(format!( - "Error decrypting variable {}: {}", - variable.path, e - )) - })? + // Use secret backend for decryption (supports both DB and Vault) + get_secret_value(db_with_opt_authed.db(), &w_id, &variable.path, &value).await? } else { "".to_string() } diff --git a/backend/windmill-common/src/global_settings.rs b/backend/windmill-common/src/global_settings.rs index 6dbc8c8ba6..c5428e5c5c 100644 --- a/backend/windmill-common/src/global_settings.rs +++ b/backend/windmill-common/src/global_settings.rs @@ -47,6 +47,7 @@ pub const JWT_SECRET_SETTING: &str = "jwt_secret"; pub const EMAIL_DOMAIN_SETTING: &str = "email_domain"; pub const OTEL_SETTING: &str = "otel"; pub const APP_WORKSPACED_ROUTE_SETTING: &str = "app_workspaced_route"; +pub const SECRET_BACKEND_SETTING: &str = "secret_backend"; pub const ENV_SETTINGS: &[&str] = &[ "DISABLE_NSJAIL", diff --git a/backend/windmill-common/src/lib.rs b/backend/windmill-common/src/lib.rs index a650f6bf0c..c61f4263cc 100644 --- a/backend/windmill-common/src/lib.rs +++ b/backend/windmill-common/src/lib.rs @@ -78,6 +78,7 @@ pub mod result_stream; pub mod runnable_settings; pub mod s3_helpers; pub mod schedule; +pub mod secret_backend; pub mod schema; pub mod scripts; pub mod server; diff --git a/backend/windmill-common/src/secret_backend/database.rs b/backend/windmill-common/src/secret_backend/database.rs new file mode 100644 index 0000000000..3dee114481 --- /dev/null +++ b/backend/windmill-common/src/secret_backend/database.rs @@ -0,0 +1,126 @@ +/* + * Author: Ruben Fiszel + * Copyright: Windmill Labs, Inc 2024 + * This file and its contents are licensed under the AGPLv3 License. + * Please see the included NOTICE for copyright information and + * LICENSE-AGPL for a copy of the license. + */ + +//! Database backend for secret storage +//! +//! This is the default backend that stores secrets encrypted in the PostgreSQL database +//! using the existing magic_crypt encryption with workspace-specific keys. + +use async_trait::async_trait; + +use crate::db::DB; +use crate::error::{Error, Result}; +use crate::variables::{build_crypt, decrypt, encrypt}; + +use super::SecretBackend; + +/// Database-backed secret storage +/// +/// This backend stores secrets encrypted in the `variable` table using +/// the workspace's encryption key. This is the default and original +/// behavior of Windmill. +pub struct DatabaseBackend { + db: DB, +} + +impl DatabaseBackend { + /// Create a new database backend + pub fn new(db: DB) -> Self { + Self { db } + } +} + +#[async_trait] +impl SecretBackend for DatabaseBackend { + async fn get_secret(&self, workspace_id: &str, path: &str) -> Result { + let variable = sqlx::query!( + "SELECT value FROM variable WHERE path = $1 AND workspace_id = $2 AND is_secret = true", + path, + workspace_id + ) + .fetch_optional(&self.db) + .await?; + + let variable = variable.ok_or_else(|| { + Error::NotFound(format!( + "Secret variable {} not found in workspace {}", + path, workspace_id + )) + })?; + + let value = variable.value; + if value.is_empty() { + return Ok(String::new()); + } + + let mc = build_crypt(&self.db, workspace_id).await?; + decrypt(&mc, value).map_err(|e| { + Error::internal_err(format!("Error decrypting variable {}: {}", path, e)) + }) + } + + async fn set_secret(&self, workspace_id: &str, path: &str, value: &str) -> Result<()> { + let mc = build_crypt(&self.db, workspace_id).await?; + let encrypted_value = encrypt(&mc, value); + + // Update the value in the database + // Note: This assumes the variable row already exists (created via the normal API) + let result = sqlx::query!( + "UPDATE variable SET value = $1 WHERE path = $2 AND workspace_id = $3 AND is_secret = true", + encrypted_value, + path, + workspace_id + ) + .execute(&self.db) + .await?; + + if result.rows_affected() == 0 { + return Err(Error::NotFound(format!( + "Secret variable {} not found in workspace {}", + path, workspace_id + ))); + } + + Ok(()) + } + + async fn delete_secret(&self, _workspace_id: &str, _path: &str) -> Result<()> { + // For database backend, deletion is handled by the normal variable deletion flow + // The encrypted value is just deleted along with the row + // This method is a no-op for database backend since the caller handles the DELETE + Ok(()) + } + + fn backend_name(&self) -> &'static str { + "database" + } +} + +/// Encrypt a value for storage in the database +/// +/// This is a convenience function for use when creating new secrets. +pub async fn encrypt_for_database(db: &DB, workspace_id: &str, value: &str) -> Result { + let mc = build_crypt(db, workspace_id).await?; + Ok(encrypt(&mc, value)) +} + +/// Decrypt a value from the database +/// +/// This is a convenience function for reading secrets. +pub async fn decrypt_from_database( + db: &DB, + workspace_id: &str, + encrypted_value: String, +) -> Result { + if encrypted_value.is_empty() { + return Ok(String::new()); + } + let mc = build_crypt(db, workspace_id).await?; + decrypt(&mc, encrypted_value) + .map_err(|e| Error::internal_err(format!("Error decrypting value: {}", e))) +} diff --git a/backend/windmill-common/src/secret_backend/mod.rs b/backend/windmill-common/src/secret_backend/mod.rs new file mode 100644 index 0000000000..772a5ebe6a --- /dev/null +++ b/backend/windmill-common/src/secret_backend/mod.rs @@ -0,0 +1,133 @@ +/* + * Author: Ruben Fiszel + * Copyright: Windmill Labs, Inc 2024 + * This file and its contents are licensed under the AGPLv3 License. + * Please see the included NOTICE for copyright information and + * LICENSE-AGPL for a copy of the license. + */ + +//! Secret Backend abstraction for storing secrets in external vaults +//! +//! This module provides a trait-based abstraction for secret storage, +//! allowing secrets to be stored in the database (default) or in external +//! vaults like HashiCorp Vault (Enterprise Edition). + +pub mod database; + +#[cfg(feature = "private")] +pub mod vault_ee; + +pub mod vault_oss; + +#[cfg(test)] +mod tests; + +#[cfg(feature = "private")] +pub use vault_ee::*; + +#[cfg(not(feature = "private"))] +pub use vault_oss::*; + +use async_trait::async_trait; +use serde::{Deserialize, Serialize}; + +use crate::error::Result; + +/// Trait for secret storage backends +/// +/// Implementations of this trait handle the storage and retrieval of secrets. +/// The default implementation stores secrets encrypted in the database. +/// Enterprise Edition supports HashiCorp Vault as an alternative backend. +#[async_trait] +pub trait SecretBackend: Send + Sync { + /// Retrieve a secret value + /// + /// # Arguments + /// * `workspace_id` - The workspace identifier + /// * `path` - The path/name of the secret variable + /// + /// # Returns + /// The decrypted secret value + async fn get_secret(&self, workspace_id: &str, path: &str) -> Result; + + /// Store a secret value + /// + /// # Arguments + /// * `workspace_id` - The workspace identifier + /// * `path` - The path/name of the secret variable + /// * `value` - The plaintext secret value to store + async fn set_secret(&self, workspace_id: &str, path: &str, value: &str) -> Result<()>; + + /// Delete a secret + /// + /// # Arguments + /// * `workspace_id` - The workspace identifier + /// * `path` - The path/name of the secret variable + async fn delete_secret(&self, workspace_id: &str, path: &str) -> Result<()>; + + /// Get the name of this backend for logging/debugging + fn backend_name(&self) -> &'static str; +} + +/// Configuration for secret storage backend +/// +/// This enum is stored in global_settings and determines which backend +/// is used for secret storage at the instance level. +#[derive(Debug, Clone, Serialize, Deserialize, PartialEq)] +#[serde(tag = "type")] +pub enum SecretBackendConfig { + /// Store secrets encrypted in the database (default behavior) + Database, + /// Store secrets in HashiCorp Vault (Enterprise Edition only) + HashiCorpVault(VaultSettings), +} + +impl Default for SecretBackendConfig { + fn default() -> Self { + SecretBackendConfig::Database + } +} + +/// Settings for HashiCorp Vault integration +#[derive(Debug, Clone, Serialize, Deserialize, PartialEq)] +pub struct VaultSettings { + /// Vault server address (e.g., "https://vault.company.com:8200") + pub address: String, + /// KV v2 mount path (e.g., "windmill") + pub mount_path: String, + /// JWT auth role name configured in Vault (used for JWT/OIDC auth) + /// Optional - if not provided, token auth is used + #[serde(skip_serializing_if = "Option::is_none")] + pub jwt_role: Option, + /// Vault Enterprise namespace (optional) + #[serde(skip_serializing_if = "Option::is_none")] + pub namespace: Option, + /// Static Vault token for testing/development (optional) + /// If provided, this is used instead of JWT authentication + #[serde(skip_serializing_if = "Option::is_none")] + pub token: Option, +} + +/// Result of a secret migration operation +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct SecretMigrationReport { + /// Total number of secrets found + pub total_secrets: usize, + /// Number of secrets successfully migrated + pub migrated_count: usize, + /// Number of secrets that failed to migrate + pub failed_count: usize, + /// Details of any failures + pub failures: Vec, +} + +/// Details of a failed secret migration +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct SecretMigrationFailure { + /// Workspace ID where the secret is located + pub workspace_id: String, + /// Path of the secret that failed to migrate + pub path: String, + /// Error message + pub error: String, +} diff --git a/backend/windmill-common/src/secret_backend/tests.rs b/backend/windmill-common/src/secret_backend/tests.rs new file mode 100644 index 0000000000..2a6962f8dd --- /dev/null +++ b/backend/windmill-common/src/secret_backend/tests.rs @@ -0,0 +1,143 @@ +/* + * Integration tests for secret backend + * + * These tests require a running Vault instance at http://127.0.0.1:8200 + * with the token "test-root-token" and a KV v2 mount at "windmill". + * + * Run Vault dev server: + * podman run -d --name vault-test --rm -p 8200:8200 \ + * -e 'VAULT_DEV_ROOT_TOKEN_ID=test-root-token' \ + * -e 'VAULT_DEV_LISTEN_ADDRESS=0.0.0.0:8200' \ + * docker.io/hashicorp/vault:latest + * + * Then enable the windmill mount: + * curl --header "X-Vault-Token: test-root-token" \ + * --request POST \ + * --data '{"type": "kv", "options": {"version": "2"}}' \ + * http://127.0.0.1:8200/v1/sys/mounts/windmill + */ + +#[cfg(test)] +mod tests { + use crate::secret_backend::{SecretBackend, VaultBackend, VaultSettings}; + + fn test_settings() -> VaultSettings { + VaultSettings { + address: "http://127.0.0.1:8200".to_string(), + mount_path: "windmill".to_string(), + jwt_role: Some("windmill-secrets".to_string()), + namespace: None, + token: Some("test-root-token".to_string()), + } + } + + #[tokio::test] + #[ignore] // Run with --ignored to execute + async fn test_vault_write_and_read() { + let settings = test_settings(); + let backend = VaultBackend::new(settings); + + let workspace_id = "test-ws-1"; + let path = "test-secret"; + let value = "my-super-secret-value"; + + // Write secret + backend + .set_secret(workspace_id, path, value) + .await + .expect("Failed to write secret"); + + // Read secret + let read_value = backend + .get_secret(workspace_id, path) + .await + .expect("Failed to read secret"); + + assert_eq!(read_value, value); + + // Delete secret + backend + .delete_secret(workspace_id, path) + .await + .expect("Failed to delete secret"); + + // Verify deleted + let result = backend.get_secret(workspace_id, path).await; + assert!(result.is_err(), "Secret should be deleted"); + } + + #[tokio::test] + #[ignore] + async fn test_vault_multiple_secrets() { + let settings = test_settings(); + let backend = VaultBackend::new(settings); + + let workspace_id = "test-ws-2"; + + // Write multiple secrets + for i in 0..5 { + let path = format!("secret-{}", i); + let value = format!("value-{}", i); + backend + .set_secret(workspace_id, &path, &value) + .await + .expect(&format!("Failed to write secret-{}", i)); + } + + // Read and verify + for i in 0..5 { + let path = format!("secret-{}", i); + let expected = format!("value-{}", i); + let value = backend + .get_secret(workspace_id, &path) + .await + .expect(&format!("Failed to read secret-{}", i)); + assert_eq!(value, expected); + } + + // Cleanup + for i in 0..5 { + let path = format!("secret-{}", i); + backend + .delete_secret(workspace_id, &path) + .await + .expect(&format!("Failed to delete secret-{}", i)); + } + } + + #[tokio::test] + #[ignore] + async fn test_vault_overwrite_secret() { + let settings = test_settings(); + let backend = VaultBackend::new(settings); + + let workspace_id = "test-ws-3"; + let path = "overwrite-test"; + + // Write initial value + backend + .set_secret(workspace_id, path, "initial-value") + .await + .expect("Failed to write initial value"); + + // Overwrite + backend + .set_secret(workspace_id, path, "new-value") + .await + .expect("Failed to overwrite"); + + // Read and verify + let value = backend + .get_secret(workspace_id, path) + .await + .expect("Failed to read"); + + assert_eq!(value, "new-value"); + + // Cleanup + backend + .delete_secret(workspace_id, path) + .await + .expect("Failed to delete"); + } +} diff --git a/backend/windmill-common/src/secret_backend/vault_oss.rs b/backend/windmill-common/src/secret_backend/vault_oss.rs new file mode 100644 index 0000000000..b338388312 --- /dev/null +++ b/backend/windmill-common/src/secret_backend/vault_oss.rs @@ -0,0 +1,107 @@ +/* + * Author: Ruben Fiszel + * Copyright: Windmill Labs, Inc 2024 + * This file and its contents are licensed under the AGPLv3 License. + * Please see the included NOTICE for copyright information and + * LICENSE-AGPL for a copy of the license. + */ + +//! HashiCorp Vault secret backend stubs (Open Source Edition) +//! +//! This module provides stub implementations for Vault integration. +//! The actual Vault integration requires Enterprise Edition. + +use std::sync::Arc; + +use crate::db::DB; +use crate::error::{Error, Result}; + +use super::{database::DatabaseBackend, SecretBackend, SecretBackendConfig, SecretMigrationReport, VaultSettings}; + +/// Stub VaultBackend for OSS - all operations return EE required error +pub struct VaultBackend; + +impl VaultBackend { + pub fn new(_settings: VaultSettings) -> Self { + Self + } +} + +#[async_trait::async_trait] +impl SecretBackend for VaultBackend { + async fn get_secret(&self, _workspace_id: &str, _path: &str) -> Result { + Err(Error::internal_err( + "HashiCorp Vault integration requires Enterprise Edition".to_string(), + )) + } + + async fn set_secret(&self, _workspace_id: &str, _path: &str, _value: &str) -> Result<()> { + Err(Error::internal_err( + "HashiCorp Vault integration requires Enterprise Edition".to_string(), + )) + } + + async fn delete_secret(&self, _workspace_id: &str, _path: &str) -> Result<()> { + Err(Error::internal_err( + "HashiCorp Vault integration requires Enterprise Edition".to_string(), + )) + } + + fn backend_name(&self) -> &'static str { + "hashicorp_vault" + } +} + +/// Create the appropriate secret backend based on configuration +/// +/// In OSS, always returns DatabaseBackend regardless of config. +/// Vault configuration is ignored with a warning. +pub async fn create_secret_backend( + db: DB, + config: &SecretBackendConfig, +) -> Result> { + match config { + SecretBackendConfig::Database => Ok(Arc::new(DatabaseBackend::new(db))), + SecretBackendConfig::HashiCorpVault(_) => { + tracing::warn!( + "HashiCorp Vault is configured but requires Enterprise Edition. \ + Falling back to database backend." + ); + Ok(Arc::new(DatabaseBackend::new(db))) + } + } +} + +/// Test connection to Vault (OSS stub) +pub async fn test_vault_connection(_settings: &VaultSettings, _db: Option<&DB>) -> Result<()> { + Err(Error::internal_err( + "HashiCorp Vault integration requires Enterprise Edition".to_string(), + )) +} + +/// Migrate secrets from database to Vault (OSS stub) +pub async fn migrate_secrets_to_vault( + _db: &DB, + _settings: &VaultSettings, +) -> Result { + Err(Error::internal_err( + "HashiCorp Vault integration requires Enterprise Edition".to_string(), + )) +} + +/// Migrate secrets from Vault back to database (OSS stub) +pub async fn migrate_secrets_to_database( + _db: &DB, + _settings: &VaultSettings, +) -> Result { + Err(Error::internal_err( + "HashiCorp Vault integration requires Enterprise Edition".to_string(), + )) +} + +/// Generate a JWT for Vault authentication (OSS stub) +pub async fn generate_vault_jwt(_db: &DB, _vault_address: &str) -> Result { + Err(Error::internal_err( + "HashiCorp Vault integration requires Enterprise Edition".to_string(), + )) +} diff --git a/frontend/package-lock.json b/frontend/package-lock.json index 8a8ad6f15e..13c9d67dde 100644 --- a/frontend/package-lock.json +++ b/frontend/package-lock.json @@ -835,7 +835,6 @@ "version": "1.7.1", "resolved": "https://registry.npmjs.org/@emnapi/core/-/core-1.7.1.tgz", "integrity": "sha512-o1uhUASyo921r2XtHYOHy7gdkGLge8ghBEQHMWmyJFoXlpU58kIrhhN3w26lpQb6dspetweapMn2CSNwQ8I4wg==", - "dev": true, "license": "MIT", "optional": true, "dependencies": { @@ -847,7 +846,6 @@ "version": "1.7.1", "resolved": "https://registry.npmjs.org/@emnapi/runtime/-/runtime-1.7.1.tgz", "integrity": "sha512-PVtJr5CmLwYAU9PZDMITZoR5iAOShYREoR45EyyLrbntV50mdePTgUn4AmOw90Ifcj+x2kRjdzr1HP3RrNiHGA==", - "dev": true, "license": "MIT", "optional": true, "dependencies": { @@ -858,7 +856,6 @@ "version": "1.1.0", "resolved": "https://registry.npmjs.org/@emnapi/wasi-threads/-/wasi-threads-1.1.0.tgz", "integrity": "sha512-WI0DdZ8xFSbgMjR1sFsKABJ/C5OnRrjT06JXbZKexJGrDuPTzZdDYfFlsgcCXCyf+suG5QU2e/y1Wo2V/OapLQ==", - "dev": true, "license": "MIT", "optional": true, "dependencies": { @@ -1348,7 +1345,6 @@ "version": "1.1.0", "resolved": "https://registry.npmjs.org/@napi-rs/wasm-runtime/-/wasm-runtime-1.1.0.tgz", "integrity": "sha512-Fq6DJW+Bb5jaWE69/qOE0D1TUN9+6uWhCeZpdnSBk14pjLcCWR7Q8n49PTSPHazM37JqrsdpEthXy2xn6jWWiA==", - "dev": true, "license": "MIT", "optional": true, "dependencies": { @@ -1550,7 +1546,6 @@ "cpu": [ "arm64" ], - "dev": true, "license": "MIT", "optional": true, "os": [ @@ -1567,7 +1562,6 @@ "cpu": [ "arm64" ], - "dev": true, "license": "MIT", "optional": true, "os": [ @@ -1584,7 +1578,6 @@ "cpu": [ "x64" ], - "dev": true, "license": "MIT", "optional": true, "os": [ @@ -1601,7 +1594,6 @@ "cpu": [ "x64" ], - "dev": true, "license": "MIT", "optional": true, "os": [ @@ -1618,7 +1610,6 @@ "cpu": [ "arm" ], - "dev": true, "license": "MIT", "optional": true, "os": [ @@ -1635,7 +1626,6 @@ "cpu": [ "arm64" ], - "dev": true, "license": "MIT", "optional": true, "os": [ @@ -1652,7 +1642,6 @@ "cpu": [ "arm64" ], - "dev": true, "license": "MIT", "optional": true, "os": [ @@ -1669,7 +1658,6 @@ "cpu": [ "x64" ], - "dev": true, "license": "MIT", "optional": true, "os": [ @@ -1686,7 +1674,6 @@ "cpu": [ "x64" ], - "dev": true, "license": "MIT", "optional": true, "os": [ @@ -1703,7 +1690,6 @@ "cpu": [ "arm64" ], - "dev": true, "license": "MIT", "optional": true, "os": [ @@ -1720,7 +1706,6 @@ "cpu": [ "wasm32" ], - "dev": true, "license": "MIT", "optional": true, "dependencies": { @@ -1737,7 +1722,6 @@ "cpu": [ "arm64" ], - "dev": true, "license": "MIT", "optional": true, "os": [ @@ -1754,7 +1738,6 @@ "cpu": [ "x64" ], - "dev": true, "license": "MIT", "optional": true, "os": [ @@ -2375,7 +2358,6 @@ "version": "0.10.1", "resolved": "https://registry.npmjs.org/@tybys/wasm-util/-/wasm-util-0.10.1.tgz", "integrity": "sha512-9tTaPJLSiejZKx+Bmog4uSubteqTvFrVrURwkmHixBo0G4seD0zUxp98E1DzUBJxLQ3NPwXrGKDiVjwx/DpPsg==", - "dev": true, "license": "MIT", "optional": true, "dependencies": { @@ -7214,7 +7196,7 @@ "version": "1.21.7", "resolved": "https://registry.npmjs.org/jiti/-/jiti-1.21.7.tgz", "integrity": "sha512-/imKNG4EbWNrVjoNC/1H5/9GFy+tqjGBHCaSsN+P2RnPqjsLmv6UD3Ej+Kj8nBWaRAwyk7kK5ZUc+OEatnTR3A==", - "dev": true, + "devOptional": true, "license": "MIT", "bin": { "jiti": "bin/jiti.js" @@ -7650,7 +7632,6 @@ "cpu": [ "arm64" ], - "dev": true, "license": "MPL-2.0", "optional": true, "os": [ @@ -7671,7 +7652,6 @@ "cpu": [ "arm64" ], - "dev": true, "license": "MPL-2.0", "optional": true, "os": [ @@ -7692,7 +7672,6 @@ "cpu": [ "x64" ], - "dev": true, "license": "MPL-2.0", "optional": true, "os": [ @@ -7713,7 +7692,6 @@ "cpu": [ "x64" ], - "dev": true, "license": "MPL-2.0", "optional": true, "os": [ @@ -7734,7 +7712,6 @@ "cpu": [ "arm" ], - "dev": true, "license": "MPL-2.0", "optional": true, "os": [ @@ -7755,7 +7732,6 @@ "cpu": [ "arm64" ], - "dev": true, "license": "MPL-2.0", "optional": true, "os": [ @@ -7776,7 +7752,6 @@ "cpu": [ "arm64" ], - "dev": true, "license": "MPL-2.0", "optional": true, "os": [ @@ -7797,7 +7772,6 @@ "cpu": [ "x64" ], - "dev": true, "license": "MPL-2.0", "optional": true, "os": [ @@ -7818,7 +7792,6 @@ "cpu": [ "x64" ], - "dev": true, "license": "MPL-2.0", "optional": true, "os": [ @@ -7839,7 +7812,6 @@ "cpu": [ "arm64" ], - "dev": true, "license": "MPL-2.0", "optional": true, "os": [ @@ -7860,7 +7832,6 @@ "cpu": [ "x64" ], - "dev": true, "license": "MPL-2.0", "optional": true, "os": [ @@ -12498,21 +12469,6 @@ } } }, - "node_modules/svelte-check/node_modules/picomatch": { - "version": "4.0.3", - "resolved": "https://registry.npmjs.org/picomatch/-/picomatch-4.0.3.tgz", - "integrity": "sha512-5gTmgEY/sqK6gFXLIsQNH19lWb4ebPDLA4SdLP7dsWkIXHWlG66oPuVvXSGFPppYZz8ZDZq0dYYrbHfBCVUb1Q==", - "dev": true, - "license": "MIT", - "optional": true, - "peer": true, - "engines": { - "node": ">=12" - }, - "funding": { - "url": "https://github.com/sponsors/jonschlinkert" - } - }, "node_modules/svelte-eslint-parser": { "version": "0.43.0", "resolved": "https://registry.npmjs.org/svelte-eslint-parser/-/svelte-eslint-parser-0.43.0.tgz", diff --git a/frontend/src/lib/components/InstanceSetting.svelte b/frontend/src/lib/components/InstanceSetting.svelte index b1920a04ff..42a77b88dd 100644 --- a/frontend/src/lib/components/InstanceSetting.svelte +++ b/frontend/src/lib/components/InstanceSetting.svelte @@ -28,6 +28,7 @@ import EEOnly from './EEOnly.svelte' import CriticalAlertChannels from './instanceSettings/CriticalAlertChannels.svelte' import SmtpSettings from './instanceSettings/SmtpSettings.svelte' + import SecretBackendConfig from './instanceSettings/SecretBackendConfig.svelte' import TextInput from './text_input/TextInput.svelte' import Label from './Label.svelte' @@ -886,6 +887,8 @@ TODO {:else if setting.fieldType == 'smtp_connect'} + {:else if setting.fieldType == 'secret_backend'} + {/if} {#if hasError} diff --git a/frontend/src/lib/components/instanceSettings.ts b/frontend/src/lib/components/instanceSettings.ts index e8c5bff667..cb8b38f95f 100644 --- a/frontend/src/lib/components/instanceSettings.ts +++ b/frontend/src/lib/components/instanceSettings.ts @@ -34,6 +34,7 @@ export interface Setting { | 'smtp_connect' | 'indexer_rates' | 'otel' + | 'secret_backend' storage: SettingStorage advancedToggle?: { label: string @@ -473,6 +474,17 @@ export const settings: Record = { fieldType: 'boolean', storage: 'setting' } + ], + 'Secret Storage': [ + { + label: 'Secret Storage Backend', + description: + 'Configure where secrets (secret variables) are stored. By default, secrets are encrypted and stored in the database. Enterprise Edition supports HashiCorp Vault as an external secret store.', + key: 'secret_backend', + fieldType: 'secret_backend', + storage: 'setting', + ee_only: 'HashiCorp Vault integration is an Enterprise Edition feature' + } ] } diff --git a/frontend/src/lib/components/instanceSettings/SecretBackendConfig.svelte b/frontend/src/lib/components/instanceSettings/SecretBackendConfig.svelte new file mode 100644 index 0000000000..c66876ad65 --- /dev/null +++ b/frontend/src/lib/components/instanceSettings/SecretBackendConfig.svelte @@ -0,0 +1,539 @@ + + +
+ +
+ + setBackendType(v)} + > + {#snippet children({ item: toggleButton })} + + + {/snippet} + + {#if vaultDisabled} +
+ HashiCorp Vault integration requires Enterprise Edition +
+ {/if} +
+ + {#if selectedType === 'Database'} +
+ +
+

Database Storage (Default)

+

+ Secrets are encrypted using workspace-specific keys and stored in the PostgreSQL database. +

+
+
+ {:else if selectedType === 'HashiCorpVault'} + +
+
+ +
+

+ HashiCorp Vault Configuration + Beta +

+

+ Store secrets in an external HashiCorp Vault instance. +

+
+
+ +
+
+ + +
+ +
+ + The KV v2 secrets engine mount path in Vault + +
+ + +
+ + setAuthMethod(v)} + > + {#snippet children({ item: toggleButton })} + + + {/snippet} + +
+ + {#if authMethod === 'token'} +
+ + Static token for authentication. Recommended only for testing/development. + +
+ {:else} +
+ + The JWT authentication role configured in Vault. + + + +
+ Vault JWT Setup Instructions +
+

Configure Vault to accept JWTs from Windmill:

+
+ 
# Enable JWT auth method
+vault auth enable jwt
+
+# Configure JWT auth with Windmill's JWKS endpoint
+vault write auth/jwt/config \
+  jwks_url="{baseUrl}/.well-known/jwks.json" \
+  bound_issuer="{baseUrl}"
+
+# Create a policy for Windmill secrets
+vault policy write windmill-secrets - <<EOF
+path "windmill/data/*" {
+  capabilities = ["create", "read", "update", "delete"]
+}
+path "windmill/metadata/*" {
+  capabilities = ["list", "delete"]
+}
+EOF
+
+# Create the JWT role
+vault write auth/jwt/role/windmill-secrets \
+  role_type="jwt" \
+  bound_audiences="{baseUrl}" \
+  user_claim="email" \
+  policies="windmill-secrets" \
+  ttl="1h"
+
+

+ Replace windmill-secrets with your role name if different. +

+
+
+
+ {/if} + +
+ + Vault Enterprise namespace (leave empty if not using namespaces) + +
+
+ + +
+
+ +
+ + +
+ + + Migrate secrets between the database and HashiCorp Vault. Original values are NOT + deleted to allow for rollback. + + +
+ +
+
+ + + +
+

Database → Vault

+

+ Decrypt secrets from database and store in Vault +

+ +
+ + +
+
+ + + +
+

Vault → Database

+

+ Read secrets from Vault and encrypt in database +

+ +
+
+
+
+
+ {/if} +
+ + + { + migrateToVaultModalOpen = false + }} + onConfirmed={migrateSecretsToVault} +> + {#snippet children()} +
+

+ This will migrate all existing secrets from the database to HashiCorp Vault. The process + will: +

+
    +
  1. Read all encrypted secrets from the database
    2. +
  2. Decrypt them using the workspace encryption keys
    3. +
  3. Store them in HashiCorp Vault under the configured mount path
    4. +
+

+ Note: Database values are NOT deleted automatically. You can manually clear them after + verifying the migration was successful. +

+

Are you sure you want to proceed?

+
+ {/snippet} + + + + { + migrateToDatabaseModalOpen = false + }} + onConfirmed={migrateSecretsToDatabase} +> + {#snippet children()} +
+

+ This will migrate all secrets from HashiCorp Vault back to the database. The process will: +

+
    +
  1. List all secrets in Vault for each workspace
    2. +
  2. Read each secret value from Vault
    3. +
  3. Encrypt and store them in the database
    4. +
+

+ Note: Vault values are NOT deleted automatically. Only secrets that already exist in the + database will be updated. +

+

Are you sure you want to proceed?

+
+ {/snippet} +