From 0f7f7c37a97efda3fa41f1856aec658425349abd Mon Sep 17 00:00:00 2001 From: HugoCasa Date: Mon, 15 Jul 2024 19:38:44 +0200 Subject: [PATCH] feat: rehydrate instance settings/configs/users from CLI (#4035) * feat: rehydrate instance settings/configs/users from CLI * fix: nits * feat: encryption key, saved instances, include all workspaces * preshot client version * Update deps.ts --- ...dfce162abdef0d94e5020b84fed424dfd1ae9.json | 15 + ...e46931ce25efc5ea0c53e160fcd98a96f4306.json | 12 + ...183c830b19a403ebb7b8be55a691675010c3.json} | 12 +- ...af58e5dd888581467c976e76f140b2c00f6d7.json | 18 + ...c758523610d1260ff59aace18abaf4869edd3.json | 12 + ...fa96f763e48316fb7420f2aad733562cab582.json | 68 +++ ...cef0b70b784c2410f3a5f51288aaea6904b8f.json | 50 ++ ...7a8c2f0eb5cc11cb2f49222f5eef4a21d8c41.json | 22 + ...28f4f735a3343fbae15d2bc3510c3308d16ba.json | 26 + ...9aa73fcb0e8cb1f5e863f31faf98e4137b5e6.json | 26 + ...0cb549a34b96554ae1872355b90304f5dcb76.json | 4 +- ...3717ee7f24abd54fab5b4a687490b7c0b3672.json | 12 + backend/windmill-api/openapi.yaml | 177 ++++++ backend/windmill-api/src/configs.rs | 20 + backend/windmill-api/src/groups.rs | 117 ++++ backend/windmill-api/src/settings.rs | 28 + backend/windmill-api/src/users.rs | 102 ++++ backend/windmill-api/src/variables.rs | 11 +- backend/windmill-api/src/workspaces.rs | 90 ++- backend/windmill-common/src/variables.rs | 12 +- cli/conf.ts | 1 + cli/deps.ts | 2 +- cli/instance.ts | 542 ++++++++++++++++++ cli/main.ts | 2 + cli/pull.ts | 7 +- cli/settings.ts | 231 +++++++- cli/sync.ts | 42 +- cli/types.ts | 10 +- cli/user.ts | 92 ++- cli/utils.ts | 4 + cli/workspace.ts | 2 +- 31 files changed, 1703 insertions(+), 66 deletions(-) create mode 100644 backend/.sqlx/query-07834003a631acac03a52a128d3dfce162abdef0d94e5020b84fed424dfd1ae9.json create mode 100644 backend/.sqlx/query-16c93e721e30b72197e6143e0ebe46931ce25efc5ea0c53e160fcd98a96f4306.json rename backend/.sqlx/{query-8626f698fd20f2da77edcd9912a6f840f49002353d5c900a25a3024d7634a89c.json => query-188534f4b29f6461b1a6214d060f183c830b19a403ebb7b8be55a691675010c3.json} (84%) create mode 100644 backend/.sqlx/query-234a278f20cb73f8ce10d2bfb67af58e5dd888581467c976e76f140b2c00f6d7.json create mode 100644 backend/.sqlx/query-39154d865946cb99219ae0139dcc758523610d1260ff59aace18abaf4869edd3.json create mode 100644 backend/.sqlx/query-598d4be56bb8c5174b279c183fffa96f763e48316fb7420f2aad733562cab582.json create mode 100644 backend/.sqlx/query-9cd6e964ba933de247ba7ddce0acef0b70b784c2410f3a5f51288aaea6904b8f.json create mode 100644 backend/.sqlx/query-9f0783aae9cf5a6de2ca783aeba7a8c2f0eb5cc11cb2f49222f5eef4a21d8c41.json create mode 100644 backend/.sqlx/query-b1f2ab29cb09a213a38a711d5c228f4f735a3343fbae15d2bc3510c3308d16ba.json create mode 100644 backend/.sqlx/query-b30a29fa6edbeacaddad2a8b79a9aa73fcb0e8cb1f5e863f31faf98e4137b5e6.json create mode 100644 backend/.sqlx/query-c7dbcb52f6b44c8fc0d6240b7d73717ee7f24abd54fab5b4a687490b7c0b3672.json create mode 100644 cli/instance.ts diff --git a/backend/.sqlx/query-07834003a631acac03a52a128d3dfce162abdef0d94e5020b84fed424dfd1ae9.json b/backend/.sqlx/query-07834003a631acac03a52a128d3dfce162abdef0d94e5020b84fed424dfd1ae9.json new file mode 100644 index 0000000000..71aa649df9 --- /dev/null +++ b/backend/.sqlx/query-07834003a631acac03a52a128d3dfce162abdef0d94e5020b84fed424dfd1ae9.json @@ -0,0 +1,15 @@ +{ + "db_name": "PostgreSQL", + "query": "INSERT INTO email_to_igroup (email, igroup) VALUES ($1, $2)", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "Varchar", + "Varchar" + ] + }, + "nullable": [] + }, + "hash": "07834003a631acac03a52a128d3dfce162abdef0d94e5020b84fed424dfd1ae9" +} diff --git a/backend/.sqlx/query-16c93e721e30b72197e6143e0ebe46931ce25efc5ea0c53e160fcd98a96f4306.json b/backend/.sqlx/query-16c93e721e30b72197e6143e0ebe46931ce25efc5ea0c53e160fcd98a96f4306.json new file mode 100644 index 0000000000..004aa63aa3 --- /dev/null +++ b/backend/.sqlx/query-16c93e721e30b72197e6143e0ebe46931ce25efc5ea0c53e160fcd98a96f4306.json @@ -0,0 +1,12 @@ +{ + "db_name": "PostgreSQL", + "query": "DELETE FROM password", + "describe": { + "columns": [], + "parameters": { + "Left": [] + }, + "nullable": [] + }, + "hash": "16c93e721e30b72197e6143e0ebe46931ce25efc5ea0c53e160fcd98a96f4306" +} diff --git a/backend/.sqlx/query-8626f698fd20f2da77edcd9912a6f840f49002353d5c900a25a3024d7634a89c.json b/backend/.sqlx/query-188534f4b29f6461b1a6214d060f183c830b19a403ebb7b8be55a691675010c3.json similarity index 84% rename from backend/.sqlx/query-8626f698fd20f2da77edcd9912a6f840f49002353d5c900a25a3024d7634a89c.json rename to backend/.sqlx/query-188534f4b29f6461b1a6214d060f183c830b19a403ebb7b8be55a691675010c3.json index d122120b9a..83139e3a86 100644 --- a/backend/.sqlx/query-8626f698fd20f2da77edcd9912a6f840f49002353d5c900a25a3024d7634a89c.json +++ b/backend/.sqlx/query-188534f4b29f6461b1a6214d060f183c830b19a403ebb7b8be55a691675010c3.json @@ -1,6 +1,6 @@ { "db_name": "PostgreSQL", - "query": "SELECT\n -- slack_team_id, \n -- slack_name, \n -- slack_command_script, \n -- CASE WHEN slack_email = 'missing@email.xyz' THEN NULL ELSE slack_email END AS slack_email,\n auto_invite_domain IS NOT NULL AS \"auto_invite_enabled!\",\n CASE WHEN auto_invite_operator IS TRUE THEN 'operator' ELSE 'developer' END AS \"auto_invite_as!\", \n CASE WHEN auto_add IS TRUE THEN 'add' ELSE 'invite' END AS \"auto_invite_mode!\", \n webhook, \n deploy_to, \n error_handler, \n openai_resource_path, \n code_completion_enabled, \n error_handler_extra_args, \n error_handler_muted_on_cancel, \n large_file_storage, \n git_sync, \n default_app,\n default_scripts \n FROM workspace_settings\n WHERE workspace_id = $1", + "query": "SELECT\n -- slack_team_id, \n -- slack_name, \n -- slack_command_script, \n -- CASE WHEN slack_email = 'missing@email.xyz' THEN NULL ELSE slack_email END AS slack_email,\n auto_invite_domain IS NOT NULL AS \"auto_invite_enabled!\",\n CASE WHEN auto_invite_operator IS TRUE THEN 'operator' ELSE 'developer' END AS \"auto_invite_as!\", \n CASE WHEN auto_add IS TRUE THEN 'add' ELSE 'invite' END AS \"auto_invite_mode!\", \n webhook, \n deploy_to, \n error_handler, \n openai_resource_path, \n code_completion_enabled, \n error_handler_extra_args, \n error_handler_muted_on_cancel, \n large_file_storage, \n git_sync,\n default_app,\n default_scripts,\n workspace.name\n FROM workspace_settings\n LEFT JOIN workspace ON workspace.id = workspace_settings.workspace_id\n WHERE workspace_id = $1", "describe": { "columns": [ { @@ -72,6 +72,11 @@ "ordinal": 13, "name": "default_scripts", "type_info": "Jsonb" + }, + { + "ordinal": 14, + "name": "name", + "type_info": "Varchar" } ], "parameters": { @@ -93,8 +98,9 @@ true, true, true, - true + true, + false ] }, - "hash": "8626f698fd20f2da77edcd9912a6f840f49002353d5c900a25a3024d7634a89c" + "hash": "188534f4b29f6461b1a6214d060f183c830b19a403ebb7b8be55a691675010c3" } diff --git a/backend/.sqlx/query-234a278f20cb73f8ce10d2bfb67af58e5dd888581467c976e76f140b2c00f6d7.json b/backend/.sqlx/query-234a278f20cb73f8ce10d2bfb67af58e5dd888581467c976e76f140b2c00f6d7.json new file mode 100644 index 0000000000..b62dba4bbe --- /dev/null +++ b/backend/.sqlx/query-234a278f20cb73f8ce10d2bfb67af58e5dd888581467c976e76f140b2c00f6d7.json @@ -0,0 +1,18 @@ +{ + "db_name": "PostgreSQL", + "query": "INSERT INTO instance_group (name, summary, id, scim_display_name, external_id) VALUES ($1, $2, $3, $4, $5)", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "Varchar", + "Varchar", + "Varchar", + "Varchar", + "Varchar" + ] + }, + "nullable": [] + }, + "hash": "234a278f20cb73f8ce10d2bfb67af58e5dd888581467c976e76f140b2c00f6d7" +} diff --git a/backend/.sqlx/query-39154d865946cb99219ae0139dcc758523610d1260ff59aace18abaf4869edd3.json b/backend/.sqlx/query-39154d865946cb99219ae0139dcc758523610d1260ff59aace18abaf4869edd3.json new file mode 100644 index 0000000000..0d58c996c3 --- /dev/null +++ b/backend/.sqlx/query-39154d865946cb99219ae0139dcc758523610d1260ff59aace18abaf4869edd3.json @@ -0,0 +1,12 @@ +{ + "db_name": "PostgreSQL", + "query": "DELETE FROM email_to_igroup", + "describe": { + "columns": [], + "parameters": { + "Left": [] + }, + "nullable": [] + }, + "hash": "39154d865946cb99219ae0139dcc758523610d1260ff59aace18abaf4869edd3" +} diff --git a/backend/.sqlx/query-598d4be56bb8c5174b279c183fffa96f763e48316fb7420f2aad733562cab582.json b/backend/.sqlx/query-598d4be56bb8c5174b279c183fffa96f763e48316fb7420f2aad733562cab582.json new file mode 100644 index 0000000000..9fbc1ed774 --- /dev/null +++ b/backend/.sqlx/query-598d4be56bb8c5174b279c183fffa96f763e48316fb7420f2aad733562cab582.json @@ -0,0 +1,68 @@ +{ + "db_name": "PostgreSQL", + "query": "SELECT email, password_hash, login_type, super_admin, verified, name, company, first_time_user, username FROM password", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "email", + "type_info": "Varchar" + }, + { + "ordinal": 1, + "name": "password_hash", + "type_info": "Varchar" + }, + { + "ordinal": 2, + "name": "login_type", + "type_info": "Varchar" + }, + { + "ordinal": 3, + "name": "super_admin", + "type_info": "Bool" + }, + { + "ordinal": 4, + "name": "verified", + "type_info": "Bool" + }, + { + "ordinal": 5, + "name": "name", + "type_info": "Varchar" + }, + { + "ordinal": 6, + "name": "company", + "type_info": "Varchar" + }, + { + "ordinal": 7, + "name": "first_time_user", + "type_info": "Bool" + }, + { + "ordinal": 8, + "name": "username", + "type_info": "Varchar" + } + ], + "parameters": { + "Left": [] + }, + "nullable": [ + false, + true, + false, + false, + false, + true, + true, + false, + true + ] + }, + "hash": "598d4be56bb8c5174b279c183fffa96f763e48316fb7420f2aad733562cab582" +} diff --git a/backend/.sqlx/query-9cd6e964ba933de247ba7ddce0acef0b70b784c2410f3a5f51288aaea6904b8f.json b/backend/.sqlx/query-9cd6e964ba933de247ba7ddce0acef0b70b784c2410f3a5f51288aaea6904b8f.json new file mode 100644 index 0000000000..ed63942f01 --- /dev/null +++ b/backend/.sqlx/query-9cd6e964ba933de247ba7ddce0acef0b70b784c2410f3a5f51288aaea6904b8f.json @@ -0,0 +1,50 @@ +{ + "db_name": "PostgreSQL", + "query": "SELECT name, summary, array_remove(array_agg(email_to_igroup.email), null) as emails, id, scim_display_name, external_id FROM email_to_igroup RIGHT JOIN instance_group ON instance_group.name = email_to_igroup.igroup GROUP BY name", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "name", + "type_info": "Varchar" + }, + { + "ordinal": 1, + "name": "summary", + "type_info": "Varchar" + }, + { + "ordinal": 2, + "name": "emails", + "type_info": "VarcharArray" + }, + { + "ordinal": 3, + "name": "id", + "type_info": "Varchar" + }, + { + "ordinal": 4, + "name": "scim_display_name", + "type_info": "Varchar" + }, + { + "ordinal": 5, + "name": "external_id", + "type_info": "Varchar" + } + ], + "parameters": { + "Left": [] + }, + "nullable": [ + false, + true, + null, + true, + true, + true + ] + }, + "hash": "9cd6e964ba933de247ba7ddce0acef0b70b784c2410f3a5f51288aaea6904b8f" +} diff --git a/backend/.sqlx/query-9f0783aae9cf5a6de2ca783aeba7a8c2f0eb5cc11cb2f49222f5eef4a21d8c41.json b/backend/.sqlx/query-9f0783aae9cf5a6de2ca783aeba7a8c2f0eb5cc11cb2f49222f5eef4a21d8c41.json new file mode 100644 index 0000000000..c1449320d9 --- /dev/null +++ b/backend/.sqlx/query-9f0783aae9cf5a6de2ca783aeba7a8c2f0eb5cc11cb2f49222f5eef4a21d8c41.json @@ -0,0 +1,22 @@ +{ + "db_name": "PostgreSQL", + "query": "INSERT INTO password(email, password_hash, login_type, super_admin, verified, name, company, first_time_user, username)\n VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9)", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "Varchar", + "Varchar", + "Varchar", + "Bool", + "Bool", + "Varchar", + "Varchar", + "Bool", + "Varchar" + ] + }, + "nullable": [] + }, + "hash": "9f0783aae9cf5a6de2ca783aeba7a8c2f0eb5cc11cb2f49222f5eef4a21d8c41" +} diff --git a/backend/.sqlx/query-b1f2ab29cb09a213a38a711d5c228f4f735a3343fbae15d2bc3510c3308d16ba.json b/backend/.sqlx/query-b1f2ab29cb09a213a38a711d5c228f4f735a3343fbae15d2bc3510c3308d16ba.json new file mode 100644 index 0000000000..7d67eebe2c --- /dev/null +++ b/backend/.sqlx/query-b1f2ab29cb09a213a38a711d5c228f4f735a3343fbae15d2bc3510c3308d16ba.json @@ -0,0 +1,26 @@ +{ + "db_name": "PostgreSQL", + "query": "SELECT name, config FROM config", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "name", + "type_info": "Varchar" + }, + { + "ordinal": 1, + "name": "config", + "type_info": "Jsonb" + } + ], + "parameters": { + "Left": [] + }, + "nullable": [ + false, + true + ] + }, + "hash": "b1f2ab29cb09a213a38a711d5c228f4f735a3343fbae15d2bc3510c3308d16ba" +} diff --git a/backend/.sqlx/query-b30a29fa6edbeacaddad2a8b79a9aa73fcb0e8cb1f5e863f31faf98e4137b5e6.json b/backend/.sqlx/query-b30a29fa6edbeacaddad2a8b79a9aa73fcb0e8cb1f5e863f31faf98e4137b5e6.json new file mode 100644 index 0000000000..f12c7413c5 --- /dev/null +++ b/backend/.sqlx/query-b30a29fa6edbeacaddad2a8b79a9aa73fcb0e8cb1f5e863f31faf98e4137b5e6.json @@ -0,0 +1,26 @@ +{ + "db_name": "PostgreSQL", + "query": "SELECT name, value FROM global_settings", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "name", + "type_info": "Varchar" + }, + { + "ordinal": 1, + "name": "value", + "type_info": "Jsonb" + } + ], + "parameters": { + "Left": [] + }, + "nullable": [ + false, + false + ] + }, + "hash": "b30a29fa6edbeacaddad2a8b79a9aa73fcb0e8cb1f5e863f31faf98e4137b5e6" +} diff --git a/backend/.sqlx/query-b3dbdfb50ee8118bdaed3164b210cb549a34b96554ae1872355b90304f5dcb76.json b/backend/.sqlx/query-b3dbdfb50ee8118bdaed3164b210cb549a34b96554ae1872355b90304f5dcb76.json index 99269c9851..54e94cfb8f 100644 --- a/backend/.sqlx/query-b3dbdfb50ee8118bdaed3164b210cb549a34b96554ae1872355b90304f5dcb76.json +++ b/backend/.sqlx/query-b3dbdfb50ee8118bdaed3164b210cb549a34b96554ae1872355b90304f5dcb76.json @@ -18,8 +18,8 @@ "Left": [] }, "nullable": [ - true, - false + false, + true ] }, "hash": "b3dbdfb50ee8118bdaed3164b210cb549a34b96554ae1872355b90304f5dcb76" diff --git a/backend/.sqlx/query-c7dbcb52f6b44c8fc0d6240b7d73717ee7f24abd54fab5b4a687490b7c0b3672.json b/backend/.sqlx/query-c7dbcb52f6b44c8fc0d6240b7d73717ee7f24abd54fab5b4a687490b7c0b3672.json new file mode 100644 index 0000000000..69a201a650 --- /dev/null +++ b/backend/.sqlx/query-c7dbcb52f6b44c8fc0d6240b7d73717ee7f24abd54fab5b4a687490b7c0b3672.json @@ -0,0 +1,12 @@ +{ + "db_name": "PostgreSQL", + "query": "DELETE FROM instance_group", + "describe": { + "columns": [], + "parameters": { + "Left": [] + }, + "nullable": [] + }, + "hash": "c7dbcb52f6b44c8fc0d6240b7d73717ee7f24abd54fab5b4a687490b7c0b3672" +} diff --git a/backend/windmill-api/openapi.yaml b/backend/windmill-api/openapi.yaml index 7c6d92926a..a63677d7ea 100644 --- a/backend/windmill-api/openapi.yaml +++ b/backend/windmill-api/openapi.yaml @@ -439,6 +439,44 @@ paths: text/plain: schema: type: string + /users/overwrite: + post: + summary: global overwrite users (require super admin and EE) + operationId: globalUsersOverwrite + tags: + - user + requestBody: + description: List of users + required: true + content: + application/json: + schema: + type: array + items: + $ref: "#/components/schemas/ExportedUser" + responses: + "200": + description: Success message + content: + text/plain: + schema: + type: string + + /users/export: + get: + summary: global export users (require super admin and EE) + operationId: globalUsersExport + tags: + - user + responses: + "200": + description: exported users + content: + application/json: + schema: + type: array + items: + $ref: "#/components/schemas/ExportedUser" /w/{workspace}/users/delete/{username}: delete: @@ -850,6 +888,23 @@ paths: schema: type: string + + /settings/list_global: + get: + summary: list global settings + operationId: listGlobalSettings + tags: + - setting + responses: + "200": + description: list of settings + content: + application/json: + schema: + type: array + items: + $ref: "#/components/schemas/GlobalSetting" + /users/email: get: summary: get current user email (if logged in) @@ -1971,6 +2026,8 @@ paths: properties: new_key: type: string + skip_reencrypt: + type: boolean required: - new_key @@ -6827,6 +6884,44 @@ paths: text/plain: schema: type: string + /groups/export: + get: + summary: export instance groups + operationId: exportInstanceGroups + tags: + - group + responses: + "200": + description: exported instance groups + content: + application/json: + schema: + type: array + items: + $ref: "#/components/schemas/ExportedInstanceGroup" + + /groups/overwrite: + post: + summary: overwrite instance groups + operationId: overwriteInstanceGroups + tags: + - group + requestBody: + description: overwrite instance groups + required: true + content: + application/json: + schema: + type: array + items: + $ref: "#/components/schemas/ExportedInstanceGroup" + responses: + "200": + description: success message + content: + text/plain: + schema: + type: string /w/{workspace}/groups/list: get: @@ -7415,6 +7510,22 @@ paths: schema: type: string + /configs/list: + get: + summary: list configs + operationId: listConfigs + tags: + - config + responses: + "200": + description: list of configs + content: + application/json: + schema: + type: array + items: + $ref: "#/components/schemas/Config" + /w/{workspace}/acls/get/{kind}/{path}: get: summary: get granular acls @@ -11121,6 +11232,72 @@ components: required: - jobs - obscured_jobs + + ExportedUser: + type: object + properties: + email: + type: string + password_hash: + type: string + super_admin: + type: boolean + verified: + type: boolean + name: + type: string + company: + type: string + first_time_user: + type: boolean + username: + type: string + required: + - email + - super_admin + - verified + - first_time_user + + GlobalSetting: + type: object + properties: + name: + type: string + value: + type: object + required: + - name + - value + + Config: + type: object + properties: + name: + type: string + config: + type: object + required: + - name + + ExportedInstanceGroup: + type: object + properties: + name: + type: string + summary: + type: string + emails: + type: array + items: + type: string + id: + type: string + scim_display_name: + type: string + external_id: + type: string + required: + - name JobSearchHit: type: object diff --git a/backend/windmill-api/src/configs.rs b/backend/windmill-api/src/configs.rs index adeb7e9fbe..aacfd49ef7 100644 --- a/backend/windmill-api/src/configs.rs +++ b/backend/windmill-api/src/configs.rs @@ -28,6 +28,7 @@ pub fn global_service() -> Router { .route("/list_worker_groups", get(list_worker_groups)) .route("/update/:name", post(update_config).delete(delete_config)) .route("/get/:name", get(get_config)) + .route("/list", get(list_configs)) } #[derive(Serialize, Deserialize, FromRow)] @@ -166,3 +167,22 @@ async fn delete_config( } Ok(format!("Deleted config {name}")) } + +#[cfg(feature = "enterprise")] +async fn list_configs( + authed: ApiAuthed, + Extension(db): Extension, +) -> error::JsonResult> { + require_super_admin(&db, &authed.email).await?; + let configs = sqlx::query_as!(Config, "SELECT name, config FROM config") + .fetch_all(&db) + .await?; + Ok(Json(configs)) +} + +#[cfg(not(feature = "enterprise"))] +async fn list_configs() -> error::JsonResult { + Err(error::Error::BadRequest( + "Config listing available only in the enterprise version".to_string(), + )) +} diff --git a/backend/windmill-api/src/groups.rs b/backend/windmill-api/src/groups.rs index dc545ab2e8..b2373c69de 100644 --- a/backend/windmill-api/src/groups.rs +++ b/backend/windmill-api/src/groups.rs @@ -50,6 +50,8 @@ pub fn global_service() -> Router { .route("/delete/:name", delete(delete_igroup)) .route("/adduser/:name", post(add_user_igroup)) .route("/removeuser/:name", post(remove_user_igroup)) + .route("/export", get(export_igroups)) + .route("/overwrite", post(overwrite_igroups)) } #[derive(FromRow, Serialize, Deserialize)] @@ -759,3 +761,118 @@ async fn remove_user( Ok(format!("Removed {} to group {}", user_username, name)) } + +#[cfg(feature = "enterprise")] +#[derive(Serialize, Deserialize)] +struct ExportedIGroup { + name: String, + #[serde(skip_serializing_if = "Option::is_none")] + summary: Option, + #[serde(skip_serializing_if = "Option::is_none")] + id: Option, + #[serde(skip_serializing_if = "Option::is_none")] + scim_display_name: Option, + #[serde(skip_serializing_if = "Option::is_none")] + external_id: Option, + #[serde(skip_serializing_if = "Option::is_none")] + emails: Option>, +} + +#[cfg(feature = "enterprise")] +async fn export_igroups( + authed: ApiAuthed, + Extension(db): Extension, +) -> JsonResult> { + require_super_admin(&db, &authed.email).await?; + let mut tx = db.begin().await?; + let igroups = sqlx::query_as!( + ExportedIGroup, + "SELECT name, summary, array_remove(array_agg(email_to_igroup.email), null) as emails, id, scim_display_name, external_id FROM email_to_igroup RIGHT JOIN instance_group ON instance_group.name = email_to_igroup.igroup GROUP BY name", + ).fetch_all(&mut *tx).await?; + + audit_log( + &mut *tx, + &authed, + "igroups.export", + ActionKind::Execute, + "global", + None, + None, + ) + .await?; + + tx.commit().await?; + + Ok(Json(igroups)) +} + +#[cfg(not(feature = "enterprise"))] +async fn export_igroups() -> JsonResult { + Err(Error::BadRequest( + "This feature is only available in the enterprise version".to_string(), + )) +} + +#[cfg(feature = "enterprise")] +async fn overwrite_igroups( + authed: ApiAuthed, + Extension(db): Extension, + Json(igroups): Json>, +) -> Result { + require_super_admin(&db, &authed.email).await?; + let mut tx = db.begin().await?; + + sqlx::query!("DELETE FROM email_to_igroup") + .execute(&mut *tx) + .await?; + + sqlx::query!("DELETE FROM instance_group") + .execute(&mut *tx) + .await?; + + for igroup in igroups.iter() { + sqlx::query!( + "INSERT INTO instance_group (name, summary, id, scim_display_name, external_id) VALUES ($1, $2, $3, $4, $5)", + igroup.name, + igroup.summary, + igroup.id, + igroup.scim_display_name, + igroup.external_id, + ) + .execute(&mut *tx) + .await?; + + if let Some(emails) = &igroup.emails { + for email in emails.iter() { + sqlx::query!( + "INSERT INTO email_to_igroup (email, igroup) VALUES ($1, $2)", + email, + igroup.name, + ) + .execute(&mut *tx) + .await?; + } + } + } + + audit_log( + &mut *tx, + &authed, + "igroups.import", + ActionKind::Create, + "global", + None, + None, + ) + .await?; + + tx.commit().await?; + Ok("Imported igroups".to_string()) +} + +#[cfg(not(feature = "enterprise"))] +async fn overwrite_igroups() -> JsonResult { + Err(Error::BadRequest( + "This feature is only available in the enterprise version".to_string(), + )) +} diff --git a/backend/windmill-api/src/settings.rs b/backend/windmill-api/src/settings.rs index 9744dd9901..5cb16cc661 100644 --- a/backend/windmill-api/src/settings.rs +++ b/backend/windmill-api/src/settings.rs @@ -40,6 +40,7 @@ pub fn global_service() -> Router { "/global/:key", post(set_global_setting).get(get_global_setting), ) + .route("/list_global", get(list_global_settings)) .route("/test_smtp", post(test_email)) .route("/test_license_key", post(test_license_key)) .route("/send_stats", post(send_stats)) @@ -263,6 +264,33 @@ pub async fn get_global_setting( Ok(Json(value.unwrap_or_else(|| serde_json::Value::Null))) } +#[cfg(feature = "enterprise")] +#[derive(Deserialize, serde::Serialize)] +struct GlobalSetting { + name: String, + value: serde_json::Value, +} + +#[cfg(feature = "enterprise")] +async fn list_global_settings( + Extension(db): Extension, + authed: ApiAuthed, +) -> JsonResult> { + require_super_admin(&db, &authed.email).await?; + let settings = sqlx::query_as!(GlobalSetting, "SELECT name, value FROM global_settings") + .fetch_all(&db) + .await?; + + Ok(Json(settings)) +} + +#[cfg(not(feature = "enterprise"))] +async fn list_global_settings() -> JsonResult { + return Err(error::Error::BadRequest( + "Listing global settings not available on community edition".to_string(), + )); +} + pub async fn send_stats(Extension(db): Extension, authed: ApiAuthed) -> Result { require_super_admin(&db, &authed.email).await?; windmill_common::stats_ee::send_stats(&"manual".to_string(), &HTTP_CLIENT, &db).await?; diff --git a/backend/windmill-api/src/users.rs b/backend/windmill-api/src/users.rs index d589f439c4..8e6e48dea7 100644 --- a/backend/windmill-api/src/users.rs +++ b/backend/windmill-api/src/users.rs @@ -104,6 +104,8 @@ pub fn global_service() -> Router { post(update_tutorial_progress).get(get_tutorial_progress), ) .route("/leave_instance", post(leave_instance)) + .route("/export", get(export_global_users)) + .route("/overwrite", post(overwrite_global_users)) // .route("/list_invite_codes", get(list_invite_codes)) // .route("/create_invite_code", post(create_invite_code)) // .route("/signup", post(signup)) @@ -2643,6 +2645,106 @@ async fn username_to_email( Ok(email) } +#[cfg(feature = "enterprise")] +#[derive(Serialize, Deserialize)] +struct ExportedGlobalUser { + email: String, + password_hash: Option, + login_type: String, + super_admin: bool, + verified: bool, + name: Option, + company: Option, + first_time_user: bool, + username: Option, +} + +#[cfg(feature = "enterprise")] +async fn export_global_users( + Extension(db): Extension, + authed: ApiAuthed, +) -> JsonResult> { + require_super_admin(&db, &authed.email).await?; + let mut tx = db.begin().await?; + let users = sqlx::query_as!( + ExportedGlobalUser, + "SELECT email, password_hash, login_type, super_admin, verified, name, company, first_time_user, username FROM password" + ) + .fetch_all(&mut *tx) + .await?; + + audit_log( + &mut *tx, + &authed, + "users.export_export", + ActionKind::Execute, + "global", + None, + None, + ) + .await?; + + tx.commit().await?; + + Ok(Json(users)) +} + +#[cfg(not(feature = "enterprise"))] +async fn export_global_users() -> JsonResult { + Err(Error::BadRequest( + "This feature is only available in the enterprise version".to_string(), + )) +} + +#[cfg(feature = "enterprise")] +async fn overwrite_global_users( + Extension(db): Extension, + authed: ApiAuthed, + Json(users): Json>, +) -> Result { + require_super_admin(&db, &authed.email).await?; + let mut tx = db.begin().await?; + sqlx::query!("DELETE FROM password") + .execute(&mut *tx) + .await?; + for user in users { + sqlx::query!( + "INSERT INTO password(email, password_hash, login_type, super_admin, verified, name, company, first_time_user, username) + VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9)", + user.email, + user.password_hash, + user.login_type, + user.super_admin, + user.verified, + user.name, + user.company, + user.first_time_user, + user.username + ) + .execute(&mut *tx) + .await?; + } + audit_log( + &mut *tx, + &authed, + "users.import_global", + ActionKind::Create, + "global", + None, + None, + ) + .await?; + tx.commit().await?; + Ok("loaded global users".to_string()) +} + +#[cfg(not(feature = "enterprise"))] +async fn overwrite_global_users() -> JsonResult { + Err(Error::BadRequest( + "This feature is only available in the enterprise version".to_string(), + )) +} + #[derive(Deserialize)] struct RenameUser { new_username: String, diff --git a/backend/windmill-api/src/variables.rs b/backend/windmill-api/src/variables.rs index f5907653e4..b6f1f0ecdc 100644 --- a/backend/windmill-api/src/variables.rs +++ b/backend/windmill-api/src/variables.rs @@ -32,7 +32,7 @@ use windmill_common::{ }; use lazy_static::lazy_static; -use magic_crypt::{MagicCrypt256, MagicCryptTrait}; +use magic_crypt::{MagicCrypt256, MagicCryptError, MagicCryptTrait}; use serde::Deserialize; use sqlx::{Postgres, Transaction}; use windmill_git_sync::{handle_deployment_metadata, DeployedObject}; @@ -656,6 +656,11 @@ pub fn encrypt(mc: &MagicCrypt256, value: &str) -> String { } pub fn decrypt(mc: &MagicCrypt256, value: String) -> Result { - mc.decrypt_base64_to_string(value) - .map_err(|e| Error::InternalErr(e.to_string())) + mc.decrypt_base64_to_string(value).map_err(|e| match e { + MagicCryptError::DecryptError(_) => Error::InternalErr( + "Could not decrypt value. The value may have been encrypted with a different key." + .to_string(), + ), + _ => Error::InternalErr(e.to_string()), + }) } diff --git a/backend/windmill-api/src/workspaces.rs b/backend/windmill-api/src/workspaces.rs index 51e4d1df39..360ff3bfbe 100644 --- a/backend/windmill-api/src/workspaces.rs +++ b/backend/windmill-api/src/workspaces.rs @@ -1350,6 +1350,7 @@ async fn get_encryption_key( #[derive(Deserialize)] struct SetEncryptionKeyRequest { new_key: String, + skip_reencrypt: Option, } async fn set_encryption_key( @@ -1375,37 +1376,40 @@ async fn set_encryption_key( ) .execute(&db) .await?; - let new_encryption_key = build_crypt(&db, w_id.as_str()).await?; - let mut truncated_new_key = request.new_key.clone(); - truncated_new_key.truncate(8); - tracing::warn!( - "Re-encrypting all secrets for workspace {}. New key is {}***", - w_id, - truncated_new_key - ); + if !request.skip_reencrypt.unwrap_or(false) { + let new_encryption_key = build_crypt(&db, w_id.as_str()).await?; - let all_variables = sqlx::query!( - "SELECT path, value, is_secret FROM variable WHERE workspace_id = $1", - w_id - ) - .fetch_all(&db) - .await?; - - for variable in all_variables { - if !variable.is_secret { - continue; - } - let decrypted_value = decrypt(&previous_encryption_key, variable.value)?; - let new_encrypted_value = encrypt(&new_encryption_key, decrypted_value.as_str()); - sqlx::query!( - "UPDATE variable SET value = $1 WHERE workspace_id = $2 AND path = $3", - new_encrypted_value, + let mut truncated_new_key = request.new_key.clone(); + truncated_new_key.truncate(8); + tracing::warn!( + "Re-encrypting all secrets for workspace {}. New key is {}***", w_id, - variable.path + truncated_new_key + ); + + let all_variables = sqlx::query!( + "SELECT path, value, is_secret FROM variable WHERE workspace_id = $1", + w_id ) - .execute(&db) + .fetch_all(&db) .await?; + + for variable in all_variables { + if !variable.is_secret { + continue; + } + let decrypted_value = decrypt(&previous_encryption_key, variable.value)?; + let new_encrypted_value = encrypt(&new_encryption_key, decrypted_value.as_str()); + sqlx::query!( + "UPDATE variable SET value = $1 WHERE workspace_id = $2 AND path = $3", + new_encrypted_value, + w_id, + variable.path + ) + .execute(&db) + .await?; + } } return Ok(()); @@ -2312,6 +2316,7 @@ struct ArchiveQueryParams { include_users: Option, include_groups: Option, include_settings: Option, + include_key: Option, default_ts: Option, } @@ -2406,6 +2411,7 @@ struct SimplifiedSettings { git_sync: Option, default_app: Option, default_scripts: Option, + name: String, } async fn tarball_workspace( @@ -2424,6 +2430,7 @@ async fn tarball_workspace( include_users, include_groups, include_settings, + include_key, default_ts, }): Query, ) -> Result<([(HeaderName, String); 2], impl IntoResponse)> { @@ -2765,20 +2772,45 @@ async fn tarball_workspace( error_handler_extra_args, error_handler_muted_on_cancel, large_file_storage, - git_sync, + git_sync, default_app, - default_scripts + default_scripts, + workspace.name FROM workspace_settings + LEFT JOIN workspace ON workspace.id = workspace_settings.workspace_id WHERE workspace_id = $1"#, &w_id ).fetch_one(&mut *tx).await?; - let settings_str = &to_string_without_metadata(&settings, true, None).unwrap(); + let settings_str = serde_json::to_value(settings) + .map(|v| serde_json::to_string_pretty(&v).ok()) + .ok() + .flatten() + .ok_or_else(|| Error::InternalErr("Error serializing settings".to_string()))?; + archive .write_to_archive(&settings_str, "settings.json") .await?; } + if include_key.unwrap_or(false) { + let key = sqlx::query_scalar!( + "SELECT key FROM workspace_key WHERE workspace_id = $1", + &w_id + ) + .fetch_one(&mut *tx) + .await?; + + let key_json = serde_json::to_value(key) + .map(|v| serde_json::to_string_pretty(&v).ok()) + .ok() + .flatten() + .ok_or_else(|| Error::InternalErr("Error serializing enryption key".to_string()))?; + archive + .write_to_archive(&key_json, "encryption_key.json") + .await?; + } + archive.finish().await?; let file = tokio::fs::File::open(&file_path).await?; diff --git a/backend/windmill-common/src/variables.rs b/backend/windmill-common/src/variables.rs index 59799439c9..a908802f4b 100644 --- a/backend/windmill-common/src/variables.rs +++ b/backend/windmill-common/src/variables.rs @@ -6,7 +6,7 @@ * LICENSE-AGPL for a copy of the license. */ -use magic_crypt::{MagicCrypt256, MagicCryptTrait}; +use magic_crypt::{MagicCrypt256, MagicCryptError, MagicCryptTrait}; use serde::{Deserialize, Serialize}; use crate::{worker::WORKER_GROUP, BASE_URL, DB}; @@ -138,9 +138,13 @@ pub async fn decrypt_value_with_mc( value: String, mc: MagicCrypt256, ) -> Result { - Ok(mc - .decrypt_base64_to_string(value) - .map_err(|e| crate::Error::InternalErr(e.to_string()))?) + Ok(mc.decrypt_base64_to_string(value).map_err(|e| match e { + MagicCryptError::DecryptError(_) => crate::error::Error::InternalErr( + "Could not decrypt value. The value may have been encrypted with a different key." + .to_string(), + ), + _ => crate::error::Error::InternalErr(e.to_string()), + })?) } pub async fn get_reserved_variables( diff --git a/cli/conf.ts b/cli/conf.ts index 0388e084ba..bbf04881a4 100644 --- a/cli/conf.ts +++ b/cli/conf.ts @@ -15,6 +15,7 @@ export interface SyncOptions { includeUsers?: boolean; includeGroups?: boolean; includeSettings?: boolean; + includeKey?: boolean; message?: string; includes?: string[]; extraIncludes?: string[]; diff --git a/cli/deps.ts b/cli/deps.ts index 19200dfcd3..5f5d809af3 100644 --- a/cli/deps.ts +++ b/cli/deps.ts @@ -1,5 +1,5 @@ // windmill -export * from "npm:windmill-client@1.319.1"; +export * from "npm:windmill-client@1.364.0"; // cliffy export { Command } from "https://deno.land/x/cliffy@v1.0.0-rc.4/command/mod.ts"; diff --git a/cli/instance.ts b/cli/instance.ts new file mode 100644 index 0000000000..48a2caba16 --- /dev/null +++ b/cli/instance.ts @@ -0,0 +1,542 @@ +import { + Select, + WorkspaceService, + path, + Confirm, + yamlStringify, + yamlParse, + Command, + setClient, +} from "./deps.ts"; +import { DelimiterStream, Input, colors, log } from "./deps.ts"; +import { loginInteractive } from "./login.ts"; +import { getRootStore } from "./store.ts"; +import { push, pull } from "./sync.ts"; +import { showDiff } from "./types.ts"; + +import { + pushInstanceUsers, + pullInstanceUsers, + pullInstanceGroups, + pushInstanceGroups, +} from "./user.ts"; +import { + add as workspaceSetup, + addWorkspace, + allWorkspaces, + removeWorkspace, +} from "./workspace.ts"; +import { + pushInstanceSettings, + pullInstanceSettings, + pullInstanceConfigs, + pushInstanceConfigs, + type SimplifiedSettings, +} from "./settings.ts"; +import { sleep, deepEqual } from "./utils.ts"; +import { GlobalOptions } from "./types.ts"; + +export interface Instance { + remote: string; + name: string; + token: string; + prefix: string; +} + +function makeInstanceStream( + readable: ReadableStream +): ReadableStream { + return readable + .pipeThrough(new DelimiterStream(new TextEncoder().encode("\n"))) + .pipeThrough(new TextDecoderStream()) + .pipeThrough( + new TransformStream({ + transform(line, controller) { + try { + if (line.length <= 2) { + return; + } + const instance = JSON.parse(line) as Instance; + controller.enqueue(instance); + } catch { + /* ignore */ + } + }, + }) + ); +} + +async function getInstanceStream() { + const file = await Deno.open((await getRootStore()) + "instances.ndjson", { + write: false, + read: true, + }); + return makeInstanceStream(file.readable); +} + +export async function allInstances(): Promise { + try { + const instanceStream = await getInstanceStream(); + const instances: Instance[] = []; + for await (const instance of instanceStream) { + instances.push(instance); + } + + return instances; + } catch (_) { + return []; + } +} +export async function addInstance() { + let remote = await Input.prompt({ + message: "Enter the remote url of this instance", + default: "https://app.windmill.dev/", + }); + remote = new URL(remote).toString(); // add trailing slash in all cases! + + const defaultName = new URL(remote).hostname; + + const name = await Input.prompt({ + message: "Enter a name for this instance", + default: defaultName, + }); + + const prefix = name.toLowerCase().replace(/[^a-z0-9]/g, ""); + + let token: string | undefined = undefined; + while (!token) { + token = await loginInteractive(remote); + } + + await appendInstance({ + name, + remote, + token, + prefix, + }); + log.info( + colors.green.underline(`Added instance ${name} with remote ${remote}!`) + ); + + return { + name, + remote, + token, + prefix, + }; +} + +async function appendInstance(instance: Instance) { + instance.remote = new URL(instance.remote).toString(); // add trailing slash in all cases! + await removeInstance(instance.name); + const file = await Deno.open((await getRootStore()) + "instances.ndjson", { + append: true, + write: true, + read: true, + create: true, + }); + await file.write(new TextEncoder().encode(JSON.stringify(instance) + "\n")); + + file.close(); +} + +async function removeInstance(name: string) { + const orgWorkspaces = await allInstances(); + + await Deno.writeTextFile( + (await getRootStore()) + "instances.ndjson", + orgWorkspaces + .filter((x) => x.name !== name) + .map((x) => JSON.stringify(x)) + .join("\n") + "\n" + ); +} + +type CompareObject = { + [K in T]: string; +}; +export function compareInstanceObjects( + fromObjects: CompareObject[], + toObjects: CompareObject[], + idProp: T, + objectName: string +) { + let changes = 0; + for (const toObject of toObjects) { + const fromMatch = fromObjects.find((u) => u[idProp] === toObject[idProp]); + if (!fromMatch) { + log.info(colors.red(`- instance ${objectName} ${toObject[idProp]}`)); + changes += 1; + } else if (!deepEqual(toObject, fromMatch)) { + log.info(colors.yellow(`~ instance ${objectName} ${toObject[idProp]}`)); + showDiff(yamlStringify(toObject), yamlStringify(fromMatch)); + changes += 1; + } + } + + for (const fromObject of fromObjects) { + const toMatch = toObjects.find((u) => u[idProp] === fromObject[idProp]); + if (!toMatch) { + log.info(colors.green(`+ instance ${objectName} ${fromObject[idProp]}`)); + changes += 1; + } + } + + return changes; +} + +type InstanceSyncOptions = { + skipUsers?: boolean; + skipSettings?: boolean; + skipConfigs?: boolean; + skipGroups?: boolean; + includeWorkspaces?: boolean; + baseUrl?: string; +}; + +async function instancePull(opts: GlobalOptions & InstanceSyncOptions) { + const instances = await allInstances(); + let instance: Instance; + if (instances.length < 1) { + instance = await addInstance(); + } else { + const choice = (await Select.prompt({ + message: "Select an instance to pull from", + options: [ + ...instances.map((i) => ({ + name: `${i.name} (${i.remote})`, + value: i.name, + })), + { name: "Add new instance", value: "new" }, + ], + })) as unknown as string; + + if (choice === "new") { + instance = await addInstance(); + } else { + instance = instances.find((i) => i.name === choice)!; + } + } + + setClient( + instance.token, + instance.remote.slice(0, instance.remote.length - 1) + ); + + log.info("Pulling instance-level changes"); + log.info(`remote (${instance.name}) -> local`); + + let uChanges = 0; + if (!opts.skipUsers) { + uChanges = (await pullInstanceUsers(true)) as number; + } + let sChanges = 0; + if (!opts.skipSettings) { + sChanges = (await pullInstanceSettings(true)) as number; + } + let cChanges = 0; + if (!opts.skipConfigs) { + cChanges = (await pullInstanceConfigs(true)) as number; + } + let gChanges = 0; + if (!opts.skipGroups) { + gChanges = (await pullInstanceGroups(true)) as number; + } + + const totalChanges = uChanges + sChanges + cChanges + gChanges; + + if (totalChanges > 0) { + const confirm = await Confirm.prompt({ + message: `Do you want to apply these ${totalChanges} instance-level changes?`, + default: true, + }); + + if (confirm) { + if (!opts.skipUsers && uChanges > 0) { + await pullInstanceUsers(); + } + if (!opts.skipSettings && sChanges > 0) { + await pullInstanceSettings(); + } + if (!opts.skipConfigs && cChanges > 0) { + await pullInstanceConfigs(); + } + if (!opts.skipGroups && gChanges > 0) { + await pullInstanceGroups(); + } + } + } else { + log.info("No instance-level changes to apply"); + } + + sleep(1000); + + if (opts.includeWorkspaces) { + log.info("\nPulling all workspaces"); + const remoteWorkspaces = await WorkspaceService.listWorkspacesAsSuperAdmin({ + page: 1, + perPage: 1000, + }); + let localWorkspaces = await allWorkspaces(); + localWorkspaces = localWorkspaces.filter((w) => + w.name.startsWith(instance.prefix + "_") + ); + const rootDir = Deno.cwd(); + for (const remoteWorkspace of remoteWorkspaces) { + log.info("\nPulling workspace " + remoteWorkspace.id); + sleep(1000); + const workspaceName = instance.prefix + "_" + remoteWorkspace.id; + await Deno.mkdir(path.join(rootDir, workspaceName), { + recursive: true, + }); + await Deno.chdir(path.join(rootDir, workspaceName)); + await addWorkspace( + { + remote: instance.remote, + name: workspaceName, + workspaceId: remoteWorkspace.id, + token: instance.token, + }, + { + token: undefined, + workspace: undefined, + } + ); + + await pull({ + workspace: workspaceName, + token: undefined, + includeGroups: true, + includeSchedules: true, + includeSettings: true, + includeUsers: true, + includeKey: true, + }); + } + + const localWorkspacesToDelete = localWorkspaces.filter( + (w) => !remoteWorkspaces.find((r) => r.id === w.workspaceId) + ); + + if (localWorkspacesToDelete.length > 0) { + const confirmDelete = await Confirm.prompt({ + message: + "Do you want to delete the local copy of workspaces that don't exist anymore on the instance?\n" + + localWorkspacesToDelete.map((w) => w.workspaceId).join(", "), + default: true, + }); + + if (confirmDelete) { + for (const workspace of localWorkspacesToDelete) { + await removeWorkspace(workspace.name, false, {}); + await Deno.remove(path.join(rootDir, workspace.name), { + recursive: true, + }); + } + } + } + + log.info(colors.green.underline.bold("All workspaces pulled")); + } +} + +async function instancePush(opts: GlobalOptions & InstanceSyncOptions) { + let instances = await allInstances(); + let instance: Instance; + if (instances.length < 1) { + instance = await addInstance(); + } else { + const choice = (await Select.prompt({ + message: "Select an instance to push to", + options: [ + ...instances.map((i) => ({ + name: `${i.name} (${i.remote})`, + value: i.name, + })), + { name: "Add new instance", value: "new" }, + ], + })) as unknown as string; + + if (choice === "new") { + instance = await addInstance(); + } else { + instance = instances.find((i) => i.name === choice)!; + } + } + + setClient( + instance.token, + instance.remote.slice(0, instance.remote.length - 1) + ); + + log.info("Pushing instance-level changes"); + log.info!(`remote (${instance.name}) <- local`); + + let uChanges = 0; + if (!opts.skipUsers) { + uChanges = (await pushInstanceUsers(true)) as number; + } + let sChanges = 0; + if (!opts.skipSettings) { + sChanges = (await pushInstanceSettings(true, opts.baseUrl)) as number; + } + let cChanges = 0; + if (!opts.skipConfigs) { + cChanges = (await pushInstanceConfigs(true)) as number; + } + let gChanges = 0; + if (!opts.skipGroups) { + gChanges = (await pushInstanceGroups(true)) as number; + } + + const totalChanges = uChanges + sChanges + cChanges + gChanges; + + if (totalChanges > 0) { + const confirm = await Confirm.prompt({ + message: `Do you want to apply these ${totalChanges} instance-level changes?`, + default: true, + }); + + if (confirm) { + if (!opts.skipUsers && uChanges > 0) { + await pushInstanceUsers(); + } + if (!opts.skipSettings && sChanges > 0) { + await pushInstanceSettings(false, opts.baseUrl); + } + if (!opts.skipConfigs && cChanges > 0) { + await pushInstanceConfigs(); + } + if (!opts.skipGroups && gChanges > 0) { + await pushInstanceGroups(); + } + } + } else { + log.info("No instance-level changes to apply"); + } + + sleep(1000); + + if (opts.includeWorkspaces) { + instances = await allInstances(); + const localPrefix = (await Select.prompt({ + message: "What is the prefix of the local workspaces you want to sync?", + options: [ + ...instances.map((i) => ({ + name: `${i.prefix} (${i.name} - ${i.remote})`, + value: i.prefix, + })), + ], + default: instance.prefix as unknown, + })) as unknown as string; + + const remoteWorkspaces = await WorkspaceService.listWorkspacesAsSuperAdmin({ + page: 1, + perPage: 1000, + }); + let localWorkspaces = await allWorkspaces(); + localWorkspaces = localWorkspaces.filter((w) => + w.name.startsWith(localPrefix + "_") + ); + + log.info("\nPushing all workspaces"); + const rootDir = Deno.cwd(); + for (const localWorkspace of localWorkspaces) { + log.info("\nPushing workspace " + localWorkspace.workspaceId); + sleep(1000); + try { + await Deno.chdir(path.join(rootDir, localWorkspace.name)); + } catch (_) { + throw new Error( + "Workspace folder not found, are you in the right directory?" + ); + } + + try { + const workspaceSettings = yamlParse( + await Deno.readTextFile("settings.yaml") + ) as SimplifiedSettings; + await workspaceSetup( + { + token: instance.token, + workspace: undefined, + create: true, + createWorkspaceName: workspaceSettings.name, + createUsername: undefined, + }, + localWorkspace.name, + localWorkspace.workspaceId, + instance.remote + ); + } catch (_) { + log.error( + "Settings file not found in workspace local folder, skipping" + ); + continue; + } + await push({ + workspace: localWorkspace.name, + token: undefined, + includeGroups: true, + includeSchedules: true, + includeSettings: true, + includeUsers: true, + includeKey: true, + }); + } + + const workspacesToDelete = remoteWorkspaces.filter( + (w) => !localWorkspaces.find((l) => l.workspaceId === w.id) + ); + if (workspacesToDelete.length > 0) { + const confirmDelete = await Confirm.prompt({ + message: + "Do you want to delete the following remote workspaces that don't exist locally?\n" + + workspacesToDelete.map((w) => w.id).join(", "), + default: true, + }); + + if (confirmDelete) { + for (const workspace of workspacesToDelete) { + await WorkspaceService.deleteWorkspace({ workspace: workspace.id }); + log.info(colors.green.underline("Deleted workspace " + workspace.id)); + } + } + } + log.info(colors.green.underline.bold("All workspaces pushed")); + } +} + +const command = new Command() + .description( + "sync local with a remote instance or the opposite (push or pull)" + ) + .action(() => + log.info("2 actions available, pull and push. Use -h to display help.") + ) + .command("pull") + .description( + "Pull instance settings, users, configs, instance groups and overwrite local" + ) + .option("--skip-users", "Skip pulling users") + .option("--skip-settings", "Skip pulling settings") + .option("--skip-configs", "Skip pulling configs (worker groups and SMTP)") + .option("--skip-groups", "Skip pulling instance groups") + .option("--include-workspaces", "Also pull workspaces") + .action(instancePull as any) + .command("push") + .description( + "Push instance settings, users, configs, group and overwrite remote" + ) + .option("--skip-users", "Skip pushing users") + .option("--skip-settings", "Skip pushing settings") + .option("--skip-configs", "Skip pushing configs (worker groups and SMTP)") + .option("--skip-groups", "Skip pushing instance groups") + .option("--include-workspaces", "Also push workspaces") + .option( + "--base-url", + "Base url to be passed to the instance settings instead of the local one" + ) + .action(instancePush as any); + +export default command; diff --git a/cli/main.ts b/cli/main.ts index 0cc1ce9203..d1c24458bd 100644 --- a/cli/main.ts +++ b/cli/main.ts @@ -20,6 +20,7 @@ import hub from "./hub.ts"; import folder from "./folder.ts"; import schedule from "./schedule.ts"; import sync from "./sync.ts"; +import instance from "./instance.ts"; import dev from "./dev.ts"; import { fetchVersion, tryResolveVersion } from "./context.ts"; import { GlobalOptions } from "./types.ts"; @@ -89,6 +90,7 @@ let command: any = new Command() .command("schedule", schedule) .command("dev", dev) .command("sync", sync) + .command("instance", instance) .command("version", "Show version information") .action(async (opts) => { diff --git a/cli/pull.ts b/cli/pull.ts index 62d17401eb..c9b71aff3b 100644 --- a/cli/pull.ts +++ b/cli/pull.ts @@ -14,6 +14,7 @@ export async function downloadZip( includeUsers?: boolean, includeGroups?: boolean, includeSettings?: boolean, + includeKey?: boolean, defaultTs?: "bun" | "deno" ): Promise { const requestHeaders: HeadersInit & { set(x: string, y: string): void } = @@ -40,9 +41,9 @@ export async function downloadZip( includeSchedules ?? false }&include_users=${includeUsers ?? false}&include_groups=${ includeGroups ?? false - }&include_settings=${includeSettings ?? false}&default_ts=${ - defaultTs ?? "bun" - }`, + }&include_settings=${includeSettings ?? false}&include_key=${ + includeKey ?? false + }&default_ts=${defaultTs ?? "bun"}`, { headers: requestHeaders, method: "GET", diff --git a/cli/settings.ts b/cli/settings.ts index a9ff102eff..061f1f0cae 100644 --- a/cli/settings.ts +++ b/cli/settings.ts @@ -1,8 +1,17 @@ +import { SettingService } from "./deps.ts"; +import { yamlStringify } from "./deps.ts"; +import { GlobalSetting } from "./deps.ts"; +import { Config } from "./deps.ts"; +import { Confirm } from "./deps.ts"; +import { colors } from "./deps.ts"; +import { ConfigService } from "./deps.ts"; +import { yamlParse } from "./deps.ts"; import { WorkspaceService, log } from "./deps.ts"; +import { compareInstanceObjects } from "./instance.ts"; import { isSuperset } from "./types.ts"; import { deepEqual } from "./utils.ts"; -interface SimplifiedSettings { +export interface SimplifiedSettings { // slack_team_id?: string; // slack_name?: string; // slack_command_script?: string; @@ -21,6 +30,7 @@ interface SimplifiedSettings { git_sync?: any; default_app?: string; default_scripts?: any; + name: string; } export async function pushWorkspaceSettings( @@ -33,6 +43,11 @@ export async function pushWorkspaceSettings( const remoteSettings = await WorkspaceService.getSettings({ workspace, }); + + const workspaceName = await WorkspaceService.getWorkspaceName({ + workspace, + }); + settings = { // slack_team_id: remoteSettings.slack_team_id, // slack_name: remoteSettings.slack_name, @@ -55,6 +70,7 @@ export async function pushWorkspaceSettings( git_sync: remoteSettings.git_sync, default_app: remoteSettings.default_app, default_scripts: remoteSettings.default_scripts, + name: workspaceName, }; } catch (err) { throw new Error(`Failed to get workspace settings: ${err}`); @@ -198,4 +214,217 @@ export async function pushWorkspaceSettings( }, }); } + + if (localSettings.name !== settings.name) { + log.debug(`Updating workspace name...`); + await WorkspaceService.changeWorkspaceName({ + workspace, + requestBody: { + new_name: localSettings.name, + }, + }); + } +} + +export async function pushWorkspaceKey( + workspace: string, + _path: string, + key: string | undefined, + localKey: string +) { + try { + key = await WorkspaceService.getWorkspaceEncryptionKey({ + workspace, + }).then((r) => r.key); + } catch (err) { + throw new Error(`Failed to get workspace encryption key: ${err}`); + } + if (localKey && key !== localKey) { + const confirm = await Confirm.prompt({ + message: + "The local workspace encryption key does not match the remote. Do you want to reencrypt all your secrets on the remote with the new key?\nSay 'no' if your local secrets are already encrypted with the new key (e.g. workspace/instance migration)\nOtherwise, say 'yes' and pull the secrets after the reencryption.\n", + default: true, + }); + log.debug(`Updating workspace encryption key...`); + await WorkspaceService.setWorkspaceEncryptionKey({ + workspace, + requestBody: { + new_key: localKey, + skip_reencrypt: !confirm, + }, + }); + } else { + log.debug(`Workspace encryption key is up to date`); + } +} + +export async function pullInstanceSettings(preview = false) { + const remoteSettings = await SettingService.listGlobalSettings(); + + if (preview) { + let localSettings: GlobalSetting[] = []; + + try { + localSettings = yamlParse( + await Deno.readTextFile("instance_settings.yaml") + ) as GlobalSetting[]; + } catch {} + + return compareInstanceObjects( + remoteSettings, + localSettings, + "name", + "setting" + ); + } else { + log.info("Pulling settings from instance"); + + await Deno.writeTextFile( + "instance_settings.yaml", + yamlStringify(remoteSettings as any) + ); + + log.info(colors.green("Settings written to instance_settings.yaml")); + } +} + +export async function pushInstanceSettings( + preview: boolean = false, + baseUrl?: string +) { + const remoteSettings = await SettingService.listGlobalSettings(); + let localSettings = (await Deno.readTextFile("instance_settings.yaml") + .then((raw) => yamlParse(raw)) + .catch(() => [])) as GlobalSetting[]; + + if (baseUrl) { + localSettings = localSettings.filter((s) => s.name !== "base_url"); + localSettings.push({ + name: "base_url", + value: baseUrl, + }); + } + + if (preview) { + return compareInstanceObjects( + localSettings, + remoteSettings, + "name", + "setting" + ); + } else { + for (const setting of localSettings) { + const remoteMatch = remoteSettings.find((s) => s.name === setting.name); + if (remoteMatch && deepEqual(remoteMatch, setting)) { + continue; + } + try { + await SettingService.setGlobal({ + key: setting.name, + requestBody: { + value: setting.value, + }, + }); + } catch (err) { + log.error(`Failed to set setting ${setting.name}: ${err}`); + } + } + + for (const remoteSetting of remoteSettings) { + const localMatch = localSettings.find( + (s) => s.name === remoteSetting.name + ); + if (!localMatch) { + try { + await SettingService.setGlobal({ + key: remoteSetting.name, + requestBody: { + value: null, + }, + }); + } catch (err) { + log.error(`Failed to delete setting ${remoteSetting.name}: ${err}`); + } + } + } + + log.info(colors.green("Settings pushed to instance")); + } +} + +export async function pullInstanceConfigs(preview = false) { + const remoteConfigs = await ConfigService.listConfigs(); + + if (preview) { + let localConfigs: Config[] = []; + try { + localConfigs = yamlParse( + await Deno.readTextFile("instance_configs.yaml") + ) as Config[]; + } catch {} + + return compareInstanceObjects( + remoteConfigs, + localConfigs, + "name", + "config" + ); + } else { + log.info("Pulling configs from instance"); + + await Deno.writeTextFile( + "instance_configs.yaml", + yamlStringify(remoteConfigs as any) + ); + + log.info(colors.green("Configs written to instance_configs.yaml")); + } +} + +export async function pushInstanceConfigs(preview: boolean = false) { + const remoteConfigs = await ConfigService.listConfigs(); + const localConfigs = (await Deno.readTextFile("instance_configs.yaml") + .then((raw) => yamlParse(raw)) + .catch(() => [])) as Config[]; + + if (preview) { + return compareInstanceObjects( + localConfigs, + remoteConfigs, + "name", + "config" + ); + } else { + log.info("Pushing configs to instance"); + for (const config of localConfigs) { + const remoteMatch = remoteConfigs.find((c) => c.name === config.name); + if (remoteMatch && deepEqual(remoteMatch, config)) { + continue; + } + try { + await ConfigService.updateConfig({ + name: config.name, + requestBody: config.config, + }); + } catch (err) { + log.error(`Failed to set config ${config.name}: ${err}`); + } + } + + for (const removeConfig of remoteConfigs) { + const localMatch = localConfigs.find((c) => c.name === removeConfig.name); + + if (!localMatch) { + try { + await ConfigService.deleteConfig({ + name: removeConfig.name, + }); + } catch (err) { + log.error(`Failed to delete config ${removeConfig.name}: ${err}`); + } + } + } + + log.info(colors.green("Configs pushed to instance")); + } } diff --git a/cli/sync.ts b/cli/sync.ts index aec38102c9..3c8ab71365 100644 --- a/cli/sync.ts +++ b/cli/sync.ts @@ -576,6 +576,7 @@ export async function elementsToMap( if (!skips.includeUsers && path.endsWith(".user" + ext)) continue; if (!skips.includeGroups && path.endsWith(".group" + ext)) continue; if (!skips.includeSettings && path === "settings" + ext) continue; + if (!skips.includeKey && path === "encryption_key") continue; if (skips.skipResources && path.endsWith(".resource" + ext)) continue; if (skips.skipVariables && path.endsWith(".variable" + ext)) continue; @@ -628,6 +629,7 @@ interface Skips { includeUsers?: boolean | undefined; includeGroups?: boolean | undefined; includeSettings?: boolean | undefined; + includeKey?: boolean | undefined; } async function compareDynFSElement( @@ -709,30 +711,32 @@ async function compareDynFSElement( function getOrderFromPath(p: string) { const typ = getTypeStrFromPath(p); - if (typ == "folder") { + if (typ == "settings") { return 0; - } else if (typ == "resource-type") { + } else if (typ == "folder") { return 1; - } else if (typ == "resource") { + } else if (typ == "resource-type") { return 2; - } else if (typ == "script") { + } else if (typ == "resource") { return 3; - } else if (typ == "flow") { + } else if (typ == "script") { return 4; - } else if (typ == "app") { + } else if (typ == "flow") { return 5; - } else if (typ == "schedule") { + } else if (typ == "app") { return 6; - } else if (typ == "variable") { + } else if (typ == "schedule") { return 7; - } else if (typ == "user") { + } else if (typ == "variable") { return 8; - } else if (typ == "group") { + } else if (typ == "user") { return 9; - } else if (typ == "settings") { + } else if (typ == "group") { return 10; - } else { + } else if (typ == "encryption_key") { return 11; + } else { + return 12; } } @@ -752,7 +756,11 @@ const isNotWmillFile = (p: string, isDirectory: boolean) => { try { const typ = getTypeStrFromPath(p); - if (typ == "resource-type" || typ == "settings") { + if ( + typ == "resource-type" || + typ == "settings" || + typ == "encryption_key" + ) { return p.includes(SEP); } else { return ( @@ -840,7 +848,7 @@ export async function ignoreF(wmillconf: { }; } -async function pull(opts: GlobalOptions & SyncOptions) { +export async function pull(opts: GlobalOptions & SyncOptions) { opts = await mergeConfigWithConfigFile(opts); if (opts.stateful) { @@ -868,6 +876,7 @@ async function pull(opts: GlobalOptions & SyncOptions) { opts.includeUsers, opts.includeGroups, opts.includeSettings, + opts.includeKey, opts.defaultTs ))!, !opts.json, @@ -1108,7 +1117,7 @@ function removeSuffix(str: string, suffix: string) { return str.slice(0, str.length - suffix.length); } -async function push(opts: GlobalOptions & SyncOptions) { +export async function push(opts: GlobalOptions & SyncOptions) { opts = await mergeConfigWithConfigFile(opts); const codebases = await listSyncCodebases(opts); if (opts.raw) { @@ -1144,6 +1153,7 @@ async function push(opts: GlobalOptions & SyncOptions) { opts.includeUsers, opts.includeGroups, opts.includeSettings, + opts.includeKey, opts.defaultTs ))!, !opts.json, @@ -1428,6 +1438,7 @@ const command = new Command() .option("--include-users", "Include syncing users") .option("--include-groups", "Include syncing groups") .option("--include-settings", "Include syncing workspace settings") + .option("--include-key", "Include workspace encryption key") .option( "-i --includes ", "Comma separated patterns to specify which file to take into account (among files that are compatible with windmill). Patterns can include * (any string until '/') and ** (any string)" @@ -1465,6 +1476,7 @@ const command = new Command() .option("--include-users", "Include syncing users") .option("--include-groups", "Include syncing groups") .option("--include-settings", "Include syncing workspace settings") + .option("--include-key", "Include workspace encryption key") .option( "-i --includes ", "Comma separated patterns to specify which file to take into account (among files that are compatible with windmill). Patterns can include * (any string until '/') and ** (any string)" diff --git a/cli/types.ts b/cli/types.ts index d94bba6cac..ecee767fa3 100644 --- a/cli/types.ts +++ b/cli/types.ts @@ -14,7 +14,7 @@ import { deepEqual } from "./utils.ts"; import { pushSchedule } from "./schedule.ts"; import { pushWorkspaceUser } from "./user.ts"; import { pushGroup } from "./user.ts"; -import { pushWorkspaceSettings } from "./settings.ts"; +import { pushWorkspaceSettings, pushWorkspaceKey } from "./settings.ts"; export interface DifferenceCreate { type: "CREATE"; @@ -132,6 +132,8 @@ export async function pushObj( await pushGroup(workspace, p, befObj, newObj); } else if (typeEnding === "settings") { await pushWorkspaceSettings(workspace, p, befObj, newObj); + } else if (typeEnding === "encryption_key") { + await pushWorkspaceKey(workspace, p, befObj, newObj); } else { throw new Error( `The item ${p} has an unrecognized type ending ${typeEnding}` @@ -168,7 +170,8 @@ export function getTypeStrFromPath( | "schedule" | "user" | "group" - | "settings" { + | "settings" + | "encryption_key" { if (p.includes(".flow" + SEP)) { return "flow"; } @@ -204,7 +207,8 @@ export function getTypeStrFromPath( typeEnding === "schedule" || typeEnding === "user" || typeEnding === "group" || - typeEnding === "settings" + typeEnding === "settings" || + typeEnding === "encryption_key" ) { return typeEnding; } else { diff --git a/cli/user.ts b/cli/user.ts index f1cd6c75c6..8c5fde04a9 100644 --- a/cli/user.ts +++ b/cli/user.ts @@ -6,6 +6,7 @@ import { removeType, removePathPrefix, } from "./types.ts"; +import { compareInstanceObjects } from "./instance.ts"; import { colors, Command, @@ -17,7 +18,13 @@ import { GroupService, WorkspaceService, GranularAclService, + yamlStringify, + yamlParse, } from "./deps.ts"; +import { ExportedUser } from "../../windmill-deno-client/windmill-api/models/ExportedUser.ts"; +import { InstanceGroup } from "./deps.ts"; +import { SettingService } from "./deps.ts"; +import { ExportedInstanceGroup } from "./deps.ts"; async function list(opts: GlobalOptions) { await requireLogin(opts); @@ -168,13 +175,18 @@ export async function pushWorkspaceUser( } else { console.log(colors.bold.yellow("Creating new user: " + email)); try { + const automatedUsernameCreation: boolean = await SettingService.getGlobal( + { + key: "automate_username_creation", + } + ); await WorkspaceService.addUser({ workspace: workspace, requestBody: { email: email, is_admin: localUser.role === "admin", operator: localUser.role === "operator", - username: localUser.username, + username: !automatedUsernameCreation ? localUser.username : undefined, }, }); } catch (e) { @@ -368,6 +380,84 @@ export async function pushGroup( } } +export async function pullInstanceUsers(preview: boolean = false) { + const remoteUsers = await UserService.globalUsersExport(); + + if (preview) { + let localUsers: ExportedUser[] = []; + try { + const raw = await Deno.readTextFile("instance_users.yaml"); + localUsers = yamlParse(raw) as ExportedUser[]; + } catch {} + return compareInstanceObjects(remoteUsers, localUsers, "email", "user"); + } else { + log.info("Pulling users from instance..."); + await Deno.writeTextFile( + "instance_users.yaml", + yamlStringify(remoteUsers as any) + ); + log.info(colors.green("Users written to instance_users.yaml")); + } +} + +export async function pushInstanceUsers(preview: boolean = false) { + const remoteUsers = await UserService.globalUsersExport(); + const localUsers = (await Deno.readTextFile("instance_users.yaml") + .then((raw) => yamlParse(raw)) + .catch(() => [])) as ExportedUser[]; + + if (preview) { + return compareInstanceObjects(localUsers, remoteUsers, "email", "user"); + } else { + log.info("Pushing users to instance..."); + await UserService.globalUsersOverwrite({ + requestBody: localUsers, + }); + + log.info(colors.green("Users pushed to the instance")); + } +} + +export async function pullInstanceGroups(preview = false) { + const remoteGroups = await GroupService.exportInstanceGroups(); + + if (preview) { + let localGroups: InstanceGroup[] = []; + try { + const raw = await Deno.readTextFile("instance_groups.yaml"); + localGroups = yamlParse(raw) as InstanceGroup[]; + } catch {} + return compareInstanceObjects(remoteGroups, localGroups, "name", "group"); + } else { + log.info("Pulling groups from instance..."); + + await Deno.writeTextFile( + "instance_groups.yaml", + yamlStringify(remoteGroups as any) + ); + + log.info(colors.green("Groups written to instance_groups.yaml")); + } +} + +export async function pushInstanceGroups(preview: boolean = false) { + const remoteGroups = await GroupService.exportInstanceGroups(); + const localGroups = (await Deno.readTextFile("instance_groups.yaml") + .then((raw) => yamlParse(raw)) + .catch(() => [])) as ExportedInstanceGroup[]; + + if (preview) { + return compareInstanceObjects(localGroups, remoteGroups, "name", "group"); + } else { + log.info("Pushing groups to instance..."); + await GroupService.overwriteInstanceGroups({ + requestBody: localGroups, + }); + + log.info(colors.green("Groups pushed to the instance")); + } +} + const command = new Command() .description("user related commands") .action(list as any) diff --git a/cli/utils.ts b/cli/utils.ts index 209ec7b8b8..8732560f72 100644 --- a/cli/utils.ts +++ b/cli/utils.ts @@ -125,3 +125,7 @@ export async function generateHashFromBuffer( export function readInlinePathSync(path: string): string { return Deno.readTextFileSync(path.replaceAll("/", SEP)); } + +export function sleep(ms: number) { + return new Promise((resolve) => setTimeout(resolve, ms)); +} diff --git a/cli/workspace.ts b/cli/workspace.ts index 1cf49efc50..956df39793 100644 --- a/cli/workspace.ts +++ b/cli/workspace.ts @@ -161,7 +161,7 @@ export async function add( opts: GlobalOptions & { create: boolean; createWorkspaceName: string | undefined; - createUsername: string; + createUsername: string | undefined; }, workspaceName: string | undefined, workspaceId: string | undefined,