From fda0e28db31fba9f71be6db7280696c3949fd7be Mon Sep 17 00:00:00 2001 From: Guillaume Bouvignies Date: Fri, 12 Jan 2024 16:19:28 +0100 Subject: [PATCH] feat: GIt sync to multiple repo (#2996) --- backend/Cargo.lock | 1 + .../20240112131322_git_sync_multi.down.sql | 2 + .../20240112131322_git_sync_multi.up.sql | 2 + backend/windmill-api/openapi-deref.yaml | 36 +-- backend/windmill-api/openapi.yaml | 10 +- backend/windmill-api/src/workspaces.rs | 4 +- backend/windmill-git-sync/Cargo.toml | 1 + backend/windmill-git-sync/src/lib.rs | 188 ++++++++-------- .../(logged)/workspace_settings/+page.svelte | 211 +++++++++++------- 9 files changed, 263 insertions(+), 192 deletions(-) create mode 100644 backend/migrations/20240112131322_git_sync_multi.down.sql create mode 100644 backend/migrations/20240112131322_git_sync_multi.up.sql diff --git a/backend/Cargo.lock b/backend/Cargo.lock index 2b6076961b..0a8f859ace 100644 --- a/backend/Cargo.lock +++ b/backend/Cargo.lock @@ -9432,6 +9432,7 @@ dependencies = [ "serde_json", "sqlx", "tracing", + "uuid 1.6.1", "windmill-common", "windmill-queue", ] diff --git a/backend/migrations/20240112131322_git_sync_multi.down.sql b/backend/migrations/20240112131322_git_sync_multi.down.sql new file mode 100644 index 0000000000..0beef40678 --- /dev/null +++ b/backend/migrations/20240112131322_git_sync_multi.down.sql @@ -0,0 +1,2 @@ +-- Add down migration script here +UPDATE workspace_settings SET git_sync = git_sync->0; \ No newline at end of file diff --git a/backend/migrations/20240112131322_git_sync_multi.up.sql b/backend/migrations/20240112131322_git_sync_multi.up.sql new file mode 100644 index 0000000000..9ff8eaadc9 --- /dev/null +++ b/backend/migrations/20240112131322_git_sync_multi.up.sql @@ -0,0 +1,2 @@ +-- Add up migration script here +UPDATE workspace_settings SET git_sync = '[]'::jsonb || git_sync diff --git a/backend/windmill-api/openapi-deref.yaml b/backend/windmill-api/openapi-deref.yaml index e3ee43430a..cf4f74ce8e 100644 --- a/backend/windmill-api/openapi-deref.yaml +++ b/backend/windmill-api/openapi-deref.yaml @@ -1,6 +1,6 @@ openapi: 3.0.3 info: - version: 1.241.0 + version: 1.242.0 title: Windmill API contact: name: Windmill Team @@ -1525,17 +1525,19 @@ paths: s3_resource_path: type: string git_sync: - type: object - properties: &ref_16 - script_path: - type: string - git_repo_resource_path: - type: string - use_individual_branch: - type: boolean - required: &ref_17 - - script_path - - git_repo_resource_path + type: array + items: + type: object + properties: &ref_16 + script_path: + type: string + git_repo_resource_path: + type: string + use_individual_branch: + type: boolean + required: &ref_17 + - script_path + - git_repo_resource_path required: - code_completion_enabled /w/{workspace}/workspaces/get_deploy_to: @@ -1891,9 +1893,11 @@ paths: type: object properties: git_sync_settings: - type: object - properties: *ref_16 - required: *ref_17 + type: array + items: + type: object + properties: *ref_16 + required: *ref_17 responses: '200': description: status @@ -10202,7 +10206,7 @@ paths: schema: {} /w/{workspace}/job_helpers/list_stored_files: get: - summary: List the file keys available in the worspace files storage (S3) + summary: List the file keys available in the workspace files storage (S3) operationId: listStoredFiles tags: - helpers diff --git a/backend/windmill-api/openapi.yaml b/backend/windmill-api/openapi.yaml index 2bd4bce8c5..53fef4922e 100644 --- a/backend/windmill-api/openapi.yaml +++ b/backend/windmill-api/openapi.yaml @@ -1193,7 +1193,9 @@ paths: large_file_storage: $ref: "#/components/schemas/LargeFileStorage" git_sync: - $ref: "#/components/schemas/WorkspaceGitSync" + type: array + items: + $ref: "#/components/schemas/WorkspaceGitSync" required: - code_completion_enabled @@ -1532,7 +1534,9 @@ paths: type: object properties: git_sync_settings: - $ref: "#/components/schemas/WorkspaceGitSync" + type: array + items: + $ref: "#/components/schemas/WorkspaceGitSync" responses: "200": @@ -6846,7 +6850,7 @@ paths: /w/{workspace}/job_helpers/list_stored_files: get: - summary: List the file keys available in the worspace files storage (S3) + summary: List the file keys available in the workspace files storage (S3) operationId: listStoredFiles tags: - helpers diff --git a/backend/windmill-api/src/workspaces.rs b/backend/windmill-api/src/workspaces.rs index 3831b911cf..55a2c99964 100644 --- a/backend/windmill-api/src/workspaces.rs +++ b/backend/windmill-api/src/workspaces.rs @@ -1044,7 +1044,7 @@ async fn edit_large_file_storage_config( #[derive(Deserialize)] struct EditGitSyncConfig { - git_sync_settings: Option, + git_sync_settings: Option>, } async fn edit_git_sync_config( @@ -1078,7 +1078,7 @@ async fn edit_git_sync_config( .await?; if let Some(git_sync_settings) = new_config.git_sync_settings { - let serialized_config = serde_json::to_value::(git_sync_settings) + let serialized_config = serde_json::to_value::>(git_sync_settings) .map_err(|err| Error::InternalErr(err.to_string()))?; sqlx::query!( diff --git a/backend/windmill-git-sync/Cargo.toml b/backend/windmill-git-sync/Cargo.toml index 074d3fec56..e3de11bcc6 100644 --- a/backend/windmill-git-sync/Cargo.toml +++ b/backend/windmill-git-sync/Cargo.toml @@ -13,6 +13,7 @@ enterprise = ["windmill-queue/enterprise"] default = [] [dependencies] +uuid.workspace = true serde.workspace = true sqlx.workspace = true serde_json.workspace = true diff --git a/backend/windmill-git-sync/src/lib.rs b/backend/windmill-git-sync/src/lib.rs index 393b07c5eb..34a848f31f 100644 --- a/backend/windmill-git-sync/src/lib.rs +++ b/backend/windmill-git-sync/src/lib.rs @@ -9,6 +9,7 @@ use std::collections::HashMap; use sqlx::{Pool, Postgres}; +use uuid::Uuid; use windmill_common::users::SUPERADMIN_SYNC_EMAIL; use windmill_common::workspaces::WorkspaceGitRepo; @@ -45,7 +46,7 @@ impl DeployedObject { } } -pub async fn handle_deployment_metadata<'c, R: rsmq_async::RsmqConnection + Send + 'c>( +pub async fn handle_deployment_metadata<'c, R: rsmq_async::RsmqConnection + Send + Clone + 'c>( email: &str, created_by: &str, db: &DB, @@ -94,122 +95,119 @@ pub async fn handle_deployment_metadata<'c, R: rsmq_async::RsmqConnection + Send )); } - let workspace_git_repo = workspace_git_repo_setting + let workspace_git_repos = workspace_git_repo_setting .unwrap() - .map(|conf| serde_json::from_value::(conf).ok()) - .flatten(); + .map(|conf| serde_json::from_value::>(conf).ok()) + .flatten() + .unwrap_or_default(); - let git_sync_job_uuid = if !skip_git_sync && workspace_git_repo.is_some() { - let workspace_git_repo = workspace_git_repo.unwrap(); + let mut git_sync_job_uuids: Vec = vec![]; + if !skip_git_sync { + for workspace_git_repo in workspace_git_repos { + let mut args: HashMap = HashMap::new(); + args.insert( + "repo_url_resource_path".to_string(), + json!(workspace_git_repo + .git_repo_resource_path + .strip_prefix("$res:")), + ); - let mut args: HashMap = HashMap::new(); - args.insert( - "repo_url_resource_path".to_string(), - json!(workspace_git_repo - .git_repo_resource_path - .strip_prefix("$res:")), - ); + if let Some(path) = obj_path { + args.insert("path".to_string(), json!(path)); + } + if let Some(parent_path) = obj_parent_path.clone() { + args.insert("parent_path".to_string(), json!(parent_path)); + } - if let Some(path) = obj_path { - args.insert("path".to_string(), json!(path)); + let message = match obj.clone() { + DeployedObject::Script { path, .. } => { + if deployment_message.as_ref().is_none() + || deployment_message.as_ref().is_some_and(|x| x.is_empty()) + { + format!("Script '{}' deployed", path) + } else { + deployment_message.clone().unwrap() + } + } + DeployedObject::Flow { path, .. } => { + if deployment_message.as_ref().is_none() + || deployment_message.as_ref().is_some_and(|x| x.is_empty()) + { + format!("Flow '{}' deployed", path) + } else { + deployment_message.clone().unwrap() + } + } + DeployedObject::App { path, .. } => { + if deployment_message.as_ref().is_none() + || deployment_message.as_ref().is_some_and(|x| x.is_empty()) + { + format!("App '{}' deployed", path) + } else { + deployment_message.clone().unwrap() + } + } + }; + + args.insert("commit_msg".to_string(), json!(message)); + args.insert( + "use_individual_branch".to_string(), + json!(workspace_git_repo.use_individual_branch.unwrap_or(false)), + ); + + let tx: PushIsolationLevel<'_, R> = + PushIsolationLevel::IsolatedRoot(db.clone(), rsmq.clone()); + + let (job_uuid, new_tx) = windmill_queue::push( + db, + tx, + &w_id, + JobPayload::DeploymentCallback { path: workspace_git_repo.script_path.clone() }, + args, + created_by, + &email, + SUPERADMIN_SYNC_EMAIL.to_string(), + None, + None, + None, + None, + None, + false, + false, + None, + true, + None, + None, + None, + None, + ) + .await?; + new_tx.commit().await?; + + git_sync_job_uuids.push(job_uuid); } - if let Some(parent_path) = obj_parent_path { - args.insert("parent_path".to_string(), json!(parent_path)); - } - - let message = match obj.clone() { - DeployedObject::Script { path, .. } => { - if deployment_message.as_ref().is_none() - || deployment_message.as_ref().is_some_and(|x| x.is_empty()) - { - format!("Script '{}' deployed", path) - } else { - deployment_message.clone().unwrap() - } - } - DeployedObject::Flow { path, .. } => { - if deployment_message.as_ref().is_none() - || deployment_message.as_ref().is_some_and(|x| x.is_empty()) - { - format!("Flow '{}' deployed", path) - } else { - deployment_message.clone().unwrap() - } - } - DeployedObject::App { path, .. } => { - if deployment_message.as_ref().is_none() - || deployment_message.as_ref().is_some_and(|x| x.is_empty()) - { - format!("App '{}' deployed", path) - } else { - deployment_message.clone().unwrap() - } - } - }; - - args.insert("commit_msg".to_string(), json!(message)); - args.insert( - "use_individual_branch".to_string(), - json!(workspace_git_repo.use_individual_branch.unwrap_or(false)), - ); - - let tx: PushIsolationLevel<'_, R> = PushIsolationLevel::IsolatedRoot(db.clone(), rsmq); - - let (job_uuid, new_tx) = windmill_queue::push( - db, - tx, - &w_id, - JobPayload::DeploymentCallback { path: workspace_git_repo.script_path.clone() }, - args, - created_by, - &email, - SUPERADMIN_SYNC_EMAIL.to_string(), - None, - None, - None, - None, - None, - false, - false, - None, - true, - None, - None, - None, - None, - ) - .await?; - new_tx.commit().await?; - Some(job_uuid) - } else { - None - }; + } // We're not persisting the default commit msg as it's pretty useless. We will persist the ones manually set by users - let job_uuids = if git_sync_job_uuid.is_some() { - vec![git_sync_job_uuid.unwrap()] - } else { - vec![] - }; - if !skip_db_insert && (deployment_message.is_some() || job_uuids.len() > 0) { + if !skip_db_insert && (deployment_message.is_some() || git_sync_job_uuids.len() > 0) { // if the git sync job hasn't been triggered, and there is not custom deployment message, there's not point adding an entry to the table match obj.clone() { DeployedObject::Script { path, hash, .. } => { sqlx::query!( "INSERT INTO deployment_metadata (workspace_id, path, script_hash, callback_job_ids, deployment_msg) VALUES ($1, $2, $3, $4, $5)", - w_id, path, hash.0, &job_uuids, deployment_message, + w_id, path, hash.0, &git_sync_job_uuids, deployment_message, ) }, DeployedObject::Flow { path, .. } => { sqlx::query!( "INSERT INTO deployment_metadata (workspace_id, path, callback_job_ids, deployment_msg) VALUES ($1, $2, $3, $4) ON CONFLICT (workspace_id, path) WHERE script_hash IS NULL AND app_version IS NULL DO UPDATE SET callback_job_ids = $3, deployment_msg = $4", - w_id, path, &job_uuids, deployment_message, + w_id, path, &git_sync_job_uuids, deployment_message, ) } DeployedObject::App { path, version, .. } => { sqlx::query!( "INSERT INTO deployment_metadata (workspace_id, path, app_version, callback_job_ids, deployment_msg) VALUES ($1, $2, $3, $4, $5)", - w_id, path, version, &job_uuids, deployment_message, + w_id, path, version, &git_sync_job_uuids, deployment_message, ) } }.execute(db) diff --git a/frontend/src/routes/(root)/(logged)/workspace_settings/+page.svelte b/frontend/src/routes/(root)/(logged)/workspace_settings/+page.svelte index add8149ecb..b2ffe78f95 100644 --- a/frontend/src/routes/(root)/(logged)/workspace_settings/+page.svelte +++ b/frontend/src/routes/(root)/(logged)/workspace_settings/+page.svelte @@ -60,13 +60,11 @@ script_path: string git_repo_resource_path: string use_individual_branch: boolean - } - let gitSyncTestJob: - | { - jobId: string - status: 'running' | 'success' | 'failure' - } - | undefined = undefined + }[] + let gitSyncTestJobs: { + jobId: string | undefined + status: 'running' | 'success' | 'failure' | undefined + }[] let codeCompletionEnabled: boolean = false let tab = ($page.url.searchParams.get('tab') as @@ -194,29 +192,29 @@ } } - async function editWindmillGitSyncSettings( - gitRepoResourcePath: string, - useIndividualBranch: boolean - ): Promise { - if (!emptyString(gitRepoResourcePath)) { - gitSyncSettings = { - script_path: 'hub/7926/sync-script-to-git-repo-windmill', - git_repo_resource_path: `$res:${gitRepoResourcePath.replace('$res:', '')}`, - use_individual_branch: useIndividualBranch + async function editWindmillGitSyncSettings(): Promise { + let alreadySeenResource: string[] = [] + let finalSettings = gitSyncSettings.map((elmt) => { + alreadySeenResource.push(elmt.git_repo_resource_path) + return { + script_path: elmt.script_path, + git_repo_resource_path: `$res:${elmt.git_repo_resource_path.replace('$res:', '')}`, + use_individual_branch: elmt.use_individual_branch } + }) + if (alreadySeenResource.some((res, index) => alreadySeenResource.indexOf(res) !== index)) { + sendUserToast('Same Git resource used more than once', true) + return + } + if (finalSettings.length > 0) { await WorkspaceService.editWorkspaceGitSyncConfig({ workspace: $workspaceStore!, requestBody: { - git_sync_settings: gitSyncSettings + git_sync_settings: finalSettings } }) sendUserToast('Workspace Git sync settings updated') } else { - gitSyncSettings = { - script_path: '', - git_repo_resource_path: '', - use_individual_branch: false - } await WorkspaceService.editWorkspaceGitSyncConfig({ workspace: $workspaceStore!, requestBody: { @@ -261,18 +259,27 @@ settings.large_file_storage?.type === LargeFileStorage.type.S3STORAGE ? settings.large_file_storage?.s3_resource_path?.replace('$res:', '') : undefined - if (settings.git_sync !== undefined && settings.git_sync !== null) { - gitSyncSettings = { - git_repo_resource_path: settings.git_sync.git_repo_resource_path.replace('$res:', ''), - script_path: settings.git_sync.script_path, - use_individual_branch: settings.git_sync.use_individual_branch ?? false - } + if ( + settings.git_sync !== undefined && + settings.git_sync !== null && + settings.git_sync.length > 0 + ) { + gitSyncSettings = settings.git_sync.map((settings) => { + return { + git_repo_resource_path: settings.git_repo_resource_path.replace('$res:', ''), + script_path: settings.script_path, + use_individual_branch: settings.use_individual_branch ?? false + } + }) + gitSyncTestJobs = settings.git_sync.map((settings) => { + return { + jobId: undefined, + status: undefined + } + }) } else { - gitSyncSettings = { - git_repo_resource_path: '', - script_path: '', - use_individual_branch: false - } + gitSyncSettings = [] + gitSyncTestJobs = [] } // check openai_client_credentials_oauth @@ -322,18 +329,19 @@ ) } - async function runGitSyncTestJob() { - if (emptyString(gitSyncSettings.script_path)) { + async function runGitSyncTestJob(settingsIdx: number) { + let gitSyncSettingsElmt = gitSyncSettings[settingsIdx] + if (emptyString(gitSyncSettingsElmt.script_path)) { return } let jobId = await JobService.runScriptByPath({ workspace: $workspaceStore!, path: 'hub/7925/git-repo-test-read-write-windmill', requestBody: { - repo_url_resource_path: gitSyncSettings.git_repo_resource_path.replace('$res:', '') + repo_url_resource_path: gitSyncSettingsElmt.git_repo_resource_path.replace('$res:', '') } }) - gitSyncTestJob = { + gitSyncTestJobs[settingsIdx] = { jobId: jobId, status: 'running' } @@ -343,7 +351,7 @@ workspace: $workspaceStore!, id: jobId }) - gitSyncTestJob!.status = testResult.success ? 'success' : 'failure' + gitSyncTestJobs[settingsIdx].status = testResult.success ? 'success' : 'failure' }, timeoutCode: async () => { try { @@ -776,57 +784,98 @@
Filtering out certain sensitive folders from the sync will be available soon. -
- {#if gitSyncSettings} - {#key gitSyncSettings} + + {#each gitSyncSettings as gitSyncSettingsElmt, idx} +
+
Repository #{idx + 1}
+
+
+ {#key gitSyncSettingsElmt} { - editWindmillGitSyncSettings(ev.detail, gitSyncSettings?.use_individual_branch) + gitSyncSettingsElmt.git_repo_resource_path = ev.detail }} /> {/key} - {/if} -
-
- {#if gitSyncTestJob !== undefined} - {#if gitSyncTestJob?.status === 'running'} - - {:else if gitSyncTestJob?.status === 'success'} - - {:else} - +
+
+ {#if gitSyncSettings.filter((settings) => settings.git_repo_resource_path === gitSyncSettingsElmt.git_repo_resource_path).length > 1} + Using the same resource twice is not allowed. {/if} - Git sync resource checked via Windmill job - - {gitSyncTestJob?.jobId} - WARNING: Only read permissions are verified. - {/if} -
+ {#if gitSyncTestJobs[idx].status !== undefined} + {#if gitSyncTestJobs[idx].status === 'running'} + + {:else if gitSyncTestJobs[idx].status === 'success'} + + {:else} + + {/if} + Git sync resource checked via Windmill job + + {gitSyncTestJobs[idx].jobId} + WARNING: Only read permissions are verified. + {/if} +
-
- {#if gitSyncSettings} - { - editWindmillGitSyncSettings(gitSyncSettings.git_repo_resource_path, ev.detail) - }} - options={{ - right: 'Create one branch per deployed script/flow/app', - rightTooltip: - "If set, Windmill will create a unique branch per script/flow/app being pushed, prefixed with 'wm_deploy/'." - }} - /> - {/if} +
+ {#if gitSyncSettings} + + {/if} +
+ {/each} + +
+
@@ -865,6 +914,16 @@ git push
+
+ +
{/if} {:else}