From 08453d48e3a527b4365257343e4e4e2bd550ff61 Mon Sep 17 00:00:00 2001 From: Guillaume Bouvignies Date: Wed, 14 Feb 2024 14:28:04 +0100 Subject: [PATCH] feat: Git sync exclude certain type per repository (#3210) * feat: Git sync exclude certain type per repository * sync secret and variable * remove bind:disabled --- backend/windmill-api/openapi.yaml | 14 ++ backend/windmill-common/src/workspaces.rs | 1 + backend/windmill-git-sync/src/lib.rs | 137 +++++++--- .../(logged)/workspace_settings/+page.svelte | 237 ++++++++++++++---- 4 files changed, 299 insertions(+), 90 deletions(-) diff --git a/backend/windmill-api/openapi.yaml b/backend/windmill-api/openapi.yaml index 080aba6847..7f3e9778be 100644 --- a/backend/windmill-api/openapi.yaml +++ b/backend/windmill-api/openapi.yaml @@ -9584,6 +9584,20 @@ components: type: string use_individual_branch: type: boolean + exclude_types_override: + type: array + items: + type: string + enum: + - script + - flow + - app + - folder + - resource + - variable + - secret + - resourcetype + - schedule required: - script_path - git_repo_resource_path diff --git a/backend/windmill-common/src/workspaces.rs b/backend/windmill-common/src/workspaces.rs index 668b93469c..7f9f74ecd8 100644 --- a/backend/windmill-common/src/workspaces.rs +++ b/backend/windmill-common/src/workspaces.rs @@ -26,4 +26,5 @@ pub struct GitRepositorySettings { pub script_path: String, pub git_repo_resource_path: String, pub use_individual_branch: Option, + pub exclude_types_override: Option>, } diff --git a/backend/windmill-git-sync/src/lib.rs b/backend/windmill-git-sync/src/lib.rs index b4ed33bf57..9778017a20 100644 --- a/backend/windmill-git-sync/src/lib.rs +++ b/backend/windmill-git-sync/src/lib.rs @@ -109,7 +109,7 @@ pub async fn handle_deployment_metadata<'c, R: rsmq_async::RsmqConnection + Send None }; - let mut skip_git_sync = if obj_path.is_none() && obj_parent_path.is_none() { + let skip_git_sync = if obj_path.is_none() && obj_parent_path.is_none() { tracing::debug!( "Ignoring {} from git sync as it's in a private user folder", obj.get_path() @@ -118,48 +118,109 @@ pub async fn handle_deployment_metadata<'c, R: rsmq_async::RsmqConnection + Send } else { false }; - - skip_git_sync = skip_git_sync - || match obj { - DeployedObject::Script { .. } => !workspace_git_sync_settings - .include_type - .iter() - .any(|element| *element == ObjectType::Script), - DeployedObject::Flow { .. } => !workspace_git_sync_settings - .include_type - .iter() - .any(|element| *element == ObjectType::Flow), - DeployedObject::App { .. } => !workspace_git_sync_settings - .include_type - .iter() - .any(|element| *element == ObjectType::App), - DeployedObject::Folder { .. } => !workspace_git_sync_settings - .include_type - .iter() - .any(|element| *element == ObjectType::Folder), - DeployedObject::Resource { .. } => !workspace_git_sync_settings - .include_type - .iter() - .any(|element| *element == ObjectType::Resource), - DeployedObject::Variable { .. } => !workspace_git_sync_settings - .include_type - .iter() - .any(|element| *element == ObjectType::Variable), - DeployedObject::ResourceType { .. } => !workspace_git_sync_settings - .include_type - .iter() - .any(|element| *element == ObjectType::ResourceType), - DeployedObject::Schedule { .. } => !workspace_git_sync_settings - .include_type - .iter() - .any(|element| *element == ObjectType::Schedule), - }; - tracing::debug!("Skipping git sync for {:?} -> {}", obj_path, skip_git_sync); let mut git_sync_job_uuids: Vec = vec![]; if !skip_git_sync { for workspace_git_repo in workspace_git_sync_settings.repositories { + // check that this type of object should indeed be synced + let skip_object_type = match obj { + DeployedObject::Script { .. } => { + !workspace_git_sync_settings + .include_type + .iter() + .any(|element| *element == ObjectType::Script) + || workspace_git_repo + .exclude_types_override + .unwrap_or_default() + .iter() + .any(|element| *element == ObjectType::Script) + } + DeployedObject::Flow { .. } => { + !workspace_git_sync_settings + .include_type + .iter() + .any(|element| *element == ObjectType::Flow) + || workspace_git_repo + .exclude_types_override + .unwrap_or_default() + .iter() + .any(|element| *element == ObjectType::Flow) + } + DeployedObject::App { .. } => { + !workspace_git_sync_settings + .include_type + .iter() + .any(|element| *element == ObjectType::App) + || workspace_git_repo + .exclude_types_override + .unwrap_or_default() + .iter() + .any(|element| *element == ObjectType::App) + } + DeployedObject::Folder { .. } => { + !workspace_git_sync_settings + .include_type + .iter() + .any(|element| *element == ObjectType::Folder) + || workspace_git_repo + .exclude_types_override + .unwrap_or_default() + .iter() + .any(|element| *element == ObjectType::Folder) + } + DeployedObject::Resource { .. } => { + !workspace_git_sync_settings + .include_type + .iter() + .any(|element| *element == ObjectType::Resource) + || workspace_git_repo + .exclude_types_override + .unwrap_or_default() + .iter() + .any(|element| *element == ObjectType::Resource) + } + DeployedObject::Variable { .. } => { + !workspace_git_sync_settings + .include_type + .iter() + .any(|element| *element == ObjectType::Variable) + || workspace_git_repo + .exclude_types_override + .unwrap_or_default() + .iter() + .any(|element| *element == ObjectType::Variable) + } + DeployedObject::ResourceType { .. } => { + !workspace_git_sync_settings + .include_type + .iter() + .any(|element| *element == ObjectType::ResourceType) + || workspace_git_repo + .exclude_types_override + .unwrap_or_default() + .iter() + .any(|element| *element == ObjectType::ResourceType) + } + DeployedObject::Schedule { .. } => { + !workspace_git_sync_settings + .include_type + .iter() + .any(|element| *element == ObjectType::Schedule) + || workspace_git_repo + .exclude_types_override + .unwrap_or_default() + .iter() + .any(|element| *element == ObjectType::Schedule) + } + }; + + if skip_object_type { + tracing::debug!("Skipping sync due to objec type filter"); + continue; + } + + // if yes, sync it let mut args: HashMap = HashMap::new(); args.insert( "repo_url_resource_path".to_string(), diff --git a/frontend/src/routes/(root)/(logged)/workspace_settings/+page.svelte b/frontend/src/routes/(root)/(logged)/workspace_settings/+page.svelte index 31f577956c..541c1ebe0c 100644 --- a/frontend/src/routes/(root)/(logged)/workspace_settings/+page.svelte +++ b/frontend/src/routes/(root)/(logged)/workspace_settings/+page.svelte @@ -42,6 +42,28 @@ import Portal from 'svelte-portal' import { fade } from 'svelte/transition' + type GitSyncTypeMap = { + scripts: boolean + flows: boolean + apps: boolean + folders: boolean + resourceTypes: boolean + resources: boolean + variables: boolean + secrets: boolean + schedules: boolean + } + type GitSyncType = + | 'script' + | 'flow' + | 'app' + | 'folder' + | 'resourcetype' + | 'resource' + | 'variable' + | 'secret' + | 'schedule' + let s3FileViewer: S3FilePicker let initialPath: string @@ -67,21 +89,12 @@ let gitSyncSettings: { include_path: string[] repositories: { + exclude_types_override: GitSyncTypeMap script_path: string git_repo_resource_path: string use_individual_branch: boolean }[] - include_type: { - scripts: boolean - flows: boolean - apps: boolean - folders: boolean - resourceTypes: boolean - resources: boolean - variables: boolean - secrets: boolean - schedules: boolean - } + include_type: GitSyncTypeMap } let gitSyncTestJobs: { jobId: string | undefined @@ -225,7 +238,9 @@ let alreadySeenResource: string[] = [] let repositories = gitSyncSettings.repositories.map((elmt) => { alreadySeenResource.push(elmt.git_repo_resource_path) + let exclude_types_override = gitSyncTypeMapToArray(elmt.exclude_types_override, true) return { + exclude_types_override: exclude_types_override, script_path: elmt.script_path, git_repo_resource_path: `$res:${elmt.git_repo_resource_path.replace('$res:', '')}`, use_individual_branch: elmt.use_individual_branch @@ -236,44 +251,7 @@ return !emptyString(elmt) }) - let include_type: ( - | 'script' - | 'flow' - | 'app' - | 'folder' - | 'resourcetype' - | 'resource' - | 'variable' - | 'secret' - | 'schedule' - )[] = [] - if (gitSyncSettings.include_type.scripts) { - include_type.push('script') - } - if (gitSyncSettings.include_type.flows) { - include_type.push('flow') - } - if (gitSyncSettings.include_type.apps) { - include_type.push('app') - } - if (gitSyncSettings.include_type.folders) { - include_type.push('folder') - } - if (gitSyncSettings.include_type.resourceTypes) { - include_type.push('resourcetype') - } - if (gitSyncSettings.include_type.resources) { - include_type.push('resource') - } - if (gitSyncSettings.include_type.variables) { - include_type.push('variable') - } - if (gitSyncSettings.include_type.secrets) { - include_type.push('secret') - } - if (gitSyncSettings.include_type.schedules) { - include_type.push('schedule') - } + let include_type = gitSyncTypeMapToArray(gitSyncSettings.include_type, true) if (alreadySeenResource.some((res, index) => alreadySeenResource.indexOf(res) !== index)) { sendUserToast('Same Git resource used more than once', true) @@ -302,6 +280,44 @@ } } + function gitSyncTypeMapToArray(typesMap: GitSyncTypeMap, expectedValue: boolean): GitSyncType[] { + let result: GitSyncType[] = [] + if (typesMap.scripts == expectedValue) { + result.push('script') + } + if (typesMap.flows == expectedValue) { + result.push('flow') + } + if (typesMap.apps == expectedValue) { + result.push('app') + } + if (typesMap.folders == expectedValue) { + result.push('folder') + } + if (typesMap.resourceTypes == expectedValue) { + result.push('resourcetype') + } + if (typesMap.resources == expectedValue) { + result.push('resource') + } + if (typesMap.variables == expectedValue) { + result.push('variable') + } + if (typesMap.secrets == expectedValue) { + result.push('secret') + } + if (typesMap.schedules == expectedValue) { + result.push('schedule') + } + return result + } + + function resetGitSyncRepositoryExclude(type: string) { + gitSyncSettings.repositories.forEach((elmt) => { + elmt.exclude_types_override[type] = false + }) + } + async function editWorkspaceDefaultApp(appPath: string | undefined): Promise { if (emptyString(appPath)) { await WorkspaceService.editWorkspaceDefaultApp({ @@ -388,7 +404,18 @@ return { git_repo_resource_path: settings.git_repo_resource_path.replace('$res:', ''), script_path: settings.script_path, - use_individual_branch: settings.use_individual_branch ?? false + use_individual_branch: settings.use_individual_branch ?? false, + exclude_types_override: { + scripts: (settings.exclude_types_override?.indexOf('script') ?? -1) >= 0, + flows: (settings.exclude_types_override?.indexOf('flow') ?? -1) >= 0, + apps: (settings.exclude_types_override?.indexOf('app') ?? -1) >= 0, + resourceTypes: (settings.exclude_types_override?.indexOf('resourcetype') ?? -1) >= 0, + resources: (settings.exclude_types_override?.indexOf('resource') ?? -1) >= 0, + variables: (settings.exclude_types_override?.indexOf('variable') ?? -1) >= 0, + secrets: (settings.exclude_types_override?.indexOf('secret') ?? -1) >= 0, + schedules: (settings.exclude_types_override?.indexOf('schedule') ?? -1) >= 0, + folders: (settings.exclude_types_override?.indexOf('folder') ?? -1) >= 0 + } } }), include_type: { @@ -421,7 +448,6 @@ } gitSyncTestJobs = [] } - console.log(gitSyncSettings) // check openai_client_credentials_oauth usingOpenaiClientCredentialsOauth = await ResourceService.existsResourceType({ @@ -1021,41 +1047,57 @@
resetGitSyncRepositoryExclude('scripts')} options={{ right: 'Scripts' }} /> resetGitSyncRepositoryExclude('flows')} options={{ right: 'Flows' }} /> resetGitSyncRepositoryExclude('apps')} options={{ right: 'Apps' }} /> resetGitSyncRepositoryExclude('folders')} options={{ right: 'Folders' }} /> resetGitSyncRepositoryExclude('resources')} options={{ right: 'Resources' }} />
{ + resetGitSyncRepositoryExclude('variables') + resetGitSyncRepositoryExclude('secrets') + if (!ev.detail) { + gitSyncSettings.include_type.secrets = false + } + }} options={{ right: 'Variables ' }} /> - resetGitSyncRepositoryExclude('secrets')} options={{ left: 'Include secrets' }} />
resetGitSyncRepositoryExclude('schedules')} options={{ right: 'Schedules' }} /> resetGitSyncRepositoryExclude('resourcetypes')} options={{ right: 'Resource Types' }} />
@@ -1136,6 +1178,86 @@ /> {/if} + +
+ {#if gitSyncSettings && Object.keys(gitSyncSettings.include_type).some((k) => gitSyncSettings.include_type[k] === true)} +
Exclude specific types for this repository only
+ {#if gitSyncSettings.include_type.scripts} + + {/if} + {#if gitSyncSettings.include_type.flows} + + {/if} + {#if gitSyncSettings.include_type.apps} + + {/if} + {#if gitSyncSettings.include_type.folders} + + {/if} + {#if gitSyncSettings.include_type.resources} + + {/if} + {#if gitSyncSettings.include_type.variables} +
+ { + if (ev.detail && gitSyncSettings.include_type.secrets) { + gitSyncRepository.exclude_types_override.secrets = true + } else if (ev.detail) { + gitSyncRepository.exclude_types_override.secrets = false + } + }} + options={{ right: 'Exclude variables ' }} + /> + {#if gitSyncSettings.include_type.secrets} + - + + {/if} +
+ {/if} + {#if gitSyncSettings.include_type.schedules} + + {/if} + {#if gitSyncSettings.include_type.resourceTypes} + + {/if} + {/if} +
{/each} {/if} @@ -1151,7 +1273,18 @@ { script_path: 'hub/7958/sync-script-to-git-repo-windmill', git_repo_resource_path: '', - use_individual_branch: false + use_individual_branch: false, + exclude_types_override: { + scripts: false, + flows: false, + apps: false, + folders: false, + resourceTypes: false, + resources: false, + variables: false, + secrets: false, + schedules: false + } } ] gitSyncTestJobs = [