starred/windmill
1
0
Fork 0
mirror of https://github.com/windmill-labs/windmill.git synced 2026-08-21 08:02:26 +00:00
Code Issues Packages Projects Releases Wiki Activity

feat: GIt sync to multiple repo (#2996)

Browse Source
This commit is contained in:
Guillaume Bouvignies
2024-01-12 16:19:28 +01:00
committed by GitHub
parent 13c5802670
commit fda0e28db3
9 changed files with 263 additions and 192 deletions
Download Patch File Download Diff File Expand all files Collapse all files
backend/Cargo.lock
Generated
+1
View File
@@ -9432,6 +9432,7 @@ dependencies = [
"serde_json",
"sqlx",
"tracing",
"uuid 1.6.1",
"windmill-common",
"windmill-queue",
]
backend/migrations/20240112131322_git_sync_multi.down.sql
+2
View File
@@ -0,0 +1,2 @@
-- Add down migration script here
UPDATE workspace_settings SET git_sync = git_sync->0;
backend/migrations/20240112131322_git_sync_multi.up.sql
+2
View File
@@ -0,0 +1,2 @@
-- Add up migration script here
UPDATE workspace_settings SET git_sync = '[]'::jsonb || git_sync
backend/windmill-api/openapi-deref.yaml
+20 -16
View File
@@ -1,6 +1,6 @@
openapi: 3.0.3
info:
version: 1.241.0
version: 1.242.0
title: Windmill API
contact:
name: Windmill Team
@@ -1525,17 +1525,19 @@ paths:
s3_resource_path:
type: string
git_sync:
type: object
properties: &ref_16
script_path:
type: string
git_repo_resource_path:
type: string
use_individual_branch:
type: boolean
required: &ref_17
- script_path
- git_repo_resource_path
type: array
items:
type: object
properties: &ref_16
script_path:
type: string
git_repo_resource_path:
type: string
use_individual_branch:
type: boolean
required: &ref_17
- script_path
- git_repo_resource_path
required:
- code_completion_enabled
/w/{workspace}/workspaces/get_deploy_to:
@@ -1891,9 +1893,11 @@ paths:
type: object
properties:
git_sync_settings:
type: object
properties: *ref_16
required: *ref_17
type: array
items:
type: object
properties: *ref_16
required: *ref_17
responses:
'200':
description: status
@@ -10202,7 +10206,7 @@ paths:
schema: {}
/w/{workspace}/job_helpers/list_stored_files:
get:
summary: List the file keys available in the worspace files storage (S3)
summary: List the file keys available in the workspace files storage (S3)
operationId: listStoredFiles
tags:
- helpers
backend/windmill-api/openapi.yaml
+7 -3
View File
@@ -1193,7 +1193,9 @@ paths:
large_file_storage:
$ref: "#/components/schemas/LargeFileStorage"
git_sync:
$ref: "#/components/schemas/WorkspaceGitSync"
type: array
items:
$ref: "#/components/schemas/WorkspaceGitSync"
required:
- code_completion_enabled
@@ -1532,7 +1534,9 @@ paths:
type: object
properties:
git_sync_settings:
$ref: "#/components/schemas/WorkspaceGitSync"
type: array
items:
$ref: "#/components/schemas/WorkspaceGitSync"
responses:
"200":
@@ -6846,7 +6850,7 @@ paths:
/w/{workspace}/job_helpers/list_stored_files:
get:
summary: List the file keys available in the worspace files storage (S3)
summary: List the file keys available in the workspace files storage (S3)
operationId: listStoredFiles
tags:
- helpers
backend/windmill-api/src/workspaces.rs
+2 -2
View File
@@ -1044,7 +1044,7 @@ async fn edit_large_file_storage_config(
#[derive(Deserialize)]
struct EditGitSyncConfig {
git_sync_settings: Option<WorkspaceGitRepo>,
git_sync_settings: Option<Vec<WorkspaceGitRepo>>,
}
async fn edit_git_sync_config(
@@ -1078,7 +1078,7 @@ async fn edit_git_sync_config(
.await?;
if let Some(git_sync_settings) = new_config.git_sync_settings {
let serialized_config = serde_json::to_value::<WorkspaceGitRepo>(git_sync_settings)
let serialized_config = serde_json::to_value::<Vec<WorkspaceGitRepo>>(git_sync_settings)
.map_err(|err| Error::InternalErr(err.to_string()))?;
sqlx::query!(
backend/windmill-git-sync/Cargo.toml
+1
View File
@@ -13,6 +13,7 @@ enterprise = ["windmill-queue/enterprise"]
default = []
[dependencies]
uuid.workspace = true
serde.workspace = true
sqlx.workspace = true
serde_json.workspace = true
backend/windmill-git-sync/src/lib.rs
+93 -95
View File
@@ -9,6 +9,7 @@
use std::collections::HashMap;
use sqlx::{Pool, Postgres};
use uuid::Uuid;
use windmill_common::users::SUPERADMIN_SYNC_EMAIL;
use windmill_common::workspaces::WorkspaceGitRepo;
@@ -45,7 +46,7 @@ impl DeployedObject {
}
}
pub async fn handle_deployment_metadata<'c, R: rsmq_async::RsmqConnection + Send + 'c>(
pub async fn handle_deployment_metadata<'c, R: rsmq_async::RsmqConnection + Send + Clone + 'c>(
email: &str,
created_by: &str,
db: &DB,
@@ -94,122 +95,119 @@ pub async fn handle_deployment_metadata<'c, R: rsmq_async::RsmqConnection + Send
));
}
let workspace_git_repo = workspace_git_repo_setting
let workspace_git_repos = workspace_git_repo_setting
.unwrap()
.map(|conf| serde_json::from_value::<WorkspaceGitRepo>(conf).ok())
.flatten();
.map(|conf| serde_json::from_value::<Vec<WorkspaceGitRepo>>(conf).ok())
.flatten()
.unwrap_or_default();
let git_sync_job_uuid = if !skip_git_sync && workspace_git_repo.is_some() {
let workspace_git_repo = workspace_git_repo.unwrap();
let mut git_sync_job_uuids: Vec<Uuid> = vec![];
if !skip_git_sync {
for workspace_git_repo in workspace_git_repos {
let mut args: HashMap<String, serde_json::Value> = HashMap::new();
args.insert(
"repo_url_resource_path".to_string(),
json!(workspace_git_repo
.git_repo_resource_path
.strip_prefix("$res:")),
);
let mut args: HashMap<String, serde_json::Value> = HashMap::new();
args.insert(
"repo_url_resource_path".to_string(),
json!(workspace_git_repo
.git_repo_resource_path
.strip_prefix("$res:")),
);
if let Some(path) = obj_path {
args.insert("path".to_string(), json!(path));
}
if let Some(parent_path) = obj_parent_path.clone() {
args.insert("parent_path".to_string(), json!(parent_path));
}
if let Some(path) = obj_path {
args.insert("path".to_string(), json!(path));
let message = match obj.clone() {
DeployedObject::Script { path, .. } => {
if deployment_message.as_ref().is_none()
|| deployment_message.as_ref().is_some_and(|x| x.is_empty())
{
format!("Script '{}' deployed", path)
} else {
deployment_message.clone().unwrap()
}
}
DeployedObject::Flow { path, .. } => {
if deployment_message.as_ref().is_none()
|| deployment_message.as_ref().is_some_and(|x| x.is_empty())
{
format!("Flow '{}' deployed", path)
} else {
deployment_message.clone().unwrap()
}
}
DeployedObject::App { path, .. } => {
if deployment_message.as_ref().is_none()
|| deployment_message.as_ref().is_some_and(|x| x.is_empty())
{
format!("App '{}' deployed", path)
} else {
deployment_message.clone().unwrap()
}
}
};
args.insert("commit_msg".to_string(), json!(message));
args.insert(
"use_individual_branch".to_string(),
json!(workspace_git_repo.use_individual_branch.unwrap_or(false)),
);
let tx: PushIsolationLevel<'_, R> =
PushIsolationLevel::IsolatedRoot(db.clone(), rsmq.clone());
let (job_uuid, new_tx) = windmill_queue::push(
db,
tx,
&w_id,
JobPayload::DeploymentCallback { path: workspace_git_repo.script_path.clone() },
args,
created_by,
&email,
SUPERADMIN_SYNC_EMAIL.to_string(),
None,
None,
None,
None,
None,
false,
false,
None,
true,
None,
None,
None,
None,
)
.await?;
new_tx.commit().await?;
git_sync_job_uuids.push(job_uuid);
}
if let Some(parent_path) = obj_parent_path {
args.insert("parent_path".to_string(), json!(parent_path));
}
let message = match obj.clone() {
DeployedObject::Script { path, .. } => {
if deployment_message.as_ref().is_none()
|| deployment_message.as_ref().is_some_and(|x| x.is_empty())
{
format!("Script '{}' deployed", path)
} else {
deployment_message.clone().unwrap()
}
}
DeployedObject::Flow { path, .. } => {
if deployment_message.as_ref().is_none()
|| deployment_message.as_ref().is_some_and(|x| x.is_empty())
{
format!("Flow '{}' deployed", path)
} else {
deployment_message.clone().unwrap()
}
}
DeployedObject::App { path, .. } => {
if deployment_message.as_ref().is_none()
|| deployment_message.as_ref().is_some_and(|x| x.is_empty())
{
format!("App '{}' deployed", path)
} else {
deployment_message.clone().unwrap()
}
}
};
args.insert("commit_msg".to_string(), json!(message));
args.insert(
"use_individual_branch".to_string(),
json!(workspace_git_repo.use_individual_branch.unwrap_or(false)),
);
let tx: PushIsolationLevel<'_, R> = PushIsolationLevel::IsolatedRoot(db.clone(), rsmq);
let (job_uuid, new_tx) = windmill_queue::push(
db,
tx,
&w_id,
JobPayload::DeploymentCallback { path: workspace_git_repo.script_path.clone() },
args,
created_by,
&email,
SUPERADMIN_SYNC_EMAIL.to_string(),
None,
None,
None,
None,
None,
false,
false,
None,
true,
None,
None,
None,
None,
)
.await?;
new_tx.commit().await?;
Some(job_uuid)
} else {
None
};
}
// We're not persisting the default commit msg as it's pretty useless. We will persist the ones manually set by users
let job_uuids = if git_sync_job_uuid.is_some() {
vec![git_sync_job_uuid.unwrap()]
} else {
vec![]
};
if !skip_db_insert && (deployment_message.is_some() || job_uuids.len() > 0) {
if !skip_db_insert && (deployment_message.is_some() || git_sync_job_uuids.len() > 0) {
// if the git sync job hasn't been triggered, and there is not custom deployment message, there's not point adding an entry to the table
match obj.clone() {
DeployedObject::Script { path, hash, .. } => {
sqlx::query!(
"INSERT INTO deployment_metadata (workspace_id, path, script_hash, callback_job_ids, deployment_msg) VALUES ($1, $2, $3, $4, $5)",
w_id, path, hash.0, &job_uuids, deployment_message,
w_id, path, hash.0, &git_sync_job_uuids, deployment_message,
)
},
DeployedObject::Flow { path, .. } => {
sqlx::query!(
"INSERT INTO deployment_metadata (workspace_id, path, callback_job_ids, deployment_msg) VALUES ($1, $2, $3, $4) ON CONFLICT (workspace_id, path) WHERE script_hash IS NULL AND app_version IS NULL DO UPDATE SET callback_job_ids = $3, deployment_msg = $4",
w_id, path, &job_uuids, deployment_message,
w_id, path, &git_sync_job_uuids, deployment_message,
)
}
DeployedObject::App { path, version, .. } => {
sqlx::query!(
"INSERT INTO deployment_metadata (workspace_id, path, app_version, callback_job_ids, deployment_msg) VALUES ($1, $2, $3, $4, $5)",
w_id, path, version, &job_uuids, deployment_message,
w_id, path, version, &git_sync_job_uuids, deployment_message,
)
}
}.execute(db)
frontend/src/routes/(root)/(logged)/workspace_settings/+page.svelte
+135 -76
View File
@@ -60,13 +60,11 @@
script_path: string
git_repo_resource_path: string
use_individual_branch: boolean
}
let gitSyncTestJob:
| {
jobId: string
status: 'running' | 'success' | 'failure'
}
| undefined = undefined
}[]
let gitSyncTestJobs: {
jobId: string | undefined
status: 'running' | 'success' | 'failure' | undefined
}[]
let codeCompletionEnabled: boolean = false
let tab =
($page.url.searchParams.get('tab') as
@@ -194,29 +192,29 @@
}
}
async function editWindmillGitSyncSettings(
gitRepoResourcePath: string,
useIndividualBranch: boolean
): Promise<void> {
if (!emptyString(gitRepoResourcePath)) {
gitSyncSettings = {
script_path: 'hub/7926/sync-script-to-git-repo-windmill',
git_repo_resource_path: `$res:${gitRepoResourcePath.replace('$res:', '')}`,
use_individual_branch: useIndividualBranch
async function editWindmillGitSyncSettings(): Promise<void> {
let alreadySeenResource: string[] = []
let finalSettings = gitSyncSettings.map((elmt) => {
alreadySeenResource.push(elmt.git_repo_resource_path)
return {
script_path: elmt.script_path,
git_repo_resource_path: `$res:${elmt.git_repo_resource_path.replace('$res:', '')}`,
use_individual_branch: elmt.use_individual_branch
}
})
if (alreadySeenResource.some((res, index) => alreadySeenResource.indexOf(res) !== index)) {
sendUserToast('Same Git resource used more than once', true)
return
}
if (finalSettings.length > 0) {
await WorkspaceService.editWorkspaceGitSyncConfig({
workspace: $workspaceStore!,
requestBody: {
git_sync_settings: gitSyncSettings
git_sync_settings: finalSettings
}
})
sendUserToast('Workspace Git sync settings updated')
} else {
gitSyncSettings = {
script_path: '',
git_repo_resource_path: '',
use_individual_branch: false
}
await WorkspaceService.editWorkspaceGitSyncConfig({
workspace: $workspaceStore!,
requestBody: {
@@ -261,18 +259,27 @@
settings.large_file_storage?.type === LargeFileStorage.type.S3STORAGE
? settings.large_file_storage?.s3_resource_path?.replace('$res:', '')
: undefined
if (settings.git_sync !== undefined && settings.git_sync !== null) {
gitSyncSettings = {
git_repo_resource_path: settings.git_sync.git_repo_resource_path.replace('$res:', ''),
script_path: settings.git_sync.script_path,
use_individual_branch: settings.git_sync.use_individual_branch ?? false
}
if (
settings.git_sync !== undefined &&
settings.git_sync !== null &&
settings.git_sync.length > 0
) {
gitSyncSettings = settings.git_sync.map((settings) => {
return {
git_repo_resource_path: settings.git_repo_resource_path.replace('$res:', ''),
script_path: settings.script_path,
use_individual_branch: settings.use_individual_branch ?? false
}
})
gitSyncTestJobs = settings.git_sync.map((settings) => {
return {
jobId: undefined,
status: undefined
}
})
} else {
gitSyncSettings = {
git_repo_resource_path: '',
script_path: '',
use_individual_branch: false
}
gitSyncSettings = []
gitSyncTestJobs = []
}
// check openai_client_credentials_oauth
@@ -322,18 +329,19 @@
)
}
async function runGitSyncTestJob() {
if (emptyString(gitSyncSettings.script_path)) {
async function runGitSyncTestJob(settingsIdx: number) {
let gitSyncSettingsElmt = gitSyncSettings[settingsIdx]
if (emptyString(gitSyncSettingsElmt.script_path)) {
return
}
let jobId = await JobService.runScriptByPath({
workspace: $workspaceStore!,
path: 'hub/7925/git-repo-test-read-write-windmill',
requestBody: {
repo_url_resource_path: gitSyncSettings.git_repo_resource_path.replace('$res:', '')
repo_url_resource_path: gitSyncSettingsElmt.git_repo_resource_path.replace('$res:', '')
}
})
gitSyncTestJob = {
gitSyncTestJobs[settingsIdx] = {
jobId: jobId,
status: 'running'
}
@@ -343,7 +351,7 @@
workspace: $workspaceStore!,
id: jobId
})
gitSyncTestJob!.status = testResult.success ? 'success' : 'failure'
gitSyncTestJobs[settingsIdx].status = testResult.success ? 'success' : 'failure'
},
timeoutCode: async () => {
try {
@@ -776,57 +784,98 @@
<br />
Filtering out certain sensitive folders from the sync will be available soon.
</Alert>
<div class="flex mt-5 mb-1 gap-1">
{#if gitSyncSettings}
{#key gitSyncSettings}
{#each gitSyncSettings as gitSyncSettingsElmt, idx}
<div class="flex mt-5 mb-1 gap-1 items-center text-xs">
<h6>Repository #{idx + 1}</h6>
<Button
color="light"
size="xs"
startIcon={{ icon: XCircle }}
iconOnly={true}
on:click={() => {
gitSyncSettings.splice(idx, 1)
gitSyncSettings = [...gitSyncSettings]
}}
/>
</div>
<div class="flex mt-5 mb-1 gap-1">
{#key gitSyncSettingsElmt}
<ResourcePicker
resourceType="git_repository"
initialValue={gitSyncSettings?.git_repo_resource_path?.split('$res:')?.[1] || ''}
initialValue={gitSyncSettingsElmt.git_repo_resource_path}
on:change={(ev) => {
editWindmillGitSyncSettings(ev.detail, gitSyncSettings?.use_individual_branch)
gitSyncSettingsElmt.git_repo_resource_path = ev.detail
}}
/>
<Button
disabled={emptyString(gitSyncSettings?.script_path)}
disabled={emptyString(gitSyncSettingsElmt.script_path)}
btnClasses="w-32 text-center"
color="dark"
on:click={() => runGitSyncTestJob()}
on:click={() => runGitSyncTestJob(idx)}
size="xs">Test connection</Button
>
{/key}
{/if}
</div>
<div class="flex mb-5 text-normal text-2xs gap-1">
{#if gitSyncTestJob !== undefined}
{#if gitSyncTestJob?.status === 'running'}
<RotateCw size={14} />
{:else if gitSyncTestJob?.status === 'success'}
<CheckCircle2 size={14} class="text-green-600" />
{:else}
<XCircle size={14} class="text-red-700" />
</div>
<div class="flex mb-5 text-normal text-2xs gap-1">
{#if gitSyncSettings.filter((settings) => settings.git_repo_resource_path === gitSyncSettingsElmt.git_repo_resource_path).length > 1}
<span class="text-red-700">Using the same resource twice is not allowed.</span>
{/if}
Git sync resource checked via Windmill job
<a target="_blank" href={`/run/${gitSyncTestJob?.jobId}?workspace=${$workspaceStore}`}>
{gitSyncTestJob?.jobId}
</a>WARNING: Only read permissions are verified.
{/if}
</div>
{#if gitSyncTestJobs[idx].status !== undefined}
{#if gitSyncTestJobs[idx].status === 'running'}
<RotateCw size={14} />
{:else if gitSyncTestJobs[idx].status === 'success'}
<CheckCircle2 size={14} class="text-green-600" />
{:else}
<XCircle size={14} class="text-red-700" />
{/if}
Git sync resource checked via Windmill job
<a
target="_blank"
href={`/run/${gitSyncTestJobs[idx].jobId}?workspace=${$workspaceStore}`}
>
{gitSyncTestJobs[idx].jobId}
</a>WARNING: Only read permissions are verified.
{/if}
</div>
<div class="flex mt-5 mb-1 gap-1">
{#if gitSyncSettings}
<Toggle
disabled={emptyString(gitSyncSettings?.git_repo_resource_path)}
bind:checked={gitSyncSettings.use_individual_branch}
on:change={(ev) => {
editWindmillGitSyncSettings(gitSyncSettings.git_repo_resource_path, ev.detail)
}}
options={{
right: 'Create one branch per deployed script/flow/app',
rightTooltip:
"If set, Windmill will create a unique branch per script/flow/app being pushed, prefixed with 'wm_deploy/'."
}}
/>
{/if}
<div class="flex mt-5 mb-1 gap-1">
{#if gitSyncSettings}
<Toggle
disabled={emptyString(gitSyncSettingsElmt?.git_repo_resource_path)}
bind:checked={gitSyncSettingsElmt.use_individual_branch}
options={{
right: 'Create one branch per deployed script/flow/app',
rightTooltip:
"If set, Windmill will create a unique branch per script/flow/app being pushed, prefixed with 'wm_deploy/'."
}}
/>
{/if}
</div>
{/each}
<div class="flex mt-5 mb-5 gap-1">
<Button
color="none"
variant="border"
on:click={() => {
gitSyncSettings = [
...gitSyncSettings,
{
script_path: 'hub/7926/sync-script-to-git-repo-windmill',
git_repo_resource_path: '',
use_individual_branch: false
}
]
gitSyncTestJobs = [
...gitSyncTestJobs,
{
jobId: undefined,
status: undefined
}
]
}}>Add connection</Button
>
</div>
<div class="bg-surface-disabled p-4 rounded-md flex flex-col gap-1">
@@ -865,6 +914,16 @@ git push</code
>
</div>
</div>
<div class="flex mt-5 mb-5 gap-1">
<Button
color="blue"
disabled={gitSyncSettings.some((elmt) => emptyString(elmt.git_repo_resource_path))}
on:click={() => {
editWindmillGitSyncSettings()
console.log('Saving git sync settings', gitSyncSettings)
}}>Save Git sync settings</Button
>
</div>
{/if}
{:else}
<div class="bg-red-100 border-l-4 border-red-600 text-orange-700 p-4 m-4" role="alert">