diff --git a/backend/.sqlx/query-206a65cd2f0da413eb7f5873f696aa3562cf23fa48c479bcba51dd421813a291.json b/backend/.sqlx/query-206a65cd2f0da413eb7f5873f696aa3562cf23fa48c479bcba51dd421813a291.json new file mode 100644 index 0000000000..ffd781d5c9 --- /dev/null +++ b/backend/.sqlx/query-206a65cd2f0da413eb7f5873f696aa3562cf23fa48c479bcba51dd421813a291.json @@ -0,0 +1,17 @@ +{ + "db_name": "PostgreSQL", + "query": "\n UPDATE workspace_settings\n SET git_sync = jsonb_set(\n git_sync,\n '{repositories}',\n (SELECT jsonb_agg(\n CASE WHEN elem->>'git_repo_resource_path' = $2\n THEN jsonb_set(\n jsonb_set(elem, '{auto_pull,last_synced_sha}', $3, true),\n '{auto_pull,last_pull_status}', $4, true)\n ELSE elem END)\n FROM jsonb_array_elements(git_sync->'repositories') AS elem)\n )\n WHERE workspace_id = $1\n ", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "Text", + "Text", + "Jsonb", + "Jsonb" + ] + }, + "nullable": [] + }, + "hash": "206a65cd2f0da413eb7f5873f696aa3562cf23fa48c479bcba51dd421813a291" +} diff --git a/backend/.sqlx/query-ec2b685c6116ac69a97c2e8b8354b572edc96014012109cac3be54627337eb7c.json b/backend/.sqlx/query-ec2b685c6116ac69a97c2e8b8354b572edc96014012109cac3be54627337eb7c.json new file mode 100644 index 0000000000..8b905a6d4c --- /dev/null +++ b/backend/.sqlx/query-ec2b685c6116ac69a97c2e8b8354b572edc96014012109cac3be54627337eb7c.json @@ -0,0 +1,26 @@ +{ + "db_name": "PostgreSQL", + "query": "SELECT workspace_id, git_sync\n FROM workspace_settings\n WHERE git_sync IS NOT NULL\n AND git_sync->'repositories' @> '[{\"auto_pull\": {\"enabled\": true}}]'::jsonb", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "workspace_id", + "type_info": "Varchar" + }, + { + "ordinal": 1, + "name": "git_sync", + "type_info": "Jsonb" + } + ], + "parameters": { + "Left": [] + }, + "nullable": [ + false, + true + ] + }, + "hash": "ec2b685c6116ac69a97c2e8b8354b572edc96014012109cac3be54627337eb7c" +} diff --git a/backend/Cargo.lock b/backend/Cargo.lock index 0837c09b54..3979c7b016 100644 --- a/backend/Cargo.lock +++ b/backend/Cargo.lock @@ -13845,6 +13845,7 @@ dependencies = [ "windmill-parser-ts", "windmill-queue", "windmill-runtime-nativets", + "windmill-store", "windmill-test-utils", "windmill-trigger", "windmill-trigger-azure", diff --git a/backend/Cargo.toml b/backend/Cargo.toml index f517c7668a..41190428c8 100644 --- a/backend/Cargo.toml +++ b/backend/Cargo.toml @@ -249,6 +249,7 @@ windmill-queue.workspace = true windmill-common = { workspace = true, default-features = false } windmill-object-store.workspace = true windmill-git-sync.workspace = true +windmill-store.workspace = true windmill-api = { workspace = true, default-features = false } windmill-api-agent-workers = { workspace = true, optional = true } windmill-api-scripts.workspace = true diff --git a/backend/src/monitor.rs b/backend/src/monitor.rs index f396726087..5911f71518 100644 --- a/backend/src/monitor.rs +++ b/backend/src/monitor.rs @@ -2885,6 +2885,17 @@ pub async fn monitor_db( } }; + // Poll git-sync repositories for new commits and pull them into the + // workspace (repo → Windmill auto-pull). Runs every 2 iterations (~60s). + let git_auto_pull_f = async { + #[cfg(feature = "private")] + if server_mode && iteration.is_some() && iteration.as_ref().unwrap().should_run(2) { + if let Some(db) = conn.as_sql() { + poll_git_auto_pull(db).await; + } + } + }; + join!( expired_items_f, zombie_jobs_f, @@ -2912,9 +2923,256 @@ pub async fn monitor_db( manage_audit_partitions_f, export_audit_logs_to_object_store_f, cleanup_scheduled_job_deletions_f, + git_auto_pull_f, ); } +/// Advisory lock id ensuring only one server replica runs the git auto-pull +/// poll at a time (adjacent to RESTART_LOCK_ID used for restart coordination). +#[cfg(feature = "private")] +const GIT_AUTO_PULL_LOCK_ID: i64 = 737_483_921; + +/// Poll every git-sync repository with auto-pull enabled and enqueue a pull when +/// the tracked branch has new commits (repo → Windmill direction). +/// +/// Runs on a single replica at a time (advisory lock) and only on +/// Enterprise-licensed instances. Detection is `git ls-remote`; GitHub-App +/// repositories are skipped here and sync via webhooks instead (phase 2). +#[cfg(feature = "private")] +pub async fn poll_git_auto_pull(db: &Pool) { + use windmill_common::ee_oss::{get_license_plan, LicensePlan}; + + if !matches!(get_license_plan().await, LicensePlan::Enterprise) { + return; + } + + let mut lock_conn = match db.acquire().await { + Ok(c) => c, + Err(e) => { + tracing::error!("git auto-pull: failed to acquire connection: {e:#}"); + return; + } + }; + let locked: bool = match sqlx::query_scalar("SELECT pg_try_advisory_lock($1)") + .bind(GIT_AUTO_PULL_LOCK_ID) + .fetch_one(&mut *lock_conn) + .await + { + Ok(v) => v, + Err(e) => { + tracing::error!("git auto-pull: advisory lock failed: {e:#}"); + return; + } + }; + if !locked { + // Another replica is already polling this tick. + return; + } + + if let Err(e) = poll_git_auto_pull_inner(db).await { + tracing::error!("git auto-pull: poll error: {e:#}"); + } + + if let Err(e) = sqlx::query("SELECT pg_advisory_unlock($1)") + .bind(GIT_AUTO_PULL_LOCK_ID) + .execute(&mut *lock_conn) + .await + { + tracing::error!("git auto-pull: advisory unlock failed: {e:#}"); + } +} + +#[cfg(feature = "private")] +async fn poll_git_auto_pull_inner(db: &Pool) -> error::Result<()> { + use windmill_common::workspaces::{AutoPullMode, AutoPullStatus, WorkspaceGitSyncSettings}; + + let rows = sqlx::query!( + r#"SELECT workspace_id, git_sync + FROM workspace_settings + WHERE git_sync IS NOT NULL + AND git_sync->'repositories' @> '[{"auto_pull": {"enabled": true}}]'::jsonb"# + ) + .fetch_all(db) + .await?; + + for row in rows { + let Some(git_sync) = row.git_sync else { + continue; + }; + let settings: WorkspaceGitSyncSettings = match serde_json::from_value(git_sync) { + Ok(s) => s, + Err(e) => { + tracing::warn!( + "git auto-pull: invalid git_sync settings for workspace {}: {e}", + row.workspace_id + ); + continue; + } + }; + + for repo in &settings.repositories { + let Some(auto_pull) = &repo.auto_pull else { + continue; + }; + if !auto_pull.enabled || auto_pull.mode == AutoPullMode::Webhook { + continue; + } + + match windmill_store::resources::get_git_repo_head_for_autopull( + db, + &row.workspace_id, + &repo.git_repo_resource_path, + ) + .await + { + Ok(Some((git_ref, sha))) => { + if !auto_pull.should_pull(&git_ref, &sha) { + continue; + } + let use_promotion_overrides = repo.use_individual_branch.unwrap_or(false); + match windmill_git_sync::enqueue_git_pull_job( + db, + &row.workspace_id, + repo, + None, + use_promotion_overrides, + ) + .await + { + Ok(job_id) => { + // Optimistically advance the synced sha so the next + // tick doesn't re-enqueue the same commit; the job id + // lets the user inspect the actual pull outcome. + let mut synced = auto_pull.last_synced_sha.clone(); + synced.insert(git_ref.clone(), sha.clone()); + let status = AutoPullStatus { + synced_sha: Some(sha.clone()), + at: chrono::Utc::now().timestamp(), + job_id: Some(job_id), + success: true, + error: None, + }; + if let Err(e) = persist_auto_pull_state( + db, + &row.workspace_id, + &repo.git_repo_resource_path, + &synced, + &status, + ) + .await + { + tracing::error!( + "git auto-pull: failed to persist state for {}/{}: {e:#}", + row.workspace_id, + repo.git_repo_resource_path + ); + } + tracing::info!( + "git auto-pull: enqueued pull {job_id} for {}/{} at {sha}", + row.workspace_id, + repo.git_repo_resource_path + ); + } + Err(e) => { + record_auto_pull_failure( + db, + &row.workspace_id, + &repo.git_repo_resource_path, + &auto_pull.last_synced_sha, + format!("failed to enqueue pull: {e}"), + ) + .await; + } + } + } + // GitHub-App repo: synced via webhook (phase 2), nothing to poll. + Ok(None) => {} + Err(e) => { + record_auto_pull_failure( + db, + &row.workspace_id, + &repo.git_repo_resource_path, + &auto_pull.last_synced_sha, + format!("ls-remote failed: {e}"), + ) + .await; + } + } + } + } + + Ok(()) +} + +/// Update the `auto_pull.last_synced_sha` and `auto_pull.last_pull_status` of a +/// single repository in `workspace_settings.git_sync`, leaving every other +/// field (and the other repositories) untouched so concurrent settings edits +/// are not clobbered. +#[cfg(feature = "private")] +async fn persist_auto_pull_state( + db: &Pool, + w_id: &str, + repo_resource_path: &str, + last_synced_sha: &std::collections::HashMap, + status: &windmill_common::workspaces::AutoPullStatus, +) -> error::Result<()> { + let synced_json = serde_json::to_value(last_synced_sha) + .map_err(|e| error::Error::internal_err(e.to_string()))?; + let status_json = + serde_json::to_value(status).map_err(|e| error::Error::internal_err(e.to_string()))?; + sqlx::query!( + r#" + UPDATE workspace_settings + SET git_sync = jsonb_set( + git_sync, + '{repositories}', + (SELECT jsonb_agg( + CASE WHEN elem->>'git_repo_resource_path' = $2 + THEN jsonb_set( + jsonb_set(elem, '{auto_pull,last_synced_sha}', $3, true), + '{auto_pull,last_pull_status}', $4, true) + ELSE elem END) + FROM jsonb_array_elements(git_sync->'repositories') AS elem) + ) + WHERE workspace_id = $1 + "#, + w_id, + repo_resource_path, + synced_json, + status_json, + ) + .execute(db) + .await?; + Ok(()) +} + +/// Record a failed auto-pull attempt in the repository's status without +/// advancing the synced sha, so the failure is visible and the next tick retries. +#[cfg(feature = "private")] +async fn record_auto_pull_failure( + db: &Pool, + w_id: &str, + repo_resource_path: &str, + last_synced_sha: &std::collections::HashMap, + error_msg: String, +) { + tracing::warn!("git auto-pull: {error_msg} for {w_id}/{repo_resource_path}"); + let status = windmill_common::workspaces::AutoPullStatus { + synced_sha: None, + at: chrono::Utc::now().timestamp(), + job_id: None, + success: false, + error: Some(error_msg), + }; + if let Err(e) = + persist_auto_pull_state(db, w_id, repo_resource_path, last_synced_sha, &status).await + { + tracing::error!( + "git auto-pull: failed to record failure for {w_id}/{repo_resource_path}: {e:#}" + ); + } +} + async fn vacuuming_tables(db: &Pool) -> error::Result<()> { sqlx::query!("VACUUM v2_job, v2_job_completed, job_result_stream_v2, job_stats, job_logs, job_perms, concurrency_key, log_file, metrics") .execute(db) diff --git a/backend/windmill-api-workspaces/src/workspaces.rs b/backend/windmill-api-workspaces/src/workspaces.rs index e0037861d3..ab5dabcc5f 100644 --- a/backend/windmill-api-workspaces/src/workspaces.rs +++ b/backend/windmill-api-workspaces/src/workspaces.rs @@ -2696,8 +2696,24 @@ async fn edit_git_sync_repository( .find(|repo| repo.git_repo_resource_path == new_config.git_repo_resource_path); if let Some(existing_repo) = repo_found { - // Update existing repository - *existing_repo = new_config.repository; + // Update existing repository, but preserve server-owned auto-pull state + // (synced sha, last pull status, webhook id/secret) so a settings save + // from the UI cannot revert what the poller/webhook layer wrote. + let mut updated = new_config.repository; + match (updated.auto_pull.as_mut(), existing_repo.auto_pull.as_ref()) { + (Some(new_ap), Some(old_ap)) => { + new_ap.last_synced_sha = old_ap.last_synced_sha.clone(); + new_ap.last_pull_status = old_ap.last_pull_status.clone(); + new_ap.webhook_id = old_ap.webhook_id; + new_ap.webhook_secret = old_ap.webhook_secret.clone(); + } + // UI omitted auto_pull (e.g. older client): keep existing config. + (None, Some(_)) => { + updated.auto_pull = existing_repo.auto_pull.clone(); + } + _ => {} + } + *existing_repo = updated; } else { // Repository doesn't exist, add it as a new repository git_sync_settings.repositories.push(new_config.repository); diff --git a/backend/windmill-api/openapi.yaml b/backend/windmill-api/openapi.yaml index a4775a6878..066f4a7f72 100644 --- a/backend/windmill-api/openapi.yaml +++ b/backend/windmill-api/openapi.yaml @@ -27616,9 +27616,60 @@ components: type: array items: $ref: "#/components/schemas/GitSyncObjectType" + auto_pull: + $ref: "#/components/schemas/AutoPullSettings" required: - git_repo_resource_path + AutoPullMode: + type: string + enum: + - auto + - webhook + - polling + + AutoPullStatus: + type: object + properties: + synced_sha: + type: string + at: + type: integer + format: int64 + job_id: + type: string + format: uuid + success: + type: boolean + error: + type: string + required: + - at + - success + + AutoPullSettings: + type: object + properties: + enabled: + type: boolean + mode: + $ref: "#/components/schemas/AutoPullMode" + poll_interval_s: + type: integer + webhook_id: + type: integer + format: int64 + webhook_secret: + type: string + last_synced_sha: + type: object + additionalProperties: + type: string + last_pull_status: + $ref: "#/components/schemas/AutoPullStatus" + required: + - enabled + MetricMetadata: type: object properties: diff --git a/backend/windmill-store/src/resources.rs b/backend/windmill-store/src/resources.rs index 85bb906715..a52ffbcf27 100644 --- a/backend/windmill-store/src/resources.rs +++ b/backend/windmill-store/src/resources.rs @@ -2578,6 +2578,71 @@ async fn get_repo_latest_commit_hash( Ok(commit_hash) } +/// Resolve a workspace git-sync repository and return its current head commit +/// for the tracked branch, for background auto-pull polling (no authed user). +/// +/// Returns `Ok(Some((ref_spec, sha)))` for a pollable repo, or `Ok(None)` for +/// repos that cannot be polled in-process — currently GitHub-App-backed repos, +/// which authenticate via an installation token at clone time and sync via +/// webhooks instead. Credentials embedded in the resource URL (including +/// `$var:` references) are resolved with the system identity, bypassing +/// per-user ACLs, since the poller runs without an authenticated request. +pub async fn get_git_repo_head_for_autopull( + db: &DB, + w_id: &str, + git_repo_resource_path: &str, +) -> Result> { + use windmill_common::db::DbWithOptAuthed; + + let resource_path = git_repo_resource_path + .strip_prefix("$res:") + .unwrap_or(git_repo_resource_path); + + let dba: DbWithOptAuthed<'_, ApiAuthed> = DbWithOptAuthed::DB { + db: db.clone(), + audit_author: windmill_common::audit::AuditAuthor { + username: "git_sync_auto_pull".to_string(), + email: windmill_common::users::SUPERADMIN_SYNC_EMAIL.to_string(), + username_override: None, + token_prefix: None, + }, + }; + + // allow_cache=true so repeated polls reuse the interpolated value instead of + // re-decrypting any `$var:` secret in the URL (and writing an audit row) on + // every tick. + let value = + get_resource_value_interpolated_internal(&dba, w_id, resource_path, None, None, true) + .await? + .ok_or_else(|| { + Error::BadRequest(format!( + "Git repository resource '{}' not found", + resource_path + )) + })?; + + if value + .get("is_github_app") + .and_then(|v| v.as_bool()) + .unwrap_or(false) + { + return Ok(None); + } + + let git_resource: GitRepositoryResource = serde_json::from_value(value) + .map_err(|e| Error::BadRequest(format!("Invalid git repository resource: {}", e)))?; + + let ref_spec = git_resource + .branch + .as_deref() + .filter(|s| !s.is_empty()) + .unwrap_or("HEAD") + .to_string(); + + let sha = get_repo_latest_commit_hash(&git_resource, None).await?; + Ok(Some((ref_spec, sha))) +} + #[cfg(all( feature = "enterprise", any(feature = "nats", feature = "kafka", feature = "sqs_trigger") diff --git a/frontend/src/lib/components/git_sync/GitSyncContext.svelte.ts b/frontend/src/lib/components/git_sync/GitSyncContext.svelte.ts index f17cdd23d4..a2c50c53cb 100644 --- a/frontend/src/lib/components/git_sync/GitSyncContext.svelte.ts +++ b/frontend/src/lib/components/git_sync/GitSyncContext.svelte.ts @@ -179,7 +179,8 @@ export function createGitSyncContext(workspace: string) { use_individual_branch: repo.use_individual_branch, group_by_folder: repo.group_by_folder, settings: repo.settings, - exclude_types_override: repo.exclude_types_override + exclude_types_override: repo.exclude_types_override, + auto_pull: repo.auto_pull } } @@ -501,7 +502,8 @@ export function createGitSyncContext(workspace: string) { use_individual_branch: repoToSave.use_individual_branch, group_by_folder: repoToSave.group_by_folder, settings: repoToSave.settings, - exclude_types_override: repoToSave.exclude_types_override + exclude_types_override: repoToSave.exclude_types_override, + auto_pull: repoToSave.auto_pull } } }) diff --git a/frontend/src/lib/components/git_sync/GitSyncRepositoryCard.svelte b/frontend/src/lib/components/git_sync/GitSyncRepositoryCard.svelte index ba5dbf4a03..7195060889 100644 --- a/frontend/src/lib/components/git_sync/GitSyncRepositoryCard.svelte +++ b/frontend/src/lib/components/git_sync/GitSyncRepositoryCard.svelte @@ -20,6 +20,7 @@ import { workspaceStore } from '$lib/stores' import type { GitSyncRepository } from './GitSyncContext.svelte' import GitSyncModeDisplay from './GitSyncModeDisplay.svelte' + import Toggle from '$lib/components/Toggle.svelte' import { ResourceService, VariableService } from '$lib/gen' let { @@ -49,6 +50,22 @@ const validation = $derived(idx !== null ? gitSyncContext.getValidation(idx) : null) const gitSyncTestJob = $derived(idx !== null ? gitSyncContext.gitSyncTestJobs?.[idx] : null) let confirmingDelete = $state(false) + + // Enable/disable automatic repo → workspace pulls, managing the optional + // auto_pull object without binding into a possibly-undefined value. + function setAutoPullEnabled(enabled: boolean) { + if (!repo) return + if (enabled) { + repo.auto_pull = { + ...(repo.auto_pull ?? {}), + enabled: true, + mode: repo.auto_pull?.mode ?? 'auto' + } + } else if (repo.auto_pull) { + repo.auto_pull = { ...repo.auto_pull, enabled: false } + } + } + let targetBranch = $state(undefined) // Default to main, will be updated when resource is available let resourceInfo = $state<{ url?: string; error?: string } | null>(null) let loadingResourceInfo = $state(false) @@ -517,6 +534,41 @@ {/if} + + + {#if !emptyString(repo.git_repo_resource_path) && !repo.legacyImported} +
+ setAutoPullEnabled(e.detail)} + /> + {#if repo.auto_pull?.enabled} +
+ {#if repo.auto_pull?.last_pull_status} + {#if repo.auto_pull.last_pull_status.success} + Last synced{repo.auto_pull.last_pull_status.synced_sha + ? ` to ${repo.auto_pull.last_pull_status.synced_sha.slice(0, 7)}` + : ''}. Windmill checks the tracked branch about every minute. + {:else} + + Last sync failed{repo.auto_pull.last_pull_status.error + ? `: ${repo.auto_pull.last_pull_status.error}` + : ''}. + + {/if} + {:else} + Windmill checks the tracked branch about every minute and deploys new commits + automatically. + {/if} +
+ {/if} +
+ {/if} {/if} {/if} {:else} diff --git a/frontend/src/lib/components/git_sync/GitSyncSuccessModal.svelte b/frontend/src/lib/components/git_sync/GitSyncSuccessModal.svelte index 5beddd05c6..266d75ba80 100644 --- a/frontend/src/lib/components/git_sync/GitSyncSuccessModal.svelte +++ b/frontend/src/lib/components/git_sync/GitSyncSuccessModal.svelte @@ -7,11 +7,7 @@ savedWithoutInit?: boolean } - let { - open = $bindable(false), - savedWithoutInit = false - }: Props = $props() - + let { open = $bindable(false), savedWithoutInit = false }: Props = $props() @@ -23,7 +19,9 @@

Git sync connection saved successfully!

-

Your repository is now configured to receive changes from Windmill.

+

Your repository is now configured to receive changes from Windmill.

@@ -32,7 +30,8 @@

Repository saved without initialization

- Only new changes will be pushed to this repository. Existing content in Windmill has not been initialized to the repository. + Only new changes will be pushed to this repository. Existing content in Windmill has not + been initialized to the repository.

{/if} @@ -41,17 +40,17 @@

- Optional: Enable automatic deployment from Git to Windmill + Deploy changes from Git back to Windmill

- To automatically deploy changes from your Git repository back to Windmill (when PRs are merged), you can set up GitHub Actions or similar CI/CD workflows. + Turn on "Automatically deploy changes from Git" on the repository to have Windmill pull new + commits into this workspace for you.

-

This setup enables:

-
    -
  • Automatic deployment to Windmill when PRs are merged
    • -
  • Full bidirectional sync between Git and Windmill
    • -
+

+ Prefer to control deployment from your own pipeline (tests, custom gating, deploy on PR + merge)? Set up GitHub Actions or similar CI/CD workflows instead. +

- Learn more + Learn more about CI-based deployment
-