feat(git-sync): poll repos and auto-pull new commits into the workspace

Phase 1 of automatic repo → Windmill sync. A monitor task (EE-licensed,
single-replica via advisory lock) git ls-remotes each auto-pull-enabled
repository ~every minute and enqueues a pull when the tracked branch moves,
reusing the {workspace_id}:git_sync concurrency key so pulls serialize with
in-flight push commits.

- windmill-store: background (no-authed) resolver get_git_repo_head_for_autopull
  that resolves the repo resource (incl. $var: refs) and ls-remotes; GitHub-App
  repos are skipped here and will sync via webhooks (phase 2).
- monitor.rs: poll/reconcile/persist with optimistic sha advance and failure
  status; targeted jsonb update so concurrent settings edits aren't clobbered.
- edit_git_sync_repository: preserve server-owned auto_pull state on UI save.
- openapi: AutoPullSettings/AutoPullMode/AutoPullStatus + auto_pull field.
- frontend: per-repo "Automatically deploy changes from Git" toggle with last
  sync status; demote the GitHub Actions link to an advanced CI option.

Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>
This commit is contained in:
hugocasa
2026-06-25 19:09:36 +02:00
parent 6f291cd8d4
commit 5c75a2f809
11 changed files with 507 additions and 20 deletions
@@ -0,0 +1,17 @@
{
"db_name": "PostgreSQL",
"query": "\n UPDATE workspace_settings\n SET git_sync = jsonb_set(\n git_sync,\n '{repositories}',\n (SELECT jsonb_agg(\n CASE WHEN elem->>'git_repo_resource_path' = $2\n THEN jsonb_set(\n jsonb_set(elem, '{auto_pull,last_synced_sha}', $3, true),\n '{auto_pull,last_pull_status}', $4, true)\n ELSE elem END)\n FROM jsonb_array_elements(git_sync->'repositories') AS elem)\n )\n WHERE workspace_id = $1\n ",
"describe": {
"columns": [],
"parameters": {
"Left": [
"Text",
"Text",
"Jsonb",
"Jsonb"
]
},
"nullable": []
},
"hash": "206a65cd2f0da413eb7f5873f696aa3562cf23fa48c479bcba51dd421813a291"
}
@@ -0,0 +1,26 @@
{
"db_name": "PostgreSQL",
"query": "SELECT workspace_id, git_sync\n FROM workspace_settings\n WHERE git_sync IS NOT NULL\n AND git_sync->'repositories' @> '[{\"auto_pull\": {\"enabled\": true}}]'::jsonb",
"describe": {
"columns": [
{
"ordinal": 0,
"name": "workspace_id",
"type_info": "Varchar"
},
{
"ordinal": 1,
"name": "git_sync",
"type_info": "Jsonb"
}
],
"parameters": {
"Left": []
},
"nullable": [
false,
true
]
},
"hash": "ec2b685c6116ac69a97c2e8b8354b572edc96014012109cac3be54627337eb7c"
}
+1
View File
@@ -13845,6 +13845,7 @@ dependencies = [
"windmill-parser-ts",
"windmill-queue",
"windmill-runtime-nativets",
"windmill-store",
"windmill-test-utils",
"windmill-trigger",
"windmill-trigger-azure",
+1
View File
@@ -249,6 +249,7 @@ windmill-queue.workspace = true
windmill-common = { workspace = true, default-features = false }
windmill-object-store.workspace = true
windmill-git-sync.workspace = true
windmill-store.workspace = true
windmill-api = { workspace = true, default-features = false }
windmill-api-agent-workers = { workspace = true, optional = true }
windmill-api-scripts.workspace = true
+258
View File
@@ -2885,6 +2885,17 @@ pub async fn monitor_db(
}
};
// Poll git-sync repositories for new commits and pull them into the
// workspace (repo → Windmill auto-pull). Runs every 2 iterations (~60s).
let git_auto_pull_f = async {
#[cfg(feature = "private")]
if server_mode && iteration.is_some() && iteration.as_ref().unwrap().should_run(2) {
if let Some(db) = conn.as_sql() {
poll_git_auto_pull(db).await;
}
}
};
join!(
expired_items_f,
zombie_jobs_f,
@@ -2912,9 +2923,256 @@ pub async fn monitor_db(
manage_audit_partitions_f,
export_audit_logs_to_object_store_f,
cleanup_scheduled_job_deletions_f,
git_auto_pull_f,
);
}
/// Advisory lock id ensuring only one server replica runs the git auto-pull
/// poll at a time (adjacent to RESTART_LOCK_ID used for restart coordination).
#[cfg(feature = "private")]
const GIT_AUTO_PULL_LOCK_ID: i64 = 737_483_921;
/// Poll every git-sync repository with auto-pull enabled and enqueue a pull when
/// the tracked branch has new commits (repo → Windmill direction).
///
/// Runs on a single replica at a time (advisory lock) and only on
/// Enterprise-licensed instances. Detection is `git ls-remote`; GitHub-App
/// repositories are skipped here and sync via webhooks instead (phase 2).
#[cfg(feature = "private")]
pub async fn poll_git_auto_pull(db: &Pool<Postgres>) {
use windmill_common::ee_oss::{get_license_plan, LicensePlan};
if !matches!(get_license_plan().await, LicensePlan::Enterprise) {
return;
}
let mut lock_conn = match db.acquire().await {
Ok(c) => c,
Err(e) => {
tracing::error!("git auto-pull: failed to acquire connection: {e:#}");
return;
}
};
let locked: bool = match sqlx::query_scalar("SELECT pg_try_advisory_lock($1)")
.bind(GIT_AUTO_PULL_LOCK_ID)
.fetch_one(&mut *lock_conn)
.await
{
Ok(v) => v,
Err(e) => {
tracing::error!("git auto-pull: advisory lock failed: {e:#}");
return;
}
};
if !locked {
// Another replica is already polling this tick.
return;
}
if let Err(e) = poll_git_auto_pull_inner(db).await {
tracing::error!("git auto-pull: poll error: {e:#}");
}
if let Err(e) = sqlx::query("SELECT pg_advisory_unlock($1)")
.bind(GIT_AUTO_PULL_LOCK_ID)
.execute(&mut *lock_conn)
.await
{
tracing::error!("git auto-pull: advisory unlock failed: {e:#}");
}
}
#[cfg(feature = "private")]
async fn poll_git_auto_pull_inner(db: &Pool<Postgres>) -> error::Result<()> {
use windmill_common::workspaces::{AutoPullMode, AutoPullStatus, WorkspaceGitSyncSettings};
let rows = sqlx::query!(
r#"SELECT workspace_id, git_sync
FROM workspace_settings
WHERE git_sync IS NOT NULL
AND git_sync->'repositories' @> '[{"auto_pull": {"enabled": true}}]'::jsonb"#
)
.fetch_all(db)
.await?;
for row in rows {
let Some(git_sync) = row.git_sync else {
continue;
};
let settings: WorkspaceGitSyncSettings = match serde_json::from_value(git_sync) {
Ok(s) => s,
Err(e) => {
tracing::warn!(
"git auto-pull: invalid git_sync settings for workspace {}: {e}",
row.workspace_id
);
continue;
}
};
for repo in &settings.repositories {
let Some(auto_pull) = &repo.auto_pull else {
continue;
};
if !auto_pull.enabled || auto_pull.mode == AutoPullMode::Webhook {
continue;
}
match windmill_store::resources::get_git_repo_head_for_autopull(
db,
&row.workspace_id,
&repo.git_repo_resource_path,
)
.await
{
Ok(Some((git_ref, sha))) => {
if !auto_pull.should_pull(&git_ref, &sha) {
continue;
}
let use_promotion_overrides = repo.use_individual_branch.unwrap_or(false);
match windmill_git_sync::enqueue_git_pull_job(
db,
&row.workspace_id,
repo,
None,
use_promotion_overrides,
)
.await
{
Ok(job_id) => {
// Optimistically advance the synced sha so the next
// tick doesn't re-enqueue the same commit; the job id
// lets the user inspect the actual pull outcome.
let mut synced = auto_pull.last_synced_sha.clone();
synced.insert(git_ref.clone(), sha.clone());
let status = AutoPullStatus {
synced_sha: Some(sha.clone()),
at: chrono::Utc::now().timestamp(),
job_id: Some(job_id),
success: true,
error: None,
};
if let Err(e) = persist_auto_pull_state(
db,
&row.workspace_id,
&repo.git_repo_resource_path,
&synced,
&status,
)
.await
{
tracing::error!(
"git auto-pull: failed to persist state for {}/{}: {e:#}",
row.workspace_id,
repo.git_repo_resource_path
);
}
tracing::info!(
"git auto-pull: enqueued pull {job_id} for {}/{} at {sha}",
row.workspace_id,
repo.git_repo_resource_path
);
}
Err(e) => {
record_auto_pull_failure(
db,
&row.workspace_id,
&repo.git_repo_resource_path,
&auto_pull.last_synced_sha,
format!("failed to enqueue pull: {e}"),
)
.await;
}
}
}
// GitHub-App repo: synced via webhook (phase 2), nothing to poll.
Ok(None) => {}
Err(e) => {
record_auto_pull_failure(
db,
&row.workspace_id,
&repo.git_repo_resource_path,
&auto_pull.last_synced_sha,
format!("ls-remote failed: {e}"),
)
.await;
}
}
}
}
Ok(())
}
/// Update the `auto_pull.last_synced_sha` and `auto_pull.last_pull_status` of a
/// single repository in `workspace_settings.git_sync`, leaving every other
/// field (and the other repositories) untouched so concurrent settings edits
/// are not clobbered.
#[cfg(feature = "private")]
async fn persist_auto_pull_state(
db: &Pool<Postgres>,
w_id: &str,
repo_resource_path: &str,
last_synced_sha: &std::collections::HashMap<String, String>,
status: &windmill_common::workspaces::AutoPullStatus,
) -> error::Result<()> {
let synced_json = serde_json::to_value(last_synced_sha)
.map_err(|e| error::Error::internal_err(e.to_string()))?;
let status_json =
serde_json::to_value(status).map_err(|e| error::Error::internal_err(e.to_string()))?;
sqlx::query!(
r#"
UPDATE workspace_settings
SET git_sync = jsonb_set(
git_sync,
'{repositories}',
(SELECT jsonb_agg(
CASE WHEN elem->>'git_repo_resource_path' = $2
THEN jsonb_set(
jsonb_set(elem, '{auto_pull,last_synced_sha}', $3, true),
'{auto_pull,last_pull_status}', $4, true)
ELSE elem END)
FROM jsonb_array_elements(git_sync->'repositories') AS elem)
)
WHERE workspace_id = $1
"#,
w_id,
repo_resource_path,
synced_json,
status_json,
)
.execute(db)
.await?;
Ok(())
}
/// Record a failed auto-pull attempt in the repository's status without
/// advancing the synced sha, so the failure is visible and the next tick retries.
#[cfg(feature = "private")]
async fn record_auto_pull_failure(
db: &Pool<Postgres>,
w_id: &str,
repo_resource_path: &str,
last_synced_sha: &std::collections::HashMap<String, String>,
error_msg: String,
) {
tracing::warn!("git auto-pull: {error_msg} for {w_id}/{repo_resource_path}");
let status = windmill_common::workspaces::AutoPullStatus {
synced_sha: None,
at: chrono::Utc::now().timestamp(),
job_id: None,
success: false,
error: Some(error_msg),
};
if let Err(e) =
persist_auto_pull_state(db, w_id, repo_resource_path, last_synced_sha, &status).await
{
tracing::error!(
"git auto-pull: failed to record failure for {w_id}/{repo_resource_path}: {e:#}"
);
}
}
async fn vacuuming_tables(db: &Pool<Postgres>) -> error::Result<()> {
sqlx::query!("VACUUM v2_job, v2_job_completed, job_result_stream_v2, job_stats, job_logs, job_perms, concurrency_key, log_file, metrics")
.execute(db)
@@ -2696,8 +2696,24 @@ async fn edit_git_sync_repository(
.find(|repo| repo.git_repo_resource_path == new_config.git_repo_resource_path);
if let Some(existing_repo) = repo_found {
// Update existing repository
*existing_repo = new_config.repository;
// Update existing repository, but preserve server-owned auto-pull state
// (synced sha, last pull status, webhook id/secret) so a settings save
// from the UI cannot revert what the poller/webhook layer wrote.
let mut updated = new_config.repository;
match (updated.auto_pull.as_mut(), existing_repo.auto_pull.as_ref()) {
(Some(new_ap), Some(old_ap)) => {
new_ap.last_synced_sha = old_ap.last_synced_sha.clone();
new_ap.last_pull_status = old_ap.last_pull_status.clone();
new_ap.webhook_id = old_ap.webhook_id;
new_ap.webhook_secret = old_ap.webhook_secret.clone();
}
// UI omitted auto_pull (e.g. older client): keep existing config.
(None, Some(_)) => {
updated.auto_pull = existing_repo.auto_pull.clone();
}
_ => {}
}
*existing_repo = updated;
} else {
// Repository doesn't exist, add it as a new repository
git_sync_settings.repositories.push(new_config.repository);
+51
View File
@@ -27616,9 +27616,60 @@ components:
type: array
items:
$ref: "#/components/schemas/GitSyncObjectType"
auto_pull:
$ref: "#/components/schemas/AutoPullSettings"
required:
- git_repo_resource_path
AutoPullMode:
type: string
enum:
- auto
- webhook
- polling
AutoPullStatus:
type: object
properties:
synced_sha:
type: string
at:
type: integer
format: int64
job_id:
type: string
format: uuid
success:
type: boolean
error:
type: string
required:
- at
- success
AutoPullSettings:
type: object
properties:
enabled:
type: boolean
mode:
$ref: "#/components/schemas/AutoPullMode"
poll_interval_s:
type: integer
webhook_id:
type: integer
format: int64
webhook_secret:
type: string
last_synced_sha:
type: object
additionalProperties:
type: string
last_pull_status:
$ref: "#/components/schemas/AutoPullStatus"
required:
- enabled
MetricMetadata:
type: object
properties:
+65
View File
@@ -2578,6 +2578,71 @@ async fn get_repo_latest_commit_hash(
Ok(commit_hash)
}
/// Resolve a workspace git-sync repository and return its current head commit
/// for the tracked branch, for background auto-pull polling (no authed user).
///
/// Returns `Ok(Some((ref_spec, sha)))` for a pollable repo, or `Ok(None)` for
/// repos that cannot be polled in-process — currently GitHub-App-backed repos,
/// which authenticate via an installation token at clone time and sync via
/// webhooks instead. Credentials embedded in the resource URL (including
/// `$var:` references) are resolved with the system identity, bypassing
/// per-user ACLs, since the poller runs without an authenticated request.
pub async fn get_git_repo_head_for_autopull(
db: &DB,
w_id: &str,
git_repo_resource_path: &str,
) -> Result<Option<(String, String)>> {
use windmill_common::db::DbWithOptAuthed;
let resource_path = git_repo_resource_path
.strip_prefix("$res:")
.unwrap_or(git_repo_resource_path);
let dba: DbWithOptAuthed<'_, ApiAuthed> = DbWithOptAuthed::DB {
db: db.clone(),
audit_author: windmill_common::audit::AuditAuthor {
username: "git_sync_auto_pull".to_string(),
email: windmill_common::users::SUPERADMIN_SYNC_EMAIL.to_string(),
username_override: None,
token_prefix: None,
},
};
// allow_cache=true so repeated polls reuse the interpolated value instead of
// re-decrypting any `$var:` secret in the URL (and writing an audit row) on
// every tick.
let value =
get_resource_value_interpolated_internal(&dba, w_id, resource_path, None, None, true)
.await?
.ok_or_else(|| {
Error::BadRequest(format!(
"Git repository resource '{}' not found",
resource_path
))
})?;
if value
.get("is_github_app")
.and_then(|v| v.as_bool())
.unwrap_or(false)
{
return Ok(None);
}
let git_resource: GitRepositoryResource = serde_json::from_value(value)
.map_err(|e| Error::BadRequest(format!("Invalid git repository resource: {}", e)))?;
let ref_spec = git_resource
.branch
.as_deref()
.filter(|s| !s.is_empty())
.unwrap_or("HEAD")
.to_string();
let sha = get_repo_latest_commit_hash(&git_resource, None).await?;
Ok(Some((ref_spec, sha)))
}
#[cfg(all(
feature = "enterprise",
any(feature = "nats", feature = "kafka", feature = "sqs_trigger")
@@ -179,7 +179,8 @@ export function createGitSyncContext(workspace: string) {
use_individual_branch: repo.use_individual_branch,
group_by_folder: repo.group_by_folder,
settings: repo.settings,
exclude_types_override: repo.exclude_types_override
exclude_types_override: repo.exclude_types_override,
auto_pull: repo.auto_pull
}
}
@@ -501,7 +502,8 @@ export function createGitSyncContext(workspace: string) {
use_individual_branch: repoToSave.use_individual_branch,
group_by_folder: repoToSave.group_by_folder,
settings: repoToSave.settings,
exclude_types_override: repoToSave.exclude_types_override
exclude_types_override: repoToSave.exclude_types_override,
auto_pull: repoToSave.auto_pull
}
}
})
@@ -20,6 +20,7 @@
import { workspaceStore } from '$lib/stores'
import type { GitSyncRepository } from './GitSyncContext.svelte'
import GitSyncModeDisplay from './GitSyncModeDisplay.svelte'
import Toggle from '$lib/components/Toggle.svelte'
import { ResourceService, VariableService } from '$lib/gen'
let {
@@ -49,6 +50,22 @@
const validation = $derived(idx !== null ? gitSyncContext.getValidation(idx) : null)
const gitSyncTestJob = $derived(idx !== null ? gitSyncContext.gitSyncTestJobs?.[idx] : null)
let confirmingDelete = $state(false)
// Enable/disable automatic repo → workspace pulls, managing the optional
// auto_pull object without binding into a possibly-undefined value.
function setAutoPullEnabled(enabled: boolean) {
if (!repo) return
if (enabled) {
repo.auto_pull = {
...(repo.auto_pull ?? {}),
enabled: true,
mode: repo.auto_pull?.mode ?? 'auto'
}
} else if (repo.auto_pull) {
repo.auto_pull = { ...repo.auto_pull, enabled: false }
}
}
let targetBranch = $state<string | undefined>(undefined) // Default to main, will be updated when resource is available
let resourceInfo = $state<{ url?: string; error?: string } | null>(null)
let loadingResourceInfo = $state(false)
@@ -517,6 +534,41 @@
</div>
{/if}
</div>
<!-- Automatic deployment from Git (repo to workspace) -->
{#if !emptyString(repo.git_repo_resource_path) && !repo.legacyImported}
<div class="mt-4 border-t border-gray-200 pt-3 dark:border-gray-700">
<Toggle
checked={repo.auto_pull?.enabled ?? false}
options={{
right: 'Automatically deploy changes from Git',
rightTooltip:
'Windmill periodically checks the tracked branch and pulls new commits into this workspace. Repositories connected through the GitHub App will sync via webhooks in a future update.'
}}
on:change={(e) => setAutoPullEnabled(e.detail)}
/>
{#if repo.auto_pull?.enabled}
<div class="text-2xs text-secondary mt-2">
{#if repo.auto_pull?.last_pull_status}
{#if repo.auto_pull.last_pull_status.success}
Last synced{repo.auto_pull.last_pull_status.synced_sha
? ` to ${repo.auto_pull.last_pull_status.synced_sha.slice(0, 7)}`
: ''}. Windmill checks the tracked branch about every minute.
{:else}
<span class="text-red-600 dark:text-red-400">
Last sync failed{repo.auto_pull.last_pull_status.error
? `: ${repo.auto_pull.last_pull_status.error}`
: ''}.
</span>
{/if}
{:else}
Windmill checks the tracked branch about every minute and deploys new commits
automatically.
{/if}
</div>
{/if}
</div>
{/if}
{/if}
{/if}
{:else}
@@ -7,11 +7,7 @@
savedWithoutInit?: boolean
}
let {
open = $bindable(false),
savedWithoutInit = false
}: Props = $props()
let { open = $bindable(false), savedWithoutInit = false }: Props = $props()
</script>
<Modal bind:open title="Git Sync Connection Saved" class="sm:max-w-4xl" cancelText="Close">
@@ -23,7 +19,9 @@
</div>
<div>
<h3 class="text-lg font-semibold text-primary">Git sync connection saved successfully!</h3>
<p class="text-sm text-secondary mt-1">Your repository is now configured to receive changes from Windmill.</p>
<p class="text-sm text-secondary mt-1"
>Your repository is now configured to receive changes from Windmill.</p
>
</div>
</div>
@@ -32,7 +30,8 @@
<div class="bg-blue-50 border border-blue-200 rounded-lg p-4">
<h4 class="font-medium text-blue-900 mb-2">Repository saved without initialization</h4>
<p class="text-sm text-blue-800">
Only new changes will be pushed to this repository. Existing content in Windmill has not been initialized to the repository.
Only new changes will be pushed to this repository. Existing content in Windmill has not
been initialized to the repository.
</p>
</div>
{/if}
@@ -41,17 +40,17 @@
<div class="bg-amber-50 border border-amber-200 rounded-lg p-4">
<h4 class="font-medium text-amber-900 mb-2 flex items-center gap-2">
<ArrowRight class="h-4 w-4" />
Optional: Enable automatic deployment from Git to Windmill
Deploy changes from Git back to Windmill
</h4>
<p class="text-sm text-amber-800 mb-3">
To automatically deploy changes from your Git repository back to Windmill (when PRs are merged), you can set up GitHub Actions or similar CI/CD workflows.
Turn on "Automatically deploy changes from Git" on the repository to have Windmill pull new
commits into this workspace for you.
</p>
<div class="flex flex-col gap-2">
<p class="text-sm text-amber-700">This setup enables:</p>
<ul class="text-sm text-amber-700 ml-4 list-disc space-y-1">
<li>Automatic deployment to Windmill when PRs are merged</li>
<li>Full bidirectional sync between Git and Windmill</li>
</ul>
<p class="text-sm text-amber-700">
Prefer to control deployment from your own pipeline (tests, custom gating, deploy on PR
merge)? Set up GitHub Actions or similar CI/CD workflows instead.
</p>
<div class="mt-3">
<a
href="https://www.windmill.dev/docs/advanced/deploy_gh_gl#github-actions-setup"
@@ -59,11 +58,10 @@
class="text-sm text-amber-700 hover:text-amber-900 underline flex items-center gap-1"
>
<ExternalLink class="h-3 w-3" />
Learn more
Learn more about CI-based deployment
</a>
</div>
</div>
</div>
</div>
</Modal>