From 900d48272173ab85f7ea1c3214d65ecfd6d97233 Mon Sep 17 00:00:00 2001 From: hugocasa Date: Thu, 3 Sep 2026 11:05:58 +0200 Subject: [PATCH] feat: open gitlab merge requests and post diff previews on them Co-Authored-By: Claude Opus 5 Claude-Session: https://claude.ai/code/session_01C1xHmkxuxYb1GYvth1BS75 --- ...d3f9e80375b52748c458854a4883a6ecc8f6.json} | 5 +- ...8c9fe4348d179add37361353fc9a02d96f2fa.json | 16 -- ...496c9448cffba5786b51e748bec0165a6519c.json | 17 ++ backend/ee-repo-ref.txt | 2 +- backend/windmill-api/openapi.yaml | 57 +++++ backend/windmill-api/src/git_sync_oss.rs | 5 + .../windmill-worker/src/result_processor.rs | 24 +- docs/git-sync-gitlab-setup.md | 99 ++++++++ .../lib/components/GitLabIntegration.svelte | 221 ++++++++++++++++++ .../src/lib/components/ResourceForm.svelte | 19 +- .../git_sync/GitSyncRepositoryCard.svelte | 54 +++-- 11 files changed, 475 insertions(+), 44 deletions(-) rename backend/.sqlx/{query-55a272a0050115f90b4fd0de1350ecdbda3fe30cc50d68dc5dfd357113f412ac.json => query-48055203c97499ab4fc4dcbe9271d3f9e80375b52748c458854a4883a6ecc8f6.json} (78%) delete mode 100644 backend/.sqlx/query-81e7f902c40d72937b494cd9f3f8c9fe4348d179add37361353fc9a02d96f2fa.json create mode 100644 backend/.sqlx/query-bbb8331348216892e4714e4338c496c9448cffba5786b51e748bec0165a6519c.json create mode 100644 docs/git-sync-gitlab-setup.md create mode 100644 frontend/src/lib/components/GitLabIntegration.svelte diff --git a/backend/.sqlx/query-55a272a0050115f90b4fd0de1350ecdbda3fe30cc50d68dc5dfd357113f412ac.json b/backend/.sqlx/query-48055203c97499ab4fc4dcbe9271d3f9e80375b52748c458854a4883a6ecc8f6.json similarity index 78% rename from backend/.sqlx/query-55a272a0050115f90b4fd0de1350ecdbda3fe30cc50d68dc5dfd357113f412ac.json rename to backend/.sqlx/query-48055203c97499ab4fc4dcbe9271d3f9e80375b52748c458854a4883a6ecc8f6.json index da8ccd58ce..b2cfb1cd89 100644 --- a/backend/.sqlx/query-55a272a0050115f90b4fd0de1350ecdbda3fe30cc50d68dc5dfd357113f412ac.json +++ b/backend/.sqlx/query-48055203c97499ab4fc4dcbe9271d3f9e80375b52748c458854a4883a6ecc8f6.json @@ -1,6 +1,6 @@ { "db_name": "PostgreSQL", - "query": "\n SELECT elem->'credential'\n FROM workspace_settings, jsonb_array_elements(git_sync->'repositories') AS elem\n WHERE workspace_id = $1 AND elem->>'git_repo_resource_path' = $2\n ", + "query": "\n SELECT elem->'credential'\n FROM workspace_settings, jsonb_array_elements(git_sync->'repositories') AS elem\n WHERE workspace_id = $1 AND elem->>'git_repo_resource_path' IN ($2, $3)\n ", "describe": { "columns": [ { @@ -11,6 +11,7 @@ ], "parameters": { "Left": [ + "Text", "Text", "Text" ] @@ -19,5 +20,5 @@ null ] }, - "hash": "55a272a0050115f90b4fd0de1350ecdbda3fe30cc50d68dc5dfd357113f412ac" + "hash": "48055203c97499ab4fc4dcbe9271d3f9e80375b52748c458854a4883a6ecc8f6" } diff --git a/backend/.sqlx/query-81e7f902c40d72937b494cd9f3f8c9fe4348d179add37361353fc9a02d96f2fa.json b/backend/.sqlx/query-81e7f902c40d72937b494cd9f3f8c9fe4348d179add37361353fc9a02d96f2fa.json deleted file mode 100644 index b1c00201f9..0000000000 --- a/backend/.sqlx/query-81e7f902c40d72937b494cd9f3f8c9fe4348d179add37361353fc9a02d96f2fa.json +++ /dev/null @@ -1,16 +0,0 @@ -{ - "db_name": "PostgreSQL", - "query": "\n UPDATE workspace_settings\n SET git_sync = jsonb_set(git_sync, '{repositories}',\n COALESCE((SELECT jsonb_agg(\n CASE WHEN elem->>'git_repo_resource_path' = $2\n THEN CASE WHEN $3::jsonb = 'null'::jsonb\n THEN elem - 'credential'\n ELSE jsonb_set(elem, '{credential}', $3) END\n ELSE elem END)\n FROM jsonb_array_elements(git_sync->'repositories') AS elem), '[]'::jsonb)\n )\n WHERE workspace_id = $1\n AND jsonb_typeof(git_sync->'repositories') = 'array'\n AND EXISTS (\n SELECT 1 FROM jsonb_array_elements(git_sync->'repositories') AS e\n WHERE e->>'git_repo_resource_path' = $2\n )\n ", - "describe": { - "columns": [], - "parameters": { - "Left": [ - "Text", - "Text", - "Jsonb" - ] - }, - "nullable": [] - }, - "hash": "81e7f902c40d72937b494cd9f3f8c9fe4348d179add37361353fc9a02d96f2fa" -} diff --git a/backend/.sqlx/query-bbb8331348216892e4714e4338c496c9448cffba5786b51e748bec0165a6519c.json b/backend/.sqlx/query-bbb8331348216892e4714e4338c496c9448cffba5786b51e748bec0165a6519c.json new file mode 100644 index 0000000000..980ba7a36c --- /dev/null +++ b/backend/.sqlx/query-bbb8331348216892e4714e4338c496c9448cffba5786b51e748bec0165a6519c.json @@ -0,0 +1,17 @@ +{ + "db_name": "PostgreSQL", + "query": "\n UPDATE workspace_settings\n SET git_sync = jsonb_set(git_sync, '{repositories}',\n COALESCE((SELECT jsonb_agg(\n CASE WHEN elem->>'git_repo_resource_path' IN ($2, $3)\n THEN CASE WHEN $4::jsonb = 'null'::jsonb\n THEN elem - 'credential'\n ELSE jsonb_set(elem, '{credential}', $4) END\n ELSE elem END)\n FROM jsonb_array_elements(git_sync->'repositories') AS elem), '[]'::jsonb)\n )\n WHERE workspace_id = $1\n AND jsonb_typeof(git_sync->'repositories') = 'array'\n AND EXISTS (\n SELECT 1 FROM jsonb_array_elements(git_sync->'repositories') AS e\n WHERE e->>'git_repo_resource_path' IN ($2, $3)\n )\n ", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "Text", + "Text", + "Text", + "Jsonb" + ] + }, + "nullable": [] + }, + "hash": "bbb8331348216892e4714e4338c496c9448cffba5786b51e748bec0165a6519c" +} diff --git a/backend/ee-repo-ref.txt b/backend/ee-repo-ref.txt index 81ca8bb347..bb04640317 100644 --- a/backend/ee-repo-ref.txt +++ b/backend/ee-repo-ref.txt @@ -1 +1 @@ -9ac2ffdee130fcff7ed45d137d8606ae983da837 +f31743e1c131fddd4bc151153eedc0621dc3a478 diff --git a/backend/windmill-api/openapi.yaml b/backend/windmill-api/openapi.yaml index 849f1eb6eb..9c85754dae 100644 --- a/backend/windmill-api/openapi.yaml +++ b/backend/windmill-api/openapi.yaml @@ -2877,6 +2877,48 @@ paths: "200": description: Successfully imported the installation + /w/{workspace}/git_sync/gitlab/projects: + post: + tags: + - Git Sync + summary: List the GitLab projects a token can sync + description: >- + Lists the projects the supplied GitLab token can push to, so a git + repository resource can be filled in without hand-writing a project + path. The token is used for this call only and is never stored. + Requires workspace admin. + operationId: listGitlabProjects + parameters: + - $ref: "#/components/parameters/WorkspaceId" + requestBody: + required: true + content: + application/json: + schema: + type: object + properties: + base_url: + type: string + description: The GitLab instance, e.g. https://gitlab.com + token: + type: string + description: A group access token or group service account personal access token + search: + type: string + description: Narrow the list to projects matching this text + required: + - base_url + - token + responses: + "200": + description: the projects the token can sync + content: + application/json: + schema: + type: array + items: + $ref: "#/components/schemas/GitlabProject" + /w/{workspace}/github_app/ghes_installation_callback: post: summary: GHES installation callback @@ -34020,6 +34062,21 @@ components: - rotatable - checked_at + GitlabProject: + type: object + description: a GitLab project a token can sync, as the resource form needs it + properties: + path_with_namespace: + type: string + description: nested group path plus project name, which is also GitLab's project id + http_url_to_repo: + type: string + default_branch: + type: string + required: + - path_with_namespace + - http_url_to_repo + AutoPullMode: type: string enum: diff --git a/backend/windmill-api/src/git_sync_oss.rs b/backend/windmill-api/src/git_sync_oss.rs index 0451d88699..d1653d9aa9 100644 --- a/backend/windmill-api/src/git_sync_oss.rs +++ b/backend/windmill-api/src/git_sync_oss.rs @@ -10,6 +10,11 @@ pub fn workspaced_service() -> Router { Router::new() } +#[cfg(not(feature = "private"))] +pub fn workspaced_git_sync_service() -> Router { + Router::new() +} + #[cfg(not(feature = "private"))] pub fn global_service() -> Router { Router::new() diff --git a/backend/windmill-worker/src/result_processor.rs b/backend/windmill-worker/src/result_processor.rs index 647d569cbe..3749bf65ea 100644 --- a/backend/windmill-worker/src/result_processor.rs +++ b/backend/windmill-worker/src/result_processor.rs @@ -1162,18 +1162,18 @@ async fn maybe_open_git_sync_deploy_pr( }; // Base = the tracked branch (resource branch, else the repo default). Also - // acts as the app-backed gate: PR creation needs the installation token. - let base = match windmill_common::git_sync_ee::get_app_repo_head_for_autopull( + // acts as the gate: PR creation needs a credential the server itself holds. + let base = match windmill_common::git_sync_ee::managed_pr_base_branch( db, workspace_id, &repo_path, ) .await { - Ok(Some((branch, _))) => branch, + Ok(Some(branch)) => branch, Ok(None) => { tracing::warn!( - "git sync PR: repo {repo_path} in {workspace_id} has a PR-on-deploy toggle set but is not GitHub-App-backed; skipping (connect the repo through the GitHub App, or use the open-pr-on-commit workflow)" + "git sync PR: repo {repo_path} in {workspace_id} has a PR-on-deploy toggle set but the server holds no credential for it; skipping (connect the repo through the GitHub App or a GitLab token, or use the open-pr-on-commit workflow)" ); return; } @@ -1461,7 +1461,7 @@ async fn maybe_post_git_sync_check( ( "neutral", "Could not compute the deploy diff".to_string(), - "Windmill could not fetch this PR's head or enough history from GitHub to compute its merge with the base. Push again to re-run this check." + "Windmill could not fetch this PR's head, or enough history, to compute its merge with the base. Push again to re-run this check." .to_string(), ) } else if pr_check_error.is_some() { @@ -1520,11 +1520,23 @@ async fn maybe_post_git_sync_check( Some(url) => format!("{summary}\n\n[See the job in Windmill]({url})"), None => summary.clone(), }; + // GitLab has no way to address a commit status by id: the name it was + // posted under, together with the commit, is what identifies it. + let check_run = windmill_common::git_sync_ee::CheckRun { + id: check.check_run_id, + head_sha: check.head_sha.clone().unwrap_or_default(), + name: if is_deploy { + windmill_common::git_sync_ee::CHECK_NAME_DEPLOY + } else { + windmill_common::git_sync_ee::CHECK_NAME_DIFF + } + .to_string(), + }; if let Err(e) = windmill_common::git_sync_ee::update_check_run( db, workspace_id, &check.repo_url, - check.check_run_id, + &check_run, conclusion, &title, &check_summary, diff --git a/docs/git-sync-gitlab-setup.md b/docs/git-sync-gitlab-setup.md new file mode 100644 index 0000000000..9557da7a3e --- /dev/null +++ b/docs/git-sync-gitlab-setup.md @@ -0,0 +1,99 @@ +# Git sync with GitLab + +GitLab has no equivalent of a GitHub App, so there is nothing to install and no +consent screen. What Windmill needs instead is one credential you create in +GitLab and paste once. With it, a GitLab repository gets the same managed +features an app-backed GitHub repository has: instant pull over a webhook, merge +requests opened on deploy, and a diff preview posted onto the merge request. + +## The credential + +Create a **group access token** on the group that owns the project (Settings → +Access tokens), or a **group service account** and a personal access token for +it. Either one is a bot identity that outlives the person who created it, which +is what you want for a credential the instance uses unattended. + +| | | +| --- | --- | +| Scope | `api` | +| Role | Developer to push deploy branches; **Maintainer** to also manage the webhook and open merge requests | +| Expiry | Required for a group access token; a group service account PAT can be non-expiring on self-managed (see below) | + +The `api` scope is what makes the token rotatable, so Windmill can renew it +before it expires. A `write_repository`-only token can still push, but Windmill +cannot inspect or renew it and reports that in the workspace's git sync settings. + +## Connecting a repository + +In the resource form for a `git_repository` resource, use the **GitLab** button: +paste the instance URL and the token, pick a project from the list, and Windmill +stores the whole remote URL, credential included, in a **secret variable** and +points the resource at it (`"url": "$var:u/you/gitlab_group_project_url"`). + +The variable indirection is what makes renewal possible: when Windmill rotates +the token it rewrites that one variable, and everything referencing it keeps +working. A URL pasted directly into the resource also syncs, but nothing can +renew it. + +## Expiry and renewal + +Windmill reads `expires_at` from the token itself and shows it on the repository +in the workspace's git sync settings. Within three weeks of expiry it rotates the +token through GitLab's own `POST /personal_access_tokens/self/rotate`, writes the +replacement back to the variable, and verifies it. Only the token can rotate +itself, so a token without `api` (or `self_rotate`) is a permanent warning rather +than something Windmill can fix. + +Rotation is deliberately never retried. GitLab revokes the old token the instant +it issues the replacement, and presenting an already-rotated token to `/rotate` +again is treated as reuse: it revokes **the whole token family, including the +live replacement**. So a rotation that succeeded at GitLab but failed to persist +is surfaced as an error to act on, not retried. + +Non-expiring tokens are possible only for a **group service account PAT** on +self-managed, with `require_personal_access_token_expiry` turned off in the +instance's application settings. A group access token is always rejected without +an `expires_at`. + +## What each managed feature needs + +| Feature | Needs | +| --- | --- | +| Instant pull | A project hook Windmill creates, so Maintainer; and a Windmill base URL GitLab can reach | +| Merge requests on deploy | Developer, plus the `api` scope | +| Diff preview on a merge request | The project hook, plus permission to post commit statuses and merge request notes | + +Instant pull falls back to checking the tracked branch about every minute when +the hook cannot be created or delivered, so nothing silently stops syncing. + +## Self-managed differences + +**Webhooks to a private network are blocked by default.** GitLab refuses to +create a hook pointing at a private or local address until an administrator +enables *Allow requests to the local network from webhooks and integrations* +(Admin → Settings → Network → Outbound requests, +`allow_local_requests_from_web_hooks_and_services`). A Windmill instance on the +same private network as GitLab needs this; without it, hook creation fails with a +"blocked" error and the repository keeps polling. + +Everything else is identical: Windmill talks to `/api/v4` and needs +no inbound access of its own beyond the hook deliveries. + +## Commit statuses create a pipeline + +GitLab has no separate check-run concept. Windmill's `Windmill diff` and +`Windmill` statuses are **commit statuses**, and posting one creates an `external` +pipeline on the project. Two consequences: + +- A project with *Pipelines must succeed* set will not let a merge request merge + while a Windmill status is still running, and will block it if the status + failed. Windmill therefore always drives a status it created to a terminal + state, and reports an informational result (for example "3 changes to deploy") + as success rather than leaving it pending. +- `allow_failure` is ignored on the commit-status endpoint, so a Windmill status + cannot be made advisory. If you do not want it gating merges, turn the diff + preview off rather than expecting it to be non-blocking. + +A commit status carries only a name, a 255-character description and a link, so +the diff itself goes in a merge request note that Windmill keeps up to date, and +the status links to the job. diff --git a/frontend/src/lib/components/GitLabIntegration.svelte b/frontend/src/lib/components/GitLabIntegration.svelte new file mode 100644 index 0000000000..efcecb151f --- /dev/null +++ b/frontend/src/lib/components/GitLabIntegration.svelte @@ -0,0 +1,221 @@ + + +{#if show} + + {#snippet trigger()} + + {/snippet} + {#snippet content({ close })} +
+
+
+
GitLab instance
+ +
+
+
Group access token
+
+ Create it in the group that owns the project, with the api scope and at least the + Developer role. Maintainer also lets Windmill manage the webhook and merge requests. +
+ +
+ Used to list projects now, then stored in a secret variable +
+
+
+
Filter projects
+ +
+
+ +
+ {#if listError} + {listError} + {/if} + {#if projects.length > 0} +
+
Project
+