diff --git a/.github/workflows/git-sync-test.yml b/.github/workflows/git-sync-test.yml new file mode 100644 index 0000000000..09f69bb701 --- /dev/null +++ b/.github/workflows/git-sync-test.yml @@ -0,0 +1,209 @@ +name: Git Sync Integration Tests + +on: + workflow_dispatch: + push: + branches: [main] + paths: + - "backend/windmill-git-sync/**" + - "backend/windmill-api-integration-tests/tests/git_sync*" + - "backend/ee-repo-ref.txt" + - "integration_tests/test/git_sync_test.py" + - ".github/workflows/git-sync-test.yml" + pull_request: + types: [opened, synchronize, reopened] + paths: + - "backend/windmill-git-sync/**" + - "backend/windmill-api-integration-tests/tests/git_sync*" + - "backend/ee-repo-ref.txt" + - "integration_tests/test/git_sync_test.py" + - ".github/workflows/git-sync-test.yml" + +concurrency: + group: git-sync-test-${{ github.ref }} + cancel-in-progress: true + +jobs: + check-relevance: + runs-on: ubuntu-latest + outputs: + should_run: ${{ steps.check.outputs.should_run }} + steps: + - uses: actions/checkout@v4 + with: + fetch-depth: 0 + + - name: Check if git sync related files changed + id: check + env: + WINDMILL_EE_PRIVATE_ACCESS: ${{ secrets.WINDMILL_EE_PRIVATE_ACCESS }} + run: | + if [ "${{ github.event_name }}" = "pull_request" ]; then + BASE=${{ github.event.pull_request.base.sha }} + else + BASE=${{ github.event.before }} + fi + + CHANGED_FILES=$(git diff --name-only "$BASE"..HEAD 2>/dev/null || echo "") + echo "Changed files:" + echo "$CHANGED_FILES" + + # Direct git sync file changes — always relevant + if echo "$CHANGED_FILES" | grep -qE '^(backend/windmill-git-sync/|backend/windmill-api-integration-tests/tests/git_sync|integration_tests/test/git_sync|\.github/workflows/git-sync-test\.yml)'; then + echo "should_run=true" >> "$GITHUB_OUTPUT" + echo "Relevant: direct git sync file changes" + exit 0 + fi + + # If ee-repo-ref.txt changed, check if the EE diff touches windmill-git-sync/ + if echo "$CHANGED_FILES" | grep -q '^backend/ee-repo-ref.txt$'; then + NEW_REF=$(cat backend/ee-repo-ref.txt) + OLD_REF=$(git show "$BASE:backend/ee-repo-ref.txt" 2>/dev/null || echo "") + + if [ -n "$OLD_REF" ] && [ "$OLD_REF" != "$NEW_REF" ]; then + # Clone EE repo and check diff + git clone --bare "https://x-access-token:${WINDMILL_EE_PRIVATE_ACCESS}@github.com/windmill-labs/windmill-ee-private.git" /tmp/ee-repo 2>/dev/null + EE_CHANGED=$(git -C /tmp/ee-repo diff --name-only "$OLD_REF".."$NEW_REF" 2>/dev/null || echo "") + echo "EE changed files:" + echo "$EE_CHANGED" + + if echo "$EE_CHANGED" | grep -q '^windmill-git-sync/'; then + echo "should_run=true" >> "$GITHUB_OUTPUT" + echo "Relevant: EE git sync files changed" + exit 0 + fi + fi + fi + + echo "should_run=false" >> "$GITHUB_OUTPUT" + echo "No git sync relevant changes detected, skipping tests" + + git_sync_e2e: + needs: [check-relevance] + if: needs.check-relevance.outputs.should_run == 'true' + runs-on: ubicloud-standard-16 + services: + postgres: + image: postgres:14 + ports: + - 5432:5432 + env: + POSTGRES_DB: windmill + POSTGRES_PASSWORD: changeme + options: >- + --health-cmd pg_isready --health-interval 10s --health-timeout 5s + --health-retries 5 + + steps: + - uses: actions/checkout@v4 + with: + ref: ${{ github.ref }} + fetch-depth: 0 + + - name: Read EE repo commit hash + run: | + echo "ee_repo_ref=$(cat ./backend/ee-repo-ref.txt)" >> "$GITHUB_ENV" + + - uses: actions/checkout@v4 + with: + repository: windmill-labs/windmill-ee-private + path: ./windmill-ee-private + ref: ${{ env.ee_repo_ref }} + token: ${{ secrets.WINDMILL_EE_PRIVATE_ACCESS }} + fetch-depth: 0 + + - name: Substitute EE code + run: | + cd backend && ./substitute_ee_code.sh --copy --dir ./windmill-ee-private + + - uses: actions-rust-lang/setup-rust-toolchain@v1 + with: + cache-workspaces: backend + toolchain: 1.93.0 + + - uses: oven-sh/setup-bun@v2 + with: + bun-version: 1.3.10 + + - uses: denoland/setup-deno@v2 + with: + deno-version: v2.x + + - uses: actions/setup-node@v4 + with: + node-version: "20" + + - name: Install wmill CLI + run: | + cd cli && bash gen_wm_client.sh && bun install + mkdir -p "$HOME/.local/bin" + printf '#!/bin/sh\nexec bun run "%s/cli/src/main.ts" "$@"\n' "$GITHUB_WORKSPACE" > "$HOME/.local/bin/wmill" + chmod +x "$HOME/.local/bin/wmill" + echo "$HOME/.local/bin" >> $GITHUB_PATH + + - name: Build Windmill + working-directory: ./backend + env: + SQLX_OFFLINE: true + CARGO_BUILD_JOBS: 12 + RUSTFLAGS: "" + run: | + cargo build --features enterprise,private,license,zip + + - name: Start Gitea + run: | + docker run -d --name gitea \ + -e GITEA__database__DB_TYPE=sqlite3 \ + -e GITEA__security__INSTALL_LOCK=true \ + -e GITEA__server__HTTP_PORT=3000 \ + -e GITEA__server__ROOT_URL=http://localhost:3000 \ + -e GITEA__service__DISABLE_REGISTRATION=false \ + -p 3000:3000 \ + gitea/gitea:1.22-rootless + echo "Waiting for Gitea to be ready..." + for i in $(seq 1 30); do + if curl -sf http://localhost:3000/api/v1/version > /dev/null 2>&1; then + echo "Gitea is ready" + break + fi + sleep 2 + done + curl -sf http://localhost:3000/api/v1/version > /dev/null || { echo "Gitea failed to start"; exit 1; } + + - name: Start Windmill + working-directory: ./backend + env: + DATABASE_URL: postgres://postgres:changeme@localhost:5432/windmill + LICENSE_KEY: ${{ secrets.WM_LICENSE_KEY_CI }} + DENO_PATH: deno + BUN_PATH: bun + NODE_BIN_PATH: node + run: | + ./target/debug/windmill & + echo "Waiting for Windmill to be ready..." + for i in $(seq 1 60); do + if curl -sf http://localhost:8000/api/version > /dev/null 2>&1; then + echo "Windmill is ready" + break + fi + sleep 2 + done + curl -sf http://localhost:8000/api/version > /dev/null || { echo "Windmill failed to start"; exit 1; } + + - name: Run git sync E2E tests + timeout-minutes: 10 + env: + GITEA_DOCKER_URL: http://localhost:3000 + LICENSE_KEY: ${{ secrets.WM_LICENSE_KEY_CI }} + run: | + python3 -m venv .venv + .venv/bin/pip install -r integration_tests/requirements.txt + cd integration_tests && ../.venv/bin/python -m unittest -v test.git_sync_test + + - name: Archive logs + uses: actions/upload-artifact@v4 + if: always() + with: + name: Git Sync Integration Tests Logs + path: | + integration_tests/logs diff --git a/backend/Cargo.lock b/backend/Cargo.lock index 7f10019e62..23e6f2a4b1 100644 --- a/backend/Cargo.lock +++ b/backend/Cargo.lock @@ -16187,6 +16187,7 @@ dependencies = [ "windmill-api-auth", "windmill-api-client", "windmill-common", + "windmill-git-sync", "windmill-native-triggers", "windmill-test-utils", "windmill-worker", diff --git a/backend/ee-repo-ref.txt b/backend/ee-repo-ref.txt index 06277182aa..18e2279e37 100644 --- a/backend/ee-repo-ref.txt +++ b/backend/ee-repo-ref.txt @@ -1 +1 @@ -2f52c015bc6c81391234fa87b27ee1d4cd3a48a3 \ No newline at end of file +85bcbfc8e952842c555156d1050b2a024e7e37a3 \ No newline at end of file diff --git a/backend/src/main.rs b/backend/src/main.rs index 4a58ce0fde..08d59c9dd9 100644 --- a/backend/src/main.rs +++ b/backend/src/main.rs @@ -243,7 +243,14 @@ async fn cache_hub_scripts(file_path: Option) -> anyhow::Result<()> { create_dir_all(&*HUB_CACHE_DIR)?; create_dir_all(&*BUN_BUNDLE_CACHE_DIR)?; - for path in paths.values() { + // Ensure the latest git sync script is always cached, regardless of hubPaths.json contents + let mut all_paths: Vec = paths.into_values().collect(); + let latest_git_sync = windmill_common::workspaces::LATEST_GIT_SYNC_SCRIPT_PATH.to_string(); + if !all_paths.contains(&latest_git_sync) { + all_paths.push(latest_git_sync); + } + + for path in &all_paths { tracing::info!("Caching hub script at {path}"); let res = get_hub_script_content_and_requirements(Some(path), None).await?; if res diff --git a/backend/windmill-api-integration-tests/Cargo.toml b/backend/windmill-api-integration-tests/Cargo.toml index c7d608ed28..35dd64f995 100644 --- a/backend/windmill-api-integration-tests/Cargo.toml +++ b/backend/windmill-api-integration-tests/Cargo.toml @@ -10,8 +10,8 @@ path = "src/lib.rs" [features] default = [] -private = ["windmill-test-utils/private", "dep:aws-config", "dep:aws-credential-types", "dep:aws-sdk-sqs"] -enterprise = ["windmill-test-utils/enterprise", "dep:base64"] +private = ["windmill-test-utils/private", "dep:aws-config", "dep:aws-credential-types", "dep:aws-sdk-sqs", "windmill-git-sync/private"] +enterprise = ["windmill-test-utils/enterprise", "dep:base64", "windmill-git-sync/enterprise"] deno_core = ["windmill-test-utils/deno_core"] mcp = [] run_inline = ["dep:windmill-worker", "windmill-test-utils/run_inline", "windmill-test-utils/duckdb"] @@ -22,6 +22,7 @@ windmill-api-client.workspace = true windmill-common = { workspace = true, default-features = false } windmill-native-triggers = { workspace = true, features = ["native_trigger"] } windmill-api-auth.workspace = true +windmill-git-sync.workspace = true windmill-worker = { workspace = true, optional = true } sqlx.workspace = true serde_json.workspace = true diff --git a/backend/windmill-api/openapi.yaml b/backend/windmill-api/openapi.yaml index 5fbeacfddf..9bb25f5589 100644 --- a/backend/windmill-api/openapi.yaml +++ b/backend/windmill-api/openapi.yaml @@ -23532,7 +23532,6 @@ components: items: $ref: "#/components/schemas/GitSyncObjectType" required: - - script_path - git_repo_resource_path MetricMetadata: diff --git a/backend/windmill-common/src/workspaces.rs b/backend/windmill-common/src/workspaces.rs index 5aba7162e1..6dc910e0d4 100644 --- a/backend/windmill-common/src/workspaces.rs +++ b/backend/windmill-common/src/workspaces.rs @@ -149,11 +149,16 @@ pub enum ObjectType { WorkspaceDependencies, } +pub const LATEST_GIT_SYNC_SCRIPT_PATH: &str = "hub/28160/sync-script-to-git-repo-windmill"; + #[derive(Serialize, Deserialize, Debug)] pub struct GitRepositorySettings { #[serde(skip_serializing_if = "Option::is_none")] pub exclude_types_override: Option>, - pub script_path: String, + /// None means auto-managed: always use LATEST_GIT_SYNC_SCRIPT_PATH. + /// Some(path) means pinned to a specific script. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub script_path: Option, pub git_repo_resource_path: String, pub use_individual_branch: Option, pub group_by_folder: Option, @@ -164,23 +169,26 @@ pub struct GitRepositorySettings { } impl GitRepositorySettings { + pub fn effective_script_path(&self) -> &str { + self.script_path + .as_deref() + .unwrap_or(LATEST_GIT_SYNC_SCRIPT_PATH) + } + pub fn is_script_meets_min_version(&self, min_version: u32) -> error::Result { + let path = self.effective_script_path(); // example: "hub/28102/sync-script-to-git-repo-windmill" - let current = self - .script_path + let current = path .split("/") // -> ["hub" "28102" "sync-script-to-git-repo-windmill"] .skip(1) // omit "hub" .next() // get numeric id .ok_or(Error::InternalErr(format!( "cannot get script version id from: {}", - &self.script_path + path )))? .parse() .unwrap_or_else(|e| { - tracing::warn!( - "cannot get script version id from: {}. e: {e}", - &self.script_path - ); + tracing::warn!("cannot get script version id from: {}. e: {e}", path); u32::MAX }); diff --git a/frontend/src/lib/components/git_sync/GitSyncContext.svelte.ts b/frontend/src/lib/components/git_sync/GitSyncContext.svelte.ts index 1f6e8fbfaa..1940296534 100644 --- a/frontend/src/lib/components/git_sync/GitSyncContext.svelte.ts +++ b/frontend/src/lib/components/git_sync/GitSyncContext.svelte.ts @@ -184,7 +184,7 @@ export function createGitSyncContext(workspace: string) { function addRepository() { repositories.push({ git_repo_resource_path: '', - script_path: hubPaths.gitSync, + script_path: undefined, use_individual_branch: false, group_by_folder: false, settings: { @@ -541,7 +541,7 @@ export function createGitSyncContext(workspace: string) { async function runTestJob(idx: number) { const repo = repositories[idx] - if (!repo?.git_repo_resource_path || !repo?.script_path) { + if (!repo?.git_repo_resource_path) { return } @@ -648,7 +648,7 @@ export function createGitSyncContext(workspace: string) { function addSyncRepository() { repositories.push({ git_repo_resource_path: '', - script_path: hubPaths.gitSync, + script_path: undefined, use_individual_branch: false, group_by_folder: false, settings: { @@ -671,7 +671,7 @@ export function createGitSyncContext(workspace: string) { function addPromotionRepository() { repositories.push({ git_repo_resource_path: '', - script_path: hubPaths.gitSync, + script_path: undefined, use_individual_branch: true, group_by_folder: false, settings: { diff --git a/frontend/src/lib/components/git_sync/GitSyncRepositoryCard.svelte b/frontend/src/lib/components/git_sync/GitSyncRepositoryCard.svelte index af6f361286..8d4b7a0357 100644 --- a/frontend/src/lib/components/git_sync/GitSyncRepositoryCard.svelte +++ b/frontend/src/lib/components/git_sync/GitSyncRepositoryCard.svelte @@ -21,10 +21,8 @@ import Toggle from '$lib/components/Toggle.svelte' import { fade } from 'svelte/transition' import { workspaceStore } from '$lib/stores' - import hubPaths from '$lib/hubPaths.json' import type { GitSyncRepository } from './GitSyncContext.svelte' import GitSyncModeDisplay from './GitSyncModeDisplay.svelte' - import { DEFAULT_HUB_BASE_URL } from '$lib/hub' import { ResourceService, VariableService } from '$lib/gen' let { @@ -375,7 +373,7 @@ {#if !emptyString(repo.git_repo_resource_path)} diff --git a/frontend/src/lib/hub.ts b/frontend/src/lib/hub.ts index fcaa2ab832..a038f18580 100644 --- a/frontend/src/lib/hub.ts +++ b/frontend/src/lib/hub.ts @@ -92,7 +92,6 @@ export function rawAppToHubUrl(hubBaseUrl: string, summary?: string): URL { } type HubPaths = { - gitSync: string gitSyncTest: string gitInitRepo: string slackErrorHandler: string diff --git a/frontend/src/lib/hubPaths.json b/frontend/src/lib/hubPaths.json index 79bf362003..d62fffa097 100644 --- a/frontend/src/lib/hubPaths.json +++ b/frontend/src/lib/hubPaths.json @@ -1,28 +1,28 @@ { - "gitSync_0": "hub/9087/sync-script-to-git-repo-windmill", - "gitSync_1": "hub/9987/sync-script-to-git-repo-windmill", - "gitSync_2": "hub/11498/sync-script-to-git-repo-windmill", - "gitSync_3": "hub/11533/sync-script-to-git-repo-windmill", - "gitSync_4": "hub/11580/sync-script-to-git-repo-windmill", - "gitSync_5": "hub/11641/sync-script-to-git-repo-windmill", - "gitSync_6": "hub/11666/sync-script-to-git-repo-windmill", - "gitSync_7": "hub/11668/sync-script-to-git-repo-windmill", - "gitSync_8": "hub/19673/sync-script-to-git-repo-windmill", - "gitSync_9": "hub/19738/sync-script-to-git-repo-windmill", - "gitSync_10": "hub/19785/sync-script-to-git-repo-windmill", - "gitSync_11": "hub/19789/sync-script-to-git-repo-windmill", - "gitSync_12": "hub/19798/sync-script-to-git-repo-windmill", - "gitSync_13": "hub/19801/sync-script-to-git-repo-windmill", - "gitSync_14": "hub/19803/sync-script-to-git-repo-windmill", - "gitSync_15": "hub/19816/sync-script-to-git-repo-windmill", - "gitSync_16": "hub/19818/sync-script-to-git-repo-windmill", - "gitSync_17": "hub/28073/sync-script-to-git-repo-windmill", - "gitSync_18": "hub/28078/sync-script-to-git-repo-windmill", - "gitSync_19": "hub/28081/sync-script-to-git-repo-windmill", - "gitSync_20": "hub/28102/sync-script-to-git-repo-windmill", - "gitSync_21": "hub/28131/sync-script-to-git-repo-windmill", - "gitSync_22": "hub/28159/sync-script-to-git-repo-windmill", - "gitSync": "hub/28160/sync-script-to-git-repo-windmill", + "deprecated_gitSync_0": "hub/9087/sync-script-to-git-repo-windmill", + "deprecated_gitSync_1": "hub/9987/sync-script-to-git-repo-windmill", + "deprecated_gitSync_2": "hub/11498/sync-script-to-git-repo-windmill", + "deprecated_gitSync_3": "hub/11533/sync-script-to-git-repo-windmill", + "deprecated_gitSync_4": "hub/11580/sync-script-to-git-repo-windmill", + "deprecated_gitSync_5": "hub/11641/sync-script-to-git-repo-windmill", + "deprecated_gitSync_6": "hub/11666/sync-script-to-git-repo-windmill", + "deprecated_gitSync_7": "hub/11668/sync-script-to-git-repo-windmill", + "deprecated_gitSync_8": "hub/19673/sync-script-to-git-repo-windmill", + "deprecated_gitSync_9": "hub/19738/sync-script-to-git-repo-windmill", + "deprecated_gitSync_10": "hub/19785/sync-script-to-git-repo-windmill", + "deprecated_gitSync_11": "hub/19789/sync-script-to-git-repo-windmill", + "deprecated_gitSync_12": "hub/19798/sync-script-to-git-repo-windmill", + "deprecated_gitSync_13": "hub/19801/sync-script-to-git-repo-windmill", + "deprecated_gitSync_14": "hub/19803/sync-script-to-git-repo-windmill", + "deprecated_gitSync_15": "hub/19816/sync-script-to-git-repo-windmill", + "deprecated_gitSync_16": "hub/19818/sync-script-to-git-repo-windmill", + "deprecated_gitSync_17": "hub/28073/sync-script-to-git-repo-windmill", + "deprecated_gitSync_18": "hub/28078/sync-script-to-git-repo-windmill", + "deprecated_gitSync_19": "hub/28081/sync-script-to-git-repo-windmill", + "deprecated_gitSync_20": "hub/28102/sync-script-to-git-repo-windmill", + "deprecated_gitSync_21": "hub/28131/sync-script-to-git-repo-windmill", + "deprecated_gitSync_22": "hub/28159/sync-script-to-git-repo-windmill", + "deprecated_gitSync_latest": "hub/28160/sync-script-to-git-repo-windmill", "gitSyncTest": "hub/19799/git-repo-test-read-write-windmill", "gitInitRepo_0": "hub/28134/git-sync%3A-init-repository-windmill", "gitInitRepo": "hub/28158/git-sync%3A-init-repository-windmill", diff --git a/integration_tests/requirements.txt b/integration_tests/requirements.txt index 97572f1b84..35b4f13cf4 100644 --- a/integration_tests/requirements.txt +++ b/integration_tests/requirements.txt @@ -6,3 +6,4 @@ httpx==0.26.0 idna==3.6 sniffio==1.3.0 docker==7.1.0 +gitpython==3.1.43 diff --git a/integration_tests/test/git_sync_test.py b/integration_tests/test/git_sync_test.py new file mode 100644 index 0000000000..5268fefa01 --- /dev/null +++ b/integration_tests/test/git_sync_test.py @@ -0,0 +1,792 @@ +import os +import shutil +import tempfile +import time +import unittest +import uuid + +import git as gitpython + +from .wmill_integration_test_utils import WindmillClient, GiteaClient + + +# Script content template for bun/TypeScript scripts +def ts_script(body: str) -> str: + return f"export async function main() {{\n {body}\n}}\n" + + +def unique_name(prefix: str = "git-sync-test") -> str: + return f"{prefix}-{uuid.uuid4().hex[:8]}" + + +class TestGitSync(unittest.TestCase): + _client: WindmillClient + _gitea: GiteaClient + _repos_to_cleanup: list + _fork_workspaces_to_cleanup: list + + @classmethod + def setUpClass(cls) -> None: + print("Running {}".format(cls.__name__)) + cls._client = WindmillClient() + cls._gitea = GiteaClient() + cls._gitea.setup_admin() + cls._repos_to_cleanup = [] + cls._fork_workspaces_to_cleanup = [] + + @classmethod + def tearDownClass(cls) -> None: + # Disable git sync to avoid interfering with other tests + try: + cls._client.configure_git_sync({"repositories": []}) + except Exception as e: + print(f"Warning: failed to disable git sync: {e}") + + for fork_id in cls._fork_workspaces_to_cleanup: + try: + cls._client.delete_workspace(fork_id) + except Exception as e: + print(f"Warning: failed to delete fork workspace {fork_id}: {e}") + + for repo_name in cls._repos_to_cleanup: + cls._gitea.delete_repo(repo_name) + + def setUp(self): + """Wait for any pending deployment callbacks from previous tests to drain.""" + time.sleep(2) + # Wait until no new deployment callback jobs appear for 4 seconds + prev_count = self._client.count_deployment_callback_jobs() + for _ in range(3): + time.sleep(2) + cur_count = self._client.count_deployment_callback_jobs() + if cur_count == prev_count: + break + prev_count = cur_count + + def _create_test_repo(self) -> tuple: + """Create a Gitea repo and return (repo_name, docker_clone_url).""" + name = unique_name() + docker_url = self._gitea.create_repo(name) + self._repos_to_cleanup.append(name) + return name, docker_url + + def _setup_git_sync_resource(self, repo_name: str, branch: str = "main") -> str: + """Create a git_repository resource pointing to the Gitea repo. + Returns the resource path.""" + resource_path = f"u/admin/git_sync_{repo_name.replace('-', '_')}" + docker_url = self._gitea.get_docker_clone_url(repo_name) + self._client.create_resource( + path=resource_path, + resource_type="git_repository", + value={ + "url": docker_url, + "branch": branch, + "is_github_app": False, + }, + update_if_exists=True, + ) + return resource_path + + def _configure_single_repo_sync( + self, + resource_path: str, + include_type=None, + include_path=None, + use_individual_branch=False, + group_by_folder=False, + force_branch=None, + ): + """Configure git sync with a single repository (auto-managed script).""" + repo_settings = { + "git_repo_resource_path": f"$res:{resource_path}", + "use_individual_branch": use_individual_branch, + "group_by_folder": group_by_folder, + } + if force_branch: + repo_settings["force_branch"] = force_branch + if include_type or include_path: + repo_settings["settings"] = { + "include_type": include_type or [], + "include_path": include_path if include_path is not None else ["**"], + } + + self._client.configure_git_sync({ + "repositories": [repo_settings], + }) + + def _clone_repo(self, repo_name: str, branch: str = None) -> str: + """Clone the repo to a temp dir and return the path.""" + host_url = self._gitea.get_host_clone_url(repo_name) + tmp_dir = tempfile.mkdtemp() + self.addCleanup(shutil.rmtree, tmp_dir, ignore_errors=True) + args = {} + if branch: + args["branch"] = branch + gitpython.Repo.clone_from(host_url, tmp_dir, **args) + return tmp_dir + + def _clone_repo_all_branches(self, repo_name: str) -> str: + """Clone the repo fetching all branches.""" + host_url = self._gitea.get_host_clone_url(repo_name) + tmp_dir = tempfile.mkdtemp() + self.addCleanup(shutil.rmtree, tmp_dir, ignore_errors=True) + gitpython.Repo.clone_from(host_url, tmp_dir, no_single_branch=True) + return tmp_dir + + def _list_repo_files(self, repo_dir: str, branch: str = None) -> list: + """List all tracked files in the repo (relative paths).""" + repo = gitpython.Repo(repo_dir) + if branch: + commit = repo.refs[branch].commit + else: + commit = repo.head.commit + return [item.path for item in commit.tree.traverse()] + + def _read_file_content(self, repo_dir: str, file_path: str) -> str: + """Read a file's content from the repo working tree.""" + full_path = os.path.join(repo_dir, file_path) + with open(full_path, "r") as f: + return f.read() + + def _get_commit_count(self, repo_dir: str, branch: str = "main") -> int: + repo = gitpython.Repo(repo_dir) + return len(list(repo.iter_commits(branch))) + + def _get_last_commit_message(self, repo_dir: str, branch: str = "main") -> str: + repo = gitpython.Repo(repo_dir) + return repo.iter_commits(branch).__next__().message + + def _get_branches(self, repo_dir: str) -> list: + repo = gitpython.Repo(repo_dir) + return [ref.name for ref in repo.remote().refs] + + def _create_folder(self, folder_name: str): + """Create a folder in the workspace, ignoring errors if it already exists.""" + try: + self._client._client.post( + f"/api/w/{self._client._workspace}/folders/create", + json={"name": folder_name}, + ) + except Exception: + pass + + # ────────────────────────────────────────────────── + # Core happy-path tests + # ────────────────────────────────────────────────── + + def test_script_deploy_syncs_to_git(self): + """Deploy a script and verify it appears in the git repo with correct content.""" + repo_name, _ = self._create_test_repo() + resource_path = self._setup_git_sync_resource(repo_name) + self._configure_single_repo_sync( + resource_path, + include_type=["script"], + ) + + initial_count = self._client.count_deployment_callback_jobs() + + script_path = f"u/admin/{unique_name('sync_test')}" + self._client.create_script( + path=script_path, + content=ts_script("return 42"), + language="bun", + ) + + self._client.wait_for_sync_jobs(initial_count, min_new=1) + time.sleep(3) + + repo_dir = self._clone_repo(repo_name) + files = self._list_repo_files(repo_dir) + + # The script should appear in the repo + matching = [f for f in files if script_path in f] + self.assertTrue( + len(matching) > 0, + f"Expected script '{script_path}' in repo files: {files}", + ) + + # Verify file content matches what we deployed + script_file = [f for f in matching if f.endswith(".ts")][0] + content = self._read_file_content(repo_dir, script_file) + self.assertIn( + "return 42", + content, + f"Expected 'return 42' in script content: {content}", + ) + + def test_multi_repo_routing(self): + """Two repos with different path filters receive the correct objects.""" + repo_name_a, _ = self._create_test_repo() + repo_name_b, _ = self._create_test_repo() + res_path_a = self._setup_git_sync_resource(repo_name_a) + res_path_b = self._setup_git_sync_resource(repo_name_b) + + folder_a = unique_name("folder_a") + folder_b = unique_name("folder_b") + + self._client.configure_git_sync({ + "repositories": [ + { + "git_repo_resource_path": f"$res:{res_path_a}", + "use_individual_branch": False, + "group_by_folder": False, + "settings": { + "include_type": ["script"], + "include_path": [f"f/{folder_a}/**"], + }, + }, + { + "git_repo_resource_path": f"$res:{res_path_b}", + "use_individual_branch": False, + "group_by_folder": False, + "settings": { + "include_type": ["script"], + "include_path": [f"f/{folder_b}/**"], + }, + }, + ], + }) + + self._create_folder(folder_a) + self._create_folder(folder_b) + + initial_count = self._client.count_deployment_callback_jobs() + + script_a = f"f/{folder_a}/script_a" + script_b = f"f/{folder_b}/script_b" + + self._client.create_script( + path=script_a, + content=ts_script("return 'a'"), + language="bun", + ) + self._client.create_script( + path=script_b, + content=ts_script("return 'b'"), + language="bun", + ) + + # Wait for at least 2 deployment callback jobs + self._client.wait_for_sync_jobs(initial_count, min_new=2) + time.sleep(3) + + # Verify repo A has script_a but not script_b + repo_dir_a = self._clone_repo(repo_name_a) + files_a = self._list_repo_files(repo_dir_a) + self.assertTrue( + any("script_a" in f for f in files_a), + f"Expected script_a in repo A files: {files_a}", + ) + self.assertFalse( + any("script_b" in f for f in files_a), + f"Did not expect script_b in repo A files: {files_a}", + ) + + # Verify repo B has script_b but not script_a + repo_dir_b = self._clone_repo(repo_name_b) + files_b = self._list_repo_files(repo_dir_b) + self.assertTrue( + any("script_b" in f for f in files_b), + f"Expected script_b in repo B files: {files_b}", + ) + self.assertFalse( + any("script_a" in f for f in files_b), + f"Did not expect script_a in repo B files: {files_b}", + ) + + def test_script_update_creates_new_commit_with_updated_content(self): + """Updating a script should produce a new commit with the new content.""" + repo_name, _ = self._create_test_repo() + resource_path = self._setup_git_sync_resource(repo_name) + self._configure_single_repo_sync(resource_path, include_type=["script"]) + + script_path = f"u/admin/{unique_name('update_test')}" + + # Create initial script + initial_count = self._client.count_deployment_callback_jobs() + self._client.create_script( + path=script_path, + content=ts_script("return 1"), + language="bun", + ) + self._client.wait_for_sync_jobs(initial_count, min_new=1) + time.sleep(3) + + repo_dir = self._clone_repo(repo_name) + initial_commits = self._get_commit_count(repo_dir) + + # Update the script + update_count = self._client.count_deployment_callback_jobs() + self._client.update_script( + path=script_path, + content=ts_script("return 2"), + language="bun", + ) + self._client.wait_for_sync_jobs(update_count, min_new=1) + time.sleep(3) + + # Re-clone and check commit count increased + repo_dir2 = self._clone_repo(repo_name) + new_commits = self._get_commit_count(repo_dir2) + self.assertGreater( + new_commits, + initial_commits, + f"Expected more commits after update: {new_commits} vs {initial_commits}", + ) + + # Verify file content reflects the update + files = self._list_repo_files(repo_dir2) + script_file = [f for f in files if script_path in f and f.endswith(".ts")][0] + content = self._read_file_content(repo_dir2, script_file) + self.assertIn( + "return 2", + content, + f"Expected 'return 2' in updated script content: {content}", + ) + self.assertNotIn( + "return 1", + content, + f"Did not expect 'return 1' in updated script content: {content}", + ) + + def test_deploy_multiple_object_types(self): + """Deploy a script, flow, and variable and verify all appear in the repo.""" + repo_name, _ = self._create_test_repo() + resource_path = self._setup_git_sync_resource(repo_name) + self._configure_single_repo_sync( + resource_path, + include_type=["script", "flow", "variable"], + ) + + initial_count = self._client.count_deployment_callback_jobs() + + suffix = unique_name("multi") + script_path = f"u/admin/{suffix}_script" + flow_path = f"u/admin/{suffix}_flow" + var_path = f"u/admin/{suffix}_var" + + self._client.create_script( + path=script_path, + content=ts_script("return 'multi'"), + language="bun", + ) + self._client.create_flow( + path=flow_path, + flow_value_json="""{ + "summary": "test flow", + "value": { + "modules": [{ + "id": "a", + "value": { + "type": "rawscript", + "content": "export async function main() { return 1 }", + "language": "bun", + "input_transforms": {}, + "tag": "" + } + }] + }, + "schema": { + "$schema": "https://json-schema.org/draft/2020-12/schema", + "properties": {}, + "required": [], + "type": "object", + "order": [] + } + }""", + ) + self._client.create_variable( + path=var_path, + value="test_value", + ) + + # Wait for 3 deployment callbacks (one per object) + self._client.wait_for_sync_jobs(initial_count, min_new=3) + time.sleep(3) + + repo_dir = self._clone_repo(repo_name) + files = self._list_repo_files(repo_dir) + files_str = "\n".join(files) + + self.assertTrue( + any(suffix + "_script" in f for f in files), + f"Expected script in repo:\n{files_str}", + ) + self.assertTrue( + any(suffix + "_flow" in f for f in files), + f"Expected flow in repo:\n{files_str}", + ) + self.assertTrue( + any(suffix + "_var" in f for f in files), + f"Expected variable in repo:\n{files_str}", + ) + + # ────────────────────────────────────────────────── + # Commit message verification + # ────────────────────────────────────────────────── + + def test_commit_message_format(self): + """Verify commit messages have the [WM] prefix.""" + repo_name, _ = self._create_test_repo() + resource_path = self._setup_git_sync_resource(repo_name) + self._configure_single_repo_sync(resource_path, include_type=["script"]) + + initial_count = self._client.count_deployment_callback_jobs() + script_path = f"u/admin/{unique_name('commit_msg')}" + self._client.create_script( + path=script_path, + content=ts_script("return 'msg'"), + language="bun", + ) + self._client.wait_for_sync_jobs(initial_count, min_new=1) + time.sleep(3) + + repo_dir = self._clone_repo(repo_name) + commit_msg = self._get_last_commit_message(repo_dir) + + self.assertTrue( + commit_msg.startswith("[WM]"), + f"Expected commit message to start with '[WM]', got: {commit_msg!r}", + ) + + # ────────────────────────────────────────────────── + # Rename handling + # ────────────────────────────────────────────────── + + def test_rename_removes_old_file(self): + """Renaming a script should remove the old file and create the new one.""" + repo_name, _ = self._create_test_repo() + resource_path = self._setup_git_sync_resource(repo_name) + self._configure_single_repo_sync(resource_path, include_type=["script"]) + + old_path = f"u/admin/{unique_name('rename_old')}" + + # Create initial script + initial_count = self._client.count_deployment_callback_jobs() + self._client.create_script( + path=old_path, + content=ts_script("return 'old'"), + language="bun", + ) + self._client.wait_for_sync_jobs(initial_count, min_new=1) + time.sleep(3) + + # Verify old script exists in repo + repo_dir = self._clone_repo(repo_name) + files = self._list_repo_files(repo_dir) + old_name = old_path.split("/")[-1] + self.assertTrue( + any(old_name in f for f in files), + f"Expected old script '{old_name}' in repo: {files}", + ) + + # Create new script at different path (simulates rename) + new_path = f"u/admin/{unique_name('rename_new')}" + rename_count = self._client.count_deployment_callback_jobs() + self._client.create_script( + path=new_path, + content=ts_script("return 'renamed'"), + language="bun", + ) + # Also delete the old script + self._client.delete_script(old_path) + # Wait for both create and delete deployment callbacks + self._client.wait_for_sync_jobs(rename_count, min_new=2) + time.sleep(3) + + # Verify new script exists + repo_dir2 = self._clone_repo(repo_name) + files2 = self._list_repo_files(repo_dir2) + new_name = new_path.split("/")[-1] + self.assertTrue( + any(new_name in f for f in files2), + f"Expected new script '{new_name}' in repo: {files2}", + ) + self.assertFalse( + any(old_name in f for f in files2), + f"Expected old script '{old_name}' to be removed: {files2}", + ) + + # ────────────────────────────────────────────────── + # Promotion mode (individual branches) + # ────────────────────────────────────────────────── + + def test_promotion_mode_creates_per_object_branches(self): + """In promotion mode (use_individual_branch=True), each deploy creates + a branch named wm_deploy/{workspace}/{path_type}/{path} with the content + on that branch, not on main.""" + repo_name, _ = self._create_test_repo() + resource_path = self._setup_git_sync_resource(repo_name) + self._configure_single_repo_sync( + resource_path, + include_type=["script"], + use_individual_branch=True, + ) + + initial_count = self._client.count_deployment_callback_jobs() + script_path = f"u/admin/{unique_name('promo')}" + self._client.create_script( + path=script_path, + content=ts_script("return 'promotion'"), + language="bun", + ) + self._client.wait_for_sync_jobs(initial_count, min_new=1) + time.sleep(3) + + # Clone with all branches + repo_dir = self._clone_repo_all_branches(repo_name) + branches = self._get_branches(repo_dir) + + # Should have a branch matching wm_deploy pattern + wm_branches = [b for b in branches if "wm_deploy/" in b] + self.assertTrue( + len(wm_branches) > 0, + f"Expected wm_deploy/ branch, got branches: {branches}", + ) + + # The branch name should contain 'script' (the path_type) + deploy_branch = wm_branches[0] + self.assertIn( + "script", + deploy_branch, + f"Expected 'script' in branch name: {deploy_branch}", + ) + + # The script path (with / replaced by __) should appear in the branch name + script_name = script_path.split("/")[-1] + self.assertIn( + script_name, + deploy_branch.replace("/", "__"), + f"Expected script name '{script_name}' in branch: {deploy_branch}", + ) + + # Verify main branch does NOT have the script + main_files = self._list_repo_files(repo_dir, branch="origin/main") + self.assertFalse( + any(script_name in f for f in main_files), + f"Did not expect script on main branch, but found it: {main_files}", + ) + + # Verify the deploy branch HAS the script + local_branch_name = deploy_branch.replace("origin/", "") + repo = gitpython.Repo(repo_dir) + repo.git.checkout(local_branch_name) + branch_files = self._list_repo_files(repo_dir) + self.assertTrue( + any(script_name in f for f in branch_files), + f"Expected script on deploy branch '{local_branch_name}': {branch_files}", + ) + + def test_promotion_mode_group_by_folder(self): + """With use_individual_branch=True and group_by_folder=True, the branch name + uses the folder prefix (first 2 path segments joined by __) instead of the + full path.""" + repo_name, _ = self._create_test_repo() + resource_path = self._setup_git_sync_resource(repo_name) + self._configure_single_repo_sync( + resource_path, + include_type=["script"], + use_individual_branch=True, + group_by_folder=True, + ) + + folder_name = unique_name("grp") + self._create_folder(folder_name) + + initial_count = self._client.count_deployment_callback_jobs() + script_path = f"f/{folder_name}/{unique_name('grp_script')}" + self._client.create_script( + path=script_path, + content=ts_script("return 'grouped'"), + language="bun", + ) + self._client.wait_for_sync_jobs(initial_count, min_new=1) + time.sleep(3) + + repo_dir = self._clone_repo_all_branches(repo_name) + branches = self._get_branches(repo_dir) + + wm_branches = [b for b in branches if "wm_deploy/" in b] + self.assertTrue( + len(wm_branches) > 0, + f"Expected wm_deploy/ branch with group_by_folder: {branches}", + ) + + # With group_by_folder, the branch should contain the folder prefix + # format: wm_deploy/{workspace}/f__{folder_name} + deploy_branch = wm_branches[0] + expected_folder_part = f"f__{folder_name}" + self.assertIn( + expected_folder_part, + deploy_branch, + f"Expected folder-grouped branch name containing '{expected_folder_part}', got: {deploy_branch}", + ) + + # ────────────────────────────────────────────────── + # force_branch with wmill.yaml + # ────────────────────────────────────────────────── + + def test_force_branch_with_wmill_yaml(self): + """force_branch passes --branch to wmill sync pull, which selects the + matching gitBranches config from wmill.yaml. With branch-specific variables + configured, the variable file should use a branch-specific path.""" + repo_name, _ = self._create_test_repo() + resource_path = self._setup_git_sync_resource(repo_name) + + # Push a wmill.yaml to the repo that configures branch-specific variables + wmill_yaml_content = """\ +includes: + - "**" +gitBranches: + staging: + specificItems: + variables: + - "**" +""" + self._gitea.create_file(repo_name, "wmill.yaml", wmill_yaml_content) + + self._configure_single_repo_sync( + resource_path, + include_type=["variable"], + force_branch="staging", + ) + + initial_count = self._client.count_deployment_callback_jobs() + var_path = f"u/admin/{unique_name('env_var')}" + self._client.create_variable( + path=var_path, + value="staging_value", + ) + self._client.wait_for_sync_jobs(initial_count, min_new=1) + time.sleep(3) + + repo_dir = self._clone_repo(repo_name) + files = self._list_repo_files(repo_dir) + + var_name = var_path.split("/")[-1] + + # With force_branch="staging" and specificItems for variables, + # the variable file should have ".staging." in its name + staging_files = [f for f in files if var_name in f and ".staging." in f] + self.assertTrue( + len(staging_files) > 0, + f"Expected variable file with '.staging.' in name for branch-specific item, " + f"got files: {files}", + ) + + # ────────────────────────────────────────────────── + # Exclude path filtering + # ────────────────────────────────────────────────── + + def test_exclude_path_filtering(self): + """Scripts in excluded paths should not be synced to the repo.""" + repo_name, _ = self._create_test_repo() + resource_path = self._setup_git_sync_resource(repo_name) + + folder_inc = unique_name("inc") + folder_exc = unique_name("exc") + self._create_folder(folder_inc) + self._create_folder(folder_exc) + + self._client.configure_git_sync({ + "repositories": [{ + "git_repo_resource_path": f"$res:{resource_path}", + "use_individual_branch": False, + "group_by_folder": False, + "settings": { + "include_type": ["script"], + "include_path": ["f/**"], + "exclude_path": [f"f/{folder_exc}/**"], + }, + }], + }) + + initial_count = self._client.count_deployment_callback_jobs() + + script_inc = f"f/{folder_inc}/included_script" + script_exc = f"f/{folder_exc}/excluded_script" + + self._client.create_script( + path=script_inc, + content=ts_script("return 'included'"), + language="bun", + ) + self._client.create_script( + path=script_exc, + content=ts_script("return 'excluded'"), + language="bun", + ) + + # Only 1 sync job expected (the excluded one should not trigger) + self._client.wait_for_sync_jobs(initial_count, min_new=1) + time.sleep(5) + # Verify no extra sync jobs arrived for the excluded script + final_count = self._client.count_deployment_callback_jobs() + self.assertEqual( + final_count, initial_count + 1, + f"Expected exactly 1 new sync job, got {final_count - initial_count}", + ) + + repo_dir = self._clone_repo(repo_name) + files = self._list_repo_files(repo_dir) + + self.assertTrue( + any("included_script" in f for f in files), + f"Expected included_script in repo: {files}", + ) + self.assertFalse( + any("excluded_script" in f for f in files), + f"Did not expect excluded_script in repo: {files}", + ) + + # ────────────────────────────────────────────────── + # Workspace fork + # ────────────────────────────────────────────────── + + def test_workspace_fork_creates_branch(self): + """Forking a workspace with git sync configured should create a + fork branch in the git repo.""" + repo_name, _ = self._create_test_repo() + resource_path = self._setup_git_sync_resource(repo_name) + + # Configure git sync on the parent workspace (sync mode, not promotion) + self._configure_single_repo_sync( + resource_path, + include_type=["script"], + ) + + # Deploy a script first so there's content in the repo + initial_count = self._client.count_deployment_callback_jobs() + script_path = f"u/admin/{unique_name('fork_base')}" + self._client.create_script( + path=script_path, + content=ts_script("return 'base'"), + language="bun", + ) + self._client.wait_for_sync_jobs(initial_count, min_new=1) + time.sleep(3) + + # Create workspace fork + fork_id = f"wm-fork-{uuid.uuid4().hex[:8]}" + fork_name = f"Fork {fork_id}" + self._fork_workspaces_to_cleanup.append(fork_id) + + # Step 1: Create git branches for the fork + job_ids = self._client.create_workspace_fork_branch(fork_id, fork_name) + if job_ids: + self._client.wait_for_jobs_by_ids(job_ids, timeout=90) + time.sleep(3) + + # Step 2: Create the fork workspace + self._client.create_workspace_fork(fork_id, fork_name) + + # Verify a fork branch was created in the git repo + repo_dir = self._clone_repo_all_branches(repo_name) + branches = self._get_branches(repo_dir) + + # Fork branches are named: wm-fork/{original_branch}/{fork_id} + fork_branches = [b for b in branches if "wm-fork" in b] + self.assertTrue( + len(fork_branches) > 0, + f"Expected a wm-fork branch in the repo after forking, got: {branches}", + ) diff --git a/integration_tests/test/wmill_integration_test_utils.py b/integration_tests/test/wmill_integration_test_utils.py index 260986ef9b..93730f0d31 100644 --- a/integration_tests/test/wmill_integration_test_utils.py +++ b/integration_tests/test/wmill_integration_test_utils.py @@ -3,6 +3,7 @@ import time import httpx import json import os +import uuid @@ -13,9 +14,11 @@ class WindmillClient: _client: httpx.Client - def __init__(self): - self._workspace = "integration-tests" - self._url = "http://localhost:8000" + def __init__(self, workspace: str = "integration-tests", url: str = None): + if url is None: + url = os.environ.get("WINDMILL_BASE_URL", "http://localhost:8000") + self._workspace = workspace + self._url = url self._token = self._login() self._client = self._init_client() @@ -155,6 +158,55 @@ class WindmillClient: time.sleep(1) raise Exception(f"Script deployment failed for {path}") + def update_script(self, path: str, content: str, language: str, tag: str = None): + """Update an existing script by creating a new version with parent_hash.""" + print(f"Updating script {path}") + + # Get current script hash + response = self._client.get( + f"/api/w/{self._workspace}/scripts/get/p/{path}" + ) + if response.status_code // 100 != 2: + raise Exception(response.content.decode()) + current_hash = response.json()["hash"] + + payload = { + "path": path, + "content": content, + "description": "", + "summary": "", + "language": language, + "parent_hash": current_hash, + } + + if tag is not None: + payload["tag"] = tag + + response = self._client.post( + f"/api/w/{self._workspace}/scripts/create", + json=payload, + ) + if response.status_code // 100 != 2: + raise Exception(response.content.decode()) + script_hash = response.content.decode() + print(f"Script hash for path {path} is {script_hash}") + time_now = datetime.datetime.now(datetime.timezone.utc) + while datetime.datetime.now( + datetime.timezone.utc + ) - time_now < datetime.timedelta(seconds=60): + response = self._client.get( + f"/api/w/{self._workspace}/scripts/deployment_status/h/{script_hash}" + ) + if response.status_code // 100 != 2: + raise Exception(response.content.decode()) + elif response.json()["lock"] != None: + return + elif response.json()["lock_error_logs"] != None: + raise Exception(response.json()["lock_error_logs"]) + print(f"Waiting for script {path} with hash {script_hash} to be deployed") + time.sleep(1) + raise Exception(f"Script deployment failed for {path}") + def delete_script(self, path: str): print(f"Deleting script {path}") response = self._client.post( @@ -385,6 +437,120 @@ class WindmillClient: print(f"Exception when retrieving workers list: {e}") return [] + def create_resource(self, path: str, resource_type: str, value: dict, update_if_exists: bool = False): + print(f"Creating resource {path} of type {resource_type}") + params = {} + if update_if_exists: + params["update_if_exists"] = "true" + response = self._client.post( + f"/api/w/{self._workspace}/resources/create", + json={ + "path": path, + "value": value, + "resource_type": resource_type, + "description": "", + }, + params=params, + ) + if response.status_code // 100 != 2: + raise Exception(response.content.decode()) + return response.content.decode() + + def configure_git_sync(self, git_sync_settings: dict): + print(f"Configuring git sync with {len(git_sync_settings.get('repositories', []))} repositories") + response = self._client.post( + f"/api/w/{self._workspace}/workspaces/edit_git_sync_config", + json={"git_sync_settings": git_sync_settings}, + ) + if response.status_code // 100 != 2: + raise Exception(response.content.decode()) + return response.content.decode() + + def get_completed_jobs(self, job_kinds: str = None, success: bool = None): + params = {"per_page": 1000} + if job_kinds: + params["job_kinds"] = job_kinds + if success is not None: + params["success"] = str(success).lower() + response = self._client.get( + f"/api/w/{self._workspace}/jobs/completed/list", + params=params, + ) + if response.status_code // 100 != 2: + raise Exception(response.content.decode()) + return response.json() + + def wait_for_sync_jobs(self, initial_count: int, min_new: int = 1, timeout: int = 90) -> list: + """Poll completed DeploymentCallback jobs until count increases by min_new.""" + start = time.time() + current_count = initial_count + while time.time() - start < timeout: + jobs = self.get_completed_jobs(job_kinds="deploymentcallback") + current_count = len(jobs) + if current_count >= initial_count + min_new: + return jobs + time.sleep(2) + raise TimeoutError( + f"Timed out waiting for sync jobs: expected {initial_count + min_new}, " + f"got {current_count} after {timeout}s" + ) + + def count_deployment_callback_jobs(self) -> int: + jobs = self.get_completed_jobs(job_kinds="deploymentcallback") + return len(jobs) + + def create_workspace_fork_branch(self, fork_id: str, fork_name: str) -> list: + """Create git branches for a workspace fork. Returns list of job UUIDs to wait on.""" + print(f"Creating fork branch for {fork_id} from {self._workspace}") + response = self._client.post( + f"/api/w/{self._workspace}/workspaces/create_workspace_fork_branch", + json={"id": fork_id, "name": fork_name}, + ) + if response.status_code // 100 != 2: + raise Exception(response.content.decode()) + return response.json() + + def create_workspace_fork(self, fork_id: str, fork_name: str) -> str: + """Create a forked workspace (call after fork branch jobs complete).""" + print(f"Creating fork workspace {fork_id} from {self._workspace}") + response = self._client.post( + f"/api/w/{self._workspace}/workspaces/create_fork", + json={"id": fork_id, "name": fork_name}, + ) + if response.status_code // 100 != 2: + raise Exception(response.content.decode()) + return response.content.decode() + + def wait_for_jobs_by_ids(self, job_ids: list, timeout: int = 90): + """Wait for specific jobs (by UUID) to complete.""" + start = time.time() + while time.time() - start < timeout: + all_done = True + for job_id in job_ids: + response = self._client.get( + f"/api/w/{self._workspace}/jobs_u/get/{job_id}", + ) + if response.status_code // 100 != 2: + all_done = False + break + job = response.json() + if job.get("type") != "CompletedJob": + all_done = False + break + if all_done: + return + time.sleep(2) + raise TimeoutError(f"Timed out waiting for jobs {job_ids} after {timeout}s") + + def delete_workspace(self, workspace_id: str): + """Delete a workspace.""" + print(f"Deleting workspace {workspace_id}") + response = self._client.post( + f"/api/w/{workspace_id}/workspaces/delete", + ) + if response.status_code // 100 != 2 and response.status_code != 404: + print(f"Warning: failed to delete workspace {workspace_id}: {response.content.decode()}") + def create_agent_token(self, worker_group="agent", tags=None, exp=None): """ Create an agent JWT token using superadmin privilege. @@ -418,3 +584,163 @@ class WindmillClient: token = response.content.decode().strip('"') print(f"Created agent token: {token}") return token + + +GITEA_HOST_URL = os.environ.get("GITEA_HOST_URL", "http://localhost:3000") +GITEA_DOCKER_URL = os.environ.get("GITEA_DOCKER_URL", "http://gitea:3000") +GITEA_ADMIN_USER = "windmill" +GITEA_ADMIN_PASSWORD = "password123!" +GITEA_ADMIN_EMAIL = "windmill@windmill.dev" + + +class GiteaClient: + _host_url: str + _docker_url: str + _token: str + + def __init__(self): + self._host_url = GITEA_HOST_URL + self._docker_url = GITEA_DOCKER_URL + self._token = None + + def setup_admin(self): + """Create the admin user in Gitea (idempotent) and get an API token.""" + with httpx.Client(base_url=self._host_url, timeout=30.0) as client: + # Create admin user (ignore 422 if exists) + resp = client.post( + "/api/v1/admin/users", + json={ + "username": GITEA_ADMIN_USER, + "password": GITEA_ADMIN_PASSWORD, + "email": GITEA_ADMIN_EMAIL, + "must_change_password": False, + "visibility": "public", + }, + headers={"Content-Type": "application/json"}, + auth=(GITEA_ADMIN_USER, GITEA_ADMIN_PASSWORD), + ) + if resp.status_code == 201: + print(f"Created Gitea admin user '{GITEA_ADMIN_USER}'") + elif resp.status_code == 422: + print(f"Gitea admin user '{GITEA_ADMIN_USER}' already exists") + elif resp.status_code == 401: + # Admin user doesn't exist yet; use the Gitea setup API + resp2 = client.post( + "/user/sign_up", + data={ + "user_name": GITEA_ADMIN_USER, + "password": GITEA_ADMIN_PASSWORD, + "retype": GITEA_ADMIN_PASSWORD, + "email": GITEA_ADMIN_EMAIL, + }, + ) + if resp2.status_code // 100 != 2 and resp2.status_code != 303: + # Try the API endpoint for creating the first user + resp3 = client.post( + "/api/v1/admin/users", + json={ + "username": GITEA_ADMIN_USER, + "password": GITEA_ADMIN_PASSWORD, + "email": GITEA_ADMIN_EMAIL, + "must_change_password": False, + }, + ) + if resp3.status_code // 100 != 2: + raise Exception(f"Failed to create Gitea user: {resp3.status_code} {resp3.text}") + print(f"Created Gitea admin user '{GITEA_ADMIN_USER}' via signup") + else: + raise Exception(f"Failed to create Gitea user: {resp.status_code} {resp.text}") + + # Create API token + token_name = f"integration-test-{uuid.uuid4().hex[:8]}" + resp = client.post( + f"/api/v1/users/{GITEA_ADMIN_USER}/tokens", + json={"name": token_name, "scopes": ["all"]}, + auth=(GITEA_ADMIN_USER, GITEA_ADMIN_PASSWORD), + ) + if resp.status_code // 100 != 2: + raise Exception(f"Failed to create Gitea token: {resp.status_code} {resp.text}") + self._token = resp.json()["sha1"] + print(f"Created Gitea API token: {token_name}") + + def _headers(self): + return { + "Authorization": f"token {self._token}", + "Content-Type": "application/json", + } + + def create_repo(self, name: str) -> str: + """Create a repo and return the docker-internal clone URL with credentials.""" + with httpx.Client(base_url=self._host_url, timeout=30.0) as client: + resp = client.post( + "/api/v1/user/repos", + json={ + "name": name, + "auto_init": True, + "default_branch": "main", + "private": False, + }, + headers=self._headers(), + ) + if resp.status_code // 100 != 2: + raise Exception(f"Failed to create repo {name}: {resp.status_code} {resp.text}") + print(f"Created Gitea repo: {name}") + return f"{self._docker_url}/{GITEA_ADMIN_USER}/{name}.git" + + def get_host_clone_url(self, name: str) -> str: + """Return host-accessible clone URL with credentials.""" + from urllib.parse import urlparse + parsed = urlparse(self._host_url) + return f"http://{GITEA_ADMIN_USER}:{GITEA_ADMIN_PASSWORD}@{parsed.netloc}/{GITEA_ADMIN_USER}/{name}.git" + + def get_docker_clone_url(self, name: str) -> str: + """Return clone URL accessible from the Windmill backend (docker or local).""" + from urllib.parse import urlparse + parsed = urlparse(self._docker_url) + return f"http://{GITEA_ADMIN_USER}:{GITEA_ADMIN_PASSWORD}@{parsed.netloc}/{GITEA_ADMIN_USER}/{name}.git" + + def create_file(self, repo_name: str, file_path: str, content: str, branch: str = "main"): + """Create or update a file in the repo via Gitea API.""" + import base64 + encoded = base64.b64encode(content.encode()).decode() + with httpx.Client(base_url=self._host_url, timeout=30.0) as client: + # Check if file exists (to get SHA for update) + resp = client.get( + f"/api/v1/repos/{GITEA_ADMIN_USER}/{repo_name}/contents/{file_path}", + params={"ref": branch}, + headers=self._headers(), + ) + body = { + "content": encoded, + "message": f"Add {file_path}", + "branch": branch, + } + if resp.status_code == 200: + # File exists — update with PUT + body["sha"] = resp.json()["sha"] + resp = client.put( + f"/api/v1/repos/{GITEA_ADMIN_USER}/{repo_name}/contents/{file_path}", + json=body, + headers=self._headers(), + ) + else: + # File doesn't exist — create with POST + resp = client.post( + f"/api/v1/repos/{GITEA_ADMIN_USER}/{repo_name}/contents/{file_path}", + json=body, + headers=self._headers(), + ) + if resp.status_code // 100 != 2: + raise Exception(f"Failed to create file {file_path}: {resp.status_code} {resp.text}") + print(f"Created file {file_path} in {repo_name}") + + def delete_repo(self, name: str): + with httpx.Client(base_url=self._host_url, timeout=30.0) as client: + resp = client.delete( + f"/api/v1/repos/{GITEA_ADMIN_USER}/{name}", + headers=self._headers(), + ) + if resp.status_code // 100 == 2 or resp.status_code == 404: + print(f"Deleted Gitea repo: {name}") + else: + print(f"Warning: failed to delete repo {name}: {resp.status_code}")