test: wait on the seed script's own git sync push in e2e tests (#11854)

* test: wait on the seed script's own git sync push in e2e tests

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>

* chore: TEMP loop git sync e2e with race diagnostics (to be dropped)

---------

Co-authored-by: Claude Opus 5.5 <noreply@anthropic.com>

.github/workflows/git-sync-test.yml

@@ -206,14 +206,21 @@ jobs:
           curl -sf http://localhost:8000/api/version > /dev/null || { echo "Windmill failed to start"; exit 1; }
 
       - name: Run git sync E2E tests
-        timeout-minutes: 10
+        timeout-minutes: 75
         env:
           GITEA_DOCKER_URL: http://localhost:3000
           LICENSE_KEY: ${{ secrets.WM_LICENSE_KEY_CI }}
         run: |
           python3 -m venv .venv
           .venv/bin/pip install -r integration_tests/requirements.txt
-          cd integration_tests && ../.venv/bin/python -m unittest -v test.git_sync_test
+          cd integration_tests
+          fails=0
+          for i in $(seq 1 8); do
+            echo "TEMP-LOOP iteration $i"
+            ../.venv/bin/python -m unittest -v test.git_sync_test || fails=$((fails+1))
+          done
+          echo "TEMP-LOOP failed iterations: $fails"
+          [ "$fails" = 0 ]
 
       - name: Archive logs
         uses: actions/upload-artifact@v4

integration_tests/test/git_sync_test.py

@@ -215,17 +215,58 @@ class GitSyncTestBase(unittest.TestCase):
                 return repo
         return None
 
+    def _new_sync_jobs(self, seen: set, path: str) -> list:
+        """Completed deploy callbacks not in `seen` whose items include `path`.
+        A callback debounced into another completes as skipped with its items
+        still in its args, so skipped ones are left out: only the survivor,
+        whose args carry the accumulated items, ran the push."""
+        jobs = []
+        for listed in self._client.get_completed_jobs(job_kinds="deploymentcallback"):
+            if listed["id"] in seen:
+                continue
+            job = self._client._client.get(
+                f"/api/w/{self._client._workspace}/jobs_u/get/{listed['id']}"
+            ).json()
+            if job.get("is_skipped"):
+                continue
+            items = (job.get("args") or {}).get("items") or []
+            if any(item.get("path") == path for item in items):
+                jobs.append(job)
+        return jobs
+
     def _deploy_seed_script(self, name_prefix: str) -> str:
-        """Deploy a script and wait for its push to git. Returns the script path."""
-        initial_count = self._client.count_deployment_callback_jobs()
+        """Deploy a script and wait for its push to git. Returns the script path.
+
+        Waits on the callback that carries this script, not on the workspace's
+        callback count: a callback still finishing from an earlier test raises
+        the count before this push lands."""
+        seen = {j["id"] for j in self._client.get_completed_jobs(job_kinds="deploymentcallback")}
         script_path = f"u/admin/{unique_name(name_prefix)}"
         self._client.create_script(
             path=script_path,
             content=ts_script("return 'seed'"),
             language="bun",
         )
-        self._client.wait_for_sync_jobs(initial_count, min_new=1)
-        time.sleep(3)
+        self._wait_until(
+            lambda: self._new_sync_jobs(seen, script_path),
+            timeout=90,
+            interval=2,
+            message=f"no push job for {script_path}",
+        )
+        # TEMP-DIAG: would the old count-based wait have returned before our push?
+        ours = self._new_sync_jobs(seen, script_path)
+        ours_ids = {j["id"] for j in ours}
+        ours_done = min(j.get("completed_at") or "" for j in ours)
+        others = [
+            (j["id"], j.get("completed_at"), (j.get("args") or {}).get("repo_url_resource_path"))
+            for j in self._client.get_completed_jobs(job_kinds="deploymentcallback")
+            if j["id"] not in seen and j["id"] not in ours_ids
+            and (j.get("completed_at") or "") < ours_done
+        ]
+        print(f"TEMP-DIAG seed={script_path} ours_done={ours_done} earlier_others={others}"
+              + (" RACE-HIT" if others else ""), flush=True)
+        failed = [j for j in self._new_sync_jobs(seen, script_path) if not j.get("success")]
+        self.assertEqual(failed, [], f"the push of {script_path} failed")
         return script_path
 
     def _repo_script_file(self, repo_name: str, script_path: str, branch: str = None) -> str:
@@ -773,17 +814,7 @@ class TestGitSync(GitSyncTestBase):
 
     def _new_sync_job_results(self, seen: set, path: str) -> list:
         """Results of the deploy callbacks not in `seen` that pushed `path`."""
-        results = []
-        for listed in self._client.get_completed_jobs(job_kinds="deploymentcallback"):
-            if listed["id"] in seen:
-                continue
-            job = self._client._client.get(
-                f"/api/w/{self._client._workspace}/jobs_u/get/{listed['id']}"
-            ).json()
-            items = (job.get("args") or {}).get("items") or []
-            if any(item.get("path") == path for item in items):
-                results.append(job.get("result"))
-        return results
+        return [job.get("result") for job in self._new_sync_jobs(seen, path)]
 
     def test_promotion_push_reports_whether_it_pushed(self):
         """The push job's result says whether a commit was pushed, and the