feat(python-client): retry a GET on 429 and 502-504 with exponential backoff (#12513)

* feat(python-client): retry a GET on 429 and 502-504 with exponential backoff

A script that reads many variables or resources at once can get a 429 or a
502/503/504 back from a busy server, and the call failed at once. Windmill.get
now retries up to 3 times, 1s, 2s and 4s apart, or after the Retry-After
seconds the server sends (capped at 60). Other statuses, and POSTs, are not
retried.

* fix(python-client): honour an HTTP-date Retry-After and test the 502/504 and 60s cap paths

* fix(python-client): never repeat the slack approval GET and bound GET retries to 15s

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>

* fix(python-client): fall back to the backoff delay on a Retry-After digit float() rejects

---------

Co-authored-by: Ruben Fiszel <ruben@windmill.dev>
Co-authored-by: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
Raunak Burrows
2026-11-19 10:02:15 +01:00
committed by GitHub
co-authored by Claude Opus 5.5 Ruben Fiszel
parent 1fa8fd0c94
commit 1c78f8d7a3
2 changed files with 114 additions and 3 deletions
@@ -0,0 +1,56 @@
"""Windmill.get retries a GET the server rejected under load, with backoff."""
import httpx
import pytest
from wmill.client import Windmill
def _client(statuses, monkeypatch):
sent, sleeps = [], []
responses = iter(statuses)
def handler(request: httpx.Request) -> httpx.Response:
sent.append(request)
status, headers = next(responses)
return httpx.Response(status, headers=headers, json="value")
monkeypatch.setattr("wmill.client.time.sleep", sleeps.append)
client = Windmill(base_url="http://wm.test", token="t", workspace="ws")
client.client = httpx.Client(base_url=client.base_url, transport=httpx.MockTransport(handler))
return client, sent, sleeps
@pytest.mark.parametrize(
"statuses, retry_budget, expected",
[
([(503, {}), (429, {"Retry-After": "5"}), (200, {})], None, (200, 3, [1, 5])),
([(503, {})] * 4, None, (503, 4, [1, 2, 4])),
# a Retry-After past the remaining budget ends the retries
([(502, {}), (504, {"Retry-After": "120"})], None, (504, 2, [1])),
([(429, {"Retry-After": "10"}), (429, {"Retry-After": "10"})], None, (429, 2, [10])),
([(503, {"Retry-After": "Fri, 01 Jan 2100 00:00:00 GMT"})], None, (503, 1, [])),
([(503, {"Retry-After": "Wed, 21 Oct 2015 07:28:00 GMT"}), (200, {})], None, (200, 2, [0])),
# "²" is a digit float() rejects: it falls back to the backoff delay
([(503, {"Retry-After": "²".encode()}), (200, {})], None, (200, 2, [1])),
([(503, {})] * 3, 3, (503, 3, [1, 2])),
([(404, {})], None, (404, 1, [])),
([(200, {})], None, (200, 1, [])),
],
)
def test_get_retries_transient_errors(statuses, retry_budget, expected, monkeypatch):
client, sent, sleeps = _client(statuses, monkeypatch)
resp = client.get(
"/w/ws/variables/get_value/u/a/b", raise_for_status=False, retry_budget=retry_budget
)
assert (resp.status_code, len(sent), sleeps) == expected
def test_slack_approval_request_is_not_repeated(monkeypatch):
# the endpoint posts the approval message: a proxy 502 after it did must not post a second
client, sent, sleeps = _client([(502, {}), (200, {})], monkeypatch)
monkeypatch.setenv("WM_JOB_ID", "job")
monkeypatch.setenv("WM_FLOW_JOB_ID", "flow")
with pytest.raises(Exception):
client.request_interactive_slack_approval("u/a/slack", "C1")
assert (len(sent), sleeps) == (1, [])
+58 -3
View File
@@ -2,6 +2,7 @@ from __future__ import annotations
import atexit
import datetime as dt
from email.utils import parsedate_to_datetime
import functools
from io import BufferedReader, BytesIO
import logging
@@ -30,6 +31,27 @@ logger = logging.getLogger("windmill_client")
JobStatus = Literal["RUNNING", "WAITING", "COMPLETED"]
_TRANSIENT_GET_STATUSES = (429, 502, 503, 504)
_GET_RETRIES = 3
# Total sleep one GET may spend retrying; a Retry-After beyond it ends the retries.
_GET_RETRY_BUDGET_SECONDS = 15.0
def _retry_after_seconds(retry_after: str, default: float) -> float:
"""Seconds a Retry-After header (delay-seconds or an HTTP-date) asks to wait."""
retry_after = retry_after.strip()
if not retry_after:
return default
if retry_after.isdecimal():
return float(retry_after)
try:
when = parsedate_to_datetime(retry_after)
except (TypeError, ValueError):
return default
if when.tzinfo is None:
when = when.replace(tzinfo=dt.timezone.utc)
return max((when - dt.datetime.now(dt.timezone.utc)).total_seconds(), 0.0)
def _sign_s3_objects_body(s3_objects: list, expiry_secs: int | None) -> dict:
# `expiry_secs` is optional but not nullable in the spec, so omit it rather than
@@ -111,19 +133,49 @@ class Windmill:
timeout=httpx.Timeout(900.0),
)
def get(self, endpoint, raise_for_status=True, **kwargs) -> httpx.Response:
def get(
self,
endpoint,
raise_for_status=True,
*,
retry_transient: bool = True,
retry_budget: float | None = None,
**kwargs,
) -> httpx.Response:
"""Make an HTTP GET request to the Windmill API.
Args:
endpoint: API endpoint path
raise_for_status: Whether to raise an exception on HTTP errors
retry_transient: Retry on 429 and 502-504 with exponential backoff, honoring
Retry-After. Pass False for an endpoint with a side effect.
retry_budget: Most seconds to spend sleeping between retries (at most 15)
**kwargs: Additional arguments passed to httpx.get
Returns:
HTTP response object
"""
endpoint = endpoint.lstrip("/")
resp = self.client.get(f"/{endpoint}", **kwargs)
# A 502/504 can come from a proxy after the server handled the request, so only
# a GET without side effects may be repeated: one with them passes retry_transient=False.
budget = _GET_RETRY_BUDGET_SECONDS
if retry_budget is not None:
budget = min(budget, max(retry_budget, 0.0))
attempt = 0
while True:
resp = self.client.get(f"/{endpoint}", **kwargs)
if (
not retry_transient
or resp.status_code not in _TRANSIENT_GET_STATUSES
or attempt == _GET_RETRIES
):
break
delay = _retry_after_seconds(resp.headers.get("Retry-After", ""), 2**attempt)
if delay > budget:
break
time.sleep(delay)
budget -= delay
attempt += 1
if raise_for_status:
try:
resp.raise_for_status()
@@ -420,7 +472,9 @@ class Windmill:
while True:
result_res = self.get(
f"/w/{self.workspace}/jobs_u/completed/get_result_maybe/{job_id}", True
f"/w/{self.workspace}/jobs_u/completed/get_result_maybe/{job_id}",
True,
retry_budget=(timeout - (time.time() - start_time)) if timeout else None,
).json()
started = result_res["started"]
@@ -1539,6 +1593,7 @@ class Windmill:
self.get(
f"/w/{workspace}/jobs/slack_approval/{os.environ.get('WM_JOB_ID', 'NO_JOB_ID')}",
params=params,
retry_transient=False,
)
def username_to_email(self, username: str) -> str: