From 1c78f8d7a38e49106adcd236f53c026855924682 Mon Sep 17 00:00:00 2001 From: Raunak Burrows Date: Thu, 19 Nov 2026 09:02:15 +0000 Subject: [PATCH] feat(python-client): retry a GET on 429 and 502-504 with exponential backoff (#12513) * feat(python-client): retry a GET on 429 and 502-504 with exponential backoff A script that reads many variables or resources at once can get a 429 or a 502/503/504 back from a busy server, and the call failed at once. Windmill.get now retries up to 3 times, 1s, 2s and 4s apart, or after the Retry-After seconds the server sends (capped at 60). Other statuses, and POSTs, are not retried. * fix(python-client): honour an HTTP-date Retry-After and test the 502/504 and 60s cap paths * fix(python-client): never repeat the slack approval GET and bound GET retries to 15s Co-Authored-By: Claude Opus 5.5 * fix(python-client): fall back to the backoff delay on a Retry-After digit float() rejects --------- Co-authored-by: Ruben Fiszel Co-authored-by: Claude Opus 5.5 --- python-client/wmill/tests/test_get_retry.py | 56 +++++++++++++++++++ python-client/wmill/wmill/client.py | 61 ++++++++++++++++++++- 2 files changed, 114 insertions(+), 3 deletions(-) create mode 100644 python-client/wmill/tests/test_get_retry.py diff --git a/python-client/wmill/tests/test_get_retry.py b/python-client/wmill/tests/test_get_retry.py new file mode 100644 index 0000000000..eb0784503f --- /dev/null +++ b/python-client/wmill/tests/test_get_retry.py @@ -0,0 +1,56 @@ +"""Windmill.get retries a GET the server rejected under load, with backoff.""" + +import httpx +import pytest + +from wmill.client import Windmill + + +def _client(statuses, monkeypatch): + sent, sleeps = [], [] + responses = iter(statuses) + + def handler(request: httpx.Request) -> httpx.Response: + sent.append(request) + status, headers = next(responses) + return httpx.Response(status, headers=headers, json="value") + + monkeypatch.setattr("wmill.client.time.sleep", sleeps.append) + client = Windmill(base_url="http://wm.test", token="t", workspace="ws") + client.client = httpx.Client(base_url=client.base_url, transport=httpx.MockTransport(handler)) + return client, sent, sleeps + + +@pytest.mark.parametrize( + "statuses, retry_budget, expected", + [ + ([(503, {}), (429, {"Retry-After": "5"}), (200, {})], None, (200, 3, [1, 5])), + ([(503, {})] * 4, None, (503, 4, [1, 2, 4])), + # a Retry-After past the remaining budget ends the retries + ([(502, {}), (504, {"Retry-After": "120"})], None, (504, 2, [1])), + ([(429, {"Retry-After": "10"}), (429, {"Retry-After": "10"})], None, (429, 2, [10])), + ([(503, {"Retry-After": "Fri, 01 Jan 2100 00:00:00 GMT"})], None, (503, 1, [])), + ([(503, {"Retry-After": "Wed, 21 Oct 2015 07:28:00 GMT"}), (200, {})], None, (200, 2, [0])), + # "²" is a digit float() rejects: it falls back to the backoff delay + ([(503, {"Retry-After": "²".encode()}), (200, {})], None, (200, 2, [1])), + ([(503, {})] * 3, 3, (503, 3, [1, 2])), + ([(404, {})], None, (404, 1, [])), + ([(200, {})], None, (200, 1, [])), + ], +) +def test_get_retries_transient_errors(statuses, retry_budget, expected, monkeypatch): + client, sent, sleeps = _client(statuses, monkeypatch) + resp = client.get( + "/w/ws/variables/get_value/u/a/b", raise_for_status=False, retry_budget=retry_budget + ) + assert (resp.status_code, len(sent), sleeps) == expected + + +def test_slack_approval_request_is_not_repeated(monkeypatch): + # the endpoint posts the approval message: a proxy 502 after it did must not post a second + client, sent, sleeps = _client([(502, {}), (200, {})], monkeypatch) + monkeypatch.setenv("WM_JOB_ID", "job") + monkeypatch.setenv("WM_FLOW_JOB_ID", "flow") + with pytest.raises(Exception): + client.request_interactive_slack_approval("u/a/slack", "C1") + assert (len(sent), sleeps) == (1, []) diff --git a/python-client/wmill/wmill/client.py b/python-client/wmill/wmill/client.py index d9af5b03b8..1dabd85684 100644 --- a/python-client/wmill/wmill/client.py +++ b/python-client/wmill/wmill/client.py @@ -2,6 +2,7 @@ from __future__ import annotations import atexit import datetime as dt +from email.utils import parsedate_to_datetime import functools from io import BufferedReader, BytesIO import logging @@ -30,6 +31,27 @@ logger = logging.getLogger("windmill_client") JobStatus = Literal["RUNNING", "WAITING", "COMPLETED"] +_TRANSIENT_GET_STATUSES = (429, 502, 503, 504) +_GET_RETRIES = 3 +# Total sleep one GET may spend retrying; a Retry-After beyond it ends the retries. +_GET_RETRY_BUDGET_SECONDS = 15.0 + + +def _retry_after_seconds(retry_after: str, default: float) -> float: + """Seconds a Retry-After header (delay-seconds or an HTTP-date) asks to wait.""" + retry_after = retry_after.strip() + if not retry_after: + return default + if retry_after.isdecimal(): + return float(retry_after) + try: + when = parsedate_to_datetime(retry_after) + except (TypeError, ValueError): + return default + if when.tzinfo is None: + when = when.replace(tzinfo=dt.timezone.utc) + return max((when - dt.datetime.now(dt.timezone.utc)).total_seconds(), 0.0) + def _sign_s3_objects_body(s3_objects: list, expiry_secs: int | None) -> dict: # `expiry_secs` is optional but not nullable in the spec, so omit it rather than @@ -111,19 +133,49 @@ class Windmill: timeout=httpx.Timeout(900.0), ) - def get(self, endpoint, raise_for_status=True, **kwargs) -> httpx.Response: + def get( + self, + endpoint, + raise_for_status=True, + *, + retry_transient: bool = True, + retry_budget: float | None = None, + **kwargs, + ) -> httpx.Response: """Make an HTTP GET request to the Windmill API. Args: endpoint: API endpoint path raise_for_status: Whether to raise an exception on HTTP errors + retry_transient: Retry on 429 and 502-504 with exponential backoff, honoring + Retry-After. Pass False for an endpoint with a side effect. + retry_budget: Most seconds to spend sleeping between retries (at most 15) **kwargs: Additional arguments passed to httpx.get Returns: HTTP response object """ endpoint = endpoint.lstrip("/") - resp = self.client.get(f"/{endpoint}", **kwargs) + # A 502/504 can come from a proxy after the server handled the request, so only + # a GET without side effects may be repeated: one with them passes retry_transient=False. + budget = _GET_RETRY_BUDGET_SECONDS + if retry_budget is not None: + budget = min(budget, max(retry_budget, 0.0)) + attempt = 0 + while True: + resp = self.client.get(f"/{endpoint}", **kwargs) + if ( + not retry_transient + or resp.status_code not in _TRANSIENT_GET_STATUSES + or attempt == _GET_RETRIES + ): + break + delay = _retry_after_seconds(resp.headers.get("Retry-After", ""), 2**attempt) + if delay > budget: + break + time.sleep(delay) + budget -= delay + attempt += 1 if raise_for_status: try: resp.raise_for_status() @@ -420,7 +472,9 @@ class Windmill: while True: result_res = self.get( - f"/w/{self.workspace}/jobs_u/completed/get_result_maybe/{job_id}", True + f"/w/{self.workspace}/jobs_u/completed/get_result_maybe/{job_id}", + True, + retry_budget=(timeout - (time.time() - start_time)) if timeout else None, ).json() started = result_res["started"] @@ -1539,6 +1593,7 @@ class Windmill: self.get( f"/w/{workspace}/jobs/slack_approval/{os.environ.get('WM_JOB_ID', 'NO_JOB_ID')}", params=params, + retry_transient=False, ) def username_to_email(self, username: str) -> str: