mirror of
https://github.com/windmill-labs/windmill.git
synced 2026-11-20 00:07:01 +00:00
feat(python-client): retry a GET on 429 and 502-504 with exponential backoff (#12513)
* feat(python-client): retry a GET on 429 and 502-504 with exponential backoff A script that reads many variables or resources at once can get a 429 or a 502/503/504 back from a busy server, and the call failed at once. Windmill.get now retries up to 3 times, 1s, 2s and 4s apart, or after the Retry-After seconds the server sends (capped at 60). Other statuses, and POSTs, are not retried. * fix(python-client): honour an HTTP-date Retry-After and test the 502/504 and 60s cap paths * fix(python-client): never repeat the slack approval GET and bound GET retries to 15s Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com> * fix(python-client): fall back to the backoff delay on a Retry-After digit float() rejects --------- Co-authored-by: Ruben Fiszel <ruben@windmill.dev> Co-authored-by: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
co-authored by
Claude Opus 5.5
Ruben Fiszel
parent
1fa8fd0c94
commit
1c78f8d7a3
@@ -0,0 +1,56 @@
|
||||
"""Windmill.get retries a GET the server rejected under load, with backoff."""
|
||||
|
||||
import httpx
|
||||
import pytest
|
||||
|
||||
from wmill.client import Windmill
|
||||
|
||||
|
||||
def _client(statuses, monkeypatch):
|
||||
sent, sleeps = [], []
|
||||
responses = iter(statuses)
|
||||
|
||||
def handler(request: httpx.Request) -> httpx.Response:
|
||||
sent.append(request)
|
||||
status, headers = next(responses)
|
||||
return httpx.Response(status, headers=headers, json="value")
|
||||
|
||||
monkeypatch.setattr("wmill.client.time.sleep", sleeps.append)
|
||||
client = Windmill(base_url="http://wm.test", token="t", workspace="ws")
|
||||
client.client = httpx.Client(base_url=client.base_url, transport=httpx.MockTransport(handler))
|
||||
return client, sent, sleeps
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"statuses, retry_budget, expected",
|
||||
[
|
||||
([(503, {}), (429, {"Retry-After": "5"}), (200, {})], None, (200, 3, [1, 5])),
|
||||
([(503, {})] * 4, None, (503, 4, [1, 2, 4])),
|
||||
# a Retry-After past the remaining budget ends the retries
|
||||
([(502, {}), (504, {"Retry-After": "120"})], None, (504, 2, [1])),
|
||||
([(429, {"Retry-After": "10"}), (429, {"Retry-After": "10"})], None, (429, 2, [10])),
|
||||
([(503, {"Retry-After": "Fri, 01 Jan 2100 00:00:00 GMT"})], None, (503, 1, [])),
|
||||
([(503, {"Retry-After": "Wed, 21 Oct 2015 07:28:00 GMT"}), (200, {})], None, (200, 2, [0])),
|
||||
# "²" is a digit float() rejects: it falls back to the backoff delay
|
||||
([(503, {"Retry-After": "²".encode()}), (200, {})], None, (200, 2, [1])),
|
||||
([(503, {})] * 3, 3, (503, 3, [1, 2])),
|
||||
([(404, {})], None, (404, 1, [])),
|
||||
([(200, {})], None, (200, 1, [])),
|
||||
],
|
||||
)
|
||||
def test_get_retries_transient_errors(statuses, retry_budget, expected, monkeypatch):
|
||||
client, sent, sleeps = _client(statuses, monkeypatch)
|
||||
resp = client.get(
|
||||
"/w/ws/variables/get_value/u/a/b", raise_for_status=False, retry_budget=retry_budget
|
||||
)
|
||||
assert (resp.status_code, len(sent), sleeps) == expected
|
||||
|
||||
|
||||
def test_slack_approval_request_is_not_repeated(monkeypatch):
|
||||
# the endpoint posts the approval message: a proxy 502 after it did must not post a second
|
||||
client, sent, sleeps = _client([(502, {}), (200, {})], monkeypatch)
|
||||
monkeypatch.setenv("WM_JOB_ID", "job")
|
||||
monkeypatch.setenv("WM_FLOW_JOB_ID", "flow")
|
||||
with pytest.raises(Exception):
|
||||
client.request_interactive_slack_approval("u/a/slack", "C1")
|
||||
assert (len(sent), sleeps) == (1, [])
|
||||
@@ -2,6 +2,7 @@ from __future__ import annotations
|
||||
|
||||
import atexit
|
||||
import datetime as dt
|
||||
from email.utils import parsedate_to_datetime
|
||||
import functools
|
||||
from io import BufferedReader, BytesIO
|
||||
import logging
|
||||
@@ -30,6 +31,27 @@ logger = logging.getLogger("windmill_client")
|
||||
|
||||
JobStatus = Literal["RUNNING", "WAITING", "COMPLETED"]
|
||||
|
||||
_TRANSIENT_GET_STATUSES = (429, 502, 503, 504)
|
||||
_GET_RETRIES = 3
|
||||
# Total sleep one GET may spend retrying; a Retry-After beyond it ends the retries.
|
||||
_GET_RETRY_BUDGET_SECONDS = 15.0
|
||||
|
||||
|
||||
def _retry_after_seconds(retry_after: str, default: float) -> float:
|
||||
"""Seconds a Retry-After header (delay-seconds or an HTTP-date) asks to wait."""
|
||||
retry_after = retry_after.strip()
|
||||
if not retry_after:
|
||||
return default
|
||||
if retry_after.isdecimal():
|
||||
return float(retry_after)
|
||||
try:
|
||||
when = parsedate_to_datetime(retry_after)
|
||||
except (TypeError, ValueError):
|
||||
return default
|
||||
if when.tzinfo is None:
|
||||
when = when.replace(tzinfo=dt.timezone.utc)
|
||||
return max((when - dt.datetime.now(dt.timezone.utc)).total_seconds(), 0.0)
|
||||
|
||||
|
||||
def _sign_s3_objects_body(s3_objects: list, expiry_secs: int | None) -> dict:
|
||||
# `expiry_secs` is optional but not nullable in the spec, so omit it rather than
|
||||
@@ -111,19 +133,49 @@ class Windmill:
|
||||
timeout=httpx.Timeout(900.0),
|
||||
)
|
||||
|
||||
def get(self, endpoint, raise_for_status=True, **kwargs) -> httpx.Response:
|
||||
def get(
|
||||
self,
|
||||
endpoint,
|
||||
raise_for_status=True,
|
||||
*,
|
||||
retry_transient: bool = True,
|
||||
retry_budget: float | None = None,
|
||||
**kwargs,
|
||||
) -> httpx.Response:
|
||||
"""Make an HTTP GET request to the Windmill API.
|
||||
|
||||
Args:
|
||||
endpoint: API endpoint path
|
||||
raise_for_status: Whether to raise an exception on HTTP errors
|
||||
retry_transient: Retry on 429 and 502-504 with exponential backoff, honoring
|
||||
Retry-After. Pass False for an endpoint with a side effect.
|
||||
retry_budget: Most seconds to spend sleeping between retries (at most 15)
|
||||
**kwargs: Additional arguments passed to httpx.get
|
||||
|
||||
Returns:
|
||||
HTTP response object
|
||||
"""
|
||||
endpoint = endpoint.lstrip("/")
|
||||
resp = self.client.get(f"/{endpoint}", **kwargs)
|
||||
# A 502/504 can come from a proxy after the server handled the request, so only
|
||||
# a GET without side effects may be repeated: one with them passes retry_transient=False.
|
||||
budget = _GET_RETRY_BUDGET_SECONDS
|
||||
if retry_budget is not None:
|
||||
budget = min(budget, max(retry_budget, 0.0))
|
||||
attempt = 0
|
||||
while True:
|
||||
resp = self.client.get(f"/{endpoint}", **kwargs)
|
||||
if (
|
||||
not retry_transient
|
||||
or resp.status_code not in _TRANSIENT_GET_STATUSES
|
||||
or attempt == _GET_RETRIES
|
||||
):
|
||||
break
|
||||
delay = _retry_after_seconds(resp.headers.get("Retry-After", ""), 2**attempt)
|
||||
if delay > budget:
|
||||
break
|
||||
time.sleep(delay)
|
||||
budget -= delay
|
||||
attempt += 1
|
||||
if raise_for_status:
|
||||
try:
|
||||
resp.raise_for_status()
|
||||
@@ -420,7 +472,9 @@ class Windmill:
|
||||
|
||||
while True:
|
||||
result_res = self.get(
|
||||
f"/w/{self.workspace}/jobs_u/completed/get_result_maybe/{job_id}", True
|
||||
f"/w/{self.workspace}/jobs_u/completed/get_result_maybe/{job_id}",
|
||||
True,
|
||||
retry_budget=(timeout - (time.time() - start_time)) if timeout else None,
|
||||
).json()
|
||||
|
||||
started = result_res["started"]
|
||||
@@ -1539,6 +1593,7 @@ class Windmill:
|
||||
self.get(
|
||||
f"/w/{workspace}/jobs/slack_approval/{os.environ.get('WM_JOB_ID', 'NO_JOB_ID')}",
|
||||
params=params,
|
||||
retry_transient=False,
|
||||
)
|
||||
|
||||
def username_to_email(self, username: str) -> str:
|
||||
|
||||
Reference in New Issue
Block a user