feat: retry a workflow-as-code task from its task options (#11013)

* feat: retry a workflow-as-code task from its task options Co-Authored-By: Claude Opus 5 (1M context) <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_01WvjgKMRNtRNPnAkg6MkiTA * fix: claim every retry attempt key up front, so a step cannot alias one Co-Authored-By: Claude Opus 5 (1M context) <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_01WvjgKMRNtRNPnAkg6MkiTA * fix: bound retry attempts, which now claim their keys up front Co-Authored-By: Claude Opus 5 (1M context) <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_01WvjgKMRNtRNPnAkg6MkiTA * fix: honour an explicit zero retry multiplier in the python client Co-Authored-By: Claude Opus 5 (1M context) <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_01WvjgKMRNtRNPnAkg6MkiTA * docs: state the retry validation rules once in the task docstring Co-Authored-By: Claude Opus 5 (1M context) <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_01WvjgKMRNtRNPnAkg6MkiTA --------- Co-authored-by: Claude Opus 5 (1M context) <noreply@anthropic.com>
2026-09-09 00:04:10 +00:00 · 2026-09-08 11:37:56 +02:00
parent 33f9828c3e
commit d3f305db98
15 changed files with 1126 additions and 106 deletions
@@ -4580,6 +4580,22 @@ def parse_sql_client_name(name: str) -> tuple[str, Optional[str]]
 # decoded back before the caller sees it: a \`\`datetime\`\` comes back as a
 # string, a tuple as a list.
 # 
+# \`\`retry\`\` re-dispatches the task after a failure, inside \`\`@workflow\`\` only.
+# Every attempt is a step of its own (\`\`call_api\`\`, \`\`call_api#2\`\`, ...) and
+# the wait between two of them is a durable sleep, so a retrying task holds no
+# worker while it backs off. Keys: \`\`attempts\`\` (retries after the first
+# failure, a whole number from 0 to 100), \`\`delay\`\` (seconds before the first
+# retry, sub-second delays dropped), \`\`multiplier\`\` (applied to the delay
+# after each attempt, 1 keeps it constant), \`\`max_delay\`\` (ceiling in
+# seconds). \`\`attempts\`\` is required, and an out-of-range or unknown key is
+# rejected where the policy is written.
+# 
+# A workflow sleeps once per round, so tasks backing off in the same fan-out
+# wait one after another rather than together: the delay before a fan-out
+# retries is the sum of every backoff pending in it, not the longest one, and
+# it grows with both the width of the fan-out and \`\`attempts\`\`. Retries with
+# no \`\`delay\`\` all go out in a single round.
+# 
 # Usage::
 # 
 #     @task
@@ -4587,10 +4603,15 @@ def parse_sql_client_name(name: str) -> tuple[str, Optional[str]]
 # 
 #     @task(path="f/external_script", timeout=600, tag="gpu")
 #     async def run_external(x: int): ...
-def task(_func = None, path: Optional[str] = None, tag: Optional[str] = None, timeout: Optional[int] = None, cache_ttl: Optional[int] = None, priority: Optional[int] = None, concurrency_limit: Optional[int] = None, concurrency_key: Optional[str] = None, concurrency_time_window_s: Optional[int] = None)
+# 
+#     @task(retry={"attempts": 3, "delay": 30, "multiplier": 2})
+#     async def call_api(payload: dict): ...
+def task(_func = None, path: Optional[str] = None, tag: Optional[str] = None, timeout: Optional[int] = None, cache_ttl: Optional[int] = None, priority: Optional[int] = None, concurrency_limit: Optional[int] = None, concurrency_key: Optional[str] = None, concurrency_time_window_s: Optional[int] = None, retry: Optional[dict] = None)

 # Create a task that dispatches to a separate Windmill script.
 # 
+# \`\`retry\`\` takes the same policy as :func:\`task\`.
+# 
 # Usage::
 # 
 #     extract = task_script("f/data/extract", timeout=600)
@@ -4598,10 +4619,12 @@ def task(_func = None, path: Optional[str] = None, tag: Optional[str] = None, ti
 #     @workflow
 #     async def main():
 #         data = await extract(url="https://...")
-def task_script(path: str, timeout: Optional[int] = None, tag: Optional[str] = None, cache_ttl: Optional[int] = None, priority: Optional[int] = None, concurrency_limit: Optional[int] = None, concurrency_key: Optional[str] = None, concurrency_time_window_s: Optional[int] = None)
+def task_script(path: str, timeout: Optional[int] = None, tag: Optional[str] = None, cache_ttl: Optional[int] = None, priority: Optional[int] = None, concurrency_limit: Optional[int] = None, concurrency_key: Optional[str] = None, concurrency_time_window_s: Optional[int] = None, retry: Optional[dict] = None)

 # Create a task that dispatches to a separate Windmill flow.
 # 
+# \`\`retry\`\` takes the same policy as :func:\`task\`.
+# 
 # Usage::
 # 
 #     pipeline = task_flow("f/etl/pipeline", priority=10)
@@ -4609,7 +4632,7 @@ def task_script(path: str, timeout: Optional[int] = None, tag: Optional[str] = N
 #     @workflow
 #     async def main():
 #         result = await pipeline(input=data)
-def task_flow(path: str, timeout: Optional[int] = None, tag: Optional[str] = None, cache_ttl: Optional[int] = None, priority: Optional[int] = None, concurrency_limit: Optional[int] = None, concurrency_key: Optional[str] = None, concurrency_time_window_s: Optional[int] = None)
+def task_flow(path: str, timeout: Optional[int] = None, tag: Optional[str] = None, cache_ttl: Optional[int] = None, priority: Optional[int] = None, concurrency_limit: Optional[int] = None, concurrency_key: Optional[str] = None, concurrency_time_window_s: Optional[int] = None, retry: Optional[dict] = None)

 # Decorator marking an async function as a workflow-as-code entry point.
 # 
@@ -6657,6 +6680,34 @@ A caught failure reads the same whether it came from a task or from a \`step()\`
 Import: \`import { workflow, task, taskScript, taskFlow, step, sleep, waitForApproval, getApprovalUrls, getResumeUrls, parallel } from "windmill-client"\`

 \`\`\`typescript
+/**
+ * Re-dispatch policy for a failed task.
+ *
+ * Every attempt is a step of its own (\`fetch\`, \`fetch#2\`, \`fetch#3\`), and the
+ * wait between two of them is a durable sleep, so a retrying task holds no
+ * worker while it backs off.
+ *
+ * A workflow sleeps once per round, so tasks backing off in the same fan-out
+ * wait one after another rather than together: the delay before a fan-out
+ * retries is the sum of every backoff pending in it, not the longest one, and
+ * it grows with both the width of the fan-out and \`attempts\`. Retries with no
+ * \`delay\` all go out in a single round.
+ */
+export interface TaskRetry {
+  /** Attempts after the first failure: \`2\` runs the task at most 3 times.
+  *  A whole number from 0 to 100; anything else is rejected where the policy
+  *  is written. */
+  attempts: number;
+  /** Seconds to wait before the first retry. Default 0, retry immediately.
+  *  Sub-second delays are dropped — a durable sleep resolves to the second. */
+  delay?: number;
+  /** Applied to the delay after each attempt: 1 (the default) keeps it
+  *  constant, 2 doubles it. */
+  multiplier?: number;
+  /** Ceiling for the delay in seconds, for a \`multiplier\` above 1. */
+  max_delay?: number;
+}
+
 export interface TaskOptions {
  timeout?: number;
  tag?: string;
@@ -6665,6 +6716,7 @@ export interface TaskOptions {
  concurrency_limit?: number;
  concurrency_key?: string;
  concurrency_time_window_s?: number;
+  retry?: TaskRetry;
 }

 /**
@@ -6682,9 +6734,11 @@ export async function getResumeUrls(approver?: string, flowLevel?: boolean): Pro
 * @example
 * const extract_data = task(async (url: string) => { ... });
 * const run_external = task("f/external_script", async (x: number) => { ... });
+ * const call_api = task(fetchOrders, { retry: { attempts: 3, delay: 30, multiplier: 2 } });
 *
 * Inside a \`workflow()\`, calling a task dispatches it as a step.
- * Outside a workflow, the function body executes directly.
+ * Outside a workflow, the function body executes directly and
+ * {@link TaskOptions} — retry included — does not apply.
 *
 * A task runs as its own job, so its result is always encoded as JSON and
 * decoded back before the caller sees it: a \`Date\` comes back as a string, a
@@ -6828,6 +6882,22 @@ def get_resume_urls(approver: str = None, flow_level: bool = None) -> dict
 # decoded back before the caller sees it: a \`\`datetime\`\` comes back as a
 # string, a tuple as a list.
 #
+# \`\`retry\`\` re-dispatches the task after a failure, inside \`\`@workflow\`\` only.
+# Every attempt is a step of its own (\`\`call_api\`\`, \`\`call_api#2\`\`, ...) and
+# the wait between two of them is a durable sleep, so a retrying task holds no
+# worker while it backs off. Keys: \`\`attempts\`\` (retries after the first
+# failure, a whole number from 0 to 100), \`\`delay\`\` (seconds before the first
+# retry, sub-second delays dropped), \`\`multiplier\`\` (applied to the delay
+# after each attempt, 1 keeps it constant), \`\`max_delay\`\` (ceiling in
+# seconds). \`\`attempts\`\` is required, and an out-of-range or unknown key is
+# rejected where the policy is written.
+#
+# A workflow sleeps once per round, so tasks backing off in the same fan-out
+# wait one after another rather than together: the delay before a fan-out
+# retries is the sum of every backoff pending in it, not the longest one, and
+# it grows with both the width of the fan-out and \`\`attempts\`\`. Retries with
+# no \`\`delay\`\` all go out in a single round.
+#
 # Usage::
 #
 #     @task
@@ -6835,10 +6905,15 @@ def get_resume_urls(approver: str = None, flow_level: bool = None) -> dict
 #
 #     @task(path="f/external_script", timeout=600, tag="gpu")
 #     async def run_external(x: int): ...
-def task(_func = None, *, path: Optional[str] = None, tag: Optional[str] = None, timeout: Optional[int] = None, cache_ttl: Optional[int] = None, priority: Optional[int] = None, concurrency_limit: Optional[int] = None, concurrency_key: Optional[str] = None, concurrency_time_window_s: Optional[int] = None)
+#
+#     @task(retry={"attempts": 3, "delay": 30, "multiplier": 2})
+#     async def call_api(payload: dict): ...
+def task(_func = None, *, path: Optional[str] = None, tag: Optional[str] = None, timeout: Optional[int] = None, cache_ttl: Optional[int] = None, priority: Optional[int] = None, concurrency_limit: Optional[int] = None, concurrency_key: Optional[str] = None, concurrency_time_window_s: Optional[int] = None, retry: Optional[dict] = None)

 # Create a task that dispatches to a separate Windmill script.
 #
+# \`\`retry\`\` takes the same policy as :func:\`task\`.
+#
 # Usage::
 #
 #     extract = task_script("f/data/extract", timeout=600)
@@ -6846,10 +6921,12 @@ def task(_func = None, *, path: Optional[str] = None, tag: Optional[str] = None,
 #     @workflow
 #     async def main():
 #         data = await extract(url="https://...")
-def task_script(path: str, *, timeout: Optional[int] = None, tag: Optional[str] = None, cache_ttl: Optional[int] = None, priority: Optional[int] = None, concurrency_limit: Optional[int] = None, concurrency_key: Optional[str] = None, concurrency_time_window_s: Optional[int] = None)
+def task_script(path: str, *, timeout: Optional[int] = None, tag: Optional[str] = None, cache_ttl: Optional[int] = None, priority: Optional[int] = None, concurrency_limit: Optional[int] = None, concurrency_key: Optional[str] = None, concurrency_time_window_s: Optional[int] = None, retry: Optional[dict] = None)

 # Create a task that dispatches to a separate Windmill flow.
 #
+# \`\`retry\`\` takes the same policy as :func:\`task\`.
+#
 # Usage::
 #
 #     pipeline = task_flow("f/etl/pipeline", priority=10)
@@ -6857,7 +6934,7 @@ def task_script(path: str, *, timeout: Optional[int] = None, tag: Optional[str]
 #     @workflow
 #     async def main():
 #         result = await pipeline(input=data)
-def task_flow(path: str, *, timeout: Optional[int] = None, tag: Optional[str] = None, cache_ttl: Optional[int] = None, priority: Optional[int] = None, concurrency_limit: Optional[int] = None, concurrency_key: Optional[str] = None, concurrency_time_window_s: Optional[int] = None)
+def task_flow(path: str, *, timeout: Optional[int] = None, tag: Optional[str] = None, cache_ttl: Optional[int] = None, priority: Optional[int] = None, concurrency_limit: Optional[int] = None, concurrency_key: Optional[str] = None, concurrency_time_window_s: Optional[int] = None, retry: Optional[dict] = None)

 # Decorator marking an async function as a workflow-as-code entry point.
 #
@@ -34,8 +34,12 @@ cp python-client/wmill/wmill/client.py \
   /tmp/windmill-mytest/cache/python_3_12/wmill==*/wmill/client.py
 find /tmp/windmill-mytest -name __pycache__ -type d -exec rm -rf {} +

-# 4. RESTART the backend — see below
-# 5. run your scenarios, and rm -rf /tmp/windmill-mytest when done
+# 4. drop the bundle snapshots — a bun job runs a bundle built from the package,
+#    cached by content hash, so a job that already ran keeps the old SDK inlined
+rm -rf /tmp/windmill-mytest/cache/bun
+
+# 5. RESTART the backend — see below
+# 6. run your scenarios, and rm -rf /tmp/windmill-mytest when done
 ```

 ## Restart the workers after injecting
@@ -1436,6 +1436,217 @@ class TestTaskOptions:
        assert step_info["concurrency_time_window_s"] == 60


+# =====================================================================
+# TASK RETRY TESTS
+# =====================================================================
+
+
+# What the failed child job leaves in ``completed_steps``.
+_FAILED = {"__wmill_error": True, "message": "boom", "result": {}}
+
+
+class TestTaskRetry:
+    """Nothing carries a retry across rounds: every round re-derives which
+    attempt comes next from the checkpoint alone."""
+
+    def test_each_failure_buys_a_backoff_sleep_and_one_more_attempt(self):
+        @task(retry={"attempts": 2, "delay": 30, "multiplier": 2})
+        async def call_api(x: int):
+            return x
+
+        @workflow
+        async def wf(x: int):
+            return await call_api(x=x)
+
+        completed = {"call_api": _FAILED}
+        r = _run_workflow(wf, {"completed_steps": dict(completed)}, {"x": 1})
+        assert r == {"type": "sleep", "key": "call_api#retry2", "seconds": 30}
+
+        completed["call_api#retry2"] = None
+        r = _run_workflow(wf, {"completed_steps": dict(completed)}, {"x": 1})
+        assert [s["key"] for s in r["steps"]] == ["call_api#2"]
+
+        # the delay grows by the multiplier for the second retry
+        completed["call_api#2"] = _FAILED
+        r = _run_workflow(wf, {"completed_steps": dict(completed)}, {"x": 1})
+        assert r == {"type": "sleep", "key": "call_api#retry3", "seconds": 60}
+
+        completed["call_api#retry3"] = None
+        r = _run_workflow(wf, {"completed_steps": dict(completed)}, {"x": 1})
+        assert [s["key"] for s in r["steps"]] == ["call_api#3"]
+
+        # attempts spent: the failure reaches the body
+        completed["call_api#3"] = _FAILED
+        with pytest.raises(TaskError):
+            _run_workflow(wf, {"completed_steps": dict(completed)}, {"x": 1})
+
+    def test_a_task_the_body_never_awaits_still_sleeps_and_retries(self):
+        # The runner dispatches unawaited task calls by flushing ``_pending``, so
+        # a backoff that only fired when awaited would drop the retry silently
+        # and report the workflow complete.
+        @task(retry={"attempts": 1, "delay": 30})
+        async def fire(x: int):
+            return x
+
+        @workflow
+        async def wf():
+            fire(x=1)
+            return "done"
+
+        r = _run_workflow(wf, {"completed_steps": {"fire": _FAILED}}, {})
+        assert r == {"type": "sleep", "key": "fire#retry2", "seconds": 30}
+
+    # ``step()`` names are arbitrary strings, so a step really can be called
+    # ``t#2``. Whichever of the two allocates second is the one renamed, and it
+    # has to be the same one in every round — hence claiming the attempt keys up
+    # front.
+    def test_an_inline_step_named_like_an_attempt_key_before_the_task_keeps_it(self):
+        @task(retry={"attempts": 1})
+        async def t(x: int):
+            return x
+
+        @workflow
+        async def wf():
+            decoy = await step("t#2", lambda: "not an attempt")
+            return [decoy, await t(x=1)]
+
+        r = _run_workflow(
+            wf, {"completed_steps": {"t#2": "not an attempt", "t": _FAILED}}, {}
+        )
+        assert [s["key"] for s in r["steps"]] == ["t#2_2"]
+
+    def test_an_inline_step_named_like_an_attempt_key_after_the_task_is_not_it(self):
+        @task(retry={"attempts": 1})
+        async def t(x: int):
+            return x
+
+        @workflow
+        async def wf():
+            pending = t(x=1)
+            decoy = await step("t#2", lambda: "not an attempt")
+            return [decoy, await pending]
+
+        # The first round records the step under the key left over after the
+        # task claimed ``t#2``, so the retry re-dispatches instead of reading
+        # the step's value.
+        r = _run_workflow(wf, {}, {})
+        assert r["key"] == "t#2_2"
+
+        r = _run_workflow(
+            wf, {"completed_steps": {"t": _FAILED, "t#2_2": "not an attempt"}}, {}
+        )
+        assert [s["key"] for s in r["steps"]] == ["t#2"]
+
+    def test_the_child_dispatched_for_an_attempt_walks_past_failure_and_backoff(self):
+        # The non-matching branch awaits a future that never resolves, so a child
+        # that walks the loop wrong parks the run until its timeout.
+        @task(retry={"attempts": 1, "delay": 30})
+        async def t(x: int):
+            return x * 10
+
+        @workflow
+        async def wf(x: int):
+            return await t(x=x)
+
+        r = _run_workflow(
+            wf,
+            {
+                "completed_steps": {"t": _FAILED, "t#retry2": None},
+                "_executing_key": "t#2",
+            },
+            {"x": 4},
+        )
+        assert r == {"type": "complete", "result": 40}
+
+    def test_a_misspelled_retry_option_raises_instead_of_being_ignored(self):
+        # The policy is a plain dict here, unlike the TS `TaskRetry` type, so
+        # nothing else would tell the author the option never took effect.
+        with pytest.raises(ValueError, match="max_delay_s"):
+
+            @task(retry={"attempts": 2, "max_delay_s": 300})
+            async def t(x: int):
+                return x
+
+    def test_an_out_of_range_attempt_count_is_rejected_where_it_is_written(self):
+        # Each attempt claims its keys before the first one is dispatched, so an
+        # unbounded count would hang the workflow allocating them.
+        for attempts in (10_000, -1, 2.5, float("inf")):
+            with pytest.raises(ValueError, match="whole number"):
+
+                @task(retry={"attempts": attempts})
+                async def t(x: int):
+                    return x
+
+    def test_a_zero_multiplier_is_honoured_not_read_as_the_default_of_one(self):
+        @task(retry={"attempts": 2, "delay": 30, "multiplier": 0})
+        async def t(x: int):
+            return x
+
+        @workflow
+        async def wf(x: int):
+            return await t(x=x)
+
+        r = _run_workflow(wf, {"completed_steps": {"t": _FAILED}}, {"x": 1})
+        assert r == {"type": "sleep", "key": "t#retry2", "seconds": 30}
+
+        # 30 * 0 — the second retry goes out with no wait at all
+        r = _run_workflow(
+            wf,
+            {"completed_steps": {"t": _FAILED, "t#retry2": None, "t#2": _FAILED}},
+            {"x": 1},
+        )
+        assert [s["key"] for s in r["steps"]] == ["t#3"]
+
+    def test_max_delay_caps_the_backoff_a_multiplier_grows(self):
+        @task(retry={"attempts": 2, "delay": 60, "multiplier": 100, "max_delay": 300})
+        async def t(x: int):
+            return x
+
+        @workflow
+        async def wf(x: int):
+            return await t(x=x)
+
+        r = _run_workflow(
+            wf,
+            {"completed_steps": {"t": _FAILED, "t#retry2": None, "t#2": _FAILED}},
+            {"x": 1},
+        )
+        assert r == {"type": "sleep", "key": "t#retry3", "seconds": 300}
+
+    def test_retry_that_succeeds_resolves_and_later_steps_keep_their_keys(self):
+        # No delay, so the retry is dispatched without a sleep round in between.
+        @task(retry={"attempts": 1})
+        async def flaky(x: int):
+            return x
+
+        @workflow
+        async def wf(x: int):
+            return await double(x=await flaky(x=x))
+
+        r = _run_workflow(
+            wf, {"completed_steps": {"flaky": _FAILED, "flaky#2": 7}}, {"x": 1}
+        )
+        assert r["steps"][0]["key"] == "double"
+        assert r["steps"][0]["args"] == {"x": 7}
+
+    def test_retrying_one_call_does_not_move_the_keys_of_the_calls_beside_it(self):
+        @task(retry={"attempts": 1})
+        async def t(x: int):
+            return x
+
+        @workflow
+        async def wf():
+            return await asyncio.gather(t(x=1), t(x=2))
+
+        r = _run_workflow(wf, {}, {})
+        assert [s["key"] for s in r["steps"]] == ["t", "t_2"]
+
+        # the first call retries as ``t#2``; the second keeps the ``t_2`` it was
+        # dispatched under, rather than being read as the first call's retry
+        r = _run_workflow(wf, {"completed_steps": {"t": _FAILED, "t_2": 20}}, {})
+        assert [s["key"] for s in r["steps"]] == ["t#2"]
+
+
 # =====================================================================
 # SLEEP TESTS
 # =====================================================================
@@ -2874,6 +2874,59 @@ def _task_error_from_marker(marker: dict, fallback_message: str) -> TaskError:
    )


+# The worker deserializes a sleep into a ``u32`` of seconds and fails the whole
+# job on anything wider, so a delay a multiplier has run away with has to be
+# capped here rather than sent.
+_MAX_SLEEP_SECONDS = 2**32 - 1
+
+_RETRY_KEYS = ("attempts", "delay", "multiplier", "max_delay")
+
+# Every attempt claims its keys before the first one is dispatched, so an
+# unbounded ``attempts`` is a workflow that hangs allocating rather than a very
+# patient one.
+_MAX_RETRY_ATTEMPTS = 100
+
+
+def _checked_retry(retry: Optional[dict]) -> Optional[dict]:
+    """Reject a policy where it is written, rather than mid-run on a replay: the
+    policy is a plain dict, so a misspelled key would otherwise be dropped in
+    silence and the task would retry on a policy nobody wrote."""
+    if retry is None:
+        return None
+    unknown = sorted(k for k in retry if k not in _RETRY_KEYS)
+    if unknown:
+        raise ValueError(
+            f"unknown retry option(s): {', '.join(unknown)}. Expected any of: {', '.join(_RETRY_KEYS)}"
+        )
+    attempts = retry.get("attempts")
+    if isinstance(attempts, bool) or not isinstance(attempts, int) or not 0 <= attempts <= _MAX_RETRY_ATTEMPTS:
+        raise ValueError(
+            f"retry attempts must be a whole number between 0 and {_MAX_RETRY_ATTEMPTS}, got {attempts!r}"
+        )
+    return retry
+
+
+def _retry_delay_seconds(retry: dict, attempt: int) -> int:
+    """Seconds to wait before retry number ``attempt`` (0 is the first retry)."""
+    base = retry.get("delay") or 0
+    if base <= 0:
+        return 0
+    # `or 1` would read an explicit `multiplier: 0` — every retry after the
+    # first going out with no wait — as the default of 1.
+    multiplier = retry.get("multiplier")
+    if multiplier is None:
+        multiplier = 1
+    try:
+        grown = base * multiplier**attempt
+    except OverflowError:
+        # A float delay times an integer multiplier raised past ~1e308.
+        grown = _MAX_SLEEP_SECONDS
+    max_delay = retry.get("max_delay")
+    if max_delay is not None:
+        grown = min(grown, max_delay)
+    return max(0, int(min(grown, _MAX_SLEEP_SECONDS)))
+
+
 _workflow_ctx: _contextvars.ContextVar["WorkflowCtx"] = _contextvars.ContextVar(
    "_workflow_ctx"
 )
@@ -2929,28 +2982,77 @@ class WorkflowCtx:

    def _next_step(self, name: str, script: str, func=None, dispatch_type: str = "inline", _task_options: Optional[dict] = None, **kwargs):
        """Return an awaitable that either resolves from cache or suspends."""
-        key = self._alloc_key(name or script or "step")
+        step_name = name or script or "step"
+        retry = (_task_options or {}).get("retry") or {}
+        # Clamped as well as validated at decoration: a policy that reached here
+        # another way must not spin the key loop below.
+        max_retries = min(max(0, int(retry.get("attempts") or 0)), _MAX_RETRY_ATTEMPTS)

+        # Claimed up front, all of them, and named off the first attempt's key:
+        # one allocated later would shift the keys of the steps beside it, and a
+        # ``step()`` named ``t#2`` — names are arbitrary — could alias one.
+        # Whichever is allocated second is the one renamed, in every round alike.
+        base_key = self._alloc_key(step_name)
+        attempt_keys = [base_key]
+        backoff_keys = []
+        for i in range(max_retries):
+            backoff_keys.append(self._alloc_key(f"{base_key}#retry{i + 2}"))
+            attempt_keys.append(self._alloc_key(f"{base_key}#{i + 2}"))
+
+        # One pass per attempt. Every attempt the checkpoint already holds is
+        # decided here — a failed one either retries (moving to the next key) or
+        # is handed back to the body — so the loop always ends at the first
+        # attempt that has yet to run.
+        attempt = 0
+        while True:
+            key = attempt_keys[attempt]
+
+            if key in self._completed:
+                val = self._completed[key]
+                if isinstance(val, dict) and val.get("__wmill_error"):
+                    if attempt < max_retries:
+                        self._retry_backoff(backoff_keys[attempt], base_key, retry, attempt)
+                        attempt += 1
+                        continue
+                    raise _task_error_from_marker(val, f"Task '{name}' failed")
+                return self._resolved(val)
+
+            if self._executing_key is not None:
+                if key == self._executing_key:
+                    return self._execute_directly(func, **kwargs)
+                else:
+                    return self._never_resolve()
+
+            print(f"\n--- WAC: {key} ---")
+            info = {"name": name or key, "script": script or key, "args": kwargs, "key": key, "dispatch_type": dispatch_type}
+            if _task_options:
+                for opt_key in ("timeout", "tag", "cache_ttl", "priority", "concurrent_limit", "concurrency_key", "concurrency_time_window_s"):
+                    if opt_key in _task_options and _task_options[opt_key] is not None:
+                        info[opt_key] = _task_options[opt_key]
+            self._pending.append(info)
+            return self._suspend()
+
+    def _retry_backoff(self, key: str, base_key: str, retry: dict, attempt: int) -> None:
+        """Wait out the backoff between two attempts of a retried task, as a
+        durable sleep, and return once there is nothing to wait for — no delay
+        configured, or the sleep already in the checkpoint.
+
+        Raises where it stands rather than from a coroutine the caller has to
+        await: a task call the body never awaits is still dispatched (the runner
+        flushes ``_pending``), so a backoff that only fired when awaited would
+        drop the retry and let the round report the workflow complete."""
+        seconds = _retry_delay_seconds(retry, attempt)
+        if seconds < 1:
+            return
        if key in self._completed:
-            val = self._completed[key]
-            if isinstance(val, dict) and val.get("__wmill_error"):
-                raise _task_error_from_marker(val, f"Task '{name}' failed")
-            return self._resolved(val)
-
+            return
+        # Child mode never raises: the parent dispatched this child only after
+        # its own round had slept, so the loop moves on to the attempt being
+        # executed.
        if self._executing_key is not None:
-            if key == self._executing_key:
-                return self._execute_directly(func, **kwargs)
-            else:
-                return self._never_resolve()
-
-        print(f"\n--- WAC: {key} ---")
-        info = {"name": name or key, "script": script or key, "args": kwargs, "key": key, "dispatch_type": dispatch_type}
-        if _task_options:
-            for opt_key in ("timeout", "tag", "cache_ttl", "priority", "concurrent_limit", "concurrency_key", "concurrency_time_window_s"):
-                if opt_key in _task_options and _task_options[opt_key] is not None:
-                    info[opt_key] = _task_options[opt_key]
-        self._pending.append(info)
-        return self._suspend()
+            return
+        print(f"\n--- WAC: sleep({key}, {seconds}s) before retrying {base_key} ---")
+        raise _StepSuspend({"mode": "sleep", "key": key, "seconds": seconds, "steps": []})

    async def _resolved(self, value):
        return value
@@ -3190,6 +3292,7 @@ def task(
    concurrency_limit: Optional[int] = None,
    concurrency_key: Optional[str] = None,
    concurrency_time_window_s: Optional[int] = None,
+    retry: Optional[dict] = None,
 ):
    """Decorator that marks a function as a workflow task.

@@ -3204,6 +3307,22 @@ def task(
    decoded back before the caller sees it: a ``datetime`` comes back as a
    string, a tuple as a list.

+    ``retry`` re-dispatches the task after a failure, inside ``@workflow`` only.
+    Every attempt is a step of its own (``call_api``, ``call_api#2``, ...) and
+    the wait between two of them is a durable sleep, so a retrying task holds no
+    worker while it backs off. Keys: ``attempts`` (retries after the first
+    failure, a whole number from 0 to 100), ``delay`` (seconds before the first
+    retry, sub-second delays dropped), ``multiplier`` (applied to the delay
+    after each attempt, 1 keeps it constant), ``max_delay`` (ceiling in
+    seconds). ``attempts`` is required, and an out-of-range or unknown key is
+    rejected where the policy is written.
+
+    A workflow sleeps once per round, so tasks backing off in the same fan-out
+    wait one after another rather than together: the delay before a fan-out
+    retries is the sum of every backoff pending in it, not the longest one, and
+    it grows with both the width of the fan-out and ``attempts``. Retries with
+    no ``delay`` all go out in a single round.
+
    Usage::

        @task
@@ -3211,6 +3330,9 @@ def task(

        @task(path="f/external_script", timeout=600, tag="gpu")
        async def run_external(x: int): ...
+
+        @task(retry={"attempts": 3, "delay": 30, "multiplier": 2})
+        async def call_api(payload: dict): ...
    """
    from inspect import signature as _sig

@@ -3222,6 +3344,7 @@ def task(
        "concurrent_limit": concurrency_limit,
        "concurrency_key": concurrency_key,
        "concurrency_time_window_s": concurrency_time_window_s,
+        "retry": _checked_retry(retry),
    }
    # Remove None values
    _task_opts = {k: v for k, v in _task_opts.items() if v is not None} or None
@@ -3316,9 +3439,12 @@ def task_script(
    concurrency_limit: Optional[int] = None,
    concurrency_key: Optional[str] = None,
    concurrency_time_window_s: Optional[int] = None,
+    retry: Optional[dict] = None,
 ):
    """Create a task that dispatches to a separate Windmill script.

+    ``retry`` takes the same policy as :func:`task`.
+
    Usage::

        extract = task_script("f/data/extract", timeout=600)
@@ -3328,7 +3454,7 @@ def task_script(
            data = await extract(url="https://...")
    """
    name = path.rsplit("/", 1)[-1]
-    _opts = {k: v for k, v in {"timeout": timeout, "tag": tag, "cache_ttl": cache_ttl, "priority": priority, "concurrent_limit": concurrency_limit, "concurrency_key": concurrency_key, "concurrency_time_window_s": concurrency_time_window_s}.items() if v is not None} or None
+    _opts = {k: v for k, v in {"timeout": timeout, "tag": tag, "cache_ttl": cache_ttl, "priority": priority, "concurrent_limit": concurrency_limit, "concurrency_key": concurrency_key, "concurrency_time_window_s": concurrency_time_window_s, "retry": _checked_retry(retry)}.items() if v is not None} or None

    def wrapper(**kwargs):
        ctx = _workflow_ctx.get(None)
@@ -3352,9 +3478,12 @@ def task_flow(
    concurrency_limit: Optional[int] = None,
    concurrency_key: Optional[str] = None,
    concurrency_time_window_s: Optional[int] = None,
+    retry: Optional[dict] = None,
 ):
    """Create a task that dispatches to a separate Windmill flow.

+    ``retry`` takes the same policy as :func:`task`.
+
    Usage::

        pipeline = task_flow("f/etl/pipeline", priority=10)
@@ -3364,7 +3493,7 @@ def task_flow(
            result = await pipeline(input=data)
    """
    name = path.rsplit("/", 1)[-1]
-    _opts = {k: v for k, v in {"timeout": timeout, "tag": tag, "cache_ttl": cache_ttl, "priority": priority, "concurrent_limit": concurrency_limit, "concurrency_key": concurrency_key, "concurrency_time_window_s": concurrency_time_window_s}.items() if v is not None} or None
+    _opts = {k: v for k, v in {"timeout": timeout, "tag": tag, "cache_ttl": cache_ttl, "priority": priority, "concurrent_limit": concurrency_limit, "concurrency_key": concurrency_key, "concurrency_time_window_s": concurrency_time_window_s, "retry": _checked_retry(retry)}.items() if v is not None} or None

    def wrapper(**kwargs):
        ctx = _workflow_ctx.get(None)
@@ -2553,6 +2553,22 @@ def parse_sql_client_name(name: str) -> tuple[str, Optional[str]]
 # decoded back before the caller sees it: a \`\`datetime\`\` comes back as a
 # string, a tuple as a list.
 # 
+# \`\`retry\`\` re-dispatches the task after a failure, inside \`\`@workflow\`\` only.
+# Every attempt is a step of its own (\`\`call_api\`\`, \`\`call_api#2\`\`, ...) and
+# the wait between two of them is a durable sleep, so a retrying task holds no
+# worker while it backs off. Keys: \`\`attempts\`\` (retries after the first
+# failure, a whole number from 0 to 100), \`\`delay\`\` (seconds before the first
+# retry, sub-second delays dropped), \`\`multiplier\`\` (applied to the delay
+# after each attempt, 1 keeps it constant), \`\`max_delay\`\` (ceiling in
+# seconds). \`\`attempts\`\` is required, and an out-of-range or unknown key is
+# rejected where the policy is written.
+# 
+# A workflow sleeps once per round, so tasks backing off in the same fan-out
+# wait one after another rather than together: the delay before a fan-out
+# retries is the sum of every backoff pending in it, not the longest one, and
+# it grows with both the width of the fan-out and \`\`attempts\`\`. Retries with
+# no \`\`delay\`\` all go out in a single round.
+# 
 # Usage::
 # 
 #     @task
@@ -2560,10 +2576,15 @@ def parse_sql_client_name(name: str) -> tuple[str, Optional[str]]
 # 
 #     @task(path="f/external_script", timeout=600, tag="gpu")
 #     async def run_external(x: int): ...
-def task(_func = None, path: Optional[str] = None, tag: Optional[str] = None, timeout: Optional[int] = None, cache_ttl: Optional[int] = None, priority: Optional[int] = None, concurrency_limit: Optional[int] = None, concurrency_key: Optional[str] = None, concurrency_time_window_s: Optional[int] = None)
+# 
+#     @task(retry={"attempts": 3, "delay": 30, "multiplier": 2})
+#     async def call_api(payload: dict): ...
+def task(_func = None, path: Optional[str] = None, tag: Optional[str] = None, timeout: Optional[int] = None, cache_ttl: Optional[int] = None, priority: Optional[int] = None, concurrency_limit: Optional[int] = None, concurrency_key: Optional[str] = None, concurrency_time_window_s: Optional[int] = None, retry: Optional[dict] = None)

 # Create a task that dispatches to a separate Windmill script.
 # 
+# \`\`retry\`\` takes the same policy as :func:\`task\`.
+# 
 # Usage::
 # 
 #     extract = task_script("f/data/extract", timeout=600)
@@ -2571,10 +2592,12 @@ def task(_func = None, path: Optional[str] = None, tag: Optional[str] = None, ti
 #     @workflow
 #     async def main():
 #         data = await extract(url="https://...")
-def task_script(path: str, timeout: Optional[int] = None, tag: Optional[str] = None, cache_ttl: Optional[int] = None, priority: Optional[int] = None, concurrency_limit: Optional[int] = None, concurrency_key: Optional[str] = None, concurrency_time_window_s: Optional[int] = None)
+def task_script(path: str, timeout: Optional[int] = None, tag: Optional[str] = None, cache_ttl: Optional[int] = None, priority: Optional[int] = None, concurrency_limit: Optional[int] = None, concurrency_key: Optional[str] = None, concurrency_time_window_s: Optional[int] = None, retry: Optional[dict] = None)

 # Create a task that dispatches to a separate Windmill flow.
 # 
+# \`\`retry\`\` takes the same policy as :func:\`task\`.
+# 
 # Usage::
 # 
 #     pipeline = task_flow("f/etl/pipeline", priority=10)
@@ -2582,7 +2605,7 @@ def task_script(path: str, timeout: Optional[int] = None, tag: Optional[str] = N
 #     @workflow
 #     async def main():
 #         result = await pipeline(input=data)
-def task_flow(path: str, timeout: Optional[int] = None, tag: Optional[str] = None, cache_ttl: Optional[int] = None, priority: Optional[int] = None, concurrency_limit: Optional[int] = None, concurrency_key: Optional[str] = None, concurrency_time_window_s: Optional[int] = None)
+def task_flow(path: str, timeout: Optional[int] = None, tag: Optional[str] = None, cache_ttl: Optional[int] = None, priority: Optional[int] = None, concurrency_limit: Optional[int] = None, concurrency_key: Optional[str] = None, concurrency_time_window_s: Optional[int] = None, retry: Optional[dict] = None)

 # Decorator marking an async function as a workflow-as-code entry point.
 # 
@@ -2661,6 +2684,34 @@ export const WAC_SDK_TYPESCRIPT = `## TypeScript Workflow-as-Code API (windmill-
 Import: \`import { workflow, task, taskScript, taskFlow, step, sleep, waitForApproval, getApprovalUrls, getResumeUrls, parallel } from "windmill-client"\`

 \`\`\`typescript
+/**
+ * Re-dispatch policy for a failed task.
+ *
+ * Every attempt is a step of its own (\`fetch\`, \`fetch#2\`, \`fetch#3\`), and the
+ * wait between two of them is a durable sleep, so a retrying task holds no
+ * worker while it backs off.
+ *
+ * A workflow sleeps once per round, so tasks backing off in the same fan-out
+ * wait one after another rather than together: the delay before a fan-out
+ * retries is the sum of every backoff pending in it, not the longest one, and
+ * it grows with both the width of the fan-out and \`attempts\`. Retries with no
+ * \`delay\` all go out in a single round.
+ */
+export interface TaskRetry {
+  /** Attempts after the first failure: \`2\` runs the task at most 3 times.
+  *  A whole number from 0 to 100; anything else is rejected where the policy
+  *  is written. */
+  attempts: number;
+  /** Seconds to wait before the first retry. Default 0, retry immediately.
+  *  Sub-second delays are dropped — a durable sleep resolves to the second. */
+  delay?: number;
+  /** Applied to the delay after each attempt: 1 (the default) keeps it
+  *  constant, 2 doubles it. */
+  multiplier?: number;
+  /** Ceiling for the delay in seconds, for a \`multiplier\` above 1. */
+  max_delay?: number;
+}
+
 export interface TaskOptions {
  timeout?: number;
  tag?: string;
@@ -2669,6 +2720,7 @@ export interface TaskOptions {
  concurrency_limit?: number;
  concurrency_key?: string;
  concurrency_time_window_s?: number;
+  retry?: TaskRetry;
 }

 /**
@@ -2686,9 +2738,11 @@ export async function getResumeUrls(approver?: string, flowLevel?: boolean): Pro
 * @example
 * const extract_data = task(async (url: string) => { ... });
 * const run_external = task("f/external_script", async (x: number) => { ... });
+ * const call_api = task(fetchOrders, { retry: { attempts: 3, delay: 30, multiplier: 2 } });
 *
 * Inside a \`workflow()\`, calling a task dispatches it as a step.
- * Outside a workflow, the function body executes directly.
+ * Outside a workflow, the function body executes directly and
+ * {@link TaskOptions} — retry included — does not apply.
 *
 * A task runs as its own job, so its result is always encoded as JSON and
 * decoded back before the caller sees it: a \`Date\` comes back as a string, a
@@ -2832,6 +2886,22 @@ def get_resume_urls(approver: str = None, flow_level: bool = None) -> dict
 # decoded back before the caller sees it: a \`\`datetime\`\` comes back as a
 # string, a tuple as a list.
 #
+# \`\`retry\`\` re-dispatches the task after a failure, inside \`\`@workflow\`\` only.
+# Every attempt is a step of its own (\`\`call_api\`\`, \`\`call_api#2\`\`, ...) and
+# the wait between two of them is a durable sleep, so a retrying task holds no
+# worker while it backs off. Keys: \`\`attempts\`\` (retries after the first
+# failure, a whole number from 0 to 100), \`\`delay\`\` (seconds before the first
+# retry, sub-second delays dropped), \`\`multiplier\`\` (applied to the delay
+# after each attempt, 1 keeps it constant), \`\`max_delay\`\` (ceiling in
+# seconds). \`\`attempts\`\` is required, and an out-of-range or unknown key is
+# rejected where the policy is written.
+#
+# A workflow sleeps once per round, so tasks backing off in the same fan-out
+# wait one after another rather than together: the delay before a fan-out
+# retries is the sum of every backoff pending in it, not the longest one, and
+# it grows with both the width of the fan-out and \`\`attempts\`\`. Retries with
+# no \`\`delay\`\` all go out in a single round.
+#
 # Usage::
 #
 #     @task
@@ -2839,10 +2909,15 @@ def get_resume_urls(approver: str = None, flow_level: bool = None) -> dict
 #
 #     @task(path="f/external_script", timeout=600, tag="gpu")
 #     async def run_external(x: int): ...
-def task(_func = None, *, path: Optional[str] = None, tag: Optional[str] = None, timeout: Optional[int] = None, cache_ttl: Optional[int] = None, priority: Optional[int] = None, concurrency_limit: Optional[int] = None, concurrency_key: Optional[str] = None, concurrency_time_window_s: Optional[int] = None)
+#
+#     @task(retry={"attempts": 3, "delay": 30, "multiplier": 2})
+#     async def call_api(payload: dict): ...
+def task(_func = None, *, path: Optional[str] = None, tag: Optional[str] = None, timeout: Optional[int] = None, cache_ttl: Optional[int] = None, priority: Optional[int] = None, concurrency_limit: Optional[int] = None, concurrency_key: Optional[str] = None, concurrency_time_window_s: Optional[int] = None, retry: Optional[dict] = None)

 # Create a task that dispatches to a separate Windmill script.
 #
+# \`\`retry\`\` takes the same policy as :func:\`task\`.
+#
 # Usage::
 #
 #     extract = task_script("f/data/extract", timeout=600)
@@ -2850,10 +2925,12 @@ def task(_func = None, *, path: Optional[str] = None, tag: Optional[str] = None,
 #     @workflow
 #     async def main():
 #         data = await extract(url="https://...")
-def task_script(path: str, *, timeout: Optional[int] = None, tag: Optional[str] = None, cache_ttl: Optional[int] = None, priority: Optional[int] = None, concurrency_limit: Optional[int] = None, concurrency_key: Optional[str] = None, concurrency_time_window_s: Optional[int] = None)
+def task_script(path: str, *, timeout: Optional[int] = None, tag: Optional[str] = None, cache_ttl: Optional[int] = None, priority: Optional[int] = None, concurrency_limit: Optional[int] = None, concurrency_key: Optional[str] = None, concurrency_time_window_s: Optional[int] = None, retry: Optional[dict] = None)

 # Create a task that dispatches to a separate Windmill flow.
 #
+# \`\`retry\`\` takes the same policy as :func:\`task\`.
+#
 # Usage::
 #
 #     pipeline = task_flow("f/etl/pipeline", priority=10)
@@ -2861,7 +2938,7 @@ def task_script(path: str, *, timeout: Optional[int] = None, tag: Optional[str]
 #     @workflow
 #     async def main():
 #         result = await pipeline(input=data)
-def task_flow(path: str, *, timeout: Optional[int] = None, tag: Optional[str] = None, cache_ttl: Optional[int] = None, priority: Optional[int] = None, concurrency_limit: Optional[int] = None, concurrency_key: Optional[str] = None, concurrency_time_window_s: Optional[int] = None)
+def task_flow(path: str, *, timeout: Optional[int] = None, tag: Optional[str] = None, cache_ttl: Optional[int] = None, priority: Optional[int] = None, concurrency_limit: Optional[int] = None, concurrency_key: Optional[str] = None, concurrency_time_window_s: Optional[int] = None, retry: Optional[dict] = None)

 # Decorator marking an async function as a workflow-as-code entry point.
 #
@@ -2708,6 +2708,22 @@ def parse_sql_client_name(name: str) -> tuple[str, Optional[str]]
 # decoded back before the caller sees it: a ``datetime`` comes back as a
 # string, a tuple as a list.
 # 
+# ``retry`` re-dispatches the task after a failure, inside ``@workflow`` only.
+# Every attempt is a step of its own (``call_api``, ``call_api#2``, ...) and
+# the wait between two of them is a durable sleep, so a retrying task holds no
+# worker while it backs off. Keys: ``attempts`` (retries after the first
+# failure, a whole number from 0 to 100), ``delay`` (seconds before the first
+# retry, sub-second delays dropped), ``multiplier`` (applied to the delay
+# after each attempt, 1 keeps it constant), ``max_delay`` (ceiling in
+# seconds). ``attempts`` is required, and an out-of-range or unknown key is
+# rejected where the policy is written.
+# 
+# A workflow sleeps once per round, so tasks backing off in the same fan-out
+# wait one after another rather than together: the delay before a fan-out
+# retries is the sum of every backoff pending in it, not the longest one, and
+# it grows with both the width of the fan-out and ``attempts``. Retries with
+# no ``delay`` all go out in a single round.
+# 
 # Usage::
 # 
 #     @task
@@ -2715,10 +2731,15 @@ def parse_sql_client_name(name: str) -> tuple[str, Optional[str]]
 # 
 #     @task(path="f/external_script", timeout=600, tag="gpu")
 #     async def run_external(x: int): ...
-def task(_func = None, path: Optional[str] = None, tag: Optional[str] = None, timeout: Optional[int] = None, cache_ttl: Optional[int] = None, priority: Optional[int] = None, concurrency_limit: Optional[int] = None, concurrency_key: Optional[str] = None, concurrency_time_window_s: Optional[int] = None)
+# 
+#     @task(retry={"attempts": 3, "delay": 30, "multiplier": 2})
+#     async def call_api(payload: dict): ...
+def task(_func = None, path: Optional[str] = None, tag: Optional[str] = None, timeout: Optional[int] = None, cache_ttl: Optional[int] = None, priority: Optional[int] = None, concurrency_limit: Optional[int] = None, concurrency_key: Optional[str] = None, concurrency_time_window_s: Optional[int] = None, retry: Optional[dict] = None)

 # Create a task that dispatches to a separate Windmill script.
 # 
+# ``retry`` takes the same policy as :func:`task`.
+# 
 # Usage::
 # 
 #     extract = task_script("f/data/extract", timeout=600)
@@ -2726,10 +2747,12 @@ def task(_func = None, path: Optional[str] = None, tag: Optional[str] = None, ti
 #     @workflow
 #     async def main():
 #         data = await extract(url="https://...")
-def task_script(path: str, timeout: Optional[int] = None, tag: Optional[str] = None, cache_ttl: Optional[int] = None, priority: Optional[int] = None, concurrency_limit: Optional[int] = None, concurrency_key: Optional[str] = None, concurrency_time_window_s: Optional[int] = None)
+def task_script(path: str, timeout: Optional[int] = None, tag: Optional[str] = None, cache_ttl: Optional[int] = None, priority: Optional[int] = None, concurrency_limit: Optional[int] = None, concurrency_key: Optional[str] = None, concurrency_time_window_s: Optional[int] = None, retry: Optional[dict] = None)

 # Create a task that dispatches to a separate Windmill flow.
 # 
+# ``retry`` takes the same policy as :func:`task`.
+# 
 # Usage::
 # 
 #     pipeline = task_flow("f/etl/pipeline", priority=10)
@@ -2737,7 +2760,7 @@ def task_script(path: str, timeout: Optional[int] = None, tag: Optional[str] = N
 #     @workflow
 #     async def main():
 #         result = await pipeline(input=data)
-def task_flow(path: str, timeout: Optional[int] = None, tag: Optional[str] = None, cache_ttl: Optional[int] = None, priority: Optional[int] = None, concurrency_limit: Optional[int] = None, concurrency_key: Optional[str] = None, concurrency_time_window_s: Optional[int] = None)
+def task_flow(path: str, timeout: Optional[int] = None, tag: Optional[str] = None, cache_ttl: Optional[int] = None, priority: Optional[int] = None, concurrency_limit: Optional[int] = None, concurrency_key: Optional[str] = None, concurrency_time_window_s: Optional[int] = None, retry: Optional[dict] = None)

 # Decorator marking an async function as a workflow-as-code entry point.
 # 
@@ -656,6 +656,22 @@ def parse_sql_client_name(name: str) -> tuple[str, Optional[str]]
 # decoded back before the caller sees it: a ``datetime`` comes back as a
 # string, a tuple as a list.
 # 
+# ``retry`` re-dispatches the task after a failure, inside ``@workflow`` only.
+# Every attempt is a step of its own (``call_api``, ``call_api#2``, ...) and
+# the wait between two of them is a durable sleep, so a retrying task holds no
+# worker while it backs off. Keys: ``attempts`` (retries after the first
+# failure, a whole number from 0 to 100), ``delay`` (seconds before the first
+# retry, sub-second delays dropped), ``multiplier`` (applied to the delay
+# after each attempt, 1 keeps it constant), ``max_delay`` (ceiling in
+# seconds). ``attempts`` is required, and an out-of-range or unknown key is
+# rejected where the policy is written.
+# 
+# A workflow sleeps once per round, so tasks backing off in the same fan-out
+# wait one after another rather than together: the delay before a fan-out
+# retries is the sum of every backoff pending in it, not the longest one, and
+# it grows with both the width of the fan-out and ``attempts``. Retries with
+# no ``delay`` all go out in a single round.
+# 
 # Usage::
 # 
 #     @task
@@ -663,10 +679,15 @@ def parse_sql_client_name(name: str) -> tuple[str, Optional[str]]
 # 
 #     @task(path="f/external_script", timeout=600, tag="gpu")
 #     async def run_external(x: int): ...
-def task(_func = None, path: Optional[str] = None, tag: Optional[str] = None, timeout: Optional[int] = None, cache_ttl: Optional[int] = None, priority: Optional[int] = None, concurrency_limit: Optional[int] = None, concurrency_key: Optional[str] = None, concurrency_time_window_s: Optional[int] = None)
+# 
+#     @task(retry={"attempts": 3, "delay": 30, "multiplier": 2})
+#     async def call_api(payload: dict): ...
+def task(_func = None, path: Optional[str] = None, tag: Optional[str] = None, timeout: Optional[int] = None, cache_ttl: Optional[int] = None, priority: Optional[int] = None, concurrency_limit: Optional[int] = None, concurrency_key: Optional[str] = None, concurrency_time_window_s: Optional[int] = None, retry: Optional[dict] = None)

 # Create a task that dispatches to a separate Windmill script.
 # 
+# ``retry`` takes the same policy as :func:`task`.
+# 
 # Usage::
 # 
 #     extract = task_script("f/data/extract", timeout=600)
@@ -674,10 +695,12 @@ def task(_func = None, path: Optional[str] = None, tag: Optional[str] = None, ti
 #     @workflow
 #     async def main():
 #         data = await extract(url="https://...")
-def task_script(path: str, timeout: Optional[int] = None, tag: Optional[str] = None, cache_ttl: Optional[int] = None, priority: Optional[int] = None, concurrency_limit: Optional[int] = None, concurrency_key: Optional[str] = None, concurrency_time_window_s: Optional[int] = None)
+def task_script(path: str, timeout: Optional[int] = None, tag: Optional[str] = None, cache_ttl: Optional[int] = None, priority: Optional[int] = None, concurrency_limit: Optional[int] = None, concurrency_key: Optional[str] = None, concurrency_time_window_s: Optional[int] = None, retry: Optional[dict] = None)

 # Create a task that dispatches to a separate Windmill flow.
 # 
+# ``retry`` takes the same policy as :func:`task`.
+# 
 # Usage::
 # 
 #     pipeline = task_flow("f/etl/pipeline", priority=10)
@@ -685,7 +708,7 @@ def task_script(path: str, timeout: Optional[int] = None, tag: Optional[str] = N
 #     @workflow
 #     async def main():
 #         result = await pipeline(input=data)
-def task_flow(path: str, timeout: Optional[int] = None, tag: Optional[str] = None, cache_ttl: Optional[int] = None, priority: Optional[int] = None, concurrency_limit: Optional[int] = None, concurrency_key: Optional[str] = None, concurrency_time_window_s: Optional[int] = None)
+def task_flow(path: str, timeout: Optional[int] = None, tag: Optional[str] = None, cache_ttl: Optional[int] = None, priority: Optional[int] = None, concurrency_limit: Optional[int] = None, concurrency_key: Optional[str] = None, concurrency_time_window_s: Optional[int] = None, retry: Optional[dict] = None)

 # Decorator marking an async function as a workflow-as-code entry point.
 # 
@@ -42,6 +42,22 @@ def get_resume_urls(approver: str = None, flow_level: bool = None) -> dict
 # decoded back before the caller sees it: a ``datetime`` comes back as a
 # string, a tuple as a list.
 #
+# ``retry`` re-dispatches the task after a failure, inside ``@workflow`` only.
+# Every attempt is a step of its own (``call_api``, ``call_api#2``, ...) and
+# the wait between two of them is a durable sleep, so a retrying task holds no
+# worker while it backs off. Keys: ``attempts`` (retries after the first
+# failure, a whole number from 0 to 100), ``delay`` (seconds before the first
+# retry, sub-second delays dropped), ``multiplier`` (applied to the delay
+# after each attempt, 1 keeps it constant), ``max_delay`` (ceiling in
+# seconds). ``attempts`` is required, and an out-of-range or unknown key is
+# rejected where the policy is written.
+#
+# A workflow sleeps once per round, so tasks backing off in the same fan-out
+# wait one after another rather than together: the delay before a fan-out
+# retries is the sum of every backoff pending in it, not the longest one, and
+# it grows with both the width of the fan-out and ``attempts``. Retries with
+# no ``delay`` all go out in a single round.
+#
 # Usage::
 #
 #     @task
@@ -49,10 +65,15 @@ def get_resume_urls(approver: str = None, flow_level: bool = None) -> dict
 #
 #     @task(path="f/external_script", timeout=600, tag="gpu")
 #     async def run_external(x: int): ...
-def task(_func = None, *, path: Optional[str] = None, tag: Optional[str] = None, timeout: Optional[int] = None, cache_ttl: Optional[int] = None, priority: Optional[int] = None, concurrency_limit: Optional[int] = None, concurrency_key: Optional[str] = None, concurrency_time_window_s: Optional[int] = None)
+#
+#     @task(retry={"attempts": 3, "delay": 30, "multiplier": 2})
+#     async def call_api(payload: dict): ...
+def task(_func = None, *, path: Optional[str] = None, tag: Optional[str] = None, timeout: Optional[int] = None, cache_ttl: Optional[int] = None, priority: Optional[int] = None, concurrency_limit: Optional[int] = None, concurrency_key: Optional[str] = None, concurrency_time_window_s: Optional[int] = None, retry: Optional[dict] = None)

 # Create a task that dispatches to a separate Windmill script.
 #
+# ``retry`` takes the same policy as :func:`task`.
+#
 # Usage::
 #
 #     extract = task_script("f/data/extract", timeout=600)
@@ -60,10 +81,12 @@ def task(_func = None, *, path: Optional[str] = None, tag: Optional[str] = None,
 #     @workflow
 #     async def main():
 #         data = await extract(url="https://...")
-def task_script(path: str, *, timeout: Optional[int] = None, tag: Optional[str] = None, cache_ttl: Optional[int] = None, priority: Optional[int] = None, concurrency_limit: Optional[int] = None, concurrency_key: Optional[str] = None, concurrency_time_window_s: Optional[int] = None)
+def task_script(path: str, *, timeout: Optional[int] = None, tag: Optional[str] = None, cache_ttl: Optional[int] = None, priority: Optional[int] = None, concurrency_limit: Optional[int] = None, concurrency_key: Optional[str] = None, concurrency_time_window_s: Optional[int] = None, retry: Optional[dict] = None)

 # Create a task that dispatches to a separate Windmill flow.
 #
+# ``retry`` takes the same policy as :func:`task`.
+#
 # Usage::
 #
 #     pipeline = task_flow("f/etl/pipeline", priority=10)
@@ -71,7 +94,7 @@ def task_script(path: str, *, timeout: Optional[int] = None, tag: Optional[str]
 #     @workflow
 #     async def main():
 #         result = await pipeline(input=data)
-def task_flow(path: str, *, timeout: Optional[int] = None, tag: Optional[str] = None, cache_ttl: Optional[int] = None, priority: Optional[int] = None, concurrency_limit: Optional[int] = None, concurrency_key: Optional[str] = None, concurrency_time_window_s: Optional[int] = None)
+def task_flow(path: str, *, timeout: Optional[int] = None, tag: Optional[str] = None, cache_ttl: Optional[int] = None, priority: Optional[int] = None, concurrency_limit: Optional[int] = None, concurrency_key: Optional[str] = None, concurrency_time_window_s: Optional[int] = None, retry: Optional[dict] = None)

 # Decorator marking an async function as a workflow-as-code entry point.
 #
@@ -3,6 +3,34 @@
 Import: `import { workflow, task, taskScript, taskFlow, step, sleep, waitForApproval, getApprovalUrls, getResumeUrls, parallel } from "windmill-client"`

 ```typescript
+/**
+ * Re-dispatch policy for a failed task.
+ *
+ * Every attempt is a step of its own (`fetch`, `fetch#2`, `fetch#3`), and the
+ * wait between two of them is a durable sleep, so a retrying task holds no
+ * worker while it backs off.
+ *
+ * A workflow sleeps once per round, so tasks backing off in the same fan-out
+ * wait one after another rather than together: the delay before a fan-out
+ * retries is the sum of every backoff pending in it, not the longest one, and
+ * it grows with both the width of the fan-out and `attempts`. Retries with no
+ * `delay` all go out in a single round.
+ */
+export interface TaskRetry {
+  /** Attempts after the first failure: `2` runs the task at most 3 times.
+  *  A whole number from 0 to 100; anything else is rejected where the policy
+  *  is written. */
+  attempts: number;
+  /** Seconds to wait before the first retry. Default 0, retry immediately.
+  *  Sub-second delays are dropped — a durable sleep resolves to the second. */
+  delay?: number;
+  /** Applied to the delay after each attempt: 1 (the default) keeps it
+  *  constant, 2 doubles it. */
+  multiplier?: number;
+  /** Ceiling for the delay in seconds, for a `multiplier` above 1. */
+  max_delay?: number;
+}
+
 export interface TaskOptions {
  timeout?: number;
  tag?: string;
@@ -11,6 +39,7 @@ export interface TaskOptions {
  concurrency_limit?: number;
  concurrency_key?: string;
  concurrency_time_window_s?: number;
+  retry?: TaskRetry;
 }

 /**
@@ -28,9 +57,11 @@ export async function getResumeUrls(approver?: string, flowLevel?: boolean): Pro
 * @example
 * const extract_data = task(async (url: string) => { ... });
 * const run_external = task("f/external_script", async (x: number) => { ... });
+ * const call_api = task(fetchOrders, { retry: { attempts: 3, delay: 30, multiplier: 2 } });
 *
 * Inside a `workflow()`, calling a task dispatches it as a step.
- * Outside a workflow, the function body executes directly.
+ * Outside a workflow, the function body executes directly and
+ * {@link TaskOptions} — retry included — does not apply.
 *
 * A task runs as its own job, so its result is always encoded as JSON and
 * decoded back before the caller sees it: a `Date` comes back as a string, a
@@ -841,6 +841,22 @@ def parse_sql_client_name(name: str) -> tuple[str, Optional[str]]
 # decoded back before the caller sees it: a ``datetime`` comes back as a
 # string, a tuple as a list.
 # 
+# ``retry`` re-dispatches the task after a failure, inside ``@workflow`` only.
+# Every attempt is a step of its own (``call_api``, ``call_api#2``, ...) and
+# the wait between two of them is a durable sleep, so a retrying task holds no
+# worker while it backs off. Keys: ``attempts`` (retries after the first
+# failure, a whole number from 0 to 100), ``delay`` (seconds before the first
+# retry, sub-second delays dropped), ``multiplier`` (applied to the delay
+# after each attempt, 1 keeps it constant), ``max_delay`` (ceiling in
+# seconds). ``attempts`` is required, and an out-of-range or unknown key is
+# rejected where the policy is written.
+# 
+# A workflow sleeps once per round, so tasks backing off in the same fan-out
+# wait one after another rather than together: the delay before a fan-out
+# retries is the sum of every backoff pending in it, not the longest one, and
+# it grows with both the width of the fan-out and ``attempts``. Retries with
+# no ``delay`` all go out in a single round.
+# 
 # Usage::
 # 
 #     @task
@@ -848,10 +864,15 @@ def parse_sql_client_name(name: str) -> tuple[str, Optional[str]]
 # 
 #     @task(path="f/external_script", timeout=600, tag="gpu")
 #     async def run_external(x: int): ...
-def task(_func = None, path: Optional[str] = None, tag: Optional[str] = None, timeout: Optional[int] = None, cache_ttl: Optional[int] = None, priority: Optional[int] = None, concurrency_limit: Optional[int] = None, concurrency_key: Optional[str] = None, concurrency_time_window_s: Optional[int] = None)
+# 
+#     @task(retry={"attempts": 3, "delay": 30, "multiplier": 2})
+#     async def call_api(payload: dict): ...
+def task(_func = None, path: Optional[str] = None, tag: Optional[str] = None, timeout: Optional[int] = None, cache_ttl: Optional[int] = None, priority: Optional[int] = None, concurrency_limit: Optional[int] = None, concurrency_key: Optional[str] = None, concurrency_time_window_s: Optional[int] = None, retry: Optional[dict] = None)

 # Create a task that dispatches to a separate Windmill script.
 # 
+# ``retry`` takes the same policy as :func:`task`.
+# 
 # Usage::
 # 
 #     extract = task_script("f/data/extract", timeout=600)
@@ -859,10 +880,12 @@ def task(_func = None, path: Optional[str] = None, tag: Optional[str] = None, ti
 #     @workflow
 #     async def main():
 #         data = await extract(url="https://...")
-def task_script(path: str, timeout: Optional[int] = None, tag: Optional[str] = None, cache_ttl: Optional[int] = None, priority: Optional[int] = None, concurrency_limit: Optional[int] = None, concurrency_key: Optional[str] = None, concurrency_time_window_s: Optional[int] = None)
+def task_script(path: str, timeout: Optional[int] = None, tag: Optional[str] = None, cache_ttl: Optional[int] = None, priority: Optional[int] = None, concurrency_limit: Optional[int] = None, concurrency_key: Optional[str] = None, concurrency_time_window_s: Optional[int] = None, retry: Optional[dict] = None)

 # Create a task that dispatches to a separate Windmill flow.
 # 
+# ``retry`` takes the same policy as :func:`task`.
+# 
 # Usage::
 # 
 #     pipeline = task_flow("f/etl/pipeline", priority=10)
@@ -870,7 +893,7 @@ def task_script(path: str, timeout: Optional[int] = None, tag: Optional[str] = N
 #     @workflow
 #     async def main():
 #         result = await pipeline(input=data)
-def task_flow(path: str, timeout: Optional[int] = None, tag: Optional[str] = None, cache_ttl: Optional[int] = None, priority: Optional[int] = None, concurrency_limit: Optional[int] = None, concurrency_key: Optional[str] = None, concurrency_time_window_s: Optional[int] = None)
+def task_flow(path: str, timeout: Optional[int] = None, tag: Optional[str] = None, cache_ttl: Optional[int] = None, priority: Optional[int] = None, concurrency_limit: Optional[int] = None, concurrency_key: Optional[str] = None, concurrency_time_window_s: Optional[int] = None, retry: Optional[dict] = None)

 # Decorator marking an async function as a workflow-as-code entry point.
 # 
@@ -246,6 +246,34 @@ A caught failure reads the same whether it came from a task or from a `step()`,
 Import: `import { workflow, task, taskScript, taskFlow, step, sleep, waitForApproval, getApprovalUrls, getResumeUrls, parallel } from "windmill-client"`

 ```typescript
+/**
+ * Re-dispatch policy for a failed task.
+ *
+ * Every attempt is a step of its own (`fetch`, `fetch#2`, `fetch#3`), and the
+ * wait between two of them is a durable sleep, so a retrying task holds no
+ * worker while it backs off.
+ *
+ * A workflow sleeps once per round, so tasks backing off in the same fan-out
+ * wait one after another rather than together: the delay before a fan-out
+ * retries is the sum of every backoff pending in it, not the longest one, and
+ * it grows with both the width of the fan-out and `attempts`. Retries with no
+ * `delay` all go out in a single round.
+ */
+export interface TaskRetry {
+  /** Attempts after the first failure: `2` runs the task at most 3 times.
+  *  A whole number from 0 to 100; anything else is rejected where the policy
+  *  is written. */
+  attempts: number;
+  /** Seconds to wait before the first retry. Default 0, retry immediately.
+  *  Sub-second delays are dropped — a durable sleep resolves to the second. */
+  delay?: number;
+  /** Applied to the delay after each attempt: 1 (the default) keeps it
+  *  constant, 2 doubles it. */
+  multiplier?: number;
+  /** Ceiling for the delay in seconds, for a `multiplier` above 1. */
+  max_delay?: number;
+}
+
 export interface TaskOptions {
  timeout?: number;
  tag?: string;
@@ -254,6 +282,7 @@ export interface TaskOptions {
  concurrency_limit?: number;
  concurrency_key?: string;
  concurrency_time_window_s?: number;
+  retry?: TaskRetry;
 }

 /**
@@ -271,9 +300,11 @@ export async function getResumeUrls(approver?: string, flowLevel?: boolean): Pro
 * @example
 * const extract_data = task(async (url: string) => { ... });
 * const run_external = task("f/external_script", async (x: number) => { ... });
+ * const call_api = task(fetchOrders, { retry: { attempts: 3, delay: 30, multiplier: 2 } });
 *
 * Inside a `workflow()`, calling a task dispatches it as a step.
- * Outside a workflow, the function body executes directly.
+ * Outside a workflow, the function body executes directly and
+ * {@link TaskOptions} — retry included — does not apply.
 *
 * A task runs as its own job, so its result is always encoded as JSON and
 * decoded back before the caller sees it: a `Date` comes back as a string, a
@@ -417,6 +448,22 @@ def get_resume_urls(approver: str = None, flow_level: bool = None) -> dict
 # decoded back before the caller sees it: a ``datetime`` comes back as a
 # string, a tuple as a list.
 #
+# ``retry`` re-dispatches the task after a failure, inside ``@workflow`` only.
+# Every attempt is a step of its own (``call_api``, ``call_api#2``, ...) and
+# the wait between two of them is a durable sleep, so a retrying task holds no
+# worker while it backs off. Keys: ``attempts`` (retries after the first
+# failure, a whole number from 0 to 100), ``delay`` (seconds before the first
+# retry, sub-second delays dropped), ``multiplier`` (applied to the delay
+# after each attempt, 1 keeps it constant), ``max_delay`` (ceiling in
+# seconds). ``attempts`` is required, and an out-of-range or unknown key is
+# rejected where the policy is written.
+#
+# A workflow sleeps once per round, so tasks backing off in the same fan-out
+# wait one after another rather than together: the delay before a fan-out
+# retries is the sum of every backoff pending in it, not the longest one, and
+# it grows with both the width of the fan-out and ``attempts``. Retries with
+# no ``delay`` all go out in a single round.
+#
 # Usage::
 #
 #     @task
@@ -424,10 +471,15 @@ def get_resume_urls(approver: str = None, flow_level: bool = None) -> dict
 #
 #     @task(path="f/external_script", timeout=600, tag="gpu")
 #     async def run_external(x: int): ...
-def task(_func = None, *, path: Optional[str] = None, tag: Optional[str] = None, timeout: Optional[int] = None, cache_ttl: Optional[int] = None, priority: Optional[int] = None, concurrency_limit: Optional[int] = None, concurrency_key: Optional[str] = None, concurrency_time_window_s: Optional[int] = None)
+#
+#     @task(retry={"attempts": 3, "delay": 30, "multiplier": 2})
+#     async def call_api(payload: dict): ...
+def task(_func = None, *, path: Optional[str] = None, tag: Optional[str] = None, timeout: Optional[int] = None, cache_ttl: Optional[int] = None, priority: Optional[int] = None, concurrency_limit: Optional[int] = None, concurrency_key: Optional[str] = None, concurrency_time_window_s: Optional[int] = None, retry: Optional[dict] = None)

 # Create a task that dispatches to a separate Windmill script.
 #
+# ``retry`` takes the same policy as :func:`task`.
+#
 # Usage::
 #
 #     extract = task_script("f/data/extract", timeout=600)
@@ -435,10 +487,12 @@ def task(_func = None, *, path: Optional[str] = None, tag: Optional[str] = None,
 #     @workflow
 #     async def main():
 #         data = await extract(url="https://...")
-def task_script(path: str, *, timeout: Optional[int] = None, tag: Optional[str] = None, cache_ttl: Optional[int] = None, priority: Optional[int] = None, concurrency_limit: Optional[int] = None, concurrency_key: Optional[str] = None, concurrency_time_window_s: Optional[int] = None)
+def task_script(path: str, *, timeout: Optional[int] = None, tag: Optional[str] = None, cache_ttl: Optional[int] = None, priority: Optional[int] = None, concurrency_limit: Optional[int] = None, concurrency_key: Optional[str] = None, concurrency_time_window_s: Optional[int] = None, retry: Optional[dict] = None)

 # Create a task that dispatches to a separate Windmill flow.
 #
+# ``retry`` takes the same policy as :func:`task`.
+#
 # Usage::
 #
 #     pipeline = task_flow("f/etl/pipeline", priority=10)
@@ -446,7 +500,7 @@ def task_script(path: str, *, timeout: Optional[int] = None, tag: Optional[str]
 #     @workflow
 #     async def main():
 #         result = await pipeline(input=data)
-def task_flow(path: str, *, timeout: Optional[int] = None, tag: Optional[str] = None, cache_ttl: Optional[int] = None, priority: Optional[int] = None, concurrency_limit: Optional[int] = None, concurrency_key: Optional[str] = None, concurrency_time_window_s: Optional[int] = None)
+def task_flow(path: str, *, timeout: Optional[int] = None, tag: Optional[str] = None, cache_ttl: Optional[int] = None, priority: Optional[int] = None, concurrency_limit: Optional[int] = None, concurrency_key: Optional[str] = None, concurrency_time_window_s: Optional[int] = None, retry: Optional[dict] = None)

 # Decorator marking an async function as a workflow-as-code entry point.
 #
@@ -1458,9 +1458,10 @@ def extract_wac_ts_sdk(ts_content: str) -> str:
        return ''

    declarations = []
-    task_options = _extract_ts_interface(ts_content, 'TaskOptions')
-    if task_options:
-        declarations.append(task_options)
+    for interface_name in ('TaskRetry', 'TaskOptions'):
+        interface = _extract_ts_interface(ts_content, interface_name)
+        if interface:
+            declarations.append(interface)

    for function_name in WAC_TS_FUNCTIONS:
        signature = _extract_ts_exported_function(ts_content, function_name)
@@ -46,7 +46,7 @@ cp "${script_dirpath}/sqlUtils.ts" "${script_dirpath}/src/"
 echo "" >> "${script_dirpath}/src/index.ts"
 echo 'export type { DenoS3LightClientSettings } from "./s3Types";' >> "${script_dirpath}/src/index.ts"
 echo "" >> "${script_dirpath}/src/index.ts"
-echo 'export { type Base64, setClient, getVariable, setVariable, getResource, setResource, getResumeUrls, setState, setProgress, getProgress, getState, getIdToken, denoS3LightClientSettings, cancelJob, loadS3FileStream, loadS3File, writeS3File, deleteS3File, signS3Objects, signS3Object, getPresignedS3PublicUrls, getPresignedS3PublicUrl, task, taskScript, taskFlow, workflow, step, sleep, parallel, waitForApproval, getApprovalUrls, type TaskOptions, type Jsonified, type JsonifiedFn, WorkflowCtx, _workflowCtx, setWorkflowCtx, StepSuspend, runScript, runScriptAsync, runScriptByPath, runScriptByHash, runScriptByPathAsync, runScriptByHashAsync, runFlow, runFlowAsync, waitJob, getRootJobId, setFlowUserState, getFlowUserState, usernameToEmail, requestInteractiveSlackApproval, type Sql, requestInteractiveTeamsApproval, appendToResultStream, streamResult, datatable, ducklake, upsertPartition, appendPartition, type DucklakeMaterializeOptions, type SqlStatement, type DatatableSqlTemplateFunction, type SqlTemplateFunction, type S3Object, type S3ObjectRecord, type S3ObjectURI, commitKafkaOffsets } from "./client";' >> "${script_dirpath}/src/index.ts"
+echo 'export { type Base64, setClient, getVariable, setVariable, getResource, setResource, getResumeUrls, setState, setProgress, getProgress, getState, getIdToken, denoS3LightClientSettings, cancelJob, loadS3FileStream, loadS3File, writeS3File, deleteS3File, signS3Objects, signS3Object, getPresignedS3PublicUrls, getPresignedS3PublicUrl, task, taskScript, taskFlow, workflow, step, sleep, parallel, waitForApproval, getApprovalUrls, type TaskOptions, type TaskRetry, type Jsonified, type JsonifiedFn, WorkflowCtx, _workflowCtx, setWorkflowCtx, StepSuspend, runScript, runScriptAsync, runScriptByPath, runScriptByHash, runScriptByPathAsync, runScriptByHashAsync, runFlow, runFlowAsync, waitJob, getRootJobId, setFlowUserState, getFlowUserState, usernameToEmail, requestInteractiveSlackApproval, type Sql, requestInteractiveTeamsApproval, appendToResultStream, streamResult, datatable, ducklake, upsertPartition, appendPartition, type DucklakeMaterializeOptions, type SqlStatement, type DatatableSqlTemplateFunction, type SqlTemplateFunction, type S3Object, type S3ObjectRecord, type S3ObjectURI, commitKafkaOffsets } from "./client";' >> "${script_dirpath}/src/index.ts"

 # Build default export by combining client utilities + services
 # This preserves backward compatibility for `import wmill from "windmill-client"`
@@ -1682,6 +1682,33 @@ export type JsonifiedFn<T extends (...args: any[]) => Promise<any>> = (
  ...args: Parameters<T>
 ) => Promise<Jsonified<Awaited<ReturnType<T>>>>;

+/** Re-dispatch policy for a failed task.
+ *
+ * Every attempt is a step of its own (`fetch`, `fetch#2`, `fetch#3`), and the
+ * wait between two of them is a durable sleep, so a retrying task holds no
+ * worker while it backs off.
+ *
+ * A workflow sleeps once per round, so tasks backing off in the same fan-out
+ * wait one after another rather than together: the delay before a fan-out
+ * retries is the sum of every backoff pending in it, not the longest one, and
+ * it grows with both the width of the fan-out and `attempts`. Retries with no
+ * `delay` all go out in a single round.
+ */
+export interface TaskRetry {
+  /** Attempts after the first failure: `2` runs the task at most 3 times.
+   *  A whole number from 0 to 100; anything else is rejected where the policy
+   *  is written. */
+  attempts: number;
+  /** Seconds to wait before the first retry. Default 0, retry immediately.
+   *  Sub-second delays are dropped — a durable sleep resolves to the second. */
+  delay?: number;
+  /** Applied to the delay after each attempt: 1 (the default) keeps it
+   *  constant, 2 doubles it. */
+  multiplier?: number;
+  /** Ceiling for the delay in seconds, for a `multiplier` above 1. */
+  max_delay?: number;
+}
+
 export interface TaskOptions {
  timeout?: number;
  tag?: string;
@@ -1690,6 +1717,48 @@ export interface TaskOptions {
  concurrency_limit?: number;
  concurrency_key?: string;
  concurrency_time_window_s?: number;
+  retry?: TaskRetry;
+}
+
+/** The worker deserializes a sleep into a `u32` of seconds and fails the whole
+ *  job on anything wider, so a delay a multiplier has run away with has to be
+ *  capped here rather than sent. */
+const MAX_SLEEP_SECONDS = 0xffffffff;
+
+/** Every attempt claims its keys before the first one is dispatched, so an
+ *  unbounded `attempts` is a workflow that hangs allocating rather than a very
+ *  patient one. */
+const MAX_RETRY_ATTEMPTS = 100;
+
+/** Rejected where the policy is written, so a workflow fails at its first line
+ *  rather than mid-run on a replay. */
+function assertUsableRetry(retry: TaskRetry | undefined): void {
+  if (retry === undefined) return;
+  const { attempts } = retry;
+  if (!Number.isInteger(attempts) || attempts < 0 || attempts > MAX_RETRY_ATTEMPTS) {
+    throw new Error(
+      `retry.attempts must be a whole number between 0 and ${MAX_RETRY_ATTEMPTS}, got ${attempts}`,
+    );
+  }
+}
+
+/** How many retries the policy asks for, defended against a value that reached
+ *  `_nextStep` without going through `assertUsableRetry`. */
+function retryAttempts(retry: TaskRetry | undefined): number {
+  const attempts = Math.trunc(retry?.attempts ?? 0) || 0;
+  return Math.min(Math.max(attempts, 0), MAX_RETRY_ATTEMPTS);
+}
+
+/** Seconds to wait before retry number `attempt` (0 is the first retry). */
+function retryDelaySeconds(retry: TaskRetry, attempt: number): number {
+  const base = retry.delay ?? 0;
+  if (!(base > 0)) return 0;
+  const grown = base * Math.pow(retry.multiplier ?? 1, attempt);
+  // Clamping against the ceiling also absorbs the `Infinity` an aggressive
+  // multiplier reaches within a few attempts. Floor, rather than round, so
+  // sub-second delays drop the same way they do in the python client.
+  const seconds = Math.floor(Math.min(retry.max_delay ?? grown, grown, MAX_SLEEP_SECONDS));
+  return seconds > 0 ? seconds : 0;
 }

 /** A step key travels as one path segment when its URLs are minted, so it must be
@@ -1777,59 +1846,103 @@ export class WorkflowCtx {
    options?: TaskOptions,
  ): PromiseLike<any> {
    this._rethrowSwallowed();
-    const key = this._allocKey(name || script || "step");
+    const stepName = name || script || "step";
+    const maxRetries = retryAttempts(options?.retry);

-    if (key in this.completed) {
-      const value = this.completed[key];
-      if (value && typeof value === "object" && (value as any).__wmill_error) {
-        const err = taskErrorFromMarker(value, `Task '${name}' failed`);
-        return { then: (_resolve: any, reject?: any) => { if (reject) reject(err); else throw err; } } as PromiseLike<any>;
+    // Claimed up front, all of them, and named off the first attempt's key: one
+    // allocated later would shift the keys of the steps beside it, and a
+    // `step()` named `t#2` — names are arbitrary — could alias one. Whichever is
+    // allocated second is the one renamed, identically in every round.
+    const baseKey = this._allocKey(stepName);
+    const attemptKeys = [baseKey];
+    const backoffKeys: string[] = [];
+    for (let i = 0; i < maxRetries; i++) {
+      backoffKeys.push(this._allocKey(`${baseKey}#retry${i + 2}`));
+      attemptKeys.push(this._allocKey(`${baseKey}#${i + 2}`));
+    }
+
+    // One pass per attempt. Every attempt the checkpoint already holds is
+    // decided here — a failed one either retries (moving to the next key) or is
+    // handed back to the body — so the loop always ends at the first attempt
+    // that has yet to run.
+    for (let attempt = 0; ; attempt++) {
+      const key = attemptKeys[attempt];
+
+      if (key in this.completed) {
+        const value = this.completed[key];
+        if (value && typeof value === "object" && (value as any).__wmill_error) {
+          if (attempt < maxRetries) {
+            this._retryBackoff(backoffKeys[attempt], baseKey, options!.retry!, attempt);
+            continue;
+          }
+          const err = taskErrorFromMarker(value, `Task '${name}' failed`);
+          return { then: (_resolve: any, reject?: any) => { if (reject) reject(err); else throw err; } } as PromiseLike<any>;
+        }
+        return { then: (resolve: any) => resolve(value) };
      }
-      return { then: (resolve: any) => resolve(value) };
-    }

-    // If this is a child job executing a specific step, return null to signal
-    // that the task wrapper should run the inner function directly
-    if (this._executingKey === key) {
-      return { then: (resolve: any) => resolve(null), _execute_directly: true } as any;
-    }
+      // If this is a child job executing a specific step, return null to signal
+      // that the task wrapper should run the inner function directly
+      if (this._executingKey === key) {
+        return { then: (resolve: any) => resolve(null), _execute_directly: true } as any;
+      }

-    // In child job mode (_executingKey is set), non-matching uncompleted steps
-    // should never resolve or throw — the matching step will throw step_complete
-    // which terminates the workflow. Returning a never-resolving thenable prevents
-    // race conditions where a non-matching step's StepSuspend fires before step_complete.
-    if (this._executingKey !== null) {
-      return { then: () => new Promise(() => {}) };
-    }
+      // In child job mode (_executingKey is set), non-matching uncompleted steps
+      // should never resolve or throw — the matching step will throw step_complete
+      // which terminates the workflow. Returning a never-resolving thenable prevents
+      // race conditions where a non-matching step's StepSuspend fires before step_complete.
+      if (this._executingKey !== null) {
+        return { then: () => new Promise(() => {}) };
+      }

-    const stepInfo: any = { name: name || key, script: script || key, args, key, dispatch_type };
-    if (options) {
-      if (options.timeout !== undefined) stepInfo.timeout = options.timeout;
-      if (options.tag !== undefined) stepInfo.tag = options.tag;
-      if (options.cache_ttl !== undefined) stepInfo.cache_ttl = options.cache_ttl;
-      if (options.priority !== undefined) stepInfo.priority = options.priority;
-      if (options.concurrency_limit !== undefined) stepInfo.concurrent_limit = options.concurrency_limit;
-      if (options.concurrency_key !== undefined) stepInfo.concurrency_key = options.concurrency_key;
-      if (options.concurrency_time_window_s !== undefined) stepInfo.concurrency_time_window_s = options.concurrency_time_window_s;
+      const stepInfo: any = { name: name || key, script: script || key, args, key, dispatch_type };
+      if (options) {
+        if (options.timeout !== undefined) stepInfo.timeout = options.timeout;
+        if (options.tag !== undefined) stepInfo.tag = options.tag;
+        if (options.cache_ttl !== undefined) stepInfo.cache_ttl = options.cache_ttl;
+        if (options.priority !== undefined) stepInfo.priority = options.priority;
+        if (options.concurrency_limit !== undefined) stepInfo.concurrent_limit = options.concurrency_limit;
+        if (options.concurrency_key !== undefined) stepInfo.concurrency_key = options.concurrency_key;
+        if (options.concurrency_time_window_s !== undefined) stepInfo.concurrency_time_window_s = options.concurrency_time_window_s;
+      }
+      this.pending.push(stepInfo);
+      return {
+        then: (): never => {
+          // Only the first .then() call throws with all accumulated steps.
+          // Subsequent calls (e.g. from Promise.all resolving other thenables)
+          // also throw (they'll be caught by the same handler).
+          if (this._suspended) return new Promise(() => {}) as never;
+          this._suspended = true;
+          const steps = [...this.pending];
+          this.pending = [];
+          const names = steps.map(s => s.name).join(", ");
+          console.log(`\n--- WAC: ${names} ---`);
+          this._raiseSuspend({
+            mode: steps.length > 1 ? "parallel" : "sequential",
+            steps,
+          });
+        },
+      };
    }
-    this.pending.push(stepInfo);
-    return {
-      then: (): never => {
-        // Only the first .then() call throws with all accumulated steps.
-        // Subsequent calls (e.g. from Promise.all resolving other thenables)
-        // also throw (they'll be caught by the same handler).
-        if (this._suspended) return new Promise(() => {}) as never;
-        this._suspended = true;
-        const steps = [...this.pending];
-        this.pending = [];
-        const names = steps.map(s => s.name).join(", ");
-        console.log(`\n--- WAC: ${names} ---`);
-        this._raiseSuspend({
-          mode: steps.length > 1 ? "parallel" : "sequential",
-          steps,
-        });
-      },
-    };
+  }
+
+  /** Wait out the backoff between two attempts of a retried task, as a durable
+   *  sleep, and return once there is nothing to wait for — no delay configured,
+   *  or the sleep already in the checkpoint.
+   *
+   *  Raises where it stands, the way `_sleep` does, rather than handing back a
+   *  thenable: a task call the body never awaits is still dispatched (the runner
+   *  flushes `pending`), so a backoff that only fired when awaited would drop
+   *  the retry and let the round report the workflow complete. */
+  private _retryBackoff(key: string, baseKey: string, retry: TaskRetry, attempt: number): void {
+    const seconds = retryDelaySeconds(retry, attempt);
+    if (seconds < 1) return;
+    if (key in this.completed) return;
+    // Child mode never raises: the parent dispatched this child only after its
+    // own round had slept, so the loop moves on to the attempt being executed.
+    if (this._executingKey !== null) return;
+    console.log(`\n--- WAC: sleep(${key}, ${seconds}s) before retrying ${baseKey} ---`);
+    this._raiseSuspend({ mode: "sleep", key, seconds, steps: [] });
  }
  /** Return and clear any pending (unawaited) steps. */
  _flushPending(): Array<{ name: string; script: string; args: Record<string, any>; key: string; dispatch_type: string }> {
@@ -2145,9 +2258,11 @@ export async function step<T>(
 * @example
 * const extract_data = task(async (url: string) => { ... });
 * const run_external = task("f/external_script", async (x: number) => { ... });
+ * const call_api = task(fetchOrders, { retry: { attempts: 3, delay: 30, multiplier: 2 } });
 *
 * Inside a `workflow()`, calling a task dispatches it as a step.
- * Outside a workflow, the function body executes directly.
+ * Outside a workflow, the function body executes directly and
+ * {@link TaskOptions} — retry included — does not apply.
 *
 * A task runs as its own job, so its result is always encoded as JSON and
 * decoded back before the caller sees it: a `Date` comes back as a string, a
@@ -2171,6 +2286,8 @@ export function task<T extends (...args: any[]) => Promise<any>>(
    taskOptions = maybeFnOrOptions as TaskOptions | undefined;
  }

+  assertUsableRetry(taskOptions?.retry);
+
  const taskName = fn.name || taskPath || "";

  // NOT async — in workflow context we return the thenable directly so that
@@ -2255,6 +2372,7 @@ export function task<T extends (...args: any[]) => Promise<any>>(
 * // inside workflow: await extract({ url: "https://..." })
 */
 export function taskScript(path: string, options?: TaskOptions): (...args: any[]) => PromiseLike<any> {
+  assertUsableRetry(options?.retry);
  const name = path.split("/").pop() || path;
  const wrapper = function (...args: any[]) {
    const ctx: WorkflowCtx | null = _workflowCtx ?? Reflect.get(globalThis, "__wmill_wf_ctx");
@@ -2280,6 +2398,7 @@ export function taskScript(path: string, options?: TaskOptions): (...args: any[]
 * // inside workflow: await pipeline({ input: data })
 */
 export function taskFlow(path: string, options?: TaskOptions): (...args: any[]) => PromiseLike<any> {
+  assertUsableRetry(options?.retry);
  const name = path.split("/").pop() || path;
  const wrapper = function (...args: any[]) {
    const ctx: WorkflowCtx | null = _workflowCtx ?? Reflect.get(globalThis, "__wmill_wf_ctx");
@@ -0,0 +1,225 @@
+/**
+ * `TaskOptions.retry`, against the real client.
+ *
+ * Run with: bun test typescript-client/tests/workflow_retry.test.ts
+ *
+ * Nothing carries a retry across rounds: every round re-derives which attempt
+ * comes next from the checkpoint alone, so these drive the rounds a worker
+ * would and assert what it would act on. Imports client.ts itself (a mirror
+ * would pin the mirror), with the two generated modules stubbed so the import
+ * works without ./build.sh.
+ */
+import { expect, test, describe, mock } from "bun:test";
+
+mock.module("../services.gen", () => ({
+  ResourceService: {},
+  VariableService: {},
+  JobService: {},
+  HelpersService: {},
+  AppService: {},
+  MetricsService: {},
+  OidcService: {},
+  UserService: {},
+  KafkaTriggerService: {},
+}));
+mock.module("../core/OpenAPI", () => ({
+  OpenAPI: { BASE: "http://localhost:8000/api", TOKEN: "tok" },
+}));
+
+const { WorkflowCtx, task, step, setWorkflowCtx, StepSuspend } = await import("../client.ts");
+import { isSuspendSignal } from "../wacError";
+
+/** What the failed child job leaves in `completed_steps`. */
+const failed = {
+  __wmill_error: true,
+  message: "boom",
+  step_key: "callApi",
+  result: { error: { name: "Error", message: "boom" } },
+};
+
+/** One round against `completed`, reduced to what the worker acts on. */
+async function round(completed: Record<string, any>, body: () => Promise<any>): Promise<any> {
+  const ctx = new WorkflowCtx({ completed_steps: completed } as any);
+  setWorkflowCtx(ctx);
+  try {
+    return { type: "complete", result: await body() };
+  } catch (e: any) {
+    if (isSuspendSignal(e, StepSuspend)) return { type: "suspend", ...e.dispatchInfo };
+    return { type: "error", error: e };
+  } finally {
+    setWorkflowCtx(null);
+  }
+}
+
+describe("task retry", () => {
+  test("each failure buys a backoff sleep and one more attempt, until attempts run out", async () => {
+    const callApi = task(async function callApi(x: number) {
+      return x;
+    }, { retry: { attempts: 2, delay: 30, multiplier: 2 } });
+    const body = () => callApi(1) as Promise<any>;
+
+    let r = await round({}, body);
+    expect(r.steps.map((s: any) => s.key)).toEqual(["callApi"]);
+
+    const completed: Record<string, any> = { callApi: failed };
+    r = await round(completed, body);
+    expect(r).toMatchObject({ mode: "sleep", key: "callApi#retry2", seconds: 30 });
+
+    completed["callApi#retry2"] = null;
+    r = await round(completed, body);
+    expect(r.steps.map((s: any) => s.key)).toEqual(["callApi#2"]);
+
+    // the delay grows by the multiplier for the second retry
+    completed["callApi#2"] = failed;
+    r = await round(completed, body);
+    expect(r).toMatchObject({ mode: "sleep", key: "callApi#retry3", seconds: 60 });
+
+    completed["callApi#retry3"] = null;
+    r = await round(completed, body);
+    expect(r.steps.map((s: any) => s.key)).toEqual(["callApi#3"]);
+
+    // attempts spent: the failure reaches the body
+    completed["callApi#3"] = failed;
+    r = await round(completed, body);
+    expect(r.type).toBe("error");
+    expect(r.error.name).toBe("TaskError");
+  });
+
+  test("retrying one call of a task does not move the keys of the calls beside it", async () => {
+    const t = task(async function t(x: number) {
+      return x;
+    }, { retry: { attempts: 1 } });
+    const body = () => Promise.all([t(1), t(2)]);
+
+    let r = await round({}, body);
+    expect(r.steps.map((s: any) => s.key)).toEqual(["t", "t_2"]);
+
+    // the first call retries as `t#2`; the second keeps the `t_2` it was
+    // dispatched under, rather than being read as the first call's retry
+    r = await round({ t: failed, t_2: 20 }, body);
+    expect(r.steps.map((s: any) => s.key)).toEqual(["t#2"]);
+  });
+
+  test("a task the body never awaits still sleeps and retries", async () => {
+    // The runner dispatches unawaited task calls by flushing `pending`, so a
+    // backoff that only fired when awaited would drop the retry silently and
+    // report the workflow complete.
+    const fire = task(async function fire() {
+      return 1;
+    }, { retry: { attempts: 1, delay: 30 } });
+    const body = async () => {
+      fire();
+      return "done";
+    };
+
+    const r = await round({ fire: failed }, body);
+    expect(r).toMatchObject({ mode: "sleep", key: "fire#retry2", seconds: 30 });
+  });
+
+  // `step()` names are arbitrary strings, so a step really can be called `t#2`.
+  // Whichever of the two allocates second is the one renamed, and it has to be
+  // the same one in every round — hence claiming the attempt keys up front.
+  test("an inline step named like an attempt key, before the task, keeps its key", async () => {
+    const t = task(async function t(x: number) {
+      return x;
+    }, { retry: { attempts: 1 } });
+    const body = async () => {
+      const decoy = await step("t#2", () => "not an attempt");
+      return [decoy, await t(1)];
+    };
+
+    const r = await round({ "t#2": "not an attempt", t: failed }, body);
+    expect(r.steps.map((s: any) => s.key)).toEqual(["t#2_2"]);
+  });
+
+  test("an inline step named like an attempt key, after the task, does not stand in for it", async () => {
+    const t = task(async function t(x: number) {
+      return x;
+    }, { retry: { attempts: 1 } });
+    const body = async () => {
+      const pending = t(1);
+      const decoy = await step("t#2", () => "not an attempt");
+      return [decoy, await pending];
+    };
+
+    // Round 1 records the step under the key left over after the task claimed
+    // `t#2`, so the retry re-dispatches instead of reading the step's value.
+    let r = await round({}, body);
+    expect(r).toMatchObject({ mode: "inline_checkpoint", key: "t#2_2" });
+
+    r = await round({ t: failed, "t#2_2": "not an attempt" }, body);
+    expect(r.steps.map((s: any) => s.key)).toEqual(["t#2"]);
+  });
+
+  test("the child dispatched for an attempt walks the loop past the failure and backoff", async () => {
+    // The non-matching branch returns a thenable that never resolves, so a
+    // child that walks the loop wrong parks the run until its timeout.
+    const t = task(async function t(x: number) {
+      return x * 10;
+    }, { retry: { attempts: 1, delay: 30 } });
+    const ctx = new WorkflowCtx({
+      completed_steps: { t: failed, "t#retry2": null },
+      _executing_key: "t#2",
+    } as any);
+    setWorkflowCtx(ctx);
+    try {
+      const suspend: any = await t(4).then(
+        () => null,
+        (e: any) => e,
+      );
+      expect(suspend.dispatchInfo).toMatchObject({ mode: "step_complete", result: 40 });
+    } finally {
+      setWorkflowCtx(null);
+    }
+  });
+
+  test("an out-of-range attempt count is rejected where it is written", () => {
+    // Each attempt claims its keys before the first one is dispatched, so an
+    // unbounded count would hang the workflow allocating them.
+    for (const attempts of [10_000, -1, 2.5, Infinity, NaN]) {
+      expect(() =>
+        task(async function t(x: number) {
+          return x;
+        }, { retry: { attempts } }),
+      ).toThrow("whole number");
+    }
+  });
+
+  test("a zero multiplier is honoured, not read as the default of 1", async () => {
+    const t = task(async function t(x: number) {
+      return x;
+    }, { retry: { attempts: 2, delay: 30, multiplier: 0 } });
+    const body = () => t(1) as Promise<any>;
+
+    let r = await round({ t: failed }, body);
+    expect(r).toMatchObject({ mode: "sleep", key: "t#retry2", seconds: 30 });
+
+    // 30 * 0 — the second retry goes out with no wait at all
+    r = await round({ t: failed, "t#retry2": null, "t#2": failed }, body);
+    expect(r.steps.map((s: any) => s.key)).toEqual(["t#3"]);
+  });
+
+  test("max_delay caps the backoff a multiplier grows", async () => {
+    const t = task(async function t(x: number) {
+      return x;
+    }, { retry: { attempts: 2, delay: 60, multiplier: 100, max_delay: 300 } });
+    const body = () => t(1) as Promise<any>;
+
+    const r = await round({ t: failed, "t#retry2": null, "t#2": failed }, body);
+    expect(r).toMatchObject({ mode: "sleep", key: "t#retry3", seconds: 300 });
+  });
+
+  test("a retry that succeeds resolves to its value, and later steps keep their own keys", async () => {
+    // No delay, so the retry is dispatched without a sleep round in between.
+    const flaky = task(async function flaky(x: number) {
+      return x;
+    }, { retry: { attempts: 1 } });
+    const double = task(async function double(v: number) {
+      return v * 2;
+    });
+    const body = async () => double((await flaky(1)) as number);
+
+    const r = await round({ flaky: failed, "flaky#2": 7 }, body);
+    expect(r.steps).toMatchObject([{ key: "double", args: { v: 7 } }]);
+  });
+});