feat: test flows from the SDKs with mocked or replaced steps, subflows included (#12538)

* feat: test flows from the SDKs with mocked or replaced steps

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>

* fix: walk only real flow modules and refuse overrides a step would ignore

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>

* fix: harden flow tests against sdk tokens, rls, test-run pins and built-in tools

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>

* fix: require the code capability to test a flow

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>

* fix: refuse overriding ai agent tools in flow tests

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>

* fix: refuse overriding the preprocessor, which a flow test never runs

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>

* feat: override steps inside subflows in flow tests

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>

* fix: gate subflow overrides on worker version and disable caching in flow tests

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>

* docs: note subflows never run their preprocessor in flow test docstrings

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>

---------

Co-authored-by: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
Ruben Fiszel
2026-11-19 14:40:33 +01:00
committed by GitHub
co-authored by Claude Opus 5.5
parent 5bf5ff28bc
commit 98439f8521
29 changed files with 1757 additions and 22 deletions
+136
View File
@@ -325,6 +325,67 @@ class Windmill:
raise Exception("path must be provided")
return self.post(endpoint, json=args, params=params).text
def test_flow_async(
self,
path: str,
args: dict = None,
steps: dict = None,
) -> str:
"""Start the deployed flow at ``path`` with some steps overridden and return its job id.
See ``test_flow`` for ``steps``.
"""
overrides = {}
uses_function = False
for step_id, o in (steps or {}).items():
if callable(o):
name = getattr(o, "__name__", None)
if not name or name == "<lambda>":
raise Exception(
f"step `{step_id}`: the replacing function must be a named, "
"top-level function of this file"
)
overrides[step_id] = {"function": name}
uses_function = True
else:
overrides[step_id] = {"return_value": o}
body = {"args": args or {}, "steps": overrides}
if uses_function:
caller_job_id = os.environ.get("WM_JOB_ID")
if not caller_job_id:
raise Exception(
"replacing a step with a function only works from a script running in Windmill"
)
body["caller_job_id"] = caller_job_id
return self.post(
f"/w/{self.workspace}/jobs/run/flow_test/{path}", json=body
).text
def test_flow(
self,
path: str,
args: dict = None,
steps: dict = None,
timeout: dt.timedelta | int | float | None = None,
verbose: bool = False,
assert_result_is_not_none: bool = False,
) -> Any:
"""Run the deployed flow at ``path`` with some steps overridden and return its result.
See the module-level ``test_flow``.
"""
job_id = self.test_flow_async(path, args, steps)
return self.wait_job(
job_id,
timeout=timeout,
verbose=verbose,
assert_result_is_not_none=assert_result_is_not_none,
)
# Not pytest tests, despite the name.
test_flow.__test__ = False
test_flow_async.__test__ = False
def run_script(
self,
path: str = None,
@@ -1777,6 +1838,81 @@ def run_flow_async(
)
@init_global_client
def test_flow(
path: str,
args: Dict[str, Any] = None,
steps: Dict[str, Any] = None,
timeout: dt.timedelta | int | float | None = None,
verbose: bool = False,
assert_result_is_not_none: bool = False,
) -> Any:
"""Run the deployed flow at ``path`` with some steps overridden, wait for it and return its
result. Meant for test scripts.
Each entry of ``steps`` maps a step id to either:
- a value: the step is skipped and returns that value, or
- a top-level function of the calling script's own file: the step runs that function instead
of its own code, with the step's inputs as keyword arguments.
A step inside a subflow is named by the path of step ids to it, ``"sub.inner"`` for step
``inner`` of the subflow that step ``sub`` calls. It applies to every call of that subflow
made by the tested flow and by the subflows it overrides; a subflow left as is still calls
the deployed one. Those subflows, and each one on the way to an override, run as you rather
than as their on-behalf-of identity. Result caching is off for the whole test run.
The flow's preprocessor does not run: ``args`` are what the flow receives after it. A subflow
never runs its own preprocessor, in a test or not. AI agent
tools cannot be overridden; override the agent step instead.
Example::
def fake_fetch(url: str):
return {"status": 200}
def main():
result = wmill.test_flow("f/flows/sync", {"user": "a"}, steps={
"fetch_users": fake_fetch, # replaced logic
"send_email": "sent", # mocked result
})
Args:
path: Flow path
args: Flow arguments
steps: Step id to mocked result, or to the function replacing the step
timeout: Maximum time to wait
verbose: Enable verbose logging
assert_result_is_not_none: Raise exception if result is None
Returns:
Flow result
"""
return _client.test_flow(
path,
args=args,
steps=steps,
timeout=timeout,
verbose=verbose,
assert_result_is_not_none=assert_result_is_not_none,
)
@init_global_client
def test_flow_async(
path: str,
args: Dict[str, Any] = None,
steps: Dict[str, Any] = None,
) -> str:
"""Like ``test_flow``, but returns the job id without waiting for the flow."""
return _client.test_flow_async(path, args=args, steps=steps)
# Not pytest tests, despite the name.
test_flow.__test__ = False
test_flow_async.__test__ = False
@init_global_client
def run_script_sync(
hash: str,