fix(wac): one failure record for tasks and steps, in every round (#10368)

* fix(wac): hand a caught task and step failure the same shape in every round

Co-Authored-By: Claude Opus 5 (1M context) <noreply@anthropic.com>

* refactor(wac): decide the failure record once, server-side

Co-Authored-By: Claude Opus 5 (1M context) <noreply@anthropic.com>

* fix(wac): leave a legacy SDK's failure marker untouched, and ship wacError to jsr

Co-Authored-By: Claude Opus 5 (1M context) <noreply@anthropic.com>

* fix(wac): carry a step's custom error fields, and bound the stack in bytes

Co-Authored-By: Claude Opus 5 (1M context) <noreply@anthropic.com>

* fix(wac): keep a step's extra fields serializable and bounded

Co-Authored-By: Claude Opus 5 (1M context) <noreply@anthropic.com>

* fix(wac): record a non-Error throw the way a task records it

Co-Authored-By: Claude Opus 5 (1M context) <noreply@anthropic.com>

* fix(wac): guard the last unguarded throw site in the step marker

Co-Authored-By: Claude Opus 5 (1M context) <noreply@anthropic.com>

* fix(wac): make failure reporting non-throwing on both clients

Co-Authored-By: Claude Opus 5 (1M context) <noreply@anthropic.com>

* fix(wac): take the step traceback the way the executor takes it

Co-Authored-By: Claude Opus 5 (1M context) <noreply@anthropic.com>

* fix(wac): contain the reads that happen before a failure is checkpointed

Co-Authored-By: Claude Opus 5 (1M context) <noreply@anthropic.com>

* fix(wac): fall back to the checkpointed marker, not the live one

Co-Authored-By: Claude Opus 5 (1M context) <noreply@anthropic.com>

* fix(wac): keep non-finite fields and hostile proxies out of the checkpoint path

Co-Authored-By: Claude Opus 5 (1M context) <noreply@anthropic.com>

* fix(wac): keep the snapshot that passed the serialization probe

Co-Authored-By: Claude Opus 5 (1M context) <noreply@anthropic.com>

* chore(wac): keep the failure-record module's surface to what is used

Co-Authored-By: Claude Opus 5 (1M context) <noreply@anthropic.com>

---------

Co-authored-by: Claude Opus 5 (1M context) <noreply@anthropic.com>
This commit is contained in:
Ruben Fiszel
2026-07-28 13:15:24 +02:00
committed by GitHub
parent 8d684c0b23
commit aeaea57ca1
17 changed files with 1143 additions and 136 deletions
+14 -3
View File
@@ -5812,6 +5812,15 @@ pub struct WacInlineCheckpointPayload {
pub duration_ms: Option<u64>,
}
#[derive(Serialize)]
pub struct WacInlineCheckpointResponse {
/// The normalized failure record, when the posted step failed. The SDK
/// raises from this rather than from its own copy, so the round that ran
/// the failing body and every replay of it read the same object.
#[serde(skip_serializing_if = "Option::is_none")]
pub failure: Option<serde_json::Value>,
}
/// Fast-path endpoint called by the WAC v2 SDKs to persist a single `step()`
/// checkpoint delta without unwinding the parent workflow subprocess.
///
@@ -5837,7 +5846,7 @@ pub async fn wac_inline_checkpoint(
Extension(db): Extension<DB>,
Path((w_id, job_id)): Path<(String, Uuid)>,
Json(payload): Json<WacInlineCheckpointPayload>,
) -> error::Result<StatusCode> {
) -> error::Result<Json<WacInlineCheckpointResponse>> {
// Enforce ephemeral-job-token binding: the presented token must be the
// one issued to *this* specific job. Regular workspace API tokens don't
// have `job_id` populated in their JWT claims, so `token_job_id` is None
@@ -5869,7 +5878,7 @@ pub async fn wac_inline_checkpoint(
let source_hash = runnable_id.map(|h| h.to_string());
let mut tx = db.begin().await?;
windmill_common::wac::persist_inline_checkpoint_delta(
let failure = windmill_common::wac::persist_inline_checkpoint_delta(
&mut tx,
&job_id,
source_hash.as_deref(),
@@ -5881,7 +5890,9 @@ pub async fn wac_inline_checkpoint(
.await?;
tx.commit().await?;
Ok(StatusCode::OK)
// Only failures come back: a successful step's result can be large, and the
// SDK already holds it.
Ok(Json(WacInlineCheckpointResponse { failure }))
}
lazy_static::lazy_static! {
+337 -2
View File
@@ -80,6 +80,172 @@ mod tests {
assert_eq!(approval_resume_id("approval_2"), 0x50d1_eeca);
assert_eq!(approval_resume_id("manager"), 0x6ee4_a469);
}
use super::wac_failure_record;
use serde_json::json;
/// The point of the function: a task failure and a step failure describing
/// the same error must be indistinguishable to the handler that catches
/// them, apart from the child job only a task has.
#[test]
fn a_task_and_a_step_failure_read_the_same() {
let from_child = wac_failure_record(
"fetch",
Some("abc-123"),
&json!({"error": {"name": "ValueError", "message": "nope", "stack": "frames"}}),
);
let from_step = wac_failure_record(
"fetch",
None,
&json!({"error": {"name": "ValueError", "message": "nope", "stack": "frames"}}),
);
assert_eq!(from_child["result"], from_step["result"]);
assert_eq!(from_child["message"], json!("nope"));
assert_eq!(from_step["message"], json!("nope"));
assert_eq!(from_child["child_job_id"], json!("abc-123"));
assert_eq!(from_step.get("child_job_id"), None);
}
/// A child job's result is whatever the failing job produced — a cancel, a
/// timeout, an executor that writes a bare string. The handler is still
/// promised `name` and `message`, so they cannot be conjured per-caller.
#[test]
fn an_unshaped_child_result_still_yields_name_and_message() {
for raw in [
json!({"error": "boom"}),
json!({"error": {"message": "boom"}}),
json!("boom"),
json!(null),
] {
let rec = wac_failure_record("s", None, &raw);
assert_eq!(rec["result"]["error"]["name"], json!("Error"), "{raw}");
assert!(
rec["result"]["error"]["message"].is_string(),
"{raw} produced no message"
);
assert_eq!(rec["result"]["error"].get("stack"), None, "{raw}");
}
// Nothing to say beats saying `{}`: the fallback exists for these.
for empty in [json!(null), json!({}), json!([]), json!({"error": ""})] {
assert_eq!(
wac_failure_record("s", None, &empty)["message"],
json!("WAC step 's' failed"),
"{empty}"
);
}
}
/// Extra keys are the failing side's own, and dropping them would lose a
/// custom error's fields; the three normalized ones still win.
#[test]
fn extra_error_fields_survive_normalization() {
let rec = wac_failure_record(
"s",
None,
&json!({"error": {"name": "HttpError", "message": "429", "code": 429, "stack": 12}}),
);
assert_eq!(rec["result"]["error"]["code"], json!(429));
assert_eq!(rec["result"]["error"]["name"], json!("HttpError"));
// a non-string stack is not something a handler can be told to read
assert_eq!(rec["result"]["error"].get("stack"), None);
}
use super::normalize_posted_step_result;
/// An SDK that predates the echoed record raises from the copy it built, so
/// rewriting what it posted would make its live round and its replays
/// disagree — the very thing being fixed here. It keeps its own shape.
#[test]
fn a_legacy_sdk_marker_is_stored_untouched() {
let legacy = json!({
"__wmill_error": true,
"message": "nope",
"step_key": "s",
"result": {"error": "nope", "type": "TypeError"},
});
assert_eq!(normalize_posted_step_result("s", legacy.clone()), legacy);
}
#[test]
fn a_current_sdk_marker_is_normalized_and_a_success_is_not() {
let posted = json!({
"__wmill_error": true,
"message": "nope",
"step_key": "s",
"result": {"error": {"name": "ValueError", "message": "nope"}},
});
let stored = normalize_posted_step_result("s", posted);
assert_eq!(stored["result"]["error"]["name"], json!("ValueError"));
assert_eq!(stored["step_key"], json!("s"));
let success = json!({"rows": 3});
assert_eq!(normalize_posted_step_result("s", success.clone()), success);
}
#[test]
fn an_oversized_stack_is_truncated() {
let rec = wac_failure_record(
"s",
None,
&json!({"error": {"message": "m", "stack": "x".repeat(100_000)}}),
);
let stack = rec["result"]["error"]["stack"].as_str().unwrap();
assert!(stack.len() < 100_000, "stack was not truncated");
assert!(stack.ends_with("... (truncated)"));
}
/// The cap bounds what lands in the checkpoint, so it has to be bytes: a
/// multibyte traceback counted in characters would be up to 4x over.
#[test]
fn the_stack_cap_counts_bytes_not_characters() {
let rec = wac_failure_record(
"s",
None,
&json!({"error": {"message": "m", "stack": "é".repeat(50_000)}}),
);
let stack = rec["result"]["error"]["stack"].as_str().unwrap();
assert!(
stack.len() <= 8 * 1024 + "\n... (truncated)".len(),
"kept {} bytes",
stack.len()
);
}
/// `extra` is the failing side's own attributes, so it can carry a response
/// body straight past the cap that exists to bound the checkpoint.
#[test]
fn an_oversized_extra_is_dropped_rather_than_stored() {
let rec = wac_failure_record(
"s",
None,
&json!({"error": {"message": "m", "extra": {"body": "x".repeat(100_000)}}}),
);
assert_eq!(rec["result"]["error"].get("extra"), None);
assert_eq!(rec["result"]["error"]["extra_omitted"], json!(true));
// one that fits is kept whole
let small = wac_failure_record(
"s",
None,
&json!({"error": {"message": "m", "extra": {"code": 429}}}),
);
assert_eq!(small["result"]["error"]["extra"], json!({"code": 429}));
assert_eq!(small["result"]["error"].get("extra_omitted"), None);
}
/// `message` is the failure's own message; `Value::to_string` on a string
/// would hand the handler `"boom"` with the JSON quotes still on it.
#[test]
fn a_bare_string_failure_keeps_its_message_unquoted() {
assert_eq!(
wac_failure_record("s", None, &json!({"error": "boom"}))["message"],
json!("boom")
);
assert_eq!(
wac_failure_record("s", None, &json!("boom"))["message"],
json!("boom")
);
}
}
/// Load the WAC checkpoint from `v2_job_status.workflow_as_code_status._checkpoint`.
@@ -139,6 +305,164 @@ pub async fn save_checkpoint(
Ok(())
}
/// Marks a `completed_steps` entry as a failure rather than a step result.
pub(crate) const WAC_ERROR_MARKER: &str = "__wmill_error";
/// Per-field budget for the two unbounded things a failure record carries, the
/// stack and `extra`. `persist_inline_checkpoint_delta` rewrites the whole
/// checkpoint on every step, so either one left unbounded is re-serialized once
/// per subsequent step for the rest of the workflow. The two are additive, so a
/// record costs at most twice this.
const MAX_CHECKPOINT_FIELD_BYTES: usize = 8 * 1024;
fn truncate_stack(stack: &str) -> String {
if stack.len() <= MAX_CHECKPOINT_FIELD_BYTES {
return stack.to_string();
}
// Byte budget, not characters: the cap exists to bound what goes into the
// checkpoint, and a multibyte traceback would otherwise be up to 4x it.
let mut cut = MAX_CHECKPOINT_FIELD_BYTES;
while cut > 0 && !stack.is_char_boundary(cut) {
cut -= 1;
}
format!("{}\n... (truncated)", &stack[..cut])
}
/// A failure's own message, without the JSON quoting `Value::to_string` puts
/// around a string. `None` when the value carries no message at all, so the
/// caller's fallback wins — a reader handed `{}` learns less than one handed
/// "WAC step 'x' failed".
fn value_message(v: &Value) -> Option<String> {
match v {
Value::Null => None,
Value::String(s) if s.is_empty() => None,
Value::String(s) => Some(s.clone()),
Value::Object(o) if o.is_empty() => None,
Value::Array(a) if a.is_empty() => None,
other => Some(other.to_string()),
}
}
fn message_only(message: Option<String>) -> serde_json::Map<String, Value> {
let mut m = serde_json::Map::new();
if let Some(message) = message {
m.insert("message".to_string(), Value::String(message));
}
m
}
/// Build the failure record a caught WAC failure reads, from whatever the
/// failing side produced.
///
/// The single place this shape is decided, for both a task failure (arriving as
/// the child job's own result) and a `step()` failure (as the SDK posted it).
/// Assembling it per caller instead is how the two come to disagree on `name`
/// or on `stack` while both claim to be one shape. `name`, `message` and
/// `stack` are normalized here; any other key the failing side attached to its
/// error is passed through untouched, so a custom error's own fields survive.
pub fn wac_failure_record(step_key: &str, child_job_id: Option<&str>, raw_result: &Value) -> Value {
let mut error = match raw_result.get("error") {
Some(Value::Object(o)) => o.clone(),
// A bare-string error (some executors), or a result not shaped like a
// failure at all: keep whatever it says as the message rather than
// dropping it.
Some(other) => message_only(value_message(other)),
None => message_only(value_message(raw_result)),
};
let name = error
.get("name")
.and_then(|v| v.as_str())
.filter(|s| !s.is_empty())
.unwrap_or("Error")
.to_string();
let message = error
.get("message")
.and_then(|v| v.as_str())
.filter(|s| !s.is_empty())
.map(|s| s.to_string())
.unwrap_or_else(|| format!("WAC step '{step_key}' failed"));
error.insert("name".to_string(), Value::String(name));
error.insert("message".to_string(), Value::String(message.clone()));
match error.get("stack").and_then(|v| v.as_str()) {
Some(stack) => {
error.insert("stack".to_string(), Value::String(truncate_stack(stack)));
}
// Never invent one, and never keep a non-string in the field a handler
// is told it can read.
None => {
error.remove("stack");
}
}
// `extra` is the failing side's own attributes, so it can hold a response
// body or a dataframe repr and route straight around the stack cap into the
// checkpoint this record is rewritten into on every later step. Dropped
// wholesale past the same budget rather than truncated, since half a
// structure is worse than a flag saying it was too big.
if let Some(extra) = error.get("extra") {
if serde_json::to_string(extra).map_or(true, |s| s.len() > MAX_CHECKPOINT_FIELD_BYTES) {
error.remove("extra");
error.insert("extra_omitted".to_string(), Value::Bool(true));
}
}
let mut record = serde_json::Map::new();
record.insert(WAC_ERROR_MARKER.to_string(), Value::Bool(true));
// `str(e)` / `e.message` reads the failure's own message whether it came
// from a task or a step; which task, and which child job, are the fields
// below rather than prose baked into the message.
record.insert("message".to_string(), Value::String(message));
record.insert("step_key".to_string(), Value::String(step_key.to_string()));
if let Some(child) = child_job_id {
record.insert("child_job_id".to_string(), Value::String(child.to_string()));
}
record.insert(
"result".to_string(),
Value::Object(
[("error".to_string(), Value::Object(error))]
.into_iter()
.collect(),
),
);
Value::Object(record)
}
/// Decide what to store for a step result an SDK posted.
///
/// A failure is normalized through `wac_failure_record`, the same function that
/// shapes task failures, so the two cannot drift apart.
///
/// A marker an older SDK posted is stored untouched instead. Those are
/// recognizable by an `error` that is a message string rather than an object,
/// and the SDK that posted one raises from the copy it built and ignores the
/// record echoed back to it. Rewriting it here would leave the round that ran
/// the failing body reading one shape and every replay of it reading another —
/// the divergence this whole mechanism exists to remove. It keeps its own shape
/// until it upgrades.
pub(crate) fn normalize_posted_step_result(key: &str, posted: Value) -> Value {
if !is_wac_failure(&posted) {
return posted;
}
let normalizable = posted
.get("result")
.and_then(|r| r.get("error"))
.map(|e| e.is_object())
.unwrap_or(false);
if !normalizable {
return posted;
}
wac_failure_record(key, None, posted.get("result").unwrap_or(&Value::Null))
}
/// Whether a `completed_steps` entry is a failure record.
pub(crate) fn is_wac_failure(value: &Value) -> bool {
value
.get(WAC_ERROR_MARKER)
.and_then(|v| v.as_bool())
.unwrap_or(false)
}
/// Process a completed child job result: add to checkpoint's completed_steps.
pub fn add_completed_step(checkpoint: &mut WacCheckpoint, step_key: &str, result: Value) {
checkpoint
@@ -225,6 +549,11 @@ pub fn add_completed_step(checkpoint: &mut WacCheckpoint, step_key: &str, result
/// multiple times per call, and the `||` merges re-serialize the whole
/// object. The two-statement Rust-side load-modify-save below is ~10×
/// faster in practice, so we keep it and rely on the SDK-level lock.
///
/// Returns the value actually stored: a failure posted by an SDK is normalized
/// through `wac_failure_record` first, so the round that ran the failing body
/// can raise from the same record every replay will read instead of building
/// its own copy of it.
pub async fn persist_inline_checkpoint_delta(
tx: &mut Transaction<'_, Postgres>,
job_id: &Uuid,
@@ -233,7 +562,7 @@ pub async fn persist_inline_checkpoint_delta(
result: Value,
started_at: Option<&str>,
duration_ms: Option<u64>,
) -> error::Result<()> {
) -> error::Result<Option<Value>> {
// Row-lock the existing checkpoint row (if any) for the duration of the
// transaction. NULL if the row doesn't exist yet — see the doc comment
// above for why the first-write race is accepted.
@@ -288,6 +617,12 @@ pub async fn persist_inline_checkpoint_delta(
"WAC v2 inline checkpoint — persisting step result"
);
let result = normalize_posted_step_result(key, result);
// Only a failure is ever read back, so only a failure is copied: a
// successful step's result can be large and moves straight into the
// checkpoint.
let failure = is_wac_failure(&result).then(|| result.clone());
add_completed_step(&mut checkpoint, key, result);
let status_json = serde_json::to_value(&checkpoint)
@@ -335,5 +670,5 @@ pub async fn persist_inline_checkpoint_delta(
.await
.map_err(|e| Error::InternalErr(format!("Failed to write step timeline: {e}")))?;
Ok(())
Ok(failure)
}
@@ -1791,13 +1791,11 @@ pub(crate) async fn handle_wac_child_completion(
step_key = %step_key,
"WAC v2 child job failed, storing error for workflow try/catch"
);
json!({
"__wmill_error": true,
"message": format!("WAC task '{}' failed (child job {})", step_key, child_job_id),
"child_job_id": child_job_id.to_string(),
"step_key": step_key,
"result": child_err,
})
windmill_common::wac::wac_failure_record(
&step_key,
Some(&child_job_id.to_string()),
&child_err,
)
};
tracing::info!(
+11 -4
View File
@@ -6506,6 +6506,8 @@ Python: \`except Exception\` is safe around WAC calls because internal suspensio
TypeScript: avoid broad \`try/catch\` around WAC SDK calls. The SDK uses an internal suspension error during initial dispatch; catching it can break workflow suspension. If a broad catch is unavoidable, rethrow internal suspension errors before handling business errors.
A caught failure reads the same whether it came from a task or from a \`step()\`, and the same in the round that ran the failing body as in every round replaying it. It carries \`step_key\`, \`child_job_id\` (absent for a \`step()\`, which runs in the workflow job and has no child job), a \`message\` that is the failure's own message, and \`result\` = \`{"error": {"name", "message", "stack"?, "extra"?}}\`. \`name\`, \`message\` and \`stack\` are the fields that read the same whichever side failed; \`name\` and \`message\` are always there, \`stack\` only when the failure had a traceback to give. \`extra\` carries the failure's own custom fields (an exception's attributes, an error's properties) and is best-effort: it is absent when there were none, and a task can report entries a step does not, so read it defensively and don't branch on its absence. \`extra\` is dropped when it is too large to keep in the checkpoint, and \`extra_omitted: true\` says so — absent \`extra\` with no \`extra_omitted\` means the failure simply had no custom fields. Branch on those, not on the original exception type: the workflow body re-runs from the top every round and a replay rebuilds the failure from the checkpoint, so nothing outside that record survives. Python raises \`TaskError\`; TypeScript throws an \`Error\` named \`TaskError\` carrying the same fields. Nothing is chained onto \`__cause__\` / \`cause\` — the traceback is in \`result.error.stack\`, and is also printed to the job log when the step fails.
## TypeScript Workflow-as-Code API (windmill-client)
@@ -6644,14 +6646,19 @@ export async function parallel<T, R>(items: T[], fn: (item: T) => PromiseLike<R>
Import: \`from wmill import workflow, task, task_script, task_flow, step, sleep, wait_for_approval, get_approval_urls, get_resume_urls, parallel, TaskError\`
\`\`\`python
# Raised when a WAC task step failed.
# Raised when a WAC \`\`task\`\` or \`\`step\`\` failed.
#
# Attributes:
# step_key: The checkpoint key of the failed step.
# child_job_id: The UUID of the failed child job.
# result: The error result from the child job.
# child_job_id: The UUID of the failed child job, or \`\`None\`\` for a
# \`\`step()\`\`, which runs in the workflow job and has no child job.
# result: \`\`{"error": {"name", "message", "stack"?, "extra"?}}\`\` — the
# same shape whether a task or a step failed. \`\`name\`\` and \`\`message\`\`
# are always present; \`\`stack\`\` only when the failure had a traceback,
# and \`\`extra\`\` only when it carried custom fields of its own, dropped
# with \`\`extra_omitted: True\`\` beside it when too large to checkpoint.
class TaskError(Exception):
def __init__(self, message: str, *, step_key: str = '', child_job_id: str = '', result = None)
def __init__(self, message: str, *, step_key: str = '', child_job_id: Optional[str] = None, result = None)
# Get URLs needed for resuming a flow after suspension.
#
+188 -11
View File
@@ -976,13 +976,21 @@ class TestRaisingInlineStepIsCheckpointed:
on the never-resolving future forever.
"""
# A failed child job reports `{"error": {"name", "message", "stack"}}`, and a
# failed step() has to be indistinguishable from it. The stack is a traceback
# string, asserted separately.
MARKER = {
"__wmill_error": True,
"message": "boom",
"step_key": "risky",
"result": {"error": "boom", "type": "ValueError"},
"result": {"error": {"name": "ValueError", "message": "boom"}},
}
@staticmethod
def _without_stack(marker: dict) -> dict:
error = {k: v for k, v in marker["result"]["error"].items() if k != "stack"}
return {**marker, "result": {**marker["result"], "error": error}}
@staticmethod
def _boom():
raise ValueError("boom")
@@ -1003,36 +1011,205 @@ class TestRaisingInlineStepIsCheckpointed:
r = _run_workflow(self._wf(), {}, {"x": 5})
assert r["type"] == "inline_checkpoint"
assert r["key"] == "risky"
assert r["result"] == self.MARKER
assert self._without_stack(r["result"]) == self.MARKER
stack = r["result"]["result"]["error"]["stack"]
# Frames only and the SDK's own `result = fn()` frame dropped, the way
# the python executor formats a failed job's stack.
assert 'raise ValueError("boom")' in stack
assert "result = fn()" not in stack
assert not stack.startswith("Traceback")
def test_custom_exception_attributes_survive_under_extra(self):
"""A failed child job reports custom attributes under ``error.extra``;
a step dropping them would make the same exception carry less depending
on how it was run."""
from wmill.client import _step_error_marker
class HttpError(ValueError):
def __init__(self):
super().__init__("429")
self.code = 429
error = _step_error_marker("k", HttpError())["result"]["error"]
assert error["extra"] == {"code": 429}
assert error["name"] == "HttpError"
assert "extra" not in _step_error_marker("k", ValueError("plain"))["result"]["error"]
def test_unserializable_attributes_do_not_cost_the_fast_path(self):
"""The fast-path POST serializes strictly, so an exception holding a
live object — ``resp.raise_for_status()`` is the common one — must not
make the marker unserializable and drop the step onto the slow path."""
import json as _json
from wmill.client import _step_error_marker
class Boom(Exception):
def __init__(self):
super().__init__("boom")
self.response = object()
self.status = 429
marker = _step_error_marker("k", Boom())
_json.dumps(marker) # raises if an attribute leaked through unserialized
assert marker["result"]["error"]["extra"]["status"] == 429
def test_an_exception_whose_str_raises_still_reports(self):
"""Every coercion of the user's exception runs inside the ``except``
reporting it, so one that raises would replace their failure with an
unrelated one and leave the step uncheckpointed."""
import json as _json
from wmill.client import _step_error_marker
class Hostile(Exception):
def __str__(self):
raise RuntimeError("cannot be rendered")
marker = _step_error_marker("k", Hostile())
_json.dumps(marker)
assert marker["result"]["error"]["name"] == "Hostile"
assert "unrepresentable" in marker["result"]["error"]["message"]
# ...including one that makes reading its own traceback raise
class HostileTraceback(Exception):
def __getattribute__(self, item):
if item == "__traceback__":
raise RuntimeError("no traceback for you")
return super().__getattribute__(item)
marker = _step_error_marker("k", HostileTraceback())
_json.dumps(marker)
assert marker["result"]["error"]["name"] == "HostileTraceback"
# ...or reading its own attributes
class HostileDict(Exception):
def __getattribute__(self, item):
if item == "__dict__":
raise RuntimeError("no attributes for you")
return super().__getattribute__(item)
marker = _step_error_marker("k", HostileDict())
_json.dumps(marker)
assert marker["result"]["error"]["name"] == "HostileDict"
# A float is serializable, so `default=` never sees NaN — it would go out
# as a bare `NaN` literal, which is not JSON and which the backend
# rejects, so the step could not be checkpointed at all.
class NotFinite(Exception):
def __init__(self):
super().__init__("nan")
self.value = float("nan")
self.limit = float("inf")
marker = _step_error_marker("k", NotFinite())
_json.dumps(marker, allow_nan=False)
assert marker["result"]["error"]["extra"] == {"value": "NaN", "limit": "Infinity"}
def test_fast_path_posts_error_and_raises_the_replay_exception(self, monkeypatch):
"""The default path: the checkpoint is POSTed and the workflow body gets
the same ``TaskError`` a replay rebuilds from the marker — raising the
original ``ValueError`` here would make ``except ValueError:`` catch on
this run and miss on the next one."""
this run and miss on the next one. ``except`` is control flow, so
anything a handler can branch on has to be identical in both rounds."""
_set_inline_fast_path_env(monkeypatch)
stub = _StubInlineClient()
class _EchoingStub(_StubInlineClient):
"""The endpoint normalizes the failure before storing it and echoes
back what it stored. The echo deliberately differs from what was
posted, so the assertions below can tell which copy was raised from."""
async def post(self, url, content=None):
await super().post(url, content=content)
stored = {**self.posted[-1]["result"], "message": "normalized by the backend"}
class _Response:
# the endpoint answers with a JSON body; a backend predating
# the echo answers without one, which is how the client tells
# "no echo" from "an echo it could not read"
headers = {"content-type": "application/json"}
def raise_for_status(self):
pass
def json(self):
return {"failure": stored}
return _Response()
stub = _EchoingStub()
posted = stub.posted
async def run():
ctx = WorkflowCtx({})
ctx._inline_http_client = stub
with pytest.raises(TaskError, match="boom") as live:
with pytest.raises(TaskError) as live:
await ctx._run_inline_step("risky", self._boom)
# ...and the replay of that very checkpoint raises the same thing.
replayed = WorkflowCtx({"completed_steps": {"risky": self.MARKER}})
with pytest.raises(TaskError, match="boom") as replay:
# The live round raised from the record the backend stored, not from
# the marker it posted: that is what keeps the two rounds identical
# even if the SDK and the backend ever build a record differently.
stored = {**posted[0]["result"], "message": "normalized by the backend"}
assert str(live.value) == "normalized by the backend"
# ...and the replay of that very record raises the same thing.
replayed = WorkflowCtx({"completed_steps": {"risky": stored}})
with pytest.raises(TaskError) as replay:
await replayed._run_inline_step("risky", self._boom)
assert type(live.value) is type(replay.value)
assert live.value.args == replay.value.args
assert live.value.result == replay.value.result == self.MARKER["result"]
assert isinstance(live.value.__cause__, ValueError)
assert live.value.result == replay.value.result == stored["result"]
assert live.value.step_key == replay.value.step_key == "risky"
# A step has no child job to name, and nothing hangs off __cause__:
# a replay has no original exception to chain, so neither round does.
assert live.value.child_job_id is replay.value.child_job_id is None
assert live.value.__cause__ is replay.value.__cause__ is None
asyncio.run(run())
assert len(posted) == 1
assert posted[0]["key"] == "risky"
assert posted[0]["result"] == self.MARKER
assert self._without_stack(posted[0]["result"]) == self.MARKER
def test_a_missing_echo_is_not_the_same_as_an_unreadable_one(self, monkeypatch):
"""A backend predating the echo answers without a JSON body and the
locally checkpointed marker stands in. A JSON body that will not parse
means the stored record exists but is unknown, so the round has to end
and let the next one read whatever the backend actually kept."""
_set_inline_fast_path_env(monkeypatch)
def _client(headers, json_impl):
class _Response:
def raise_for_status(self):
pass
_Response.headers = headers
_Response.json = json_impl
class _Client(_StubInlineClient):
async def post(self, url, content=None):
await super().post(url, content=content)
return _Response()
return _Client()
def _boom_json(self):
raise ValueError("not json")
async def run():
# no JSON body: the fast path still completes, raising the failure
ctx = WorkflowCtx({})
ctx._inline_http_client = _client({}, _boom_json)
with pytest.raises(TaskError):
await ctx._run_inline_step("risky", self._boom)
# a JSON body that will not parse: fall through to the suspend path
ctx = WorkflowCtx({})
ctx._inline_http_client = _client(
{"content-type": "application/json"}, _boom_json
)
with pytest.raises(_StepSuspend) as suspend:
await ctx._run_inline_step("risky", self._boom)
assert suspend.value.dispatch_info["key"] == "risky"
asyncio.run(run())
def test_replay_reraises_and_does_not_hang(self):
checkpoint = {
+145 -28
View File
@@ -2654,6 +2654,8 @@ def parse_sql_client_name(name: str) -> tuple[str, Optional[str]]:
import asyncio as _asyncio
import contextvars as _contextvars
import sys as _sys
import traceback as _traceback
def _assert_usable_step_key(key: str, what: str) -> None:
@@ -2687,21 +2689,62 @@ class _StepFailure(BaseException):
class TaskError(Exception):
"""Raised when a WAC task step failed.
"""Raised when a WAC ``task`` or ``step`` failed.
Attributes:
step_key: The checkpoint key of the failed step.
child_job_id: The UUID of the failed child job.
result: The error result from the child job.
child_job_id: The UUID of the failed child job, or ``None`` for a
``step()``, which runs in the workflow job and has no child job.
result: ``{"error": {"name", "message", "stack"?, "extra"?}}`` — the
same shape whether a task or a step failed. ``name`` and ``message``
are always present; ``stack`` only when the failure had a traceback,
and ``extra`` only when it carried custom fields of its own, dropped
with ``extra_omitted: True`` beside it when too large to checkpoint.
"""
def __init__(self, message: str, *, step_key: str = "", child_job_id: str = "", result=None):
def __init__(self, message: str, *, step_key: str = "", child_job_id: Optional[str] = None, result=None):
super().__init__(message)
self.step_key = step_key
self.child_job_id = child_job_id
self.result = result
def _safe_str(o) -> str:
"""``str()`` on the failing side's own object, which can raise in turn — a
detached ORM row, a proxy over a closed connection, an ``__str__`` that
itself fails. Every coercion here runs inside the ``except`` that is
reporting the user's failure, so an escape would replace their error with an
unrelated one and skip the checkpoint entirely."""
try:
return str(o)
except Exception:
return f"<unrepresentable {type(o).__name__}>"
def _step_error_stack(exc: BaseException) -> str:
"""The traceback of a failed ``step()`` body, formatted the way the python
executor formats a failed job's: frames only, and the frame that called into
the user's code dropped. Here that first frame is ``_run_inline_step``'s own
``result = fn()``, the counterpart of the generated wrapper frame the
executor strips, so a step's stack and a task's stack read alike.
Taken from ``sys.exc_info()`` the way the executor takes it, falling back to
the attribute: an exception overriding ``__getattribute__`` makes reading
``__traceback__`` raise, and this runs inside the ``except`` reporting the
user's failure, so an escape would lose both their error and the checkpoint.
"""
tb = _sys.exc_info()[2]
if tb is None:
try:
tb = exc.__traceback__
except Exception:
return ""
try:
return "".join(_traceback.format_tb(tb)[1:]).strip()
except Exception:
return ""
def _json_round_trip(value):
"""Put a value through the checkpoint's encoding without checkpointing it, so
the paths that never persist anything still hand back the shape the ones that
@@ -2711,23 +2754,69 @@ def _json_round_trip(value):
def _step_error_marker(key: str, exc: BaseException) -> dict:
"""Serialize a failed ``step()`` body into the ``__wmill_error`` marker that
task failures also use, so it can be stored in ``completed_steps``."""
task failures also use, so it can be stored in ``completed_steps``.
The marker's final shape is decided by the backend (``wac_failure_record``),
which normalizes task failures through the same function; what is built here
is the raw material plus the envelope the backend recognizes."""
error = {"name": type(exc).__name__, "message": _safe_str(exc)}
stack = _step_error_stack(exc)
if stack:
error["stack"] = stack
# Custom attributes go under ``extra``, the same key the python executor uses
# for a failed child job, so an exception carrying e.g. a ``code`` keeps it
# whether it failed as a task or as a step.
#
# Coerced through ``default=str`` the way the executor writes its own error:
# the fast-path POST serializes strictly, and the commonest failing step
# there is — ``resp.raise_for_status()``, whose ``__dict__`` holds a request
# and a response object — would otherwise fail to serialize and silently
# drop every such failure onto the slow suspend-and-replay path.
# ``getattr``'s default only swallows ``AttributeError``; an exception
# overriding ``__getattribute__`` raises whatever it likes from this read.
try:
extra = getattr(exc, "__dict__", None)
except Exception:
extra = None
if extra:
# ``default`` is where json hands back the objects it cannot represent,
# and ``str()`` on a detached ORM row or a proxy over a closed
# connection raises in turn. This runs inside the ``except`` that is
# reporting the user's failure, so an escape here would replace their
# error with an unrelated one and skip the checkpoint entirely.
# Narrow: what json raises for something it cannot represent. A broader
# catch would hide a mistake in this function as a silently missing
# field, which is how it read before.
try:
# ``parse_constant`` catches the one thing ``default`` cannot: a
# float is serializable, so ``NaN``/``Infinity`` pass through as
# bare literals that are not JSON and that the backend's extractor
# rejects — taking the whole checkpoint down with them. Kept as
# their text rather than dropped, so the attribute still says
# something.
error["extra"] = json.loads(
json.dumps(extra, default=_safe_str), parse_constant=lambda c: c
)
except (TypeError, ValueError, RecursionError):
pass
return {
"__wmill_error": True,
"message": str(exc),
"message": _safe_str(exc),
"step_key": key,
"result": {"error": str(exc), "type": type(exc).__name__},
"result": {"error": error},
}
def _step_error_from_marker(marker: dict, name: str) -> TaskError:
"""Rebuild the exception a failed step raises. Both the run that produced the
failure and every later replay go through here, so a workflow's ``except``
clauses see the same type either way."""
def _task_error_from_marker(marker: dict, fallback_message: str) -> TaskError:
"""Rebuild the exception a failed task or step raises. The run that produced
the failure and every later replay go through here: ``except`` is control
flow, ``@workflow`` re-runs its body from the top every round, so a handler
that branches on the failure it caught must be handed the same thing in
every round or it dispatches different tasks on the way back."""
return TaskError(
marker.get("message", f"Step '{name}' failed"),
marker.get("message") or fallback_message,
step_key=marker.get("step_key", ""),
child_job_id=marker.get("child_job_id", ""),
child_job_id=marker.get("child_job_id"),
result=marker.get("result"),
)
@@ -2792,12 +2881,7 @@ class WorkflowCtx:
if key in self._completed:
val = self._completed[key]
if isinstance(val, dict) and val.get("__wmill_error"):
raise TaskError(
val.get("message", f"Task '{name}' failed"),
step_key=val.get("step_key", ""),
child_job_id=val.get("child_job_id", ""),
result=val.get("result"),
)
raise _task_error_from_marker(val, f"Task '{name}' failed")
return self._resolved(val)
if self._executing_key is not None:
@@ -2904,7 +2988,7 @@ class WorkflowCtx:
if key in self._completed:
val = self._completed[key]
if isinstance(val, dict) and val.get("__wmill_error"):
raise _step_error_from_marker(val, name)
raise _task_error_from_marker(val, f"Step '{name}' failed")
return val
if self._executing_key is not None:
@@ -2916,16 +3000,25 @@ class WorkflowCtx:
t0 = _time_mod.monotonic()
# A raised step still has to reach ``completed_steps``, or a replay with
# ``_executing_key`` set finds nothing recorded and parks forever on the
# ``_asyncio.Future()`` above. ``_StepSuspend`` and ``CancelledError`` are
# ``BaseException``, so they pass through untouched.
step_error: Optional[Exception] = None
# ``_asyncio.Future()`` above. The control-flow signals (``_StepSuspend``,
# ``_StepFailure``) and ``CancelledError`` are ``BaseException``, so they
# pass through untouched.
step_failed = False
try:
result = fn()
if _asyncio.iscoroutine(result):
result = await result
except Exception as _exc:
step_error = _exc
step_failed = True
result = _step_error_marker(key, _exc)
# The failure is reported as a value from here on, so nothing else
# prints the traceback. Without this a step that fails and is never
# caught leaves a job log whose deepest frame is inside this client.
print(f"--- WAC: {key} failed ---")
print(f"{type(_exc).__name__}: {_safe_str(_exc)}")
_step_stack = result["result"]["error"].get("stack")
if _step_stack:
print(_step_stack)
duration_ms = int((_time_mod.monotonic() - t0) * 1000)
# Fast path: POST the delta to the new per-job API endpoint and return
@@ -2943,6 +3036,7 @@ class WorkflowCtx:
_token = os.environ.get("WM_TOKEN")
if _fast_path_enabled and _job_id and _workspace and _base and _token:
_fast_path_ok = False
_stored_failure = None
_replay_result = None
try:
# ``default=str`` is the encoder the worker wrapper uses on the
@@ -2978,6 +3072,20 @@ class WorkflowCtx:
content=_payload,
)
_resp.raise_for_status()
if step_failed:
# The backend normalizes the failure before storing it,
# and hands back what it stored. Raising from that, not
# from the marker posted above, is what makes this round
# and every replay read the same record even if the two
# sides ever disagree about how to build one.
#
# A backend predating the echo answers without a JSON
# body, and the round-tripped marker below stands in. A
# JSON body that will not parse is different: the record
# may already be committed and its content is unknown,
# so let it raise and take the suspend path instead.
if "json" in _resp.headers.get("content-type", ""):
_stored_failure = (_resp.json() or {}).get("failure")
_fast_path_ok = True
except Exception as _e:
logger.info(
@@ -2987,12 +3095,21 @@ class WorkflowCtx:
)
# fall through to the legacy suspend path
if _fast_path_ok:
# Raise what a replay would rebuild from the marker, never the
# Raise what a replay would rebuild from the record, never the
# original: a replay cannot reconstruct the original type, so
# raising it here would make ``except ValueError:`` catch on this
# run and miss on the next. ``__cause__`` is for tracebacks only.
if step_error is not None:
raise _step_error_from_marker(result, name) from step_error
# run and miss on the next. Nothing is chained onto
# ``__cause__`` for the same reason — the traceback a replay can
# still show is in ``result["error"]["stack"]``. ``_stored_failure``
# is None against a backend that predates the echoed record,
# which is what ``_replay_result`` below stands in for.
if step_failed:
# ``_replay_result``, not ``result``: the fallback has to be
# what the checkpoint holds, so the round that ran the body
# reads what every replay of it will.
raise _task_error_from_marker(
_stored_failure or _replay_result, f"Step '{name}' failed"
)
# Return the round trip of what was checkpointed, never the
# in-memory value: handing back the live object would let the
# round that ran the body branch on a type — tuple, datetime —
+11 -4
View File
@@ -1073,6 +1073,8 @@ Let task errors fail the workflow unless the user asks for recovery logic.
Python: \`except Exception\` is safe around WAC calls because internal suspension inherits from \`BaseException\`. Avoid bare \`except:\` in workflow code. If the user asks for recovery logic around failed child work, catch \`TaskError\` from \`wmill\` for task failures.
TypeScript: avoid broad \`try/catch\` around WAC SDK calls. The SDK uses an internal suspension error during initial dispatch; catching it can break workflow suspension. If a broad catch is unavoidable, rethrow internal suspension errors before handling business errors.
A caught failure reads the same whether it came from a task or from a \`step()\`, and the same in the round that ran the failing body as in every round replaying it. It carries \`step_key\`, \`child_job_id\` (absent for a \`step()\`, which runs in the workflow job and has no child job), a \`message\` that is the failure's own message, and \`result\` = \`{"error": {"name", "message", "stack"?, "extra"?}}\`. \`name\`, \`message\` and \`stack\` are the fields that read the same whichever side failed; \`name\` and \`message\` are always there, \`stack\` only when the failure had a traceback to give. \`extra\` carries the failure's own custom fields (an exception's attributes, an error's properties) and is best-effort: it is absent when there were none, and a task can report entries a step does not, so read it defensively and don't branch on its absence. \`extra\` is dropped when it is too large to keep in the checkpoint, and \`extra_omitted: true\` says so — absent \`extra\` with no \`extra_omitted\` means the failure simply had no custom fields. Branch on those, not on the original exception type: the workflow body re-runs from the top every round and a replay rebuilds the failure from the checkpoint, so nothing outside that record survives. Python raises \`TaskError\`; TypeScript throws an \`Error\` named \`TaskError\` carrying the same fields. Nothing is chained onto \`__cause__\` / \`cause\` — the traceback is in \`result.error.stack\`, and is also printed to the job log when the step fails.
`;
export const FLOW_CHAT_SPECIAL_MODULES = `## Special Modules
@@ -2622,14 +2624,19 @@ export const WAC_SDK_PYTHON = `## Python Workflow-as-Code API (wmill)
Import: \`from wmill import workflow, task, task_script, task_flow, step, sleep, wait_for_approval, get_approval_urls, get_resume_urls, parallel, TaskError\`
\`\`\`python
# Raised when a WAC task step failed.
# Raised when a WAC \`\`task\`\` or \`\`step\`\` failed.
#
# Attributes:
# step_key: The checkpoint key of the failed step.
# child_job_id: The UUID of the failed child job.
# result: The error result from the child job.
# child_job_id: The UUID of the failed child job, or \`\`None\`\` for a
# \`\`step()\`\`, which runs in the workflow job and has no child job.
# result: \`\`{"error": {"name", "message", "stack"?, "extra"?}}\`\` — the
# same shape whether a task or a step failed. \`\`name\`\` and \`\`message\`\`
# are always present; \`\`stack\`\` only when the failure had a traceback,
# and \`\`extra\`\` only when it carried custom fields of its own, dropped
# with \`\`extra_omitted: True\`\` beside it when too large to checkpoint.
class TaskError(Exception):
def __init__(self, message: str, *, step_key: str = '', child_job_id: str = '', result = None)
def __init__(self, message: str, *, step_key: str = '', child_job_id: Optional[str] = None, result = None)
# Get URLs needed for resuming a flow after suspension.
#
@@ -3,14 +3,19 @@
Import: `from wmill import workflow, task, task_script, task_flow, step, sleep, wait_for_approval, get_approval_urls, get_resume_urls, parallel, TaskError`
```python
# Raised when a WAC task step failed.
# Raised when a WAC ``task`` or ``step`` failed.
#
# Attributes:
# step_key: The checkpoint key of the failed step.
# child_job_id: The UUID of the failed child job.
# result: The error result from the child job.
# child_job_id: The UUID of the failed child job, or ``None`` for a
# ``step()``, which runs in the workflow job and has no child job.
# result: ``{"error": {"name", "message", "stack"?, "extra"?}}`` — the
# same shape whether a task or a step failed. ``name`` and ``message``
# are always present; ``stack`` only when the failure had a traceback,
# and ``extra`` only when it carried custom fields of its own, dropped
# with ``extra_omitted: True`` beside it when too large to checkpoint.
class TaskError(Exception):
def __init__(self, message: str, *, step_key: str = '', child_job_id: str = '', result = None)
def __init__(self, message: str, *, step_key: str = '', child_job_id: Optional[str] = None, result = None)
# Get URLs needed for resuming a flow after suspension.
#
@@ -238,6 +238,8 @@ Python: `except Exception` is safe around WAC calls because internal suspension
TypeScript: avoid broad `try/catch` around WAC SDK calls. The SDK uses an internal suspension error during initial dispatch; catching it can break workflow suspension. If a broad catch is unavoidable, rethrow internal suspension errors before handling business errors.
A caught failure reads the same whether it came from a task or from a `step()`, and the same in the round that ran the failing body as in every round replaying it. It carries `step_key`, `child_job_id` (absent for a `step()`, which runs in the workflow job and has no child job), a `message` that is the failure's own message, and `result` = `{"error": {"name", "message", "stack"?, "extra"?}}`. `name`, `message` and `stack` are the fields that read the same whichever side failed; `name` and `message` are always there, `stack` only when the failure had a traceback to give. `extra` carries the failure's own custom fields (an exception's attributes, an error's properties) and is best-effort: it is absent when there were none, and a task can report entries a step does not, so read it defensively and don't branch on its absence. `extra` is dropped when it is too large to keep in the checkpoint, and `extra_omitted: true` says so — absent `extra` with no `extra_omitted` means the failure simply had no custom fields. Branch on those, not on the original exception type: the workflow body re-runs from the top every round and a replay rebuilds the failure from the checkpoint, so nothing outside that record survives. Python raises `TaskError`; TypeScript throws an `Error` named `TaskError` carrying the same fields. Nothing is chained onto `__cause__` / `cause` — the traceback is in `result.error.stack`, and is also printed to the job log when the step fails.
## TypeScript Workflow-as-Code API (windmill-client)
@@ -376,14 +378,19 @@ export async function parallel<T, R>(items: T[], fn: (item: T) => PromiseLike<R>
Import: `from wmill import workflow, task, task_script, task_flow, step, sleep, wait_for_approval, get_approval_urls, get_resume_urls, parallel, TaskError`
```python
# Raised when a WAC task step failed.
# Raised when a WAC ``task`` or ``step`` failed.
#
# Attributes:
# step_key: The checkpoint key of the failed step.
# child_job_id: The UUID of the failed child job.
# result: The error result from the child job.
# child_job_id: The UUID of the failed child job, or ``None`` for a
# ``step()``, which runs in the workflow job and has no child job.
# result: ``{"error": {"name", "message", "stack"?, "extra"?}}`` — the
# same shape whether a task or a step failed. ``name`` and ``message``
# are always present; ``stack`` only when the failure had a traceback,
# and ``extra`` only when it carried custom fields of its own, dropped
# with ``extra_omitted: True`` beside it when too large to checkpoint.
class TaskError(Exception):
def __init__(self, message: str, *, step_key: str = '', child_job_id: str = '', result = None)
def __init__(self, message: str, *, step_key: str = '', child_job_id: Optional[str] = None, result = None)
# Get URLs needed for resuming a flow after suspension.
#
+2
View File
@@ -185,3 +185,5 @@ Let task errors fail the workflow unless the user asks for recovery logic.
Python: `except Exception` is safe around WAC calls because internal suspension inherits from `BaseException`. Avoid bare `except:` in workflow code. If the user asks for recovery logic around failed child work, catch `TaskError` from `wmill` for task failures.
TypeScript: avoid broad `try/catch` around WAC SDK calls. The SDK uses an internal suspension error during initial dispatch; catching it can break workflow suspension. If a broad catch is unavoidable, rethrow internal suspension errors before handling business errors.
A caught failure reads the same whether it came from a task or from a `step()`, and the same in the round that ran the failing body as in every round replaying it. It carries `step_key`, `child_job_id` (absent for a `step()`, which runs in the workflow job and has no child job), a `message` that is the failure's own message, and `result` = `{"error": {"name", "message", "stack"?, "extra"?}}`. `name`, `message` and `stack` are the fields that read the same whichever side failed; `name` and `message` are always there, `stack` only when the failure had a traceback to give. `extra` carries the failure's own custom fields (an exception's attributes, an error's properties) and is best-effort: it is absent when there were none, and a task can report entries a step does not, so read it defensively and don't branch on its absence. `extra` is dropped when it is too large to keep in the checkpoint, and `extra_omitted: true` says so — absent `extra` with no `extra_omitted` means the failure simply had no custom fields. Branch on those, not on the original exception type: the workflow body re-runs from the top every round and a replay rebuilds the failure from the checkpoint, so nothing outside that record survives. Python raises `TaskError`; TypeScript throws an `Error` named `TaskError` carrying the same fields. Nothing is chained onto `__cause__` / `cause` — the traceback is in `result.error.stack`, and is also printed to the job log when the step fails.
+6
View File
@@ -107,6 +107,12 @@ def extract_ts_functions(content: str) -> list[dict]:
if not return_type:
return_type = 'Promise<void>' if is_async else 'void'
# `@internal` marks an export that exists for another module or for a
# test to reach, not for a user to call. The SDK reference these prompts
# become is a user-facing API list, so it must not advertise them.
if jsdoc_raw and '@internal' in jsdoc_raw:
continue
docstring = clean_jsdoc(jsdoc_raw) if jsdoc_raw else ''
seen_names.add(name)
functions.append({
+1
View File
@@ -10,6 +10,7 @@ npx --yes @hey-api/openapi-ts@0.43.0 --input "${script_dirpath}/../backend/wind
sed -i 's/get \[Symbol\.toStringTag\]() {/get \[Symbol\.toStringTag\]() : string {/g' "${script_dirpath}/src/core/CancelablePromise.ts"
cp "${script_dirpath}/client.ts" "${script_dirpath}/src/"
cp "${script_dirpath}/wacError.ts" "${script_dirpath}/src/"
cp "${script_dirpath}/s3Types.ts" "${script_dirpath}/src/"
cp "${script_dirpath}/sqlUtils.ts" "${script_dirpath}/src/"
echo "" >> "${script_dirpath}/src/index.ts"
+1
View File
@@ -35,6 +35,7 @@ fi
cp "${script_dirpath}/client.ts" "${script_dirpath}/src/"
cp "${script_dirpath}/wacError.ts" "${script_dirpath}/src/"
cp "${script_dirpath}/s3Types.ts" "${script_dirpath}/src/"
cp "${script_dirpath}/sqlUtils.ts" "${script_dirpath}/src/"
echo "" >> "${script_dirpath}/src/index.ts"
+42 -40
View File
@@ -12,6 +12,7 @@ import {
KafkaTriggerService,
} from "./services.gen";
import { OpenAPI } from "./core/OpenAPI";
import { isSuspendSignal, stepErrorMarker, taskErrorFromMarker } from "./wacError";
// import type { DenoS3LightClientSettings } from "./index";
import {
DenoS3LightClientSettings,
@@ -1520,32 +1521,6 @@ export class StepSuspend extends Error {
}
}
/** Serialize a failed `step()` body into the `__wmill_error` marker that task
* failures also use, so it can be stored in `completed_steps`. */
function stepErrorMarker(key: string, e: unknown): Record<string, any> {
const message = e instanceof Error ? e.message : String(e);
// Constructor name, not `e.name`: a `class MyError extends Error {}` that
// never assigns `this.name` reports "Error", which would make the same
// failure read as `MyError` in the python client and `Error` here.
const type = e instanceof Error ? (e.constructor?.name ?? e.name) : typeof e;
return { __wmill_error: true, message, step_key: key, result: { error: message, type } };
}
/** Rebuild the error a failed step throws. Both the run that produced the
* failure and every later replay go through here, so a workflow's catch block
* sees the same shape either way. */
function stepErrorFromMarker(marker: any, name: string): Error {
const err = new Error(marker?.message || `Step '${name}' failed`);
// Matches the python client, which raises TaskError here; the failed body's
// own type stays in `result.type`. Keeps a failed job's serialized error
// identical across the two languages.
err.name = "TaskError";
(err as any).result = marker?.result;
(err as any).step_key = marker?.step_key;
(err as any).child_job_id = marker?.child_job_id;
return err;
}
/** Values `JSON.stringify` cannot represent: it omits the property holding one.
* The type-level half of `checkpointableResult`, which nulls them at the top
* level, where there is no key to omit. */
@@ -1763,11 +1738,7 @@ export class WorkflowCtx {
if (key in this.completed) {
const value = this.completed[key];
if (value && typeof value === "object" && (value as any).__wmill_error) {
const err = new Error((value as any).message || `Task '${name}' failed`);
err.name = "TaskError";
(err as any).result = (value as any).result;
(err as any).step_key = (value as any).step_key;
(err as any).child_job_id = (value as any).child_job_id;
const err = taskErrorFromMarker(value, `Task '${name}' failed`);
return { then: (_resolve: any, reject?: any) => { if (reject) reject(err); else throw err; } } as PromiseLike<any>;
}
return { then: (resolve: any) => resolve(value) };
@@ -1894,7 +1865,7 @@ export class WorkflowCtx {
if (key in this.completed) {
const value = this.completed[key];
if (value && typeof value === "object" && (value as any).__wmill_error) {
throw stepErrorFromMarker(value, name);
throw taskErrorFromMarker(value, `Step '${name}' failed`);
}
return value as T;
}
@@ -1912,15 +1883,24 @@ export class WorkflowCtx {
// never-resolving promise above. A nested StepSuspend is control flow,
// not a step failure.
let result: T;
let stepError: unknown;
let errored = false;
try {
result = await fn();
} catch (e) {
if ((e as any)?.name === "StepSuspend" || e instanceof StepSuspend) throw e;
if (isSuspendSignal(e, StepSuspend)) throw e;
errored = true;
stepError = e;
result = stepErrorMarker(key, e) as any;
// The failure is reported as a value from here on, so nothing else prints
// the stack. Without this a step that throws and is never caught leaves a
// job log whose deepest frame is inside this client.
console.log(`--- WAC: ${key} failed ---`);
// Only from the marker, which already coerced the thrown value once and
// survived it. Reaching for `e` again here would re-run the coercion that
// `stepErrorMarker` guards, and throw out of this catch before the failure
// is ever checkpointed.
console.log(
(result as any)?.result?.error?.stack ?? (result as any)?.message ?? "step failed",
);
}
const durationMs = Date.now() - t0;
@@ -1988,6 +1968,22 @@ export class WorkflowCtx {
if (!resp.ok) {
throw new Error(`inline_checkpoint API ${resp.status}`);
}
if (!errored) return undefined;
// The backend normalizes the failure before storing it, and hands
// back what it stored. Throwing from that, not from the marker posted
// above, is what makes this round and every replay read the same
// record even if the two sides ever disagree about how to build one.
//
// A backend predating the echo answers without a JSON body; the
// caller then falls back to the round trip of what it posted. A JSON
// body we cannot read is different: the normalized record may already
// be committed and we do not know what it says, so let this reject
// and take the suspend path, where the next round reads whatever the
// backend actually stored.
if (!(resp.headers.get("content-type") ?? "").includes("json")) {
return undefined;
}
return (await resp.json())?.failure;
} finally {
clearTimeout(t);
}
@@ -1995,8 +1991,9 @@ export class WorkflowCtx {
// Swallow chain errors so a past failure does not poison future awaits.
this._inlineChain = chainTail.catch(() => {});
let fastPathOk = false;
let storedFailure: any;
try {
await chainTail;
storedFailure = await chainTail;
fastPathOk = true;
} catch (e) {
console.log(
@@ -2005,12 +2002,17 @@ export class WorkflowCtx {
// fall through to the legacy suspend path below
}
if (fastPathOk) {
// Throw what a replay would rebuild from the marker, never the
// Throw what a replay would rebuild from the record, never the
// original: a replay cannot reconstruct the original type, so throwing
// it here would match `e instanceof TypeError` on this run and miss on
// the next. `cause` is for logging only — absent on replay.
// the next. Nothing is attached to `cause` for the same reason — the
// stack a replay can still show is in `result.error.stack`.
if (errored) {
throw Object.assign(stepErrorFromMarker(result, name), { cause: stepError });
// `checkpointed`, not `result`: against a backend with no echo the
// fallback still has to be what the checkpoint holds, or an `extra`
// carrying a Date reads as a Date now and as its ISO string on every
// replay — the divergence the success path below already avoids.
throw taskErrorFromMarker(storedFailure ?? checkpointed, `Step '${name}' failed`);
}
// Return the round trip of what was checkpointed, never the in-memory
// value: handing back the live object would let the round that ran the
@@ -2155,7 +2157,7 @@ export function task<T extends (...args: any[]) => Promise<any>>(
try {
result = await fn(...args);
} catch (e) {
if ((e as any)?.name === "StepSuspend" || e instanceof StepSuspend) throw e;
if (isSuspendSignal(e, StepSuspend)) throw e;
ctx._raiseStepFailure(e);
}
ctx._raiseSuspend({ mode: "step_complete", steps: [], result: checkpointableResult(result) });
+1 -1
View File
@@ -3,7 +3,7 @@
"version": "1.773.0",
"exports": "./src/index.ts",
"publish": {
"exclude": ["!src", "./s3Types.ts", "./sqlUtils.ts", "./client.ts"]
"exclude": ["!src", "./s3Types.ts", "./sqlUtils.ts", "./client.ts", "./wacError.ts"]
},
"license": "Apache-2.0"
}
+200 -28
View File
@@ -5,6 +5,11 @@
*/
import { expect, test, describe } from "bun:test";
// The two functions that decide what a caught failure looks like come from the
// shipped module, not from the mirror below: they are what drifted from the
// backend's shape before, so a copy of them here would guard nothing.
import { isSuspendSignal, stepErrorMarker, taskErrorFromMarker } from "../wacError";
// --- Inline SDK (mirrors client.ts implementation) ---
class StepSuspend extends Error {
@@ -79,11 +84,7 @@ class WorkflowCtx {
if (key in this.completed) {
const value = this.completed[key];
if (value && typeof value === "object" && (value as any).__wmill_error) {
const err = new Error((value as any).message || `Task '${name}' failed`);
err.name = "TaskError";
(err as any).result = (value as any).result;
(err as any).step_key = (value as any).step_key;
(err as any).child_job_id = (value as any).child_job_id;
const err = taskErrorFromMarker(value, `Task '${name}' failed`);
return { then: (_resolve: any, reject?: any) => { if (reject) reject(err); else throw err; } };
}
return { then: (resolve: any) => resolve(value) };
@@ -157,10 +158,7 @@ class WorkflowCtx {
if (key in this.completed) {
const value = this.completed[key];
if (value && typeof value === "object" && (value as any).__wmill_error) {
const err = new Error((value as any).message || `Step '${name}' failed`);
err.name = "TaskError";
(err as any).result = (value as any).result;
throw err;
throw taskErrorFromMarker(value, `Step '${name}' failed`);
}
return value as T;
}
@@ -170,22 +168,11 @@ class WorkflowCtx {
}
let result: any;
let errored = false;
try {
result = await fn();
} catch (e: any) {
if (e?.name === "StepSuspend" || e instanceof StepSuspend) throw e;
errored = true;
const message = e instanceof Error ? e.message : String(e);
result = {
__wmill_error: true,
message,
step_key: key,
result: {
error: message,
type: e instanceof Error ? (e.constructor?.name ?? e.name) : typeof e,
},
};
if (isSuspendSignal(e, StepSuspend)) throw e;
result = stepErrorMarker(key, e);
}
this._raiseSuspend({
mode: "inline_checkpoint",
@@ -248,7 +235,7 @@ function task<T extends (...args: any[]) => Promise<any>>(
try {
result = await fn(...args);
} catch (e: any) {
if (e?.name === "StepSuspend" || e instanceof StepSuspend) throw e;
if (isSuspendSignal(e, StepSuspend)) throw e;
ctx._raiseStepFailure(e);
}
ctx._raiseSuspend({
@@ -1415,11 +1402,18 @@ describe("error propagation via __wmill_error marker", () => {
// _executingKey set, reaches the unrecorded key, and parks on the
// never-resolving promise forever.
describe("throwing inline step is checkpointed", () => {
// A failed child job reports `{ error: { name, message, stack } }`, and a
// failed step() has to be indistinguishable from it. The stack is a
// JS stack string, asserted separately.
const marker = {
__wmill_error: true,
message: "boom",
step_key: "step_0",
result: { error: "boom", type: "TypeError" },
result: { error: { name: "TypeError", message: "boom" } },
};
const withoutStack = (m: any) => {
const { stack, ...error } = m.result.error;
return { ...m, result: { ...m.result, error } };
};
// The workflow body catches — the shape a failing step is written for, and
@@ -1450,7 +1444,8 @@ describe("throwing inline step is checkpointed", () => {
expect(caught).toBeInstanceOf(StepSuspend);
expect(caught.dispatchInfo.mode).toBe("inline_checkpoint");
expect(caught.dispatchInfo.key).toBe("step_0");
expect(caught.dispatchInfo.result).toEqual(marker);
expect(withoutStack(caught.dispatchInfo.result)).toEqual(marker);
expect(caught.dispatchInfo.result.result.error.stack).toContain("TypeError: boom");
});
test("a swallowed suspend still reaches the runner", async () => {
@@ -1459,7 +1454,7 @@ describe("throwing inline step is checkpointed", () => {
const result = await runWorkflow(catchingWf(), {}, [5]);
expect(result.type).toBe("inline_checkpoint");
expect(result.key).toBe("step_0");
expect(result.result).toEqual(marker);
expect(withoutStack(result.result)).toEqual(marker);
});
test("a swallowed suspend from a succeeding step still reaches the runner", async () => {
@@ -1476,6 +1471,176 @@ describe("throwing inline step is checkpointed", () => {
expect(result.result).toBe(42);
});
// The backend records `name: e.name` for a failed child job
// (bun_executor.rs). A step reporting the constructor name instead would make
// the same failure read differently depending on whether it ran as a task or
// as a step, in the one field handlers are told to branch on.
test("error.name is e.name, as a failed child job reports it", async () => {
class MyError extends Error {}
const unnamed = stepErrorMarker("k", new MyError("boom"));
expect(unnamed.result.error.name).toBe("Error");
class NamedError extends Error {
name = "NamedError";
}
const named = stepErrorMarker("k", new NamedError("boom"));
expect(named.result.error.name).toBe("NamedError");
const domish = Object.assign(new Error("aborted"), { name: "AbortError" });
expect(stepErrorMarker("k", domish).result.error.name).toBe("AbortError");
});
// A failed child job reports custom properties under `error.extra`
// (bun_executor.rs). A step dropping them would make the same error carry
// less information depending on how it was run.
test("custom error properties survive under error.extra, as a task's do", async () => {
const e = Object.assign(new Error("429"), { code: 429, retryAfter: 5 });
const marker = stepErrorMarker("k", e);
expect(marker.result.error.extra).toEqual({ code: 429, retryAfter: 5 });
// the named fields the executors report separately are not duplicated
expect(marker.result.error.extra.message).toBeUndefined();
expect(marker.result.error.extra.stack).toBeUndefined();
expect(stepErrorMarker("k", new Error("plain")).result.error.extra).toBeUndefined();
});
// The marker is stringified while the workflow is still running (checkpoint
// POST, then wrapper output). A property that can't survive that would end
// the job instead of reaching the user's catch.
test("a property that cannot be serialized is dropped, not propagated", () => {
const circular: any = { name: "req" };
circular.self = circular;
const withCircular = Object.assign(new Error("boom"), { code: 429, request: circular });
const marker = stepErrorMarker("k", withCircular);
expect(() => JSON.stringify(marker)).not.toThrow();
expect(marker.result.error.extra).toEqual({ code: 429 });
const withThrowingAccessor = new Error("boom");
Object.defineProperty(withThrowingAccessor, "boobytrap", {
enumerable: true,
get() {
throw new Error("read me and die");
},
});
expect(() => stepErrorMarker("k", withThrowingAccessor)).not.toThrow();
});
// A task executor reads `name`/`message`/`stack` off whatever was thrown, not
// off an Error instance, so a step must too or a handler can tell the two
// apart in the fields the contract tells it to branch on.
test("a non-Error throw records what a task would record", () => {
const thrown = stepErrorMarker("k", { name: "Thrown", message: "boom", code: 429 });
expect(thrown.result.error.name).toBe("Thrown");
expect(thrown.result.error.message).toBe("boom");
expect(thrown.result.error.extra).toEqual({ code: 429 });
// A string carries none of the three, so the record is left for the backend
// to fill — the same fallback a task throwing a string produces. Its
// character indices are not custom fields.
const str = stepErrorMarker("k", "boom");
expect(str.result.error).toEqual({});
// `String()` on a value with no `toString` to reach throws in turn, and
// this runs inside the catch reporting the user's failure.
expect(() => stepErrorMarker("k", Object.create(null))).not.toThrow();
});
// Reporting a failure must not be able to fail: every read of the thrown
// value happens inside the catch that is reporting it, so an escape replaces
// the user's error with an unrelated one and leaves the step uncheckpointed.
test("a hostile thrown value cannot make failure reporting throw", () => {
const hostile = new Proxy(
{},
{
get() {
throw new Error("get trap");
},
ownKeys() {
throw new Error("ownKeys trap");
},
},
);
expect(() => stepErrorMarker("k", hostile)).not.toThrow();
expect(() => JSON.stringify(stepErrorMarker("k", hostile))).not.toThrow();
const throwingToString = { toString() { throw new Error("no"); } };
expect(() => stepErrorMarker("k", throwingToString)).not.toThrow();
});
// Probing the original value only proves it serialized once. The marker is
// serialized again to reach the checkpoint, and by then the failure has
// nowhere left to go, so what survived the probe is what gets kept.
test("a property that serializes only once cannot break the checkpoint", () => {
let calls = 0;
const onceOnly = {
toJSON() {
if (calls++ > 0) throw new Error("second time");
return { ok: true };
},
};
const marker = stepErrorMarker("k", Object.assign(new Error("boom"), { payload: onceOnly }));
expect(marker.result.error.extra.payload).toEqual({ ok: true });
// re-serialized on the way to the checkpoint, and again by the wrapper
expect(() => JSON.stringify(marker)).not.toThrow();
expect(() => JSON.stringify(marker)).not.toThrow();
});
// The caller reads `.name` off the thrown value to spot a suspend, before it
// ever reaches the hardened marker. A hostile value escaping there leaves the
// step uncheckpointed and a later replay parks on it forever.
test("a hostile throw still reaches the checkpoint through _runInlineStep", async () => {
const hostile = new Proxy(
{},
{
get() {
throw new Error("get trap");
},
ownKeys() {
throw new Error("ownKeys trap");
},
},
);
const ctx = new WorkflowCtx({});
let caught: any;
try {
await ctx._runInlineStep("risky", () => {
throw hostile;
});
} catch (e) {
caught = e;
}
// the suspend carrying the checkpoint, not the hostile value itself
expect(caught).toBeInstanceOf(StepSuspend);
expect(caught.dispatchInfo.key).toBe("step_0");
expect(caught.dispatchInfo.result.__wmill_error).toBe(true);
});
// `instanceof` consults a proxy's `getPrototypeOf` trap, so the suspend check
// itself can throw — before anything is checkpointed.
test("suspend detection survives a value that refuses to be inspected", () => {
const hostilePrototype = new Proxy(new StepSuspend({ mode: "sequential" }), {
getPrototypeOf() {
throw new Error("getPrototypeOf trap");
},
});
// the name is still readable, so it is still recognised as the signal
expect(isSuspendSignal(hostilePrototype, StepSuspend)).toBe(true);
const opaque = new Proxy(
{},
{
getPrototypeOf() {
throw new Error("getPrototypeOf trap");
},
get() {
throw new Error("get trap");
},
},
);
expect(() => isSuspendSignal(opaque, StepSuspend)).not.toThrow();
expect(isSuspendSignal(opaque, StepSuspend)).toBe(false);
});
test("a replayed step failure is named TaskError, like the python client", async () => {
const ctx = new WorkflowCtx({ completed_steps: { step_0: marker } });
let caught: any;
@@ -1485,8 +1650,15 @@ describe("throwing inline step is checkpointed", () => {
caught = e;
}
expect(`${caught.name}: ${caught.message}`).toBe("TaskError: boom");
// the failing body's own type stays addressable here
expect(caught.result).toEqual({ error: "boom", type: "TypeError" });
// the failing body's own type stays addressable here, in the shape a
// failed task hands over too
expect(caught.result).toEqual({ error: { name: "TypeError", message: "boom" } });
expect(caught.step_key).toBe("step_0");
// a step runs in the workflow job, so there is no child job to name
expect(caught.child_job_id).toBeUndefined();
// nothing is chained onto `cause`: a replay has no original error to
// chain, so neither round does
expect(caught.cause).toBeUndefined();
});
test("a child job cannot swallow its own completion signal", async () => {
+159
View File
@@ -0,0 +1,159 @@
// Workflow-as-Code failure record, client side.
//
// Deliberately dependency-free so the tests can import it directly: the rest of
// client.ts pulls in the generated API modules, which is why the workflow test
// suite re-implements WorkflowCtx inline. What a caught failure looks like, and
// what is a failure at all rather than the SDK's own control flow, is decided
// here — so it is decided against the shipped code rather than against a copy.
/** Error properties the executors already report as named fields, so they must
* not be repeated inside `extra`. Kept identical to the skip-list in the
* generated bun/deno job wrappers. */
const SERIALIZED_ERROR_FIELDS = [
"line",
"name",
"stack",
"column",
"message",
"sourceURL",
"originalLine",
"originalColumn",
];
/** Read a property off a value that may fight back: a proxy `get` trap or a
* throwing accessor turns an ordinary field read into an exception. Callers
* are on the failure-reporting path, where nothing has been checkpointed yet. */
function safeRead(o: any, k: string): unknown {
try {
return o?.[k];
} catch {
return undefined;
}
}
/** @internal
* Whether a caught value is the SDK's own suspend signal, which must be
* rethrown rather than reported as a step failure.
*
* Both halves of the check can throw on a hostile value: `instanceof` consults
* a proxy's `getPrototypeOf` trap, and reading `.name` its `get` trap. This
* runs before anything has been checkpointed, so neither may escape — a value
* that fights back is simply not a suspend. */
export function isSuspendSignal(e: unknown, suspendCtor: Function): boolean {
try {
if (e instanceof (suspendCtor as any)) return true;
} catch {
// a proxy refusing to be inspected is not the SDK's own signal
}
return safeRead(e, "name") === "StepSuspend";
}
/** @internal
* Serialize a failed `step()` body into the `__wmill_error` marker that task
* failures also use, so it can be stored in `completed_steps`.
*
* The record's final shape is decided by the backend (`wac_failure_record`),
* which normalizes task failures through the same function; what is built here
* is the raw material plus the envelope the backend recognizes. */
export function stepErrorMarker(key: string, e: unknown): Record<string, any> {
const thrown = e as any;
// The three named fields are read off whatever was thrown, exactly as the
// bun/deno wrappers read them for a failed child job — `e.name`, `e.message`,
// `e.stack`, whether or not it is an `Error`. Anything else here lets a
// handler tell a task from a step in the fields it is told to branch on: a
// thrown `{ name, message, code }` keeps all three as a task, and a
// `class MyError extends Error {}` that never assigns `this.name` reports
// `Error` as a task, so both must here too. What is absent stays absent and
// the backend fills it, which is what a task with no usable fields gets.
//
// `throw "boom"` therefore reports the backend's fallback message rather than
// "boom", because a task throwing a string is equally lossy — its `e.message`
// is undefined too. Recovering the text here, from the marker's top-level
// `message`, would split `message` between a task and a step, which is the
// thing this record exists to prevent. The executors are where a primitive
// throw could keep its text for both.
const rawName = safeRead(thrown, "name");
const rawMessage = safeRead(thrown, "message");
const rawStack = safeRead(thrown, "stack");
const name = typeof rawName === "string" ? rawName : undefined;
const message = typeof rawMessage === "string" ? rawMessage : undefined;
const stack = typeof rawStack === "string" ? rawStack : undefined;
const error: Record<string, any> = {};
if (name) error.name = name;
if (message !== undefined) error.message = message;
if (stack) error.stack = stack;
// Custom properties go under `extra`, the same key and the same skip-list the
// bun/deno executors use for a failed child job, so an error carrying e.g. a
// `code` keeps it whether it failed as a task or as a step.
//
// Unlike the executors, which serialize once as the job dies, this record is
// stringified while the workflow is still running — once for the checkpoint
// POST and again for the wrapper's output. A property that cannot survive
// that would take the whole workflow down instead of reaching the `catch` the
// user wrote, so each one has to prove it round-trips before it is kept.
// `AxiosError.request` is the everyday example: it is circular via
// `socket._httpMessage`. Reading the property can throw too, since
// `getOwnPropertyNames` returns accessors and this invokes them.
// Objects only: on a primitive `Object.getOwnPropertyNames` returns its
// character indices, and `extra: {0: "b", 1: "o", …}` is noise the checkpoint
// would then have to carry.
if (thrown !== null && typeof thrown === "object") {
const extra: Record<string, any> = {};
// `ownKeys` is a proxy trap too, so even listing the properties can throw.
let keys: string[] = [];
try {
keys = Object.getOwnPropertyNames(thrown);
} catch {
keys = [];
}
for (const k of keys) {
if (SERIALIZED_ERROR_FIELDS.includes(k)) continue;
try {
// The snapshot, not the original: probing the original only proves it
// serialized *once*. A `toJSON` that succeeds and then throws — or one
// whose output depends on state that moves — would pass here and break
// the checkpoint encoding afterwards, when the failure has nowhere left
// to go. Keeping what the probe produced makes the check binding.
// `undefined` means the key would be omitted anyway (a function, a
// symbol, an explicit undefined), and `JSON.parse` rejects it for us.
extra[k] = JSON.parse(JSON.stringify(thrown[k]));
} catch {
// unreadable or unserializable: the failure itself still gets reported
}
}
if (Object.keys(extra).length > 0) error.extra = extra;
}
// `String(e)` throws in turn on a value with no `toString` to reach —
// `Object.create(null)`, a proxy that rejects the coercion — and this runs
// inside the catch that is reporting the user's failure, so an escape here
// replaces their error with an unrelated one and skips the checkpoint.
let fallback: string;
try {
fallback = String(e);
} catch {
fallback = `unrepresentable ${typeof e} thrown`;
}
return {
__wmill_error: true,
message: message ?? fallback,
step_key: key,
result: { error },
};
}
/** @internal
* Rebuild the error a failed task or step throws. The run that produced the
* failure and every later replay go through here: a catch is control flow,
* `workflow()` re-runs its body from the top every round, so a handler that
* branches on the failure it caught must be handed the same thing in every
* round or it dispatches different tasks on the way back. */
export function taskErrorFromMarker(marker: any, fallbackMessage: string): Error {
const err = new Error(marker?.message || fallbackMessage);
// Matches the python client, which raises TaskError here. Keeps a failed
// job's serialized error identical across the two languages.
err.name = "TaskError";
(err as any).result = marker?.result;
(err as any).step_key = marker?.step_key;
(err as any).child_job_id = marker?.child_job_id;
return err;
}