fix: dispatch workflow-as-code tasks from a deployed flow's inline step (#11146)

* fix: dispatch workflow-as-code tasks from a deployed flow's inline step

* fix: give a workflow-as-code task its own result-cache key

* fix: key a cached workflow-as-code task on its name and arguments

* fix: hash a cached workflow-as-code task's arguments like any job's

* chore: regenerate system prompts for the task cache_ttl docs

* fix: key a cached workflow-as-code task on its step key, not its name

* fix: key a cached workflow-as-code task on a fingerprint of its code

* fix: keep the task() doc attached to task()

* fix: key a cached inline task on its step key and the workflow input

* docs: cache_ttl has no effect on a taskFlow target
This commit is contained in:
Ruben Fiszel
2026-09-16 09:58:29 +02:00
committed by GitHub
parent a48ae656ae
commit e8078f2a96
16 changed files with 341 additions and 20 deletions
+94
View File
@@ -1209,6 +1209,100 @@ export function main() {
Ok(())
}
/// A deployed flow runs an inline step as the `flow_node` its deploy rewrote it into,
/// a `FlowScript` job rather than the preview job the editor runs. A workflow-as-code
/// step's `task()` children must dispatch from that kind too, as re-runs of the same
/// node, or the step passes its editor test and fails once deployed.
///
/// The step is cached: a child that shared the parent's result-cache key would hand
/// its own result (`10`) back to the parent on resume, in place of the workflow's.
#[sqlx::test(fixtures("base", "wac_flow_script"))]
async fn test_bun_wac_task_dispatch_from_flow_script(db: Pool<Postgres>) -> anyhow::Result<()> {
use windmill_common::flows::FlowNodeId;
initialize_tracing().await;
let server = ApiServer::start(db.clone()).await?;
let port = server.addr.port();
let node = FlowNodeId(3000000000000011);
let job = RunJob::from(JobPayload::FlowScript {
id: node,
path: "f/system/wac_flow_script/a".to_string(),
language: ScriptLang::Bun,
cache_ttl: Some(60),
cache_ignore_s3_path: None,
dedicated_worker: None,
concurrency_settings: windmill_common::runnable_settings::ConcurrencySettings::default(),
})
.arg("n", serde_json::json!(5))
.run_until_complete(&db, false, port)
.await;
assert_eq!(
job.json_result().unwrap(),
serde_json::json!({"doubled": 10})
);
let children: Vec<(String, Option<i64>, Option<i32>)> = sqlx::query_as(
"SELECT kind::text, runnable_id, cache_ttl FROM v2_job WHERE parent_job = $1",
)
.bind(job.id)
.fetch_all(&db)
.await?;
assert_eq!(
children,
vec![("flowscript".to_string(), Some(node.0), None)],
"the task child re-runs the parent's flow node, outside the result cache"
);
Ok(())
}
/// `task(fn, { cache_ttl })` on an inline task of a deployed flow's step: the child runs
/// the parent's code with the parent's arguments, so its result-cache key carries its
/// step key, or the parent and every sibling would read its result back as their own.
#[sqlx::test(fixtures("base", "wac_flow_script"))]
async fn test_bun_wac_inline_task_cache_is_per_task(db: Pool<Postgres>) -> anyhow::Result<()> {
use windmill_common::flows::FlowNodeId;
initialize_tracing().await;
let server = ApiServer::start(db.clone()).await?;
let port = server.addr.port();
let payload = || JobPayload::FlowScript {
id: FlowNodeId(3000000000000012),
path: "f/system/wac_flow_script/a".to_string(),
language: ScriptLang::Bun,
cache_ttl: None,
cache_ignore_s3_path: None,
dedicated_worker: None,
concurrency_settings: windmill_common::runnable_settings::ConcurrencySettings::default(),
};
let mut children_from_cache = Vec::new();
for _ in 0..2 {
let job = RunJob::from(payload())
.arg("n", serde_json::json!(5))
.run_until_complete(&db, false, port)
.await;
assert_eq!(
job.json_result().unwrap(),
serde_json::json!({"doubled": 10, "tripled": 15})
);
let from_cache: i64 = sqlx::query_scalar(
"SELECT count(*) FROM job_logs l JOIN v2_job j ON j.id = l.job_id \
WHERE j.parent_job = $1 AND l.logs LIKE '%found in cache%'",
)
.bind(job.id)
.fetch_one(&db)
.await?;
children_from_cache.push(from_cache);
}
assert_eq!(
children_from_cache,
vec![0, 2],
"the second run serves each task from its own cache entry"
);
Ok(())
}
// ============================================================================
// Environment Variable Tests
// ============================================================================
+53
View File
@@ -0,0 +1,53 @@
-- A deployed flow whose inline bun step is workflow-as-code calling task(), in the
-- shape the deploy leaves behind: the RawScript module rewritten into a flow_node that
-- the step then runs as a FlowScript job. No lock, so the worker resolves
-- windmill-client at run time like the other bun fixtures.
INSERT INTO public.flow(workspace_id, summary, description, path, versions, schema, value, edited_by) VALUES (
'test-workspace', '', '',
'f/system/wac_flow_script',
'{}',
'{"$schema":"https://json-schema.org/draft/2020-12/schema","properties":{"n":{"type":"integer","description":""}},"required":[],"type":"object"}',
'{"modules":[{"id":"a","value":{"type":"flowscript","id":3000000000000011,"language":"bun","input_transforms":{"n":{"expr":"flow_input.n","type":"javascript"}}}}]}',
'system'
);
INSERT INTO public.flow_node(id, workspace_id, path, hash_v2, lock, code) VALUES (
3000000000000011,
'test-workspace',
'f/system/wac_flow_script',
'0000000000000000000000000000000000000000000000000000000000000011',
NULL,
E'import { workflow, task } from "windmill-client";
const double = task(async (n: number) => {
return n * 2;
});
export const main = workflow(async (n: number) => {
const d = await double(n);
return { doubled: d };
});'
);
-- The same flow's step with two tasks that cache their own result.
INSERT INTO public.flow_node(id, workspace_id, path, hash_v2, lock, code) VALUES (
3000000000000012,
'test-workspace',
'f/system/wac_flow_script',
'0000000000000000000000000000000000000000000000000000000000000012',
NULL,
E'import { workflow, task } from "windmill-client";
const double = task(async (n: number) => {
return n * 2;
}, { cache_ttl: 60 });
const triple = task(async (n: number) => {
return n * 3;
}, { cache_ttl: 60 });
export const main = workflow(async (n: number) => {
const d = await double(n);
const t = await triple(n);
return { doubled: d, tripled: t };
});'
);
+48 -16
View File
@@ -2874,15 +2874,17 @@ pub async fn handle_wac_v2_output(
.collect();
// Resolve job_payload once (same for all children since they re-run
// the parent script)
// the parent script). The step's cache setting is for the workflow's
// result; a task is cached only through its own `cache_ttl` option,
// under a key of its own (see `cached_result_path`).
let job_payload_template = match job.kind {
JobKind::Script => {
if let Some(hash) = job.runnable_id {
Ok(JobPayload::ScriptHash {
hash,
path: job.runnable_path.clone().unwrap_or_default(),
cache_ttl: job.cache_ttl,
cache_ignore_s3_path: job.cache_ignore_s3_path,
cache_ttl: None,
cache_ignore_s3_path: None,
dedicated_worker: None,
language: job.script_lang.unwrap_or(ScriptLang::Bun),
priority: job.priority,
@@ -2897,6 +2899,27 @@ pub async fn handle_wac_v2_output(
))
}
}
// A deployed flow runs an inline step as the `flow_node` its deploy
// rewrote it into; the child re-runs that node the way a `Script`
// child re-runs its hash, so `runnable_id` (the checkpoint's source
// hash) stays the same across parent and children.
JobKind::FlowScript => {
if let Some(id) = job.runnable_id {
Ok(JobPayload::FlowScript {
id: windmill_common::flows::FlowNodeId(id.0),
path: job.runnable_path.clone().unwrap_or_default(),
language: job.script_lang.unwrap_or(ScriptLang::Bun),
cache_ttl: None,
cache_ignore_s3_path: None,
dedicated_worker: None,
concurrency_settings: ConcurrencySettings::default(),
})
} else {
Err(error::Error::internal_err(
"WAC v2 FlowScript job missing runnable_id".to_string(),
))
}
}
JobKind::Preview => {
let row: Option<(Option<String>, Option<String>)> = sqlx::query_as(
"SELECT raw_code, raw_lock FROM v2_job WHERE id = $1 AND workspace_id = $2",
@@ -2912,8 +2935,8 @@ pub async fn handle_wac_v2_output(
hash: None,
language: job.script_lang.unwrap_or(ScriptLang::Bun),
lock: lock,
cache_ttl: job.cache_ttl,
cache_ignore_s3_path: job.cache_ignore_s3_path,
cache_ttl: None,
cache_ignore_s3_path: None,
dedicated_worker: None,
concurrency_settings: ConcurrencySettingsWithCustom::default(),
debouncing_settings: DebouncingSettings::default(),
@@ -3012,6 +3035,12 @@ pub async fn handle_wac_v2_output(
let mut pushed_ids: Vec<Uuid> = Vec::with_capacity(num_steps);
let push_result: error::Result<()> = async {
for (step, (_, child_uuid)) in steps.iter().zip(job_ids.iter()) {
// A task with a runnable of its own (a deployed script or flow) queues
// at that runnable's priority; any other task is the parent's code and
// queues at the parent's.
let own_runnable = matches!(step.dispatch_type.as_str(), "script" | "flow")
&& !step.script.starts_with("./");
// Resolve job payload based on dispatch_type
let (job_payload, child_args, is_external, on_behalf_of) =
match step.dispatch_type.as_str() {
@@ -3025,8 +3054,8 @@ pub async fn handle_wac_v2_output(
hash: None,
language: module.language,
lock: module.lock,
cache_ttl: job.cache_ttl,
cache_ignore_s3_path: job.cache_ignore_s3_path,
cache_ttl: None,
cache_ignore_s3_path: None,
dedicated_worker: None,
concurrency_settings: ConcurrencySettingsWithCustom::default(),
debouncing_settings: DebouncingSettings::default(),
@@ -3110,7 +3139,8 @@ pub async fn handle_wac_v2_output(
let mut job_payload = job_payload;
if let Some(cache_ttl) = step.cache_ttl {
match &mut job_payload {
JobPayload::ScriptHash { cache_ttl: ref mut ct, .. } => {
JobPayload::ScriptHash { cache_ttl: ref mut ct, .. }
| JobPayload::FlowScript { cache_ttl: ref mut ct, .. } => {
*ct = Some(cache_ttl)
}
JobPayload::Code(ref mut code) => code.cache_ttl = Some(cache_ttl),
@@ -3122,7 +3152,8 @@ pub async fn handle_wac_v2_output(
|| step.concurrency_time_window_s.is_some()
{
match &mut job_payload {
JobPayload::ScriptHash { concurrency_settings: ref mut cs, .. } => {
JobPayload::ScriptHash { concurrency_settings: ref mut cs, .. }
| JobPayload::FlowScript { concurrency_settings: ref mut cs, .. } => {
if let Some(limit) = step.concurrent_limit {
cs.concurrent_limit = Some(limit);
}
@@ -3188,13 +3219,14 @@ pub async fn handle_wac_v2_output(
job.visible_to_owner,
step.tag.clone().or_else(|| Some(job.tag.clone())),
step.timeout.or(job.timeout),
None, // flow_step_id
step.priority, // priority_override
None, // authed
false, // running
None, // end_user_email
None, // trigger
None, // suspended_mode
None, // flow_step_id
step.priority
.or(if own_runnable { None } else { job.priority }),
None, // authed
false, // running
None, // end_user_email
None, // trigger
None, // suspended_mode
)
.await?;
+28 -2
View File
@@ -1559,7 +1559,7 @@ pub async fn cached_result_path(
client: &AuthedClient,
job: &MiniPulledJob,
raw_data: Option<&RawData>,
) -> String {
) -> windmill_common::error::Result<String> {
let mut hasher = sha2::Sha256::new();
hasher.update(&[job.kind as u8]);
if let Some(ScriptHash(hash)) = job.runnable_id {
@@ -1574,6 +1574,13 @@ pub async fn cached_result_path(
_ => {}
}
}
// A workflow-as-code task child runs its parent's code with the parent's
// arguments; the step it executes is what tells its result from the parent's
// and from its siblings'.
if let Some(step_key) = wac_executing_key(db, job).await? {
hasher.update(b"wac_step:");
hasher.update(step_key.as_bytes());
}
hash_args(
db,
client,
@@ -1584,7 +1591,26 @@ pub async fn cached_result_path(
job.cache_ignore_s3_path.unwrap_or(false),
)
.await;
format!("g/results/{:064x}", hasher.finalize())
Ok(format!("g/results/{:064x}", hasher.finalize()))
}
/// The checkpoint step key a workflow-as-code parent seeded for this child at push
/// time; `None` for any job that is not such a child.
async fn wac_executing_key(
db: &DB,
job: &MiniPulledJob,
) -> windmill_common::error::Result<Option<String>> {
if job.parent_job.is_none() || job.flow_step_id.is_some() {
return Ok(None);
}
let key: Option<Option<String>> = sqlx::query_scalar(
"SELECT workflow_as_code_status->'_checkpoint'->>'_executing_key' \
FROM v2_job_status WHERE id = $1",
)
.bind(job.id)
.fetch_optional(db)
.await?;
Ok(key.flatten())
}
#[cfg(feature = "parquet")]
+1 -1
View File
@@ -4645,7 +4645,7 @@ pub async fn handle_queued_job(
let cached_res_path = if job.cache_ttl.is_some() {
match conn {
Connection::Sql(db) => {
Some(cached_result_path(db, &client, &job, preview_data.as_ref()).await)
Some(cached_result_path(db, &client, &job, preview_data.as_ref()).await?)
}
Connection::Http(_) => None,
}
+2 -1
View File
@@ -2005,7 +2005,8 @@ pub async fn update_flow_status_after_job_completion_internal(
if flow_job.cache_ttl.is_some() && success {
let flow = RawData::Flow(flow_data.clone());
let cached_res_path = cached_result_path(db, client, &flow_job, Some(&flow)).await;
let cached_res_path =
cached_result_path(db, client, &flow_job, Some(&flow)).await?;
save_in_cache(
db,
+23
View File
@@ -4608,6 +4608,14 @@ def parse_sql_client_name(name: str) -> tuple[str, Optional[str]]
# it grows with both the width of the fan-out and \`\`attempts\`\`. Retries with
# no \`\`delay\`\` all go out in a single round.
#
# \`\`cache_ttl\`\` serves a previous result of the task for that many seconds
# instead of running it again. A task is keyed on its step key (its name and
# call order) and the workflow's input, not on the arguments it is called
# with, so cache one only when whether it runs, and what it receives, follow
# from the workflow's input alone. A \`\`task_script\`\` target is keyed on the
# arguments it is called with. It has no effect on a \`\`task_flow\`\` target,
# which keeps its flow's own cache policy.
#
# Usage::
#
# @task
@@ -6742,6 +6750,13 @@ export interface TaskRetry {
export interface TaskOptions {
timeout?: number;
tag?: string;
/** Seconds during which a previous result of this task is served instead of
* running it again. A task written inline in the workflow is keyed on its
* step key (its name and call order) and the workflow's input, not on the
* arguments it is called with, so cache one only when whether it runs, and
* what it receives, follow from the workflow's input alone. A \`taskScript\`
* target is keyed on the arguments it is called with. It has no effect on a
* \`taskFlow\` target, which keeps its flow's own cache policy. */
cache_ttl?: number;
priority?: number;
concurrency_limit?: number;
@@ -6933,6 +6948,14 @@ def get_resume_urls(approver: str = None, flow_level: bool = None) -> dict
# it grows with both the width of the fan-out and \`\`attempts\`\`. Retries with
# no \`\`delay\`\` all go out in a single round.
#
# \`\`cache_ttl\`\` serves a previous result of the task for that many seconds
# instead of running it again. A task is keyed on its step key (its name and
# call order) and the workflow's input, not on the arguments it is called
# with, so cache one only when whether it runs, and what it receives, follow
# from the workflow's input alone. A \`\`task_script\`\` target is keyed on the
# arguments it is called with. It has no effect on a \`\`task_flow\`\` target,
# which keeps its flow's own cache policy.
#
# Usage::
#
# @task
+8
View File
@@ -3327,6 +3327,14 @@ def task(
it grows with both the width of the fan-out and ``attempts``. Retries with
no ``delay`` all go out in a single round.
``cache_ttl`` serves a previous result of the task for that many seconds
instead of running it again. A task is keyed on its step key (its name and
call order) and the workflow's input, not on the arguments it is called
with, so cache one only when whether it runs, and what it receives, follow
from the workflow's input alone. A ``task_script`` target is keyed on the
arguments it is called with. It has no effect on a ``task_flow`` target,
which keeps its flow's own cache policy.
Usage::
@task
+23
View File
@@ -2588,6 +2588,14 @@ def parse_sql_client_name(name: str) -> tuple[str, Optional[str]]
# it grows with both the width of the fan-out and \`\`attempts\`\`. Retries with
# no \`\`delay\`\` all go out in a single round.
#
# \`\`cache_ttl\`\` serves a previous result of the task for that many seconds
# instead of running it again. A task is keyed on its step key (its name and
# call order) and the workflow's input, not on the arguments it is called
# with, so cache one only when whether it runs, and what it receives, follow
# from the workflow's input alone. A \`\`task_script\`\` target is keyed on the
# arguments it is called with. It has no effect on a \`\`task_flow\`\` target,
# which keeps its flow's own cache policy.
#
# Usage::
#
# @task
@@ -2738,6 +2746,13 @@ export interface TaskRetry {
export interface TaskOptions {
timeout?: number;
tag?: string;
/** Seconds during which a previous result of this task is served instead of
* running it again. A task written inline in the workflow is keyed on its
* step key (its name and call order) and the workflow's input, not on the
* arguments it is called with, so cache one only when whether it runs, and
* what it receives, follow from the workflow's input alone. A \`taskScript\`
* target is keyed on the arguments it is called with. It has no effect on a
* \`taskFlow\` target, which keeps its flow's own cache policy. */
cache_ttl?: number;
priority?: number;
concurrency_limit?: number;
@@ -2929,6 +2944,14 @@ def get_resume_urls(approver: str = None, flow_level: bool = None) -> dict
# it grows with both the width of the fan-out and \`\`attempts\`\`. Retries with
# no \`\`delay\`\` all go out in a single round.
#
# \`\`cache_ttl\`\` serves a previous result of the task for that many seconds
# instead of running it again. A task is keyed on its step key (its name and
# call order) and the workflow's input, not on the arguments it is called
# with, so cache one only when whether it runs, and what it receives, follow
# from the workflow's input alone. A \`\`task_script\`\` target is keyed on the
# arguments it is called with. It has no effect on a \`\`task_flow\`\` target,
# which keeps its flow's own cache policy.
#
# Usage::
#
# @task
+8
View File
@@ -2728,6 +2728,14 @@ def parse_sql_client_name(name: str) -> tuple[str, Optional[str]]
# it grows with both the width of the fan-out and ``attempts``. Retries with
# no ``delay`` all go out in a single round.
#
# ``cache_ttl`` serves a previous result of the task for that many seconds
# instead of running it again. A task is keyed on its step key (its name and
# call order) and the workflow's input, not on the arguments it is called
# with, so cache one only when whether it runs, and what it receives, follow
# from the workflow's input alone. A ``task_script`` target is keyed on the
# arguments it is called with. It has no effect on a ``task_flow`` target,
# which keeps its flow's own cache policy.
#
# Usage::
#
# @task
@@ -672,6 +672,14 @@ def parse_sql_client_name(name: str) -> tuple[str, Optional[str]]
# it grows with both the width of the fan-out and ``attempts``. Retries with
# no ``delay`` all go out in a single round.
#
# ``cache_ttl`` serves a previous result of the task for that many seconds
# instead of running it again. A task is keyed on its step key (its name and
# call order) and the workflow's input, not on the arguments it is called
# with, so cache one only when whether it runs, and what it receives, follow
# from the workflow's input alone. A ``task_script`` target is keyed on the
# arguments it is called with. It has no effect on a ``task_flow`` target,
# which keeps its flow's own cache policy.
#
# Usage::
#
# @task
@@ -58,6 +58,14 @@ def get_resume_urls(approver: str = None, flow_level: bool = None) -> dict
# it grows with both the width of the fan-out and ``attempts``. Retries with
# no ``delay`` all go out in a single round.
#
# ``cache_ttl`` serves a previous result of the task for that many seconds
# instead of running it again. A task is keyed on its step key (its name and
# call order) and the workflow's input, not on the arguments it is called
# with, so cache one only when whether it runs, and what it receives, follow
# from the workflow's input alone. A ``task_script`` target is keyed on the
# arguments it is called with. It has no effect on a ``task_flow`` target,
# which keeps its flow's own cache policy.
#
# Usage::
#
# @task
@@ -34,6 +34,13 @@ export interface TaskRetry {
export interface TaskOptions {
timeout?: number;
tag?: string;
/** Seconds during which a previous result of this task is served instead of
* running it again. A task written inline in the workflow is keyed on its
* step key (its name and call order) and the workflow's input, not on the
* arguments it is called with, so cache one only when whether it runs, and
* what it receives, follow from the workflow's input alone. A `taskScript`
* target is keyed on the arguments it is called with. It has no effect on a
* `taskFlow` target, which keeps its flow's own cache policy. */
cache_ttl?: number;
priority?: number;
concurrency_limit?: number;
@@ -857,6 +857,14 @@ def parse_sql_client_name(name: str) -> tuple[str, Optional[str]]
# it grows with both the width of the fan-out and ``attempts``. Retries with
# no ``delay`` all go out in a single round.
#
# ``cache_ttl`` serves a previous result of the task for that many seconds
# instead of running it again. A task is keyed on its step key (its name and
# call order) and the workflow's input, not on the arguments it is called
# with, so cache one only when whether it runs, and what it receives, follow
# from the workflow's input alone. A ``task_script`` target is keyed on the
# arguments it is called with. It has no effect on a ``task_flow`` target,
# which keeps its flow's own cache policy.
#
# Usage::
#
# @task
@@ -277,6 +277,13 @@ export interface TaskRetry {
export interface TaskOptions {
timeout?: number;
tag?: string;
/** Seconds during which a previous result of this task is served instead of
* running it again. A task written inline in the workflow is keyed on its
* step key (its name and call order) and the workflow's input, not on the
* arguments it is called with, so cache one only when whether it runs, and
* what it receives, follow from the workflow's input alone. A `taskScript`
* target is keyed on the arguments it is called with. It has no effect on a
* `taskFlow` target, which keeps its flow's own cache policy. */
cache_ttl?: number;
priority?: number;
concurrency_limit?: number;
@@ -468,6 +475,14 @@ def get_resume_urls(approver: str = None, flow_level: bool = None) -> dict
# it grows with both the width of the fan-out and ``attempts``. Retries with
# no ``delay`` all go out in a single round.
#
# ``cache_ttl`` serves a previous result of the task for that many seconds
# instead of running it again. A task is keyed on its step key (its name and
# call order) and the workflow's input, not on the arguments it is called
# with, so cache one only when whether it runs, and what it receives, follow
# from the workflow's input alone. A ``task_script`` target is keyed on the
# arguments it is called with. It has no effect on a ``task_flow`` target,
# which keeps its flow's own cache policy.
#
# Usage::
#
# @task
+7
View File
@@ -1712,6 +1712,13 @@ export interface TaskRetry {
export interface TaskOptions {
timeout?: number;
tag?: string;
/** Seconds during which a previous result of this task is served instead of
* running it again. A task written inline in the workflow is keyed on its
* step key (its name and call order) and the workflow's input, not on the
* arguments it is called with, so cache one only when whether it runs, and
* what it receives, follow from the workflow's input alone. A `taskScript`
* target is keyed on the arguments it is called with. It has no effect on a
* `taskFlow` target, which keeps its flow's own cache policy. */
cache_ttl?: number;
priority?: number;
concurrency_limit?: number;