From dd9f633764bcef0451eb3fb3580389a91dfa99ad Mon Sep 17 00:00:00 2001 From: Guilhem Date: Tue, 17 Nov 2026 12:12:27 +0100 Subject: [PATCH] fix: read agent tool inputs from the flow running the agent step (#12457) * fix: read flow_input and results of the subflow in agent tool inputs Co-Authored-By: Claude Opus 5.5 * fix: resolve the owning flow for nested agent tool inputs Co-Authored-By: Claude Opus 5.5 * fix: keep nested agent output out of the chat conversation Co-Authored-By: Claude Opus 5.5 * fix: give a first-step agent its flow's args as previous_result Co-Authored-By: Claude Opus 5.5 * fix: read flow_user_state of the run in nested agent tool inputs Co-Authored-By: Claude Opus 5.5 * test: pin first-step previous_result and nested flow_user_state for agent tools Co-Authored-By: Claude Opus 5.5 --------- Co-authored-by: Claude Opus 5.5 --- ...64b4bdd2ef4727b664f0fb9044eaa46a454db.json | 46 +++++ ...6a710d0efd77dac4aedeb88d72240e816530b.json | 34 ---- backend/tests/ai_agent_tool_scope.rs | 181 ++++++++++++++++++ backend/windmill-worker/src/ai/tools.rs | 4 +- backend/windmill-worker/src/ai/utils.rs | 55 +++--- backend/windmill-worker/src/ai_executor.rs | 44 +++-- 6 files changed, 291 insertions(+), 73 deletions(-) create mode 100644 backend/.sqlx/query-5355271bf33a45184745d3f06c064b4bdd2ef4727b664f0fb9044eaa46a454db.json delete mode 100644 backend/.sqlx/query-9008f9abb70a9a07e38acb20bea6a710d0efd77dac4aedeb88d72240e816530b.json create mode 100644 backend/tests/ai_agent_tool_scope.rs diff --git a/backend/.sqlx/query-5355271bf33a45184745d3f06c064b4bdd2ef4727b664f0fb9044eaa46a454db.json b/backend/.sqlx/query-5355271bf33a45184745d3f06c064b4bdd2ef4727b664f0fb9044eaa46a454db.json new file mode 100644 index 0000000000..4728a15cd6 --- /dev/null +++ b/backend/.sqlx/query-5355271bf33a45184745d3f06c064b4bdd2ef4727b664f0fb9044eaa46a454db.json @@ -0,0 +1,46 @@ +{ + "db_name": "PostgreSQL", + "query": "\n SELECT\n f.args as \"args: Json>>\",\n fs.flow_status as \"step_flow_status: Json\",\n rs.flow_status as \"run_status: Json\",\n r.runnable_path,\n r.id as root_job_id\n FROM v2_job f\n LEFT JOIN v2_job_status fs ON fs.id = f.id\n INNER JOIN v2_job r\n ON r.id = COALESCE(f.root_job, f.flow_innermost_root_job, f.parent_job, f.id)\n LEFT JOIN v2_job_status rs ON rs.id = r.id\n WHERE f.id = $1\n ", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "args: Json>>", + "type_info": "Jsonb" + }, + { + "ordinal": 1, + "name": "step_flow_status: Json", + "type_info": "Jsonb" + }, + { + "ordinal": 2, + "name": "run_status: Json", + "type_info": "Jsonb" + }, + { + "ordinal": 3, + "name": "runnable_path", + "type_info": "Varchar" + }, + { + "ordinal": 4, + "name": "root_job_id", + "type_info": "Uuid" + } + ], + "parameters": { + "Left": [ + "Uuid" + ] + }, + "nullable": [ + true, + true, + true, + true, + false + ] + }, + "hash": "5355271bf33a45184745d3f06c064b4bdd2ef4727b664f0fb9044eaa46a454db" +} diff --git a/backend/.sqlx/query-9008f9abb70a9a07e38acb20bea6a710d0efd77dac4aedeb88d72240e816530b.json b/backend/.sqlx/query-9008f9abb70a9a07e38acb20bea6a710d0efd77dac4aedeb88d72240e816530b.json deleted file mode 100644 index 758da55ad2..0000000000 --- a/backend/.sqlx/query-9008f9abb70a9a07e38acb20bea6a710d0efd77dac4aedeb88d72240e816530b.json +++ /dev/null @@ -1,34 +0,0 @@ -{ - "db_name": "PostgreSQL", - "query": "\n SELECT\n j.args as \"args: Json>>\",\n js.flow_status as \"flow_status: Json\",\n j.runnable_path\n FROM v2_job_status js\n INNER JOIN v2_job j ON j.id = js.id\n WHERE js.id = $1\n ", - "describe": { - "columns": [ - { - "ordinal": 0, - "name": "args: Json>>", - "type_info": "Jsonb" - }, - { - "ordinal": 1, - "name": "flow_status: Json", - "type_info": "Jsonb" - }, - { - "ordinal": 2, - "name": "runnable_path", - "type_info": "Varchar" - } - ], - "parameters": { - "Left": [ - "Uuid" - ] - }, - "nullable": [ - true, - true, - true - ] - }, - "hash": "9008f9abb70a9a07e38acb20bea6a710d0efd77dac4aedeb88d72240e816530b" -} diff --git a/backend/tests/ai_agent_tool_scope.rs b/backend/tests/ai_agent_tool_scope.rs new file mode 100644 index 0000000000..7037a297fc --- /dev/null +++ b/backend/tests/ai_agent_tool_scope.rs @@ -0,0 +1,181 @@ +use axum::{routing::post, Json, Router}; +use serde_json::{json, Value}; +use sqlx::{Pool, Postgres}; +use std::time::Duration; +use uuid::Uuid; +use windmill_common::{flows::FlowValue, jobs::JobPayload}; +use windmill_test_utils::{ + completed_job, in_test_worker, listen_for_completed_jobs, ApiServer, RunJob, StreamFind, +}; + +const CONVERSATION: Uuid = Uuid::from_u128(0xc0de); + +/// A model that calls every tool it is offered once, then answers. +async fn model(Json(body): Json) -> ([(&'static str, &'static str); 1], String) { + let finished = body["messages"] + .as_array() + .unwrap() + .iter() + .any(|m| m["role"] == "tool"); + let delta = if finished { + json!({"role":"assistant", "content":"done"}) + } else { + let calls: Vec = body["tools"] + .as_array() + .unwrap() + .iter() + .enumerate() + .map(|(i, tool)| { + json!({ + "index":i, "id":format!("call_{i}"), "type":"function", + "function":{"name":tool["function"]["name"], "arguments":"{}"} + }) + }) + .collect(); + json!({"role":"assistant", "tool_calls":calls}) + }; + let event = json!({"choices":[{"index":0, "delta":delta, "finish_reason":null}]}); + let end = json!({"choices":[{"index":0, "delta":{}, "finish_reason":if finished {"stop"} else {"tool_calls"}}]}); + ( + [("content-type", "text/event-stream")], + format!("data: {event}\n\ndata: {end}\n\ndata: [DONE]\n\n"), + ) +} + +fn record_tool(id: &str) -> Value { + json!({"id":id,"summary":"record","value":{ + "type":"rawscript", "language":"bun", + "input_transforms":{ + "topic":{"type":"javascript","expr":"flow_input.iter.value"}, + "prev":{"type":"javascript","expr":"results.a"}, + "pr":{"type":"javascript","expr":"previous_result"}, + "state":{"type":"javascript","expr":"flow_user_state('k')"} + }, + "content":"export async function main(topic: string, prev: string, pr: string, state: string) { return `${topic}|${prev}|${pr}|${state}` }" + }}) +} + +/// A loop body runs as its own flow job, as a subflow does: a tool's `flow_input` and `results` +/// must be that flow's, as a regular step's are, not the root run's. A nested agent's tools read +/// the same flow, past the agent calling it, and the run's user state. An agent first in its flow +/// gets that flow's args as `previous_result`, as a regular first step does. In a chat run, only the calling agent writes to the +/// conversation, and every row it writes belongs to the run. +#[sqlx::test(fixtures("base"))] +#[serial_test::serial] +async fn agent_tool_reads_inputs_of_the_flow_running_the_step( + db: Pool, +) -> anyhow::Result<()> { + std::env::set_var("ALLOW_PRIVATE_AI_BASE_URLS", "true"); + let server = ApiServer::start(db.clone()).await?; + let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await?; + let base = format!("http://{}", listener.local_addr()?); + let stub = tokio::spawn(async move { + let router = Router::new().route("/v1/chat/completions", post(model)); + axum::serve(listener, router).await.unwrap(); + }); + let agent_inputs = json!({ + "provider":{"type":"static","value":{"kind":"customai","model":"scope-test","resource":{"base_url":format!("{base}/v1")}}}, + "user_message":{"type":"static","value":"record"} + }); + let flow: FlowValue = serde_json::from_value( + json!({"chat_input_enabled":true,"modules":[{"id":"loop","value":{ + "type":"forloopflow", + "iterator":{"type":"javascript","expr":"['t1', 't2']"}, + "skip_failures":false, + "modules":[ + {"id":"first","value":{ + "type":"aiagent", + "tools":[{"id":"first_record","summary":"record","value":{ + "type":"rawscript", "language":"bun", + "input_transforms":{"pr":{"type":"javascript","expr":"previous_result"}}, + "content":"export async function main(pr: any) { return `first|${pr.iter.value}` }" + }}], + "input_transforms":agent_inputs.clone() + }}, + {"id":"a","value":{ + "type":"rawscript", "language":"bun", + "input_transforms":{"topic":{"type":"javascript","expr":"flow_input.iter.value"}}, + "content":"export async function main(topic: string) { return 'seed-' + topic }" + }}, + {"id":"agent","value":{ + "type":"aiagent", + "tools":[ + record_tool("record"), + {"id":"inner","summary":"inner","value":{ + "type":"aiagent", + "tools":[record_tool("inner_record")], + "input_transforms":agent_inputs.clone() + }} + ], + "input_transforms":agent_inputs + }} + ] + }}]}), + )?; + let id = RunJob::from(JobPayload::RawFlow { + value: flow, + path: Some("u/test/agent_tool_scope".into()), + restarted_from: None, + }) + .push(&db) + .await; + // What the API does for a chat turn: open the conversation and give the run its id. + sqlx::query( + "INSERT INTO flow_conversation (id, workspace_id, flow_path, created_by) + VALUES ($1, 'test-workspace', 'u/test/agent_tool_scope', 'test-user')", + ) + .bind(CONVERSATION) + .execute(&db) + .await?; + sqlx::query( + "UPDATE v2_job_status SET flow_status = jsonb_set(flow_status, '{memory_id}', to_jsonb($2::uuid)) + || jsonb_build_object('user_states', jsonb_build_object('k', 'v')) + WHERE id = $1", + ) + .bind(id) + .bind(CONVERSATION) + .execute(&db) + .await?; + let notifications = listen_for_completed_jobs(&db).await; + tokio::time::timeout( + Duration::from_secs(60), + in_test_worker(&db, notifications.find(&id), server.addr.port()), + ) + .await?; + let job = completed_job(id, &db).await; + stub.abort(); + server.close().await?; + assert!(job.success, "{:?}", job.result); + let recorded: Vec<(String, Value)> = sqlx::query_as( + "SELECT j.runnable_path, c.result FROM v2_job j JOIN v2_job_completed c USING(id) + WHERE j.runnable_path LIKE '%/tools/%record' ORDER BY c.result::text, j.runnable_path", + ) + .fetch_all(&db) + .await? + .into_iter() + .map(|(path, result): (String, sqlx::types::Json)| (path, result.0)) + .collect(); + let results: Vec<&Value> = recorded.iter().map(|(_, r)| r).collect(); + assert_eq!( + results, + vec![ + &json!("first|t1"), + &json!("first|t2"), + &json!("t1|seed-t1|seed-t1|v"), + &json!("t1|seed-t1|seed-t1|v"), + &json!("t2|seed-t2|seed-t2|v"), + &json!("t2|seed-t2|seed-t2|v") + ], + "{recorded:?}" + ); + // A row the nested agent wrote would carry the calling agent's job as its turn. + let turns: Vec = sqlx::query_scalar( + "SELECT turn_job_id FROM flow_conversation_message WHERE conversation_id = $1", + ) + .bind(CONVERSATION) + .fetch_all(&db) + .await?; + assert!(!turns.is_empty()); + assert!(turns.iter().all(|turn| *turn == id), "{turns:?}"); + Ok(()) +} diff --git a/backend/windmill-worker/src/ai/tools.rs b/backend/windmill-worker/src/ai/tools.rs index dab9ec3356..693c39926b 100644 --- a/backend/windmill-worker/src/ai/tools.rs +++ b/backend/windmill-worker/src/ai/tools.rs @@ -1460,14 +1460,14 @@ async fn add_tool_message_to_chat( let chat_enabled = ctx .flow_context - .flow_status + .run_status .as_ref() .and_then(|fs| fs.chat_input_enabled) .unwrap_or(false); if chat_enabled { if let Some(memory_id) = ctx .flow_context - .flow_status + .run_status .as_ref() .and_then(|fs| fs.memory_id) { diff --git a/backend/windmill-worker/src/ai/utils.rs b/backend/windmill-worker/src/ai/utils.rs index f1d09302ca..387d37d7ac 100644 --- a/backend/windmill-worker/src/ai/utils.rs +++ b/backend/windmill-worker/src/ai/utils.rs @@ -14,7 +14,7 @@ use windmill_common::{ db::DB, error::Error, flow_conversations::{add_message_to_conversation_tx, MessageExtras, MessageType}, - flow_status::AgentAction, + flow_status::{AgentAction, FlowStatus}, flows::{InputTransform, Step}, jobs::JobKind, scripts::{ScriptHash, ScriptLang}, @@ -141,47 +141,56 @@ pub async fn get_flow_job_runnable_and_raw_flow( #[derive(Debug, Clone, Default)] pub struct FlowContext { + /// `flow_input` of the flow running the step. pub flow_inputs: Option>>, - pub flow_status: Option, + /// Status of the flow running the step, which `results.` and the previous result read. + pub step_flow_status: Option, + /// Status of the root run, which holds the chat settings and the run's memory id. + pub run_status: Option, /// Path of the flow the run started from, which scopes a string memory id. pub flow_path: Option, + /// The root run, whose user state `flow_user_state` reads. + pub root_job_id: Option, } -/// Get flow context (chat settings + args + flow_status) from root flow's job data -pub async fn get_flow_context(db: &DB, job: &MiniPulledJob) -> FlowContext { - let root_job_id = job - .root_job - .or(job.flow_innermost_root_job) - .or(job.parent_job); - - let Some(root_job_id) = root_job_id else { - return FlowContext::default(); - }; - +/// Read the context an agent step evaluates its tools in. `flow_input` and `results` are those of +/// `flow_job_id`, the flow the agent is a step of, as for any other step: inside a subflow that is +/// not the root run. Chat and memory belong to the whole run. +/// +/// The root is resolved from the flow job, as `get_root_job_id` does: a nested agent's own +/// `root_job` names the agent calling it, which has no status row. +pub async fn get_flow_context(db: &DB, job: &MiniPulledJob, flow_job_id: &Uuid) -> FlowContext { match sqlx::query!( r#" SELECT - j.args as "args: Json>>", - js.flow_status as "flow_status: Json", - j.runnable_path - FROM v2_job_status js - INNER JOIN v2_job j ON j.id = js.id - WHERE js.id = $1 + f.args as "args: Json>>", + fs.flow_status as "step_flow_status: Json", + rs.flow_status as "run_status: Json", + r.runnable_path, + r.id as root_job_id + FROM v2_job f + LEFT JOIN v2_job_status fs ON fs.id = f.id + INNER JOIN v2_job r + ON r.id = COALESCE(f.root_job, f.flow_innermost_root_job, f.parent_job, f.id) + LEFT JOIN v2_job_status rs ON rs.id = r.id + WHERE f.id = $1 "#, - root_job_id + flow_job_id ) .fetch_optional(db) .await { Ok(Some(row)) => FlowContext { flow_inputs: row.args.map(|j| j.0), - flow_status: row.flow_status.map(|j| j.0), + step_flow_status: row.step_flow_status.map(|j| j.0), + run_status: row.run_status.map(|j| j.0), flow_path: row.runnable_path, + root_job_id: Some(row.root_job_id), }, Ok(None) => { tracing::warn!( - "No flow context found for root job {} (agent job {}), returning default", - root_job_id, + "No flow context found for flow job {} (agent job {}), returning default", + flow_job_id, job.id ); FlowContext::default() diff --git a/backend/windmill-worker/src/ai_executor.rs b/backend/windmill-worker/src/ai_executor.rs index e9259e533b..2f72794753 100644 --- a/backend/windmill-worker/src/ai_executor.rs +++ b/backend/windmill-worker/src/ai_executor.rs @@ -1270,6 +1270,7 @@ pub async fn handle_ai_agent_job( db, conn, job, + &flow_job_id, flow_status_job.as_ref(), Some(flow_step_id.as_str()), &args, @@ -1285,7 +1286,9 @@ pub async fn handle_ai_agent_job( killpill_rx, has_stream, has_websearch, - omit_output_from_conversation, + // A nested agent answers its caller, whose tool message carries that answer into the + // chat; its own rows would also be tied to the calling agent's job, not the turn. + omit_output_from_conversation || direct_parent_job_kind == JobKind::AIAgent, preserve_step_tags, cancel_rx, tool_abort_handles.clone(), @@ -1641,6 +1644,8 @@ pub async fn run_agent( // agent job and flow data job: &MiniPulledJob, + // the flow the agent is a step of, past the calling agent for a nested agent tool + flow_job_id: &Uuid, parent_job: Option<&Uuid>, flow_step_id_override: Option<&str>, args: &AIAgentArgs, @@ -1719,7 +1724,7 @@ pub async fn run_agent( )); // Fetch flow context for input transforms context, chat and memory - let mut flow_context = get_flow_context(db, job).await; + let mut flow_context = get_flow_context(db, job, flow_job_id).await; // Set when the job parked on a request or on background jobs and has been pulled again. Read // whatever the roster says now: a linked agent can lose a tool while one of its runs is parked. let resumed = match load_checkpoint(db, &job.id).await? { @@ -1766,10 +1771,7 @@ pub async fn run_agent( // The run's memory id is also the chat conversation id, which a step's own memory id never // replaces. - let conversation_id = flow_context - .flow_status - .as_ref() - .and_then(|fs| fs.memory_id); + let conversation_id = flow_context.run_status.as_ref().and_then(|fs| fs.memory_id); let (history, history_notes) = resolve_history_source( args, conversation_id, @@ -1847,11 +1849,20 @@ pub async fn run_agent( // Extract previous step result only if any tool needs it let previous_result = { if any_tool_needs_previous_result(&tools) { - if let Some(ref flow_status) = flow_context.flow_status { - get_previous_job_result(db, &job.workspace_id, flow_status) - .await - .ok() - .flatten() + if let Some(ref flow_status) = flow_context.step_flow_status { + // As for a regular step, the first step of a flow (a subflow, a loop iteration) + // gets that flow's args: no step precedes it. + if flow_status.step == 0 { + flow_context + .flow_inputs + .as_ref() + .and_then(|args| serde_json::value::to_raw_value(args).ok()) + } else { + get_previous_job_result(db, &job.workspace_id, flow_status) + .await + .ok() + .flatten() + } } else { None } @@ -1862,13 +1873,18 @@ pub async fn run_agent( // Build IdContext for results.stepId syntax let id_context = { - if let Some(ref flow_status) = flow_context.flow_status { + if let Some(ref flow_status) = flow_context.step_flow_status { // Get the step ID from the AI agent's flow step let previous_id = effective_flow_step_id .map(str::to_string) .unwrap_or_else(|| "unknown".to_string()); - Some(get_transform_context(job, &previous_id, flow_status)) + let mut id_context = get_transform_context(job, &previous_id, flow_status); + // The agent's own root names the calling agent for a nested agent. + if let Some(root) = flow_context.root_job_id { + id_context.root_flow_job = root; + } + Some(id_context) } else { None } @@ -2028,7 +2044,7 @@ pub async fn run_agent( let mut stream_event_processor = new_stream_event_processor(&final_events_str); let chat_enabled = flow_context - .flow_status + .run_status .as_ref() .and_then(|fs| fs.chat_input_enabled) .unwrap_or(false);