mirror of
https://github.com/windmill-labs/windmill.git
synced 2026-11-20 00:07:01 +00:00
fix: read agent tool inputs from the flow running the agent step (#12457)
* fix: read flow_input and results of the subflow in agent tool inputs Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com> * fix: resolve the owning flow for nested agent tool inputs Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com> * fix: keep nested agent output out of the chat conversation Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com> * fix: give a first-step agent its flow's args as previous_result Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com> * fix: read flow_user_state of the run in nested agent tool inputs Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com> * test: pin first-step previous_result and nested flow_user_state for agent tools Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com> --------- Co-authored-by: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
co-authored by
Claude Opus 5.5
parent
88ef4e9f3c
commit
dd9f633764
+46
@@ -0,0 +1,46 @@
|
||||
{
|
||||
"db_name": "PostgreSQL",
|
||||
"query": "\n SELECT\n f.args as \"args: Json<HashMap<String, Box<RawValue>>>\",\n fs.flow_status as \"step_flow_status: Json<FlowStatus>\",\n rs.flow_status as \"run_status: Json<FlowStatus>\",\n r.runnable_path,\n r.id as root_job_id\n FROM v2_job f\n LEFT JOIN v2_job_status fs ON fs.id = f.id\n INNER JOIN v2_job r\n ON r.id = COALESCE(f.root_job, f.flow_innermost_root_job, f.parent_job, f.id)\n LEFT JOIN v2_job_status rs ON rs.id = r.id\n WHERE f.id = $1\n ",
|
||||
"describe": {
|
||||
"columns": [
|
||||
{
|
||||
"ordinal": 0,
|
||||
"name": "args: Json<HashMap<String, Box<RawValue>>>",
|
||||
"type_info": "Jsonb"
|
||||
},
|
||||
{
|
||||
"ordinal": 1,
|
||||
"name": "step_flow_status: Json<FlowStatus>",
|
||||
"type_info": "Jsonb"
|
||||
},
|
||||
{
|
||||
"ordinal": 2,
|
||||
"name": "run_status: Json<FlowStatus>",
|
||||
"type_info": "Jsonb"
|
||||
},
|
||||
{
|
||||
"ordinal": 3,
|
||||
"name": "runnable_path",
|
||||
"type_info": "Varchar"
|
||||
},
|
||||
{
|
||||
"ordinal": 4,
|
||||
"name": "root_job_id",
|
||||
"type_info": "Uuid"
|
||||
}
|
||||
],
|
||||
"parameters": {
|
||||
"Left": [
|
||||
"Uuid"
|
||||
]
|
||||
},
|
||||
"nullable": [
|
||||
true,
|
||||
true,
|
||||
true,
|
||||
true,
|
||||
false
|
||||
]
|
||||
},
|
||||
"hash": "5355271bf33a45184745d3f06c064b4bdd2ef4727b664f0fb9044eaa46a454db"
|
||||
}
|
||||
-34
@@ -1,34 +0,0 @@
|
||||
{
|
||||
"db_name": "PostgreSQL",
|
||||
"query": "\n SELECT\n j.args as \"args: Json<HashMap<String, Box<RawValue>>>\",\n js.flow_status as \"flow_status: Json<windmill_common::flow_status::FlowStatus>\",\n j.runnable_path\n FROM v2_job_status js\n INNER JOIN v2_job j ON j.id = js.id\n WHERE js.id = $1\n ",
|
||||
"describe": {
|
||||
"columns": [
|
||||
{
|
||||
"ordinal": 0,
|
||||
"name": "args: Json<HashMap<String, Box<RawValue>>>",
|
||||
"type_info": "Jsonb"
|
||||
},
|
||||
{
|
||||
"ordinal": 1,
|
||||
"name": "flow_status: Json<windmill_common::flow_status::FlowStatus>",
|
||||
"type_info": "Jsonb"
|
||||
},
|
||||
{
|
||||
"ordinal": 2,
|
||||
"name": "runnable_path",
|
||||
"type_info": "Varchar"
|
||||
}
|
||||
],
|
||||
"parameters": {
|
||||
"Left": [
|
||||
"Uuid"
|
||||
]
|
||||
},
|
||||
"nullable": [
|
||||
true,
|
||||
true,
|
||||
true
|
||||
]
|
||||
},
|
||||
"hash": "9008f9abb70a9a07e38acb20bea6a710d0efd77dac4aedeb88d72240e816530b"
|
||||
}
|
||||
@@ -0,0 +1,181 @@
|
||||
use axum::{routing::post, Json, Router};
|
||||
use serde_json::{json, Value};
|
||||
use sqlx::{Pool, Postgres};
|
||||
use std::time::Duration;
|
||||
use uuid::Uuid;
|
||||
use windmill_common::{flows::FlowValue, jobs::JobPayload};
|
||||
use windmill_test_utils::{
|
||||
completed_job, in_test_worker, listen_for_completed_jobs, ApiServer, RunJob, StreamFind,
|
||||
};
|
||||
|
||||
const CONVERSATION: Uuid = Uuid::from_u128(0xc0de);
|
||||
|
||||
/// A model that calls every tool it is offered once, then answers.
|
||||
async fn model(Json(body): Json<Value>) -> ([(&'static str, &'static str); 1], String) {
|
||||
let finished = body["messages"]
|
||||
.as_array()
|
||||
.unwrap()
|
||||
.iter()
|
||||
.any(|m| m["role"] == "tool");
|
||||
let delta = if finished {
|
||||
json!({"role":"assistant", "content":"done"})
|
||||
} else {
|
||||
let calls: Vec<Value> = body["tools"]
|
||||
.as_array()
|
||||
.unwrap()
|
||||
.iter()
|
||||
.enumerate()
|
||||
.map(|(i, tool)| {
|
||||
json!({
|
||||
"index":i, "id":format!("call_{i}"), "type":"function",
|
||||
"function":{"name":tool["function"]["name"], "arguments":"{}"}
|
||||
})
|
||||
})
|
||||
.collect();
|
||||
json!({"role":"assistant", "tool_calls":calls})
|
||||
};
|
||||
let event = json!({"choices":[{"index":0, "delta":delta, "finish_reason":null}]});
|
||||
let end = json!({"choices":[{"index":0, "delta":{}, "finish_reason":if finished {"stop"} else {"tool_calls"}}]});
|
||||
(
|
||||
[("content-type", "text/event-stream")],
|
||||
format!("data: {event}\n\ndata: {end}\n\ndata: [DONE]\n\n"),
|
||||
)
|
||||
}
|
||||
|
||||
fn record_tool(id: &str) -> Value {
|
||||
json!({"id":id,"summary":"record","value":{
|
||||
"type":"rawscript", "language":"bun",
|
||||
"input_transforms":{
|
||||
"topic":{"type":"javascript","expr":"flow_input.iter.value"},
|
||||
"prev":{"type":"javascript","expr":"results.a"},
|
||||
"pr":{"type":"javascript","expr":"previous_result"},
|
||||
"state":{"type":"javascript","expr":"flow_user_state('k')"}
|
||||
},
|
||||
"content":"export async function main(topic: string, prev: string, pr: string, state: string) { return `${topic}|${prev}|${pr}|${state}` }"
|
||||
}})
|
||||
}
|
||||
|
||||
/// A loop body runs as its own flow job, as a subflow does: a tool's `flow_input` and `results`
|
||||
/// must be that flow's, as a regular step's are, not the root run's. A nested agent's tools read
|
||||
/// the same flow, past the agent calling it, and the run's user state. An agent first in its flow
|
||||
/// gets that flow's args as `previous_result`, as a regular first step does. In a chat run, only the calling agent writes to the
|
||||
/// conversation, and every row it writes belongs to the run.
|
||||
#[sqlx::test(fixtures("base"))]
|
||||
#[serial_test::serial]
|
||||
async fn agent_tool_reads_inputs_of_the_flow_running_the_step(
|
||||
db: Pool<Postgres>,
|
||||
) -> anyhow::Result<()> {
|
||||
std::env::set_var("ALLOW_PRIVATE_AI_BASE_URLS", "true");
|
||||
let server = ApiServer::start(db.clone()).await?;
|
||||
let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await?;
|
||||
let base = format!("http://{}", listener.local_addr()?);
|
||||
let stub = tokio::spawn(async move {
|
||||
let router = Router::new().route("/v1/chat/completions", post(model));
|
||||
axum::serve(listener, router).await.unwrap();
|
||||
});
|
||||
let agent_inputs = json!({
|
||||
"provider":{"type":"static","value":{"kind":"customai","model":"scope-test","resource":{"base_url":format!("{base}/v1")}}},
|
||||
"user_message":{"type":"static","value":"record"}
|
||||
});
|
||||
let flow: FlowValue = serde_json::from_value(
|
||||
json!({"chat_input_enabled":true,"modules":[{"id":"loop","value":{
|
||||
"type":"forloopflow",
|
||||
"iterator":{"type":"javascript","expr":"['t1', 't2']"},
|
||||
"skip_failures":false,
|
||||
"modules":[
|
||||
{"id":"first","value":{
|
||||
"type":"aiagent",
|
||||
"tools":[{"id":"first_record","summary":"record","value":{
|
||||
"type":"rawscript", "language":"bun",
|
||||
"input_transforms":{"pr":{"type":"javascript","expr":"previous_result"}},
|
||||
"content":"export async function main(pr: any) { return `first|${pr.iter.value}` }"
|
||||
}}],
|
||||
"input_transforms":agent_inputs.clone()
|
||||
}},
|
||||
{"id":"a","value":{
|
||||
"type":"rawscript", "language":"bun",
|
||||
"input_transforms":{"topic":{"type":"javascript","expr":"flow_input.iter.value"}},
|
||||
"content":"export async function main(topic: string) { return 'seed-' + topic }"
|
||||
}},
|
||||
{"id":"agent","value":{
|
||||
"type":"aiagent",
|
||||
"tools":[
|
||||
record_tool("record"),
|
||||
{"id":"inner","summary":"inner","value":{
|
||||
"type":"aiagent",
|
||||
"tools":[record_tool("inner_record")],
|
||||
"input_transforms":agent_inputs.clone()
|
||||
}}
|
||||
],
|
||||
"input_transforms":agent_inputs
|
||||
}}
|
||||
]
|
||||
}}]}),
|
||||
)?;
|
||||
let id = RunJob::from(JobPayload::RawFlow {
|
||||
value: flow,
|
||||
path: Some("u/test/agent_tool_scope".into()),
|
||||
restarted_from: None,
|
||||
})
|
||||
.push(&db)
|
||||
.await;
|
||||
// What the API does for a chat turn: open the conversation and give the run its id.
|
||||
sqlx::query(
|
||||
"INSERT INTO flow_conversation (id, workspace_id, flow_path, created_by)
|
||||
VALUES ($1, 'test-workspace', 'u/test/agent_tool_scope', 'test-user')",
|
||||
)
|
||||
.bind(CONVERSATION)
|
||||
.execute(&db)
|
||||
.await?;
|
||||
sqlx::query(
|
||||
"UPDATE v2_job_status SET flow_status = jsonb_set(flow_status, '{memory_id}', to_jsonb($2::uuid))
|
||||
|| jsonb_build_object('user_states', jsonb_build_object('k', 'v'))
|
||||
WHERE id = $1",
|
||||
)
|
||||
.bind(id)
|
||||
.bind(CONVERSATION)
|
||||
.execute(&db)
|
||||
.await?;
|
||||
let notifications = listen_for_completed_jobs(&db).await;
|
||||
tokio::time::timeout(
|
||||
Duration::from_secs(60),
|
||||
in_test_worker(&db, notifications.find(&id), server.addr.port()),
|
||||
)
|
||||
.await?;
|
||||
let job = completed_job(id, &db).await;
|
||||
stub.abort();
|
||||
server.close().await?;
|
||||
assert!(job.success, "{:?}", job.result);
|
||||
let recorded: Vec<(String, Value)> = sqlx::query_as(
|
||||
"SELECT j.runnable_path, c.result FROM v2_job j JOIN v2_job_completed c USING(id)
|
||||
WHERE j.runnable_path LIKE '%/tools/%record' ORDER BY c.result::text, j.runnable_path",
|
||||
)
|
||||
.fetch_all(&db)
|
||||
.await?
|
||||
.into_iter()
|
||||
.map(|(path, result): (String, sqlx::types::Json<Value>)| (path, result.0))
|
||||
.collect();
|
||||
let results: Vec<&Value> = recorded.iter().map(|(_, r)| r).collect();
|
||||
assert_eq!(
|
||||
results,
|
||||
vec![
|
||||
&json!("first|t1"),
|
||||
&json!("first|t2"),
|
||||
&json!("t1|seed-t1|seed-t1|v"),
|
||||
&json!("t1|seed-t1|seed-t1|v"),
|
||||
&json!("t2|seed-t2|seed-t2|v"),
|
||||
&json!("t2|seed-t2|seed-t2|v")
|
||||
],
|
||||
"{recorded:?}"
|
||||
);
|
||||
// A row the nested agent wrote would carry the calling agent's job as its turn.
|
||||
let turns: Vec<Uuid> = sqlx::query_scalar(
|
||||
"SELECT turn_job_id FROM flow_conversation_message WHERE conversation_id = $1",
|
||||
)
|
||||
.bind(CONVERSATION)
|
||||
.fetch_all(&db)
|
||||
.await?;
|
||||
assert!(!turns.is_empty());
|
||||
assert!(turns.iter().all(|turn| *turn == id), "{turns:?}");
|
||||
Ok(())
|
||||
}
|
||||
@@ -1460,14 +1460,14 @@ async fn add_tool_message_to_chat(
|
||||
|
||||
let chat_enabled = ctx
|
||||
.flow_context
|
||||
.flow_status
|
||||
.run_status
|
||||
.as_ref()
|
||||
.and_then(|fs| fs.chat_input_enabled)
|
||||
.unwrap_or(false);
|
||||
if chat_enabled {
|
||||
if let Some(memory_id) = ctx
|
||||
.flow_context
|
||||
.flow_status
|
||||
.run_status
|
||||
.as_ref()
|
||||
.and_then(|fs| fs.memory_id)
|
||||
{
|
||||
|
||||
@@ -14,7 +14,7 @@ use windmill_common::{
|
||||
db::DB,
|
||||
error::Error,
|
||||
flow_conversations::{add_message_to_conversation_tx, MessageExtras, MessageType},
|
||||
flow_status::AgentAction,
|
||||
flow_status::{AgentAction, FlowStatus},
|
||||
flows::{InputTransform, Step},
|
||||
jobs::JobKind,
|
||||
scripts::{ScriptHash, ScriptLang},
|
||||
@@ -141,47 +141,56 @@ pub async fn get_flow_job_runnable_and_raw_flow(
|
||||
|
||||
#[derive(Debug, Clone, Default)]
|
||||
pub struct FlowContext {
|
||||
/// `flow_input` of the flow running the step.
|
||||
pub flow_inputs: Option<HashMap<String, Box<RawValue>>>,
|
||||
pub flow_status: Option<windmill_common::flow_status::FlowStatus>,
|
||||
/// Status of the flow running the step, which `results.<step>` and the previous result read.
|
||||
pub step_flow_status: Option<FlowStatus>,
|
||||
/// Status of the root run, which holds the chat settings and the run's memory id.
|
||||
pub run_status: Option<FlowStatus>,
|
||||
/// Path of the flow the run started from, which scopes a string memory id.
|
||||
pub flow_path: Option<String>,
|
||||
/// The root run, whose user state `flow_user_state` reads.
|
||||
pub root_job_id: Option<Uuid>,
|
||||
}
|
||||
|
||||
/// Get flow context (chat settings + args + flow_status) from root flow's job data
|
||||
pub async fn get_flow_context(db: &DB, job: &MiniPulledJob) -> FlowContext {
|
||||
let root_job_id = job
|
||||
.root_job
|
||||
.or(job.flow_innermost_root_job)
|
||||
.or(job.parent_job);
|
||||
|
||||
let Some(root_job_id) = root_job_id else {
|
||||
return FlowContext::default();
|
||||
};
|
||||
|
||||
/// Read the context an agent step evaluates its tools in. `flow_input` and `results` are those of
|
||||
/// `flow_job_id`, the flow the agent is a step of, as for any other step: inside a subflow that is
|
||||
/// not the root run. Chat and memory belong to the whole run.
|
||||
///
|
||||
/// The root is resolved from the flow job, as `get_root_job_id` does: a nested agent's own
|
||||
/// `root_job` names the agent calling it, which has no status row.
|
||||
pub async fn get_flow_context(db: &DB, job: &MiniPulledJob, flow_job_id: &Uuid) -> FlowContext {
|
||||
match sqlx::query!(
|
||||
r#"
|
||||
SELECT
|
||||
j.args as "args: Json<HashMap<String, Box<RawValue>>>",
|
||||
js.flow_status as "flow_status: Json<windmill_common::flow_status::FlowStatus>",
|
||||
j.runnable_path
|
||||
FROM v2_job_status js
|
||||
INNER JOIN v2_job j ON j.id = js.id
|
||||
WHERE js.id = $1
|
||||
f.args as "args: Json<HashMap<String, Box<RawValue>>>",
|
||||
fs.flow_status as "step_flow_status: Json<FlowStatus>",
|
||||
rs.flow_status as "run_status: Json<FlowStatus>",
|
||||
r.runnable_path,
|
||||
r.id as root_job_id
|
||||
FROM v2_job f
|
||||
LEFT JOIN v2_job_status fs ON fs.id = f.id
|
||||
INNER JOIN v2_job r
|
||||
ON r.id = COALESCE(f.root_job, f.flow_innermost_root_job, f.parent_job, f.id)
|
||||
LEFT JOIN v2_job_status rs ON rs.id = r.id
|
||||
WHERE f.id = $1
|
||||
"#,
|
||||
root_job_id
|
||||
flow_job_id
|
||||
)
|
||||
.fetch_optional(db)
|
||||
.await
|
||||
{
|
||||
Ok(Some(row)) => FlowContext {
|
||||
flow_inputs: row.args.map(|j| j.0),
|
||||
flow_status: row.flow_status.map(|j| j.0),
|
||||
step_flow_status: row.step_flow_status.map(|j| j.0),
|
||||
run_status: row.run_status.map(|j| j.0),
|
||||
flow_path: row.runnable_path,
|
||||
root_job_id: Some(row.root_job_id),
|
||||
},
|
||||
Ok(None) => {
|
||||
tracing::warn!(
|
||||
"No flow context found for root job {} (agent job {}), returning default",
|
||||
root_job_id,
|
||||
"No flow context found for flow job {} (agent job {}), returning default",
|
||||
flow_job_id,
|
||||
job.id
|
||||
);
|
||||
FlowContext::default()
|
||||
|
||||
@@ -1270,6 +1270,7 @@ pub async fn handle_ai_agent_job(
|
||||
db,
|
||||
conn,
|
||||
job,
|
||||
&flow_job_id,
|
||||
flow_status_job.as_ref(),
|
||||
Some(flow_step_id.as_str()),
|
||||
&args,
|
||||
@@ -1285,7 +1286,9 @@ pub async fn handle_ai_agent_job(
|
||||
killpill_rx,
|
||||
has_stream,
|
||||
has_websearch,
|
||||
omit_output_from_conversation,
|
||||
// A nested agent answers its caller, whose tool message carries that answer into the
|
||||
// chat; its own rows would also be tied to the calling agent's job, not the turn.
|
||||
omit_output_from_conversation || direct_parent_job_kind == JobKind::AIAgent,
|
||||
preserve_step_tags,
|
||||
cancel_rx,
|
||||
tool_abort_handles.clone(),
|
||||
@@ -1641,6 +1644,8 @@ pub async fn run_agent(
|
||||
|
||||
// agent job and flow data
|
||||
job: &MiniPulledJob,
|
||||
// the flow the agent is a step of, past the calling agent for a nested agent tool
|
||||
flow_job_id: &Uuid,
|
||||
parent_job: Option<&Uuid>,
|
||||
flow_step_id_override: Option<&str>,
|
||||
args: &AIAgentArgs,
|
||||
@@ -1719,7 +1724,7 @@ pub async fn run_agent(
|
||||
));
|
||||
|
||||
// Fetch flow context for input transforms context, chat and memory
|
||||
let mut flow_context = get_flow_context(db, job).await;
|
||||
let mut flow_context = get_flow_context(db, job, flow_job_id).await;
|
||||
// Set when the job parked on a request or on background jobs and has been pulled again. Read
|
||||
// whatever the roster says now: a linked agent can lose a tool while one of its runs is parked.
|
||||
let resumed = match load_checkpoint(db, &job.id).await? {
|
||||
@@ -1766,10 +1771,7 @@ pub async fn run_agent(
|
||||
|
||||
// The run's memory id is also the chat conversation id, which a step's own memory id never
|
||||
// replaces.
|
||||
let conversation_id = flow_context
|
||||
.flow_status
|
||||
.as_ref()
|
||||
.and_then(|fs| fs.memory_id);
|
||||
let conversation_id = flow_context.run_status.as_ref().and_then(|fs| fs.memory_id);
|
||||
let (history, history_notes) = resolve_history_source(
|
||||
args,
|
||||
conversation_id,
|
||||
@@ -1847,11 +1849,20 @@ pub async fn run_agent(
|
||||
// Extract previous step result only if any tool needs it
|
||||
let previous_result = {
|
||||
if any_tool_needs_previous_result(&tools) {
|
||||
if let Some(ref flow_status) = flow_context.flow_status {
|
||||
get_previous_job_result(db, &job.workspace_id, flow_status)
|
||||
.await
|
||||
.ok()
|
||||
.flatten()
|
||||
if let Some(ref flow_status) = flow_context.step_flow_status {
|
||||
// As for a regular step, the first step of a flow (a subflow, a loop iteration)
|
||||
// gets that flow's args: no step precedes it.
|
||||
if flow_status.step == 0 {
|
||||
flow_context
|
||||
.flow_inputs
|
||||
.as_ref()
|
||||
.and_then(|args| serde_json::value::to_raw_value(args).ok())
|
||||
} else {
|
||||
get_previous_job_result(db, &job.workspace_id, flow_status)
|
||||
.await
|
||||
.ok()
|
||||
.flatten()
|
||||
}
|
||||
} else {
|
||||
None
|
||||
}
|
||||
@@ -1862,13 +1873,18 @@ pub async fn run_agent(
|
||||
|
||||
// Build IdContext for results.stepId syntax
|
||||
let id_context = {
|
||||
if let Some(ref flow_status) = flow_context.flow_status {
|
||||
if let Some(ref flow_status) = flow_context.step_flow_status {
|
||||
// Get the step ID from the AI agent's flow step
|
||||
let previous_id = effective_flow_step_id
|
||||
.map(str::to_string)
|
||||
.unwrap_or_else(|| "unknown".to_string());
|
||||
|
||||
Some(get_transform_context(job, &previous_id, flow_status))
|
||||
let mut id_context = get_transform_context(job, &previous_id, flow_status);
|
||||
// The agent's own root names the calling agent for a nested agent.
|
||||
if let Some(root) = flow_context.root_job_id {
|
||||
id_context.root_flow_job = root;
|
||||
}
|
||||
Some(id_context)
|
||||
} else {
|
||||
None
|
||||
}
|
||||
@@ -2028,7 +2044,7 @@ pub async fn run_agent(
|
||||
let mut stream_event_processor = new_stream_event_processor(&final_events_str);
|
||||
|
||||
let chat_enabled = flow_context
|
||||
.flow_status
|
||||
.run_status
|
||||
.as_ref()
|
||||
.and_then(|fs| fs.chat_input_enabled)
|
||||
.unwrap_or(false);
|
||||
|
||||
Reference in New Issue
Block a user