diff --git a/backend/.sqlx/query-392b256dd7f36ec3736fc2f9d564dc198a6b69d62fb244691af0601e542b6ac3.json b/backend/.sqlx/query-392b256dd7f36ec3736fc2f9d564dc198a6b69d62fb244691af0601e542b6ac3.json new file mode 100644 index 0000000000..1b5aa1bd1f --- /dev/null +++ b/backend/.sqlx/query-392b256dd7f36ec3736fc2f9d564dc198a6b69d62fb244691af0601e542b6ac3.json @@ -0,0 +1,23 @@ +{ + "db_name": "PostgreSQL", + "query": "SELECT EXISTS(SELECT 1 FROM resource WHERE workspace_id = $1 AND path = $2 AND resource_type = 'ai_agent')", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "exists", + "type_info": "Bool" + } + ], + "parameters": { + "Left": [ + "Text", + "Text" + ] + }, + "nullable": [ + null + ] + }, + "hash": "392b256dd7f36ec3736fc2f9d564dc198a6b69d62fb244691af0601e542b6ac3" +} diff --git a/backend/windmill-api-flow-conversations/src/lib.rs b/backend/windmill-api-flow-conversations/src/lib.rs index 7320721ea5..2ed3531cca 100644 --- a/backend/windmill-api-flow-conversations/src/lib.rs +++ b/backend/windmill-api-flow-conversations/src/lib.rs @@ -14,7 +14,7 @@ pub use windmill_common::flow_conversations::FlowConversation; use windmill_common::{ db::{UserDB, DB}, error::{JsonResult, Result}, - flow_conversations::{running_turns, MessageType, RunningTurn}, + flow_conversations::{running_turns, MessageType, RunningTurn, ANONYMOUS_CONVERSATION_OWNER}, utils::{not_found_if_none, paginate, truncate_with_ellipsis, Pagination}, }; @@ -96,7 +96,8 @@ async fn list_conversations( "created_by", "is_test", ]) - .and_where_eq("workspace_id", "?".bind(&w_id)); + .and_where_eq("workspace_id", "?".bind(&w_id)) + .and_where_ne("created_by", "?".bind(&ANONYMOUS_CONVERSATION_OWNER)); if let Some(flow_path) = &query.flow_path { sqlb.and_where_eq("flow_path", "?".bind(flow_path)); diff --git a/backend/windmill-api-jobs/src/execution.rs b/backend/windmill-api-jobs/src/execution.rs index da77acbf7c..53c318b70d 100644 --- a/backend/windmill-api-jobs/src/execution.rs +++ b/backend/windmill-api-jobs/src/execution.rs @@ -916,6 +916,35 @@ pub async fn handle_chat_conversation_messages( is_test: bool, // The run's args, for the files the message carried. args: &HashMap>, +) -> error::Result<()> { + file_chat_turn( + tx, + &authed.username, + authed.is_admin, + w_id, + flow_path, + run_query, + user_message_raw, + job_id, + is_test, + args, + ) + .await +} + +/// [`handle_chat_conversation_messages`] for a conversation owned by `owner`, which need not be a +/// user: an anonymous app visitor's is owned by `ANONYMOUS_CONVERSATION_OWNER`. +pub async fn file_chat_turn( + tx: &mut sqlx::Transaction<'_, sqlx::Postgres>, + owner: &str, + is_admin: bool, + w_id: &str, + flow_path: &str, + run_query: &RunJobQuery, + user_message_raw: Option<&Box>, + job_id: Uuid, + is_test: bool, + args: &HashMap>, ) -> error::Result<()> { // Names the query parameter rather than the field: it is not a flow argument, and // supplying it as one is the first thing tried on reading `memory_id is required`. @@ -945,8 +974,8 @@ pub async fn handle_chat_conversation_messages( tx, w_id, flow_path, - &authed.username, - authed.is_admin, + owner, + is_admin, &user_message, memory_id, is_test, diff --git a/backend/windmill-api/openapi.yaml b/backend/windmill-api/openapi.yaml index ddc7b70832..6996562aac 100644 --- a/backend/windmill-api/openapi.yaml +++ b/backend/windmill-api/openapi.yaml @@ -17107,6 +17107,12 @@ paths: user_attachments: type: array items: {} + previous_messages: + description: | + The history to answer from, as chat messages, when the agent's memory is off. + An agent with managed memory reads its conversation instead and ignores this. + type: array + items: {} responses: "201": description: job created diff --git a/backend/windmill-api/src/agent_runs.rs b/backend/windmill-api/src/agent_runs.rs index f1f85985ea..3bf65c9808 100644 --- a/backend/windmill-api/src/agent_runs.rs +++ b/backend/windmill-api/src/agent_runs.rs @@ -68,51 +68,9 @@ pub(crate) async fn run_agent_as( crate::jobs::check_license_key_valid().await?; // Read through the caller's own permissions: reading the agent is what allows running it. - let mut tx = user_db.clone().begin(authed).await?; - let value = sqlx::query_scalar!( - "SELECT value AS \"value: sqlx::types::Json\" - FROM resource WHERE workspace_id = $1 AND path = $2 AND resource_type = 'ai_agent'", - w_id, - path - ) - .fetch_optional(&mut *tx) - .await?; - tx.commit().await?; - let Some(sqlx::types::Json(mut value)) = not_found_if_none(value, "Agent", path)? else { - return Err(Error::BadRequest(format!( - "Agent {path} has no configuration" - ))); - }; - // Who the agent runs as and who can ask it in Slack, not inputs: its identity when it opts - // in, as a deployed flow does. - let obj = value.as_object_mut(); - let (agent_obo, run_obo) = obj - .map(|o| { - o.remove("slack_anyone"); - (o.remove("on_behalf_of"), o.remove("run_on_behalf_of")) - }) - .unwrap_or_default(); - let agent_obo = agent_obo - .filter(|_| run_obo.and_then(|v| v.as_bool()).unwrap_or(false)) - .and_then(|v| v.as_str().map(str::to_string)); - let config = config_to_draft(value)?; - - // A chat turn is filed where the agent's chat lists its conversations, which a flow cannot - // name (flow paths carry no `.`), so the two never share a conversation list. let chat = run_query.memory_id.is_some(); - let run_path = if chat { - format!("{path}.chat") - } else { - path.to_string() - }; - // A step memory id would replace the conversation's, and the agent would forget the turns - // before; history comes from the conversation alone. - let flow_value = agent_step_flow( - &config, - AGENT_NODE_ID, - chat, - &["memory_id", "previous_messages"], - )?; + let AgentTurn { flow_value, run_path, agent_obo } = + agent_turn(authed, user_db, w_id, path, chat).await?; let on_behalf_of = windmill_common::on_behalf_of_from_permissioned_as(agent_obo.as_deref(), w_id, db).await?; @@ -210,11 +168,73 @@ pub(crate) async fn run_agent_as( Ok(uuid) } +/// A turn of a saved agent, as the flow that runs it. +pub(crate) struct AgentTurn { + pub flow_value: windmill_common::flows::FlowValue, + /// Where the turn is filed: a chat turn under `.chat`, which a flow cannot name (flow + /// paths carry no `.`), so the agent's conversations never share a list with a flow's. + pub run_path: String, + /// The identity the agent runs as when it opts in (`run_on_behalf_of`). A caller that runs it + /// as some other identity, such as an app's, ignores it. + pub agent_obo: Option, +} + +/// Reads the agent through `authed`'s own permissions and builds the flow of one of its turns. +/// Who it may see and which inputs or slack settings it carries are not inputs of the run. +pub(crate) async fn agent_turn( + authed: &ApiAuthed, + user_db: &UserDB, + w_id: &str, + path: &str, + chat: bool, +) -> Result { + let mut tx = user_db.clone().begin(authed).await?; + let value = sqlx::query_scalar!( + "SELECT value AS \"value: sqlx::types::Json\" + FROM resource WHERE workspace_id = $1 AND path = $2 AND resource_type = 'ai_agent'", + w_id, + path + ) + .fetch_optional(&mut *tx) + .await?; + tx.commit().await?; + let Some(sqlx::types::Json(mut value)) = not_found_if_none(value, "Agent", path)? else { + return Err(Error::BadRequest(format!( + "Agent {path} has no configuration" + ))); + }; + let obj = value.as_object_mut(); + let (agent_obo, run_obo) = obj + .map(|o| { + o.remove("slack_anyone"); + (o.remove("on_behalf_of"), o.remove("run_on_behalf_of")) + }) + .unwrap_or_default(); + let agent_obo = agent_obo + .filter(|_| run_obo.and_then(|v| v.as_bool()).unwrap_or(false)) + .and_then(|v| v.as_str().map(str::to_string)); + let config = config_to_draft(value)?; + let run_path = if chat { + format!("{path}.chat") + } else { + path.to_string() + }; + // A step memory id would replace the conversation's, and the agent would forget the turns + // before; history comes from the conversation alone. + let flow_value = agent_step_flow( + &config, + AGENT_NODE_ID, + chat, + &["memory_id", "previous_messages"], + )?; + Ok(AgentTurn { flow_value, run_path, agent_obo }) +} + /// The step fetches the message's files with its job's token, so a run as the agent's identity /// would read any file that identity can: a caller attaches only what they could download /// themselves, checked as `sign_s3_objects` checks a key before signing it. #[cfg(all(feature = "parquet", feature = "private"))] -async fn check_attachments_readable( +pub(crate) async fn check_attachments_readable( db: &DB, authed: &ApiAuthed, w_id: &str, @@ -224,14 +244,9 @@ async fn check_attachments_readable( get_large_file_storage, get_workspace_s3_resource_and_check_paths, }; use windmill_common::db::DbWithOptAuthed; - use windmill_types::s3::{S3Object, S3Permission}; + use windmill_types::s3::S3Permission; - let Some(raw) = args.get("user_attachments") else { - return Ok(()); - }; - let attachments: Option> = serde_json::from_str(raw.get()) - .map_err(|e| Error::BadRequest(format!("invalid user_attachments: {e}")))?; - for attachment in attachments.iter().flatten().filter(|a| !a.s3.is_empty()) { + for attachment in run_attachments(args)? { // The path check is skipped, not failed, when the storage does not resolve. if get_large_file_storage(db, w_id, attachment.storage.clone()) .await? @@ -254,10 +269,39 @@ async fn check_attachments_readable( Ok(()) } +/// Every file the step fetches with its job's token: the message's attachments, and the +/// `s3_object` parts of the history a caller supplies, which the worker downloads alike. +pub(crate) fn run_attachments( + args: &HashMap>, +) -> Result> { + use windmill_ai::ai_types::{ContentPart, OpenAIContent, OpenAIMessage}; + + let mut files: Vec = match args.get("user_attachments") { + Some(raw) => serde_json::from_str::>>(raw.get()) + .map_err(|e| Error::BadRequest(format!("invalid user_attachments: {e}")))? + .unwrap_or_default(), + None => vec![], + }; + if let Some(raw) = args.get("previous_messages") { + let messages = serde_json::from_str::>>(raw.get()) + .map_err(|e| Error::BadRequest(format!("invalid previous_messages: {e}")))?; + for message in messages.into_iter().flatten() { + if let Some(OpenAIContent::Parts(parts)) = message.content { + files.extend(parts.into_iter().filter_map(|part| match part { + ContentPart::S3Object { s3_object } => Some(s3_object), + _ => None, + })); + } + } + } + files.retain(|f| !f.s3.is_empty()); + Ok(files) +} + /// The agent as a one-step flow, validated by deserializing through `FlowValue` rather than /// trusted as raw JSON. Its own transforms are the step's, minus `without`, and the run's inputs -/// supply the message and the attachments over the top. Always inlined, never a link to the -/// resource: a linked step would resolve the agent when it runs, not as it was read here. +/// supply the message, the attachments and any history over the top. Always inlined, never a link +/// to the resource: a linked step would resolve the agent when it runs, not as it was read here. pub(crate) fn agent_step_flow( config: &AgentDraft, id: &str, @@ -271,7 +315,7 @@ pub(crate) fn agent_step_flow( for key in without { input_transforms.remove(*key); } - for key in ["user_message", "user_attachments"] { + for key in ["user_message", "user_attachments", "previous_messages"] { input_transforms.insert( key.to_string(), serde_json::json!({ "type": "javascript", "expr": format!("flow_input.{}", key) }), @@ -292,3 +336,25 @@ pub(crate) fn agent_step_flow( } Ok(serde_json::from_value(flow)?) } + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn run_attachments_include_the_files_a_supplied_history_quotes() { + let args: HashMap> = serde_json::from_value(serde_json::json!({ + "user_attachments": [{ "s3": "u/a/attached.png" }], + "previous_messages": [ + { "role": "user", "content": [ + { "type": "text", "text": "look" }, + { "type": "s3_object", "s3_object": { "s3": "u/b/quoted.png" } } + ]}, + { "role": "assistant", "content": "ok" } + ] + })) + .unwrap(); + let keys: Vec<_> = run_attachments(&args).unwrap().into_iter().map(|f| f.s3).collect(); + assert_eq!(keys, ["u/a/attached.png", "u/b/quoted.png"]); + } +} diff --git a/backend/windmill-api/src/ai_evals/run.rs b/backend/windmill-api/src/ai_evals/run.rs index 410e92f515..6b219e5597 100644 --- a/backend/windmill-api/src/ai_evals/run.rs +++ b/backend/windmill-api/src/ai_evals/run.rs @@ -255,6 +255,8 @@ fn agent_module(config: &AgentDraft) -> Result { .entry("input_transforms") .or_insert_with(|| serde_json::json!({})); if let Some(transforms) = transforms.as_object_mut() { + // A case carries no history: each one is asked fresh. + transforms.remove("previous_messages"); for key in ["user_message", "user_attachments"] { transforms.insert( key.to_string(), diff --git a/backend/windmill-api/src/apps.rs b/backend/windmill-api/src/apps.rs index 9f58638046..bc0124b685 100644 --- a/backend/windmill-api/src/apps.rs +++ b/backend/windmill-api/src/apps.rs @@ -58,8 +58,8 @@ use windmill_audit::ActionKind; use windmill_common::roles::{caller_capabilities, AuthorLevel}; use windmill_common::{ apps::{ - app_value_has_inline_script, app_value_runnable_paths, AppScriptId, ListAppQuery, - APP_WORKSPACED_ROUTE, + app_value_has_inline_script, app_value_runnable_paths, AppRunnableKind, AppScriptId, + ListAppQuery, APP_WORKSPACED_ROUTE, }, auth::{APP_EMBED_TOKEN_LABEL_PREFIX, RAW_APP_SDK_TOKEN_LABEL_PREFIX, TOKEN_PREFIX_LEN}, cache, @@ -2307,14 +2307,14 @@ pub(crate) enum BuilderAppWrite { } /// The half of the builder-app check that needs no DB (see `docs/operator-builder-rights.md`). -/// Returns every runnable the app can invoke, as `(is_flow, path)`, from both the policy's +/// Returns every runnable the app can invoke, as `(kind, path)`, from both the policy's /// triggerables and the value's path runnables: they are not the same list. fn check_operator_composed_app( write: BuilderAppWrite, raw_app: bool, value: Option<&RawValue>, policy: Option<&mut Policy>, -) -> Result> { +) -> Result> { let verb = if write == BuilderAppWrite::Deploy { "deploy" } else { @@ -2325,7 +2325,7 @@ fn check_operator_composed_app( "Operators with builder rights can only author full-code apps".to_string(), )); } - let mut referenced: Vec<(bool, String)> = Vec::new(); + let mut referenced: Vec<(AppRunnableKind, String)> = Vec::new(); if let Some(value) = value { // The text stored, NULs stripped: a stripped NUL can rename a key into one this reads. let value: serde_json::Value = @@ -2386,9 +2386,9 @@ fn check_operator_composed_app( Ok(referenced) } -/// The runnables a builder app's policy authorizes, as `(is_flow, path)`; refuses a policy that +/// The runnables a builder app's policy authorizes, as `(kind, path)`; refuses a policy that /// pins inline code. -fn checked_policy_runnable_paths(policy: &Policy) -> Result> { +fn checked_policy_runnable_paths(policy: &Policy) -> Result> { // A `rawscript/` triggerable is the deployed app's authorization to run caller-supplied // `raw_code` hashing to it: pinning one hands a builder arbitrary code execution through an // app whose value passed the inline-script check. A composition-only app has none. @@ -2425,21 +2425,24 @@ fn checked_policy_runnable_paths(policy: &Policy) -> Result> std::iter::once(key.as_str()) .chain(key.match_indices(':').map(|(i, _)| &key[i + 1..])) .filter_map(|candidate| { - candidate - .strip_prefix("script/") - .map(|p| (false, p.to_string())) - .or_else(|| { - candidate - .strip_prefix("flow/") - .map(|p| (true, p.to_string())) - }) + [ + ("script/", AppRunnableKind::Script), + ("flow/", AppRunnableKind::Flow), + ("agent/", AppRunnableKind::Agent), + ] + .into_iter() + .find_map(|(prefix, kind)| { + candidate + .strip_prefix(prefix) + .map(|p| (kind, p.to_string())) + }) }) .collect::>() }) .collect()) } -fn refuse_hub_runnables(referenced: &[(bool, String)]) -> Result<()> { +fn refuse_hub_runnables(referenced: &[(AppRunnableKind, String)]) -> Result<()> { for (_, path) in referenced { if path.starts_with("hub/") { return Err(Error::PermissionDenied(format!( @@ -2494,7 +2497,7 @@ async fn require_runnables_readable( db: &DB, user_db: &UserDB, w_id: &str, - referenced: Vec<(bool, String)>, + referenced: Vec<(AppRunnableKind, String)>, ) -> Result<()> { if referenced.is_empty() { return Ok(()); @@ -2504,26 +2507,33 @@ async fn require_runnables_readable( // whichever version the caller happens to see. Resolved before the transaction opens so no // request holds two pool connections at once. let mut hashes = Vec::with_capacity(referenced.len()); - for (is_flow, path) in &referenced { - hashes.push(if *is_flow { - None - } else { + for (kind, path) in &referenced { + hashes.push(if *kind == AppRunnableKind::Script { windmill_common::get_latest_script_hash(db, path, w_id).await? + } else { + None }); } let mut tx = user_db.clone().begin(authed).await?; - for ((is_flow, path), hash) in referenced.into_iter().zip(hashes) { - let readable = if is_flow { - sqlx::query_scalar!( + for ((kind, path), hash) in referenced.into_iter().zip(hashes) { + let readable = match kind { + AppRunnableKind::Flow => sqlx::query_scalar!( "SELECT EXISTS(SELECT 1 FROM flow WHERE workspace_id = $1 AND path = $2)", w_id, path, ) .fetch_one(&mut *tx) .await? - .unwrap_or(false) - } else { - match hash { + .unwrap_or(false), + AppRunnableKind::Agent => sqlx::query_scalar!( + "SELECT EXISTS(SELECT 1 FROM resource WHERE workspace_id = $1 AND path = $2 AND resource_type = 'ai_agent')", + w_id, + path, + ) + .fetch_one(&mut *tx) + .await? + .unwrap_or(false), + AppRunnableKind::Script => match hash { Some(hash) => sqlx::query_scalar!( "SELECT EXISTS(SELECT 1 FROM script WHERE workspace_id = $1 AND hash = $2)", w_id, @@ -2533,17 +2543,21 @@ async fn require_runnables_readable( .await? .unwrap_or(false), None => false, - } + }, }; // One message for every cause: telling "no runnable version" apart from "not readable" // would reveal that a path the caller cannot read exists. if !readable { - return Err(Error::PermissionDenied(if is_flow { - format!("Flow {path} does not exist or is not readable by you") - } else { - format!( + return Err(Error::PermissionDenied(match kind { + AppRunnableKind::Flow => { + format!("Flow {path} does not exist or is not readable by you") + } + AppRunnableKind::Agent => { + format!("Agent {path} does not exist or is not readable by you") + } + AppRunnableKind::Script => format!( "Script {path} does not exist, is not readable by you, or has not finished deploying" - ) + ), })); } } @@ -4529,6 +4543,7 @@ async fn execute_component( let previews_hub = payload.path.as_deref().is_some_and(|p| { p.strip_prefix("script/") .or_else(|| p.strip_prefix("flow/")) + .or_else(|| p.strip_prefix("agent/")) .unwrap_or(p) .starts_with("hub/") }); @@ -4545,6 +4560,7 @@ async fn execute_component( let runnable_path = p .strip_prefix("script/") .or_else(|| p.strip_prefix("flow/")) + .or_else(|| p.strip_prefix("agent/")) .unwrap_or(p); require_path_read_access_for_preview(authed, &Some(runnable_path.to_string()))?; } @@ -4880,6 +4896,36 @@ async fn execute_component( // the app identity. Drop it; the legitimate value rides in `extra`. payload.args.remove("_TEMP_SCRIPT_REFS"); + // An agent runnable takes the message and its files, plus optionally the conversation they + // belong to, which names the agent's memory rather than an input of its run. Without it the + // run is a single one, as on the direct agent endpoint. + let agent_path = payload + .path + .as_deref() + .and_then(|p| p.strip_prefix("agent/")) + .map(str::to_string); + let agent_memory_id = if agent_path.is_some() { + let memory_id = payload + .args + .remove("memory_id") + .map(|v| { + serde_json::from_str::>(v.get()) + .map_err(|_| Error::BadRequest("memory_id must be a string".to_string())) + }) + .transpose()? + .flatten() + .filter(|m| !m.trim().is_empty()); + payload.args.retain(|k, _| { + matches!( + k.as_str(), + "user_message" | "user_attachments" | "previous_messages" + ) + }); + memory_id + } else { + None + }; + let (mut args, job_id) = build_args( policy, policy_triggerables, @@ -4918,9 +4964,64 @@ async fn execute_component( } .filter(|t| !t.is_empty()) }; + if agent_path.is_some() { + #[cfg(feature = "enterprise")] + crate::jobs::check_license_key_valid().await?; + let has_attachments = !crate::agent_runs::run_attachments(&args.args)?.is_empty(); + match opt_authed.as_ref() { + // The agent reads the files with the app's token: a viewer attaches only what they + // could download themselves. + Some(_viewer) if has_attachments => { + #[cfg(all(feature = "parquet", feature = "private"))] + crate::agent_runs::check_attachments_readable(&db, _viewer, &w_id, &args.args) + .await?; + } + None if has_attachments => { + return Err(Error::BadRequest( + "an anonymous viewer cannot send files to an agent".to_string(), + )); + } + _ => {} + } + } + // Filed under the viewer, who must therefore be kept past the push below. + let agent_viewer = agent_path.as_ref().and(opt_authed.clone()); + let mut agent_run_path = None; + let component = payload.component.clone(); let (job_payload, tag, _runnable_on_behalf_of) = match (payload.path, payload.raw_code, payload.id) { + (Some(_), None, None) if agent_path.is_some() => { + let agent = agent_path.as_deref().unwrap_or_default(); + // Read as the identity the app runs as: reading the agent is what allows running + // it. The app's identity is also the run's, never the agent's own on-behalf-of. + let run_as = windmill_api_auth::fetch_api_authed_from_permissioned_as( + permissioned_as.clone(), + email.clone(), + &w_id, + &db, + None, + ) + .await?; + let turn = crate::agent_runs::agent_turn( + &run_as, + &user_db, + &w_id, + agent, + agent_memory_id.is_some(), + ) + .await?; + agent_run_path = Some(turn.run_path.clone()); + ( + JobPayload::RawFlow { + value: turn.flow_value, + path: Some(turn.run_path), + restarted_from: None, + }, + None, + None, + ) + } // flow or script: (Some(path), None, None) => { get_payload_tag_from_prefixed_path(&path, &db, &w_id).await? @@ -5061,6 +5162,44 @@ async fn execute_component( .await?; } + if let (Some(run_path), Some(memory_id)) = (agent_run_path.as_deref(), agent_memory_id) { + // A signed-in viewer's turn is filed under them: nobody else may continue their + // conversation. An anonymous visitor's id is derived per app so that it can name no + // signed-in user's conversation (a bare UUID is used as is). Either way the + // conversation's row lock refuses a turn while one runs, and retention collects its + // memory with its messages. + let (owner, is_admin, memory_id) = match agent_viewer.as_ref() { + Some(viewer) => (viewer.username.clone(), viewer.is_admin, memory_id), + None => ( + windmill_common::flow_conversations::ANONYMOUS_CONVERSATION_OWNER.to_string(), + false, + windmill_common::flow_conversations::memory_key( + &w_id, + run_path, + &format!("anonymous:{path}:{memory_id}"), + ) + .to_string(), + ), + }; + let run_query = RunJobQuery { memory_id: Some(memory_id), ..Default::default() }; + if let Some(key) = run_query.memory_key(&w_id, run_path) { + crate::jobs::set_flow_memory_id(&mut tx, uuid, key).await?; + } + crate::jobs::file_chat_turn( + &mut tx, + &owner, + is_admin, + &w_id, + run_path, + &run_query, + args.args.get("user_message"), + uuid, + is_preview, + &args.args, + ) + .await?; + } + // Apply runnable query parameters if provided if let Some(ref run_query) = payload.run_query_params { if let Some(flow_path) = flow_path.as_deref() { @@ -6923,7 +7062,7 @@ mod embed_token_tests { #[cfg(test)] mod operator_app_tests { - use super::{check_operator_composed_app, BuilderAppWrite, Policy}; + use super::{check_operator_composed_app, AppRunnableKind, BuilderAppWrite, Policy}; use windmill_common::error::Result; use windmill_common::worker::to_raw_value; @@ -6941,7 +7080,10 @@ mod operator_app_tests { }}) } - fn composed_app(value: serde_json::Value, policy: &mut Policy) -> Result> { + fn composed_app( + value: serde_json::Value, + policy: &mut Policy, + ) -> Result> { let value = to_raw_value(&value); check_operator_composed_app(BuilderAppWrite::Deploy, true, Some(&value), Some(policy)) } @@ -6959,7 +7101,10 @@ mod operator_app_tests { let referenced = composed_app(clean.clone(), &mut policy).unwrap(); assert_eq!(policy.sandbox, Some(true)); // The runnables the policy authorizes are handed back for the caller to authorize. - assert_eq!(referenced, vec![(false, "f/x/s".to_string())]); + assert_eq!( + referenced, + vec![(AppRunnableKind::Script, "f/x/s".to_string())] + ); // A hub reference is unreviewed code, refused like it is in a flow. let mut policy = builder_policy( @@ -6996,12 +7141,20 @@ mod operator_app_tests { let mut policy = builder_policy(serde_json::json!({})); assert_eq!( composed_app(path_runnable("f/x/s", "script"), &mut policy).unwrap(), - vec![(false, "f/x/s".to_string())] + vec![(AppRunnableKind::Script, "f/x/s".to_string())] ); let mut policy = builder_policy(serde_json::json!({})); assert_eq!( composed_app(path_runnable("f/x/f", "flow"), &mut policy).unwrap(), - vec![(true, "f/x/f".to_string())] + vec![(AppRunnableKind::Flow, "f/x/f".to_string())] + ); + // An agent is reported from both surfaces, for the caller to check it can read it. + let mut policy = builder_policy( + serde_json::json!({"a:agent/f/x/bot": {"static_inputs": {}, "one_of_inputs": {}}}), + ); + assert_eq!( + composed_app(path_runnable("f/x/bot", "agent"), &mut policy).unwrap(), + vec![(AppRunnableKind::Agent, "f/x/bot".to_string())] ); let mut policy = builder_policy(serde_json::json!({})); assert!(composed_app(path_runnable("hub/1/x", "hubscript"), &mut policy).is_err()); @@ -7018,7 +7171,7 @@ mod operator_app_tests { ); assert_eq!( composed_app(clean.clone(), &mut policy).unwrap(), - vec![(false, "f/x/s".to_string())] + vec![(AppRunnableKind::Script, "f/x/s".to_string())] ); // `Viewer` mode makes `execute_component` accept any script/flow path, triggerables or diff --git a/backend/windmill-api/src/apps_raw_bundler.ts b/backend/windmill-api/src/apps_raw_bundler.ts index 965a9289b5..47460be3a6 100644 --- a/backend/windmill-api/src/apps_raw_bundler.ts +++ b/backend/windmill-api/src/apps_raw_bundler.ts @@ -135,7 +135,7 @@ export async function main( const nonEmpty = (v: unknown) => typeof v === 'string' && v.length > 0 // The prefixes `execute_component` resolves a run against; anything else is a // grant no run can match. - const RUN_TYPES = ['script', 'flow', 'hubscript'] + const RUN_TYPES = ['script', 'flow', 'hubscript', 'agent'] const malformed = Object.entries(runnables ?? {}) .filter(([, r]) => r != null) .filter(([, r]) => { diff --git a/backend/windmill-common/src/apps.rs b/backend/windmill-common/src/apps.rs index 4e25cec17d..9ce4326212 100644 --- a/backend/windmill-common/src/apps.rs +++ b/backend/windmill-common/src/apps.rs @@ -34,14 +34,23 @@ pub fn app_value_has_inline_script(value: &Value) -> bool { } } -/// Every workspace runnable the app value points a component at, as `(is_flow, path)`. +/// What a path runnable of an app runs. +#[derive(Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord)] +pub enum AppRunnableKind { + Script, + Flow, + /// A saved AI agent (an `ai_agent` resource), run as one chat turn. + Agent, +} + +/// Every workspace runnable the app value points a component at, as `(kind, path)`. /// /// This is what the deployed bundle actually asks `execute_component` to run: it resolves a /// `runnable_id` against the stored `runnables` and sends `{runType}/{path}`. The policy's /// triggerables are a separate surface, so both have to be authorized. Anything not plain text /// is refused, never skipped: the editor compares `type` loosely (`["path"] == "path"`) and /// interpolates `path` (`["u/x"]` → `u/x`), so a skipped entry still runs. -pub fn app_value_runnable_paths(value: &Value) -> error::Result> { +pub fn app_value_runnable_paths(value: &Value) -> error::Result> { let bad = |msg: String| error::Error::BadRequest(msg); let runnables = match value.get("runnables") { None | Some(Value::Null) => return Ok(vec![]), @@ -61,12 +70,13 @@ pub fn app_value_runnable_paths(value: &Value) -> error::Result true, - Some(Value::String(t)) if t == "script" || t == "hubscript" => false, + let kind = match runnable.get("runType") { + Some(Value::String(t)) if t == "flow" => AppRunnableKind::Flow, + Some(Value::String(t)) if t == "script" || t == "hubscript" => AppRunnableKind::Script, + Some(Value::String(t)) if t == "agent" => AppRunnableKind::Agent, _ => return Err(bad(format!("Unsupported runType for runnable {path}"))), }; - out.push((is_flow, path.clone())); + out.push((kind, path.clone())); } Ok(out) } diff --git a/backend/windmill-common/src/flow_conversations.rs b/backend/windmill-common/src/flow_conversations.rs index 571877ead2..12b3daf2ec 100644 --- a/backend/windmill-common/src/flow_conversations.rs +++ b/backend/windmill-common/src/flow_conversations.rs @@ -14,6 +14,11 @@ use crate::utils::truncate_with_ellipsis; /// Changing it detaches every memory stored under a string memory id. const MEMORY_ID_NAMESPACE: Uuid = Uuid::from_u128(0x6f1c2d4e_8a3b_5c7d_9e0f_1a2b3c4d5e6f); +/// Owns anonymous app visitors' agent conversations, as `created_by` names an anonymous app run. +/// It is also a valid username, so listing conversations leaves these out: a user of that name +/// would otherwise list every anonymous visitor's. +pub const ANONYMOUS_CONVERSATION_OWNER: &str = "anonymous"; + /// Memory is stored and carried in `flow_status.memory_id` as a uuid, which names the same memory /// wherever it is passed, as a chat conversation id must. Any other string names a memory through a /// name-based (v5) uuid scoped to its workspace and flow, so the same key in two flows or two diff --git a/chat-sdk/README.md b/chat-sdk/README.md index 006d5407d8..2106d35bee 100644 --- a/chat-sdk/README.md +++ b/chat-sdk/README.md @@ -148,6 +148,45 @@ const chat = useWindmillChat({ flowPath: 'f/support/assistant' }) - **`wmill app dev`**: there is no viewer session on the dev server, so pass `baseUrl`, `workspace` and `token` explicitly during development. +### A saved agent, through an agent runnable + +To chat with a saved AI agent (an `ai_agent` resource), give the app an **agent +runnable**: a path runnable of type `agent` naming the agent, picked under "AI agents" +in the editor, or in `raw_app.yaml`: + +```yaml +runnables: + chat: + type: agent + path: f/support/agent +``` + +Each call runs one turn of the agent's chat as the app, so a sandboxed app declares no +frontend SDK scope, viewers get no consent prompt and need no access to the agent. +Name the runnable and pass the app's generated `wmill.ts`, which the chat goes through: + +```tsx +import { useWindmillChat } from '@windmill-labs/chat-sdk/react' +import * as wmill from './wmill' + +const chat = useWindmillChat({ wmill, runnable: 'chat' }) +``` + +`runnable` is checked against the runnables `wmill.ts` declares, and names the local +history, so two agent runnables in one app keep separate conversations. Going through +`wmill.ts` is what makes the same code work in a deployed app and under `wmill app dev`. + +`createChat` and the assistant-ui runtime take the same options; the AI SDK transport +does not. The runnable takes `user_message` and `memory_id`, the conversation id, which +the agent's memory is keyed by, so it remembers earlier turns (called from `wmill.ts` +without `memory_id`, it is a single run with no conversation). The agent needs managed +memory: one whose memory is off takes `previous_messages` instead, the history you keep. A signed-in viewer's turns +are filed under them, so nobody else can continue their conversations; an anonymous +visitor's are kept apart per app. The chat makes no request of its own: history is +`'local'` (the app's own storage), `stop()` stops following without cancelling the turn, +a message sent while it still runs is refused with `TurnRunningError`, and attachments +are refused. + ## Any framework `createChat` returns a store: `subscribe` calls the listener immediately and on every @@ -178,7 +217,7 @@ await chat.sendMessage('Hello') | Option | | |---|---| -| `flowPath` | Path of the deployed flow, e.g. `f/support/assistant`. Required. | +| `flowPath` | Path of the deployed flow, e.g. `f/support/assistant`. Required unless `follow` is set. | | `baseUrl` | The Windmill origin. Detected inside a raw app. | | `workspace` | Detected inside a raw app. | | `token` | A token, or a function returning one (called before every request, so it can fetch a short-lived token from your backend). Omit it inside a raw app. | @@ -190,6 +229,8 @@ await chat.sendMessage('Hello') | `pollDelayMs` | How often, in ms, the server polls a running turn for the stream (Enterprise; 50 at the fastest, other servers ignore it). Unset, the server relaxes from 100 ms to 3 s over a long turn; set it when tokens must keep flowing at that pace. | | `onFinish`, `onError` | Called when a turn has its answer, or could not run at all. | | `run` | Runs the flow for a turn yourself and returns the job id, instead of the deployed flow at `flowPath` (Windmill's editor chats with an undeployed flow through a preview run this way). Pass `memory_id` = the conversation id. | +| `follow` | With `run`: follows the job through your own client, calling `onStream` with each chunk of its result stream and resolving with its result. The chat then makes no request of its own; `runnable` sets both for a raw app's agent runnable, see [A saved agent](#a-saved-agent-through-an-agent-runnable). | +| `runnable`, `wmill` | In a raw app, chat through the agent runnable of that name, going through the app's generated module (`import * as wmill from './wmill'`). Replaces `flowPath`, `run` and `follow`. | ## State @@ -294,7 +335,8 @@ That store is keyed by the **Windmill user**, so it fits a viewer session or a t issued per user. With one token shared by every visitor of a site, all visitors would see each other's conversations. For that setup use `history: 'local'` (the default with an explicit `token`): the conversation list and messages stay in the browser's -`localStorage`, per Windmill instance, workspace and flow. `'none'` keeps nothing +`localStorage`, per Windmill instance, workspace and flow, and the chat starts with that +list in `state.conversations`, without a `loadConversations()` call. `'none'` keeps nothing beyond the page. When the default `'server'` mode turns out unreadable (a token or sandboxed app @@ -333,7 +375,8 @@ custom integrations. When asked to add a chat over a Windmill flow: the flow must be deployed with chat mode on. Pick the entry point from the table at the top (`useChat` → `@windmill-labs/chat-sdk/ai-sdk`, assistant-ui → `@windmill-labs/chat-sdk/assistant-ui`, otherwise `@windmill-labs/chat-sdk/react`). Inside a -Windmill raw app pass only `flowPath`. Elsewhere pass `baseUrl`, `workspace` and a +Windmill raw app pass only `flowPath`; for a saved agent, use an `agent` runnable and pass +`{ wmill, runnable }`. Elsewhere pass `baseUrl`, `workspace` and a `token`; for a public page use a token scoped to `jobs:run:flows:` and leave `history` at its default. Render `role`, `content`, `pending`, `success` and `tool.status`; never build the SSE handling yourself. diff --git a/chat-sdk/src/assistant-ui.ts b/chat-sdk/src/assistant-ui.ts index b9a76db679..04ab23793e 100644 --- a/chat-sdk/src/assistant-ui.ts +++ b/chat-sdk/src/assistant-ui.ts @@ -7,7 +7,7 @@ import { } from '@assistant-ui/react' import { useEffect, useMemo } from 'react' import { useWindmillChat } from './react' -import type { ChatMessage, ChatOptions } from './types' +import type { ChatMessage, RawAppWmill, TypedChatOptions } from './types' import { parseJsonOr } from './utils' /** One assistant turn: the assistant and tool rows between two user messages. */ @@ -17,7 +17,7 @@ export interface WindmillTurn { messages: ChatMessage[] } -export interface WindmillRuntimeOptions extends ChatOptions { +export type WindmillRuntimeOptions = TypedChatOptions & { /** Load the conversation list on mount so `ThreadListPrimitive` has something to show. Default true. */ threadList?: boolean } @@ -27,7 +27,7 @@ export interface WindmillRuntimeOptions extends ChatOptions { * Conversations become threads, so the thread list primitives switch, create and * delete Windmill conversations. */ -export function useWindmillRuntime(options: WindmillRuntimeOptions): AssistantRuntime { +export function useWindmillRuntime(options: WindmillRuntimeOptions): AssistantRuntime { const chat = useWindmillChat(options) const turns = useMemo(() => groupTurns(chat.messages), [chat.messages]) const withThreadList = options.threadList !== false diff --git a/chat-sdk/src/attachments.ts b/chat-sdk/src/attachments.ts index ac5e71c7d3..a11b0106ba 100644 --- a/chat-sdk/src/attachments.ts +++ b/chat-sdk/src/attachments.ts @@ -88,10 +88,9 @@ export async function uploadAttachments( attachments.map(async (attachment, index) => { try { const blob = attachmentBlob(attachment) - const filename = storedAttachmentName( - attachment.name || `attachment-${index + 1}`, - blob.type - ) + // The last segment only: the name may come from whoever sent the file. + const name = attachment.name?.split(/[\\/]/).pop()?.replace(/^\.+$/, '') + const filename = storedAttachmentName(name || `attachment-${index + 1}`, blob.type) const { file_key } = await api.uploadFile(`${prefix}/${index}/${filename}`, blob, { contentType: blob.type, signal: batch.signal diff --git a/chat-sdk/src/chat.ts b/chat-sdk/src/chat.ts index dfef438f33..2d48d383a0 100644 --- a/chat-sdk/src/chat.ts +++ b/chat-sdk/src/chat.ts @@ -11,11 +11,13 @@ import { resolveConfig, type ResolvedConfig } from './config' import { followJob } from './follow' import { createLocalHistory, type LocalHistory } from './history' import { uploadAttachments } from './attachments' -import type { AgentStreamEvent } from './stream' +import { createStreamEventParser, type AgentStreamEvent } from './stream' import type { Chat, ChatMessage, ChatOptions, + RawAppWmill, + TypedChatOptions, ChatState, Conversation, SendMessageOptions, @@ -62,7 +64,7 @@ interface Turn { streamedText: boolean } -export function createChat(options: ChatOptions): Chat { +export function createChat(options: TypedChatOptions): Chat { return new ChatImpl(options) } @@ -102,7 +104,9 @@ class ChatImpl implements Chat { messages: [], status: 'idle', error: undefined, - conversations: [], + // Browser history is read synchronously, so a chat starts with its list as server history + // cannot: an app that never calls `loadConversations` still shows its conversations. + conversations: this.#config.history === 'local' ? this.#local.listConversations() : [], history: this.#config.history, loadingMessages: false, hasMoreMessages: false @@ -134,6 +138,9 @@ class ChatImpl implements Chat { if (attachments.length > 0 && !attachmentsInput) { throw new Error('windmill-chat: attachments need `attachmentsInput`, the flow input that takes them') } + if (attachments.length > 0 && this.#config.follow) { + throw new Error('windmill-chat: attachments upload with the SDK\'s own requests, which `follow` replaces') + } if (attachmentsInput && !attachmentsInput.multiple && attachments.length > 1) { // Uploading all of them would run with the first and leave the rest stranded in storage. throw new Error(`windmill-chat: \`${attachmentsInput.name}\` holds one file; got ${attachments.length}`) @@ -338,7 +345,7 @@ class ChatImpl implements Chat { this.#set({ messages: finalized(this.#state.messages), status: 'idle' }) this.#persistLocal() } - if (turn.jobId) { + if (turn.jobId && !this.#config.follow) { // Needs `jobs:write` on a token; the stream is closed either way. await this.#api.cancelJob(turn.jobId).catch(() => {}) } @@ -577,6 +584,27 @@ class ChatImpl implements Chat { async #follow(turn: Turn, onStreamStart: () => void): Promise { let started = false + const { follow } = this.#config + if (follow) { + const { signal } = turn.controller + const parser = createStreamEventParser() + const onStream = (chunk: string) => { + if (signal.aborted) return + if (!started) { + started = true + onStreamStart() + } + this.#applyEvents(turn, parser.push(chunk)) + } + // A caller's follow may not honor the signal; the turn stops waiting on it either way. + const aborted = new Promise((_, reject) => { + if (signal.aborted) reject(abortError()) + signal.addEventListener('abort', () => reject(abortError()), { once: true }) + }) + const result = await Promise.race([follow(turn.jobId!, onStream, { signal }), aborted]) + this.#applyEvents(turn, parser.flush()) + return result + } for await (const event of followJob(this.#api, turn.jobId!, { signal: turn.controller.signal })) { if (event.type === 'completed') { if (event.streamLost) this.#dropCutRound(turn) @@ -722,8 +750,9 @@ class ChatImpl implements Chat { let messages = this.#state.messages let failed = false if (isErrorResult(result)) { - // The envelope is also a legitimate result shape; the job's own status decides. - failed = await this.#api + // The envelope is also a legitimate result shape; the job's own status decides, when it + // can be read. + failed = this.#config.follow ? true : await this.#api .getCompletedResult(turn.jobId!, turn.controller.signal) .then((r) => r.success === false) .catch(() => true) diff --git a/chat-sdk/src/config.ts b/chat-sdk/src/config.ts index 1903149d58..98cf97f2b2 100644 --- a/chat-sdk/src/config.ts +++ b/chat-sdk/src/config.ts @@ -1,4 +1,5 @@ import type { ChatOptions, FetchLike, HistoryMode, StorageLike, TokenSource } from './types' +import { rawAppRunnable } from './runnable' export interface ResolvedConfig { flowPath: string @@ -15,6 +16,7 @@ export interface ResolvedConfig { pageSize: number pollDelayMs: number | undefined run: ChatOptions['run'] + follow: ChatOptions['follow'] onFinish: ChatOptions['onFinish'] onError: ChatOptions['onError'] } @@ -51,13 +53,31 @@ export function detectRawApp(): RawAppContext | undefined { } export function resolveConfig(options: ChatOptions): ResolvedConfig { - if (!options.flowPath) throw new Error('windmill-chat: flowPath is required') + if (options.runnable !== undefined) { + if (!options.wmill) throw new Error('windmill-chat: `runnable` needs `wmill`, the app\'s generated module') + if (options.flowPath || options.run || options.follow) { + throw new Error('windmill-chat: `runnable` replaces `flowPath`, `run` and `follow`') + } + // Unsandboxed apps share their origin's storage, so the app's path keeps their histories apart. + const app = (globalThis as { ctx?: { path?: unknown } }).ctx?.path + options = { + ...options, + ...rawAppRunnable(options.wmill.backendAsync[options.runnable], options.wmill.streamJob), + flowPath: `runnable:${typeof app === 'string' ? `${app}:` : ''}${options.runnable}` + } + } + const follow = options.follow + if (follow && !options.run) throw new Error('windmill-chat: `follow` needs `run`, which starts the job it follows') + if (follow && options.history === 'server') { + throw new Error('windmill-chat: server history is read with the SDK\'s own requests, which `follow` replaces') + } + if (!options.flowPath && !follow) throw new Error('windmill-chat: flowPath is required') const explicitToken = options.token !== undefined const detected = options.baseUrl && options.workspace && explicitToken ? undefined : detectRawApp() const baseUrl = options.baseUrl ?? detected?.baseUrl const workspace = options.workspace ?? detected?.workspace - if (!baseUrl || !workspace) { + if ((!baseUrl || !workspace) && !follow) { throw new Error( 'windmill-chat: pass baseUrl and workspace. They are only detected inside a raw app: an unsandboxed one on the Windmill origin, or a sandboxed one whose policy declares frontend SDK scopes.' ) @@ -66,11 +86,11 @@ export function resolveConfig(options: ChatOptions): ResolvedConfig { const token = options.token ?? (options.baseUrl === undefined || options.baseUrl === detected?.baseUrl ? detected?.token : undefined) return { - flowPath: options.flowPath, - baseUrl, - workspace, + flowPath: options.flowPath ?? '', + baseUrl: baseUrl ?? '', + workspace: workspace ?? '', token, - history: options.history ?? (explicitToken ? 'local' : 'server'), + history: options.history ?? (explicitToken || follow ? 'local' : 'server'), historyExplicit: options.history !== undefined, inputs: options.inputs ?? {}, fetch: options.fetch, @@ -79,6 +99,7 @@ export function resolveConfig(options: ChatOptions): ResolvedConfig { pageSize: options.pageSize ?? 50, pollDelayMs: options.pollDelayMs, run: options.run, + follow, onFinish: options.onFinish, onError: options.onError } diff --git a/chat-sdk/src/index.ts b/chat-sdk/src/index.ts index a050edce37..d05a5ab217 100644 --- a/chat-sdk/src/index.ts +++ b/chat-sdk/src/index.ts @@ -31,6 +31,8 @@ export type { Conversation, FetchLike, HistoryMode, + RawAppWmill, + TypedChatOptions, SendMessageOptions, StorageLike, TokenSource, diff --git a/chat-sdk/src/react.ts b/chat-sdk/src/react.ts index c32324ee18..4159008ce7 100644 --- a/chat-sdk/src/react.ts +++ b/chat-sdk/src/react.ts @@ -1,6 +1,6 @@ import { useEffect, useMemo, useRef, useSyncExternalStore } from 'react' import { createChat } from './chat' -import type { Chat, ChatOptions, ChatState } from './types' +import type { Chat, ChatOptions, ChatState, RawAppWmill, TypedChatOptions } from './types' export type UseWindmillChat = ChatState & Pick< @@ -19,22 +19,23 @@ export type UseWindmillChat = ChatState & /** * A chat on a chat-mode flow. The chat is created once per `flowPath`, `baseUrl`, - * `workspace`, `history`, `storageKey`, credential and presence of `run`, and + * `workspace`, `history`, `storageKey`, credential and presence of `run` and `follow`, and * destroyed on unmount. A credential change is a new user, whose chat must not * carry the previous one's state: a different token string, or a switch between no * token, a token string and a token function, all recreate it. A token function is * read through a ref on every call, so a new closure per render changes what the * next call runs and nothing else; pass a `storageKey` per user when local history - * must not be shared. `run`, the callbacks and `inputs` are read the same way: the + * must not be shared. `run`, `follow`, the callbacks and `inputs` are read the same way: the * latest render's values go with the next message. */ -export function useWindmillChat(options: ChatOptions): UseWindmillChat { - const latest = useRef(options) +export function useWindmillChat(options: TypedChatOptions): UseWindmillChat { + const latest = useRef(options) latest.current = options const credential = typeof options.token === 'function' ? 'fn' : typeof options.token === 'string' ? `str:${options.token}` : 'none' // A custom runner replaces the deployed flow call, so its presence is part of what the chat is. const customRun = options.run !== undefined + const customFollow = options.follow !== undefined const chat = useMemo( () => createChat({ @@ -49,11 +50,14 @@ export function useWindmillChat(options: ChatOptions): UseWindmillChat { } : options.token, run: customRun ? (args, turn) => (latest.current.run ?? options.run!)(args, turn) : undefined, + follow: customFollow + ? (jobId, onStream, turn) => (latest.current.follow ?? options.follow!)(jobId, onStream, turn) + : undefined, onFinish: (turn) => latest.current.onFinish?.(turn), onError: (error, turn) => latest.current.onError?.(error, turn) }), // eslint-disable-next-line react-hooks/exhaustive-deps - [options.flowPath, options.baseUrl, options.workspace, options.history, options.storageKey, credential, customRun] + [options.flowPath, options.runnable, options.wmill, options.baseUrl, options.workspace, options.history, options.storageKey, credential, customRun, customFollow] ) useEffect(() => () => chat.destroy(), [chat]) const state = useSyncExternalStore(chat.subscribe, chat.getState, chat.getState) diff --git a/chat-sdk/src/runnable.ts b/chat-sdk/src/runnable.ts new file mode 100644 index 0000000000..08ddf592e7 --- /dev/null +++ b/chat-sdk/src/runnable.ts @@ -0,0 +1,29 @@ +import { turnRunningError } from './api' +import type { ChatOptions } from './types' + +/** + * `run` and `follow` over a raw app's agent runnable (a path runnable of type `agent`, naming + * a saved AI agent), behind the `runnable` option. `memory_id` is the conversation id, the + * agent's memory key. + */ +export function rawAppRunnable( + // The generated signature is the runnable's own. + start: (args: any) => Promise, + streamJob: (jobId: string, onUpdate?: (update: { new_result_stream?: string }) => void) => Promise +): Required> { + return { + run: async (args, { conversationId }) => { + try { + return String(await start({ ...args, memory_id: conversationId })) + } catch (e) { + // The host relays the server's refusal as the error's text, possibly behind a stack. + const text = e instanceof Error ? e.message : String(e) + throw turnRunningError(text.slice(text.indexOf('{'), text.lastIndexOf('}') + 1)) ?? e + } + }, + follow: (jobId, onStream) => + streamJob(jobId, (update) => { + if (update.new_result_stream) onStream(update.new_result_stream) + }) + } +} diff --git a/chat-sdk/src/types.ts b/chat-sdk/src/types.ts index 180238e50c..ad9f30cbfb 100644 --- a/chat-sdk/src/types.ts +++ b/chat-sdk/src/types.ts @@ -88,9 +88,33 @@ export type StorageLike = Pick export type FetchLike = (input: string | URL | Request, init?: RequestInit) => Promise +/** A raw app's generated `wmill.ts`, as far as a chat uses it. */ +export interface RawAppWmill { + backendAsync: Record Promise> + streamJob: (jobId: string, onUpdate?: (update: { new_result_stream?: string }) => void) => Promise +} + +/** `ChatOptions` whose `runnable` must name one of `wmill`'s runnables. */ +export type TypedChatOptions = ChatOptions & { + wmill?: W + runnable?: keyof W['backendAsync'] & string +} + export interface ChatOptions { - /** Path of a deployed flow with chat mode enabled, e.g. `f/support/assistant`. */ - flowPath: string + /** + * Path of a deployed flow with chat mode enabled, e.g. `f/support/assistant`. Required + * unless `runnable` or `follow` is set, where it only names the local history. + */ + flowPath?: string + /** + * In a raw app, chat through its agent runnable of this name instead of a flow: + * `{ wmill, runnable: 'chat' }`. Each turn runs as the app, so a sandboxed app needs no + * frontend SDK scope and its viewers no access to the agent; the chat follows it as + * with `follow`. It also names the local history, so each runnable keeps its own. + */ + runnable?: string + /** The app's generated module, `import * as wmill from './wmill'`, which `runnable` goes through. */ + wmill?: RawAppWmill /** Windmill origin, e.g. `https://app.windmill.dev`. Detected inside a raw app. */ baseUrl?: string /** Detected inside a raw app. */ @@ -128,6 +152,15 @@ export interface ChatOptions { * Windmill's own editor uses this to chat with an undeployed flow through a preview run. */ run?: (args: Record, turn: { conversationId: string; signal: AbortSignal }) => Promise + /** + * Follows the job `run` returned through your own client instead of the SDK's requests: + * call `onStream` with each chunk of the job's result stream and resolve with its result. + * With it the chat makes no request of its own, so it needs no credential, `baseUrl` or + * `workspace`, and in exchange history is `local` (or `none`), `stop()` stops following + * without cancelling the job, and attachments are refused. `runnable` sets both for a raw + * app's agent runnable. + */ + follow?: (jobId: string, onStream: (chunk: string) => void, turn: { signal: AbortSignal }) => Promise /** Called once a turn has its answer (a failed flow included: its error is the answer). */ onFinish?: (turn: { conversationId: string; jobId?: string; messages: ChatMessage[] }) => void /** Called when a turn could not run or be followed; `state.error` holds the same error. */ diff --git a/chat-sdk/test/chat.test.ts b/chat-sdk/test/chat.test.ts index 3845c91ac7..6c88af5a36 100644 --- a/chat-sdk/test/chat.test.ts +++ b/chat-sdk/test/chat.test.ts @@ -1829,6 +1829,7 @@ describe('createChat with server history', () => { await chat.selectConversation(older) await new Promise((r) => setTimeout(r, 400)) const again = createChat(options({ token: 'tok', storage }, fetch)) + expect(again.getState().conversations.map((c) => c.id)).toEqual([newer, older]) expect((await again.loadConversations()).map((c) => c.id)).toEqual([newer, older]) }) diff --git a/chat-sdk/test/runnable.test.ts b/chat-sdk/test/runnable.test.ts new file mode 100644 index 0000000000..4dcaa60b84 --- /dev/null +++ b/chat-sdk/test/runnable.test.ts @@ -0,0 +1,68 @@ +import { describe, expect, test } from 'bun:test' +import { TurnRunningError } from '../src/api' +import { createChat } from '../src/chat' +import { memoryStorage, ndjson } from './support' + +describe('raw app agent runnable', () => { + test('chats through the runnable without a request of its own', async () => { + const calls: unknown[] = [] + let refuse = false + const start = async (args: unknown) => { + // As the host relays a 409: the server's body behind the error's stack. + if (refuse) { + const body = { error: 'still answering', running_turn: { job_id: 'job-1', user_seq: 1 } } + throw new Error(`Error: ${JSON.stringify(body)}\n at x`) + } + calls.push(args) + return `job-${calls.length}` + } + const streamJob = async (jobId: string, onUpdate?: (u: { new_result_stream?: string }) => void) => { + const stream = ndjson({ type: 'token_delta', content: 'answer ' }, { type: 'token_delta', content: jobId }) + onUpdate?.({ new_result_stream: stream.slice(0, 10) }) + onUpdate?.({ new_result_stream: stream.slice(10) }) + if (jobId === 'job-3') return { error: { name: 'Error', message: 'agent failed' } } + return { output: `answer ${jobId}`, messages: [] } + } + const storage = memoryStorage() + const wmill = { backendAsync: { chat: start, other: start }, streamJob } + const chat = createChat({ + wmill, + runnable: 'chat', + storage, + fetch: () => Promise.reject(new Error('no request expected')) + }) + expect(chat.getState().history).toBe('local') + await chat.sendMessage('hi') + await chat.sendMessage('again') + const conversationId = chat.getState().conversationId + expect(calls).toEqual([ + { user_message: 'hi', memory_id: conversationId }, + { user_message: 'again', memory_id: conversationId } + ]) + expect(chat.getState().messages.map((m) => m.content)).toEqual(['hi', 'answer job-1', 'again', 'answer job-2']) + + await chat.sendMessage('third') + const messages = chat.getState().messages + const last = messages[messages.length - 1] + expect([last.content, last.success]).toEqual(['agent failed', false]) + + // A turn still running elsewhere takes the message back rather than failing a turn. + refuse = true + await expect(chat.sendMessage('fourth')).rejects.toBeInstanceOf(TurnRunningError) + expect(chat.getState().messages).toEqual(messages) + + // Each runnable keeps its own history: another agent never sees this conversation. + expect(createChat({ wmill, runnable: 'chat', storage }).getState().conversations).toHaveLength(1) + expect(createChat({ wmill, runnable: 'other', storage }).getState().conversations).toEqual([]) + // Nor does the same runnable name in another app, whose path the host puts in `ctx`. + const g = globalThis as { ctx?: unknown } + g.ctx = { path: 'u/admin/other_app' } + try { + expect(createChat({ wmill, runnable: 'chat', storage }).getState().conversations).toEqual([]) + } finally { + delete g.ctx + } + // @ts-expect-error a name the app's module does not declare + createChat({ wmill, runnable: 'missing', storage }) + }) +}) diff --git a/cli/src/commands/app/dev.ts b/cli/src/commands/app/dev.ts index fbeba2914f..eb9bb055d6 100644 --- a/cli/src/commands/app/dev.ts +++ b/cli/src/commands/app/dev.ts @@ -1711,6 +1711,49 @@ export function buildWmillTs( return windmillUtils.genWmillTs(runnables); } +/** + * The inputs of an agent runnable, after the agent's memory (see the frontend's + * `agentRunnableSchema`): `memory_id` when it keeps managed memory, `previous_messages` when it + * is off, and neither for an older `manual` setting. + */ +function agentRunnableSchema(memory: any) { + const managed = + memory?.kind === "compaction" || + ((memory?.kind === "window" || memory?.kind === "auto") && + Boolean(memory.context_length)); + const legacy = memory?.kind === "manual"; + const history: Record = {}; + if (managed) { + history.memory_id = { + type: "string", + description: + "The conversation the message belongs to: reuse it to continue one, or omit it for a single run", + }; + } else if (!legacy) { + history.previous_messages = { + type: "array", + description: + "The history to answer from, as chat messages: this agent keeps no memory", + items: { type: "object" }, + }; + } + return { + $schema: "https://json-schema.org/draft/2020-12/schema", + type: "object", + properties: { + user_message: { type: "string", description: "The message to answer" }, + ...history, + user_attachments: { + type: "array", + description: "Files the message carries, as object storage references", + items: { type: "object", resourceType: "s3object" }, + }, + }, + required: ["user_message"], + order: ["user_message", ...Object.keys(history), "user_attachments"], + }; +} + /** * Fetches schemas from the Windmill API for path-based runnables (script / flow). * Returns a map of runnableId -> schema. Path-based runnables don't store their @@ -1741,6 +1784,12 @@ export async function fetchPathRunnableSchemas( path: runnable.path, }); if (script.schema) schemas[runnableId] = script.schema; + } else if (runnable.runType === "agent") { + const agent = await wmill.getResource({ + workspace: workspaceId, + path: runnable.path, + }); + schemas[runnableId] = agentRunnableSchema((agent.value as any)?.memory); } else if (runnable.runType === "flow") { const flow = await wmill.getFlowByPath({ workspace: workspaceId, @@ -1771,7 +1820,8 @@ function convertRunnablesToApiFormat(runnables: Record): void { if ( runnable?.type === "script" || runnable?.type === "hubscript" || - runnable?.type === "flow" + runnable?.type === "flow" || + runnable?.type === "agent" ) { // Convert from file format to API format // { type: "script" } -> { type: "path", runType: "script" } @@ -2029,9 +2079,11 @@ async function streamJobWithSSE( ws: WebSocket, baseUrl: string, token: string, + streamOffset?: number, ): Promise { const sseUrl = - `${baseUrl}api/w/${workspace}/jobs_u/getupdate_sse/${jobId}?fast=true`; + `${baseUrl}api/w/${workspace}/jobs_u/getupdate_sse/${jobId}?fast=true` + + (streamOffset !== undefined ? `&stream_offset=${streamOffset}` : ""); const extraHeaders = getHeaders(); const response = await fetch(sseUrl, { @@ -2057,6 +2109,7 @@ async function streamJobWithSSE( const decoder = new TextDecoder(); let buffer = ""; + let lastOffset = streamOffset; try { while (true) { @@ -2076,8 +2129,17 @@ async function streamJobWithSSE( if (type === "ping" || type === "timeout") { if (type === "timeout") { + // The server ends every stream after a while; the job goes on, so it is resumed. reader.cancel(); - return; + return streamJobWithSSE( + workspace, + jobId, + reqId, + ws, + baseUrl, + token, + lastOffset, + ); } continue; } @@ -2108,6 +2170,9 @@ async function streamJobWithSSE( return; } + if (update.stream_offset !== undefined) { + lastOffset = update.stream_offset; + } // Send stream update if there's new stream data if (update.new_result_stream !== undefined) { ws.send( diff --git a/cli/src/commands/app/metadata.ts b/cli/src/commands/app/metadata.ts index 1981949404..5f16e84220 100644 --- a/cli/src/commands/app/metadata.ts +++ b/cli/src/commands/app/metadata.ts @@ -14,7 +14,7 @@ export type Runnable = } | { type: "path" | "runnableByPath"; - runType: "script" | "hubscript" | "flow"; + runType: "script" | "hubscript" | "flow" | "agent"; path: string; fields?: Record; }; diff --git a/cli/src/commands/app/raw_apps.ts b/cli/src/commands/app/raw_apps.ts index ec526f2af2..570b03f949 100644 --- a/cli/src/commands/app/raw_apps.ts +++ b/cli/src/commands/app/raw_apps.ts @@ -230,7 +230,8 @@ export async function loadRunnablesFromBackend( } else if ( runnable?.type === "script" || runnable?.type === "hubscript" || - runnable?.type === "flow" + runnable?.type === "flow" || + runnable?.type === "agent" ) { // For path-based runnables, convert from file format to API format // { type: "script" } -> { type: "path", runType: "script" } diff --git a/cli/src/guidance/skills.gen.ts b/cli/src/guidance/skills.gen.ts index 7d0cf5c1cc..d932957a20 100644 --- a/cli/src/guidance/skills.gen.ts +++ b/cli/src/guidance/skills.gen.ts @@ -6839,7 +6839,7 @@ An app can be demoed by recording a session: every interaction becomes a step ca Apply it to customer data, internal notes and anything else a viewer of the demo should not see. It costs nothing when the app is never recorded. -### Chat UIs over a flow in chat mode +### Chat UIs over a flow in chat mode or a saved agent A flow deployed with chat mode on is a chat backend (streaming answer, tool calls, memory, conversation history). Do not drive it through a runnable: add \`@windmill-labs/chat-sdk\` to \`package.json\` and use it directly, it detects the app's Windmill and credential. @@ -6852,9 +6852,31 @@ const chat = useWindmillChat({ flowPath: 'f/support/assistant' }) \`@windmill-labs/chat-sdk/ai-sdk\` gives a \`ChatTransport\` for the Vercel AI SDK's \`useChat\`, \`@windmill-labs/chat-sdk/assistant-ui\` a runtime for assistant-ui. The flow must be deployed, not a draft. A sandboxed app needs \`jobs:run\` in its frontend SDK scopes, plus \`flow_conversations:write\` for the conversation sidebar; without them the chat keeps history in the browser. +#### A saved agent: an \`agent\` runnable + +Chat with a saved AI agent (an \`ai_agent\` resource) through an \`agent\` runnable, a path runnable whose \`path\` is the agent's. Each call runs one turn of the agent's chat as the app, so a sandboxed app declares no frontend SDK scope, viewers get no consent prompt and need no access to the agent. Prefer it in every app, sandboxed or not: + +In \`raw_app.yaml\`: + +\`\`\`yaml +runnables: + chat: + type: agent + path: f/support/agent +\`\`\` + +\`\`\`tsx +import { useWindmillChat } from '@windmill-labs/chat-sdk/react' +import * as wmill from './wmill' + +const chat = useWindmillChat({ wmill, runnable: 'chat' }) +\`\`\` + +\`runnable\` is the runnable's key; each runnable keeps its own conversations. The runnable takes \`user_message\` and \`memory_id\`, the conversation id, which the chat fills in: the agent remembers earlier turns of a conversation, and a signed-in viewer's conversations are theirs alone. Called directly as \`backend.({ user_message })\` without \`memory_id\`, it is a single run with no conversation, for a one-off question. The chat needs an agent with managed memory. An agent whose memory is off keeps none: its runnable takes \`previous_messages\` (chat messages) instead of \`memory_id\`, and the app keeps the history itself. The chat keeps its transcript in the app's own storage. \`chat.stop()\` stops following without cancelling the turn, and a message sent while it still runs is refused with \`TurnRunningError\`. Attachments are not supported this way. + ## Backend runnables -Each runnable has a unique key (used to call it from the frontend) and one of four types: +Each runnable has a unique key (used to call it from the frontend) and one of five types: | Type | What it is | |---|---| @@ -6862,6 +6884,7 @@ Each runnable has a unique key (used to call it from the frontend) and one of fo | \`script\` | Reference to an existing workspace script by path. | | \`flow\` | Reference to an existing workspace flow by path. | | \`hubscript\` | Reference to a hub script by path. | +| \`agent\` | Reference to a saved AI agent by path. Each call is one turn of its chat (see the chat section above). | ### Inline runnables @@ -6903,9 +6926,9 @@ An inline runnable runs as an ordinary Windmill job. \`import * as wmill from 'w Use only \`wmill\` functions the SDK actually exports; for an endpoint none of them covers, the generated service classes (\`JobService\`, \`ScriptService\`, ...) are importable from \`windmill-client\`. What is not available is a name you guessed at: \`getBaseUrl\` and \`getWorkspaceToken\` are inventions, not API. -### Path runnables (script / flow / hubscript) +### Path runnables (script / flow / hubscript / agent) -When \`type\` is \`script\`, \`flow\`, or \`hubscript\`, the runnable just stores a \`path\` to an existing workspace or hub item — no inline code. The referenced item's input/output schema becomes the runnable's surface. +When \`type\` is \`script\`, \`flow\`, \`hubscript\` or \`agent\`, the runnable just stores a \`path\` to an existing workspace or hub item — no inline code. The referenced item's input/output schema becomes the runnable's surface; an agent's is \`user_message\` and \`user_attachments\`, plus \`memory_id\` when the agent keeps managed memory or \`previous_messages\` when its memory is off. Before writing an inline runnable, look for a workspace script or flow that already does the job, or a Hub script (a prebuilt integration at \`hub///\`) for a third-party service, and reference it instead of copying its logic. @@ -6914,7 +6937,7 @@ Before writing an inline runnable, look for a workspace script or flow that alre This decides whether an app works before anything is deployed: - **Inline runnables run the app's current code.** The editor sends the runnable's source with each request, so an inline runnable works in the preview with nothing deployed. -- **Path runnables (\`script\` / \`flow\` / \`hubscript\`) run the DEPLOYED item at that path.** So do \`wmill.runFlow\`, \`wmill.runFlowAsync\` and \`wmill.runScriptByPath\` called from inside a runnable. A draft — including a draft you just created — does not exist for them. +- **Path runnables (\`script\` / \`flow\` / \`hubscript\` / \`agent\`) run the DEPLOYED item at that path.** So do \`wmill.runFlow\`, \`wmill.runFlowAsync\` and \`wmill.runScriptByPath\` called from inside a runnable. A draft — including a draft you just created — does not exist for them. So an app wired to a flow you just wrote does nothing until **that flow is deployed**. The app itself does NOT have to be deployed for this: the preview runs the app's draft, so the referenced flow is the only thing that has to exist deployed. @@ -7024,7 +7047,7 @@ def main(user_id: str): 7. **Mark sensitive UI with \`data-wm-no-record\`** — it is what keeps that data out of a recorded demo; passwords are handled for you. 8. **Reach for \`backendAsync\` + \`waitJob\`** for long work — never a hand-written job-polling runnable. 9. **Deploy what a path runnable points at** — a path runnable aimed at a draft fails at runtime; tell the user what needs deploying. -10. **Use \`@windmill-labs/chat-sdk\` for a chat over a chat-mode flow** — never a runnable that runs the flow and polls its stream. +10. **Use \`@windmill-labs/chat-sdk\` for a chat over a chat-mode flow or a saved agent** — never hand-roll a runnable that runs it and polls its stream; a saved agent is an \`agent\` runnable passed as \`{ wmill, runnable }\`. `, "triggers": `--- name: triggers diff --git a/docs/operator-builder-rights.md b/docs/operator-builder-rights.md index 44a012131f..8445fa0dde 100644 --- a/docs/operator-builder-rights.md +++ b/docs/operator-builder-rights.md @@ -66,7 +66,8 @@ resolves the runnable it runs on the root handle, so `validate_operator_composed referenced path under the caller's RLS and refuses hub ones. For a script it checks the version a run would pick: grants are per version, and a path reused after its versions were archived starts with none while the archived ones keep theirs. But it has to check **two** surfaces, -because they are not the same list: the policy's `script/` and `flow/` triggerables, +because they are not the same list: the policy's `script/`, `flow/` and +`agent/` triggerables (an agent is checked as an `ai_agent` resource the caller can read), and the by-path entries of the app value's `runnables` map, which is what the editor resolves a `runnable_id` against and sends. Read only that map, and refuse an entry whose `type`, `path` or `runType` is not text. @@ -77,8 +78,8 @@ at run time for `component = "a:b"`. Every colon is a possible split, so the che suffix that parses as a runnable rather than guessing which one a request will use. What makes those checks bind is that **`ExecutionMode::Viewer` is refused for a builder app**. In -Viewer mode `execute_component` falls back to a default triggerable for any `script/`/`flow/` -path, so the policy stops being the list of what the app may invoke, and the job runs as the +Viewer mode `execute_component` falls back to a default triggerable for any `script/`/`flow/`/ +`agent/` path, so the policy stops being the list of what the app may invoke, and the job runs as the *viewer*: an admin who merely opened the app would run anything in the workspace as themselves. `Publisher`, `Guest` and `Anonymous` have no such fallback; all three run as the publisher. If you ever relax the Viewer refusal, the deploy-time path checks above stop being an authorization boundary. diff --git a/frontend/src/lib/components/DeployWorkspace.svelte b/frontend/src/lib/components/DeployWorkspace.svelte index daf944ff37..4c7fae7ccd 100644 --- a/frontend/src/lib/components/DeployWorkspace.svelte +++ b/frontend/src/lib/components/DeployWorkspace.svelte @@ -283,6 +283,8 @@ result.push({ kind: 'script', path: runnable.path }) } else if (runnable.runType == 'flow') { result.push({ kind: 'flow', path: runnable.path }) + } else if (runnable.runType == 'agent') { + result.push({ kind: 'resource', path: runnable.path }) } } } diff --git a/frontend/src/lib/components/apps/editor/inlineScriptsPanel/EmptyInlineScript.svelte b/frontend/src/lib/components/apps/editor/inlineScriptsPanel/EmptyInlineScript.svelte index b7521372f8..ff15d6a4fc 100644 --- a/frontend/src/lib/components/apps/editor/inlineScriptsPanel/EmptyInlineScript.svelte +++ b/frontend/src/lib/components/apps/editor/inlineScriptsPanel/EmptyInlineScript.svelte @@ -101,14 +101,14 @@ - +
- + {#if !composeOnlyApps} - + {/if}
diff --git a/frontend/src/lib/components/apps/editor/inlineScriptsPanel/InlineScriptRunnableByPath.svelte b/frontend/src/lib/components/apps/editor/inlineScriptsPanel/InlineScriptRunnableByPath.svelte index 02f4524830..e3552d0e79 100644 --- a/frontend/src/lib/components/apps/editor/inlineScriptsPanel/InlineScriptRunnableByPath.svelte +++ b/frontend/src/lib/components/apps/editor/inlineScriptsPanel/InlineScriptRunnableByPath.svelte @@ -2,7 +2,7 @@ import { createBubbler, stopPropagation } from 'svelte/legacy' const bubble = createBubbler() - import { Button, Drawer, DrawerContent } from '$lib/components/common' + import { Alert, Button, Drawer, DrawerContent } from '$lib/components/common' import FlowGraphViewer from '$lib/components/FlowGraphViewer.svelte' import Skeleton from '$lib/components/common/skeleton/Skeleton.svelte' import FlowModuleScript from '$lib/components/flows/content/FlowModuleScript.svelte' @@ -32,7 +32,11 @@ import Popover from '$lib/components/meltComponents/Popover.svelte' import ScriptEditorDrawer from '$lib/components/flows/content/ScriptEditorDrawer.svelte' import FlowEditorDrawer from '$lib/components/flows/content/FlowEditorDrawer.svelte' - import { FlowService, ScriptService, type OpenFlow } from '$lib/gen' + import { FlowService, ResourceService, ScriptService, type OpenFlow } from '$lib/gen' + import Badge from '$lib/components/common/badge/Badge.svelte' + import { summarizeAgentBrain, type AgentTool } from '$lib/components/flows/agentResourceUtils' + import { agentRunnableSchema } from '$lib/components/raw_apps/agentRunnable' + import { agentMemoryMode } from '$lib/components/flows/agentFormFields' import { replaceScriptPlaceholderWithItsValues } from '$lib/hub' import { useOperatingWorkspace } from '$lib/components/operatingWorkspace.svelte' @@ -149,6 +153,47 @@ } } + let agentInfo: + | { + description?: string + rows: { label: string; value: string }[] + tools: string[] + memoryOff: boolean + } + | undefined = $state() + + async function refreshAgent(runnable: RunnableByPath) { + hubFlowPreview = undefined + try { + const agent = await ResourceService.getResource({ + workspace: $operatingWorkspace ?? '', + path: runnable.path + }) + const config = agent.value as any + agentInfo = { + description: agent.description ?? undefined, + rows: summarizeAgentBrain(config), + tools: ((config?.tools ?? []) as AgentTool[]).map( + (t) => t.summary || t.value?.tool_type || t.id + ), + memoryOff: agentMemoryMode(config?.memory) === 'off' + } + const schema = agentRunnableSchema(config?.memory) + if (!deepEqual(runnable.schema, schema)) runnable.schema = schema + // Every input of a turn comes from its caller: a static one would be pinned by the + // deployed policy, answering every viewer's message with the same one. + if ( + Object.values(fields ?? {}).some((f) => f?.type !== 'user') || + !deepEqual(Object.keys(fields ?? {}).sort(), Object.keys(schema.properties).sort()) + ) { + fields = computeFields(schema, true, {}) + } + } catch (e) { + notFound = true + console.error(e) + } + } + async function fork(path: string) { let { content, language, schema } = await getScriptByPath(path) if (!schema || typeof schema == 'string' || Object.keys(schema).length == 0) { @@ -205,6 +250,8 @@ refreshScript(runnable) } else if (runnable.runType == 'flow') { refreshFlow(runnable) + } else if (runnable.runType == 'agent') { + refreshAgent(runnable) } else { hubFlowPreview = undefined } @@ -352,6 +399,17 @@ Details {/if} + {:else if runnable.runType == 'agent'} + {:else} - {/snippet} - {#snippet content()} - {#if runnable.runType == 'flow' && isHubFlowPath(runnable.path)} - Since this is a reference to a hub flow, cache settings are managed from the flow after - you fork it into your workspace. - {:else} - Since this is a reference to a workspace {runnable.runType}, set the cache in the - {runnable.runType} settings directly by editing it. The cache will be shared by any app or - flow that uses this {runnable.runType}. - {/if} - {/snippet} - + {#if runnable.runType != 'agent'} + + {#snippet trigger()} + + {/snippet} + {#snippet content()} + {#if runnable.runType == 'flow' && isHubFlowPath(runnable.path)} + Since this is a reference to a hub flow, cache settings are managed from the flow after + you fork it into your workspace. + {:else} + Since this is a reference to a workspace {runnable.runType}, set the cache in the + {runnable.runType} settings directly by editing it. The cache will be shared by any app or + flow that uses this {runnable.runType}. + {/if} + {/snippet} + + {/if} {/if} + {:else if runnable.runType == 'agent'} +
+ {#if agentInfo?.memoryOff} + + Each call is answered on its own inputs, so a chat would forget every turn. Pass + previous_messages to give it the history. + + {/if} + {#if agentInfo?.description} +

{agentInfo.description}

+ {/if} +
+ {#each agentInfo?.rows ?? [] as row (row.label)} +
+
{row.label}
+
{row.value}
+
+ {/each} + {#if agentInfo?.tools.length} +
+
Tools
+
+ {#each agentInfo.tools as tool, i (i)} + {tool} + {/each} +
+
+ {/if} +
+
{:else} Unrecognized runType {runnable.runType} {/if} diff --git a/frontend/src/lib/components/apps/editor/settingsPanel/mainInput/RunnableSelector.svelte b/frontend/src/lib/components/apps/editor/settingsPanel/mainInput/RunnableSelector.svelte index 5845965198..f741f5511d 100644 --- a/frontend/src/lib/components/apps/editor/settingsPanel/mainInput/RunnableSelector.svelte +++ b/frontend/src/lib/components/apps/editor/settingsPanel/mainInput/RunnableSelector.svelte @@ -2,11 +2,17 @@ import { Button, Drawer, DrawerContent, Tab, Tabs } from '$lib/components/common' import PickHubScript from '$lib/components/flows/pickers/PickHubScript.svelte' import PickHubFlow from '$lib/components/flows/pickers/PickHubFlow.svelte' - import { Building, Globe2, MousePointer, Plus } from 'lucide-svelte' + import { Bot, Building, Globe2, MousePointer, Plus } from 'lucide-svelte' import InlineScriptList from './InlineScriptList.svelte' - import type { InlineScript, Runnable, StaticAppInput } from '$lib/components/apps/inputType' + import type { + InlineScript, + PathRunType, + Runnable, + StaticAppInput + } from '$lib/components/apps/inputType' import WorkspaceScriptList from './WorkspaceScriptList.svelte' import WorkspaceFlowList from './WorkspaceFlowList.svelte' + import WorkspaceAgentList from './WorkspaceAgentList.svelte' import { createEventDispatcher, untrack } from 'svelte' import type { Schema } from '$lib/common' import { emptySchema } from '$lib/utils' @@ -18,7 +24,13 @@ const roleCaps = useRoleCapabilities() import { buildPathRunnableSelection } from './runnableSelectorUtils' - type TabType = 'hubscripts' | 'hubflows' | 'workspacescripts' | 'workspaceflows' | 'inlinescripts' + type TabType = + | 'hubscripts' + | 'hubflows' + | 'workspacescripts' + | 'workspaceflows' + | 'workspaceagents' + | 'inlinescripts' interface Props { defaultUserInput?: boolean @@ -47,6 +59,13 @@ ) let filter: string = $state('') let picker: Drawer | undefined = $state() + const selectLabel = $derived( + onlyFlow + ? 'Select a flow' + : rawApps + ? 'Select a script, flow or agent' + : 'Select a script or flow' + ) const dispatch = createEventDispatcher<{ pick: { @@ -57,7 +76,7 @@ async function loadSchemaFromTriggerable( path: string, - runType: 'script' | 'flow' | 'hubscript' + runType: PathRunType ): Promise<{ schema: Schema; summary: string | undefined }> { const schema = await loadSchema($operatingWorkspace!, path, runType) if (!schema.schema.order) { @@ -94,6 +113,20 @@ }) } + async function pickAgent(path: string) { + const selection = buildPathRunnableSelection( + path, + 'agent', + await loadSchemaFromTriggerable(path, 'agent'), + defaultUserInput, + rawApps + ) + dispatch('pick', { + runnable: selection.runnable, + fields: selection.fields + }) + } + async function pickHubScript(path: string) { const selection = buildPathRunnableSelection( path, @@ -154,22 +187,25 @@ - +
{#if !onlyFlow} {#if !rawApps} - + {/if} - + + {/if} + + {#if rawApps && !onlyFlow} + {/if} - {#if $roleCaps.apps !== 'compose'} {#if !onlyFlow} - + {/if} - + {/if}
@@ -186,6 +222,8 @@ pickScript(e.detail)} /> {:else if tab == 'workspaceflows'} pickFlow(e.detail)} /> + {:else if tab == 'workspaceagents'} + pickAgent(e.detail)} /> {:else if tab == 'hubscripts'} pickHubScript(e.detail.path)} /> {:else if tab == 'hubflows'} @@ -218,6 +256,6 @@ startIcon={{ icon: MousePointer }} btnClasses="truncate w-full" > - {#if onlyFlow}Select a flow{:else}Select a script or flow{/if} + {selectLabel}
diff --git a/frontend/src/lib/components/apps/editor/settingsPanel/mainInput/WorkspaceAgentList.svelte b/frontend/src/lib/components/apps/editor/settingsPanel/mainInput/WorkspaceAgentList.svelte new file mode 100644 index 0000000000..4335fedaf8 --- /dev/null +++ b/frontend/src/lib/components/apps/editor/settingsPanel/mainInput/WorkspaceAgentList.svelte @@ -0,0 +1,83 @@ + + + (emptyString(x.description) ? x.path : x.description + ' (' + x.path + ')')} +/> +
+ e.stopPropagation() + }} + /> +
+ +{#if agents} + {#if filteredItems.length == 0} + + {:else} +
    + {#each filteredItems as item (item)} +
  • + +
    • + {/each} +
+ {/if} +{:else} + {#each Array(10).fill(0) as _} + + {/each} +{/if} diff --git a/frontend/src/lib/components/apps/editor/settingsPanel/mainInput/runnableSelectorUtils.ts b/frontend/src/lib/components/apps/editor/settingsPanel/mainInput/runnableSelectorUtils.ts index 44e989e925..66eb1e21cf 100644 --- a/frontend/src/lib/components/apps/editor/settingsPanel/mainInput/runnableSelectorUtils.ts +++ b/frontend/src/lib/components/apps/editor/settingsPanel/mainInput/runnableSelectorUtils.ts @@ -1,5 +1,5 @@ import type { Schema } from '$lib/common' -import type { Runnable, StaticAppInput } from '$lib/components/apps/inputType' +import type { PathRunType, Runnable, StaticAppInput } from '$lib/components/apps/inputType' import { schemaToInputsSpec } from '$lib/components/apps/utils' import { defaultIfEmptyString } from '$lib/utils' @@ -10,7 +10,7 @@ export type LoadedRunnableSchema = { export function buildPathRunnableSelection( path: string, - runType: 'script' | 'flow' | 'hubscript', + runType: PathRunType, loadedSchema: LoadedRunnableSchema, defaultUserInput: boolean, rawApps: boolean diff --git a/frontend/src/lib/components/apps/inputType.ts b/frontend/src/lib/components/apps/inputType.ts index 8501372216..99aa5d8418 100644 --- a/frontend/src/lib/components/apps/inputType.ts +++ b/frontend/src/lib/components/apps/inputType.ts @@ -136,11 +136,14 @@ export type TemplateV2Input = { connections: InputConnectionEval[] } +/** `agent` (a saved AI agent run as one chat turn) exists in raw apps only. */ +export type PathRunType = 'script' | 'flow' | 'hubscript' | 'agent' + export type RunnableByPath = { name: string path: string schema: any - runType: 'script' | 'flow' | 'hubscript' + runType: PathRunType type: 'runnableByPath' | 'path' } diff --git a/frontend/src/lib/components/copilot/chat/app/core.ts b/frontend/src/lib/components/copilot/chat/app/core.ts index 166df65a16..5a0ff5cc6d 100644 --- a/frontend/src/lib/components/copilot/chat/app/core.ts +++ b/frontend/src/lib/components/copilot/chat/app/core.ts @@ -22,7 +22,7 @@ import { appDatatableRole, sdkDatatableCall } from '$lib/components/raw_apps/dat import { RAW_APP_BASE } from '$system_prompts' // Backend runnable types -export type BackendRunnableType = 'script' | 'flow' | 'hubscript' | 'inline' +export type BackendRunnableType = 'script' | 'flow' | 'hubscript' | 'agent' | 'inline' export interface InlineScript { language: 'bun' | 'python3' @@ -311,9 +311,9 @@ const getSetBackendRunnableSchema = memo(() => ), name: z.string().describe('A short summary/description of what the runnable does'), type: z - .enum(['script', 'flow', 'hubscript', 'inline']) + .enum(['script', 'flow', 'hubscript', 'agent', 'inline']) .describe( - 'The type of runnable: "inline" for custom code, "script" for workspace script, "flow" for workspace flow, "hubscript" for hub script' + 'The type of runnable: "inline" for custom code, "script" for workspace script, "flow" for workspace flow, "hubscript" for hub script, "agent" for a saved AI agent (one chat turn)' ), staticInputs: z .record(z.string(), z.any()) @@ -328,7 +328,7 @@ const getSetBackendRunnableSchema = memo(() => .string() .optional() .describe( - 'Required when type is "script", "flow", or "hubscript". The fully qualified path to the workspace script/flow or hub script.' + 'Required when type is "script", "flow", "hubscript" or "agent". The fully qualified path to the workspace script/flow, hub script or AI agent.' ), viewerDataAccess: z .boolean() @@ -723,6 +723,11 @@ export const getAppTools = memo((): Tool[] => [ } const parsedArgs = getSetBackendRunnableSchema().parse(cleanedArgs) + if (parsedArgs.type === 'agent' && Object.keys(parsedArgs.staticInputs ?? {}).length > 0) { + const errorMsg = `An "agent" runnable takes no staticInputs: each turn's inputs come from the app.` + toolCallbacks.setToolStatus(toolId, { content: errorMsg, error: errorMsg }) + return errorMsg + } toolCallbacks.setToolStatus(toolId, { content: `Setting backend runnable '${parsedArgs.key}'...` }) @@ -1087,6 +1092,16 @@ When creating a backend runnable with \`set_backend_runnable\`: } \`\`\` +5. **For saved AI agents** (type: "agent"), one chat turn per call, taking \`user_message\` and either \`memory_id\` (the conversation id, for an agent with managed memory) or \`previous_messages\` (the history, for one whose memory is off); no staticInputs: + \`\`\` + { + key: "chat", + name: "Chat with the support agent", + type: "agent", + path: "f/support/agent" + } + \`\`\` + ## Instructions 1. Use the smallest context needed. If the target file or runnable is clear, inspect only that item with \`get_frontend_file(path)\` or \`get_backend_runnable(key)\`. diff --git a/frontend/src/lib/components/copilot/chat/global/core.test.ts b/frontend/src/lib/components/copilot/chat/global/core.test.ts index 546459cee2..61bdea9dc9 100644 --- a/frontend/src/lib/components/copilot/chat/global/core.test.ts +++ b/frontend/src/lib/components/copilot/chat/global/core.test.ts @@ -2548,6 +2548,38 @@ describe('global AI tools', () => { expect(asPath.inlineScript).toBeUndefined() }) + // A static input is pinned by the deployed policy: carried into an agent runnable, it would + // replace every viewer's message. + it('drops inherited static inputs when a runnable becomes an agent', async () => { + seedBackendDraft( + 'raw_app', + 'u/admin/agent_conv_app', + { summary: 'Converted', files: {}, runnables: {}, data: { tables: [] } }, + { workspace: WORKSPACE } + ) + await callGlobalTool('write_app_runnable', { + path: 'u/admin/agent_conv_app', + key: 'chat', + runnable: { + name: 'Run the flow', + type: 'flow', + path: 'u/admin/hello_flow', + staticInputs: { user_message: 'pinned' } + } + }) + await callGlobalTool('write_app_runnable', { + path: 'u/admin/agent_conv_app', + key: 'chat', + runnable: { name: 'Chat', type: 'agent', path: 'u/admin/assistant' } + }) + + const asAgent = getBackendDraft('raw_app', 'u/admin/agent_conv_app', { + workspace: WORKSPACE + }).runnables.chat + expect(asAgent.runType).toBe('agent') + expect(asAgent.fields).toEqual({}) + }) + it("resets a path runnable's schema when it is retargeted, and keeps it when it is not", async () => { // The editor populates `schema` from the item the runnable points at, and // genWmillTs types `backend.(args)` from it — so it must not outlive the target. diff --git a/frontend/src/lib/components/copilot/chat/global/core.ts b/frontend/src/lib/components/copilot/chat/global/core.ts index 03356a51f0..66af4ab332 100644 --- a/frontend/src/lib/components/copilot/chat/global/core.ts +++ b/frontend/src/lib/components/copilot/chat/global/core.ts @@ -180,7 +180,7 @@ import { fileTools } from '../files/fileTools' import type { AttachedFilesStore } from '../files/attachedFiles.svelte' import { artifactTools } from '../artifacts/artifactTools' import type { SessionArtifactsStore } from '../artifacts/artifactsState.svelte' -import type { Runnable } from '$lib/components/apps/inputType' +import type { PathRunType, Runnable } from '$lib/components/apps/inputType' import { UserDraft } from '$lib/userDraft.svelte' import { emptySchema } from '$lib/utils' import type { Schema } from '$lib/common' @@ -1076,9 +1076,9 @@ const backendRunnableSchema = z .object({ name: z.string().describe('Short summary/description of what the runnable does.'), type: z - .enum(['inline', 'script', 'flow', 'hubscript']) + .enum(['inline', 'script', 'flow', 'hubscript', 'agent']) .describe( - 'Runnable kind: "inline" for custom code stored on the app; "script"/"flow" for a workspace runnable; "hubscript" for a hub script.' + 'Runnable kind: "inline" for custom code stored on the app; "script"/"flow" for a workspace runnable; "hubscript" for a hub script; "agent" for a saved AI agent, run as one chat turn.' ), staticInputs: z .record(z.string(), z.any()) @@ -1097,7 +1097,7 @@ const backendRunnableSchema = z .string() .optional() .describe( - 'Required when type is "script", "flow", or "hubscript". Workspace path of the runnable.' + 'Required when type is "script", "flow", "hubscript" or "agent". Workspace path of the runnable.' ), viewerDataAccess: z .boolean() @@ -2008,7 +2008,7 @@ type BackendRunnableInput = z.infer type PersistedRunnable = { name: string type: 'inline' | 'path' | 'runnableByName' | 'runnableByPath' - runType?: 'script' | 'flow' | 'hubscript' + runType?: PathRunType path?: string inlineScript?: { language: string; content: string } fields?: Record @@ -2041,6 +2041,7 @@ function convertPersistedToBackendRunnable( } else if (persisted.type === 'path' || persisted.type === 'runnableByPath') { if (persisted.runType === 'flow') out.type = 'flow' else if (persisted.runType === 'hubscript') out.type = 'hubscript' + else if (persisted.runType === 'agent') out.type = 'agent' else out.type = 'script' out.path = persisted.path } @@ -2062,14 +2063,19 @@ function buildPersistedRunnable( input: BackendRunnableInput, existing: PersistedRunnable | undefined ): PersistedRunnable { - const fields = input.staticInputs - ? Object.fromEntries( - Object.entries(input.staticInputs).map(([k, v]) => [ - k, - { type: 'static', value: v, fieldType: 'object' } - ]) - ) - : (existing?.fields ?? {}) + // An agent runnable takes every input from the app: a static one inherited from the kind it was + // converted from would be pinned by the deployed policy over every viewer's message. + const fields = + input.type === 'agent' + ? {} + : input.staticInputs + ? Object.fromEntries( + Object.entries(input.staticInputs).map(([k, v]) => [ + k, + { type: 'static', value: v, fieldType: 'object' } + ]) + ) + : (existing?.fields ?? {}) // Converting between kinds must drop the other kind's fields. Every consumer dispatches on // `type` (isRunnableByName / isRunnableByPath), so a leftover does not change which code @@ -2110,7 +2116,15 @@ function buildPersistedRunnable( } if (!input.path) { - throw new Error('path is required when runnable type is "script", "flow", or "hubscript".') + throw new Error( + 'path is required when runnable type is "script", "flow", "hubscript" or "agent".' + ) + } + // A static input is pinned by the deployed policy: every viewer's turn would send the same one. + if (input.type === 'agent' && Object.keys(input.staticInputs ?? {}).length > 0) { + throw new Error( + 'An "agent" runnable takes no staticInputs: each turn\'s inputs come from the app.' + ) } return { ...carried, @@ -9425,9 +9439,16 @@ async function undeployedRunnableTargets( ? runnable?.runType : runnable?.type const path = runnable?.path - if ((type !== 'script' && type !== 'flow') || !path) continue + if ((type !== 'script' && type !== 'flow' && type !== 'agent') || !path) continue try { - if (await deployedItemExists(workspace, type, path, undefined)) continue + if (type === 'agent') { + // A saved agent is an `ai_agent` resource; another resource at the path runs nothing. + const found = await ResourceService.getResource({ workspace, path }).catch((e) => { + if ((e as { status?: number } | null | undefined)?.status === 404) return undefined + throw e + }) + if (found?.resource_type === 'ai_agent') continue + } else if (await deployedItemExists(workspace, type, path, undefined)) continue } catch { // The probe is advisory: a lookup failure must never block the write or // the deploy it annotates. diff --git a/frontend/src/lib/components/flows/content/AgentEditorHost.svelte b/frontend/src/lib/components/flows/content/AgentEditorHost.svelte index b902b69bfa..cdec419e5a 100644 --- a/frontend/src/lib/components/flows/content/AgentEditorHost.svelte +++ b/frontend/src/lib/components/flows/content/AgentEditorHost.svelte @@ -47,7 +47,8 @@ AGENT_EDITOR_RUN_INPUTS, AGENT_TOOLS_ROW, DEFAULT_AGENT_MEMORY, - keepsManagedMemory + keepsManagedMemory, + agentMemoryMode } from '../agentFormFields' import { toolDisplayName, type AgentTool } from '../agentToolUtils' import { useAgentDraft, type AgentRunAs } from '../agentDraft.svelte' @@ -243,6 +244,23 @@ // apart from the echo of their own write. let lastArgs = $state(undefined) + // With memory off a run keeps no history of its own, so the form also takes the one to answer + // from; managed memory and an older setting would ignore one. + let formSchema = $derived( + agentMemoryMode((agentValue?.input_transforms?.memory as any)?.value) !== 'off' + ? AGENT_CHAT_SCHEMA + : { + ...AGENT_CHAT_SCHEMA, + properties: { + ...AGENT_CHAT_SCHEMA.properties, + previous_messages: { + type: 'array', + items: { type: 'object' }, + description: 'The conversation so far, as chat messages, for the agent to answer from' + } + } + } + ) // Picked on the first load, after which it is the reader's: following the memory setting live // would pull them out of a chat for switching memory off in the form. let testMode = $state<'form' | 'chat' | undefined>(undefined) @@ -603,7 +621,7 @@
{@render viewForm?.({ - schema: AGENT_CHAT_SCHEMA, + schema: formSchema, run: runOnce, loading: runLoading, actions: onOpenConfig ? configurationButton : undefined diff --git a/frontend/src/lib/components/flows/pickers/PickHubFlow.svelte b/frontend/src/lib/components/flows/pickers/PickHubFlow.svelte index 8c07ad1506..ff926c0c5d 100644 --- a/frontend/src/lib/components/flows/pickers/PickHubFlow.svelte +++ b/frontend/src/lib/components/flows/pickers/PickHubFlow.svelte @@ -60,7 +60,7 @@ {@render children?.()} + import { onDestroy } from 'svelte' import { executeRunnable } from '../apps/components/helpers/executeRunnable' import { userStore } from '$lib/stores' import { waitJob } from '../waitJob' @@ -54,6 +55,9 @@ // Job ids launched by this app instance — see `gateJobIds`. const launchedJobs = new Set() + // A stream resumed on every server timeout would otherwise outlive the app it answers. + let destroyed = false + onDestroy(() => (destroyed = true)) /** For a host that loads this component after the bundle may already have sent requests. */ export function handleMessage(event: MessageEvent) { @@ -124,48 +128,64 @@ args[key] = `$ctx:${field.ctx}` } } - const uuid = await executeRunnable( - runnable, - workspace, - undefined, - $userStore?.username, - path, - runnable_id, - { - component: runnable_id, - args, - force_viewer_allow_user_resources: editor - ? undefinedIfEmpty( - Object.keys(runnable?.fields ?? {}).filter( - (k) => - runnable?.fields?.[k]?.type == 'user' && - runnable?.fields?.[k]?.allowUserResources + let uuid: string + try { + uuid = await executeRunnable( + runnable, + workspace, + undefined, + $userStore?.username, + path, + runnable_id, + { + component: runnable_id, + args, + force_viewer_allow_user_resources: editor + ? undefinedIfEmpty( + Object.keys(runnable?.fields ?? {}).filter( + (k) => + runnable?.fields?.[k]?.type == 'user' && + runnable?.fields?.[k]?.allowUserResources + ) ) - ) - : undefined, - force_viewer_one_of_fields: undefined, - force_viewer_static_fields: editor - ? Object.fromEntries( - Object.entries(runnable?.fields ?? {}) - .filter(([k, v]) => v.type == 'static') - .map(([k, v]) => [k, v?.['value']]) - ) - : undefined, - force_viewer_sensitive_inputs: editor - ? undefinedIfEmpty( - Object.keys(runnable?.fields ?? {}).filter( - (k) => runnable?.fields?.[k]?.type == 'user' && runnable?.fields?.[k]?.sensitive + : undefined, + force_viewer_one_of_fields: undefined, + force_viewer_static_fields: editor + ? Object.fromEntries( + Object.entries(runnable?.fields ?? {}) + .filter(([k, v]) => v.type == 'static') + .map(([k, v]) => [k, v?.['value']]) ) - ) - : undefined, - force_viewer_delete_after_secs: editor ? runnable?.delete_after_secs : undefined, - on_behalf_of: editor ? previewOnBehalfOf : undefined, - viewer_data_access: editor ? runnable?.viewer_data_access || undefined : undefined - }, - undefined, - undefined, - editor ? rawAppPreviewModules(runnables, runnable_id) : undefined - ) + : undefined, + force_viewer_sensitive_inputs: editor + ? undefinedIfEmpty( + Object.keys(runnable?.fields ?? {}).filter( + (k) => + runnable?.fields?.[k]?.type == 'user' && runnable?.fields?.[k]?.sensitive + ) + ) + : undefined, + force_viewer_delete_after_secs: editor ? runnable?.delete_after_secs : undefined, + on_behalf_of: editor ? previewOnBehalfOf : undefined, + viewer_data_access: editor ? runnable?.viewer_data_access || undefined : undefined + }, + undefined, + undefined, + editor ? rawAppPreviewModules(runnables, runnable_id) : undefined + ) + } catch (e) { + // A run refused before it started still answers the app, which would otherwise + // wait for ever; the refusal's body says why (a 409 names the turn still running). + const body = (e as { body?: unknown })?.body + const message = + typeof body === 'string' + ? body + : body + ? JSON.stringify(body) + : String((e as Error)?.message ?? e) + respond({ result: { message }, error: true }) + return + } launchedJobs.add(uuid) let job: JobById = { component: runnable_id, created_at: Date.now(), job: uuid } if (data.type == 'backendAsync') { @@ -209,97 +229,105 @@ ) return } - const params = new URLSearchParams() - params.set('fast', 'true') - params.set('only_result', 'true') + let lastOffset: number | undefined + const connect = (streamOffset?: number) => { + const params = new URLSearchParams() + params.set('fast', 'true') + params.set('only_result', 'true') + if (streamOffset !== undefined) params.set('stream_offset', String(streamOffset)) - const sseUrl = `/api/w/${workspace}/jobs_u/getupdate_sse/${jobId}?${params.toString()}` - const eventSource = new EventSource(sseUrl) + const sseUrl = `/api/w/${workspace}/jobs_u/getupdate_sse/${jobId}?${params.toString()}` + const eventSource = new EventSource(sseUrl) - eventSource.onmessage = (sseEvent) => { - try { - const update = JSON.parse(sseEvent.data) - const type = update.type + eventSource.onmessage = (sseEvent) => { + try { + const update = JSON.parse(sseEvent.data) + const type = update.type - if (type === 'ping' || type === 'timeout') { - if (type === 'timeout') { - eventSource.close() + if (type === 'ping' || type === 'timeout') { + if (type === 'timeout') { + // The server ends every stream after a while; the job goes on, so it is resumed. + eventSource.close() + if (!destroyed) connect(lastOffset) + } + return } - return - } - if (type === 'error') { - eventSource.close() - sourceWindow?.postMessage( - { - type: 'streamJobRes', - reqId, - error: true, - result: { message: update.error || 'SSE error' } - }, - '*' - ) - return - } + if (type === 'error') { + eventSource.close() + sourceWindow?.postMessage( + { + type: 'streamJobRes', + reqId, + error: true, + result: { message: update.error || 'SSE error' } + }, + '*' + ) + return + } - if (type === 'not_found') { - eventSource.close() - sourceWindow?.postMessage( - { - type: 'streamJobRes', - reqId, - error: true, - result: { message: 'Job not found' } - }, - '*' - ) - return - } + if (type === 'not_found') { + eventSource.close() + sourceWindow?.postMessage( + { + type: 'streamJobRes', + reqId, + error: true, + result: { message: 'Job not found' } + }, + '*' + ) + return + } - // Send stream update if there's new stream data - if (update.new_result_stream !== undefined) { - sourceWindow?.postMessage( - { - type: 'streamJobUpdate', - reqId, - new_result_stream: update.new_result_stream, - stream_offset: update.stream_offset - }, - '*' - ) - } + // Send stream update if there's new stream data + if (update.stream_offset !== undefined) lastOffset = update.stream_offset + if (update.new_result_stream !== undefined) { + sourceWindow?.postMessage( + { + type: 'streamJobUpdate', + reqId, + new_result_stream: update.new_result_stream, + stream_offset: update.stream_offset + }, + '*' + ) + } - // Check if job is completed - if (update.completed) { - eventSource.close() - sourceWindow?.postMessage( - { - type: 'streamJobRes', - reqId, - error: false, - result: update.only_result - }, - '*' - ) + // Check if job is completed + if (update.completed) { + eventSource.close() + sourceWindow?.postMessage( + { + type: 'streamJobRes', + reqId, + error: false, + result: update.only_result + }, + '*' + ) + } + } catch (parseErr) { + console.warn('Failed to parse SSE data:', parseErr) } - } catch (parseErr) { - console.warn('Failed to parse SSE data:', parseErr) + } + + eventSource.onerror = (error) => { + console.warn('SSE stream error:', error) + eventSource.close() + sourceWindow?.postMessage( + { + type: 'streamJobRes', + reqId, + error: true, + result: { message: 'SSE connection error' } + }, + '*' + ) } } - - eventSource.onerror = (error) => { - console.warn('SSE stream error:', error) - eventSource.close() - sourceWindow?.postMessage( - { - type: 'streamJobRes', - reqId, - error: true, - result: { message: 'SSE connection error' } - }, - '*' - ) - } + connect() } } diff --git a/frontend/src/lib/components/raw_apps/RawAppEditor.svelte b/frontend/src/lib/components/raw_apps/RawAppEditor.svelte index eec6fc3b0e..3d22087317 100644 --- a/frontend/src/lib/components/raw_apps/RawAppEditor.svelte +++ b/frontend/src/lib/components/raw_apps/RawAppEditor.svelte @@ -40,7 +40,7 @@ import { getAppliedDarkModeVariant, type DarkModeVariant } from '$lib/darkModeVariant' import RawAppSidebar from './RawAppSidebar.svelte' import type { Modules } from './RawAppModules.svelte' - import { isRunnableByName, isRunnableByPath } from '../apps/inputType' + import { isRunnableByName, isRunnableByPath, type PathRunType } from '../apps/inputType' import { aiChatManager, AIMode } from '../copilot/chat/AIChatManager.svelte' import { onMount, onDestroy, untrack } from 'svelte' import type { @@ -917,6 +917,8 @@ addEventListener('message', function (e) { backendRunnable.type = 'flow' } else if (runnable.runType === 'hubscript') { backendRunnable.type = 'hubscript' + } else if (runnable.runType === 'agent') { + backendRunnable.type = 'agent' } else { backendRunnable.type = 'script' } @@ -1230,7 +1232,7 @@ addEventListener('message', function (e) { runnables[key] = { name: runnable.name, type: 'path', - runType: runnable.type as 'script' | 'flow' | 'hubscript', + runType: runnable.type as PathRunType, path: runnable.path, fields: runnable.staticInputs ? Object.fromEntries( @@ -1861,7 +1863,7 @@ addEventListener('message', function (e) { // The deployed app's `window.ctx`: the viewer's membership in the app's workspace. function previewCtx() { - return $state.snapshot({ ctx: operatingUser.current, workspace: opWorkspace }) + return $state.snapshot({ ctx: operatingUser.current, workspace: opWorkspace, path }) } // A build fed before the viewer is known keeps that viewer until the next one. Outside diff --git a/frontend/src/lib/components/raw_apps/RawAppInlineScriptRunnable.svelte b/frontend/src/lib/components/raw_apps/RawAppInlineScriptRunnable.svelte index 86b2ea57ed..71a3dae951 100644 --- a/frontend/src/lib/components/raw_apps/RawAppInlineScriptRunnable.svelte +++ b/frontend/src/lib/components/raw_apps/RawAppInlineScriptRunnable.svelte @@ -21,7 +21,7 @@ import SchemaForm from '../SchemaForm.svelte' import RunnableJobPanelInner from '../apps/editor/RunnableJobPanelInner.svelte' import JobLoader from '../JobLoader.svelte' - import { FlowService, type Job, type OpenFlow, type ScriptLang } from '$lib/gen' + import { FlowService, JobService, type Job, type OpenFlow, type ScriptLang } from '$lib/gen' import { getScriptByPath } from '$lib/scripts' import { slide } from 'svelte/transition' import { DebugToolbar, DebugPanel, debugState } from '$lib/components/debug' @@ -224,6 +224,22 @@ } else { await jobLoader.runFlowByPath(runnable.path, args) } + } else if (runnable.runType == 'agent') { + // A turn of the agent as it is deployed, as the editor: the direct run has no way + // to ask for the app's identity. + const path = runnable.path + const { user_message, memory_id, user_attachments, previous_messages } = args as Record< + string, + any + > + await jobLoader.abstractRun(() => + JobService.runAgent({ + workspace: opWs!, + path, + memoryId: memory_id || undefined, + requestBody: { user_message, user_attachments, previous_messages } + }) + ) } else if (runnable.runType == 'script' || runnable.runType == 'hubscript') { if (previewOnBehalfOf) { const script = await getScriptByPath(runnable.path, opWs) diff --git a/frontend/src/lib/components/raw_apps/RawAppPreview.svelte b/frontend/src/lib/components/raw_apps/RawAppPreview.svelte index 36c3fb360c..1c82cdcd33 100644 --- a/frontend/src/lib/components/raw_apps/RawAppPreview.svelte +++ b/frontend/src/lib/components/raw_apps/RawAppPreview.svelte @@ -134,7 +134,7 @@ const html = unsandboxedRawAppHtml( workspace, secret, - { ctx: u, workspace }, + { ctx: u, workspace, path }, window.location.origin, pageHash() ) @@ -207,7 +207,7 @@ // Same shape as the unsandboxed wrapper: always the object, so // `window.ctx.workspace` works for anonymous viewers too. Snapshotted: a // caller's `$state` user is a proxy, which postMessage cannot clone. - ctx: { ctx: $state.snapshot(user), workspace }, + ctx: { ctx: $state.snapshot(user), workspace, path }, initialHash, storage: { local: bundleStorage ?? {}, session: {} }, // The wrapper turns this into `window.process.env` before it injects the diff --git a/frontend/src/lib/components/raw_apps/agentRunnable.ts b/frontend/src/lib/components/raw_apps/agentRunnable.ts new file mode 100644 index 0000000000..cca1964197 --- /dev/null +++ b/frontend/src/lib/components/raw_apps/agentRunnable.ts @@ -0,0 +1,43 @@ +import type { Schema, SchemaProperty } from '$lib/common' +import { agentMemoryMode, keepsManagedMemory } from '$lib/components/flows/agentFormFields' + +/** + * The inputs of a raw app's agent runnable, after the agent's memory as its own page reads it: a + * chat (`memory_id`, the conversation and memory key) when it keeps managed memory, single runs + * answered from the history the app passes (`previous_messages`) when it is off, and neither for an + * older `manual` setting (`memory_id` is the run's, which an older `auto` one keeps memory under). + * The server accepts both either way (see `execute_component`), so a schema gone stale only + * mistypes the inputs. + */ +export function agentRunnableSchema(memory: unknown): Schema { + const history: Record = {} + if (keepsManagedMemory(memory)) { + history.memory_id = { + type: 'string', + description: + 'The conversation the message belongs to: reuse it to continue one, or omit it for a single run' + } + } + if (agentMemoryMode(memory) === 'off') { + history.previous_messages = { + type: 'array', + description: 'The history to answer from, as chat messages: this agent keeps no memory', + items: { type: 'object' } + } + } + return { + $schema: 'https://json-schema.org/draft/2020-12/schema', + type: 'object', + properties: { + user_message: { type: 'string', description: 'The message to answer' }, + ...history, + user_attachments: { + type: 'array', + description: 'Files the message carries, as object storage references', + items: { type: 'object', resourceType: 's3object' } + } + }, + required: ['user_message'], + order: ['user_message', ...Object.keys(history), 'user_attachments'] + } +} diff --git a/frontend/src/lib/components/workspaceSettings/projectBundle.ts b/frontend/src/lib/components/workspaceSettings/projectBundle.ts index 8adedeb559..047f7ee7fa 100644 --- a/frontend/src/lib/components/workspaceSettings/projectBundle.ts +++ b/frontend/src/lib/components/workspaceSettings/projectBundle.ts @@ -148,6 +148,8 @@ function runnableRef(node: Record): Ref | undefined { if (!isRunnableByPath(node as any) || typeof node.path !== 'string') return undefined if (node.runType === 'flow') return { kind: 'flow', path: node.path } if (node.runType === 'script') return { kind: 'script', path: node.path } + // A saved agent is an `ai_agent` resource. + if (node.runType === 'agent') return { kind: 'resource', path: node.path } return undefined // hubscript -> external hub, ignored } diff --git a/frontend/src/lib/infer.ts b/frontend/src/lib/infer.ts index 9e9cddeb00..3c1d30bc87 100644 --- a/frontend/src/lib/infer.ts +++ b/frontend/src/lib/infer.ts @@ -1,11 +1,14 @@ import { ScriptService, + ResourceService, type MainArgSignature, FlowService, type Script, type AssetUsageAccessType } from '$lib/gen' import { get, writable } from 'svelte/store' +import type { PathRunType } from './components/apps/inputType.js' +import { agentRunnableSchema } from './components/raw_apps/agentRunnable.js' import type { Schema, SupportedLanguage } from './common.js' import { type DynamicInput, @@ -658,9 +661,15 @@ async function inferSchemaIfNecessary(script: Script) { export async function loadSchema( workspace: string, path: string, - runType: 'script' | 'flow' | 'hubscript' + runType: PathRunType ): Promise<{ schema: Schema; summary: string | undefined }> { - if (runType === 'script') { + if (runType === 'agent') { + const agent = await ResourceService.getResource({ workspace, path }) + return { + schema: agentRunnableSchema((agent.value as any)?.memory), + summary: agent.description || undefined + } + } else if (runType === 'script') { const script = await ScriptService.getScriptByPath({ workspace, path diff --git a/system_prompts/auto-generated/prompts.ts b/system_prompts/auto-generated/prompts.ts index 5b8c3cbe8c..b97fc5c3d9 100644 --- a/system_prompts/auto-generated/prompts.ts +++ b/system_prompts/auto-generated/prompts.ts @@ -1054,7 +1054,7 @@ An app can be demoed by recording a session: every interaction becomes a step ca Apply it to customer data, internal notes and anything else a viewer of the demo should not see. It costs nothing when the app is never recorded. -### Chat UIs over a flow in chat mode +### Chat UIs over a flow in chat mode or a saved agent A flow deployed with chat mode on is a chat backend (streaming answer, tool calls, memory, conversation history). Do not drive it through a runnable: add \`@windmill-labs/chat-sdk\` to \`package.json\` and use it directly, it detects the app's Windmill and credential. @@ -1067,9 +1067,26 @@ const chat = useWindmillChat({ flowPath: 'f/support/assistant' }) \`@windmill-labs/chat-sdk/ai-sdk\` gives a \`ChatTransport\` for the Vercel AI SDK's \`useChat\`, \`@windmill-labs/chat-sdk/assistant-ui\` a runtime for assistant-ui. The flow must be deployed, not a draft. A sandboxed app needs \`jobs:run\` in its frontend SDK scopes, plus \`flow_conversations:write\` for the conversation sidebar; without them the chat keeps history in the browser. +#### A saved agent: an \`agent\` runnable + +Chat with a saved AI agent (an \`ai_agent\` resource) through an \`agent\` runnable, a path runnable whose \`path\` is the agent's. Each call runs one turn of the agent's chat as the app, so a sandboxed app declares no frontend SDK scope, viewers get no consent prompt and need no access to the agent. Prefer it in every app, sandboxed or not: + +\`\`\`json +{ "chat": { "type": "agent", "path": "f/support/agent" } } +\`\`\` + +\`\`\`tsx +import { useWindmillChat } from '@windmill-labs/chat-sdk/react' +import * as wmill from './wmill' + +const chat = useWindmillChat({ wmill, runnable: 'chat' }) +\`\`\` + +\`runnable\` is the runnable's key; each runnable keeps its own conversations. The runnable takes \`user_message\` and \`memory_id\`, the conversation id, which the chat fills in: the agent remembers earlier turns of a conversation, and a signed-in viewer's conversations are theirs alone. Called directly as \`backend.({ user_message })\` without \`memory_id\`, it is a single run with no conversation, for a one-off question. The chat needs an agent with managed memory. An agent whose memory is off keeps none: its runnable takes \`previous_messages\` (chat messages) instead of \`memory_id\`, and the app keeps the history itself. The chat keeps its transcript in the app's own storage. \`chat.stop()\` stops following without cancelling the turn, and a message sent while it still runs is refused with \`TurnRunningError\`. Attachments are not supported this way. + ## Backend runnables -Each runnable has a unique key (used to call it from the frontend) and one of four types: +Each runnable has a unique key (used to call it from the frontend) and one of five types: | Type | What it is | |---|---| @@ -1077,6 +1094,7 @@ Each runnable has a unique key (used to call it from the frontend) and one of fo | \`script\` | Reference to an existing workspace script by path. | | \`flow\` | Reference to an existing workspace flow by path. | | \`hubscript\` | Reference to a hub script by path. | +| \`agent\` | Reference to a saved AI agent by path. Each call is one turn of its chat (see the chat section above). | ### Inline runnables @@ -1117,9 +1135,9 @@ An inline runnable runs as an ordinary Windmill job. \`import * as wmill from 'w Use only \`wmill\` functions the SDK actually exports; for an endpoint none of them covers, the generated service classes (\`JobService\`, \`ScriptService\`, ...) are importable from \`windmill-client\`. What is not available is a name you guessed at: \`getBaseUrl\` and \`getWorkspaceToken\` are inventions, not API. -### Path runnables (script / flow / hubscript) +### Path runnables (script / flow / hubscript / agent) -When \`type\` is \`script\`, \`flow\`, or \`hubscript\`, the runnable just stores a \`path\` to an existing workspace or hub item — no inline code. The referenced item's input/output schema becomes the runnable's surface. +When \`type\` is \`script\`, \`flow\`, \`hubscript\` or \`agent\`, the runnable just stores a \`path\` to an existing workspace or hub item — no inline code. The referenced item's input/output schema becomes the runnable's surface; an agent's is \`user_message\` and \`user_attachments\`, plus \`memory_id\` when the agent keeps managed memory or \`previous_messages\` when its memory is off. Before writing an inline runnable, look for a workspace script or flow that already does the job, or a Hub script (a prebuilt integration at \`hub///\`) for a third-party service, and reference it instead of copying its logic. @@ -1128,7 +1146,7 @@ Before writing an inline runnable, look for a workspace script or flow that alre This decides whether an app works before anything is deployed: - **Inline runnables run the app's current code.** The editor sends the runnable's source with each request, so an inline runnable works in the preview with nothing deployed. -- **Path runnables (\`script\` / \`flow\` / \`hubscript\`) run the DEPLOYED item at that path.** So do \`wmill.runFlow\`, \`wmill.runFlowAsync\` and \`wmill.runScriptByPath\` called from inside a runnable. A draft — including a draft you just created — does not exist for them. +- **Path runnables (\`script\` / \`flow\` / \`hubscript\` / \`agent\`) run the DEPLOYED item at that path.** So do \`wmill.runFlow\`, \`wmill.runFlowAsync\` and \`wmill.runScriptByPath\` called from inside a runnable. A draft — including a draft you just created — does not exist for them. So an app wired to a flow you just wrote does nothing until **that flow is deployed**. The app itself does NOT have to be deployed for this: the preview runs the app's draft, so the referenced flow is the only thing that has to exist deployed. @@ -1227,7 +1245,7 @@ def main(user_id: str): 7. **Mark sensitive UI with \`data-wm-no-record\`** — it is what keeps that data out of a recorded demo; passwords are handled for you. 8. **Reach for \`backendAsync\` + \`waitJob\`** for long work — never a hand-written job-polling runnable. 9. **Deploy what a path runnable points at** — a path runnable aimed at a draft fails at runtime; tell the user what needs deploying. -10. **Use \`@windmill-labs/chat-sdk\` for a chat over a chat-mode flow** — never a runnable that runs the flow and polls its stream. +10. **Use \`@windmill-labs/chat-sdk\` for a chat over a chat-mode flow or a saved agent** — never hand-roll a runnable that runs it and polls its stream; a saved agent is an \`agent\` runnable passed as \`{ wmill, runnable }\`. `; export const PIPELINE_BASE = `# Data pipeline authoring diff --git a/system_prompts/auto-generated/skills/raw-app/SKILL.md b/system_prompts/auto-generated/skills/raw-app/SKILL.md index df5e7ece6a..f45b85402d 100644 --- a/system_prompts/auto-generated/skills/raw-app/SKILL.md +++ b/system_prompts/auto-generated/skills/raw-app/SKILL.md @@ -380,7 +380,7 @@ An app can be demoed by recording a session: every interaction becomes a step ca Apply it to customer data, internal notes and anything else a viewer of the demo should not see. It costs nothing when the app is never recorded. -### Chat UIs over a flow in chat mode +### Chat UIs over a flow in chat mode or a saved agent A flow deployed with chat mode on is a chat backend (streaming answer, tool calls, memory, conversation history). Do not drive it through a runnable: add `@windmill-labs/chat-sdk` to `package.json` and use it directly, it detects the app's Windmill and credential. @@ -393,9 +393,31 @@ const chat = useWindmillChat({ flowPath: 'f/support/assistant' }) `@windmill-labs/chat-sdk/ai-sdk` gives a `ChatTransport` for the Vercel AI SDK's `useChat`, `@windmill-labs/chat-sdk/assistant-ui` a runtime for assistant-ui. The flow must be deployed, not a draft. A sandboxed app needs `jobs:run` in its frontend SDK scopes, plus `flow_conversations:write` for the conversation sidebar; without them the chat keeps history in the browser. +#### A saved agent: an `agent` runnable + +Chat with a saved AI agent (an `ai_agent` resource) through an `agent` runnable, a path runnable whose `path` is the agent's. Each call runs one turn of the agent's chat as the app, so a sandboxed app declares no frontend SDK scope, viewers get no consent prompt and need no access to the agent. Prefer it in every app, sandboxed or not: + +In `raw_app.yaml`: + +```yaml +runnables: + chat: + type: agent + path: f/support/agent +``` + +```tsx +import { useWindmillChat } from '@windmill-labs/chat-sdk/react' +import * as wmill from './wmill' + +const chat = useWindmillChat({ wmill, runnable: 'chat' }) +``` + +`runnable` is the runnable's key; each runnable keeps its own conversations. The runnable takes `user_message` and `memory_id`, the conversation id, which the chat fills in: the agent remembers earlier turns of a conversation, and a signed-in viewer's conversations are theirs alone. Called directly as `backend.({ user_message })` without `memory_id`, it is a single run with no conversation, for a one-off question. The chat needs an agent with managed memory. An agent whose memory is off keeps none: its runnable takes `previous_messages` (chat messages) instead of `memory_id`, and the app keeps the history itself. The chat keeps its transcript in the app's own storage. `chat.stop()` stops following without cancelling the turn, and a message sent while it still runs is refused with `TurnRunningError`. Attachments are not supported this way. + ## Backend runnables -Each runnable has a unique key (used to call it from the frontend) and one of four types: +Each runnable has a unique key (used to call it from the frontend) and one of five types: | Type | What it is | |---|---| @@ -403,6 +425,7 @@ Each runnable has a unique key (used to call it from the frontend) and one of fo | `script` | Reference to an existing workspace script by path. | | `flow` | Reference to an existing workspace flow by path. | | `hubscript` | Reference to a hub script by path. | +| `agent` | Reference to a saved AI agent by path. Each call is one turn of its chat (see the chat section above). | ### Inline runnables @@ -444,9 +467,9 @@ An inline runnable runs as an ordinary Windmill job. `import * as wmill from 'wi Use only `wmill` functions the SDK actually exports; for an endpoint none of them covers, the generated service classes (`JobService`, `ScriptService`, ...) are importable from `windmill-client`. What is not available is a name you guessed at: `getBaseUrl` and `getWorkspaceToken` are inventions, not API. -### Path runnables (script / flow / hubscript) +### Path runnables (script / flow / hubscript / agent) -When `type` is `script`, `flow`, or `hubscript`, the runnable just stores a `path` to an existing workspace or hub item — no inline code. The referenced item's input/output schema becomes the runnable's surface. +When `type` is `script`, `flow`, `hubscript` or `agent`, the runnable just stores a `path` to an existing workspace or hub item — no inline code. The referenced item's input/output schema becomes the runnable's surface; an agent's is `user_message` and `user_attachments`, plus `memory_id` when the agent keeps managed memory or `previous_messages` when its memory is off. Before writing an inline runnable, look for a workspace script or flow that already does the job, or a Hub script (a prebuilt integration at `hub///`) for a third-party service, and reference it instead of copying its logic. @@ -455,7 +478,7 @@ Before writing an inline runnable, look for a workspace script or flow that alre This decides whether an app works before anything is deployed: - **Inline runnables run the app's current code.** The editor sends the runnable's source with each request, so an inline runnable works in the preview with nothing deployed. -- **Path runnables (`script` / `flow` / `hubscript`) run the DEPLOYED item at that path.** So do `wmill.runFlow`, `wmill.runFlowAsync` and `wmill.runScriptByPath` called from inside a runnable. A draft — including a draft you just created — does not exist for them. +- **Path runnables (`script` / `flow` / `hubscript` / `agent`) run the DEPLOYED item at that path.** So do `wmill.runFlow`, `wmill.runFlowAsync` and `wmill.runScriptByPath` called from inside a runnable. A draft — including a draft you just created — does not exist for them. So an app wired to a flow you just wrote does nothing until **that flow is deployed**. The app itself does NOT have to be deployed for this: the preview runs the app's draft, so the referenced flow is the only thing that has to exist deployed. @@ -565,4 +588,4 @@ def main(user_id: str): 7. **Mark sensitive UI with `data-wm-no-record`** — it is what keeps that data out of a recorded demo; passwords are handled for you. 8. **Reach for `backendAsync` + `waitJob`** for long work — never a hand-written job-polling runnable. 9. **Deploy what a path runnable points at** — a path runnable aimed at a draft fails at runtime; tell the user what needs deploying. -10. **Use `@windmill-labs/chat-sdk` for a chat over a chat-mode flow** — never a runnable that runs the flow and polls its stream. +10. **Use `@windmill-labs/chat-sdk` for a chat over a chat-mode flow or a saved agent** — never hand-roll a runnable that runs it and polls its stream; a saved agent is an `agent` runnable passed as `{ wmill, runnable }`. diff --git a/system_prompts/base/raw-app.md b/system_prompts/base/raw-app.md index dc70d0b94d..69c03594db 100644 --- a/system_prompts/base/raw-app.md +++ b/system_prompts/base/raw-app.md @@ -139,7 +139,7 @@ An app can be demoed by recording a session: every interaction becomes a step ca Apply it to customer data, internal notes and anything else a viewer of the demo should not see. It costs nothing when the app is never recorded. -### Chat UIs over a flow in chat mode +### Chat UIs over a flow in chat mode or a saved agent A flow deployed with chat mode on is a chat backend (streaming answer, tool calls, memory, conversation history). Do not drive it through a runnable: add `@windmill-labs/chat-sdk` to `package.json` and use it directly, it detects the app's Windmill and credential. @@ -152,9 +152,38 @@ const chat = useWindmillChat({ flowPath: 'f/support/assistant' }) `@windmill-labs/chat-sdk/ai-sdk` gives a `ChatTransport` for the Vercel AI SDK's `useChat`, `@windmill-labs/chat-sdk/assistant-ui` a runtime for assistant-ui. The flow must be deployed, not a draft. A sandboxed app needs `jobs:run` in its frontend SDK scopes, plus `flow_conversations:write` for the conversation sidebar; without them the chat keeps history in the browser. +#### A saved agent: an `agent` runnable + +Chat with a saved AI agent (an `ai_agent` resource) through an `agent` runnable, a path runnable whose `path` is the agent's. Each call runs one turn of the agent's chat as the app, so a sandboxed app declares no frontend SDK scope, viewers get no consent prompt and need no access to the agent. Prefer it in every app, sandboxed or not: + + +In `raw_app.yaml`: + +```yaml +runnables: + chat: + type: agent + path: f/support/agent +``` + + +```json +{ "chat": { "type": "agent", "path": "f/support/agent" } } +``` + + +```tsx +import { useWindmillChat } from '@windmill-labs/chat-sdk/react' +import * as wmill from './wmill' + +const chat = useWindmillChat({ wmill, runnable: 'chat' }) +``` + +`runnable` is the runnable's key; each runnable keeps its own conversations. The runnable takes `user_message` and `memory_id`, the conversation id, which the chat fills in: the agent remembers earlier turns of a conversation, and a signed-in viewer's conversations are theirs alone. Called directly as `backend.({ user_message })` without `memory_id`, it is a single run with no conversation, for a one-off question. The chat needs an agent with managed memory. An agent whose memory is off keeps none: its runnable takes `previous_messages` (chat messages) instead of `memory_id`, and the app keeps the history itself. The chat keeps its transcript in the app's own storage. `chat.stop()` stops following without cancelling the turn, and a message sent while it still runs is refused with `TurnRunningError`. Attachments are not supported this way. + ## Backend runnables -Each runnable has a unique key (used to call it from the frontend) and one of four types: +Each runnable has a unique key (used to call it from the frontend) and one of five types: | Type | What it is | |---|---| @@ -162,6 +191,7 @@ Each runnable has a unique key (used to call it from the frontend) and one of fo | `script` | Reference to an existing workspace script by path. | | `flow` | Reference to an existing workspace flow by path. | | `hubscript` | Reference to a hub script by path. | +| `agent` | Reference to a saved AI agent by path. Each call is one turn of its chat (see the chat section above). | ### Inline runnables @@ -205,9 +235,9 @@ An inline runnable runs as an ordinary Windmill job. `import * as wmill from 'wi Use only `wmill` functions the SDK actually exports; for an endpoint none of them covers, the generated service classes (`JobService`, `ScriptService`, ...) are importable from `windmill-client`. What is not available is a name you guessed at: `getBaseUrl` and `getWorkspaceToken` are inventions, not API. -### Path runnables (script / flow / hubscript) +### Path runnables (script / flow / hubscript / agent) -When `type` is `script`, `flow`, or `hubscript`, the runnable just stores a `path` to an existing workspace or hub item — no inline code. The referenced item's input/output schema becomes the runnable's surface. +When `type` is `script`, `flow`, `hubscript` or `agent`, the runnable just stores a `path` to an existing workspace or hub item — no inline code. The referenced item's input/output schema becomes the runnable's surface; an agent's is `user_message` and `user_attachments`, plus `memory_id` when the agent keeps managed memory or `previous_messages` when its memory is off. Before writing an inline runnable, look for a workspace script or flow that already does the job, or a Hub script (a prebuilt integration at `hub///`) for a third-party service, and reference it instead of copying its logic. @@ -216,7 +246,7 @@ Before writing an inline runnable, look for a workspace script or flow that alre This decides whether an app works before anything is deployed: - **Inline runnables run the app's current code.** The editor sends the runnable's source with each request, so an inline runnable works in the preview with nothing deployed. -- **Path runnables (`script` / `flow` / `hubscript`) run the DEPLOYED item at that path.** So do `wmill.runFlow`, `wmill.runFlowAsync` and `wmill.runScriptByPath` called from inside a runnable. A draft — including a draft you just created — does not exist for them. +- **Path runnables (`script` / `flow` / `hubscript` / `agent`) run the DEPLOYED item at that path.** So do `wmill.runFlow`, `wmill.runFlowAsync` and `wmill.runScriptByPath` called from inside a runnable. A draft — including a draft you just created — does not exist for them. So an app wired to a flow you just wrote does nothing until **that flow is deployed**. The app itself does NOT have to be deployed for this: the preview runs the app's draft, so the referenced flow is the only thing that has to exist deployed. @@ -345,4 +375,4 @@ def main(user_id: str): 7. **Mark sensitive UI with `data-wm-no-record`** — it is what keeps that data out of a recorded demo; passwords are handled for you. 8. **Reach for `backendAsync` + `waitJob`** for long work — never a hand-written job-polling runnable. 9. **Deploy what a path runnable points at** — a path runnable aimed at a draft fails at runtime; tell the user what needs deploying. -10. **Use `@windmill-labs/chat-sdk` for a chat over a chat-mode flow** — never a runnable that runs the flow and polls its stream. +10. **Use `@windmill-labs/chat-sdk` for a chat over a chat-mode flow or a saved agent** — never hand-roll a runnable that runs it and polls its stream; a saved agent is an `agent` runnable passed as `{ wmill, runnable }`.