diff --git a/backend/.sqlx/query-07a005f0f9e80a156cd2a5a0ae39a1fabeaa167818206a25abfe31d5582f942a.json b/backend/.sqlx/query-07a005f0f9e80a156cd2a5a0ae39a1fabeaa167818206a25abfe31d5582f942a.json new file mode 100644 index 0000000000..635ec2b94d --- /dev/null +++ b/backend/.sqlx/query-07a005f0f9e80a156cd2a5a0ae39a1fabeaa167818206a25abfe31d5582f942a.json @@ -0,0 +1,15 @@ +{ + "db_name": "PostgreSQL", + "query": "DELETE FROM flow_conversation c\n WHERE c.id = ANY($1)\n AND c.workspace_id = $2\n AND NOT EXISTS (\n SELECT 1 FROM flow_conversation_message m WHERE m.conversation_id = c.id\n )", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "UuidArray", + "Text" + ] + }, + "nullable": [] + }, + "hash": "07a005f0f9e80a156cd2a5a0ae39a1fabeaa167818206a25abfe31d5582f942a" +} diff --git a/backend/.sqlx/query-1c3473a0f9f6b6148b2c975f9f05bdefedf8a51c4e6ddf0eca367b9cc778d051.json b/backend/.sqlx/query-318ed7a45d8326e3ecbc9727ee3812adaeeead8b39b007a32badadf67ec5ca17.json similarity index 69% rename from backend/.sqlx/query-1c3473a0f9f6b6148b2c975f9f05bdefedf8a51c4e6ddf0eca367b9cc778d051.json rename to backend/.sqlx/query-318ed7a45d8326e3ecbc9727ee3812adaeeead8b39b007a32badadf67ec5ca17.json index fb27bd9446..ae6ffd4e3c 100644 --- a/backend/.sqlx/query-1c3473a0f9f6b6148b2c975f9f05bdefedf8a51c4e6ddf0eca367b9cc778d051.json +++ b/backend/.sqlx/query-318ed7a45d8326e3ecbc9727ee3812adaeeead8b39b007a32badadf67ec5ca17.json @@ -1,6 +1,6 @@ { "db_name": "PostgreSQL", - "query": "SELECT id, conversation_id, message_type as \"message_type: MessageType\", content, job_id, created_at, created_seq, step_name, success\n FROM (\n SELECT id, conversation_id, message_type, content, job_id, created_at, created_seq, step_name, success\n FROM flow_conversation_message\n WHERE conversation_id = $1\n ORDER BY created_seq DESC\n LIMIT $2 OFFSET $3\n ) AS messages\n ORDER BY created_seq ASC\n ", + "query": "SELECT id, conversation_id, message_type as \"message_type: MessageType\", content, job_id, created_at, created_seq, step_name, success, tool_arguments, tool_result, reasoning\n FROM flow_conversation_message\n WHERE conversation_id = $1\n AND created_seq > $2\n ORDER BY created_seq ASC\n LIMIT $3\n ", "describe": { "columns": [ { @@ -58,6 +58,21 @@ "ordinal": 8, "name": "success", "type_info": "Bool" + }, + { + "ordinal": 9, + "name": "tool_arguments", + "type_info": "Text" + }, + { + "ordinal": 10, + "name": "tool_result", + "type_info": "Text" + }, + { + "ordinal": 11, + "name": "reasoning", + "type_info": "Text" } ], "parameters": { @@ -76,8 +91,11 @@ false, false, true, - false + false, + true, + true, + true ] }, - "hash": "1c3473a0f9f6b6148b2c975f9f05bdefedf8a51c4e6ddf0eca367b9cc778d051" + "hash": "318ed7a45d8326e3ecbc9727ee3812adaeeead8b39b007a32badadf67ec5ca17" } diff --git a/backend/.sqlx/query-462d2b2822b185a6f51fafcfa957cb3b31ee6b69abae79a214dddba0dee4425c.json b/backend/.sqlx/query-462d2b2822b185a6f51fafcfa957cb3b31ee6b69abae79a214dddba0dee4425c.json deleted file mode 100644 index 0c4d90b073..0000000000 --- a/backend/.sqlx/query-462d2b2822b185a6f51fafcfa957cb3b31ee6b69abae79a214dddba0dee4425c.json +++ /dev/null @@ -1,15 +0,0 @@ -{ - "db_name": "PostgreSQL", - "query": "DELETE FROM flow_conversation_message m\n USING flow_conversation c\n WHERE m.conversation_id = c.id AND c.workspace_id = $1 AND m.job_id = ANY($2)", - "describe": { - "columns": [], - "parameters": { - "Left": [ - "Text", - "UuidArray" - ] - }, - "nullable": [] - }, - "hash": "462d2b2822b185a6f51fafcfa957cb3b31ee6b69abae79a214dddba0dee4425c" -} diff --git a/backend/.sqlx/query-c383cc023714b361d10c10e8fef1fc148ab1da942951ee9ffdddaecee76a6be9.json b/backend/.sqlx/query-48c8522a4fed219c5011f4ba63c81cfe028a8b2a32bd790840cef65c452a8c31.json similarity index 77% rename from backend/.sqlx/query-c383cc023714b361d10c10e8fef1fc148ab1da942951ee9ffdddaecee76a6be9.json rename to backend/.sqlx/query-48c8522a4fed219c5011f4ba63c81cfe028a8b2a32bd790840cef65c452a8c31.json index 56a3642faa..1b50ef4134 100644 --- a/backend/.sqlx/query-c383cc023714b361d10c10e8fef1fc148ab1da942951ee9ffdddaecee76a6be9.json +++ b/backend/.sqlx/query-48c8522a4fed219c5011f4ba63c81cfe028a8b2a32bd790840cef65c452a8c31.json @@ -1,6 +1,6 @@ { "db_name": "PostgreSQL", - "query": "SELECT id, workspace_id, flow_path, title, created_at, updated_at, created_by\n FROM flow_conversation\n WHERE id = $1 AND workspace_id = $2", + "query": "SELECT id, workspace_id, flow_path, title, created_at, updated_at, created_by, is_test\n FROM flow_conversation\n WHERE id = $1 AND workspace_id = $2", "describe": { "columns": [ { @@ -37,6 +37,11 @@ "ordinal": 6, "name": "created_by", "type_info": "Varchar" + }, + { + "ordinal": 7, + "name": "is_test", + "type_info": "Bool" } ], "parameters": { @@ -52,8 +57,9 @@ true, false, false, + false, false ] }, - "hash": "c383cc023714b361d10c10e8fef1fc148ab1da942951ee9ffdddaecee76a6be9" + "hash": "48c8522a4fed219c5011f4ba63c81cfe028a8b2a32bd790840cef65c452a8c31" } diff --git a/backend/.sqlx/query-b1a9a433e577133869c067b2ce383fc6ce4e9df307feb5fd3edc0d1276d61ff1.json b/backend/.sqlx/query-55dbe12954489532644b91e67b2a9df7ede61e050d3e634b795aaf7f60b3b05d.json similarity index 63% rename from backend/.sqlx/query-b1a9a433e577133869c067b2ce383fc6ce4e9df307feb5fd3edc0d1276d61ff1.json rename to backend/.sqlx/query-55dbe12954489532644b91e67b2a9df7ede61e050d3e634b795aaf7f60b3b05d.json index 1818efc0c0..2ae7033d0d 100644 --- a/backend/.sqlx/query-b1a9a433e577133869c067b2ce383fc6ce4e9df307feb5fd3edc0d1276d61ff1.json +++ b/backend/.sqlx/query-55dbe12954489532644b91e67b2a9df7ede61e050d3e634b795aaf7f60b3b05d.json @@ -1,6 +1,6 @@ { "db_name": "PostgreSQL", - "query": "INSERT INTO flow_conversation_message (conversation_id, message_type, content, job_id, step_name, success)\n VALUES ($1, $2, $3, $4, $5, $6)", + "query": "INSERT INTO flow_conversation_message (conversation_id, message_type, content, job_id, step_name, success, tool_arguments, tool_result, reasoning)\n VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9)", "describe": { "columns": [], "parameters": { @@ -21,10 +21,13 @@ "Text", "Uuid", "Varchar", - "Bool" + "Bool", + "Text", + "Text", + "Text" ] }, "nullable": [] }, - "hash": "b1a9a433e577133869c067b2ce383fc6ce4e9df307feb5fd3edc0d1276d61ff1" + "hash": "55dbe12954489532644b91e67b2a9df7ede61e050d3e634b795aaf7f60b3b05d" } diff --git a/backend/.sqlx/query-5b9c9eb64051f291fed4be9bc0b0cc0aef2e7bde732899976eddac36a2da7658.json b/backend/.sqlx/query-5b9c9eb64051f291fed4be9bc0b0cc0aef2e7bde732899976eddac36a2da7658.json new file mode 100644 index 0000000000..8864b75b79 --- /dev/null +++ b/backend/.sqlx/query-5b9c9eb64051f291fed4be9bc0b0cc0aef2e7bde732899976eddac36a2da7658.json @@ -0,0 +1,24 @@ +{ + "db_name": "PostgreSQL", + "query": "UPDATE flow_conversation SET title = $1, updated_at = updated_at\n WHERE id = $2 AND workspace_id = $3\n RETURNING id", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "id", + "type_info": "Uuid" + } + ], + "parameters": { + "Left": [ + "Varchar", + "Uuid", + "Text" + ] + }, + "nullable": [ + false + ] + }, + "hash": "5b9c9eb64051f291fed4be9bc0b0cc0aef2e7bde732899976eddac36a2da7658" +} diff --git a/backend/.sqlx/query-69bfbe9b39414b724488532cc3b3659915d9fcb3d58f16532aaffe06c44ec976.json b/backend/.sqlx/query-69bfbe9b39414b724488532cc3b3659915d9fcb3d58f16532aaffe06c44ec976.json new file mode 100644 index 0000000000..52105ca605 --- /dev/null +++ b/backend/.sqlx/query-69bfbe9b39414b724488532cc3b3659915d9fcb3d58f16532aaffe06c44ec976.json @@ -0,0 +1,23 @@ +{ + "db_name": "PostgreSQL", + "query": "DELETE FROM flow_conversation_message m\n USING flow_conversation c\n WHERE m.conversation_id = c.id AND c.workspace_id = $1 AND m.job_id = ANY($2)\n RETURNING m.conversation_id", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "conversation_id", + "type_info": "Uuid" + } + ], + "parameters": { + "Left": [ + "Text", + "UuidArray" + ] + }, + "nullable": [ + false + ] + }, + "hash": "69bfbe9b39414b724488532cc3b3659915d9fcb3d58f16532aaffe06c44ec976" +} diff --git a/backend/.sqlx/query-6bd23a98838e3eec309e6b696edc776bd56fc9dae1238b3272557d1562400dbe.json b/backend/.sqlx/query-79776c1a15e41edecb1a3cf749ff7e6c10efc85dc73473470078ad28f4c7cee1.json similarity index 72% rename from backend/.sqlx/query-6bd23a98838e3eec309e6b696edc776bd56fc9dae1238b3272557d1562400dbe.json rename to backend/.sqlx/query-79776c1a15e41edecb1a3cf749ff7e6c10efc85dc73473470078ad28f4c7cee1.json index d7f5fc45d4..202267cd13 100644 --- a/backend/.sqlx/query-6bd23a98838e3eec309e6b696edc776bd56fc9dae1238b3272557d1562400dbe.json +++ b/backend/.sqlx/query-79776c1a15e41edecb1a3cf749ff7e6c10efc85dc73473470078ad28f4c7cee1.json @@ -1,6 +1,6 @@ { "db_name": "PostgreSQL", - "query": "INSERT INTO flow_conversation (id, workspace_id, flow_path, created_by, title)\n VALUES ($1, $2, $3, $4, $5)\n RETURNING id, workspace_id, flow_path, title, created_at, updated_at, created_by", + "query": "INSERT INTO flow_conversation (id, workspace_id, flow_path, created_by, title, is_test)\n VALUES ($1, $2, $3, $4, $5, $6)\n RETURNING id, workspace_id, flow_path, title, created_at, updated_at, created_by, is_test", "describe": { "columns": [ { @@ -37,6 +37,11 @@ "ordinal": 6, "name": "created_by", "type_info": "Varchar" + }, + { + "ordinal": 7, + "name": "is_test", + "type_info": "Bool" } ], "parameters": { @@ -45,7 +50,8 @@ "Varchar", "Varchar", "Varchar", - "Varchar" + "Varchar", + "Bool" ] }, "nullable": [ @@ -55,8 +61,9 @@ true, false, false, + false, false ] }, - "hash": "6bd23a98838e3eec309e6b696edc776bd56fc9dae1238b3272557d1562400dbe" + "hash": "79776c1a15e41edecb1a3cf749ff7e6c10efc85dc73473470078ad28f4c7cee1" } diff --git a/backend/.sqlx/query-89ea81b765550cf665e30533efc9672f8c98d2579fc72c07752361cc5fd683dc.json b/backend/.sqlx/query-89ea81b765550cf665e30533efc9672f8c98d2579fc72c07752361cc5fd683dc.json new file mode 100644 index 0000000000..7a7110030c --- /dev/null +++ b/backend/.sqlx/query-89ea81b765550cf665e30533efc9672f8c98d2579fc72c07752361cc5fd683dc.json @@ -0,0 +1,15 @@ +{ + "db_name": "PostgreSQL", + "query": "DELETE FROM ai_agent_memory a\n USING flow_conversation c\n WHERE c.id = ANY($1)\n AND c.workspace_id = $2\n AND a.conversation_id = c.id\n AND a.workspace_id = c.workspace_id\n AND NOT EXISTS (\n SELECT 1 FROM flow_conversation_message m WHERE m.conversation_id = c.id\n )", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "UuidArray", + "Text" + ] + }, + "nullable": [] + }, + "hash": "89ea81b765550cf665e30533efc9672f8c98d2579fc72c07752361cc5fd683dc" +} diff --git a/backend/.sqlx/query-90b910da8d00a7c7bcf29c167e38e44eb1c0062a8241dc8fe0ed3dd95b65f89a.json b/backend/.sqlx/query-90b910da8d00a7c7bcf29c167e38e44eb1c0062a8241dc8fe0ed3dd95b65f89a.json new file mode 100644 index 0000000000..1555f3683d --- /dev/null +++ b/backend/.sqlx/query-90b910da8d00a7c7bcf29c167e38e44eb1c0062a8241dc8fe0ed3dd95b65f89a.json @@ -0,0 +1,14 @@ +{ + "db_name": "PostgreSQL", + "query": "DELETE FROM flow_conversation c\n WHERE c.id = ANY($1)\n AND NOT EXISTS (\n SELECT 1 FROM flow_conversation_message m WHERE m.conversation_id = c.id\n )", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "UuidArray" + ] + }, + "nullable": [] + }, + "hash": "90b910da8d00a7c7bcf29c167e38e44eb1c0062a8241dc8fe0ed3dd95b65f89a" +} diff --git a/backend/.sqlx/query-967f52005f4a044b3a2e9f02ceadf90dad5246681bde6caaa633b85a5e8b2352.json b/backend/.sqlx/query-967f52005f4a044b3a2e9f02ceadf90dad5246681bde6caaa633b85a5e8b2352.json new file mode 100644 index 0000000000..5161f716ff --- /dev/null +++ b/backend/.sqlx/query-967f52005f4a044b3a2e9f02ceadf90dad5246681bde6caaa633b85a5e8b2352.json @@ -0,0 +1,22 @@ +{ + "db_name": "PostgreSQL", + "query": "DELETE FROM flow_conversation_message WHERE job_id = ANY($1) RETURNING conversation_id", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "conversation_id", + "type_info": "Uuid" + } + ], + "parameters": { + "Left": [ + "UuidArray" + ] + }, + "nullable": [ + false + ] + }, + "hash": "967f52005f4a044b3a2e9f02ceadf90dad5246681bde6caaa633b85a5e8b2352" +} diff --git a/backend/.sqlx/query-a1b23f3e62c6433d95cdac58215741a51ca0ce66bf1c674097705cf2e1b72eff.json b/backend/.sqlx/query-a1b23f3e62c6433d95cdac58215741a51ca0ce66bf1c674097705cf2e1b72eff.json new file mode 100644 index 0000000000..d8fdd5140b --- /dev/null +++ b/backend/.sqlx/query-a1b23f3e62c6433d95cdac58215741a51ca0ce66bf1c674097705cf2e1b72eff.json @@ -0,0 +1,14 @@ +{ + "db_name": "PostgreSQL", + "query": "DELETE FROM ai_agent_memory a\n USING flow_conversation c\n WHERE c.id = ANY($1)\n AND a.conversation_id = c.id\n AND a.workspace_id = c.workspace_id\n AND NOT EXISTS (\n SELECT 1 FROM flow_conversation_message m WHERE m.conversation_id = c.id\n )", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "UuidArray" + ] + }, + "nullable": [] + }, + "hash": "a1b23f3e62c6433d95cdac58215741a51ca0ce66bf1c674097705cf2e1b72eff" +} diff --git a/backend/.sqlx/query-bfdd60b42e32bd81e2d20b327462893147b4e5ff078531de36147d908132d636.json b/backend/.sqlx/query-bfdd60b42e32bd81e2d20b327462893147b4e5ff078531de36147d908132d636.json deleted file mode 100644 index f9fbc7a58b..0000000000 --- a/backend/.sqlx/query-bfdd60b42e32bd81e2d20b327462893147b4e5ff078531de36147d908132d636.json +++ /dev/null @@ -1,14 +0,0 @@ -{ - "db_name": "PostgreSQL", - "query": "DELETE FROM flow_conversation_message WHERE job_id = ANY($1)", - "describe": { - "columns": [], - "parameters": { - "Left": [ - "UuidArray" - ] - }, - "nullable": [] - }, - "hash": "bfdd60b42e32bd81e2d20b327462893147b4e5ff078531de36147d908132d636" -} diff --git a/backend/.sqlx/query-e8802be9203c1e88a06e337260ccca029380139f89a01a89033e36a6ed9ac082.json b/backend/.sqlx/query-d6c65ce1d443d1e1783818d36c7eabd9633bbe5219bfd9683f336de19763cb58.json similarity index 62% rename from backend/.sqlx/query-e8802be9203c1e88a06e337260ccca029380139f89a01a89033e36a6ed9ac082.json rename to backend/.sqlx/query-d6c65ce1d443d1e1783818d36c7eabd9633bbe5219bfd9683f336de19763cb58.json index a3374d6cdf..d3d76191f8 100644 --- a/backend/.sqlx/query-e8802be9203c1e88a06e337260ccca029380139f89a01a89033e36a6ed9ac082.json +++ b/backend/.sqlx/query-d6c65ce1d443d1e1783818d36c7eabd9633bbe5219bfd9683f336de19763cb58.json @@ -1,6 +1,6 @@ { "db_name": "PostgreSQL", - "query": "SELECT id, conversation_id, message_type as \"message_type: MessageType\", content, job_id, created_at, created_seq, step_name, success\n FROM flow_conversation_message\n WHERE conversation_id = $1\n AND created_seq > $2\n ORDER BY created_seq ASC\n LIMIT $3\n ", + "query": "SELECT id, conversation_id, message_type as \"message_type: MessageType\", content, job_id, created_at, created_seq, step_name, success, tool_arguments, tool_result, reasoning\n FROM (\n SELECT id, conversation_id, message_type, content, job_id, created_at, created_seq, step_name, success, tool_arguments, tool_result, reasoning\n FROM flow_conversation_message\n WHERE conversation_id = $1\n ORDER BY created_seq DESC\n LIMIT $2 OFFSET $3\n ) AS messages\n ORDER BY created_seq ASC\n ", "describe": { "columns": [ { @@ -58,6 +58,21 @@ "ordinal": 8, "name": "success", "type_info": "Bool" + }, + { + "ordinal": 9, + "name": "tool_arguments", + "type_info": "Text" + }, + { + "ordinal": 10, + "name": "tool_result", + "type_info": "Text" + }, + { + "ordinal": 11, + "name": "reasoning", + "type_info": "Text" } ], "parameters": { @@ -76,8 +91,11 @@ false, false, true, - false + false, + true, + true, + true ] }, - "hash": "e8802be9203c1e88a06e337260ccca029380139f89a01a89033e36a6ed9ac082" + "hash": "d6c65ce1d443d1e1783818d36c7eabd9633bbe5219bfd9683f336de19763cb58" } diff --git a/backend/migrations/20260907131755_flow_conversation_is_test.down.sql b/backend/migrations/20260907131755_flow_conversation_is_test.down.sql new file mode 100644 index 0000000000..aa186105cf --- /dev/null +++ b/backend/migrations/20260907131755_flow_conversation_is_test.down.sql @@ -0,0 +1 @@ +ALTER TABLE flow_conversation DROP COLUMN is_test; diff --git a/backend/migrations/20260907131755_flow_conversation_is_test.up.sql b/backend/migrations/20260907131755_flow_conversation_is_test.up.sql new file mode 100644 index 0000000000..df4ddb8981 --- /dev/null +++ b/backend/migrations/20260907131755_flow_conversation_is_test.up.sql @@ -0,0 +1,27 @@ +-- A chat run from the flow editor's test panel is stored exactly like one from the +-- deployed flow, so the two were indistinguishable once written. Marking them lets the +-- lists tell a trial apart from a real conversation. +ALTER TABLE flow_conversation ADD COLUMN is_test BOOLEAN NOT NULL DEFAULT false; + +-- Existing rows: a conversation whose messages came from a flowpreview run was a test. +-- Derived once here because the job is purged on retention, after which the origin of an +-- old conversation is unknowable. +-- +-- Walked to the root job rather than matched directly: an existing message row never holds +-- the flow job itself. Only this migration's release starts storing it on the user row, and +-- the rows written before it point at the step that produced them — the AI agent's job for +-- an answer, the tool's own job for a tool call — whose kind is never 'flowpreview'. +-- +-- `root_job` first, matching `get_root_job_id` (windmill-worker/src/common.rs): only it +-- reaches the top of the run. `flow_innermost_root_job` stops at the closest flow scope by +-- design, so an agent inside a subflow would land on that subflow's 'flow' row and the +-- conversation would read as deployed. +UPDATE flow_conversation c +SET is_test = true +WHERE EXISTS ( + SELECT 1 FROM flow_conversation_message m + JOIN v2_job j ON j.id = m.job_id + JOIN v2_job root + ON root.id = coalesce(j.root_job, j.flow_innermost_root_job, j.parent_job, j.id) + WHERE m.conversation_id = c.id AND root.kind = 'flowpreview' +); diff --git a/backend/migrations/20260908110914_flow_conversation_message_tool_call.down.sql b/backend/migrations/20260908110914_flow_conversation_message_tool_call.down.sql new file mode 100644 index 0000000000..56f20ba722 --- /dev/null +++ b/backend/migrations/20260908110914_flow_conversation_message_tool_call.down.sql @@ -0,0 +1,3 @@ +ALTER TABLE flow_conversation_message DROP COLUMN tool_arguments; +ALTER TABLE flow_conversation_message DROP COLUMN tool_result; +ALTER TABLE flow_conversation_message DROP COLUMN reasoning; diff --git a/backend/migrations/20260908110914_flow_conversation_message_tool_call.up.sql b/backend/migrations/20260908110914_flow_conversation_message_tool_call.up.sql new file mode 100644 index 0000000000..2ada36e14a --- /dev/null +++ b/backend/migrations/20260908110914_flow_conversation_message_tool_call.up.sql @@ -0,0 +1,9 @@ +-- A tool row's call and result are read back from the tool's own job, which an MCP tool +-- and a provider-native tool never have: they run inside the agent's job. For those the +-- row is the only record, so it carries the call itself. +ALTER TABLE flow_conversation_message ADD COLUMN tool_arguments TEXT; +ALTER TABLE flow_conversation_message ADD COLUMN tool_result TEXT; + +-- The thinking that produced an answer is streamed, never returned in the response body, +-- so it exists nowhere once the stream is over. +ALTER TABLE flow_conversation_message ADD COLUMN reasoning TEXT; diff --git a/backend/summarized_schema.txt b/backend/summarized_schema.txt index 41dd70ca93..444a358740 100644 --- a/backend/summarized_schema.txt +++ b/backend/summarized_schema.txt @@ -97,10 +97,10 @@ email_trigger: path(char), local_part(char), workspaced_local_part(bool), script favorite: usr(char), workspace_id(char), path(char), favorite_kind(favorite_kind) flow: workspace_id(char), path(char), summary(text), description(text), value(jsonb), edited_by(char), edited_at(ts), archived(bool), schema(json), extra_perms(jsonb), dependency_job(uuid), draft_only(bool), tag(char), ws_error_handler_muted(bool), dedicated_worker(bool), timeout(int), visible_to_runner_only(bool), concurrency_key(char), versions(bigint[]), on_behalf_of(varchar), on_behalf_of_email(text), lock_error_logs(text), labels(text[]) FK: (workspace_id) -> workspace(id) -flow_conversation: id(uuid), workspace_id(char), flow_path(char), title(char), created_at(ts), updated_at(ts), created_by(char) +flow_conversation: id(uuid), workspace_id(char), flow_path(char), title(char), created_at(ts), updated_at(ts), created_by(char), is_test(bool) FK: (workspace_id) -> workspace(id) -flow_conversation_message: id(uuid), conversation_id(uuid), message_type(message_type), content(text), job_id(uuid), created_at(ts), created_seq(int8), step_name(char), success(bool) - FK: (conversation_id) -> flow_conversation(id) | (job_id) -> v2_job(id) +flow_conversation_message: id(uuid), conversation_id(uuid), message_type(message_type), content(text), job_id(uuid), created_at(ts), created_seq(int8), step_name(char), success(bool), tool_arguments(text), tool_result(text), reasoning(text) + FK: (conversation_id) -> flow_conversation(id) flow_iterator_data: job_id(uuid), itered(jsonb) flow_node: id(bigint), workspace_id(char), hash(bigint), path(char), lock(text), code(text), flow(jsonb), hash_v2(char(64)) FK: (path, workspace_id) -> flow(path, workspace_id) | (workspace_id) -> workspace(id) diff --git a/backend/tests/v2_job_delete_orphans.rs b/backend/tests/v2_job_delete_orphans.rs index 95cd1673b6..288ad8ccf0 100644 --- a/backend/tests/v2_job_delete_orphans.rs +++ b/backend/tests/v2_job_delete_orphans.rs @@ -37,7 +37,7 @@ async fn seed_side_rows(db: &Pool, ws: &str, job_id: Uuid) -> anyhow:: .bind(ws) .execute(db) .await?; - // created_seq is assigned by a trigger; inserting a value is rejected. + // created_seq is an identity column; supplying a value is rejected. sqlx::query( "INSERT INTO flow_conversation_message (conversation_id, message_type, content, job_id) VALUES ($1, 'assistant', 'hi', $2)", @@ -121,6 +121,90 @@ async fn test_delete_jobs_removes_side_rows(db: Pool) -> anyhow::Resul Ok(()) } +/// (conversation rows, agent-memory rows) for one conversation. +async fn conversation_and_memory_counts( + db: &Pool, + conversation_id: Uuid, +) -> anyhow::Result<(i64, i64)> { + Ok(( + count( + db, + "SELECT count(*) FROM flow_conversation WHERE id = $1", + conversation_id, + ) + .await?, + count( + db, + "SELECT count(*) FROM ai_agent_memory WHERE conversation_id = $1", + conversation_id, + ) + .await?, + )) +} + +/// A conversation outlives the jobs behind its messages until the last one goes: only then +/// are the row and the agent's memory for it left with nothing, and only then are they +/// deleted. Both halves matter — the surviving half is what a single data-modifying CTE +/// would break, since its emptiness check would read the snapshot from before the delete. +#[sqlx::test(fixtures("base"))] +async fn test_delete_jobs_removes_a_conversation_once_its_last_message_goes( + db: Pool, +) -> anyhow::Result<()> { + initialize_tracing().await; + + let first_job = Uuid::new_v4(); + let second_job = Uuid::new_v4(); + insert_job(&db, WS, first_job).await?; + insert_job(&db, WS, second_job).await?; + + let conv_id = Uuid::new_v4(); + sqlx::query( + "INSERT INTO flow_conversation (id, workspace_id, flow_path, created_by) + VALUES ($1, $2, 'f/flow', 'test-user')", + ) + .bind(conv_id) + .bind(WS) + .execute(&db) + .await?; + for job_id in [first_job, second_job] { + sqlx::query( + "INSERT INTO flow_conversation_message (conversation_id, message_type, content, job_id) + VALUES ($1, 'assistant', 'hi', $2)", + ) + .bind(conv_id) + .bind(job_id) + .execute(&db) + .await?; + } + sqlx::query( + "INSERT INTO ai_agent_memory (workspace_id, conversation_id, step_id, messages) + VALUES ($1, $2, 'a', '[]'::jsonb)", + ) + .bind(WS) + .bind(conv_id) + .execute(&db) + .await?; + + let mut conn = db.acquire().await?; + windmill_common::jobs::delete_jobs(&mut conn, &[first_job]).await?; + drop(conn); + assert_eq!( + conversation_and_memory_counts(&db, conv_id).await?, + (1, 1), + "a conversation with a message left must survive, memory included" + ); + + let mut conn = db.acquire().await?; + windmill_common::jobs::delete_jobs(&mut conn, &[second_job]).await?; + drop(conn); + assert_eq!( + conversation_and_memory_counts(&db, conv_id).await?, + (0, 0), + "the last message going should take the conversation and its memory" + ); + Ok(()) +} + #[sqlx::test(fixtures("base"))] async fn test_clear_schedule_removes_side_rows(db: Pool) -> anyhow::Result<()> { initialize_tracing().await; @@ -192,6 +276,74 @@ async fn test_workspace_delete_removes_side_rows(db: Pool) -> anyhow:: Ok(()) } +/// The purge endpoint carries its own copy of the emptied-conversation rule, so it gets the +/// same guard: the conversation and its memory go with the last message, and not before. +#[sqlx::test(fixtures("base"))] +async fn test_jobs_export_delete_removes_a_conversation_once_its_last_message_goes( + db: Pool, +) -> anyhow::Result<()> { + initialize_tracing().await; + + let first_job = Uuid::new_v4(); + let second_job = Uuid::new_v4(); + insert_job(&db, WS, first_job).await?; + insert_job(&db, WS, second_job).await?; + + let conv_id = Uuid::new_v4(); + sqlx::query( + "INSERT INTO flow_conversation (id, workspace_id, flow_path, created_by) + VALUES ($1, $2, 'f/flow', 'test-user')", + ) + .bind(conv_id) + .bind(WS) + .execute(&db) + .await?; + for job_id in [first_job, second_job] { + sqlx::query( + "INSERT INTO flow_conversation_message (conversation_id, message_type, content, job_id) + VALUES ($1, 'assistant', 'hi', $2)", + ) + .bind(conv_id) + .bind(job_id) + .execute(&db) + .await?; + } + sqlx::query( + "INSERT INTO ai_agent_memory (workspace_id, conversation_id, step_id, messages) + VALUES ($1, $2, 'a', '[]'::jsonb)", + ) + .bind(WS) + .bind(conv_id) + .execute(&db) + .await?; + + let server = ApiServer::start(db.clone()).await?; + let port = server.addr.port(); + let purge = |job_id: Uuid| async move { + reqwest::Client::new() + .post(format!("http://localhost:{port}/api/w/{WS}/jobs/delete")) + .header("Authorization", "Bearer SECRET_TOKEN") + .json(&[job_id]) + .send() + .await + }; + + assert!(purge(first_job).await?.status().is_success()); + assert_eq!( + conversation_and_memory_counts(&db, conv_id).await?, + (1, 1), + "a conversation with a message left must survive the purge endpoint too" + ); + + assert!(purge(second_job).await?.status().is_success()); + assert_eq!( + conversation_and_memory_counts(&db, conv_id).await?, + (0, 0), + "the last message going should take the conversation and its memory" + ); + Ok(()) +} + /// The `/jobs/delete` purge endpoint must scope every side-table delete to the path /// workspace. A `test-workspace` admin passing a job id from another workspace must not be /// able to delete that workspace's job or side rows (the side tables no longer cascade, so diff --git a/backend/windmill-ai/src/providers/anthropic.rs b/backend/windmill-ai/src/providers/anthropic.rs index 486b08b739..038aef2c2e 100644 --- a/backend/windmill-ai/src/providers/anthropic.rs +++ b/backend/windmill-ai/src/providers/anthropic.rs @@ -767,6 +767,7 @@ impl QueryBuilder for AnthropicQueryBuilder { let AnthropicSSEParser { accumulated_content, + accumulated_reasoning, accumulated_tool_calls, events_str, annotations, @@ -790,6 +791,7 @@ impl QueryBuilder for AnthropicQueryBuilder { } else { Some(accumulated_content) }, + reasoning: (!accumulated_reasoning.is_empty()).then_some(accumulated_reasoning), tool_calls: accumulated_tool_calls.into_values().collect(), events_str: Some(events_str), annotations, diff --git a/backend/windmill-ai/src/providers/bedrock.rs b/backend/windmill-ai/src/providers/bedrock.rs index cf977f2484..adee6e5053 100644 --- a/backend/windmill-ai/src/providers/bedrock.rs +++ b/backend/windmill-ai/src/providers/bedrock.rs @@ -1188,6 +1188,13 @@ impl BedrockQueryBuilder { Some(accumulated_text) }; + // The block folded for replay is also what the reader sees as thinking. Read out + // before the block itself moves into the tool calls below. + let reasoning_text = reasoning + .as_ref() + .and_then(|r| r.reasoning_text.clone()) + .filter(|t| !t.is_empty()); + let tool_calls = streaming_tool_calls_to_openai( accumulated_tool_calls.into_values().collect(), reasoning, @@ -1195,6 +1202,7 @@ impl BedrockQueryBuilder { Ok(ParsedResponse::Text { content, + reasoning: reasoning_text, tool_calls, events_str: if events_str.is_empty() { None diff --git a/backend/windmill-ai/src/providers/google_ai.rs b/backend/windmill-ai/src/providers/google_ai.rs index 21428903fa..8320ef5661 100644 --- a/backend/windmill-ai/src/providers/google_ai.rs +++ b/backend/windmill-ai/src/providers/google_ai.rs @@ -666,6 +666,7 @@ impl QueryBuilder for GoogleAIQueryBuilder { let GeminiSSEParser { accumulated_content, + accumulated_reasoning, accumulated_tool_calls, mut events_str, stream_event_processor, @@ -698,6 +699,7 @@ impl QueryBuilder for GoogleAIQueryBuilder { } else { Some(accumulated_content) }, + reasoning: (!accumulated_reasoning.is_empty()).then_some(accumulated_reasoning), tool_calls: accumulated_tool_calls.into_values().collect(), events_str: Some(events_str), annotations, diff --git a/backend/windmill-ai/src/providers/openai.rs b/backend/windmill-ai/src/providers/openai.rs index 34185a8852..8fa65dca21 100644 --- a/backend/windmill-ai/src/providers/openai.rs +++ b/backend/windmill-ai/src/providers/openai.rs @@ -538,6 +538,9 @@ impl QueryBuilder for OpenAIQueryBuilder { } else { Some(parser.accumulated_content) }, + // The Responses stream has no reasoning-summary event in + // `OpenAIResponsesSSEEvent`, so nothing thinks out loud on this path yet. + reasoning: None, tool_calls: parser.accumulated_tool_calls.into_values().collect(), events_str: Some(parser.events_str), annotations: parser.annotations, diff --git a/backend/windmill-ai/src/providers/other.rs b/backend/windmill-ai/src/providers/other.rs index bc2d8f2992..b6040d8d79 100644 --- a/backend/windmill-ai/src/providers/other.rs +++ b/backend/windmill-ai/src/providers/other.rs @@ -251,6 +251,7 @@ impl QueryBuilder for OtherQueryBuilder { let OpenAISSEParser { accumulated_content, + accumulated_reasoning, accumulated_tool_calls, mut events_str, stream_event_processor, @@ -277,6 +278,7 @@ impl QueryBuilder for OtherQueryBuilder { } else { Some(accumulated_content) }, + reasoning: (!accumulated_reasoning.is_empty()).then_some(accumulated_reasoning), tool_calls: accumulated_tool_calls.into_values().collect(), events_str: Some(events_str), annotations: Vec::new(), diff --git a/backend/windmill-ai/src/query_builder.rs b/backend/windmill-ai/src/query_builder.rs index c8f5fbb092..60ece300a3 100644 --- a/backend/windmill-ai/src/query_builder.rs +++ b/backend/windmill-ai/src/query_builder.rs @@ -33,6 +33,8 @@ pub struct BuildRequestArgs<'a> { pub enum ParsedResponse { Text { content: Option, + /// The thinking the model streamed before the answer, when it emitted any. + reasoning: Option, tool_calls: Vec, events_str: Option, annotations: Vec, diff --git a/backend/windmill-ai/src/sse.rs b/backend/windmill-ai/src/sse.rs index 96692d0173..986642fd7e 100644 --- a/backend/windmill-ai/src/sse.rs +++ b/backend/windmill-ai/src/sse.rs @@ -135,6 +135,8 @@ pub trait SSEParser { pub struct OpenAISSEParser { pub accumulated_content: String, + /// The thinking streamed before the answer, kept so it can be stored with it. + pub accumulated_reasoning: String, pub accumulated_tool_calls: HashMap, pub events_str: String, pub stream_event_processor: Box, @@ -146,6 +148,7 @@ impl OpenAISSEParser { pub fn new(stream_event_processor: Box) -> Self { Self { accumulated_content: String::new(), + accumulated_reasoning: String::new(), accumulated_tool_calls: HashMap::new(), events_str: String::new(), stream_event_processor, @@ -175,6 +178,7 @@ impl SSEParser for OpenAISSEParser { if let Some(mut choices) = event.choices.filter(|s| !s.is_empty()) { if let Some(delta) = choices.remove(0).delta { if let Some(reasoning) = delta.reasoning_content.filter(|s| !s.is_empty()) { + self.accumulated_reasoning.push_str(&reasoning); let event = StreamingEvent::ReasoningTokenDelta { content: reasoning }; self.stream_event_processor .send(event, &mut self.events_str) @@ -353,6 +357,8 @@ enum ContentBlockState { /// Anthropic SSE Parser for streaming responses pub struct AnthropicSSEParser { pub accumulated_content: String, + /// The thinking streamed before the answer, kept so it can be stored with it. + pub accumulated_reasoning: String, pub accumulated_tool_calls: HashMap, pub events_str: String, pub stream_event_processor: Box, @@ -375,6 +381,7 @@ impl AnthropicSSEParser { pub fn new(stream_event_processor: Box) -> Self { Self { accumulated_content: String::new(), + accumulated_reasoning: String::new(), accumulated_tool_calls: HashMap::new(), events_str: String::new(), stream_event_processor, @@ -455,6 +462,7 @@ impl SSEParser for AnthropicSSEParser { .thinking .get_or_insert_with(String::new) .push_str(&thinking); + self.accumulated_reasoning.push_str(&thinking); self.stream_event_processor .send( StreamingEvent::ReasoningTokenDelta { content: thinking }, @@ -523,6 +531,7 @@ impl SSEParser for AnthropicSSEParser { .thinking .get_or_insert_with(String::new) .push_str(&thinking); + self.accumulated_reasoning.push_str(&thinking); self.stream_event_processor .send( StreamingEvent::ReasoningTokenDelta { content: thinking }, @@ -590,6 +599,8 @@ impl SSEParser for AnthropicSSEParser { /// `windmill_ai::ai_google` so the logic can be shared with the API proxy. pub struct GeminiSSEParser { pub accumulated_content: String, + /// The thinking streamed before the answer, kept so it can be stored with it. + pub accumulated_reasoning: String, pub accumulated_tool_calls: HashMap, pub events_str: String, pub stream_event_processor: Box, @@ -603,6 +614,7 @@ impl GeminiSSEParser { pub fn new(stream_event_processor: Box) -> Self { Self { accumulated_content: String::new(), + accumulated_reasoning: String::new(), accumulated_tool_calls: HashMap::new(), events_str: String::new(), stream_event_processor, @@ -621,6 +633,7 @@ impl SSEParser for GeminiSSEParser { }; if let Some(reasoning) = parsed.reasoning.filter(|s| !s.is_empty()) { + self.accumulated_reasoning.push_str(&reasoning); self.stream_event_processor .send( StreamingEvent::ReasoningTokenDelta { content: reasoning }, diff --git a/backend/windmill-api-flow-conversations/src/lib.rs b/backend/windmill-api-flow-conversations/src/lib.rs index e85af5b83b..e380906298 100644 --- a/backend/windmill-api-flow-conversations/src/lib.rs +++ b/backend/windmill-api-flow-conversations/src/lib.rs @@ -1,6 +1,6 @@ use axum::{ extract::{Path, Query}, - routing::{delete, get}, + routing::{delete, get, post}, Extension, Json, Router, }; use chrono::{DateTime, Utc}; @@ -15,13 +15,14 @@ use windmill_common::{ db::{UserDB, DB}, error::{JsonResult, Result}, flow_conversations::MessageType, - utils::{not_found_if_none, paginate, Pagination}, + utils::{not_found_if_none, paginate, truncate_with_ellipsis, Pagination}, }; pub fn workspaced_service() -> Router { Router::new() .route("/list", get(list_conversations)) .route("/delete/{conversation_id}", delete(delete_conversation)) + .route("/update/{conversation_id}", post(update_conversation)) .route("/{conversation_id}/messages", get(list_messages)) } @@ -36,11 +37,31 @@ pub struct FlowConversationMessage { pub created_seq: i64, pub step_name: Option, pub success: bool, + /// The call behind a tool row whose tool has no job of its own — an MCP tool, or a + /// provider-native one. Read back from the job otherwise, and null here. + pub tool_arguments: Option, + pub tool_result: Option, + /// The thinking that produced an answer, streamed by the provider and stored here + /// because nothing else keeps it. + pub reasoning: Option, +} + +/// Which conversations a listing holds. A test chat was started from the editor's test +/// panel; a deployed one from the flow itself. +#[derive(Deserialize, Default, Clone, Copy)] +#[serde(rename_all = "lowercase")] +pub enum ConversationKind { + Test, + /// The default: a deployed flow's chat should not surface someone's trial runs. + #[default] + Deployed, + All, } #[derive(Deserialize)] pub struct ListConversationsQuery { pub flow_path: Option, + pub kind: Option, } #[derive(Deserialize)] @@ -67,6 +88,7 @@ async fn list_conversations( "created_at", "updated_at", "created_by", + "is_test", ]) .and_where_eq("workspace_id", "?".bind(&w_id)); @@ -74,6 +96,16 @@ async fn list_conversations( sqlb.and_where_eq("flow_path", "?".bind(flow_path)); } + match query.kind.unwrap_or_default() { + ConversationKind::Test => { + sqlb.and_where_eq("is_test", "true"); + } + ConversationKind::Deployed => { + sqlb.and_where_eq("is_test", "false"); + } + ConversationKind::All => {} + } + sqlb.order_by("updated_at", true) .limit(per_page as i64) .offset(offset as i64); @@ -101,7 +133,7 @@ async fn delete_conversation( // Verify the conversation exists and belongs to the user let conversation = sqlx::query_as!( FlowConversation, - "SELECT id, workspace_id, flow_path, title, created_at, updated_at, created_by + "SELECT id, workspace_id, flow_path, title, created_at, updated_at, created_by, is_test FROM flow_conversation WHERE id = $1 AND workspace_id = $2", conversation_id, @@ -148,6 +180,44 @@ async fn delete_conversation( Ok(format!("Conversation {} deleted", conversation_id)) } +#[derive(Deserialize)] +pub struct UpdateConversation { + /// The chat's name. Set from the first message when the chat is created, and left + /// alone afterwards, so a typed one stays typed. + pub title: String, +} + +async fn update_conversation( + authed: ApiAuthed, + Extension(user_db): Extension, + Path((w_id, conversation_id)): Path<(String, Uuid)>, + Json(update): Json, +) -> Result { + // The column is VARCHAR(255) and the helper appends an ellipsis to what it cuts, so the + // bound it takes is three short of the column's. A longer title would otherwise reach + // Postgres as a 22001 and come back a 500. + let title = truncate_with_ellipsis(update.title.trim(), 252); + + let mut tx = user_db.clone().begin(&authed).await?; + + let updated = sqlx::query_scalar!( + "UPDATE flow_conversation SET title = $1, updated_at = updated_at + WHERE id = $2 AND workspace_id = $3 + RETURNING id", + title, + conversation_id, + &w_id + ) + .fetch_optional(&mut *tx) + .await?; + + not_found_if_none(updated, "Conversation", conversation_id.to_string())?; + + tx.commit().await?; + + Ok(format!("Conversation {} updated", conversation_id)) +} + async fn list_messages( authed: ApiAuthed, Extension(user_db): Extension, @@ -178,7 +248,7 @@ async fn list_messages( let messages = if let Some(after_seq) = query.after_seq { sqlx::query_as!( FlowConversationMessage, - r#"SELECT id, conversation_id, message_type as "message_type: MessageType", content, job_id, created_at, created_seq, step_name, success + r#"SELECT id, conversation_id, message_type as "message_type: MessageType", content, job_id, created_at, created_seq, step_name, success, tool_arguments, tool_result, reasoning FROM flow_conversation_message WHERE conversation_id = $1 AND created_seq > $2 @@ -195,9 +265,9 @@ async fn list_messages( // Fetch messages for this conversation, oldest first, but reverse the order of the messages for easy rendering on the frontend sqlx::query_as!( FlowConversationMessage, - r#"SELECT id, conversation_id, message_type as "message_type: MessageType", content, job_id, created_at, created_seq, step_name, success + r#"SELECT id, conversation_id, message_type as "message_type: MessageType", content, job_id, created_at, created_seq, step_name, success, tool_arguments, tool_result, reasoning FROM ( - SELECT id, conversation_id, message_type, content, job_id, created_at, created_seq, step_name, success + SELECT id, conversation_id, message_type, content, job_id, created_at, created_seq, step_name, success, tool_arguments, tool_result, reasoning FROM flow_conversation_message WHERE conversation_id = $1 ORDER BY created_seq DESC diff --git a/backend/windmill-api-jobs/src/execution.rs b/backend/windmill-api-jobs/src/execution.rs index e897bf5eaf..09557c4e0c 100644 --- a/backend/windmill-api-jobs/src/execution.rs +++ b/backend/windmill-api-jobs/src/execution.rs @@ -668,10 +668,17 @@ pub async fn handle_chat_conversation_messages( flow_path: &str, run_query: &RunJobQuery, user_message_raw: Option<&Box>, + job_id: Uuid, + is_test: bool, ) -> error::Result<()> { + // Names the query parameter rather than the field: it is not a flow argument, and + // supplying it as one is the first thing tried on reading `memory_id is required`. let memory_id = run_query.memory_id.ok_or_else(|| { windmill_common::error::Error::BadRequest( - "memory_id is required for chat-enabled flows".to_string(), + "memory_id is required for chat-enabled flows. Pass it as the `memory_id` query \ + parameter, not as a flow argument: it names the conversation the turn belongs to, \ + so a fresh UUID starts one and reusing a UUID continues it." + .to_string(), ) })?; @@ -695,17 +702,22 @@ pub async fn handle_chat_conversation_messages( &authed.username, &user_message, memory_id, + is_test, ) .await?; + // The run this message started. Its args are the only record of what the message + // carried besides its text — attachments and every other flow input — and nothing + // written later points at them: an assistant row holds the AI agent step's job. add_message_to_conversation_tx( tx, memory_id, - None, + Some(job_id), &user_message, MessageType::User, None, true, + None, ) .await?; @@ -826,6 +838,8 @@ pub async fn run_flow<'c>( &flow_path.to_string(), &run_query, args.args.get("user_message"), + uuid, + false, ) .await?; } diff --git a/backend/windmill-api-jobs/src/jobs_export.rs b/backend/windmill-api-jobs/src/jobs_export.rs index 05f5a51bbc..8b80edc349 100644 --- a/backend/windmill-api-jobs/src/jobs_export.rs +++ b/backend/windmill-api-jobs/src/jobs_export.rs @@ -692,16 +692,53 @@ pub async fn delete_jobs( .await? .rows_affected(); - let conversation_message_deleted = sqlx::query!( + // One row per message deleted, so the conversation of a chat losing several appears + // several times: the count is taken before the dedup below. + let mut conversation_ids: Vec = sqlx::query_scalar!( "DELETE FROM flow_conversation_message m USING flow_conversation c - WHERE m.conversation_id = c.id AND c.workspace_id = $1 AND m.job_id = ANY($2)", + WHERE m.conversation_id = c.id AND c.workspace_id = $1 AND m.job_id = ANY($2) + RETURNING m.conversation_id", &w_id, &job_ids ) - .execute(&mut *tx) - .await? - .rows_affected(); + .fetch_all(&mut *tx) + .await?; + let conversation_message_deleted = conversation_ids.len() as u64; + + // Same rule as retention (windmill_common::jobs::delete_jobs): a conversation with no + // messages left goes, and the agent's memory for it with it. + conversation_ids.sort_unstable(); + conversation_ids.dedup(); + if !conversation_ids.is_empty() { + sqlx::query!( + "DELETE FROM ai_agent_memory a + USING flow_conversation c + WHERE c.id = ANY($1) + AND c.workspace_id = $2 + AND a.conversation_id = c.id + AND a.workspace_id = c.workspace_id + AND NOT EXISTS ( + SELECT 1 FROM flow_conversation_message m WHERE m.conversation_id = c.id + )", + &conversation_ids, + &w_id + ) + .execute(&mut *tx) + .await?; + sqlx::query!( + "DELETE FROM flow_conversation c + WHERE c.id = ANY($1) + AND c.workspace_id = $2 + AND NOT EXISTS ( + SELECT 1 FROM flow_conversation_message m WHERE m.conversation_id = c.id + )", + &conversation_ids, + &w_id + ) + .execute(&mut *tx) + .await?; + } // Resolutions are not exported, so a delete-then-reimport of the same UUID would // otherwise resurrect the old annotation on a job that never carried one. diff --git a/backend/windmill-api/openapi.yaml b/backend/windmill-api/openapi.yaml index aee198e92a..a8dda2b5f0 100644 --- a/backend/windmill-api/openapi.yaml +++ b/backend/windmill-api/openapi.yaml @@ -12403,6 +12403,15 @@ paths: in: query schema: type: string + - name: kind + description: which conversations to list - the flow editor's test chats, the deployed flow's own, or both + in: query + schema: + type: string + enum: + - test + - deployed + - all responses: "200": description: flow conversations list @@ -12413,6 +12422,40 @@ paths: items: $ref: "#/components/schemas/FlowConversation" + /w/{workspace}/flow_conversations/update/{conversation_id}: + post: + summary: rename flow conversation + operationId: updateFlowConversation + tags: + - flow_conversations + parameters: + - $ref: "#/components/parameters/WorkspaceId" + - name: conversation_id + description: conversation id + in: path + required: true + schema: + type: string + format: uuid + requestBody: + required: true + content: + application/json: + schema: + type: object + required: [title] + properties: + title: + type: string + description: the chat's name + responses: + "200": + description: flow conversation updated + content: + text/plain: + schema: + type: string + /w/{workspace}/flow_conversations/delete/{conversation_id}: delete: summary: delete flow conversation @@ -27958,7 +28001,7 @@ components: FlowConversation: type: object required: - [id, workspace_id, flow_path, created_at, updated_at, created_by] + [id, workspace_id, flow_path, created_at, updated_at, created_by, is_test] properties: id: type: string @@ -27985,6 +28028,9 @@ components: created_by: type: string description: Username who created the conversation + is_test: + type: boolean + description: Started from the flow editor's test panel rather than a deployed run FlowConversationMessage: type: object @@ -28024,6 +28070,25 @@ components: success: type: boolean description: Whether the message is a success + tool_arguments: + type: string + nullable: true + description: >- + The call, for a tool that runs inside the agent's job and so has none of its + own: an MCP tool, or a provider-native one such as web search. A Windmill tool + is a script or flow with its own job, and its call is read from there instead. + tool_result: + type: string + nullable: true + description: >- + What that same call returned, including the citations of a provider-native web + search. Null on a call that failed, which `success` reports. + reasoning: + type: string + nullable: true + description: >- + The thinking behind this answer. Stored because it reaches the job result only + as `wm_stream`, which a step with `streaming: false` never accumulates. EndpointTool: type: object diff --git a/backend/windmill-api/src/jobs.rs b/backend/windmill-api/src/jobs.rs index 84968ebbcc..e31b12a53f 100644 --- a/backend/windmill-api/src/jobs.rs +++ b/backend/windmill-api/src/jobs.rs @@ -9553,6 +9553,9 @@ async fn run_preview_flow_job( &flow_path, &run_query, user_message.as_ref(), + uuid, + // Run from the editor's test panel: a trial, not a real conversation. + true, ) .await?; } diff --git a/backend/windmill-common/src/flow_conversations.rs b/backend/windmill-common/src/flow_conversations.rs index 21b1f56389..2e111a48bb 100644 --- a/backend/windmill-common/src/flow_conversations.rs +++ b/backend/windmill-common/src/flow_conversations.rs @@ -26,6 +26,8 @@ pub struct FlowConversation { pub created_at: DateTime, pub updated_at: DateTime, pub created_by: String, + /// Started from the flow editor's test panel rather than a deployed run. + pub is_test: bool, } pub async fn get_or_create_conversation_with_id( @@ -35,11 +37,12 @@ pub async fn get_or_create_conversation_with_id( username: &str, title: &str, conversation_id: Uuid, + is_test: bool, ) -> Result { // Check if conversation already exists let existing_conversation = sqlx::query_as!( FlowConversation, - "SELECT id, workspace_id, flow_path, title, created_at, updated_at, created_by + "SELECT id, workspace_id, flow_path, title, created_at, updated_at, created_by, is_test FROM flow_conversation WHERE id = $1 AND workspace_id = $2", conversation_id, @@ -58,14 +61,15 @@ pub async fn get_or_create_conversation_with_id( // Create new conversation with provided ID let conversation = sqlx::query_as!( FlowConversation, - "INSERT INTO flow_conversation (id, workspace_id, flow_path, created_by, title) - VALUES ($1, $2, $3, $4, $5) - RETURNING id, workspace_id, flow_path, title, created_at, updated_at, created_by", + "INSERT INTO flow_conversation (id, workspace_id, flow_path, created_by, title, is_test) + VALUES ($1, $2, $3, $4, $5, $6) + RETURNING id, workspace_id, flow_path, title, created_at, updated_at, created_by, is_test", conversation_id, w_id, flow_path, username, - title + title, + is_test ) .fetch_one(&mut **tx) .await?; @@ -73,6 +77,17 @@ pub async fn get_or_create_conversation_with_id( Ok(conversation) } +/// What a row carries beyond its text, for the parts of a turn no job can be asked for. +/// An MCP or provider-native tool runs inside the agent's job, which holds every call of +/// the turn and nothing tying one to a row. Thinking reaches that job's result only as +/// `wm_stream`, which a step with `streaming: false` never accumulates. +#[derive(Debug, Clone, Default)] +pub struct MessageExtras { + pub tool_arguments: Option, + pub tool_result: Option, + pub reasoning: Option, +} + /// Add a message to a conversation using an existing transaction /// If the conversation doesn't exist, logs a warning and returns Ok (no error thrown) /// This allows memory_id to be used for agent memory without requiring a conversation @@ -84,6 +99,7 @@ pub async fn add_message_to_conversation_tx( message_type: MessageType, step_name: Option<&str>, success: bool, + extras: Option<&MessageExtras>, ) -> Result<()> { // Check if conversation exists first let conversation_exists = sqlx::query!( @@ -104,14 +120,17 @@ pub async fn add_message_to_conversation_tx( // Insert the message sqlx::query!( - "INSERT INTO flow_conversation_message (conversation_id, message_type, content, job_id, step_name, success) - VALUES ($1, $2, $3, $4, $5, $6)", + "INSERT INTO flow_conversation_message (conversation_id, message_type, content, job_id, step_name, success, tool_arguments, tool_result, reasoning) + VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9)", conversation_id, message_type as MessageType, content, job_id, step_name, - success + success, + extras.and_then(|e| e.tool_arguments.as_deref()), + extras.and_then(|e| e.tool_result.as_deref()), + extras.and_then(|e| e.reasoning.as_deref()) ) .execute(&mut **tx) .await?; diff --git a/backend/windmill-common/src/jobs.rs b/backend/windmill-common/src/jobs.rs index 5541615838..fa6bd60df1 100644 --- a/backend/windmill-common/src/jobs.rs +++ b/backend/windmill-common/src/jobs.rs @@ -485,12 +485,45 @@ pub async fn delete_jobs(conn: &mut sqlx::PgConnection, ids: &[uuid::Uuid]) -> e ) .execute(&mut *conn) .await?; - sqlx::query!( - "DELETE FROM flow_conversation_message WHERE job_id = ANY($1)", + let mut conversation_ids: Vec = sqlx::query_scalar!( + "DELETE FROM flow_conversation_message WHERE job_id = ANY($1) RETURNING conversation_id", ids ) - .execute(&mut *conn) + .fetch_all(&mut *conn) .await?; + conversation_ids.sort_unstable(); + conversation_ids.dedup(); + if !conversation_ids.is_empty() { + // A conversation is a view over its messages: once the last one goes with its job, + // the row and the agent's memory for it are all that is left, and nothing else + // collects them — `ai_agent_memory` carries no job id for retention to match on. + // Two statements rather than one CTE: a data-modifying CTE reads the snapshot from + // before the delete above, so every conversation would still look non-empty. + // Memory first, since it reads the conversation row for its workspace. + sqlx::query!( + "DELETE FROM ai_agent_memory a + USING flow_conversation c + WHERE c.id = ANY($1) + AND a.conversation_id = c.id + AND a.workspace_id = c.workspace_id + AND NOT EXISTS ( + SELECT 1 FROM flow_conversation_message m WHERE m.conversation_id = c.id + )", + &conversation_ids + ) + .execute(&mut *conn) + .await?; + sqlx::query!( + "DELETE FROM flow_conversation c + WHERE c.id = ANY($1) + AND NOT EXISTS ( + SELECT 1 FROM flow_conversation_message m WHERE m.conversation_id = c.id + )", + &conversation_ids + ) + .execute(&mut *conn) + .await?; + } sqlx::query!("DELETE FROM zombie_job_counter WHERE job_id = ANY($1)", ids) .execute(&mut *conn) .await?; diff --git a/backend/windmill-worker/src/ai/tools.rs b/backend/windmill-worker/src/ai/tools.rs index 9c15229ba6..cd18482e88 100644 --- a/backend/windmill-worker/src/ai/tools.rs +++ b/backend/windmill-worker/src/ai/tools.rs @@ -33,7 +33,7 @@ use windmill_common::{ client::AuthedClient, db::DB, error::Error, - flow_conversations::MessageType, + flow_conversations::{MessageExtras, MessageType}, flow_status::AgentAction, flows::FlowModuleValue, worker::{to_raw_value, Connection}, @@ -235,9 +235,24 @@ async fn execute_mcp_tool_call( update_flow_status_module_with_actions_success(ctx.db, parent_job, true).await?; } - // Add tool message to conversation if chat_input_enabled + // An MCP tool runs inside the agent's job, whose result holds every call of the + // turn and nothing tying one of them to this row: same job id for all of them, + // no call id on the row. Kept here so the card shows this call — and the row + // names that job, so retention sweeps it with every other row of the turn. let content = format!("Used {} tool", tool_call.function.name); - add_tool_message_to_chat(ctx, None, &content, true).await; + let agent_job_id = ctx.job.id; + add_tool_message_to_chat( + ctx, + Some(agent_job_id), + &content, + true, + Some(MessageExtras { + tool_arguments: Some(tool_call.function.arguments.clone()), + tool_result: Some(result_str), + ..Default::default() + }), + ) + .await; } Err(e) => { let error_msg = format!("MCP tool error: {}", e); @@ -272,7 +287,18 @@ async fn execute_mcp_tool_call( } // Add tool message to conversation if chat_input_enabled - add_tool_message_to_chat(ctx, None, &error_msg, false).await; + let agent_job_id = ctx.job.id; + add_tool_message_to_chat( + ctx, + Some(agent_job_id), + &error_msg, + false, + Some(MessageExtras { + tool_arguments: Some(tool_call.function.arguments.clone()), + ..Default::default() + }), + ) + .await; } } @@ -681,7 +707,7 @@ async fn handle_tool_execution_error( } // Add tool message to conversation if chat_input_enabled (error case) - add_tool_message_to_chat(ctx, Some(job_id), &error_message, false).await; + add_tool_message_to_chat(ctx, Some(job_id), &error_message, false, None).await; Ok(()) } @@ -806,7 +832,7 @@ async fn handle_tool_execution_success( format!("Error executing {}", tool_call.function.name) }; - add_tool_message_to_chat(ctx, Some(job_id), &content, success).await; + add_tool_message_to_chat(ctx, Some(job_id), &content, success, None).await; Ok(()) } @@ -814,9 +840,16 @@ async fn handle_tool_execution_success( /// Add tool message to conversation if chat is enabled async fn add_tool_message_to_chat( ctx: &mut ToolExecutionContext<'_>, + // The job this row belongs to: the tool's own where it has one, else the agent's, which + // is the job it ran inside. Every row names one so that retention collects the whole + // turn — `delete_jobs` removes messages by `job_id = ANY(..)` (there is no FK on the + // column; `drop_v2_job_side_table_cascades` dropped it), and a row naming no job would + // survive every purge and leave a conversation that can never become empty. tool_job_id: Option, content: &str, success: bool, + // Only for a tool with no job of its own; a Windmill tool's call is read from its job. + extras: Option, ) { if ctx.omit_output_from_conversation { return; @@ -852,6 +885,7 @@ async fn add_tool_message_to_chat( MessageType::Tool, &step_name, success, + extras.as_ref(), ) .await { diff --git a/backend/windmill-worker/src/ai/utils.rs b/backend/windmill-worker/src/ai/utils.rs index 51a00e65a7..198e91750c 100644 --- a/backend/windmill-worker/src/ai/utils.rs +++ b/backend/windmill-worker/src/ai/utils.rs @@ -13,7 +13,7 @@ use windmill_common::flows::FlowModuleValue; use windmill_common::{ db::DB, error::Error, - flow_conversations::{add_message_to_conversation_tx, MessageType}, + flow_conversations::{add_message_to_conversation_tx, MessageExtras, MessageType}, flow_status::AgentAction, flows::{InputTransform, Step}, jobs::JobKind, @@ -209,6 +209,7 @@ pub async fn add_message_to_conversation( message_type: MessageType, step_name: &Option, success: bool, + extras: Option<&MessageExtras>, ) -> Result<(), Error> { let mut tx = db.begin().await?; add_message_to_conversation_tx( @@ -219,6 +220,7 @@ pub async fn add_message_to_conversation( message_type, step_name.as_deref(), success, + extras, ) .await?; tx.commit().await?; diff --git a/backend/windmill-worker/src/ai_executor.rs b/backend/windmill-worker/src/ai_executor.rs index cd17c57f59..0d3524beee 100644 --- a/backend/windmill-worker/src/ai_executor.rs +++ b/backend/windmill-worker/src/ai_executor.rs @@ -40,7 +40,7 @@ use windmill_common::{ client::AuthedClient, db::DB, error::{self, Error}, - flow_conversations::MessageType, + flow_conversations::{MessageExtras, MessageType}, flow_status::AgentAction, flows::{AgentTool, FlowModule, FlowModuleValue, InputTransform, ToolValue}, get_latest_hash_for_path, @@ -1355,6 +1355,7 @@ pub async fn run_agent( match parsed { ParsedResponse::Text { content: response_content, + reasoning: response_reasoning, tool_calls, events_str, annotations, @@ -1394,6 +1395,13 @@ pub async fn run_agent( let db_clone = db.clone(); let message_content = "Used websearch tool successfully".to_string(); let step_name = step_name.clone(); + // The search ran inside the provider's call, so this job's args + // describe the agent, not the search: its sources reach the row + // only if they are written here. + let extras = (!annotations.is_empty()).then(|| MessageExtras { + tool_result: serde_json::to_string(&annotations).ok(), + ..Default::default() + }); tokio::spawn(async move { if let Err(e) = add_message_to_conversation( &db_clone, @@ -1403,6 +1411,7 @@ pub async fn run_agent( MessageType::Tool, &step_name, true, + extras.as_ref(), ) .await { @@ -1446,6 +1455,11 @@ pub async fn run_agent( let db_clone = db.clone(); let message_content = response_content.clone(); let step_name = step_name.clone(); + // The thinking is streamed and never returned in a response + // body, so the answer's row is the only place it can be kept. + let extras = response_reasoning.clone().map(|reasoning| { + MessageExtras { reasoning: Some(reasoning), ..Default::default() } + }); // Spawn task because we do not need to wait for the result tokio::spawn(async move { @@ -1457,6 +1471,7 @@ pub async fn run_agent( MessageType::Assistant, &step_name, true, + extras.as_ref(), ) .await { @@ -1471,6 +1486,44 @@ pub async fn run_agent( } } + // An iteration that answered with tool calls has no message row to carry its + // thinking, and the next iteration's row holds only its own. Stored on a row + // of its own so a reader sees what led to the call. + if persist_output_to_conversation + && response_content.as_deref().unwrap_or("").is_empty() + { + if let (Some(memory_id), Some(reasoning)) = + (memory_id, response_reasoning.clone()) + { + let agent_job_id = job.id; + let db_clone = db.clone(); + let step_name = step_name.clone(); + tokio::spawn(async move { + if let Err(e) = add_message_to_conversation( + &db_clone, + &memory_id, + Some(agent_job_id), + "", + MessageType::Assistant, + &step_name, + true, + Some(&MessageExtras { + reasoning: Some(reasoning), + ..Default::default() + }), + ) + .await + { + tracing::warn!( + "Failed to add reasoning message to conversation {}: {}", + memory_id, + e + ); + } + }); + } + } + if tool_calls.is_empty() { break; } else if i == max_iterations - 1 { @@ -1584,6 +1637,7 @@ pub async fn run_agent( MessageType::Assistant, &step_name, true, + None, ) .await { diff --git a/backend/windmill-worker/src/worker_flow.rs b/backend/windmill-worker/src/worker_flow.rs index cbda50c7bc..808e9317ad 100644 --- a/backend/windmill-worker/src/worker_flow.rs +++ b/backend/windmill-worker/src/worker_flow.rs @@ -2235,6 +2235,7 @@ async fn add_tool_message_to_conversation( MessageType::Assistant, None, success, + None, ) .await?; tx.commit().await?; diff --git a/cli/src/guidance/skills.gen.ts b/cli/src/guidance/skills.gen.ts index f646b08f8c..c50c4ee44f 100644 --- a/cli/src/guidance/skills.gen.ts +++ b/cli/src/guidance/skills.gen.ts @@ -5268,9 +5268,59 @@ tool, \`websearch\` for web search. } \`\`\` -- \`provider\` is a static object, not a bare resource string: \`{ "kind": , +- \`provider\` is an object, not a bare resource string: \`{ "kind": , "resource": "$res:", "model": }\`. Required unless the module links to a saved - agent through \`value.agent\` + agent through \`value.agent\`. Static is right for a flow run from a form; a chat flow wires its + fields to flow inputs instead — see below + +### Chat-Mode Flows + +A flow with \`value.chat_input_enabled: true\` is run from a chat instead of a form: the composer +sends one message per turn and renders the conversation. It needs a required \`user_message\` string +input, read by the agent. + +**A static \`provider\` gives a chat that cannot change its model.** Feed it from flow inputs +instead, either way round: one input carrying the whole object (\`"expr": "flow_input.model_config"\`) +makes every field editable, or wire it field by field to fix some and expose others. A field the +chat can write becomes a control in the composer — a provider picker, a model list, a thinking +slider — and a field left static is fixed, with no control drawn for it. Two fields are +exceptions: \`kind\`, which the button writes only together with \`resource\` since a provider is +picked as a pair, and \`reasoning_effort\`, which gets a slider only for a model Windmill has +thinking levels for. Either way the field stays askable under Configure inputs, so nothing +the run needs becomes unreachable. +\`user_attachments\` works the same way: point it at an s3-object input and the composer gets a +paperclip. + +\`\`\`json +{ + "id": "chat_agent", + "value": { + "type": "aiagent", + "input_transforms": { + "provider": { + "type": "javascript", + "expr": "({ kind: 'anthropic', resource: '$res:f/ai/claude', model: flow_input.model, reasoning_effort: flow_input.thinking })" + }, + "user_message": { "type": "javascript", "expr": "flow_input.user_message" }, + "user_attachments": { "type": "javascript", "expr": "flow_input.files" }, + "memory": { "type": "static", "value": { "kind": "auto", "context_length": 10 } }, + "streaming": { "type": "static", "value": true }, + "output_type": { "type": "static", "value": "text" } + }, + "tools": [] + } +} +\`\`\` + +- \`memory\` is what lets the agent see earlier turns; without it every message starts from nothing +- \`streaming\` on makes the answer and its thinking appear token by token instead of all at once +- Running one needs a \`memory_id\` **query parameter** — not a flow argument — naming the + conversation the turn belongs to: a fresh UUID starts one, reusing a UUID continues it. The chat + supplies it itself; a run driven any other way has to pass it or the server refuses the job +- Wiring field by field means one object literal whose values are literals or bare \`flow_input.x\` + references. A spread, a call or a computed key leaves the composer unable to tell which input + feeds which field, so it offers no control at all — a bare \`flow_input.x\` for the whole object + is read instead as that one input carrying every field ### Tool Naming Rules diff --git a/frontend/src/lib/common.ts b/frontend/src/lib/common.ts index 0d76b1a0b8..4f82fd252e 100644 --- a/frontend/src/lib/common.ts +++ b/frontend/src/lib/common.ts @@ -45,6 +45,8 @@ export interface SchemaProperty { required?: string[] showExpr?: string hideWhenChatEnabled?: boolean + /** Why the oneOf variant is chat mode's to pick. Set = selector disabled, reason shown. */ + lockOneOfWhenChatEnabled?: string password?: boolean order?: string[] nullable?: boolean diff --git a/frontend/src/lib/components/AIReasoningEffortPicker.svelte b/frontend/src/lib/components/AIReasoningEffortPicker.svelte index 3ebf94c9bf..39b4e80011 100644 --- a/frontend/src/lib/components/AIReasoningEffortPicker.svelte +++ b/frontend/src/lib/components/AIReasoningEffortPicker.svelte @@ -27,7 +27,7 @@ let capability = $derived( provider && model ? getReasoningCapability(provider, model) - : { supported: false, levels: [], canDisable: false } + : { supported: false, levels: [], canDisable: false, known: false } ) // The token that turns reasoning off on a model that reasons by default diff --git a/frontend/src/lib/components/ArgInput.svelte b/frontend/src/lib/components/ArgInput.svelte index b7001059c2..eaca31c761 100644 --- a/frontend/src/lib/components/ArgInput.svelte +++ b/frontend/src/lib/components/ArgInput.svelte @@ -123,6 +123,8 @@ workspace?: string | undefined s3StorageConfigured?: boolean chatInputEnabled?: boolean + /** Why the oneOf variant is fixed. Set = the selector is disabled and says so. */ + oneOfLockedReason?: string actions?: import('svelte').Snippet innerBottomSnippet?: import('svelte').Snippet fieldHeaderActions?: import('svelte').Snippet @@ -184,6 +186,7 @@ workspace = undefined, s3StorageConfigured = true, chatInputEnabled = false, + oneOfLockedReason = undefined, actions, innerBottomSnippet, fieldHeaderActions, @@ -1104,11 +1107,15 @@ {:else if inputCat == 'object' || inputCat == 'resource-object' || isListJson} {#if oneOf && oneOf.length >= 2}
+ {#if oneOfLockedReason !== undefined} +
{oneOfLockedReason}
+ {/if} {#if oneOf && oneOf.length >= 2} { oneOfSelected = detail const selectedObjProperties = diff --git a/frontend/src/lib/components/EditableSchemaForm.svelte b/frontend/src/lib/components/EditableSchemaForm.svelte index 67dfeed0b8..bd3399d9d2 100644 --- a/frontend/src/lib/components/EditableSchemaForm.svelte +++ b/frontend/src/lib/components/EditableSchemaForm.svelte @@ -43,6 +43,8 @@ interface Props { schema: Schema | any hiddenArgs?: string[] + /** Fields another part of the app owns: shown, but not renameable, deletable or retypeable. */ + lockedArgs?: string[] args?: Record shouldHideNoInputs?: boolean noVariablePicker?: boolean @@ -89,6 +91,7 @@ let { schema = $bindable(), hiddenArgs = [], + lockedArgs = [], args = $bindable(undefined), shouldHideNoInputs = false, noVariablePicker = false, @@ -587,6 +590,7 @@ > {#if keys.length > 0} {#each keys as argName, i (argName)} + {@const locked = lockedArgs.includes(argName)}
@@ -605,7 +609,7 @@ >
{argName} - {#if !uiOnly} + {#if !uiOnly && !locked}
{#snippet trigger()} @@ -654,7 +658,7 @@ Required {/if} - {#if !uiOnly} + {#if !uiOnly && !locked}
{/if} {/if} -
+ +
{#if flowHasChanged()}
[ key, diff --git a/frontend/src/lib/components/InputTransformSchemaForm.svelte b/frontend/src/lib/components/InputTransformSchemaForm.svelte index 465e173f1f..31f2eb0054 100644 --- a/frontend/src/lib/components/InputTransformSchemaForm.svelte +++ b/frontend/src/lib/components/InputTransformSchemaForm.svelte @@ -8,7 +8,7 @@ import type { PickableProperties } from './flows/previousResults' import InputTransformForm from './InputTransformForm.svelte' import InputTransformPickers from './InputTransformPickers.svelte' - import { useS3StorageConfigured } from './inputTransformEnv.svelte' + import { useWorkspaceStorageConfigured } from './inputTransformEnv.svelte' import type ItemPicker from './ItemPicker.svelte' import type VariableEditor from './VariableEditor.svelte' import ResizeTransitionWrapper from './common/ResizeTransitionWrapper.svelte' @@ -86,7 +86,7 @@ let itemPicker: ItemPicker | undefined = $state(undefined) let variableEditor: VariableEditor | undefined = $state(undefined) - const s3Storage = useS3StorageConfigured(() => ws) + const s3Storage = useWorkspaceStorageConfigured(() => ws) let keys: string[] = $state([]) $effect(() => { diff --git a/frontend/src/lib/components/ScrollFade.svelte b/frontend/src/lib/components/ScrollFade.svelte new file mode 100644 index 0000000000..8726ea95ab --- /dev/null +++ b/frontend/src/lib/components/ScrollFade.svelte @@ -0,0 +1,66 @@ + + + diff --git a/frontend/src/lib/components/chat/utils.ts b/frontend/src/lib/components/chat/utils.ts index 60f59fce10..f2fc7c3ac1 100644 --- a/frontend/src/lib/components/chat/utils.ts +++ b/frontend/src/lib/components/chat/utils.ts @@ -1,30 +1,89 @@ +/** + * The AI agent's streamed events, as the worker writes them. + * + * One SSE chunk can carry several lines, so parsing returns a list: a chunk holding a + * tool call and its result must not collapse to whichever came last. Mirrors + * `StreamingEvent` in backend/windmill-ai/src/types.rs (tagged `type`, snake_case). + */ +export type StreamEvent = + | { kind: 'token'; content: string } + | { kind: 'reasoning'; content: string } + | { kind: 'tool_call'; callId: string; name: string } + | { kind: 'tool_arguments'; callId: string; name: string; arguments: string } + | { kind: 'tool_execution'; callId: string; name: string } + | { kind: 'tool_result'; callId: string; name: string; result: string; success: boolean } + +export function parseStreamEvents(streamData: string): StreamEvent[] { + const events: StreamEvent[] = [] + for (const line of streamData.trim().split('\n')) { + if (!line.trim()) continue + let parsed: any + try { + parsed = JSON.parse(line) + } catch (e) { + console.error('Failed to parse stream line:', line, e) + continue + } + switch (parsed?.type) { + case 'token_delta': + if (parsed.content) events.push({ kind: 'token', content: parsed.content }) + break + case 'reasoning_token_delta': + if (parsed.content) events.push({ kind: 'reasoning', content: parsed.content }) + break + case 'tool_call': + events.push({ kind: 'tool_call', callId: parsed.call_id, name: parsed.function_name }) + break + case 'tool_call_arguments': + events.push({ + kind: 'tool_arguments', + callId: parsed.call_id, + name: parsed.function_name, + arguments: parsed.arguments ?? '' + }) + break + case 'tool_execution': + events.push({ kind: 'tool_execution', callId: parsed.call_id, name: parsed.function_name }) + break + case 'tool_result': + events.push({ + kind: 'tool_result', + callId: parsed.call_id, + name: parsed.function_name, + result: parsed.result ?? '', + success: parsed.success !== false + }) + break + } + } + return events +} + +/** One-line summary of a tool call, for a surface with no room for the call itself. */ +export function toolSummary(name: string, success: boolean): string { + return success ? `Used ${name} tool` : `Failed to use ${name} tool` +} + +/** + * Flattened view of a chunk, for callers that render a single running string. + * Keeps the shape AppChat has always consumed. + */ export function parseStreamDeltas(streamData: string): { content: string type?: string success?: boolean } { - const lines = streamData.trim().split('\n') let content = '' let type = 'message' let success = true - - for (const line of lines) { - if (!line.trim()) continue - try { - const parsed = JSON.parse(line) - if (parsed.type === 'tool_result') { - type = 'tool_result' - success = parsed.success - const toolName = parsed.function_name - content = success ? `Used ${toolName} tool` : `Failed to use ${toolName} tool` - } - if (parsed.type === 'token_delta' && parsed.content) { - content += parsed.content - } - } catch (e) { - console.error('Failed to parse stream line:', line, e) + for (const event of parseStreamEvents(streamData)) { + if (event.kind === 'token') { + content += event.content + } else if (event.kind === 'tool_result') { + type = 'tool_result' + success = event.success + content = toolSummary(event.name, event.success) } } - return { content, type, success } } diff --git a/frontend/src/lib/components/common/badge/UnreadCountBadge.svelte b/frontend/src/lib/components/common/badge/UnreadCountBadge.svelte new file mode 100644 index 0000000000..d3d45454f8 --- /dev/null +++ b/frontend/src/lib/components/common/badge/UnreadCountBadge.svelte @@ -0,0 +1,53 @@ + + +{#if count > 0} + + {count > 9 ? '9+' : count} + +{/if} + + diff --git a/frontend/src/lib/components/copilot/ChatModelSettings.svelte b/frontend/src/lib/components/copilot/ChatModelSettings.svelte new file mode 100644 index 0000000000..2cbdc38d61 --- /dev/null +++ b/frontend/src/lib/components/copilot/ChatModelSettings.svelte @@ -0,0 +1,293 @@ + + +{#snippet trigger()} +
+ +
+{/snippet} + +{#snippet section(sec: ChoiceSection, item: MeltItem)} +
{sec.label}
+ {#if sec.loading} +
+ Loading... +
+ {:else if sec.options.length === 0} +
{sec.emptyMessage ?? 'Nothing to choose from'}
+ {:else} +
+ {#each sec.options as option (option.key)} + option.onSelect()}> + {option.label} + {#if option.hint} + {option.hint} + {/if} + {#if option.selected} + + {/if} + + {/each} +
+ {/if} +{/snippet} + +{#snippet rows(items: Item[], item: MeltItem, builders: MeltBuilders)} + {#each items.filter((row) => !row.hide) as row (row.displayName)} + {#if row.separatorTop} +
+ {/if} + {#if row.submenuItems} + + + {:else} + row.action?.(e)}> + {#if row.icon} + + {/if} + {row.displayName} + {#if row.selected} + + {/if} + + {/if} + {/each} +{/snippet} + +{#if config.readOnly} + {@render trigger()} +{:else} + + {#snippet buttonReplacement()} + {@render trigger()} + {/snippet} + {#snippet menu({ item, builders, close })} +
+ {#if config.topItems} +
+ {@render rows(config.topItems(close), item, builders)} +
+ {/if} + {#each config.sections ?? [] as sec (sec.label)} +
+ {@render section(sec, item)} +
+ {/each} + {#if reasoning} +
+ {#if controlState === 'fixed'} + {}} + unsupportedReason={fixedReason} + /> + {:else if controlState === 'awaiting-model'} + {}} + unsupportedReason="Pick a model first" + /> + {:else if controlState === 'unknown'} + +
+
Thinking
+ {#key reasoning.value} + reasoning?.onSelect(e.currentTarget.value.trim()), + // Capture, not bubble: Svelte delegates `keydown` to the root, which sits + // above the menu — so a bubble handler here would run only after melt's own + // listener had read the key as typeahead and moved focus. A capture key is + // not delegatable, so this becomes a real listener on the input and sees the + // event first. + onkeydowncapture: (e) => { + // Escape cancels: let it reach the menu with the value untouched. + if (e.key === 'Escape') return + // Tab closes the menu, unmounting this field before focus moves, so no + // change event would ever fire. Commit on the way past. + if (e.key === 'Tab') { + reasoning?.onSelect(e.currentTarget.value.trim()) + return + } + // Enter means done: commit and close, rather than leaving the menu open + // around a field the commit is about to rebuild. + if (e.key === 'Enter') { + e.preventDefault() + reasoning?.onSelect(e.currentTarget.value.trim()) + close() + return + } + // Everything else is typing; the menu reads loose keys as typeahead. + e.stopPropagation() + } + }} + /> + {/key} +
+ Windmill has no thinking levels for this provider — type what it accepts. +
+
+ {:else if controlState === 'ladder'} + + effortSlider?.adjust(e)} + class="block group" + > + (stop === reasoning?.offToken ? 'off' : stop)} + overrideLabel={stops.includes(currentStop) ? undefined : effortLabel} + /> + + {:else} + + {}} + unsupportedReason="Not supported by this model" + /> + {/if} +
+ {/if} + {#if config.bottomItems} +
+ {@render rows(config.bottomItems(close), item, builders)} +
+ {/if} +
+ {/snippet} + +{/if} diff --git a/frontend/src/lib/components/copilot/ReasoningEffortSlider.svelte b/frontend/src/lib/components/copilot/ReasoningEffortSlider.svelte new file mode 100644 index 0000000000..fb184bf9c4 --- /dev/null +++ b/frontend/src/lib/components/copilot/ReasoningEffortSlider.svelte @@ -0,0 +1,172 @@ + + +{#if unsupportedReason} + +
+
Thinking
+
{unsupportedReason}
+
+{:else} +
+ Thinking + {overrideLabel ?? format(current)} +
+ {#if stops.length > 1} + +
+ onSelect(stops[+e.currentTarget.value])} + onclick={(e) => { + // `click`, not `pointerup`: it is the event that means pressed and released on + // the track, so a press that began on the row above cannot commit an effort + // nobody chose. Only the click that moved nothing — any other stop has already + // committed through `oninput`, and doing it again would write it twice. + if (!hasPosition && +e.currentTarget.value === stopIndex) { + onSelect(stops[stopIndex]) + } + }} + use:isolatePointer + class="lean-range no-default-style w-full" + aria-label="Reasoning effort" + /> +
+ {/if} +{/if} + + diff --git a/frontend/src/lib/components/copilot/chat/AIChatDisplay.svelte b/frontend/src/lib/components/copilot/chat/AIChatDisplay.svelte index 7de7dd32dc..2a81be584f 100644 --- a/frontend/src/lib/components/copilot/chat/AIChatDisplay.svelte +++ b/frontend/src/lib/components/copilot/chat/AIChatDisplay.svelte @@ -35,6 +35,7 @@ import ChatQuickActions from './ChatQuickActions.svelte' import ContextUsageIndicator from './ContextUsageIndicator.svelte' import AIChatModelSettings from './AIChatModelSettings.svelte' + import ScrollFade from '$lib/components/ScrollFade.svelte' import AssistantSettingsModal from './AssistantSettingsModal.svelte' import { SkillsMenu } from './skills/skillsMenu.svelte' import { McpMenu } from '$lib/components/mcp/mcpMenu.svelte' @@ -44,6 +45,7 @@ import Markdown from 'svelte-exmarkdown' import { twMerge } from 'tailwind-merge' import { AIAutonomyMode, AIMode } from './AIChatManager.svelte' + import { getChatViewHost } from './chatViewHost' import { getAiChatManager } from './aiChatManagerContext' import ChatTypingIndicator from './ChatTypingIndicator.svelte' import AIChatInput from './AIChatInput.svelte' @@ -68,6 +70,9 @@ import { base } from '$lib/base' const MAX_YOLO_TOOLTIP_TOOLS = 8 + const chatHost = getChatViewHost() + // The skill and MCP menus take an AIChatManager itself, which the seam deliberately + // doesn't carry. They render only under GLOBAL, which a non-copilot host never sets. const aiChatManager = getAiChatManager() // The user spent their one-time free Windmill AI grant: there is no model left to send @@ -174,8 +179,12 @@ wideLayout = false, emptyHint, inputPreface, + footerSettings, initialInstructions = undefined, - onDraftChange = undefined + onDraftChange = undefined, + placeholder = undefined, + scrollElement = $bindable(), + onTranscriptScroll = undefined }: { messages: DisplayMessage[] pastChats: { id: string; title: string }[] @@ -202,9 +211,18 @@ wideLayout?: boolean emptyHint?: Snippet inputPreface?: Snippet + /** The settings control at the footer's right edge, where the copilot puts its + * model picker. A host that configures its turn elsewhere replaces it here. */ + footerSettings?: Snippet // Seed / observe the main composer's draft text (see AIChatInput). initialInstructions?: string onDraftChange?: (text: string) => void + /** Composer placeholder. Falls back to the per-AI-mode wording. */ + placeholder?: string + /** The transcript's scroll container. A host that paginates older messages + * needs it to measure and restore the scroll position. */ + scrollElement?: HTMLDivElement | undefined + onTranscriptScroll?: () => void } = $props() let aiChatInput: AIChatInput | undefined = $state() @@ -223,7 +241,7 @@ let panelEl: HTMLDivElement | undefined = $state() $effect(() => { function onWindowKeydownCapture(e: KeyboardEvent) { - if (e.key !== 'Escape' || !aiChatManager.loading) return + if (e.key !== 'Escape' || !chatHost.loading) return const active = document.activeElement const focusOnChat = !active || active === document.body || (panelEl?.contains(active) ?? false) @@ -231,22 +249,21 @@ // row alone stops the turn — wherever it is mounted, since the preview panel holds the // form outside `panelEl`. Matched by call: two chats can be loading at once, and one's // row must not answer for the other. - if (aiChatManager.hasPendingRunForm) { + if (chatHost.hasPendingRunForm) { const row = active?.closest('[data-run-form-actions]') const toolCallId = row?.getAttribute('data-run-form-actions') - if (!toolCallId || !aiChatManager.isRunFormPending(toolCallId)) return + if (!toolCallId || !chatHost.isRunFormPending(toolCallId)) return } else if (!focusOnChat) return e.preventDefault() // Immediate form: other chat panels' identical listeners must not // also cancel on body focus, nor a drawer/modal close on this press. e.stopImmediatePropagation() - aiChatManager.cancel() + chatHost.cancel() } window.addEventListener('keydown', onWindowKeydownCapture, true) return () => window.removeEventListener('keydown', onWindowKeydownCapture, true) }) - let scrollEl: HTMLDivElement | undefined = $state() // Programmatic-scroll guard. `scrollDown()` triggers an async `scroll` // event; if a token-append between the scrollTo and the dispatch makes // scrollHeight grow, the gap can briefly exceed STICK_TO_BOTTOM_PX and @@ -259,22 +276,23 @@ // Instant scroll — smooth would animate every token append, racing with // the next scrollDown and confusing the onscroll bottom-detection below. function scrollDown() { - if (!scrollEl) return + if (!scrollElement) return programmaticScrollAt = Date.now() - scrollEl.scrollTo({ top: scrollEl.scrollHeight, behavior: 'auto' }) + scrollElement.scrollTo({ top: scrollElement.scrollHeight, behavior: 'auto' }) } let height = $state(0) $effect(() => { - if (aiChatManager.automaticScroll && height) { + if (chatHost.automaticScroll && height) { scrollDown() } // Recompute the scroll-to-latest visibility on every content-height // change. `onScroll` only fires for actual scroll events, so without // this the arrow can go stale when content grows past the threshold // while auto-scroll is disabled (user scrolled up mid-stream). - if (scrollEl && height) { - const distance = scrollEl.scrollHeight - scrollEl.scrollTop - scrollEl.clientHeight + if (scrollElement && height) { + const distance = + scrollElement.scrollHeight - scrollElement.scrollTop - scrollElement.clientHeight showScrollToLatest = distance > SCROLL_TO_LATEST_THRESHOLD_PX } }) @@ -289,8 +307,9 @@ const SCROLL_TO_LATEST_THRESHOLD_PX = 200 let showScrollToLatest = $state(false) function onScroll() { - if (!scrollEl) return - const distance = scrollEl.scrollHeight - scrollEl.scrollTop - scrollEl.clientHeight + if (!scrollElement) return + const distance = + scrollElement.scrollHeight - scrollElement.scrollTop - scrollElement.clientHeight // Always refresh the arrow visibility — even during the cooldown, // because clicking the arrow itself triggers a programmatic scroll // whose only event would otherwise be swallowed, leaving the arrow @@ -303,14 +322,15 @@ return } if (distance <= STICK_TO_BOTTOM_PX) { - aiChatManager.enableAutomaticScroll() + chatHost.enableAutomaticScroll() } else { - aiChatManager.disableAutomaticScroll() + chatHost.disableAutomaticScroll() } + onTranscriptScroll?.() } function submitSuggestion(suggestion: string) { - aiChatManager.sendRequest({ instructions: suggestion }) + chatHost.sendRequest({ instructions: suggestion }) } export function focusInput() { @@ -319,35 +339,39 @@ $effect(() => { if (aiChatInput) { - aiChatManager.setAiChatInput(aiChatInput) + chatHost.setAiChatInput(aiChatInput) } return () => { - aiChatManager.setAiChatInput(null) + chatHost.setAiChatInput(null) } }) // Also shown for a run held by another tab, labeled with where it is: the // dots say a turn is in flight even before the reader reaches the footer // note. Remote runs pause nothing and offer no Stop — this tab can't cancel. - const showTypingIndicator = $derived(aiChatManager.loading || aiChatManager.runHeldElsewhere) + const showTypingIndicator = $derived(chatHost.loading || chatHost.runHeldElsewhere) // The manual `@` context-picker button. Shown in SCRIPT/FLOW (workspace items + // code blocks) and APP (datatables, frontend files). Hidden in GLOBAL — there // `@`-context is still invoked inline by typing `@` in the input, so the button // is redundant. NAVIGATOR/ASK/API don't take @-context at all. const showContextPicker = $derived( - aiChatManager.mode === AIMode.SCRIPT || - aiChatManager.mode === AIMode.FLOW || - aiChatManager.mode === AIMode.APP + chatHost.mode === AIMode.SCRIPT || chatHost.mode === AIMode.FLOW || chatHost.mode === AIMode.APP ) - // File attachment is GLOBAL-mode only. - const canAttachFiles = $derived(aiChatManager.mode === AIMode.GLOBAL && !disabled) - // Steers the OS file picker toward text + image formats (soft hint; both attach - // to the message — text files after a content sniff). - const TEXT_FILE_ACCEPT = - 'image/*,text/*,.txt,.csv,.tsv,.json,.jsonl,.ndjson,.md,.markdown,.log,.yaml,.yml,.toml,.ini,.cfg,.conf,.env,.xml,.html,.htm,.css,.js,.mjs,.cjs,.ts,.tsx,.jsx,.py,.rb,.rs,.go,.java,.kt,.c,.h,.cpp,.cc,.cs,.php,.sh,.bash,.zsh,.sql,.svelte,.vue,.dockerfile' + // Why attaching is off, when this chat takes attachments but cannot right now. The `+` is + // kept and disabled rather than dropped: the input is the composer's either way, so the + // reader has to be able to see here why nothing can be attached. + const attachmentsOffReason = $derived( + chatHost.supportsMessageAttachments ? chatHost.attachmentsUnavailableReason : undefined + ) + const canAttachFiles = $derived( + chatHost.supportsMessageAttachments && !disabled && !attachmentsOffReason + ) + // Folders are linked as session-wide assets, which only a host that reads files in + // the browser can do — a host running the turn server-side takes attachments only. + const canLinkFolders = $derived(chatHost.supportsLinkedFolders && !disabled) let fileInputEl = $state(null) let folderInputEl = $state(null) let dragDepth = $state(0) @@ -373,12 +397,12 @@ } async function handleAddFiles(files: FileList | FileToAttach[]) { - const { added, rejected } = await aiChatManager.attachedFiles.addFiles(files) + const { added, rejected } = await chatHost.attachedFiles.addFiles(files) reportAddResult(added, rejected) } async function addDirHandle(dir: FileSystemDirectoryHandle) { - const { added, rejected } = await aiChatManager.attachedFiles.addFolder(dir) + const { added, rejected } = await chatHost.attachedFiles.addFolder(dir) reportAddResult(added, rejected) } @@ -467,12 +491,16 @@ handles.length === 0 ? flatFiles : await Promise.all(handles.filter(isFileHandle).map((h) => h.getFile())) - // Loose text files attach to the message, like images. - const textFiles = looseFiles.filter((f) => !isImageFile(f)) - if (textFiles.length > 0) await aiChatInput?.addTextFiles(textFiles) + // Loose files attach to the message, like images. + await attachNonImageFiles(looseFiles.filter((f) => !isImageFile(f))) // Folders link as a live handle. - for (const h of handles.filter(isDirectoryHandle)) { - await addDirHandle(h) + const dirs = handles.filter(isDirectoryHandle) + if (dirs.length > 0 && !canLinkFolders) { + sendUserToast('Folders cannot be attached in this chat — drop individual files.', true) + } else { + for (const h of dirs) { + await addDirHandle(h) + } } } else { // Fallback (no File System Access API): snapshot dropped files AND folders by walking @@ -497,25 +525,35 @@ topLevelText.push(file) } } - if (folderEntries.length > 0) await handleAddFiles(folderEntries) - if (topLevelText.length > 0) await aiChatInput?.addTextFiles(topLevelText) + if (folderEntries.length > 0) { + if (canLinkFolders) await handleAddFiles(folderEntries) + else sendUserToast('Folders cannot be attached in this chat — drop individual files.', true) + } + await attachNonImageFiles(topLevelText) } } async function onFileInputChange(e: Event) { const input = e.currentTarget as HTMLInputElement if (input.files && input.files.length > 0) { - const picked = Array.from(input.files) - const imageFiles = picked.filter(isImageFile) - const textFiles = picked.filter((f) => !isImageFile(f)) - // Reserved before the text work is awaited — see onPanelDrop. - const imageWork = imageFiles.length > 0 ? aiChatInput?.addImages(imageFiles) : undefined - if (textFiles.length > 0) await aiChatInput?.addTextFiles(textFiles) - await imageWork + await attachPickedFiles(Array.from(input.files)) } input.value = '' // allow re-selecting the same file } + async function attachNonImageFiles(files: File[]) { + await aiChatInput?.addNonImageFiles(files) + } + + async function attachPickedFiles(picked: File[]) { + const imageFiles = picked.filter(isImageFile) + const others = picked.filter((f) => !isImageFile(f)) + // Reserved before the other work is awaited — see onPanelDrop. + const imageWork = imageFiles.length > 0 ? aiChatInput?.addImages(imageFiles) : undefined + await attachNonImageFiles(others) + await imageWork + } + function onFolderInputChange(e: Event) { const input = e.currentTarget as HTMLInputElement // webkitdirectory files carry webkitRelativePath (`folder/sub/file`); addFiles groups @@ -524,9 +562,9 @@ input.value = '' } const autonomyAvailability = $derived({ - autoAcceptEditsAvailable: aiChatManager.autoAcceptEditsAvailable, - autoAcceptToolConfirmationsAvailable: aiChatManager.autoAcceptToolConfirmationsAvailable, - planModeAvailable: aiChatManager.planModeAvailable + autoAcceptEditsAvailable: chatHost.autoAcceptEditsAvailable, + autoAcceptToolConfirmationsAvailable: chatHost.autoAcceptToolConfirmationsAvailable, + planModeAvailable: chatHost.planModeAvailable }) const availableAutonomyModeOptions = $derived( autonomyModeOptions.filter((option) => option.isAvailable(autonomyAvailability)) @@ -534,8 +572,8 @@ // Fall back to ask-permission when the persisted mode isn't applicable in the // current AI mode (e.g. auto-accept edits while in a mode without edits). const effectiveAutonomyMode = $derived( - availableAutonomyModeOptions.some((option) => option.mode === aiChatManager.autonomyMode) - ? aiChatManager.autonomyMode + availableAutonomyModeOptions.some((option) => option.mode === chatHost.autonomyMode) + ? chatHost.autonomyMode : AIAutonomyMode.DEFAULT ) const showAutonomyModeSelector = $derived(!disabled && availableAutonomyModeOptions.length > 1) @@ -544,13 +582,25 @@ // The typing-dots indicator implies the AI is busy, which is misleading while // the loop is parked on the user; surface a text pill instead so users know to // act on the tool above. - const waitingForUserAction = $derived(aiChatManager.loading && !!pendingUserAction(messages)) + /** + * An agent step's answer hangs its icon in the margin beside the text, so the column has + * to carry enough padding for it to land in. Widened on both sides, not just the left: + * the column is centred, and padding one side alone would shift the text off centre. + */ + const agentGutter = $derived(messages.some((m) => m.role === 'assistant' && m.stepName)) + const columnClass = $derived( + wideLayout + ? `w-full max-w-3xl mx-auto ${agentGutter ? 'px-8' : 'px-7'}` + : `w-full max-w-2xl mx-auto ${agentGutter ? 'px-8' : 'px-3'}` + ) + + const waitingForUserAction = $derived(chatHost.loading && !!pendingUserAction(messages)) // Gated on `loading` because a card restored from history still looks parked: // its resolver left with the old page, so the composer must not advertise an // answer it cannot deliver. const pendingQuestionToolCallId = $derived.by(() => { - if (!aiChatManager.loading) { + if (!chatHost.loading) { return undefined } const pending = pendingUserActionDetail(messages) @@ -559,14 +609,14 @@ // Get app context for display when in APP mode const appContext = $derived.by((): SelectedContext | undefined => { - if (aiChatManager.mode !== AIMode.APP || !aiChatManager.appAiChatHelpers) { + if (chatHost.mode !== AIMode.APP || !chatHost.appAiChatHelpers) { return undefined } - return aiChatManager.appAiChatHelpers.getSelectedContext() + return chatHost.appAiChatHelpers.getSelectedContext() }) const yoloBypassedTools = $derived.by(() => { - return aiChatManager.tools + return chatHost.tools .filter((tool) => tool.requiresConfirmation === true || tool.bypassedByAutoAccept === true) .map((tool) => ({ name: tool.def.function.name, @@ -583,8 +633,7 @@ Math.max(0, yoloBypassedTools.length - visibleYoloBypassedTools.length) ) const showFlowPendingActionControls = $derived( - (aiChatManager.flowAiChatHelpers?.hasPendingChanges() ?? false) && - !aiChatManager.autoAcceptEditsActive + (chatHost.flowAiChatHelpers?.hasPendingChanges() ?? false) && !chatHost.autoAcceptEditsActive ) // A disabled state with no message (a remote hold, a spent free grant) keeps // the footer toolbar in place — swapping it for an empty strip would make @@ -592,11 +641,16 @@ // a real message (archived, AI off) still shows it, hold or not, matching // the precedence disabledMessage itself encodes. const footerMessageShown = $derived(disabled && disabledMessage !== '') + // `canAttachFiles` belongs in the group too: in GLOBAL mode the `+` always has the + // context picker or the autonomy selector beside it, but a host with attachments and + // nothing else would lose the group and the `+` with it. const showFooterLeftControls = $derived( !footerMessageShown && - (showContextPicker || + (canAttachFiles || + attachmentsOffReason !== undefined || + showContextPicker || showAutonomyModeSelector || - (aiChatManager.mode === AIMode.SCRIPT && hasDiff)) + (chatHost.mode === AIMode.SCRIPT && hasDiff)) ) @@ -694,12 +748,12 @@ the panel, or the Escape-to-stop focus check would wrongly reject them. --> {#each pastChats as chat (chat.id)}
{:else}
- {#if aiChatManager.mode === AIMode.GLOBAL} + {#if chatHost.mode === AIMode.GLOBAL} {/if} {#if !hideModeSelector} {/if} - {#if aiChatManager.mode === AIMode.APP} + {#if chatHost.mode === AIMode.APP} {/if} - - - {#if aiChatManager.mode === AIMode.GLOBAL} + {#if chatHost.supportsModelSettings} + + + {/if} + {@render footerSettings?.()} + {#if chatHost.mode === AIMode.GLOBAL} {/if} - {#if aiChatManager.mode === AIMode.APP && appContext && (appContext.inspectorElement || appContext.codeSelection)} + {#if chatHost.mode === AIMode.APP && appContext && (appContext.inspectorElement || appContext.codeSelection)} {#if appContext.inspectorElement}
{/if}
- {#if (aiChatManager.mode === AIMode.NAVIGATOR || aiChatManager.mode === AIMode.ASK) && suggestions.length > 0 && messages.filter((m) => m.role === 'user').length === 0 && !disabled} + {#if (chatHost.mode === AIMode.NAVIGATOR || chatHost.mode === AIMode.ASK) && suggestions.length > 0 && messages.filter((m) => m.role === 'user').length === 0 && !disabled}
{#each suggestions as suggestion (suggestion)} diff --git a/frontend/src/lib/components/copilot/chat/AIChatInput.svelte b/frontend/src/lib/components/copilot/chat/AIChatInput.svelte index 7f4514cd41..d15c3ecf88 100644 --- a/frontend/src/lib/components/copilot/chat/AIChatInput.svelte +++ b/frontend/src/lib/components/copilot/chat/AIChatInput.svelte @@ -11,7 +11,10 @@ type ContextElement } from './context' import { AIMode } from './AIChatManager.svelte' - import { CHAT_INPUT_PADDING, getAiChatManager } from './aiChatManagerContext' + import { CHAT_INPUT_PADDING } from './aiChatManagerContext' + import { getChatViewHost } from './chatViewHost' + import { composerBoxClass, COMPOSER_FIELD_RESET } from './composerBox' + import { getAiChatManager } from './aiChatManagerContext' import { formatMention } from './mention' import { twMerge } from 'tailwind-merge' import { tick, untrack, type Snippet } from 'svelte' @@ -40,12 +43,22 @@ textByteLength, type AttachedTextFile } from './textFileUtils' + import { + fileToAttachedBlob, + matchesAccept, + MAX_ATTACHED_BLOBS, + MAX_BLOB_BYTES, + type AttachedBlob + } from './blobUtils' import { MessageDraft } from './messageDraft.svelte' import ExpandableImage, { isImageViewerOpen } from '$lib/components/common/image/ExpandableImage.svelte' - const aiChatManager = getAiChatManager() + const chatHost = getChatViewHost() + // Resolved here, not where it is used: getContext is only legal during component + // initialisation, and the mention consumer below runs inside the send gesture. + const chatManager = getAiChatManager() interface Props { availableContext: ContextElement[] @@ -65,15 +78,15 @@ showContext?: boolean bottomRightSnippet?: Snippet onKeyDown?: (e: KeyboardEvent) => void - // When provided, overrides `aiChatManager.loading` for the send/stop + // When provided, overrides `chatHost.loading` for the send/stop // button — useful for callers driving their own request lifecycle // (e.g. the inline ⌘K widget runs requests outside the global - // `aiChatManager.loading` flag). + // `chatHost.loading` flag). loading?: boolean - // Called when the user clicks Stop. Defaults to `aiChatManager.cancel()`. + // Called when the user clicks Stop. Defaults to `chatHost.cancel()`. onCancel?: () => void // Observe the composer draft as it changes (the text is local state — - // `aiChatManager.instructions` only carries programmatic prompts). Used by + // `chatHost.instructions` only carries programmatic prompts). Used by // sessions to persist the typed-but-unsent prompt with the session draft. onDraftChange?: (text: string) => void // tool_call_id of the askUserQuestion the turn is parked on, when it is. A @@ -132,7 +145,7 @@ // The composer unlocks by itself when the other tab's turn ends, so the // placeholder names what it is waiting on (the typing indicator says // where the run is). - if (aiChatManager.runHeldElsewhere) { + if (chatHost.runHeldElsewhere) { return 'Waiting for the turn in the other tab to finish' } if (pendingQuestionToolCallId !== undefined) { @@ -147,7 +160,7 @@ return placeholder } - switch (aiChatManager.mode) { + switch (chatHost.mode) { case AIMode.SCRIPT: return 'Modify this script...' case AIMode.FLOW: @@ -208,31 +221,88 @@ : undefined ) + /** + * Free slots in one attachment lane, against both that lane's own cap and any limit the + * host's consumer imposes on the turn as a whole — a flow input holding a single file + * caps images and blobs together, not one each. In-flight decodes count: two drops that + * both read the staged count before either resolves would claim the same slots twice. + */ + function attachmentSlots(laneCap: number, laneStaged: number): number { + const laneRemaining = laneCap - laneStaged + const turnCap = chatHost.maxMessageAttachments + if (turnCap === undefined) return laneRemaining + const staged = + draft.images.length + + pendingImages + + draft.files.length + + pendingFiles + + draft.blobs.length + + pendingBlobs + // A queue counts too: what is held mid-run merges into one turn on flush, so a + // second file accepted now would be dropped there instead of refused here. + const queued = + chatHost.queuedImages.length + chatHost.queuedFiles.length + chatHost.queuedBlobs.length + return Math.min(laneRemaining, Math.max(0, turnCap - staged - queued)) + } + + /** What to say when the host's own limit is the one that bit. */ + function turnCapMessage(): string { + const turnCap = chatHost.maxMessageAttachments + return turnCap === 1 + ? 'This chat sends one attachment per message.' + : `This chat sends up to ${turnCap} attachments per message.` + } + + /** Why some of what was picked did not fit, naming whichever limit actually bit. */ + function skippedMessage(laneCap: number, lane: 'images' | 'files', skipped: number): string { + return chatHost.maxMessageAttachments !== undefined + ? `${turnCapMessage()} ${skipped} file(s) were not attached.` + : `You can attach up to ${laneCap} ${lane}; ${skipped} were skipped.` + } + // Images being decoded right now. Holds off sending so a message can never go // out without an attachment the user already dropped, and reserves cap slots // against a concurrent drop. let pendingImages = $state(0) - /** Attach dropped/pasted image files (downscaled + bounded). GLOBAL mode only. */ + /** Attach dropped/pasted image files (downscaled + bounded). */ export async function addImages(files: (File | Blob)[]) { - if (aiChatManager.mode !== AIMode.GLOBAL) return + if (!chatHost.supportsMessageAttachments) return + // Attaching can be off despite the chat taking attachments — no object storage to + // upload to, say. The `+` renders disabled with the reason; a drop and a paste reach + // here instead, and would otherwise become a chip that only fails once sent. + const unavailable = chatHost.attachmentsUnavailableReason + if (unavailable) { + sendUserToast(unavailable, true) + return + } const imageFiles = files.filter(isImageFile) if (imageFiles.length === 0) return - // tryGetCurrentModel returns undefined instead of throwing: this runs from a - // drop/paste handler that can't surface a rejection. - const model = tryGetCurrentModel() - // Only known text-only models fail this, so attaching would certainly 400 the - // next turn — refuse rather than warn and send it anyway. - if (model && !modelSupportsVision(model.provider, model.model)) { - sendUserToast(`${model.model} can't read images. Switch to a vision model first.`, true) - return + // The vision check is about the model this composer's own turn will hit, so it + // only applies to a host that picks that model. Elsewhere the model is chosen + // in the flow and tryGetCurrentModel would answer for the wrong one. + if (chatHost.supportsModelSettings) { + // tryGetCurrentModel returns undefined instead of throwing: this runs from a + // drop/paste handler that can't surface a rejection. + const model = tryGetCurrentModel() + // Only known text-only models fail this, so attaching would certainly 400 the + // next turn — refuse rather than warn and send it anyway. + if (model && !modelSupportsVision(model.provider, model.model)) { + sendUserToast(`${model.model} can't read images. Switch to a vision model first.`, true) + return + } } // Count decodes already in flight: two drops that both read the image count // before either resolves would each claim the same free slots and overshoot // the cap. - const remaining = MAX_ATTACHED_IMAGES - draft.images.length - pendingImages + const remaining = attachmentSlots(MAX_ATTACHED_IMAGES, draft.images.length + pendingImages) if (remaining <= 0) { - sendUserToast(`You can attach up to ${MAX_ATTACHED_IMAGES} images.`, true) + sendUserToast( + chatHost.maxMessageAttachments !== undefined + ? turnCapMessage() + : `You can attach up to ${MAX_ATTACHED_IMAGES} images.`, + true + ) return } const oversized = imageFiles.filter((f) => f.size > MAX_IMAGE_BYTES) @@ -245,7 +315,7 @@ const batch = usable.slice(0, remaining) if (batch.length < usable.length) { sendUserToast( - `You can attach up to ${MAX_ATTACHED_IMAGES} images; ${usable.length - batch.length} were skipped.`, + skippedMessage(MAX_ATTACHED_IMAGES, 'images', usable.length - batch.length), true ) } @@ -308,17 +378,22 @@ draft.files.reduce((sum, f) => sum + textByteLength(f.content), 0) + pendingFileBytes ) $effect(() => { - aiChatManager.setComposerStaged(composerKey, editingMessageIndex, stagedBytes) + chatHost.setComposerStaged(composerKey, editingMessageIndex, stagedBytes) }) - $effect(() => () => aiChatManager.clearComposerStaged(composerKey)) + $effect(() => () => chatHost.clearComposerStaged(composerKey)) - /** Attach dropped/picked text files (sniffed + bounded). GLOBAL mode only. */ + /** Attach dropped/picked text files (sniffed + bounded). */ export async function addTextFiles(candidates: File[]) { - if (aiChatManager.mode !== AIMode.GLOBAL) return + if (!chatHost.supportsMessageAttachments) return if (candidates.length === 0) return - const remaining = MAX_ATTACHED_FILES - draft.files.length - pendingFiles + const remaining = attachmentSlots(MAX_ATTACHED_FILES, draft.files.length + pendingFiles) if (remaining <= 0) { - sendUserToast(`You can attach up to ${MAX_ATTACHED_FILES} files.`, true) + sendUserToast( + chatHost.maxMessageAttachments !== undefined + ? turnCapMessage() + : `You can attach up to ${MAX_ATTACHED_FILES} files.`, + true + ) return } const oversized = candidates.filter((f) => f.size > MAX_TEXT_FILE_BYTES) @@ -333,10 +408,7 @@ if (usable.length === 0) return let batch = usable.slice(0, remaining) if (batch.length < usable.length) { - sendUserToast( - `You can attach up to ${MAX_ATTACHED_FILES} files; ${usable.length - batch.length} were skipped.`, - true - ) + sendUserToast(skippedMessage(MAX_ATTACHED_FILES, 'files', usable.length - batch.length), true) } // Conversation-level byte budget: transcript + queue + every live // composer's stage (this one and, mid-edit, the other) + this composer's @@ -346,7 +418,7 @@ // stage stands in for it, so counting both would charge those bytes twice. let budget = MAX_CONVERSATION_FILE_BYTES - - aiChatManager.attachmentBytesExcluding(composerKey) - + chatHost.attachmentBytesExcluding(composerKey) - draft.files.reduce((sum, f) => sum + textByteLength(f.content), 0) - pendingFileBytes const withinBudget: File[] = [] @@ -388,7 +460,7 @@ // from the budget — the decoded sizes replace it. const liveBudget = MAX_CONVERSATION_FILE_BYTES - - aiChatManager.attachmentBytesExcluding(composerKey) - + chatHost.attachmentBytesExcluding(composerKey) - draft.files.reduce((sum, f) => sum + textByteLength(f.content), 0) - (pendingFileBytes - reservedBytes) const { droppedAtBudget } = draft.addFiles(reads, liveBudget) @@ -410,6 +482,86 @@ draft.files = draft.files.filter((_, i) => i !== index) } + // Blobs being read right now — same send-hold/slot-reservation role as pendingImages. + let pendingBlobs = $state(0) + + /** + * Attach non-image files through the lane the host actually reads: a host that decodes + * them takes text, one that forwards them verbatim (to object storage) takes blobs, and + * its narrower `accept` is re-applied because a drop and a paste both bypass the picker's + * own filtering. Every way of attaching goes through here, so no route can take the lane + * the host ignores and drop the file at send. + */ + export async function addNonImageFiles(files: File[]) { + if (files.length === 0) return + // Attaching can be off despite the chat taking attachments — no object storage to + // upload to, say. The `+` renders disabled with the reason; a drop and a paste reach + // here instead, and would otherwise become a chip that only fails once sent. + const unavailable = chatHost.attachmentsUnavailableReason + if (unavailable) { + sendUserToast(unavailable, true) + return + } + if (!chatHost.attachmentsAsBlobs) { + await addTextFiles(files) + return + } + const allowed = files.filter((f) => matchesAccept(f, chatHost.attachmentAccept)) + if (allowed.length < files.length) { + sendUserToast( + `${files.length - allowed.length} file(s) skipped — this chat accepts ${chatHost.attachmentAccept}.`, + true + ) + } + await addBlobs(allowed) + } + + /** Attach files the host takes verbatim (a PDF, say). Kept out of addTextFiles: + * that one decodes to a string and drops anything the binary sniff rejects. */ + export async function addBlobs(candidates: File[]) { + if (!chatHost.supportsMessageAttachments) return + if (candidates.length === 0) return + const oversized = candidates.filter((f) => f.size > MAX_BLOB_BYTES) + if (oversized.length > 0) { + const mb = Math.round(MAX_BLOB_BYTES / 1_000_000) + sendUserToast(`${oversized.length} file(s) over ${mb}MB were skipped.`, true) + } + const usable = candidates.filter((f) => f.size <= MAX_BLOB_BYTES) + if (usable.length === 0) return + const remaining = attachmentSlots(MAX_ATTACHED_BLOBS, draft.blobs.length + pendingBlobs) + if (remaining <= 0) { + sendUserToast( + chatHost.maxMessageAttachments !== undefined + ? turnCapMessage() + : `You can attach up to ${MAX_ATTACHED_BLOBS} files.`, + true + ) + return + } + const batch = usable.slice(0, remaining) + if (batch.length < usable.length) { + sendUserToast(skippedMessage(MAX_ATTACHED_BLOBS, 'files', usable.length - batch.length), true) + } + pendingBlobs += batch.length + try { + const added: AttachedBlob[] = [] + for (const file of batch) { + try { + added.push(await fileToAttachedBlob(file)) + } catch (e) { + sendUserToast(`Could not read ${file.name}`, true) + } + } + if (added.length > 0) draft.addBlobs(added) + } finally { + pendingBlobs -= batch.length + } + } + + function removeBlob(index: number) { + draft.blobs = draft.blobs.filter((_, i) => i !== index) + } + // App mode @ mention state let showAppContextTooltip = $state(false) let appContextTooltipWord = $state('') @@ -420,9 +572,9 @@ // Modes that show the rich textarea with @-context support (workspace // scripts, workspace flows, code blocks, DBs, etc.). const isContextEnabledMode = $derived( - aiChatManager.mode === AIMode.SCRIPT || - aiChatManager.mode === AIMode.FLOW || - aiChatManager.mode === AIMode.GLOBAL + chatHost.mode === AIMode.SCRIPT || + chatHost.mode === AIMode.FLOW || + chatHost.mode === AIMode.GLOBAL ) const domSelectorChips = $derived( @@ -491,7 +643,8 @@ // Attachments still decoding/reading (or mid-drop-routing) count as // occupancy too — they belong to a draft the user started even though // their lane is still empty. - if (pendingImages > 0 || pendingFiles > 0 || ingestionHolds > 0) return false + if (pendingImages > 0 || pendingFiles > 0 || pendingBlobs > 0 || ingestionHolds > 0) + return false if ( !draft.replaceIfEmpty({ text: value, @@ -514,15 +667,17 @@ export function prependText( text: string, restoredImages: AttachedImage[] = [], - restoredFiles: AttachedTextFile[] = [] + restoredFiles: AttachedTextFile[] = [], + restoredBlobs: AttachedBlob[] = [] ): boolean { // mergedIntoDraft: the restored text landed on top of a draft the user was // already writing — both instructions now share one composer, so the caller // must keep both their contexts rather than replacing one with the other. - const { mergedIntoDraft, droppedImages, droppedFiles } = draft.prepend({ + const { mergedIntoDraft, droppedImages, droppedFiles, droppedBlobs } = draft.prepend({ text, images: restoredImages, - files: restoredFiles + files: restoredFiles, + blobs: restoredBlobs }) if (droppedImages > 0) { sendUserToast( @@ -536,6 +691,12 @@ true ) } + if (droppedBlobs > 0) { + sendUserToast( + `You can attach up to ${MAX_ATTACHED_BLOBS} files; ${droppedBlobs} restored file(s) were dropped.`, + true + ) + } focusInput() return mergedIntoDraft } @@ -551,14 +712,14 @@ * the composer. The conversation is left untouched — resending creates a new * message, unlike the bubble's edit pencil which rewinds the conversation. */ function recallLastSentMessage(): boolean { - const messages = aiChatManager.displayMessages + const messages = chatHost.displayMessages for (let i = messages.length - 1; i >= 0; i--) { const message = messages[i] if (message.role !== 'user' || message.synthetic) continue // Images come from the stored turn, never the bubble: a provider // rejection strips them from history while the bubble keeps its copy, // and recalling that copy would re-attach the refused image. - const images = aiChatManager.storedImages(i) ?? [] + const images = chatHost.storedImages(i) ?? [] // Eligibility looks at the bubble, though: the last thing the user // actually sent is the recall boundary, so a context-only turn (GLOBAL // allows text-free sends with chips) recalls its chips, and a turn @@ -582,8 +743,7 @@ // count against the conversation budget — re-admit them instead of // copying, or resending would blow past MAX_CONVERSATION_FILE_BYTES. if (message.files?.length) { - const budget = - MAX_CONVERSATION_FILE_BYTES - aiChatManager.attachmentBytesExcluding(composerKey) + const budget = MAX_CONVERSATION_FILE_BYTES - chatHost.attachmentBytesExcluding(composerKey) const { droppedAtBudget } = draft.addFiles(message.files, budget) if (droppedAtBudget > 0) { const mb = Math.round(MAX_CONVERSATION_FILE_BYTES / 1_000_000) @@ -654,10 +814,10 @@ if ( contextElement.type === 'app_datatable' && - aiChatManager.mode === AIMode.APP && - aiChatManager.appAiChatHelpers + chatHost.mode === AIMode.APP && + chatHost.appAiChatHelpers ) { - const appAiChatHelpers = aiChatManager.appAiChatHelpers + const appAiChatHelpers = chatHost.appAiChatHelpers appAiChatHelpers.addTableToWhitelist( contextElement.datatableName, contextElement.schemaName, @@ -699,14 +859,21 @@ * consuming past them would hand this message a mention the user picked for * the next one. */ function consumeMentionsIfGlobal() { - if (aiChatManager.mode !== AIMode.GLOBAL) return - aiChatManager.contextManager?.consumeMentionContext() + if (chatHost.mode !== AIMode.GLOBAL) return + // The mention context belongs to the copilot's own ContextManager, which only + // the manager has — the GLOBAL guard above means this host is always it. + chatManager.contextManager?.consumeMentionContext() } function sendRequest() { // The send button is disabled while decoding, but Enter reaches here directly. // Sending now would drop the in-flight attachments onto the following message. - if (pendingImages > 0 || pendingFiles > 0 || ingestionHolds > 0) { + if (pendingImages > 0 || pendingFiles > 0 || pendingBlobs > 0 || ingestionHolds > 0) { + return + } + // A host whose consumer needs a message of its own refuses an attachment-only + // turn. Returning before `take()` keeps the chips where the user put them. + if (chatHost.requiresMessageText && draft.text.trim() === '') { return } // Read before `take()` empties the draft the id derives from, and only take @@ -715,7 +882,7 @@ const answeredQuestionId = questionAnsweredBySend if ( answeredQuestionId && - aiChatManager.handleUserQuestionAnswer(answeredQuestionId, [ + chatHost.handleUserQuestionAnswer(answeredQuestionId, [ expanded(chatDraft(draft.text.trim(), draft.pastes)) ]) ) { @@ -727,7 +894,7 @@ contextTextareaComponent?.clearForSend() return } - if (aiChatManager.loading) { + if (chatHost.loading) { // Queue the message instead of silently discarding it — it is // auto-sent when the streaming turn completes successfully. // Editing-while-loading keeps the old discard behavior. Paste @@ -738,14 +905,15 @@ // chips picked at press time. if ( editingMessageIndex === null && - (!draft.isEmpty || (aiChatManager.mode === AIMode.GLOBAL && selectedContext.length > 0)) + (!draft.isEmpty || (chatHost.mode === AIMode.GLOBAL && selectedContext.length > 0)) ) { const sent = draft.take() - aiChatManager.queueMessage( + chatHost.queueMessage( expanded(chatDraft(sent.text, sent.pastes)), sent.images, [...selectedContext], - sent.files + sent.files, + sent.blobs ) // Consumed at enqueue, not at flush: the entry above pinned them. consumeMentionsIfGlobal() @@ -758,7 +926,7 @@ // message's original chips), so send exactly what's shown — the user may // have added or removed chips. const sent = draft.take() - aiChatManager.restartGeneration( + chatHost.restartGeneration( editingMessageIndex, sent.text, sent.pastes, @@ -771,13 +939,17 @@ const sent = draft.take() // Pin before consuming: the manager falls back to the live selection only // when given no override, and the consume below empties it. - const carried = aiChatManager.mode === AIMode.GLOBAL ? [...selectedContext] : undefined + const carried = chatHost.mode === AIMode.GLOBAL ? [...selectedContext] : undefined consumeMentionsIfGlobal() - aiChatManager.sendRequest({ + // A host that refuses the turn puts the draft back itself (see AIChatManager's + // restoreToInput and FlowChatViewHost's upload failure): restoring here too + // would double the text and every attachment. + chatHost.sendRequest({ instructions: sent.text, pastes: sent.pastes, images: sent.images, files: sent.files, + blobs: sent.blobs, contextOverride: carried, contextOverrideOrigin: carried ? 'pinned' : undefined }) @@ -999,37 +1171,67 @@ updateAppTooltipPosition(appTooltipCurrentViewNumber) } }) + + /** + * Clipboard files on the plain composer. ContextTextarea does this for the rich one; a + * host that attaches but renders the plain field would otherwise take files from the `+` + * and from a drop and silently ignore the same file pasted. + * + * Only when the clipboard carries no text, as there: a spreadsheet or browser copy puts a + * bitmap alongside the text, and pasting a cell range must paste the cells. + */ + function handlePlainPaste(e: ClipboardEvent) { + if (!chatHost.supportsMessageAttachments) return + if ((e.clipboardData?.getData('text/plain') ?? '').trim()) return + const pasted = Array.from(e.clipboardData?.files ?? []) + const images = pasted.filter((f) => f.type.startsWith('image/')) + const others = pasted.filter((f) => !f.type.startsWith('image/')) + if (images.length === 0 && others.length === 0) return + e.preventDefault() + if (images.length > 0) void addImages(images) + if (others.length > 0) void addNonImageFiles(others) + } {#snippet sendStopButton()} - {@const isLoading = (loading ?? aiChatManager.loading) && !questionAnsweredBySend} + {@const isLoading = (loading ?? chatHost.loading) && !questionAnsweredBySend} {@const emptyDraft = draft.isEmpty} + {@const needsText = chatHost.requiresMessageText && draft.text.trim() === ''} + + {@const needsTextForAttachment = needsText && !emptyDraft} {@const sendDisabled = disabled || pendingImages > 0 || pendingFiles > 0 || + pendingBlobs > 0 || ingestionHolds > 0 || + needsText || (emptyDraft && (onSendRequest !== undefined || - aiChatManager.mode !== AIMode.GLOBAL || + chatHost.mode !== AIMode.GLOBAL || selectedContext.length === 0))} diff --git a/frontend/src/lib/components/copilot/chat/AIChatModelSettings.svelte b/frontend/src/lib/components/copilot/chat/AIChatModelSettings.svelte index 9dd94fa4ea..3dab316768 100644 --- a/frontend/src/lib/components/copilot/chat/AIChatModelSettings.svelte +++ b/frontend/src/lib/components/copilot/chat/AIChatModelSettings.svelte @@ -1,10 +1,12 @@ {#snippet externalLinkIcon()} {/snippet} - - {#snippet buttonReplacement()} -
- -
- {/snippet} - {#snippet menu({ item, builders, close })} -
- - {#if promptSettings} - - {/if} + -
-
Model
-
- {#each models as m (m.provider + m.model)} - selectModel(m)} - > - {m.model} - {#if m.model === providerModel.model && m.provider === providerModel.provider} - - {/if} - - {/each} -
- -
- {#if capability.supported} - - -
- Thinking - {currentStop} -
- {#if stops.length > 1} - -
- selectReasoning(stops[+e.currentTarget.value])} - use:isolatePointer - class="lean-range no-default-style w-full" - aria-label="Reasoning effort" - /> -
- {/if} - - {:else} - -
-
Thinking
-
Not supported by this model
-
- {/if} - - - (thinkingPreferences.expandByDefault = !thinkingPreferences.expandByDefault)} - > - Always expand thinking - {#if thinkingPreferences.expandByDefault} - - {/if} - -
- {/snippet} - - - {#if promptSettings} {/if} - - diff --git a/frontend/src/lib/components/copilot/chat/AskUserQuestionDisplay.svelte b/frontend/src/lib/components/copilot/chat/AskUserQuestionDisplay.svelte index 403b42864d..17cc2bb4c1 100644 --- a/frontend/src/lib/components/copilot/chat/AskUserQuestionDisplay.svelte +++ b/frontend/src/lib/components/copilot/chat/AskUserQuestionDisplay.svelte @@ -4,7 +4,7 @@ import { CircleHelp, ArrowUp, Plus, Square, SquareCheck } from 'lucide-svelte' import Button from '$lib/components/common/button/Button.svelte' import TextInput from '$lib/components/text_input/TextInput.svelte' - import { getAiChatManager } from './aiChatManagerContext' + import { getChatViewHost } from './chatViewHost' import type { UserQuestionDisplay } from './shared' // Sessions inject a per-pane `AIChatManager` via context; outside of @@ -12,7 +12,7 @@ // this, answers clicked inside a session would dispatch to the singleton's // pending callbacks map (which doesn't have the session manager's question // callback), and the AI loop would stall. - const aiChatManager = getAiChatManager() + const chatHost = getChatViewHost() interface Props { toolCallId: string @@ -93,14 +93,14 @@ } return } - aiChatManager.handleUserQuestionAnswer(toolCallId, [choice]) + chatHost.handleUserQuestionAnswer(toolCallId, [choice]) } function submitPicked() { if (!multiSelect || picked.size === 0) { return } - aiChatManager.handleUserQuestionAnswer(toolCallId, [...picked]) + chatHost.handleUserQuestionAnswer(toolCallId, [...picked]) } function submitCustomAnswer() { @@ -119,7 +119,7 @@ return } - aiChatManager.handleUserQuestionAnswer(toolCallId, [answer]) + chatHost.handleUserQuestionAnswer(toolCallId, [answer]) } function handleChoiceKeydown(event: KeyboardEvent, choice: string, index: number) { diff --git a/frontend/src/lib/components/copilot/chat/AssistantMessage.svelte b/frontend/src/lib/components/copilot/chat/AssistantMessage.svelte index f06941c2a4..e68540955c 100644 --- a/frontend/src/lib/components/copilot/chat/AssistantMessage.svelte +++ b/frontend/src/lib/components/copilot/chat/AssistantMessage.svelte @@ -12,6 +12,11 @@ workspaceItemRegistry } from './workspaceItems.svelte' import { markdownProse } from '$lib/components/markdownProse' + import DisplayResult from '$lib/components/DisplayResult.svelte' + import { Bot, ExternalLink } from 'lucide-svelte' + import CopyButton from '$lib/components/common/button/CopyButton.svelte' + import { base } from '$lib/base' + import { displayDate } from '$lib/utils' interface Props { message: DisplayMessage @@ -22,6 +27,21 @@ let { message, workspace }: Props = $props() + // The run this answer came out of. Only a flow chat has one — a copilot turn runs in + // the browser — so the footer is absent rather than empty elsewhere. + const jobId = $derived(message.role === 'assistant' ? message.jobId : undefined) + const createdAt = $derived(message.role === 'assistant' ? message.createdAt : undefined) + const runHref = $derived(jobId ? `${base}/run/${jobId}?workspace=${workspace}` : undefined) + // Today's answers show the time alone; the day earns its place only on a conversation + // read back later. Resolved at render, so a chat left open across midnight keeps + // yesterday's format until it is reopened. + const timestamp = $derived.by(() => { + if (!createdAt) return undefined + const at = new Date(createdAt) + const today = new Date().toDateString() === at.toDateString() + return displayDate(at, false, !today) + }) + const reasoning = $derived( message.role === 'assistant' ? message.reasoning?.trim() || undefined : undefined ) @@ -60,6 +80,20 @@ return rest === 0 ? `${minutes}m` : `${minutes}m ${rest}s` } + const stepName = $derived(message.role === 'assistant' ? message.stepName : undefined) + + // A flow step can return a file rather than text; the raw JSON would be + // unreadable, so hand it to the result viewer instead of the markdown renderer. + const s3Object = $derived.by(() => { + if (!message.content.startsWith('{')) return undefined + try { + const parsed = JSON.parse(message.content) + return parsed?.type === 'windmill_s3_object' && parsed?.s3 ? parsed : undefined + } catch { + return undefined + } + }) + const candidatePaths = $derived(extractCandidatePaths(message.content)) const rendererPlugin = { renderer: { @@ -98,6 +132,19 @@ }) + +{#if stepName} +
+ + {stepName} +
+{/if} + {#if reasoning} {/if} -{#if message.content} +{#if s3Object} + +{:else if message.content}
{/if} + +{#if message.content} + +
+ + {#if timestamp} + {timestamp} + {/if} + {#if runHref} + + job {jobId?.slice(0, 8)} + + + {/if} +
+{/if} diff --git a/frontend/src/lib/components/copilot/chat/ContextTextarea.svelte b/frontend/src/lib/components/copilot/chat/ContextTextarea.svelte index 8ae3b51b56..976b969b3f 100644 --- a/frontend/src/lib/components/copilot/chat/ContextTextarea.svelte +++ b/frontend/src/lib/components/copilot/chat/ContextTextarea.svelte @@ -1,4 +1,5 @@ - - -
+
{@render leading?.()} @@ -830,11 +817,7 @@ {placeholder} class={twMerge( 'textarea-input resize-none caret-black dark:caret-white overflow-clip', - // The box (border/ring) lives on the wrapper; kill the textarea's own - // @tailwindcss/forms border, focus ring, and background so only the - // wrapper reads as the field. - '!border-transparent !bg-transparent !shadow-none focus:!border-transparent focus:!ring-0', - 'disabled:cursor-not-allowed disabled:placeholder:text-disabled', + COMPOSER_FIELD_RESET, CHAT_INPUT_PADDING, className )} diff --git a/frontend/src/lib/components/copilot/chat/ContextUsageIndicator.svelte b/frontend/src/lib/components/copilot/chat/ContextUsageIndicator.svelte index 96b452120b..cd6c58b2a3 100644 --- a/frontend/src/lib/components/copilot/chat/ContextUsageIndicator.svelte +++ b/frontend/src/lib/components/copilot/chat/ContextUsageIndicator.svelte @@ -1,16 +1,16 @@ -{#if aiChatManager.queuedMessage || aiChatManager.queuedImages.length > 0 || aiChatManager.queuedFiles.length > 0 || (aiChatManager.queuedContext?.length ?? 0) > 0} +{#if chatHost.queuedMessage || chatHost.queuedImages.length > 0 || chatHost.queuedFiles.length > 0 || chatHost.queuedBlobs.length > 0 || (chatHost.queuedContext?.length ?? 0) > 0}
- {#if aiChatManager.queuedMessage || aiChatManager.queuedImages.length > 0 || aiChatManager.queuedFiles.length > 0} + {#if chatHost.queuedMessage || chatHost.queuedImages.length > 0 || chatHost.queuedFiles.length > 0 || chatHost.queuedBlobs.length > 0} - {:else if aiChatManager.queuedContext?.length} + {:else if chatHost.queuedContext?.length}
- {#each aiChatManager.queuedContext as element (contextElementKey(element))} + {#each chatHost.queuedContext as element (contextElementKey(element))} {/each}
@@ -82,7 +97,7 @@ iconOnly title="Remove queued message and put it back in the input" startIcon={{ icon: X }} - on:click={() => aiChatManager.dequeueMessage()} + on:click={() => chatHost.dequeueMessage()} />
{/if} diff --git a/frontend/src/lib/components/copilot/chat/ToolConfirmationFooter.svelte b/frontend/src/lib/components/copilot/chat/ToolConfirmationFooter.svelte index 699b79a292..fcf943cb34 100644 --- a/frontend/src/lib/components/copilot/chat/ToolConfirmationFooter.svelte +++ b/frontend/src/lib/components/copilot/chat/ToolConfirmationFooter.svelte @@ -1,7 +1,7 @@ diff --git a/frontend/src/lib/components/copilot/chat/ToolExecutionDisplay.svelte b/frontend/src/lib/components/copilot/chat/ToolExecutionDisplay.svelte index 620357ff87..a0d16043a4 100644 --- a/frontend/src/lib/components/copilot/chat/ToolExecutionDisplay.svelte +++ b/frontend/src/lib/components/copilot/chat/ToolExecutionDisplay.svelte @@ -21,9 +21,9 @@ } from './planMode' import { Button } from '$lib/components/common' import { markdownProse } from '$lib/components/markdownProse' - import { getAiChatManager } from './aiChatManagerContext' + import { getChatViewHost } from './chatViewHost' - const aiChatManager = getAiChatManager() + const chatHost = getChatViewHost() import { isActiveUserQuestion, type ToolDisplayMessage } from './shared' import ChatCollapsibleCard from './ChatCollapsibleCard.svelte' import { twMerge } from 'tailwind-merge' @@ -69,7 +69,7 @@ const planLabel = $derived((planState && planCopy?.[planState]) ?? '') const planDoc = $derived( message.planArtifactId - ? aiChatManager.artifacts.artifacts.find((a) => a.id === message.planArtifactId) + ? chatHost.artifacts.artifacts.find((a) => a.id === message.planArtifactId) : undefined ) // The version this card wrote, not the document's current one, since later proposals move it on. @@ -201,7 +201,7 @@ title="Open this plan in the side panel: {planDoc.name}" startIcon={{ icon: FileText, classes: PLAN_MODE_TEXT_COLOR }} endIcon={{ icon: PanelRight }} - on:click={() => aiChatManager.openArtifact?.(planDoc.id, planDoc.name, planCardVersion)} + on:click={() => chatHost.openArtifact?.(planDoc.id, planDoc.name, planCardVersion)} > Plan diff --git a/frontend/src/lib/components/copilot/chat/blobUtils.test.ts b/frontend/src/lib/components/copilot/chat/blobUtils.test.ts new file mode 100644 index 0000000000..2c3b50a64a --- /dev/null +++ b/frontend/src/lib/components/copilot/chat/blobUtils.test.ts @@ -0,0 +1,42 @@ +import { describe, expect, it } from 'vitest' +import { dataUrlToBlob, matchesAccept } from './blobUtils' + +function file(name: string, type: string): File { + return new File(['x'], name, { type }) +} + +describe('dataUrlToBlob', () => { + // The bytes are re-uploaded verbatim, so a decode that drops or shifts one is a + // corrupted file the reader only discovers downstream. + it('decodes base64 back to the exact bytes', async () => { + const bytes = new Uint8Array([0x00, 0xff, 0x10, 0x89, 0x50]) + const b64 = btoa(String.fromCharCode(...bytes)) + const blob = dataUrlToBlob(`data:application/pdf;base64,${b64}`) + expect(blob.type).toBe('application/pdf') + expect(new Uint8Array(await blob.arrayBuffer())).toEqual(bytes) + }) + + it('percent-decodes a url that is not base64', async () => { + const blob = dataUrlToBlob('data:text/plain,hello%20world') + expect(blob.type).toBe('text/plain') + expect(await blob.text()).toBe('hello world') + }) + + it('falls back to a media type when the url names none', async () => { + expect(dataUrlToBlob('data:;base64,QQ==').type).toBe('application/octet-stream') + expect(dataUrlToBlob('data:;base64,QQ==', 'image/png').type).toBe('image/png') + }) +}) + +describe('matchesAccept', () => { + it('matches an extension, a type wildcard and an exact media type', () => { + expect(matchesAccept(file('report.PDF', ''), '.pdf')).toBe(true) + expect(matchesAccept(file('shot.png', 'image/png'), 'image/*')).toBe(true) + expect(matchesAccept(file('shot.png', 'image/png'), 'image/png')).toBe(true) + }) + + it('refuses a file no pattern covers, and allows everything when the list is empty', () => { + expect(matchesAccept(file('notes.txt', 'text/plain'), '.pdf, image/*')).toBe(false) + expect(matchesAccept(file('notes.txt', 'text/plain'), '')).toBe(true) + }) +}) diff --git a/frontend/src/lib/components/copilot/chat/blobUtils.ts b/frontend/src/lib/components/copilot/chat/blobUtils.ts new file mode 100644 index 0000000000..e2eacd1680 --- /dev/null +++ b/frontend/src/lib/components/copilot/chat/blobUtils.ts @@ -0,0 +1,79 @@ +/** + * Message-scoped attachments that are neither an image nor readable text — a PDF + * being the case that matters. They ride the composer next to images and files, + * as chips cleared on send, and reach the host through `ChatSendRequestOptions`. + * + * The bytes are kept verbatim, unlike an image (which normalises to a bounded + * PNG/JPEG for the model) and unlike a text file (which is decoded to a string): + * a host that forwards these to object storage has to upload what the user + * picked, not a re-encoding of it. + */ + +/** Blobs one message may carry — the same slot cap images and text files use. */ +export const MAX_ATTACHED_BLOBS = 8 + +/** + * Per-blob byte cap. The data URL sits in composer state until send, so this + * bounds what one message can hold in memory; a host uploading elsewhere pays + * the same bytes again on the wire. + */ +export const MAX_BLOB_BYTES = 20_000_000 + +export type AttachedBlob = { + name: string + /** The file's own media type, verbatim — the upload's Content-Type depends on it. */ + mediaType: string + /** `data:;base64,<...>` of the original bytes. */ + dataUrl: string + size: number +} + +/** + * Whether a file satisfies an `accept` list — the same list the OS picker gets, applied + * again on drop, where the browser enforces nothing. + */ +export function matchesAccept(file: File, accept: string): boolean { + const patterns = accept + .split(',') + .map((p) => p.trim().toLowerCase()) + .filter(Boolean) + if (patterns.length === 0) return true + const type = file.type.toLowerCase() + const name = file.name.toLowerCase() + return patterns.some((pattern) => { + if (pattern.startsWith('.')) return name.endsWith(pattern) + if (pattern.endsWith('/*')) return type.startsWith(pattern.slice(0, -1)) + return type === pattern + }) +} + +export async function fileToAttachedBlob(file: File): Promise { + const dataUrl = await new Promise((resolve, reject) => { + const reader = new FileReader() + reader.onload = () => resolve(String(reader.result)) + reader.onerror = () => reject(reader.error ?? new Error(`Could not read ${file.name}`)) + reader.readAsDataURL(file) + }) + return { + name: file.name, + mediaType: file.type || 'application/octet-stream', + dataUrl, + size: file.size + } +} + +/** The bytes behind a `data:` URL, for a host that has to re-upload them. */ +export function dataUrlToBlob(dataUrl: string, fallbackType = 'application/octet-stream'): Blob { + const comma = dataUrl.indexOf(',') + const header = dataUrl.slice(5, comma) + const isBase64 = header.endsWith(';base64') + const mediaType = (isBase64 ? header.slice(0, -';base64'.length) : header) || fallbackType + const payload = dataUrl.slice(comma + 1) + if (!isBase64) { + return new Blob([decodeURIComponent(payload)], { type: mediaType }) + } + const binary = atob(payload) + const bytes = new Uint8Array(binary.length) + for (let i = 0; i < binary.length; i++) bytes[i] = binary.charCodeAt(i) + return new Blob([bytes], { type: mediaType }) +} diff --git a/frontend/src/lib/components/copilot/chat/chatViewHost.ts b/frontend/src/lib/components/copilot/chat/chatViewHost.ts new file mode 100644 index 0000000000..84daaffe24 --- /dev/null +++ b/frontend/src/lib/components/copilot/chat/chatViewHost.ts @@ -0,0 +1,177 @@ +import { getContext, setContext } from 'svelte' +import type { AIMode, AIAutonomyMode } from './AIChatManager.svelte' +import { getAiChatManager } from './aiChatManagerContext' +import type { DisplayMessage, Tool } from './shared' +import type { ContextElement } from './context' +import type { AttachedBlob } from './blobUtils' +import type { AttachedImage } from './imageUtils' +import type { AttachedTextFile } from './textFileUtils' +import type { PasteAttachment } from './pasteTokens' +import type { AttachedFilesStore } from './files/attachedFiles.svelte' +import type { SessionArtifactsStore } from './artifacts/artifactsState.svelte' +import type { ArtifactVersionTarget } from '$lib/components/sessions/previewRouter' +import type { FlowAIChatHelpers } from './flow/core' +import type { AppAIChatHelpers } from './app/core' +import type AIChatInput from './AIChatInput.svelte' + +export type ChatSendRequestOptions = { + instructions?: string + pastes?: PasteAttachment[] + images?: AttachedImage[] + files?: AttachedTextFile[] + blobs?: AttachedBlob[] + /** Selected-context snapshot for this turn, in place of the live selection. Set + * whenever a send settles its context ahead of the turn. A host with no context + * of its own ignores it. */ + contextOverride?: ContextElement[] + /** Where `contextOverride` came from. 'pinned': chips picked for THIS message, so + * they are consumed from the live selection on send. 'replay': an edit or retry + * resending an older message's context, already consumed long ago. */ + contextOverrideOrigin?: 'pinned' | 'replay' + /** The conversation this turn belongs to, where that is not the one on screen — a + * queued message going out after its own chat's run finished. Hosts with a single + * conversation ignore it. */ + conversationId?: string +} + +/** + * What the chat view components (AIChatDisplay and everything it renders) need + * from whatever is driving the conversation. AIChatManager implements it for the + * copilot's own LLM loop; FlowChatViewHost implements it over a flow run's + * conversation so both chats render through the same components. + * + * Each affordance is gated by the field that answers for it — attachments by + * `supportsMessageAttachments`, the model button by `supportsModelSettings`, and so on — + * so a host turns on exactly what it can serve, and a new one is a matter of answering + * these fields rather than of being a copilot. `mode` is the exception, still read + * directly for chrome that only the copilot has. + */ +export interface ChatViewHost { + // Transcript + displayMessages: DisplayMessage[] + /** API-level messages. Only the count is read (context usage visibility). */ + messages: readonly unknown[] + contextTokens: number + /** The workspace a message's paths and jobs resolve against, which a fork session + * pins away from the navigated one. */ + readonly operatingWorkspace: string | undefined + loading: boolean + /** A turn this tab can neither follow nor stop, held by another tab on the same chat. */ + readonly runHeldElsewhere: boolean + loadingLabel: string | undefined + compacting: boolean + currentReply: string + currentReasoning: string + currentReasoningActive: boolean + readonly reasoningHiddenIndicatorLabel: string | undefined + readonly automaticScroll: boolean + enableAutomaticScroll: () => void + disableAutomaticScroll: () => void + + // Composer + instructions: string + readonly sendInFlight: boolean + /** Resolves to whether the draft was consumed as a turn. */ + sendRequest: (options?: ChatSendRequestOptions) => Promise + cancel: (reason?: string) => void + setAiChatInput: (aiChatInput: AIChatInput | null) => void + readonly queuedMessage: string + queuedContext: ContextElement[] | undefined + readonly queuedImages: AttachedImage[] + readonly queuedFiles: AttachedTextFile[] + readonly queuedBlobs: AttachedBlob[] + queueMessage: ( + text: string, + images?: AttachedImage[], + context?: ContextElement[], + files?: AttachedTextFile[], + blobs?: AttachedBlob[] + ) => void + dequeueMessage: () => void + setComposerStaged: (key: string, editingIndex: number | null, bytes: number) => void + clearComposerStaged: (key: string) => void + attachmentBytesExcluding: (selfKey: string) => number + + // Per-message actions + storedImages: (displayMessageIndex: number) => AttachedImage[] | undefined + retryRequest: (messageIndex: number) => void + restartGeneration: ( + displayMessageIndex: number, + newContent?: string, + pastes?: PasteAttachment[], + images?: AttachedImage[], + editedContext?: ContextElement[], + files?: AttachedTextFile[] + ) => void | Promise + handleUserQuestionAnswer: (toolId: string, choices: string[]) => boolean + handleToolConfirmation: (toolId: string, confirmed: boolean) => void + /** A tool is waiting on a run form the user is filling in. Escape belongs to that form + * then, not to the turn — see AIChatDisplay's window handler. */ + readonly hasPendingRunForm: boolean + isRunFormPending: (toolCallId: string) => boolean + + // Copilot-only surfaces. Left undefined/false by hosts that have no LLM loop + // of their own; the chrome they drive hides itself. + mode?: AIMode + isSessionChat: boolean + /** Model + reasoning picker. Off where the model is configured elsewhere. */ + supportsModelSettings: boolean + /** Click a user message to edit and resend it. Needs a host that can rewind + * its own transcript, which a host replaying a server-side run cannot. */ + supportsMessageEditing: boolean + /** The `+` menu's file entry and drag-and-drop onto the panel. Attachments ride + * one message; where they go afterwards is the host's business (see sendRequest). */ + supportsMessageAttachments: boolean + /** + * Why attaching is off right now, when the host would otherwise take attachments. Distinct + * from `supportsMessageAttachments` being false, which means this chat never takes them: + * here the composer keeps the control and says what is missing, because moving the input + * elsewhere would only offer an editor that cannot work either. + */ + attachmentsUnavailableReason?: string + /** The turn needs text: attachments alone cannot be sent. True where the consumer + * requires a message of its own — an AI agent step refuses a run with neither a + * `user_message` nor manual memory. */ + requiresMessageText: boolean + /** The `+` menu's folder entries, backed by `attachedFiles`. A linked folder is a + * live handle on the user's disk, so only a host reading files in the browser has one. */ + supportsLinkedFolders: boolean + /** `accept` for the file picker, and the drop filter. A host whose consumer only + * understands some formats narrows it so the rest are refused rather than ignored. */ + attachmentAccept: string + /** How many attachments one turn can carry, when the consumer holds a fixed number — + * a flow input that is a single file, say. Undefined means no limit. Enforced at the + * picker and on drop, so what the composer shows is what the turn actually sends. */ + maxMessageAttachments?: number + /** Take non-image attachments verbatim (`blobs`) instead of decoding them to text. + * True where the bytes are forwarded somewhere — object storage — rather than read + * in the browser. */ + attachmentsAsBlobs: boolean + tools: Tool[] + autonomyMode: AIAutonomyMode + setAutonomyMode: (mode: AIAutonomyMode) => void + readonly autoAcceptEditsActive: boolean + readonly autoAcceptEditsAvailable: boolean + readonly autoAcceptToolConfirmationsAvailable: boolean + readonly planModeAvailable: boolean + attachedFiles: AttachedFilesStore + artifacts: SessionArtifactsStore + openArtifact?: (artifactId: string, name: string, version?: ArtifactVersionTarget) => void + flowAiChatHelpers?: FlowAIChatHelpers + appAiChatHelpers?: AppAIChatHelpers +} + +const CHAT_VIEW_HOST_CONTEXT_KEY = 'chatViewHost' + +export function setChatViewHost(host: ChatViewHost) { + setContext(CHAT_VIEW_HOST_CONTEXT_KEY, host) +} + +/** + * Resolve the host driving the chat in this subtree. Falls back to the + * AIChatManager (scoped instance or app-wide singleton) so every existing + * copilot chat keeps working without setting anything. + */ +export function getChatViewHost(): ChatViewHost { + return getContext(CHAT_VIEW_HOST_CONTEXT_KEY) ?? getAiChatManager() +} diff --git a/frontend/src/lib/components/copilot/chat/composerBox.ts b/frontend/src/lib/components/copilot/chat/composerBox.ts new file mode 100644 index 0000000000..f85783d988 --- /dev/null +++ b/frontend/src/lib/components/copilot/chat/composerBox.ts @@ -0,0 +1,27 @@ +/** + * The composer's box, shared by both of AIChatInput's branches — the rich + * ContextTextarea and the plain textarea a host without @-context gets. + * + * Border and rounding live on the WRAPPER, never on the field, so the chip rows + * (context, files, images) sit inside the box above the text. The field's own + * @tailwindcss/forms border, ring and background are neutralised so only the + * wrapper reads as the input. + * + * The disabled treatment is on the wrapper for the same reason: `disabled` on the + * field alone leaves it looking exactly like a usable one, so the only cue that + * typing is refused is placeholder text the eye reads as an invitation. + */ + +const BOX_BASE = 'w-full scroll-pb-2 rounded-md border border-border-light transition-colors' + +export function composerBoxClass(disabled: boolean = false): string { + return `${BOX_BASE} ${ + disabled + ? 'bg-surface-disabled cursor-not-allowed' + : 'bg-surface-input focus-within:border-border-selected' + }` +} + +/** Applied to the field inside the box; without it the field draws a second border. */ +export const COMPOSER_FIELD_RESET = + '!border-transparent !bg-transparent !shadow-none focus:!border-transparent focus:!ring-0 disabled:cursor-not-allowed disabled:placeholder:text-disabled' diff --git a/frontend/src/lib/components/copilot/chat/flow/FlowAIChat.svelte b/frontend/src/lib/components/copilot/chat/flow/FlowAIChat.svelte index c6bed6988c..8195e2bb25 100644 --- a/frontend/src/lib/components/copilot/chat/flow/FlowAIChat.svelte +++ b/frontend/src/lib/components/copilot/chat/flow/FlowAIChat.svelte @@ -1,4 +1,5 @@ - { - showChatModeWarning = false - chatInputEnabled = false - }} -> -

- Enabling Chat Mode will replace all existing flow inputs with a single - user_message - parameter. -

-

- Your current input configuration will be lost. Are you sure you want to continue? -

- + +{#snippet inputsEditButton(open: boolean, toggle: () => void)} + + + {#snippet children({ item })} + + + {/snippet} + {/if}
{/if} @@ -696,11 +761,15 @@
{#if flowStore.val.value?.chat_input_enabled}
- {#if showAdditionalInputs} -
+ {#if chatPanelTab === 'inputs'} + +
{#snippet openEditTab()} - + {@render inputsAddTrigger()} {/snippet} {/snippet}
{/if} - + +
+ +
{:else}
@@ -815,22 +884,9 @@
{#snippet close_button()} -
@@ -884,12 +940,7 @@ }} > {#snippet trigger()} -
- -
+ {@render inputsAddTrigger()} {/snippet} {/if} diff --git a/frontend/src/lib/components/flows/conversations/FlowChat.svelte b/frontend/src/lib/components/flows/conversations/FlowChat.svelte index 160af2bf27..fc1f2f51e2 100644 --- a/frontend/src/lib/components/flows/conversations/FlowChat.svelte +++ b/frontend/src/lib/components/flows/conversations/FlowChat.svelte @@ -1,102 +1,133 @@ -
- {#if chat && chatState} - {#if !hideSidebar} - - {/if} + +
+ + +
- {/if} +
diff --git a/frontend/src/lib/components/flows/conversations/FlowChatInterface.svelte b/frontend/src/lib/components/flows/conversations/FlowChatInterface.svelte index 22fe037728..47ba31e30b 100644 --- a/frontend/src/lib/components/flows/conversations/FlowChatInterface.svelte +++ b/frontend/src/lib/components/flows/conversations/FlowChatInterface.svelte @@ -1,79 +1,53 @@ - -{#if additionalInputsSchema} +{#if modalSchema} {#snippet actions()} @@ -164,82 +215,84 @@ {/if} -
- -
- {#if deploymentInProgress} - - {/if} - {#if chatState.loadingMessages && chatState.messages.length === 0} -
- -
- {:else if chatState.messages.length === 0} -
- -

Start a conversation

-

Send a message to run the flow and see the results

-
+{#snippet emptyHint()} +
+ {#if manager.isLoadingMessages} + {:else} -
- {#each chatState.messages as message (message.id)} - - {/each} - {#if busy} -
- - Processing... -
- {/if} -
- {/if} -
- - -
- {#if additionalInputsSchema} -
-
- - {#if hasMissingRequired} - - {/if} + +

Start a conversation

+

Send a message to run the flow and see the results

+ {#if !emptyString(description)} + +
+
-
+ {/if} {/if} -
- { - if (e.key === 'Enter' && !e.shiftKey && !e.isComposing) { - e.preventDefault() - handleSendMessage() - } - }} - showCancelButton={busy} - onCancel={() => chat.stop()} - sendTitle={deploymentInProgress ? 'Deployment in progress' : 'Send message (Enter)'} - /> -
+{/snippet} + +{#snippet footerSettings()} + {#if modalSchema} +
+ + {#if modalMissingRequired} + + {/if} +
+ {/if} + {#if modelWiring} + + {/if} +{/snippet} + + +
0} + class:min-h-64={chatHost.displayMessages.length === 0} +> + {}} + deletePastChat={() => {}} + saveAndClear={() => {}} + />
diff --git a/frontend/src/lib/components/flows/conversations/FlowChatManager.svelte.ts b/frontend/src/lib/components/flows/conversations/FlowChatManager.svelte.ts new file mode 100644 index 0000000000..5266464941 --- /dev/null +++ b/frontend/src/lib/components/flows/conversations/FlowChatManager.svelte.ts @@ -0,0 +1,1256 @@ +import type { + FlowConversation, + FlowConversationMessage, + ListFlowConversationsData +} from '$lib/gen/types.gen' +import { FlowConversationsService, JobService } from '$lib/gen' +import { sendUserToast } from '$lib/toast' +import { waitJob } from '$lib/components/waitJob' +import { tick } from 'svelte' +import InfiniteList from '$lib/components/InfiniteList.svelte' +import { workspaceStore, userStore } from '$lib/stores' +import { get } from 'svelte/store' +import { parseStreamEvents } from '$lib/components/chat/utils' +import { randomUUID } from '$lib/utils/uuid' +import { + prefersInstantReveal, + TypewriterReveal +} from '$lib/components/copilot/chat/typewriterReveal' +import { + appendRevealed, + applyStreamEvent, + emptyTurnState, + turnFailed, + type TurnState +} from './turnTranscript' + +export interface ChatMessage extends FlowConversationMessage { + loading?: boolean + streaming?: boolean + /** + * The tool a row's call belongs to, as the stream reports it. Local to a running turn: + * afterwards the name comes from the summary the server stored, and the call itself + * from the tool's own job (see toolCallContext) or from the row's own + * `tool_arguments` / `tool_result` when the tool had no job. + */ + tool_name?: string +} + +export interface ConversationWithDraft extends FlowConversation { + isDraft?: boolean +} + +/** A chat run from the editor's test panel, one started on the deployed flow, or both. */ +export type ConversationKind = NonNullable + +/** What the sidebar shows for one chat. */ +export type ConversationStatus = 'running' | 'error' | 'queued' | 'idle' + +/** What the chat renders about one conversation's turn. */ +type TurnStatus = { + isLoading: boolean + isWaitingForResponse: boolean + /** + * A turn is on its way but has no job yet — attachments uploading, say. `isLoading` + * only covers the run itself, and the gap between them is long enough to change what + * the send lands in. + */ + isDispatchingTurn: boolean + /** The thinking of the turn in flight, until it is attached to the answer it produced. */ + currentReasoning: string + /** The model is reasoning: true from the first thinking token until the answer starts. */ + isReasoningActive: boolean + jobId?: string +} + +/** + * The machinery one conversation's live turn runs on. Deliberately not `$state`: nothing + * renders from it, and an EventSource or a TypewriterReveal has no business behind a proxy. + */ +type TurnRuntime = { + eventSource?: EventSource + pollingInterval?: ReturnType + // What the turn has written so far — which row is open, and the text in it. Held here + // rather than in the stream handler's locals: the typewriter reveals on animation + // frames, long after the chunk that delivered the text was applied. + turn: TurnState + // The worker's events reach us in bursts — the provider batches tokens, and the SSE + // endpoint ships whatever accumulated — so display is paced separately from arrival, + // exactly as the session chat does it. Answer and thinking pace independently. + replyReveal: TypewriterReveal + reasoningReveal: TypewriterReveal + // How far into the stream this turn has read. Sent back on reconnect so a resumed + // stream continues after the deltas already rendered rather than replaying them. The + // offset indexes `streamJobId`'s stream alone. + streamOffset?: number + streamJobId?: string +} + +function emptyStatus(): TurnStatus { + return { + isLoading: false, + isWaitingForResponse: false, + isDispatchingTurn: false, + currentReasoning: '', + isReasoningActive: false + } +} + +export class FlowChatManager { + // State + inputMessage = $state('') + isLoadingMessages = $state(false) + messagesContainer = $state(undefined) + inputElement = $state(undefined) + loadingMoreMessages = $state(false) + conversations = $state([]) + deletingConversationId = $state(undefined) + isSidebarExpanded = $state(false) + /** + * Whether a turn may start while another conversation's is still running. + * + * Only the deployed chat, whose run helper posts a job and holds nothing. The editor's + * test panel drives the flow graph, the progress bar and the step history from one run + * at a time, so a second turn there would move the graph off the one being watched. + */ + allowsParallelTurns = $state(false) + + /** Each conversation's rows, live ones included, so a turn keeps writing while the + * reader is in another chat. Doubles as the load cache: rows here are never re-fetched. */ + #rowsById = $state>({}) + /** How far back each conversation has been paged. Held per conversation for the same + * reason the rows are: a cached chat keeps its scrollback when the reader returns to it, + * and one global counter would ask the server for the page number of whichever chat was + * scrolled last. */ + #pagedTo = $state>({}) + #hasMoreById = $state>({}) + #status = $state>({}) + #runtime = new Map() + + /** Row ids are temp- prefixed: the sweep after a run keeps only what the server stored. */ + #newRowId = () => 'temp-' + randomUUID() + + /** Every row id the chat still holds, across conversations. What a per-row store prunes + * against, so a background chat's rows are not mistaken for gone. */ + get liveRowIds(): Set { + const ids = new Set() + for (const rows of Object.values(this.#rowsById)) for (const row of rows) ids.add(row.id) + return ids + } + + /** Whether the open chat has older rows to fetch — what the scroll handler acts on. */ + get hasMoreMessages(): boolean { + return this.selectedConversationId + ? (this.#hasMoreById[this.selectedConversationId] ?? false) + : false + } + + #rowsOf(conversationId: string): ChatMessage[] { + return this.#rowsById[conversationId] ?? [] + } + + #statusOf(conversationId: string): TurnStatus { + return this.#status[conversationId] ?? emptyStatus() + } + + /** The status record to write into, created on first use. */ + #liveStatus(conversationId: string): TurnStatus { + this.#status[conversationId] ??= emptyStatus() + return this.#status[conversationId] + } + + #liveRuntime(conversationId: string): TurnRuntime { + const existing = this.#runtime.get(conversationId) + if (existing) return existing + const created: TurnRuntime = { + turn: emptyTurnState(conversationId), + replyReveal: new TypewriterReveal({ + onReveal: (chunk) => this.#reveal(conversationId, 'answer', chunk), + instant: prefersInstantReveal() + }), + reasoningReveal: new TypewriterReveal({ + onReveal: (chunk) => this.#reveal(conversationId, 'reasoning', chunk), + instant: prefersInstantReveal() + }) + } + this.#runtime.set(conversationId, created) + return created + } + + #reveal(conversationId: string, kind: 'answer' | 'reasoning', chunk: string) { + const runtime = this.#liveRuntime(conversationId) + const step = appendRevealed( + { rows: this.#rowsOf(conversationId), state: runtime.turn }, + kind, + chunk, + this.#newRowId + ) + this.#rowsById[conversationId] = step.rows + runtime.turn = step.state + if (kind === 'reasoning') + this.#liveStatus(conversationId).currentReasoning = step.state.reasoning + } + + /** + * Name the run a turn started, on both the status and the message that began it. + * + * The server writes its own user row carrying this job id, but the poller drops user rows + * and the temp sweep keeps them, so the row on screen never becomes that one — it holds + * the id only after a reload fetches the transcript fresh. Stamping it here is what makes + * a retry replay the turn it is looking at rather than run the text again with whatever + * the composer holds later. + */ + #nameTurnJob(conversationId: string, userRowId: string, jobId: string) { + this.#liveStatus(conversationId).jobId = jobId + this.#rowsById[conversationId] = this.#rowsOf(conversationId).map((row) => + row.id === userRowId ? { ...row, job_id: jobId } : row + ) + } + + /** Reveal everything buffered now, so the row is whole before the turn moves on. */ + #flushReveals(conversationId: string) { + const runtime = this.#runtime.get(conversationId) + runtime?.replyReveal.flush() + runtime?.reasoningReveal.flush() + } + + // The open conversation's turn, which is what the composer and the transcript render. + get messages(): ChatMessage[] { + return this.selectedConversationId ? this.#rowsOf(this.selectedConversationId) : [] + } + set messages(rows: ChatMessage[]) { + if (this.selectedConversationId) this.#rowsById[this.selectedConversationId] = rows + } + get isLoading(): boolean { + return this.selectedConversationId + ? this.#statusOf(this.selectedConversationId).isLoading + : false + } + get isWaitingForResponse(): boolean { + return this.selectedConversationId + ? this.#statusOf(this.selectedConversationId).isWaitingForResponse + : false + } + get isDispatchingTurn(): boolean { + return this.selectedConversationId + ? this.#statusOf(this.selectedConversationId).isDispatchingTurn + : false + } + get currentReasoning(): string { + return this.selectedConversationId + ? this.#statusOf(this.selectedConversationId).currentReasoning + : '' + } + get isReasoningActive(): boolean { + return this.selectedConversationId + ? this.#statusOf(this.selectedConversationId).isReasoningActive + : false + } + get currentJobId(): string | undefined { + return this.selectedConversationId + ? this.#statusOf(this.selectedConversationId).jobId + : undefined + } + + /** Mark a turn as dispatching before it has a job — the upload the composer awaits. */ + setDispatching(conversationId: string, dispatching: boolean) { + this.#liveStatus(conversationId).isDispatchingTurn = dispatching + } + + /** Whether this conversation has a turn being dispatched or running. */ + isConversationBusy(conversationId: string): boolean { + const status = this.#statusOf(conversationId) + return status.isLoading || status.isWaitingForResponse || status.isDispatchingTurn + } + + /** The conversations with a turn in flight right now. */ + get runningConversationIds(): string[] { + return Object.keys(this.#status).filter((id) => this.isConversationBusy(id)) + } + + /** + * Whether a message is waiting to go out in this chat. The composer owns the queue — + * it is the thing that took the draft — and the sidebar shows it per row. + */ + hasQueuedMessage?: (conversationId: string) => boolean + + /** + * How many rows each chat held when the reader last had it open. Compared against what + * it holds now for the unread count, exactly as the session sidebar does it. In memory + * rather than on the row: what is unread is what arrived while this page was open. + */ + #lastSeenCount = $state>({}) + + /** + * Rows that landed in this chat since the reader last looked at it. The open chat is + * being read, so it is never unread — which is also what makes the watermark cheap: + * it is written once, on the way out of a chat, rather than on every row that arrives. + */ + unreadCount(conversationId: string): number { + if (conversationId === this.selectedConversationId) return 0 + const seen = this.#lastSeenCount[conversationId] ?? 0 + return Math.max(0, this.#rowsOf(conversationId).length - seen) + } + + /** + * Unread across every chat, for the collapsed rail. Summed over the conversations whose + * rows this page holds — an unopened chat has none, and counting it would mean fetching + * every row of every chat to put a number on a button. + */ + get totalUnread(): number { + return Object.keys(this.#rowsById).reduce((total, id) => total + this.unreadCount(id), 0) + } + + /** + * Forget a conversation entirely: its turn, its rows and what was read of them. + * + * Leaving a chat marks it read, but a chat that leaves the *list* has nothing left to + * read — and its rows would otherwise keep counting toward the rail's unread badge with + * no row in the sidebar that could ever clear them. + */ + #forget(conversationId: string) { + this.endTurn(conversationId) + this.#runtime.delete(conversationId) + delete this.#rowsById[conversationId] + delete this.#status[conversationId] + delete this.#lastSeenCount[conversationId] + delete this.#pagedTo[conversationId] + delete this.#hasMoreById[conversationId] + } + + #markSeen(conversationId: string | undefined) { + if (!conversationId) return + this.#lastSeenCount[conversationId] = this.#rowsOf(conversationId).length + } + + /** + * What the sidebar shows for one chat. Only says `error` for a conversation whose rows + * are loaded: an unopened chat has none, and fetching every row of every chat to put a + * triangle on one is not worth the requests. + */ + conversationStatus(conversationId: string): ConversationStatus { + if (this.isConversationBusy(conversationId)) return 'running' + if (this.hasQueuedMessage?.(conversationId)) return 'queued' + const rows = this.#rowsById[conversationId] + if (rows) { + for (let i = rows.length - 1; i >= 0; i--) { + if (rows[i].message_type !== 'user') continue + return turnFailed(rows, i) ? 'error' : 'idle' + } + } + return 'idle' + } + /** + * Which conversations the list holds. The editor shows its own test chats, since + * testing is what happens there; a deployed flow shows the chats its users started, + * so nobody's trial runs are mixed into them. + */ + conversationKind = $state('deployed') + /** + * Whether the sidebar offers the kind filter. Only the editor does: a deployed flow has + * no test chats of its own to show, and offering to list someone's trial runs there + * would put editor scratch in front of the flow's users. + */ + canFilterConversationKind = $state(false) + /** + * What this surface's own runs are, which the filter does not change: the editor runs + * previews, the flow page runs the deployed flow. A conversation is fixed to one kind + * at creation, so a turn sent from here into a conversation of the other kind would be + * stored as part of it and the mixing would be invisible afterwards. + */ + surfaceKind = $state>('deployed') + selectedConversationId = $state(undefined) + conversationListComponent = $state(undefined) + + // Private state + #scrollTimeout: ReturnType | undefined = undefined + #perPage = 50 + + // Options + #onRunFlow?: ( + userMessage: string, + conversationId: string, + additionalInputs?: Record + ) => Promise + #useStreaming = $state(false) + #path = $state(undefined) + + // When the flow editor runs as an AI-session live editor, it acts on a workspace + // that can differ from the nav store. FlowChat.svelte wires this to + // FlowEditorContext.opWorkspace so workspace-scoped calls hit the acting workspace. + operatingWorkspace?: () => string | undefined + + #workspace(): string | undefined { + return this.operatingWorkspace?.() ?? get(workspaceStore) + } + + initialize( + onRunFlow: ( + userMessage: string, + conversationId: string, + additionalInputs?: Record + ) => Promise, + path: string, + useStreaming: boolean = false + ) { + this.#onRunFlow = onRunFlow + this.#path = path + this.#useStreaming = useStreaming + } + + updateConversationId(conversationId: string | undefined) { + this.selectedConversationId = conversationId + } + + /** + * Called when a turn finishes on its own, so whatever was typed into that chat while it + * ran can go out. This is the only thing that sends a queued message, so it fires from + * a settled run and nothing else: a stream that merely dropped leaves the flow job + * running on a worker, and the next turn must not start alongside it. The chat closing + * and a cancel are both silent here — a cancel instead hands the queue back to the + * composer, since Stop should not arm the next turn. + */ + onTurnSettled?: (conversationId: string) => void + + /** Stop following one conversation's turn and forget what it was mid-way through. */ + endTurn(conversationId: string, options?: { settled?: boolean }) { + const runtime = this.#runtime.get(conversationId) + if (runtime) { + runtime.replyReveal.reset() + runtime.reasoningReveal.reset() + runtime.turn = emptyTurnState(conversationId) + runtime.eventSource?.close() + runtime.eventSource = undefined + if (runtime.pollingInterval) { + clearInterval(runtime.pollingInterval) + runtime.pollingInterval = undefined + } + } + const status = this.#liveStatus(conversationId) + status.currentReasoning = '' + status.isReasoningActive = false + status.isLoading = false + status.isWaitingForResponse = false + status.jobId = undefined + if (options?.settled) this.onTurnSettled?.(conversationId) + } + + /** Every turn this chat is following. Called when the chat itself goes away. */ + cleanup() { + for (const conversationId of Object.keys(this.#status)) this.endTurn(conversationId) + } + + // Public methods for component to call + fillInputMessage(message: string) { + this.inputMessage = message + } + + focusInput() { + this.inputElement?.focus() + } + + clearMessages() { + this.messages = [] + this.inputMessage = '' + } + + async createConversation({ clearMessages = true }: { clearMessages?: boolean }) { + // Leaving a chat is leaving it read, however the selection moves — this path sets it + // directly rather than going through `selectConversation`, and without this the rows + // the reader just watched arrive would be counted as unread behind them. + this.#markSeen(this.selectedConversationId) + // Check if there's already a draft conversation + const existingDraft = this.conversations.find((c) => c.isDraft) + if (existingDraft) { + // Select the existing draft instead of creating a new one + this.selectedConversationId = existingDraft.id + this.clearMessages() + return existingDraft.id + } + const newConversationId = randomUUID() + this.selectedConversationId = newConversationId + + // Create a new conversation object and add it to the top of the list + const newConversation: ConversationWithDraft = { + id: newConversationId, + workspace_id: this.#workspace()!, + flow_path: this.#path!, + title: 'New chat', + created_at: new Date().toISOString(), + updated_at: new Date().toISOString(), + created_by: get(userStore)!.username!, + // The kind the first turn will give it: a draft started here runs on this surface. + is_test: this.surfaceKind === 'test', + isDraft: true + } + + // Prepend to conversations list + this.conversations = [newConversation, ...this.conversations] + // Clear messages in the chat interface + if (clearMessages) { + this.clearMessages() + } + this.focusInput() + + return newConversationId + } + + setupInfiniteList() { + this.conversationListComponent?.setLoader((page, perPage) => + this.loadConversations(page, perPage) + ) + this.conversationListComponent?.setDeleteItemFn((id) => this.deleteConversation(id)) + } + + async selectConversation(conversationId: string, isDraft?: boolean) { + // Everything the chat being left holds has been in front of the reader. + this.#markSeen(this.selectedConversationId) + this.selectedConversationId = conversationId + // Load conversation messages into chat interface + if (isDraft) { + // For draft conversations, just clear messages (don't try to load from backend) + this.clearMessages() + } else { + // For persisted conversations, load messages from backend + await this.loadConversationMessages(conversationId) + } + } + + /** + * Open this flow's most recent conversation, unless the caller already chose one. + * + * Every turn is stored the moment it runs — a preview from the editor exactly like a + * deployed run — so a chat that has been used before should come back to it instead of + * to an empty pane, which reads as having lost it. + */ + async selectLatestConversation() { + if (this.selectedConversationId || !this.#workspace() || !this.#path) return + const [latest] = await this.loadConversations(1, 1) + // Re-checked after the await: a message sent meanwhile has already opened its own. + if (!latest || this.selectedConversationId) return + await this.selectConversation(latest.id) + } + + /** + * Narrow the list to one kind of chat and reload it. + * + * The open conversation goes with it when it is not of the new kind: the composer sends + * into whatever is selected, and a test run appended to a deployed conversation would be + * stored as part of it — `get_or_create_conversation_with_id` keeps the row's own + * `is_test`, so the mixing would be invisible afterwards. + */ + async setConversationKind(kind: ConversationKind) { + if (this.conversationKind === kind) return + // A turn in flight writes into the conversation it started in — the stream appends + // rows, the poller merges more — so the selection cannot be moved under it. Held + // from the moment the composer starts dispatching, since an upload runs before the + // job exists and a send landing after the switch would start a fresh conversation. + // The control is disabled meanwhile; this is the same rule where it is enforced. + if (this.isTurnInFlight) return + this.conversationKind = kind + const open = this.conversations.find((c) => c.id === this.selectedConversationId) + const stillListed = + open === undefined || + open.isDraft === true || + kind === 'all' || + (kind === 'test') === (open.is_test === true) + if (!stillListed) { + // Still a real chat, just not one this filter lists: what was read of it stays read. + this.#markSeen(this.selectedConversationId) + this.selectedConversationId = undefined + } + await this.refreshConversations() + } + + // A name typed on a chat that has not run yet, kept until its first turn creates the row + // server-side: that insert titles the conversation from the message, and the refresh + // which follows would otherwise replace the typed name with it. + #draftTitle: { id: string; title: string } | undefined = undefined + + /** Write a title to the server and to the row the list holds. */ + async #writeConversationTitle(conversationId: string, title: string) { + try { + await FlowConversationsService.updateFlowConversation({ + workspace: this.#workspace()!, + conversationId, + requestBody: { title } + }) + this.conversations = this.conversations.map((c) => + c.id === conversationId ? { ...c, title } : c + ) + return true + } catch (error) { + console.error('Failed to rename conversation:', error) + sendUserToast('Failed to rename conversation', true) + return false + } + } + + /** Rename a chat. The list holds the row, so it is patched rather than reloaded. */ + async renameConversation(conversationId: string, title: string) { + const trimmed = title.trim() + const current = this.conversations.find((c) => c.id === conversationId) + if (!current || trimmed === '' || trimmed === current.title) return + // A chat that has never run is local to this list; there is nothing to rename yet. + // The name is held instead, and written once the first turn creates the row. + if (current.isDraft) { + this.#draftTitle = { id: conversationId, title: trimmed } + this.conversations = this.conversations.map((c) => + c.id === conversationId ? { ...c, title: trimmed } : c + ) + return + } + await this.#writeConversationTitle(conversationId, trimmed) + } + + /** No job came of the send, so nothing is in flight and nothing is waiting on one. */ + #turnFailedToStart(conversationId: string) { + const status = this.#liveStatus(conversationId) + status.isLoading = false + status.isWaitingForResponse = false + status.isDispatchingTurn = false + } + + /** + * Why the composer must stay shut, when the open conversation belongs to the other + * surface. Reading such a chat is fine; adding to it from here is not. + */ + get wrongKindReason(): string | undefined { + const open = this.conversations.find((c) => c.id === this.selectedConversationId) + if (!open || open.isDraft || (open.is_test === true) === (this.surfaceKind === 'test')) + return undefined + return this.surfaceKind === 'test' + ? 'This chat belongs to the deployed flow. Start a new chat to test.' + : 'This chat was run from the editor. Start a new chat to continue here.' + } + + /** + * The open conversation has a turn being dispatched or running: nothing may move the + * conversation under it. Other chats are unaffected where turns run in parallel. + */ + get isTurnInFlight(): boolean { + return this.selectedConversationId + ? this.isConversationBusy(this.selectedConversationId) + : false + } + + /** + * Why this conversation cannot be opened right now. Only where a turn holds the whole + * surface: the editor's panel shows one run on the graph, so leaving the chat that + * started it would leave the graph describing a conversation nobody is reading. + */ + lockedReason(conversationId: string): string | undefined { + if (this.allowsParallelTurns) return undefined + const running = this.runningConversationIds.find((id) => id !== conversationId) + return running ? 'Wait for the current answer to switch conversation' : undefined + } + + /** Why a new chat cannot be started right now. Same rule, with no conversation yet. */ + get newChatReason(): string | undefined { + if (this.allowsParallelTurns) return undefined + return this.runningConversationIds.length > 0 + ? 'Wait for the current answer to start a new chat' + : undefined + } + + async refreshConversations() { + await this.conversationListComponent?.loadData('forceRefresh') + } + + // Only used by InfiniteList + private async deleteConversation(conversationId: string) { + try { + this.deletingConversationId = conversationId + await FlowConversationsService.deleteFlowConversation({ + workspace: this.#workspace()!, + conversationId + }) + if (this.selectedConversationId === conversationId) { + this.selectedConversationId = undefined + } + this.#forget(conversationId) + sendUserToast('Conversation deleted successfully') + } catch (error) { + console.error('Failed to delete conversation:', error) + sendUserToast('Failed to delete conversation', true) + throw error + } finally { + this.deletingConversationId = undefined + } + } + + /** Stop the open conversation's turn. Every other chat's keeps running. */ + async cancelCurrentJob() { + const conversationId = this.selectedConversationId + if (!this.#workspace() || !conversationId) { + return + } + const jobId = this.#statusOf(conversationId).jobId + + try { + if (jobId) { + await JobService.cancelQueuedJob({ + workspace: this.#workspace()!, + id: jobId, + requestBody: {} + }) + sendUserToast(`Job ${jobId} cancelled`) + } + } catch (error) { + console.error('Error cancelling job:', error) + sendUserToast('Could not cancel job', true) + } finally { + this.endTurn(conversationId) + } + } + + async loadConversationMessages(conversationId?: string) { + await this.loadMessages(true, conversationId) + } + + // Only used by InfiniteList + private async loadConversations(page: number, perPage: number) { + if (!this.#workspace() || !this.#path) return [] + + try { + const response = await FlowConversationsService.listFlowConversations({ + workspace: this.#workspace()!, + flowPath: this.#path, + kind: this.conversationKind, + page: page, + perPage: perPage + }) + + return response + } catch (error) { + console.error('Failed to load conversations:', error) + sendUserToast('Failed to load conversations', true) + return [] + } + } + + // Message loading + private async loadMessages(reset: boolean, conversationId?: string) { + let conversationIdToUse = conversationId ?? this.selectedConversationId + if (!this.#workspace() || !conversationIdToUse) return + + if (reset) { + // Rows already held are either what a previous load fetched or what a turn is + // writing right now; either way re-fetching would drop a live turn's temp rows. + if (this.#rowsById[conversationIdToUse]) { + return + } + this.isLoadingMessages = true + } else { + this.loadingMoreMessages = true + } + + const pageToFetch = reset ? 1 : (this.#pagedTo[conversationIdToUse] ?? 1) + 1 + + try { + const previousScrollHeight = this.messagesContainer?.scrollHeight || 0 + + const response = await FlowConversationsService.listConversationMessages({ + workspace: this.#workspace()!, + conversationId: conversationIdToUse, + page: pageToFetch, + perPage: this.#perPage + }) + + if (reset) { + this.#rowsById[conversationIdToUse] = response + this.#pagedTo[conversationIdToUse] = 1 + this.isLoadingMessages = false + await new Promise((resolve) => setTimeout(resolve, 100)) + this.scrollToBottom() + } else { + // Held rows win: a page refetched after a switch back would otherwise be + // prepended a second time. + const held = this.#rowsOf(conversationIdToUse) + const heldIds = new Set(held.map((m) => m.id)) + this.#rowsById[conversationIdToUse] = [ + ...response.filter((m) => !heldIds.has(m.id)), + ...held + ] + this.#pagedTo[conversationIdToUse] = pageToFetch + // Restore scroll position + await new Promise((resolve) => setTimeout(resolve, 50)) + if (this.messagesContainer) { + this.messagesContainer.scrollTop = + this.messagesContainer.scrollHeight - previousScrollHeight + } + } + + this.#hasMoreById[conversationIdToUse] = response.length === this.#perPage + } catch (error) { + console.error('Failed to load messages:', error) + sendUserToast('Failed to load messages: ' + error) + } finally { + this.isLoadingMessages = false + this.loadingMoreMessages = false + } + } + + handleScroll = () => { + if (this.#scrollTimeout) clearTimeout(this.#scrollTimeout) + + this.#scrollTimeout = setTimeout(() => { + if (!this.messagesContainer || !this.hasMoreMessages || this.loadingMoreMessages) return + + if (this.messagesContainer.scrollTop <= 10) { + this.loadMessages(false) + } + }, 200) + } + + scrollToBottom() { + if (this.messagesContainer) { + this.messagesContainer.scrollTop = this.messagesContainer.scrollHeight + } + } + + private scrollToUserMessage(messageId: string) { + if (!this.messagesContainer) return + const messageElement = this.messagesContainer.querySelector(`[data-message-id="${messageId}"]`) + if (messageElement) { + messageElement.scrollIntoView({ behavior: 'smooth', block: 'start' }) + } + } + + private getLastPersistedMessageSeq(conversationId: string) { + const rows = this.#rowsOf(conversationId) + for (let i = rows.length - 1; i >= 0; i--) { + const message = rows[i] + if (!message.id.startsWith('temp-')) { + return message.created_seq + } + } + + return undefined + } + + // Polling + private async pollJobResult(conversationId: string, jobId: string) { + try { + await waitJob(jobId, this.#workspace()) + } catch (error) { + console.error('Error polling job result:', error) + } finally { + // Do a final poll to get all messages from database + try { + await this.pollConversationMessages(conversationId, { removeTempMessages: true }) + } catch {} + this.endTurn(conversationId, { settled: true }) + } + } + + private async pollConversationMessages( + conversationId: string, + options?: { isNewConversation?: boolean; removeTempMessages?: boolean } + ) { + if (!this.#workspace()) return + + try { + const lastSeq = this.getLastPersistedMessageSeq(conversationId) + const response = await FlowConversationsService.listConversationMessages({ + workspace: this.#workspace()!, + conversationId: conversationId, + page: 1, + perPage: 50, + afterSeq: lastSeq + }) + + if (options?.isNewConversation) { + await this.refreshConversations() + } + + // Written to this conversation's rows, not the open one's: a turn keeps landing + // rows while the reader is in another chat. + const filteredResponse = response.filter((msg) => msg.message_type !== 'user') + for (const msg of filteredResponse) { + const rows = this.#rowsOf(conversationId) + if (!rows.find((m) => m.id === msg.id)) { + this.#rowsById[conversationId] = [...rows, msg] + } + } + + // Only remove temporary messages when explicitly requested (e.g., after job completion) + // During streaming, we keep temp messages to avoid them disappearing due to race conditions + if (options?.removeTempMessages) { + this.#rowsById[conversationId] = this.#rowsOf(conversationId).filter( + (msg) => !msg.id.startsWith('temp-') || msg.message_type === 'user' + ) + } + } catch (error) { + console.error('Polling error:', error) + } + } + + private startPolling(conversationId: string, isNewConversation?: boolean) { + const runtime = this.#liveRuntime(conversationId) + if (runtime.pollingInterval) return + runtime.pollingInterval = setInterval(() => { + this.pollConversationMessages(conversationId, { isNewConversation }) + }, 500) // Poll every 0.5 seconds + setTimeout( + () => { + this.stopPolling(conversationId) + }, + 2 * 60 * 1000 + ) // Stop polling after 2 minutes + } + + private stopPolling(conversationId: string) { + const runtime = this.#runtime.get(conversationId) + if (runtime?.pollingInterval) { + clearInterval(runtime.pollingInterval) + runtime.pollingInterval = undefined + } + } + + // Message sending + /** + * Send `inputMessage` as a turn. `onUserRow` is called with the id of the row added + * for it, which is the only moment that id is knowable: the caller needs it to hang + * what the composer sent — attachments, other inputs — on a row that has no job yet. + * + * Returns whether a run actually started, so a caller that spent the composer's draft + * on it can put the draft back. + */ + async sendMessage( + additionalInputs?: Record, + onUserRow?: (rowId: string) => void, + /** + * The conversation the turn was started in. Passed by a caller that had to await + * something first — an attachment upload — since the reader can select another + * conversation while it runs, and the turn belongs to the one they sent it from. + */ + pinnedConversationId?: string + ): Promise { + // Generate a new conversation ID if we don't have one + let currentConversationId = pinnedConversationId ?? this.selectedConversationId + if (!currentConversationId) { + const newConversationId = await this.createConversation({ clearMessages: false }) + currentConversationId = newConversationId + } + + if (!currentConversationId) { + console.error('No conversation ID found') + return false + } + + // One turn per conversation: a second one would run against the same agent memory, + // and the two would interleave into one transcript. A message typed meanwhile is + // queued by the composer and flushed when this one settles. + if (this.isConversationBusy(currentConversationId)) return false + // Where a turn holds the whole surface, another chat's run is equally a reason not + // to start: the editor's graph can only follow one. + if (!this.allowsParallelTurns && this.runningConversationIds.length > 0) return false + + const isNewConversation = this.#rowsOf(currentConversationId).length === 0 + + // Reset state for new message + this.stopPolling(currentConversationId) + + const userMessage: ChatMessage = { + id: `temp-${randomUUID()}`, + content: this.inputMessage.trim(), + created_at: new Date().toISOString(), + created_seq: 0, + message_type: 'user', + conversation_id: currentConversationId + } + + this.#rowsById[currentConversationId] = [...this.#rowsOf(currentConversationId), userMessage] + onUserRow?.(userMessage.id) + const messageContent = this.inputMessage.trim() + this.inputMessage = '' + const status = this.#liveStatus(currentConversationId) + status.isLoading = true + status.isWaitingForResponse = true + + // This turn's own answer, not shared state: a queued follow-up can flush while this + // one is still finishing, and re-enter sendMessage before it reads the result. + let started = false + try { + await tick() + this.scrollToUserMessage(userMessage.id) + + if (this.#useStreaming && this.#path) { + started = await this.handleStreamingMessage( + messageContent, + currentConversationId, + isNewConversation, + userMessage.id, + additionalInputs + ) + } else { + started = await this.handlePollingMessage( + messageContent, + currentConversationId, + isNewConversation, + userMessage.id, + additionalInputs + ) + } + } catch (error) { + console.error('Error running flow:', error) + sendUserToast('Failed to run flow: ' + error, true) + // A turn that never started leaves nothing to wait for. Said here as well as in + // the finally because the streaming path keeps `isLoading` for its own stream, + // and without this the composer and the sidebar stay locked until a reload. + this.#turnFailedToStart(currentConversationId) + started = false + } finally { + if (!this.#useStreaming) { + status.isLoading = false + } + } + + // The row now exists, titled from the message by the server. A name typed while it + // was a draft has to be written over that — unconditionally, not through + // renameConversation: on the streaming path this runs before any refresh, so the + // local row still carries the typed name and an equality check would skip the write. + // Cleared only once it lands, so a failed run keeps the name for the next attempt. + // Only when a turn actually ran: nothing created the row otherwise, so the write + // would 404 and stack a rename failure on top of the real one. + if (started && this.#draftTitle?.id === currentConversationId) { + const { title } = this.#draftTitle + if (await this.#writeConversationTitle(currentConversationId, title)) { + this.#draftTitle = undefined + } + } + + await tick() + this.focusInput() + if (!started) { + // Nothing ran, so the row claiming a turn has to go with it — the caller puts + // the message back in the composer. + this.#rowsById[currentConversationId] = this.#rowsOf(currentConversationId).filter( + (m) => m.id !== userMessage.id + ) + return false + } + return true + } + + /** Answers whether a job was actually started. */ + private async handleStreamingMessage( + messageContent: string, + currentConversationId: string, + isNewConversation: boolean, + userRowId: string, + additionalInputs?: Record + ): Promise { + const runtime = this.#liveRuntime(currentConversationId) + // Close any existing EventSource + runtime.eventSource?.close() + + // Track stream state for this message + runtime.turn = emptyTurnState(currentConversationId) + runtime.replyReveal.reset() + runtime.reasoningReveal.reset() + runtime.streamOffset = undefined + runtime.streamJobId = undefined + + try { + const jobId = await this.#onRunFlow?.(messageContent, currentConversationId, additionalInputs) + if (!jobId) { + console.error('No jobId returned from onRunFlow') + this.#turnFailedToStart(currentConversationId) + return false + } + // What Stop cancels. Set from the flow job now rather than waiting for + // `flow_stream_job_id`, which stays null until a step starts streaming — Stop + // pressed before the first token would otherwise end the turn on screen and leave + // the flow running. + this.#nameTurnJob(currentConversationId, userRowId, jobId) + + // start polling + this.startPolling(currentConversationId, isNewConversation) + + this.#followJob(currentConversationId, jobId) + } catch (error) { + // Everything that can throw here happens before the stream is live — the run + // request itself (which the deployed page's launcher throws from), or building + // the EventSource. Either way no turn ran. + console.error('Stream connection error:', error) + sendUserToast('Failed to connect to stream', true) + this.endTurn(currentConversationId) + this.#turnFailedToStart(currentConversationId) + return false + } + return true + } + + /** + * Follow a job that is already running. The server ends every stream after + * `TIMEOUT_SSE_STREAM`, so a timeout re-enters here on the same job rather than starting + * a second run — which is what it used to do, leaving two runs writing one conversation. + */ + #followJob(currentConversationId: string, jobId: string) { + const runtime = this.#liveRuntime(currentConversationId) + const status = this.#liveStatus(currentConversationId) + runtime.eventSource?.close() + + const streamUrl = `/api/w/${this.#workspace()}/jobs_u/getupdate_sse/${jobId}` + const url = new URL(streamUrl, window.location.origin) + url.searchParams.set('poll_delay_ms', '50') + url.searchParams.set('fast', 'true') + url.searchParams.set('only_result', 'true') + // Resume after what is already on screen; without it the stream replays from the + // start and the answer gains a second copy of everything rendered so far. + if (runtime.streamOffset !== undefined) { + url.searchParams.set('stream_offset', runtime.streamOffset.toString()) + } + const eventSource = new EventSource(url.toString()) + runtime.eventSource = eventSource + let isCompleted = false + + eventSource.onmessage = async (event) => { + try { + const data = JSON.parse(event.data) + const type = data.type + + // The server ends the stream on its own clock; re-attach to the same job. + if (type === 'timeout') { + eventSource.close() + runtime.eventSource = undefined + this.#followJob(currentConversationId, jobId) + return + } + + // Handle ping - just ignore + if (type === 'ping') { + return + } + + // Handle error + if (type === 'error') { + eventSource.close() + runtime.eventSource = undefined + console.error('SSE error:', data) + sendUserToast('Stream error: ' + (data.error || 'Unknown error'), true) + this.endTurn(currentConversationId) + return + } + + // Handle not found + if (type === 'not_found') { + eventSource.close() + runtime.eventSource = undefined + console.error('Job not found') + sendUserToast('Job not found', true) + this.endTurn(currentConversationId) + return + } + + if (type === 'update') { + if (data.flow_stream_job_id) { + status.jobId = data.flow_stream_job_id + if (data.flow_stream_job_id !== runtime.streamJobId) { + const offsetFromOtherJob = + runtime.streamJobId !== undefined && runtime.streamOffset !== undefined + runtime.streamJobId = data.flow_stream_job_id + if (offsetFromOtherJob) { + // The offset indexes the previous sub-job's stream — a retried last step + // gets a new one — so this connection skipped the new job's first + // chunks. Drop this delta and re-attach from the start of that stream. + runtime.streamOffset = undefined + eventSource.close() + runtime.eventSource = undefined + this.#followJob(currentConversationId, jobId) + return + } + } + } + if (data.stream_offset !== undefined) { + runtime.streamOffset = data.stream_offset + } + // Process new stream content + if (data.new_result_stream) { + // Stop polling since we are receiving last step streaming + this.stopPolling(currentConversationId) + // One chunk can carry several events, so each is applied in turn: a + // chunk holding a call and its result must produce both. + for (const event of parseStreamEvents(data.new_result_stream)) { + if (event.kind === 'reasoning') { + status.isReasoningActive = true + runtime.reasoningReveal.push(event.content) + } else if (event.kind === 'token') { + runtime.replyReveal.push(event.content) + } else { + // Whatever the pacing still holds belongs to the row before the tool — + // thinking that led straight to the call included — so it is revealed + // before the event that closes that row. + if (event.kind === 'tool_call' || event.kind === 'tool_execution') { + this.#flushReveals(currentConversationId) + status.currentReasoning = '' + status.isReasoningActive = false + } + const step = applyStreamEvent( + { rows: this.#rowsOf(currentConversationId), state: runtime.turn }, + event, + this.#newRowId + ) + this.#rowsById[currentConversationId] = step.rows + runtime.turn = step.state + } + } + } + + // Handle completion + if (data.completed) { + isCompleted = true + // Anything still buffered would be dropped by the temp-row sweep below. + this.#flushReveals(currentConversationId) + // Do a final poll to get all messages from database + await this.pollConversationMessages(currentConversationId, { + removeTempMessages: true + }) + this.endTurn(currentConversationId, { settled: true }) + } + } + } catch (error) { + console.error('Error processing stream event:', error) + } + } + + eventSource.onerror = (error) => { + if (isCompleted) return + console.error('EventSource error:', error) + sendUserToast('Stream error occurred', true) + this.endTurn(currentConversationId) + } + } + + /** Answers whether a job was actually started. */ + private async handlePollingMessage( + messageContent: string, + currentConversationId: string, + isNewConversation: boolean, + userRowId: string, + additionalInputs?: Record + ): Promise { + const jobId = await this.#onRunFlow?.(messageContent, currentConversationId, additionalInputs) + if (!jobId) { + console.error('No jobId returned from onRunFlow') + this.#turnFailedToStart(currentConversationId) + return false + } + + // Store the current job ID so it can be cancelled + this.#nameTurnJob(currentConversationId, userRowId, jobId) + + if (isNewConversation) { + await this.refreshConversations() + } + + // Start polling for intermediate messages in non-streaming mode too + this.startPolling(currentConversationId) + this.pollJobResult(currentConversationId, jobId) + return true + } +} + +export const createFlowChatManager = () => new FlowChatManager() diff --git a/frontend/src/lib/components/flows/conversations/FlowChatManager.test.ts b/frontend/src/lib/components/flows/conversations/FlowChatManager.test.ts new file mode 100644 index 0000000000..3ba9098691 --- /dev/null +++ b/frontend/src/lib/components/flows/conversations/FlowChatManager.test.ts @@ -0,0 +1,257 @@ +import { describe, it, expect, vi, beforeEach, afterEach } from 'vitest' +import { FlowConversationsService } from '$lib/gen' +import { createFlowChatManager } from './FlowChatManager.svelte' + +vi.mock('$lib/gen', () => ({ + FlowConversationsService: { + listConversationMessages: vi.fn(), + deleteFlowConversation: vi.fn() + }, + JobService: {}, + FlowService: {} +})) +vi.mock('$lib/toast', () => ({ sendUserToast: vi.fn() })) +vi.mock('$lib/stores', () => ({ + userStore: { subscribe: (run: (v: unknown) => void) => (run({ username: 'admin' }), () => {}) }, + workspaceStore: { subscribe: (run: (v: unknown) => void) => (run('ws'), () => {}) } +})) + +const rows = (conversationId: string, count: number) => + Array.from({ length: count }, (_, i) => ({ + id: `${conversationId}-${i}`, + content: '', + created_at: new Date().toISOString(), + created_seq: i, + conversation_id: conversationId, + message_type: i % 2 === 0 ? 'user' : 'assistant' + })) + +function managerWithRows() { + const manager = createFlowChatManager() + manager.operatingWorkspace = () => 'ws' + ;(manager as any).initialize(vi.fn(), 'u/admin/flow', false) + return manager +} + +/** + * The unread badge is driven by a watermark the manager writes when the reader leaves a + * chat. What it must never do is keep counting a chat the reader can no longer reach — + * there would be no row left in the sidebar to clear it. + */ +describe('unread bookkeeping', () => { + beforeEach(() => { + vi.mocked(FlowConversationsService.listConversationMessages).mockReset() + vi.mocked(FlowConversationsService.deleteFlowConversation).mockReset() + }) + + it('counts rows loaded for a chat the reader is not in', async () => { + const manager = managerWithRows() + manager.selectedConversationId = 'open' + vi.mocked(FlowConversationsService.listConversationMessages).mockResolvedValue( + rows('a', 2) as any + ) + await manager.loadConversationMessages('a') + + expect(manager.unreadCount('a')).toBe(2) + expect(manager.totalUnread).toBe(2) + }) + + it('stops counting a deleted chat', async () => { + const manager = managerWithRows() + manager.selectedConversationId = 'open' + vi.mocked(FlowConversationsService.listConversationMessages).mockResolvedValue( + rows('a', 2) as any + ) + await manager.loadConversationMessages('a') + expect(manager.totalUnread).toBe(2) + + vi.mocked(FlowConversationsService.deleteFlowConversation).mockResolvedValue(undefined as any) + await (manager as any).deleteConversation('a') + + expect(manager.totalUnread).toBe(0) + expect(manager.unreadCount('a')).toBe(0) + }) + + it('treats the open chat as read', async () => { + const manager = managerWithRows() + vi.mocked(FlowConversationsService.listConversationMessages).mockResolvedValue( + rows('a', 3) as any + ) + await manager.selectConversation('a') + + expect(manager.unreadCount('a')).toBe(0) + expect(manager.totalUnread).toBe(0) + }) + + // The message the reader just typed came back as unread behind them, because this path + // moves the selection without going through `selectConversation`. + it('marks the chat being left as read when a new one is started', async () => { + const manager = managerWithRows() + vi.mocked(FlowConversationsService.listConversationMessages).mockResolvedValue( + rows('a', 2) as any + ) + await manager.selectConversation('a') + await manager.createConversation({ clearMessages: true }) + + expect(manager.unreadCount('a')).toBe(0) + }) +}) + +/** + * A queued message goes out when the turn ahead of it reaches a terminal state — not + * merely when the chat stops looking busy. The stream dropping is the case that separates + * the two: `onerror` ends the turn's client-side state, but the flow job it was following + * keeps running on a worker, and starting the next turn there would interleave two runs + * over one conversation's agent memory. + */ +describe('queued turns wait for a settled run', () => { + it('does not flush when a turn ends without settling', () => { + const manager = managerWithRows() + const flushed: string[] = [] + manager.onTurnSettled = (id) => flushed.push(id) + + manager.endTurn('a') + + expect(flushed).toEqual([]) + }) + + it('flushes the chat whose run settled, naming it', () => { + const manager = managerWithRows() + const flushed: string[] = [] + manager.onTurnSettled = (id) => flushed.push(id) + + manager.endTurn('a', { settled: true }) + + expect(flushed).toEqual(['a']) + }) +}) + +/** + * The server ends every SSE stream on its own clock (`TIMEOUT_SSE_STREAM`, 60s by default) + * and expects the client to re-attach. Re-entering the run-starting path instead spawned a + * second flow job a minute, each one writing the same conversation — so this pins that a + * timeout follows the job it already has, and resumes where the last one stopped. + */ +describe('an SSE timeout re-attaches instead of re-running', () => { + class FakeEventSource { + static opened: string[] = [] + static live: FakeEventSource[] = [] + onmessage: ((e: { data: string }) => void) | null = null + onerror: ((e: unknown) => void) | null = null + closed = false + constructor(url: string) { + FakeEventSource.opened.push(url) + FakeEventSource.live.push(this) + } + close() { + this.closed = true + } + emit(payload: unknown) { + this.onmessage?.({ data: JSON.stringify(payload) }) + } + } + + let realEventSource: unknown + let live: ReturnType | undefined + + beforeEach(() => { + FakeEventSource.opened = [] + FakeEventSource.live = [] + realEventSource = (globalThis as any).EventSource + ;(globalThis as any).EventSource = FakeEventSource + // `test-setup.ts` makes `window` be `globalThis`, which has no `location` — so the + // stream URL's base is what is missing, not the window itself. + ;(globalThis as any).location = { origin: 'http://localhost' } + }) + + afterEach(() => { + // The turn is still running: its 500ms poll interval would otherwise keep calling + // into the manager after the test. Here rather than in the body, which a failing + // assertion would skip. + live?.cleanup() + live = undefined + ;(globalThis as any).EventSource = realEventSource + delete (globalThis as any).location + }) + + it('follows the same job and carries the offset forward', async () => { + const manager = (live = managerWithRows()) + const onRunFlow = vi.fn(async () => 'job-1') + ;(manager as any).initialize(onRunFlow, 'u/admin/flow', true) + manager.operatingWorkspace = () => 'ws' + + manager.selectedConversationId = 'a' + manager.inputMessage = 'hello' + await manager.sendMessage(undefined, undefined, 'a') + + expect(onRunFlow).toHaveBeenCalledTimes(1) + // Stop has something to cancel before any token arrives: the flow job is named as + // soon as it is enqueued, not when the streaming step starts. + expect(manager.currentJobId).toBe('job-1') + + FakeEventSource.live[0].emit({ type: 'update', stream_offset: 42 }) + FakeEventSource.live[0].emit({ type: 'timeout' }) + + // No second run, and the reconnect resumes rather than replaying the answer. + expect(onRunFlow).toHaveBeenCalledTimes(1) + expect(FakeEventSource.opened).toHaveLength(2) + expect(FakeEventSource.opened[0]).toContain('/job-1') + expect(FakeEventSource.opened[1]).toContain('/job-1') + expect(FakeEventSource.opened[1]).toContain('stream_offset=42') + }) +}) + +/** Why the row has to be stamped at all is on `#nameTurnJob`; this pins that it is, and + * that it lands in the conversation the turn was sent to. */ +describe('a sent message names the run it started', () => { + let realEventSource: unknown + let live: ReturnType | undefined + + beforeEach(() => { + vi.mocked(FlowConversationsService.listConversationMessages).mockResolvedValue([] as any) + realEventSource = (globalThis as any).EventSource + ;(globalThis as any).EventSource = class { + onmessage: unknown = null + onerror: unknown = null + close() {} + } + ;(globalThis as any).location = { origin: 'http://localhost' } + }) + + afterEach(() => { + live?.cleanup() + live = undefined + ;(globalThis as any).EventSource = realEventSource + delete (globalThis as any).location + }) + + it('stamps the row the turn began with, so the transcript can replay it', async () => { + const manager = (live = managerWithRows()) + ;(manager as any).initialize(vi.fn(async () => 'job-7'), 'u/admin/flow', true) + manager.operatingWorkspace = () => 'ws' + manager.selectedConversationId = 'a' + manager.inputMessage = 'hello' + + await manager.sendMessage(undefined, undefined, 'a') + + const userRow = manager.messages.find((m) => m.message_type === 'user') + expect(userRow?.job_id).toBe('job-7') + }) + + // A queued message flushes into the chat it was typed in, which by then need not be the + // one on screen — the row and its job must both land there, not in the open chat. + it('stamps the row in the conversation the turn was sent to, not the open one', async () => { + const manager = (live = managerWithRows()) + ;(manager as any).initialize(vi.fn(async () => 'job-8'), 'u/admin/flow', true) + manager.operatingWorkspace = () => 'ws' + manager.selectedConversationId = 'open-chat' + manager.inputMessage = 'sent to the background chat' + + await manager.sendMessage(undefined, undefined, 'background-chat') + + expect(manager.messages).toEqual([]) + await manager.selectConversation('background-chat') + const userRow = manager.messages.find((m) => m.message_type === 'user') + expect(userRow?.job_id).toBe('job-8') + }) +}) diff --git a/frontend/src/lib/components/flows/conversations/FlowChatModelSettings.svelte b/frontend/src/lib/components/flows/conversations/FlowChatModelSettings.svelte new file mode 100644 index 0000000000..46aa2bcdd1 --- /dev/null +++ b/frontend/src/lib/components/flows/conversations/FlowChatModelSettings.svelte @@ -0,0 +1,277 @@ + + +{#snippet providerSummary()} + {#if resourcePath} + {resourcePath} + {/if} +{/snippet} + +{#if resourceEditable} + { + resourcesVersion++ + if (e.detail) { + pendingResourcePath = e.detail + } + }} + /> +{/if} + + diff --git a/frontend/src/lib/components/flows/conversations/FlowConversationsSidebar.svelte b/frontend/src/lib/components/flows/conversations/FlowConversationsSidebar.svelte index 02c2581db1..116e79e54a 100644 --- a/frontend/src/lib/components/flows/conversations/FlowConversationsSidebar.svelte +++ b/frontend/src/lib/components/flows/conversations/FlowConversationsSidebar.svelte @@ -1,204 +1,373 @@ +{#snippet statusDot(conversation: ConversationWithDraft)} + + + {#snippet resting()} + + {/snippet} + +{/snippet} +
-
+
- - + {#if !emptyString(description)} + + + + + {#snippet text()} + + + {/snippet} + + {/if} +
+ +
-
New chat
- + + {#if manager.canFilterConversationKind} + + {#snippet trigger()} + +
- {#if !expanded} + {#if !manager.isSidebarExpanded}
{/if} -
- {#if draftShown && expanded} -
- -
- {/if} +
- {#snippet customRow({ item: conversation })} - {#if expanded} + {#snippet customRow({ item: conversation, hover })} + {#if manager.isSidebarExpanded}
- + onClick={() => manager.selectConversation(conversation.id, conversation.isDraft)} + selected={manager.selectedConversationId === conversation.id} + disabled={!!rowLocked(conversation)} + title={rowLocked(conversation)} + btnClasses="transition-all duration-150 group gap-2" + > + + {@render statusDot(conversation)} + {@const unread = manager.unreadCount(conversation.id)} + 0 ? 'font-semibold text-primary' : '' + )} + > + {getConversationTitle(conversation)} + + {#if manager.conversationStatus(conversation.id) === 'queued' || unread > 0} + + {#if manager.conversationStatus(conversation.id) === 'queued'} + + {/if} + + + {/if} + + {#if !rowLocked(conversation)} + + +
e.stopPropagation()} + > + rowActions(conversation)} size="xs" /> +
+ {/if} + + {/if}
{/if} {/snippet} {#snippet empty()} - {#if !draftShown} -
-

No conversations yet

-
- {/if} +
+

No conversations yet

+
{/snippet}
diff --git a/frontend/src/lib/components/flows/conversations/agentChatInputs.test.ts b/frontend/src/lib/components/flows/conversations/agentChatInputs.test.ts new file mode 100644 index 0000000000..0e7927c89d --- /dev/null +++ b/frontend/src/lib/components/flows/conversations/agentChatInputs.test.ts @@ -0,0 +1,531 @@ +import { describe, expect, it } from 'vitest' +import { + agentModelGap, + agentModelWiringInputs, + composerOwnedInputs, + attachmentsTargetFor, + parseProviderTransform, + resolveAgentChatInputs, + resolveAgentModelWiring, + withoutRejectedEffort +} from './agentChatInputs' +import type { FlowModule } from '$lib/gen' + +/** An agent step with the given input transforms, as the editor stores them. */ +function agentWith(input_transforms: Record): FlowModule { + return { + id: 'a', + value: { type: 'aiagent', tools: [], input_transforms } + } as unknown as FlowModule +} + +function agent(expr: string): FlowModule { + return { + id: 'a', + value: { + type: 'aiagent', + tools: [], + input_transforms: { provider: { type: 'javascript', expr } } + } + } as unknown as FlowModule +} + +describe('parseProviderTransform', () => { + it('reads a whole-object reference', () => { + expect(parseProviderTransform({ type: 'javascript', expr: 'flow_input.model' })).toEqual({ + whole: 'model', + fields: {}, + fixed: {} + }) + }) + + it('splits a partial expression into wired and fixed fields', () => { + const wiring = parseProviderTransform({ + type: 'javascript', + expr: "{ kind: 'anthropic', resource: '$res:u/admin/claude', model: flow_input.model, reasoning_effort: flow_input.thinking }" + }) + expect(wiring).toEqual({ + fields: { model: 'model', reasoning_effort: 'thinking' }, + fixed: { kind: 'anthropic', resource: '$res:u/admin/claude' } + }) + }) + + // The regression this detector exists for: counting flow_input references would read + // this as one input carrying the whole provider object, and the composer would write + // {kind, resource, model} into an input the expression uses as the model name. + it('does not mistake a single-reference partial expression for a whole-object one', () => { + const wiring = parseProviderTransform({ + type: 'javascript', + expr: "{ kind: 'anthropic', resource: '$res:u/admin/claude', model: flow_input.model }" + }) + expect(wiring?.whole).toBeUndefined() + expect(wiring?.fields).toEqual({ model: 'model' }) + }) + + // The flow editor's JS field commonly holds a parenthesised object, which is how an + // author writes one without it reading as a block. + it('accepts a parenthesised object expression', () => { + const wiring = parseProviderTransform({ + type: 'javascript', + expr: `({ + "kind": "anthropic", + "resource": "$res:u/admin/anthropic_windmill_codegen", + "model": "claude-sonnet-5", + "reasoning_effort": flow_input.thinking +})` + }) + expect(wiring).toEqual({ + fields: { reasoning_effort: 'thinking' }, + fixed: { + kind: 'anthropic', + resource: '$res:u/admin/anthropic_windmill_codegen', + model: 'claude-sonnet-5' + } + }) + }) + + it('treats a static provider as entirely fixed', () => { + expect( + parseProviderTransform({ + type: 'static', + value: { kind: 'openai', resource: '$res:u/admin/oai', model: 'gpt-5.6' } + }) + ).toEqual({ + fields: {}, + fixed: { kind: 'openai', resource: '$res:u/admin/oai', model: 'gpt-5.6' } + }) + }) + + it.each([ + ['{ ...base, model: flow_input.model }', 'a spread could supply any field'], + ['{ model: pickModel(flow_input.x) }', 'a call is not classifiable'], + ['flow_input.model + 1', 'not a bare reference'], + ['{ model: ', 'unparseable'] + ])('gives up on %s', (expr) => { + expect(parseProviderTransform({ type: 'javascript', expr } as any)).toBeUndefined() + }) +}) + +describe('resolveAgentChatInputs', () => { + const schema = { properties: { files: { type: 'array' } }, required: [] } + const reader = (name: string) => + agentWith({ user_attachments: { type: 'javascript', expr: `flow_input.${name}` } }) + // Every agent step carries a placeholder transform for each key of AI_AGENT_SCHEMA + // (loadSchemaFromModule writes them back onto the module), so an agent that reads + // nothing must not be mistaken for one reading a different input. + const seeded = () => agentWith({ user_attachments: { type: 'static', value: undefined } }) + + it('promotes the input one agent reads', () => { + expect(resolveAgentChatInputs([reader('files')], schema).map((i) => i.name)).toEqual(['files']) + }) + + it('still promotes it when another agent leaves the field unwired', () => { + expect(resolveAgentChatInputs([reader('files'), seeded()], schema).map((i) => i.name)).toEqual([ + 'files' + ]) + }) + + it('promotes nothing when two agents read different inputs', () => { + const twoInputs = { + properties: { files: { type: 'array' }, docs: { type: 'array' } }, + required: [] + } + expect(resolveAgentChatInputs([reader('files'), reader('docs')], twoInputs)).toEqual([]) + }) +}) + +describe('agentModelWiringInputs', () => { + // The button writes `kind` only alongside a resource, since a provider is picked as a + // pair. Hiding a kind input it cannot write would leave the run without one. + it('keeps a kind input the model button cannot write', () => { + const wiring = resolveAgentModelWiring([ + agent(`({ kind: flow_input.k, "resource": "$res:u/admin/claude", model: flow_input.m })`) + ]) + expect(agentModelWiringInputs(wiring)).toEqual(['m']) + }) + + it('hides a kind input it writes with the resource', () => { + const wiring = resolveAgentModelWiring([ + agent(`({ kind: flow_input.k, resource: flow_input.r, model: flow_input.m })`) + ]) + expect(agentModelWiringInputs(wiring)?.sort()).toEqual(['k', 'm', 'r']) + }) + + // The button always draws a thinking control — a ladder, a typed token, or why there is + // neither — so a wired effort is the button's whatever the registry knows about the model. + it('always claims a wired reasoning_effort, whatever the provider', () => { + const custom = resolveAgentModelWiring([ + agent( + `({ "kind": "customai", "resource": "$res:u/admin/custom", model: flow_input.m, reasoning_effort: flow_input.thinking })` + ) + ]) + expect(agentModelWiringInputs(custom)?.sort()).toEqual(['m', 'thinking']) + + const known = resolveAgentModelWiring([ + agent( + `({ "kind": "openai", "resource": "$res:u/admin/oai", "model": "gpt-4o", reasoning_effort: flow_input.thinking })` + ) + ]) + expect(agentModelWiringInputs(known)).toEqual(['thinking']) + }) +}) + +// The modal is whatever this does not return, so the two can no longer disagree about an +// input — and what a control can do *right now* is deliberately not part of the answer. +describe('composerOwnedInputs', () => { + const wiring = () => + resolveAgentModelWiring([ + agent(`({ "kind": "openai", "resource": "$res:u/admin/oai", model: flow_input.m })`) + ]) + + it('claims the model wiring and the attachments target together', () => { + expect(composerOwnedInputs(wiring(), { name: 'files' })?.sort()).toEqual(['files', 'm']) + }) + + it('claims the attachments input whatever the workspace can do with it', () => { + // No object storage is a state the paperclip shows, not a handover: the modal could + // not upload either, and the run would fail fetching a key typed there. + expect(composerOwnedInputs(undefined, { name: 'files' })).toEqual(['files']) + }) + + it('leaves an input no control fits to the modal', () => { + // `attachmentsTargetFor` returns nothing for a shape the paperclip cannot write — a + // plain string key, say — and then the input is genuinely the modal's. + expect(composerOwnedInputs(wiring(), undefined)).toEqual(['m']) + }) +}) + +describe('resolveAgentModelWiring', () => { + const fixedResource = `"kind": "anthropic", "resource": "$res:u/admin/claude"` + + it('drives a field every agent reads from the same input', () => { + const wiring = resolveAgentModelWiring([ + agent( + `({ ${fixedResource}, "model": "claude-sonnet-5", reasoning_effort: flow_input.thinking })` + ), + agent( + `({ ${fixedResource}, "model": "claude-opus-5", reasoning_effort: flow_input.thinking })` + ) + ]) + expect(wiring?.fields).toEqual({ reasoning_effort: 'thinking' }) + // The agents run different models, so there is no single one to name. + expect(wiring?.fixed).toEqual({ kind: 'anthropic', resource: '$res:u/admin/claude' }) + }) + + it('drops a field the agents disagree about', () => { + const wiring = resolveAgentModelWiring([ + agent(`({ ${fixedResource}, reasoning_effort: flow_input.thinking })`), + agent(`({ ${fixedResource}, reasoning_effort: flow_input.other })`) + ]) + expect(wiring?.fields.reasoning_effort).toBeUndefined() + }) + + // A nested agent is the parent agent's tool, not a step the reader is talking to: the + // graph walks it as a child module, and counting it here would defeat the chat's own + // model control (this is the shape of the all-tools example flow). + it("ignores an agent carried as another agent's tool", () => { + const parent = agent('flow_input.model') + ;(parent.value as any).tools = [ + { + id: 'summarize', + value: { + tool_type: 'flowmodule', + type: 'aiagent', + tools: [], + input_transforms: { + provider: { type: 'static', value: { kind: 'anthropic', model: 'claude-sonnet-5' } } + } + } + } + ] + expect(resolveAgentModelWiring([parent])).toEqual({ + whole: 'model', + fields: {}, + fixed: {}, + someAgentCannotRun: false + }) + }) + + // The control writes one flow input; an agent that fixes the field instead never reads + // it, so offering the control would move one agent and leave the other where it was. + it('does not offer a field one agent wires and another fixes', () => { + const wiring = resolveAgentModelWiring([ + agent(`({ ${fixedResource}, model: flow_input.model })`), + agent(`({ ${fixedResource}, "model": "claude-opus-5" })`) + ]) + expect(wiring?.fields.model).toBeUndefined() + expect(wiring?.fixed.model).toBeUndefined() + }) + + // Disagreeing about the model is not the same as having no model: the flow runs, on a + // different one per agent, and the composer has nothing to fix. + it('says nothing about a model the agents merely disagree about', () => { + const wiring = resolveAgentModelWiring([ + agent(`({ ${fixedResource}, "model": "claude-sonnet-5" })`), + agent(`({ ${fixedResource}, "model": "claude-opus-5" })`) + ]) + expect(agentModelGap(wiring)).toBeUndefined() + }) + + it('still reports an agent with nothing to call', () => { + expect( + agentModelGap(resolveAgentModelWiring([agent(`({ "kind": "openai", "model": "" })`)])) + ).toBe('Pick a provider and model on the AI agent step to use this chat.') + }) + + // An expression the parser cannot account for could supply anything, so the agents it + // belongs to cannot be spoken for either. + it("offers nothing when one agent's provider cannot be read", () => { + expect( + resolveAgentModelWiring([ + agent(`({ ${fixedResource}, model: flow_input.model })`), + agent(`({ ...base, model: flow_input.model })`) + ]) + ).toBeUndefined() + }) + + // Disagreement is not the same as absence, but an agent with an empty model still + // cannot run, however well its neighbour is configured. + it('keeps warning when one agent has no model and another does', () => { + const wiring = resolveAgentModelWiring([ + agent(`({ ${fixedResource}, "model": "claude-sonnet-5" })`), + agent(`({ ${fixedResource}, "model": "" })`) + ]) + expect(agentModelGap(wiring)).toBe( + 'Pick a provider and model on the AI agent step to use this chat.' + ) + }) + + it('refuses a flow mixing whole-object and field-by-field wiring', () => { + expect( + resolveAgentModelWiring([ + agent('flow_input.provider'), + agent(`({ ${fixedResource}, model: flow_input.model })`) + ]) + ).toBeUndefined() + }) +}) + +describe('attachmentsTargetFor', () => { + const input = (property: Record) => + ({ name: 'files', key: 'user_attachments', property }) as any + + it('takes a list of s3 files, and says it holds several', () => { + expect( + attachmentsTargetFor(input({ type: 'array', items: { resourceType: 's3object' } })) + ).toEqual({ name: 'files', multiple: true }) + }) + + it('takes a single s3 file', () => { + expect(attachmentsTargetFor(input({ format: 'resource-s3_object' }))).toEqual({ + name: 'files', + multiple: false + }) + }) + + // The transform can build the s3 object itself, promoting an input that holds a key + // rather than a file. Uploading into it would write an object where a string is declared. + it('offers no paperclip where the input cannot hold a file', () => { + expect(attachmentsTargetFor(input({ type: 'string' }))).toBeUndefined() + expect( + attachmentsTargetFor(input({ type: 'array', items: { type: 'string' } })) + ).toBeUndefined() + expect(attachmentsTargetFor(undefined)).toBeUndefined() + }) +}) + +describe('withoutRejectedEffort', () => { + const wiring = (fields: Record, fixed: Record = {}) => + ({ fields, fixed }) as any + + // The live 400 this guards: Anthropic turns any effort into adaptive thinking, which + // Haiku rejects outright ("adaptive thinking is not supported on this model"). + it('drops an effort the chosen model rejects', () => { + const values = { model: 'claude-haiku-4-5-20251001', reasoning_effort: 'high' } + expect( + withoutRejectedEffort( + wiring({ model: 'model', reasoning_effort: 'reasoning_effort' }, { kind: 'anthropic' }), + values + ) + ).toEqual({ model: 'claude-haiku-4-5-20251001', reasoning_effort: '' }) + }) + + it('keeps an effort the model takes', () => { + const values = { model: 'claude-sonnet-5', reasoning_effort: 'high' } + expect( + withoutRejectedEffort( + wiring({ model: 'model', reasoning_effort: 'reasoning_effort' }, { kind: 'anthropic' }), + values + ) + ).toBe(values) + }) + + // Clearing on a guess would override the author's own default. + it('leaves the value alone for a family the registry cannot speak for', () => { + const values = { model: 'some-model', reasoning_effort: 'high' } + expect( + withoutRejectedEffort( + wiring({ model: 'model', reasoning_effort: 'reasoning_effort' }, { kind: 'customai' }), + values + ) + ).toBe(values) + }) + + // A flow that wires `provider` as one object keeps the effort inside it, so reading + // `fields.reasoning_effort` finds nothing and the 400 would go out unchecked. + it('clears the effort inside a whole-object provider input', () => { + const wiring = resolveAgentModelWiring([agent('flow_input.provider')]) + const values = { + provider: { + kind: 'anthropic', + model: 'claude-haiku-4-5-20251001', + reasoning_effort: 'high' + } + } + expect(withoutRejectedEffort(wiring, values)).toEqual({ + provider: { + kind: 'anthropic', + model: 'claude-haiku-4-5-20251001', + reasoning_effort: '' + } + }) + }) + + it('leaves a whole-object provider alone when the model takes the effort', () => { + const wiring = resolveAgentModelWiring([agent('flow_input.provider')]) + const values = { + provider: { kind: 'anthropic', model: 'claude-sonnet-5', reasoning_effort: 'high' } + } + expect(withoutRejectedEffort(wiring, values)).toBe(values) + }) +}) + +/** + * The shape agent chat is usually built in: one agent answers the reader, others do work of + * their own in branches. A sub-agent never sees the message, so what it runs on is not a + * setting this conversation has. + */ +describe('agents that do not read the message', () => { + const answerer = (provider: string) => + ({ + id: 'answerer', + value: { + type: 'aiagent', + tools: [], + input_transforms: { + user_message: { type: 'javascript', expr: 'flow_input.user_message' }, + provider: { type: 'javascript', expr: provider } + } + } + }) as unknown as FlowModule + const subAgent = (provider: string, message = "'critique: ' + results.answerer") => + ({ + id: 'critic', + value: { + type: 'aiagent', + tools: [], + input_transforms: { + user_message: { type: 'javascript', expr: message }, + provider: { type: 'javascript', expr: provider } + } + } + }) as unknown as FlowModule + + const wired = `({ kind: 'anthropic', resource: '$res:u/admin/c', model: flow_input.model })` + const fixed = `({ kind: 'anthropic', resource: '$res:u/admin/c', model: 'claude-sonnet-5' })` + + it('keeps the model control when only a sub-agent fixes its own model', () => { + const wiring = resolveAgentModelWiring([answerer(wired), subAgent(fixed)]) + expect(wiring?.fields.model).toBe('model') + }) + + // Two agents both answering the reader still have to agree: either might be the one + // that replies, so a control moving one of them would be a lie about the other. + it('still needs agreement among the agents that do read the message', () => { + const wiring = resolveAgentModelWiring([ + answerer(wired), + subAgent(fixed, 'flow_input.user_message') + ]) + expect(wiring?.fields.model).toBeUndefined() + }) + + // An author wraps the message in context freely; that is still the agent being talked to. + it('counts an agent whose prompt embeds the message', () => { + const wrapped = subAgent(fixed, "'Answer politely: ' + flow_input.user_message") + const wiring = resolveAgentModelWiring([answerer(wired), wrapped]) + expect(wiring?.fields.model).toBeUndefined() + }) + + // Nothing to scope to means the flow is shaped in some way this cannot read, so every + // agent counts again rather than none. + it('falls back to every agent when none reads the message', () => { + const wiring = resolveAgentModelWiring([subAgent(wired), subAgent(fixed)]) + expect(wiring?.fields.model).toBeUndefined() + }) + + // The editor writes the dot form, but an author may hand-edit either. A shape this does + // not recognise drops that agent out of the unanimity check it should be part of. + it.each([ + ["flow_input['user_message']", 'bracket access'], + ['flow_input?.user_message', 'optional chaining'], + ["'Answer politely: ' + flow_input.user_message", 'embedded in a prompt'], + ['flow_input.user_message + flow_input.tone', 'read alongside another input'] + ])('treats %s as reading the message', (message) => { + const wiring = resolveAgentModelWiring([ + answerer(wired), + subAgent(fixed, message as string) + ]) + expect(wiring?.fields.model).toBeUndefined() + }) + + // A trailing line comment used to swallow the closing paren the parser adds, which made + // the whole expression unreadable and dropped the agent out of the check. + it.each([ + ['flow_input.user_message // the message', 'a trailing comment'], + ['flow_input.user_message\n// why', 'a comment on its own last line'] + ])('still reads the message with %s', (message) => { + const wiring = resolveAgentModelWiring([ + answerer(wired), + subAgent(fixed, message as string) + ]) + expect(wiring?.fields.model).toBeUndefined() + }) + + it.each([ + ['// see flow_input.user_message', 'a mention in a comment'], + ["'flow_input.user_message'", 'a mention in a string'], + ['flow_input.user_message_extra', 'a different input with the same prefix'] + ])('does not treat %s as reading the message', (message) => { + const wiring = resolveAgentModelWiring([ + answerer(wired), + subAgent(fixed, message as string) + ]) + expect(wiring?.fields.model).toBe('model') + }) +}) + +// An author annotating their own provider must not lose the model control for it: a +// comment beside the expression is not another expression. +describe('parseProviderTransform with comments', () => { + it.each([ + ['flow_input.provider // the one to use', 'a trailing line comment'], + ['flow_input.provider /* the one to use */', 'a trailing block comment'], + ['/* pick one */ flow_input.provider', 'a leading comment'] + ])('reads a whole-object reference despite %s', (expr) => { + expect(parseProviderTransform({ type: 'javascript', expr })?.whole).toBe('provider') + }) + + it('reads an object literal with a comment inside it', () => { + const wiring = parseProviderTransform({ + type: 'javascript', + expr: `({ kind: 'anthropic', /* fixed */ resource: '$res:u/admin/c', model: flow_input.model })` + }) + expect(wiring?.fields.model).toBe('model') + }) + + // The check exists to reject an expression with something else beside it; a second + // expression is still something else. + it('still refuses a second expression beside it', () => { + expect(parseProviderTransform({ type: 'javascript', expr: 'flow_input.provider, 1' })).toBeUndefined() + }) +}) diff --git a/frontend/src/lib/components/flows/conversations/agentChatInputs.ts b/frontend/src/lib/components/flows/conversations/agentChatInputs.ts new file mode 100644 index 0000000000..15520d1747 --- /dev/null +++ b/frontend/src/lib/components/flows/conversations/agentChatInputs.ts @@ -0,0 +1,536 @@ +import type { AIProvider, FlowModule, InputTransform } from '$lib/gen' +import { getReasoningCapability } from '$lib/components/copilot/reasoningRegistry' +import { parseExpressionAt } from 'acorn' + +/** + * The flow's own AI agent steps, including those inside loops and branches but never one + * carried as another agent's tool. + * + * The graph walks an agent's tools as if they were child steps (flowTree.ts), which is + * right for the graph and wrong here: a tool agent's provider belongs to the agent that + * calls it, not to the chat. Counting it would let a nested agent's fixed model defeat the + * composer's model control on the step the reader is actually talking to. + */ +function agentSteps(modules: FlowModule[] | undefined): FlowModule[] { + const found: FlowModule[] = [] + const walk = (mods: FlowModule[]) => { + for (const module of mods) { + const value = module.value as any + if (value?.type === 'aiagent') { + found.push(module) + continue + } + if (value?.type === 'forloopflow' || value?.type === 'whileloopflow') { + walk(value.modules ?? []) + } else if (value?.type === 'branchone') { + walk(value.default ?? []) + for (const branch of value.branches ?? []) walk(branch.modules ?? []) + } else if (value?.type === 'branchall') { + for (const branch of value.branches ?? []) walk(branch.modules ?? []) + } + } + } + walk(modules ?? []) + return found +} + +/** Block and line comments removed, so what is left is only what affects the value. */ +function withoutComments(source: string): string { + return source.replace(/\/\*[\s\S]*?\*\//g, '').replace(/\/\/[^\n]*/g, '') +} + +/** + * An expression wrapped so acorn will read it: parenthesised, because a leading `{` would + * otherwise parse as a block, and on its own line, because a trailing `// comment` would + * otherwise swallow the closing paren and make the whole thing unparseable. + */ +function parenthesised(expr: string): string { + return `(\n${expr}\n)` +} + +/** The flow input the server requires on a chat-enabled flow, and stores as the message. */ +const MESSAGE_INPUT = 'user_message' + +/** + * The agents the reader is talking to: the ones the chat's message is fed to. + * + * A flow commonly runs one agent on the message and others on work of their own — a + * critic reading `results.x`, a classifier in a branch. Those never see what was typed, + * so what they run on is not a setting this conversation has: letting one of them differ + * on the model would take the model control away from the agent that does answer. + * + * The message need not be the whole prompt — an author wraps it in context freely — so + * this asks whether the expression reads it at all. A flow where no agent reads it is one + * shaped in some way this cannot speak for, and every agent counts again rather than none. + */ +function chatFacingAgents(modules: FlowModule[] | undefined): FlowModule[] { + const agents = agentSteps(modules) + const facing = agents.filter((module) => { + const transform = (module.value as any).input_transforms?.[MESSAGE_INPUT] + return transform?.type === 'javascript' && readsFlowInput(transform.expr, MESSAGE_INPUT) + }) + return facing.length > 0 ? facing : agents +} + +/** + * Whether an expression reads `flow_input.` anywhere in it. + * + * Parsed rather than matched: the author may write `flow_input['user_message']` as readily + * as the dot form the editor emits, and a mention inside a comment or a string is not a + * read. Reading two inputs is still a read of each, so this cannot use `flowInputRef`, + * which answers a different question — which single input feeds a field. + */ +function readsFlowInput(expr: string, name: string): boolean { + let root: unknown + try { + root = parseExpressionAt(parenthesised(expr), 0, { ecmaVersion: 'latest' }) + } catch { + return false + } + let found = false + const visit = (node: any) => { + if (found || !node || typeof node !== 'object') return + if (Array.isArray(node)) { + node.forEach(visit) + return + } + if (flowInputName(node) === name) { + found = true + return + } + for (const key of Object.keys(node)) { + if (key === 'type' || key === 'start' || key === 'end') continue + visit(node[key]) + } + } + visit(root) + return found +} + +/** + * AI agent inputs the chat composer can drive. + * + * The composer never edits the flow: an agent field is reachable only when the author + * wired a flow input to it, so what the chip writes is a run input like any other. That + * also keeps it working on the deployed chat, where the reader has no write access, and + * on a step linked to an `ai_agent` resource, where every field but `user_message` / + * `user_attachments` comes from the resource and is not overridable at all. + * + * Only what the person chatting legitimately owns turn to turn belongs here, which today + * is the files they attach and nothing else. `system_prompt`, `temperature` and + * `max_completion_tokens` shape how the agent behaves for everyone who runs the flow — + * surfacing them per conversation invites tuning the flow from the chat instead of + * fixing it in the editor. They stay flow settings, reachable through Configure inputs + * when the author deliberately exposes them. `max_iterations` is absent for the same + * reason, and because it caps the tool-use loop rather than a single generation. The + * model has its own control, resolved separately through `resolveAgentModelWiring`. + */ +export const AGENT_CHAT_INPUT_KEYS = ['user_attachments'] as const + +export type AgentChatInputKey = (typeof AGENT_CHAT_INPUT_KEYS)[number] + +/** The key that rides one message rather than the conversation, which the composer + * clears on send. A later key of the other sort would not be this one. */ +export const PER_TURN_AGENT_CHAT_INPUT_KEY: AgentChatInputKey = 'user_attachments' + +export type AgentChatInput = { + /** Flow input property feeding the agent field. */ + name: string + key: AgentChatInputKey + /** The flow input's own schema entry — the chip renders it with the same editor the modal would. */ + property: Record +} + +const FLOW_INPUT_REF = /flow_input\??\.([A-Za-z_$][\w$]*)/g + +/** + * The flow input a transform is fed by, when exactly one feeds it. + * + * The expression need not be a bare pass-through — a step commonly reshapes what it + * reads, e.g. `(flow_input.files || []).map(f => ({ bucket: f.storage, key: f.s3 }))`. + * Writing that input is still right, because the expression consumes it. Two or more + * inputs are ambiguous: the composer would have no way to say which one it is editing. + */ +export function flowInputRef(transform: InputTransform | undefined): string | undefined { + if (transform?.type !== 'javascript') return undefined + const names = new Set([...transform.expr.matchAll(FLOW_INPUT_REF)].map((match) => match[1])) + return names.size === 1 ? [...names][0] : undefined +} + +/** A provider value as the agent stores it. */ +export type AgentModel = { kind?: string; model?: string; reasoning_effort?: string } + +/** The provider fields the composer can read or drive. */ +const PROVIDER_FIELDS = ['kind', 'resource', 'model', 'reasoning_effort'] as const +export type ProviderField = (typeof PROVIDER_FIELDS)[number] + +/** + * How an AI agent step's `provider` is supplied, field by field. + * + * The agent takes one `provider` object, so an author who wants the chat to choose only + * the model writes the rest as literals around it: + * + * { kind: 'anthropic', resource: '$res:u/admin/claude', model: flow_input.model } + * + * `fields` names the flow input behind each field the author exposed, `fixed` holds the + * literals, and `whole` covers the plain `flow_input.x` case where one input carries the + * entire object. Reading them apart is what lets the composer offer exactly the knobs the + * flow exposed — and stops a partial expression from being mistaken for a whole-object + * one, which would write a provider object into an input the flow reads as a model name. + */ +export type AgentModelWiring = { + whole?: string + fields: Partial> + fixed: Partial> + /** + * One of the agents names no resource or no model of its own and no flow input feeds + * it, so that agent's run fails whatever the others do. Held apart from the fields, + * which describe what the composer may offer. + */ + someAgentCannotRun?: boolean +} + +/** The flow input behind `flow_input.x`, `flow_input?.x` or `flow_input['x']`. */ +function flowInputName(node: any): string | undefined { + const member = node?.type === 'ChainExpression' ? node.expression : node + if (member?.type !== 'MemberExpression') return undefined + if (member.object?.type !== 'Identifier' || member.object.name !== 'flow_input') return undefined + if (!member.computed && member.property?.type === 'Identifier') return member.property.name + if (member.computed && member.property?.type === 'Literal') { + return typeof member.property.value === 'string' ? member.property.value : undefined + } + return undefined +} + +/** A property's name, for the plain `key:` and `'key':` forms only. */ +function propertyKey(property: any): string | undefined { + if (property?.type !== 'Property' || property.computed) return undefined + if (property.key?.type === 'Identifier') return property.key.name + if (property.key?.type === 'Literal' && typeof property.key.value === 'string') { + return property.key.value + } + return undefined +} + +/** + * Read a `provider` input transform. Anything this cannot account for in full returns + * undefined rather than a guess: the composer then leaves the field alone instead of + * writing into an expression it does not understand. + */ +export function parseProviderTransform( + transform: InputTransform | undefined +): AgentModelWiring | undefined { + if (transform?.type === 'static') { + const value = transform.value + if (!value || typeof value !== 'object') return undefined + const fixed: AgentModelWiring['fixed'] = {} + for (const field of PROVIDER_FIELDS) { + if (value[field] !== undefined) fixed[field] = value[field] + } + return { fields: {}, fixed } + } + if (transform?.type !== 'javascript') return undefined + + // The author's own text may already be wrapped, so any balanced surround is fine — what + // the span check rejects is an expression with something else beside it. + const source = parenthesised(transform.expr) + let node: any + try { + node = parseExpressionAt(source, 0, { ecmaVersion: 'latest' }) + } catch { + return undefined + } + // A comment beside the expression is not another expression: the author annotating their + // own provider must not cost them the model control. Dropped before the check so what + // remains is only what would change the value — and so a paren inside a comment is not + // counted as one of the wrapping pair. + const before = withoutComments(source.slice(0, node.start)) + const after = withoutComments(source.slice(node.end)) + if (!/^[\s(]*$/.test(before) || !/^[\s)]*$/.test(after)) return undefined + if ((before.match(/\(/g)?.length ?? 0) !== (after.match(/\)/g)?.length ?? 0)) return undefined + + const whole = flowInputName(node) + if (whole) return { whole, fields: {}, fixed: {} } + if (node.type !== 'ObjectExpression') return undefined + + const fields: AgentModelWiring['fields'] = {} + const fixed: AgentModelWiring['fixed'] = {} + for (const property of node.properties) { + const key = propertyKey(property) + // A spread or a computed key could supply any field, so nothing here is knowable. + if (!key) return undefined + if (!(PROVIDER_FIELDS as readonly string[]).includes(key)) continue + const name = flowInputName(property.value) + if (name) { + fields[key as ProviderField] = name + } else if (property.value?.type === 'Literal') { + fixed[key as ProviderField] = property.value.value + } else { + return undefined + } + } + return { fields, fixed } +} + +/** How one agent supplies a provider field: from an input, as a literal, or not at all. */ +type FieldSupply = + | { kind: 'wired'; name: string } + | { kind: 'fixed'; value: any } + | { kind: 'absent' } + +/** Whether one agent supplies a field with nothing usable: no input, and no literal. */ +function agentFieldEmpty(wiring: AgentModelWiring, field: ProviderField): boolean { + if (wiring.fields[field] !== undefined) return false + const value = wiring.fixed[field] + return value === undefined || value === '' +} + +function fieldSupply(wiring: AgentModelWiring, field: ProviderField): FieldSupply { + const name = wiring.fields[field] + if (name !== undefined) return { kind: 'wired', name } + const value = wiring.fixed[field] + if (value !== undefined) return { kind: 'fixed', value } + return { kind: 'absent' } +} + +/** + * The provider wiring the chat can act on, across every AI agent in the flow. + * + * With several agents a field is drivable when they agree on it: one flow input feeding + * it, or one literal fixing it. Where they disagree there is no single value to show or + * write, so that field is dropped and the others still work. A flow mixing whole-object + * and field-by-field wiring is ambiguous throughout and yields nothing. + */ +export function resolveAgentModelWiring( + modules: FlowModule[] | undefined +): AgentModelWiring | undefined { + const agents = chatFacingAgents(modules) + const parsed = agents.map((agent) => + parseProviderTransform((agent.value as any).input_transforms?.['provider']) + ) + if (parsed.length === 0) return undefined + // An agent whose provider cannot be read is an agent the composer cannot speak for: + // dropping it would let the rest declare a control that governs only some of them. + if (parsed.some((wiring) => wiring === undefined)) return undefined + const wirings = parsed as AgentModelWiring[] + // Whether any single agent has nothing to call, which stays true however the others + // are wired — the gap message is about that agent, not about their agreement. + const someAgentCannotRun = wirings.some( + (wiring) => + !wiring.whole && (agentFieldEmpty(wiring, 'resource') || agentFieldEmpty(wiring, 'model')) + ) + if (wirings.length === 1) return { ...wirings[0], someAgentCannotRun } + + const wholes = new Set(wirings.map((w) => w.whole)) + if (wholes.size === 1 && !wholes.has(undefined)) { + return { whole: [...wholes][0], fields: {}, fixed: {}, someAgentCannotRun } + } + if (wirings.some((w) => w.whole !== undefined)) return undefined + + const fields: AgentModelWiring['fields'] = {} + const fixed: AgentModelWiring['fixed'] = {} + for (const field of PROVIDER_FIELDS) { + // Every agent has to supply the field the same way for the composer to speak for + // them all. One wired name among agents that otherwise fix it is not agreement: + // the control would move that one agent and leave the others where they are. + const supplies = new Set(wirings.map((w) => JSON.stringify(fieldSupply(w, field)))) + // Disagreement leaves the field neither editable nor known: a control offered here + // would govern one agent while the rest ran on something else. + if (supplies.size > 1) continue + const supply: FieldSupply = JSON.parse([...supplies][0]) + if (supply.kind === 'wired') fields[field] = supply.name + else if (supply.kind === 'fixed') fixed[field] = supply.value + } + return { fields, fixed, someAgentCannotRun } +} + +/** + * Why the chat cannot run, when the agent's own provider is incomplete. + * + * A freshly added agent carries `{ kind: 'openai', model: '', resource: '' }`, so it names + * a provider kind while having nothing to call — the run fails and the chat can do nothing + * about it, because no flow input feeds either field. Saying so beats a dead model button. + * A field the flow exposes is never a gap: the reader picks it in the composer. + */ +export function agentModelGap(wiring: AgentModelWiring | undefined): string | undefined { + // No agent, several of them, or an expression we cannot read: not ours to judge. + if (!wiring || wiring.whole) return undefined + // Asked of each agent rather than of what they agree on: agents that merely disagree + // about the model all have one, and the message would be false — while an agent with + // an empty model still cannot run, however well the others are configured. + return wiring.someAgentCannotRun + ? 'Pick a provider and model on the AI agent step to use this chat.' + : undefined +} + +/** + * The flow inputs the model button actually writes, so the modal does not ask for them a + * second time — and, just as much, so it still asks for the ones the button cannot reach. + * + * `kind` is the one to watch: the button writes it only alongside a resource, since a + * provider is picked as a pair. A flow that wires `kind` to an input while fixing the + * resource leaves the button nothing to write it with, and hiding it would leave the run + * without a provider kind and no way to supply one. + * + * `reasoning_effort` needs no such condition: the button always draws a thinking control, + * whatever it can say about the model — a ladder, a typed token, or why there is neither — so + * a wired effort is always the button's. Asking for it in the modal as well would be a second + * editor for a field that already has one. + */ +export function agentModelWiringInputs(wiring: AgentModelWiring | undefined): string[] { + if (!wiring) return [] + if (wiring.whole) return [wiring.whole] + const driven: ProviderField[] = ['resource', 'model', 'reasoning_effort'] + if (wiring.fields.resource !== undefined) driven.push('kind') + return driven.map((field) => wiring.fields[field]).filter((name): name is string => !!name) +} + +/** + * The flow inputs the composer edits, and therefore the ones the Configure-inputs modal must + * not ask for. The modal is whatever is left, so this is the single answer to "who edits + * this" rather than a list kept in step with the controls that render. + * + * Only the shape of the flow decides it. Whether a control can act *right now* — no object + * storage to upload to, no rules for a provider's thinking levels — is a state that control + * shows, not a reason to hand the input to an editor that would be no more able. + */ +export function composerOwnedInputs( + wiring: AgentModelWiring | undefined, + attachmentsTarget: { name: string } | undefined +): string[] { + return [...agentModelWiringInputs(wiring), ...(attachmentsTarget ? [attachmentsTarget.name] : [])] +} + +/** Whether a schema entry holds an s3 file, as the flow input editor recognises one. */ +function holdsS3File(property: Record | undefined): boolean { + return ( + property?.format === 'resource-s3_object' || + property?.resourceType === 's3object' || + property?.resourceType === 's3_object' + ) +} + +/** + * Where the composer's attachments go, or nothing when there is nowhere they fit. + * + * The agent reads `user_attachments` through a transform that may reshape what it takes, so + * the flow input feeding it is not necessarily an s3 field: an expression building the s3 + * object itself promotes a plain string. Writing `{ s3, filename }` into that input fails at + * run time, so the paperclip appears only where the schema says the value belongs. + */ +export function attachmentsTargetFor( + input: AgentChatInput | undefined +): { name: string; multiple: boolean } | undefined { + if (!input) return undefined + if (holdsS3File(input.property)) return { name: input.name, multiple: false } + return input.property?.type === 'array' && holdsS3File(input.property.items) + ? { name: input.name, multiple: true } + : undefined +} + +export function isEmptyAgentChatInputValue(value: any): boolean { + if (value === undefined || value === null || value === '') return true + return Array.isArray(value) && value.length === 0 +} + +/** + * The flow inputs that an AI agent step reads directly into one of its chat-relevant + * fields. Several agents may resolve to the same flow input; it is one chip either way, + * and one that stays unambiguous however many agents read it. + */ +export function resolveAgentChatInputs( + modules: FlowModule[] | undefined, + additionalInputsSchema: Record | undefined +): AgentChatInput[] { + const properties = additionalInputsSchema?.properties + if (!modules || !properties) return [] + + // One input per key, and only when every agent reading that key reads the same one: + // the composer writes a single flow input, so promoting one of two would feed one + // agent and leave the other with nothing — while hiding both from the modal, where + // the reader could at least have filled them in. + const namesPerKey = new Map>() + for (const module of chatFacingAgents(modules)) { + const transforms = (module.value as any).input_transforms ?? {} + for (const key of AGENT_CHAT_INPUT_KEYS) { + const transform = transforms[key] + // An agent that feeds the key from anything but a flow input — a literal, another + // step's result, or the empty placeholder every agent step carries for the keys of + // AI_AGENT_SCHEMA — is not reading an input, so it has no say in which one the + // composer drives. + if (transform?.type !== 'javascript' || !transform.expr.includes('flow_input')) continue + const name = flowInputRef(transform) + // A name the schema doesn't declare has no field to promote, and one expression + // reading two inputs names none: either way this agent reads something the + // composer cannot drive, which is what disagreement means here. + const usable = name && name in properties ? name : undefined + const names = namesPerKey.get(key) ?? new Set() + names.add(usable) + namesPerKey.set(key, names) + } + } + + const keyOf = new Map() + for (const [key, names] of namesPerKey) { + if (names.size !== 1) continue + const name = [...names][0] + if (name === undefined || keyOf.has(name)) continue + keyOf.set(name, key) + } + + return [...keyOf.entries()].map(([name, key]) => ({ + name, + key, + property: properties[name] + })) +} + +/** + * The run's inputs with a reasoning effort the chosen model cannot take removed. + * + * `effortPatch` reconciles the two when the reader switches model, which covers the only + * way the composer can put them out of step. It is not the only way they get out of step: + * a value stored from an earlier visit, a default the flow author wrote, or a model chosen + * before the effort was, all arrive already mismatched — and the provider answers a + * mismatch with a 400 that names neither input ("adaptive thinking is not supported on this + * model"). Checked here, where the run's arguments are settled, so every route is covered. + * + * Only where the registry positively knows the model rejects it. An unknown family keeps + * whatever the author wrote: dropping a value on a guess would override their own default. + */ +export function withoutRejectedEffort( + wiring: AgentModelWiring | undefined, + values: Record +): Record { + if (!wiring) return values + + // One input carrying the whole provider object: the three fields are read from it and + // the effort is cleared inside it, since that is where the agent will look for them. + if (wiring.whole) { + const provider = values[wiring.whole] + if (!provider || typeof provider !== 'object') return values + if (!rejectsEffort(provider.kind, provider.model, provider.reasoning_effort)) return values + return { ...values, [wiring.whole]: { ...provider, reasoning_effort: '' } } + } + + const effortInput = wiring.fields.reasoning_effort + if (!effortInput) return values + const kindInput = wiring.fields.kind + const modelInput = wiring.fields.model + const rejected = rejectsEffort( + kindInput ? values[kindInput] : wiring.fixed.kind, + modelInput ? values[modelInput] : wiring.fixed.model, + values[effortInput] + ) + return rejected ? { ...values, [effortInput]: '' } : values +} + +/** Whether the registry positively says this model will not take this effort. */ +function rejectsEffort(provider: unknown, model: unknown, effort: unknown): boolean { + if (typeof effort !== 'string' || effort === '') return false + if (typeof provider !== 'string' || typeof model !== 'string' || !provider || !model) { + return false + } + const capability = getReasoningCapability(provider as AIProvider, model) + return capability.known && !capability.supported +} diff --git a/frontend/src/lib/components/flows/conversations/attachmentNames.test.ts b/frontend/src/lib/components/flows/conversations/attachmentNames.test.ts new file mode 100644 index 0000000000..3204e57237 --- /dev/null +++ b/frontend/src/lib/components/flows/conversations/attachmentNames.test.ts @@ -0,0 +1,31 @@ +import { describe, expect, it } from 'vitest' +import { storedAttachmentName } from './attachmentNames' + +/** + * The worker reads an attachment's media type from the object key and nothing else, so a + * key whose extension disagrees with the bytes reaches the provider mislabelled. + */ +describe('storedAttachmentName', () => { + // The composer re-encodes images, so the picked extension is the one that lies. + it('renames a re-encoded image to the type it was encoded as', () => { + expect(storedAttachmentName('photo.webp', 'image/png')).toBe('photo.png') + expect(storedAttachmentName('holiday.png', 'image/jpeg')).toBe('holiday.jpg') + }) + + it('gives an extension to a name that has none', () => { + expect(storedAttachmentName('attachment-1', 'image/png')).toBe('attachment-1.png') + expect(storedAttachmentName('contract', 'application/pdf')).toBe('contract.pdf') + }) + + it('replaces only the last extension', () => { + expect(storedAttachmentName('report.2026.final.webp', 'image/png')).toBe( + 'report.2026.final.png' + ) + }) + + // Blobs upload byte for byte, so a type we do not re-encode keeps the name as picked. + it('leaves a type it does not re-encode alone', () => { + expect(storedAttachmentName('notes.csv', 'text/csv')).toBe('notes.csv') + expect(storedAttachmentName('archive.zip', 'application/zip')).toBe('archive.zip') + }) +}) diff --git a/frontend/src/lib/components/flows/conversations/attachmentNames.ts b/frontend/src/lib/components/flows/conversations/attachmentNames.ts new file mode 100644 index 0000000000..05ecc68fda --- /dev/null +++ b/frontend/src/lib/components/flows/conversations/attachmentNames.ts @@ -0,0 +1,28 @@ +/** + * What a chat attachment is stored under in object storage. + * + * The worker reads an attachment's media type from the object key and nothing else — + * `mime_guess::from_path` in `windmill-ai/src/image_handler.rs`, falling back to + * `image/png` when it can read no extension — and never from the content type stored + * beside it. So the key's extension is a claim about the bytes, and it has to be true. + */ + +/** The extension each type the composer can send must be stored under. */ +const EXTENSION_BY_MEDIA_TYPE: Record = { + 'image/png': 'png', + 'image/jpeg': 'jpg', + 'application/pdf': 'pdf' +} + +/** + * The composer re-encodes every image to PNG or JPEG, so keeping the picked `photo.webp` + * would hand the provider PNG bytes labelled webp, which Anthropic rejects outright; and a + * PDF picked without an extension would be read back as the `image/png` fallback. A type + * not listed is left as picked — blobs upload byte for byte, so their name is already true. + */ +export function storedAttachmentName(filename: string, mediaType: string): string { + const extension = EXTENSION_BY_MEDIA_TYPE[mediaType] + if (!extension) return filename + const stem = filename.replace(/\.[^./]+$/, '') + return `${stem || filename}.${extension}` +} diff --git a/frontend/src/lib/components/flows/conversations/flowChatViewHost.svelte.ts b/frontend/src/lib/components/flows/conversations/flowChatViewHost.svelte.ts new file mode 100644 index 0000000000..fd9abc79d6 --- /dev/null +++ b/frontend/src/lib/components/flows/conversations/flowChatViewHost.svelte.ts @@ -0,0 +1,702 @@ +import type { + ChatSendRequestOptions, + ChatViewHost +} from '$lib/components/copilot/chat/chatViewHost' +import type { DisplayMessage } from '$lib/components/copilot/chat/shared' +import type { ChatMessage, FlowChatManager } from './FlowChatManager.svelte' +import { AIAutonomyMode } from '$lib/components/copilot/chat/AIChatManager.svelte' +import { isPlanCardTool } from '$lib/components/copilot/chat/planMode' +import { ToolCallStore, type ToolCallDetails } from './toolCallContext.svelte' +import { AttachedFilesStore } from '$lib/components/copilot/chat/files/attachedFiles.svelte' +import { SessionArtifactsStore } from '$lib/components/copilot/chat/artifacts/artifactsState.svelte' +import { dataUrlToBlob, type AttachedBlob } from '$lib/components/copilot/chat/blobUtils' +import { storedAttachmentName } from './attachmentNames' +import type { AttachedImage } from '$lib/components/copilot/chat/imageUtils' +import type { AttachedTextFile } from '$lib/components/copilot/chat/textFileUtils' +import { HelpersService, JobService } from '$lib/gen' +import { sendUserToast } from '$lib/toast' +import { randomUUID } from '$lib/utils/uuid' +import { turnFailed } from './turnTranscript' +import { + argsToMessageInputs, + attachmentsToMessageInputs, + MessageInputsStore, + type MessageInputs +} from './messageInputContext.svelte' + +/** A row that needs no job behind it. */ +const EMPTY_TOOL_CALL: ToolCallDetails = {} + +/** What an AI agent step reads out of `user_attachments`. */ +type S3Attachment = { s3: string; filename?: string } + +/** The flow input the composer's attachments feed, and whether it holds a list. */ +export type AttachmentsTarget = { name: string; multiple: boolean } + +export type FlowChatViewHostOptions = { + additionalInputs?: () => Record | undefined + attachmentsTarget?: () => AttachmentsTarget | undefined + workspace?: () => string | undefined + /** Why attaching is off despite the flow taking attachments — no object storage, say. + * Undefined while the workspace has not answered: an explanation must not be a guess. */ + attachmentsUnavailable?: () => string | undefined + /** Flow inputs the composer renders a control of its own for, so a message does not + * repeat them as context chips. */ + inputsShownInComposer?: () => string[] + /** The flow's input schema, which says which of a run's arguments are secret. */ + inputsSchema?: () => { properties?: Record } | undefined +} + +/** + * A tool's arguments and result reach us as strings: the provider's JSON for the call, and + * whatever the tool returned, which is often but not always JSON. Parsed where it parses so + * the card can fold it, kept verbatim where it does not. + */ +function parseToolPayload(raw: string | null | undefined): any { + if (raw === undefined || raw === null || raw === '') return undefined + try { + return JSON.parse(raw) + } catch { + return raw + } +} + +function toDisplayMessage( + message: ChatMessage, + userIndex: number, + showStepNames: boolean, + inputs: MessageInputsStore, + toolCalls: ToolCallStore, + failed: boolean, + pendingInputs: MessageInputs | undefined +): DisplayMessage { + switch (message.message_type) { + case 'user': { + // What the turn ran with — the message row itself keeps only the text. Renders + // through the same lanes the copilot uses. + // What this tab sent wins wherever it has it, and the job answers for the rest: + // a row read back from the server on a later visit, a conversation reopened. + // The row is named with its job as soon as the run starts, so a send that kept + // nothing would switch lanes mid-run and fetch arguments it had just handed over. + const { images, contextElements } = + pendingInputs ?? + (message.job_id ? inputs.get(message.job_id) : { images: [], contextElements: [] }) + return { + role: 'user', + index: userIndex, + content: message.content, + // Drives the shared Retry button: the turn this message started failed. + error: failed || undefined, + images: images.length > 0 ? images : undefined, + contextElements: contextElements.length > 0 ? contextElements : undefined + } + } + case 'tool': { + const failed = message.success === false + // The same three details reach a row from one of two places, never both: the row + // itself for a tool with no job of its own, and for one still streaming; the + // tool's own job otherwise. + const fromRow: ToolCallDetails = { + toolName: message.tool_name, + parameters: parseToolPayload(message.tool_arguments), + result: parseToolPayload(message.tool_result) + } + // A row that carries its own call must not ask a job for it: an MCP tool runs + // inside the agent's job and names it, so the answer would be the agent's own + // arguments and result rather than the tool's. (It also saves a fetch per row + // when a conversation opens.) + const carriesItsOwnCall = fromRow.parameters !== undefined || fromRow.result !== undefined + const fromJob = carriesItsOwnCall ? EMPTY_TOOL_CALL : toolCalls.get(message.job_id) + const toolName = fromRow.toolName ?? fromJob.toolName + const parameters = fromRow.parameters ?? fromJob.parameters + const result = failed ? undefined : (fromRow.result ?? fromJob.result) + return { + role: 'tool', + tool_call_id: message.id, + content: message.content, + // Withheld for the copilot's two plan-mode names: `toolName` is what makes + // ToolExecutionDisplay render a plan card, and an agent tool that happened to + // share one would silently become one. + toolName: isPlanCardTool(toolName) ? undefined : toolName, + parameters, + result, + // The card's fold is opt-in (ToolExecutionDisplay reads showDetails), so it is + // offered only when there is a call or a result behind it to reveal. + showDetails: parameters !== undefined || result !== undefined, + error: failed ? message.content : undefined, + isLoading: message.loading + } + } + default: + return { + role: 'assistant', + content: message.content, + streaming: message.streaming, + reasoning: message.reasoning ?? undefined, + stepName: showStepNames ? (message.step_name ?? undefined) : undefined, + // The run behind the answer, so a reader can open what produced it. Absent on + // the temp message a stream builds, which has no job id until it settles. + jobId: message.job_id || undefined, + createdAt: message.created_at + } + } +} + +/** + * Renders a flow run's conversation through the AI session chat components. The turn is a + * flow job rather than an LLM call this host makes, so what it can offer is whatever it can + * write back into the run's arguments — each field below says for itself (see ChatViewHost). + */ +export class FlowChatViewHost implements ChatViewHost { + #manager: FlowChatManager + #options: FlowChatViewHostOptions + + constructor(manager: FlowChatManager, options: FlowChatViewHostOptions = {}) { + this.#manager = manager + this.#options = options + // The queue belongs to the chat it was typed into, and that chat's run can finish + // while the reader is in another one — so the manager says which turn settled rather + // than the composer watching the open chat. + manager.onTurnSettled = (conversationId) => this.flushQueuedMessage(conversationId) + // The sidebar marks a chat with something waiting to go out; the queue lives here. + manager.hasQueuedMessage = (conversationId) => !!this.#queues[conversationId]?.text.trim() + } + + // The step name says which AI agent step wrote a message, so it only tells the + // reader anything once a conversation holds more than one. Counted over the + // transcript rather than over the flow's current steps: a conversation outlives + // edits to the flow, so it can carry labels from a shape the flow no longer has. + #showStepNames = $derived.by( + () => new Set(this.#manager.messages.map((m) => m.step_name).filter(Boolean)).size > 1 + ) + + // The inputs a turn was sent with, by the id of the row the composer added for it. + // The row is named with its job once the run starts, so the job could answer this too — + // but that is a fetch for arguments this tab just sent, and one that fails would blank + // the row for the rest of the session. Every send records, not only one carrying files: + // a row with no entry switches to the job lane the moment it is named, which renders it + // empty for the length of a round trip. + #sentInputs = $state>({}) + + #messageInputs = new MessageInputsStore( + () => this.#options.workspace?.(), + () => this.#options.inputsSchema?.(), + () => new Set(this.#options.inputsShownInComposer?.() ?? []) + ) + #toolCalls = new ToolCallStore(() => this.#options.workspace?.()) + + displayMessages = $derived.by(() => { + let userIndex = 0 + const showStepNames = this.#showStepNames + const messages = this.#manager.messages + return messages.map((message, i) => + toDisplayMessage( + message, + message.message_type === 'user' ? userIndex++ : -1, + showStepNames, + this.#messageInputs, + this.#toolCalls, + message.message_type === 'user' && turnFailed(messages, i), + this.#sentInputs[message.id] + ) + ) + }) + + /** + * Run the turn at this transcript position again, as it ran the first time: its own + * arguments, read back from its job, rather than whatever the composer holds now. The + * row shows the attachments and inputs it ran with, so a retry that quietly used today's + * settings would run something other than what the reader is looking at. + * + * The composer's own controls are the exception. What they edit — the provider, the + * model, the thinking level — is on screen beside the transcript rather than on the row, + * and a model that cannot answer is one of the likelier reasons a turn failed. Changing + * it and pressing Retry has to run the new one, or the retry fails the same way with no + * sign of why. + * + * A job that has been purged can no longer say what it ran with; the composer is then + * the only account left, and the row shows nothing either, so the two still agree. + */ + retryRequest = async (messageIndex: number) => { + const message = this.#manager.messages[messageIndex] + if (!message || message.message_type !== 'user' || this.loading || this.#readingReplayArgs) + return + // The turn belongs to the chat it was clicked in. Read before the fetch below, since + // the reader can select another conversation while it runs — the same reason the + // upload path pins it. + const conversationId = this.#manager.selectedConversationId + const workspace = this.#options.workspace?.() + let replayArgs: Record | undefined + if (message.job_id && workspace) { + this.#readingReplayArgs = true + try { + const original = (await JobService.getJobArgs({ + workspace, + id: message.job_id + })) as Record + // `user_message` is the message itself, passed as the instructions below. + const { user_message: _sent, ...rest } = original ?? {} + const composerOwned = this.#options.inputsShownInComposer?.() ?? [] + const current = this.#options.additionalInputs?.() ?? {} + for (const name of composerOwned) { + if (name in current) rest[name] = current[name] + else delete rest[name] + } + replayArgs = rest + } catch (error) { + // Only a job that is gone justifies running something else. Anything else — + // a network blip, a 500 — would substitute a different turn silently, which + // is the whole thing this guards against. + if ((error as { status?: number })?.status !== 404) { + sendUserToast('Could not read what that turn ran with. Try again.', true) + return + } + } finally { + this.#readingReplayArgs = false + } + } + // A send typed while the arguments were being read has already started a turn here, + // and `sendRequest` would refuse this one by handing its text to the composer as if + // the reader had typed it. + if (conversationId && this.#manager.isConversationBusy(conversationId)) { + sendUserToast('That chat started another turn. Retry once it finishes.', true) + return + } + void this.sendRequest({ instructions: message.content, conversationId }, replayArgs) + } + messages: readonly unknown[] = [] + contextTokens = 0 + operatingWorkspace = $derived.by(() => this.#options.workspace?.()) + /** A retry reading back the turn it is about to replay. Read only by `retryRequest`, so a + * plain field: nothing renders from it. Deliberately not part of `loading`, which renders + * Stop — there is no run yet to stop, and Stop would cancel the failed turn's own job and + * take the chat's queue back. A normal send in this window still gets ahead of the retry; + * the toast below is what says so. */ + #readingReplayArgs = false + loading = $derived.by( + () => + this.#manager.isLoading || + this.#manager.isWaitingForResponse || + this.#manager.isDispatchingTurn + ) + // A flow run is followed from its job, so another tab holds nothing this one can't read. + runHeldElsewhere = false + loadingLabel = undefined + compacting = false + currentReply = '' + // The turn's thinking while it streams; it moves onto the answer once that starts. + currentReasoning = $derived.by(() => this.#manager.currentReasoning) + currentReasoningActive = $derived.by(() => this.#manager.isReasoningActive) + reasoningHiddenIndicatorLabel = undefined + + #automaticScroll = $state(true) + get automaticScroll() { + return this.#automaticScroll + } + enableAutomaticScroll = () => { + this.#automaticScroll = true + } + disableAutomaticScroll = () => { + this.#automaticScroll = false + } + + instructions = '' + // The flow run is the send: it is in flight for as long as the job is. + get sendInFlight() { + return this.#manager.isLoading + } + /** + * Chats whose in-flight send the reader stopped before it had a job to cancel. Read only + * by the send that set it aside, so a plain Set: nothing renders from this. + */ + #abortedSends = new Set() + + /** + * Give a spent draft back after a send that did not run. The composer took it before + * calling, so something has to. + * + * The composer belongs to whichever chat is on screen, so a turn that was not the open + * one's goes back to its own queue instead: a flush fires when *its* chat's run settles, + * which with turns running side by side can be while the reader is somewhere else, and + * dropping the text into the composer there would put it in the wrong conversation. + */ + #restoreToComposer(options: ChatSendRequestOptions) { + const conversationId = options.conversationId ?? this.#manager.selectedConversationId + // Back to the queue whenever that chat has one, open or not. The composer is the + // right home for a spent draft only while nothing is waiting behind it: anything + // queued was typed later, and that chat's next settled run sends its queue whole — + // so putting the older draft in the composer would run the two out of order. + const waiting = conversationId ? this.#queueOf(conversationId) : undefined + const queueHasMore = + !!waiting && (!!waiting.text.trim() || waiting.images.length > 0 || waiting.blobs.length > 0) + if ( + conversationId && + (queueHasMore || conversationId !== this.#manager.selectedConversationId) + ) { + this.#enqueue( + conversationId, + options.instructions ?? '', + options.images ?? [], + options.blobs ?? [], + 'front' + ) + return + } + this.#aiChatInput?.prependText( + options.instructions ?? '', + options.images ?? [], + [], + options.blobs ?? [] + ) + } + + /** + * `replayArgs` are a failed turn's own run arguments, read back from its job. They stand + * in for the composer's current inputs — attachments included — so a retry runs the turn + * that failed rather than a new one wearing its text. + */ + sendRequest = async (options: ChatSendRequestOptions = {}, replayArgs?: Record) => { + const text = options.instructions?.trim() ?? '' + const args = { ...(replayArgs ?? this.#options.additionalInputs?.() ?? {}) } + const target = this.#options.attachmentsTarget?.() + // Where the paperclip is the input's editor, the stored settings have no say over it: + // a value saved while the modal owned it — before this workspace had object storage — + // would otherwise ride along on every later message. Attachments are set below or not + // at all. Where the modal still owns it, what the reader typed there stands. A replay + // is the exception: its attachments are the ones the failed turn ran with. + if (target && this.supportsMessageAttachments && !replayArgs) { + delete args[target.name] + } + let images = options.images ?? [] + let blobs = options.blobs ?? [] + // The composer refuses an attachment-only send (requiresMessageText), so this is + // the same rule at the other end: nothing runs without a message. + if (!text) return false + // And nothing runs into a conversation belonging to the other surface: the composer + // is shut for it, but a queued turn could have been written before it was opened. + // Reads the open conversation, which is the turn's own except for a flush into a + // background chat — reachable only if a surface ever both runs turns in parallel and + // lists both kinds of chat. No surface does today; one that did would need this to + // take `conversationId`. + const wrongKind = this.#manager.wrongKindReason + if (wrongKind) { + sendUserToast(wrongKind, true) + this.#restoreToComposer(options) + return false + } + // The per-turn cap again, at the place the truncation would happen: the composer + // enforces it as files are attached, but a queue built over several turns arrives + // here as one send, and a scalar input keeps `uploaded[0]` — uploading the rest + // would strand them in storage while the transcript claimed they went. + const cap = this.maxMessageAttachments + if (cap !== undefined && images.length + blobs.length > cap) { + const dropped = images.length + blobs.length - cap + images = images.slice(0, cap) + blobs = blobs.slice(0, Math.max(0, cap - images.length)) + sendUserToast( + cap === 1 + ? `This chat sends one attachment per message; ${dropped} file(s) were not sent.` + : `This chat sends up to ${cap} attachments per message; ${dropped} file(s) were not sent.`, + true + ) + } + const attachments = [...images, ...blobs] + // Settled before the upload below, and created here when there is none yet rather than + // left to `sendMessage` afterwards: everything that has to name this turn while it + // uploads — its busy state, a Stop, a message typed behind it — needs an id to name it + // by, and a send with none had no way to be marked, stopped or queued against. + // Read before the upload for the same reason: the reader can pick another chat while + // it runs, and the turn belongs to the one they sent it from. + const conversationId = + options.conversationId ?? + this.#manager.selectedConversationId ?? + (await this.#manager.createConversation({ clearMessages: false })) + // Whether the composer's own files are what this turn's attachment input holds. A + // replay's do not: its attachments are the ones its job already has. + let attachedLocally = false + if (target && attachments.length > 0) { + // Named, not "the open chat": the reader can switch while the upload runs, and + // where turns run in parallel the other chat has its own status to keep. + this.#manager.setDispatching(conversationId, true) + this.#abortedSends.delete(conversationId) + try { + const uploaded = await this.#uploadAttachments(attachments) + args[target.name] = target.multiple ? uploaded : uploaded[0] + } catch (e) { + sendUserToast( + `Could not upload the attachments: ${e instanceof Error ? e.message : String(e)}`, + true + ) + // The composer already took the draft; without this the turn is simply lost. The + // id is the one captured before the upload, not whatever is open now: the reader + // can switch while it runs, and the draft belongs to the chat they sent it from. + this.#restoreToComposer({ ...options, conversationId }) + return false + } finally { + this.#manager.setDispatching(conversationId, false) + } + // Stop pressed while the upload ran has no job to cancel yet, so it is honoured + // here — the run has not started, and starting it now would execute a message the + // reader already took back. + if (this.#abortedSends.delete(conversationId)) { + this.#restoreToComposer({ ...options, conversationId }) + return false + } + attachedLocally = true + } + const sentInputs = this.#describeSentInputs(args, images, blobs, attachedLocally) + + this.#manager.inputMessage = text + const started = await this.#manager.sendMessage( + Object.keys(args).length > 0 || replayArgs || this.#options.additionalInputs?.() + ? args + : undefined, + (rowId) => { + if (!sentInputs) return + // Keep entries for rows some conversation still holds, not just the open one: + // a queued message flushes into the chat it was typed in, which by then need + // not be on screen, and pruning against the open chat would drop its chips. + const live = this.#manager.liveRowIds + const kept = Object.fromEntries( + Object.entries(this.#sentInputs).filter(([id]) => live.has(id)) + ) + this.#sentInputs = { ...kept, [rowId]: sentInputs } + }, + conversationId + ) + if (!started) { + // The upload succeeded and the run did not, so the composer's draft was spent on + // nothing. The uploaded objects stay where they are — a resend uploads its own, + // under its own prefix — but what the reader wrote comes back, to the chat it was + // written in rather than the one open by now. + this.#restoreToComposer({ ...options, conversationId }) + return false + } + return true + } + + /** + * The chips a row shows for the turn just sent, built from the arguments going out + * through the same split `argsToMessageInputs` makes of a job's arguments. + * + * Files the composer attached are the exception: the data URLs are still in hand and + * render with no network, where the args hold S3 references `argsToMessageInputs` would + * turn into `download_s3_file` links for bytes this tab already has. So that one input + * is described from what was attached and every other from the args. A replay attached + * nothing locally and is described entirely from the args, which are its own job's. + */ + #describeSentInputs( + args: Record, + images: AttachedImage[], + blobs: AttachedBlob[], + attachedLocally: boolean + ): MessageInputs | undefined { + const workspace = this.#options.workspace?.() + // Without one there is no way to build a file's link, so the job lane answers instead. + if (!workspace) return undefined + const target = this.#options.attachmentsTarget?.() + const shownElsewhere = new Set(this.#options.inputsShownInComposer?.() ?? []) + if (attachedLocally && target) shownElsewhere.add(target.name) + const fromArgs = argsToMessageInputs( + workspace, + args, + this.#options.inputsSchema?.(), + shownElsewhere + ) + const attached = attachedLocally + ? attachmentsToMessageInputs(images, blobs) + : { images: [], contextElements: [] } + return { + images: [...attached.images, ...fromArgs.images], + contextElements: [...attached.contextElements, ...fromArgs.contextElements] + } + } + + /** + * Put each attachment in the workspace's object storage and hand back what the + * agent reads. The flow runs on a worker, so the bytes have to exist somewhere + * the worker can fetch — unlike the copilot, which sends them from the browser. + */ + async #uploadAttachments( + attachments: { name?: string; dataUrl: string; mediaType?: string }[] + ): Promise { + const workspace = this.#options.workspace?.() + if (!workspace) throw new Error('no workspace') + // One prefix per turn keeps a re-attached filename from overwriting the copy an + // earlier message still refers to. + const prefix = `windmill_chat_uploads/${randomUUID()}` + return Promise.all( + attachments.map(async (attachment, index) => { + const blob = dataUrlToBlob(attachment.dataUrl, attachment.mediaType) + const filename = storedAttachmentName( + attachment.name ?? `attachment-${index + 1}`, + blob.type + ) + const { file_key } = await HelpersService.fileUpload({ + workspace, + fileKey: `${prefix}/${filename}`, + contentType: blob.type, + requestBody: blob + }) + return { s3: file_key, filename } + }) + ) + } + + cancel = () => { + // Stop means stop: what was typed during the run goes back to the composer rather + // than waiting there to go out after some later turn settles. + this.dequeueMessage() + // A send still uploading has no job to cancel; it reads this once the upload lands. + const open = this.#manager.selectedConversationId + if (open) this.#abortedSends.add(open) + void this.#manager.cancelCurrentJob() + } + // Typed off the interface: a Svelte component's own type resolves differently + // across import specifiers, and the two would then not be assignable. + #aiChatInput: Parameters[0] = null + setAiChatInput: ChatViewHost['setAiChatInput'] = (aiChatInput) => { + this.#aiChatInput = aiChatInput + } + + // A message typed while a chat is running waits here with its attachments and goes out + // whole when that chat's run finishes (see flushQueuedMessage). Held per conversation: + // turns run side by side, so a message typed into one must not ride out of another. + #queues = $state< + Record + >({}) + queuedContext = undefined + queuedFiles: AttachedTextFile[] = [] + + #queueOf(conversationId: string | undefined) { + return ( + (conversationId ? this.#queues[conversationId] : undefined) ?? { + text: '', + images: [] as AttachedImage[], + blobs: [] as AttachedBlob[] + } + ) + } + get queuedMessage(): string { + return this.#queueOf(this.#manager.selectedConversationId).text + } + get queuedImages(): AttachedImage[] { + return this.#queueOf(this.#manager.selectedConversationId).images + } + get queuedBlobs(): AttachedBlob[] { + return this.#queueOf(this.#manager.selectedConversationId).blobs + } + + queueMessage = ( + text: string, + images: AttachedImage[] = [], + _context?: unknown, + _files?: unknown, + blobs: AttachedBlob[] = [] + ) => { + this.#enqueue(this.#manager.selectedConversationId, text, images, blobs) + } + + /** + * Add to what a chat already has waiting, rather than replacing it: the reader can type + * again while a flush of the previous queue is still uploading, and that second message + * is in the queue by the time a failed flush hands the first one back. + */ + #enqueue( + conversationId: string | undefined, + text: string, + images: AttachedImage[], + blobs: AttachedBlob[], + /** Where this belongs in what is already waiting. A draft handed back by a send that + * did not run was typed before anything queued behind it, and goes back in front. */ + at: 'end' | 'front' = 'end' + ) { + if (!conversationId) return + const trimmed = text.trim() + if (!trimmed && images.length === 0 && blobs.length === 0) return + const queue = this.#queueOf(conversationId) + const joined = !trimmed + ? queue.text + : !queue.text + ? trimmed + : at === 'front' + ? `${trimmed}\n${queue.text}` + : `${queue.text}\n${trimmed}` + this.#queues[conversationId] = { + text: joined, + images: at === 'front' ? [...images, ...queue.images] : [...queue.images, ...images], + blobs: at === 'front' ? [...blobs, ...queue.blobs] : [...queue.blobs, ...blobs] + } + } + /** Put the queued draft back in the composer, attachments included. */ + dequeueMessage = () => { + const { text, images, blobs } = this.#takeQueue(this.#manager.selectedConversationId) + if (!text && images.length === 0 && blobs.length === 0) return + this.#aiChatInput?.prependText(text, images, [], blobs) + } + #takeQueue(conversationId: string | undefined) { + const taken = this.#queueOf(conversationId) + if (conversationId) delete this.#queues[conversationId] + return taken + } + /** Send whatever was typed during the run. Called once that chat's run settles. */ + flushQueuedMessage = (conversationId = this.#manager.selectedConversationId) => { + // Same rule as sendRequest, read before the queue is drained: a turn with no message + // cannot run, and taking the queue for it would drop the attachments on the floor. + if (!this.#queueOf(conversationId).text.trim()) return + const { text, images, blobs } = this.#takeQueue(conversationId) + void this.sendRequest({ instructions: text, images, blobs, conversationId }) + } + setComposerStaged = () => {} + clearComposerStaged = () => {} + attachmentBytesExcluding = () => 0 + + storedImages = () => undefined + restartGeneration = () => {} + handleUserQuestionAnswer = () => false + handleToolConfirmation = () => {} + // A flow's tools take their arguments from the model, never from a form the reader fills. + hasPendingRunForm = false + isRunFormPending = () => false + + mode = undefined + isSessionChat = false + supportsModelSettings = false + supportsMessageEditing = false + // The turn is a flow run, and an AI agent step refuses one with neither a + // `user_message` nor manual memory (ai_executor.rs) — so files alone cannot be sent. + requiresMessageText = true + // Attachments go to object storage for the worker to read, so a linked folder — + // a live handle on the user's own disk — has no meaning here. + // The input's shape alone: whether this chat takes attachments at all is a fact about the + // flow, not about the workspace. Object storage decides whether it can right now, which is + // `attachmentsUnavailableReason` — a state on the control rather than a reason to move the + // input to the modal, where a file picker would be just as unable to upload. + get supportsMessageAttachments() { + return !!this.#options.attachmentsTarget?.() + } + get attachmentsUnavailableReason() { + return this.#options.attachmentsUnavailable?.() + } + supportsLinkedFolders = false + attachmentsAsBlobs = true + // A scalar flow input holds one file; sending more would upload every one and run with + // the first, leaving the rest orphaned in storage and the transcript claiming otherwise. + get maxMessageAttachments() { + return this.#options.attachmentsTarget?.()?.multiple === false ? 1 : undefined + } + // What a provider actually takes. Anthropic's document block accepts base64 + // `application/pdf` and nothing else, so the wider set `is_document_mime` + // (windmill-ai/src/ai_types.rs) claims — csv, html, plain, docx, xlsx — is rejected with a + // 400 rather than read. Widen this only alongside a worker that inlines text as text. + attachmentAccept = 'image/*,application/pdf,.pdf' + tools = [] + autonomyMode = AIAutonomyMode.DEFAULT + setAutonomyMode = () => {} + autoAcceptEditsActive = false + autoAcceptEditsAvailable = false + autoAcceptToolConfirmationsAvailable = false + planModeAvailable = false + attachedFiles = new AttachedFilesStore() + artifacts = new SessionArtifactsStore() +} diff --git a/frontend/src/lib/components/flows/conversations/flowChatViewHost.test.ts b/frontend/src/lib/components/flows/conversations/flowChatViewHost.test.ts new file mode 100644 index 0000000000..8cae62485a --- /dev/null +++ b/frontend/src/lib/components/flows/conversations/flowChatViewHost.test.ts @@ -0,0 +1,380 @@ +import { describe, it, expect, vi, beforeEach } from 'vitest' +import { HelpersService, JobService } from '$lib/gen' +import { FlowChatViewHost } from './flowChatViewHost.svelte' + +vi.mock('$lib/gen', () => ({ + HelpersService: { fileUpload: vi.fn() }, + FlowConversationsService: { listConversationMessages: vi.fn(), deleteFlowConversation: vi.fn() }, + JobService: { getJobArgs: vi.fn() }, + FlowService: {} +})) +vi.mock('$lib/toast', () => ({ sendUserToast: vi.fn() })) +// The host imports it only for the AIAutonomyMode enum, and the real module pulls the +// editor in behind it. +vi.mock('$lib/components/copilot/chat/AIChatManager.svelte', () => ({ + AIAutonomyMode: { DEFAULT: 'default' }, + AIMode: { GLOBAL: 'global' } +})) + +/** Just enough manager for the send protocol: the host only touches these. */ +function stubManager(selectedConversationId: string | undefined) { + return { + selectedConversationId, + inputMessage: '', + messages: [] as unknown[], + isLoading: false, + isWaitingForResponse: false, + isDispatchingTurn: false, + currentReasoning: '', + isReasoningActive: false, + wrongKindReason: undefined, + liveRowIds: new Set(), + dispatching: [] as { id: string; on: boolean }[], + setDispatching(id: string, on: boolean) { + this.dispatching.push({ id, on }) + }, + createConversation: vi.fn(async function (this: any) { + this.selectedConversationId = 'made-for-the-send' + return 'made-for-the-send' + }), + sendMessage: vi.fn(async () => true), + cancelCurrentJob: vi.fn(async () => {}), + conversationStatus: () => 'idle', + unreadCount: () => 0, + busy: false, + isConversationBusy(this: any) { + return this.busy + } + } +} + +const host = (manager: ReturnType) => + new FlowChatViewHost(manager as any, { + workspace: () => 'ws', + attachmentsTarget: () => ({ name: 'files', multiple: true }), + additionalInputs: () => ({}) + }) + +const anAttachment = { + name: 'shot.png', + dataUrl: 'data:image/png;base64,iVBORw0KGgo=', + mediaType: 'image/png' +} + +/** + * The window between the composer taking a draft and the flow job existing. Everything that + * has to name the turn during it — its busy state, a Stop, a message typed behind it — needs + * a conversation to name it by, and a first message has none until one is made. + */ +describe('a send whose attachments are still uploading', () => { + beforeEach(() => { + vi.mocked(HelpersService.fileUpload).mockReset() + }) + + it('marks the conversation it was sent from, not whichever is open when the upload lands', async () => { + const manager = stubManager('a') + vi.mocked(HelpersService.fileUpload).mockImplementation(async () => { + manager.selectedConversationId = 'b' + return { file_key: 'k' } as any + }) + await host(manager).sendRequest({ instructions: 'hi', blobs: [anAttachment] as any }) + + expect(manager.dispatching).toEqual([ + { id: 'a', on: true }, + { id: 'a', on: false } + ]) + expect(manager.sendMessage.mock.calls[0]?.[2]).toBe('a') + }) + + // Stop has no job to cancel yet, so it has to be honoured when the upload lands — + // otherwise the run starts and the reader watches a message they took back execute. + it('does not run after a Stop pressed while it uploaded', async () => { + const manager = stubManager('a') + const chatHost = host(manager) + vi.mocked(HelpersService.fileUpload).mockImplementation(async () => { + chatHost.cancel() + return { file_key: 'k' } as any + }) + + const started = await chatHost.sendRequest({ + instructions: 'stop me', + blobs: [anAttachment] as any + }) + + expect(started).toBe(false) + expect(manager.sendMessage).not.toHaveBeenCalled() + }) + + // The first message of a chat has no conversation until one is made. Left to + // `sendMessage` afterwards, nothing in this window had an id to work with. + it('creates the conversation before uploading, so a first message can be stopped too', async () => { + const manager = stubManager(undefined) + const chatHost = host(manager) + vi.mocked(HelpersService.fileUpload).mockImplementation(async () => { + chatHost.cancel() + return { file_key: 'k' } as any + }) + + const started = await chatHost.sendRequest({ + instructions: 'first message', + blobs: [anAttachment] as any + }) + + expect(manager.createConversation).toHaveBeenCalled() + expect(manager.dispatching[0]?.id).toBe('made-for-the-send') + expect(started).toBe(false) + expect(manager.sendMessage).not.toHaveBeenCalled() + }) +}) + +/** + * Where a refused draft lands. The composer holds whichever chat is on screen, so it is the + * right home only when the turn was that chat's and nothing is queued behind it. + */ +describe('a send that did not run', () => { + beforeEach(() => { + vi.mocked(HelpersService.fileUpload).mockReset() + vi.mocked(HelpersService.fileUpload).mockResolvedValue({ file_key: 'k' } as any) + }) + + it('hands the draft back to the composer when it was the open chat with nothing waiting', async () => { + const manager = stubManager('a') + manager.sendMessage = vi.fn(async () => false) + const chatHost = host(manager) + const prepended: string[] = [] + chatHost.setAiChatInput({ prependText: (text: string) => prepended.push(text) } as any) + + await chatHost.sendRequest({ instructions: 'came back', blobs: [anAttachment] as any }) + + expect(prepended).toEqual(['came back']) + }) + + // The composer on screen belongs to another conversation by now, and anything queued was + // typed later — so the draft joins its own chat's queue, in front of what followed it. + it('queues it in front of its own chat when that chat is not the open one', async () => { + const manager = stubManager('a') + manager.sendMessage = vi.fn(async () => false) + const chatHost = host(manager) + const prepended: string[] = [] + chatHost.setAiChatInput({ prependText: (text: string) => prepended.push(text) } as any) + chatHost.queueMessage('typed after') + manager.selectedConversationId = 'b' + + await chatHost.sendRequest({ + instructions: 'sent first', + blobs: [anAttachment] as any, + conversationId: 'a' + }) + + expect(prepended).toEqual([]) + manager.selectedConversationId = 'a' + expect(chatHost.queuedMessage).toBe('sent first\ntyped after') + }) + + // The open chat with something already queued — reachable in the editor, where a turn + // running in another chat refuses this one. The composer would put the older draft + // beside the newer text, and the next settled run would send them inverted. + it('queues it in front even when it is the open chat, if something is waiting', async () => { + const manager = stubManager('a') + manager.sendMessage = vi.fn(async () => false) + const chatHost = host(manager) + const prepended: string[] = [] + chatHost.setAiChatInput({ prependText: (text: string) => prepended.push(text) } as any) + chatHost.queueMessage('typed after') + + await chatHost.sendRequest({ instructions: 'sent first', blobs: [anAttachment] as any }) + + expect(prepended).toEqual([]) + expect(chatHost.queuedMessage).toBe('sent first\ntyped after') + }) +}) + +/** + * Retry runs the turn that failed, not a new one wearing its text. The row shows the + * attachments and inputs it ran with — read back from its job — so a retry that quietly + * used the composer's current settings would run something other than what is on screen. + */ +describe('retrying a failed turn', () => { + const failedRow = { + id: 'row-1', + message_type: 'user', + content: 'summarise this', + job_id: 'job-that-failed' + } + + beforeEach(() => { + vi.mocked(JobService.getJobArgs).mockReset() + }) + + it('runs with the arguments that turn ran with, not the ones on screen now', async () => { + const manager = stubManager('a') + manager.messages = [failedRow] as any + const chatHost = new FlowChatViewHost(manager as any, { + workspace: () => 'ws', + attachmentsTarget: () => ({ name: 'files', multiple: true }), + // What the composer holds now, which must not be what the retry runs with. + additionalInputs: () => ({ tone: 'breezy', files: [] }) + }) + vi.mocked(JobService.getJobArgs).mockResolvedValue({ + user_message: 'summarise this', + tone: 'formal', + files: [{ s3: 'windmill_chat_uploads/abc/report.pdf' }] + } as any) + + await chatHost.retryRequest(0) + + expect(JobService.getJobArgs).toHaveBeenCalledWith({ workspace: 'ws', id: 'job-that-failed' }) + // The original attachment and the original input; `user_message` goes as the text. + expect(manager.sendMessage.mock.calls[0]?.[0]).toEqual({ + tone: 'formal', + files: [{ s3: 'windmill_chat_uploads/abc/report.pdf' }] + }) + }) + + // A purged job can no longer say what it ran with, and refusing to retry would strand + // the reader on a failed turn. The composer is the only account left, and the row shows + // nothing either, so the two still agree. + it('falls back to the composer when the original run is gone', async () => { + const manager = stubManager('a') + manager.messages = [failedRow] as any + const chatHost = new FlowChatViewHost(manager as any, { + workspace: () => 'ws', + additionalInputs: () => ({ tone: 'breezy' }) + }) + vi.mocked(JobService.getJobArgs).mockRejectedValue({ status: 404 }) + + await chatHost.retryRequest(0) + + expect(manager.sendMessage.mock.calls[0]?.[0]).toEqual({ tone: 'breezy' }) + }) + + // Anything other than a purged job leaves the turn unknowable rather than gone, and + // running the text with today's settings would be the silent substitution this guards. + it('refuses rather than running something else when the read fails', async () => { + const manager = stubManager('a') + manager.messages = [failedRow] as any + const chatHost = new FlowChatViewHost(manager as any, { + workspace: () => 'ws', + additionalInputs: () => ({ tone: 'breezy' }) + }) + vi.mocked(JobService.getJobArgs).mockRejectedValue({ status: 500 }) + + await chatHost.retryRequest(0) + + expect(manager.sendMessage).not.toHaveBeenCalled() + }) + + // A turn started in that chat while the arguments were being read: the retry must say so + // rather than let `sendRequest` refuse it by dropping its text into the composer. + it('says so when a send got ahead of it', async () => { + const manager = stubManager('a') + manager.messages = [failedRow] as any + const chatHost = new FlowChatViewHost(manager as any, { + workspace: () => 'ws', + additionalInputs: () => ({}) + }) + vi.mocked(JobService.getJobArgs).mockImplementation(async () => { + manager.busy = true + return { user_message: 'summarise this' } as any + }) + + await chatHost.retryRequest(0) + + expect(manager.sendMessage).not.toHaveBeenCalled() + }) + + // The flag that refuses a second Retry is deliberately not part of `loading`, so nothing + // disables the button — this is what stops two fetches racing into two runs. + it('refuses a second Retry while the first is still reading', async () => { + const manager = stubManager('a') + manager.messages = [failedRow] as any + const chatHost = new FlowChatViewHost(manager as any, { + workspace: () => 'ws', + additionalInputs: () => ({}) + }) + let release: (v: any) => void = () => {} + vi.mocked(JobService.getJobArgs).mockReturnValue( + new Promise((resolve) => (release = resolve)) as any + ) + + const first = chatHost.retryRequest(0) + await chatHost.retryRequest(0) + release({ user_message: 'summarise this' }) + await first + + expect(JobService.getJobArgs).toHaveBeenCalledTimes(1) + expect(manager.sendMessage).toHaveBeenCalledTimes(1) + }) + + // The reader can pick another chat while the arguments are being read back, and the + // replay belongs to the one they clicked Retry in — the same window the upload path pins. + it('runs in the chat it was clicked in, not the one open when the read lands', async () => { + const manager = stubManager('a') + manager.messages = [failedRow] as any + const chatHost = new FlowChatViewHost(manager as any, { + workspace: () => 'ws', + additionalInputs: () => ({}) + }) + vi.mocked(JobService.getJobArgs).mockImplementation(async () => { + manager.selectedConversationId = 'b' + return { user_message: 'summarise this', tone: 'formal' } as any + }) + + await chatHost.retryRequest(0) + + expect(manager.sendMessage.mock.calls[0]?.[2]).toBe('a') + }) + + // The provider and model are edited beside the transcript, not on the row — and a model + // that cannot answer is a likely reason the turn failed, so Retry must run the new one. + it('replays the turn but with the model the composer now holds', async () => { + const manager = stubManager('a') + manager.messages = [failedRow] as any + const chatHost = new FlowChatViewHost(manager as any, { + workspace: () => 'ws', + inputsShownInComposer: () => ['model'], + additionalInputs: () => ({ model: 'claude-sonnet-5', tone: 'breezy' }) + }) + vi.mocked(JobService.getJobArgs).mockResolvedValue({ + user_message: 'summarise this', + model: 'a-model-that-failed', + tone: 'formal' + } as any) + + await chatHost.retryRequest(0) + + expect(manager.sendMessage.mock.calls[0]?.[0]).toEqual({ + model: 'claude-sonnet-5', + tone: 'formal' + }) + }) +}) + +// The row is named with its job as soon as the run starts, so a send that kept nothing +// would swap to the job lane mid-run and render empty for the length of a round trip. +describe('the inputs a row shows for the turn just sent', () => { + it('come from what was sent, with no fetch, once the row has its job', async () => { + vi.mocked(JobService.getJobArgs).mockReset() + const manager = stubManager('a') + const row = { id: 'temp-1', message_type: 'user', content: 'bonjour', job_id: undefined } + manager.messages = [row] + manager.liveRowIds = new Set(['temp-1']) + manager.sendMessage = vi.fn(async (_args: any, nameRow: any) => { + nameRow?.('temp-1') + return true + }) as any + const chatHost = new FlowChatViewHost(manager as any, { + workspace: () => 'ws', + additionalInputs: () => ({ tone: 'formal' }) + }) + + await chatHost.sendRequest({ instructions: 'bonjour' }) + // What `#nameTurnJob` does once the flow job exists. + row.job_id = 'job-1' as any + + expect(chatHost.displayMessages[0]?.contextElements).toEqual([ + { type: 'attached_file', title: 'tone', content: 'formal' } + ]) + expect(JobService.getJobArgs).not.toHaveBeenCalled() + }) +}) diff --git a/frontend/src/lib/components/flows/conversations/jobBackedStore.svelte.ts b/frontend/src/lib/components/flows/conversations/jobBackedStore.svelte.ts new file mode 100644 index 0000000000..1ecd103464 --- /dev/null +++ b/frontend/src/lib/components/flows/conversations/jobBackedStore.svelte.ts @@ -0,0 +1,81 @@ +/** + * What a row can only learn from the job behind it, fetched once and kept while mounted. + * + * A conversation row stores the little it must; the rest — a tool call's arguments and + * result, the attachments a message ran with — already exists on that turn's job. Reading + * it back keeps one copy of the data instead of two, at the cost of a fetch per row, and + * of the same three rules wherever it is done: ask once, answer empty until it lands, and + * cache the empty answer when the job is gone so a purged run is not re-fetched forever. + * + * The rules live here; what to fetch and how to read it is the caller's. + */ + +/** + * Fetches allowed out at once. A page of conversation rows asks for all of its jobs in the + * same render, and the answers only fill chips in below text that is already on screen. + */ +const MAX_CONCURRENT = 6 + +export class JobBackedStore { + #workspace: () => string | undefined + #load: (workspace: string, jobId: string) => Promise + #empty: T + #byJob = $state>({}) + #inFlight = new Set() + #waiting: string[] = [] + #running = 0 + + constructor( + workspace: () => string | undefined, + empty: T, + load: (workspace: string, jobId: string) => Promise + ) { + this.#workspace = workspace + this.#empty = empty + this.#load = load + } + + /** What the job holds, fetching on first ask. Empty until it lands. */ + get(jobId: string | null | undefined): T { + if (!jobId) return this.#empty + const cached = this.#byJob[jobId] + if (cached) return cached + this.#enqueue(jobId) + return this.#empty + } + + #enqueue(jobId: string) { + if (this.#inFlight.has(jobId)) return + this.#inFlight.add(jobId) + this.#waiting.push(jobId) + this.#pump() + } + + #pump() { + while (this.#running < MAX_CONCURRENT && this.#waiting.length > 0) { + const jobId = this.#waiting.shift()! + this.#running++ + void this.#fetch(jobId).finally(() => { + this.#running-- + this.#pump() + }) + } + } + + async #fetch(jobId: string) { + const workspace = this.#workspace() + if (!workspace) { + // Neither cached nor in flight, so the row asks again once a workspace is known. + this.#inFlight.delete(jobId) + return + } + try { + this.#byJob = { ...this.#byJob, [jobId]: await this.#load(workspace, jobId) } + } catch { + // A purged job, or one this user cannot read: the row keeps what it stored. + this.#byJob = { ...this.#byJob, [jobId]: this.#empty } + } finally { + this.#inFlight.delete(jobId) + } + } +} diff --git a/frontend/src/lib/components/flows/conversations/messageInputContext.svelte.ts b/frontend/src/lib/components/flows/conversations/messageInputContext.svelte.ts new file mode 100644 index 0000000000..496bcac352 --- /dev/null +++ b/frontend/src/lib/components/flows/conversations/messageInputContext.svelte.ts @@ -0,0 +1,142 @@ +/** + * The inputs a chat message ran with, recovered from its job. + * + * A conversation message stores only its text, so what the user attached and which + * settings the turn used exist nowhere but the run's arguments. The user row carries + * the flow job id, whose args are the raw run arguments — `user_message` plus every + * other flow input. + * + * Fetched lazily and kept in memory only: a purged job leaves a dangling id and the + * turn simply shows no inputs, which is honest — the arguments are gone. + */ +import { JobService } from '$lib/gen' +import { JobBackedStore } from './jobBackedStore.svelte' +import { base } from '$lib/base' +import { redactFileArgs, redactSecretArgs } from '$lib/components/job_args' +import { + createAttachedFileContextElement, + type ContextElement +} from '$lib/components/copilot/chat/context' +import type { AttachedImage } from '$lib/components/copilot/chat/imageUtils' + +type S3Ref = { s3: string; filename?: string; storage?: string } + +const IMAGE_EXTENSIONS = ['.png', '.jpg', '.jpeg', '.gif', '.webp', '.bmp', '.svg', '.avif'] + +function isS3Ref(value: any): value is S3Ref { + return !!value && typeof value === 'object' && typeof value.s3 === 'string' && value.s3 !== '' +} + +function s3Refs(value: any): S3Ref[] { + if (isS3Ref(value)) return [value] + if (Array.isArray(value)) return value.filter(isS3Ref) + return [] +} + +function displayName(ref: S3Ref): string { + return ref.filename ?? ref.s3.split('/').pop() ?? ref.s3 +} + +function looksLikeImage(ref: S3Ref): boolean { + const name = displayName(ref).toLowerCase() + return IMAGE_EXTENSIONS.some((ext) => name.endsWith(ext)) +} + +/** Same-origin, cookie-authed GET — usable directly as an , no blob fetch. */ +function downloadUrl(workspace: string, ref: S3Ref): string { + const params = new URLSearchParams({ file_key: ref.s3 }) + if (ref.storage) params.set('storage', ref.storage) + return `${base}/api/w/${workspace}/job_helpers/download_s3_file?${params.toString()}` +} + +/** A scalar input, summarised for a chip. Objects are left to the file/JSON branches. */ +function scalarSummary(value: any): string | undefined { + if (value === undefined || value === null || value === '') return undefined + if (typeof value === 'string') return value + if (typeof value === 'number' || typeof value === 'boolean') return String(value) + return undefined +} + +export type MessageInputs = { images: AttachedImage[]; contextElements: ContextElement[] } + +const EMPTY: MessageInputs = { images: [], contextElements: [] } + +/** + * Split a turn's run arguments into the lanes a user message renders: image + * thumbnails, and a chip per remaining input. `user_message` is the bubble itself. + */ +export function argsToMessageInputs( + workspace: string, + args: Record | undefined, + schema: { properties?: Record } | undefined, + shownElsewhere: ReadonlySet = new Set() +): MessageInputs { + if (!args) return EMPTY + const images: AttachedImage[] = [] + const contextElements: ContextElement[] = [] + // A chip is text on screen, so it goes through the same redaction the run page and the + // copilot apply to a job's arguments: a password input must not be readable here, and a + // base64 file is unreadable anyway. + const shown = redactFileArgs(redactSecretArgs(args, schema), schema) + for (const [name, value] of Object.entries(shown)) { + if (name === 'user_message') continue + // An input the composer has its own control for — the model button's provider fields — + // is already on screen, and repeating it under every message is noise. + if (shownElsewhere.has(name)) continue + const files = s3Refs(value) + if (files.length > 0) { + for (const ref of files) { + if (looksLikeImage(ref)) { + images.push({ + dataUrl: downloadUrl(workspace, ref), + mediaType: 'image/png', + name: displayName(ref) + }) + } else { + contextElements.push( + createAttachedFileContextElement(displayName(ref), `Attached file · ${ref.s3}`) + ) + } + } + continue + } + const summary = scalarSummary(value) + if (summary !== undefined) { + contextElements.push(createAttachedFileContextElement(name, summary)) + } + } + return images.length > 0 || contextElements.length > 0 ? { images, contextElements } : EMPTY +} + +/** + * The same lanes, built from what the composer just sent. The turn in flight has no + * job yet, so its row cannot read its inputs back from one; the data URLs are still + * in hand, so the thumbnails need no fetch. + */ +export function attachmentsToMessageInputs( + images: AttachedImage[], + blobs: { name: string }[] +): MessageInputs { + const contextElements = blobs.map((blob) => + createAttachedFileContextElement(blob.name, `Attached file · ${blob.name}`) + ) + return images.length > 0 || contextElements.length > 0 ? { images, contextElements } : EMPTY +} + +/** The run arguments behind the transcript's user rows. One fetch per turn while mounted. */ +export class MessageInputsStore extends JobBackedStore { + constructor( + workspace: () => string | undefined, + schema: () => { properties?: Record } | undefined, + shownElsewhere: () => ReadonlySet + ) { + super(workspace, EMPTY, async (ws, jobId) => + argsToMessageInputs( + ws, + (await JobService.getJobArgs({ workspace: ws, id: jobId })) as any, + schema(), + shownElsewhere() + ) + ) + } +} diff --git a/frontend/src/lib/components/flows/conversations/messageInputContext.test.ts b/frontend/src/lib/components/flows/conversations/messageInputContext.test.ts new file mode 100644 index 0000000000..d5f4535a00 --- /dev/null +++ b/frontend/src/lib/components/flows/conversations/messageInputContext.test.ts @@ -0,0 +1,47 @@ +import { describe, expect, it } from 'vitest' +import { argsToMessageInputs } from './messageInputContext.svelte' + +const SCHEMA = { + properties: { + token: { type: 'string', password: true }, + city: { type: 'string' }, + report: { type: 'object', format: 'resource-s3_object' } + } +} + +function summaries(elements: { title?: string; content?: string }[]) { + return elements.map((e) => e.content) +} + +describe('argsToMessageInputs', () => { + it('never puts a secret input on screen', () => { + const { contextElements } = argsToMessageInputs( + 'ws', + { token: 'hunter2', city: 'Paris' }, + SCHEMA + ) + expect(summaries(contextElements as any)).toEqual(['', 'Paris']) + }) + + // Only images get a thumbnail lane; every other s3 file is a chip naming its key. + it('splits attachments into thumbnails and file chips', () => { + const { images, contextElements } = argsToMessageInputs( + 'ws', + { report: [{ s3: 'a/shot.png' }, { s3: 'a/notes.pdf' }] }, + SCHEMA + ) + expect(images.map((i) => i.name)).toEqual(['shot.png']) + expect(images[0].dataUrl).toContain('file_key=a%2Fshot.png') + expect(contextElements).toHaveLength(1) + }) + + it('leaves out the message and anything the composer already shows', () => { + const { contextElements } = argsToMessageInputs( + 'ws', + { user_message: 'hi', city: 'Paris' }, + SCHEMA, + new Set(['city']) + ) + expect(contextElements).toEqual([]) + }) +}) diff --git a/frontend/src/lib/components/flows/conversations/toolCallContext.svelte.ts b/frontend/src/lib/components/flows/conversations/toolCallContext.svelte.ts new file mode 100644 index 0000000000..ffb76bca13 --- /dev/null +++ b/frontend/src/lib/components/flows/conversations/toolCallContext.svelte.ts @@ -0,0 +1,56 @@ +/** + * What a tool call ran with and returned, recovered from its job. + * + * A conversation row stores only a summary sentence ("Used X tool"), but a Windmill tool + * runs as its own job, and that job already holds everything the card needs: `args` are + * the arguments the model supplied, `result` is what came back, and `script_path` names + * the tool. Reading them there keeps one copy of the data instead of two. + * + * Two kinds of tool are out of reach and keep the sentence: an MCP tool, and a + * provider-native one (web search). Neither gets a job of its own — both run inside the + * agent's — so both rows name the agent's job, which retention needs to collect them. Its + * args are the agent's configuration and its result the agent's answer, so reading them + * would show confidently wrong details: an `aiagent` job is ignored here, and an MCP row + * stores its own call on the row instead (`tool_arguments` / `tool_result`). + */ +import { JobService } from '$lib/gen' +import { JobBackedStore } from './jobBackedStore.svelte' + +export type ToolCallDetails = { + toolName?: string + parameters?: any + result?: any +} + +const EMPTY: ToolCallDetails = {} + +/** The tool's own name, which is the last segment of the job's path. */ +function toolNameFromPath(path: string | undefined): string | undefined { + const name = path?.split('/').filter(Boolean).pop() + return name && name !== '' ? name : undefined +} + +export function jobToToolCallDetails(job: any): ToolCallDetails { + // The agent's own job means this row is a tool that ran inside it — MCP or provider-native + // — and names that job so retention can collect it. Its args describe the agent and its + // result is the agent's answer, so reading either would show confidently wrong details. + if (!job || job.job_kind === 'aiagent') return EMPTY + const parameters = + job.args && typeof job.args === 'object' && Object.keys(job.args).length > 0 + ? job.args + : undefined + return { + toolName: toolNameFromPath(job.script_path), + parameters, + result: job.result + } +} + +/** The tool jobs behind the transcript's tool rows. One fetch per row while mounted. */ +export class ToolCallStore extends JobBackedStore { + constructor(workspace: () => string | undefined) { + super(workspace, EMPTY, async (ws, jobId) => + jobToToolCallDetails(await JobService.getJob({ workspace: ws, id: jobId, noLogs: true })) + ) + } +} diff --git a/frontend/src/lib/components/flows/conversations/toolCallContext.test.ts b/frontend/src/lib/components/flows/conversations/toolCallContext.test.ts new file mode 100644 index 0000000000..7a0c85056e --- /dev/null +++ b/frontend/src/lib/components/flows/conversations/toolCallContext.test.ts @@ -0,0 +1,38 @@ +import { describe, expect, it } from 'vitest' +import { jobToToolCallDetails } from './toolCallContext.svelte' + +/** + * Every conversation row names a job so retention can collect it, which means a tool that + * ran inside the agent — MCP, or provider-native web search — points at the agent's own + * job. Reading that job for the call would show the agent's configuration as the tool's + * arguments and the agent's answer as its result. + */ +describe('jobToToolCallDetails', () => { + it('reads nothing from the agent job a jobless tool points at', () => { + expect( + jobToToolCallDetails({ + job_kind: 'aiagent', + script_path: 'f/chat/agent', + args: { user_message: 'hi', provider: { model: 'claude-sonnet-5' } }, + result: 'the agent answer' + }) + ).toEqual({}) + }) + + it('reads the call from a tool that has a job of its own', () => { + expect( + jobToToolCallDetails({ + job_kind: 'script', + script_path: 'f/tools/search_docs', + args: { query: 'retention' }, + result: ['a', 'b'] + }) + ).toEqual({ toolName: 'search_docs', parameters: { query: 'retention' }, result: ['a', 'b'] }) + }) + + it('treats an argument-less call as having none rather than an empty object', () => { + expect( + jobToToolCallDetails({ job_kind: 'script', script_path: 'f/t/now', args: {} }).parameters + ).toBeUndefined() + }) +}) diff --git a/frontend/src/lib/components/flows/conversations/turnTranscript.test.ts b/frontend/src/lib/components/flows/conversations/turnTranscript.test.ts new file mode 100644 index 0000000000..c8dfdb2325 --- /dev/null +++ b/frontend/src/lib/components/flows/conversations/turnTranscript.test.ts @@ -0,0 +1,136 @@ +import { describe, expect, it } from 'vitest' +import { + appendRevealed, + applyStreamEvent, + emptyTurnState, + turnFailed, + type TurnStep +} from './turnTranscript' +import type { ChatMessage } from './FlowChatManager.svelte' +import type { StreamEvent } from '$lib/components/chat/utils' + +function start(): TurnStep { + return { rows: [], state: emptyTurnState('conv') } +} + +let n = 0 +const nextId = () => `row-${++n}` + +function apply(step: TurnStep, events: StreamEvent[]): TurnStep { + return events.reduce((acc, event) => applyStreamEvent(acc, event, nextId), step) +} + +describe('turn transcript', () => { + it('keeps thinking that arrives in the same chunk as the tool call it led to', () => { + // The regression: reasoning revealed and a tool call applied back to back, which is + // how one SSE chunk delivers "thought about it, then called the tool". + let step = appendRevealed(start(), 'reasoning', 'Checking the issue first.', nextId) + step = apply(step, [{ kind: 'tool_call', callId: 'c1', name: 'mcp_linear_get_issue' }]) + + expect(step.rows.map((r) => r.message_type)).toEqual(['assistant', 'tool']) + expect(step.rows[0].reasoning).toBe('Checking the issue first.') + expect(step.rows[0].streaming).toBe(false) + }) + + it('starts a new answer row after a tool call instead of extending the one before it', () => { + let step = appendRevealed(start(), 'answer', 'Let me look. ', nextId) + step = apply(step, [ + { kind: 'tool_call', callId: 'c1', name: 'get_time' }, + { kind: 'tool_result', callId: 'c1', name: 'get_time', result: '{}', success: true } + ]) + step = appendRevealed(step, 'answer', 'It is noon.', nextId) + + expect(step.rows.map((r) => r.content)).toEqual([ + 'Let me look. ', + 'Used get_time tool', + 'It is noon.' + ]) + }) + + it('lands a tool call and its result on one row', () => { + const step = apply(start(), [ + { kind: 'tool_call', callId: 'c1', name: 'get_time' }, + { kind: 'tool_arguments', callId: 'c1', name: 'get_time', arguments: '{"tz":"UTC"}' }, + { kind: 'tool_execution', callId: 'c1', name: 'get_time' }, + { kind: 'tool_result', callId: 'c1', name: 'get_time', result: '{"now":1}', success: true } + ]) + + const tools = step.rows.filter((r) => r.message_type === 'tool') + expect(tools).toHaveLength(1) + expect(tools[0].tool_arguments).toBe('{"tz":"UTC"}') + expect(tools[0].tool_result).toBe('{"now":1}') + expect(tools[0].loading).toBe(false) + }) + + it('marks a failed tool call on its row', () => { + const step = apply(start(), [ + { kind: 'tool_call', callId: 'c1', name: 'get_time' }, + { kind: 'tool_result', callId: 'c1', name: 'get_time', result: 'boom', success: false } + ]) + + const tool = step.rows.find((r) => r.message_type === 'tool') + expect(tool?.success).toBe(false) + expect(tool?.content).toBe('Failed to use get_time tool') + }) + + it('grows one answer row as text is revealed', () => { + let step = appendRevealed(start(), 'answer', 'Hel', nextId) + step = appendRevealed(step, 'answer', 'lo', nextId) + + expect(step.rows).toHaveLength(1) + expect(step.rows[0].content).toBe('Hello') + expect(step.rows[0].streaming).toBe(true) + }) +}) + +/** + * Gates the Retry button, which on a flow re-runs the whole thing — side effects included — + * so it has to mean "this turn produced no answer", not "something inside it went wrong". + */ +describe('turnFailed', () => { + const row = (over: Partial): ChatMessage => + ({ id: 'x', message_type: 'assistant', content: '', ...over }) as ChatMessage + + it('is false when a tool failed but the agent went on to answer', () => { + const messages = [ + row({ message_type: 'user' }), + row({ message_type: 'tool', success: false }), + row({ message_type: 'assistant', success: true }) + ] + expect(turnFailed(messages, 0)).toBe(false) + }) + + it('is true when the turn ends on a failure', () => { + const messages = [ + row({ message_type: 'user' }), + row({ message_type: 'tool', success: true }), + row({ message_type: 'assistant', success: false }) + ] + expect(turnFailed(messages, 0)).toBe(true) + }) + + it('reports nothing while the turn is still running', () => { + const messages = [ + row({ message_type: 'user' }), + row({ message_type: 'tool', success: false }), + row({ message_type: 'assistant', streaming: true }) + ] + expect(turnFailed(messages, 0)).toBe(false) + }) + + // The window stops at the next user message, so a later turn's failure is not this one's. + it('does not read past the next user message', () => { + const messages = [ + row({ message_type: 'user' }), + row({ message_type: 'assistant', success: true }), + row({ message_type: 'user' }), + row({ message_type: 'assistant', success: false }) + ] + expect(turnFailed(messages, 0)).toBe(false) + expect(turnFailed(messages, 2)).toBe(true) + }) + + it('is false for a turn that has produced nothing yet', () => { + expect(turnFailed([row({ message_type: 'user' })], 0)).toBe(false) + }) +}) diff --git a/frontend/src/lib/components/flows/conversations/turnTranscript.ts b/frontend/src/lib/components/flows/conversations/turnTranscript.ts new file mode 100644 index 0000000000..ca1f2d9fdf --- /dev/null +++ b/frontend/src/lib/components/flows/conversations/turnTranscript.ts @@ -0,0 +1,188 @@ +/** + * The rows a turn is made of, as its events arrive. + * + * A run reports itself as a stream of events — thinking, answer tokens, a tool call, its + * result — and the transcript is what those become: an assistant row that grows, a card + * per tool call, the thinking that preceded either. The rules are small but ordered, and + * getting them wrong is invisible until someone watches a real run: thinking that arrives + * in the same chunk as the tool call it led to has to keep its row, and a row that a tool + * interrupted must not take the next answer's text. + * + * Kept as plain data in, plain data out — no EventSource, no runes, no toasts — so those + * rules can be exercised without a socket. FlowChatManager owns the connection and the + * pacing and calls in here; nothing in this file knows either exists. + */ +import type { ChatMessage } from './FlowChatManager.svelte' +import type { StreamEvent } from '$lib/components/chat/utils' +import { toolSummary } from '$lib/components/chat/utils' + +/** + * What the reducer remembers between events: the assistant row currently open and the + * text revealed into it, and which row each tool call is writing to. + */ +export type TurnState = { + conversationId: string + /** The assistant row taking text right now, or '' when the next text opens a new one. */ + assistantId: string + content: string + reasoning: string + /** Row id per tool call id: a call arrives as up to four events that share one card. */ + toolRowIds: Record +} + +export type TurnStep = { rows: ChatMessage[]; state: TurnState } + +export function emptyTurnState(conversationId: string): TurnState { + return { conversationId, assistantId: '', content: '', reasoning: '', toolRowIds: {} } +} + +function newRow(state: TurnState, id: string, patch: Partial): ChatMessage { + return { + id, + content: '', + created_at: new Date().toISOString(), + created_seq: 0, + conversation_id: state.conversationId, + job_id: '', + message_type: 'assistant', + loading: false, + streaming: false, + ...patch + } as ChatMessage +} + +/** Everything still streaming is finished: nothing further will be appended to it. */ +function settleStreamingRows(rows: ChatMessage[]): ChatMessage[] { + return rows.map((row) => (row.streaming ? { ...row, streaming: false } : row)) +} + +/** + * Text revealed into the turn's answer row, opening one if none is. Reasoning opens it + * too: thinking comes before the first answer token, and a tool call can arrive before + * that token ever does. + */ +export function appendRevealed( + { rows, state }: TurnStep, + kind: 'answer' | 'reasoning', + chunk: string, + newId: () => string +): TurnStep { + if (chunk === '') return { rows, state } + const next: TurnState = + kind === 'answer' + ? { ...state, content: state.content + chunk } + : { ...state, reasoning: state.reasoning + chunk } + const reasoning = next.reasoning === '' ? undefined : next.reasoning + + if (next.assistantId === '') { + const id = newId() + return { + rows: [...rows, newRow(next, id, { content: next.content, streaming: true, reasoning })], + state: { ...next, assistantId: id } + } + } + return { + rows: rows.map((row) => + row.id === next.assistantId ? { ...row, content: next.content, reasoning } : row + ), + state: next + } +} + +/** Create or patch the row for one tool call; its four events share the one card. */ +function upsertToolRow( + { rows, state }: TurnStep, + callId: string, + patch: Partial, + newId: () => string +): TurnStep { + const existing = state.toolRowIds[callId] + if (existing) { + return { + rows: rows.map((row) => (row.id === existing ? { ...row, ...patch } : row)), + state + } + } + const id = newId() + return { + rows: [...rows, newRow(state, id, { message_type: 'tool', success: true, ...patch })], + state: { ...state, toolRowIds: { ...state.toolRowIds, [callId]: id } } + } +} + +/** + * One event applied. Text events are not handled here: they reach the transcript through + * `appendRevealed` once the pacing has decided how much of them to show. + */ +export function applyStreamEvent( + step: TurnStep, + event: StreamEvent, + newId: () => string +): TurnStep { + switch (event.kind) { + case 'tool_call': + case 'tool_execution': { + // The row the answer was going into is finished, and the text that had been + // revealed into it stays there: the next answer starts its own row. Whatever the + // pacing still holds must have been flushed before this call, or it would land + // on the row after the tool card instead of the one before it. + const settled: TurnStep = { + rows: settleStreamingRows(step.rows), + state: { ...step.state, assistantId: '', content: '', reasoning: '' } + } + return upsertToolRow( + settled, + event.callId, + { tool_name: event.name, content: `Running ${event.name}`, loading: true }, + newId + ) + } + case 'tool_arguments': + return upsertToolRow( + step, + event.callId, + { tool_name: event.name, tool_arguments: event.arguments }, + newId + ) + case 'tool_result': + return upsertToolRow( + step, + event.callId, + { + tool_name: event.name, + tool_result: event.result, + content: toolSummary(event.name, event.success), + success: event.success, + loading: false + }, + newId + ) + default: + return step + } +} + +/** + * Whether the turn a user message started ended without an answer. + * + * Read from the turn's last row and no other. A tool that fails mid-turn is handed back to + * the agent, which routinely recovers and answers, so an unsuccessful tool row says nothing + * about the turn — and this drives the Retry button, which in the copilot means "the request + * never went through" rather than "something inside it went wrong". Offering it for a turn + * that answered would invite running the whole flow a second time, side effects and all. + * + * A turn with no row at all is not a failure to report: a run that dies anywhere still ends + * in an assistant row carrying the error (`worker_flow.rs`, on flow completion), so the only + * turns with nothing after the user message are the ones still waiting for their first row. + */ +export function turnFailed(messages: ChatMessage[], userIndex: number): boolean { + let last: ChatMessage | undefined + for (let i = userIndex + 1; i < messages.length; i++) { + const message = messages[i] + if (message.message_type === 'user') break + // Still going, so the turn has no outcome to report yet. + if (message.streaming || message.loading) return false + last = message + } + return last?.success === false +} diff --git a/frontend/src/lib/components/flows/flowInfers.ts b/frontend/src/lib/components/flows/flowInfers.ts index 958f6903bd..1a0bc8afc5 100644 --- a/frontend/src/lib/components/flows/flowInfers.ts +++ b/frontend/src/lib/components/flows/flowInfers.ts @@ -39,6 +39,12 @@ export const AI_AGENT_SCHEMA: Schema = { }, memory: { type: 'object', + // Chat mode keys memory on the conversation, so a chat whose agent has memory off + // forgets every turn. Enabling chat mode sets `auto`; this keeps it there. A step + // sitting at `off` stays switchable, or a flow that reached that state before — + // an agent added to an already-chat-enabled flow — would have no way out of it. + lockOneOfWhenChatEnabled: + "Chat mode keys this agent's history on the conversation, so memory stays on while it is enabled.", description: 'History sent between the system message and the user message.', oneOf: [ { diff --git a/frontend/src/lib/components/inputTransformEnv.svelte.ts b/frontend/src/lib/components/inputTransformEnv.svelte.ts index 50df45a15a..c8d34dc391 100644 --- a/frontend/src/lib/components/inputTransformEnv.svelte.ts +++ b/frontend/src/lib/components/inputTransformEnv.svelte.ts @@ -1,11 +1,31 @@ -import { CancelError, WorkspaceService } from '$lib/gen' +import { CancelError, WorkspaceService, type LargeFileStorage } from '$lib/gen' import { resource } from 'runed' /** - * Whether the workspace has S3 storage configured, for the fields that warn without it. Call during - * component initialisation and read `.current` where the answer is used. + * Whether the workspace has large-file storage the upload endpoints can resolve. Every + * kind counts, not only S3: Azure Blob, Azure Workload Identity, S3 via AWS OIDC and GCS + * all go through the same object-store abstraction, so reading `s3_resource_path` alone + * calls a perfectly good workspace unconfigured. */ -export function useS3StorageConfigured(ws: () => string | undefined): { +function storageConfigured(storage: LargeFileStorage | undefined): boolean { + if (!storage) return false + return ( + storage.type !== undefined || + storage.s3_resource_path !== undefined || + storage.azure_blob_resource_path !== undefined || + storage.gcs_resource_path !== undefined + ) +} + +/** + * Whether the workspace can store uploaded files. Call during component initialisation and + * read `.current` where the answer is used. + * + * Assumed configured until this workspace's own answer lands, so a surface that says "no + * storage configured" never flashes that on navigation, nor claims it merely because the + * fetch failed. + */ +export function useWorkspaceStorageConfigured(ws: () => string | undefined): { readonly current: boolean } { const settings = resource(ws, async (ws, _previousWs, { onCleanup }) => { @@ -24,12 +44,10 @@ export function useS3StorageConfigured(ws: () => string | undefined): { } }) - // Assume configured until this workspace's own answer lands: the warning must not - // linger from the previous workspace, nor appear merely because the fetch failed. const configured = $derived.by(() => { const loaded = settings.current return loaded && loaded.ws === ws() - ? loaded.settings.large_file_storage?.s3_resource_path !== undefined + ? storageConfigured(loaded.settings.large_file_storage) : true }) diff --git a/frontend/src/lib/components/sessions/SessionPicker.svelte b/frontend/src/lib/components/sessions/SessionPicker.svelte index e5b62da0eb..c634129f74 100644 --- a/frontend/src/lib/components/sessions/SessionPicker.svelte +++ b/frontend/src/lib/components/sessions/SessionPicker.svelte @@ -51,6 +51,7 @@ resetSessionPreviewTabs } from './sessionRuntime.svelte' import SessionStatusDot from './SessionStatusDot.svelte' + import UnreadCountBadge from '$lib/components/common/badge/UnreadCountBadge.svelte' import { buildWorkspaceHierarchy } from '$lib/utils/workspaceHierarchy' import SessionFilterMenu from './SessionFilterMenu.svelte' import { Menu, Menubar, MenuItem } from '$lib/components/meltComponents' @@ -801,14 +802,7 @@ {#if draft} {/if} - {#if unread > 0} - - {unread > 9 ? '9+' : unread} - - {/if} + {/if} @@ -1075,14 +1069,7 @@ {#if draft} {/if} - {#if unread > 0} - - {unread > 9 ? '9+' : unread} - - {/if} + {/if} diff --git a/frontend/src/lib/components/sessions/SessionStatusDot.svelte b/frontend/src/lib/components/sessions/SessionStatusDot.svelte index 646163bcaf..3d37d41d54 100644 --- a/frontend/src/lib/components/sessions/SessionStatusDot.svelte +++ b/frontend/src/lib/components/sessions/SessionStatusDot.svelte @@ -8,8 +8,18 @@ let { status, isFork, - forkDetached = false - }: { status: SessionChatStatus; isFork: boolean; forkDetached?: boolean } = $props() + forkDetached = false, + resting, + restingTitle + }: { + status: SessionChatStatus + isFork: boolean + forkDetached?: boolean + /** What the slot shows when there is no live signal. Sessions leave it unset and get + * the workspace/fork mark below; another list passes its own resting mark. */ + resting?: import('svelte').Snippet + restingTitle?: string + } = $props() const statusTooltip: Record = { idle: 'No chat activity', @@ -39,7 +49,9 @@ : 'Root workspace session' ) - const title = $derived(liveOverride ? statusTooltip[status] : persistentTitle) + const title = $derived( + liveOverride ? statusTooltip[status] : (restingTitle ?? persistentTitle) + ) @@ -55,6 +67,8 @@ {:else if status === 'error'} + {:else if resting} + {@render resting()} {:else if isFork} {#if forkDetached} diff --git a/frontend/src/routes/(root)/(logged)/flows/get/[...path]/+page.svelte b/frontend/src/routes/(root)/(logged)/flows/get/[...path]/+page.svelte index 6a6e731c11..1db5b25e97 100644 --- a/frontend/src/routes/(root)/(logged)/flows/get/[...path]/+page.svelte +++ b/frontend/src/routes/(root)/(logged)/flows/get/[...path]/+page.svelte @@ -84,6 +84,7 @@ onEditInForkClick } from '$lib/utils/editInFork' import { isCloudHosted } from '$lib/cloud' + import { agentStreamingEnabled } from '$lib/components/flows/agentFormFields' let flow: Flow | undefined = $state() let can_write = $state(false) @@ -522,6 +523,12 @@ let showEditButtons = $state(false) let mainButtons = $derived(getMainButtons(flow, args)) let chatInputEnabled = $derived(flow?.value?.chat_input_enabled ?? false) + let shouldUseStreaming = $derived.by(() => { + const modules = flow?.value?.modules + const lastModule = modules && modules.length > 0 ? modules[modules.length - 1] : undefined + if (lastModule?.value?.type !== 'aiagent') return false + return agentStreamingEnabled(lastModule.value) + }) @@ -630,63 +637,74 @@
- {#if flow?.path} - - {/if} + + +
*:first-child]:mt-3' : 'contents'} + > + {#if flow?.path} + + {/if} - {#if flow?.archived} - This flow was archived -
- {/if} + {#if flow?.archived} + This flow was archived +
+ {/if} - {#if pinnedVersion !== undefined} - - This is a historical version of the flow, not the latest. - - View latest - - -
- {/if} + {#if pinnedVersion !== undefined} + + This is a historical version of the flow, not the latest. + + View latest + + +
+ {/if} - {#if !emptyString(flow?.description)} -
- -
-
- {/if} + + {#if !chatInputEnabled && !emptyString(flow?.description)} +
+ +
+
+ {/if} - {#if deploymentInProgress} -
- - - Deployment in progress - {#if deploymentJobId} - view job - {/if} - -
- {/if} - {#if flow.lock_error_logs && flow.lock_error_logs != ''} - -

- This flow has not been deployed successfully because of the following errors: -

- - -
- {/if} + {#if deploymentInProgress} +
+ + + Deployment in progress + {#if deploymentJobId} + view job + {/if} + +
+ {/if} + {#if flow.lock_error_logs && flow.lock_error_logs != ''} + +

+ This flow has not been deployed successfully because of the following errors: +

+ + +
+ {/if} +
{#if chatInputEnabled} @@ -694,7 +712,13 @@ onRunFlow={runFlowForChat} {deploymentInProgress} path={flow?.path ?? ''} + description={flow?.description} + useStreaming={shouldUseStreaming} inputSchema={flow?.schema} + flowModules={flow?.value?.modules} + wideLayout + frame="none" + parallelTurns /> {:else} {@const hasSchema = diff --git a/system_prompts/auto-generated/flow.md b/system_prompts/auto-generated/flow.md index d24db93d53..da32bb4ac3 100644 --- a/system_prompts/auto-generated/flow.md +++ b/system_prompts/auto-generated/flow.md @@ -102,9 +102,59 @@ tool, `websearch` for web search. } ``` -- `provider` is a static object, not a bare resource string: `{ "kind": , +- `provider` is an object, not a bare resource string: `{ "kind": , "resource": "$res:", "model": }`. Required unless the module links to a saved - agent through `value.agent` + agent through `value.agent`. Static is right for a flow run from a form; a chat flow wires its + fields to flow inputs instead — see below + +### Chat-Mode Flows + +A flow with `value.chat_input_enabled: true` is run from a chat instead of a form: the composer +sends one message per turn and renders the conversation. It needs a required `user_message` string +input, read by the agent. + +**A static `provider` gives a chat that cannot change its model.** Feed it from flow inputs +instead, either way round: one input carrying the whole object (`"expr": "flow_input.model_config"`) +makes every field editable, or wire it field by field to fix some and expose others. A field the +chat can write becomes a control in the composer — a provider picker, a model list, a thinking +slider — and a field left static is fixed, with no control drawn for it. Two fields are +exceptions: `kind`, which the button writes only together with `resource` since a provider is +picked as a pair, and `reasoning_effort`, which gets a slider only for a model Windmill has +thinking levels for. Either way the field stays askable under Configure inputs, so nothing +the run needs becomes unreachable. +`user_attachments` works the same way: point it at an s3-object input and the composer gets a +paperclip. + +```json +{ + "id": "chat_agent", + "value": { + "type": "aiagent", + "input_transforms": { + "provider": { + "type": "javascript", + "expr": "({ kind: 'anthropic', resource: '$res:f/ai/claude', model: flow_input.model, reasoning_effort: flow_input.thinking })" + }, + "user_message": { "type": "javascript", "expr": "flow_input.user_message" }, + "user_attachments": { "type": "javascript", "expr": "flow_input.files" }, + "memory": { "type": "static", "value": { "kind": "auto", "context_length": 10 } }, + "streaming": { "type": "static", "value": true }, + "output_type": { "type": "static", "value": "text" } + }, + "tools": [] + } +} +``` + +- `memory` is what lets the agent see earlier turns; without it every message starts from nothing +- `streaming` on makes the answer and its thinking appear token by token instead of all at once +- Running one needs a `memory_id` **query parameter** — not a flow argument — naming the + conversation the turn belongs to: a fresh UUID starts one, reusing a UUID continues it. The chat + supplies it itself; a run driven any other way has to pass it or the server refuses the job +- Wiring field by field means one object literal whose values are literals or bare `flow_input.x` + references. A spread, a call or a computed key leaves the composer unable to tell which input + feeds which field, so it offers no control at all — a bare `flow_input.x` for the whole object + is read instead as that one input carrying every field ### Tool Naming Rules diff --git a/system_prompts/auto-generated/prompts.ts b/system_prompts/auto-generated/prompts.ts index 3529c99041..fe59b8a5c1 100644 --- a/system_prompts/auto-generated/prompts.ts +++ b/system_prompts/auto-generated/prompts.ts @@ -133,9 +133,59 @@ tool, \`websearch\` for web search. } \`\`\` -- \`provider\` is a static object, not a bare resource string: \`{ "kind": , +- \`provider\` is an object, not a bare resource string: \`{ "kind": , "resource": "$res:", "model": }\`. Required unless the module links to a saved - agent through \`value.agent\` + agent through \`value.agent\`. Static is right for a flow run from a form; a chat flow wires its + fields to flow inputs instead — see below + +### Chat-Mode Flows + +A flow with \`value.chat_input_enabled: true\` is run from a chat instead of a form: the composer +sends one message per turn and renders the conversation. It needs a required \`user_message\` string +input, read by the agent. + +**A static \`provider\` gives a chat that cannot change its model.** Feed it from flow inputs +instead, either way round: one input carrying the whole object (\`"expr": "flow_input.model_config"\`) +makes every field editable, or wire it field by field to fix some and expose others. A field the +chat can write becomes a control in the composer — a provider picker, a model list, a thinking +slider — and a field left static is fixed, with no control drawn for it. Two fields are +exceptions: \`kind\`, which the button writes only together with \`resource\` since a provider is +picked as a pair, and \`reasoning_effort\`, which gets a slider only for a model Windmill has +thinking levels for. Either way the field stays askable under Configure inputs, so nothing +the run needs becomes unreachable. +\`user_attachments\` works the same way: point it at an s3-object input and the composer gets a +paperclip. + +\`\`\`json +{ + "id": "chat_agent", + "value": { + "type": "aiagent", + "input_transforms": { + "provider": { + "type": "javascript", + "expr": "({ kind: 'anthropic', resource: '$res:f/ai/claude', model: flow_input.model, reasoning_effort: flow_input.thinking })" + }, + "user_message": { "type": "javascript", "expr": "flow_input.user_message" }, + "user_attachments": { "type": "javascript", "expr": "flow_input.files" }, + "memory": { "type": "static", "value": { "kind": "auto", "context_length": 10 } }, + "streaming": { "type": "static", "value": true }, + "output_type": { "type": "static", "value": "text" } + }, + "tools": [] + } +} +\`\`\` + +- \`memory\` is what lets the agent see earlier turns; without it every message starts from nothing +- \`streaming\` on makes the answer and its thinking appear token by token instead of all at once +- Running one needs a \`memory_id\` **query parameter** — not a flow argument — naming the + conversation the turn belongs to: a fresh UUID starts one, reusing a UUID continues it. The chat + supplies it itself; a run driven any other way has to pass it or the server refuses the job +- Wiring field by field means one object literal whose values are literals or bare \`flow_input.x\` + references. A spread, a call or a computed key leaves the composer unable to tell which input + feeds which field, so it offers no control at all — a bare \`flow_input.x\` for the whole object + is read instead as that one input carrying every field ### Tool Naming Rules diff --git a/system_prompts/auto-generated/skills/write-flow/SKILL.md b/system_prompts/auto-generated/skills/write-flow/SKILL.md index 8c7b7bfec5..4712d2d05c 100644 --- a/system_prompts/auto-generated/skills/write-flow/SKILL.md +++ b/system_prompts/auto-generated/skills/write-flow/SKILL.md @@ -190,9 +190,59 @@ tool, `websearch` for web search. } ``` -- `provider` is a static object, not a bare resource string: `{ "kind": , +- `provider` is an object, not a bare resource string: `{ "kind": , "resource": "$res:", "model": }`. Required unless the module links to a saved - agent through `value.agent` + agent through `value.agent`. Static is right for a flow run from a form; a chat flow wires its + fields to flow inputs instead — see below + +### Chat-Mode Flows + +A flow with `value.chat_input_enabled: true` is run from a chat instead of a form: the composer +sends one message per turn and renders the conversation. It needs a required `user_message` string +input, read by the agent. + +**A static `provider` gives a chat that cannot change its model.** Feed it from flow inputs +instead, either way round: one input carrying the whole object (`"expr": "flow_input.model_config"`) +makes every field editable, or wire it field by field to fix some and expose others. A field the +chat can write becomes a control in the composer — a provider picker, a model list, a thinking +slider — and a field left static is fixed, with no control drawn for it. Two fields are +exceptions: `kind`, which the button writes only together with `resource` since a provider is +picked as a pair, and `reasoning_effort`, which gets a slider only for a model Windmill has +thinking levels for. Either way the field stays askable under Configure inputs, so nothing +the run needs becomes unreachable. +`user_attachments` works the same way: point it at an s3-object input and the composer gets a +paperclip. + +```json +{ + "id": "chat_agent", + "value": { + "type": "aiagent", + "input_transforms": { + "provider": { + "type": "javascript", + "expr": "({ kind: 'anthropic', resource: '$res:f/ai/claude', model: flow_input.model, reasoning_effort: flow_input.thinking })" + }, + "user_message": { "type": "javascript", "expr": "flow_input.user_message" }, + "user_attachments": { "type": "javascript", "expr": "flow_input.files" }, + "memory": { "type": "static", "value": { "kind": "auto", "context_length": 10 } }, + "streaming": { "type": "static", "value": true }, + "output_type": { "type": "static", "value": "text" } + }, + "tools": [] + } +} +``` + +- `memory` is what lets the agent see earlier turns; without it every message starts from nothing +- `streaming` on makes the answer and its thinking appear token by token instead of all at once +- Running one needs a `memory_id` **query parameter** — not a flow argument — naming the + conversation the turn belongs to: a fresh UUID starts one, reusing a UUID continues it. The chat + supplies it itself; a run driven any other way has to pass it or the server refuses the job +- Wiring field by field means one object literal whose values are literals or bare `flow_input.x` + references. A spread, a call or a computed key leaves the composer unable to tell which input + feeds which field, so it offers no control at all — a bare `flow_input.x` for the whole object + is read instead as that one input carrying every field ### Tool Naming Rules diff --git a/system_prompts/base/flow-base.md b/system_prompts/base/flow-base.md index e70ac3d42a..58c0f6bc8c 100644 --- a/system_prompts/base/flow-base.md +++ b/system_prompts/base/flow-base.md @@ -102,9 +102,59 @@ tool, `websearch` for web search. } ``` -- `provider` is a static object, not a bare resource string: `{ "kind": , +- `provider` is an object, not a bare resource string: `{ "kind": , "resource": "$res:", "model": }`. Required unless the module links to a saved - agent through `value.agent` + agent through `value.agent`. Static is right for a flow run from a form; a chat flow wires its + fields to flow inputs instead — see below + +### Chat-Mode Flows + +A flow with `value.chat_input_enabled: true` is run from a chat instead of a form: the composer +sends one message per turn and renders the conversation. It needs a required `user_message` string +input, read by the agent. + +**A static `provider` gives a chat that cannot change its model.** Feed it from flow inputs +instead, either way round: one input carrying the whole object (`"expr": "flow_input.model_config"`) +makes every field editable, or wire it field by field to fix some and expose others. A field the +chat can write becomes a control in the composer — a provider picker, a model list, a thinking +slider — and a field left static is fixed, with no control drawn for it. Two fields are +exceptions: `kind`, which the button writes only together with `resource` since a provider is +picked as a pair, and `reasoning_effort`, which gets a slider only for a model Windmill has +thinking levels for. Either way the field stays askable under Configure inputs, so nothing +the run needs becomes unreachable. +`user_attachments` works the same way: point it at an s3-object input and the composer gets a +paperclip. + +```json +{ + "id": "chat_agent", + "value": { + "type": "aiagent", + "input_transforms": { + "provider": { + "type": "javascript", + "expr": "({ kind: 'anthropic', resource: '$res:f/ai/claude', model: flow_input.model, reasoning_effort: flow_input.thinking })" + }, + "user_message": { "type": "javascript", "expr": "flow_input.user_message" }, + "user_attachments": { "type": "javascript", "expr": "flow_input.files" }, + "memory": { "type": "static", "value": { "kind": "auto", "context_length": 10 } }, + "streaming": { "type": "static", "value": true }, + "output_type": { "type": "static", "value": "text" } + }, + "tools": [] + } +} +``` + +- `memory` is what lets the agent see earlier turns; without it every message starts from nothing +- `streaming` on makes the answer and its thinking appear token by token instead of all at once +- Running one needs a `memory_id` **query parameter** — not a flow argument — naming the + conversation the turn belongs to: a fresh UUID starts one, reusing a UUID continues it. The chat + supplies it itself; a run driven any other way has to pass it or the server refuses the job +- Wiring field by field means one object literal whose values are literals or bare `flow_input.x` + references. A spread, a call or a computed key leaves the composer unable to tell which input + feeds which field, so it offers no control at all — a bare `flow_input.x` for the whole object + is read instead as that one input carrying every field ### Tool Naming Rules