From 56b8ce1bd04fbb300aea5239668c8e0e9cd8a235 Mon Sep 17 00:00:00 2001 From: Guilhem Lemouel Date: Tue, 1 Sep 2026 15:38:22 +0200 Subject: [PATCH] wip: flow chat composer settings + message context --- backend/Cargo.toml | 2 +- backend/windmill-api-jobs/src/execution.rs | 7 +- backend/windmill-api/src/jobs.rs | 1 + frontend/package-lock.json | 44 +---- .../lib/components/FlowPreviewContent.svelte | 1 + .../copilot/chat/AIChatDisplay.svelte | 119 ++++++++---- .../copilot/chat/AIChatInput.svelte | 173 +++++++++++++----- .../copilot/chat/AIChatManager.svelte.ts | 14 ++ .../lib/components/copilot/chat/blobUtils.ts | 79 ++++++++ .../components/copilot/chat/chatViewHost.ts | 15 ++ .../copilot/chat/messageDraft.svelte.ts | 46 ++++- .../components/flows/content/FlowInput.svelte | 1 + .../AgentChatInputSubmenu.svelte | 111 +++++++++++ .../flows/conversations/FlowChat.svelte | 15 +- .../conversations/FlowChatInterface.svelte | 157 +++++++++++----- .../conversations/FlowChatSettings.svelte | 142 ++++++++++++++ .../flows/conversations/agentChatInputs.ts | 155 ++++++++++++++++ .../conversations/flowChatViewHost.svelte.ts | 128 +++++++++++-- .../messageInputContext.svelte.ts | 136 ++++++++++++++ .../(logged)/flows/get/[...path]/+page.svelte | 1 + 20 files changed, 1160 insertions(+), 187 deletions(-) create mode 100644 frontend/src/lib/components/copilot/chat/blobUtils.ts create mode 100644 frontend/src/lib/components/flows/conversations/AgentChatInputSubmenu.svelte create mode 100644 frontend/src/lib/components/flows/conversations/FlowChatSettings.svelte create mode 100644 frontend/src/lib/components/flows/conversations/agentChatInputs.ts create mode 100644 frontend/src/lib/components/flows/conversations/messageInputContext.svelte.ts diff --git a/backend/Cargo.toml b/backend/Cargo.toml index 86940805fc..b00dc5c081 100644 --- a/backend/Cargo.toml +++ b/backend/Cargo.toml @@ -115,7 +115,7 @@ strip = "none" [features] default = [] -private = ["windmill-api/private", "windmill-api-agent-workers?/private", "windmill-autoscaling/private", "windmill-common/private", "windmill-dep-map/private", "windmill-object-store/private", "windmill-git-sync/private", "windmill-indexer/private", "windmill-operator?/private", "windmill-queue/private", "windmill-worker/private", "windmill-test-utils/private"] +private = ["windmill-api/private", "windmill-api-agent-workers?/private", "windmill-autoscaling/private", "windmill-common/private", "windmill-dep-map/private", "windmill-object-store/private", "windmill-git-sync/private", "windmill-indexer?/private", "windmill-operator?/private", "windmill-queue/private", "windmill-worker/private", "windmill-test-utils/private"] agent_worker_server = ["windmill-api/agent_worker_server", "dep:windmill-api-agent-workers", "windmill-test-utils/agent_worker_server"] enterprise = ["windmill-worker/enterprise", "windmill-queue/enterprise", "windmill-api/enterprise", "windmill-api-agent-workers?/enterprise", "dep:windmill-autoscaling", "windmill-autoscaling/enterprise", "windmill-git-sync/enterprise", "windmill-common/prometheus", "windmill-common/enterprise", "windmill-object-store/enterprise", "license"] local_reports = ["windmill-common/local_reports"] diff --git a/backend/windmill-api-jobs/src/execution.rs b/backend/windmill-api-jobs/src/execution.rs index fbfd656cdf..d17dcfa15f 100644 --- a/backend/windmill-api-jobs/src/execution.rs +++ b/backend/windmill-api-jobs/src/execution.rs @@ -632,6 +632,7 @@ pub async fn handle_chat_conversation_messages( flow_path: &str, run_query: &RunJobQuery, user_message_raw: Option<&Box>, + job_id: Uuid, ) -> error::Result<()> { let memory_id = run_query.memory_id.ok_or_else(|| { windmill_common::error::Error::BadRequest( @@ -662,10 +663,13 @@ pub async fn handle_chat_conversation_messages( ) .await?; + // The run this message started. Its args are the only record of what the message + // carried besides its text — attachments and every other flow input — and nothing + // written later points at them: an assistant row holds the AI agent step's job. add_message_to_conversation_tx( tx, memory_id, - None, + Some(job_id), &user_message, MessageType::User, None, @@ -790,6 +794,7 @@ pub async fn run_flow<'c>( &flow_path.to_string(), &run_query, args.args.get("user_message"), + uuid, ) .await?; } diff --git a/backend/windmill-api/src/jobs.rs b/backend/windmill-api/src/jobs.rs index a30cffbfbf..050887ce0f 100644 --- a/backend/windmill-api/src/jobs.rs +++ b/backend/windmill-api/src/jobs.rs @@ -9257,6 +9257,7 @@ async fn run_preview_flow_job( &flow_path, &run_query, user_message.as_ref(), + uuid, ) .await?; } diff --git a/frontend/package-lock.json b/frontend/package-lock.json index e6549e22ef..e8d65ecdaa 100644 --- a/frontend/package-lock.json +++ b/frontend/package-lock.json @@ -1755,7 +1755,6 @@ "cpu": [ "arm64" ], - "dev": true, "license": "MIT", "optional": true, "os": [ @@ -1772,7 +1771,6 @@ "cpu": [ "arm64" ], - "dev": true, "license": "MIT", "optional": true, "os": [ @@ -1789,7 +1787,6 @@ "cpu": [ "x64" ], - "dev": true, "license": "MIT", "optional": true, "os": [ @@ -1806,7 +1803,6 @@ "cpu": [ "x64" ], - "dev": true, "license": "MIT", "optional": true, "os": [ @@ -1823,7 +1819,6 @@ "cpu": [ "arm" ], - "dev": true, "license": "MIT", "optional": true, "os": [ @@ -1840,7 +1835,6 @@ "cpu": [ "arm64" ], - "dev": true, "license": "MIT", "optional": true, "os": [ @@ -1857,7 +1851,6 @@ "cpu": [ "arm64" ], - "dev": true, "license": "MIT", "optional": true, "os": [ @@ -1874,7 +1867,6 @@ "cpu": [ "ppc64" ], - "dev": true, "license": "MIT", "optional": true, "os": [ @@ -1891,7 +1883,6 @@ "cpu": [ "s390x" ], - "dev": true, "license": "MIT", "optional": true, "os": [ @@ -1908,7 +1899,6 @@ "cpu": [ "x64" ], - "dev": true, "license": "MIT", "optional": true, "os": [ @@ -1925,7 +1915,6 @@ "cpu": [ "x64" ], - "dev": true, "license": "MIT", "optional": true, "os": [ @@ -1942,7 +1931,6 @@ "cpu": [ "arm64" ], - "dev": true, "license": "MIT", "optional": true, "os": [ @@ -1959,7 +1947,6 @@ "cpu": [ "arm64" ], - "dev": true, "license": "MIT", "optional": true, "os": [ @@ -1976,7 +1963,6 @@ "cpu": [ "x64" ], - "dev": true, "license": "MIT", "optional": true, "os": [ @@ -7583,7 +7569,7 @@ "version": "1.21.7", "resolved": "https://registry.npmjs.org/jiti/-/jiti-1.21.7.tgz", "integrity": "sha512-/imKNG4EbWNrVjoNC/1H5/9GFy+tqjGBHCaSsN+P2RnPqjsLmv6UD3Ej+Kj8nBWaRAwyk7kK5ZUc+OEatnTR3A==", - "dev": true, + "devOptional": true, "license": "MIT", "bin": { "jiti": "bin/jiti.js" @@ -8279,7 +8265,6 @@ "cpu": [ "arm64" ], - "dev": true, "license": "MPL-2.0", "optional": true, "os": [ @@ -8300,7 +8285,6 @@ "cpu": [ "arm64" ], - "dev": true, "license": "MPL-2.0", "optional": true, "os": [ @@ -8321,7 +8305,6 @@ "cpu": [ "x64" ], - "dev": true, "license": "MPL-2.0", "optional": true, "os": [ @@ -8342,7 +8325,6 @@ "cpu": [ "x64" ], - "dev": true, "license": "MPL-2.0", "optional": true, "os": [ @@ -8363,7 +8345,6 @@ "cpu": [ "arm" ], - "dev": true, "license": "MPL-2.0", "optional": true, "os": [ @@ -8384,7 +8365,6 @@ "cpu": [ "arm64" ], - "dev": true, "license": "MPL-2.0", "optional": true, "os": [ @@ -8405,7 +8385,6 @@ "cpu": [ "arm64" ], - "dev": true, "license": "MPL-2.0", "optional": true, "os": [ @@ -8426,7 +8405,6 @@ "cpu": [ "x64" ], - "dev": true, "license": "MPL-2.0", "optional": true, "os": [ @@ -8447,7 +8425,6 @@ "cpu": [ "x64" ], - "dev": true, "license": "MPL-2.0", "optional": true, "os": [ @@ -8468,7 +8445,6 @@ "cpu": [ "arm64" ], - "dev": true, "license": "MPL-2.0", "optional": true, "os": [ @@ -8489,7 +8465,6 @@ "cpu": [ "x64" ], - "dev": true, "license": "MPL-2.0", "optional": true, "os": [ @@ -13195,21 +13170,6 @@ } } }, - "node_modules/svelte-check/node_modules/picomatch": { - "version": "4.0.7", - "resolved": "https://registry.npmjs.org/picomatch/-/picomatch-4.0.7.tgz", - "integrity": "sha512-qcJu88Q2IWqJsDD529JKMdwGm/dvInW4HvQnRwiH9JtihJvzGOscDtHE3x1pBKeUOTysQ8kVmLnJ2kJu7yhcGA==", - "dev": true, - "license": "MIT", - "optional": true, - "peer": true, - "engines": { - "node": ">=12" - }, - "funding": { - "url": "https://github.com/sponsors/jonschlinkert" - } - }, "node_modules/svelte-eslint-parser": { "version": "0.43.0", "resolved": "https://registry.npmjs.org/svelte-eslint-parser/-/svelte-eslint-parser-0.43.0.tgz", @@ -13989,7 +13949,7 @@ "version": "5.9.3", "resolved": "https://registry.npmjs.org/typescript/-/typescript-5.9.3.tgz", "integrity": "sha512-jl1vZzPDinLr9eUt3J/t7V6FgNEw9QjvBPdysz9KfQDD41fQrC2Y4vKQdiaUpFT4bXlb1RHhLpp8wtm6M5TgSw==", - "dev": true, + "devOptional": true, "license": "Apache-2.0", "bin": { "tsc": "bin/tsc", diff --git a/frontend/src/lib/components/FlowPreviewContent.svelte b/frontend/src/lib/components/FlowPreviewContent.svelte index d895cb4e2e..acb9ac3780 100644 --- a/frontend/src/lib/components/FlowPreviewContent.svelte +++ b/frontend/src/lib/components/FlowPreviewContent.svelte @@ -484,6 +484,7 @@ hideSidebar={true} path={$pathStore} inputSchema={flowStore.val.schema} + flowModules={flowStore.val.value?.modules} /> {:else} diff --git a/frontend/src/lib/components/copilot/chat/AIChatDisplay.svelte b/frontend/src/lib/components/copilot/chat/AIChatDisplay.svelte index 79dcc3c2f6..933900cf59 100644 --- a/frontend/src/lib/components/copilot/chat/AIChatDisplay.svelte +++ b/frontend/src/lib/components/copilot/chat/AIChatDisplay.svelte @@ -50,6 +50,7 @@ import type { SelectedContext } from './app/core' import { type FileToAttach } from './files/attachedFiles.svelte' import { isImageFile } from './imageUtils' + import { matchesAccept } from './blobUtils' import { hasFileSystemAccess, pickDirectory, @@ -157,6 +158,8 @@ wideLayout = false, emptyHint, inputPreface, + footerControls, + footerSettings, initialInstructions = undefined, onDraftChange = undefined, placeholder = undefined, @@ -188,6 +191,11 @@ wideLayout?: boolean emptyHint?: Snippet inputPreface?: Snippet + /** Extra controls at the head of the composer's footer row. */ + footerControls?: Snippet + /** The settings control at the footer's right edge, where the copilot puts its + * model picker. A host that configures its turn elsewhere replaces it here. */ + footerSettings?: Snippet // Seed / observe the main composer's draft text (see AIChatInput). initialInstructions?: string onDraftChange?: (text: string) => void @@ -319,12 +327,10 @@ chatHost.mode === AIMode.SCRIPT || chatHost.mode === AIMode.FLOW || chatHost.mode === AIMode.APP ) - // File attachment is GLOBAL-mode only. - const canAttachFiles = $derived(chatHost.mode === AIMode.GLOBAL && !disabled) - // Steers the OS file picker toward text + image formats (soft hint; both attach - // to the message — text files after a content sniff). - const TEXT_FILE_ACCEPT = - 'image/*,text/*,.txt,.csv,.tsv,.json,.jsonl,.ndjson,.md,.markdown,.log,.yaml,.yml,.toml,.ini,.cfg,.conf,.env,.xml,.html,.htm,.css,.js,.mjs,.cjs,.ts,.tsx,.jsx,.py,.rb,.rs,.go,.java,.kt,.c,.h,.cpp,.cc,.cs,.php,.sh,.bash,.zsh,.sql,.svelte,.vue,.dockerfile' + const canAttachFiles = $derived(chatHost.supportsMessageAttachments && !disabled) + // Folders are linked as session-wide assets, which only a host that reads files in + // the browser can do — a host running the turn server-side takes attachments only. + const canLinkFolders = $derived(chatHost.supportsLinkedFolders && !disabled) let fileInputEl = $state(null) let folderInputEl = $state(null) let dragDepth = $state(0) @@ -444,12 +450,16 @@ handles.length === 0 ? flatFiles : await Promise.all(handles.filter(isFileHandle).map((h) => h.getFile())) - // Loose text files attach to the message, like images. - const textFiles = looseFiles.filter((f) => !isImageFile(f)) - if (textFiles.length > 0) await aiChatInput?.addTextFiles(textFiles) + // Loose files attach to the message, like images. + await attachNonImageFiles(looseFiles.filter((f) => !isImageFile(f))) // Folders link as a live handle. - for (const h of handles.filter(isDirectoryHandle)) { - await addDirHandle(h) + const dirs = handles.filter(isDirectoryHandle) + if (dirs.length > 0 && !canLinkFolders) { + sendUserToast('Folders cannot be attached in this chat — drop individual files.', true) + } else { + for (const h of dirs) { + await addDirHandle(h) + } } } else { // Fallback (no File System Access API): snapshot dropped files AND folders by walking @@ -474,25 +484,53 @@ topLevelText.push(file) } } - if (folderEntries.length > 0) await handleAddFiles(folderEntries) - if (topLevelText.length > 0) await aiChatInput?.addTextFiles(topLevelText) + if (folderEntries.length > 0) { + if (canLinkFolders) await handleAddFiles(folderEntries) + else sendUserToast('Folders cannot be attached in this chat — drop individual files.', true) + } + await attachNonImageFiles(topLevelText) } } async function onFileInputChange(e: Event) { const input = e.currentTarget as HTMLInputElement if (input.files && input.files.length > 0) { - const picked = Array.from(input.files) - const imageFiles = picked.filter(isImageFile) - const textFiles = picked.filter((f) => !isImageFile(f)) - // Reserved before the text work is awaited — see onPanelDrop. - const imageWork = imageFiles.length > 0 ? aiChatInput?.addImages(imageFiles) : undefined - if (textFiles.length > 0) await aiChatInput?.addTextFiles(textFiles) - await imageWork + await attachPickedFiles(Array.from(input.files)) } input.value = '' // allow re-selecting the same file } + /** + * Route non-image files to the lane the host reads. A host that decodes them + * takes text; one that forwards them verbatim (to object storage, say) takes + * blobs, and its narrower `accept` is re-applied here because a drop bypasses + * the picker's own filtering. + */ + async function attachNonImageFiles(files: File[]) { + if (files.length === 0) return + if (chatHost.attachmentsAsBlobs) { + const allowed = files.filter((f) => matchesAccept(f, chatHost.attachmentAccept)) + if (allowed.length < files.length) { + sendUserToast( + `${files.length - allowed.length} file(s) skipped — this chat accepts ${chatHost.attachmentAccept}.`, + true + ) + } + await aiChatInput?.addBlobs(allowed) + } else { + await aiChatInput?.addTextFiles(files) + } + } + + async function attachPickedFiles(picked: File[]) { + const imageFiles = picked.filter(isImageFile) + const others = picked.filter((f) => !isImageFile(f)) + // Reserved before the other work is awaited — see onPanelDrop. + const imageWork = imageFiles.length > 0 ? aiChatInput?.addImages(imageFiles) : undefined + await attachNonImageFiles(others) + await imageWork + } + function onFolderInputChange(e: Event) { const input = e.currentTarget as HTMLInputElement // webkitdirectory files carry webkitRelativePath (`folder/sub/file`); addFiles groups @@ -562,9 +600,14 @@ const showFlowPendingActionControls = $derived( (chatHost.flowAiChatHelpers?.hasPendingChanges() ?? false) && !chatHost.autoAcceptEditsActive ) + // Everything the left group can hold. `canAttachFiles` belongs here too: in GLOBAL + // mode the `+` always has the context picker or the autonomy selector beside it, but + // a host with attachments and nothing else would lose the group and the `+` with it. const showFooterLeftControls = $derived( !disabled && - (showContextPicker || + (footerControls !== undefined || + canAttachFiles || + showContextPicker || showAutonomyModeSelector || (chatHost.mode === AIMode.SCRIPT && hasDiff)) ) @@ -825,6 +868,7 @@ the panel, or the Escape-to-stop focus check would wrongly reject them. --> > {#if showFooterLeftControls}
+ {@render footerControls?.()} {#if showContextPicker && !disabled} {#snippet trigger()} @@ -886,19 +930,23 @@ the panel, or the Escape-to-stop focus check would wrongly reject them. --> linkFiles() } }, - { - // A real (live) link needs the File System Access API; without it the - // folder is only snapshotted, so call it "Add folder", not "Link folder". - displayName: canUseFsAccess ? 'Link folder' : 'Add folder', - icon: Folder, - tooltip: canUseFsAccess - ? 'Linked live — the assistant reads the folder’s current files from disk and refreshes each turn.' - : 'Loaded as a snapshot — the folder’s files are copied into your browser (they won’t auto-update). For a live link that refreshes from disk, use a Chromium-based browser (Chrome, Edge).', - action: () => { - plusMenuOpen = false - linkFolder() - } - }, + ...(canLinkFolders + ? [ + { + // A real (live) link needs the File System Access API; without it the + // folder is only snapshotted, so call it "Add folder", not "Link folder". + displayName: canUseFsAccess ? 'Link folder' : 'Add folder', + icon: Folder, + tooltip: canUseFsAccess + ? 'Linked live — the assistant reads the folder’s current files from disk and refreshes each turn.' + : 'Loaded as a snapshot — the folder’s files are copied into your browser (they won’t auto-update). For a live link that refreshes from disk, use a Chromium-based browser (Chrome, Edge).', + action: () => { + plusMenuOpen = false + linkFolder() + } + } + ] + : []), ...(chatHost.mode === AIMode.GLOBAL && mcpConnections ? [ { @@ -946,7 +994,7 @@ the panel, or the Escape-to-stop focus check would wrongly reject them. --> bind:this={fileInputEl} type="file" multiple - accept={TEXT_FILE_ACCEPT} + accept={chatHost.attachmentAccept} class="hidden no-default-style" onchange={onFileInputChange} /> @@ -1050,6 +1098,7 @@ the panel, or the Escape-to-stop focus check would wrongly reject them. --> {#if chatHost.supportsModelSettings} {/if} + {@render footerSettings?.()} {#if chatHost.mode === AIMode.GLOBAL} {/if} diff --git a/frontend/src/lib/components/copilot/chat/AIChatInput.svelte b/frontend/src/lib/components/copilot/chat/AIChatInput.svelte index ecdc3fd4b0..67d603f210 100644 --- a/frontend/src/lib/components/copilot/chat/AIChatInput.svelte +++ b/frontend/src/lib/components/copilot/chat/AIChatInput.svelte @@ -41,6 +41,12 @@ textByteLength, type AttachedTextFile } from './textFileUtils' + import { + fileToAttachedBlob, + MAX_ATTACHED_BLOBS, + MAX_BLOB_BYTES, + type AttachedBlob + } from './blobUtils' import { MessageDraft } from './messageDraft.svelte' import ExpandableImage, { isImageViewerOpen @@ -208,19 +214,24 @@ // against a concurrent drop. let pendingImages = $state(0) - /** Attach dropped/pasted image files (downscaled + bounded). GLOBAL mode only. */ + /** Attach dropped/pasted image files (downscaled + bounded). */ export async function addImages(files: (File | Blob)[]) { - if (chatHost.mode !== AIMode.GLOBAL) return + if (!chatHost.supportsMessageAttachments) return const imageFiles = files.filter(isImageFile) if (imageFiles.length === 0) return - // tryGetCurrentModel returns undefined instead of throwing: this runs from a - // drop/paste handler that can't surface a rejection. - const model = tryGetCurrentModel() - // Only known text-only models fail this, so attaching would certainly 400 the - // next turn — refuse rather than warn and send it anyway. - if (model && !modelSupportsVision(model.provider, model.model)) { - sendUserToast(`${model.model} can't read images. Switch to a vision model first.`, true) - return + // The vision check is about the model this composer's own turn will hit, so it + // only applies to a host that picks that model. Elsewhere the model is chosen + // in the flow and tryGetCurrentModel would answer for the wrong one. + if (chatHost.supportsModelSettings) { + // tryGetCurrentModel returns undefined instead of throwing: this runs from a + // drop/paste handler that can't surface a rejection. + const model = tryGetCurrentModel() + // Only known text-only models fail this, so attaching would certainly 400 the + // next turn — refuse rather than warn and send it anyway. + if (model && !modelSupportsVision(model.provider, model.model)) { + sendUserToast(`${model.model} can't read images. Switch to a vision model first.`, true) + return + } } // Count decodes already in flight: two drops that both read the image count // before either resolves would each claim the same free slots and overshoot @@ -309,7 +320,7 @@ /** Attach dropped/picked text files (sniffed + bounded). GLOBAL mode only. */ export async function addTextFiles(candidates: File[]) { - if (chatHost.mode !== AIMode.GLOBAL) return + if (!chatHost.supportsMessageAttachments) return if (candidates.length === 0) return const remaining = MAX_ATTACHED_FILES - draft.files.length - pendingFiles if (remaining <= 0) { @@ -405,6 +416,53 @@ draft.files = draft.files.filter((_, i) => i !== index) } + // Blobs being read right now — same send-hold/slot-reservation role as pendingImages. + let pendingBlobs = $state(0) + + /** Attach files the host takes verbatim (a PDF, say). Kept out of addTextFiles: + * that one decodes to a string and drops anything the binary sniff rejects. */ + export async function addBlobs(candidates: File[]) { + if (!chatHost.supportsMessageAttachments) return + if (candidates.length === 0) return + const oversized = candidates.filter((f) => f.size > MAX_BLOB_BYTES) + if (oversized.length > 0) { + const mb = Math.round(MAX_BLOB_BYTES / 1_000_000) + sendUserToast(`${oversized.length} file(s) over ${mb}MB were skipped.`, true) + } + const usable = candidates.filter((f) => f.size <= MAX_BLOB_BYTES) + if (usable.length === 0) return + const remaining = MAX_ATTACHED_BLOBS - draft.blobs.length - pendingBlobs + if (remaining <= 0) { + sendUserToast(`You can attach up to ${MAX_ATTACHED_BLOBS} files.`, true) + return + } + const batch = usable.slice(0, remaining) + if (batch.length < usable.length) { + sendUserToast( + `You can attach up to ${MAX_ATTACHED_BLOBS} files; ${usable.length - batch.length} were skipped.`, + true + ) + } + pendingBlobs += batch.length + try { + const added: AttachedBlob[] = [] + for (const file of batch) { + try { + added.push(await fileToAttachedBlob(file)) + } catch (e) { + sendUserToast(`Could not read ${file.name}`, true) + } + } + if (added.length > 0) draft.addBlobs(added) + } finally { + pendingBlobs -= batch.length + } + } + + function removeBlob(index: number) { + draft.blobs = draft.blobs.filter((_, i) => i !== index) + } + // App mode @ mention state let showAppContextTooltip = $state(false) let appContextTooltipWord = $state('') @@ -486,7 +544,7 @@ // Attachments still decoding/reading (or mid-drop-routing) count as // occupancy too — they belong to a draft the user started even though // their lane is still empty. - if (pendingImages > 0 || pendingFiles > 0 || ingestionHolds > 0) return false + if (pendingImages > 0 || pendingFiles > 0 || pendingBlobs > 0 || ingestionHolds > 0) return false if ( !draft.replaceIfEmpty({ text: value, @@ -691,7 +749,7 @@ function sendRequest() { // The send button is disabled while decoding, but Enter reaches here directly. // Sending now would drop the in-flight attachments onto the following message. - if (pendingImages > 0 || pendingFiles > 0 || ingestionHolds > 0) { + if (pendingImages > 0 || pendingFiles > 0 || pendingBlobs > 0 || ingestionHolds > 0) { return } // Read before `take()` empties the draft the id derives from, and only take @@ -752,7 +810,8 @@ instructions: sent.text, pastes: sent.pastes, images: sent.images, - files: sent.files + files: sent.files, + blobs: sent.blobs }) // clearForSend() pre-zaps the textarea's mention-sync so the wipe // doesn't drop `selectedContext` before `AIChatManager.beforeSend` @@ -987,6 +1046,7 @@ disabled || pendingImages > 0 || pendingFiles > 0 || + pendingBlobs > 0 || ingestionHolds > 0 || (emptyDraft && (onSendRequest !== undefined || @@ -1015,7 +1075,7 @@ thumbnails get their own row (different height). --> {#snippet badgeRow()} {@const contextChips = showContext ? selectedContext : domSelectorChips} - {#if contextChips.length > 0 || draft.files.length > 0 || pendingFiles > 0} + {#if contextChips.length > 0 || draft.files.length > 0 || pendingFiles > 0 || draft.blobs.length > 0 || pendingBlobs > 0}
{#each contextChips as element (contextKey(element))} removeFile(i)} /> {/each} - {#each { length: pendingFiles } as _, i (i)} + + {#each draft.blobs as blob, i (i)} + removeBlob(i)} + /> + {/each} + {#each { length: pendingFiles + pendingBlobs } as _, i (i)}
void addImages(pasted) : undefined} - onTextFiles={chatHost.mode === AIMode.GLOBAL + onTextFiles={chatHost.supportsMessageAttachments ? (pasted) => void addTextFiles(pasted) : undefined} {availableContext} @@ -1236,30 +1309,46 @@ {/if} {:else} -
- - {#if !bottomRightSnippet} -
- {@render sendStopButton()} -
- {/if} + +
+ {@render badgeRow()} + {@render imageChipsRow()} +
+ + {#if !bottomRightSnippet} +
+ {@render sendStopButton()} +
+ {/if} +
{/if} {#if bottomRightSnippet} diff --git a/frontend/src/lib/components/copilot/chat/AIChatManager.svelte.ts b/frontend/src/lib/components/copilot/chat/AIChatManager.svelte.ts index 1fae48fc80..eb1c845f48 100644 --- a/frontend/src/lib/components/copilot/chat/AIChatManager.svelte.ts +++ b/frontend/src/lib/components/copilot/chat/AIChatManager.svelte.ts @@ -411,6 +411,20 @@ export class AIChatManager implements ChatViewHost { // affordances apply here. See ChatViewHost for hosts where they don't. supportsModelSettings = true supportsMessageEditing = true + // Attachments and linked folders are GLOBAL-mode affordances. Declared as + // getters because `mode` changes under a mounted composer. + get supportsMessageAttachments() { + return this.mode === AIMode.GLOBAL + } + get supportsLinkedFolders() { + return this.mode === AIMode.GLOBAL + } + // Steers the OS file picker toward text + image formats (soft hint; both attach + // to the message — text files after a content sniff). + // The copilot reads attachments in the browser, so non-image files decode to text. + attachmentsAsBlobs = false + attachmentAccept = + 'image/*,text/*,.txt,.csv,.tsv,.json,.jsonl,.ndjson,.md,.markdown,.log,.yaml,.yml,.toml,.ini,.cfg,.conf,.env,.xml,.html,.htm,.css,.js,.mjs,.cjs,.ts,.tsx,.jsx,.py,.rb,.rs,.go,.java,.kt,.c,.h,.cpp,.cc,.cs,.php,.sh,.bash,.zsh,.sql,.svelte,.vue,.dockerfile' /** Files the user attached to the current GLOBAL-mode conversation. */ attachedFiles = new AttachedFilesStore() /** Markdown artifacts the copilot created for the current session. */ diff --git a/frontend/src/lib/components/copilot/chat/blobUtils.ts b/frontend/src/lib/components/copilot/chat/blobUtils.ts new file mode 100644 index 0000000000..e2eacd1680 --- /dev/null +++ b/frontend/src/lib/components/copilot/chat/blobUtils.ts @@ -0,0 +1,79 @@ +/** + * Message-scoped attachments that are neither an image nor readable text — a PDF + * being the case that matters. They ride the composer next to images and files, + * as chips cleared on send, and reach the host through `ChatSendRequestOptions`. + * + * The bytes are kept verbatim, unlike an image (which normalises to a bounded + * PNG/JPEG for the model) and unlike a text file (which is decoded to a string): + * a host that forwards these to object storage has to upload what the user + * picked, not a re-encoding of it. + */ + +/** Blobs one message may carry — the same slot cap images and text files use. */ +export const MAX_ATTACHED_BLOBS = 8 + +/** + * Per-blob byte cap. The data URL sits in composer state until send, so this + * bounds what one message can hold in memory; a host uploading elsewhere pays + * the same bytes again on the wire. + */ +export const MAX_BLOB_BYTES = 20_000_000 + +export type AttachedBlob = { + name: string + /** The file's own media type, verbatim — the upload's Content-Type depends on it. */ + mediaType: string + /** `data:;base64,<...>` of the original bytes. */ + dataUrl: string + size: number +} + +/** + * Whether a file satisfies an `accept` list — the same list the OS picker gets, applied + * again on drop, where the browser enforces nothing. + */ +export function matchesAccept(file: File, accept: string): boolean { + const patterns = accept + .split(',') + .map((p) => p.trim().toLowerCase()) + .filter(Boolean) + if (patterns.length === 0) return true + const type = file.type.toLowerCase() + const name = file.name.toLowerCase() + return patterns.some((pattern) => { + if (pattern.startsWith('.')) return name.endsWith(pattern) + if (pattern.endsWith('/*')) return type.startsWith(pattern.slice(0, -1)) + return type === pattern + }) +} + +export async function fileToAttachedBlob(file: File): Promise { + const dataUrl = await new Promise((resolve, reject) => { + const reader = new FileReader() + reader.onload = () => resolve(String(reader.result)) + reader.onerror = () => reject(reader.error ?? new Error(`Could not read ${file.name}`)) + reader.readAsDataURL(file) + }) + return { + name: file.name, + mediaType: file.type || 'application/octet-stream', + dataUrl, + size: file.size + } +} + +/** The bytes behind a `data:` URL, for a host that has to re-upload them. */ +export function dataUrlToBlob(dataUrl: string, fallbackType = 'application/octet-stream'): Blob { + const comma = dataUrl.indexOf(',') + const header = dataUrl.slice(5, comma) + const isBase64 = header.endsWith(';base64') + const mediaType = (isBase64 ? header.slice(0, -';base64'.length) : header) || fallbackType + const payload = dataUrl.slice(comma + 1) + if (!isBase64) { + return new Blob([decodeURIComponent(payload)], { type: mediaType }) + } + const binary = atob(payload) + const bytes = new Uint8Array(binary.length) + for (let i = 0; i < binary.length; i++) bytes[i] = binary.charCodeAt(i) + return new Blob([bytes], { type: mediaType }) +} diff --git a/frontend/src/lib/components/copilot/chat/chatViewHost.ts b/frontend/src/lib/components/copilot/chat/chatViewHost.ts index e462db3605..6eecb960c9 100644 --- a/frontend/src/lib/components/copilot/chat/chatViewHost.ts +++ b/frontend/src/lib/components/copilot/chat/chatViewHost.ts @@ -3,6 +3,7 @@ import type { AIMode, AIAutonomyMode } from './AIChatManager.svelte' import { getAiChatManager } from './aiChatManagerContext' import type { DisplayMessage, Tool } from './shared' import type { ContextElement } from './context' +import type { AttachedBlob } from './blobUtils' import type { AttachedImage } from './imageUtils' import type { AttachedTextFile } from './textFileUtils' import type { PasteAttachment } from './pasteTokens' @@ -18,6 +19,7 @@ export type ChatSendRequestOptions = { pastes?: PasteAttachment[] images?: AttachedImage[] files?: AttachedTextFile[] + blobs?: AttachedBlob[] } /** @@ -92,6 +94,19 @@ export interface ChatViewHost { /** Click a user message to edit and resend it. Needs a host that can rewind * its own transcript, which a host replaying a server-side run cannot. */ supportsMessageEditing: boolean + /** The `+` menu's file entry and drag-and-drop onto the panel. Attachments ride + * one message; where they go afterwards is the host's business (see sendRequest). */ + supportsMessageAttachments: boolean + /** The `+` menu's folder entries, backed by `attachedFiles`. A linked folder is a + * live handle on the user's disk, so only a host reading files in the browser has one. */ + supportsLinkedFolders: boolean + /** `accept` for the file picker, and the drop filter. A host whose consumer only + * understands some formats narrows it so the rest are refused rather than ignored. */ + attachmentAccept: string + /** Take non-image attachments verbatim (`blobs`) instead of decoding them to text. + * True where the bytes are forwarded somewhere — object storage — rather than read + * in the browser. */ + attachmentsAsBlobs: boolean tools: Tool[] autonomyMode: AIAutonomyMode setAutonomyMode: (mode: AIAutonomyMode) => void diff --git a/frontend/src/lib/components/copilot/chat/messageDraft.svelte.ts b/frontend/src/lib/components/copilot/chat/messageDraft.svelte.ts index 264a438d2b..b64711dccf 100644 --- a/frontend/src/lib/components/copilot/chat/messageDraft.svelte.ts +++ b/frontend/src/lib/components/copilot/chat/messageDraft.svelte.ts @@ -1,6 +1,6 @@ /** - * A message draft: the four lanes that ship together with one send — text, - * pastes, images, text files. Every place a draft accumulates or moves + * A message draft: the five lanes that ship together with one send — text, + * pastes, images, text files, blobs. Every place a draft accumulates or moves * (composer attach, queue append, dequeue restore, failure restore) goes * through this type, so the draft rules — file dedupe by source identity, * courtesy rename, attachment slot caps, all-lanes-move-together — live here @@ -10,6 +10,7 @@ * manager-wide state — enforced at the composer until it moves into the * store) and @context/DOM picks (ContextManager owns their lifecycle). */ +import { MAX_ATTACHED_BLOBS, type AttachedBlob } from './blobUtils' import { MAX_ATTACHED_IMAGES, type AttachedImage } from './imageUtils' import type { PasteAttachment } from './pasteTokens' import { @@ -19,12 +20,13 @@ import { type AttachedTextFile } from './textFileUtils' -/** A draft's four lanes as plain data — what moves between owners. */ +/** A draft's five lanes as plain data — what moves between owners. */ export interface DraftSnapshot { text: string pastes: PasteAttachment[] images: AttachedImage[] files: AttachedTextFile[] + blobs: AttachedBlob[] } export class MessageDraft { @@ -32,12 +34,14 @@ export class MessageDraft { pastes = $state([]) images = $state([]) files = $state([]) + blobs = $state([]) constructor(seed?: Partial) { if (seed?.text) this.text = seed.text if (seed?.pastes) this.pastes = [...seed.pastes] if (seed?.images) this.images = [...seed.images] if (seed?.files) this.files = [...seed.files] + if (seed?.blobs) this.blobs = [...seed.blobs] } get isEmpty(): boolean { @@ -45,12 +49,13 @@ export class MessageDraft { this.text.trim() === '' && this.pastes.length === 0 && this.images.length === 0 && - this.files.length === 0 + this.files.length === 0 && + this.blobs.length === 0 ) } get hasAttachments(): boolean { - return this.images.length > 0 || this.files.length > 0 + return this.images.length > 0 || this.files.length > 0 || this.blobs.length > 0 } /** Files joining a draft always fold (dedupe by source identity, courtesy @@ -83,6 +88,14 @@ export class MessageDraft { return dropped } + /** Blobs join up to the slot cap. Returns the dropped count (caller toasts). */ + addBlobs(blobs: AttachedBlob[]): number { + const merged = [...this.blobs, ...blobs] + const dropped = Math.max(0, merged.length - MAX_ATTACHED_BLOBS) + this.blobs = merged.slice(0, MAX_ATTACHED_BLOBS) + return dropped + } + /** * Merge a restored draft on top of this one (queued-message delete, restore * after a cancelled/errored turn): the restored draft was written FIRST, so @@ -91,10 +104,16 @@ export class MessageDraft { * Returns whether text merged onto a non-empty draft (the caller must then * keep both drafts' context), plus dropped counts for toasts. */ - prepend(restored: { text: string; images?: AttachedImage[]; files?: AttachedTextFile[] }): { + prepend(restored: { + text: string + images?: AttachedImage[] + files?: AttachedTextFile[] + blobs?: AttachedBlob[] + }): { mergedIntoDraft: boolean droppedImages: number droppedFiles: number + droppedBlobs: number } { const mergedIntoDraft = !!restored.text && !!this.text.trim() // An attachment-only restore has empty text; prepending would only add blank lines. @@ -115,7 +134,13 @@ export class MessageDraft { droppedFiles = Math.max(0, merged.length - MAX_ATTACHED_FILES) this.files = merged.slice(0, MAX_ATTACHED_FILES) } - return { mergedIntoDraft, droppedImages, droppedFiles } + let droppedBlobs = 0 + if (restored.blobs?.length) { + const merged = [...restored.blobs, ...this.blobs] + droppedBlobs = Math.max(0, merged.length - MAX_ATTACHED_BLOBS) + this.blobs = merged.slice(0, MAX_ATTACHED_BLOBS) + } + return { mergedIntoDraft, droppedImages, droppedFiles, droppedBlobs } } /** Replace the draft with a snapshot, but only when it is empty — an occupied @@ -132,16 +157,18 @@ export class MessageDraft { this.pastes = [...(snapshot.pastes ?? [])] this.images = [...(snapshot.images ?? [])] this.files = [...(snapshot.files ?? [])] + this.blobs = [...(snapshot.blobs ?? [])] } - /** Snapshot and clear atomically — the four lanes always move together, so no + /** Snapshot and clear atomically — the five lanes always move together, so no * call site can take one and forget another. */ take(): DraftSnapshot { const snapshot: DraftSnapshot = { text: this.text, pastes: this.pastes, images: this.images, - files: this.files + files: this.files, + blobs: this.blobs } this.clear() return snapshot @@ -152,5 +179,6 @@ export class MessageDraft { this.pastes = [] this.images = [] this.files = [] + this.blobs = [] } } diff --git a/frontend/src/lib/components/flows/content/FlowInput.svelte b/frontend/src/lib/components/flows/content/FlowInput.svelte index d780dede0a..1c7cf13fd7 100644 --- a/frontend/src/lib/components/flows/content/FlowInput.svelte +++ b/frontend/src/lib/components/flows/content/FlowInput.svelte @@ -760,6 +760,7 @@ hideSidebar={true} useStreaming={shouldUseStreaming} inputSchema={flowStore.val.schema} + flowModules={flowStore.val.value?.modules} />
{:else} diff --git a/frontend/src/lib/components/flows/conversations/AgentChatInputSubmenu.svelte b/frontend/src/lib/components/flows/conversations/AgentChatInputSubmenu.svelte new file mode 100644 index 0000000000..02e825a340 --- /dev/null +++ b/frontend/src/lib/components/flows/conversations/AgentChatInputSubmenu.svelte @@ -0,0 +1,111 @@ + + + + +{#if $subOpen} +
+
+ +
+
+{/if} diff --git a/frontend/src/lib/components/flows/conversations/FlowChat.svelte b/frontend/src/lib/components/flows/conversations/FlowChat.svelte index 127e33dbc4..ee7115cbef 100644 --- a/frontend/src/lib/components/flows/conversations/FlowChat.svelte +++ b/frontend/src/lib/components/flows/conversations/FlowChat.svelte @@ -5,6 +5,7 @@ import FlowChatInterface from './FlowChatInterface.svelte' import { getContext, untrack } from 'svelte' import type { FlowEditorContext } from '../types' + import type { FlowModule } from '$lib/gen' interface Props { onRunFlow: ( @@ -17,6 +18,8 @@ path: string hideSidebar?: boolean inputSchema?: Record + /** The flow's modules, used to find which inputs an AI agent step reads directly. */ + flowModules?: FlowModule[] /** Wider centered column, for the full-page chat. */ wideLayout?: boolean } @@ -28,6 +31,7 @@ path, hideSidebar = false, inputSchema = undefined, + flowModules = undefined, wideLayout = false }: Props = $props() @@ -56,17 +60,21 @@ } }) - // Derive additional inputs schema (excluding user_message) for chat mode + // Everything the chat asks for beyond the message itself. `user_message` is the + // composer: the server requires that exact argument on a chat-enabled flow and + // stores it as the conversation's message (execution.rs:644), so the name is a + // contract rather than the author's choice. const additionalInputsSchema = $derived.by(() => { const props = inputSchema?.properties ?? {} - const filtered = Object.fromEntries(Object.entries(props).filter(([k]) => k !== 'user_message')) + const messageInput = 'user_message' + const filtered = Object.fromEntries(Object.entries(props).filter(([k]) => k !== messageInput)) if (Object.keys(filtered).length === 0) return undefined const required = inputSchema?.required const requiredArray: string[] = Array.isArray(required) ? required : [] return { ...inputSchema, properties: filtered, - required: requiredArray.filter((k: string) => k !== 'user_message') + required: requiredArray.filter((k: string) => k !== messageInput) } }) @@ -79,6 +87,7 @@ {manager} {deploymentInProgress} {additionalInputsSchema} + {flowModules} {path} {wideLayout} /> diff --git a/frontend/src/lib/components/flows/conversations/FlowChatInterface.svelte b/frontend/src/lib/components/flows/conversations/FlowChatInterface.svelte index 8756353777..2c04351e59 100644 --- a/frontend/src/lib/components/flows/conversations/FlowChatInterface.svelte +++ b/frontend/src/lib/components/flows/conversations/FlowChatInterface.svelte @@ -1,6 +1,6 @@ - -{#if additionalInputsSchema} +{#if modalSchema} {#snippet actions()} @@ -126,25 +205,17 @@
{/snippet} -{#snippet inputPreface()} - {#if additionalInputsSchema} -
-
- - {#if hasMissingRequired} - - {/if} -
-
- {/if} +{#snippet footerSettings()} + {/snippet} + + {#if inputsMissingRequired} + + {/if} +
+ {/snippet} + {#snippet menu({ item, builders })} +
+ {#each inputs as input (input.name)} + onChange(input.name, value)} + {builders} + meltItem={item} + {workspace} + {helperScript} + /> + {/each} + + {#if staticModel && !modelInput} +
+
Model
+
{staticModel.model}
+
Set in the flow
+ {/if} + {#if onOpenInputs} +
+ + {/if} +
+ {/snippet} + diff --git a/frontend/src/lib/components/flows/conversations/agentChatInputs.ts b/frontend/src/lib/components/flows/conversations/agentChatInputs.ts new file mode 100644 index 0000000000..a5dd85d6dd --- /dev/null +++ b/frontend/src/lib/components/flows/conversations/agentChatInputs.ts @@ -0,0 +1,155 @@ +import type { FlowModule, InputTransform } from '$lib/gen' +import { getAllModules } from '../flowExplorer' +import { Bot, Hash, Paperclip, ScrollText, Thermometer } from 'lucide-svelte' +import type { ComponentType } from 'svelte' + +/** + * AI agent inputs the chat composer can drive, in footer display order. + * + * The composer never edits the flow: an agent field is reachable only when the author + * wired a flow input to it, so what the chip writes is a run input like any other. That + * also keeps it working on the deployed chat, where the reader has no write access, and + * on a step linked to an `ai_agent` resource, where every field but `user_message` / + * `user_attachments` comes from the resource and is not overridable at all. + * + * `max_iterations` is deliberately absent: it caps the agent's tool-use loop rather than + * a single generation, so it belongs with the flow's settings, not the model's. + */ +export const AGENT_CHAT_INPUT_KEYS = [ + 'user_attachments', + 'provider', + 'system_prompt', + 'temperature', + 'max_completion_tokens' +] as const + +export type AgentChatInputKey = (typeof AGENT_CHAT_INPUT_KEYS)[number] + +/** `user_attachments` is per-turn by definition; the rest are conversation settings. */ +export const PER_TURN_AGENT_CHAT_INPUT_KEY: AgentChatInputKey = 'user_attachments' + +export type AgentChatInput = { + /** Flow input property feeding the agent field. */ + name: string + key: AgentChatInputKey + /** The flow input's own schema entry — the chip renders it with the same editor the modal would. */ + property: Record + required: boolean +} + +export const AGENT_CHAT_INPUT_META: Record< + AgentChatInputKey, + { icon: ComponentType; label: string; summarize: (value: any) => string | undefined } +> = { + user_attachments: { + icon: Paperclip, + label: 'Attach', + summarize: (value) => { + const count = Array.isArray(value) ? value.length : value ? 1 : 0 + return count > 0 ? String(count) : undefined + } + }, + provider: { + icon: Bot, + label: 'Model', + summarize: (value) => (typeof value?.model === 'string' ? value.model : undefined) + }, + system_prompt: { + icon: ScrollText, + label: 'System prompt', + summarize: (value) => (typeof value === 'string' && value.trim() !== '' ? 'set' : undefined) + }, + temperature: { + icon: Thermometer, + label: 'Temperature', + summarize: (value) => (typeof value === 'number' ? String(value) : undefined) + }, + max_completion_tokens: { + icon: Hash, + label: 'Max tokens', + summarize: (value) => (typeof value === 'number' ? String(value) : undefined) + } +} + +const FLOW_INPUT_REF = /flow_input\??\.([A-Za-z_$][\w$]*)/g + +/** + * The flow input a transform is fed by, when exactly one feeds it. + * + * The expression need not be a bare pass-through — a step commonly reshapes what it + * reads, e.g. `(flow_input.files || []).map(f => ({ bucket: f.storage, key: f.s3 }))`. + * Writing that input is still right, because the expression consumes it. Two or more + * inputs are ambiguous: the composer would have no way to say which one it is editing. + */ +export function flowInputRef(transform: InputTransform | undefined): string | undefined { + if (transform?.type !== 'javascript') return undefined + const names = new Set( + [...transform.expr.matchAll(FLOW_INPUT_REF)].map((match) => match[1]) + ) + return names.size === 1 ? [...names][0] : undefined +} + +/** A provider value as the agent stores it. */ +export type AgentModel = { kind?: string; model?: string; reasoning_effort?: string } + +/** + * The model the flow already fixes, when it fixes exactly one: a single AI agent step + * whose `provider` is a static value. Named on the settings trigger the way the session + * chat names its own model, but not editable — no flow input feeds it, so there is + * nothing the composer could write. + */ +export function resolveStaticAgentModel( + modules: FlowModule[] | undefined +): AgentModel | undefined { + const agents = getAllModules(modules ?? []).filter((m) => m.value.type === 'aiagent') + if (agents.length !== 1) return undefined + const provider = (agents[0].value as any).input_transforms?.['provider'] + if (provider?.type !== 'static') return undefined + const value = provider.value + return typeof value?.model === 'string' + ? { kind: value.kind, model: value.model, reasoning_effort: value.reasoning_effort } + : undefined +} + +export function isEmptyAgentChatInputValue(value: any): boolean { + if (value === undefined || value === null || value === '') return true + return Array.isArray(value) && value.length === 0 +} + +/** + * The flow inputs that an AI agent step reads directly into one of its chat-relevant + * fields. Several agents may resolve to the same flow input; it is one chip either way, + * and one that stays unambiguous however many agents read it. + */ +export function resolveAgentChatInputs( + modules: FlowModule[] | undefined, + additionalInputsSchema: Record | undefined +): AgentChatInput[] { + const properties = additionalInputsSchema?.properties + if (!modules || !properties) return [] + const required: string[] = Array.isArray(additionalInputsSchema?.required) + ? additionalInputsSchema.required + : [] + + const keyOf = new Map() + for (const module of getAllModules(modules)) { + if (module.value.type !== 'aiagent') continue + const transforms = module.value.input_transforms ?? {} + for (const key of AGENT_CHAT_INPUT_KEYS) { + const name = flowInputRef(transforms[key]) + // A name the schema doesn't declare has no field to promote, and `user_message` + // is already the composer itself. + if (!name || !(name in properties) || keyOf.has(name)) continue + keyOf.set(name, key) + } + } + + return [...keyOf.entries()] + .map(([name, key]) => ({ + name, + key, + property: properties[name], + required: required.includes(name) + })) + .sort((a, b) => AGENT_CHAT_INPUT_KEYS.indexOf(a.key) - AGENT_CHAT_INPUT_KEYS.indexOf(b.key)) +} diff --git a/frontend/src/lib/components/flows/conversations/flowChatViewHost.svelte.ts b/frontend/src/lib/components/flows/conversations/flowChatViewHost.svelte.ts index 4a1ba24648..eb15b21b30 100644 --- a/frontend/src/lib/components/flows/conversations/flowChatViewHost.svelte.ts +++ b/frontend/src/lib/components/flows/conversations/flowChatViewHost.svelte.ts @@ -7,15 +7,47 @@ import type { ChatMessage, FlowChatManager } from './FlowChatManager.svelte' import { AIAutonomyMode } from '$lib/components/copilot/chat/AIChatManager.svelte' import { AttachedFilesStore } from '$lib/components/copilot/chat/files/attachedFiles.svelte' import { SessionArtifactsStore } from '$lib/components/copilot/chat/artifacts/artifactsState.svelte' +import { dataUrlToBlob } from '$lib/components/copilot/chat/blobUtils' +import { HelpersService } from '$lib/gen' +import { sendUserToast } from '$lib/toast' +import { randomUUID } from '$lib/utils/uuid' +import { MessageInputsStore } from './messageInputContext.svelte' + +/** What an AI agent step reads out of `user_attachments`. */ +type S3Attachment = { s3: string; filename?: string } + +/** The flow input the composer's attachments feed, and whether it holds a list. */ +export type AttachmentsTarget = { name: string; multiple: boolean } + +export type FlowChatViewHostOptions = { + additionalInputs?: () => Record | undefined + /** Called once the turn is dispatched, to clear anything that rides one message. */ + onSent?: () => void + attachmentsTarget?: () => AttachmentsTarget | undefined + workspace?: () => string | undefined + /** Off when the workspace has no object storage — there is nowhere to upload to. */ + canAttach?: () => boolean +} function toDisplayMessage( message: ChatMessage, userIndex: number, - showStepNames: boolean + showStepNames: boolean, + inputs: MessageInputsStore ): DisplayMessage { switch (message.message_type) { - case 'user': - return { role: 'user', index: userIndex, content: message.content } + case 'user': { + // What the turn ran with, read back from its job — the message row itself + // keeps only the text. Renders through the same lanes the copilot uses. + const { images, contextElements } = inputs.get(message.job_id) + return { + role: 'user', + index: userIndex, + content: message.content, + images: images.length > 0 ? images : undefined, + contextElements: contextElements.length > 0 ? contextElements : undefined + } + } case 'tool': // Both producers of a tool row — the agent executor and the frontend's own // stream parser — write the whole message as a one-line description of the @@ -46,14 +78,11 @@ function toDisplayMessage( */ export class FlowChatViewHost implements ChatViewHost { #manager: FlowChatManager - #additionalInputs: () => Record | undefined + #options: FlowChatViewHostOptions - constructor( - manager: FlowChatManager, - additionalInputs: () => Record | undefined = () => undefined - ) { + constructor(manager: FlowChatManager, options: FlowChatViewHostOptions = {}) { this.#manager = manager - this.#additionalInputs = additionalInputs + this.#options = options } // The step name says which AI agent step wrote a message, so it only tells the @@ -64,16 +93,25 @@ export class FlowChatViewHost implements ChatViewHost { () => new Set(this.#manager.messages.map((m) => m.step_name).filter(Boolean)).size > 1 ) + #messageInputs = new MessageInputsStore(() => this.#options.workspace?.()) + displayMessages = $derived.by(() => { let userIndex = 0 const showStepNames = this.#showStepNames return this.#manager.messages.map((message) => - toDisplayMessage(message, message.message_type === 'user' ? userIndex++ : -1, showStepNames) + toDisplayMessage( + message, + message.message_type === 'user' ? userIndex++ : -1, + showStepNames, + this.#messageInputs + ) ) }) messages: readonly unknown[] = [] contextTokens = 0 - loading = $derived.by(() => this.#manager.isLoading || this.#manager.isWaitingForResponse) + loading = $derived.by( + () => this.#manager.isLoading || this.#manager.isWaitingForResponse || this.#uploading + ) loadingLabel = undefined compacting = false currentReply = '' @@ -97,13 +135,70 @@ export class FlowChatViewHost implements ChatViewHost { get sendInFlight() { return this.#manager.isLoading } + // True while attachments are uploading — before the flow job exists, so + // `manager.isLoading` cannot cover it and the composer would look idle. + #uploading = $state(false) + sendRequest = async (options: ChatSendRequestOptions = {}) => { const text = options.instructions?.trim() + // The conversation's message is the flow's `user_message`, which the manager + // requires; an attachment-only turn has nothing to send. if (!text) return false + + const args = { ...(this.#options.additionalInputs?.() ?? {}) } + const target = this.#options.attachmentsTarget?.() + const attachments = [...(options.images ?? []), ...(options.blobs ?? [])] + if (target && attachments.length > 0) { + this.#uploading = true + try { + const uploaded = await this.#uploadAttachments(attachments) + args[target.name] = target.multiple ? uploaded : uploaded[0] + } catch (e) { + sendUserToast( + `Could not upload the attachments: ${e instanceof Error ? e.message : String(e)}`, + true + ) + return false + } finally { + this.#uploading = false + } + } + this.#manager.inputMessage = text - await this.#manager.sendMessage(this.#additionalInputs()) + this.#options.onSent?.() + await this.#manager.sendMessage( + Object.keys(args).length > 0 || this.#options.additionalInputs?.() ? args : undefined + ) return true } + + /** + * Put each attachment in the workspace's object storage and hand back what the + * agent reads. The flow runs on a worker, so the bytes have to exist somewhere + * the worker can fetch — unlike the copilot, which sends them from the browser. + */ + async #uploadAttachments( + attachments: { name?: string; dataUrl: string; mediaType?: string }[] + ): Promise { + const workspace = this.#options.workspace?.() + if (!workspace) throw new Error('no workspace') + // One prefix per turn keeps a re-attached filename from overwriting the copy an + // earlier message still refers to. + const prefix = `windmill_chat_uploads/${randomUUID()}` + return Promise.all( + attachments.map(async (attachment, index) => { + const filename = attachment.name ?? `attachment-${index + 1}` + const blob = dataUrlToBlob(attachment.dataUrl, attachment.mediaType) + const { file_key } = await HelpersService.fileUpload({ + workspace, + fileKey: `${prefix}/${filename}`, + contentType: blob.type, + requestBody: blob + }) + return { s3: file_key, filename } + }) + ) + } cancel = () => { void this.#manager.cancelCurrentJob() } @@ -144,6 +239,15 @@ export class FlowChatViewHost implements ChatViewHost { isSessionChat = false supportsModelSettings = false supportsMessageEditing = false + // Attachments go to object storage for the worker to read, so a linked folder — + // a live handle on the user's own disk — has no meaning here. + get supportsMessageAttachments() { + return (this.#options.canAttach?.() ?? false) && !!this.#options.attachmentsTarget?.() + } + supportsLinkedFolders = false + attachmentsAsBlobs = true + // What an AI agent step accepts (AI_AGENT_SCHEMA.user_attachments). + attachmentAccept = 'image/*,application/pdf,.pdf' tools = [] autonomyMode = AIAutonomyMode.DEFAULT setAutonomyMode = () => {} diff --git a/frontend/src/lib/components/flows/conversations/messageInputContext.svelte.ts b/frontend/src/lib/components/flows/conversations/messageInputContext.svelte.ts new file mode 100644 index 0000000000..ad25db31ec --- /dev/null +++ b/frontend/src/lib/components/flows/conversations/messageInputContext.svelte.ts @@ -0,0 +1,136 @@ +/** + * The inputs a chat message ran with, recovered from its job. + * + * A conversation message stores only its text, so what the user attached and which + * settings the turn used exist nowhere but the run's arguments. The user row carries + * the flow job id, whose args are the raw run arguments — `user_message` plus every + * other flow input. + * + * Fetched lazily and kept in memory only: a purged job leaves a dangling id and the + * turn simply shows no inputs, which is honest — the arguments are gone. + */ +import { JobService } from '$lib/gen' +import { base } from '$lib/base' +import { + createAttachedFileContextElement, + type ContextElement +} from '$lib/components/copilot/chat/context' +import type { AttachedImage } from '$lib/components/copilot/chat/imageUtils' + +type S3Ref = { s3: string; filename?: string; storage?: string } + +const IMAGE_EXTENSIONS = ['.png', '.jpg', '.jpeg', '.gif', '.webp', '.bmp', '.svg', '.avif'] + +function isS3Ref(value: any): value is S3Ref { + return !!value && typeof value === 'object' && typeof value.s3 === 'string' && value.s3 !== '' +} + +function s3Refs(value: any): S3Ref[] { + if (isS3Ref(value)) return [value] + if (Array.isArray(value)) return value.filter(isS3Ref) + return [] +} + +function displayName(ref: S3Ref): string { + return ref.filename ?? ref.s3.split('/').pop() ?? ref.s3 +} + +function looksLikeImage(ref: S3Ref): boolean { + const name = displayName(ref).toLowerCase() + return IMAGE_EXTENSIONS.some((ext) => name.endsWith(ext)) +} + +/** Same-origin, cookie-authed GET — usable directly as an , no blob fetch. */ +function downloadUrl(workspace: string, ref: S3Ref): string { + const params = new URLSearchParams({ file_key: ref.s3 }) + if (ref.storage) params.set('storage', ref.storage) + return `${base}/api/w/${workspace}/job_helpers/download_s3_file?${params.toString()}` +} + +/** A scalar input, summarised for a chip. Objects are left to the file/JSON branches. */ +function scalarSummary(value: any): string | undefined { + if (value === undefined || value === null || value === '') return undefined + if (typeof value === 'string') return value + if (typeof value === 'number' || typeof value === 'boolean') return String(value) + return undefined +} + +export type MessageInputs = { images: AttachedImage[]; contextElements: ContextElement[] } + +const EMPTY: MessageInputs = { images: [], contextElements: [] } + +/** + * Split a turn's run arguments into the lanes a user message renders: image + * thumbnails, and a chip per remaining input. `user_message` is the bubble itself. + */ +export function argsToMessageInputs( + workspace: string, + args: Record | undefined +): MessageInputs { + if (!args) return EMPTY + const images: AttachedImage[] = [] + const contextElements: ContextElement[] = [] + for (const [name, value] of Object.entries(args)) { + if (name === 'user_message') continue + const files = s3Refs(value) + if (files.length > 0) { + for (const ref of files) { + if (looksLikeImage(ref)) { + images.push({ + dataUrl: downloadUrl(workspace, ref), + mediaType: 'image/png', + name: displayName(ref) + }) + } else { + contextElements.push( + createAttachedFileContextElement(displayName(ref), `Attached file · ${ref.s3}`) + ) + } + } + continue + } + const summary = scalarSummary(value) + if (summary !== undefined) { + contextElements.push(createAttachedFileContextElement(name, summary)) + } + } + return images.length > 0 || contextElements.length > 0 ? { images, contextElements } : EMPTY +} + +/** + * Per-conversation cache of run arguments by job id. One fetch per turn, kept only + * for as long as the chat is mounted. + */ +export class MessageInputsStore { + #workspace: () => string | undefined + #byJob = $state>({}) + #inFlight = new Set() + + constructor(workspace: () => string | undefined) { + this.#workspace = workspace + } + + /** What the turn ran with, fetching on first ask. Empty until the args land. */ + get(jobId: string | null | undefined): MessageInputs { + if (!jobId) return EMPTY + const cached = this.#byJob[jobId] + if (cached) return cached + void this.#load(jobId) + return EMPTY + } + + async #load(jobId: string) { + const workspace = this.#workspace() + if (!workspace || this.#inFlight.has(jobId)) return + this.#inFlight.add(jobId) + try { + const args = await JobService.getJobArgs({ workspace, id: jobId }) + this.#byJob = { ...this.#byJob, [jobId]: argsToMessageInputs(workspace, args as any) } + } catch { + // A purged job, or one this user cannot read: the turn shows no inputs. + this.#byJob = { ...this.#byJob, [jobId]: EMPTY } + } finally { + this.#inFlight.delete(jobId) + } + } +} diff --git a/frontend/src/routes/(root)/(logged)/flows/get/[...path]/+page.svelte b/frontend/src/routes/(root)/(logged)/flows/get/[...path]/+page.svelte index b363363a8b..3152461df6 100644 --- a/frontend/src/routes/(root)/(logged)/flows/get/[...path]/+page.svelte +++ b/frontend/src/routes/(root)/(logged)/flows/get/[...path]/+page.svelte @@ -705,6 +705,7 @@ path={flow?.path ?? ''} useStreaming={shouldUseStreaming} inputSchema={flow?.schema} + flowModules={flow?.value?.modules} wideLayout /> {:else}