wip: flow chat composer settings + message context

This commit is contained in:
Guilhem Lemouel
2026-09-01 15:38:22 +02:00
parent c8350480df
commit 56b8ce1bd0
20 changed files with 1160 additions and 187 deletions
+1 -1
View File
@@ -115,7 +115,7 @@ strip = "none"
[features]
default = []
private = ["windmill-api/private", "windmill-api-agent-workers?/private", "windmill-autoscaling/private", "windmill-common/private", "windmill-dep-map/private", "windmill-object-store/private", "windmill-git-sync/private", "windmill-indexer/private", "windmill-operator?/private", "windmill-queue/private", "windmill-worker/private", "windmill-test-utils/private"]
private = ["windmill-api/private", "windmill-api-agent-workers?/private", "windmill-autoscaling/private", "windmill-common/private", "windmill-dep-map/private", "windmill-object-store/private", "windmill-git-sync/private", "windmill-indexer?/private", "windmill-operator?/private", "windmill-queue/private", "windmill-worker/private", "windmill-test-utils/private"]
agent_worker_server = ["windmill-api/agent_worker_server", "dep:windmill-api-agent-workers", "windmill-test-utils/agent_worker_server"]
enterprise = ["windmill-worker/enterprise", "windmill-queue/enterprise", "windmill-api/enterprise", "windmill-api-agent-workers?/enterprise", "dep:windmill-autoscaling", "windmill-autoscaling/enterprise", "windmill-git-sync/enterprise", "windmill-common/prometheus", "windmill-common/enterprise", "windmill-object-store/enterprise", "license"]
local_reports = ["windmill-common/local_reports"]
+6 -1
View File
@@ -632,6 +632,7 @@ pub async fn handle_chat_conversation_messages(
flow_path: &str,
run_query: &RunJobQuery,
user_message_raw: Option<&Box<serde_json::value::RawValue>>,
job_id: Uuid,
) -> error::Result<()> {
let memory_id = run_query.memory_id.ok_or_else(|| {
windmill_common::error::Error::BadRequest(
@@ -662,10 +663,13 @@ pub async fn handle_chat_conversation_messages(
)
.await?;
// The run this message started. Its args are the only record of what the message
// carried besides its text — attachments and every other flow input — and nothing
// written later points at them: an assistant row holds the AI agent step's job.
add_message_to_conversation_tx(
tx,
memory_id,
None,
Some(job_id),
&user_message,
MessageType::User,
None,
@@ -790,6 +794,7 @@ pub async fn run_flow<'c>(
&flow_path.to_string(),
&run_query,
args.args.get("user_message"),
uuid,
)
.await?;
}
+1
View File
@@ -9257,6 +9257,7 @@ async fn run_preview_flow_job(
&flow_path,
&run_query,
user_message.as_ref(),
uuid,
)
.await?;
}
+2 -42
View File
@@ -1755,7 +1755,6 @@
"cpu": [
"arm64"
],
"dev": true,
"license": "MIT",
"optional": true,
"os": [
@@ -1772,7 +1771,6 @@
"cpu": [
"arm64"
],
"dev": true,
"license": "MIT",
"optional": true,
"os": [
@@ -1789,7 +1787,6 @@
"cpu": [
"x64"
],
"dev": true,
"license": "MIT",
"optional": true,
"os": [
@@ -1806,7 +1803,6 @@
"cpu": [
"x64"
],
"dev": true,
"license": "MIT",
"optional": true,
"os": [
@@ -1823,7 +1819,6 @@
"cpu": [
"arm"
],
"dev": true,
"license": "MIT",
"optional": true,
"os": [
@@ -1840,7 +1835,6 @@
"cpu": [
"arm64"
],
"dev": true,
"license": "MIT",
"optional": true,
"os": [
@@ -1857,7 +1851,6 @@
"cpu": [
"arm64"
],
"dev": true,
"license": "MIT",
"optional": true,
"os": [
@@ -1874,7 +1867,6 @@
"cpu": [
"ppc64"
],
"dev": true,
"license": "MIT",
"optional": true,
"os": [
@@ -1891,7 +1883,6 @@
"cpu": [
"s390x"
],
"dev": true,
"license": "MIT",
"optional": true,
"os": [
@@ -1908,7 +1899,6 @@
"cpu": [
"x64"
],
"dev": true,
"license": "MIT",
"optional": true,
"os": [
@@ -1925,7 +1915,6 @@
"cpu": [
"x64"
],
"dev": true,
"license": "MIT",
"optional": true,
"os": [
@@ -1942,7 +1931,6 @@
"cpu": [
"arm64"
],
"dev": true,
"license": "MIT",
"optional": true,
"os": [
@@ -1959,7 +1947,6 @@
"cpu": [
"arm64"
],
"dev": true,
"license": "MIT",
"optional": true,
"os": [
@@ -1976,7 +1963,6 @@
"cpu": [
"x64"
],
"dev": true,
"license": "MIT",
"optional": true,
"os": [
@@ -7583,7 +7569,7 @@
"version": "1.21.7",
"resolved": "https://registry.npmjs.org/jiti/-/jiti-1.21.7.tgz",
"integrity": "sha512-/imKNG4EbWNrVjoNC/1H5/9GFy+tqjGBHCaSsN+P2RnPqjsLmv6UD3Ej+Kj8nBWaRAwyk7kK5ZUc+OEatnTR3A==",
"dev": true,
"devOptional": true,
"license": "MIT",
"bin": {
"jiti": "bin/jiti.js"
@@ -8279,7 +8265,6 @@
"cpu": [
"arm64"
],
"dev": true,
"license": "MPL-2.0",
"optional": true,
"os": [
@@ -8300,7 +8285,6 @@
"cpu": [
"arm64"
],
"dev": true,
"license": "MPL-2.0",
"optional": true,
"os": [
@@ -8321,7 +8305,6 @@
"cpu": [
"x64"
],
"dev": true,
"license": "MPL-2.0",
"optional": true,
"os": [
@@ -8342,7 +8325,6 @@
"cpu": [
"x64"
],
"dev": true,
"license": "MPL-2.0",
"optional": true,
"os": [
@@ -8363,7 +8345,6 @@
"cpu": [
"arm"
],
"dev": true,
"license": "MPL-2.0",
"optional": true,
"os": [
@@ -8384,7 +8365,6 @@
"cpu": [
"arm64"
],
"dev": true,
"license": "MPL-2.0",
"optional": true,
"os": [
@@ -8405,7 +8385,6 @@
"cpu": [
"arm64"
],
"dev": true,
"license": "MPL-2.0",
"optional": true,
"os": [
@@ -8426,7 +8405,6 @@
"cpu": [
"x64"
],
"dev": true,
"license": "MPL-2.0",
"optional": true,
"os": [
@@ -8447,7 +8425,6 @@
"cpu": [
"x64"
],
"dev": true,
"license": "MPL-2.0",
"optional": true,
"os": [
@@ -8468,7 +8445,6 @@
"cpu": [
"arm64"
],
"dev": true,
"license": "MPL-2.0",
"optional": true,
"os": [
@@ -8489,7 +8465,6 @@
"cpu": [
"x64"
],
"dev": true,
"license": "MPL-2.0",
"optional": true,
"os": [
@@ -13195,21 +13170,6 @@
}
}
},
"node_modules/svelte-check/node_modules/picomatch": {
"version": "4.0.7",
"resolved": "https://registry.npmjs.org/picomatch/-/picomatch-4.0.7.tgz",
"integrity": "sha512-qcJu88Q2IWqJsDD529JKMdwGm/dvInW4HvQnRwiH9JtihJvzGOscDtHE3x1pBKeUOTysQ8kVmLnJ2kJu7yhcGA==",
"dev": true,
"license": "MIT",
"optional": true,
"peer": true,
"engines": {
"node": ">=12"
},
"funding": {
"url": "https://github.com/sponsors/jonschlinkert"
}
},
"node_modules/svelte-eslint-parser": {
"version": "0.43.0",
"resolved": "https://registry.npmjs.org/svelte-eslint-parser/-/svelte-eslint-parser-0.43.0.tgz",
@@ -13989,7 +13949,7 @@
"version": "5.9.3",
"resolved": "https://registry.npmjs.org/typescript/-/typescript-5.9.3.tgz",
"integrity": "sha512-jl1vZzPDinLr9eUt3J/t7V6FgNEw9QjvBPdysz9KfQDD41fQrC2Y4vKQdiaUpFT4bXlb1RHhLpp8wtm6M5TgSw==",
"dev": true,
"devOptional": true,
"license": "Apache-2.0",
"bin": {
"tsc": "bin/tsc",
@@ -484,6 +484,7 @@
hideSidebar={true}
path={$pathStore}
inputSchema={flowStore.val.schema}
flowModules={flowStore.val.value?.modules}
/>
</div>
{:else}
@@ -50,6 +50,7 @@
import type { SelectedContext } from './app/core'
import { type FileToAttach } from './files/attachedFiles.svelte'
import { isImageFile } from './imageUtils'
import { matchesAccept } from './blobUtils'
import {
hasFileSystemAccess,
pickDirectory,
@@ -157,6 +158,8 @@
wideLayout = false,
emptyHint,
inputPreface,
footerControls,
footerSettings,
initialInstructions = undefined,
onDraftChange = undefined,
placeholder = undefined,
@@ -188,6 +191,11 @@
wideLayout?: boolean
emptyHint?: Snippet
inputPreface?: Snippet
/** Extra controls at the head of the composer's footer row. */
footerControls?: Snippet
/** The settings control at the footer's right edge, where the copilot puts its
* model picker. A host that configures its turn elsewhere replaces it here. */
footerSettings?: Snippet
// Seed / observe the main composer's draft text (see AIChatInput).
initialInstructions?: string
onDraftChange?: (text: string) => void
@@ -319,12 +327,10 @@
chatHost.mode === AIMode.SCRIPT || chatHost.mode === AIMode.FLOW || chatHost.mode === AIMode.APP
)
// File attachment is GLOBAL-mode only.
const canAttachFiles = $derived(chatHost.mode === AIMode.GLOBAL && !disabled)
// Steers the OS file picker toward text + image formats (soft hint; both attach
// to the message — text files after a content sniff).
const TEXT_FILE_ACCEPT =
'image/*,text/*,.txt,.csv,.tsv,.json,.jsonl,.ndjson,.md,.markdown,.log,.yaml,.yml,.toml,.ini,.cfg,.conf,.env,.xml,.html,.htm,.css,.js,.mjs,.cjs,.ts,.tsx,.jsx,.py,.rb,.rs,.go,.java,.kt,.c,.h,.cpp,.cc,.cs,.php,.sh,.bash,.zsh,.sql,.svelte,.vue,.dockerfile'
const canAttachFiles = $derived(chatHost.supportsMessageAttachments && !disabled)
// Folders are linked as session-wide assets, which only a host that reads files in
// the browser can do — a host running the turn server-side takes attachments only.
const canLinkFolders = $derived(chatHost.supportsLinkedFolders && !disabled)
let fileInputEl = $state<HTMLInputElement | null>(null)
let folderInputEl = $state<HTMLInputElement | null>(null)
let dragDepth = $state(0)
@@ -444,12 +450,16 @@
handles.length === 0
? flatFiles
: await Promise.all(handles.filter(isFileHandle).map((h) => h.getFile()))
// Loose text files attach to the message, like images.
const textFiles = looseFiles.filter((f) => !isImageFile(f))
if (textFiles.length > 0) await aiChatInput?.addTextFiles(textFiles)
// Loose files attach to the message, like images.
await attachNonImageFiles(looseFiles.filter((f) => !isImageFile(f)))
// Folders link as a live handle.
for (const h of handles.filter(isDirectoryHandle)) {
await addDirHandle(h)
const dirs = handles.filter(isDirectoryHandle)
if (dirs.length > 0 && !canLinkFolders) {
sendUserToast('Folders cannot be attached in this chat — drop individual files.', true)
} else {
for (const h of dirs) {
await addDirHandle(h)
}
}
} else {
// Fallback (no File System Access API): snapshot dropped files AND folders by walking
@@ -474,25 +484,53 @@
topLevelText.push(file)
}
}
if (folderEntries.length > 0) await handleAddFiles(folderEntries)
if (topLevelText.length > 0) await aiChatInput?.addTextFiles(topLevelText)
if (folderEntries.length > 0) {
if (canLinkFolders) await handleAddFiles(folderEntries)
else sendUserToast('Folders cannot be attached in this chat — drop individual files.', true)
}
await attachNonImageFiles(topLevelText)
}
}
async function onFileInputChange(e: Event) {
const input = e.currentTarget as HTMLInputElement
if (input.files && input.files.length > 0) {
const picked = Array.from(input.files)
const imageFiles = picked.filter(isImageFile)
const textFiles = picked.filter((f) => !isImageFile(f))
// Reserved before the text work is awaited — see onPanelDrop.
const imageWork = imageFiles.length > 0 ? aiChatInput?.addImages(imageFiles) : undefined
if (textFiles.length > 0) await aiChatInput?.addTextFiles(textFiles)
await imageWork
await attachPickedFiles(Array.from(input.files))
}
input.value = '' // allow re-selecting the same file
}
/**
* Route non-image files to the lane the host reads. A host that decodes them
* takes text; one that forwards them verbatim (to object storage, say) takes
* blobs, and its narrower `accept` is re-applied here because a drop bypasses
* the picker's own filtering.
*/
async function attachNonImageFiles(files: File[]) {
if (files.length === 0) return
if (chatHost.attachmentsAsBlobs) {
const allowed = files.filter((f) => matchesAccept(f, chatHost.attachmentAccept))
if (allowed.length < files.length) {
sendUserToast(
`${files.length - allowed.length} file(s) skipped — this chat accepts ${chatHost.attachmentAccept}.`,
true
)
}
await aiChatInput?.addBlobs(allowed)
} else {
await aiChatInput?.addTextFiles(files)
}
}
async function attachPickedFiles(picked: File[]) {
const imageFiles = picked.filter(isImageFile)
const others = picked.filter((f) => !isImageFile(f))
// Reserved before the other work is awaited — see onPanelDrop.
const imageWork = imageFiles.length > 0 ? aiChatInput?.addImages(imageFiles) : undefined
await attachNonImageFiles(others)
await imageWork
}
function onFolderInputChange(e: Event) {
const input = e.currentTarget as HTMLInputElement
// webkitdirectory files carry webkitRelativePath (`folder/sub/file`); addFiles groups
@@ -562,9 +600,14 @@
const showFlowPendingActionControls = $derived(
(chatHost.flowAiChatHelpers?.hasPendingChanges() ?? false) && !chatHost.autoAcceptEditsActive
)
// Everything the left group can hold. `canAttachFiles` belongs here too: in GLOBAL
// mode the `+` always has the context picker or the autonomy selector beside it, but
// a host with attachments and nothing else would lose the group and the `+` with it.
const showFooterLeftControls = $derived(
!disabled &&
(showContextPicker ||
(footerControls !== undefined ||
canAttachFiles ||
showContextPicker ||
showAutonomyModeSelector ||
(chatHost.mode === AIMode.SCRIPT && hasDiff))
)
@@ -825,6 +868,7 @@ the panel, or the Escape-to-stop focus check would wrongly reject them. -->
>
{#if showFooterLeftControls}
<div class="flex flex-row items-center gap-x-1.5 min-w-0 flex-wrap">
{@render footerControls?.()}
{#if showContextPicker && !disabled}
<Popover placement="bottom-start">
{#snippet trigger()}
@@ -886,19 +930,23 @@ the panel, or the Escape-to-stop focus check would wrongly reject them. -->
linkFiles()
}
},
{
// A real (live) link needs the File System Access API; without it the
// folder is only snapshotted, so call it "Add folder", not "Link folder".
displayName: canUseFsAccess ? 'Link folder' : 'Add folder',
icon: Folder,
tooltip: canUseFsAccess
? 'Linked live — the assistant reads the folders current files from disk and refreshes each turn.'
: 'Loaded as a snapshot — the folders files are copied into your browser (they wont auto-update). For a live link that refreshes from disk, use a Chromium-based browser (Chrome, Edge).',
action: () => {
plusMenuOpen = false
linkFolder()
}
},
...(canLinkFolders
? [
{
// A real (live) link needs the File System Access API; without it the
// folder is only snapshotted, so call it "Add folder", not "Link folder".
displayName: canUseFsAccess ? 'Link folder' : 'Add folder',
icon: Folder,
tooltip: canUseFsAccess
? 'Linked live — the assistant reads the folders current files from disk and refreshes each turn.'
: 'Loaded as a snapshot — the folders files are copied into your browser (they wont auto-update). For a live link that refreshes from disk, use a Chromium-based browser (Chrome, Edge).',
action: () => {
plusMenuOpen = false
linkFolder()
}
}
]
: []),
...(chatHost.mode === AIMode.GLOBAL && mcpConnections
? [
{
@@ -946,7 +994,7 @@ the panel, or the Escape-to-stop focus check would wrongly reject them. -->
bind:this={fileInputEl}
type="file"
multiple
accept={TEXT_FILE_ACCEPT}
accept={chatHost.attachmentAccept}
class="hidden no-default-style"
onchange={onFileInputChange}
/>
@@ -1050,6 +1098,7 @@ the panel, or the Escape-to-stop focus check would wrongly reject them. -->
{#if chatHost.supportsModelSettings}
<AIChatModelSettings />
{/if}
{@render footerSettings?.()}
{#if chatHost.mode === AIMode.GLOBAL}
<McpConnections bind:this={mcpConnections} />
{/if}
@@ -41,6 +41,12 @@
textByteLength,
type AttachedTextFile
} from './textFileUtils'
import {
fileToAttachedBlob,
MAX_ATTACHED_BLOBS,
MAX_BLOB_BYTES,
type AttachedBlob
} from './blobUtils'
import { MessageDraft } from './messageDraft.svelte'
import ExpandableImage, {
isImageViewerOpen
@@ -208,19 +214,24 @@
// against a concurrent drop.
let pendingImages = $state(0)
/** Attach dropped/pasted image files (downscaled + bounded). GLOBAL mode only. */
/** Attach dropped/pasted image files (downscaled + bounded). */
export async function addImages(files: (File | Blob)[]) {
if (chatHost.mode !== AIMode.GLOBAL) return
if (!chatHost.supportsMessageAttachments) return
const imageFiles = files.filter(isImageFile)
if (imageFiles.length === 0) return
// tryGetCurrentModel returns undefined instead of throwing: this runs from a
// drop/paste handler that can't surface a rejection.
const model = tryGetCurrentModel()
// Only known text-only models fail this, so attaching would certainly 400 the
// next turn — refuse rather than warn and send it anyway.
if (model && !modelSupportsVision(model.provider, model.model)) {
sendUserToast(`${model.model} can't read images. Switch to a vision model first.`, true)
return
// The vision check is about the model this composer's own turn will hit, so it
// only applies to a host that picks that model. Elsewhere the model is chosen
// in the flow and tryGetCurrentModel would answer for the wrong one.
if (chatHost.supportsModelSettings) {
// tryGetCurrentModel returns undefined instead of throwing: this runs from a
// drop/paste handler that can't surface a rejection.
const model = tryGetCurrentModel()
// Only known text-only models fail this, so attaching would certainly 400 the
// next turn — refuse rather than warn and send it anyway.
if (model && !modelSupportsVision(model.provider, model.model)) {
sendUserToast(`${model.model} can't read images. Switch to a vision model first.`, true)
return
}
}
// Count decodes already in flight: two drops that both read the image count
// before either resolves would each claim the same free slots and overshoot
@@ -309,7 +320,7 @@
/** Attach dropped/picked text files (sniffed + bounded). GLOBAL mode only. */
export async function addTextFiles(candidates: File[]) {
if (chatHost.mode !== AIMode.GLOBAL) return
if (!chatHost.supportsMessageAttachments) return
if (candidates.length === 0) return
const remaining = MAX_ATTACHED_FILES - draft.files.length - pendingFiles
if (remaining <= 0) {
@@ -405,6 +416,53 @@
draft.files = draft.files.filter((_, i) => i !== index)
}
// Blobs being read right now — same send-hold/slot-reservation role as pendingImages.
let pendingBlobs = $state(0)
/** Attach files the host takes verbatim (a PDF, say). Kept out of addTextFiles:
* that one decodes to a string and drops anything the binary sniff rejects. */
export async function addBlobs(candidates: File[]) {
if (!chatHost.supportsMessageAttachments) return
if (candidates.length === 0) return
const oversized = candidates.filter((f) => f.size > MAX_BLOB_BYTES)
if (oversized.length > 0) {
const mb = Math.round(MAX_BLOB_BYTES / 1_000_000)
sendUserToast(`${oversized.length} file(s) over ${mb}MB were skipped.`, true)
}
const usable = candidates.filter((f) => f.size <= MAX_BLOB_BYTES)
if (usable.length === 0) return
const remaining = MAX_ATTACHED_BLOBS - draft.blobs.length - pendingBlobs
if (remaining <= 0) {
sendUserToast(`You can attach up to ${MAX_ATTACHED_BLOBS} files.`, true)
return
}
const batch = usable.slice(0, remaining)
if (batch.length < usable.length) {
sendUserToast(
`You can attach up to ${MAX_ATTACHED_BLOBS} files; ${usable.length - batch.length} were skipped.`,
true
)
}
pendingBlobs += batch.length
try {
const added: AttachedBlob[] = []
for (const file of batch) {
try {
added.push(await fileToAttachedBlob(file))
} catch (e) {
sendUserToast(`Could not read ${file.name}`, true)
}
}
if (added.length > 0) draft.addBlobs(added)
} finally {
pendingBlobs -= batch.length
}
}
function removeBlob(index: number) {
draft.blobs = draft.blobs.filter((_, i) => i !== index)
}
// App mode @ mention state
let showAppContextTooltip = $state(false)
let appContextTooltipWord = $state('')
@@ -486,7 +544,7 @@
// Attachments still decoding/reading (or mid-drop-routing) count as
// occupancy too — they belong to a draft the user started even though
// their lane is still empty.
if (pendingImages > 0 || pendingFiles > 0 || ingestionHolds > 0) return false
if (pendingImages > 0 || pendingFiles > 0 || pendingBlobs > 0 || ingestionHolds > 0) return false
if (
!draft.replaceIfEmpty({
text: value,
@@ -691,7 +749,7 @@
function sendRequest() {
// The send button is disabled while decoding, but Enter reaches here directly.
// Sending now would drop the in-flight attachments onto the following message.
if (pendingImages > 0 || pendingFiles > 0 || ingestionHolds > 0) {
if (pendingImages > 0 || pendingFiles > 0 || pendingBlobs > 0 || ingestionHolds > 0) {
return
}
// Read before `take()` empties the draft the id derives from, and only take
@@ -752,7 +810,8 @@
instructions: sent.text,
pastes: sent.pastes,
images: sent.images,
files: sent.files
files: sent.files,
blobs: sent.blobs
})
// clearForSend() pre-zaps the textarea's mention-sync so the wipe
// doesn't drop `selectedContext` before `AIChatManager.beforeSend`
@@ -987,6 +1046,7 @@
disabled ||
pendingImages > 0 ||
pendingFiles > 0 ||
pendingBlobs > 0 ||
ingestionHolds > 0 ||
(emptyDraft &&
(onSendRequest !== undefined ||
@@ -1015,7 +1075,7 @@
thumbnails get their own row (different height). -->
{#snippet badgeRow()}
{@const contextChips = showContext ? selectedContext : domSelectorChips}
{#if contextChips.length > 0 || draft.files.length > 0 || pendingFiles > 0}
{#if contextChips.length > 0 || draft.files.length > 0 || pendingFiles > 0 || draft.blobs.length > 0 || pendingBlobs > 0}
<div class="flex flex-row flex-wrap items-center gap-1 px-2.5 pt-2">
{#each contextChips as element (contextKey(element))}
<ContextElementBadge
@@ -1034,7 +1094,19 @@
onDelete={() => removeFile(i)}
/>
{/each}
{#each { length: pendingFiles } as _, i (i)}
<!-- Blobs are shown by the same badge as text files. Their preview line stands
in for content the badge cannot render (a PDF has no text to show). -->
{#each draft.blobs as blob, i (i)}
<ContextElementBadge
contextElement={createAttachedFileContextElement(
blob.name,
`${blob.mediaType} · ${Math.max(1, Math.round(blob.size / 1024))} KB`
)}
deletable
onDelete={() => removeBlob(i)}
/>
{/each}
{#each { length: pendingFiles + pendingBlobs } as _, i (i)}
<div
class="h-6 w-24 rounded-md border bg-surface flex items-center justify-center"
title="Reading file..."
@@ -1099,6 +1171,7 @@
draft.isEmpty &&
pendingImages === 0 &&
pendingFiles === 0 &&
pendingBlobs === 0 &&
ingestionHolds === 0
) {
// Shell-style recall: ArrowUp in the empty main composer pulls the
@@ -1135,10 +1208,10 @@
bind:this={contextTextareaComponent}
bind:value={draft.text}
bind:pastes={draft.pastes}
onImageFiles={chatHost.mode === AIMode.GLOBAL
onImageFiles={chatHost.supportsMessageAttachments
? (pasted) => void addImages(pasted)
: undefined}
onTextFiles={chatHost.mode === AIMode.GLOBAL
onTextFiles={chatHost.supportsMessageAttachments
? (pasted) => void addTextFiles(pasted)
: undefined}
{availableContext}
@@ -1236,30 +1309,46 @@
</Portal>
{/if}
{:else}
<div class={twMerge('relative w-full scroll-pb-2 pt-2', className)}>
<textarea
bind:this={instructionsTextareaComponent}
bind:value={draft.text}
use:autosize={{ maxHeight: '40vh' }}
onkeydown={(e) => {
if (onKeyDown) {
onKeyDown(e)
}
if (e.key === 'Enter' && !e.shiftKey) {
e.preventDefault()
sendRequest()
}
}}
rows={1}
placeholder={modePlaceholder}
class={twMerge('resize-none', CHAT_INPUT_PADDING)}
{disabled}
></textarea>
{#if !bottomRightSnippet}
<div class="absolute bottom-1 right-1">
{@render sendStopButton()}
</div>
{/if}
<!-- Mirrors ContextTextarea's box: border + rounded on the wrapper, not on the
textarea, so the same chip rows sit INSIDE the box above the text. Without
this the rows would float on the page background and read as a different
composer from the session chat's. -->
<div
class="w-full scroll-pb-2 bg-surface-input rounded-md border border-border-light focus-within:border-border-selected transition-colors"
>
{@render badgeRow()}
{@render imageChipsRow()}
<div class={twMerge('relative w-full', className)}>
<textarea
bind:this={instructionsTextareaComponent}
bind:value={draft.text}
use:autosize={{ maxHeight: '40vh' }}
onkeydown={(e) => {
if (onKeyDown) {
onKeyDown(e)
}
if (e.key === 'Enter' && !e.shiftKey) {
e.preventDefault()
sendRequest()
}
}}
rows={1}
placeholder={modePlaceholder}
class={twMerge(
'resize-none',
// The box lives on the wrapper; kill the textarea's own forms border,
// focus ring and background so only the wrapper reads as the field.
'!border-transparent !bg-transparent !shadow-none focus:!border-transparent focus:!ring-0',
CHAT_INPUT_PADDING
)}
{disabled}
></textarea>
{#if !bottomRightSnippet}
<div class="absolute bottom-1 right-1">
{@render sendStopButton()}
</div>
{/if}
</div>
</div>
{/if}
{#if bottomRightSnippet}
@@ -411,6 +411,20 @@ export class AIChatManager implements ChatViewHost {
// affordances apply here. See ChatViewHost for hosts where they don't.
supportsModelSettings = true
supportsMessageEditing = true
// Attachments and linked folders are GLOBAL-mode affordances. Declared as
// getters because `mode` changes under a mounted composer.
get supportsMessageAttachments() {
return this.mode === AIMode.GLOBAL
}
get supportsLinkedFolders() {
return this.mode === AIMode.GLOBAL
}
// Steers the OS file picker toward text + image formats (soft hint; both attach
// to the message — text files after a content sniff).
// The copilot reads attachments in the browser, so non-image files decode to text.
attachmentsAsBlobs = false
attachmentAccept =
'image/*,text/*,.txt,.csv,.tsv,.json,.jsonl,.ndjson,.md,.markdown,.log,.yaml,.yml,.toml,.ini,.cfg,.conf,.env,.xml,.html,.htm,.css,.js,.mjs,.cjs,.ts,.tsx,.jsx,.py,.rb,.rs,.go,.java,.kt,.c,.h,.cpp,.cc,.cs,.php,.sh,.bash,.zsh,.sql,.svelte,.vue,.dockerfile'
/** Files the user attached to the current GLOBAL-mode conversation. */
attachedFiles = new AttachedFilesStore()
/** Markdown artifacts the copilot created for the current session. */
@@ -0,0 +1,79 @@
/**
* Message-scoped attachments that are neither an image nor readable text a PDF
* being the case that matters. They ride the composer next to images and files,
* as chips cleared on send, and reach the host through `ChatSendRequestOptions`.
*
* The bytes are kept verbatim, unlike an image (which normalises to a bounded
* PNG/JPEG for the model) and unlike a text file (which is decoded to a string):
* a host that forwards these to object storage has to upload what the user
* picked, not a re-encoding of it.
*/
/** Blobs one message may carry — the same slot cap images and text files use. */
export const MAX_ATTACHED_BLOBS = 8
/**
* Per-blob byte cap. The data URL sits in composer state until send, so this
* bounds what one message can hold in memory; a host uploading elsewhere pays
* the same bytes again on the wire.
*/
export const MAX_BLOB_BYTES = 20_000_000
export type AttachedBlob = {
name: string
/** The file's own media type, verbatim — the upload's Content-Type depends on it. */
mediaType: string
/** `data:<mediaType>;base64,<...>` of the original bytes. */
dataUrl: string
size: number
}
/**
* Whether a file satisfies an `accept` list the same list the OS picker gets, applied
* again on drop, where the browser enforces nothing.
*/
export function matchesAccept(file: File, accept: string): boolean {
const patterns = accept
.split(',')
.map((p) => p.trim().toLowerCase())
.filter(Boolean)
if (patterns.length === 0) return true
const type = file.type.toLowerCase()
const name = file.name.toLowerCase()
return patterns.some((pattern) => {
if (pattern.startsWith('.')) return name.endsWith(pattern)
if (pattern.endsWith('/*')) return type.startsWith(pattern.slice(0, -1))
return type === pattern
})
}
export async function fileToAttachedBlob(file: File): Promise<AttachedBlob> {
const dataUrl = await new Promise<string>((resolve, reject) => {
const reader = new FileReader()
reader.onload = () => resolve(String(reader.result))
reader.onerror = () => reject(reader.error ?? new Error(`Could not read ${file.name}`))
reader.readAsDataURL(file)
})
return {
name: file.name,
mediaType: file.type || 'application/octet-stream',
dataUrl,
size: file.size
}
}
/** The bytes behind a `data:` URL, for a host that has to re-upload them. */
export function dataUrlToBlob(dataUrl: string, fallbackType = 'application/octet-stream'): Blob {
const comma = dataUrl.indexOf(',')
const header = dataUrl.slice(5, comma)
const isBase64 = header.endsWith(';base64')
const mediaType = (isBase64 ? header.slice(0, -';base64'.length) : header) || fallbackType
const payload = dataUrl.slice(comma + 1)
if (!isBase64) {
return new Blob([decodeURIComponent(payload)], { type: mediaType })
}
const binary = atob(payload)
const bytes = new Uint8Array(binary.length)
for (let i = 0; i < binary.length; i++) bytes[i] = binary.charCodeAt(i)
return new Blob([bytes], { type: mediaType })
}
@@ -3,6 +3,7 @@ import type { AIMode, AIAutonomyMode } from './AIChatManager.svelte'
import { getAiChatManager } from './aiChatManagerContext'
import type { DisplayMessage, Tool } from './shared'
import type { ContextElement } from './context'
import type { AttachedBlob } from './blobUtils'
import type { AttachedImage } from './imageUtils'
import type { AttachedTextFile } from './textFileUtils'
import type { PasteAttachment } from './pasteTokens'
@@ -18,6 +19,7 @@ export type ChatSendRequestOptions = {
pastes?: PasteAttachment[]
images?: AttachedImage[]
files?: AttachedTextFile[]
blobs?: AttachedBlob[]
}
/**
@@ -92,6 +94,19 @@ export interface ChatViewHost {
/** Click a user message to edit and resend it. Needs a host that can rewind
* its own transcript, which a host replaying a server-side run cannot. */
supportsMessageEditing: boolean
/** The `+` menu's file entry and drag-and-drop onto the panel. Attachments ride
* one message; where they go afterwards is the host's business (see sendRequest). */
supportsMessageAttachments: boolean
/** The `+` menu's folder entries, backed by `attachedFiles`. A linked folder is a
* live handle on the user's disk, so only a host reading files in the browser has one. */
supportsLinkedFolders: boolean
/** `accept` for the file picker, and the drop filter. A host whose consumer only
* understands some formats narrows it so the rest are refused rather than ignored. */
attachmentAccept: string
/** Take non-image attachments verbatim (`blobs`) instead of decoding them to text.
* True where the bytes are forwarded somewhere object storage rather than read
* in the browser. */
attachmentsAsBlobs: boolean
tools: Tool<any>[]
autonomyMode: AIAutonomyMode
setAutonomyMode: (mode: AIAutonomyMode) => void
@@ -1,6 +1,6 @@
/**
* A message draft: the four lanes that ship together with one send text,
* pastes, images, text files. Every place a draft accumulates or moves
* A message draft: the five lanes that ship together with one send text,
* pastes, images, text files, blobs. Every place a draft accumulates or moves
* (composer attach, queue append, dequeue restore, failure restore) goes
* through this type, so the draft rules file dedupe by source identity,
* courtesy rename, attachment slot caps, all-lanes-move-together live here
@@ -10,6 +10,7 @@
* manager-wide state enforced at the composer until it moves into the
* store) and @context/DOM picks (ContextManager owns their lifecycle).
*/
import { MAX_ATTACHED_BLOBS, type AttachedBlob } from './blobUtils'
import { MAX_ATTACHED_IMAGES, type AttachedImage } from './imageUtils'
import type { PasteAttachment } from './pasteTokens'
import {
@@ -19,12 +20,13 @@ import {
type AttachedTextFile
} from './textFileUtils'
/** A draft's four lanes as plain data — what moves between owners. */
/** A draft's five lanes as plain data — what moves between owners. */
export interface DraftSnapshot {
text: string
pastes: PasteAttachment[]
images: AttachedImage[]
files: AttachedTextFile[]
blobs: AttachedBlob[]
}
export class MessageDraft {
@@ -32,12 +34,14 @@ export class MessageDraft {
pastes = $state<PasteAttachment[]>([])
images = $state<AttachedImage[]>([])
files = $state<AttachedTextFile[]>([])
blobs = $state<AttachedBlob[]>([])
constructor(seed?: Partial<DraftSnapshot>) {
if (seed?.text) this.text = seed.text
if (seed?.pastes) this.pastes = [...seed.pastes]
if (seed?.images) this.images = [...seed.images]
if (seed?.files) this.files = [...seed.files]
if (seed?.blobs) this.blobs = [...seed.blobs]
}
get isEmpty(): boolean {
@@ -45,12 +49,13 @@ export class MessageDraft {
this.text.trim() === '' &&
this.pastes.length === 0 &&
this.images.length === 0 &&
this.files.length === 0
this.files.length === 0 &&
this.blobs.length === 0
)
}
get hasAttachments(): boolean {
return this.images.length > 0 || this.files.length > 0
return this.images.length > 0 || this.files.length > 0 || this.blobs.length > 0
}
/** Files joining a draft always fold (dedupe by source identity, courtesy
@@ -83,6 +88,14 @@ export class MessageDraft {
return dropped
}
/** Blobs join up to the slot cap. Returns the dropped count (caller toasts). */
addBlobs(blobs: AttachedBlob[]): number {
const merged = [...this.blobs, ...blobs]
const dropped = Math.max(0, merged.length - MAX_ATTACHED_BLOBS)
this.blobs = merged.slice(0, MAX_ATTACHED_BLOBS)
return dropped
}
/**
* Merge a restored draft on top of this one (queued-message delete, restore
* after a cancelled/errored turn): the restored draft was written FIRST, so
@@ -91,10 +104,16 @@ export class MessageDraft {
* Returns whether text merged onto a non-empty draft (the caller must then
* keep both drafts' context), plus dropped counts for toasts.
*/
prepend(restored: { text: string; images?: AttachedImage[]; files?: AttachedTextFile[] }): {
prepend(restored: {
text: string
images?: AttachedImage[]
files?: AttachedTextFile[]
blobs?: AttachedBlob[]
}): {
mergedIntoDraft: boolean
droppedImages: number
droppedFiles: number
droppedBlobs: number
} {
const mergedIntoDraft = !!restored.text && !!this.text.trim()
// An attachment-only restore has empty text; prepending would only add blank lines.
@@ -115,7 +134,13 @@ export class MessageDraft {
droppedFiles = Math.max(0, merged.length - MAX_ATTACHED_FILES)
this.files = merged.slice(0, MAX_ATTACHED_FILES)
}
return { mergedIntoDraft, droppedImages, droppedFiles }
let droppedBlobs = 0
if (restored.blobs?.length) {
const merged = [...restored.blobs, ...this.blobs]
droppedBlobs = Math.max(0, merged.length - MAX_ATTACHED_BLOBS)
this.blobs = merged.slice(0, MAX_ATTACHED_BLOBS)
}
return { mergedIntoDraft, droppedImages, droppedFiles, droppedBlobs }
}
/** Replace the draft with a snapshot, but only when it is empty an occupied
@@ -132,16 +157,18 @@ export class MessageDraft {
this.pastes = [...(snapshot.pastes ?? [])]
this.images = [...(snapshot.images ?? [])]
this.files = [...(snapshot.files ?? [])]
this.blobs = [...(snapshot.blobs ?? [])]
}
/** Snapshot and clear atomically the four lanes always move together, so no
/** Snapshot and clear atomically the five lanes always move together, so no
* call site can take one and forget another. */
take(): DraftSnapshot {
const snapshot: DraftSnapshot = {
text: this.text,
pastes: this.pastes,
images: this.images,
files: this.files
files: this.files,
blobs: this.blobs
}
this.clear()
return snapshot
@@ -152,5 +179,6 @@ export class MessageDraft {
this.pastes = []
this.images = []
this.files = []
this.blobs = []
}
}
@@ -760,6 +760,7 @@
hideSidebar={true}
useStreaming={shouldUseStreaming}
inputSchema={flowStore.val.schema}
flowModules={flowStore.val.value?.modules}
/>
</div>
{:else}
@@ -0,0 +1,111 @@
<script lang="ts">
import { untrack } from 'svelte'
import { melt } from '@melt-ui/svelte'
import { twMerge } from 'tailwind-merge'
import { ChevronRight } from 'lucide-svelte'
import type { MenubarMenuElements, createDropdownMenu } from '@melt-ui/svelte'
import SchemaForm from '$lib/components/SchemaForm.svelte'
import { deepEqual } from 'fast-equals'
import { type DynamicInput } from '$lib/utils'
import { AGENT_CHAT_INPUT_META, type AgentChatInput } from './agentChatInputs'
interface Props {
input: AgentChatInput
value: any
onChange: (value: any) => void
builders: ReturnType<typeof createDropdownMenu>['builders']
meltItem: MenubarMenuElements['item']
workspace?: string
helperScript?: DynamicInput.HelperScript
}
let { input, value, onChange, builders, workspace, helperScript }: Props = $props()
const {
elements: { subTrigger, subMenu },
states: { subOpen }
} = untrack(() => builders).createSubmenu()
const meta = $derived(AGENT_CHAT_INPUT_META[input.key])
const summary = $derived(meta.summarize(value))
// A one-property schema, so the submenu holds the exact editor the Configure-inputs
// modal would render for this input.
const fieldSchema = $derived({
$schema: 'https://json-schema.org/draft/2020-12/schema',
type: 'object',
properties: { [input.name]: input.property },
required: input.required ? [input.name] : [],
order: [input.name]
})
// SchemaForm writes into `args` in place, so it cannot drive a function binding.
// Both directions are kept in step: out on an edit, and back in when the owner
// resets the value from outside.
let synced = $state.snapshot(value)
let args = $state<Record<string, any>>({ [input.name]: synced })
$effect(() => {
const edited = $state.snapshot(args[input.name])
if (!deepEqual(edited, synced)) {
synced = edited
onChange(edited)
}
})
$effect(() => {
const incoming = $state.snapshot(value)
if (
!deepEqual(
incoming,
untrack(() => synced)
)
) {
synced = incoming
args = { [input.name]: incoming }
}
})
// Melt's roving focus blurs the focused element on pointermove, which would abort a
// native drag or steal focus mid-typing. Direct listeners so they run before melt's.
function isolatePointer(node: HTMLElement) {
const stop = (e: Event) => e.stopPropagation()
node.addEventListener('pointerdown', stop)
node.addEventListener('pointermove', stop)
node.addEventListener('keydown', stop)
return {
destroy() {
node.removeEventListener('pointerdown', stop)
node.removeEventListener('pointermove', stop)
node.removeEventListener('keydown', stop)
}
}
}
</script>
<button
use:melt={$subTrigger}
class={twMerge(
'px-4 py-2 text-primary font-normal hover:bg-surface-hover cursor-pointer text-xs transition-colors w-full',
'data-[highlighted]:bg-surface-hover',
'flex flex-row gap-2 items-center rounded-sm'
)}
>
<meta.icon size={14} class="shrink-0" />
<p class="truncate grow min-w-0 whitespace-nowrap text-left">{meta.label}</p>
{#if summary}
<span class="shrink-0 text-tertiary truncate max-w-[80px]">{summary}</span>
{/if}
<ChevronRight size={14} class="ml-auto shrink-0 text-tertiary" />
</button>
{#if $subOpen}
<div
use:melt={$subMenu}
class="z-[6000] bg-surface-tertiary dark:border w-72 origin-top-right rounded-lg shadow-lg focus:outline-none p-3"
>
<div use:isolatePointer>
<SchemaForm schema={fieldSchema} bind:args {helperScript} {workspace} />
</div>
</div>
{/if}
@@ -5,6 +5,7 @@
import FlowChatInterface from './FlowChatInterface.svelte'
import { getContext, untrack } from 'svelte'
import type { FlowEditorContext } from '../types'
import type { FlowModule } from '$lib/gen'
interface Props {
onRunFlow: (
@@ -17,6 +18,8 @@
path: string
hideSidebar?: boolean
inputSchema?: Record<string, any>
/** The flow's modules, used to find which inputs an AI agent step reads directly. */
flowModules?: FlowModule[]
/** Wider centered column, for the full-page chat. */
wideLayout?: boolean
}
@@ -28,6 +31,7 @@
path,
hideSidebar = false,
inputSchema = undefined,
flowModules = undefined,
wideLayout = false
}: Props = $props()
@@ -56,17 +60,21 @@
}
})
// Derive additional inputs schema (excluding user_message) for chat mode
// Everything the chat asks for beyond the message itself. `user_message` is the
// composer: the server requires that exact argument on a chat-enabled flow and
// stores it as the conversation's message (execution.rs:644), so the name is a
// contract rather than the author's choice.
const additionalInputsSchema = $derived.by(() => {
const props = inputSchema?.properties ?? {}
const filtered = Object.fromEntries(Object.entries(props).filter(([k]) => k !== 'user_message'))
const messageInput = 'user_message'
const filtered = Object.fromEntries(Object.entries(props).filter(([k]) => k !== messageInput))
if (Object.keys(filtered).length === 0) return undefined
const required = inputSchema?.required
const requiredArray: string[] = Array.isArray(required) ? required : []
return {
...inputSchema,
properties: filtered,
required: requiredArray.filter((k: string) => k !== 'user_message')
required: requiredArray.filter((k: string) => k !== messageInput)
}
})
</script>
@@ -79,6 +87,7 @@
{manager}
{deploymentInProgress}
{additionalInputsSchema}
{flowModules}
{path}
{wideLayout}
/>
@@ -1,6 +1,6 @@
<script lang="ts">
import { Button } from '$lib/components/common'
import { Loader2, MessageCircle, Settings2 } from 'lucide-svelte'
import { Loader2, MessageCircle } from 'lucide-svelte'
import { FlowChatManager } from './FlowChatManager.svelte'
import { FlowChatViewHost } from './flowChatViewHost.svelte'
import AIChatDisplay from '$lib/components/copilot/chat/AIChatDisplay.svelte'
@@ -8,11 +8,23 @@
import Modal from '$lib/components/common/modal/Modal.svelte'
import SchemaForm from '$lib/components/SchemaForm.svelte'
import { type DynamicInput } from '$lib/utils'
import { CancelError, WorkspaceService, type FlowModule } from '$lib/gen'
import { workspaceStore } from '$lib/stores'
import { resource } from 'runed'
import FlowChatSettings from './FlowChatSettings.svelte'
import {
isEmptyAgentChatInputValue,
PER_TURN_AGENT_CHAT_INPUT_KEY,
resolveAgentChatInputs,
resolveStaticAgentModel
} from './agentChatInputs'
interface Props {
manager: FlowChatManager
deploymentInProgress?: boolean
additionalInputsSchema?: Record<string, any>
/** The flow's modules, used to find which inputs an AI agent step reads directly. */
flowModules?: FlowModule[]
path: string
wideLayout?: boolean
}
@@ -21,6 +33,7 @@
manager,
deploymentInProgress = false,
additionalInputsSchema,
flowModules,
path,
wideLayout = false
}: Props = $props()
@@ -35,14 +48,75 @@
return undefined
})
// Inputs an AI agent step reads straight out of the flow input get a composer chip
// instead of a modal field; the rest stay in the modal.
const agentChatInputs = $derived(resolveAgentChatInputs(flowModules, additionalInputsSchema))
// The composer's attachments feed this input; it never appears as a chip or a
// modal field, because the paperclip is its editor.
const attachmentsInput = $derived(
agentChatInputs.find((input) => input.key === PER_TURN_AGENT_CHAT_INPUT_KEY)
)
const attachmentsTarget = $derived(
attachmentsInput
? { name: attachmentsInput.name, multiple: attachmentsInput.property?.type === 'array' }
: undefined
)
const chatWorkspace = $derived(manager.operatingWorkspace?.() ?? $workspaceStore)
// Uploading needs the workspace's object storage; without one the `+` stays hidden
// rather than failing on drop. Assumed absent until this workspace's answer lands.
const settings = resource(
() => chatWorkspace,
async (ws, _previous, { onCleanup }) => {
if (!ws) return undefined
const req = WorkspaceService.getPublicSettings({ workspace: ws })
onCleanup(() => req.cancel())
try {
return { ws, settings: await req }
} catch (err) {
if (!(err instanceof CancelError)) {
console.error('Failed to fetch workspace settings:', err)
}
return undefined
}
}
)
const s3StorageConfigured = $derived.by(() => {
const loaded = settings.current
if (!loaded || loaded.ws !== chatWorkspace) return false
return loaded.settings.large_file_storage?.s3_resource_path !== undefined
})
const settingInputs = $derived(
agentChatInputs.filter((input) => input.key !== PER_TURN_AGENT_CHAT_INPUT_KEY)
)
const staticModel = $derived(resolveStaticAgentModel(flowModules))
const modalSchema = $derived.by(() => {
if (!additionalInputsSchema) return undefined
const promoted = new Set(agentChatInputs.map((input) => input.name))
const properties = Object.fromEntries(
Object.entries(additionalInputsSchema.properties ?? {}).filter(([key]) => !promoted.has(key))
)
if (Object.keys(properties).length === 0) return undefined
const required: string[] = Array.isArray(additionalInputsSchema.required)
? additionalInputsSchema.required
: []
return {
...additionalInputsSchema,
properties,
required: required.filter((key) => !promoted.has(key))
}
})
// LocalStorage helpers
const STORAGE_KEY_PREFIX = 'windmill_flow_chat_inputs_'
// State for additional inputs modal
let showInputsModal = $state(false)
let additionalInputsValues = $state<Record<string, any> | undefined>(
loadInputsFromStorage() ?? undefined
)
// Conversation settings, persisted per flow. Attachments are absent by construction:
// they ride the composer's own draft and are cleared with it on send.
let inputValues = $state<Record<string, any>>(loadInputsFromStorage() ?? {})
let modalDraft = $state<Record<string, any>>({})
function getStorageKey(): string {
return `${STORAGE_KEY_PREFIX}${path}`
@@ -66,20 +140,28 @@
}
}
function setInputValue(name: string, value: any) {
inputValues = { ...inputValues, [name]: value }
saveInputsToStorage(inputValues)
}
function handleModalConfirm() {
saveInputsToStorage(additionalInputsValues ?? {})
inputValues = { ...inputValues, ...modalDraft }
saveInputsToStorage(inputValues)
showInputsModal = false
}
function openInputsModal() {
const stored = loadInputsFromStorage()
if (stored) additionalInputsValues = stored
modalDraft = { ...(loadInputsFromStorage() ?? inputValues) }
showInputsModal = true
}
const chatHost = new FlowChatViewHost(manager, () =>
additionalInputsSchema ? (loadInputsFromStorage() ?? additionalInputsValues) : undefined
)
const chatHost = new FlowChatViewHost(manager, {
additionalInputs: () => (additionalInputsSchema ? { ...inputValues } : undefined),
attachmentsTarget: () => attachmentsTarget,
workspace: () => chatWorkspace,
canAttach: () => s3StorageConfigured
})
setChatViewHost(chatHost)
// A message typed mid-run is held by the host; send it once the run settles.
@@ -89,24 +171,21 @@
}
})
const hasMissingRequired = $derived.by(() => {
if (!additionalInputsSchema?.required?.length) return false
const values = additionalInputsValues ?? {}
return additionalInputsSchema.required.some(
(field: string) =>
values[field] === undefined || values[field] === '' || values[field] === null
const modalMissingRequired = $derived.by(() => {
if (!modalSchema?.required?.length) return false
return modalSchema.required.some((field: string) =>
isEmptyAgentChatInputValue(inputValues[field])
)
})
</script>
<!-- Additional Inputs Modal -->
{#if additionalInputsSchema}
{#if modalSchema}
<Modal title="Configure inputs" bind:open={showInputsModal}>
<SchemaForm
schema={additionalInputsSchema}
bind:args={additionalInputsValues}
schema={modalSchema}
bind:args={modalDraft}
helperScript={dynamicInputHelperScript}
workspace={manager.operatingWorkspace?.()}
workspace={chatWorkspace}
/>
{#snippet actions()}
<Button onClick={handleModalConfirm} variant="accent">Save</Button>
@@ -126,25 +205,17 @@
</div>
{/snippet}
{#snippet inputPreface()}
{#if additionalInputsSchema}
<div class="flex items-center justify-end w-full mb-1">
<div class="relative">
<Button
unifiedSize="xs"
variant="default"
startIcon={{ icon: Settings2 }}
title="Inputs"
onClick={openInputsModal}
>
Inputs
</Button>
{#if hasMissingRequired}
<span class="absolute -top-1 -right-1 w-2 h-2 bg-yellow-500 rounded-full"></span>
{/if}
</div>
</div>
{/if}
{#snippet footerSettings()}
<FlowChatSettings
inputs={settingInputs}
values={inputValues}
onChange={setInputValue}
{staticModel}
onOpenInputs={modalSchema ? openInputsModal : undefined}
inputsMissingRequired={modalMissingRequired}
workspace={chatWorkspace}
helperScript={dynamicInputHelperScript}
/>
{/snippet}
<!-- The transcript scroller fills its flex row, which needs a height to resolve
@@ -167,7 +238,9 @@
hideModeSelector
{wideLayout}
{emptyHint}
{inputPreface}
footerSettings={settingInputs.length > 0 || modalSchema || staticModel
? footerSettings
: undefined}
placeholder="Send a message to run the flow"
disabled={deploymentInProgress}
disabledMessage="Deployment in progress"
@@ -0,0 +1,142 @@
<script lang="ts">
import { ChevronDown, Settings2, SlidersHorizontal } from 'lucide-svelte'
import DropdownV2 from '$lib/components/DropdownV2.svelte'
import Button from '$lib/components/common/button/Button.svelte'
import { type DynamicInput } from '$lib/utils'
import AgentChatInputSubmenu from './AgentChatInputSubmenu.svelte'
import { type AgentChatInput, type AgentModel } from './agentChatInputs'
interface Props {
/** Wired agent fields, one submenu each. */
inputs: AgentChatInput[]
values: Record<string, any>
onChange: (name: string, value: any) => void
/** The model the flow fixes, when it fixes one. Named on the trigger, not editable. */
staticModel?: AgentModel
/** Opens the Configure-inputs modal, when the flow has inputs no agent field reads. */
onOpenInputs?: () => void
inputsMissingRequired?: boolean
workspace?: string
helperScript?: DynamicInput.HelperScript
}
let {
inputs,
values,
onChange,
staticModel,
onOpenInputs,
inputsMissingRequired = false,
workspace,
helperScript
}: Props = $props()
let menuOpen = $state(false)
// The chosen model where the flow exposes one, else the model it fixes. With neither —
// several agents, or a model computed per run — there is no single one to name, so the
// trigger says what it is instead: settings.
const modelInput = $derived(inputs.find((input) => input.key === 'provider'))
const model = $derived.by((): AgentModel | undefined => {
const chosen = modelInput ? values[modelInput.name] : undefined
return typeof chosen?.model === 'string' ? chosen : staticModel
})
// The trigger resizes when a value changes while the menu is open, which would shift a
// bottom-end popover anchored to its right edge. Freeze the width for as long as it is open.
let triggerEl: HTMLElement | undefined = $state(undefined)
let lockedWidth = $state<number | undefined>(undefined)
$effect(() => {
if (menuOpen) {
if (lockedWidth === undefined && triggerEl) {
lockedWidth = triggerEl.getBoundingClientRect().width
}
} else {
lockedWidth = undefined
}
})
</script>
<DropdownV2
customMenu
placement="bottom-end"
fixedHeight={false}
closeOnItemClick={false}
bind:open={menuOpen}
>
{#snippet buttonReplacement()}
<div
bind:this={triggerEl}
class="relative"
style={lockedWidth !== undefined ? `width: ${lockedWidth}px` : undefined}
>
<!-- With a single model the trigger reads exactly as the copilot's does:
the model, then its reasoning effort. Otherwise there is no one model to
name and it falls back to what it opens. -->
<Button
nonCaptureEvent
unifiedSize="2xs"
variant="subtle"
startIcon={model ? undefined : { icon: Settings2 }}
endIcon={{ icon: ChevronDown }}
btnClasses="w-full max-w-[200px] text-secondary font-normal"
title={model ? 'Model & agent settings' : 'Agent settings'}
>
{#if model}
<span class="flex items-center gap-1 min-w-0">
<span class="truncate">{model.model}</span>
{#if model.reasoning_effort}
<span class="shrink-0 text-tertiary">· {model.reasoning_effort}</span>
{/if}
</span>
{:else}
<span class="truncate">Settings</span>
{/if}
</Button>
{#if inputsMissingRequired}
<span class="absolute -top-0.5 -right-0.5 w-2 h-2 bg-yellow-500 rounded-full"></span>
{/if}
</div>
{/snippet}
{#snippet menu({ item, builders })}
<div
class="bg-surface-tertiary dark:border w-64 origin-top-right rounded-lg shadow-lg focus:outline-none py-1 text-xs"
>
{#each inputs as input (input.name)}
<AgentChatInputSubmenu
{input}
value={values[input.name]}
onChange={(value) => onChange(input.name, value)}
{builders}
meltItem={item}
{workspace}
{helperScript}
/>
{/each}
<!-- A model the flow fixes is shown but not offered: no flow input feeds it, so
changing it here would mean editing the flow. -->
{#if staticModel && !modelInput}
<div class="my-1 border-t border-border-light"></div>
<div class="px-3 pt-1.5 pb-1 text-2xs uppercase tracking-wide text-secondary">Model</div>
<div class="px-3 pb-1 text-primary truncate">{staticModel.model}</div>
<div class="px-3 pb-1.5 text-2xs text-tertiary">Set in the flow</div>
{/if}
{#if onOpenInputs}
<div class="my-1 border-t border-border-light"></div>
<button
class="px-4 py-2 text-primary font-normal hover:bg-surface-hover cursor-pointer text-xs transition-colors w-full flex flex-row gap-2 items-center rounded-sm"
onclick={() => {
menuOpen = false
onOpenInputs?.()
}}
>
<SlidersHorizontal size={14} class="shrink-0" />
<p class="truncate grow min-w-0 text-left">Other inputs</p>
{#if inputsMissingRequired}
<span class="w-2 h-2 bg-yellow-500 rounded-full shrink-0"></span>
{/if}
</button>
{/if}
</div>
{/snippet}
</DropdownV2>
@@ -0,0 +1,155 @@
import type { FlowModule, InputTransform } from '$lib/gen'
import { getAllModules } from '../flowExplorer'
import { Bot, Hash, Paperclip, ScrollText, Thermometer } from 'lucide-svelte'
import type { ComponentType } from 'svelte'
/**
* AI agent inputs the chat composer can drive, in footer display order.
*
* The composer never edits the flow: an agent field is reachable only when the author
* wired a flow input to it, so what the chip writes is a run input like any other. That
* also keeps it working on the deployed chat, where the reader has no write access, and
* on a step linked to an `ai_agent` resource, where every field but `user_message` /
* `user_attachments` comes from the resource and is not overridable at all.
*
* `max_iterations` is deliberately absent: it caps the agent's tool-use loop rather than
* a single generation, so it belongs with the flow's settings, not the model's.
*/
export const AGENT_CHAT_INPUT_KEYS = [
'user_attachments',
'provider',
'system_prompt',
'temperature',
'max_completion_tokens'
] as const
export type AgentChatInputKey = (typeof AGENT_CHAT_INPUT_KEYS)[number]
/** `user_attachments` is per-turn by definition; the rest are conversation settings. */
export const PER_TURN_AGENT_CHAT_INPUT_KEY: AgentChatInputKey = 'user_attachments'
export type AgentChatInput = {
/** Flow input property feeding the agent field. */
name: string
key: AgentChatInputKey
/** The flow input's own schema entry — the chip renders it with the same editor the modal would. */
property: Record<string, any>
required: boolean
}
export const AGENT_CHAT_INPUT_META: Record<
AgentChatInputKey,
{ icon: ComponentType; label: string; summarize: (value: any) => string | undefined }
> = {
user_attachments: {
icon: Paperclip,
label: 'Attach',
summarize: (value) => {
const count = Array.isArray(value) ? value.length : value ? 1 : 0
return count > 0 ? String(count) : undefined
}
},
provider: {
icon: Bot,
label: 'Model',
summarize: (value) => (typeof value?.model === 'string' ? value.model : undefined)
},
system_prompt: {
icon: ScrollText,
label: 'System prompt',
summarize: (value) => (typeof value === 'string' && value.trim() !== '' ? 'set' : undefined)
},
temperature: {
icon: Thermometer,
label: 'Temperature',
summarize: (value) => (typeof value === 'number' ? String(value) : undefined)
},
max_completion_tokens: {
icon: Hash,
label: 'Max tokens',
summarize: (value) => (typeof value === 'number' ? String(value) : undefined)
}
}
const FLOW_INPUT_REF = /flow_input\??\.([A-Za-z_$][\w$]*)/g
/**
* The flow input a transform is fed by, when exactly one feeds it.
*
* The expression need not be a bare pass-through a step commonly reshapes what it
* reads, e.g. `(flow_input.files || []).map(f => ({ bucket: f.storage, key: f.s3 }))`.
* Writing that input is still right, because the expression consumes it. Two or more
* inputs are ambiguous: the composer would have no way to say which one it is editing.
*/
export function flowInputRef(transform: InputTransform | undefined): string | undefined {
if (transform?.type !== 'javascript') return undefined
const names = new Set(
[...transform.expr.matchAll(FLOW_INPUT_REF)].map((match) => match[1])
)
return names.size === 1 ? [...names][0] : undefined
}
/** A provider value as the agent stores it. */
export type AgentModel = { kind?: string; model?: string; reasoning_effort?: string }
/**
* The model the flow already fixes, when it fixes exactly one: a single AI agent step
* whose `provider` is a static value. Named on the settings trigger the way the session
* chat names its own model, but not editable no flow input feeds it, so there is
* nothing the composer could write.
*/
export function resolveStaticAgentModel(
modules: FlowModule[] | undefined
): AgentModel | undefined {
const agents = getAllModules(modules ?? []).filter((m) => m.value.type === 'aiagent')
if (agents.length !== 1) return undefined
const provider = (agents[0].value as any).input_transforms?.['provider']
if (provider?.type !== 'static') return undefined
const value = provider.value
return typeof value?.model === 'string'
? { kind: value.kind, model: value.model, reasoning_effort: value.reasoning_effort }
: undefined
}
export function isEmptyAgentChatInputValue(value: any): boolean {
if (value === undefined || value === null || value === '') return true
return Array.isArray(value) && value.length === 0
}
/**
* The flow inputs that an AI agent step reads directly into one of its chat-relevant
* fields. Several agents may resolve to the same flow input; it is one chip either way,
* and one that stays unambiguous however many agents read it.
*/
export function resolveAgentChatInputs(
modules: FlowModule[] | undefined,
additionalInputsSchema: Record<string, any> | undefined
): AgentChatInput[] {
const properties = additionalInputsSchema?.properties
if (!modules || !properties) return []
const required: string[] = Array.isArray(additionalInputsSchema?.required)
? additionalInputsSchema.required
: []
const keyOf = new Map<string, AgentChatInputKey>()
for (const module of getAllModules(modules)) {
if (module.value.type !== 'aiagent') continue
const transforms = module.value.input_transforms ?? {}
for (const key of AGENT_CHAT_INPUT_KEYS) {
const name = flowInputRef(transforms[key])
// A name the schema doesn't declare has no field to promote, and `user_message`
// is already the composer itself.
if (!name || !(name in properties) || keyOf.has(name)) continue
keyOf.set(name, key)
}
}
return [...keyOf.entries()]
.map(([name, key]) => ({
name,
key,
property: properties[name],
required: required.includes(name)
}))
.sort((a, b) => AGENT_CHAT_INPUT_KEYS.indexOf(a.key) - AGENT_CHAT_INPUT_KEYS.indexOf(b.key))
}
@@ -7,15 +7,47 @@ import type { ChatMessage, FlowChatManager } from './FlowChatManager.svelte'
import { AIAutonomyMode } from '$lib/components/copilot/chat/AIChatManager.svelte'
import { AttachedFilesStore } from '$lib/components/copilot/chat/files/attachedFiles.svelte'
import { SessionArtifactsStore } from '$lib/components/copilot/chat/artifacts/artifactsState.svelte'
import { dataUrlToBlob } from '$lib/components/copilot/chat/blobUtils'
import { HelpersService } from '$lib/gen'
import { sendUserToast } from '$lib/toast'
import { randomUUID } from '$lib/utils/uuid'
import { MessageInputsStore } from './messageInputContext.svelte'
/** What an AI agent step reads out of `user_attachments`. */
type S3Attachment = { s3: string; filename?: string }
/** The flow input the composer's attachments feed, and whether it holds a list. */
export type AttachmentsTarget = { name: string; multiple: boolean }
export type FlowChatViewHostOptions = {
additionalInputs?: () => Record<string, any> | undefined
/** Called once the turn is dispatched, to clear anything that rides one message. */
onSent?: () => void
attachmentsTarget?: () => AttachmentsTarget | undefined
workspace?: () => string | undefined
/** Off when the workspace has no object storage — there is nowhere to upload to. */
canAttach?: () => boolean
}
function toDisplayMessage(
message: ChatMessage,
userIndex: number,
showStepNames: boolean
showStepNames: boolean,
inputs: MessageInputsStore
): DisplayMessage {
switch (message.message_type) {
case 'user':
return { role: 'user', index: userIndex, content: message.content }
case 'user': {
// What the turn ran with, read back from its job — the message row itself
// keeps only the text. Renders through the same lanes the copilot uses.
const { images, contextElements } = inputs.get(message.job_id)
return {
role: 'user',
index: userIndex,
content: message.content,
images: images.length > 0 ? images : undefined,
contextElements: contextElements.length > 0 ? contextElements : undefined
}
}
case 'tool':
// Both producers of a tool row — the agent executor and the frontend's own
// stream parser — write the whole message as a one-line description of the
@@ -46,14 +78,11 @@ function toDisplayMessage(
*/
export class FlowChatViewHost implements ChatViewHost {
#manager: FlowChatManager
#additionalInputs: () => Record<string, any> | undefined
#options: FlowChatViewHostOptions
constructor(
manager: FlowChatManager,
additionalInputs: () => Record<string, any> | undefined = () => undefined
) {
constructor(manager: FlowChatManager, options: FlowChatViewHostOptions = {}) {
this.#manager = manager
this.#additionalInputs = additionalInputs
this.#options = options
}
// The step name says which AI agent step wrote a message, so it only tells the
@@ -64,16 +93,25 @@ export class FlowChatViewHost implements ChatViewHost {
() => new Set(this.#manager.messages.map((m) => m.step_name).filter(Boolean)).size > 1
)
#messageInputs = new MessageInputsStore(() => this.#options.workspace?.())
displayMessages = $derived.by(() => {
let userIndex = 0
const showStepNames = this.#showStepNames
return this.#manager.messages.map((message) =>
toDisplayMessage(message, message.message_type === 'user' ? userIndex++ : -1, showStepNames)
toDisplayMessage(
message,
message.message_type === 'user' ? userIndex++ : -1,
showStepNames,
this.#messageInputs
)
)
})
messages: readonly unknown[] = []
contextTokens = 0
loading = $derived.by(() => this.#manager.isLoading || this.#manager.isWaitingForResponse)
loading = $derived.by(
() => this.#manager.isLoading || this.#manager.isWaitingForResponse || this.#uploading
)
loadingLabel = undefined
compacting = false
currentReply = ''
@@ -97,13 +135,70 @@ export class FlowChatViewHost implements ChatViewHost {
get sendInFlight() {
return this.#manager.isLoading
}
// True while attachments are uploading — before the flow job exists, so
// `manager.isLoading` cannot cover it and the composer would look idle.
#uploading = $state(false)
sendRequest = async (options: ChatSendRequestOptions = {}) => {
const text = options.instructions?.trim()
// The conversation's message is the flow's `user_message`, which the manager
// requires; an attachment-only turn has nothing to send.
if (!text) return false
const args = { ...(this.#options.additionalInputs?.() ?? {}) }
const target = this.#options.attachmentsTarget?.()
const attachments = [...(options.images ?? []), ...(options.blobs ?? [])]
if (target && attachments.length > 0) {
this.#uploading = true
try {
const uploaded = await this.#uploadAttachments(attachments)
args[target.name] = target.multiple ? uploaded : uploaded[0]
} catch (e) {
sendUserToast(
`Could not upload the attachments: ${e instanceof Error ? e.message : String(e)}`,
true
)
return false
} finally {
this.#uploading = false
}
}
this.#manager.inputMessage = text
await this.#manager.sendMessage(this.#additionalInputs())
this.#options.onSent?.()
await this.#manager.sendMessage(
Object.keys(args).length > 0 || this.#options.additionalInputs?.() ? args : undefined
)
return true
}
/**
* Put each attachment in the workspace's object storage and hand back what the
* agent reads. The flow runs on a worker, so the bytes have to exist somewhere
* the worker can fetch unlike the copilot, which sends them from the browser.
*/
async #uploadAttachments(
attachments: { name?: string; dataUrl: string; mediaType?: string }[]
): Promise<S3Attachment[]> {
const workspace = this.#options.workspace?.()
if (!workspace) throw new Error('no workspace')
// One prefix per turn keeps a re-attached filename from overwriting the copy an
// earlier message still refers to.
const prefix = `windmill_chat_uploads/${randomUUID()}`
return Promise.all(
attachments.map(async (attachment, index) => {
const filename = attachment.name ?? `attachment-${index + 1}`
const blob = dataUrlToBlob(attachment.dataUrl, attachment.mediaType)
const { file_key } = await HelpersService.fileUpload({
workspace,
fileKey: `${prefix}/${filename}`,
contentType: blob.type,
requestBody: blob
})
return { s3: file_key, filename }
})
)
}
cancel = () => {
void this.#manager.cancelCurrentJob()
}
@@ -144,6 +239,15 @@ export class FlowChatViewHost implements ChatViewHost {
isSessionChat = false
supportsModelSettings = false
supportsMessageEditing = false
// Attachments go to object storage for the worker to read, so a linked folder —
// a live handle on the user's own disk — has no meaning here.
get supportsMessageAttachments() {
return (this.#options.canAttach?.() ?? false) && !!this.#options.attachmentsTarget?.()
}
supportsLinkedFolders = false
attachmentsAsBlobs = true
// What an AI agent step accepts (AI_AGENT_SCHEMA.user_attachments).
attachmentAccept = 'image/*,application/pdf,.pdf'
tools = []
autonomyMode = AIAutonomyMode.DEFAULT
setAutonomyMode = () => {}
@@ -0,0 +1,136 @@
/**
* The inputs a chat message ran with, recovered from its job.
*
* A conversation message stores only its text, so what the user attached and which
* settings the turn used exist nowhere but the run's arguments. The user row carries
* the flow job id, whose args are the raw run arguments `user_message` plus every
* other flow input.
*
* Fetched lazily and kept in memory only: a purged job leaves a dangling id and the
* turn simply shows no inputs, which is honest the arguments are gone.
*/
import { JobService } from '$lib/gen'
import { base } from '$lib/base'
import {
createAttachedFileContextElement,
type ContextElement
} from '$lib/components/copilot/chat/context'
import type { AttachedImage } from '$lib/components/copilot/chat/imageUtils'
type S3Ref = { s3: string; filename?: string; storage?: string }
const IMAGE_EXTENSIONS = ['.png', '.jpg', '.jpeg', '.gif', '.webp', '.bmp', '.svg', '.avif']
function isS3Ref(value: any): value is S3Ref {
return !!value && typeof value === 'object' && typeof value.s3 === 'string' && value.s3 !== ''
}
function s3Refs(value: any): S3Ref[] {
if (isS3Ref(value)) return [value]
if (Array.isArray(value)) return value.filter(isS3Ref)
return []
}
function displayName(ref: S3Ref): string {
return ref.filename ?? ref.s3.split('/').pop() ?? ref.s3
}
function looksLikeImage(ref: S3Ref): boolean {
const name = displayName(ref).toLowerCase()
return IMAGE_EXTENSIONS.some((ext) => name.endsWith(ext))
}
/** Same-origin, cookie-authed GET — usable directly as an <img src>, no blob fetch. */
function downloadUrl(workspace: string, ref: S3Ref): string {
const params = new URLSearchParams({ file_key: ref.s3 })
if (ref.storage) params.set('storage', ref.storage)
return `${base}/api/w/${workspace}/job_helpers/download_s3_file?${params.toString()}`
}
/** A scalar input, summarised for a chip. Objects are left to the file/JSON branches. */
function scalarSummary(value: any): string | undefined {
if (value === undefined || value === null || value === '') return undefined
if (typeof value === 'string') return value
if (typeof value === 'number' || typeof value === 'boolean') return String(value)
return undefined
}
export type MessageInputs = { images: AttachedImage[]; contextElements: ContextElement[] }
const EMPTY: MessageInputs = { images: [], contextElements: [] }
/**
* Split a turn's run arguments into the lanes a user message renders: image
* thumbnails, and a chip per remaining input. `user_message` is the bubble itself.
*/
export function argsToMessageInputs(
workspace: string,
args: Record<string, any> | undefined
): MessageInputs {
if (!args) return EMPTY
const images: AttachedImage[] = []
const contextElements: ContextElement[] = []
for (const [name, value] of Object.entries(args)) {
if (name === 'user_message') continue
const files = s3Refs(value)
if (files.length > 0) {
for (const ref of files) {
if (looksLikeImage(ref)) {
images.push({
dataUrl: downloadUrl(workspace, ref),
mediaType: 'image/png',
name: displayName(ref)
})
} else {
contextElements.push(
createAttachedFileContextElement(displayName(ref), `Attached file · ${ref.s3}`)
)
}
}
continue
}
const summary = scalarSummary(value)
if (summary !== undefined) {
contextElements.push(createAttachedFileContextElement(name, summary))
}
}
return images.length > 0 || contextElements.length > 0 ? { images, contextElements } : EMPTY
}
/**
* Per-conversation cache of run arguments by job id. One fetch per turn, kept only
* for as long as the chat is mounted.
*/
export class MessageInputsStore {
#workspace: () => string | undefined
#byJob = $state<Record<string, MessageInputs>>({})
#inFlight = new Set<string>()
constructor(workspace: () => string | undefined) {
this.#workspace = workspace
}
/** What the turn ran with, fetching on first ask. Empty until the args land. */
get(jobId: string | null | undefined): MessageInputs {
if (!jobId) return EMPTY
const cached = this.#byJob[jobId]
if (cached) return cached
void this.#load(jobId)
return EMPTY
}
async #load(jobId: string) {
const workspace = this.#workspace()
if (!workspace || this.#inFlight.has(jobId)) return
this.#inFlight.add(jobId)
try {
const args = await JobService.getJobArgs({ workspace, id: jobId })
this.#byJob = { ...this.#byJob, [jobId]: argsToMessageInputs(workspace, args as any) }
} catch {
// A purged job, or one this user cannot read: the turn shows no inputs.
this.#byJob = { ...this.#byJob, [jobId]: EMPTY }
} finally {
this.#inFlight.delete(jobId)
}
}
}
@@ -705,6 +705,7 @@
path={flow?.path ?? ''}
useStreaming={shouldUseStreaming}
inputSchema={flow?.schema}
flowModules={flow?.value?.modules}
wideLayout
/>
{:else}