From bb89e45a7f900e904896539823ef69dfbadca5dd Mon Sep 17 00:00:00 2001 From: Guilhem Lemouel Date: Thu, 17 Sep 2026 08:39:01 +0200 Subject: [PATCH] feat: carry uploaded attachments on the pending chat user message Co-Authored-By: Claude Opus 5 (1M context) --- chat-sdk/README.md | 6 ++-- chat-sdk/src/attachments.ts | 6 ++-- chat-sdk/src/chat.ts | 7 +++++ chat-sdk/src/index.ts | 1 + chat-sdk/src/types.ts | 8 +++-- chat-sdk/test/attachments.test.ts | 30 +++++++++++++++++++ .../conversations/flowChatViewHost.svelte.ts | 4 +-- 7 files changed, 53 insertions(+), 9 deletions(-) diff --git a/chat-sdk/README.md b/chat-sdk/README.md index 899ca49acb..0d747f8f20 100644 --- a/chat-sdk/README.md +++ b/chat-sdk/README.md @@ -244,8 +244,10 @@ await chat.sendMessage('What does this contract say?', { Each file is uploaded to the workspace's object storage under `windmill_uploads/chat///` and handed to the input as `{ s3, filename }` -objects (the object for a single-file input). The name's extension is corrected to the -file's media type for PNG, JPEG and PDF, because the worker reads the type off the key. +objects (the object for a single-file input). Once the uploads return, the pending user +message lists them in `attachments`, as `{ input, s3, filename }` references. The name's +extension is corrected to the file's media type for PNG, JPEG and PDF, because the worker +reads the type off the key. A failed upload rejects `sendMessage` before any run starts, and `stop()` during the upload aborts it; both leave the transcript as it was. The workspace needs object storage set up. With Enterprise advanced storage permissions, the user needs read, write diff --git a/chat-sdk/src/attachments.ts b/chat-sdk/src/attachments.ts index 176c8d6637..10d14eb6df 100644 --- a/chat-sdk/src/attachments.ts +++ b/chat-sdk/src/attachments.ts @@ -1,5 +1,5 @@ import type { WindmillChatApi } from './api' -import type { ChatAttachment } from './types' +import type { AttachmentUpload } from './types' import { abortError, isAbortError } from './utils' /** @@ -45,7 +45,7 @@ export function storedAttachmentName(filename: string, mediaType: string): strin } /** The bytes of an attachment as a Blob carrying its media type. */ -export function attachmentBlob(attachment: ChatAttachment): Blob { +export function attachmentBlob(attachment: AttachmentUpload): Blob { const data = attachment.data if (typeof data !== 'string') { return attachment.mediaType && attachment.mediaType !== data.type @@ -83,7 +83,7 @@ function dataUrlToBlob(dataUrl: string, fallbackType: string): Blob { */ export async function uploadAttachments( api: WindmillChatApi, - attachments: ChatAttachment[], + attachments: AttachmentUpload[], turnId: string, signal?: AbortSignal ): Promise { diff --git a/chat-sdk/src/chat.ts b/chat-sdk/src/chat.ts index e32ab25104..e42bae0505 100644 --- a/chat-sdk/src/chat.ts +++ b/chat-sdk/src/chat.ts @@ -155,6 +155,13 @@ class ChatImpl implements Chat { // and `stop()` can abort the upload, while a second send is refused as usual. const uploaded = await uploadAttachments(this.#api, attachments, randomId(), turn.controller.signal) args[attachmentsInput.name] = attachmentsInput.multiple ? uploaded : uploaded[0] + // Shown on the pending message until its server row replaces it, carrying its own. + const carried = uploaded.map((u) => ({ input: attachmentsInput.name, s3: u.s3, filename: u.filename })) + if (this.#turnActive(turn)) { + this.#set({ + messages: this.#state.messages.map((m) => (m.id === turn.userMessageId ? { ...m, attachments: carried } : m)) + }) + } } turn.started = true // Listed only once the run is asked for: a send that never runs (an upload that failed diff --git a/chat-sdk/src/index.ts b/chat-sdk/src/index.ts index 67edf67920..2a8e5ea1c6 100644 --- a/chat-sdk/src/index.ts +++ b/chat-sdk/src/index.ts @@ -16,6 +16,7 @@ export { extractChatAnswer, conversationIdFor } from './utils' export { storedAttachmentName, uploadAttachments, CHAT_UPLOADS_PREFIX, type UploadedAttachment } from './attachments' export type { AttachmentsInput, + AttachmentUpload, Chat, ChatAttachment, ChatMessage, diff --git a/chat-sdk/src/types.ts b/chat-sdk/src/types.ts index 3e37a08721..666b0aa623 100644 --- a/chat-sdk/src/types.ts +++ b/chat-sdk/src/types.ts @@ -26,6 +26,8 @@ export interface ToolInvocation { status: 'running' | 'success' | 'error' } +export interface ChatAttachment { input: string; s3: string; storage?: string; filename?: string } + export interface ChatMessage { id: string role: ChatRole @@ -39,6 +41,8 @@ export interface ChatMessage { jobId?: string /** The flow step that produced the message. */ stepName?: string + /** The files a user message carried, as object-storage references. */ + attachments?: ChatAttachment[] /** True while the message is optimistic or still streaming. */ pending: boolean /** Id of the persisted row once the server has it; `id` itself never changes, so list keys stay stable. */ @@ -119,7 +123,7 @@ export interface ChatOptions { } /** A file sent with a message. It is uploaded to the workspace's object storage before the run starts. */ -export interface ChatAttachment { +export interface AttachmentUpload { /** Kept as the last segment of the stored key, its extension corrected to the media type for PNG, JPEG and PDF. */ name: string /** The bytes: a Blob, or a `data:` URL of them. */ @@ -142,7 +146,7 @@ export interface SendMessageOptions { * the way an AI agent step reads `user_attachments`. A failed upload rejects `sendMessage` * and the run never starts; `stop()` during the upload does the same with an `AbortError`. */ - attachments?: ChatAttachment[] + attachments?: AttachmentUpload[] /** Required with `attachments`. With `multiple: false`, more than one attachment is refused before anything uploads. */ attachmentsInput?: AttachmentsInput } diff --git a/chat-sdk/test/attachments.test.ts b/chat-sdk/test/attachments.test.ts index f66f2e33d8..e609dc391c 100644 --- a/chat-sdk/test/attachments.test.ts +++ b/chat-sdk/test/attachments.test.ts @@ -353,4 +353,34 @@ describe('sendMessage with attachments', () => { ).rejects.toMatchObject({ name: 'AbortError' }) expect(calls).toHaveLength(0) }) + + test('the pending user message carries its uploaded files before the run returns', async () => { + let releaseRun: (r: Response) => void = () => {} + const { fetch, calls } = fetchMock( + upload, + (c) => + c.method === 'POST' && c.url.pathname === `/api/w/ws/jobs/run/f/${FLOW}` + ? new Promise((resolve) => (releaseRun = resolve)) + : undefined, + answer + ) + const chat = createChat(options(fetch)) + const sending = chat.sendMessage('read these', { + attachments: [ + { name: 'photo.webp', data: png }, + { name: 'contract', data: pdf } + ], + attachmentsInput: { name: 'files', multiple: true } + }) + while (runs(calls).length === 0) await new Promise((r) => setTimeout(r, 1)) + const keys = uploads(calls).map((c) => c.url.searchParams.get('file_key')!) + const pending = chat.getState().messages.find((m) => m.role === 'user')! + expect(pending.pending).toBe(true) + expect(pending.attachments).toEqual([ + { input: 'files', s3: keys[0], filename: 'photo.png' }, + { input: 'files', s3: keys[1], filename: 'contract.pdf' } + ]) + releaseRun(text('job-1')) + await sending + }) }) diff --git a/frontend/src/lib/components/flows/conversations/flowChatViewHost.svelte.ts b/frontend/src/lib/components/flows/conversations/flowChatViewHost.svelte.ts index 5f92e5fc9e..faead079cc 100644 --- a/frontend/src/lib/components/flows/conversations/flowChatViewHost.svelte.ts +++ b/frontend/src/lib/components/flows/conversations/flowChatViewHost.svelte.ts @@ -1,4 +1,4 @@ -import type { Chat, ChatAttachment, ChatMessage, ChatState } from 'windmill-chat' +import type { AttachmentUpload, Chat, ChatMessage, ChatState } from 'windmill-chat' import type { ChatSendRequestOptions, ChatViewHost @@ -269,7 +269,7 @@ export class FlowChatViewHost implements ChatViewHost { true ) } - const attachments: ChatAttachment[] = target + const attachments: AttachmentUpload[] = target ? [...images, ...blobs].map((attachment, index) => ({ name: attachment.name ?? `attachment-${index + 1}`, data: attachment.dataUrl,