diff --git a/chat-sdk/README.md b/chat-sdk/README.md index a233425d14..2f71b62416 100644 --- a/chat-sdk/README.md +++ b/chat-sdk/README.md @@ -181,7 +181,7 @@ await chat.sendMessage('Hello') | `workspace` | Detected inside a raw app. | | `token` | A token, or a function returning one (called before every request, so it can fetch a short-lived token from your backend). Omit it inside a raw app. | | `history` | `'server'`, `'local'` or `'none'`, see [History](#history). Defaults to `'server'` with a viewer session and `'local'` with an explicit `token`. | -| `inputs` | Extra flow inputs sent with every message. `sendMessage(text, { inputs })` adds per-message ones. | +| `inputs` | Extra flow inputs sent with every message. `sendMessage(text, { inputs })` adds per-message ones, and `{ attachments, attachmentsInput }` files, see [Attachments](#attachments). | | `storageKey` | Namespace for `local` history, e.g. the signed-in user's id. Local history is per browser and per flow; without it, users sharing a browser share it. | | `fetch`, `storage` | Replacements for the globals, for tests and unusual runtimes. | | `pageSize` | Messages and conversations per page of server history. Default 50. | @@ -224,12 +224,32 @@ A turn goes `submitted` (the flow is queued) → `streaming` (the answer is arri answer, an `assistant` message with `success: false`. `status: 'error'` (with `error` set) means the turn could not run or be followed at all, such as a refused request. -Methods: `sendMessage(text, { inputs? })`, `stop()`, `newConversation()`, +Methods: `sendMessage(text, { inputs?, attachments?, attachmentsInput? })`, `stop()`, `newConversation()`, `selectConversation(id)`, `loadConversations({ page?, perPage? })`, `deleteConversation(id)`, `loadOlderMessages()`, `destroy()`. Switching conversations stops following the current answer; the flow keeps running and, with server history, its answer is there when you come back. +## Attachments + +A flow whose AI agent step reads `user_attachments` from an `s3object[]` (or a single +`s3object`) flow input takes files with a message: + +```ts +await chat.sendMessage('What does this contract say?', { + attachments: [{ name: file.name, data: file }], // a Blob/File, or a `data:` URL + attachmentsInput: { name: 'files', multiple: true } +}) +``` + +Each file is uploaded to the workspace's object storage under +`windmill_chat_uploads///` and handed to the input as `{ s3, filename }` +objects (the object for a single-file input). The name's extension is corrected to the +file's media type for PNG, JPEG and PDF, because the worker reads the type off the key. +A failed upload rejects `sendMessage` before any run starts, and `stop()` during the +upload aborts it; both leave the transcript as it was. The workspace needs object +storage set up, and the token needs to be allowed to upload. + ## History Windmill stores every conversation of a chat-mode flow, and each Windmill user sees diff --git a/chat-sdk/src/api.ts b/chat-sdk/src/api.ts index afc50ff0a1..18e457c6c6 100644 --- a/chat-sdk/src/api.ts +++ b/chat-sdk/src/api.ts @@ -193,6 +193,27 @@ export class WindmillChatApi { return (await res.json()) as FlowConversationMessage[] } + /** + * Puts bytes in the workspace's object storage under `fileKey` and returns the key they were + * stored under (the server may rewrite it). Needs the workspace to have object storage set. + */ + async uploadFile( + fileKey: string, + body: Blob, + options: { contentType?: string; signal?: AbortSignal } = {} + ): Promise<{ file_key: string }> { + const query: Record = { file_key: fileKey } + if (options.contentType) query.content_type = options.contentType + const res = await this.#request('job_helpers/upload_s3_file', { + method: 'POST', + query, + raw: body, + contentType: options.contentType || 'application/octet-stream', + signal: options.signal + }) + return (await res.json()) as { file_key: string } + } + async deleteConversation(conversationId: string): Promise { await this.#request(`flow_conversations/delete/${encodeURIComponent(conversationId)}`, { method: 'DELETE' @@ -204,7 +225,11 @@ export class WindmillChatApi { init: { method?: string query?: Record + /** JSON-encoded. */ body?: unknown + /** Sent as is, under `contentType`. */ + raw?: Blob + contentType?: string accept?: string signal?: AbortSignal } = {} @@ -215,13 +240,14 @@ export class WindmillChatApi { const headers: Record = {} if (init.accept) headers['Accept'] = init.accept if (init.body !== undefined) headers['Content-Type'] = 'application/json' + else if (init.raw !== undefined) headers['Content-Type'] = init.contentType ?? 'application/octet-stream' const token = typeof this.#token === 'function' ? await this.#token() : this.#token if (token) headers['Authorization'] = `Bearer ${token}` const res = await this.#fetch(url.toString(), { method: init.method ?? 'GET', headers, - body: init.body === undefined ? undefined : JSON.stringify(init.body), + body: init.body === undefined ? init.raw : JSON.stringify(init.body), // A token must not be paired with ambient cookies; without one, the cookie is // the credential and only rides same-origin requests. credentials: token ? 'omit' : 'same-origin', diff --git a/chat-sdk/src/attachments.ts b/chat-sdk/src/attachments.ts new file mode 100644 index 0000000000..b00cf521f6 --- /dev/null +++ b/chat-sdk/src/attachments.ts @@ -0,0 +1,96 @@ +import type { WindmillChatApi } from './api' +import type { ChatAttachment } from './types' + +/** Where a chat's uploads live in the workspace's object storage. */ +export const CHAT_UPLOADS_PREFIX = 'windmill_chat_uploads' + +/** What an AI agent step reads out of `user_attachments`. */ +export interface UploadedAttachment { + s3: string + filename: string +} + +/** + * The extension each type a chat composer sends must be stored under. + * + * The worker reads an attachment's media type from the object key and nothing else — + * `mime_guess::from_path` in `windmill-ai/src/image_handler.rs`, falling back to + * `image/png` when it can read no extension — and never from the content type stored + * beside it. So the key's extension is a claim about the bytes, and it has to be true. + */ +const EXTENSION_BY_MEDIA_TYPE: Record = { + 'image/png': 'png', + 'image/jpeg': 'jpg', + 'application/pdf': 'pdf' +} + +/** + * The name an attachment is stored under. A composer commonly re-encodes every image to PNG + * or JPEG, so keeping the picked `photo.webp` would hand the provider PNG bytes labelled webp, + * which Anthropic rejects outright; and a PDF picked without an extension would be read back + * as the `image/png` fallback. A type not listed is left as picked — other files upload byte + * for byte, so their name is already true. + */ +export function storedAttachmentName(filename: string, mediaType: string): string { + const extension = EXTENSION_BY_MEDIA_TYPE[mediaType] + if (!extension) return filename + const stem = filename.replace(/\.[^./]+$/, '') + return `${stem || filename}.${extension}` +} + +/** The bytes of an attachment as a Blob carrying its media type. */ +export function attachmentBlob(attachment: ChatAttachment): Blob { + const data = attachment.data + if (typeof data !== 'string') { + return attachment.mediaType && attachment.mediaType !== data.type + ? new Blob([data], { type: attachment.mediaType }) + : data + } + return dataUrlToBlob(data, attachment.mediaType ?? 'application/octet-stream') +} + +function dataUrlToBlob(dataUrl: string, fallbackType: string): Blob { + const comma = dataUrl.indexOf(',') + if (!dataUrl.startsWith('data:') || comma === -1) { + throw new Error('windmill-chat: an attachment given as a string must be a data: URL') + } + const header = dataUrl.slice(5, comma) + const isBase64 = header.endsWith(';base64') + const mediaType = (isBase64 ? header.slice(0, -';base64'.length) : header) || fallbackType + const payload = dataUrl.slice(comma + 1) + if (!isBase64) return new Blob([decodeURIComponent(payload)], { type: mediaType }) + const binary = atob(payload) + const bytes = new Uint8Array(binary.length) + for (let i = 0; i < binary.length; i++) bytes[i] = binary.charCodeAt(i) + return new Blob([bytes], { type: mediaType }) +} + +/** + * Put each attachment in the workspace's object storage and hand back what the agent reads. + * The flow runs on a worker, so the bytes have to exist somewhere the worker can fetch. + * + * A prefix per turn, and a segment per attachment inside it. The turn's prefix keeps a + * re-attached filename off the copy an earlier message still points at; the segment does the + * same within one turn, where two files can arrive under one name and would otherwise race to + * a single key and leave the agent reading one of them twice. The name itself stays the last + * segment, so anything that reads a name off the key still sees what the user attached. + */ +export async function uploadAttachments( + api: WindmillChatApi, + attachments: ChatAttachment[], + turnId: string, + signal?: AbortSignal +): Promise { + const prefix = `${CHAT_UPLOADS_PREFIX}/${turnId}` + return Promise.all( + attachments.map(async (attachment, index) => { + const blob = attachmentBlob(attachment) + const filename = storedAttachmentName(attachment.name || `attachment-${index + 1}`, blob.type) + const { file_key } = await api.uploadFile(`${prefix}/${index}/${filename}`, blob, { + contentType: blob.type, + signal + }) + return { s3: file_key, filename } + }) + ) +} diff --git a/chat-sdk/src/chat.ts b/chat-sdk/src/chat.ts index dda8f10d05..0d39123ed3 100644 --- a/chat-sdk/src/chat.ts +++ b/chat-sdk/src/chat.ts @@ -7,6 +7,7 @@ import { import { resolveConfig, type ResolvedConfig } from './config' import { followJob } from './follow' import { createLocalHistory, type LocalHistory } from './history' +import { uploadAttachments } from './attachments' import type { AgentStreamEvent } from './stream' import type { Chat, @@ -14,6 +15,7 @@ import type { ChatOptions, ChatState, Conversation, + SendMessageOptions, ToolInvocation } from './types' import { @@ -39,6 +41,10 @@ interface Turn { conversationId: string /** Id of the turn's user message; the answer is whatever follows it. */ userMessageId: string + /** The turn opened the conversation; withdrawing it closes the conversation again. */ + isNew: boolean + /** The run was asked for. Before that, a failure or a stop withdraws the turn instead of failing it. */ + started: boolean jobId?: string /** The flow job and its step jobs; a persisted answer carries one of them as `job_id`. */ jobIds?: Set @@ -97,21 +103,25 @@ class ChatImpl implements Chat { } } - sendMessage = async ( - text: string, - options: { inputs?: Record } = {} - ): Promise => { + sendMessage = async (text: string, options: SendMessageOptions = {}): Promise => { const content = text.trim() if (!content) return if (this.#turn) { throw new Error('windmill-chat: a message is already being answered; call stop() first') } + const attachments = options.attachments ?? [] + const attachmentsInput = options.attachmentsInput + if (attachments.length > 0 && !attachmentsInput) { + throw new Error('windmill-chat: attachments need `attachmentsInput`, the flow input that takes them') + } const isNew = this.#state.conversationId === undefined const conversationId = this.#state.conversationId ?? randomId() const turn: Turn = { controller: new AbortController(), conversationId, userMessageId: `pending-${randomId()}`, + isNew, + started: false, streamedText: false } this.#turn = turn @@ -134,7 +144,14 @@ class ChatImpl implements Chat { this.#rememberConversation() try { - const args = { ...this.#config.inputs, ...options.inputs, user_message: content } + const args: Record = { ...this.#config.inputs, ...options.inputs, user_message: content } + if (attachmentsInput && attachments.length > 0) { + // Uploaded with the turn already shown as submitted: the message is in the transcript + // and `stop()` can abort the upload, while a second send is refused as usual. + const uploaded = await uploadAttachments(this.#api, attachments, randomId(), turn.controller.signal) + args[attachmentsInput.name] = attachmentsInput.multiple ? uploaded : uploaded[0] + } + turn.started = true const context = { memoryId: conversationId, conversationId, signal: turn.controller.signal } turn.jobId = this.#config.run ? await this.#config.run(args, context) @@ -148,6 +165,13 @@ class ChatImpl implements Chat { } await this.#finishTurn(turn, result, isNew) } catch (e) { + if (!turn.started) { + // Nothing ran: the message is withdrawn rather than shown as a failed turn, and the + // caller gets the reason (an upload that failed, or the AbortError of a stop()). + if (this.#turn === turn) this.#turn = undefined + this.#withdrawTurn(turn) + throw e + } // stop() and a conversation switch abort the turn and settle the state themselves. if (turn.controller.signal.aborted || isAbortError(e)) return this.#failTurn(turn, e) @@ -160,6 +184,12 @@ class ChatImpl implements Chat { const turn = this.#turn if (!turn) return this.#detachTurn() + if (!turn.started) { + // Still uploading its attachments: there is no run to cancel, and the message the + // reader took back must not stay in the transcript as sent. + this.#withdrawTurn(turn) + return + } if (this.#state.conversationId === turn.conversationId) { this.#set({ messages: finalized(this.#state.messages), status: 'idle' }) this.#persistLocal() @@ -545,6 +575,29 @@ class ChatImpl implements Chat { } } + /** + * Undo what `sendMessage` showed for a turn that never ran: its user message, and the + * conversation it opened when there was none. Only while that conversation is still the + * one on screen; a switch meanwhile has already left it behind. + */ + #withdrawTurn(turn: Turn): void { + if (this.#state.conversationId !== turn.conversationId) return + const messages = this.#state.messages.filter((m) => m.id !== turn.userMessageId) + if (turn.isNew) { + if (this.#state.history === 'local') this.#local.deleteConversation(turn.conversationId) + this.#set({ + conversationId: undefined, + conversations: this.#state.conversations.filter((c) => c.id !== turn.conversationId), + messages, + status: 'idle', + error: undefined + }) + return + } + this.#set({ messages, status: 'idle', error: undefined }) + this.#persistLocal() + } + #failTurn(turn: Turn, e: unknown): void { if (!this.#turnActive(turn)) return const error = toError(e) diff --git a/chat-sdk/src/index.ts b/chat-sdk/src/index.ts index 353c814154..67edf67920 100644 --- a/chat-sdk/src/index.ts +++ b/chat-sdk/src/index.ts @@ -13,8 +13,11 @@ export { export { parseStreamEvents, createStreamEventParser, type AgentStreamEvent } from './stream' export { followJob, type FollowEvent } from './follow' export { extractChatAnswer, conversationIdFor } from './utils' +export { storedAttachmentName, uploadAttachments, CHAT_UPLOADS_PREFIX, type UploadedAttachment } from './attachments' export type { + AttachmentsInput, Chat, + ChatAttachment, ChatMessage, ChatOptions, ChatRole, @@ -23,6 +26,7 @@ export type { Conversation, FetchLike, HistoryMode, + SendMessageOptions, StorageLike, TokenSource, ToolInvocation diff --git a/chat-sdk/src/types.ts b/chat-sdk/src/types.ts index 92b8dfb89b..0d98398db6 100644 --- a/chat-sdk/src/types.ts +++ b/chat-sdk/src/types.ts @@ -118,12 +118,45 @@ export interface ChatOptions { onError?: (error: Error, turn: { conversationId: string; jobId?: string }) => void } +/** A file sent with a message. It is uploaded to the workspace's object storage before the run starts. */ +export interface ChatAttachment { + /** Kept as the last segment of the stored key, its extension corrected to the media type for PNG, JPEG and PDF. */ + name: string + /** The bytes: a Blob, or a `data:` URL of them. */ + data: Blob | string + /** The file's media type. Defaults to the Blob's own type, or the data URL's. */ + mediaType?: string +} + +/** The flow input the uploaded attachments are handed to: an `s3object` (`multiple: false`) or an `s3object[]`. */ +export interface AttachmentsInput { + name: string + multiple: boolean +} + +export interface SendMessageOptions { + /** Extra flow inputs for this message, on top of `ChatOptions.inputs`. */ + inputs?: Record + /** + * Files to upload and hand to the flow as `{ s3, filename }` objects in `attachmentsInput`, + * the way an AI agent step reads `user_attachments`. A failed upload rejects `sendMessage` + * and the run never starts; `stop()` during the upload does the same with an `AbortError`. + */ + attachments?: ChatAttachment[] + /** Required with `attachments`. */ + attachmentsInput?: AttachmentsInput +} + export interface Chat { getState(): ChatState /** Calls `listener` now and on every change; returns the unsubscribe function (Svelte store contract). */ subscribe(listener: (state: ChatState) => void): () => void - /** Sends a message in the current conversation, starting one when there is none. Resolves when the answer is complete. */ - sendMessage(text: string, options?: { inputs?: Record }): Promise + /** + * Sends a message in the current conversation, starting one when there is none. Resolves + * when the answer is complete. Rejects when the message could not be sent at all — a turn + * already running, an attachment that failed to upload — without touching the transcript. + */ + sendMessage(text: string, options?: SendMessageOptions): Promise /** Stops following the answer and asks Windmill to cancel the run. */ stop(): Promise newConversation(): void diff --git a/chat-sdk/test/attachments.test.ts b/chat-sdk/test/attachments.test.ts new file mode 100644 index 0000000000..c960aa935f --- /dev/null +++ b/chat-sdk/test/attachments.test.ts @@ -0,0 +1,204 @@ +import { describe, expect, test } from 'bun:test' +import { storedAttachmentName } from '../src/attachments' +import { createChat } from '../src/chat' +import type { ChatOptions } from '../src/types' +import { abortError } from '../src/utils' +import { fetchMock, json, memoryStorage, sse, text, type RecordedCall, type Route } from './support' + +const BASE = 'http://wm.test' +const FLOW = 'f/chat/agent' +const UPLOAD_PATH = '/api/w/ws/job_helpers/upload_s3_file' + +const run: Route = (c) => + c.method === 'POST' && c.url.pathname === `/api/w/ws/jobs/run/f/${FLOW}` + ? text('job-1') + : undefined + +/** Stores under the key it was asked to, like the server with a `file_key`. */ +const upload: Route = (c) => + c.url.pathname === UPLOAD_PATH + ? json({ file_key: c.url.searchParams.get('file_key') }) + : undefined + +const answer: Route = (c) => + c.url.pathname === '/api/w/ws/jobs_u/getupdate_sse/job-1' + ? sse([ + { + type: 'update', + completed: true, + only_result: { output: 'ok', messages: [] } + } + ]) + : undefined + +function options(fetch: ChatOptions['fetch']): ChatOptions { + return { + flowPath: FLOW, + baseUrl: BASE, + workspace: 'ws', + token: 'tok', + fetch, + storage: memoryStorage() + } +} + +const uploads = (calls: RecordedCall[]) => calls.filter((c) => c.url.pathname === UPLOAD_PATH) +const runs = (calls: RecordedCall[]) => + calls.filter((c) => c.url.pathname.startsWith('/api/w/ws/jobs/run/')) + +const png = new Blob([new Uint8Array([0x89, 0x50, 0x4e, 0x47])], { + type: 'image/png' +}) +const pdf = new Blob(['%PDF-1.7'], { type: 'application/pdf' }) + +describe('storedAttachmentName', () => { + // The worker reads the media type from the key's extension, so it has to match the bytes. + test('renames a re-encoded image and gives a bare name its extension', () => { + expect(storedAttachmentName('photo.webp', 'image/png')).toBe('photo.png') + expect(storedAttachmentName('holiday.png', 'image/jpeg')).toBe('holiday.jpg') + expect(storedAttachmentName('contract', 'application/pdf')).toBe('contract.pdf') + expect(storedAttachmentName('report.2026.final.webp', 'image/png')).toBe( + 'report.2026.final.png' + ) + }) + + test('leaves a type it does not know alone', () => { + expect(storedAttachmentName('notes.csv', 'text/csv')).toBe('notes.csv') + }) +}) + +describe('sendMessage with attachments', () => { + test('uploads each file under the turn prefix and hands the list to the input', async () => { + const { fetch, calls } = fetchMock(upload, run, answer) + const chat = createChat(options(fetch)) + + await chat.sendMessage('read these', { + inputs: { locale: 'fr' }, + attachments: [ + { name: 'photo.webp', data: png }, + { name: 'contract', data: pdf }, + // A data URL is decoded to its bytes; the mediaType names what they are. + { + name: 'photo.webp', + data: `data:image/png;base64,${btoa('\x89PNG')}` + } + ], + attachmentsInput: { name: 'files', multiple: true } + }) + + const keys = uploads(calls).map((c) => c.url.searchParams.get('file_key')!) + expect(keys).toHaveLength(3) + const prefix = keys[0].split('/').slice(0, 2).join('/') + expect(prefix).toMatch(/^windmill_chat_uploads\/[0-9a-f-]{36}$/) + expect(keys).toEqual([ + `${prefix}/0/photo.png`, + `${prefix}/1/contract.pdf`, + `${prefix}/2/photo.png` + ]) + expect(uploads(calls).map((c) => c.url.searchParams.get('content_type'))).toEqual([ + 'image/png', + 'application/pdf', + 'image/png' + ]) + expect(uploads(calls).map((c) => c.headers['content-type'])).toEqual([ + 'image/png', + 'application/pdf', + 'image/png' + ]) + expect(new Uint8Array(await uploads(calls)[2].raw!.arrayBuffer())).toEqual( + new Uint8Array([0x89, 0x50, 0x4e, 0x47]) + ) + + expect(runs(calls)[0].body).toEqual({ + locale: 'fr', + user_message: 'read these', + files: [ + { s3: `${prefix}/0/photo.png`, filename: 'photo.png' }, + { s3: `${prefix}/1/contract.pdf`, filename: 'contract.pdf' }, + { s3: `${prefix}/2/photo.png`, filename: 'photo.png' } + ] + }) + expect(chat.getState().status).toBe('idle') + }) + + test('hands a single object to an input that holds one file', async () => { + const { fetch, calls } = fetchMock(upload, run, answer) + const chat = createChat(options(fetch)) + await chat.sendMessage('read this', { + attachments: [{ name: 'contract.pdf', data: pdf }], + attachmentsInput: { name: 'file', multiple: false } + }) + const body = runs(calls)[0].body as Record + expect(body.file).toEqual({ + s3: expect.stringMatching(/\/0\/contract\.pdf$/), + filename: 'contract.pdf' + }) + }) + + test('refuses attachments without an input to put them in', async () => { + const { fetch, calls } = fetchMock(upload, run, answer) + const chat = createChat(options(fetch)) + await expect( + chat.sendMessage('hi', { attachments: [{ name: 'a.pdf', data: pdf }] }) + ).rejects.toThrow('attachmentsInput') + expect(calls).toHaveLength(0) + }) + + test('a failed upload rejects without a run, and withdraws the message', async () => { + const { fetch, calls } = fetchMock( + (c) => (c.url.pathname === UPLOAD_PATH ? text('no object storage', 500) : undefined), + run, + answer + ) + const chat = createChat(options(fetch)) + const statuses: string[] = [] + chat.subscribe((s) => statuses.push(s.status)) + + await expect( + chat.sendMessage('read this', { + attachments: [{ name: 'contract.pdf', data: pdf }], + attachmentsInput: { name: 'files', multiple: true } + }) + ).rejects.toThrow('no object storage') + + expect(runs(calls)).toHaveLength(0) + // Shown as submitted while uploading, then withdrawn whole: no message, no conversation. + expect(statuses).toContain('submitted') + const state = chat.getState() + expect(state.status).toBe('idle') + expect(state.messages).toEqual([]) + expect(state.conversationId).toBeUndefined() + expect(state.conversations).toEqual([]) + // The chat is free for the next message. + await chat.sendMessage('plain') + expect(runs(calls)).toHaveLength(1) + }) + + test('stop() during the upload aborts it and withdraws the message', async () => { + const { fetch, calls } = fetchMock( + (c) => + c.url.pathname === UPLOAD_PATH + ? new Promise((_, reject) => + c.signal!.addEventListener('abort', () => reject(abortError())) + ) + : undefined, + run, + answer + ) + const chat = createChat(options(fetch)) + const sending = chat.sendMessage('read this', { + attachments: [{ name: 'contract.pdf', data: pdf }], + attachmentsInput: { name: 'files', multiple: true } + }) + await new Promise((r) => setTimeout(r, 0)) + expect(chat.getState().status).toBe('submitted') + await chat.stop() + await expect(sending).rejects.toMatchObject({ name: 'AbortError' }) + expect(runs(calls)).toHaveLength(0) + expect(chat.getState()).toMatchObject({ + status: 'idle', + messages: [], + conversationId: undefined + }) + }) +}) diff --git a/chat-sdk/test/support.ts b/chat-sdk/test/support.ts index c794e50f3d..e198f19355 100644 --- a/chat-sdk/test/support.ts +++ b/chat-sdk/test/support.ts @@ -5,6 +5,9 @@ export interface RecordedCall { url: URL headers: Record body: unknown + /** A body sent as is rather than as JSON (an upload). */ + raw?: Blob + signal?: AbortSignal } export type Route = (call: RecordedCall) => Response | Promise | undefined @@ -20,7 +23,9 @@ export function fetchMock(...routes: Route[]): { fetch: FetchLike; calls: Record headers: Object.fromEntries( Object.entries((init?.headers as Record) ?? {}).map(([k, v]) => [k.toLowerCase(), v]) ), - body: typeof init?.body === 'string' ? JSON.parse(init.body) : undefined + body: typeof init?.body === 'string' ? JSON.parse(init.body) : undefined, + raw: init?.body instanceof Blob ? init.body : undefined, + signal: init?.signal ?? undefined } calls.push(call) for (const route of routes) { diff --git a/frontend/src/lib/components/FlowPreviewContent.svelte b/frontend/src/lib/components/FlowPreviewContent.svelte index f6f6da0cc9..cd2460b1b0 100644 --- a/frontend/src/lib/components/FlowPreviewContent.svelte +++ b/frontend/src/lib/components/FlowPreviewContent.svelte @@ -473,6 +473,7 @@ hideSidebar={true} path={$pathStore} inputSchema={flowStore.val.schema} + flowModules={flowStore.val.value?.modules} /> {:else} diff --git a/frontend/src/lib/components/InputTransformSchemaForm.svelte b/frontend/src/lib/components/InputTransformSchemaForm.svelte index 465e173f1f..31f2eb0054 100644 --- a/frontend/src/lib/components/InputTransformSchemaForm.svelte +++ b/frontend/src/lib/components/InputTransformSchemaForm.svelte @@ -8,7 +8,7 @@ import type { PickableProperties } from './flows/previousResults' import InputTransformForm from './InputTransformForm.svelte' import InputTransformPickers from './InputTransformPickers.svelte' - import { useS3StorageConfigured } from './inputTransformEnv.svelte' + import { useWorkspaceStorageConfigured } from './inputTransformEnv.svelte' import type ItemPicker from './ItemPicker.svelte' import type VariableEditor from './VariableEditor.svelte' import ResizeTransitionWrapper from './common/ResizeTransitionWrapper.svelte' @@ -86,7 +86,7 @@ let itemPicker: ItemPicker | undefined = $state(undefined) let variableEditor: VariableEditor | undefined = $state(undefined) - const s3Storage = useS3StorageConfigured(() => ws) + const s3Storage = useWorkspaceStorageConfigured(() => ws) let keys: string[] = $state([]) $effect(() => { diff --git a/frontend/src/lib/components/copilot/chat/AIChatDisplay.svelte b/frontend/src/lib/components/copilot/chat/AIChatDisplay.svelte index 82dcbc2dcd..0085ca701f 100644 --- a/frontend/src/lib/components/copilot/chat/AIChatDisplay.svelte +++ b/frontend/src/lib/components/copilot/chat/AIChatDisplay.svelte @@ -361,7 +361,15 @@ chatHost.mode === AIMode.SCRIPT || chatHost.mode === AIMode.FLOW || chatHost.mode === AIMode.APP ) - const canAttachFiles = $derived(chatHost.supportsMessageAttachments && !disabled) + // Why attaching is off, when this chat takes attachments but cannot right now. The `+` is + // kept and disabled rather than dropped: the input is the composer's either way, so the + // reader has to be able to see here why nothing can be attached. + const attachmentsOffReason = $derived( + chatHost.supportsMessageAttachments ? chatHost.attachmentsUnavailableReason : undefined + ) + const canAttachFiles = $derived( + chatHost.supportsMessageAttachments && !disabled && !attachmentsOffReason + ) // Folders are linked as session-wide assets, which only a host that reads files in // the browser can do — a host running the turn server-side takes attachments only. const canLinkFolders = $derived(chatHost.supportsLinkedFolders && !disabled) @@ -430,24 +438,32 @@ return Array.from(e.dataTransfer?.types ?? []).includes('Files') } + // A drop is claimed while attaching is off for a stated reason, too: the browser would + // otherwise navigate to the dropped file, and the reader is owed the reason instead. + const panelTakesDrops = $derived(canAttachFiles || attachmentsOffReason !== undefined) + function onPanelDragEnter(e: DragEvent) { - if (!canAttachFiles || !dragHasFiles(e)) return + if (!panelTakesDrops || !dragHasFiles(e)) return e.preventDefault() dragDepth++ } function onPanelDragOver(e: DragEvent) { - if (!canAttachFiles || !dragHasFiles(e)) return + if (!panelTakesDrops || !dragHasFiles(e)) return e.preventDefault() if (e.dataTransfer) e.dataTransfer.dropEffect = 'copy' } function onPanelDragLeave(_e: DragEvent) { - if (!canAttachFiles) return + if (!panelTakesDrops) return dragDepth = Math.max(0, dragDepth - 1) } async function onPanelDrop(e: DragEvent) { dragDepth = 0 - if (!canAttachFiles || !dragHasFiles(e)) return + if (!panelTakesDrops || !dragHasFiles(e)) return e.preventDefault() + if (attachmentsOffReason) { + sendUserToast(attachmentsOffReason, true) + return + } const dt = e.dataTransfer if (!dt) return // Images and loose text files attach to the message; folders link as session @@ -484,9 +500,8 @@ handles.length === 0 ? flatFiles : await Promise.all(handles.filter(isFileHandle).map((h) => h.getFile())) - // Loose text files attach to the message, like images. - const textFiles = looseFiles.filter((f) => !isImageFile(f)) - if (textFiles.length > 0) await aiChatInput?.addTextFiles(textFiles) + // Loose files attach to the message, like images. + await attachNonImageFiles(looseFiles.filter((f) => !isImageFile(f))) // Folders link as a live handle. const dirs = handles.filter(isDirectoryHandle) if (dirs.length > 0 && !canLinkFolders) { @@ -523,24 +538,31 @@ if (canLinkFolders) await handleAddFiles(folderEntries) else sendUserToast('Folders cannot be attached in this chat — drop individual files.', true) } - if (topLevelText.length > 0) await aiChatInput?.addTextFiles(topLevelText) + await attachNonImageFiles(topLevelText) } } async function onFileInputChange(e: Event) { const input = e.currentTarget as HTMLInputElement if (input.files && input.files.length > 0) { - const picked = Array.from(input.files) - const imageFiles = picked.filter(isImageFile) - const textFiles = picked.filter((f) => !isImageFile(f)) - // Reserved before the text work is awaited — see onPanelDrop. - const imageWork = imageFiles.length > 0 ? aiChatInput?.addImages(imageFiles) : undefined - if (textFiles.length > 0) await aiChatInput?.addTextFiles(textFiles) - await imageWork + await attachPickedFiles(Array.from(input.files)) } input.value = '' // allow re-selecting the same file } + async function attachNonImageFiles(files: File[]) { + await aiChatInput?.addNonImageFiles(files) + } + + async function attachPickedFiles(picked: File[]) { + const imageFiles = picked.filter(isImageFile) + const others = picked.filter((f) => !isImageFile(f)) + // Reserved before the other work is awaited — see onPanelDrop. + const imageWork = imageFiles.length > 0 ? aiChatInput?.addImages(imageFiles) : undefined + await attachNonImageFiles(others) + await imageWork + } + function onFolderInputChange(e: Event) { const input = e.currentTarget as HTMLInputElement // webkitdirectory files carry webkitRelativePath (`folder/sub/file`); addFiles groups @@ -622,6 +644,7 @@ const showFooterLeftControls = $derived( !footerMessageShown && (canAttachFiles || + attachmentsOffReason !== undefined || showContextPicker || showAutonomyModeSelector || (chatHost.mode === AIMode.SCRIPT && hasDiff)) @@ -991,7 +1014,21 @@ the panel, or the Escape-to-stop focus check would wrongly reject them. --> {/snippet} {/if} - {#if canAttachFiles} + {#if attachmentsOffReason} + +