diff --git a/frontend/src/lib/components/FlowPreviewContent.svelte b/frontend/src/lib/components/FlowPreviewContent.svelte index 23efc93695..75d960ba18 100644 --- a/frontend/src/lib/components/FlowPreviewContent.svelte +++ b/frontend/src/lib/components/FlowPreviewContent.svelte @@ -419,7 +419,7 @@ >
schemaFormWithArgPicker?.resetSelected()} + onReject={() => schemaFormWithArgPicker?.resetSelected()} {inputSelected} />
diff --git a/frontend/src/lib/components/JsonEditor.svelte b/frontend/src/lib/components/JsonEditor.svelte index 24b41f9e55..800cfb0224 100644 --- a/frontend/src/lib/components/JsonEditor.svelte +++ b/frontend/src/lib/components/JsonEditor.svelte @@ -4,6 +4,7 @@ import SimpleEditor from '$lib/components/SimpleEditor.svelte' import { createEventDispatcher } from 'svelte' import { createDispatcherIfMounted } from '$lib/createDispatcherIfMounted' + export let code: string | undefined export let value: any = undefined export let error = '' diff --git a/frontend/src/lib/components/RunForm.svelte b/frontend/src/lib/components/RunForm.svelte index 7b36d50f51..a7ced0f78e 100644 --- a/frontend/src/lib/components/RunForm.svelte +++ b/frontend/src/lib/components/RunForm.svelte @@ -13,11 +13,13 @@ import TimeAgo from './TimeAgo.svelte' import Popover from './meltComponents/Popover.svelte' - import { Calendar, CornerDownLeft } from 'lucide-svelte' + import { Calendar, Check, CornerDownLeft } from 'lucide-svelte' import RunFormAdvancedPopup from './RunFormAdvancedPopup.svelte' import { page } from '$app/stores' import { replaceState } from '$app/navigation' import JsonInputs from '$lib/components/JsonInputs.svelte' + import TriggerableByAI from './TriggerableByAI.svelte' + import InputSelectedBadge from './schema/InputSelectedBadge.svelte' export let runnable: | { @@ -59,6 +61,8 @@ let reloadArgs = 0 let jsonEditor: JsonInputs | undefined = undefined let schemaHeight = 0 + let showInputSelectedBadge = false + let savedPreviousArgs: Record | undefined = undefined export async function setArgs(nargs: Record) { args = nargs @@ -97,6 +101,49 @@ } + 0 ? runnable?.summary : runnable?.path}. + ## Script description: ${runnable?.description ?? ''}. + ## Schema used: ${JSON.stringify(runnable?.schema)}. + ## Current args: ${JSON.stringify(args)}}`} + onTrigger={(value) => { + savedPreviousArgs = args + setArgs(JSON.parse(value ?? '{}')) + showInputSelectedBadge = true + }} + showAnimation={false} +/> + +{#snippet acceptButton()} + +{/snippet} + +{#if showInputSelectedBadge} + { + setArgs(savedPreviousArgs ?? {}) + savedPreviousArgs = undefined + showInputSelectedBadge = false + }} + /> +{/if}
{#if detailed} {#if runnable} @@ -137,6 +184,10 @@
{:else} +

Loading...

{/if} {/if} @@ -174,7 +225,7 @@ ? { type: 'hash', hash: runnable.hash - } + } : undefined} prettifyHeader {noVariablePicker} diff --git a/frontend/src/lib/components/ScriptBuilder.svelte b/frontend/src/lib/components/ScriptBuilder.svelte index c1065d356f..025902906b 100644 --- a/frontend/src/lib/components/ScriptBuilder.svelte +++ b/frontend/src/lib/components/ScriptBuilder.svelte @@ -12,6 +12,7 @@ } from '$lib/gen' import { inferArgs } from '$lib/infer' import { initialCode } from '$lib/script_helpers' + import AIFormSettings from './copilot/AIFormSettings.svelte' import { defaultScripts, enterpriseLicense, @@ -143,6 +144,7 @@ let metadataOpen = !neverShowMeta && (showMeta || + searchParams.get('metadata_open') == 'true' || (initialPath == '' && searchParams.get('state') == undefined && searchParams.get('collab') == undefined)) @@ -1047,6 +1049,13 @@ }} /> + {#if script.schema} +
+ +
+ {/if}
diff --git a/frontend/src/lib/components/TriggerableByAI.svelte b/frontend/src/lib/components/TriggerableByAI.svelte index f4569d844e..0c6e2a049b 100644 --- a/frontend/src/lib/components/TriggerableByAI.svelte +++ b/frontend/src/lib/components/TriggerableByAI.svelte @@ -1,11 +1,18 @@ + { + goto(href) + }} +/>
+ import { Button } from '$lib/components/common' + import { Pencil } from 'lucide-svelte' + import { aiChatManager } from './chat/AIChatManager.svelte' + import AskAiButton from './AskAiButton.svelte' + + interface Props { + onEditInstructions: () => void + instructions: string + runnableType: 'script' | 'flow' + } + + const { onEditInstructions, instructions, runnableType }: Props = $props() + + async function fillFormWithAI() { + aiChatManager.openChat() + aiChatManager.askAi(`Analyze the ${runnableType} form on this page and fill the inputs for me`) + } + + +
+
+

Fill the inputs with AI

+ +
+
+

+ {instructions + ? 'Instructions: ' + instructions + : 'No AI instructions provided. Click edit to add guidance for AI form filling.'} +

+
+
diff --git a/frontend/src/lib/components/copilot/AIFormSettings.svelte b/frontend/src/lib/components/copilot/AIFormSettings.svelte new file mode 100644 index 0000000000..3454c192c0 --- /dev/null +++ b/frontend/src/lib/components/copilot/AIFormSettings.svelte @@ -0,0 +1,37 @@ + + +
+ { + if (prompt !== undefined) { + prompt = undefined + } else { + prompt = '' + } + }} + options={{ + right: 'Enable filling script inputs with AI' + }} + /> + {#if prompt !== undefined} + + {/if} +
diff --git a/frontend/src/lib/components/copilot/chat/AIChatManager.svelte.ts b/frontend/src/lib/components/copilot/chat/AIChatManager.svelte.ts index edb407093f..9f79b0cbff 100644 --- a/frontend/src/lib/components/copilot/chat/AIChatManager.svelte.ts +++ b/frontend/src/lib/components/copilot/chat/AIChatManager.svelte.ts @@ -32,7 +32,10 @@ import { askTools, prepareAskSystemMessage } from './ask/core' type TriggerablesMap = Record< string, - { description: string; onTrigger: ((value?: string) => void) | undefined } + { + description: string + onTrigger: ((value?: string) => void) | undefined + } > export enum AIMode { diff --git a/frontend/src/lib/components/copilot/chat/navigator/core.ts b/frontend/src/lib/components/copilot/chat/navigator/core.ts index e438d13501..be07b37c41 100644 --- a/frontend/src/lib/components/copilot/chat/navigator/core.ts +++ b/frontend/src/lib/components/copilot/chat/navigator/core.ts @@ -6,6 +6,9 @@ import type { } from 'openai/resources/index.mjs' import type { Tool } from '../shared' import { aiChatManager } from '../AIChatManager.svelte' +import { ResourceService } from '$lib/gen' +import { workspaceStore } from '$lib/stores' +import { get } from 'svelte/store' export const CHAT_SYSTEM_PROMPT = ` You are Windmill's intelligent assistant, designed to help users navigate the application and answer questions about its functionality. It is your only purpose to help the user in the context of the windmill application. @@ -23,6 +26,9 @@ INSTRUCTIONS: - Use get_triggerable_components to understand available options, and then trigger the components using trigger_component. Then wait a moment before rescanning the current page, and then continue with the next step. Do this 5 times max. - Make sure you navigated as far as possible before responding to the user. Always use get_triggerable_components one last time to make sure you didn't miss anything. - If you are not able to fulfill the user's request after 5 attempts, redirect the user to the documentation. +- If you are asked to fill a form or act on an input, input the existing json object and change the fields the user asked you to change. Take into account the prompt_for_ai field of the schema to know what and how to do changes. Then tell the user that you have updated the form, and ask him to review the changes before running the script or flow. +- For form inputs where format starts with "resource-" and is not "resource-obj", fetch the available resources using get_available_resources, and then use the resource_path prefixed with "$res:" to fill the input. +- If you are not sure about an input, set the ones you are sure about, and then ask the user for the value of the input you are not sure about. GENERAL PRINCIPLES: - Be concise but thorough @@ -33,6 +39,7 @@ GENERAL PRINCIPLES: - When you do not find what you are looking for on the current page, go to the home page by looking for the "Home" component, then scan the components again. IMPORTANT CONSIDERATIONS: +- The user might have changed the page in the middle of the conversation, so make sure you rescan the page on each user request instead of just responding that you cannot find what the user is asking for. - If you navigate to a script creation page, consider this: - The page opens with the settings drawer open. After doing the changes mentioned by the user, close the settings drawer. - Then if the user has described what he wanted the script to do, switch to script mode with the change_mode tool, and use the new tools you'll have access to to edit the script. @@ -99,12 +106,13 @@ const EXECUTE_COMMAND_TOOL: ChatCompletionTool = { type: 'string', description: 'Value to pass to the AI-triggerable component trigger function' }, - description: { + actionTaken: { type: 'string', - description: 'Description of the component' + description: + 'Short description of the action taken. Can be clicked, filled, etc. Includes which component was triggered.' } }, - required: ['id', 'description'] + required: ['id', 'actionTaken'] } } } @@ -122,6 +130,24 @@ const GET_CURRENT_PAGE_NAME_TOOL: ChatCompletionTool = { } } +const GET_AVAILABLE_RESOURCES_TOOL: ChatCompletionTool = { + type: 'function', + function: { + name: 'get_available_resources', + description: 'Get the available resources to the user', + parameters: { + type: 'object', + properties: { + resource_type: { + type: 'string', + description: 'The type of resource to get, separated by ","' + } + }, + required: ['resource_type'] + } + } +} + function getTriggerableComponents(): string { try { // Get components registered in the triggerablesByAI store @@ -135,7 +161,12 @@ function getTriggerableComponents(): string { // List each registered component with its ID and description Object.entries(registeredComponents).forEach(([id, component], index) => { - result += `[${index}] ID: "${id}" - Description: ${component.description} - Triggerable: ${component.onTrigger ? 'Yes' : 'No'}\n` + result += ` + [${index}] + ID: "${id}" + Description: ${component.description} + Triggerable: ${component.onTrigger ? 'Yes' : 'No'} + \n` }) return result @@ -234,14 +265,22 @@ async function getDocumentation(args: { request: string }): Promise { return data.choices[0].message.content } +async function getAvailableResources(args: { resource_type: string }): Promise { + const resources = await ResourceService.listResource({ + workspace: get(workspaceStore) as string, + resourceType: args.resource_type + }) + return JSON.stringify(resources) +} + const triggerComponentTool: Tool<{}> = { def: EXECUTE_COMMAND_TOOL, fn: async ({ args, toolId, toolCallbacks }) => { - toolCallbacks.onToolCall(toolId, 'Clicking on component...') + toolCallbacks.onToolCall(toolId, 'Triggering component...') const result = triggerComponent(args) toolCallbacks.onFinishToolCall( toolId, - 'Clicked ' + args.description.charAt(0).toLowerCase() + args.description.slice(1) + args.actionTaken.charAt(0).toUpperCase() + args.actionTaken.slice(1) ) return result } @@ -250,9 +289,9 @@ const triggerComponentTool: Tool<{}> = { const getTriggerableComponentsTool: Tool<{}> = { def: GET_TRIGGERABLE_COMPONENTS_TOOL, fn: async ({ toolId, toolCallbacks }) => { - toolCallbacks.onToolCall(toolId, 'Looking for screen components...') + toolCallbacks.onToolCall(toolId, 'Scanning the page...') const components = getTriggerableComponents() - toolCallbacks.onFinishToolCall(toolId, 'Retrieved screen components') + toolCallbacks.onFinishToolCall(toolId, 'Scanned the page') return components } } @@ -270,9 +309,31 @@ export const getDocumentationTool: Tool<{}> = { def: GET_DOCUMENTATION_TOOL, fn: async ({ args, toolId, toolCallbacks }) => { toolCallbacks.onToolCall(toolId, 'Getting documentation...') - const docResult = await getDocumentation(args) - toolCallbacks.onFinishToolCall(toolId, 'Retrieved documentation') - return docResult + try { + const docResult = await getDocumentation(args) + toolCallbacks.onFinishToolCall(toolId, 'Retrieved documentation') + return docResult + } catch (error) { + toolCallbacks.onFinishToolCall(toolId, 'Error getting documentation') + console.error('Error getting documentation:', error) + return 'Failed to get documentation, pursuing with the user request...' + } + } +} + +const getAvailableResourcesTool: Tool<{}> = { + def: GET_AVAILABLE_RESOURCES_TOOL, + fn: async ({ args, toolId, toolCallbacks }) => { + toolCallbacks.onToolCall(toolId, 'Getting available resources...') + try { + const resources = await getAvailableResources(args) + toolCallbacks.onFinishToolCall(toolId, 'Retrieved available resources') + return resources + } catch (error) { + toolCallbacks.onFinishToolCall(toolId, 'Error getting available resources') + console.error('Error getting available resources:', error) + return 'Failed to get available resources, pursuing with the user request...' + } } } @@ -280,7 +341,8 @@ export const navigatorTools: Tool<{}>[] = [ getTriggerableComponentsTool, triggerComponentTool, getDocumentationTool, - getCurrentPageNameTool + getCurrentPageNameTool, + getAvailableResourcesTool ] export function prepareNavigatorSystemMessage(): ChatCompletionSystemMessageParam { diff --git a/frontend/src/lib/components/flows/content/FlowSettings.svelte b/frontend/src/lib/components/flows/content/FlowSettings.svelte index 85b604bf5d..31af301b12 100644 --- a/frontend/src/lib/components/flows/content/FlowSettings.svelte +++ b/frontend/src/lib/components/flows/content/FlowSettings.svelte @@ -18,6 +18,7 @@ import MetadataGen from '$lib/components/copilot/MetadataGen.svelte' import Badge from '$lib/components/Badge.svelte' import { AlertTriangle } from 'lucide-svelte' + import AIFormSettings from '$lib/components/copilot/AIFormSettings.svelte' export let noEditor: boolean @@ -112,6 +113,10 @@ }} /> + + {#if $flowStore.schema} + + {/if}
diff --git a/frontend/src/lib/components/schema/InputSelectedBadge.svelte b/frontend/src/lib/components/schema/InputSelectedBadge.svelte index 9538fffc95..7abadf14fd 100644 --- a/frontend/src/lib/components/schema/InputSelectedBadge.svelte +++ b/frontend/src/lib/components/schema/InputSelectedBadge.svelte @@ -3,28 +3,46 @@ import { twMerge } from 'tailwind-merge' import Button from '$lib/components/common/button/Button.svelte' import { X } from 'lucide-svelte' + import type { Snippet } from 'svelte' - export let inputSelected: 'history' | 'captures' | 'saved' | undefined = undefined + interface Props { + inputSelected: 'history' | 'captures' | 'saved' | 'ai' | undefined + labelColor?: string + className?: string + acceptButton?: Snippet + onReject: () => void + } + + let { inputSelected, className = '', acceptButton, onReject, labelColor = '' }: Props = $props()
-

+

Using {inputSelected === 'history' ? 'historic' : inputSelected === 'captures' - ? 'captures' - : 'saved'} input arguments + ? 'captures' + : inputSelected === 'ai' + ? 'AI generated' + : 'saved'} input arguments

+ {#if acceptButton} + {@render acceptButton()} + {/if}
diff --git a/frontend/src/routes/(root)/(logged)/+layout.svelte b/frontend/src/routes/(root)/(logged)/+layout.svelte index bc26efb9e4..0c8b40c8c8 100644 --- a/frontend/src/routes/(root)/(logged)/+layout.svelte +++ b/frontend/src/routes/(root)/(logged)/+layout.svelte @@ -463,6 +463,7 @@ label="Ask AI" class="!text-xs" iconClasses="!text-violet-400 dark:!text-violet-400" + shortcut={`${getModifierKey()}L`} /> @@ -534,6 +535,7 @@ label="Ask AI" class="!text-xs" iconClasses="!text-violet-400 dark:!text-violet-400" + shortcut={`${getModifierKey()}L`} /> @@ -645,6 +647,7 @@ label="Ask AI" class="!text-xs" iconClasses="!text-violet-400 dark:!text-violet-400" + shortcut={`${getModifierKey()}L`} /> diff --git a/frontend/src/routes/(root)/(logged)/flows/get/[...path]/+page.svelte b/frontend/src/routes/(root)/(logged)/flows/get/[...path]/+page.svelte index 55bf1d5d1c..443e73fc87 100644 --- a/frontend/src/routes/(root)/(logged)/flows/get/[...path]/+page.svelte +++ b/frontend/src/routes/(root)/(logged)/flows/get/[...path]/+page.svelte @@ -23,6 +23,7 @@ import { sendUserToast } from '$lib/toast' import DeployWorkspaceDrawer from '$lib/components/DeployWorkspaceDrawer.svelte' import SavedInputsV2 from '$lib/components/SavedInputsV2.svelte' + import AIFormAssistant from '$lib/components/copilot/AIFormAssistant.svelte' import { FolderOpen, Archive, @@ -512,7 +513,7 @@
{ + onReject={() => { savedInputsV2?.resetSelected() }} {inputSelected} @@ -532,6 +533,16 @@ />
+ {#if flow.schema?.prompt_for_ai !== undefined} + { + goto(`/flows/edit/${flow?.path}`) + }} + runnableType="flow" + /> + {/if} +
{ + onReject={() => { savedInputsV2?.resetSelected() }} {inputSelected} @@ -695,6 +696,16 @@ />
+ {#if script?.schema?.prompt_for_ai !== undefined} + { + goto(`/scripts/edit/${script?.path}?metadata_open=true`) + }} + runnableType="script" + /> + {/if} +