From 61a3c81d5d2358dcf6dc44ea8a0a94dd7e53dd33 Mon Sep 17 00:00:00 2001 From: centdix <40307056+centdix@users.noreply.github.com> Date: Mon, 15 Dec 2025 18:29:06 +0100 Subject: [PATCH] chore(appchat): improve prompt and tools (#7376) * nit flow * better prompt * remove files from user message * truncated files * nit * f --- .../copilot/chat/AIChatManager.svelte.ts | 1 - .../chat/__tests__/app/appEvalRunner.ts | 6 +- .../lib/components/copilot/chat/app/core.ts | 178 ++++++------------ .../copilot/chat/flow/FlowAIChat.svelte | 15 +- .../lib/components/copilot/chat/flow/core.ts | 40 ++-- 5 files changed, 82 insertions(+), 158 deletions(-) diff --git a/frontend/src/lib/components/copilot/chat/AIChatManager.svelte.ts b/frontend/src/lib/components/copilot/chat/AIChatManager.svelte.ts index 8eae180b5f..7a6a68c289 100644 --- a/frontend/src/lib/components/copilot/chat/AIChatManager.svelte.ts +++ b/frontend/src/lib/components/copilot/chat/AIChatManager.svelte.ts @@ -657,7 +657,6 @@ class AIChatManager { case AIMode.APP: userMessage = prepareAppUserMessage( oldInstructions, - this.appAiChatHelpers?.getFiles(), this.appAiChatHelpers?.getSelectedContext() ) break diff --git a/frontend/src/lib/components/copilot/chat/__tests__/app/appEvalRunner.ts b/frontend/src/lib/components/copilot/chat/__tests__/app/appEvalRunner.ts index bb0de14bc8..3f0da73c92 100644 --- a/frontend/src/lib/components/copilot/chat/__tests__/app/appEvalRunner.ts +++ b/frontend/src/lib/components/copilot/chat/__tests__/app/appEvalRunner.ts @@ -73,11 +73,7 @@ export async function runAppEval( const model = resolveModel(options?.variant, options?.model) // Build user message - const userMessage = prepareAppUserMessage( - userPrompt, - helpers.getFiles(), - helpers.getSelectedContext() - ) + const userMessage = prepareAppUserMessage(userPrompt, helpers.getSelectedContext()) // Run the base evaluation const rawResult = await runEval({ diff --git a/frontend/src/lib/components/copilot/chat/app/core.ts b/frontend/src/lib/components/copilot/chat/app/core.ts index ec305c51a4..3187eeb16c 100644 --- a/frontend/src/lib/components/copilot/chat/app/core.ts +++ b/frontend/src/lib/components/copilot/chat/app/core.ts @@ -89,6 +89,9 @@ export interface AppAIChatHelpers { // ============= Utility ============= +/** Maximum characters per file in get_files tool */ +const BATCH_FILE_SIZE_LIMIT = 2500 + /** Memoize a factory function - the factory is only called once, on first access */ const memo = (factory: () => T): (() => T) => { let cached: T | undefined @@ -97,15 +100,6 @@ const memo = (factory: () => T): (() => T) => { // ============= Frontend File Tools ============= -const getListFrontendFilesSchema = memo(() => z.object({})) -const getListFrontendFilesToolDef = memo(() => - createToolDef( - getListFrontendFilesSchema(), - 'list_frontend_files', - 'List all frontend file paths in the raw app. Returns an array of file paths without content. Use this for overview, then get specific files.' - ) -) - const getGetFrontendFileSchema = memo(() => z.object({ path: z @@ -121,15 +115,6 @@ const getGetFrontendFileToolDef = memo(() => ) ) -const getGetFrontendFilesSchema = memo(() => z.object({})) -const getGetFrontendFilesToolDef = memo(() => - createToolDef( - getGetFrontendFilesSchema(), - 'get_frontend_files', - 'Get all frontend files in the raw app. Returns a record of file paths to their content. Use list_frontend_files + get_frontend_file for large apps.' - ) -) - const getSetFrontendFileSchema = memo(() => z.object({ path: z @@ -163,15 +148,6 @@ const getDeleteFrontendFileToolDef = memo(() => // ============= Backend Runnable Tools ============= -const getListBackendRunnablesSchema = memo(() => z.object({})) -const getListBackendRunnablesToolDef = memo(() => - createToolDef( - getListBackendRunnablesSchema(), - 'list_backend_runnables', - 'List all backend runnable keys and names in the raw app. Returns an array without full content. Use this for overview, then get specific runnables.' - ) -) - const getGetBackendRunnableSchema = memo(() => z.object({ key: z.string().describe('The key of the backend runnable to get') @@ -185,15 +161,6 @@ const getGetBackendRunnableToolDef = memo(() => ) ) -const getGetBackendRunnablesSchema = memo(() => z.object({})) -const getGetBackendRunnablesToolDef = memo(() => - createToolDef( - getGetBackendRunnablesSchema(), - 'get_backend_runnables', - 'Get all backend runnables in the raw app. Returns a record of runnable keys to their configuration. Use list_backend_runnables + get_backend_runnable for large apps.' - ) -) - const getInlineScriptSchema = memo(() => z.object({ language: z.enum(['bun', 'python3']).describe('The language of the inline script'), @@ -273,7 +240,7 @@ const getGetFilesToolDef = memo(() => createToolDef( getGetFilesSchema(), 'get_files', - 'Get all files in the raw app, including both frontend files and backend runnables as separate records.' + 'Get an overview of all files in the app. Content may be truncated for large apps - use get_frontend_file or get_backend_runnable for full content of specific files.' ) ) @@ -443,7 +410,7 @@ function formatLintResultResponse(message: string, lintResult: LintResult): stri // ============= Tools Array ============= export const getAppTools = memo((): Tool[] => [ - // Combined files tool + // Combined files tool (with per-file truncation for large apps) { def: getGetFilesToolDef(), fn: async ({ helpers, toolId, toolCallbacks }) => { @@ -451,10 +418,50 @@ export const getAppTools = memo((): Tool[] => [ const files = helpers.getFiles() const frontendCount = Object.keys(files.frontend).length const backendCount = Object.keys(files.backend).length + + // Truncate each file individually to BATCH_FILE_SIZE_LIMIT + let anyTruncated = false + const truncatedFiles: AppFiles = { + frontend: {}, + backend: {} + } + + for (const [path, content] of Object.entries(files.frontend)) { + if (content.length > BATCH_FILE_SIZE_LIMIT) { + truncatedFiles.frontend[path] = + content.slice(0, BATCH_FILE_SIZE_LIMIT) + '\n... [TRUNCATED]' + anyTruncated = true + } else { + truncatedFiles.frontend[path] = content + } + } + + for (const [key, runnable] of Object.entries(files.backend)) { + const runnableCopy = { ...runnable } + if (runnableCopy.inlineScript?.content) { + const content = runnableCopy.inlineScript.content + if (content.length > BATCH_FILE_SIZE_LIMIT) { + runnableCopy.inlineScript = { + ...runnableCopy.inlineScript, + content: content.slice(0, BATCH_FILE_SIZE_LIMIT) + '\n... [TRUNCATED]' + } + anyTruncated = true + } + } + truncatedFiles.backend[key] = runnableCopy + } + toolCallbacks.setToolStatus(toolId, { - content: `Retrieved ${frontendCount} frontend files and ${backendCount} backend runnables` + content: `Retrieved ${frontendCount} frontend files and ${backendCount} backend runnables${anyTruncated ? ' (some truncated)' : ''}` }) - return JSON.stringify(files, null, 2) + + let result = JSON.stringify(truncatedFiles, null, 2) + if (anyTruncated) { + result += + '\n\nNote: Some file contents were truncated. Use get_frontend_file(path) or get_backend_runnable(key) to get full content.' + } + + return result } }, // Selected context tool @@ -473,16 +480,7 @@ export const getAppTools = memo((): Tool[] => [ return JSON.stringify(context, null, 2) } }, - // Frontend tools - list - { - def: getListFrontendFilesToolDef(), - fn: async ({ helpers, toolId, toolCallbacks }) => { - toolCallbacks.setToolStatus(toolId, { content: 'Listing frontend files...' }) - const paths = helpers.listFrontendFiles() - toolCallbacks.setToolStatus(toolId, { content: `Found ${paths.length} frontend files` }) - return JSON.stringify(paths, null, 2) - } - }, + // Frontend tools { def: getGetFrontendFileToolDef(), fn: async ({ args, helpers, toolId, toolCallbacks }) => { @@ -502,16 +500,6 @@ export const getAppTools = memo((): Tool[] => [ return content } }, - { - def: getGetFrontendFilesToolDef(), - fn: async ({ helpers, toolId, toolCallbacks }) => { - toolCallbacks.setToolStatus(toolId, { content: 'Getting frontend files...' }) - const files = helpers.getFrontendFiles() - const fileCount = Object.keys(files).length - toolCallbacks.setToolStatus(toolId, { content: `Retrieved ${fileCount} frontend files` }) - return JSON.stringify(files, null, 2) - } - }, { def: getSetFrontendFileToolDef(), fn: async ({ args, helpers, toolId, toolCallbacks }) => { @@ -548,16 +536,7 @@ export const getAppTools = memo((): Tool[] => [ return `Frontend file '${parsedArgs.path}' has been deleted successfully` } }, - // Backend tools - list - { - def: getListBackendRunnablesToolDef(), - fn: async ({ helpers, toolId, toolCallbacks }) => { - toolCallbacks.setToolStatus(toolId, { content: 'Listing backend runnables...' }) - const list = helpers.listBackendRunnables() - toolCallbacks.setToolStatus(toolId, { content: `Found ${list.length} backend runnables` }) - return JSON.stringify(list, null, 2) - } - }, + // Backend tools { def: getGetBackendRunnableToolDef(), fn: async ({ args, helpers, toolId, toolCallbacks }) => { @@ -577,16 +556,6 @@ export const getAppTools = memo((): Tool[] => [ return JSON.stringify(runnable, null, 2) } }, - { - def: getGetBackendRunnablesToolDef(), - fn: async ({ helpers, toolId, toolCallbacks }) => { - toolCallbacks.setToolStatus(toolId, { content: 'Getting backend runnables...' }) - const runnables = helpers.getBackendRunnables() - const count = Object.keys(runnables).length - toolCallbacks.setToolStatus(toolId, { content: `Retrieved ${count} backend runnables` }) - return JSON.stringify(runnables, null, 2) - } - }, { def: getSetBackendRunnableToolDef(), fn: async ({ args, helpers, toolId, toolCallbacks }) => { @@ -707,15 +676,11 @@ For inline scripts, the code must have a \`main\` function as its entrypoint. ## Available Tools ### File Management -- \`get_files()\`: Get both frontend files and backend runnables (use for small apps or full overview) -- \`list_frontend_files()\`: List all frontend file paths without content (efficient for large apps) -- \`get_frontend_file(path)\`: Get content of a specific frontend file -- \`get_frontend_files()\`: Get all frontend files with content (use list + get for large apps) +- \`get_files()\`: Get an overview of all files (content may be truncated for large files) +- \`get_frontend_file(path)\`: Get full content of a specific frontend file - \`set_frontend_file(path, content)\`: Create or update a frontend file. Returns lint diagnostics. - \`delete_frontend_file(path)\`: Delete a frontend file -- \`list_backend_runnables()\`: List all backend runnable keys and names (efficient for large apps) - \`get_backend_runnable(key)\`: Get full configuration of a specific backend runnable -- \`get_backend_runnables()\`: Get all backend runnables (use list + get for large apps) - \`set_backend_runnable(key, name, type, ...)\`: Create or update a backend runnable. Returns lint diagnostics. - \`delete_backend_runnable(key)\`: Delete a backend runnable @@ -726,12 +691,6 @@ For inline scripts, the code must have a \`main\` function as its entrypoint. - \`list_workspace_runnables(query, type?)\`: Search workspace scripts and flows - \`search_hub_scripts(query)\`: Search hub scripts -### Best Practices -For large apps with many files or runnables: -1. Use \`list_frontend_files()\` or \`list_backend_runnables()\` first to get an overview -2. Then use \`get_frontend_file(path)\` or \`get_backend_runnable(key)\` to inspect specific items -3. This approach is more efficient and avoids overwhelming the context with too much content - ## Backend Runnable Configuration When creating a backend runnable with \`set_backend_runnable\`: @@ -780,15 +739,6 @@ When creating a backend runnable with \`set_backend_runnable\`: } \`\`\` -## Instructions - -Follow the user instructions carefully. When creating a new app: -1. First use \`get_files\` to see the current state (includes wmill.d.ts showing how to call backend functions) -2. Create frontend files using \`set_frontend_file\`. This returns lint diagnostics. -3. Create backend runnables using \`set_backend_runnable\`. This returns lint diagnostics. -4. Use \`lint()\` to check for errors at any time -5. Use \`list_workspace_runnables\` or \`search_hub_scripts\` to find existing scripts/flows to reuse -6. Always fix any lint errors before finishing Windmill expects all backend runnable calls to use an object parameter structure. For example for: \`\`\`typescript @@ -808,7 +758,15 @@ await backend.myFunction() When you are using the windmill-client, do not forget that as id for variables or resources, those are path that are of the form \'u//\' or \'f//\'. -Explain what you're doing as you work. Show file contents before setting them when making significant changes. +## Instructions + +1. Start with \`get_files()\` to get an overview of all frontend files and backend runnables (content may be truncated for large files) +2. Use \`get_frontend_file(path)\` or \`get_backend_runnable(key)\` to get full content of specific files when needed +3. Make changes using \`set_frontend_file\` and \`set_backend_runnable\`. These return lint diagnostics. +4. Use \`lint()\` at the end to check for and fix any remaining errors + +When creating a new app, use \`list_workspace_runnables\` or \`search_hub_scripts\` to find existing scripts/flows to reuse. + ` if (customPrompt?.trim()) { @@ -823,7 +781,6 @@ Explain what you're doing as you work. Show file contents before setting them wh export function prepareAppUserMessage( instructions: string, - files?: AppFiles, selectedContext?: SelectedContext ): ChatCompletionUserMessageParam { let content = '' @@ -837,21 +794,6 @@ export function prepareAppUserMessage( } } - if (files) { - if (Object.keys(files.frontend).length > 0) { - content += `## CURRENT FRONTEND FILES:\n` - for (const [path, fileContent] of Object.entries(files.frontend)) { - content += `\n### ${path}\n\`\`\`\n${fileContent}\n\`\`\`\n` - } - content += '\n' - } - - if (Object.keys(files.backend).length > 0) { - content += `## CURRENT BACKEND RUNNABLES:\n` - content += '\`\`\`json\n' + JSON.stringify(files.backend, null, 2) + '\n\`\`\`\n\n' - } - } - content += `## INSTRUCTIONS:\n${instructions}` return { diff --git a/frontend/src/lib/components/copilot/chat/flow/FlowAIChat.svelte b/frontend/src/lib/components/copilot/chat/flow/FlowAIChat.svelte index 60a8b873b4..d190596b16 100644 --- a/frontend/src/lib/components/copilot/chat/flow/FlowAIChat.svelte +++ b/frontend/src/lib/components/copilot/chat/flow/FlowAIChat.svelte @@ -175,16 +175,18 @@ schema: Record | undefined ) => { try { - if (modules) { - // Restore inline script references back to full content - const restoredModules = restoreInlineScriptReferences(modules) - - // Take snapshot of current flowStore BEFORE making changes + if (modules || schema) { + // Take snapshot of current flowStore and set as beforeFlow if (!diffManager?.hasPendingChanges) { const snapshot = $state.snapshot(flowStore).val diffManager?.setBeforeFlow(snapshot) + diffManager?.setEditMode(true) } + } + if (modules) { + // Restore inline script references back to full content + const restoredModules = restoreInlineScriptReferences(modules) // Directly modify flowStore (immediate effect) flowStore.val.value.modules = restoredModules } @@ -194,10 +196,7 @@ flowStore.val.schema = schema } - diffManager?.setEditMode(true) - // Refresh the state store to update UI - // The $effect in FlowGraphV2 will automatically sync currentFlow and currentInputSchema refreshStateStore(flowStore) } catch (error) { console.error('setFlowJson error:', error) diff --git a/frontend/src/lib/components/copilot/chat/flow/core.ts b/frontend/src/lib/components/copilot/chat/flow/core.ts index 414c67c860..9a0fa98185 100644 --- a/frontend/src/lib/components/copilot/chat/flow/core.ts +++ b/frontend/src/lib/components/copilot/chat/flow/core.ts @@ -10,7 +10,6 @@ import type { ChatCompletionSystemMessageParam, ChatCompletionUserMessageParam } from 'openai/resources/chat/completions.mjs' -import type { ChatCompletionTool as ChatCompletionFunctionTool } from 'openai/resources/chat/completions.mjs' import { z } from 'zod' import uFuzzy from '@leeoniya/ufuzzy' import { emptyString } from '$lib/utils' @@ -120,29 +119,17 @@ const getInstructionsForCodeGenerationToolDef = createToolDef( ) // Using string for modules and schema because Gemini-2.5-flash performs better with strings (MALFORMED_FUNCTION_CALL errors happens more often with objects) -const setFlowJsonToolDef: ChatCompletionFunctionTool = { - type: 'function', - function: { - name: 'set_flow_json', - description: - 'Set the entire flow by providing the complete flow object. This replaces all existing modules and schema.', - strict: false, - parameters: { - type: 'object', - properties: { - modules: { - type: 'string', - description: 'JSON string containing the flow modules' - }, - schema: { - type: 'string', - description: 'JSON string containing the flow input schema' - } - }, - required: [] - } - } -} +const setFlowJsonToolSchema = z.object({ + modules: z.string().optional().nullable().describe('JSON string containing the flow modules'), + schema: z.string().optional().nullable().describe('JSON string containing the flow input schema') +}) + +const setFlowJsonToolDef = createToolDef( + setFlowJsonToolSchema, + 'set_flow_json', + 'Set the entire flow by providing the complete flow object. This replaces all existing modules and schema.', + { strict: false } +) class WorkspaceScriptsSearch { private uf: uFuzzy @@ -210,7 +197,8 @@ const testRunStepSchema = z.object({ const testRunStepToolDef = createToolDef( testRunStepSchema, 'test_run_step', - 'Execute a test run of a specific step in the flow' + 'Execute a test run of a specific step in the flow', + { strict: false } ) const inspectInlineScriptSchema = z.object({ @@ -335,7 +323,7 @@ export const flowTools: Tool[] = [ }, { // set strict to false to avoid issues with open ai models - def: { ...testRunStepToolDef, function: { ...testRunStepToolDef.function, strict: false } }, + def: testRunStepToolDef, fn: async ({ args, workspace, helpers, toolCallbacks, toolId }) => { const { flow } = helpers.getFlowAndSelectedId()