diff --git a/frontend/src/lib/components/copilot/chat/AIChatManager.svelte.ts b/frontend/src/lib/components/copilot/chat/AIChatManager.svelte.ts index 5b0e549dd0..5502788024 100644 --- a/frontend/src/lib/components/copilot/chat/AIChatManager.svelte.ts +++ b/frontend/src/lib/components/copilot/chat/AIChatManager.svelte.ts @@ -61,7 +61,12 @@ import { runChatLoop } from './chatLoop' import type { ReviewChangesOpts } from './monaco-adapter' import { getCurrentModel, tryGetCurrentModel, getCombinedCustomPrompt } from '$lib/aiStore' import type { WorkspaceMutationTarget } from './workspaceTools' -import { globalToolsFor, prepareGlobalSystemMessage, prepareGlobalUserMessage } from './global/core' +import { + globalToolsFor, + prepareGlobalSystemMessage, + prepareGlobalUserMessage, + type GlobalToolHelpers +} from './global/core' import { isGlobalAiEnabled } from './global/gate' // If the estimated token usage is greater than the model context window - the threshold, we delete the oldest message @@ -512,7 +517,11 @@ export class AIChatManager { previewTools: this.isSessionChat }) this.tools = globalToolsFor({ sessionPreview: this.isSessionChat }) - this.helpers = this.isSessionChat ? { sessionId: this.sessionId } : {} + this.helpers = { + ...(this.isSessionChat ? { sessionId: this.sessionId } : {}), + testActiveFlow: async (args?: Record) => + this.flowAiChatHelpers?.testFlow(args, this.sessionId) + } satisfies GlobalToolHelpers } else if (mode === AIMode.APP) { const customPrompt = getCombinedCustomPrompt(mode) this.systemMessage = prepareAppSystemMessage(customPrompt) diff --git a/frontend/src/lib/components/copilot/chat/flow/core.ts b/frontend/src/lib/components/copilot/chat/flow/core.ts index 9de82b4132..92883fab0f 100644 --- a/frontend/src/lib/components/copilot/chat/flow/core.ts +++ b/frontend/src/lib/components/copilot/chat/flow/core.ts @@ -1,10 +1,4 @@ -import { - ScriptService, - type FlowModule, - type InputTransform, - type RawScript, - JobService -} from '$lib/gen' +import { type FlowModule, type InputTransform, type RawScript } from '$lib/gen' import type { ChatCompletionSystemMessageParam, ChatCompletionUserMessageParam @@ -30,7 +24,8 @@ import { formatScriptLintResult, type ScriptLintResult, createSearchWorkspaceTool, - createGetRunnableDetailsTool + createGetRunnableDetailsTool, + executeFlowStepTestRun } from '../shared' import { createWorkspaceMutationTools } from '../workspaceTools' import type { ContextElement } from '../context' @@ -442,99 +437,14 @@ export const flowTools: Tool[] = [ const stepId = args.stepId const stepArgs = args.args || {} - // Find the step in the flow (includes preprocessor/failure modules) - let targetModule: FlowModule | undefined = findModuleInFlow(flow.value, stepId) ?? undefined - - if (!targetModule) { - toolCallbacks.setToolStatus(toolId, { - content: `Step '${stepId}' not found in flow`, - error: `Step with id '${stepId}' does not exist in the current flow` - }) - throw new Error( - `Step with id '${stepId}' not found in flow. Available steps: ${(flow.value.modules ?? []).map((m: FlowModule) => m.id).join(', ')}` - ) - } - - const module = targetModule - const moduleValue = module.value - - if (moduleValue.type === 'rawscript') { - // Test raw script step - - return executeTestRun({ - jobStarter: () => - JobService.runScriptPreview({ - workspace: workspace, - requestBody: { - content: moduleValue.content ?? '', - language: moduleValue.language, - args: - module.id === SPECIAL_MODULE_IDS.PREPROCESSOR - ? { _ENTRYPOINT_OVERRIDE: 'preprocessor', ...stepArgs } - : stepArgs - } - }), - workspace, - toolCallbacks, - toolId, - startMessage: `Starting test run of step '${stepId}'...`, - contextName: 'script' - }) - } else if (moduleValue.type === 'script') { - // Test script step - need to get the script content - const script = moduleValue.hash - ? await ScriptService.getScriptByHash({ - workspace: workspace, - hash: moduleValue.hash - }) - : await ScriptService.getScriptByPath({ - workspace: workspace, - path: moduleValue.path - }) - - return executeTestRun({ - jobStarter: () => - JobService.runScriptPreview({ - workspace: workspace, - requestBody: { - content: script.content, - language: script.language, - args: - module.id === SPECIAL_MODULE_IDS.PREPROCESSOR - ? { _ENTRYPOINT_OVERRIDE: 'preprocessor', ...stepArgs } - : stepArgs - } - }), - workspace, - toolCallbacks, - toolId, - startMessage: `Starting test run of script step '${stepId}'...`, - contextName: 'script' - }) - } else if (moduleValue.type === 'flow') { - // Test flow step - return executeTestRun({ - jobStarter: () => - JobService.runFlowByPath({ - workspace: workspace, - path: moduleValue.path, - requestBody: stepArgs - }), - workspace, - toolCallbacks, - toolId, - startMessage: `Starting test run of flow step '${stepId}'...`, - contextName: 'flow' - }) - } else { - toolCallbacks.setToolStatus(toolId, { - content: `Step type '${moduleValue.type}' not supported for testing`, - error: `Cannot test step of type '${moduleValue.type}'` - }) - throw new Error( - `Cannot test step of type '${moduleValue.type}'. Supported types: rawscript, script, flow` - ) - } + return executeFlowStepTestRun({ + flowValue: flow.value, + stepId, + args: stepArgs, + workspace, + toolCallbacks, + toolId + }) }, requiresConfirmation: true, confirmationMessage: 'Run flow step test', diff --git a/frontend/src/lib/components/copilot/chat/global/core.test.ts b/frontend/src/lib/components/copilot/chat/global/core.test.ts index b7a53f5604..08ec244aa2 100644 --- a/frontend/src/lib/components/copilot/chat/global/core.test.ts +++ b/frontend/src/lib/components/copilot/chat/global/core.test.ts @@ -45,11 +45,30 @@ vi.mock('$lib/gen', async () => { ScriptService: wrapService(actual.ScriptService, { existsScriptByPath: vi.fn(async () => false), createScript: vi.fn(async () => 'created'), + getScriptByPath: vi.fn(async () => { + throw new Error('getScriptByPath mock not configured') + }), + getScriptByHash: vi.fn(async () => { + throw new Error('getScriptByHash mock not configured') + }), getScriptByPathWithDraft: vi.fn(async () => { throw new Error('getScriptByPathWithDraft mock not configured') }), + queryHubScripts: vi.fn(async () => []), + getHubScriptContentByPath: vi.fn(async () => ''), listScripts: vi.fn(async () => []) }), + JobService: wrapService(actual.JobService, { + runScriptPreview: vi.fn(async () => 'job-script-preview'), + runFlowPreview: vi.fn(async () => 'job-flow-preview'), + runFlowByPath: vi.fn(async () => 'job-flow-by-path'), + getJob: vi.fn(async () => ({ + type: 'CompletedJob', + success: true, + result: { ok: true }, + logs: 'test logs' + })) + }), FlowService: wrapService(actual.FlowService, { existsFlowByPath: vi.fn(async () => false), createFlow: vi.fn(async () => 'created'), @@ -124,6 +143,7 @@ import { AppService, FlowService, HttpTriggerService, + JobService, ResourceService, ScheduleService, ScriptService, @@ -170,6 +190,17 @@ function localStorageSnapshot(): string { return values.join('\n') } +async function withCompletedTestJob(run: () => Promise): Promise { + vi.useFakeTimers() + try { + const promise = run() + await vi.advanceTimersByTimeAsync(1000) + return await promise + } finally { + vi.useRealTimers() + } +} + describe('global AI tools', () => { beforeEach(() => { __resetUserDraftForTesting() @@ -178,6 +209,41 @@ describe('global AI tools', () => { vi.clearAllMocks() }) + it('exposes hub search and path-aware test tools', () => { + const names = globalTools.map((tool) => tool.def.function.name) + + expect(names).toContain('search_hub_scripts') + expect(names).toContain('test_run_script') + expect(names).toContain('test_run_flow') + expect(names).toContain('test_run_step') + }) + + it('searches hub scripts without fetching script contents', async () => { + vi.mocked(ScriptService.queryHubScripts).mockResolvedValueOnce([ + { + version_id: 7, + app: 'slack', + summary: 'Send Message' + } + ] as any) + + const raw = await callGlobalTool('search_hub_scripts', { + query: 'slack message' + }) + + expect(ScriptService.queryHubScripts).toHaveBeenCalledWith({ + text: 'slack message', + kind: 'script' + }) + expect(ScriptService.getHubScriptContentByPath).not.toHaveBeenCalled() + expect(JSON.parse(raw)).toEqual([ + { + path: 'hub/7/slack/send_message', + summary: 'Send Message' + } + ]) + }) + it('redacts variable draft values when reading workspace items', async () => { await callGlobalTool('write_variable', { path: 'f/secrets/api_key', @@ -1266,6 +1332,284 @@ describe('global AI tools', () => { expect(item.value.value).toBeUndefined() }) + it('test_run_script previews local draft script content by path', async () => { + const content = 'export async function main(name: string) {\n\treturn `hello ${name}`\n}' + await callGlobalTool('write_script', { + path: 'f/scripts/draft-test', + summary: 'Draft test script', + language: 'bun', + content + }) + + const result = await withCompletedTestJob(() => + callGlobalTool('test_run_script', { + path: 'f/scripts/draft-test', + args: { name: 'Ada' } + }) + ) + + expect(JobService.runScriptPreview).toHaveBeenCalledWith({ + workspace: WORKSPACE, + requestBody: { + path: 'f/scripts/draft-test', + content, + args: { name: 'Ada' }, + language: 'bun' + } + }) + expect(ScriptService.getScriptByPath).not.toHaveBeenCalled() + expect(result).toContain('Result (SUCCESS)') + expect(result).toContain('test logs') + }) + + it('test_run_script previews deployed script content when no local draft exists', async () => { + vi.mocked(ScriptService.getScriptByPath).mockResolvedValueOnce({ + path: 'f/scripts/deployed-test', + summary: 'Deployed test script', + content: 'def main(name):\n return name', + language: 'python3' + } as any) + + await withCompletedTestJob(() => + callGlobalTool('test_run_script', { + path: 'f/scripts/deployed-test', + args: { name: 'Grace' } + }) + ) + + expect(ScriptService.getScriptByPath).toHaveBeenCalledWith({ + workspace: WORKSPACE, + path: 'f/scripts/deployed-test' + }) + expect(JobService.runScriptPreview).toHaveBeenCalledWith({ + workspace: WORKSPACE, + requestBody: { + path: 'f/scripts/deployed-test', + content: 'def main(name):\n return name', + args: { name: 'Grace' }, + language: 'python3' + } + }) + }) + + it('test_run_flow previews local draft flow content by path', async () => { + const modules = [{ id: 'start', value: { type: 'identity' } }] + await callGlobalTool('write_flow', { + path: 'f/flows/draft-test', + summary: 'Draft test flow', + modules: JSON.stringify(modules) + }) + + await withCompletedTestJob(() => + callGlobalTool('test_run_flow', { + path: 'f/flows/draft-test', + args: { name: 'Ada' } + }) + ) + + expect(FlowService.getFlowByPath).not.toHaveBeenCalled() + expect(JobService.runFlowPreview).toHaveBeenCalledWith({ + workspace: WORKSPACE, + requestBody: { + path: 'f/flows/draft-test', + value: { modules }, + args: { name: 'Ada' } + } + }) + }) + + it('test_run_flow previews deployed flow content when no local draft exists', async () => { + const modules = [{ id: 'deployed_start', value: { type: 'identity' } }] + vi.mocked(FlowService.getFlowByPath).mockResolvedValueOnce({ + path: 'f/flows/deployed-test', + summary: 'Deployed test flow', + value: { modules }, + schema: {} + } as any) + + await withCompletedTestJob(() => + callGlobalTool('test_run_flow', { + path: 'f/flows/deployed-test', + args: { name: 'Grace' } + }) + ) + + expect(FlowService.getFlowByPath).toHaveBeenCalledWith({ + workspace: WORKSPACE, + path: 'f/flows/deployed-test' + }) + expect(JobService.runFlowPreview).toHaveBeenCalledWith({ + workspace: WORKSPACE, + requestBody: { + path: 'f/flows/deployed-test', + value: { modules }, + args: { name: 'Grace' } + } + }) + }) + + it('test_run_flow uses the live flow editor test hook when the active editor matches the path', async () => { + UserDraft.save( + 'flow', + '', + { + path: 'u/admin/live_flow', + summary: 'Live flow', + value: { modules: [{ id: 'live_step', value: { type: 'identity' } }] }, + schema: {}, + edited_by: '', + edited_at: '', + archived: false, + extra_perms: {} + }, + { workspace: WORKSPACE } + ) + UserDraft.setLiveEditorDraft({ + workspace: WORKSPACE, + itemKind: 'flow', + storagePath: '', + effectivePath: 'u/admin/live_flow' + }) + const testActiveFlow = vi.fn(async () => 'job-live-flow') + + const result = await withCompletedTestJob(() => + callGlobalTool( + 'test_run_flow', + { + path: 'u/admin/live_flow', + args: { name: 'Ada' } + }, + toolCallbacks, + { testActiveFlow } + ) + ) + + expect(testActiveFlow).toHaveBeenCalledWith({ name: 'Ada' }) + expect(FlowService.getFlowByPath).not.toHaveBeenCalled() + expect(JobService.runFlowPreview).not.toHaveBeenCalled() + expect(result).toContain('Result (SUCCESS)') + }) + + it('test_run_step previews rawscript steps from the local draft flow', async () => { + const content = 'export async function main(name: string) {\n\treturn name.toUpperCase()\n}' + await callGlobalTool('write_flow', { + path: 'f/flows/rawscript-step', + summary: 'Flow with rawscript', + modules: JSON.stringify([ + { + id: 'format_name', + value: { + type: 'rawscript', + language: 'bun', + content, + input_transforms: {} + } + } + ]) + }) + + await withCompletedTestJob(() => + callGlobalTool('test_run_step', { + path: 'f/flows/rawscript-step', + stepId: 'format_name', + args: { name: 'Ada' } + }) + ) + + expect(JobService.runScriptPreview).toHaveBeenCalledWith({ + workspace: WORKSPACE, + requestBody: { + content, + language: 'bun', + args: { name: 'Ada' } + } + }) + }) + + it('test_run_step prefers local script drafts for script steps', async () => { + const content = 'export async function main(name: string) {\n\treturn `draft ${name}`\n}' + await callGlobalTool('write_script', { + path: 'f/scripts/step-script', + summary: 'Step script', + language: 'bun', + content + }) + await callGlobalTool('write_flow', { + path: 'f/flows/script-step', + summary: 'Flow with script step', + modules: JSON.stringify([ + { + id: 'call_script', + value: { + type: 'script', + path: 'f/scripts/step-script', + input_transforms: {} + } + } + ]) + }) + + await withCompletedTestJob(() => + callGlobalTool('test_run_step', { + path: 'f/flows/script-step', + stepId: 'call_script', + args: { name: 'Ada' } + }) + ) + + expect(ScriptService.getScriptByPath).not.toHaveBeenCalled() + expect(JobService.runScriptPreview).toHaveBeenCalledWith({ + workspace: WORKSPACE, + requestBody: { + path: 'f/scripts/step-script', + content, + language: 'bun', + args: { name: 'Ada' } + } + }) + }) + + it('test_run_step previews local draft subflows for flow steps', async () => { + const nestedModules = [{ id: 'nested_start', value: { type: 'identity' } }] + await callGlobalTool('write_flow', { + path: 'f/flows/nested-draft', + summary: 'Nested draft flow', + modules: JSON.stringify(nestedModules) + }) + await callGlobalTool('write_flow', { + path: 'f/flows/parent-flow', + summary: 'Parent flow', + modules: JSON.stringify([ + { + id: 'call_flow', + value: { + type: 'flow', + path: 'f/flows/nested-draft', + input_transforms: {} + } + } + ]) + }) + + await withCompletedTestJob(() => + callGlobalTool('test_run_step', { + path: 'f/flows/parent-flow', + stepId: 'call_flow', + args: { name: 'Ada' } + }) + ) + + expect(JobService.runFlowByPath).not.toHaveBeenCalled() + expect(JobService.runFlowPreview).toHaveBeenCalledWith({ + workspace: WORKSPACE, + requestBody: { + path: 'f/flows/nested-draft', + value: { modules: nestedModules }, + args: { name: 'Ada' } + } + }) + }) + it('asks the user a question and returns the selected answer', async () => { const callbacks: ToolCallbacks = { setToolStatus: vi.fn(), @@ -1384,6 +1728,9 @@ describe('prepareGlobalSystemMessage', () => { expect(content).toContain( 'Use discard_local_draft to remove an unsaved local draft, including the matching open editor draft' ) + expect(content).toContain( + 'After creating or editing a script or flow draft, run test_run_script, test_run_flow, or test_run_step' + ) expect(content).toContain('If the user message includes an ACTIVE EDITOR section') expect(content).not.toContain('AI draft') expect(content).not.toContain('UserDraft') diff --git a/frontend/src/lib/components/copilot/chat/global/core.ts b/frontend/src/lib/components/copilot/chat/global/core.ts index 164ae4d4fc..fb4edd990c 100644 --- a/frontend/src/lib/components/copilot/chat/global/core.ts +++ b/frontend/src/lib/components/copilot/chat/global/core.ts @@ -4,6 +4,7 @@ import { FlowService, GcpTriggerService, HttpTriggerService, + JobService, KafkaTriggerService, MqttTriggerService, NatsTriggerService, @@ -55,6 +56,9 @@ import type { import { z } from 'zod' import { createToolDef, + createSearchHubScriptsTool, + executeFlowStepTestRun, + executeTestRun, findAndReplace, type CreatedResourceTriggerKind, type Tool, @@ -387,6 +391,49 @@ const patchFlowJsonSchema = z.object({ ) }) +const testRunArgsSchema = z + .record(z.string(), z.any()) + .nullable() + .optional() + .describe('Arguments to pass to the runnable. Omit or pass null when no arguments are needed.') + +const testRunScriptSchema = z.object({ + path: z.string().describe('Workspace path of the script to test.'), + args: testRunArgsSchema +}) + +const testRunScriptToolDef = createToolDef( + testRunScriptSchema, + 'test_run_script', + 'Execute a preview-style test run of a script by path, preferring local draft content when it exists.', + { strict: false } +) + +const testRunFlowSchema = z.object({ + path: z.string().describe('Workspace path of the flow to test.'), + args: testRunArgsSchema +}) + +const testRunFlowToolDef = createToolDef( + testRunFlowSchema, + 'test_run_flow', + 'Execute a preview-style test run of a flow by path, preferring local draft content when it exists.', + { strict: false } +) + +const testRunStepSchema = z.object({ + path: z.string().describe('Workspace path of the flow containing the step to test.'), + stepId: z.string().describe('The id of the step/module to test.'), + args: testRunArgsSchema +}) + +const testRunStepToolDef = createToolDef( + testRunStepSchema, + 'test_run_step', + 'Execute a test run of one step in a flow by path, preferring local draft flow/script content when it exists.', + { strict: false } +) + // ============= App tools (raw apps) ============= const backendRunnableSchema = z @@ -553,6 +600,7 @@ Rules: - Variable values are never readable. For secrets, create a secret variable and reference it from resources as "$var:path/to/variable". - Use search_resource_types before write_resource. - Use get_instructions before writing scripts, flows, resources, or apps. For scripts, pass the target language. +- After creating or editing a script or flow draft, run test_run_script, test_run_flow, or test_run_step with representative args before reporting that it works. These tools prefer local drafts, so testing does not require deployment. - When a required decision is ambiguous, use askUserQuestion with two to ten clear proposed answer strings instead of guessing. The user can also type a custom answer when none of the proposed answers fit. - Keep context targeted.${ previewTools @@ -1332,6 +1380,7 @@ export const globalTools: Tool<{}>[] = [ return getInstructions(parsed.subject, parsed.language) } }, + createSearchHubScriptsTool(false), { def: createToolDef( askUserQuestionSchema, @@ -1556,6 +1605,39 @@ export const globalTools: Tool<{}>[] = [ return patchFlowJson(parsed, ctx) } }, + { + def: testRunScriptToolDef, + fn: async (ctx) => { + const parsed = testRunScriptSchema.parse(ctx.args) + return testRunScriptByPath(parsed, ctx) + }, + requiresConfirmation: true, + confirmationMessage: 'Run script test', + showDetails: true, + autoCollapseDetails: false + }, + { + def: testRunFlowToolDef, + fn: async (ctx) => { + const parsed = testRunFlowSchema.parse(ctx.args) + return testRunFlowByPath(parsed, ctx) + }, + requiresConfirmation: true, + confirmationMessage: 'Run flow test', + showDetails: true, + autoCollapseDetails: false + }, + { + def: testRunStepToolDef, + fn: async (ctx) => { + const parsed = testRunStepSchema.parse(ctx.args) + return testRunFlowStepByPath(parsed, ctx) + }, + requiresConfirmation: true, + confirmationMessage: 'Run flow step test', + showDetails: true, + autoCollapseDetails: false + }, { def: createToolDef( deployWorkspaceItemSchema, @@ -1834,9 +1916,23 @@ type WriteDraftCtx = { // handlers below would route a backgrounded session's tool call to whatever // session the user happens to be viewing. export type SessionToolHelpers = { sessionId?: string } +export type GlobalToolHelpers = SessionToolHelpers & { + testActiveFlow?: (args?: Record) => Promise +} function sessionIdFromCtx(ctx: { helpers?: unknown }): string | undefined { - return (ctx.helpers as SessionToolHelpers | undefined)?.sessionId + return (ctx.helpers as GlobalToolHelpers | undefined)?.sessionId +} + +function activeFlowTestFromCtx( + ctx: { workspace: string; helpers?: unknown }, + path: string +): GlobalToolHelpers['testActiveFlow'] | undefined { + const activeEditor = getActiveGlobalEditorContext(ctx.workspace) + if (activeEditor?.type !== 'flow' || activeEditor.path !== path) { + return undefined + } + return (ctx.helpers as GlobalToolHelpers | undefined)?.testActiveFlow } export type OpenPreviewHandler = (req: { @@ -2468,6 +2564,136 @@ async function setFlowModuleCode( ) } +function normalizeTestRunArgs(args: Record | null | undefined): Record { + return args ?? {} +} + +function flowDraftValueForPreview(flowDraft: FlowDraftValue): FlowValue { + return flowDraftAsEditableInput(flowDraft).value +} + +async function loadScriptForFlowStep( + moduleValue: { path: string; hash?: string }, + workspace: string +): Promise<{ content: string; language: ScriptLang }> { + const draft = getGlobalDraft(workspace, 'script', moduleValue.path) + if (draft) { + if (typeof draft.value !== 'string' || !draft.language) { + throw new Error(`Draft script "${moduleValue.path}" is missing content or language.`) + } + return { content: draft.value, language: draft.language } + } + + const script = moduleValue.hash + ? await ScriptService.getScriptByHash({ workspace, hash: moduleValue.hash }) + : await ScriptService.getScriptByPath({ workspace, path: moduleValue.path }) + return { content: script.content, language: script.language } +} + +async function loadDraftFlowPreviewValue( + path: string, + workspace: string +): Promise { + if (!getGlobalDraft(workspace, 'flow', path)) { + return undefined + } + const nestedFlow = await loadFlowDraftValue(path, workspace) + return flowDraftValueForPreview(nestedFlow.flow) +} + +async function testRunScriptByPath( + args: z.infer, + ctx: WriteDraftCtx +): Promise { + const { workspace, toolId, toolCallbacks } = ctx + const script = await loadScriptForEdit(args.path, workspace) + const testArgs = normalizeTestRunArgs(args.args) + + return executeTestRun({ + jobStarter: () => + JobService.runScriptPreview({ + workspace, + requestBody: { + path: args.path, + content: script.content, + args: testArgs, + language: script.language + } + }), + workspace, + toolCallbacks, + toolId, + startMessage: `Running test for script "${args.path}"...`, + contextName: 'script' + }) +} + +async function testRunFlowByPath( + args: z.infer, + ctx: WriteDraftCtx +): Promise { + const { workspace, toolId, toolCallbacks } = ctx + const testArgs = normalizeTestRunArgs(args.args) + const testActiveFlow = activeFlowTestFromCtx(ctx, args.path) + + if (testActiveFlow) { + return executeTestRun({ + jobStarter: async () => { + const jobId = await testActiveFlow(testArgs) + if (!jobId) { + throw new Error('Failed to start test run - active flow editor returned undefined') + } + return jobId + }, + workspace, + toolCallbacks, + toolId, + startMessage: `Starting live editor flow test run for "${args.path}"...`, + contextName: 'flow' + }) + } + + const flow = await loadFlowDraftValue(args.path, workspace) + + return executeTestRun({ + jobStarter: () => + JobService.runFlowPreview({ + workspace, + requestBody: { + path: args.path, + value: flowDraftValueForPreview(flow.flow), + args: testArgs + } + }), + workspace, + toolCallbacks, + toolId, + startMessage: `Starting flow test run for "${args.path}"...`, + contextName: 'flow' + }) +} + +async function testRunFlowStepByPath( + args: z.infer, + ctx: WriteDraftCtx +): Promise { + const { workspace, toolId, toolCallbacks } = ctx + const flow = await loadFlowDraftValue(args.path, workspace) + const flowValue = flowDraftValueForPreview(flow.flow) + const testArgs = normalizeTestRunArgs(args.args) + + return executeFlowStepTestRun({ + flowValue, + stepId: args.stepId, + args: testArgs, + workspace, + toolCallbacks, + toolId, + loadScript: loadScriptForFlowStep, + loadFlowPreviewValue: loadDraftFlowPreviewValue + }) +} + async function initApp( args: { path: string @@ -3203,7 +3429,8 @@ export function prepareGlobalUserMessage( (context) => context.type === 'workspace_script' || context.type === 'workspace_flow' ) const activeEditor = - options.activeEditor ?? (options.workspace ? getActiveGlobalEditorContext(options.workspace) : undefined) + options.activeEditor ?? + (options.workspace ? getActiveGlobalEditorContext(options.workspace) : undefined) let content = '' if (activeEditor) { diff --git a/frontend/src/lib/components/copilot/chat/shared.ts b/frontend/src/lib/components/copilot/chat/shared.ts index 88ed20d664..b5018d2a2c 100644 --- a/frontend/src/lib/components/copilot/chat/shared.ts +++ b/frontend/src/lib/components/copilot/chat/shared.ts @@ -19,7 +19,7 @@ import { get } from 'svelte/store' import type { CodePieceElement, ContextElement, FlowModuleCodePieceElement } from './context' import { workspaceStore } from '$lib/stores' import type { ExtendedOpenFlow } from '$lib/components/flows/types' -import { findModuleInModules } from '$lib/components/flows/flowTree' +import { findModuleInFlow, findModuleInModules } from '$lib/components/flows/flowTree' import type { FunctionParameters } from 'openai/resources/shared.mjs' import { z } from 'zod' import { @@ -27,7 +27,9 @@ import { FlowService, JobService, type CompletedJob, + type FlowValue, type FlowModule, + type ScriptLang, type Script, type Flow } from '$lib/gen' @@ -1082,6 +1084,162 @@ export async function executeTestRun(config: TestRunConfig): Promise { } } +type FlowStepScriptLoader = ( + moduleValue: { path: string; hash?: string }, + workspace: string +) => Promise<{ content: string; language: ScriptLang }> + +type FlowStepPreviewLoader = (path: string, workspace: string) => Promise + +export type FlowStepTestRunConfig = { + flowValue: FlowValue + stepId: string + args?: Record | null + workspace: string + toolCallbacks: ToolCallbacks + toolId: string + loadScript?: FlowStepScriptLoader + loadFlowPreviewValue?: FlowStepPreviewLoader +} + +function normalizeFlowStepArgs(args: Record | null | undefined): Record { + return args ?? {} +} + +function flowStepArgsForModule(moduleId: string, args: Record): Record { + return moduleId === SPECIAL_MODULE_IDS.PREPROCESSOR + ? { _ENTRYPOINT_OVERRIDE: 'preprocessor', ...args } + : args +} + +function getAvailableFlowStepIds(flowValue: FlowValue): string { + return [ + ...(flowValue.modules ?? []).map((module: FlowModule) => module.id), + ...(flowValue.preprocessor_module ? [flowValue.preprocessor_module.id] : []), + ...(flowValue.failure_module ? [flowValue.failure_module.id] : []) + ].join(', ') +} + +async function loadDeployedScriptForFlowStep( + moduleValue: { path: string; hash?: string }, + workspace: string +): Promise<{ content: string; language: ScriptLang }> { + const script = moduleValue.hash + ? await ScriptService.getScriptByHash({ workspace, hash: moduleValue.hash }) + : await ScriptService.getScriptByPath({ workspace, path: moduleValue.path }) + return { content: script.content, language: script.language } +} + +export async function executeFlowStepTestRun({ + flowValue, + stepId, + args, + workspace, + toolCallbacks, + toolId, + loadScript = loadDeployedScriptForFlowStep, + loadFlowPreviewValue +}: FlowStepTestRunConfig): Promise { + const targetModule = findModuleInFlow(flowValue, stepId) ?? undefined + + if (!targetModule) { + toolCallbacks.setToolStatus(toolId, { + content: `Step "${stepId}" not found in flow`, + error: `Step with id "${stepId}" does not exist in the current flow` + }) + throw new Error( + `Step with id "${stepId}" not found in flow. Available steps: ${getAvailableFlowStepIds(flowValue)}` + ) + } + + const moduleValue = targetModule.value + const stepArgs = normalizeFlowStepArgs(args) + + if (moduleValue.type === 'rawscript') { + return executeTestRun({ + jobStarter: () => + JobService.runScriptPreview({ + workspace, + requestBody: { + content: moduleValue.content ?? '', + language: moduleValue.language, + args: flowStepArgsForModule(targetModule.id, stepArgs) + } + }), + workspace, + toolCallbacks, + toolId, + startMessage: `Starting test run of step "${stepId}"...`, + contextName: 'script' + }) + } + + if (moduleValue.type === 'script') { + const script = await loadScript(moduleValue, workspace) + return executeTestRun({ + jobStarter: () => + JobService.runScriptPreview({ + workspace, + requestBody: { + path: moduleValue.path, + content: script.content, + language: script.language, + args: flowStepArgsForModule(targetModule.id, stepArgs) + } + }), + workspace, + toolCallbacks, + toolId, + startMessage: `Starting test run of script step "${stepId}"...`, + contextName: 'script' + }) + } + + if (moduleValue.type === 'flow') { + const previewValue = await loadFlowPreviewValue?.(moduleValue.path, workspace) + if (previewValue) { + return executeTestRun({ + jobStarter: () => + JobService.runFlowPreview({ + workspace, + requestBody: { + path: moduleValue.path, + value: previewValue, + args: stepArgs + } + }), + workspace, + toolCallbacks, + toolId, + startMessage: `Starting test run of draft flow step "${stepId}"...`, + contextName: 'flow' + }) + } + + return executeTestRun({ + jobStarter: () => + JobService.runFlowByPath({ + workspace, + path: moduleValue.path, + requestBody: stepArgs + }), + workspace, + toolCallbacks, + toolId, + startMessage: `Starting test run of flow step "${stepId}"...`, + contextName: 'flow' + }) + } + + toolCallbacks.setToolStatus(toolId, { + content: `Step type "${moduleValue.type}" not supported for testing`, + error: `Cannot test step of type "${moduleValue.type}"` + }) + throw new Error( + `Cannot test step of type "${moduleValue.type}". Supported types: rawscript, script, flow` + ) +} + function formatLogs(logs: string | undefined): undefined | string { if (logs && logs.trim()) { if (logs.length <= MAX_LOG_LENGTH) {