diff --git a/ai_evals/adapters/frontend/core/flow/fileHelpers.ts b/ai_evals/adapters/frontend/core/flow/fileHelpers.ts index 6148541526..f05a527b23 100644 --- a/ai_evals/adapters/frontend/core/flow/fileHelpers.ts +++ b/ai_evals/adapters/frontend/core/flow/fileHelpers.ts @@ -6,8 +6,7 @@ import type { FlowAIChatHelpers } from '../../../../../frontend/src/lib/componen import type { ScriptLintResult } from '../../../../../frontend/src/lib/components/copilot/chat/shared' import { findModuleById } from '../../../../../frontend/src/lib/components/copilot/chat/shared' import { - inlineScriptStore, - restoreInlineScriptReferences + createInlineScriptSession } from '../../../../../frontend/src/lib/components/copilot/chat/flow/inlineScriptsUtils' import { registerBenchmarkWorkspaceRunnables, @@ -50,6 +49,7 @@ export async function createFlowFileHelpers( type: 'object' } } + const inlineScriptSession = createInlineScriptSession() const flowFilePath = workspaceRoot ? join(workspaceRoot, 'flow.json') : null @@ -74,6 +74,7 @@ export async function createFlowFileHelpers( const module = findModuleById(flow.value.modules, id) return module ? [module] : [] }, + inlineScriptSession, setSnapshot: () => {}, revertToSnapshot: () => {}, setCode: async (id: string, code: string) => { @@ -81,7 +82,7 @@ export async function createFlowFileHelpers( if (module && module.value.type === 'rawscript') { module.value.content = code } - inlineScriptStore.set(id, code) + inlineScriptSession.set(id, code) await persistFlow() }, setFlowJson: async ( @@ -89,7 +90,13 @@ export async function createFlowFileHelpers( schema: Record | undefined ) => { if (modules) { - flow.value.modules = restoreInlineScriptReferences(modules) + flow.value.modules = inlineScriptSession.restoreInlineScriptReferences(modules) + const unresolvedRefs = inlineScriptSession.findUnresolvedInlineScriptRefs(flow.value.modules) + if (unresolvedRefs.length > 0) { + throw new Error( + `Unresolved inline script references: ${unresolvedRefs.join(', ')}` + ) + } } if (schema !== undefined) { flow.schema = schema diff --git a/ai_evals/adapters/frontend/core/flow/flowEvalRunner.ts b/ai_evals/adapters/frontend/core/flow/flowEvalRunner.ts index b73e2d5ac0..bf63d5c0d1 100644 --- a/ai_evals/adapters/frontend/core/flow/flowEvalRunner.ts +++ b/ai_evals/adapters/frontend/core/flow/flowEvalRunner.ts @@ -60,7 +60,12 @@ export async function runFlowEval( const systemMessage = prepareFlowSystemMessage() const tools = flowTools as ProductionTool[] const model = options?.model ?? 'claude-haiku-4-5-20251001' - const userMessage = prepareFlowUserMessage(userPrompt, helpers.getFlowAndSelectedId(), []) + const userMessage = prepareFlowUserMessage( + userPrompt, + helpers.getFlowAndSelectedId(), + [], + helpers.inlineScriptSession + ) const rawResult = await runEval({ userPrompt, diff --git a/frontend/src/lib/components/copilot/chat/AIChatManager.svelte.ts b/frontend/src/lib/components/copilot/chat/AIChatManager.svelte.ts index 140bbf4f14..b17b09bd12 100644 --- a/frontend/src/lib/components/copilot/chat/AIChatManager.svelte.ts +++ b/frontend/src/lib/components/copilot/chat/AIChatManager.svelte.ts @@ -452,7 +452,9 @@ class AIChatManager { } else if (this.mode === AIMode.FLOW) { return prepareFlowUserMessage( pendingPrompt, - this.flowAiChatHelpers!.getFlowAndSelectedId() + this.flowAiChatHelpers!.getFlowAndSelectedId(), + [], + this.flowAiChatHelpers!.inlineScriptSession ) } else if (this.mode === AIMode.NAVIGATOR) { return prepareNavigatorUserMessage(pendingPrompt) @@ -650,7 +652,8 @@ class AIChatManager { userMessage = prepareFlowUserMessage( oldInstructions, this.flowAiChatHelpers!.getFlowAndSelectedId(), - oldSelectedContext + oldSelectedContext, + this.flowAiChatHelpers!.inlineScriptSession ) break case AIMode.NAVIGATOR: diff --git a/frontend/src/lib/components/copilot/chat/flow/FlowAIChat.svelte b/frontend/src/lib/components/copilot/chat/flow/FlowAIChat.svelte index 1a567db370..8fd4b35003 100644 --- a/frontend/src/lib/components/copilot/chat/flow/FlowAIChat.svelte +++ b/frontend/src/lib/components/copilot/chat/flow/FlowAIChat.svelte @@ -5,7 +5,7 @@ import { dfs } from '$lib/components/flows/previousResults' import type { FlowModule, InputTransform, OpenFlow } from '$lib/gen' import type { FlowAIChatHelpers } from './core' - import { restoreInlineScriptReferences } from './inlineScriptsUtils' + import { createInlineScriptSession } from './inlineScriptsUtils' import { loadSchemaFromModule } from '$lib/components/flows/flowInfers' import { aiChatManager } from '../AIChatManager.svelte' import { refreshStateStore } from '$lib/svelte5Utils.svelte' @@ -27,6 +27,7 @@ const selectedId = $derived(selectionManager.getSelectedId()) const { exprsToSet } = getContext('FlowCopilotContext') ?? {} + const inlineScriptSession = createInlineScriptSession() // Get diffManager from the graph const diffManager = $derived(flowModuleSchemaMap?.getDiffManager()) @@ -62,6 +63,7 @@ } return flowStore.val.value.modules }, + inlineScriptSession, setSnapshot: (snapshot: ExtendedOpenFlow) => { diffManager?.setBeforeFlow(snapshot) }, @@ -103,6 +105,7 @@ // 2. Apply the code change module.value.content = code + inlineScriptSession.set(id, code) const { input_transforms, schema } = await loadSchemaFromModule(module) module.value.input_transforms = input_transforms refreshStateStore(flowStore) @@ -216,7 +219,13 @@ if (modules) { // Restore inline script references back to full content - const restoredModules = restoreInlineScriptReferences(modules) + const restoredModules = inlineScriptSession.restoreInlineScriptReferences(modules) + const unresolvedRefs = inlineScriptSession.findUnresolvedInlineScriptRefs(restoredModules) + if (unresolvedRefs.length > 0) { + throw new Error( + `Unresolved inline script references: ${unresolvedRefs.join(', ')}` + ) + } // Directly modify flowStore (immediate effect) flowStore.val.value.modules = restoredModules } diff --git a/frontend/src/lib/components/copilot/chat/flow/core.ts b/frontend/src/lib/components/copilot/chat/flow/core.ts index f695da0aa1..e8fd54020a 100644 --- a/frontend/src/lib/components/copilot/chat/flow/core.ts +++ b/frontend/src/lib/components/copilot/chat/flow/core.ts @@ -34,7 +34,7 @@ import { } from '../shared' import type { ContextElement } from '../context' import type { ExtendedOpenFlow } from '$lib/components/flows/types' -import { inlineScriptStore, extractAndReplaceInlineScripts } from './inlineScriptsUtils' +import type { InlineScriptSession } from './inlineScriptsUtils' import { flowModulesSchema } from './openFlowZod' import { collectAllModuleIdsFromArray } from './utils' import { getFlowPrompt } from '$system_prompts' @@ -247,6 +247,7 @@ export interface FlowAIChatHelpers { // flow context getFlowAndSelectedId: () => { flow: ExtendedOpenFlow; selectedId: string } getModules: (id?: string) => FlowModule[] + inlineScriptSession: InlineScriptSession // snapshot management (AI sets this when making changes) /** Set the before flow snapshot */ @@ -581,7 +582,7 @@ export const flowTools: Tool[] = [ }, { def: inspectInlineScriptToolDef, - fn: async ({ args, toolCallbacks, toolId }) => { + fn: async ({ args, helpers, toolCallbacks, toolId }) => { const parsedArgs = inspectInlineScriptSchema.parse(args) const moduleId = parsedArgs.moduleId @@ -589,7 +590,7 @@ export const flowTools: Tool[] = [ content: `Retrieving inline script content for module '${moduleId}'...` }) - const content = inlineScriptStore.get(moduleId) + const content = helpers.inlineScriptSession.get(moduleId) if (content === undefined) { toolCallbacks.setToolStatus(toolId, { @@ -623,9 +624,6 @@ export const flowTools: Tool[] = [ toolCallbacks.setToolStatus(toolId, { content: `Setting code for module '${moduleId}'...` }) - // Update store to keep it coherent (for subsequent set_flow_json calls with references) - inlineScriptStore.set(moduleId, code) - // Update the flow directly via helper await helpers.setCode(moduleId, code) @@ -1057,7 +1055,8 @@ You have access to the following contexts: export function prepareFlowUserMessage( instructions: string, flowAndSelectedId?: { flow: ExtendedOpenFlow; selectedId: string }, - selectedContext: ContextElement[] = [] + selectedContext: ContextElement[] = [], + inlineScriptSession?: InlineScriptSession ): ChatCompletionUserMessageParam { const flow = flowAndSelectedId?.flow const selectedId = flowAndSelectedId?.selectedId @@ -1075,10 +1074,13 @@ ${instructions}` } const codePieces = selectedContext.filter((c) => c.type === 'flow_module_code_piece') + const scriptSession = inlineScriptSession // Clear the inline script store and extract inline scripts for token optimization - inlineScriptStore.clear() - const optimizedModules = extractAndReplaceInlineScripts(flow.value.modules) + scriptSession?.clear() + const optimizedModules = scriptSession + ? scriptSession.extractAndReplaceInlineScripts(flow.value.modules) + : flow.value.modules // Apply code pieces to the optimized modules (returns YAML string) const flowModulesYaml = applyCodePiecesToFlowModules(codePieces, optimizedModules) @@ -1086,7 +1088,7 @@ ${instructions}` // Handle preprocessor and failure modules let optimizedPreprocessor = flow.value.preprocessor_module if (optimizedPreprocessor?.value?.type === 'rawscript' && optimizedPreprocessor.value.content) { - inlineScriptStore.set(optimizedPreprocessor.id, optimizedPreprocessor.value.content) + scriptSession?.set(optimizedPreprocessor.id, optimizedPreprocessor.value.content) optimizedPreprocessor = { ...optimizedPreprocessor, value: { @@ -1098,7 +1100,7 @@ ${instructions}` let optimizedFailure = flow.value.failure_module if (optimizedFailure?.value?.type === 'rawscript' && optimizedFailure.value.content) { - inlineScriptStore.set(optimizedFailure.id, optimizedFailure.value.content) + scriptSession?.set(optimizedFailure.id, optimizedFailure.value.content) optimizedFailure = { ...optimizedFailure, value: { diff --git a/frontend/src/lib/components/copilot/chat/flow/inlineScriptsUtils.ts b/frontend/src/lib/components/copilot/chat/flow/inlineScriptsUtils.ts index 9a124c44d7..4180b7e36b 100644 --- a/frontend/src/lib/components/copilot/chat/flow/inlineScriptsUtils.ts +++ b/frontend/src/lib/components/copilot/chat/flow/inlineScriptsUtils.ts @@ -1,10 +1,17 @@ import type { FlowModule } from '$lib/gen' -/** - * Storage for inline scripts extracted from flow modules. - * Maps module IDs to their rawscript content for token-efficient transmission to AI. - */ -class InlineScriptStore { +export interface InlineScriptSession { + clear(): void + set(moduleId: string, content: string): void + get(moduleId: string): string | undefined + has(moduleId: string): boolean + getAll(): Record + extractAndReplaceInlineScripts(modules: FlowModule[]): FlowModule[] + restoreInlineScriptReferences(modules: FlowModule[]): FlowModule[] + findUnresolvedInlineScriptRefs(modules: FlowModule[]): string[] +} + +class DefaultInlineScriptSession implements InlineScriptSession { private scripts: Map = new Map() clear() { @@ -26,15 +33,28 @@ class InlineScriptStore { getAll(): Record { return Object.fromEntries(this.scripts.entries()) } + + extractAndReplaceInlineScripts(modules: FlowModule[]): FlowModule[] { + return extractAndReplaceInlineScripts(modules, this) + } + + restoreInlineScriptReferences(modules: FlowModule[]): FlowModule[] { + return restoreInlineScriptReferences(modules, this) + } + + findUnresolvedInlineScriptRefs(modules: FlowModule[]): string[] { + return findUnresolvedInlineScriptRefs(modules) + } } -export const inlineScriptStore = new InlineScriptStore() +export function createInlineScriptSession(): InlineScriptSession { + return new DefaultInlineScriptSession() +} -/** - * Recursively extracts all rawscript content from flow modules and stores them. - * Replaces the content with references like "inline_script.{module_id}". - */ -export function extractAndReplaceInlineScripts(modules: FlowModule[]): FlowModule[] { +function extractAndReplaceInlineScripts( + modules: FlowModule[], + session: Pick +): FlowModule[] { if (!modules || !Array.isArray(modules)) { return [] } @@ -43,52 +63,45 @@ export function extractAndReplaceInlineScripts(modules: FlowModule[]): FlowModul const newModule = { ...module } if (newModule.value.type === 'rawscript' && newModule.value.content) { - // Store the original content - inlineScriptStore.set(module.id, newModule.value.content) - - // Replace with reference + session.set(module.id, newModule.value.content) newModule.value = { ...newModule.value, content: `inline_script.${module.id}` } } else if (newModule.value.type === 'forloopflow' || newModule.value.type === 'whileloopflow') { - // Recursively process nested modules in loops if (newModule.value.modules) { newModule.value = { ...newModule.value, - modules: extractAndReplaceInlineScripts(newModule.value.modules) + modules: extractAndReplaceInlineScripts(newModule.value.modules, session) } } } else if (newModule.value.type === 'branchone') { - // Process branches and default modules if (newModule.value.branches) { newModule.value = { ...newModule.value, branches: newModule.value.branches.map((branch) => ({ ...branch, - modules: branch.modules ? extractAndReplaceInlineScripts(branch.modules) : [] + modules: branch.modules ? extractAndReplaceInlineScripts(branch.modules, session) : [] })) } } if (newModule.value.default) { newModule.value = { ...newModule.value, - default: extractAndReplaceInlineScripts(newModule.value.default) + default: extractAndReplaceInlineScripts(newModule.value.default, session) } } } else if (newModule.value.type === 'branchall') { - // Process all branches if (newModule.value.branches) { newModule.value = { ...newModule.value, branches: newModule.value.branches.map((branch) => ({ ...branch, - modules: branch.modules ? extractAndReplaceInlineScripts(branch.modules) : [] + modules: branch.modules ? extractAndReplaceInlineScripts(branch.modules, session) : [] })) } } } else if (newModule.value.type === 'aiagent') { - // Process AI agent tools if (newModule.value.tools) { newModule.value = { ...newModule.value, @@ -102,7 +115,7 @@ export function extractAndReplaceInlineScripts(modules: FlowModule[]): FlowModul 'content' in tool.value && tool.value.content ) { - inlineScriptStore.set(tool.id, tool.value.content as string) + session.set(tool.id, tool.value.content as string) return { ...tool, value: { @@ -121,70 +134,58 @@ export function extractAndReplaceInlineScripts(modules: FlowModule[]): FlowModul }) } -/** - * Recursively restores inline script references back to their full content. - * If content matches pattern "inline_script.{id}", looks up and restores the original. - * If content doesn't match (new/modified script), keeps it as-is. - */ -export function restoreInlineScriptReferences(modules: FlowModule[]): FlowModule[] { +function restoreInlineScriptReferences( + modules: FlowModule[], + session: Pick +): FlowModule[] { return modules.map((module) => { const newModule = { ...module } if (newModule.value.type === 'rawscript' && newModule.value.content) { - const content = newModule.value.content - // Check if it's a reference - const match = content.match(/^inline_script\.(.+)$/) + const match = newModule.value.content.match(/^inline_script\.(.+)$/) if (match) { - const moduleId = match[1] - const storedContent = inlineScriptStore.get(moduleId) + const storedContent = session.get(match[1]) if (storedContent !== undefined) { - // Restore original content newModule.value = { ...newModule.value, content: storedContent } } - // If not found in store, keep the reference as-is (shouldn't happen normally) } - // If not a reference, it's new/modified content - keep as-is } else if (newModule.value.type === 'forloopflow' || newModule.value.type === 'whileloopflow') { - // Recursively process nested modules in loops if (newModule.value.modules) { newModule.value = { ...newModule.value, - modules: restoreInlineScriptReferences(newModule.value.modules) + modules: restoreInlineScriptReferences(newModule.value.modules, session) } } } else if (newModule.value.type === 'branchone') { - // Process branches and default modules if (newModule.value.branches) { newModule.value = { ...newModule.value, branches: newModule.value.branches.map((branch) => ({ ...branch, - modules: branch.modules ? restoreInlineScriptReferences(branch.modules) : [] + modules: branch.modules ? restoreInlineScriptReferences(branch.modules, session) : [] })) } } if (newModule.value.default) { newModule.value = { ...newModule.value, - default: restoreInlineScriptReferences(newModule.value.default) + default: restoreInlineScriptReferences(newModule.value.default, session) } } } else if (newModule.value.type === 'branchall') { - // Process all branches if (newModule.value.branches) { newModule.value = { ...newModule.value, branches: newModule.value.branches.map((branch) => ({ ...branch, - modules: branch.modules ? restoreInlineScriptReferences(branch.modules) : [] + modules: branch.modules ? restoreInlineScriptReferences(branch.modules, session) : [] })) } } } else if (newModule.value.type === 'aiagent') { - // Process AI agent tools if (newModule.value.tools) { newModule.value = { ...newModule.value, @@ -198,11 +199,9 @@ export function restoreInlineScriptReferences(modules: FlowModule[]): FlowModule 'content' in tool.value && tool.value.content ) { - const content = tool.value.content as string - const match = content.match(/^inline_script\.(.+)$/) + const match = (tool.value.content as string).match(/^inline_script\.(.+)$/) if (match) { - const toolId = match[1] - const storedContent = inlineScriptStore.get(toolId) + const storedContent = session.get(match[1]) if (storedContent !== undefined) { return { ...tool, @@ -224,11 +223,7 @@ export function restoreInlineScriptReferences(modules: FlowModule[]): FlowModule }) } -/** - * Recursively finds any unresolved inline script references in flow modules. - * Returns array of module IDs that still have `inline_script.{id}` patterns. - */ -export function findUnresolvedInlineScriptRefs(modules: FlowModule[]): string[] { +function findUnresolvedInlineScriptRefs(modules: FlowModule[]): string[] { const unresolvedRefs: string[] = [] function checkModule(module: FlowModule) { @@ -257,7 +252,6 @@ export function findUnresolvedInlineScriptRefs(modules: FlowModule[]): string[] }) } } else if (module.value.type === 'aiagent') { - // Check AI agent tools if (module.value.tools) { for (const tool of module.value.tools) { if (