diff --git a/ai_evals/adapters/frontend/core/flow/fileHelpers.ts b/ai_evals/adapters/frontend/core/flow/fileHelpers.ts index 10c050949b..f09b5caee0 100644 --- a/ai_evals/adapters/frontend/core/flow/fileHelpers.ts +++ b/ai_evals/adapters/frontend/core/flow/fileHelpers.ts @@ -89,13 +89,14 @@ export async function createFlowFileHelpers( preprocessorModule, failureModule }) => { - applyFlowJsonUpdate(flow, inlineScriptSession, { + const result = applyFlowJsonUpdate(flow, inlineScriptSession, { modules, schema, preprocessorModule, failureModule }) await persistFlow() + return result } const helpers: FlowAIChatHelpers = { diff --git a/ai_evals/adapters/frontend/core/shared/baseEvalRunner.ts b/ai_evals/adapters/frontend/core/shared/baseEvalRunner.ts index 8aaece305f..8d8b07fea7 100644 --- a/ai_evals/adapters/frontend/core/shared/baseEvalRunner.ts +++ b/ai_evals/adapters/frontend/core/shared/baseEvalRunner.ts @@ -145,6 +145,20 @@ export async function runEval( skipResponsesApi: modelProvider.provider !== 'openai' }) + if (result.hitMaxIterations) { + return { + success: false, + output: getOutput(), + error: `Reached max turns (${maxIterations})`, + tokenUsage: result.tokenUsage, + toolCallsCount, + toolsCalled, + toolCallDetails, + iterations: Math.max(1, result.addedMessages.filter((m) => m.role === 'assistant').length), + messages + } + } + return { success: true, output: getOutput(), diff --git a/ai_evals/cases/flow.yaml b/ai_evals/cases/flow.yaml index d4ede9c833..4879562981 100644 --- a/ai_evals/cases/flow.yaml +++ b/ai_evals/cases/flow.yaml @@ -189,6 +189,8 @@ If validation passes, let the save continue normally. Update `save_results` so it uses the validation outcome instead of bypassing it. initial: ai_evals/fixtures/frontend/flow/initial/test5_initial.json + runtime: + maxTurns: 8 validate: topLevelStepIds: - fetch_data diff --git a/ai_evals/core/runSuite.ts b/ai_evals/core/runSuite.ts index d5486e3f55..6fe0a5c607 100644 --- a/ai_evals/core/runSuite.ts +++ b/ai_evals/core/runSuite.ts @@ -100,6 +100,7 @@ async function runCaseAttempts(input: { const initial = await input.modeRunner.loadInitial(input.evalCase.initialPath); const expected = await input.modeRunner.loadExpected(input.evalCase.expectedPath); const run = await input.modeRunner.run(input.evalCase.prompt, initial, { + evalCase: input.evalCase, caseId: input.evalCase.id, caseNumber: input.caseIndex + 1, totalCases: input.totalCases, @@ -181,6 +182,7 @@ async function runCaseAttempts(input: { actual: run.actual, run, context: { + evalCase: input.evalCase, caseId: input.evalCase.id, caseNumber: input.caseIndex + 1, totalCases: input.totalCases, diff --git a/ai_evals/core/types.ts b/ai_evals/core/types.ts index 13a77557d4..36f63d8cbb 100644 --- a/ai_evals/core/types.ts +++ b/ai_evals/core/types.ts @@ -8,6 +8,7 @@ export interface EvalCaseRuntimeBackendPreview { } export interface EvalCaseRuntimeSpec { + maxTurns?: number; backendPreview?: EvalCaseRuntimeBackendPreview; } @@ -109,6 +110,7 @@ export interface ModeRunOutput { } export interface ModeRunContext { + evalCase?: EvalCase; caseId: string; caseNumber: number; totalCases: number; diff --git a/ai_evals/modes/app.ts b/ai_evals/modes/app.ts index 66bf139c44..4ad7284246 100644 --- a/ai_evals/modes/app.ts +++ b/ai_evals/modes/app.ts @@ -23,6 +23,7 @@ export function createAppModeRunner( const result = await runAppEval(prompt, getFrontendApiKey(modelConfig.provider), { initialFrontend: initial?.frontend, initialBackend: initial?.backend as AppFiles["backend"] | undefined, + maxIterations: context.evalCase?.runtime?.maxTurns, provider: modelConfig.provider, model: modelConfig.model, runContext: context, diff --git a/ai_evals/modes/flow.ts b/ai_evals/modes/flow.ts index 174bb296da..f0a099adc3 100644 --- a/ai_evals/modes/flow.ts +++ b/ai_evals/modes/flow.ts @@ -37,6 +37,7 @@ export function createFlowModeRunner( const result = await runFlowEval(prompt, getFrontendApiKey(modelConfig.provider), { initialFlow: initial?.flowFixture, workspaceFixtures: initial?.workspace, + maxIterations: context.evalCase?.runtime?.maxTurns, provider: modelConfig.provider, model: modelConfig.model, runContext: context, diff --git a/ai_evals/modes/script.ts b/ai_evals/modes/script.ts index 1670488734..cf6739a9bf 100644 --- a/ai_evals/modes/script.ts +++ b/ai_evals/modes/script.ts @@ -29,6 +29,7 @@ export function createScriptModeRunner( const result = await runScriptEval(prompt, getFrontendApiKey(modelConfig.provider), { initialScript: initial, + maxIterations: context.evalCase?.runtime?.maxTurns, provider: modelConfig.provider, model: modelConfig.model, runContext: context, diff --git a/frontend/src/lib/components/copilot/chat/chatLoop.ts b/frontend/src/lib/components/copilot/chat/chatLoop.ts index 3c27317be3..a624698cc9 100644 --- a/frontend/src/lib/components/copilot/chat/chatLoop.ts +++ b/frontend/src/lib/components/copilot/chat/chatLoop.ts @@ -55,6 +55,7 @@ export interface ChatLoopConfig { export interface ChatLoopResult { addedMessages: ChatCompletionMessageParam[] tokenUsage: ChatTokenUsage + hitMaxIterations: boolean } export async function runChatLoop(config: ChatLoopConfig): Promise { @@ -74,9 +75,11 @@ export async function runChatLoop(config: ChatLoopConfig): Promise= maxIterations) { + hitMaxIterations = true break } iterations++ @@ -218,5 +221,5 @@ export async function runChatLoop(config: ChatLoopConfig): Promise