fix: cap frontend eval chat turns

ai_evals/adapters/frontend/core/flow/fileHelpers.ts

@@ -89,13 +89,14 @@ export async function createFlowFileHelpers(
 		preprocessorModule,
 		failureModule
 	}) => {
-		applyFlowJsonUpdate(flow, inlineScriptSession, {
+		const result = applyFlowJsonUpdate(flow, inlineScriptSession, {
 			modules,
 			schema,
 			preprocessorModule,
 			failureModule
 		})
 		await persistFlow()
+		return result
 	}
 
 	const helpers: FlowAIChatHelpers = {

ai_evals/adapters/frontend/core/shared/baseEvalRunner.ts

@@ -145,6 +145,20 @@ export async function runEval<THelpers, TOutput>(
 			skipResponsesApi: modelProvider.provider !== 'openai'
 		})
 
+		if (result.hitMaxIterations) {
+			return {
+				success: false,
+				output: getOutput(),
+				error: `Reached max turns (${maxIterations})`,
+				tokenUsage: result.tokenUsage,
+				toolCallsCount,
+				toolsCalled,
+				toolCallDetails,
+				iterations: Math.max(1, result.addedMessages.filter((m) => m.role === 'assistant').length),
+				messages
+			}
+		}
+
 		return {
 			success: true,
 			output: getOutput(),

ai_evals/cases/flow.yaml

@@ -189,6 +189,8 @@
     If validation passes, let the save continue normally.
     Update `save_results` so it uses the validation outcome instead of bypassing it.
   initial: ai_evals/fixtures/frontend/flow/initial/test5_initial.json
+  runtime:
+    maxTurns: 8
   validate:
     topLevelStepIds:
       - fetch_data

ai_evals/core/runSuite.ts

@@ -100,6 +100,7 @@ async function runCaseAttempts<TInitial, TExpected, TActual>(input: {
       const initial = await input.modeRunner.loadInitial(input.evalCase.initialPath);
       const expected = await input.modeRunner.loadExpected(input.evalCase.expectedPath);
       const run = await input.modeRunner.run(input.evalCase.prompt, initial, {
+        evalCase: input.evalCase,
         caseId: input.evalCase.id,
         caseNumber: input.caseIndex + 1,
         totalCases: input.totalCases,
@@ -181,6 +182,7 @@ async function runCaseAttempts<TInitial, TExpected, TActual>(input: {
             actual: run.actual,
             run,
             context: {
+              evalCase: input.evalCase,
               caseId: input.evalCase.id,
               caseNumber: input.caseIndex + 1,
               totalCases: input.totalCases,

ai_evals/core/types.ts

@@ -8,6 +8,7 @@ export interface EvalCaseRuntimeBackendPreview {
 }
 
 export interface EvalCaseRuntimeSpec {
+  maxTurns?: number;
   backendPreview?: EvalCaseRuntimeBackendPreview;
 }
 
@@ -109,6 +110,7 @@ export interface ModeRunOutput<TActual> {
 }
 
 export interface ModeRunContext {
+  evalCase?: EvalCase;
   caseId: string;
   caseNumber: number;
   totalCases: number;

ai_evals/modes/app.ts

@@ -23,6 +23,7 @@ export function createAppModeRunner(
       const result = await runAppEval(prompt, getFrontendApiKey(modelConfig.provider), {
         initialFrontend: initial?.frontend,
         initialBackend: initial?.backend as AppFiles["backend"] | undefined,
+        maxIterations: context.evalCase?.runtime?.maxTurns,
         provider: modelConfig.provider,
         model: modelConfig.model,
         runContext: context,

ai_evals/modes/flow.ts

@@ -37,6 +37,7 @@ export function createFlowModeRunner(
       const result = await runFlowEval(prompt, getFrontendApiKey(modelConfig.provider), {
         initialFlow: initial?.flowFixture,
         workspaceFixtures: initial?.workspace,
+        maxIterations: context.evalCase?.runtime?.maxTurns,
         provider: modelConfig.provider,
         model: modelConfig.model,
         runContext: context,

ai_evals/modes/script.ts

@@ -29,6 +29,7 @@ export function createScriptModeRunner(
 
       const result = await runScriptEval(prompt, getFrontendApiKey(modelConfig.provider), {
         initialScript: initial,
+        maxIterations: context.evalCase?.runtime?.maxTurns,
         provider: modelConfig.provider,
         model: modelConfig.model,
         runContext: context,

frontend/src/lib/components/copilot/chat/chatLoop.ts

@@ -55,6 +55,7 @@ export interface ChatLoopConfig {
 export interface ChatLoopResult {
 	addedMessages: ChatCompletionMessageParam[]
 	tokenUsage: ChatTokenUsage
+	hitMaxIterations: boolean
 }
 
 export async function runChatLoop(config: ChatLoopConfig): Promise<ChatLoopResult> {
@@ -74,9 +75,11 @@ export async function runChatLoop(config: ChatLoopConfig): Promise<ChatLoopResul
 	const addedMessages: ChatCompletionMessageParam[] = []
 	let tokenUsage = emptyChatTokenUsage()
 	let iterations = 0
+	let hitMaxIterations = false
 
 	while (true) {
 		if (maxIterations !== undefined && iterations >= maxIterations) {
+			hitMaxIterations = true
 			break
 		}
 		iterations++
@@ -218,5 +221,5 @@ export async function runChatLoop(config: ChatLoopConfig): Promise<ChatLoopResul
 		}
 	}
 
-	return { addedMessages, tokenUsage }
+	return { addedMessages, tokenUsage, hitMaxIterations }
 }