From 06c0b964f95d1166dd1dbf2208c5edbbcb273e11 Mon Sep 17 00:00:00 2001 From: centdix Date: Thu, 9 Apr 2026 13:30:49 +0200 Subject: [PATCH] refactor: simplify flow benchmark validation Co-Authored-By: Claude Opus 4.5 --- ai_evals/cases/flow.yaml | 145 +++++++++---- ai_evals/core/cases.ts | 2 + ai_evals/core/judge.ts | 17 +- ai_evals/core/runSuite.ts | 1 + ai_evals/core/types.ts | 20 +- ai_evals/core/validators.ts | 414 +++++++++--------------------------- ai_evals/modes/flow.ts | 3 +- 7 files changed, 225 insertions(+), 377 deletions(-) diff --git a/ai_evals/cases/flow.yaml b/ai_evals/cases/flow.yaml index aa838b931a..cdb53c696a 100644 --- a/ai_evals/cases/flow.yaml +++ b/ai_evals/cases/flow.yaml @@ -3,6 +3,10 @@ Create a flow that takes two numbers, `a` and `b`, and returns their sum. Keep it simple and use a single step named `sum_numbers`. expected: ai_evals/fixtures/frontend/flow/expected/test0_sum_two_numbers.json + judgeChecklist: + - "the flow takes `a` and `b` as inputs" + - "the main step is named `sum_numbers`" + - the flow returns the sum of the two numbers - id: flow-test1-reuse-existing-script prompt: |- @@ -11,6 +15,10 @@ The flow should take `a` and `b` as inputs and use a single step named `sum_numbers`. initial: ai_evals/fixtures/frontend/flow/initial/test1_reuse_existing_script_initial.json expected: ai_evals/fixtures/frontend/flow/expected/test1_reuse_existing_script.json + judgeChecklist: + - "the flow takes `a` and `b` as inputs" + - "the main step is named `sum_numbers`" + - the flow reuses the existing workspace script instead of rewriting the addition logic - id: flow-test2-call-existing-subflow prompt: |- @@ -19,6 +27,10 @@ Use a single step named `call_add_numbers`. initial: ai_evals/fixtures/frontend/flow/initial/test2_call_existing_subflow_initial.json expected: ai_evals/fixtures/frontend/flow/expected/test2_call_existing_subflow.json + judgeChecklist: + - "the parent flow takes `a` and `b` as inputs" + - "the main step is named `call_add_numbers`" + - the parent flow delegates to an existing workspace subflow instead of inlining the addition logic - id: flow-test3-branchone-routing prompt: |- @@ -27,6 +39,13 @@ Free, pro, and enterprise requests should go to different queues, and unknown tiers should fall back to a default queue. Name the main routing step `route_by_tier`. expected: ai_evals/fixtures/frontend/flow/expected/test3_branchone_routing.json + judgeChecklist: + - "the input schema includes a string field named `tier`" + - "the main routing step is named `route_by_tier`" + - free requests go to a free queue + - pro requests go to a pro queue + - enterprise requests go to an enterprise queue + - unknown tiers fall back to a default queue - id: flow-test4-order-processing-loop prompt: |- @@ -56,50 +75,13 @@ - customer_email - shipping_address resolveResultsRefs: true - modules: - - name: validates order item price and quantity - inputRefsContainAny: - - items - - order - codeContainsAll: - - price - - quantity - - name: calculates totals with 8 percent tax - inputRefsContainAny: - - items - - order - codeContainsAll: - - "0.08" - - name: checks inventory for each item - inputRefsContainAny: - - items - - itemDetails - - order - codeContainsAll: - - available - codeContainsAny: - - ".map(" - - "forEach(" - - "for (" - - name: creates shipment outcome - codeContainsAny: - - shipment_id - - SHIP- - - name: creates backorder outcome - codeContainsAny: - - backorder_id - - BO- - - name: sends confirmation to the customer - inputRefsContainAny: - - customer_email - - order - codeContainsAny: - - customer_email - - customerEmail - - name: returns final summary using totals and confirmation - inputRefsContainAll: - - calculate_total - - send_confirmation + judgeChecklist: + - the flow validates that every item has a positive price and quantity + - the flow calculates the order total with 8% tax + - the flow checks inventory for each item using placeholder availability data + - the flow creates a shipment if everything is in stock, otherwise a backorder + - the flow sends a confirmation using placeholder email logic + - the flow returns a final order summary with the resulting status - id: flow-test5-parallel-data-pipeline prompt: |- @@ -116,7 +98,16 @@ - 70 to 89 goes to a secondary database with a warning - below 70 goes to quarantine and triggers an alert - return a processing report with total records, quality score, and destination - expected: ai_evals/fixtures/frontend/flow/expected/test3.json + judgeChecklist: + - the flow loads a placeholder configuration listing three external sources + - the flow fetches placeholder records from each source + - the flow cleans and validates each source's records + - the flow combines everything into one dataset + - the flow computes an overall quality score + - scores of 90 or above go to the primary database + - scores from 70 to 89 go to a secondary database with a warning + - scores below 70 go to quarantine and trigger an alert + - the final report includes total records, quality score, and destination - id: flow-test6-ai-agent-tools prompt: |- @@ -131,7 +122,16 @@ - open a support ticket when needed After that, log the interaction and return the assistant's response along with any actions it took. - expected: ai_evals/fixtures/frontend/flow/expected/test4.json + judgeChecklist: + - "the input schema includes `customer_id` and `query_text`" + - the flow loads the customer's profile and order history + - the flow uses an AI assistant step + - the assistant can look up orders + - the assistant can check refund eligibility + - the assistant can search FAQs + - the assistant can open a support ticket + - the flow logs the interaction + - the final output returns the assistant response along with any actions taken or resulting support action details - id: flow-test7-simple-modification prompt: |- @@ -143,6 +143,12 @@ Update `save_results` so it handles the validation result correctly. initial: ai_evals/fixtures/frontend/flow/initial/test5_initial.json expected: ai_evals/fixtures/frontend/flow/expected/test5_modify_simple.json + judgeChecklist: + - the updated flow keeps the original fetch and process steps intact + - "a `validate_data` step is added after `process_data`" + - "`validate_data` checks that the processed data array is not empty" + - "empty data returns an error object with the message `No data to save`" + - "`save_results` handles the validation result correctly" - id: flow-test8-branching-in-loop prompt: |- @@ -154,7 +160,14 @@ Keep the existing processing as a fallback for unknown order types. Each path should return the orderId, shipping cost, and shipping type. initial: ai_evals/fixtures/frontend/flow/initial/test6_initial.json - expected: ai_evals/fixtures/frontend/flow/expected/test6_modify_medium.json + judgeChecklist: + - "the existing `loop_orders` flow still handles per-order processing" + - exact branching topology is not required as long as `loop_orders` handles the order types correctly + - express orders are marked as priority and use a shipping cost of 15.99 + - standard orders use a shipping cost of 5.99 + - pickup orders use a shipping cost of 0 and are treated as no shipping required + - unknown order types still follow a fallback path + - "each processed order returns `orderId`, `shippingCost`, and `shippingType`" - id: flow-test9-parallel-refactor prompt: |- @@ -166,6 +179,13 @@ Keep `get_item` as the first step and `return_result` as the last step. initial: ai_evals/fixtures/frontend/flow/initial/test7_initial.json expected: ai_evals/fixtures/frontend/flow/expected/test7_modify_complex.json + judgeChecklist: + - "the updated flow keeps `get_item` as the first step" + - "the updated flow keeps `return_result` as the last step" + - "`enrich_price`, `enrich_inventory`, and `enrich_reviews` run independently rather than sequentially" + - each enrichment path returns a fallback value if it fails + - "`combine_data` merges the enrichment results" + - "`combine_data` sets `hasFallbacks` when any fallback was used" - id: flow-test10-while-loop-counter prompt: |- @@ -174,6 +194,11 @@ Name the looping step `count_until_target`. Once the target is reached, return the final counter value. expected: ai_evals/fixtures/frontend/flow/expected/test10_while_loop_counter.json + judgeChecklist: + - "the input schema includes a number field named `target`" + - "the looping step is named `count_until_target`" + - the flow keeps incrementing a counter until the target is reached + - the final output returns the final counter value - id: flow-test11-preprocessor-and-failure-handler prompt: |- @@ -183,6 +208,16 @@ The main step should be named `process_event` and return a simple success object. If anything fails, return a compact error object with the error message and the failing step id. expected: ai_evals/fixtures/frontend/flow/expected/test11_preprocessor_failure.json + validate: + requireSpecialModules: + - preprocessor_module + - failure_module + judgeChecklist: + - the flow trims the payload before the main processing runs + - the flow rejects empty payload strings + - "the main step is named `process_event`" + - "`process_event` returns a simple success object" + - failures return a compact error object with the error message and failing step id - id: flow-test12-approval-step prompt: |- @@ -193,3 +228,19 @@ One approval should be enough to continue. After approval, add a final step named `finalize_purchase` that returns an approved status object. expected: ai_evals/fixtures/frontend/flow/expected/test12_approval_step.json + validate: + schemaRequiredPaths: + - requester_email + - amount + requireSuspendSteps: + - id: request_approval + requiredEvents: 1 + resumeRequiredStringFieldAnyOf: + - comment + - approver_comment + judgeChecklist: + - "the flow includes an approval step named `request_approval`" + - "`request_approval` pauses the flow and asks the approver for a comment" + - one approval is enough to continue + - "the flow includes a final step named `finalize_purchase`" + - "`finalize_purchase` returns an approved status object after approval" diff --git a/ai_evals/core/cases.ts b/ai_evals/core/cases.ts index 877ca44ed3..96a1dfbfb6 100644 --- a/ai_evals/core/cases.ts +++ b/ai_evals/core/cases.ts @@ -13,6 +13,7 @@ interface RawEvalCase { initial?: string; expected?: string; validate?: FlowValidationSpec; + judgeChecklist?: string[]; } export function getRepoRoot(): string { @@ -34,6 +35,7 @@ export async function loadCases(mode: EvalMode): Promise { initialPath: resolveFixturePath(entry.initial), expectedPath: resolveFixturePath(entry.expected), validate: entry.validate, + judgeChecklist: entry.judgeChecklist, })); } diff --git a/ai_evals/core/judge.ts b/ai_evals/core/judge.ts index 6308ddee82..cae66721c6 100644 --- a/ai_evals/core/judge.ts +++ b/ai_evals/core/judge.ts @@ -8,6 +8,7 @@ const JUDGE_TOOL_NAME = "submit_judgement"; export async function judgeOutput(input: { mode: EvalMode; prompt: string; + checklist?: string[]; initial?: unknown; expected?: unknown; actual: unknown; @@ -30,7 +31,10 @@ export async function judgeOutput(input: { "You evaluate benchmark outputs for Windmill AI generation.", "Deterministic checks already run separately. Focus on whether the final output satisfies the user request.", "If expected state is provided, treat it as a valid example and reward semantically equivalent outputs.", - "Be strict about missing requested functionality. Inputs, module ids, and other details do not need to be exactly the same, but the functionality must be the same.", + "If a checklist is provided, treat it as the explicit acceptance criteria for this case.", + "Be strict about missing requested functionality.", + "When the prompt wording is ambiguous, prefer the checklist over inferred structural requirements.", + "Do not require exact ids, exact topology, or exact field names unless the prompt, checklist, or expected state clearly requires them.", `Always respond by calling the ${JUDGE_TOOL_NAME} tool exactly once.`, ].join("\n\n"); @@ -40,6 +44,9 @@ export async function judgeOutput(input: { "User prompt:", input.prompt, "", + "Checklist:", + formatChecklist(input.checklist), + "", "Initial state:", formatJsonBlock(input.initial), "", @@ -126,6 +133,14 @@ function formatJsonBlock(value: unknown): string { return JSON.stringify(value, null, 2); } +function formatChecklist(checklist: string[] | undefined): string { + if (!checklist || checklist.length === 0) { + return "(none)"; + } + + return checklist.map((item) => `- ${item}`).join("\n"); +} + function normalizeScore(value: number): number { if (!Number.isFinite(value)) { return 0; diff --git a/ai_evals/core/runSuite.ts b/ai_evals/core/runSuite.ts index 19a3f62324..4b3fa71223 100644 --- a/ai_evals/core/runSuite.ts +++ b/ai_evals/core/runSuite.ts @@ -160,6 +160,7 @@ async function runCaseAttempts(input: { const judge = await judgeOutput({ mode: input.modeRunner.mode, prompt: input.evalCase.prompt, + checklist: input.evalCase.judgeChecklist, initial, expected, actual: run.actual, diff --git a/ai_evals/core/types.ts b/ai_evals/core/types.ts index 465698c0f0..c1fa81a1a9 100644 --- a/ai_evals/core/types.ts +++ b/ai_evals/core/types.ts @@ -2,25 +2,18 @@ export const EVAL_MODES = ["cli", "flow", "script", "app"] as const; export type EvalMode = (typeof EVAL_MODES)[number]; -export interface FlowModuleValidation { - name: string; - typeAnyOf?: string[]; - inputRefsAny?: string[]; - inputRefsAll?: string[]; - inputRefsContainAny?: string[]; - inputRefsContainAll?: string[]; - codeContainsAny?: string[]; - codeContainsAll?: string[]; - codeRegexAll?: string[]; -} - export interface FlowValidationSpec { schemaRequiredPaths?: string[]; schemaAnyOf?: Array<{ requiredPaths: string[]; }>; resolveResultsRefs?: boolean; - modules?: FlowModuleValidation[]; + requireSpecialModules?: Array<"preprocessor_module" | "failure_module">; + requireSuspendSteps?: Array<{ + id: string; + requiredEvents?: number; + resumeRequiredStringFieldAnyOf?: string[]; + }>; } export interface EvalCase { @@ -29,6 +22,7 @@ export interface EvalCase { initialPath?: string; expectedPath?: string; validate?: FlowValidationSpec; + judgeChecklist?: string[]; } export interface BenchmarkCheck { diff --git a/ai_evals/core/validators.ts b/ai_evals/core/validators.ts index 5b1df1a260..873d848f79 100644 --- a/ai_evals/core/validators.ts +++ b/ai_evals/core/validators.ts @@ -1,5 +1,5 @@ import ts from "typescript"; -import type { BenchmarkCheck, FlowModuleValidation, FlowValidationSpec } from "./types"; +import type { BenchmarkCheck, FlowValidationSpec } from "./types"; export interface ScriptState { path: string; @@ -81,11 +81,20 @@ export function validateScriptState(input: { export function validateFlowState(input: { actual: FlowState; initial?: FlowState; - expected?: FlowState; validate?: FlowValidationSpec; }): BenchmarkCheck[] { const actualModules = getFlowModules(input.actual); - const checks: BenchmarkCheck[] = [check("flow has modules", actualModules.length > 0)]; + const placeholderModuleIds = getInlineScriptPlaceholderModuleIds(input.actual); + const checks: BenchmarkCheck[] = [ + check("flow has modules", actualModules.length > 0), + check( + "flow has no inline placeholder code", + placeholderModuleIds.length === 0, + placeholderModuleIds.length > 0 + ? `placeholder content in: ${placeholderModuleIds.join(", ")}` + : undefined + ), + ]; if (input.initial) { checks.push( @@ -96,146 +105,6 @@ export function validateFlowState(input: { ); } - if (!input.expected) { - if (input.validate) { - checks.push(...validateFlowRequirements(input.actual, input.validate)); - } - return checks; - } - - const expectedModules = getFlowModules(input.expected); - const actualTypes = new Set(actualModules.map((module) => getModuleType(module) ?? "")); - const expectedTypes = new Set(expectedModules.map((module) => getModuleType(module) ?? "")); - const missingTypes = [...expectedTypes].filter((type) => type && !actualTypes.has(type)); - const actualTopLevelIds = getTopLevelFlowModuleIds(input.actual); - const expectedTopLevelIds = getTopLevelFlowModuleIds(input.expected); - const missingIds = expectedTopLevelIds.filter((id) => !actualTopLevelIds.includes(id)); - const expectedSchemaRootType = getSchemaRootType(input.expected.schema); - const actualSchemaRootType = getSchemaRootType(input.actual.schema); - const actualModuleById = new Map( - actualModules - .map((module) => [typeof module.id === "string" ? module.id : "", module] as const) - .filter(([id]) => id.length > 0) - ); - - checks.push( - check( - "flow includes expected module types", - missingTypes.length === 0, - missingTypes.length > 0 ? `missing: ${missingTypes.join(", ")}` : undefined - ) - ); - - if (expectedSchemaRootType) { - checks.push( - check( - "flow schema root type matches expected", - actualSchemaRootType === expectedSchemaRootType, - `expected ${expectedSchemaRootType}, got ${actualSchemaRootType ?? "(missing)"}` - ) - ); - } - - if (expectedTopLevelIds.length > 0) { - checks.push( - check( - "flow includes expected top-level step ids", - missingIds.length === 0, - missingIds.length > 0 ? `missing: ${missingIds.join(", ")}` : undefined - ) - ); - } - - for (const expectedModule of expectedModules) { - const expectedId = typeof expectedModule.id === "string" ? expectedModule.id : null; - if (!expectedId) { - continue; - } - - const actualModule = actualModuleById.get(expectedId); - if (!actualModule) { - continue; - } - - const expectedType = getModuleType(expectedModule); - if (expectedType) { - checks.push( - check( - `${expectedId} type matches expected`, - getModuleType(actualModule) === expectedType, - `expected ${expectedType}, got ${getModuleType(actualModule) ?? "(missing)"}` - ) - ); - } - - const expectedPath = getModulePath(expectedModule); - if (expectedPath) { - checks.push( - check( - `${expectedId} path matches expected`, - getModulePath(actualModule) === expectedPath, - `expected ${expectedPath}, got ${getModulePath(actualModule) ?? "(missing)"}` - ) - ); - } - - if (hasSuspendConfig(expectedModule)) { - checks.push(check(`${expectedId} includes suspend config`, hasSuspendConfig(actualModule))); - } - - const expectedContinueOnError = getContinueOnError(expectedModule); - if (expectedContinueOnError !== undefined) { - checks.push( - check( - `${expectedId} continue_on_error matches expected`, - getContinueOnError(actualModule) === expectedContinueOnError - ) - ); - } - - if (hasRetryConfig(expectedModule)) { - checks.push(check(`${expectedId} includes retry config`, hasRetryConfig(actualModule))); - } - - const expectedParallel = getModuleParallel(expectedModule); - if (expectedParallel !== undefined) { - checks.push( - check( - `${expectedId} parallel matches expected`, - getModuleParallel(actualModule) === expectedParallel - ) - ); - } - } - - for (const specialKey of ["preprocessor_module", "failure_module"] as const) { - const expectedSpecial = getSpecialFlowModule(input.expected, specialKey); - if (!expectedSpecial) { - continue; - } - - const actualSpecial = getSpecialFlowModule(input.actual, specialKey); - checks.push(check(`${specialKey} exists`, Boolean(actualSpecial))); - if (!actualSpecial) { - continue; - } - - const expectedType = getModuleType(expectedSpecial); - if (expectedType) { - checks.push( - check( - `${specialKey} type matches expected`, - getModuleType(actualSpecial) === expectedType, - `expected ${expectedType}, got ${getModuleType(actualSpecial) ?? "(missing)"}` - ) - ); - } - - if (hasRetryConfig(expectedSpecial)) { - checks.push(check(`${specialKey} includes retry config`, hasRetryConfig(actualSpecial))); - } - } - if (input.validate) { checks.push(...validateFlowRequirements(input.actual, input.validate)); } @@ -413,24 +282,55 @@ function validateFlowRequirements( ); } - for (const requirement of validate.modules ?? []) { - const matched = findMatchingFlowModule(flow, requirement); + for (const specialModule of validate.requireSpecialModules ?? []) { checks.push( check( - requirement.name, - Boolean(matched), - matched ? undefined : describeMissingModuleRequirement(requirement) + `${specialModule} exists`, + Boolean(getSpecialFlowModule(flow, specialModule)) ) ); } - return checks; -} + for (const suspendStep of validate.requireSuspendSteps ?? []) { + const module = findFlowModuleById(flow, suspendStep.id); + checks.push(check(`${suspendStep.id} step exists`, Boolean(module))); + if (!module) { + continue; + } -function getTopLevelFlowModuleIds(flow: FlowState): string[] { - return getFlowModules(flow) - .map((module) => module.id) - .filter((value): value is string => typeof value === "string"); + checks.push(check(`${suspendStep.id} includes suspend config`, hasSuspendConfig(module))); + if (!hasSuspendConfig(module)) { + continue; + } + + if (suspendStep.requiredEvents !== undefined) { + checks.push( + check( + `${suspendStep.id} requires ${suspendStep.requiredEvents} approval event${suspendStep.requiredEvents === 1 ? "" : "s"}`, + getSuspendRequiredEvents(module) === suspendStep.requiredEvents, + `expected ${suspendStep.requiredEvents}, got ${getSuspendRequiredEvents(module) ?? "(missing)"}` + ) + ); + } + + if ( + suspendStep.resumeRequiredStringFieldAnyOf && + suspendStep.resumeRequiredStringFieldAnyOf.length > 0 + ) { + const requiredFields = getSuspendResumeRequiredStringFields(module); + checks.push( + check( + `${suspendStep.id} resume form requires one accepted comment field`, + suspendStep.resumeRequiredStringFieldAnyOf.some((field) => + requiredFields.includes(field) + ), + `required one of [${suspendStep.resumeRequiredStringFieldAnyOf.join(", ")}], got [${requiredFields.join(", ")}]` + ) + ); + } + } + + return checks; } function hasSchemaPath(schema: Record | undefined, dottedPath: string): boolean { @@ -616,111 +516,6 @@ function validateNestedResultsRefPath( } } -function findMatchingFlowModule( - flow: FlowState, - requirement: FlowModuleValidation -): Record | null { - for (const module of getAllFlowModules(flow)) { - if (matchesFlowModuleRequirement(module, requirement)) { - return module; - } - } - return null; -} - -function matchesFlowModuleRequirement( - module: Record, - requirement: FlowModuleValidation -): boolean { - if (requirement.typeAnyOf && requirement.typeAnyOf.length > 0) { - const actualType = getModuleType(module); - if (!actualType || !requirement.typeAnyOf.includes(actualType)) { - return false; - } - } - - const inputRefs = getModuleInputRefs(module); - if (requirement.inputRefsAny && requirement.inputRefsAny.length > 0) { - if (!requirement.inputRefsAny.some((ref) => inputRefs.includes(ref))) { - return false; - } - } - - if (requirement.inputRefsAll && requirement.inputRefsAll.length > 0) { - if (!requirement.inputRefsAll.every((ref) => inputRefs.includes(ref))) { - return false; - } - } - - if (requirement.inputRefsContainAny && requirement.inputRefsContainAny.length > 0) { - if (!requirement.inputRefsContainAny.some((snippet) => inputRefs.some((ref) => ref.includes(snippet)))) { - return false; - } - } - - if (requirement.inputRefsContainAll && requirement.inputRefsContainAll.length > 0) { - if (!requirement.inputRefsContainAll.every((snippet) => inputRefs.some((ref) => ref.includes(snippet)))) { - return false; - } - } - - const code = getModuleCode(module); - if (requirement.codeContainsAny && requirement.codeContainsAny.length > 0) { - if (!code || !requirement.codeContainsAny.some((snippet) => code.includes(snippet))) { - return false; - } - } - - if (requirement.codeContainsAll && requirement.codeContainsAll.length > 0) { - if (!code || !requirement.codeContainsAll.every((snippet) => code.includes(snippet))) { - return false; - } - } - - if (requirement.codeRegexAll && requirement.codeRegexAll.length > 0) { - if (!code) { - return false; - } - for (const pattern of requirement.codeRegexAll) { - const regex = new RegExp(pattern, "m"); - if (!regex.test(code)) { - return false; - } - } - } - - return true; -} - -function describeMissingModuleRequirement(requirement: FlowModuleValidation): string { - const parts: string[] = []; - if (requirement.typeAnyOf?.length) { - parts.push(`type in [${requirement.typeAnyOf.join(", ")}]`); - } - if (requirement.inputRefsAny?.length) { - parts.push(`any input refs [${requirement.inputRefsAny.join(", ")}]`); - } - if (requirement.inputRefsAll?.length) { - parts.push(`all input refs [${requirement.inputRefsAll.join(", ")}]`); - } - if (requirement.inputRefsContainAny?.length) { - parts.push(`any input refs containing [${requirement.inputRefsContainAny.join(", ")}]`); - } - if (requirement.inputRefsContainAll?.length) { - parts.push(`all input refs containing [${requirement.inputRefsContainAll.join(", ")}]`); - } - if (requirement.codeContainsAny?.length) { - parts.push(`code contains any [${requirement.codeContainsAny.join(", ")}]`); - } - if (requirement.codeContainsAll?.length) { - parts.push(`code contains all [${requirement.codeContainsAll.join(", ")}]`); - } - if (requirement.codeRegexAll?.length) { - parts.push(`code matches [${requirement.codeRegexAll.join(", ")}]`); - } - return parts.join("; "); -} - function getAllFlowModules(flow: FlowState): Array> { const modules: Array> = []; const specialModules = ["preprocessor_module", "failure_module"] as const; @@ -774,6 +569,30 @@ function collectNestedModules(module: Record): Array | null { + for (const module of getAllFlowModules(flow)) { + if (module.id === id) { + return module; + } + } + return null; +} + +function getInlineScriptPlaceholderModuleIds(flow: FlowState): string[] { + return getAllFlowModules(flow).flatMap((module) => { + const code = getModuleCode(module)?.trim(); + if (!code || !/^inline_script\.[A-Za-z0-9_-]+$/.test(code)) { + return []; + } + + if (typeof module.id === "string" && module.id.length > 0) { + return [module.id]; + } + + return ["(unnamed)"]; + }); +} + function getImmediateNestedModuleIds(module: Record): string[] { const ids: string[] = []; const value = isObjectRecord(module.value) ? module.value : null; @@ -803,22 +622,6 @@ function getImmediateNestedModuleIds(module: Record): string[] return ids; } -function getModuleInputRefs(module: Record): string[] { - const value = isObjectRecord(module.value) ? module.value : null; - const inputTransforms = isObjectRecord(value?.input_transforms) ? value?.input_transforms : null; - if (!inputTransforms) { - return []; - } - - return Object.values(inputTransforms) - .flatMap((entry) => { - if (!isObjectRecord(entry) || typeof entry.expr !== "string") { - return []; - } - return [entry.expr.trim()]; - }); -} - function getModuleCode(module: Record): string | null { const value = isObjectRecord(module.value) ? module.value : null; return typeof value?.content === "string" ? value.content : null; @@ -853,52 +656,35 @@ function getModuleType(module: Record): string | null { : null; } -function getModulePath(module: Record): string | null { - const value = module.value; - if (!value || typeof value !== "object") { - return null; - } - return typeof (value as Record).path === "string" - ? ((value as Record).path) - : null; -} - function hasSuspendConfig(module: Record): boolean { return typeof module.suspend === "object" && module.suspend !== null; } -function getContinueOnError(module: Record): boolean | undefined { - return typeof module.continue_on_error === "boolean" - ? (module.continue_on_error as boolean) - : undefined; +function getSuspendRequiredEvents(module: Record): number | null { + const suspend = isObjectRecord(module.suspend) ? module.suspend : null; + return typeof suspend?.required_events === "number" ? suspend.required_events : null; } -function hasRetryConfig(module: Record): boolean { - return typeof module.retry === "object" && module.retry !== null; -} +function getSuspendResumeRequiredStringFields(module: Record): string[] { + const suspend = isObjectRecord(module.suspend) ? module.suspend : null; + const resumeForm = isObjectRecord(suspend?.resume_form) ? suspend.resume_form : null; + const schema = isObjectRecord(resumeForm?.schema) ? resumeForm.schema : null; + const required = Array.isArray(schema?.required) ? schema.required : []; + const properties = isObjectRecord(schema?.properties) ? schema.properties : null; + if (!properties) { + return []; + } -function getModuleParallel(module: Record): boolean | undefined { - const value = module.value; - if (!value || typeof value !== "object") { - return undefined; - } - return typeof (value as Record).parallel === "boolean" - ? ((value as Record).parallel) - : undefined; -} - -function getSchemaRootType(schema: Record | undefined): string | null { - if (!schema || typeof schema !== "object") { - return null; - } - const properties = (schema.properties ?? {}) as Record; - const root = properties["root"]; - if (!root || typeof root !== "object") { - return null; - } - return typeof (root as { type?: unknown }).type === "string" - ? ((root as { type: string }).type) - : null; + return required.flatMap((field) => { + if (typeof field !== "string") { + return []; + } + const property = properties[field]; + if (!isObjectRecord(property) || property.type !== "string") { + return []; + } + return [field]; + }); } function appStatesEqual(left: AppFilesState, right: AppFilesState): boolean { diff --git a/ai_evals/modes/flow.ts b/ai_evals/modes/flow.ts index 4e74da4f42..69f1653b22 100644 --- a/ai_evals/modes/flow.ts +++ b/ai_evals/modes/flow.ts @@ -55,11 +55,10 @@ export function createFlowModeRunner( skillsInvoked: [], }; }, - validate({ evalCase, actual, initial, expected }) { + validate({ evalCase, actual, initial }) { return validateFlowState({ actual, initial: initial?.flow, - expected, validate: evalCase.validate, }); },