mirror of
https://github.com/windmill-labs/windmill.git
synced 2026-08-21 08:02:26 +00:00
chore(appchat): add tests pipeline (#7374)
* draft test app * gitignore * add app test pipeline * add lot of tests * add variant * remove unrelated changes * fix * fix
This commit is contained in:
centdix
GitHub
+2
-1
@@ -14,4 +14,5 @@ static/tsdocs/
|
||||
static/ui_builder/
|
||||
ui_builder.tar.gz
|
||||
ui_builder_serve/
|
||||
src/lib/components/copilot/chat/flow/__tests__/eval/results/
|
||||
src/lib/components/copilot/chat/__tests__/flow/results/
|
||||
src/lib/components/copilot/chat/__tests__/app/results/
|
||||
|
||||
@@ -0,0 +1,224 @@
|
||||
import { describe, expect, it } from 'vitest'
|
||||
import { runVariantComparison, writeAppComparisonResults } from './appEvalRunner'
|
||||
import { BASELINE_VARIANT, STREAMLINED_VARIANT } from './variants'
|
||||
import { loadAppFixtureForEval } from './appFixtureLoader'
|
||||
// @ts-ignore - Node.js path
|
||||
import { dirname, join } from 'path'
|
||||
// @ts-ignore - Node.js url
|
||||
import { fileURLToPath } from 'url'
|
||||
|
||||
// Get API key from environment - tests will be skipped if not set
|
||||
// @ts-ignore
|
||||
const OPENROUTER_API_KEY = process.env.OPENROUTER_API_KEY
|
||||
|
||||
// Skip all tests if no API key is provided
|
||||
const describeWithApiKey = OPENROUTER_API_KEY ? describe : describe.skip
|
||||
|
||||
// Get __dirname equivalent for ES modules
|
||||
const __filename = fileURLToPath(import.meta.url)
|
||||
const __dirname = dirname(__filename)
|
||||
|
||||
const MODELS = ['google/gemini-2.5-flash', 'anthropic/claude-haiku-4.5', 'openai/gpt-4o']
|
||||
const VARIANTS = [
|
||||
...MODELS.map((model) => ({
|
||||
...BASELINE_VARIANT,
|
||||
model,
|
||||
name: `baseline-${model.replace('/', '-')}`
|
||||
})),
|
||||
...MODELS.map((model) => ({
|
||||
...STREAMLINED_VARIANT,
|
||||
model,
|
||||
name: `streamlined-${model.replace('/', '-')}`
|
||||
}))
|
||||
]
|
||||
|
||||
describeWithApiKey('App Chat LLM Evaluation', () => {
|
||||
const TEST_TIMEOUT = 120_000
|
||||
if (!OPENROUTER_API_KEY) {
|
||||
console.warn('OPENROUTER_API_KEY is not set, skipping tests')
|
||||
}
|
||||
|
||||
it(
|
||||
'test1: creates a simple counter app',
|
||||
async () => {
|
||||
const USER_PROMPT = `Create a counter app with increment/decrement buttons`
|
||||
const results = await runVariantComparison(USER_PROMPT, VARIANTS, OPENROUTER_API_KEY!)
|
||||
// Write results to files
|
||||
const { summaryPath, appPaths } = await writeAppComparisonResults(USER_PROMPT, results)
|
||||
console.log(`\nResults written to: ${summaryPath}`)
|
||||
console.log(`App files: ${appPaths.join(', ')}`)
|
||||
|
||||
expect(true).toBe(true)
|
||||
},
|
||||
TEST_TIMEOUT
|
||||
)
|
||||
|
||||
it(
|
||||
'test2: modifies existing counter app to add reset button',
|
||||
async () => {
|
||||
// Load initial app from fixture folder
|
||||
const { initialFrontend, initialBackend } = await loadAppFixtureForEval(
|
||||
join(__dirname, 'initial', 'test1_counter_app')
|
||||
)
|
||||
|
||||
const USER_PROMPT = `Add a reset button that sets the counter back to 0`
|
||||
const results = await runVariantComparison(USER_PROMPT, VARIANTS, OPENROUTER_API_KEY!, {
|
||||
initialFrontend,
|
||||
initialBackend
|
||||
})
|
||||
// Write results to files
|
||||
const { summaryPath, appPaths } = await writeAppComparisonResults(USER_PROMPT, results)
|
||||
console.log(`\nResults written to: ${summaryPath}`)
|
||||
console.log(`App files: ${appPaths.join(', ')}`)
|
||||
|
||||
expect(true).toBe(true)
|
||||
},
|
||||
TEST_TIMEOUT
|
||||
)
|
||||
|
||||
// ==================== Shopping Cart Tests ====================
|
||||
|
||||
it(
|
||||
'test3: shopping cart - add quantity selector',
|
||||
async () => {
|
||||
const { initialFrontend, initialBackend } = await loadAppFixtureForEval(
|
||||
join(__dirname, 'initial', 'shopping_cart')
|
||||
)
|
||||
|
||||
const USER_PROMPT = `Add a quantity selector (+ and - buttons) to each cart item so users can adjust quantities without removing and re-adding items`
|
||||
const results = await runVariantComparison(USER_PROMPT, VARIANTS, OPENROUTER_API_KEY!, {
|
||||
initialFrontend,
|
||||
initialBackend
|
||||
})
|
||||
|
||||
const { summaryPath, appPaths } = await writeAppComparisonResults(USER_PROMPT, results)
|
||||
console.log(`\nResults written to: ${summaryPath}`)
|
||||
console.log(`App files: ${appPaths.join(', ')}`)
|
||||
|
||||
expect(true).toBe(true)
|
||||
},
|
||||
TEST_TIMEOUT
|
||||
)
|
||||
|
||||
it(
|
||||
'test4: shopping cart - add discount code',
|
||||
async () => {
|
||||
const { initialFrontend, initialBackend } = await loadAppFixtureForEval(
|
||||
join(__dirname, 'initial', 'shopping_cart')
|
||||
)
|
||||
|
||||
const USER_PROMPT = `Add a discount code input field in the cart. When the code "SAVE10" is entered, apply a 10% discount to the total`
|
||||
const results = await runVariantComparison(USER_PROMPT, VARIANTS, OPENROUTER_API_KEY!, {
|
||||
initialFrontend,
|
||||
initialBackend
|
||||
})
|
||||
|
||||
const { summaryPath, appPaths } = await writeAppComparisonResults(USER_PROMPT, results)
|
||||
console.log(`\nResults written to: ${summaryPath}`)
|
||||
console.log(`App files: ${appPaths.join(', ')}`)
|
||||
|
||||
expect(true).toBe(true)
|
||||
},
|
||||
TEST_TIMEOUT
|
||||
)
|
||||
|
||||
// ==================== File Manager Tests ====================
|
||||
|
||||
it(
|
||||
'test5: file manager - add search bar',
|
||||
async () => {
|
||||
const { initialFrontend, initialBackend } = await loadAppFixtureForEval(
|
||||
join(__dirname, 'initial', 'file_manager')
|
||||
)
|
||||
|
||||
const USER_PROMPT = `Add a search bar in the toolbar that filters files and folders by name as the user types`
|
||||
const results = await runVariantComparison(USER_PROMPT, VARIANTS, OPENROUTER_API_KEY!, {
|
||||
initialFrontend,
|
||||
initialBackend
|
||||
})
|
||||
|
||||
const { summaryPath, appPaths } = await writeAppComparisonResults(USER_PROMPT, results)
|
||||
console.log(`\nResults written to: ${summaryPath}`)
|
||||
console.log(`App files: ${appPaths.join(', ')}`)
|
||||
|
||||
expect(true).toBe(true)
|
||||
},
|
||||
TEST_TIMEOUT
|
||||
)
|
||||
|
||||
it(
|
||||
'test6: file manager - show file details',
|
||||
async () => {
|
||||
const { initialFrontend, initialBackend } = await loadAppFixtureForEval(
|
||||
join(__dirname, 'initial', 'file_manager')
|
||||
)
|
||||
|
||||
const USER_PROMPT = `Show file size (formatted as KB/MB) and modified date in the file list for each item`
|
||||
const results = await runVariantComparison(USER_PROMPT, VARIANTS, OPENROUTER_API_KEY!, {
|
||||
initialFrontend,
|
||||
initialBackend
|
||||
})
|
||||
|
||||
const { summaryPath, appPaths } = await writeAppComparisonResults(USER_PROMPT, results)
|
||||
console.log(`\nResults written to: ${summaryPath}`)
|
||||
console.log(`App files: ${appPaths.join(', ')}`)
|
||||
|
||||
expect(true).toBe(true)
|
||||
},
|
||||
TEST_TIMEOUT
|
||||
)
|
||||
|
||||
it(
|
||||
'test7: file manager - add select all checkbox',
|
||||
async () => {
|
||||
const { initialFrontend, initialBackend } = await loadAppFixtureForEval(
|
||||
join(__dirname, 'initial', 'file_manager')
|
||||
)
|
||||
|
||||
const USER_PROMPT = `Add a "Select All" checkbox in the file list header and individual checkboxes for each file. Add a "Delete Selected" button that appears when items are selected`
|
||||
const results = await runVariantComparison(USER_PROMPT, VARIANTS, OPENROUTER_API_KEY!, {
|
||||
initialFrontend,
|
||||
initialBackend
|
||||
})
|
||||
|
||||
const { summaryPath, appPaths } = await writeAppComparisonResults(USER_PROMPT, results)
|
||||
console.log(`\nResults written to: ${summaryPath}`)
|
||||
console.log(`App files: ${appPaths.join(', ')}`)
|
||||
|
||||
expect(true).toBe(true)
|
||||
},
|
||||
TEST_TIMEOUT
|
||||
)
|
||||
|
||||
// ==================== From-Scratch Creation Tests ====================
|
||||
|
||||
it(
|
||||
'test8: create quiz app from scratch',
|
||||
async () => {
|
||||
const USER_PROMPT = `Create a multiple choice quiz app with 5 questions about general knowledge. Show one question at a time with 4 answer options. Track the score and show results at the end with percentage correct.`
|
||||
const results = await runVariantComparison(USER_PROMPT, VARIANTS, OPENROUTER_API_KEY!)
|
||||
|
||||
const { summaryPath, appPaths } = await writeAppComparisonResults(USER_PROMPT, results)
|
||||
console.log(`\nResults written to: ${summaryPath}`)
|
||||
console.log(`App files: ${appPaths.join(', ')}`)
|
||||
|
||||
expect(true).toBe(true)
|
||||
},
|
||||
TEST_TIMEOUT
|
||||
)
|
||||
|
||||
it(
|
||||
'test9: create recipe book from scratch',
|
||||
async () => {
|
||||
const USER_PROMPT = `Create a recipe book app where users can add recipes with a name, ingredients list, and instructions. Include a search bar to filter recipes by name and the ability to delete recipes.`
|
||||
const results = await runVariantComparison(USER_PROMPT, VARIANTS, OPENROUTER_API_KEY!)
|
||||
|
||||
const { summaryPath, appPaths } = await writeAppComparisonResults(USER_PROMPT, results)
|
||||
console.log(`\nResults written to: ${summaryPath}`)
|
||||
console.log(`App files: ${appPaths.join(', ')}`)
|
||||
|
||||
expect(true).toBe(true)
|
||||
},
|
||||
TEST_TIMEOUT
|
||||
)
|
||||
})
|
||||
@@ -0,0 +1,166 @@
|
||||
import OpenAI from 'openai'
|
||||
import type { AppFiles, BackendRunnable } from '../../app/core'
|
||||
import { BASE_EVALUATOR_RESPONSE_FORMAT } from '../shared'
|
||||
import type { EvaluationResult } from '../shared'
|
||||
|
||||
/**
|
||||
* Expected app structure for evaluation.
|
||||
*/
|
||||
export interface ExpectedApp {
|
||||
frontend: Record<string, string>
|
||||
backend: Record<string, BackendRunnable>
|
||||
}
|
||||
|
||||
/**
|
||||
* Initial app state for evaluation context.
|
||||
*/
|
||||
export interface InitialApp {
|
||||
frontend: Record<string, string>
|
||||
backend: Record<string, BackendRunnable>
|
||||
}
|
||||
|
||||
/**
|
||||
* System prompt for evaluating app generation without a reference expected app.
|
||||
* Evaluates based on user request fulfillment and appropriate modifications to initial state.
|
||||
*/
|
||||
const APP_GENERATION_EVALUATOR_SYSTEM_PROMPT = `You are an expert evaluator for Windmill Raw App definitions. Your task is to evaluate a generated app based on:
|
||||
1. The original user request/prompt
|
||||
2. The initial app state (if any) - this is what the app looked like before the AI made changes
|
||||
|
||||
## Windmill Raw App Context
|
||||
- Raw Apps consist of frontend files and backend runnables
|
||||
- Frontend files are TypeScript/JavaScript files bundled with esbuild (entrypoint: index.tsx)
|
||||
- Backend runnables can be: inline scripts (TypeScript/Python), workspace scripts, workspace flows, or hub scripts
|
||||
- Frontend calls backend using \`await backend.<runnable_key>(args...)\`
|
||||
- Each backend runnable has a key (identifier), name (description), type, and configuration
|
||||
|
||||
## Backend Runnable Types
|
||||
- **inline**: Custom code with \`inlineScript.language\` and \`inlineScript.content\`
|
||||
- **script**: Workspace script reference with \`path\`
|
||||
- **flow**: Workspace flow reference with \`path\`
|
||||
- **hubscript**: Hub script reference with \`path\`
|
||||
|
||||
## Evaluation Criteria
|
||||
1. **User Request Fulfillment**: Does the generated app address ALL requirements from the user's original prompt?
|
||||
- Are all requested features implemented?
|
||||
- Does the frontend UI match the requirements?
|
||||
- Are the correct backend runnables created?
|
||||
2. **Appropriate Modifications** (if initial app was provided):
|
||||
- Were the changes made relevant to the user's request?
|
||||
- Was existing functionality preserved where appropriate?
|
||||
- Were only necessary changes made (no unnecessary removals or additions)?
|
||||
3. **Frontend Structure**: Are the frontend files correctly organized and implemented?
|
||||
- Is the code valid TypeScript/JavaScript?
|
||||
- Are components properly structured?
|
||||
- Are backend calls correctly made?
|
||||
4. **Backend Structure**: Are the backend runnables correctly configured?
|
||||
- Do inline scripts have proper main functions?
|
||||
- Are types and paths correct for non-inline runnables?
|
||||
5. **Integration**: Does the frontend correctly call the backend?
|
||||
- Are the runnable keys correctly referenced?
|
||||
- Are arguments passed correctly?
|
||||
6. **Code Quality**: Is the code functionally correct and well-structured?
|
||||
|
||||
## Important Notes
|
||||
- Focus on whether the user's request was fulfilled, not on stylistic preferences
|
||||
- If an initial app was provided, evaluate the appropriateness of the changes made
|
||||
- For new apps (no initial state), evaluate completeness and correctness
|
||||
- Extra helper functions or slightly different approaches can still score high if they accomplish the goal
|
||||
|
||||
${BASE_EVALUATOR_RESPONSE_FORMAT}`
|
||||
|
||||
/**
|
||||
* Evaluates how well a generated app fulfills the user's request, considering any initial app state.
|
||||
* This evaluator does not require an expected reference app - it evaluates based on the request alone.
|
||||
*
|
||||
* @param userPrompt The original user request
|
||||
* @param generatedApp The app generated by the AI
|
||||
* @param initialApp Optional initial app state (what the app looked like before AI changes)
|
||||
* @returns Evaluation result with score, statement, and missing requirements
|
||||
*/
|
||||
export async function evaluateAppGeneration(
|
||||
userPrompt: string,
|
||||
generatedApp: AppFiles,
|
||||
initialApp?: InitialApp
|
||||
): Promise<EvaluationResult> {
|
||||
// @ts-ignore
|
||||
const apiKey = process.env.OPENROUTER_API_KEY
|
||||
|
||||
const client = new OpenAI({ baseURL: 'https://openrouter.ai/api/v1', apiKey })
|
||||
|
||||
let userMessage = `## User's Original Request
|
||||
${userPrompt}
|
||||
|
||||
`
|
||||
|
||||
if (initialApp) {
|
||||
userMessage += `## Initial App State (before AI modifications)
|
||||
\`\`\`json
|
||||
${JSON.stringify(initialApp, null, 2)}
|
||||
\`\`\`
|
||||
|
||||
`
|
||||
} else {
|
||||
userMessage += `## Initial App State
|
||||
No initial app was provided - this is a new app created from scratch.
|
||||
|
||||
`
|
||||
}
|
||||
|
||||
userMessage += `## Generated App
|
||||
\`\`\`json
|
||||
${JSON.stringify(generatedApp, null, 2)}
|
||||
\`\`\`
|
||||
|
||||
Please evaluate how well the generated app:
|
||||
1. Fulfills ALL requirements from the user's original request
|
||||
2. ${initialApp ? 'Makes appropriate modifications to the initial app state' : 'Implements a complete and correct new app'}`
|
||||
|
||||
try {
|
||||
const response = await client.chat.completions.create({
|
||||
model: 'anthropic/claude-sonnet-4.5',
|
||||
messages: [
|
||||
{ role: 'system', content: APP_GENERATION_EVALUATOR_SYSTEM_PROMPT },
|
||||
{ role: 'user', content: userMessage }
|
||||
],
|
||||
temperature: 0
|
||||
})
|
||||
|
||||
const content = response.choices[0]?.message?.content
|
||||
if (!content) {
|
||||
return {
|
||||
success: false,
|
||||
resemblanceScore: 0,
|
||||
statement: 'No response from evaluator',
|
||||
error: 'Empty response from LLM'
|
||||
}
|
||||
}
|
||||
|
||||
// Parse JSON response - handle potential markdown code blocks
|
||||
let jsonContent = content.trim()
|
||||
if (jsonContent.startsWith('```')) {
|
||||
jsonContent = jsonContent.replace(/^```(?:json)?\n?/, '').replace(/\n?```$/, '')
|
||||
}
|
||||
|
||||
const parsed = JSON.parse(jsonContent) as {
|
||||
resemblanceScore: number
|
||||
statement: string
|
||||
missingRequirements?: string[]
|
||||
}
|
||||
|
||||
return {
|
||||
success: true,
|
||||
resemblanceScore: Math.max(0, Math.min(100, Math.round(parsed.resemblanceScore))),
|
||||
statement: parsed.statement,
|
||||
missingRequirements: parsed.missingRequirements ?? []
|
||||
}
|
||||
} catch (err) {
|
||||
const errorMessage = err instanceof Error ? err.message : String(err)
|
||||
return {
|
||||
success: false,
|
||||
resemblanceScore: 0,
|
||||
statement: 'Evaluation failed',
|
||||
error: errorMessage
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,119 @@
|
||||
import type {
|
||||
AppAIChatHelpers,
|
||||
AppFiles,
|
||||
BackendRunnable,
|
||||
LintResult,
|
||||
SelectedContext
|
||||
} from '../../app/core'
|
||||
|
||||
/**
|
||||
* Creates an empty lint result (no errors or warnings).
|
||||
*/
|
||||
function createEmptyLintResult(): LintResult {
|
||||
return {
|
||||
errorCount: 0,
|
||||
warningCount: 0,
|
||||
errors: { frontend: {}, backend: {} },
|
||||
warnings: { frontend: {}, backend: {} }
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Creates mock AppAIChatHelpers for eval testing.
|
||||
* Tracks app files state in memory and allows tool functions to modify it.
|
||||
*/
|
||||
export function createAppEvalHelpers(
|
||||
initialFrontend: Record<string, string> = {},
|
||||
initialBackend: Record<string, BackendRunnable> = {}
|
||||
) {
|
||||
// In-memory state
|
||||
let frontend: Record<string, string> = { ...initialFrontend }
|
||||
let backend: Record<string, BackendRunnable> = { ...initialBackend }
|
||||
let snapshotId = 0
|
||||
const snapshots: Map<number, { frontend: Record<string, string>; backend: Record<string, BackendRunnable> }> = new Map()
|
||||
|
||||
const helpers: AppAIChatHelpers = {
|
||||
// Frontend file operations
|
||||
listFrontendFiles: () => Object.keys(frontend),
|
||||
|
||||
getFrontendFile: (path: string) => frontend[path],
|
||||
|
||||
getFrontendFiles: () => ({ ...frontend }),
|
||||
|
||||
setFrontendFile: (path: string, content: string) => {
|
||||
frontend[path] = content
|
||||
// Return mock lint result - in real usage this would validate the file
|
||||
return createEmptyLintResult()
|
||||
},
|
||||
|
||||
deleteFrontendFile: (path: string) => {
|
||||
delete frontend[path]
|
||||
},
|
||||
|
||||
// Backend runnable operations
|
||||
listBackendRunnables: () => {
|
||||
return Object.entries(backend).map(([key, runnable]) => ({
|
||||
key,
|
||||
name: runnable.name
|
||||
}))
|
||||
},
|
||||
|
||||
getBackendRunnable: (key: string) => backend[key],
|
||||
|
||||
getBackendRunnables: () => ({ ...backend }),
|
||||
|
||||
setBackendRunnable: async (key: string, runnable: BackendRunnable) => {
|
||||
backend[key] = runnable
|
||||
// Return mock lint result - in real usage this would validate the runnable
|
||||
return createEmptyLintResult()
|
||||
},
|
||||
|
||||
deleteBackendRunnable: (key: string) => {
|
||||
delete backend[key]
|
||||
},
|
||||
|
||||
// Combined view
|
||||
getFiles: (): AppFiles => ({
|
||||
frontend: { ...frontend },
|
||||
backend: { ...backend }
|
||||
}),
|
||||
|
||||
getSelectedContext: (): SelectedContext => ({
|
||||
type: 'none'
|
||||
}),
|
||||
|
||||
// Snapshot management
|
||||
snapshot: () => {
|
||||
const id = ++snapshotId
|
||||
snapshots.set(id, {
|
||||
frontend: { ...frontend },
|
||||
backend: { ...backend }
|
||||
})
|
||||
return id
|
||||
},
|
||||
|
||||
revertToSnapshot: (id: number) => {
|
||||
const snap = snapshots.get(id)
|
||||
if (snap) {
|
||||
frontend = { ...snap.frontend }
|
||||
backend = { ...snap.backend }
|
||||
}
|
||||
},
|
||||
|
||||
// Linting
|
||||
lint: () => {
|
||||
// Return mock lint result - no actual linting in eval
|
||||
return createEmptyLintResult()
|
||||
}
|
||||
}
|
||||
|
||||
return {
|
||||
helpers,
|
||||
getFiles: (): AppFiles => ({
|
||||
frontend: { ...frontend },
|
||||
backend: { ...backend }
|
||||
}),
|
||||
getFrontend: () => ({ ...frontend }),
|
||||
getBackend: () => ({ ...backend })
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,168 @@
|
||||
import type { AppFiles, BackendRunnable, AppAIChatHelpers } from '../../app/core'
|
||||
import { getAppTools, prepareAppSystemMessage, prepareAppUserMessage } from '../../app/core'
|
||||
import { createAppEvalHelpers } from './appEvalHelpers'
|
||||
import { evaluateAppGeneration, type InitialApp } from './appEvalComparison'
|
||||
import {
|
||||
runEval,
|
||||
resolveSystemPrompt,
|
||||
resolveTools,
|
||||
resolveModel,
|
||||
type VariantConfig,
|
||||
type BaseEvalResult,
|
||||
type EvaluationResult,
|
||||
type Tool,
|
||||
type VariantDefaults
|
||||
} from '../shared'
|
||||
import { writeAppComparisonResultsToFolders } from './appResultsWriter'
|
||||
|
||||
// Re-export for convenience
|
||||
export type { InitialApp } from './appEvalComparison'
|
||||
|
||||
/**
|
||||
* App-specific evaluation result.
|
||||
*/
|
||||
export interface AppEvalResult extends BaseEvalResult<AppFiles> {
|
||||
/** Alias for output to maintain API compatibility */
|
||||
files: AppFiles
|
||||
}
|
||||
|
||||
/**
|
||||
* Options for running an app evaluation.
|
||||
*/
|
||||
export interface AppEvalOptions {
|
||||
initialFrontend?: Record<string, string>
|
||||
initialBackend?: Record<string, BackendRunnable>
|
||||
model?: string
|
||||
customSystemPrompt?: string
|
||||
maxIterations?: number
|
||||
variant?: VariantConfig
|
||||
/** Whether to evaluate the generated app with LLM. Default: true. Set to false to skip evaluation. */
|
||||
evaluateWithLLM?: boolean
|
||||
}
|
||||
|
||||
/**
|
||||
* App-specific variant defaults.
|
||||
*/
|
||||
const appDefaults: VariantDefaults<AppAIChatHelpers> = {
|
||||
prepareSystemMessage: prepareAppSystemMessage,
|
||||
tools: getAppTools() as Tool<AppAIChatHelpers>[]
|
||||
}
|
||||
|
||||
/**
|
||||
* Runs an app chat evaluation with real OpenAI API calls.
|
||||
* Executes tool calls using the actual app tools from core.ts or variant-configured tools.
|
||||
*/
|
||||
export async function runAppEval(
|
||||
userPrompt: string,
|
||||
openaiApiKey: string,
|
||||
options?: AppEvalOptions
|
||||
): Promise<AppEvalResult> {
|
||||
const { helpers, getFiles } = createAppEvalHelpers(
|
||||
options?.initialFrontend ?? {},
|
||||
options?.initialBackend ?? {}
|
||||
)
|
||||
|
||||
// Resolve variant configuration
|
||||
const variantName = options?.variant?.name ?? 'baseline'
|
||||
const systemMessage = resolveSystemPrompt(
|
||||
options?.variant,
|
||||
appDefaults,
|
||||
options?.customSystemPrompt
|
||||
)
|
||||
const { toolDefs, tools } = resolveTools(options?.variant, appDefaults)
|
||||
const model = resolveModel(options?.variant, options?.model)
|
||||
|
||||
// Build user message
|
||||
const userMessage = prepareAppUserMessage(
|
||||
userPrompt,
|
||||
helpers.getFiles(),
|
||||
helpers.getSelectedContext()
|
||||
)
|
||||
|
||||
// Run the base evaluation
|
||||
const rawResult = await runEval({
|
||||
userPrompt,
|
||||
systemMessage,
|
||||
userMessage,
|
||||
toolDefs,
|
||||
tools,
|
||||
helpers,
|
||||
apiKey: openaiApiKey,
|
||||
getOutput: getFiles,
|
||||
options: {
|
||||
maxIterations: options?.maxIterations,
|
||||
model,
|
||||
workspace: 'test-workspace'
|
||||
}
|
||||
})
|
||||
|
||||
// Run LLM evaluation unless explicitly disabled
|
||||
let evaluationResult: EvaluationResult | undefined
|
||||
if (options?.evaluateWithLLM !== false) {
|
||||
const generatedApp = getFiles()
|
||||
const initialApp: InitialApp | undefined =
|
||||
options?.initialFrontend || options?.initialBackend
|
||||
? {
|
||||
frontend: options.initialFrontend ?? {},
|
||||
backend: options.initialBackend ?? {}
|
||||
}
|
||||
: undefined
|
||||
evaluationResult = await evaluateAppGeneration(userPrompt, generatedApp, initialApp)
|
||||
}
|
||||
|
||||
return {
|
||||
...rawResult,
|
||||
variantName,
|
||||
files: rawResult.output,
|
||||
evaluationResult
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Runs the same prompt against multiple variants sequentially for comparison.
|
||||
* Returns results in the same order as the input variants.
|
||||
*/
|
||||
export async function runVariantComparison(
|
||||
userPrompt: string,
|
||||
variants: VariantConfig[],
|
||||
openaiApiKey: string,
|
||||
baseOptions?: Omit<AppEvalOptions, 'variant'>
|
||||
): Promise<AppEvalResult[]> {
|
||||
const results: AppEvalResult[] = await Promise.all(
|
||||
variants.map(async (variant) => {
|
||||
return await runAppEval(userPrompt, openaiApiKey, {
|
||||
...baseOptions,
|
||||
variant
|
||||
})
|
||||
})
|
||||
)
|
||||
return results
|
||||
}
|
||||
|
||||
/**
|
||||
* Writes app comparison results to a folder-based structure.
|
||||
* Each variant gets its own folder with frontend/, backend/, and details.json.
|
||||
*/
|
||||
export async function writeAppComparisonResults(
|
||||
userPrompt: string,
|
||||
results: AppEvalResult[],
|
||||
outputDir?: string
|
||||
): Promise<{ summaryPath: string; appPaths: string[] }> {
|
||||
// @ts-ignore
|
||||
const { dirname, join } = await import('path')
|
||||
// @ts-ignore
|
||||
const { fileURLToPath } = await import('url')
|
||||
|
||||
const __filename = fileURLToPath(import.meta.url)
|
||||
const __dirname = dirname(__filename)
|
||||
|
||||
const resultsDir = outputDir ?? join(__dirname, 'results')
|
||||
|
||||
const result = await writeAppComparisonResultsToFolders({
|
||||
userPrompt,
|
||||
results,
|
||||
outputDir: resultsDir
|
||||
})
|
||||
|
||||
return { summaryPath: result.summaryPath, appPaths: result.variantPaths }
|
||||
}
|
||||
@@ -0,0 +1,170 @@
|
||||
import type { AppFiles, BackendRunnable, InlineScript } from '../../app/core'
|
||||
|
||||
/**
|
||||
* Backend runnable metadata stored in meta.json files.
|
||||
*/
|
||||
interface BackendMeta {
|
||||
name: string
|
||||
language: 'bun' | 'python3'
|
||||
}
|
||||
|
||||
/**
|
||||
* Recursively reads all files in a directory and returns them as a record.
|
||||
* File paths are relative to the base directory with a leading '/'.
|
||||
*/
|
||||
async function readFilesRecursively(
|
||||
dir: string,
|
||||
basePath: string = ''
|
||||
): Promise<Record<string, string>> {
|
||||
// @ts-ignore - Node.js fs/promises
|
||||
const { readdir, readFile } = await import('fs/promises')
|
||||
// @ts-ignore - Node.js path
|
||||
const { join } = await import('path')
|
||||
|
||||
const result: Record<string, string> = {}
|
||||
const entries = await readdir(dir, { withFileTypes: true })
|
||||
|
||||
for (const entry of entries) {
|
||||
const fullPath = join(dir, entry.name)
|
||||
const relativePath = basePath ? `${basePath}/${entry.name}` : `/${entry.name}`
|
||||
|
||||
if (entry.isDirectory()) {
|
||||
const subFiles = await readFilesRecursively(fullPath, relativePath)
|
||||
Object.assign(result, subFiles)
|
||||
} else {
|
||||
const content = await readFile(fullPath, 'utf-8')
|
||||
result[relativePath] = content
|
||||
}
|
||||
}
|
||||
|
||||
return result
|
||||
}
|
||||
|
||||
/**
|
||||
* Loads frontend files from a directory.
|
||||
* All files are read recursively and paths become keys with leading '/'.
|
||||
*/
|
||||
async function loadFrontend(frontendPath: string): Promise<Record<string, string>> {
|
||||
// @ts-ignore - Node.js fs/promises
|
||||
const { access } = await import('fs/promises')
|
||||
|
||||
try {
|
||||
await access(frontendPath)
|
||||
} catch {
|
||||
// Directory doesn't exist, return empty
|
||||
return {}
|
||||
}
|
||||
|
||||
return readFilesRecursively(frontendPath)
|
||||
}
|
||||
|
||||
/**
|
||||
* Loads backend runnables from a directory.
|
||||
* Each subdirectory is a runnable with:
|
||||
* - main.ts or main.py: The code content
|
||||
* - meta.json: Metadata { name, language }
|
||||
*/
|
||||
async function loadBackend(backendPath: string): Promise<Record<string, BackendRunnable>> {
|
||||
// @ts-ignore - Node.js fs/promises
|
||||
const { readdir, readFile, access } = await import('fs/promises')
|
||||
// @ts-ignore - Node.js path
|
||||
const { join } = await import('path')
|
||||
|
||||
try {
|
||||
await access(backendPath)
|
||||
} catch {
|
||||
// Directory doesn't exist, return empty
|
||||
return {}
|
||||
}
|
||||
|
||||
const result: Record<string, BackendRunnable> = {}
|
||||
const entries = await readdir(backendPath, { withFileTypes: true })
|
||||
|
||||
for (const entry of entries) {
|
||||
if (!entry.isDirectory()) continue
|
||||
|
||||
const runnableKey = entry.name
|
||||
const runnablePath = join(backendPath, entry.name)
|
||||
|
||||
// Read meta.json
|
||||
const metaPath = join(runnablePath, 'meta.json')
|
||||
let meta: BackendMeta
|
||||
try {
|
||||
const metaContent = await readFile(metaPath, 'utf-8')
|
||||
meta = JSON.parse(metaContent)
|
||||
} catch {
|
||||
console.warn(`Missing or invalid meta.json for runnable '${runnableKey}', skipping`)
|
||||
continue
|
||||
}
|
||||
|
||||
// Find and read the main file (main.ts or main.py)
|
||||
const runnableFiles = await readdir(runnablePath)
|
||||
const mainFile = runnableFiles.find((f) => f === 'main.ts' || f === 'main.py')
|
||||
|
||||
if (!mainFile) {
|
||||
console.warn(`No main.ts or main.py found for runnable '${runnableKey}', skipping`)
|
||||
continue
|
||||
}
|
||||
|
||||
const content = await readFile(join(runnablePath, mainFile), 'utf-8')
|
||||
|
||||
const inlineScript: InlineScript = {
|
||||
language: meta.language,
|
||||
content
|
||||
}
|
||||
|
||||
result[runnableKey] = {
|
||||
name: meta.name,
|
||||
type: 'inline',
|
||||
inlineScript
|
||||
}
|
||||
}
|
||||
|
||||
return result
|
||||
}
|
||||
|
||||
/**
|
||||
* Loads an app fixture from a directory structure.
|
||||
*
|
||||
* Expected structure:
|
||||
* ```
|
||||
* fixturePath/
|
||||
* ├── frontend/
|
||||
* │ └── index.tsx # → frontend["/index.tsx"]
|
||||
* │ └── components/
|
||||
* │ └── Button.tsx # → frontend["/components/Button.tsx"]
|
||||
* └── backend/
|
||||
* └── incrementCounter/
|
||||
* ├── main.ts # The code content
|
||||
* └── meta.json # { "name": "...", "language": "bun" }
|
||||
* ```
|
||||
*
|
||||
* @param fixturePath - Path to the fixture directory
|
||||
* @returns AppFiles object with frontend and backend
|
||||
*/
|
||||
export async function loadAppFixture(fixturePath: string): Promise<AppFiles> {
|
||||
// @ts-ignore - Node.js path
|
||||
const { join } = await import('path')
|
||||
|
||||
const frontend = await loadFrontend(join(fixturePath, 'frontend'))
|
||||
const backend = await loadBackend(join(fixturePath, 'backend'))
|
||||
|
||||
return { frontend, backend }
|
||||
}
|
||||
|
||||
/**
|
||||
* Loads an app fixture and returns the separate frontend and backend objects.
|
||||
* Convenience function for use with runAppEval options.
|
||||
*/
|
||||
export async function loadAppFixtureForEval(
|
||||
fixturePath: string
|
||||
): Promise<{
|
||||
initialFrontend: Record<string, string>
|
||||
initialBackend: Record<string, BackendRunnable>
|
||||
}> {
|
||||
const { frontend, backend } = await loadAppFixture(fixturePath)
|
||||
return {
|
||||
initialFrontend: frontend,
|
||||
initialBackend: backend
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,247 @@
|
||||
import type { AppFiles, BackendRunnable } from '../../app/core'
|
||||
import type { AppEvalResult } from './appEvalRunner'
|
||||
import { generateTimestamp } from '../shared'
|
||||
|
||||
/**
|
||||
* Writes frontend files to a folder, preserving directory structure.
|
||||
* File paths like "/components/Button.tsx" become "frontend/components/Button.tsx"
|
||||
*/
|
||||
async function writeFrontendFiles(
|
||||
frontend: Record<string, string>,
|
||||
frontendPath: string
|
||||
): Promise<void> {
|
||||
// @ts-ignore - Node.js fs/promises
|
||||
const { writeFile, mkdir } = await import('fs/promises')
|
||||
// @ts-ignore - Node.js path
|
||||
const { join, dirname } = await import('path')
|
||||
|
||||
for (const [filePath, content] of Object.entries(frontend)) {
|
||||
// Remove leading slash and join with frontend path
|
||||
const relativePath = filePath.startsWith('/') ? filePath.slice(1) : filePath
|
||||
const fullPath = join(frontendPath, relativePath)
|
||||
|
||||
// Ensure parent directory exists
|
||||
await mkdir(dirname(fullPath), { recursive: true })
|
||||
|
||||
await writeFile(fullPath, content)
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Writes backend runnables to a folder structure.
|
||||
* Each runnable becomes a folder with main.ts/main.py and meta.json
|
||||
*/
|
||||
async function writeBackendRunnables(
|
||||
backend: Record<string, BackendRunnable>,
|
||||
backendPath: string
|
||||
): Promise<void> {
|
||||
// @ts-ignore - Node.js fs/promises
|
||||
const { writeFile, mkdir } = await import('fs/promises')
|
||||
// @ts-ignore - Node.js path
|
||||
const { join } = await import('path')
|
||||
|
||||
for (const [key, runnable] of Object.entries(backend)) {
|
||||
const runnablePath = join(backendPath, key)
|
||||
await mkdir(runnablePath, { recursive: true })
|
||||
|
||||
// Write meta.json
|
||||
const meta: { name: string; language?: string; type?: string; path?: string } = {
|
||||
name: runnable.name
|
||||
}
|
||||
|
||||
if (runnable.type === 'inline' && runnable.inlineScript) {
|
||||
meta.language = runnable.inlineScript.language
|
||||
|
||||
// Write main file
|
||||
const extension = runnable.inlineScript.language === 'python3' ? 'py' : 'ts'
|
||||
const mainPath = join(runnablePath, `main.${extension}`)
|
||||
await writeFile(mainPath, runnable.inlineScript.content)
|
||||
} else {
|
||||
// For non-inline runnables, store type and path in meta
|
||||
meta.type = runnable.type
|
||||
if (runnable.path) {
|
||||
meta.path = runnable.path
|
||||
}
|
||||
}
|
||||
|
||||
const metaPath = join(runnablePath, 'meta.json')
|
||||
await writeFile(metaPath, JSON.stringify(meta, null, '\t'))
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Writes app files (frontend + backend) to a folder structure.
|
||||
*/
|
||||
async function writeAppToFolder(appFiles: AppFiles, folderPath: string): Promise<void> {
|
||||
// @ts-ignore - Node.js path
|
||||
const { join } = await import('path')
|
||||
|
||||
if (Object.keys(appFiles.frontend).length > 0) {
|
||||
await writeFrontendFiles(appFiles.frontend, join(folderPath, 'frontend'))
|
||||
}
|
||||
|
||||
if (Object.keys(appFiles.backend).length > 0) {
|
||||
await writeBackendRunnables(appFiles.backend, join(folderPath, 'backend'))
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Parameters for writing app comparison results.
|
||||
*/
|
||||
export interface WriteAppResultsParams {
|
||||
userPrompt: string
|
||||
results: AppEvalResult[]
|
||||
outputDir: string
|
||||
}
|
||||
|
||||
/**
|
||||
* Writes app comparison results to a folder-based structure.
|
||||
*
|
||||
* Creates:
|
||||
* ```
|
||||
* results/{timestamp}/
|
||||
* ├── summary.md
|
||||
* └── {variant_name}/
|
||||
* ├── details.json # Metadata (toolsCalled, evaluationResult, etc.)
|
||||
* ├── frontend/ # Frontend files
|
||||
* │ └── index.tsx
|
||||
* └── backend/ # Backend runnables
|
||||
* └── myFunction/
|
||||
* ├── main.ts
|
||||
* └── meta.json
|
||||
* ```
|
||||
*/
|
||||
export async function writeAppComparisonResultsToFolders(
|
||||
params: WriteAppResultsParams
|
||||
): Promise<{ summaryPath: string; variantPaths: string[] }> {
|
||||
// @ts-ignore - Node.js fs/promises
|
||||
const { writeFile, mkdir } = await import('fs/promises')
|
||||
// @ts-ignore - Node.js path
|
||||
const { join } = await import('path')
|
||||
|
||||
const { userPrompt, results, outputDir } = params
|
||||
const timestamp = generateTimestamp()
|
||||
|
||||
// Ensure results directory exists
|
||||
await mkdir(outputDir, { recursive: true })
|
||||
const resultFolder = join(outputDir, timestamp)
|
||||
await mkdir(resultFolder, { recursive: true })
|
||||
|
||||
// Check if any results have evaluation data
|
||||
const hasEvaluation = results.some((r) => r.evaluationResult)
|
||||
|
||||
// Build summary markdown
|
||||
const summaryLines: string[] = [
|
||||
`# App Eval Results - ${timestamp}`,
|
||||
'',
|
||||
'## User Prompt',
|
||||
'```',
|
||||
userPrompt.trim(),
|
||||
'```',
|
||||
'',
|
||||
'## Results',
|
||||
''
|
||||
]
|
||||
|
||||
// Add results table header based on whether evaluation data exists
|
||||
if (hasEvaluation) {
|
||||
summaryLines.push(
|
||||
'| Variant | Success | Total Tokens | Tool Calls | Iterations | Resemblance Score |'
|
||||
)
|
||||
summaryLines.push(
|
||||
'|---------|---------|--------------|------------|------------|-------------------|'
|
||||
)
|
||||
} else {
|
||||
summaryLines.push('| Variant | Success | Total Tokens | Tool Calls | Iterations |')
|
||||
summaryLines.push('|---------|---------|--------------|------------|------------|')
|
||||
}
|
||||
|
||||
for (const result of results) {
|
||||
const baseRow = `| ${result.variantName} | ${result.success} | ${result.tokenUsage.total} | ${result.toolsCalled.length} | ${result.iterations}`
|
||||
if (hasEvaluation) {
|
||||
const score = result.evaluationResult?.resemblanceScore ?? 'N/A'
|
||||
summaryLines.push(`${baseRow} | ${score} |`)
|
||||
} else {
|
||||
summaryLines.push(`${baseRow} |`)
|
||||
}
|
||||
}
|
||||
|
||||
// Add evaluation details section if available
|
||||
if (hasEvaluation) {
|
||||
summaryLines.push('')
|
||||
summaryLines.push('## Evaluation Details')
|
||||
summaryLines.push('')
|
||||
for (const result of results) {
|
||||
if (result.evaluationResult) {
|
||||
summaryLines.push(`### ${result.variantName}`)
|
||||
summaryLines.push('')
|
||||
summaryLines.push(`**Score:** ${result.evaluationResult.resemblanceScore}/100`)
|
||||
summaryLines.push('')
|
||||
summaryLines.push(`**Statement:** ${result.evaluationResult.statement}`)
|
||||
summaryLines.push('')
|
||||
if (
|
||||
result.evaluationResult.missingRequirements &&
|
||||
result.evaluationResult.missingRequirements.length > 0
|
||||
) {
|
||||
summaryLines.push('**Missing Requirements:**')
|
||||
for (const req of result.evaluationResult.missingRequirements) {
|
||||
summaryLines.push(`- ${req}`)
|
||||
}
|
||||
summaryLines.push('')
|
||||
}
|
||||
if (result.evaluationResult.error) {
|
||||
summaryLines.push(`**Error:** ${result.evaluationResult.error}`)
|
||||
summaryLines.push('')
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Add errors section for failed variants
|
||||
const failedResults = results.filter((r) => !r.success && r.error)
|
||||
if (failedResults.length > 0) {
|
||||
summaryLines.push('')
|
||||
summaryLines.push('## Errors')
|
||||
summaryLines.push('')
|
||||
for (const result of failedResults) {
|
||||
summaryLines.push(`### ${result.variantName}`)
|
||||
summaryLines.push('')
|
||||
summaryLines.push('```')
|
||||
summaryLines.push(result.error!)
|
||||
summaryLines.push('```')
|
||||
summaryLines.push('')
|
||||
}
|
||||
}
|
||||
|
||||
const variantPaths: string[] = []
|
||||
|
||||
// Write each variant to its own folder
|
||||
for (const result of results) {
|
||||
const variantFolder = join(resultFolder, result.variantName)
|
||||
await mkdir(variantFolder, { recursive: true })
|
||||
variantPaths.push(variantFolder)
|
||||
|
||||
// Write details.json (metadata without app files)
|
||||
const details = {
|
||||
variantName: result.variantName,
|
||||
success: result.success,
|
||||
error: result.error ?? null,
|
||||
evaluationResult: result.evaluationResult ?? null,
|
||||
toolsCalled: result.toolsCalled,
|
||||
toolCallDetails: result.toolCallDetails,
|
||||
tokenUsage: result.tokenUsage,
|
||||
iterations: result.iterations,
|
||||
messages: result.messages
|
||||
}
|
||||
await writeFile(join(variantFolder, 'details.json'), JSON.stringify(details, null, '\t'))
|
||||
|
||||
// Write app files to frontend/ and backend/ folders
|
||||
await writeAppToFolder(result.files, variantFolder)
|
||||
}
|
||||
|
||||
// Write summary markdown file
|
||||
const summaryPath = join(resultFolder, 'summary.md')
|
||||
await writeFile(summaryPath, summaryLines.join('\n'))
|
||||
|
||||
return { summaryPath, variantPaths }
|
||||
}
|
||||
+22
@@ -0,0 +1,22 @@
|
||||
interface Folder {
|
||||
id: string
|
||||
name: string
|
||||
parentId: string | null
|
||||
}
|
||||
|
||||
export async function main({
|
||||
name,
|
||||
parentId
|
||||
}: {
|
||||
name: string
|
||||
parentId: string | null
|
||||
}): Promise<Folder> {
|
||||
// In a real implementation, this would create a folder in storage
|
||||
const newFolder: Folder = {
|
||||
id: `folder_${Date.now()}`,
|
||||
name,
|
||||
parentId
|
||||
}
|
||||
|
||||
return newFolder
|
||||
}
|
||||
+4
@@ -0,0 +1,4 @@
|
||||
{
|
||||
"name": "Create Folder",
|
||||
"language": "bun"
|
||||
}
|
||||
+12
@@ -0,0 +1,12 @@
|
||||
export async function main({
|
||||
id,
|
||||
type
|
||||
}: {
|
||||
id: string
|
||||
type: 'file' | 'folder'
|
||||
}): Promise<{ success: boolean }> {
|
||||
// In a real implementation, this would delete from storage
|
||||
console.log(`Deleting ${type} with id: ${id}`)
|
||||
|
||||
return { success: true }
|
||||
}
|
||||
+4
@@ -0,0 +1,4 @@
|
||||
{
|
||||
"name": "Delete Item",
|
||||
"language": "bun"
|
||||
}
|
||||
+24
@@ -0,0 +1,24 @@
|
||||
interface FileItem {
|
||||
id: string
|
||||
name: string
|
||||
type: 'file' | 'folder'
|
||||
size?: number
|
||||
modifiedAt: string
|
||||
parentId: string | null
|
||||
}
|
||||
|
||||
// Mock file system data
|
||||
const mockFiles: FileItem[] = [
|
||||
{ id: 'f1', name: 'Documents', type: 'folder', modifiedAt: '2024-01-15', parentId: null },
|
||||
{ id: 'f2', name: 'Images', type: 'folder', modifiedAt: '2024-01-10', parentId: null },
|
||||
{ id: 'f3', name: 'readme.txt', type: 'file', size: 1024, modifiedAt: '2024-01-20', parentId: null },
|
||||
{ id: 'f4', name: 'report.pdf', type: 'file', size: 52400, modifiedAt: '2024-01-18', parentId: 'f1' },
|
||||
{ id: 'f5', name: 'notes.txt', type: 'file', size: 256, modifiedAt: '2024-01-12', parentId: 'f1' },
|
||||
{ id: 'f6', name: 'photo1.jpg', type: 'file', size: 2048000, modifiedAt: '2024-01-08', parentId: 'f2' },
|
||||
{ id: 'f7', name: 'photo2.jpg', type: 'file', size: 1536000, modifiedAt: '2024-01-09', parentId: 'f2' },
|
||||
{ id: 'f8', name: 'Projects', type: 'folder', modifiedAt: '2024-01-05', parentId: 'f1' }
|
||||
]
|
||||
|
||||
export async function main({ folderId }: { folderId: string | null }): Promise<FileItem[]> {
|
||||
return mockFiles.filter((file) => file.parentId === folderId)
|
||||
}
|
||||
+4
@@ -0,0 +1,4 @@
|
||||
{
|
||||
"name": "List Files",
|
||||
"language": "bun"
|
||||
}
|
||||
+33
@@ -0,0 +1,33 @@
|
||||
interface Folder {
|
||||
id: string
|
||||
name: string
|
||||
parentId: string | null
|
||||
children: Folder[]
|
||||
}
|
||||
|
||||
// Mock folder structure
|
||||
const mockFolders: Folder[] = [
|
||||
{
|
||||
id: 'f1',
|
||||
name: 'Documents',
|
||||
parentId: null,
|
||||
children: [
|
||||
{
|
||||
id: 'f8',
|
||||
name: 'Projects',
|
||||
parentId: 'f1',
|
||||
children: []
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
id: 'f2',
|
||||
name: 'Images',
|
||||
parentId: null,
|
||||
children: []
|
||||
}
|
||||
]
|
||||
|
||||
export async function main(): Promise<Folder[]> {
|
||||
return mockFolders
|
||||
}
|
||||
+4
@@ -0,0 +1,4 @@
|
||||
{
|
||||
"name": "List Folders",
|
||||
"language": "bun"
|
||||
}
|
||||
+14
@@ -0,0 +1,14 @@
|
||||
export async function main({
|
||||
id,
|
||||
type,
|
||||
targetFolderId
|
||||
}: {
|
||||
id: string
|
||||
type: 'file' | 'folder'
|
||||
targetFolderId: string | null
|
||||
}): Promise<{ success: boolean }> {
|
||||
// In a real implementation, this would move the item in storage
|
||||
console.log(`Moving ${type} ${id} to folder: ${targetFolderId ?? 'root'}`)
|
||||
|
||||
return { success: true }
|
||||
}
|
||||
+4
@@ -0,0 +1,4 @@
|
||||
{
|
||||
"name": "Move Item",
|
||||
"language": "bun"
|
||||
}
|
||||
+14
@@ -0,0 +1,14 @@
|
||||
export async function main({
|
||||
id,
|
||||
type,
|
||||
newName
|
||||
}: {
|
||||
id: string
|
||||
type: 'file' | 'folder'
|
||||
newName: string
|
||||
}): Promise<{ success: boolean; name: string }> {
|
||||
// In a real implementation, this would rename in storage
|
||||
console.log(`Renaming ${type} ${id} to: ${newName}`)
|
||||
|
||||
return { success: true, name: newName }
|
||||
}
|
||||
+4
@@ -0,0 +1,4 @@
|
||||
{
|
||||
"name": "Rename Item",
|
||||
"language": "bun"
|
||||
}
|
||||
+26
@@ -0,0 +1,26 @@
|
||||
import React from 'react'
|
||||
|
||||
interface BreadcrumbProps {
|
||||
path: { id: string | null; name: string }[]
|
||||
onNavigate: (folderId: string | null, folderName: string) => void
|
||||
}
|
||||
|
||||
export const Breadcrumb: React.FC<BreadcrumbProps> = ({ path, onNavigate }) => {
|
||||
return (
|
||||
<div className="px-4 py-2 bg-white border-b flex items-center gap-2 text-sm">
|
||||
{path.map((item, index) => (
|
||||
<React.Fragment key={item.id ?? 'root'}>
|
||||
{index > 0 && <span className="text-gray-400">/</span>}
|
||||
<button
|
||||
onClick={() => onNavigate(item.id, item.name)}
|
||||
className={`hover:text-blue-600 ${
|
||||
index === path.length - 1 ? 'font-medium text-gray-900' : 'text-gray-600'
|
||||
}`}
|
||||
>
|
||||
{item.name}
|
||||
</button>
|
||||
</React.Fragment>
|
||||
))}
|
||||
</div>
|
||||
)
|
||||
}
|
||||
+79
@@ -0,0 +1,79 @@
|
||||
import React, { useState } from 'react'
|
||||
import type { FileItem as FileItemType } from '../index'
|
||||
|
||||
interface FileItemProps {
|
||||
item: FileItemType
|
||||
onDelete: (item: FileItemType) => void
|
||||
onRename: (item: FileItemType, newName: string) => void
|
||||
onFolderOpen: (folder: FileItemType) => void
|
||||
}
|
||||
|
||||
export const FileItem: React.FC<FileItemProps> = ({ item, onDelete, onRename, onFolderOpen }) => {
|
||||
const [isRenaming, setIsRenaming] = useState(false)
|
||||
const [newName, setNewName] = useState(item.name)
|
||||
|
||||
const handleRename = () => {
|
||||
if (newName.trim() && newName !== item.name) {
|
||||
onRename(item, newName.trim())
|
||||
}
|
||||
setIsRenaming(false)
|
||||
}
|
||||
|
||||
const handleDoubleClick = () => {
|
||||
if (item.type === 'folder') {
|
||||
onFolderOpen(item)
|
||||
}
|
||||
}
|
||||
|
||||
const icon = item.type === 'folder' ? '📁' : '📄'
|
||||
const formattedDate = new Date(item.modifiedAt).toLocaleDateString()
|
||||
|
||||
return (
|
||||
<div
|
||||
className="grid grid-cols-12 gap-4 px-4 py-2 border-b hover:bg-gray-50 items-center"
|
||||
onDoubleClick={handleDoubleClick}
|
||||
>
|
||||
<div className="col-span-6 flex items-center gap-2">
|
||||
<span>{icon}</span>
|
||||
{isRenaming ? (
|
||||
<input
|
||||
type="text"
|
||||
value={newName}
|
||||
onChange={(e) => setNewName(e.target.value)}
|
||||
onBlur={handleRename}
|
||||
onKeyDown={(e) => e.key === 'Enter' && handleRename()}
|
||||
className="border rounded px-2 py-1 flex-1"
|
||||
autoFocus
|
||||
/>
|
||||
) : (
|
||||
<span
|
||||
className={item.type === 'folder' ? 'cursor-pointer hover:text-blue-600' : ''}
|
||||
>
|
||||
{item.name}
|
||||
</span>
|
||||
)}
|
||||
</div>
|
||||
|
||||
<div className="col-span-2 text-gray-500 text-sm">
|
||||
{item.type === 'folder' ? 'Folder' : 'File'}
|
||||
</div>
|
||||
|
||||
<div className="col-span-2 text-gray-500 text-sm">{formattedDate}</div>
|
||||
|
||||
<div className="col-span-2 flex gap-2">
|
||||
<button
|
||||
onClick={() => setIsRenaming(true)}
|
||||
className="text-blue-500 hover:text-blue-700 text-sm"
|
||||
>
|
||||
Rename
|
||||
</button>
|
||||
<button
|
||||
onClick={() => onDelete(item)}
|
||||
className="text-red-500 hover:text-red-700 text-sm"
|
||||
>
|
||||
Delete
|
||||
</button>
|
||||
</div>
|
||||
</div>
|
||||
)
|
||||
}
|
||||
+46
@@ -0,0 +1,46 @@
|
||||
import React from 'react'
|
||||
import { FileItem as FileItemComponent } from './FileItem'
|
||||
import type { FileItem } from '../index'
|
||||
|
||||
interface FileListProps {
|
||||
files: FileItem[]
|
||||
onDelete: (item: FileItem) => void
|
||||
onRename: (item: FileItem, newName: string) => void
|
||||
onFolderOpen: (folder: FileItem) => void
|
||||
}
|
||||
|
||||
export const FileList: React.FC<FileListProps> = ({ files, onDelete, onRename, onFolderOpen }) => {
|
||||
if (files.length === 0) {
|
||||
return (
|
||||
<div className="text-center text-gray-500 py-8">This folder is empty</div>
|
||||
)
|
||||
}
|
||||
|
||||
// Sort: folders first, then files
|
||||
const sortedFiles = [...files].sort((a, b) => {
|
||||
if (a.type === 'folder' && b.type !== 'folder') return -1
|
||||
if (a.type !== 'folder' && b.type === 'folder') return 1
|
||||
return a.name.localeCompare(b.name)
|
||||
})
|
||||
|
||||
return (
|
||||
<div className="bg-white rounded-lg border">
|
||||
<div className="grid grid-cols-12 gap-4 px-4 py-2 border-b bg-gray-50 font-medium text-sm text-gray-600">
|
||||
<div className="col-span-6">Name</div>
|
||||
<div className="col-span-2">Type</div>
|
||||
<div className="col-span-2">Modified</div>
|
||||
<div className="col-span-2">Actions</div>
|
||||
</div>
|
||||
|
||||
{sortedFiles.map((file) => (
|
||||
<FileItemComponent
|
||||
key={file.id}
|
||||
item={file}
|
||||
onDelete={onDelete}
|
||||
onRename={onRename}
|
||||
onFolderOpen={onFolderOpen}
|
||||
/>
|
||||
))}
|
||||
</div>
|
||||
)
|
||||
}
|
||||
+56
@@ -0,0 +1,56 @@
|
||||
import React from 'react'
|
||||
import type { Folder } from '../index'
|
||||
|
||||
interface FolderTreeProps {
|
||||
folders: Folder[]
|
||||
currentFolderId: string | null
|
||||
onFolderSelect: (folderId: string | null, folderName: string) => void
|
||||
depth?: number
|
||||
}
|
||||
|
||||
export const FolderTree: React.FC<FolderTreeProps> = ({
|
||||
folders,
|
||||
currentFolderId,
|
||||
onFolderSelect,
|
||||
depth = 0
|
||||
}) => {
|
||||
return (
|
||||
<div>
|
||||
{depth === 0 && (
|
||||
<div
|
||||
className={`px-4 py-2 cursor-pointer hover:bg-gray-100 ${
|
||||
currentFolderId === null ? 'bg-blue-100 text-blue-700' : ''
|
||||
}`}
|
||||
onClick={() => onFolderSelect(null, 'Root')}
|
||||
>
|
||||
<span className="mr-2">📁</span>
|
||||
Root
|
||||
</div>
|
||||
)}
|
||||
|
||||
{folders.map((folder) => (
|
||||
<div key={folder.id}>
|
||||
<div
|
||||
className={`px-4 py-2 cursor-pointer hover:bg-gray-100 ${
|
||||
currentFolderId === folder.id ? 'bg-blue-100 text-blue-700' : ''
|
||||
}`}
|
||||
style={{ paddingLeft: `${(depth + 1) * 16 + 16}px` }}
|
||||
onClick={() => onFolderSelect(folder.id, folder.name)}
|
||||
>
|
||||
<span className="mr-2">📁</span>
|
||||
{folder.name}
|
||||
</div>
|
||||
|
||||
{folder.children && folder.children.length > 0 && (
|
||||
<FolderTree
|
||||
folders={folder.children}
|
||||
currentFolderId={currentFolderId}
|
||||
onFolderSelect={onFolderSelect}
|
||||
depth={depth + 1}
|
||||
/>
|
||||
)}
|
||||
</div>
|
||||
))}
|
||||
</div>
|
||||
)
|
||||
}
|
||||
+59
@@ -0,0 +1,59 @@
|
||||
import React, { useState } from 'react'
|
||||
|
||||
interface ToolbarProps {
|
||||
onCreateFolder: (name: string) => void
|
||||
}
|
||||
|
||||
export const Toolbar: React.FC<ToolbarProps> = ({ onCreateFolder }) => {
|
||||
const [isCreating, setIsCreating] = useState(false)
|
||||
const [folderName, setFolderName] = useState('')
|
||||
|
||||
const handleCreate = () => {
|
||||
if (folderName.trim()) {
|
||||
onCreateFolder(folderName.trim())
|
||||
setFolderName('')
|
||||
setIsCreating(false)
|
||||
}
|
||||
}
|
||||
|
||||
return (
|
||||
<div className="px-4 py-3 bg-white border-b flex items-center gap-4">
|
||||
{isCreating ? (
|
||||
<div className="flex items-center gap-2">
|
||||
<input
|
||||
type="text"
|
||||
value={folderName}
|
||||
onChange={(e) => setFolderName(e.target.value)}
|
||||
onKeyDown={(e) => e.key === 'Enter' && handleCreate()}
|
||||
placeholder="Folder name"
|
||||
className="border rounded px-3 py-1"
|
||||
autoFocus
|
||||
/>
|
||||
<button
|
||||
onClick={handleCreate}
|
||||
className="px-3 py-1 bg-blue-500 text-white rounded hover:bg-blue-600"
|
||||
>
|
||||
Create
|
||||
</button>
|
||||
<button
|
||||
onClick={() => {
|
||||
setIsCreating(false)
|
||||
setFolderName('')
|
||||
}}
|
||||
className="px-3 py-1 text-gray-600 hover:text-gray-800"
|
||||
>
|
||||
Cancel
|
||||
</button>
|
||||
</div>
|
||||
) : (
|
||||
<button
|
||||
onClick={() => setIsCreating(true)}
|
||||
className="px-3 py-1 bg-blue-500 text-white rounded hover:bg-blue-600 flex items-center gap-2"
|
||||
>
|
||||
<span>+</span>
|
||||
New Folder
|
||||
</button>
|
||||
)}
|
||||
</div>
|
||||
)
|
||||
}
|
||||
+119
@@ -0,0 +1,119 @@
|
||||
import React, { useState, useEffect } from 'react'
|
||||
import { backend } from 'wmill'
|
||||
import { FolderTree } from './components/FolderTree'
|
||||
import { FileList } from './components/FileList'
|
||||
import { Breadcrumb } from './components/Breadcrumb'
|
||||
import { Toolbar } from './components/Toolbar'
|
||||
|
||||
export interface FileItem {
|
||||
id: string
|
||||
name: string
|
||||
type: 'file' | 'folder'
|
||||
size?: number
|
||||
modifiedAt: string
|
||||
parentId: string | null
|
||||
}
|
||||
|
||||
export interface Folder {
|
||||
id: string
|
||||
name: string
|
||||
parentId: string | null
|
||||
children: Folder[]
|
||||
}
|
||||
|
||||
const App = () => {
|
||||
const [folders, setFolders] = useState<Folder[]>([])
|
||||
const [files, setFiles] = useState<FileItem[]>([])
|
||||
const [currentFolderId, setCurrentFolderId] = useState<string | null>(null)
|
||||
const [path, setPath] = useState<{ id: string | null; name: string }[]>([
|
||||
{ id: null, name: 'Root' }
|
||||
])
|
||||
const [loading, setLoading] = useState(true)
|
||||
|
||||
useEffect(() => {
|
||||
loadFolders()
|
||||
}, [])
|
||||
|
||||
useEffect(() => {
|
||||
loadFiles(currentFolderId)
|
||||
}, [currentFolderId])
|
||||
|
||||
const loadFolders = async () => {
|
||||
const data = await backend.listFolders()
|
||||
setFolders(data)
|
||||
}
|
||||
|
||||
const loadFiles = async (folderId: string | null) => {
|
||||
setLoading(true)
|
||||
const data = await backend.listFiles({ folderId })
|
||||
setFiles(data)
|
||||
setLoading(false)
|
||||
}
|
||||
|
||||
const handleFolderSelect = (folderId: string | null, folderName: string) => {
|
||||
setCurrentFolderId(folderId)
|
||||
|
||||
if (folderId === null) {
|
||||
setPath([{ id: null, name: 'Root' }])
|
||||
} else {
|
||||
// Find if folder is already in path
|
||||
const existingIndex = path.findIndex((p) => p.id === folderId)
|
||||
if (existingIndex >= 0) {
|
||||
setPath(path.slice(0, existingIndex + 1))
|
||||
} else {
|
||||
setPath([...path, { id: folderId, name: folderName }])
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
const handleCreateFolder = async (name: string) => {
|
||||
await backend.createFolder({ name, parentId: currentFolderId })
|
||||
await loadFolders()
|
||||
await loadFiles(currentFolderId)
|
||||
}
|
||||
|
||||
const handleDeleteItem = async (item: FileItem) => {
|
||||
await backend.deleteItem({ id: item.id, type: item.type })
|
||||
await loadFolders()
|
||||
await loadFiles(currentFolderId)
|
||||
}
|
||||
|
||||
const handleRenameItem = async (item: FileItem, newName: string) => {
|
||||
await backend.renameItem({ id: item.id, type: item.type, newName })
|
||||
await loadFolders()
|
||||
await loadFiles(currentFolderId)
|
||||
}
|
||||
|
||||
return (
|
||||
<div className="flex h-screen bg-gray-100">
|
||||
<div className="w-64 bg-white border-r overflow-auto">
|
||||
<div className="p-4 font-semibold border-b">Folders</div>
|
||||
<FolderTree
|
||||
folders={folders}
|
||||
currentFolderId={currentFolderId}
|
||||
onFolderSelect={handleFolderSelect}
|
||||
/>
|
||||
</div>
|
||||
|
||||
<div className="flex-1 flex flex-col">
|
||||
<Toolbar onCreateFolder={handleCreateFolder} />
|
||||
<Breadcrumb path={path} onNavigate={handleFolderSelect} />
|
||||
|
||||
<div className="flex-1 p-4 overflow-auto">
|
||||
{loading ? (
|
||||
<div className="text-center text-gray-500 py-8">Loading...</div>
|
||||
) : (
|
||||
<FileList
|
||||
files={files}
|
||||
onDelete={handleDeleteItem}
|
||||
onRename={handleRenameItem}
|
||||
onFolderOpen={(folder) => handleFolderSelect(folder.id, folder.name)}
|
||||
/>
|
||||
)}
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
)
|
||||
}
|
||||
|
||||
export default App
|
||||
+34
@@ -0,0 +1,34 @@
|
||||
interface Product {
|
||||
id: string
|
||||
name: string
|
||||
price: number
|
||||
image: string
|
||||
}
|
||||
|
||||
interface CartItem {
|
||||
product: Product
|
||||
quantity: number
|
||||
}
|
||||
|
||||
export async function main({
|
||||
cart,
|
||||
product
|
||||
}: {
|
||||
cart: CartItem[]
|
||||
product: Product
|
||||
}): Promise<CartItem[]> {
|
||||
const existingIndex = cart.findIndex((item) => item.product.id === product.id)
|
||||
|
||||
if (existingIndex >= 0) {
|
||||
// Increment quantity if already in cart
|
||||
const updatedCart = [...cart]
|
||||
updatedCart[existingIndex] = {
|
||||
...updatedCart[existingIndex],
|
||||
quantity: updatedCart[existingIndex].quantity + 1
|
||||
}
|
||||
return updatedCart
|
||||
}
|
||||
|
||||
// Add new item to cart
|
||||
return [...cart, { product, quantity: 1 }]
|
||||
}
|
||||
+4
@@ -0,0 +1,4 @@
|
||||
{
|
||||
"name": "Add to Cart",
|
||||
"language": "bun"
|
||||
}
|
||||
+17
@@ -0,0 +1,17 @@
|
||||
interface Product {
|
||||
id: string
|
||||
name: string
|
||||
price: number
|
||||
image: string
|
||||
}
|
||||
|
||||
interface CartItem {
|
||||
product: Product
|
||||
quantity: number
|
||||
}
|
||||
|
||||
export async function main({ items }: { items: CartItem[] }): Promise<number> {
|
||||
return items.reduce((total, item) => {
|
||||
return total + item.product.price * item.quantity
|
||||
}, 0)
|
||||
}
|
||||
+4
@@ -0,0 +1,4 @@
|
||||
{
|
||||
"name": "Calculate Total",
|
||||
"language": "bun"
|
||||
}
|
||||
+41
@@ -0,0 +1,41 @@
|
||||
export async function main() {
|
||||
// Mock product catalog
|
||||
return [
|
||||
{
|
||||
id: '1',
|
||||
name: 'Wireless Headphones',
|
||||
price: 79.99,
|
||||
image: 'https://picsum.photos/seed/headphones/300/200'
|
||||
},
|
||||
{
|
||||
id: '2',
|
||||
name: 'Smart Watch',
|
||||
price: 199.99,
|
||||
image: 'https://picsum.photos/seed/watch/300/200'
|
||||
},
|
||||
{
|
||||
id: '3',
|
||||
name: 'Laptop Stand',
|
||||
price: 49.99,
|
||||
image: 'https://picsum.photos/seed/stand/300/200'
|
||||
},
|
||||
{
|
||||
id: '4',
|
||||
name: 'Mechanical Keyboard',
|
||||
price: 129.99,
|
||||
image: 'https://picsum.photos/seed/keyboard/300/200'
|
||||
},
|
||||
{
|
||||
id: '5',
|
||||
name: 'USB-C Hub',
|
||||
price: 39.99,
|
||||
image: 'https://picsum.photos/seed/hub/300/200'
|
||||
},
|
||||
{
|
||||
id: '6',
|
||||
name: 'Webcam HD',
|
||||
price: 89.99,
|
||||
image: 'https://picsum.photos/seed/webcam/300/200'
|
||||
}
|
||||
]
|
||||
}
|
||||
+4
@@ -0,0 +1,4 @@
|
||||
{
|
||||
"name": "Get Products",
|
||||
"language": "bun"
|
||||
}
|
||||
+21
@@ -0,0 +1,21 @@
|
||||
interface Product {
|
||||
id: string
|
||||
name: string
|
||||
price: number
|
||||
image: string
|
||||
}
|
||||
|
||||
interface CartItem {
|
||||
product: Product
|
||||
quantity: number
|
||||
}
|
||||
|
||||
export async function main({
|
||||
cart,
|
||||
productId
|
||||
}: {
|
||||
cart: CartItem[]
|
||||
productId: string
|
||||
}): Promise<CartItem[]> {
|
||||
return cart.filter((item) => item.product.id !== productId)
|
||||
}
|
||||
+4
@@ -0,0 +1,4 @@
|
||||
{
|
||||
"name": "Remove from Cart",
|
||||
"language": "bun"
|
||||
}
|
||||
+51
@@ -0,0 +1,51 @@
|
||||
import React from 'react'
|
||||
import type { CartItem } from '../index'
|
||||
|
||||
interface CartProps {
|
||||
items: CartItem[]
|
||||
total: number
|
||||
onRemoveItem: (productId: string) => void
|
||||
}
|
||||
|
||||
export const Cart: React.FC<CartProps> = ({ items, total, onRemoveItem }) => {
|
||||
return (
|
||||
<div className="flex flex-col h-full">
|
||||
<h2 className="text-xl font-bold mb-4">Cart</h2>
|
||||
|
||||
{items.length === 0 ? (
|
||||
<p className="text-gray-500 text-center py-8">Your cart is empty</p>
|
||||
) : (
|
||||
<>
|
||||
<div className="flex-1 overflow-auto">
|
||||
{items.map((item) => (
|
||||
<div
|
||||
key={item.product.id}
|
||||
className="flex items-center justify-between py-3 border-b"
|
||||
>
|
||||
<div className="flex-1">
|
||||
<p className="font-medium">{item.product.name}</p>
|
||||
<p className="text-sm text-gray-500">
|
||||
${item.product.price.toFixed(2)} x {item.quantity}
|
||||
</p>
|
||||
</div>
|
||||
<button
|
||||
onClick={() => onRemoveItem(item.product.id)}
|
||||
className="text-red-500 hover:text-red-700 px-2"
|
||||
>
|
||||
Remove
|
||||
</button>
|
||||
</div>
|
||||
))}
|
||||
</div>
|
||||
|
||||
<div className="border-t pt-4 mt-4">
|
||||
<div className="flex justify-between text-lg font-bold">
|
||||
<span>Total:</span>
|
||||
<span>${total.toFixed(2)}</span>
|
||||
</div>
|
||||
</div>
|
||||
</>
|
||||
)}
|
||||
</div>
|
||||
)
|
||||
}
|
||||
+27
@@ -0,0 +1,27 @@
|
||||
import React from 'react'
|
||||
import type { Product } from '../index'
|
||||
|
||||
interface ProductCardProps {
|
||||
product: Product
|
||||
onAddToCart: (product: Product) => void
|
||||
}
|
||||
|
||||
export const ProductCard: React.FC<ProductCardProps> = ({ product, onAddToCart }) => {
|
||||
return (
|
||||
<div className="border rounded-lg p-4 bg-white shadow-sm">
|
||||
<img
|
||||
src={product.image}
|
||||
alt={product.name}
|
||||
className="w-full h-48 object-cover rounded mb-4"
|
||||
/>
|
||||
<h3 className="font-semibold text-lg">{product.name}</h3>
|
||||
<p className="text-gray-600 mb-4">${product.price.toFixed(2)}</p>
|
||||
<button
|
||||
onClick={() => onAddToCart(product)}
|
||||
className="w-full py-2 bg-blue-500 text-white rounded hover:bg-blue-600 transition"
|
||||
>
|
||||
Add to Cart
|
||||
</button>
|
||||
</div>
|
||||
)
|
||||
}
|
||||
+18
@@ -0,0 +1,18 @@
|
||||
import React from 'react'
|
||||
import { ProductCard } from './ProductCard'
|
||||
import type { Product } from '../index'
|
||||
|
||||
interface ProductListProps {
|
||||
products: Product[]
|
||||
onAddToCart: (product: Product) => void
|
||||
}
|
||||
|
||||
export const ProductList: React.FC<ProductListProps> = ({ products, onAddToCart }) => {
|
||||
return (
|
||||
<div className="grid grid-cols-1 md:grid-cols-2 lg:grid-cols-3 gap-6">
|
||||
{products.map((product) => (
|
||||
<ProductCard key={product.id} product={product} onAddToCart={onAddToCart} />
|
||||
))}
|
||||
</div>
|
||||
)
|
||||
}
|
||||
+81
@@ -0,0 +1,81 @@
|
||||
import React, { useState, useEffect } from 'react'
|
||||
import { backend } from 'wmill'
|
||||
import { ProductList } from './components/ProductList'
|
||||
import { Cart } from './components/Cart'
|
||||
|
||||
export interface Product {
|
||||
id: string
|
||||
name: string
|
||||
price: number
|
||||
image: string
|
||||
}
|
||||
|
||||
export interface CartItem {
|
||||
product: Product
|
||||
quantity: number
|
||||
}
|
||||
|
||||
const App = () => {
|
||||
const [products, setProducts] = useState<Product[]>([])
|
||||
const [cartItems, setCartItems] = useState<CartItem[]>([])
|
||||
const [total, setTotal] = useState(0)
|
||||
const [loading, setLoading] = useState(true)
|
||||
|
||||
useEffect(() => {
|
||||
loadProducts()
|
||||
}, [])
|
||||
|
||||
useEffect(() => {
|
||||
updateTotal()
|
||||
}, [cartItems])
|
||||
|
||||
const loadProducts = async () => {
|
||||
setLoading(true)
|
||||
const data = await backend.getProducts()
|
||||
setProducts(data)
|
||||
setLoading(false)
|
||||
}
|
||||
|
||||
const updateTotal = async () => {
|
||||
if (cartItems.length === 0) {
|
||||
setTotal(0)
|
||||
return
|
||||
}
|
||||
const newTotal = await backend.calculateTotal({ items: cartItems })
|
||||
setTotal(newTotal)
|
||||
}
|
||||
|
||||
const handleAddToCart = async (product: Product) => {
|
||||
const updatedCart = await backend.addToCart({
|
||||
cart: cartItems,
|
||||
product
|
||||
})
|
||||
setCartItems(updatedCart)
|
||||
}
|
||||
|
||||
const handleRemoveFromCart = async (productId: string) => {
|
||||
const updatedCart = await backend.removeFromCart({
|
||||
cart: cartItems,
|
||||
productId
|
||||
})
|
||||
setCartItems(updatedCart)
|
||||
}
|
||||
|
||||
if (loading) {
|
||||
return <div className="p-8 text-center">Loading products...</div>
|
||||
}
|
||||
|
||||
return (
|
||||
<div className="flex h-screen">
|
||||
<div className="flex-1 p-6 overflow-auto">
|
||||
<h1 className="text-2xl font-bold mb-6">Shop</h1>
|
||||
<ProductList products={products} onAddToCart={handleAddToCart} />
|
||||
</div>
|
||||
<div className="w-80 border-l bg-gray-50 p-6">
|
||||
<Cart items={cartItems} total={total} onRemoveItem={handleRemoveFromCart} />
|
||||
</div>
|
||||
</div>
|
||||
)
|
||||
}
|
||||
|
||||
export default App
|
||||
+3
@@ -0,0 +1,3 @@
|
||||
export async function main({ currentCount }: { currentCount: number }) {
|
||||
return currentCount - 1
|
||||
}
|
||||
+4
@@ -0,0 +1,4 @@
|
||||
{
|
||||
"name": "Decrement Counter",
|
||||
"language": "bun"
|
||||
}
|
||||
+3
@@ -0,0 +1,3 @@
|
||||
export async function main({ currentCount }: { currentCount: number }) {
|
||||
return currentCount + 1
|
||||
}
|
||||
+4
@@ -0,0 +1,4 @@
|
||||
{
|
||||
"name": "Increment Counter",
|
||||
"language": "bun"
|
||||
}
|
||||
+38
@@ -0,0 +1,38 @@
|
||||
import React, { useState } from 'react'
|
||||
import { backend } from 'wmill'
|
||||
|
||||
const CounterApp = () => {
|
||||
const [count, setCount] = useState(0)
|
||||
|
||||
const increment = async () => {
|
||||
const newCount = await backend.incrementCounter({ currentCount: count })
|
||||
setCount(newCount)
|
||||
}
|
||||
|
||||
const decrement = async () => {
|
||||
const newCount = await backend.decrementCounter({ currentCount: count })
|
||||
setCount(newCount)
|
||||
}
|
||||
|
||||
return (
|
||||
<div className="p-4">
|
||||
<h1 className="text-2xl font-bold mb-4">Counter: {count}</h1>
|
||||
<div className="flex gap-2">
|
||||
<button
|
||||
onClick={decrement}
|
||||
className="px-4 py-2 bg-red-500 text-white rounded hover:bg-red-600"
|
||||
>
|
||||
Decrement
|
||||
</button>
|
||||
<button
|
||||
onClick={increment}
|
||||
className="px-4 py-2 bg-green-500 text-white rounded hover:bg-green-600"
|
||||
>
|
||||
Increment
|
||||
</button>
|
||||
</div>
|
||||
</div>
|
||||
)
|
||||
}
|
||||
|
||||
export default CounterApp
|
||||
@@ -0,0 +1,12 @@
|
||||
import type { VariantConfig } from '../../shared'
|
||||
|
||||
/**
|
||||
* Baseline variant - uses the production system prompt and all tools.
|
||||
* This is the default configuration that matches the actual app chat implementation.
|
||||
*/
|
||||
export const BASELINE_VARIANT: VariantConfig = {
|
||||
name: 'baseline',
|
||||
description: 'Production configuration with default system prompt and all tools',
|
||||
systemPrompt: { type: 'default' },
|
||||
tools: { type: 'default' }
|
||||
}
|
||||
@@ -0,0 +1,6 @@
|
||||
// Re-export all variant configurations
|
||||
export { BASELINE_VARIANT } from './baseline'
|
||||
export { STREAMLINED_VARIANT } from './streamlined'
|
||||
|
||||
// Re-export types for convenience
|
||||
export type { VariantConfig } from '../../shared'
|
||||
@@ -0,0 +1,144 @@
|
||||
import type { VariantConfig } from '../../shared'
|
||||
import type { Tool } from '../../shared'
|
||||
import type { AppAIChatHelpers } from '../../../app/core'
|
||||
import { getAppTools } from '../../../app/core'
|
||||
|
||||
// Tool names to remove (batch-fetch tools)
|
||||
const TOOLS_TO_REMOVE = ['get_files', 'get_frontend_files', 'get_backend_runnables']
|
||||
|
||||
/**
|
||||
* Build the streamlined tools by filtering out batch-fetch tools.
|
||||
*/
|
||||
function buildStreamlinedTools(): Tool<AppAIChatHelpers>[] {
|
||||
const defaultTools = getAppTools()
|
||||
return defaultTools.filter((t) => !TOOLS_TO_REMOVE.includes(t.def.function.name))
|
||||
}
|
||||
|
||||
/**
|
||||
* Streamlined system prompt - simplified instructions focused on:
|
||||
* 1. Reading relevant files first
|
||||
* 2. Making changes with appropriate tools
|
||||
* 3. Using lint at the end to fix errors
|
||||
*/
|
||||
const STREAMLINED_SYSTEM_PROMPT = `You are a helpful assistant that creates and edits apps on the Windmill platform. Apps are defined as a collection of files that contains both the frontend and the backend.
|
||||
|
||||
## App Structure
|
||||
|
||||
### Frontend
|
||||
- The frontend is bundled using esbuild with entrypoint \`index.tsx\`
|
||||
- Frontend files are managed separately from backend runnables
|
||||
- The \`wmill.d.ts\` file is generated automatically from the backend runnables shape
|
||||
|
||||
### Backend
|
||||
Backend runnables can be of different types:
|
||||
- **inline**: Custom code written directly in the app (TypeScript/Bun or Python)
|
||||
- **script**: Reference to a workspace script by path
|
||||
- **flow**: Reference to a workspace flow by path
|
||||
- **hubscript**: Reference to a hub script by path
|
||||
|
||||
Frontend calls backend using \`await backend.<runnable_key>(args...)\`.
|
||||
|
||||
For inline scripts, the code must have a \`main\` function as its entrypoint.
|
||||
|
||||
## Available Tools
|
||||
|
||||
### File Management
|
||||
- \`list_frontend_files()\`: List all frontend file paths (use this first to see what exists)
|
||||
- \`get_frontend_file(path)\`: Get content of a specific frontend file
|
||||
- \`set_frontend_file(path, content)\`: Create or update a frontend file. Returns lint diagnostics.
|
||||
- \`delete_frontend_file(path)\`: Delete a frontend file
|
||||
- \`list_backend_runnables()\`: List all backend runnable keys and names (use this first to see what exists)
|
||||
- \`get_backend_runnable(key)\`: Get full configuration of a specific backend runnable
|
||||
- \`set_backend_runnable(key, name, type, ...)\`: Create or update a backend runnable. Returns lint diagnostics.
|
||||
- \`delete_backend_runnable(key)\`: Delete a backend runnable
|
||||
|
||||
### Linting
|
||||
- \`lint()\`: Lint all files. Returns errors/warnings grouped by frontend/backend.
|
||||
|
||||
### Discovery
|
||||
- \`list_workspace_runnables(query, type?)\`: Search workspace scripts and flows
|
||||
- \`search_hub_scripts(query)\`: Search hub scripts
|
||||
|
||||
## Backend Runnable Configuration
|
||||
|
||||
When creating a backend runnable with \`set_backend_runnable\`:
|
||||
|
||||
1. **For inline scripts** (type: "inline"):
|
||||
\`\`\`
|
||||
{
|
||||
key: "myFunction",
|
||||
name: "Does something useful",
|
||||
type: "inline",
|
||||
inlineScript: {
|
||||
language: "bun", // or "python3"
|
||||
content: "export async function main(arg1: string) { return result; }"
|
||||
}
|
||||
}
|
||||
\`\`\`
|
||||
|
||||
2. **For workspace scripts** (type: "script"):
|
||||
\`\`\`
|
||||
{
|
||||
key: "sendEmail",
|
||||
name: "Send email via SMTP",
|
||||
type: "script",
|
||||
path: "f/folder/send_email",
|
||||
staticInputs: { smtp_server: "mail.example.com" } // optional pre-filled inputs
|
||||
}
|
||||
\`\`\`
|
||||
|
||||
3. **For workspace flows** (type: "flow"):
|
||||
\`\`\`
|
||||
{
|
||||
key: "processOrder",
|
||||
name: "Process customer order",
|
||||
type: "flow",
|
||||
path: "f/folder/process_order_flow"
|
||||
}
|
||||
\`\`\`
|
||||
|
||||
4. **For hub scripts** (type: "hubscript"):
|
||||
\`\`\`
|
||||
{
|
||||
key: "slackMessage",
|
||||
name: "Send Slack message",
|
||||
type: "hubscript",
|
||||
path: "hub/123/slack/send_message"
|
||||
}
|
||||
\`\`\`
|
||||
|
||||
## Instructions
|
||||
|
||||
1. Start by reading relevant files to understand the current state
|
||||
2. Make changes using the appropriate tools
|
||||
3. Use \`lint()\` at the end to check for and fix any errors
|
||||
|
||||
Windmill expects all backend runnable calls to use an object parameter structure. For example for:
|
||||
\`\`\`typescript
|
||||
export async function main(arg1: string, arg2: string, arg3: number, arg4: { field1: string, field2: number }) {
|
||||
...
|
||||
}
|
||||
\`\`\`
|
||||
|
||||
You would call it like this:
|
||||
\`\`\`typescript
|
||||
await backend.myFunction({ arg1: 'value1', arg2: 'value2', arg3: 3, arg4: { field1: 'value1', field2: 2 } })
|
||||
\`\`\`
|
||||
If the runnable has no parameters, you can call it without an object:
|
||||
\`\`\`typescript
|
||||
await backend.myFunction()
|
||||
\`\`\`
|
||||
|
||||
When you are using the windmill-client, do not forget that as id for variables or resources, those are path that are of the form 'u/<user>/<name>' or 'f/<folder>/<name>'.
|
||||
`
|
||||
|
||||
/**
|
||||
* Streamlined variant - removes batch-fetch tools and uses simplified instructions.
|
||||
* Forces the model to read individual files before making changes.
|
||||
*/
|
||||
export const STREAMLINED_VARIANT: VariantConfig = {
|
||||
name: 'streamlined',
|
||||
description: 'No batch tools - forces reading individual files before making changes',
|
||||
systemPrompt: { type: 'custom', content: STREAMLINED_SYSTEM_PROMPT },
|
||||
tools: { type: 'custom', tools: buildStreamlinedTools() }
|
||||
}
|
||||
+8
-9
@@ -1,6 +1,5 @@
|
||||
import { describe, it, expect } from 'vitest'
|
||||
import { runVariantComparison, type ExpectedFlow } from './evalRunner'
|
||||
import { writeComparisonResults } from './evalResultsWriter'
|
||||
import { runVariantComparison, writeFlowComparisonResults, type ExpectedFlow } from './flowEvalRunner'
|
||||
import { BASELINE_VARIANT, MINIMAL_SINGLE_TOOL_VARIANT } from './variants'
|
||||
// @ts-ignore - JSON import
|
||||
import expectedTest1 from './expected/test1.json'
|
||||
@@ -71,7 +70,7 @@ STEP 5: Return action taken for each user
|
||||
})
|
||||
|
||||
// Write results to files
|
||||
const { summaryPath, flowPaths } = await writeComparisonResults(USER_PROMPT, results)
|
||||
const { summaryPath, flowPaths } = await writeFlowComparisonResults(USER_PROMPT, results)
|
||||
console.log(`\nResults written to: ${summaryPath}`)
|
||||
console.log(`Flow files: ${flowPaths.join(', ')}`)
|
||||
|
||||
@@ -117,7 +116,7 @@ STEP 7: Return final order summary with status
|
||||
expectedFlow: expectedTest2 as ExpectedFlow
|
||||
})
|
||||
|
||||
const { summaryPath, flowPaths } = await writeComparisonResults(USER_PROMPT, results)
|
||||
const { summaryPath, flowPaths } = await writeFlowComparisonResults(USER_PROMPT, results)
|
||||
console.log(`\nResults written to: ${summaryPath}`)
|
||||
console.log(`Flow files: ${flowPaths.join(', ')}`)
|
||||
|
||||
@@ -166,7 +165,7 @@ STEP 6: Return processing report with statistics (total records, quality score,
|
||||
expectedFlow: expectedTest3 as ExpectedFlow
|
||||
})
|
||||
|
||||
const { summaryPath, flowPaths } = await writeComparisonResults(USER_PROMPT, results)
|
||||
const { summaryPath, flowPaths } = await writeFlowComparisonResults(USER_PROMPT, results)
|
||||
console.log(`\nResults written to: ${summaryPath}`)
|
||||
console.log(`Flow files: ${flowPaths.join(', ')}`)
|
||||
|
||||
@@ -215,7 +214,7 @@ STEP 5: Return the agent's response and any actions taken
|
||||
expectedFlow: expectedTest4 as ExpectedFlow
|
||||
})
|
||||
|
||||
const { summaryPath, flowPaths } = await writeComparisonResults(USER_PROMPT, results)
|
||||
const { summaryPath, flowPaths } = await writeFlowComparisonResults(USER_PROMPT, results)
|
||||
console.log(`\nResults written to: ${summaryPath}`)
|
||||
console.log(`Flow files: ${flowPaths.join(', ')}`)
|
||||
|
||||
@@ -263,7 +262,7 @@ Modify this existing flow to add error handling:
|
||||
expectedFlow: expectedTest5 as ExpectedFlow
|
||||
})
|
||||
|
||||
const { summaryPath, flowPaths } = await writeComparisonResults(USER_PROMPT, results)
|
||||
const { summaryPath, flowPaths } = await writeFlowComparisonResults(USER_PROMPT, results)
|
||||
console.log(`\nResults written to: ${summaryPath}`)
|
||||
console.log(`Flow files: ${flowPaths.join(', ')}`)
|
||||
|
||||
@@ -309,7 +308,7 @@ Modify the order processing loop to handle different order types:
|
||||
expectedFlow: expectedTest6 as ExpectedFlow
|
||||
})
|
||||
|
||||
const { summaryPath, flowPaths } = await writeComparisonResults(USER_PROMPT, results)
|
||||
const { summaryPath, flowPaths } = await writeFlowComparisonResults(USER_PROMPT, results)
|
||||
console.log(`\nResults written to: ${summaryPath}`)
|
||||
console.log(`Flow files: ${flowPaths.join(', ')}`)
|
||||
|
||||
@@ -355,7 +354,7 @@ Refactor this flow for better performance by parallelizing the enrichment steps:
|
||||
expectedFlow: expectedTest7 as ExpectedFlow
|
||||
})
|
||||
|
||||
const { summaryPath, flowPaths } = await writeComparisonResults(USER_PROMPT, results)
|
||||
const { summaryPath, flowPaths } = await writeFlowComparisonResults(USER_PROMPT, results)
|
||||
console.log(`\nResults written to: ${summaryPath}`)
|
||||
console.log(`Flow files: ${flowPaths.join(', ')}`)
|
||||
|
||||
@@ -0,0 +1,72 @@
|
||||
import type { FlowModule } from '$lib/gen'
|
||||
import { evaluateWithLLM, BASE_EVALUATOR_RESPONSE_FORMAT } from '../shared'
|
||||
import type { EvaluationResult } from '../shared'
|
||||
|
||||
/**
|
||||
* Expected flow structure for evaluation.
|
||||
*/
|
||||
export interface ExpectedFlow {
|
||||
summary?: string
|
||||
value: {
|
||||
modules: FlowModule[]
|
||||
}
|
||||
schema?: Record<string, any>
|
||||
}
|
||||
|
||||
/**
|
||||
* Flow-specific evaluator system prompt.
|
||||
*/
|
||||
const FLOW_EVALUATOR_SYSTEM_PROMPT = `You are an expert evaluator for Windmill flow definitions. Your task is to evaluate a generated flow against:
|
||||
1. The original user request/prompt
|
||||
2. An expected reference flow
|
||||
|
||||
## Windmill Flow Context
|
||||
- Flows consist of modules (steps) that execute sequentially
|
||||
- Module types include: rawscript, forloopflow, branchone, branchall, script, flow, aiagent
|
||||
- Each module has an id, value (containing type and config), and may have input_transforms
|
||||
- input_transforms connect modules using expressions like "results.previous_step". Valid input_transforms are: static, javascript. Valid variables in javascript expressions are: results, flow_input, flow_input.iter.value (for forloopflow), flow_input.iter.index (for forloopflow).
|
||||
- forloopflow contains nested modules that execute per iteration with access to flow_input.iter.value
|
||||
- branchone executes first matching branch, branchall executes all matching branches
|
||||
- Branches have conditional expressions (expr) that determine execution
|
||||
- aiagent modules contain tools array with tool definitions
|
||||
|
||||
## Evaluation Criteria
|
||||
1. **User Request Fulfillment**: Does the generated flow address ALL requirements from the user's original prompt?
|
||||
- Are all requested steps present?
|
||||
- Are the requested features implemented (loops, branches, specific logic)?
|
||||
- Does the schema match what the user requested for inputs?
|
||||
2. **Structure**: Are the module types and nesting structure appropriate for the task?
|
||||
3. **Logic**: Does the flow accomplish the intended logical task?
|
||||
4. **Connections**: Are input_transforms connecting data correctly between steps?
|
||||
5. **Completeness**: Are all required steps present with no major omissions?
|
||||
6. **Code Quality**: Is the code functionally correct (exact syntax doesn't need to match)?
|
||||
|
||||
## Important Notes
|
||||
- Minor differences in variable names, code formatting, or exact wording are acceptable
|
||||
- Focus on functional equivalence, not character-by-character matching
|
||||
- The generated flow should achieve the same outcome as described in the user request
|
||||
- Extra helper steps or slightly different approaches can still score high if they accomplish the goal
|
||||
- If the user requested specific module types (like aiagent), verify they are used correctly
|
||||
|
||||
${BASE_EVALUATOR_RESPONSE_FORMAT}`
|
||||
|
||||
/**
|
||||
* Evaluates how well a generated flow matches an expected flow and user request using an LLM.
|
||||
* Returns a resemblance score (0-100), a qualitative statement, and any missing requirements.
|
||||
*/
|
||||
export async function evaluateFlowComparison(
|
||||
generatedFlow: ExpectedFlow,
|
||||
expectedFlow: ExpectedFlow,
|
||||
userPrompt: string
|
||||
): Promise<EvaluationResult> {
|
||||
// @ts-ignore
|
||||
const apiKey = process.env.OPENROUTER_API_KEY
|
||||
|
||||
return evaluateWithLLM({
|
||||
userPrompt,
|
||||
generatedOutput: generatedFlow,
|
||||
expectedOutput: expectedFlow,
|
||||
evaluatorSystemPrompt: FLOW_EVALUATOR_SYSTEM_PROMPT,
|
||||
apiKey
|
||||
})
|
||||
}
|
||||
+4
-4
@@ -1,14 +1,14 @@
|
||||
import type { FlowAIChatHelpers } from '../../core'
|
||||
import type { FlowAIChatHelpers } from '../../flow/core'
|
||||
import type { FlowModule, InputTransform } from '$lib/gen'
|
||||
import type { ExtendedOpenFlow } from '$lib/components/flows/types'
|
||||
import { findModuleById } from '../../../shared'
|
||||
import { inlineScriptStore, restoreInlineScriptReferences } from '../../inlineScriptsUtils'
|
||||
import { findModuleById } from '../../shared'
|
||||
import { inlineScriptStore, restoreInlineScriptReferences } from '../../flow/inlineScriptsUtils'
|
||||
|
||||
/**
|
||||
* Creates mock FlowAIChatHelpers for eval testing.
|
||||
* Tracks flow state in memory and allows tool functions to modify it.
|
||||
*/
|
||||
export function createEvalHelpers(
|
||||
export function createFlowEvalHelpers(
|
||||
initialModules: FlowModule[] = [],
|
||||
initialSchema?: Record<string, any>
|
||||
) {
|
||||
@@ -0,0 +1,173 @@
|
||||
import type { FlowModule } from '$lib/gen'
|
||||
import type { ExtendedOpenFlow } from '$lib/components/flows/types'
|
||||
import { flowTools, prepareFlowSystemMessage, prepareFlowUserMessage, type FlowAIChatHelpers } from '../../flow/core'
|
||||
import { createFlowEvalHelpers } from './flowEvalHelpers'
|
||||
import { evaluateFlowComparison, type ExpectedFlow } from './flowEvalComparison'
|
||||
import {
|
||||
runEval,
|
||||
resolveSystemPrompt,
|
||||
resolveTools,
|
||||
resolveModel,
|
||||
writeComparisonResults,
|
||||
type VariantConfig,
|
||||
type BaseEvalResult,
|
||||
type EvaluationResult,
|
||||
type Tool,
|
||||
type VariantDefaults
|
||||
} from '../shared'
|
||||
|
||||
// Re-export for convenience
|
||||
export type { ExpectedFlow } from './flowEvalComparison'
|
||||
|
||||
/**
|
||||
* Flow-specific evaluation result.
|
||||
*/
|
||||
export interface FlowEvalResult extends BaseEvalResult<ExtendedOpenFlow> {
|
||||
/** Alias for output to maintain API compatibility */
|
||||
flow: ExtendedOpenFlow
|
||||
}
|
||||
|
||||
/**
|
||||
* Options for running a flow evaluation.
|
||||
*/
|
||||
export interface FlowEvalOptions {
|
||||
initialModules?: FlowModule[]
|
||||
initialSchema?: Record<string, any>
|
||||
model?: string
|
||||
customSystemPrompt?: string
|
||||
maxIterations?: number
|
||||
variant?: VariantConfig
|
||||
expectedFlow?: ExpectedFlow
|
||||
}
|
||||
|
||||
/**
|
||||
* Flow-specific variant defaults.
|
||||
*/
|
||||
const flowDefaults: VariantDefaults<FlowAIChatHelpers> = {
|
||||
prepareSystemMessage: prepareFlowSystemMessage,
|
||||
tools: flowTools as Tool<FlowAIChatHelpers>[]
|
||||
}
|
||||
|
||||
/**
|
||||
* Runs a flow chat evaluation with real OpenAI API calls.
|
||||
* Executes tool calls using the actual flowTools from core.ts or variant-configured tools.
|
||||
*/
|
||||
export async function runFlowEval(
|
||||
userPrompt: string,
|
||||
openaiApiKey: string,
|
||||
options?: FlowEvalOptions
|
||||
): Promise<FlowEvalResult> {
|
||||
const { helpers, getFlow } = createFlowEvalHelpers(
|
||||
options?.initialModules ?? [],
|
||||
options?.initialSchema
|
||||
)
|
||||
|
||||
// Resolve variant configuration
|
||||
const variantName = options?.variant?.name ?? 'baseline'
|
||||
const systemMessage = resolveSystemPrompt(options?.variant, flowDefaults, options?.customSystemPrompt)
|
||||
const { toolDefs, tools } = resolveTools(options?.variant, flowDefaults)
|
||||
const model = resolveModel(options?.variant, options?.model)
|
||||
|
||||
// Build user message
|
||||
const userMessage = prepareFlowUserMessage(userPrompt, helpers.getFlowAndSelectedId(), [])
|
||||
|
||||
// Run the base evaluation
|
||||
const rawResult = await runEval({
|
||||
userPrompt,
|
||||
systemMessage,
|
||||
userMessage,
|
||||
toolDefs,
|
||||
tools,
|
||||
helpers,
|
||||
apiKey: openaiApiKey,
|
||||
getOutput: getFlow,
|
||||
options: {
|
||||
maxIterations: options?.maxIterations,
|
||||
model,
|
||||
workspace: 'test-workspace'
|
||||
}
|
||||
})
|
||||
|
||||
// Run evaluation if expected flow is provided
|
||||
let evaluationResult: EvaluationResult | undefined
|
||||
if (options?.expectedFlow) {
|
||||
const generatedFlow = getFlow()
|
||||
evaluationResult = await evaluateFlowComparison(
|
||||
{
|
||||
summary: generatedFlow.summary,
|
||||
value: { modules: generatedFlow.value.modules },
|
||||
schema: generatedFlow.schema
|
||||
},
|
||||
options.expectedFlow,
|
||||
userPrompt
|
||||
)
|
||||
}
|
||||
|
||||
return {
|
||||
...rawResult,
|
||||
variantName,
|
||||
flow: rawResult.output,
|
||||
evaluationResult
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Runs the same prompt against multiple variants sequentially for comparison.
|
||||
* Returns results in the same order as the input variants.
|
||||
*/
|
||||
export async function runVariantComparison(
|
||||
userPrompt: string,
|
||||
variants: VariantConfig[],
|
||||
openaiApiKey: string,
|
||||
baseOptions?: Omit<FlowEvalOptions, 'variant'>
|
||||
): Promise<FlowEvalResult[]> {
|
||||
const results: FlowEvalResult[] = await Promise.all(
|
||||
variants.map(async (variant) => {
|
||||
return await runFlowEval(userPrompt, openaiApiKey, {
|
||||
...baseOptions,
|
||||
variant
|
||||
})
|
||||
})
|
||||
)
|
||||
return results
|
||||
}
|
||||
|
||||
/**
|
||||
* Writes flow comparison results to files.
|
||||
*/
|
||||
export async function writeFlowComparisonResults(
|
||||
userPrompt: string,
|
||||
results: FlowEvalResult[],
|
||||
outputDir?: string
|
||||
): Promise<{ summaryPath: string; flowPaths: string[] }> {
|
||||
// @ts-ignore
|
||||
const { dirname, join } = await import('path')
|
||||
// @ts-ignore
|
||||
const { fileURLToPath } = await import('url')
|
||||
|
||||
const __filename = fileURLToPath(import.meta.url)
|
||||
const __dirname = dirname(__filename)
|
||||
|
||||
const resultsDir = outputDir ?? join(__dirname, 'results')
|
||||
|
||||
const result = await writeComparisonResults({
|
||||
userPrompt,
|
||||
results,
|
||||
outputDir: resultsDir,
|
||||
formatOutput: (flow: ExtendedOpenFlow) => ({
|
||||
summary: flow.summary ?? '',
|
||||
value: {
|
||||
modules: flow.value.modules
|
||||
},
|
||||
schema: flow.schema ?? {
|
||||
$schema: 'https://json-schema.org/draft/2020-12/schema',
|
||||
properties: {},
|
||||
required: [],
|
||||
type: 'object'
|
||||
}
|
||||
}),
|
||||
outputLabel: 'flow'
|
||||
})
|
||||
|
||||
return { summaryPath: result.summaryPath, flowPaths: result.outputPaths }
|
||||
}
|
||||
+1
-1
@@ -1,4 +1,4 @@
|
||||
import type { VariantConfig } from '../evalVariants'
|
||||
import type { VariantConfig } from '../../shared'
|
||||
|
||||
/**
|
||||
* Baseline variant - uses the production system prompt and all tools.
|
||||
+1
-1
@@ -3,4 +3,4 @@ export { BASELINE_VARIANT } from './baseline'
|
||||
export { MINIMAL_SINGLE_TOOL_VARIANT, setFlowJsonTool } from './minimal-single-tool'
|
||||
|
||||
// Re-export types for convenience
|
||||
export type { VariantConfig } from '../evalVariants'
|
||||
export type { VariantConfig } from '../../shared'
|
||||
+7
-7
@@ -1,8 +1,8 @@
|
||||
import type { VariantConfig } from '../evalVariants'
|
||||
import type { Tool } from '../../../../shared'
|
||||
import type { FlowAIChatHelpers } from '../../../core'
|
||||
import { flowTools } from '../../../core'
|
||||
import openFlowSchema from '../../../openFlow.json'
|
||||
import type { VariantConfig } from '../../shared'
|
||||
import type { Tool } from '../../shared'
|
||||
import type { FlowAIChatHelpers } from '../../../flow/core'
|
||||
import { flowTools } from '../../../flow/core'
|
||||
import openFlowSchema from '../../../flow/openFlow.json'
|
||||
|
||||
/**
|
||||
* IDs of the granular flow editing tools that should be replaced by set_flow_json.
|
||||
@@ -48,7 +48,7 @@ export const setFlowJsonTool: Tool<FlowAIChatHelpers> = {
|
||||
}
|
||||
},
|
||||
fn: async ({ args, helpers }) => {
|
||||
const { modules, schema } = args
|
||||
const { modules, schema } = args as { modules: any[]; schema?: Record<string, any> }
|
||||
await helpers.setFlowJson(modules, schema)
|
||||
return `Flow updated with ${modules.length} module(s): [${modules.map((m: any) => m.id).join(', ')}]`
|
||||
}
|
||||
@@ -61,7 +61,7 @@ export const setFlowJsonTool: Tool<FlowAIChatHelpers> = {
|
||||
*/
|
||||
function buildMinimalSingleToolTools(): Tool<FlowAIChatHelpers>[] {
|
||||
// Get all production tools except flow editing tools
|
||||
const utilityTools = flowTools.filter(
|
||||
const utilityTools = (flowTools as Tool<FlowAIChatHelpers>[]).filter(
|
||||
(t) => !FLOW_EDITING_TOOL_NAMES.includes(t.def.function.name)
|
||||
)
|
||||
|
||||
+58
-119
@@ -1,91 +1,82 @@
|
||||
import OpenAI, { APIError } from 'openai'
|
||||
import type { ChatCompletionMessageParam } from 'openai/resources/chat/completions.mjs'
|
||||
import { prepareFlowUserMessage } from '../../core'
|
||||
import { createEvalHelpers } from './evalHelpers'
|
||||
import type { FlowModule } from '$lib/gen'
|
||||
import type { ExtendedOpenFlow } from '$lib/components/flows/types'
|
||||
import type { ToolCallbacks } from '../../../shared'
|
||||
import { type VariantConfig, resolveSystemPrompt, resolveTools, resolveModel } from './evalVariants'
|
||||
import { evaluateFlowComparison, type EvalComparisonResult } from './evalFlowComparison'
|
||||
import type { ChatCompletionMessageParam, ChatCompletionSystemMessageParam } from 'openai/resources/chat/completions.mjs'
|
||||
import type { ChatCompletionTool } from 'openai/resources/chat/completions.mjs'
|
||||
import type { TokenUsage, ToolCallDetail, EvalRunnerOptions } from './types'
|
||||
import type { Tool } from './baseVariants'
|
||||
|
||||
// Re-export for convenience
|
||||
export type { EvalComparisonResult } from './evalFlowComparison'
|
||||
|
||||
export interface ToolCallDetail {
|
||||
name: string
|
||||
arguments: Record<string, any>
|
||||
}
|
||||
|
||||
export interface EvalResult {
|
||||
/**
|
||||
* Result from a single eval run (before domain-specific evaluation).
|
||||
*/
|
||||
export interface RawEvalResult<TOutput> {
|
||||
success: boolean
|
||||
flow: ExtendedOpenFlow
|
||||
output: TOutput
|
||||
error?: string
|
||||
tokenUsage: {
|
||||
prompt: number
|
||||
completion: number
|
||||
total: number
|
||||
}
|
||||
tokenUsage: TokenUsage
|
||||
toolCallsCount: number
|
||||
toolsCalled: string[]
|
||||
toolCallDetails: ToolCallDetail[]
|
||||
iterations: number
|
||||
variantName: string
|
||||
evaluationResult?: EvalComparisonResult
|
||||
messages: ChatCompletionMessageParam[]
|
||||
}
|
||||
|
||||
export interface ExpectedFlow {
|
||||
summary?: string
|
||||
value: {
|
||||
modules: FlowModule[]
|
||||
}
|
||||
schema?: Record<string, any>
|
||||
}
|
||||
|
||||
export interface EvalOptions {
|
||||
initialModules?: FlowModule[]
|
||||
initialSchema?: Record<string, any>
|
||||
model?: string
|
||||
customSystemPrompt?: string
|
||||
maxIterations?: number
|
||||
variant?: VariantConfig
|
||||
expectedFlow?: ExpectedFlow
|
||||
/**
|
||||
* Parameters for running a base evaluation.
|
||||
*/
|
||||
export interface RunEvalParams<THelpers, TOutput> {
|
||||
/** The user's prompt/instruction */
|
||||
userPrompt: string
|
||||
/** System message for the LLM */
|
||||
systemMessage: ChatCompletionSystemMessageParam
|
||||
/** User message for the LLM */
|
||||
userMessage: ChatCompletionMessageParam
|
||||
/** Tool definitions for the LLM API */
|
||||
toolDefs: ChatCompletionTool[]
|
||||
/** Full tool implementations for execution */
|
||||
tools: Tool<THelpers>[]
|
||||
/** Domain-specific helpers for tool execution */
|
||||
helpers: THelpers
|
||||
/** API key for OpenRouter */
|
||||
apiKey: string
|
||||
/** Function to get the current output state */
|
||||
getOutput: () => TOutput
|
||||
/** Optional configuration */
|
||||
options?: EvalRunnerOptions
|
||||
}
|
||||
|
||||
/**
|
||||
* Runs a flow chat evaluation with real OpenAI API calls.
|
||||
* Executes tool calls using the actual flowTools from core.ts or variant-configured tools.
|
||||
* Runs a generic evaluation with real LLM API calls.
|
||||
* Executes tool calls in a loop until the LLM stops calling tools.
|
||||
*
|
||||
* This is the core execution loop shared across all chat eval tests.
|
||||
*/
|
||||
export async function runFlowEval(
|
||||
userPrompt: string,
|
||||
openaiApiKey: string,
|
||||
options?: EvalOptions
|
||||
): Promise<EvalResult> {
|
||||
const client = new OpenAI({ baseURL: 'https://openrouter.ai/api/v1', apiKey: openaiApiKey })
|
||||
const { helpers, getFlow } = createEvalHelpers(
|
||||
options?.initialModules ?? [],
|
||||
options?.initialSchema
|
||||
)
|
||||
export async function runEval<THelpers, TOutput>(
|
||||
params: RunEvalParams<THelpers, TOutput>
|
||||
): Promise<RawEvalResult<TOutput>> {
|
||||
const {
|
||||
systemMessage,
|
||||
userMessage,
|
||||
toolDefs,
|
||||
tools,
|
||||
helpers,
|
||||
apiKey,
|
||||
getOutput,
|
||||
options
|
||||
} = params
|
||||
|
||||
// Resolve variant configuration
|
||||
const variantName = options?.variant?.name ?? 'baseline'
|
||||
const systemMessage = resolveSystemPrompt(options?.variant, options?.customSystemPrompt)
|
||||
const { toolDefs, tools } = resolveTools(options?.variant)
|
||||
const model = resolveModel(options?.variant, options?.model)
|
||||
|
||||
// Build user message
|
||||
const userMessage = prepareFlowUserMessage(userPrompt, helpers.getFlowAndSelectedId(), [])
|
||||
const client = new OpenAI({ baseURL: 'https://openrouter.ai/api/v1', apiKey })
|
||||
const model = options?.model ?? 'gpt-4o'
|
||||
const maxIterations = options?.maxIterations ?? 20
|
||||
const workspace = options?.workspace ?? 'test-workspace'
|
||||
|
||||
const messages: ChatCompletionMessageParam[] = [systemMessage, userMessage]
|
||||
const totalTokens = { prompt: 0, completion: 0, total: 0 }
|
||||
const totalTokens: TokenUsage = { prompt: 0, completion: 0, total: 0 }
|
||||
let toolCallsCount = 0
|
||||
const toolsCalled: string[] = []
|
||||
const toolCallDetails: ToolCallDetail[] = []
|
||||
let iterations = 0
|
||||
const maxIterations = options?.maxIterations ?? 20
|
||||
|
||||
// No-op tool callbacks for eval
|
||||
const toolCallbacks: ToolCallbacks = {
|
||||
const toolCallbacks = {
|
||||
setToolStatus: () => {},
|
||||
removeToolStatus: () => {}
|
||||
}
|
||||
@@ -155,7 +146,7 @@ export async function runFlowEval(
|
||||
toolCallDetails.push({ name: toolCall.function.name, arguments: args })
|
||||
const result = await tool.fn({
|
||||
args,
|
||||
workspace: 'test-workspace',
|
||||
workspace,
|
||||
helpers,
|
||||
toolCallbacks,
|
||||
toolId: toolCall.id
|
||||
@@ -176,27 +167,14 @@ export async function runFlowEval(
|
||||
}
|
||||
}
|
||||
|
||||
// Run evaluation if expected flow is provided
|
||||
let evaluationResult: EvalComparisonResult | undefined
|
||||
if (options?.expectedFlow) {
|
||||
const generatedFlow = getFlow()
|
||||
evaluationResult = await evaluateFlowComparison(
|
||||
generatedFlow,
|
||||
options.expectedFlow,
|
||||
userPrompt
|
||||
)
|
||||
}
|
||||
|
||||
return {
|
||||
success: true,
|
||||
flow: getFlow(),
|
||||
output: getOutput(),
|
||||
tokenUsage: totalTokens,
|
||||
toolCallsCount,
|
||||
toolsCalled,
|
||||
toolCallDetails,
|
||||
iterations,
|
||||
variantName,
|
||||
evaluationResult,
|
||||
messages
|
||||
}
|
||||
} catch (err) {
|
||||
@@ -218,55 +196,16 @@ export async function runFlowEval(
|
||||
errorMessage = String(err)
|
||||
}
|
||||
|
||||
// Still run evaluation on partial content if expected flow is provided
|
||||
let evaluationResult: EvalComparisonResult | undefined
|
||||
if (options?.expectedFlow) {
|
||||
try {
|
||||
const generatedFlow = getFlow()
|
||||
evaluationResult = await evaluateFlowComparison(
|
||||
generatedFlow,
|
||||
options.expectedFlow,
|
||||
userPrompt
|
||||
)
|
||||
} catch (evalErr) {
|
||||
// If evaluation itself fails, just log it and continue
|
||||
console.error('Evaluation failed:', evalErr)
|
||||
}
|
||||
}
|
||||
|
||||
return {
|
||||
success: false,
|
||||
flow: getFlow(),
|
||||
output: getOutput(),
|
||||
error: errorMessage,
|
||||
tokenUsage: totalTokens,
|
||||
toolCallsCount,
|
||||
toolsCalled,
|
||||
toolCallDetails,
|
||||
iterations,
|
||||
variantName,
|
||||
evaluationResult,
|
||||
messages
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Runs the same prompt against multiple variants sequentially for comparison.
|
||||
* Returns results in the same order as the input variants.
|
||||
*/
|
||||
export async function runVariantComparison(
|
||||
userPrompt: string,
|
||||
variants: VariantConfig[],
|
||||
openaiApiKey: string,
|
||||
baseOptions?: Omit<EvalOptions, 'variant'>
|
||||
): Promise<EvalResult[]> {
|
||||
const results: EvalResult[] = await Promise.all(
|
||||
variants.map(async (variant) => {
|
||||
return await runFlowEval(userPrompt, openaiApiKey, {
|
||||
...baseOptions,
|
||||
variant
|
||||
})
|
||||
})
|
||||
)
|
||||
return results
|
||||
}
|
||||
@@ -0,0 +1,126 @@
|
||||
import OpenAI from 'openai'
|
||||
import type { EvaluationResult } from './types'
|
||||
|
||||
/**
|
||||
* Parameters for LLM-based evaluation.
|
||||
*/
|
||||
export interface EvaluateParams {
|
||||
/** The user's original request/prompt */
|
||||
userPrompt: string
|
||||
/** The generated output to evaluate */
|
||||
generatedOutput: unknown
|
||||
/** The expected/reference output */
|
||||
expectedOutput: unknown
|
||||
/** Domain-specific system prompt for the evaluator */
|
||||
evaluatorSystemPrompt: string
|
||||
/** API key for OpenRouter */
|
||||
apiKey: string
|
||||
/** Model to use for evaluation (default: 'anthropic/claude-sonnet-4.5') */
|
||||
model?: string
|
||||
}
|
||||
|
||||
/**
|
||||
* Base evaluator system prompt template.
|
||||
* Domain-specific evaluators should build on this structure.
|
||||
*/
|
||||
export const BASE_EVALUATOR_RESPONSE_FORMAT = `
|
||||
## Response Format
|
||||
You MUST respond with valid JSON only, no additional text:
|
||||
{
|
||||
"resemblanceScore": <0-100 integer>,
|
||||
"statement": "<brief 1-2 sentence summary of how well the output matches the user request and expected output>",
|
||||
"missingRequirements": ["<list any requirements from user prompt that are missing or incorrectly implemented>"]
|
||||
}
|
||||
|
||||
Score guidelines:
|
||||
- 90-100: Fully addresses user request, functionally equivalent to expected output
|
||||
- 70-89: Addresses most user requirements, same overall structure with minor differences
|
||||
- 50-69: Partially addresses user request, achieves similar goal but different approach
|
||||
- 30-49: Missing significant requirements from user request
|
||||
- 0-29: Does not address user request or significantly incorrect`
|
||||
|
||||
/**
|
||||
* Evaluates how well a generated output matches an expected output using an LLM.
|
||||
* Returns a resemblance score (0-100), a qualitative statement, and any missing requirements.
|
||||
*
|
||||
* @param params Evaluation parameters including prompts, outputs, and API configuration
|
||||
* @returns Evaluation result with score, statement, and missing requirements
|
||||
*/
|
||||
export async function evaluateWithLLM(params: EvaluateParams): Promise<EvaluationResult> {
|
||||
const {
|
||||
userPrompt,
|
||||
generatedOutput,
|
||||
expectedOutput,
|
||||
evaluatorSystemPrompt,
|
||||
apiKey,
|
||||
model = 'anthropic/claude-sonnet-4.5'
|
||||
} = params
|
||||
|
||||
const client = new OpenAI({ baseURL: 'https://openrouter.ai/api/v1', apiKey })
|
||||
|
||||
const userMessage = `## User's Original Request
|
||||
${userPrompt}
|
||||
|
||||
## Expected Reference Output
|
||||
\`\`\`json
|
||||
${JSON.stringify(expectedOutput, null, 2)}
|
||||
\`\`\`
|
||||
|
||||
## Generated Output
|
||||
\`\`\`json
|
||||
${JSON.stringify(generatedOutput, null, 2)}
|
||||
\`\`\`
|
||||
|
||||
Please evaluate how well the generated output:
|
||||
1. Fulfills ALL requirements from the user's original request
|
||||
2. Matches the structure and logic of the expected reference output`
|
||||
|
||||
try {
|
||||
const response = await client.chat.completions.create({
|
||||
model,
|
||||
messages: [
|
||||
{ role: 'system', content: evaluatorSystemPrompt },
|
||||
{ role: 'user', content: userMessage }
|
||||
],
|
||||
temperature: 0
|
||||
})
|
||||
|
||||
const content = response.choices[0]?.message?.content
|
||||
if (!content) {
|
||||
return {
|
||||
success: false,
|
||||
resemblanceScore: 0,
|
||||
statement: 'No response from evaluator',
|
||||
error: 'Empty response from LLM'
|
||||
}
|
||||
}
|
||||
|
||||
// Parse JSON response - handle potential markdown code blocks
|
||||
let jsonContent = content.trim()
|
||||
if (jsonContent.startsWith('```')) {
|
||||
// Remove markdown code block wrapper
|
||||
jsonContent = jsonContent.replace(/^```(?:json)?\n?/, '').replace(/\n?```$/, '')
|
||||
}
|
||||
|
||||
const parsed = JSON.parse(jsonContent) as {
|
||||
resemblanceScore: number
|
||||
statement: string
|
||||
missingRequirements?: string[]
|
||||
}
|
||||
|
||||
return {
|
||||
success: true,
|
||||
resemblanceScore: Math.max(0, Math.min(100, Math.round(parsed.resemblanceScore))),
|
||||
statement: parsed.statement,
|
||||
missingRequirements: parsed.missingRequirements ?? []
|
||||
}
|
||||
} catch (err) {
|
||||
const errorMessage = err instanceof Error ? err.message : String(err)
|
||||
return {
|
||||
success: false,
|
||||
resemblanceScore: 0,
|
||||
statement: 'Evaluation failed',
|
||||
error: errorMessage
|
||||
}
|
||||
}
|
||||
}
|
||||
+35
-34
@@ -4,36 +4,48 @@ import { writeFile, mkdir } from 'fs/promises'
|
||||
import { join, dirname } from 'path'
|
||||
// @ts-ignore
|
||||
import { fileURLToPath } from 'url'
|
||||
import type { EvalResult } from './evalRunner'
|
||||
|
||||
const __filename = fileURLToPath(import.meta.url)
|
||||
const __dirname = dirname(__filename)
|
||||
import type { BaseEvalResult } from './types'
|
||||
|
||||
/**
|
||||
* Generates a timestamp string suitable for filenames.
|
||||
* Format: 2024-01-15T10-30-45-123Z (ISO but with dashes instead of colons)
|
||||
*/
|
||||
function generateTimestamp(): string {
|
||||
export function generateTimestamp(): string {
|
||||
return new Date().toISOString().replace(/:/g, '-')
|
||||
}
|
||||
|
||||
/**
|
||||
* Parameters for writing comparison results.
|
||||
*/
|
||||
export interface WriteResultsParams<TOutput> {
|
||||
/** User prompt that was tested */
|
||||
userPrompt: string
|
||||
/** Results from all variants */
|
||||
results: BaseEvalResult<TOutput>[]
|
||||
/** Directory to write results to */
|
||||
outputDir: string
|
||||
/** Function to format domain-specific output for JSON files */
|
||||
formatOutput: (output: TOutput) => unknown
|
||||
/** Label for the output type (e.g., 'flow', 'app') */
|
||||
outputLabel?: string
|
||||
}
|
||||
|
||||
/**
|
||||
* Writes comparison results to files in the results folder.
|
||||
* Creates:
|
||||
* - {timestamp}.md - Summary with prompt and results table
|
||||
* - {timestamp}_{variant_name}.json - Flow JSON for each variant
|
||||
* - summary.md - Summary with prompt and results table
|
||||
* - {variant_name}.json - Full result with metadata for each variant
|
||||
* - {variant_name}_{outputLabel}.json - Clean output for each variant
|
||||
*/
|
||||
export async function writeComparisonResults(
|
||||
userPrompt: string,
|
||||
results: EvalResult[],
|
||||
outputDir?: string
|
||||
): Promise<{ summaryPath: string; flowPaths: string[] }> {
|
||||
const resultsDir = outputDir ?? join(__dirname, 'results')
|
||||
export async function writeComparisonResults<TOutput>(
|
||||
params: WriteResultsParams<TOutput>
|
||||
): Promise<{ summaryPath: string; outputPaths: string[] }> {
|
||||
const { userPrompt, results, outputDir, formatOutput, outputLabel = 'output' } = params
|
||||
const timestamp = generateTimestamp()
|
||||
|
||||
// Ensure results directory exists
|
||||
await mkdir(resultsDir, { recursive: true })
|
||||
const resultFolder = join(resultsDir, timestamp)
|
||||
await mkdir(outputDir, { recursive: true })
|
||||
const resultFolder = join(outputDir, timestamp)
|
||||
await mkdir(resultFolder, { recursive: true })
|
||||
|
||||
// Check if any results have evaluation data
|
||||
@@ -122,15 +134,15 @@ export async function writeComparisonResults(
|
||||
}
|
||||
}
|
||||
|
||||
const flowPaths: string[] = []
|
||||
const outputPaths: string[] = []
|
||||
|
||||
for (const result of results) {
|
||||
const resultFilename = `${result.variantName}.json`
|
||||
const resultPath = join(resultFolder, resultFilename)
|
||||
flowPaths.push(resultPath)
|
||||
outputPaths.push(resultPath)
|
||||
|
||||
const flowFilename = `${result.variantName}_flow.json`
|
||||
const flowPath = join(resultFolder, flowFilename)
|
||||
const outputFilename = `${result.variantName}_${outputLabel}.json`
|
||||
const outputPath = join(resultFolder, outputFilename)
|
||||
|
||||
// Write result JSON file (with metadata)
|
||||
const resultData = {
|
||||
@@ -144,25 +156,14 @@ export async function writeComparisonResults(
|
||||
}
|
||||
await writeFile(resultPath, JSON.stringify(resultData, null, 2))
|
||||
|
||||
// Write flow definition JSON file (clean flow format)
|
||||
const flowData = {
|
||||
summary: result.flow.summary ?? '',
|
||||
value: {
|
||||
modules: result.flow.value.modules
|
||||
},
|
||||
schema: result.flow.schema ?? {
|
||||
$schema: 'https://json-schema.org/draft/2020-12/schema',
|
||||
properties: {},
|
||||
required: [],
|
||||
type: 'object'
|
||||
}
|
||||
}
|
||||
await writeFile(flowPath, JSON.stringify(flowData, null, 2))
|
||||
// Write clean output JSON file (domain-specific format)
|
||||
const outputData = formatOutput(result.output)
|
||||
await writeFile(outputPath, JSON.stringify(outputData, null, 2))
|
||||
}
|
||||
|
||||
// Write summary markdown file
|
||||
const summaryPath = join(resultFolder, `summary.md`)
|
||||
await writeFile(summaryPath, summaryLines.join('\n'))
|
||||
|
||||
return { summaryPath, flowPaths }
|
||||
return { summaryPath, outputPaths }
|
||||
}
|
||||
@@ -0,0 +1,108 @@
|
||||
import type {
|
||||
ChatCompletionFunctionTool,
|
||||
ChatCompletionSystemMessageParam
|
||||
} from 'openai/resources/chat/completions.mjs'
|
||||
import type { ChatCompletionTool } from 'openai/resources/chat/completions.mjs'
|
||||
import type { VariantConfig } from './types'
|
||||
|
||||
/**
|
||||
* Generic tool interface that matches the structure used across chat modules.
|
||||
*/
|
||||
export interface Tool<THelpers> {
|
||||
def: ChatCompletionFunctionTool
|
||||
fn: (params: {
|
||||
args: Record<string, unknown>
|
||||
workspace: string
|
||||
helpers: THelpers
|
||||
toolCallbacks: {
|
||||
setToolStatus: (...args: unknown[]) => void
|
||||
removeToolStatus: (...args: unknown[]) => void
|
||||
}
|
||||
toolId: string
|
||||
}) => Promise<string>
|
||||
}
|
||||
|
||||
/**
|
||||
* Domain-specific defaults for variant resolution.
|
||||
*/
|
||||
export interface VariantDefaults<THelpers> {
|
||||
/** Function to prepare system message, optionally with custom prompt */
|
||||
prepareSystemMessage: (customPrompt?: string) => ChatCompletionSystemMessageParam
|
||||
/** Available tools for the domain */
|
||||
tools: Tool<THelpers>[]
|
||||
}
|
||||
|
||||
/**
|
||||
* Resolves system prompt from variant config.
|
||||
* Returns the appropriate ChatCompletionSystemMessageParam based on config.
|
||||
*/
|
||||
export function resolveSystemPrompt<THelpers>(
|
||||
variant: VariantConfig | undefined,
|
||||
defaults: VariantDefaults<THelpers>,
|
||||
fallbackCustomPrompt?: string
|
||||
): ChatCompletionSystemMessageParam {
|
||||
if (!variant?.systemPrompt || variant.systemPrompt.type === 'default') {
|
||||
return defaults.prepareSystemMessage(fallbackCustomPrompt)
|
||||
}
|
||||
|
||||
if (variant.systemPrompt.type === 'default-with-custom') {
|
||||
return defaults.prepareSystemMessage(variant.systemPrompt.custom)
|
||||
}
|
||||
|
||||
// type === 'custom'
|
||||
return {
|
||||
role: 'system',
|
||||
content: variant.systemPrompt.content
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Resolves tools from variant config.
|
||||
* Returns both the tool definitions (for API) and full tools (for execution).
|
||||
*/
|
||||
export function resolveTools<THelpers>(
|
||||
variant: VariantConfig | undefined,
|
||||
defaults: VariantDefaults<THelpers>
|
||||
): {
|
||||
toolDefs: ChatCompletionTool[]
|
||||
tools: Tool<THelpers>[]
|
||||
} {
|
||||
if (!variant?.tools || variant.tools.type === 'default') {
|
||||
return {
|
||||
toolDefs: defaults.tools.map((t) => t.def),
|
||||
tools: defaults.tools
|
||||
}
|
||||
}
|
||||
|
||||
if (variant.tools.type === 'subset') {
|
||||
const includeList = (variant.tools as { type: 'subset'; include: string[] }).include
|
||||
const subset = defaults.tools.filter((t) => includeList.includes(t.def.function.name))
|
||||
return {
|
||||
toolDefs: subset.map((t) => t.def),
|
||||
tools: subset
|
||||
}
|
||||
}
|
||||
|
||||
if (variant.tools.type === 'custom') {
|
||||
// Custom tools are typed as unknown[] in base VariantConfig but domain-specific
|
||||
// code should ensure they are the correct Tool<THelpers> type
|
||||
const customTools = variant.tools.tools as Tool<THelpers>[]
|
||||
return {
|
||||
toolDefs: customTools.map((t) => t.def),
|
||||
tools: customTools
|
||||
}
|
||||
}
|
||||
|
||||
// Default fallback
|
||||
return {
|
||||
toolDefs: defaults.tools.map((t) => t.def),
|
||||
tools: defaults.tools
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Resolves model from variant config with fallback.
|
||||
*/
|
||||
export function resolveModel(variant?: VariantConfig, fallback?: string): string {
|
||||
return variant?.model ?? fallback ?? 'gpt-4o'
|
||||
}
|
||||
@@ -0,0 +1,28 @@
|
||||
// Types
|
||||
export type {
|
||||
TokenUsage,
|
||||
ToolCallDetail,
|
||||
EvaluationResult,
|
||||
BaseEvalResult,
|
||||
VariantConfig,
|
||||
EvalRunnerOptions,
|
||||
ToolCallbacks
|
||||
} from './types'
|
||||
|
||||
export { createNoOpToolCallbacks } from './types'
|
||||
|
||||
// Variant resolution
|
||||
export type { Tool, VariantDefaults } from './baseVariants'
|
||||
export { resolveSystemPrompt, resolveTools, resolveModel } from './baseVariants'
|
||||
|
||||
// Eval runner
|
||||
export type { RawEvalResult, RunEvalParams } from './baseEvalRunner'
|
||||
export { runEval } from './baseEvalRunner'
|
||||
|
||||
// LLM evaluator
|
||||
export type { EvaluateParams } from './baseLLMEvaluator'
|
||||
export { evaluateWithLLM, BASE_EVALUATOR_RESPONSE_FORMAT } from './baseLLMEvaluator'
|
||||
|
||||
// Results writer
|
||||
export type { WriteResultsParams } from './baseResultsWriter'
|
||||
export { writeComparisonResults, generateTimestamp } from './baseResultsWriter'
|
||||
@@ -0,0 +1,104 @@
|
||||
import type { ChatCompletionMessageParam } from 'openai/resources/chat/completions.mjs'
|
||||
|
||||
/**
|
||||
* Token usage tracking for LLM calls.
|
||||
*/
|
||||
export interface TokenUsage {
|
||||
prompt: number
|
||||
completion: number
|
||||
total: number
|
||||
}
|
||||
|
||||
/**
|
||||
* Details of a single tool call made during evaluation.
|
||||
*/
|
||||
export interface ToolCallDetail {
|
||||
name: string
|
||||
arguments: Record<string, unknown>
|
||||
}
|
||||
|
||||
/**
|
||||
* Result of LLM-based comparison/evaluation.
|
||||
*/
|
||||
export interface EvaluationResult {
|
||||
success: boolean
|
||||
resemblanceScore: number
|
||||
statement: string
|
||||
missingRequirements?: string[]
|
||||
error?: string
|
||||
}
|
||||
|
||||
/**
|
||||
* Base evaluation result that can be extended for domain-specific outputs.
|
||||
* @template TOutput The domain-specific output type (e.g., flow definition, app files)
|
||||
*/
|
||||
export interface BaseEvalResult<TOutput> {
|
||||
success: boolean
|
||||
output: TOutput
|
||||
error?: string
|
||||
tokenUsage: TokenUsage
|
||||
toolCallsCount: number
|
||||
toolsCalled: string[]
|
||||
toolCallDetails: ToolCallDetail[]
|
||||
iterations: number
|
||||
variantName: string
|
||||
evaluationResult?: EvaluationResult
|
||||
messages: ChatCompletionMessageParam[]
|
||||
}
|
||||
|
||||
/**
|
||||
* Base configuration for a variant in eval testing.
|
||||
* Allows customizing system prompt, tools, and model for comparison.
|
||||
*
|
||||
* Note: Domain-specific variants may extend this with custom tool configurations.
|
||||
* See flow/flowEvalVariants.ts for an example with custom tools.
|
||||
*/
|
||||
export interface VariantConfig {
|
||||
name: string
|
||||
description?: string
|
||||
|
||||
/** System prompt configuration */
|
||||
systemPrompt?:
|
||||
| { type: 'default' }
|
||||
| { type: 'default-with-custom'; custom: string }
|
||||
| { type: 'custom'; content: string }
|
||||
|
||||
/** Tools configuration - basic types supported by shared code */
|
||||
tools?:
|
||||
| { type: 'default' }
|
||||
| { type: 'subset'; include: string[] }
|
||||
| { type: 'custom'; tools: unknown[] }
|
||||
|
||||
/** Model to use (default: 'gpt-4o') */
|
||||
model?: string
|
||||
}
|
||||
|
||||
/**
|
||||
* Options for running an evaluation.
|
||||
*/
|
||||
export interface EvalRunnerOptions {
|
||||
/** Maximum iterations for tool call loop (default: 20) */
|
||||
maxIterations?: number
|
||||
/** Model to use for LLM calls */
|
||||
model?: string
|
||||
/** Workspace ID for tool calls */
|
||||
workspace?: string
|
||||
}
|
||||
|
||||
/**
|
||||
* No-op tool callbacks for eval testing.
|
||||
*/
|
||||
export interface ToolCallbacks {
|
||||
setToolStatus: (id: string, status: { content?: string; result?: string; error?: string }) => void
|
||||
removeToolStatus: (id: string) => void
|
||||
}
|
||||
|
||||
/**
|
||||
* Creates no-op tool callbacks for eval testing.
|
||||
*/
|
||||
export function createNoOpToolCallbacks(): ToolCallbacks {
|
||||
return {
|
||||
setToolStatus: () => {},
|
||||
removeToolStatus: () => {}
|
||||
}
|
||||
}
|
||||
@@ -1,145 +0,0 @@
|
||||
import OpenAI from 'openai'
|
||||
|
||||
export interface EvalComparisonResult {
|
||||
success: boolean
|
||||
resemblanceScore: number
|
||||
statement: string
|
||||
missingRequirements?: string[]
|
||||
error?: string
|
||||
}
|
||||
|
||||
interface ExpectedFlow {
|
||||
summary?: string
|
||||
value: {
|
||||
modules: unknown[]
|
||||
}
|
||||
schema?: unknown
|
||||
}
|
||||
|
||||
const EVALUATOR_SYSTEM_PROMPT = `You are an expert evaluator for Windmill flow definitions. Your task is to evaluate a generated flow against:
|
||||
1. The original user request/prompt
|
||||
2. An expected reference flow
|
||||
|
||||
## Windmill Flow Context
|
||||
- Flows consist of modules (steps) that execute sequentially
|
||||
- Module types include: rawscript, forloopflow, branchone, branchall, script, flow, aiagent
|
||||
- Each module has an id, value (containing type and config), and may have input_transforms
|
||||
- input_transforms connect modules using expressions like "results.previous_step". Valid input_transforms are: static, javascript. Valid variables in javascript expressions are: results, flow_input, flow_input.iter.value (for forloopflow), flow_input.iter.index (for forloopflow).
|
||||
- forloopflow contains nested modules that execute per iteration with access to flow_input.iter.value
|
||||
- branchone executes first matching branch, branchall executes all matching branches
|
||||
- Branches have conditional expressions (expr) that determine execution
|
||||
- aiagent modules contain tools array with tool definitions
|
||||
|
||||
## Evaluation Criteria
|
||||
1. **User Request Fulfillment**: Does the generated flow address ALL requirements from the user's original prompt?
|
||||
- Are all requested steps present?
|
||||
- Are the requested features implemented (loops, branches, specific logic)?
|
||||
- Does the schema match what the user requested for inputs?
|
||||
2. **Structure**: Are the module types and nesting structure appropriate for the task?
|
||||
3. **Logic**: Does the flow accomplish the intended logical task?
|
||||
4. **Connections**: Are input_transforms connecting data correctly between steps?
|
||||
5. **Completeness**: Are all required steps present with no major omissions?
|
||||
6. **Code Quality**: Is the code functionally correct (exact syntax doesn't need to match)?
|
||||
|
||||
## Important Notes
|
||||
- Minor differences in variable names, code formatting, or exact wording are acceptable
|
||||
- Focus on functional equivalence, not character-by-character matching
|
||||
- The generated flow should achieve the same outcome as described in the user request
|
||||
- Extra helper steps or slightly different approaches can still score high if they accomplish the goal
|
||||
- If the user requested specific module types (like aiagent), verify they are used correctly
|
||||
|
||||
## Response Format
|
||||
You MUST respond with valid JSON only, no additional text:
|
||||
{
|
||||
"resemblanceScore": <0-100 integer>,
|
||||
"statement": "<brief 1-2 sentence summary of how well the flow matches the user request and expected flow>",
|
||||
"missingRequirements": ["<list any requirements from user prompt that are missing or incorrectly implemented>"]
|
||||
}
|
||||
|
||||
Score guidelines:
|
||||
- 90-100: Fully addresses user request, functionally equivalent to expected flow
|
||||
- 70-89: Addresses most user requirements, same overall structure with minor differences
|
||||
- 50-69: Partially addresses user request, achieves similar goal but different approach
|
||||
- 30-49: Missing significant requirements from user request
|
||||
- 0-29: Does not address user request or significantly incorrect`
|
||||
|
||||
/**
|
||||
* Evaluates how well a generated flow matches an expected flow and user request using an LLM.
|
||||
* Returns a resemblance score (0-100), a qualitative statement, and any missing requirements.
|
||||
*/
|
||||
export async function evaluateFlowComparison(
|
||||
generatedFlow: ExpectedFlow,
|
||||
expectedFlow: ExpectedFlow,
|
||||
userPrompt: string
|
||||
): Promise<EvalComparisonResult> {
|
||||
const model = 'anthropic/claude-sonnet-4.5'
|
||||
// @ts-ignore
|
||||
const apiKey = process.env.OPENROUTER_API_KEY
|
||||
const client = new OpenAI({ baseURL: 'https://openrouter.ai/api/v1', apiKey })
|
||||
|
||||
const userMessage = `## User's Original Request
|
||||
${userPrompt}
|
||||
|
||||
## Expected Reference Flow
|
||||
\`\`\`json
|
||||
${JSON.stringify(expectedFlow, null, 2)}
|
||||
\`\`\`
|
||||
|
||||
## Generated Flow
|
||||
\`\`\`json
|
||||
${JSON.stringify(generatedFlow, null, 2)}
|
||||
\`\`\`
|
||||
|
||||
Please evaluate how well the generated flow:
|
||||
1. Fulfills ALL requirements from the user's original request
|
||||
2. Matches the structure and logic of the expected reference flow`
|
||||
|
||||
try {
|
||||
const response = await client.chat.completions.create({
|
||||
model,
|
||||
messages: [
|
||||
{ role: 'system', content: EVALUATOR_SYSTEM_PROMPT },
|
||||
{ role: 'user', content: userMessage }
|
||||
],
|
||||
temperature: 0
|
||||
})
|
||||
|
||||
const content = response.choices[0]?.message?.content
|
||||
if (!content) {
|
||||
return {
|
||||
success: false,
|
||||
resemblanceScore: 0,
|
||||
statement: 'No response from evaluator',
|
||||
error: 'Empty response from LLM'
|
||||
}
|
||||
}
|
||||
|
||||
// Parse JSON response - handle potential markdown code blocks
|
||||
let jsonContent = content.trim()
|
||||
if (jsonContent.startsWith('```')) {
|
||||
// Remove markdown code block wrapper
|
||||
jsonContent = jsonContent.replace(/^```(?:json)?\n?/, '').replace(/\n?```$/, '')
|
||||
}
|
||||
|
||||
const parsed = JSON.parse(jsonContent) as {
|
||||
resemblanceScore: number
|
||||
statement: string
|
||||
missingRequirements?: string[]
|
||||
}
|
||||
|
||||
return {
|
||||
success: true,
|
||||
resemblanceScore: Math.max(0, Math.min(100, Math.round(parsed.resemblanceScore))),
|
||||
statement: parsed.statement,
|
||||
missingRequirements: parsed.missingRequirements ?? []
|
||||
}
|
||||
} catch (err) {
|
||||
const errorMessage = err instanceof Error ? err.message : String(err)
|
||||
return {
|
||||
success: false,
|
||||
resemblanceScore: 0,
|
||||
statement: 'Evaluation failed',
|
||||
error: errorMessage
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -1,97 +0,0 @@
|
||||
import type { ChatCompletionSystemMessageParam } from 'openai/resources/chat/completions.mjs'
|
||||
import type { ChatCompletionTool } from 'openai/resources/chat/completions.mjs'
|
||||
import { flowTools, prepareFlowSystemMessage } from '../../core'
|
||||
import type { Tool } from '../../../shared'
|
||||
import type { FlowAIChatHelpers } from '../../core'
|
||||
|
||||
/**
|
||||
* Configuration for a variant in eval testing.
|
||||
* Allows customizing system prompt, tools, and model for comparison.
|
||||
*/
|
||||
export interface VariantConfig {
|
||||
name: string
|
||||
description?: string
|
||||
|
||||
/** System prompt configuration */
|
||||
systemPrompt?:
|
||||
| { type: 'default' }
|
||||
| { type: 'default-with-custom'; custom: string }
|
||||
| { type: 'custom'; content: string }
|
||||
|
||||
/** Tools configuration */
|
||||
tools?:
|
||||
| { type: 'default' }
|
||||
| { type: 'subset'; include: string[] }
|
||||
| { type: 'custom'; tools: Tool<FlowAIChatHelpers>[] }
|
||||
|
||||
/** Model to use (default: 'gpt-4o') */
|
||||
model?: string
|
||||
}
|
||||
|
||||
/**
|
||||
* Resolves system prompt from variant config.
|
||||
* Returns the appropriate ChatCompletionSystemMessageParam based on config.
|
||||
*/
|
||||
export function resolveSystemPrompt(
|
||||
variant?: VariantConfig,
|
||||
fallbackCustomPrompt?: string
|
||||
): ChatCompletionSystemMessageParam {
|
||||
if (!variant?.systemPrompt || variant.systemPrompt.type === 'default') {
|
||||
return prepareFlowSystemMessage(fallbackCustomPrompt)
|
||||
}
|
||||
|
||||
if (variant.systemPrompt.type === 'default-with-custom') {
|
||||
return prepareFlowSystemMessage(variant.systemPrompt.custom)
|
||||
}
|
||||
|
||||
// type === 'custom'
|
||||
return {
|
||||
role: 'system',
|
||||
content: variant.systemPrompt.content
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Resolves tools from variant config.
|
||||
* Returns both the tool definitions (for API) and full tools (for execution).
|
||||
*/
|
||||
export function resolveTools(variant?: VariantConfig): {
|
||||
toolDefs: ChatCompletionTool[]
|
||||
tools: Tool<FlowAIChatHelpers>[]
|
||||
} {
|
||||
if (!variant?.tools || variant.tools.type === 'default') {
|
||||
return {
|
||||
toolDefs: flowTools.map((t) => t.def),
|
||||
tools: flowTools
|
||||
}
|
||||
}
|
||||
|
||||
if (variant.tools.type === 'subset') {
|
||||
const subset = flowTools.filter(
|
||||
(t) =>
|
||||
variant.tools!.type === 'subset' &&
|
||||
(variant.tools as { type: 'subset'; include: string[] }).include.includes(
|
||||
t.def.function.name
|
||||
)
|
||||
)
|
||||
return {
|
||||
toolDefs: subset.map((t) => t.def),
|
||||
tools: subset
|
||||
}
|
||||
}
|
||||
|
||||
// type === 'custom'
|
||||
const customTools = variant.tools.tools
|
||||
|
||||
return {
|
||||
toolDefs: customTools.map((t) => t.def),
|
||||
tools: customTools
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Resolves model from variant config with fallback.
|
||||
*/
|
||||
export function resolveModel(variant?: VariantConfig, fallback?: string): string {
|
||||
return variant?.model ?? fallback ?? 'gpt-4o'
|
||||
}
|
||||
Reference in New Issue
Block a user