From 33e4a8a7ec2843980fa3785acb4a6db764d90a05 Mon Sep 17 00:00:00 2001 From: centdix Date: Mon, 1 Jun 2026 13:10:22 +0200 Subject: [PATCH] test: cover global flow preview ids --- ai_evals/adapters/frontend/mockBackend.ts | 48 +++++ .../adapters/frontend/vitestAdapter.test.ts | 186 +++++++++++++++++- 2 files changed, 233 insertions(+), 1 deletion(-) diff --git a/ai_evals/adapters/frontend/mockBackend.ts b/ai_evals/adapters/frontend/mockBackend.ts index be4868d8d4..c8212326a9 100644 --- a/ai_evals/adapters/frontend/mockBackend.ts +++ b/ai_evals/adapters/frontend/mockBackend.ts @@ -32,11 +32,19 @@ type BenchmarkCompletedJob = CompletedJob & { type: 'CompletedJob' } const benchmarkWorkspaces = new Set() const benchmarkWorkspaceRunnables = new Map() const benchmarkJobs = new Map() +let lastBenchmarkFlowPreviewRequest: + | { + workspace: string + requestBody: Record + memoryId?: string + } + | undefined export function resetBenchmarkMockBackend(): void { benchmarkWorkspaces.clear() benchmarkWorkspaceRunnables.clear() benchmarkJobs.clear() + lastBenchmarkFlowPreviewRequest = undefined } export function registerBenchmarkWorkspace(workspace: string): void { @@ -227,6 +235,46 @@ export function runBenchmarkFlowByPath(input: { }) } +export function runBenchmarkFlowPreview(input: { + workspace: string + requestBody: { + path?: string + value?: Record + args?: Record + } + memoryId?: string +}): string { + lastBenchmarkFlowPreviewRequest = { + workspace: input.workspace, + requestBody: structuredClone(input.requestBody), + memoryId: input.memoryId + } + + return createBenchmarkCompletedJob({ + workspace: input.workspace, + jobKind: 'flowpreview', + success: true, + scriptPath: input.requestBody.path, + args: input.requestBody.args, + result: { + path: input.requestBody.path, + args: input.requestBody.args ?? {}, + mocked: true + }, + logs: 'Mock benchmark flow preview completed successfully.' + }) +} + +export function getLastBenchmarkFlowPreviewRequest(): + | { + workspace: string + requestBody: Record + memoryId?: string + } + | undefined { + return lastBenchmarkFlowPreviewRequest +} + export function previewBenchmarkSchedule(input: { requestBody?: Record }): Record { diff --git a/ai_evals/adapters/frontend/vitestAdapter.test.ts b/ai_evals/adapters/frontend/vitestAdapter.test.ts index 92c43aab06..0e8457717c 100644 --- a/ai_evals/adapters/frontend/vitestAdapter.test.ts +++ b/ai_evals/adapters/frontend/vitestAdapter.test.ts @@ -1,4 +1,4 @@ -import { expect, it, vi } from 'vitest' +import { beforeEach, describe, expect, it, vi } from 'vitest' // @ts-ignore - Node.js fs/promises import { mkdir, writeFile } from 'fs/promises' // @ts-ignore - Node.js path @@ -44,6 +44,7 @@ vi.mock('$lib/gen', async () => { createBenchmarkSchedule, previewBenchmarkSchedule, runBenchmarkFlowByPath, + runBenchmarkFlowPreview, runBenchmarkScriptPreview } = await import('./mockBackend') @@ -168,6 +169,22 @@ vi.mock('$lib/gen', async () => { args: data.requestBody }) : actual.JobService.runFlowByPath(data), + runFlowPreview: async (data: { + workspace: string + memoryId?: string + requestBody?: { + path?: string + value?: Record + args?: Record + } + }) => + hasBenchmarkWorkspace(data.workspace) + ? runBenchmarkFlowPreview({ + workspace: data.workspace, + requestBody: data.requestBody ?? {}, + memoryId: data.memoryId + }) + : actual.JobService.runFlowPreview(data), getJob: async (data: { workspace: string; id: string }) => { if (hasBenchmarkWorkspace(data.workspace)) { const job = getBenchmarkCompletedJob(data.workspace, data.id) @@ -362,6 +379,173 @@ vi.mock('$lib/gen', async () => { } }) +describe('global flow preview tools in ai_evals', () => { + beforeEach(async () => { + const { resetBenchmarkMockBackend } = await import('./mockBackend') + const { __resetUserDraftForTesting } = await import('../../../frontend/src/lib/userDraft.svelte') + resetBenchmarkMockBackend() + __resetUserDraftForTesting() + localStorage.removeItem('wm_dev_global_ai') + }) + + it('does not pass the AI session id as a flow preview memory id', async () => { + const { + getLastBenchmarkFlowPreviewRequest, + registerBenchmarkWorkspaceRunnables, + resetBenchmarkMockBackend, + unregisterBenchmarkWorkspace + } = await import('./mockBackend') + const { globalTools } = await import('../../../frontend/src/lib/components/copilot/chat/global/core') + const { processToolCall } = await import('../../../frontend/src/lib/components/copilot/chat/shared') + const workspace = 'ai-evals-global-flow-preview' + + registerBenchmarkWorkspaceRunnables(workspace, { + flows: [ + { + path: 'f/evals/global/current_invoice_flow', + summary: 'Current invoice flow', + value: { + modules: [{ id: 'calculate_total', value: { type: 'identity' } }] + } + } + ] + }) + + try { + const response = await processToolCall({ + tools: globalTools, + toolCall: { + id: 'tool-test-run-flow', + type: 'function', + function: { + name: 'test_run_flow', + arguments: JSON.stringify({ + path: 'f/evals/global/current_invoice_flow', + args: { subtotal: 100 } + }) + } + }, + helpers: { sessionId: 'htc1xouxd96dcyo6ruqo39' }, + workspace, + toolCallbacks: { + setToolStatus: vi.fn(), + shouldAutoAcceptToolConfirmations: () => true + } + }) + + expect(response.content).toContain('Result (SUCCESS)') + expect(getLastBenchmarkFlowPreviewRequest()).toMatchObject({ + workspace, + memoryId: undefined, + requestBody: { + path: 'f/evals/global/current_invoice_flow', + args: { subtotal: 100 } + } + }) + } finally { + unregisterBenchmarkWorkspace(workspace) + resetBenchmarkMockBackend() + } + }) + + it('uses the active flow test hook with args only', async () => { + const { + createBenchmarkCompletedJob, + registerBenchmarkWorkspace, + unregisterBenchmarkWorkspace + } = await import('./mockBackend') + const { globalTools } = await import('../../../frontend/src/lib/components/copilot/chat/global/core') + const { processToolCall } = await import('../../../frontend/src/lib/components/copilot/chat/shared') + const { UserDraft } = await import('../../../frontend/src/lib/userDraft.svelte') + const workspace = 'ai-evals-global-live-flow-preview' + const path = 'f/evals/global/current_invoice_flow' + const testActiveFlow = vi.fn(async (args?: Record) => + createBenchmarkCompletedJob({ + workspace, + jobKind: 'flowpreview', + args, + result: { path, args: args ?? {}, liveEditor: true }, + logs: 'Mock live editor flow preview completed successfully.' + }) + ) + + registerBenchmarkWorkspace(workspace) + UserDraft.setLiveEditorDraft({ + workspace, + itemKind: 'flow', + storagePath: '', + effectivePath: path + }) + + try { + const response = await processToolCall({ + tools: globalTools, + toolCall: { + id: 'tool-live-flow-test', + type: 'function', + function: { + name: 'test_run_flow', + arguments: JSON.stringify({ + path, + args: { subtotal: 200 } + }) + } + }, + helpers: { + sessionId: 'htc1xouxd96dcyo6ruqo39', + testActiveFlow + }, + workspace, + toolCallbacks: { + setToolStatus: vi.fn(), + shouldAutoAcceptToolConfirmations: () => true + } + }) + + expect(response.content).toContain('Result (SUCCESS)') + expect(testActiveFlow.mock.calls).toEqual([[{ subtotal: 200 }]]) + } finally { + UserDraft.clearLiveEditorDraft('flow', { workspace, storagePath: '' }) + unregisterBenchmarkWorkspace(workspace) + } + }) + + it('wires session global flow tests without a conversation id', async () => { + const { AIChatManager, AIMode } = await import( + '../../../frontend/src/lib/components/copilot/chat/AIChatManager.svelte' + ) + const testFlow = vi.fn(async () => 'job-flow-preview') + const manager = new AIChatManager() + + localStorage.setItem('wm_dev_global_ai', '1') + manager.isSessionChat = true + manager.sessionId = 'htc1xouxd96dcyo6ruqo39' + manager.setFlowHelpers({ + getFlowAndSelectedId: vi.fn(), + getRootModules: vi.fn(), + inlineScriptSession: { get: vi.fn(), set: vi.fn(), clear: vi.fn() }, + setSnapshot: vi.fn(), + revertToSnapshot: vi.fn(), + setCode: vi.fn(), + setFlowJson: vi.fn(), + getFlowInputsSchema: vi.fn(), + updateExprsToSet: vi.fn(), + acceptAllModuleActions: vi.fn(), + rejectAllModuleActions: vi.fn(), + hasPendingChanges: vi.fn(() => false), + selectStep: vi.fn(), + testFlow, + getLintErrors: vi.fn() + } as any) + + manager.changeMode(AIMode.GLOBAL) + const jobId = await manager.helpers.testActiveFlow({ subtotal: 300 }) + + expect(jobId).toBe('job-flow-preview') + expect(testFlow.mock.calls).toEqual([[{ subtotal: 300 }]]) + }) +}) + const benchmarkOutputPath = process.env.WMILL_FRONTEND_AI_EVAL_OUTPUT_PATH const benchmarkIt = benchmarkOutputPath ? it : it.skip