From e41440b344d94ebe5746a89237c3d1be522aa59d Mon Sep 17 00:00:00 2001 From: Ruben Fiszel Date: Thu, 23 Jul 2026 01:12:55 +0200 Subject: [PATCH] feat(ai-chat): email triggers in flow/script chat + trigger-intent eval guards (WIN-2228) (#10267) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * test(ai-evals): guard implicit trigger/schedule intent in flow chat Investigation of WIN-2228 (does flow AI chat understand it should create a flow AND its associated triggers): the flow-editor chat already exposes create_schedule and create_trigger (10 kinds), both confirmation-gated, and an A/B eval shows the model already recognizes IMPLICIT trigger intent reliably (12/12 across two new cases on the current prompt) without naming a "schedule" or "trigger". Add two ai_evals flow cases that phrase the trigger intent implicitly, to guard that recognition against future prompt/tool regressions. These are not redundant with the existing explicit cases (flow-test15/16): a trial system prompt addition that spelled out a deployment prerequisite regressed the HTTP case from 6/6 to 2/6 (the model deferred instead of creating the trigger), which these cases caught. No prompt change ships: the addition showed no measured benefit over baseline and the fuller version regressed behavior. Fixes WIN-2228 Co-Authored-By: Claude Opus 4.8 (1M context) * feat(ai-chat): support email triggers in flow/script create_trigger The chat's create_trigger tool exposed 10 trigger kinds but not email, even though the backend supports email triggers and the chat's open-resource drawer was already wired for them (CreatedResourceActionDrawers, the 'email' CreatedResourceTriggerKind). So when asked to make a flow run on incoming email, the model had no email kind and substituted an HTTP trigger it mislabeled as email. Add email as a create_trigger kind (generator + regenerated zod schema + triggerConfigs → EmailTriggerService.createEmailTrigger). Email triggering only works once an instance superadmin has stood up an SMTP server and set the `email_domain` global setting, so guard the create path: read `email_domain` (readable by any authed user; returns null when unset) and, when it is not configured, return role-aware setup guidance instead of a failing create — pointing a superadmin to Instance settings and a regular user to ask a superadmin, both with the docs link. When configured, create the trigger and report the resulting inbound email address. userStore and the email-address helper are lazy-imported so the chat tools module does not drag in the heavy $lib/stores graph at load. Guarded by unit tests for both branches (shared.test.ts) and an ai_evals case (flow-test19); the model now calls create_trigger(kind=email) 3/3 on a natural "run when an email is received" prompt. Fixes WIN-2228 Co-Authored-By: Claude Opus 4.8 (1M context) * fix(ai-chat): address codex review on email trigger + eval guards - [P1] Default `workspaced_local_part` on the email trigger request body before it is sent, not only when formatting the success address. The column is BOOLEAN NOT NULL, so a request omitting it (the model may) was rejected by the backend. Assert the defaulted `false` in the happy-path unit test. - [P2] Tighten the implicit-intent eval guards so they validate the requested configuration, not just tool selection + path prefix: flow-test17 now checks the cron time (07:30) and UTC timezone; flow-test18 checks kind=http, POST method, no auth, and the route path. Cases still pass 9/9 (sonnet). Co-Authored-By: Claude Opus 4.8 (1M context) --------- Co-authored-by: Claude Opus 4.8 (1M context) --- ai_evals/cases/flow.yaml | 82 +++++++++++ .../components/copilot/chat/shared.test.ts | 134 +++++++++++++++++- .../components/copilot/chat/workspaceTools.ts | 79 ++++++++++- .../copilot/chat/workspaceToolsZod.gen.ts | 32 +++++ system_prompts/generate.py | 2 + 5 files changed, 324 insertions(+), 5 deletions(-) diff --git a/ai_evals/cases/flow.yaml b/ai_evals/cases/flow.yaml index a21ae81f17..fa2ca3bee7 100644 --- a/ai_evals/cases/flow.yaml +++ b/ai_evals/cases/flow.yaml @@ -480,3 +480,85 @@ judgeChecklist: - "the flow includes a final top-level step named `webhook_response`" - "`webhook_response` returns `ok: true` and the order summary" + +- id: flow-test17-implicit-schedule-intent + prompt: |- + I want this order processing flow to run on its own every morning at 07:30 UTC. + Set that up for me. Do not ask me for the flow path. + initial: ai_evals/fixtures/frontend/flow/initial/scheduled_order_flow.json + toolExpect: + requiredToolsUsed: + - create_schedule + toolCallArgs: + - tool: create_schedule + field: path + stringStartsWithAnyOf: + - f/ + - u/ + stringMustNotStartWithAnyOf: + - schedules/ + - tool: create_schedule + field: schedule + stringIncludesAnyOf: + - 30 7 + - tool: create_schedule + field: timezone + stringIncludesAnyOf: + - UTC + skipJudge: true + judgeChecklist: + - "a schedule is created for the flow that runs daily at 07:30 UTC" + +- id: flow-test18-implicit-http-trigger-intent + prompt: |- + I need to be able to kick off this order processing flow by sending it an HTTP POST + from an external system, with no authentication. Use route path `ai-evals/order-processing-implicit`. + Do not ask me for the flow path. + initial: ai_evals/fixtures/frontend/flow/initial/scheduled_order_flow.json + toolExpect: + requiredToolsUsed: + - create_trigger + toolCallArgs: + - tool: create_trigger + field: path + stringStartsWithAnyOf: + - f/ + - u/ + stringMustNotStartWithAnyOf: + - schedules/ + - tool: create_trigger + field: kind + stringStartsWithAnyOf: + - http + - tool: create_trigger + field: config.http_method + stringIncludesAnyOf: + - post + - tool: create_trigger + field: config.authentication_method + stringIncludesAnyOf: + - none + - tool: create_trigger + field: config.route_path + stringIncludesAnyOf: + - ai-evals/order-processing-implicit + skipJudge: true + judgeChecklist: + - "an HTTP trigger is created for the flow that accepts unauthenticated POST requests" + +- id: flow-test19-implicit-email-trigger-intent + prompt: |- + Make this order processing flow run automatically whenever an email is received. + Do not ask me for the flow path. + initial: ai_evals/fixtures/frontend/flow/initial/scheduled_order_flow.json + toolExpect: + requiredToolsUsed: + - create_trigger + toolCallArgs: + - tool: create_trigger + field: kind + stringStartsWithAnyOf: + - email + skipJudge: true + judgeChecklist: + - "an email trigger (kind email) is created, or the user is told how to enable email triggering on the instance" diff --git a/frontend/src/lib/components/copilot/chat/shared.test.ts b/frontend/src/lib/components/copilot/chat/shared.test.ts index ec6febca80..473c3d3335 100644 --- a/frontend/src/lib/components/copilot/chat/shared.test.ts +++ b/frontend/src/lib/components/copilot/chat/shared.test.ts @@ -7,8 +7,23 @@ vi.mock('monaco-editor', () => ({ editor: {} })) +const userHolder = vi.hoisted(() => ({ + current: { is_super_admin: true } as { is_super_admin: boolean } +})) + vi.mock('$lib/stores', () => ({ - workspaceStore: { subscribe: () => () => undefined } + workspaceStore: { subscribe: () => () => undefined }, + userStore: { + subscribe: (run: (value: { is_super_admin: boolean }) => void) => { + run(userHolder.current) + return () => {} + } + } +})) + +vi.mock('$lib/components/triggers/email/utils', () => ({ + getEmailAddress: (localPart: string, _wlp: boolean, _wsId: string, domain: string) => + `${localPart}@${domain}` })) vi.mock('$lib/components/flows/flowTree', () => ({ @@ -31,7 +46,10 @@ vi.mock('$lib/gen', () => ({ MqttTriggerService: { createMqttTrigger: vi.fn() }, SqsTriggerService: { createSqsTrigger: vi.fn() }, GcpTriggerService: { createGcpTrigger: vi.fn() }, - AzureTriggerService: { createAzureTrigger: vi.fn() } + AzureTriggerService: { createAzureTrigger: vi.fn() }, + AmqpTriggerService: { createAmqpTrigger: vi.fn() }, + EmailTriggerService: { createEmailTrigger: vi.fn() }, + SettingService: { getGlobal: vi.fn() } })) vi.mock('$lib/utils', () => ({ @@ -555,6 +573,118 @@ describe('processToolCall', () => { ) }) + it('email trigger: guides the user to set up email triggering when unconfigured', async () => { + const gen = (await import('$lib/gen')) as any + const { processToolCall } = await import('./shared') + const { createWorkspaceMutationTools } = await import('./workspaceTools') + const tools = createWorkspaceMutationTools() + + gen.SettingService.getGlobal.mockReset() + gen.SettingService.getGlobal.mockResolvedValue(null) + gen.EmailTriggerService.createEmailTrigger.mockReset() + + const call = (id: string) => + processToolCall({ + tools, + toolCall: { + id, + type: 'function', + function: { + name: 'create_trigger', + arguments: JSON.stringify({ + kind: 'email', + path: 'f/triggers/email_current', + config: { local_part: 'orders' } + }) + } + }, + helpers: { + getWorkspaceMutationTarget: () => ({ + kind: 'flow', + path: 'f/flows/current', + deployed: true + }) + }, + workspace: 'test-workspace', + toolCallbacks: { + setToolStatus: vi.fn(), + removeToolStatus: vi.fn(), + requestConfirmation: vi.fn().mockResolvedValue(true) + } + }) + + userHolder.current = { is_super_admin: true } + const superadminResult = await call('call_email_super') + expect(gen.EmailTriggerService.createEmailTrigger).not.toHaveBeenCalled() + expect(superadminResult.content).toContain('not set up') + expect(superadminResult.content).toContain('As a superadmin') + + userHolder.current = { is_super_admin: false } + const memberResult = await call('call_email_member') + expect(gen.EmailTriggerService.createEmailTrigger).not.toHaveBeenCalled() + expect(memberResult.content).toContain('Ask an instance superadmin') + }) + + it('email trigger: creates it and reports the address when email triggering is configured', async () => { + const gen = (await import('$lib/gen')) as any + const { processToolCall } = await import('./shared') + const { createWorkspaceMutationTools } = await import('./workspaceTools') + const tools = createWorkspaceMutationTools() + + gen.SettingService.getGlobal.mockReset() + gen.SettingService.getGlobal.mockResolvedValue('mail.example.com') + gen.EmailTriggerService.createEmailTrigger.mockReset() + gen.EmailTriggerService.createEmailTrigger.mockResolvedValue('email-created') + + const result = await processToolCall({ + tools, + toolCall: { + id: 'call_email_ok', + type: 'function', + function: { + name: 'create_trigger', + arguments: JSON.stringify({ + kind: 'email', + path: 'f/triggers/email_current', + config: { local_part: 'orders' } + }) + } + }, + helpers: { + getWorkspaceMutationTarget: () => ({ + kind: 'flow', + path: 'f/flows/current', + deployed: true + }) + }, + workspace: 'test-workspace', + toolCallbacks: { + setToolStatus: vi.fn(), + removeToolStatus: vi.fn(), + requestConfirmation: vi.fn().mockResolvedValue(true) + } + }) + + expect(gen.EmailTriggerService.createEmailTrigger).toHaveBeenCalledWith({ + workspace: 'test-workspace', + requestBody: expect.objectContaining({ + local_part: 'orders', + // defaulted before the request is sent; the backend column is NOT NULL + workspaced_local_part: false, + script_path: 'f/flows/current', + is_flow: true + }) + }) + expect(JSON.parse(result.content as string)).toEqual( + expect.objectContaining({ + success: true, + kind: 'email', + email_address: 'orders@mail.example.com', + backend_result: 'email-created' + }) + ) + }) + it('surfaces workspace mutation tool execution errors to the user', async () => { const gen = (await import('$lib/gen')) as any const { processToolCall } = await import('./shared') diff --git a/frontend/src/lib/components/copilot/chat/workspaceTools.ts b/frontend/src/lib/components/copilot/chat/workspaceTools.ts index a976b7dab6..dabe63b5bc 100644 --- a/frontend/src/lib/components/copilot/chat/workspaceTools.ts +++ b/frontend/src/lib/components/copilot/chat/workspaceTools.ts @@ -1,5 +1,6 @@ import { AzureTriggerService, + EmailTriggerService, GcpTriggerService, HttpTriggerService, KafkaTriggerService, @@ -8,10 +9,12 @@ import { NatsTriggerService, PostgresTriggerService, ScheduleService, + SettingService, SqsTriggerService, WebsocketTriggerService, type AzureTriggerData, type GcpTriggerData, + type NewEmailTrigger, type NewHttpTrigger, type NewKafkaTrigger, type NewMqttTrigger, @@ -51,6 +54,7 @@ type TriggerRequestByKind = { sqs: NewSqsTrigger gcp: GcpTriggerData azure: AzureTriggerData + email: NewEmailTrigger } type TriggerRequestBody = TriggerRequestByKind[TriggerKind] @@ -117,7 +121,7 @@ const createScheduleToolDef = createToolDef( const createTriggerToolDef = createToolDef( createTriggerToolSchema, 'create_trigger', - 'Create a trigger for the current script or flow.', + 'Create a trigger for the current script or flow. For an email trigger (kind "email"), config.local_part is the local part of the receiving address (before the @); the tool reports the full address on success. Email triggers require email triggering to be configured on the instance — if it is not, the tool returns setup guidance instead of creating one.', { strict: false } ) @@ -181,6 +185,12 @@ const triggerConfigs = { requestSchema: triggerRequestSchemas.azure as z.ZodType, create: (data: { workspace: string; requestBody: AzureTriggerData }) => AzureTriggerService.createAzureTrigger(data) + }, + email: { + label: 'Email trigger', + requestSchema: triggerRequestSchemas.email as z.ZodType, + create: (data: { workspace: string; requestBody: NewEmailTrigger }) => + EmailTriggerService.createEmailTrigger(data) } } satisfies { [K in TriggerKind]: { @@ -321,6 +331,39 @@ const createScheduleTool: Tool = { } } +const EMAIL_TRIGGER_DOCS = 'https://windmill.dev/docs/advanced/email_triggers' + +type EmailTriggerAvailability = + | { available: true; domain: string } + | { available: false; hint: string } + +/** + * Email triggers only work once an instance superadmin has stood up an SMTP + * server forwarding to Windmill and set the `email_domain` global setting + * (readable by any authed user). When it is unset the create call would fail + * opaquely, so we surface actionable, role-aware setup guidance instead. + */ +async function resolveEmailTriggerAvailability(): Promise { + let emailDomain: unknown + try { + emailDomain = await SettingService.getGlobal({ key: 'email_domain' }) + } catch { + emailDomain = undefined + } + if (typeof emailDomain === 'string' && emailDomain.trim() !== '') { + return { available: true, domain: emailDomain } + } + const [{ get }, { userStore }] = await Promise.all([ + import('svelte/store'), + import('$lib/stores') + ]) + const isSuperadmin = get(userStore)?.is_super_admin ?? false + const hint = isSuperadmin + ? `Email triggering is not set up on this instance yet, so no email trigger was created. As a superadmin, enable it: run an SMTP server that forwards inbound mail to Windmill and set the "email_domain" instance setting (Instance settings). See ${EMAIL_TRIGGER_DOCS}. Once configured, ask again and I will create the trigger.` + : `Email triggering is not set up on this instance yet, so no email trigger was created. Ask an instance superadmin to enable it: they need to run an SMTP server that forwards inbound mail to Windmill and set the "email_domain" instance setting. See ${EMAIL_TRIGGER_DOCS}. Once it is configured, ask again and I will create the trigger.` + return { available: false, hint } +} + const createTriggerTool: Tool = { def: createTriggerToolDef, requiresConfirmation: true, @@ -342,22 +385,52 @@ const createTriggerTool: Tool = { triggerConfig.label ) + let emailDomain: string | undefined + if (parsedArgs.kind === 'email') { + // `workspaced_local_part` maps to a NOT NULL column; the model may omit it, + // so default it here before the request is sent, not just when formatting the address. + const emailBody = requestBody as NewEmailTrigger + emailBody.workspaced_local_part = emailBody.workspaced_local_part ?? false + const availability = await resolveEmailTriggerAvailability() + if (!availability.available) { + toolCallbacks.setToolStatus(toolId, { + content: availability.hint, + isLoading: false, + needsConfirmation: false + }) + return availability.hint + } + emailDomain = availability.domain + } + toolCallbacks.setToolStatus(toolId, { content: `Creating ${triggerConfig.label} "${requestBody.path}"...` }) try { const result = await triggerConfig.create({ workspace, requestBody } as never) const targetKind = getActionTargetKind(requestBody.is_flow) + const emailAddress = + parsedArgs.kind === 'email' && emailDomain !== undefined + ? (await import('$lib/components/triggers/email/utils')).getEmailAddress( + (requestBody as NewEmailTrigger).local_part, + (requestBody as NewEmailTrigger).workspaced_local_part ?? false, + workspace, + emailDomain + ) + : undefined const toolResult = { success: true, kind: parsedArgs.kind, path: requestBody.path, target_path: requestBody.script_path, target_kind: targetKind, - backend_result: result + backend_result: result, + ...(emailAddress ? { email_address: emailAddress } : {}) } toolCallbacks.setToolStatus(toolId, { - content: `Created ${triggerConfig.label} "${requestBody.path}"`, + content: emailAddress + ? `Created ${triggerConfig.label} "${requestBody.path}" (send email to ${emailAddress})` + : `Created ${triggerConfig.label} "${requestBody.path}"`, result: toolResult, actions: [ createOpenTriggerAction( diff --git a/frontend/src/lib/components/copilot/chat/workspaceToolsZod.gen.ts b/frontend/src/lib/components/copilot/chat/workspaceToolsZod.gen.ts index 79883d2dd9..6aa40bce75 100644 --- a/frontend/src/lib/components/copilot/chat/workspaceToolsZod.gen.ts +++ b/frontend/src/lib/components/copilot/chat/workspaceToolsZod.gen.ts @@ -434,6 +434,35 @@ export const azureTriggerRequestSchema = z.object({ "labels": z.array(z.string()).optional() }).describe("Data for creating or updating an Azure Event Grid trigger.") +export const emailTriggerRequestSchema = z.object({ + "path": z.string(), + "script_path": z.string(), + "local_part": z.string(), + "workspaced_local_part": z.boolean().optional(), + "is_flow": z.boolean(), + "error_handler_path": z.string().optional(), + "error_handler_args": z.record(z.string(), z.any()).describe("The arguments to pass to the script or flow").optional(), + "retry": z.object({ + "constant": z.object({ + "attempts": z.number().int().describe("Number of retry attempts").optional(), + "seconds": z.number().int().describe("Seconds to wait between retries").optional() + }).describe("Retry with constant delay between attempts").optional(), + "exponential": z.object({ + "attempts": z.number().int().describe("Number of retry attempts").optional(), + "multiplier": z.number().int().describe("Multiplier for exponential backoff").optional(), + "seconds": z.number().int().gte(1).describe("Initial delay in seconds").optional(), + "random_factor": z.number().int().gte(0).lte(100).describe("Random jitter percentage (0-100) to avoid thundering herd").optional() + }).describe("Retry with exponential backoff (delay doubles each time)").optional(), + "retry_if": z.object({ + "expr": z.string().describe("JavaScript expression that returns true to retry. Has access to 'result' and 'error' variables") + }).describe("Conditional retry based on error or result").optional() + }).describe("Retry configuration for failed module executions").optional(), + "mode": z.enum(["enabled", "disabled", "suspended"]).describe("job trigger mode").optional(), + "permissioned_as": z.string().describe("The user or group this trigger runs as. Used during deployment to preserve the original trigger owner.").optional(), + "preserve_permissioned_as": z.boolean().describe("When true and the caller is a member of the 'wm_deployers' group, preserves the original permissioned_as value instead of overwriting it.").optional(), + "labels": z.array(z.string()).optional() +}) + export const variableRequestSchema = z.object({ "path": z.string().describe("The path to the variable"), "value": z.string().describe("The value of the variable"), @@ -466,6 +495,7 @@ export const triggerRequestSchemas = { sqs: sqsTriggerRequestSchema, gcp: gcpTriggerRequestSchema, azure: azureTriggerRequestSchema, + email: emailTriggerRequestSchema, } as const const triggerPathSchema = z.string().min(1).describe("The unique Windmill path for this trigger. Must be of the form `u//` or `f//`. This is the trigger object path, not the HTTP route path.") @@ -482,6 +512,7 @@ export const createTriggerToolSchema = z.object({ "sqs", "gcp", "azure", + "email", ]), path: triggerPathSchema, config: z.union([ @@ -495,5 +526,6 @@ export const createTriggerToolSchema = z.object({ sqsTriggerRequestSchema.omit({ path: true, script_path: true, is_flow: true }), gcpTriggerRequestSchema.omit({ path: true, script_path: true, is_flow: true }), azureTriggerRequestSchema.omit({ path: true, script_path: true, is_flow: true }), + emailTriggerRequestSchema.omit({ path: true, script_path: true, is_flow: true }), ]) }) diff --git a/system_prompts/generate.py b/system_prompts/generate.py index d315e3a97b..3ee993147e 100644 --- a/system_prompts/generate.py +++ b/system_prompts/generate.py @@ -832,6 +832,7 @@ WORKSPACE_TOOL_ZOD_SCHEMAS = [ ('NewSqsTrigger', 'sqsTriggerRequestSchema'), ('GcpTriggerData', 'gcpTriggerRequestSchema'), ('AzureTriggerData', 'azureTriggerRequestSchema'), + ('NewEmailTrigger', 'emailTriggerRequestSchema'), ('CreateVariable', 'variableRequestSchema'), ('CreateResource', 'resourceRequestSchema'), ] @@ -847,6 +848,7 @@ WORKSPACE_TOOL_TRIGGER_SCHEMAS = [ ('sqs', 'sqsTriggerRequestSchema'), ('gcp', 'gcpTriggerRequestSchema'), ('azure', 'azureTriggerRequestSchema'), + ('email', 'emailTriggerRequestSchema'), ] WORKSPACE_TOOL_ZOD_OUTPUT_PATH = (