feat: instrument AI fill/fix, evals, agents and the debugger (#10853)

* feat: track AI fill, AI fix, evals, reusable agents and debugger usage

Co-Authored-By: Claude Opus 5 <noreply@anthropic.com>

* chore: pin ee ref to the feature_usage registry commit

Co-Authored-By: Claude Opus 5 <noreply@anthropic.com>

* chore: update ee-repo-ref to c3b6f62ea579a3583d4b474e9885c77104cfc87e

This commit updates the EE repository reference after PR #745 was merged in windmill-ee-private.

Previous ee-repo-ref: 77992910929188a854eadc06ee45971877b6f954

New ee-repo-ref: c3b6f62ea579a3583d4b474e9885c77104cfc87e

Automated by sync-ee-ref workflow.

---------

Co-authored-by: Claude Opus 5 <noreply@anthropic.com>
Co-authored-by: windmill-internal-app[bot] <windmill-internal-app[bot]@users.noreply.github.com>
Co-authored-by: Ruben Fiszel <ruben@windmill.dev>

backend/ee-repo-ref.txt

@@ -1 +1 @@
-b58ad414b098d3d7787001a352bfbb13e43a335f
+c3b6f62ea579a3583d4b474e9885c77104cfc87e

backend/windmill-api-debug/src/lib.rs

@@ -468,6 +468,9 @@ async fn sign_debug_request(
 
     // Parse the language
     let script_lang: ScriptLang = request.language.parse().unwrap_or(ScriptLang::Bun);
+    // Taken from the parsed language, not the request's string: the telemetry key vocabulary has
+    // to stay the closed set of languages rather than whatever a caller sent.
+    let lang_key = script_lang.as_str();
 
     // Hash the code (we don't include full code in JWT to keep it small)
     let mut hasher = Sha256::new();
@@ -578,6 +581,8 @@ async fn sign_debug_request(
 
     tx.commit().await?;
 
+    windmill_common::feature_usage::log_feature_usage("debugger", "session", lang_key);
+
     Ok(Json(SignedDebugPayload {
         token,
         code: request.code,

backend/windmill-api/src/ai_evals/datasets.rs

@@ -190,6 +190,8 @@ pub async fn create_dataset(
     }
     tx.commit().await?;
 
+    windmill_common::feature_usage::log_feature_usage("ai_agent_eval", "dataset_created", "");
+
     Ok(format!("Created eval dataset {}", payload.path))
 }
 

backend/windmill-api/src/ai_evals/run.rs

@@ -707,6 +707,17 @@ pub async fn run_experiment(
             .await?;
         return Err(e);
     }
+    // Which state of the agent was measured is the whole key vocabulary: it is what separates
+    // running what is deployed from measuring edits or an older version.
+    windmill_common::feature_usage::log_feature_usage(
+        "ai_agent_eval",
+        "run",
+        match subject.kind {
+            EvalSubjectKind::Agent => "agent",
+            EvalSubjectKind::AgentDraft => "agent_draft",
+            EvalSubjectKind::AgentVersion => "agent_version",
+        },
+    );
     Ok(experiment_id.to_string())
 }
 

docs/feature-telemetry.md

@@ -4,10 +4,10 @@
 anonymous usage-stats payload. It answers "does anyone use this, and which variant do they pick"
 without any identifying data leaving the instance.
 
-It currently carries 21 registered actions across nine features (`ai_session`, `ai_chat`,
-`flow_editor`, `flow_run`, `flow_step`, `trigger`, `command_script`, `hub_script`,
-`usage_meter`). Nearly all of the product is uninstrumented, so new user-facing work is the
-opportunity to change that.
+It currently carries 28 registered actions across fourteen features (`ai_session`, `ai_chat`,
+`ai_fix`, `ai_agent`, `ai_agent_eval`, `flow_editor`, `flow_run`, `flow_step`, `run_form`,
+`debugger`, `trigger`, `command_script`, `hub_script`, `usage_meter`). Nearly all of the
+product is uninstrumented, so new user-facing work is the opportunity to change that.
 
 ## When to instrument
 

frontend/src/lib/components/InstanceSettings.svelte

@@ -1069,8 +1069,9 @@
 						<li>git sync repo count (sync vs promotion mode)</li>
 						<li
 							>feature usage (counts of which product features are used, including AI provider and
-							model identifiers, the names of public hub scripts used, and the plan tier and quota
-							shown when the execution meter is opened, last 30 days)</li
+							model identifiers, the names of public hub scripts used, the languages debug sessions
+							are started for, and the plan tier and quota shown when the execution meter is
+							opened, last 30 days)</li
 						>
 						<li
 							>feature adoption (counts of which flow, script, trigger and worker features your
@@ -1121,8 +1122,9 @@
 						<li>development instance status</li>
 						<li
 							>feature usage (counts of which product features are used, including AI provider and
-							model identifiers, the names of public hub scripts used, and the plan tier and quota
-							shown when the execution meter is opened, last 30 days)</li
+							model identifiers, the names of public hub scripts used, the languages debug sessions
+							are started for, and the plan tier and quota shown when the execution meter is
+							opened, last 30 days)</li
 						>
 						<li
 							>feature adoption (counts of which flow, script, trigger and worker features your

frontend/src/lib/components/copilot/AIFormAssistant.svelte

@@ -5,6 +5,7 @@
 	import OpenInSessionButton from '$lib/components/sessions/OpenInSessionButton.svelte'
 	import { AIBtnClasses } from './chat/AIButtonStyle'
 	import { workspaceStore } from '$lib/stores'
+	import { logFeatureUsage } from '$lib/utils/featureUsage'
 
 	interface Props {
 		onEditInstructions: () => void
@@ -15,7 +16,16 @@
 
 	const { onEditInstructions, instructions, runnableType, path }: Props = $props()
 
+	// Anonymous counter for this card being acted on, keyed by what it sits above. The two
+	// branches share one counter: they are the same intent, and which of them is on screen
+	// follows the user's session gate rather than a choice made here. `beforeOpen` is the
+	// hand-off's only per-click hook, and it runs before anything can turn the click away.
+	function logAsked() {
+		logFeatureUsage('run_form', 'ai_fill', { key: runnableType })
+	}
+
 	async function fillFormWithAI() {
+		logAsked()
 		aiChatManager.openChat()
 		aiChatManager.askAi(`Analyze the ${runnableType} form on this page and fill the inputs for me`)
 	}
@@ -29,6 +39,7 @@
 			? {
 					target: { kind: runnableType, path } as const,
 					workspaceId: $workspaceStore ?? undefined,
+					beforeOpen: logAsked,
 					seedPrompt:
 						`Run the deployed ${runnableType} \`${path}\` for me. Pick sensible inputs, ` +
 						`tell me what you chose, then run it.` +

frontend/src/lib/components/copilot/ScriptFix.svelte

@@ -13,6 +13,7 @@
 	import { getOpenInSessionHandoff } from '$lib/components/sessions/openInSessionContext'
 	import { AIBtnClasses } from './chat/AIButtonStyle'
 	import { getContext } from 'svelte'
+	import { logFeatureUsage } from '$lib/utils/featureUsage'
 
 	let {
 		lang,
@@ -45,9 +46,28 @@
 			? `Fix this error in ${what}:\n\n\`\`\`\n${error}\n\`\`\``
 			: `Fix the error from the last run of ${what}.`
 	})
+	// Anonymous counter for a failing run being handed to AI, keyed by where the run was. All
+	// three branches below report the same action: which one is on screen follows the session
+	// gate and whether a chat is already beside this panel, not a choice made here.
+	function logAiFix() {
+		logFeatureUsage('ai_fix', 'requested', { key: moduleId ? 'flow_step' : 'script' })
+	}
+
 	const sessionSource = $derived.by(() => {
 		const source = handoff?.source({ moduleId })
-		return source ? { ...source, seedPrompt, autoSend: true } : undefined
+		if (!source) return undefined
+		// The counter wraps the editor's own hook rather than replacing it: that hook persists
+		// the draft the session opens on, so dropping it would fix an older copy of the code.
+		const editorBeforeOpen = source.beforeOpen
+		return {
+			...source,
+			seedPrompt,
+			autoSend: true,
+			beforeOpen: async () => {
+				logAiFix()
+				await editorBeforeOpen?.()
+			}
+		}
 	})
 
 	// Inside a session pane the chat is already beside this panel, so there is
@@ -65,7 +85,10 @@
 			color="light"
 			spacingSize="xs2"
 			startIcon={{ icon: WandSparkles }}
-			on:click={() => sessionScopedManager.sendOrQueue(seedPrompt)}
+			on:click={() => {
+				logAiFix()
+				sessionScopedManager.sendOrQueue(seedPrompt)
+			}}
 			btnClasses={AIBtnClasses('default')}
 		>
 			AI Fix
@@ -99,6 +122,7 @@
 								startIcon={{ icon: WandSparkles }}
 								on:click={() => {
 									if ($copilotInfo.enabled) {
+										logAiFix()
 										aiChatManager.fix()
 									}
 								}}

frontend/src/lib/components/copilot/StepInputGen.svelte

@@ -10,6 +10,7 @@
 	import { dfs } from '../flows/dfs'
 	import { yamlStringifyExceptKeys } from './utils'
 	import type { FlowCopilotContext } from './flow'
+	import { logStepInputFill } from './stepInputFillTelemetry'
 	import { stepInputCompletionEnabled } from '$lib/stores'
 	import type { SchemaProperty } from '$lib/common'
 	import FlowCopilotInputsModal from './FlowCopilotInputsModal.svelte'
@@ -66,6 +67,7 @@
 		if (generatedContent.length > 0 || loading) {
 			return
 		}
+		logStepInputFill('single')
 		abortController = new AbortController()
 		loading = true
 		const flow: Flow = JSON.parse(JSON.stringify(flowStore.val))

frontend/src/lib/components/copilot/StepInputsGen.svelte

@@ -11,6 +11,7 @@
 	import { sendUserToast } from '$lib/toast'
 	import Button from '../common/button/Button.svelte'
 	import type { FlowCopilotContext } from './flow'
+	import { logStepInputFill } from './stepInputFillTelemetry'
 	import { Check, ExternalLink, Loader2, Wand2 } from 'lucide-svelte'
 	import { stepInputCompletionEnabled } from '$lib/stores'
 	import { copilotInfo } from '$lib/aiStore'
@@ -44,6 +45,7 @@
 		if (Object.keys($generatedExprs || {}).length > 0 || loading) {
 			return
 		}
+		logStepInputFill('all')
 		abortController = new AbortController()
 		loading = true
 		stepInputsLoading?.set(true)

frontend/src/lib/components/copilot/stepInputFillTelemetry.ts

@@ -0,0 +1,13 @@
+import { logFeatureUsage } from '$lib/utils/featureUsage'
+
+// Anonymous counters for the AI filling of a step's inputs. Same rules as every other
+// `logFeatureUsage` caller: aggregated counts only, and the two keys below are the whole
+// vocabulary — no argument name, expression or step id ever reaches here.
+
+/** Which filler the user reached for: the per-field one, or the one above the whole form. */
+export type StepInputFillScope = 'single' | 'all'
+
+/** Counted where the user asks for a suggestion, not where one is accepted. */
+export function logStepInputFill(scope: StepInputFillScope): void {
+	logFeatureUsage('flow_step', 'ai_fill', { key: scope })
+}

frontend/src/lib/components/flows/agentTelemetry.ts

@@ -0,0 +1,19 @@
+import { logFeatureUsage } from '$lib/utils/featureUsage'
+
+// Anonymous counters for the reusable-agent lifecycle (`docs/reusable-ai-agents.md`). Same rules
+// as every other `logFeatureUsage` caller: aggregated counts only, and the four keys below are
+// the whole vocabulary — no agent path, prompt, model or tool ever reaches here.
+
+export type ReusableAgentEvent =
+	/** A step was saved as a new reusable agent. */
+	| 'saved'
+	/** Edits to a linked agent were written back, propagating to every flow using it. */
+	| 'updated'
+	/** A saved agent was picked into a new step. */
+	| 'linked'
+	/** A linked step was forked back into a standalone agent. */
+	| 'unlinked'
+
+export function logReusableAgentUsage(event: ReusableAgentEvent): void {
+	logFeatureUsage('ai_agent', 'reusable', { key: event })
+}

frontend/src/lib/components/flows/content/AgentResourceBar.svelte

@@ -31,6 +31,7 @@
 		linkedToolsScope
 	} from '../linkedAgentToolsStore.svelte'
 	import { getAgentEdit, getAgentEditingPath, setAgentEditingPath } from '../agentEditStore.svelte'
+	import { logReusableAgentUsage } from '../agentTelemetry'
 	import { claimLinkedToolsFetch } from '../flowState'
 	import type { AgentTool as AgentToolStrict } from '../agentToolUtils'
 	import { resource } from 'runed'
@@ -338,6 +339,7 @@
 			const linked = await persist(newPath, description)
 			saveDrawer?.closeDrawer()
 			if (linked) {
+				logReusableAgentUsage(updating ? 'updated' : 'saved')
 				sendUserToast(updating ? `Updated agent ${newPath}` : `Saved reusable agent ${newPath}`)
 			}
 		} catch (e) {
@@ -356,6 +358,7 @@
 		const path = editingPath
 		try {
 			if (await persist(path)) {
+				logReusableAgentUsage('updated')
 				sendUserToast(`Updated agent ${path}`)
 			}
 		} catch (e) {
@@ -419,6 +422,7 @@
 			const fork = await forkFromResource(true)
 			if (fork) {
 				setAgentEditingPath(tools, undefined)
+				logReusableAgentUsage('unlinked')
 				sendUserToast('Forked agent. Its configuration was copied into this step')
 			} else {
 				sendUserToast('The step changed while loading the agent, so nothing was unlinked', true)

frontend/src/lib/components/flows/map/InsertModuleInner.svelte

@@ -14,6 +14,7 @@
 	import { ResourceService } from '$lib/gen'
 	import { workspaceStore } from '$lib/stores'
 	import type { FlowEditorContext } from '../types'
+	import { logReusableAgentUsage } from '../agentTelemetry'
 	import { BotIcon, Loader2, Plus } from 'lucide-svelte'
 
 	const dispatch = createEventDispatcher()
@@ -271,6 +272,7 @@
 						<Button
 							onClick={() => {
 								dispatch('close')
+								logReusableAgentUsage('linked')
 								dispatch('new', { kind: 'aiagent', agentPath: agent.path })
 							}}
 							role="menuitem"