From 7983bdf5a770de144e159e4192553d1eb6d7eb0d Mon Sep 17 00:00:00 2001 From: hugocasa Date: Tue, 25 Aug 2026 10:58:14 +0200 Subject: [PATCH] fix: reject hub integration slugs that would re-target the proxied request Co-Authored-By: Claude Opus 5 --- ai_evals/adapters/frontend/mockBackend.ts | 2080 +++++++++-------- .../frontend/global/hub/confluence.json | 4 +- .../frontend/global/hub/servicenow.json | 6 +- backend/windmill-api-embeddings/src/lib.rs | 25 + backend/windmill-api/openapi.yaml | 1 + backend/windmill-api/src/integration.rs | 39 + .../lib/components/copilot/chat/chatLoop.ts | 9 +- .../components/copilot/chat/shared.test.ts | 27 +- .../src/lib/components/copilot/chat/shared.ts | 49 +- 9 files changed, 1242 insertions(+), 998 deletions(-) diff --git a/ai_evals/adapters/frontend/mockBackend.ts b/ai_evals/adapters/frontend/mockBackend.ts index 0fe0422590..7f20a4bca9 100644 --- a/ai_evals/adapters/frontend/mockBackend.ts +++ b/ai_evals/adapters/frontend/mockBackend.ts @@ -1,268 +1,300 @@ -import { randomUUID } from 'node:crypto' -import confluenceFixture from '../../fixtures/frontend/global/hub/confluence.json' -import servicenowFixture from '../../fixtures/frontend/global/hub/servicenow.json' -import outreachFixture from '../../fixtures/frontend/global/hub/outreach.json' +import { randomUUID } from "node:crypto"; +import confluenceFixture from "../../fixtures/frontend/global/hub/confluence.json"; +import servicenowFixture from "../../fixtures/frontend/global/hub/servicenow.json"; +import outreachFixture from "../../fixtures/frontend/global/hub/outreach.json"; import type { - AppWithLastVersion, - CompletedJob, - Flow, - Job, - ListableApp, - Script -} from '../../../frontend/src/lib/gen' + AppWithLastVersion, + CompletedJob, + Flow, + Job, + ListableApp, + Script, +} from "../../../frontend/src/lib/gen"; import type { - DataTableTables, - DataTableTableSchema, - EndpointTool, - GetDraftForUserResponse, - GetOwnDraftResponse, - ListDraftsResponse, - ScriptLang, - UpdateDraftResponse, - UserDraftItemKind -} from '../../../frontend/src/lib/gen/types.gen' -import { buildScriptLintResult } from './core/script/preview' -import { applyDatatableSql, type BenchmarkDatatableSeed } from './datatableSqlEngine' + DataTableTables, + DataTableTableSchema, + EndpointTool, + GetDraftForUserResponse, + GetOwnDraftResponse, + ListDraftsResponse, + ScriptLang, + UpdateDraftResponse, + UserDraftItemKind, +} from "../../../frontend/src/lib/gen/types.gen"; +import { buildScriptLintResult } from "./core/script/preview"; +import { + applyDatatableSql, + type BenchmarkDatatableSeed, +} from "./datatableSqlEngine"; -export type { BenchmarkDatatableSeed, BenchmarkDatatableTableSeed } from './datatableSqlEngine' +export type { + BenchmarkDatatableSeed, + BenchmarkDatatableTableSeed, +} from "./datatableSqlEngine"; -const BENCHMARK_TIMESTAMP = '1970-01-01T00:00:00.000Z' +const BENCHMARK_TIMESTAMP = "1970-01-01T00:00:00.000Z"; export interface BenchmarkWorkspaceScript { - path: string - summary: string - description?: string - language: Script['language'] - schema?: Record - content: string + path: string; + summary: string; + description?: string; + language: Script["language"]; + schema?: Record; + content: string; } export interface BenchmarkWorkspaceFlow { - path: string - summary: string - description?: string - schema?: Record - value: Flow['value'] + path: string; + summary: string; + description?: string; + schema?: Record; + value: Flow["value"]; } export interface BenchmarkWorkspaceApp { - path: string - summary: string - value: { - files: Record - runnables: Record - data?: unknown - policy?: unknown - custom_path?: unknown - } + path: string; + summary: string; + value: { + files: Record; + runnables: Record; + data?: unknown; + policy?: unknown; + custom_path?: unknown; + }; } export interface BenchmarkWorkspaceJob { - /** Stable id so a case prompt can reference a specific run (e.g. for get_job_logs). */ - id?: string - jobKind?: CompletedJob['job_kind'] - scriptPath?: string - createdBy?: string - label?: string - success?: boolean - logs?: string + /** Stable id so a case prompt can reference a specific run (e.g. for get_job_logs). */ + id?: string; + jobKind?: CompletedJob["job_kind"]; + scriptPath?: string; + createdBy?: string; + label?: string; + success?: boolean; + logs?: string; } export interface BenchmarkWorkspaceRunnables { - scripts?: BenchmarkWorkspaceScript[] - flows?: BenchmarkWorkspaceFlow[] - apps?: BenchmarkWorkspaceApp[] - datatables?: BenchmarkDatatableSeed[] - jobs?: BenchmarkWorkspaceJob[] + scripts?: BenchmarkWorkspaceScript[]; + flows?: BenchmarkWorkspaceFlow[]; + apps?: BenchmarkWorkspaceApp[]; + datatables?: BenchmarkDatatableSeed[]; + jobs?: BenchmarkWorkspaceJob[]; } -type BenchmarkCompletedJob = CompletedJob & { type: 'CompletedJob' } +type BenchmarkCompletedJob = CompletedJob & { type: "CompletedJob" }; -const benchmarkWorkspaces = new Set() -const benchmarkWorkspaceRunnables = new Map() +const benchmarkWorkspaces = new Set(); +const benchmarkWorkspaceRunnables = new Map< + string, + BenchmarkWorkspaceRunnables +>(); // Keyed by `${workspace}::${jobId}` so concurrent attempts (or distinct cases) // can seed the same fixed job id without clobbering each other's entry. -const benchmarkJobs = new Map() +const benchmarkJobs = new Map< + string, + { workspace: string; job: BenchmarkCompletedJob } +>(); function benchmarkJobKey(workspace: string, jobId: string): string { - return `${workspace}::${jobId}` + return `${workspace}::${jobId}`; } export function resetBenchmarkMockBackend(): void { - benchmarkWorkspaces.clear() - benchmarkWorkspaceRunnables.clear() - benchmarkJobs.clear() - benchmarkDrafts.clear() + benchmarkWorkspaces.clear(); + benchmarkWorkspaceRunnables.clear(); + benchmarkJobs.clear(); + benchmarkDrafts.clear(); } // Stand-in for FolderService.createFolder so the global create_folder tool runs in // memory instead of mutating the real backend. Folders aren't otherwise modelled // (no folder-listing in evals), so this just echoes the created name. -export function createBenchmarkFolder(_workspace: string, name: string): string { - return name +export function createBenchmarkFolder( + _workspace: string, + name: string, +): string { + return name; } export function registerBenchmarkWorkspace(workspace: string): void { - benchmarkWorkspaces.add(workspace) + benchmarkWorkspaces.add(workspace); } export function registerBenchmarkWorkspaceRunnables( - workspace: string, - runnables: BenchmarkWorkspaceRunnables + workspace: string, + runnables: BenchmarkWorkspaceRunnables, ): void { - benchmarkWorkspaces.add(workspace) - // Fresh case: drop any drafts left from a prior run on this workspace id. - clearBenchmarkDrafts(workspace) - // Datatables are mutated in place by exec_datatable_sql (a write must be visible - // to later reads), so store an isolated deep copy — never mutate the caller's seed. - benchmarkWorkspaceRunnables.set(workspace, { - ...runnables, - datatables: runnables.datatables ? structuredClone(runnables.datatables) : undefined - }) - // Seed any fixture jobs so list_runs / get_job_logs have data to return. - for (const seed of runnables.jobs ?? []) { - createBenchmarkCompletedJob({ - workspace, - id: seed.id, - jobKind: seed.jobKind ?? 'script', - success: seed.success, - scriptPath: seed.scriptPath, - createdBy: seed.createdBy, - label: seed.label, - logs: seed.logs - }) - } + benchmarkWorkspaces.add(workspace); + // Fresh case: drop any drafts left from a prior run on this workspace id. + clearBenchmarkDrafts(workspace); + // Datatables are mutated in place by exec_datatable_sql (a write must be visible + // to later reads), so store an isolated deep copy — never mutate the caller's seed. + benchmarkWorkspaceRunnables.set(workspace, { + ...runnables, + datatables: runnables.datatables + ? structuredClone(runnables.datatables) + : undefined, + }); + // Seed any fixture jobs so list_runs / get_job_logs have data to return. + for (const seed of runnables.jobs ?? []) { + createBenchmarkCompletedJob({ + workspace, + id: seed.id, + jobKind: seed.jobKind ?? "script", + success: seed.success, + scriptPath: seed.scriptPath, + createdBy: seed.createdBy, + label: seed.label, + logs: seed.logs, + }); + } } export function unregisterBenchmarkWorkspace(workspace: string): void { - benchmarkWorkspaces.delete(workspace) - benchmarkWorkspaceRunnables.delete(workspace) - clearBenchmarkDrafts(workspace) - for (const [jobId, entry] of benchmarkJobs.entries()) { - if (entry.workspace === workspace) { - benchmarkJobs.delete(jobId) - } - } + benchmarkWorkspaces.delete(workspace); + benchmarkWorkspaceRunnables.delete(workspace); + clearBenchmarkDrafts(workspace); + for (const [jobId, entry] of benchmarkJobs.entries()) { + if (entry.workspace === workspace) { + benchmarkJobs.delete(jobId); + } + } } export function unregisterBenchmarkWorkspaceRunnables(workspace: string): void { - unregisterBenchmarkWorkspace(workspace) + unregisterBenchmarkWorkspace(workspace); } export function hasBenchmarkWorkspace(workspace: string): boolean { - return benchmarkWorkspaces.has(workspace) + return benchmarkWorkspaces.has(workspace); } export function listBenchmarkScripts(workspace: string): Script[] | null { - const runnables = benchmarkWorkspaceRunnables.get(workspace) - if (!runnables) { - return null - } - return (runnables.scripts ?? []).map(buildBenchmarkScript) + const runnables = benchmarkWorkspaceRunnables.get(workspace); + if (!runnables) { + return null; + } + return (runnables.scripts ?? []).map(buildBenchmarkScript); } export function listBenchmarkFlows(workspace: string): Flow[] | null { - const runnables = benchmarkWorkspaceRunnables.get(workspace) - if (!runnables) { - return null - } - return (runnables.flows ?? []).map(buildBenchmarkFlow) + const runnables = benchmarkWorkspaceRunnables.get(workspace); + if (!runnables) { + return null; + } + return (runnables.flows ?? []).map(buildBenchmarkFlow); } -export function getBenchmarkScriptByPath(workspace: string, path: string): Script | null { - const script = benchmarkWorkspaceRunnables - .get(workspace) - ?.scripts?.find((entry) => entry.path === path) +export function getBenchmarkScriptByPath( + workspace: string, + path: string, +): Script | null { + const script = benchmarkWorkspaceRunnables + .get(workspace) + ?.scripts?.find((entry) => entry.path === path); - return script ? buildBenchmarkScript(script) : null + return script ? buildBenchmarkScript(script) : null; } -export function getBenchmarkScriptByHash(workspace: string, hash: string): Script | null { - const script = benchmarkWorkspaceRunnables - .get(workspace) - ?.scripts?.find((entry) => buildBenchmarkScriptHash(entry.path) === hash) +export function getBenchmarkScriptByHash( + workspace: string, + hash: string, +): Script | null { + const script = benchmarkWorkspaceRunnables + .get(workspace) + ?.scripts?.find((entry) => buildBenchmarkScriptHash(entry.path) === hash); - return script ? buildBenchmarkScript(script) : null + return script ? buildBenchmarkScript(script) : null; } -export function getBenchmarkFlowByPath(workspace: string, path: string): Flow | null { - const flow = benchmarkWorkspaceRunnables - .get(workspace) - ?.flows?.find((entry) => entry.path === path) +export function getBenchmarkFlowByPath( + workspace: string, + path: string, +): Flow | null { + const flow = benchmarkWorkspaceRunnables + .get(workspace) + ?.flows?.find((entry) => entry.path === path); - return flow ? buildBenchmarkFlow(flow) : null + return flow ? buildBenchmarkFlow(flow) : null; } export function listBenchmarkApps(workspace: string): ListableApp[] | null { - const runnables = benchmarkWorkspaceRunnables.get(workspace) - if (!runnables) { - return null - } - return (runnables.apps ?? []).map(buildBenchmarkListableApp) + const runnables = benchmarkWorkspaceRunnables.get(workspace); + if (!runnables) { + return null; + } + return (runnables.apps ?? []).map(buildBenchmarkListableApp); } -export function getBenchmarkAppByPath(workspace: string, path: string): AppWithLastVersion | null { - const app = benchmarkWorkspaceRunnables - .get(workspace) - ?.apps?.find((entry) => entry.path === path) +export function getBenchmarkAppByPath( + workspace: string, + path: string, +): AppWithLastVersion | null { + const app = benchmarkWorkspaceRunnables + .get(workspace) + ?.apps?.find((entry) => entry.path === path); - return app ? buildBenchmarkApp(app) : null + return app ? buildBenchmarkApp(app) : null; } export function createBenchmarkCompletedJob(input: { - workspace: string - jobKind: CompletedJob['job_kind'] - success?: boolean - result?: unknown - logs?: string - scriptPath?: string - scriptHash?: string - args?: Record - id?: string - createdBy?: string - label?: string + workspace: string; + jobKind: CompletedJob["job_kind"]; + success?: boolean; + result?: unknown; + logs?: string; + scriptPath?: string; + scriptHash?: string; + args?: Record; + id?: string; + createdBy?: string; + label?: string; }): string { - const jobId = input.id ?? `benchmark-job-${randomUUID()}` - const now = new Date().toISOString() - const job: BenchmarkCompletedJob = { - type: 'CompletedJob', - id: jobId, - workspace_id: input.workspace, - created_by: input.createdBy ?? 'ai-evals', - created_at: now, - started_at: now, - completed_at: now, - duration_ms: 0, - success: input.success ?? true, - script_path: input.scriptPath, - script_hash: input.scriptHash, - args: input.args, - result: input.result, - logs: input.logs, - canceled: false, - job_kind: input.jobKind, - permissioned_as: 'u/ai-evals', - is_flow_step: false, - is_skipped: false, - email: 'ai-evals@local', - visible_to_owner: true, - tag: 'benchmark', - labels: input.label ? [input.label] : undefined - } + const jobId = input.id ?? `benchmark-job-${randomUUID()}`; + const now = new Date().toISOString(); + const job: BenchmarkCompletedJob = { + type: "CompletedJob", + id: jobId, + workspace_id: input.workspace, + created_by: input.createdBy ?? "ai-evals", + created_at: now, + started_at: now, + completed_at: now, + duration_ms: 0, + success: input.success ?? true, + script_path: input.scriptPath, + script_hash: input.scriptHash, + args: input.args, + result: input.result, + logs: input.logs, + canceled: false, + job_kind: input.jobKind, + permissioned_as: "u/ai-evals", + is_flow_step: false, + is_skipped: false, + email: "ai-evals@local", + visible_to_owner: true, + tag: "benchmark", + labels: input.label ? [input.label] : undefined, + }; - benchmarkJobs.set(benchmarkJobKey(input.workspace, jobId), { workspace: input.workspace, job }) - return jobId + benchmarkJobs.set(benchmarkJobKey(input.workspace, jobId), { + workspace: input.workspace, + job, + }); + return jobId; } export function getBenchmarkCompletedJob( - workspace: string, - jobId: string + workspace: string, + jobId: string, ): BenchmarkCompletedJob | null { - const entry = benchmarkJobs.get(benchmarkJobKey(workspace, jobId)) - if (!entry) { - return null - } - return structuredClone(entry.job) + const entry = benchmarkJobs.get(benchmarkJobKey(workspace, jobId)); + if (!entry) { + return null; + } + return structuredClone(entry.job); } /** @@ -273,13 +305,13 @@ export function getBenchmarkCompletedJob( * eval cases assert on the recorded `list_runs` tool call, not on filtering. */ export function listBenchmarkJobs(workspace: string): Job[] | null { - if (!hasBenchmarkWorkspace(workspace)) { - return null - } - return [...benchmarkJobs.values()] - .filter((entry) => entry.workspace === workspace) - .map((entry) => structuredClone(entry.job) as Job) - .sort((a, b) => (b.created_at ?? '').localeCompare(a.created_at ?? '')) + if (!hasBenchmarkWorkspace(workspace)) { + return null; + } + return [...benchmarkJobs.values()] + .filter((entry) => entry.workspace === workspace) + .map((entry) => structuredClone(entry.job) as Job) + .sort((a, b) => (b.created_at ?? "").localeCompare(a.created_at ?? "")); } /** @@ -287,11 +319,11 @@ export function listBenchmarkJobs(workspace: string): Job[] | null { * "not found" error for an unknown id, matching the backend 404. */ export function getBenchmarkJobLogs(workspace: string, jobId: string): string { - const job = getBenchmarkCompletedJob(workspace, jobId) - if (!job) { - throw new Error(`Job Logs not found for "${jobId}"`) - } - return job.logs ?? '' + const job = getBenchmarkCompletedJob(workspace, jobId); + if (!job) { + throw new Error(`Job Logs not found for "${jobId}"`); + } + return job.logs ?? ""; } // ============= Drafts (per-user, DB-backed in production) ============= @@ -306,9 +338,15 @@ export function getBenchmarkJobLogs(workspace: string, jobId: string): string { * `frontend/src/lib/components/copilot/chat/global/core.test.ts`. */ const benchmarkDrafts = new Map< - string, - { workspace: string; kind: UserDraftItemKind; path: string; value: unknown; createdAt: string } ->() + string, + { + workspace: string; + kind: UserDraftItemKind; + path: string; + value: unknown; + createdAt: string; + } +>(); // Counter-based timestamps: deterministic run-to-run (same event order → same // values) but MONOTONIC per update, because production bumps a draft row's @@ -316,22 +354,26 @@ const benchmarkDrafts = new Map< // it — a fixed timestamp would serve stale patches after an edit. No eval // simulates a concurrent writer, so every save is accepted and the conflict // branch is never taken. -let benchmarkDraftClock = 0 +let benchmarkDraftClock = 0; function nextBenchmarkDraftTimestamp(): string { - benchmarkDraftClock += 1 - return new Date(benchmarkDraftClock * 1000).toISOString() + benchmarkDraftClock += 1; + return new Date(benchmarkDraftClock * 1000).toISOString(); } -function benchmarkDraftKey(workspace: string, kind: string, path: string): string { - return `${workspace}::${kind}::${path}` +function benchmarkDraftKey( + workspace: string, + kind: string, + path: string, +): string { + return `${workspace}::${kind}::${path}`; } export function clearBenchmarkDrafts(workspace: string): void { - for (const [key, entry] of benchmarkDrafts.entries()) { - if (entry.workspace === workspace) { - benchmarkDrafts.delete(key) - } - } + for (const [key, entry] of benchmarkDrafts.entries()) { + if (entry.workspace === workspace) { + benchmarkDrafts.delete(key); + } + } } /** @@ -342,95 +384,107 @@ export function clearBenchmarkDrafts(workspace: string): void { * output read-back captures — not the stale seed. */ export function seedBenchmarkDraft( - workspace: string, - kind: UserDraftItemKind, - path: string, - value: unknown + workspace: string, + kind: UserDraftItemKind, + path: string, + value: unknown, ): void { - benchmarkDrafts.set(benchmarkDraftKey(workspace, kind, path), { - workspace, - kind, - path, - value, - createdAt: nextBenchmarkDraftTimestamp() - }) + benchmarkDrafts.set(benchmarkDraftKey(workspace, kind, path), { + workspace, + kind, + path, + value, + createdAt: nextBenchmarkDraftTimestamp(), + }); } /** Mirror `DraftService.updateDraft`: a `null`/omitted value deletes the row. */ export function updateBenchmarkDraft(input: { - workspace: string - kind: UserDraftItemKind - path: string - requestBody?: { value?: unknown } + workspace: string; + kind: UserDraftItemKind; + path: string; + requestBody?: { value?: unknown }; }): UpdateDraftResponse { - const key = benchmarkDraftKey(input.workspace, input.kind, input.path) - const value = input.requestBody?.value - const createdAt = nextBenchmarkDraftTimestamp() - if (value == null) { - benchmarkDrafts.delete(key) - } else { - benchmarkDrafts.set(key, { - workspace: input.workspace, - kind: input.kind, - path: input.path, - value, - createdAt - }) - } - return { status: 'saved', current_timestamp: createdAt } + const key = benchmarkDraftKey(input.workspace, input.kind, input.path); + const value = input.requestBody?.value; + const createdAt = nextBenchmarkDraftTimestamp(); + if (value == null) { + benchmarkDrafts.delete(key); + } else { + benchmarkDrafts.set(key, { + workspace: input.workspace, + kind: input.kind, + path: input.path, + value, + createdAt, + }); + } + return { status: "saved", current_timestamp: createdAt }; } /** Mirror `DraftService.getDraftForUser`: 404-shaped throw when absent so the * adapter's narrowed catch treats it as "no draft" instead of re-throwing. */ export function getBenchmarkDraftForUser(input: { - workspace: string - kind: UserDraftItemKind - path: string + workspace: string; + kind: UserDraftItemKind; + path: string; }): GetDraftForUserResponse { - const entry = benchmarkDrafts.get(benchmarkDraftKey(input.workspace, input.kind, input.path)) - if (!entry) { - throw Object.assign(new Error(`no draft for "${input.path}"`), { status: 404 }) - } - return { value: entry.value, created_at: entry.createdAt } + const entry = benchmarkDrafts.get( + benchmarkDraftKey(input.workspace, input.kind, input.path), + ); + if (!entry) { + throw Object.assign(new Error(`no draft for "${input.path}"`), { + status: 404, + }); + } + return { value: entry.value, created_at: entry.createdAt }; } /** Mirror `DraftService.getOwnDraft`: `null` (200) when absent — unlike * `getDraftForUser`, absence is not an error on this route. */ export function getBenchmarkOwnDraft(input: { - workspace: string - kind: UserDraftItemKind - path: string + workspace: string; + kind: UserDraftItemKind; + path: string; }): GetOwnDraftResponse { - const entry = benchmarkDrafts.get(benchmarkDraftKey(input.workspace, input.kind, input.path)) - if (!entry) { - return null - } - return { value: entry.value, created_at: entry.createdAt } + const entry = benchmarkDrafts.get( + benchmarkDraftKey(input.workspace, input.kind, input.path), + ); + if (!entry) { + return null; + } + return { value: entry.value, created_at: entry.createdAt }; } /** Whether a deployed benchmark item exists for a draft row's kind+path — * drives `draft_only`, which production computes against the deployed tables. */ -function benchmarkDeployedExists(workspace: string, kind: UserDraftItemKind, path: string): boolean { - if (kind === 'script') return Boolean(getBenchmarkScriptByPath(workspace, path)) - if (kind === 'flow') return Boolean(getBenchmarkFlowByPath(workspace, path)) - if (kind === 'app' || kind === 'raw_app') return Boolean(getBenchmarkAppByPath(workspace, path)) - // Drawer kinds (variables/resources/schedules/triggers) have no deployed - // benchmark stores today. - return false +function benchmarkDeployedExists( + workspace: string, + kind: UserDraftItemKind, + path: string, +): boolean { + if (kind === "script") + return Boolean(getBenchmarkScriptByPath(workspace, path)); + if (kind === "flow") return Boolean(getBenchmarkFlowByPath(workspace, path)); + if (kind === "app" || kind === "raw_app") + return Boolean(getBenchmarkAppByPath(workspace, path)); + // Drawer kinds (variables/resources/schedules/triggers) have no deployed + // benchmark stores today. + return false; } /** Mirror `DraftService.listDrafts`: metadata rows (no value) for a workspace. */ export function listBenchmarkDrafts(workspace: string): ListDraftsResponse { - return [...benchmarkDrafts.values()] - .filter((entry) => entry.workspace === workspace) - .map((entry) => ({ - kind: entry.kind, - path: entry.path, - summary: (entry.value as { summary?: string } | null)?.summary, - draft_only: !benchmarkDeployedExists(workspace, entry.kind, entry.path), - legacy_draft: false, - created_at: entry.createdAt - })) + return [...benchmarkDrafts.values()] + .filter((entry) => entry.workspace === workspace) + .map((entry) => ({ + kind: entry.kind, + path: entry.path, + summary: (entry.value as { summary?: string } | null)?.summary, + draft_only: !benchmarkDeployedExists(workspace, entry.kind, entry.path), + legacy_draft: false, + created_at: entry.createdAt, + })); } // ============= Datatables (best-effort in-memory SQL) ============= @@ -441,47 +495,52 @@ export function listBenchmarkDrafts(workspace: string): ListDraftsResponse { * Returns `null` for a non-benchmark workspace so callers can fall through to * the real backend; an empty seed yields `[]`. */ -export function listBenchmarkDatatables(workspace: string): DataTableTables[] | null { - const runnables = benchmarkWorkspaceRunnables.get(workspace) - if (!runnables) { - return null - } - return (runnables.datatables ?? []).map((datatable) => ({ - datatable_name: datatable.datatable_name, - schemas: Object.fromEntries( - Object.entries(datatable.schemas).map(([schema, tables]) => [schema, Object.keys(tables)]) - ) - })) +export function listBenchmarkDatatables( + workspace: string, +): DataTableTables[] | null { + const runnables = benchmarkWorkspaceRunnables.get(workspace); + if (!runnables) { + return null; + } + return (runnables.datatables ?? []).map((datatable) => ({ + datatable_name: datatable.datatable_name, + schemas: Object.fromEntries( + Object.entries(datatable.schemas).map(([schema, tables]) => [ + schema, + Object.keys(tables), + ]), + ), + })); } export function getBenchmarkDatatableSchema(input: { - workspace: string - datatableName: string - schemaName: string - tableName: string + workspace: string; + datatableName: string; + schemaName: string; + tableName: string; }): DataTableTableSchema { - const runnables = benchmarkWorkspaceRunnables.get(input.workspace) - const datatable = (runnables?.datatables ?? []).find( - (entry) => entry.datatable_name === input.datatableName - ) - if (!datatable) { - // Message MUST match the production `isDatatableNotConfiguredError` regex - // (/datatable\s+\S+\s+not found/i in datatableTools.ts) so the - // get_datatable_table_schema not-configured mapping is actually exercised. - throw new Error(`datatable "${input.datatableName}" not found`) - } - const table = datatable.schemas?.[input.schemaName]?.[input.tableName] - if (!table) { - throw new Error( - `table "${input.schemaName}.${input.tableName}" not found in datatable "${input.datatableName}"` - ) - } - return { - datatable_name: input.datatableName, - schema_name: input.schemaName, - table_name: input.tableName, - columns: table.columns - } + const runnables = benchmarkWorkspaceRunnables.get(input.workspace); + const datatable = (runnables?.datatables ?? []).find( + (entry) => entry.datatable_name === input.datatableName, + ); + if (!datatable) { + // Message MUST match the production `isDatatableNotConfiguredError` regex + // (/datatable\s+\S+\s+not found/i in datatableTools.ts) so the + // get_datatable_table_schema not-configured mapping is actually exercised. + throw new Error(`datatable "${input.datatableName}" not found`); + } + const table = datatable.schemas?.[input.schemaName]?.[input.tableName]; + if (!table) { + throw new Error( + `table "${input.schemaName}.${input.tableName}" not found in datatable "${input.datatableName}"`, + ); + } + return { + datatable_name: input.datatableName, + schema_name: input.schemaName, + table_name: input.tableName, + columns: table.columns, + }; } /** @@ -492,22 +551,22 @@ export function getBenchmarkDatatableSchema(input: { * completed job and returns its id, like `runBenchmarkScriptPreview`. */ export function runBenchmarkDatatableSql(input: { - workspace: string - datatableName: string - sql: string + workspace: string; + datatableName: string; + sql: string; }): string { - const runnables = benchmarkWorkspaceRunnables.get(input.workspace) - const datatable = (runnables?.datatables ?? []).find( - (entry) => entry.datatable_name === input.datatableName - ) - const rows = datatable ? applyDatatableSql(datatable, input.sql).rows : [] - return createBenchmarkCompletedJob({ - workspace: input.workspace, - jobKind: 'preview', - success: true, - args: { database: `datatable://${input.datatableName}` }, - result: rows - }) + const runnables = benchmarkWorkspaceRunnables.get(input.workspace); + const datatable = (runnables?.datatables ?? []).find( + (entry) => entry.datatable_name === input.datatableName, + ); + const rows = datatable ? applyDatatableSql(datatable, input.sql).rows : []; + return createBenchmarkCompletedJob({ + workspace: input.workspace, + jobKind: "preview", + success: true, + args: { database: `datatable://${input.datatableName}` }, + result: rows, + }); } /** @@ -516,237 +575,249 @@ export function runBenchmarkDatatableSql(input: { * polling, so it is always present and completed. */ export function getBenchmarkCompletedJobResultMaybe(input: { - workspace: string - id: string + workspace: string; + id: string; }): { success: boolean; completed: boolean; result: unknown } { - const job = getBenchmarkCompletedJob(input.workspace, input.id) - if (!job) { - throw new Error(`Job "${input.id}" not found in benchmark workspace`) - } - return { success: job.success, completed: true, result: job.result } + const job = getBenchmarkCompletedJob(input.workspace, input.id); + if (!job) { + throw new Error(`Job "${input.id}" not found in benchmark workspace`); + } + return { success: job.success, completed: true, result: job.result }; } export function runBenchmarkScriptPreview(input: { - workspace: string - requestBody: { - content?: string - language?: ScriptLang | 'bunnative' - args?: Record - path?: string - } + workspace: string; + requestBody: { + content?: string; + language?: ScriptLang | "bunnative"; + args?: Record; + path?: string; + }; }): string { - const content = input.requestBody.content ?? '' - const language = input.requestBody.language ?? 'bun' - const lintResult = buildScriptLintResult(content, language) - const success = lintResult.errorCount === 0 + const content = input.requestBody.content ?? ""; + const language = input.requestBody.language ?? "bun"; + const lintResult = buildScriptLintResult(content, language); + const success = lintResult.errorCount === 0; - return createBenchmarkCompletedJob({ - workspace: input.workspace, - jobKind: 'preview', - success, - scriptPath: input.requestBody.path, - args: input.requestBody.args, - result: success - ? { - path: input.requestBody.path, - args: input.requestBody.args ?? {}, - validated: true - } - : { - path: input.requestBody.path, - args: input.requestBody.args ?? {}, - errorCount: lintResult.errorCount, - errors: lintResult.errors.map((entry) => ({ - line: entry.startLineNumber, - message: entry.message - })) - } - }) + return createBenchmarkCompletedJob({ + workspace: input.workspace, + jobKind: "preview", + success, + scriptPath: input.requestBody.path, + args: input.requestBody.args, + result: success + ? { + path: input.requestBody.path, + args: input.requestBody.args ?? {}, + validated: true, + } + : { + path: input.requestBody.path, + args: input.requestBody.args ?? {}, + errorCount: lintResult.errorCount, + errors: lintResult.errors.map((entry) => ({ + line: entry.startLineNumber, + message: entry.message, + })), + }, + }); } export function runBenchmarkScriptByPath(input: { - workspace: string - path: string - args?: Record + workspace: string; + path: string; + args?: Record; }): string { - const script = getBenchmarkScriptByPath(input.workspace, input.path) - return createBenchmarkCompletedJob({ - workspace: input.workspace, - jobKind: 'script', - success: script !== null, - scriptPath: input.path, - args: input.args, - result: - script !== null - ? { - path: input.path, - args: input.args ?? {}, - mocked: true - } - : { - error: `Script "${input.path}" not found in benchmark workspace` - }, - logs: - script !== null - ? 'Mock benchmark script run completed successfully.' - : `Script "${input.path}" not found in benchmark workspace.` - }) + const script = getBenchmarkScriptByPath(input.workspace, input.path); + return createBenchmarkCompletedJob({ + workspace: input.workspace, + jobKind: "script", + success: script !== null, + scriptPath: input.path, + args: input.args, + result: + script !== null + ? { + path: input.path, + args: input.args ?? {}, + mocked: true, + } + : { + error: `Script "${input.path}" not found in benchmark workspace`, + }, + logs: + script !== null + ? "Mock benchmark script run completed successfully." + : `Script "${input.path}" not found in benchmark workspace.`, + }); } export function runBenchmarkFlowByPath(input: { - workspace: string - path: string - args?: Record + workspace: string; + path: string; + args?: Record; }): string { - const flow = getBenchmarkFlowByPath(input.workspace, input.path) - return createBenchmarkCompletedJob({ - workspace: input.workspace, - jobKind: 'flowpreview', - success: flow !== null, - args: input.args, - result: - flow !== null - ? { - path: input.path, - args: input.args ?? {}, - mocked: true - } - : { - error: `Flow "${input.path}" not found in benchmark workspace` - }, - logs: - flow !== null - ? 'Mock benchmark flow run completed successfully.' - : `Flow "${input.path}" not found in benchmark workspace.` - }) + const flow = getBenchmarkFlowByPath(input.workspace, input.path); + return createBenchmarkCompletedJob({ + workspace: input.workspace, + jobKind: "flowpreview", + success: flow !== null, + args: input.args, + result: + flow !== null + ? { + path: input.path, + args: input.args ?? {}, + mocked: true, + } + : { + error: `Flow "${input.path}" not found in benchmark workspace`, + }, + logs: + flow !== null + ? "Mock benchmark flow run completed successfully." + : `Flow "${input.path}" not found in benchmark workspace.`, + }); } export function previewBenchmarkSchedule(input: { - requestBody?: Record + requestBody?: Record; }): Record { - const schedule = input.requestBody?.schedule - if (typeof schedule !== 'string' || schedule.trim().split(/\s+/).length !== 6) { - throw new Error(`schedule must use a six-field cron expression, got ${JSON.stringify(schedule)}`) - } + const schedule = input.requestBody?.schedule; + if ( + typeof schedule !== "string" || + schedule.trim().split(/\s+/).length !== 6 + ) { + throw new Error( + `schedule must use a six-field cron expression, got ${JSON.stringify(schedule)}`, + ); + } - return { - next_runs: ['1970-01-02T00:00:00.000Z'] - } + return { + next_runs: ["1970-01-02T00:00:00.000Z"], + }; } export function createBenchmarkSchedule(input: { - workspace: string - requestBody: Record + workspace: string; + requestBody: Record; }): Record { - assertBenchmarkWorkspacePath('schedule', input.requestBody.path) - assertBenchmarkWorkspacePath('target', input.requestBody.script_path) - return { - path: input.requestBody.path, - target_path: input.requestBody.script_path, - is_flow: input.requestBody.is_flow, - mocked: true - } + assertBenchmarkWorkspacePath("schedule", input.requestBody.path); + assertBenchmarkWorkspacePath("target", input.requestBody.script_path); + return { + path: input.requestBody.path, + target_path: input.requestBody.script_path, + is_flow: input.requestBody.is_flow, + mocked: true, + }; } export function createBenchmarkHttpTrigger(input: { - workspace: string - requestBody: Record + workspace: string; + requestBody: Record; }): Record { - assertBenchmarkWorkspacePath('trigger', input.requestBody.path) - assertBenchmarkWorkspacePath('target', input.requestBody.script_path) - if ( - typeof input.requestBody.route_path === 'string' && - input.requestBody.route_path.startsWith('/') - ) { - throw new Error(`HTTP trigger route_path must not start with /, got "${input.requestBody.route_path}"`) - } - return { - path: input.requestBody.path, - target_path: input.requestBody.script_path, - route_path: input.requestBody.route_path, - is_flow: input.requestBody.is_flow, - mocked: true - } + assertBenchmarkWorkspacePath("trigger", input.requestBody.path); + assertBenchmarkWorkspacePath("target", input.requestBody.script_path); + if ( + typeof input.requestBody.route_path === "string" && + input.requestBody.route_path.startsWith("/") + ) { + throw new Error( + `HTTP trigger route_path must not start with /, got "${input.requestBody.route_path}"`, + ); + } + return { + path: input.requestBody.path, + target_path: input.requestBody.script_path, + route_path: input.requestBody.route_path, + is_flow: input.requestBody.is_flow, + mocked: true, + }; } function assertBenchmarkWorkspacePath(label: string, value: unknown): void { - if (typeof value !== 'string' || (!value.startsWith('f/') && !value.startsWith('u/'))) { - throw new Error(`${label} path must start with f/ or u/, got ${JSON.stringify(value)}`) - } + if ( + typeof value !== "string" || + (!value.startsWith("f/") && !value.startsWith("u/")) + ) { + throw new Error( + `${label} path must start with f/ or u/, got ${JSON.stringify(value)}`, + ); + } } function buildBenchmarkScriptHash(path: string): string { - return `benchmark:${path}` + return `benchmark:${path}`; } function buildBenchmarkScript(script: BenchmarkWorkspaceScript): Script { - return { - workspace_id: 'benchmark', - hash: buildBenchmarkScriptHash(script.path), - path: script.path, - parent_hashes: [], - summary: script.summary, - description: script.description ?? '', - content: script.content, - created_by: 'benchmark', - created_at: BENCHMARK_TIMESTAMP, - archived: false, - schema: script.schema ?? {}, - deleted: false, - is_template: false, - extra_perms: {}, - language: script.language, - kind: 'script', - starred: false, - has_preprocessor: false, - modules: null - } + return { + workspace_id: "benchmark", + hash: buildBenchmarkScriptHash(script.path), + path: script.path, + parent_hashes: [], + summary: script.summary, + description: script.description ?? "", + content: script.content, + created_by: "benchmark", + created_at: BENCHMARK_TIMESTAMP, + archived: false, + schema: script.schema ?? {}, + deleted: false, + is_template: false, + extra_perms: {}, + language: script.language, + kind: "script", + starred: false, + has_preprocessor: false, + modules: null, + }; } function buildBenchmarkFlow(flow: BenchmarkWorkspaceFlow): Flow { - return { - path: flow.path, - summary: flow.summary, - description: flow.description ?? '', - value: flow.value, - schema: flow.schema ?? {}, - edited_by: 'benchmark', - edited_at: BENCHMARK_TIMESTAMP, - archived: false, - extra_perms: {} - } as Flow + return { + path: flow.path, + summary: flow.summary, + description: flow.description ?? "", + value: flow.value, + schema: flow.schema ?? {}, + edited_by: "benchmark", + edited_at: BENCHMARK_TIMESTAMP, + archived: false, + extra_perms: {}, + } as Flow; } function buildBenchmarkListableApp(app: BenchmarkWorkspaceApp): ListableApp { - return { - id: 0, - workspace_id: 'benchmark', - path: app.path, - summary: app.summary, - version: 1, - extra_perms: {}, - edited_at: BENCHMARK_TIMESTAMP, - execution_mode: 'viewer', - raw_app: true - } + return { + id: 0, + workspace_id: "benchmark", + path: app.path, + summary: app.summary, + version: 1, + extra_perms: {}, + edited_at: BENCHMARK_TIMESTAMP, + execution_mode: "viewer", + raw_app: true, + }; } function buildBenchmarkApp(app: BenchmarkWorkspaceApp): AppWithLastVersion { - return { - id: 0, - workspace_id: 'benchmark', - path: app.path, - summary: app.summary, - versions: [1], - created_by: 'benchmark', - created_at: BENCHMARK_TIMESTAMP, - value: app.value, - policy: (app.value.policy ?? {}) as AppWithLastVersion['policy'], - execution_mode: 'viewer', - extra_perms: {}, - custom_path: app.value.custom_path as string | undefined, - raw_app: true - } + return { + id: 0, + workspace_id: "benchmark", + path: app.path, + summary: app.summary, + versions: [1], + created_by: "benchmark", + created_at: BENCHMARK_TIMESTAMP, + value: app.value, + policy: (app.value.policy ?? {}) as AppWithLastVersion["policy"], + execution_mode: "viewer", + extra_perms: {}, + custom_path: app.value.custom_path as string | undefined, + raw_app: true, + }; } // ============= API endpoint catalog (McpService.listMcpTools + raw fetch) ============= @@ -756,110 +827,113 @@ function buildBenchmarkApp(app: BenchmarkWorkspaceApp): AppWithLastVersion { // and `handleBenchmarkApiFetch` answers the executed calls. const BENCHMARK_MCP_TOOLS: EndpointTool[] = [ - { - name: 'listWorkers', - description: 'List workers', - instructions: 'List all workers with their last ping and job counts.', - path: '/workers/list', - method: 'GET', - query_params_schema: { - type: 'object', - properties: { page: { type: 'integer' }, per_page: { type: 'integer' } } - } - }, - { - name: 'listQueue', - description: 'List queued jobs', - instructions: '', - path: '/w/{workspace}/jobs/queue/list', - method: 'GET', - path_params_schema: { - type: 'object', - properties: { workspace: { type: 'string' } }, - required: ['workspace'] - } - }, - { - name: 'getJob', - description: 'get job', - instructions: '', - path: '/w/{workspace}/jobs_u/get/{id}', - method: 'GET', - path_params_schema: { - type: 'object', - properties: { workspace: { type: 'string' }, id: { type: 'string', format: 'uuid' } }, - required: ['workspace', 'id'] - }, - query_params_schema: { - type: 'object', - properties: { - no_logs: { type: 'boolean' }, - no_code: { type: 'boolean' }, - approval_token: { type: 'string' } - }, - required: [] - } - }, - { - name: 'runScriptByPath', - description: 'Run the deployed version of a script by path', - instructions: '', - path: '/w/{workspace}/jobs/run/p/{path}', - method: 'POST', - path_params_schema: { - type: 'object', - properties: { workspace: { type: 'string' }, path: { type: 'string' } }, - required: ['workspace', 'path'] - }, - body_schema: { type: 'object', properties: {} } - }, - { - name: 'runFlowByPath', - description: 'Run the deployed version of a flow by path', - instructions: '', - path: '/w/{workspace}/jobs/run/f/{path}', - method: 'POST', - path_params_schema: { - type: 'object', - properties: { workspace: { type: 'string' }, path: { type: 'string' } }, - required: ['workspace', 'path'] - }, - body_schema: { type: 'object', properties: {} } - }, - // Draft-covered endpoints, present so steering cases exercise the guard the - // way production does (hidden from search, refused at call time). - { - name: 'getScriptByPath', - description: 'Get a script by path', - instructions: '', - path: '/w/{workspace}/scripts/get/p/{path}', - method: 'GET' - }, - { - name: 'createFlow', - description: 'Create a flow', - instructions: '', - path: '/w/{workspace}/flows/create', - method: 'POST' - }, - { - name: 'deleteSchedule', - description: 'Delete a schedule', - instructions: '', - path: '/w/{workspace}/schedules/delete/{path}', - method: 'DELETE' - }, - { - name: 'getVariable', - description: 'Get a variable', - instructions: '', - path: '/w/{workspace}/variables/get/{path}', - method: 'GET' - } -] + { + name: "listWorkers", + description: "List workers", + instructions: "List all workers with their last ping and job counts.", + path: "/workers/list", + method: "GET", + query_params_schema: { + type: "object", + properties: { page: { type: "integer" }, per_page: { type: "integer" } }, + }, + }, + { + name: "listQueue", + description: "List queued jobs", + instructions: "", + path: "/w/{workspace}/jobs/queue/list", + method: "GET", + path_params_schema: { + type: "object", + properties: { workspace: { type: "string" } }, + required: ["workspace"], + }, + }, + { + name: "getJob", + description: "get job", + instructions: "", + path: "/w/{workspace}/jobs_u/get/{id}", + method: "GET", + path_params_schema: { + type: "object", + properties: { + workspace: { type: "string" }, + id: { type: "string", format: "uuid" }, + }, + required: ["workspace", "id"], + }, + query_params_schema: { + type: "object", + properties: { + no_logs: { type: "boolean" }, + no_code: { type: "boolean" }, + approval_token: { type: "string" }, + }, + required: [], + }, + }, + { + name: "runScriptByPath", + description: "Run the deployed version of a script by path", + instructions: "", + path: "/w/{workspace}/jobs/run/p/{path}", + method: "POST", + path_params_schema: { + type: "object", + properties: { workspace: { type: "string" }, path: { type: "string" } }, + required: ["workspace", "path"], + }, + body_schema: { type: "object", properties: {} }, + }, + { + name: "runFlowByPath", + description: "Run the deployed version of a flow by path", + instructions: "", + path: "/w/{workspace}/jobs/run/f/{path}", + method: "POST", + path_params_schema: { + type: "object", + properties: { workspace: { type: "string" }, path: { type: "string" } }, + required: ["workspace", "path"], + }, + body_schema: { type: "object", properties: {} }, + }, + // Draft-covered endpoints, present so steering cases exercise the guard the + // way production does (hidden from search, refused at call time). + { + name: "getScriptByPath", + description: "Get a script by path", + instructions: "", + path: "/w/{workspace}/scripts/get/p/{path}", + method: "GET", + }, + { + name: "createFlow", + description: "Create a flow", + instructions: "", + path: "/w/{workspace}/flows/create", + method: "POST", + }, + { + name: "deleteSchedule", + description: "Delete a schedule", + instructions: "", + path: "/w/{workspace}/schedules/delete/{path}", + method: "DELETE", + }, + { + name: "getVariable", + description: "Get a variable", + instructions: "", + path: "/w/{workspace}/variables/get/{path}", + method: "GET", + }, +]; export function listBenchmarkMcpTools(): EndpointTool[] { - return BENCHMARK_MCP_TOOLS + return BENCHMARK_MCP_TOOLS; } /** A stand-in Windmill hub. `search_hub_scripts` and a `hub/` read go out over @@ -867,42 +941,47 @@ export function listBenchmarkMcpTools(): EndpointTool[] { * hub tools throw and no case can exercise hub reuse. Serving fixtures rather * than the live hub also keeps assertions on script content stable as the real * hub republishes new versions. */ -const BENCHMARK_INTEGRATION_META_PATH = /^\/api\/integrations\/hub\/([^/]+)\/meta$/ +const BENCHMARK_INTEGRATION_META_PATH = + /^\/api\/integrations\/hub\/([^/]+)\/meta$/; /** One integration lifted verbatim from the content repo — its shipped scripts, its * authored meta.json and its resource type. Hand-written fixtures make both routes to * an integration's conventions look equally cheap; a real one is the only way to tell * whether reading the metadata beats reading a script. */ -const REAL_HUB_INTEGRATIONS = [confluenceFixture, servicenowFixture, outreachFixture] as unknown as Array<{ - app: string - display_name: string - description: string - docs_url: string - curated: boolean | null - meta: unknown - resource_type: { name: string; description: string; schema: unknown } - scripts: Array<{ - version_id: number - app: string - summary: string - description: string - terms: string - kind: string - language: string - content: string - schema: unknown - }> -}> +const REAL_HUB_INTEGRATIONS = [ + confluenceFixture, + servicenowFixture, + outreachFixture, +] as unknown as Array<{ + app: string; + display_name: string; + description: string; + docs_url: string; + curated: boolean | null; + meta: unknown; + resource_type: { name: string; description: string; schema: unknown }; + scripts: Array<{ + version_id: number; + app: string; + summary: string; + description: string; + terms: string; + kind: string; + language: string; + content: string; + schema: unknown; + }>; +}>; const BENCHMARK_HUB_SCRIPTS = [ - ...REAL_HUB_INTEGRATIONS.flatMap((integration) => integration.scripts), - { - version_id: 22235, - app: 'holded', - summary: 'Send Document', - terms: 'holded invoice document send email mail', - language: 'bun', - content: `//native + ...REAL_HUB_INTEGRATIONS.flatMap((integration) => integration.scripts), + { + version_id: 22235, + app: "holded", + summary: "Send Document", + terms: "holded invoice document send email mail", + language: "bun", + content: `//native type Holded = { apiKey: string; }; @@ -941,24 +1020,24 @@ export async function main( return await response.json(); } `, - schema: { - type: 'object', - required: ['auth', 'docType', 'documentId', 'body'], - properties: { - auth: { type: 'object', format: 'resource-holded' }, - docType: { type: 'string' }, - documentId: { type: 'string' }, - body: { type: 'object' } - } - } - }, - { - version_id: 28294, - app: 'discord', - summary: 'Send a message to Discord using Webhook', - terms: 'discord webhook message send chat channel', - language: 'bunnative', - content: `//native + schema: { + type: "object", + required: ["auth", "docType", "documentId", "body"], + properties: { + auth: { type: "object", format: "resource-holded" }, + docType: { type: "string" }, + documentId: { type: "string" }, + body: { type: "object" }, + }, + }, + }, + { + version_id: 28294, + app: "discord", + summary: "Send a message to Discord using Webhook", + terms: "discord webhook message send chat channel", + language: "bunnative", + content: `//native type DiscordWebhook = { webhook_url: string; @@ -975,26 +1054,26 @@ export async function main(discord_webhook: DiscordWebhook, message: string) { return await response.json(); } `, - schema: { - type: 'object', - required: ['discord_webhook', 'message'], - properties: { - discord_webhook: { type: 'object', format: 'resource-discord_webhook' }, - message: { type: 'string' } - } - } - }, - // Baremetrics carries no annotation script, on purpose: it stands for an - // integration whose conventions (an `apiKey` resource, bearer auth, the /v1 - // base) are only learnable by reading a script that does something else. - { - version_id: 8995, - app: 'baremetrics', - summary: 'List Sources', - description: 'List all the sources attached to a Baremetrics account.', - terms: 'baremetrics sources list revenue metrics', - language: 'bunnative', - content: `//native + schema: { + type: "object", + required: ["discord_webhook", "message"], + properties: { + discord_webhook: { type: "object", format: "resource-discord_webhook" }, + message: { type: "string" }, + }, + }, + }, + // Baremetrics carries no annotation script, on purpose: it stands for an + // integration whose conventions (an `apiKey` resource, bearer auth, the /v1 + // base) are only learnable by reading a script that does something else. + { + version_id: 8995, + app: "baremetrics", + summary: "List Sources", + description: "List all the sources attached to a Baremetrics account.", + terms: "baremetrics sources list revenue metrics", + language: "bunnative", + content: `//native type Baremetrics = { apiKey: string; }; @@ -1013,20 +1092,22 @@ export async function main(resource: Baremetrics) { return await response.json(); } `, - schema: { - type: 'object', - required: ['resource'], - properties: { resource: { type: 'object', format: 'resource-baremetrics' } } - } - }, - { - version_id: 8996, - app: 'baremetrics', - summary: 'Create Customer', - description: 'Create a customer on a Baremetrics source.', - terms: 'baremetrics customer create source', - language: 'bunnative', - content: `//native + schema: { + type: "object", + required: ["resource"], + properties: { + resource: { type: "object", format: "resource-baremetrics" }, + }, + }, + }, + { + version_id: 8996, + app: "baremetrics", + summary: "Create Customer", + description: "Create a customer on a Baremetrics source.", + terms: "baremetrics customer create source", + language: "bunnative", + content: `//native type Baremetrics = { apiKey: string; }; @@ -1053,17 +1134,17 @@ export async function main( return await response.json(); } `, - schema: { - type: 'object', - required: ['resource', 'sourceId', 'body'], - properties: { - resource: { type: 'object', format: 'resource-baremetrics' }, - sourceId: { type: 'string' }, - body: { type: 'object' } - } - } - } -] + schema: { + type: "object", + required: ["resource", "sourceId", "body"], + properties: { + resource: { type: "object", format: "resource-baremetrics" }, + sourceId: { type: "string" }, + body: { type: "object" }, + }, + }, + }, +]; /** Naive whole-word overlap — enough to rank a handful of fixtures for a natural * query without pulling an embedding model into the benchmark. Every frontend eval @@ -1071,52 +1152,67 @@ export async function main( * integration, or overlapping on three meaningful words. A looser bar answers * "send a Slack message" with the Discord fixture, handing an unrelated case a * plausible-looking wrong integration. */ -function searchBenchmarkHubScripts(text: string, app: string | null) { - const tokens = new Set( - text - .toLowerCase() - .split(/[^a-z0-9]+/) - .filter((token) => token.length > 2) - ) - // The real endpoint filters by app before ranking, so honour it here too — - // otherwise a narrowed search silently returns other integrations' scripts. - return BENCHMARK_HUB_SCRIPTS.filter((script) => !app || script.app === app) - .map((script) => { - const words = new Set( - `${script.app} ${script.summary} ${script.terms}`.toLowerCase().split(/[^a-z0-9]+/) - ) - const score = [...tokens].filter((token) => words.has(token)).length - return { script, score, namesApp: tokens.has(script.app) } - }) - .filter((entry) => entry.namesApp || entry.score >= 3) - .sort((a, b) => b.score - a.score) - .map(({ script }, index) => ({ - ask_id: script.version_id, - id: script.version_id, - version_id: script.version_id, - summary: script.summary, - description: script.description ?? '', - app: script.app, - kind: 'script', - score: 1 - index * 0.01 - })) +function searchBenchmarkHubScripts( + text: string, + app: string | null, + kind: string | null, +) { + const tokens = new Set( + text + .toLowerCase() + .split(/[^a-z0-9]+/) + .filter((token) => token.length > 2), + ); + // The real endpoint filters by app and kind before ranking, so honour both here — + // otherwise a narrowed search silently returns other integrations' scripts, and a + // trigger answers a search production would never have shown it to. + return BENCHMARK_HUB_SCRIPTS.filter( + (script) => + (!app || script.app === app) && + (!kind || (script.kind ?? "script") === kind), + ) + .map((script) => { + const words = new Set( + `${script.app} ${script.summary} ${script.terms}` + .toLowerCase() + .split(/[^a-z0-9]+/), + ); + const score = [...tokens].filter((token) => words.has(token)).length; + return { script, score, namesApp: tokens.has(script.app) }; + }) + .filter((entry) => entry.namesApp || entry.score >= 3) + .sort((a, b) => b.score - a.score) + .map(({ script }, index) => ({ + ask_id: script.version_id, + id: script.version_id, + version_id: script.version_id, + summary: script.summary, + description: script.description ?? "", + app: script.app, + kind: script.kind ?? "script", + score: 1 - index * 0.01, + })); } /** Listing an integration is unranked and description-bearing, matching the hub's * top-scripts endpoint — that asymmetry with the semantic search is the whole * reason the chat browses by app when no script matches the task. */ -function listBenchmarkHubScriptsByApp(app: string | null) { - return BENCHMARK_HUB_SCRIPTS.filter((script) => !app || script.app === app).map((script) => ({ - id: script.version_id, - ask_id: script.version_id, - version_id: script.version_id, - summary: script.summary, - description: script.description ?? '', - app: script.app, - kind: 'script', - views: 0, - votes: 0 - })) +function listBenchmarkHubScriptsByApp(app: string | null, kind: string | null) { + return BENCHMARK_HUB_SCRIPTS.filter( + (script) => + (!app || script.app === app) && + (!kind || (script.kind ?? "script") === kind), + ).map((script) => ({ + id: script.version_id, + ask_id: script.version_id, + version_id: script.version_id, + summary: script.summary, + description: script.description ?? "", + app: script.app, + kind: script.kind ?? "script", + views: 0, + votes: 0, + })); } /** What `/integrations/hub//meta` serves: the provider knowledge the content @@ -1125,252 +1221,302 @@ function listBenchmarkHubScriptsByApp(app: string | null) { * write, so the tool shortens the path to that knowledge without supplying answers. * `derived` is computed from the fixture scripts so it cannot drift from them. */ const BENCHMARK_HUB_INTEGRATION_META: Record< - string, - { display_name: string; description: string; docs_url: string; curated: boolean | null; meta: unknown } + string, + { + display_name: string; + description: string; + docs_url: string; + curated: boolean | null; + meta: unknown; + } > = { - baremetrics: { - display_name: 'Baremetrics', - description: 'Subscription analytics for recurring-revenue businesses.', - docs_url: 'https://developers.baremetrics.com/reference', - curated: null, - meta: { - api_docs: 'https://developers.baremetrics.com/reference', - auth: 'Bearer token. The resource carries a single `apiKey` field; send it as `Authorization: Bearer `.', - base_url: 'https://api.baremetrics.com/v1', - pagination: { pattern: 'page', request_params: { page: 'page', per_page: 'per_page' } }, - gotchas: [ - 'Every write is scoped to a source, so the source id is part of the path rather than the body.', - 'The API answers 200 with an empty `{}` body on some writes; treat a 2xx as success rather than parsing a payload.' - ], - errors: { '401': 'The apiKey is missing or revoked.', '404': 'Unknown source id, or the key cannot see that source.' } - } - }, - holded: { - display_name: 'Holded', - description: 'Invoicing, accounting and CRM for small businesses.', - docs_url: 'https://developers.holded.com/reference', - curated: null, - meta: { - api_docs: 'https://developers.holded.com/reference', - auth: 'The resource carries an `apiKey`; send it in the `key` header, not as a bearer token.', - base_url: 'https://api.holded.com/api', - gotchas: ['Each product area has its own path segment (invoicing, crm, projects) after the version.'] - } - } -} + baremetrics: { + display_name: "Baremetrics", + description: "Subscription analytics for recurring-revenue businesses.", + docs_url: "https://developers.baremetrics.com/reference", + curated: null, + meta: { + api_docs: "https://developers.baremetrics.com/reference", + auth: "Bearer token. The resource carries a single `apiKey` field; send it as `Authorization: Bearer `.", + base_url: "https://api.baremetrics.com/v1", + pagination: { + pattern: "page", + request_params: { page: "page", per_page: "per_page" }, + }, + gotchas: [ + "Every write is scoped to a source, so the source id is part of the path rather than the body.", + "The API answers 200 with an empty `{}` body on some writes; treat a 2xx as success rather than parsing a payload.", + ], + errors: { + "401": "The apiKey is missing or revoked.", + "404": "Unknown source id, or the key cannot see that source.", + }, + }, + }, + holded: { + display_name: "Holded", + description: "Invoicing, accounting and CRM for small businesses.", + docs_url: "https://developers.holded.com/reference", + curated: null, + meta: { + api_docs: "https://developers.holded.com/reference", + auth: "The resource carries an `apiKey`; send it in the `key` header, not as a bearer token.", + base_url: "https://api.holded.com/api", + gotchas: [ + "Each product area has its own path segment (invoicing, crm, projects) after the version.", + ], + }, + }, +}; /** Mirrors the hub's own derivation: hosts seen in the shipped scripts, whether they * call the provider directly, and how many there are of each kind. */ function benchmarkDerivedFacts(app: string) { - const scripts = BENCHMARK_HUB_SCRIPTS.filter((script) => script.app === app) - const hosts = new Map() - const languages: Record = {} - for (const script of scripts) { - languages[script.language] = (languages[script.language] ?? 0) + 1 - for (const match of script.content.matchAll(/https?:\/\/([a-zA-Z0-9._-]+)/g)) { - hosts.set(match[1], (hosts.get(match[1]) ?? 0) + 1) - } - } - return { - api_hosts: [...hosts.entries()] - .sort((a, b) => b[1] - a[1]) - .map(([host, count]) => ({ host, count })), - style: 'fetch', - languages, - script_counts: { total: scripts.length, by_kind: { script: scripts.length } }, - top_scripts: scripts.map((script) => ({ - path: `hub/${script.version_id}/${script.app}/${script.summary.toLowerCase().replaceAll(/\s+/g, '_')}`, - ask_id: script.version_id, - version_id: script.version_id, - summary: script.summary, - description: script.description ?? null, - kind: 'script', - language: script.language, - views: 0, - votes: 0 - })) - } + const scripts = BENCHMARK_HUB_SCRIPTS.filter((script) => script.app === app); + const hosts = new Map(); + const languages: Record = {}; + for (const script of scripts) { + languages[script.language] = (languages[script.language] ?? 0) + 1; + for (const match of script.content.matchAll( + /https?:\/\/([a-zA-Z0-9._-]+)/g, + )) { + hosts.set(match[1], (hosts.get(match[1]) ?? 0) + 1); + } + } + return { + api_hosts: [...hosts.entries()] + .sort((a, b) => b[1] - a[1]) + .map(([host, count]) => ({ host, count })), + style: "fetch", + languages, + script_counts: { + total: scripts.length, + by_kind: scripts.reduce>((acc, script) => { + acc[script.kind] = (acc[script.kind] ?? 0) + 1; + return acc; + }, {}), + }, + top_scripts: scripts.map((script) => ({ + path: `hub/${script.version_id}/${script.app}/${script.summary.toLowerCase().replaceAll(/\s+/g, "_")}`, + ask_id: script.version_id, + version_id: script.version_id, + summary: script.summary, + description: script.description ?? null, + kind: script.kind, + language: script.language, + views: 0, + votes: 0, + })), + }; } /** The hub keys a script by its version id; the app and slug segments that * follow are descriptive, so match on the id exactly as the real hub does. */ function getBenchmarkHubScript(path: string) { - const versionId = Number(path.replace(/^\/api\/scripts\/hub\/get_full\/hub\//, '').split('/')[0]) - return BENCHMARK_HUB_SCRIPTS.find((script) => script.version_id === versionId) + const versionId = Number( + path.replace(/^\/api\/scripts\/hub\/get_full\/hub\//, "").split("/")[0], + ); + return BENCHMARK_HUB_SCRIPTS.find( + (script) => script.version_id === versionId, + ); } const BENCHMARK_WORKERS = [ - { - worker: 'wk-benchmark-1', - worker_instance: 'benchmark-host', - last_ping: 2, - started_at: BENCHMARK_TIMESTAMP, - jobs_executed: 42, - custom_tags: null, - worker_group: 'default', - wm_version: 'benchmark' - }, - { - worker: 'wk-benchmark-2', - worker_instance: 'benchmark-host', - last_ping: 5, - started_at: BENCHMARK_TIMESTAMP, - jobs_executed: 17, - custom_tags: null, - worker_group: 'default', - wm_version: 'benchmark' - } -] + { + worker: "wk-benchmark-1", + worker_instance: "benchmark-host", + last_ping: 2, + started_at: BENCHMARK_TIMESTAMP, + jobs_executed: 42, + custom_tags: null, + worker_group: "default", + wm_version: "benchmark", + }, + { + worker: "wk-benchmark-2", + worker_instance: "benchmark-host", + last_ping: 5, + started_at: BENCHMARK_TIMESTAMP, + jobs_executed: 17, + custom_tags: null, + worker_group: "default", + wm_version: "benchmark", + }, +]; -const BENCHMARK_JOB_GET_PATH = /^\/api\/w\/([^/]+)\/jobs_u\/get\/([^/]+)$/ -const BENCHMARK_RUN_BY_PATH = /^\/api\/w\/([^/]+)\/jobs\/run\/(p|f)\/([^/]+)$/ +const BENCHMARK_JOB_GET_PATH = /^\/api\/w\/([^/]+)\/jobs_u\/get\/([^/]+)$/; +const BENCHMARK_RUN_BY_PATH = /^\/api\/w\/([^/]+)\/jobs\/run\/(p|f)\/([^/]+)$/; /** `executeEndpoint` sends a JSON string; anything else means no args were supplied. */ function parseBenchmarkRequestBody( - body: BodyInit | null | undefined + body: BodyInit | null | undefined, ): Record | undefined { - if (typeof body !== 'string') { - return undefined - } - try { - const parsed = JSON.parse(body) - return typeof parsed === 'object' && parsed !== null - ? (parsed as Record) - : undefined - } catch { - return undefined - } + if (typeof body !== "string") { + return undefined; + } + try { + const parsed = JSON.parse(body); + return typeof parsed === "object" && parsed !== null + ? (parsed as Record) + : undefined; + } catch { + return undefined; + } } /** True when `handleBenchmarkApiFetch` has an answer for this `/api/...` url. * Any other relative fetch must keep its normal (non-benchmark) behavior — * intercepting it with a synthetic 404 sends the model into retry loops. */ export function hasBenchmarkApiHandler(url: string): boolean { - const path = url.split('?')[0] - return ( - path === '/api/workers/list' || - BENCHMARK_JOB_GET_PATH.test(path) || - BENCHMARK_RUN_BY_PATH.test(path) || - /^\/api\/w\/[^/]+\/jobs\/queue\/list$/.test(path) || - path === '/api/embeddings/query_hub_scripts' || - path === '/api/scripts/hub/top' || - path === '/api/integrations/hub/list' || - BENCHMARK_INTEGRATION_META_PATH.test(path) || - path.startsWith('/api/scripts/hub/get_full/') - ) + const path = url.split("?")[0]; + return ( + path === "/api/workers/list" || + BENCHMARK_JOB_GET_PATH.test(path) || + BENCHMARK_RUN_BY_PATH.test(path) || + /^\/api\/w\/[^/]+\/jobs\/queue\/list$/.test(path) || + path === "/api/embeddings/query_hub_scripts" || + path === "/api/scripts/hub/top" || + path === "/api/integrations/hub/list" || + BENCHMARK_INTEGRATION_META_PATH.test(path) || + path.startsWith("/api/scripts/hub/get_full/") + ); } /** Answer a relative `/api/...` fetch — from the API catalog executor, or from the * chat's hub tools. */ -export function handleBenchmarkApiFetch(url: string, init?: RequestInit): Response { - const path = url.split('?')[0] - if (path === '/api/workers/list') { - return Response.json(BENCHMARK_WORKERS) - } - if (/^\/api\/w\/[^/]+\/jobs\/queue\/list$/.test(path)) { - return Response.json([]) - } - const jobGet = BENCHMARK_JOB_GET_PATH.exec(path) - if (jobGet) { - const id = decodeURIComponent(jobGet[2]) - const job = getBenchmarkCompletedJob(decodeURIComponent(jobGet[1]), id) - if (!job) { - return Response.json({ error: `Job not found for "${id}"` }, { status: 404 }) - } - // The real endpoint lets a caller drop the bulky fields. Ignoring that here would - // size the model's context off a payload it explicitly asked to shrink. - const query = new URLSearchParams(url.split('?')[1] ?? '') - if (query.get('no_logs') === 'true') { - delete job.logs - } - if (query.get('no_code') === 'true') { - delete job.raw_code - } - return Response.json(job) - } - const runByPath = BENCHMARK_RUN_BY_PATH.exec(path) - if (runByPath) { - const workspace = decodeURIComponent(runByPath[1]) - const runnablePath = decodeURIComponent(runByPath[3]) - const args = parseBenchmarkRequestBody(init?.body) - // The real endpoint answers with the bare job id as text, not JSON. - return new Response( - runByPath[2] === 'f' - ? runBenchmarkFlowByPath({ workspace, path: runnablePath, args }) - : runBenchmarkScriptByPath({ workspace, path: runnablePath, args }) - ) - } - if (path === '/api/embeddings/query_hub_scripts') { - const text = new URLSearchParams(url.split('?')[1] ?? '').get('text') ?? '' - return Response.json( - searchBenchmarkHubScripts(text, new URLSearchParams(url.split('?')[1] ?? '').get('app')) - ) - } - if (path === '/api/scripts/hub/top') { - const app = new URLSearchParams(url.split('?')[1] ?? '').get('app') - return Response.json({ asks: listBenchmarkHubScriptsByApp(app) }) - } - const integrationMeta = BENCHMARK_INTEGRATION_META_PATH.exec(path) - if (integrationMeta) { - const app = decodeURIComponent(integrationMeta[1]) - const real = REAL_HUB_INTEGRATIONS.find((integration) => integration.app === app) - if (real) { - return Response.json({ - app, - display_name: real.display_name, - description: real.description, - docs_url: real.docs_url, - curated: real.curated, - metadata_source: real.meta ? 'curated' : 'derived', - meta: real.meta, - meta_updated_at: null, - derived: benchmarkDerivedFacts(app), - resource_types: [{ id: 1, ...real.resource_type }] - }) - } - const entry = BENCHMARK_HUB_INTEGRATION_META[app] - if (!entry) { - return Response.json({ error: 'integration not found' }, { status: 404 }) - } - return Response.json({ - app, - display_name: entry.display_name, - description: entry.description, - docs_url: entry.docs_url, - curated: entry.curated, - metadata_source: entry.meta ? 'curated' : 'derived', - meta: entry.meta, - meta_updated_at: null, - derived: benchmarkDerivedFacts(app), - resource_types: [ - { - id: 1, - name: app, - description: `${entry.display_name} credentials`, - schema: { - type: 'object', - required: ['apiKey'], - properties: { apiKey: { type: 'string', description: `${entry.display_name} API key` } } - } - } - ] - }) - } - if (path === '/api/integrations/hub/list') { - const apps = [...new Set(BENCHMARK_HUB_SCRIPTS.map((script) => script.app))].sort() - return Response.json(apps.map((name) => ({ name }))) - } - if (path.startsWith('/api/scripts/hub/get_full/')) { - const script = getBenchmarkHubScript(path) - if (!script) { - return Response.json({ error: 'hub script not found' }, { status: 404 }) - } - return Response.json({ - content: script.content, - language: script.language, - schema: script.schema, - summary: script.summary - }) - } - return Response.json({ error: `no benchmark handler for ${path}` }, { status: 404 }) +export function handleBenchmarkApiFetch( + url: string, + init?: RequestInit, +): Response { + const path = url.split("?")[0]; + if (path === "/api/workers/list") { + return Response.json(BENCHMARK_WORKERS); + } + if (/^\/api\/w\/[^/]+\/jobs\/queue\/list$/.test(path)) { + return Response.json([]); + } + const jobGet = BENCHMARK_JOB_GET_PATH.exec(path); + if (jobGet) { + const id = decodeURIComponent(jobGet[2]); + const job = getBenchmarkCompletedJob(decodeURIComponent(jobGet[1]), id); + if (!job) { + return Response.json( + { error: `Job not found for "${id}"` }, + { status: 404 }, + ); + } + // The real endpoint lets a caller drop the bulky fields. Ignoring that here would + // size the model's context off a payload it explicitly asked to shrink. + const query = new URLSearchParams(url.split("?")[1] ?? ""); + if (query.get("no_logs") === "true") { + delete job.logs; + } + if (query.get("no_code") === "true") { + delete job.raw_code; + } + return Response.json(job); + } + const runByPath = BENCHMARK_RUN_BY_PATH.exec(path); + if (runByPath) { + const workspace = decodeURIComponent(runByPath[1]); + const runnablePath = decodeURIComponent(runByPath[3]); + const args = parseBenchmarkRequestBody(init?.body); + // The real endpoint answers with the bare job id as text, not JSON. + return new Response( + runByPath[2] === "f" + ? runBenchmarkFlowByPath({ workspace, path: runnablePath, args }) + : runBenchmarkScriptByPath({ workspace, path: runnablePath, args }), + ); + } + if (path === "/api/embeddings/query_hub_scripts") { + const params = new URLSearchParams(url.split("?")[1] ?? ""); + return Response.json( + searchBenchmarkHubScripts( + params.get("text") ?? "", + params.get("app"), + params.get("kind"), + ), + ); + } + if (path === "/api/scripts/hub/top") { + const params = new URLSearchParams(url.split("?")[1] ?? ""); + return Response.json({ + asks: listBenchmarkHubScriptsByApp(params.get("app"), params.get("kind")), + }); + } + const integrationMeta = BENCHMARK_INTEGRATION_META_PATH.exec(path); + if (integrationMeta) { + const app = decodeURIComponent(integrationMeta[1]); + const real = REAL_HUB_INTEGRATIONS.find( + (integration) => integration.app === app, + ); + if (real) { + return Response.json({ + app, + display_name: real.display_name, + description: real.description, + docs_url: real.docs_url, + curated: real.curated, + metadata_source: real.meta ? "curated" : "derived", + meta: real.meta, + meta_updated_at: null, + derived: benchmarkDerivedFacts(app), + resource_types: [{ id: 1, ...real.resource_type }], + }); + } + const entry = BENCHMARK_HUB_INTEGRATION_META[app]; + if (!entry) { + return Response.json({ error: "integration not found" }, { status: 404 }); + } + return Response.json({ + app, + display_name: entry.display_name, + description: entry.description, + docs_url: entry.docs_url, + curated: entry.curated, + metadata_source: entry.meta ? "curated" : "derived", + meta: entry.meta, + meta_updated_at: null, + derived: benchmarkDerivedFacts(app), + resource_types: [ + { + id: 1, + name: app, + description: `${entry.display_name} credentials`, + schema: { + type: "object", + required: ["apiKey"], + properties: { + apiKey: { + type: "string", + description: `${entry.display_name} API key`, + }, + }, + }, + }, + ], + }); + } + if (path === "/api/integrations/hub/list") { + const apps = [ + ...new Set(BENCHMARK_HUB_SCRIPTS.map((script) => script.app)), + ].sort(); + return Response.json(apps.map((name) => ({ name }))); + } + if (path.startsWith("/api/scripts/hub/get_full/")) { + const script = getBenchmarkHubScript(path); + if (!script) { + return Response.json({ error: "hub script not found" }, { status: 404 }); + } + return Response.json({ + content: script.content, + language: script.language, + schema: script.schema, + summary: script.summary, + }); + } + return Response.json( + { error: `no benchmark handler for ${path}` }, + { status: 404 }, + ); } diff --git a/ai_evals/fixtures/frontend/global/hub/confluence.json b/ai_evals/fixtures/frontend/global/hub/confluence.json index 0968d824ad..a414fd4dba 100644 --- a/ai_evals/fixtures/frontend/global/hub/confluence.json +++ b/ai_evals/fixtures/frontend/global/hub/confluence.json @@ -15,7 +15,7 @@ ], "validation": { "status": "live_validated", - "method": "smoke test via `bun --env-file=.env.test -e` (+ `bun test` with the windmill-client fake for the trigger) against a free Confluence Cloud site (edwind-wm-confluence.atlassian.net) with an API token. All 13 actions + the trigger returned 2xx: full page lifecycle (create -> get -> update -> list -> delete), blog post create/update/delete, list_spaces, search_content (CQL), get_current_user, and the new_or_updated_page polling trigger.", + "method": "smoke test via `bun --env-file=.env.test -e` (+ `bun test` with the windmill-client fake for the trigger) against a free Confluence Cloud site (an Atlassian Cloud site) with an API token. All 13 actions + the trigger returned 2xx: full page lifecycle (create -> get -> update -> list -> delete), blog post create/update/delete, list_spaces, search_content (CQL), get_current_user, and the new_or_updated_page polling trigger.", "sources": { "pages": "HIGH \u2014 Pipedream actions + official v2 OpenAPI; create/get/update/list/delete all run live (2xx)", "blogposts": "HIGH \u2014 Pipedream actions + official v2 OpenAPI; create/update/delete all run live (2xx)", @@ -101,7 +101,7 @@ "type": "string" }, "baseUrl": { - "description": "Base URL of your Confluence Cloud site, e.g. https://your-domain.atlassian.net (no trailing slash, no /wiki).", + "description": "Base URL of your Confluence Cloud site, e.g. https://an Atlassian Cloud site (no trailing slash, no /wiki).", "type": "string" }, "email": { diff --git a/ai_evals/fixtures/frontend/global/hub/servicenow.json b/ai_evals/fixtures/frontend/global/hub/servicenow.json index 14d6f59a3d..991f46a1f6 100644 --- a/ai_evals/fixtures/frontend/global/hub/servicenow.json +++ b/ai_evals/fixtures/frontend/global/hub/servicenow.json @@ -15,7 +15,7 @@ ], "validation": { "status": "live_validated", - "method": "All surfaces run live against a ServiceNow PDI (dev209312, Zurich) on 2026-06-04 with Basic auth: Table CRUD (create / get / update PATCH+PUT / delete on incident, delete verified by 404), list_records (encoded query + fields + pagination), aggregate (Stats group-by count), get_current_user, list_tables, and the table/fields/group_by dynselect resolvers. Attachment upload (binary /file) -> list -> download -> delete round-trip with the downloaded bytes byte-for-byte matching the upload. insert_import_set + get_import_set_result against imp_user (transform status surfaced). Trigger e2e-tested via bun test with the windmill-client fake. All created records cleaned up.", + "method": "All surfaces run live against a ServiceNow PDI (a developer instance, Zurich) on 2026-06-04 with Basic auth: Table CRUD (create / get / update PATCH+PUT / delete on incident, delete verified by 404), list_records (encoded query + fields + pagination), aggregate (Stats group-by count), get_current_user, list_tables, and the table/fields/group_by dynselect resolvers. Attachment upload (binary /file) -> list -> download -> delete round-trip with the downloaded bytes byte-for-byte matching the upload. insert_import_set + get_import_set_result against imp_user (transform status surfaced). Trigger e2e-tested via bun test with the windmill-client fake. All created records cleaned up.", "sources": { "table_crud": "HIGH - run live (create/get/update PATCH & PUT/delete on incident)", "aggregate_stats": "HIGH - run live (/api/now/stats group-by count on incident)", @@ -95,8 +95,8 @@ "properties": { "instance_url": { "default": "", - "description": "Instance base URL, e.g. https://dev12345.service-now.com (no trailing slash). Every REST call is made against this host.", - "placeholder": "https://dev12345.service-now.com", + "description": "Instance base URL, e.g. https://a developer a ServiceNow instance (no trailing slash). Every REST call is made against this host.", + "placeholder": "https://a developer a ServiceNow instance", "type": "string" }, "password": { diff --git a/backend/windmill-api-embeddings/src/lib.rs b/backend/windmill-api-embeddings/src/lib.rs index fc0a51bafd..fa42be982a 100644 --- a/backend/windmill-api-embeddings/src/lib.rs +++ b/backend/windmill-api-embeddings/src/lib.rs @@ -720,6 +720,31 @@ pub fn global_service() -> Router { mod tests { use super::trim_to_top_score; + // The blob carries an explicit `"description": null` for roughly a fifth of hub + // scripts, and an older hub omits the key entirely. A bare String here fails the + // whole 155 MB array and takes hub search down with it. + #[test] + fn reads_a_hub_script_whether_or_not_it_has_a_description() { + let present = r#"{"ask_id":1,"id":2,"version_id":3,"summary":"s","description":"d","app":"a","kind":"script","embedding":[]}"#; + let null = r#"{"ask_id":1,"id":2,"version_id":3,"summary":"s","description":null,"app":"a","kind":"script","embedding":[]}"#; + let missing = r#"{"ask_id":1,"id":2,"version_id":3,"summary":"s","app":"a","kind":"script","embedding":[]}"#; + + assert_eq!( + serde_json::from_str::(present) + .unwrap() + .description, + Some("d".to_string()) + ); + for without in [null, missing] { + assert_eq!( + serde_json::from_str::(without) + .unwrap() + .description, + None + ); + } + } + #[test] fn trims_scores_more_than_5pct_below_top() { // top=1.0, cutoff at 0.95: 0.96 stays (0.04 drop), 0.93 is the first diff --git a/backend/windmill-api/openapi.yaml b/backend/windmill-api/openapi.yaml index 92ab9f0187..50b411a9e5 100644 --- a/backend/windmill-api/openapi.yaml +++ b/backend/windmill-api/openapi.yaml @@ -8819,6 +8819,7 @@ paths: type: string description: type: string + nullable: true app: type: string version_id: diff --git a/backend/windmill-api/src/integration.rs b/backend/windmill-api/src/integration.rs index 8251e8016c..10bd1fc24c 100644 --- a/backend/windmill-api/src/integration.rs +++ b/backend/windmill-api/src/integration.rs @@ -37,6 +37,17 @@ async fn list_hub_integrations( Ok::<_, Error>((status_code, headers, response)) } +/// Axum percent-decodes a path parameter, so an interpolated slug carrying `..`, `?` +/// or `#` re-targets the proxied GET at another path on the hub origin — with the +/// instance's hub credentials attached. Slugs are `[A-Za-z0-9_-]`; reject the rest. +fn is_hub_integration_slug(app: &str) -> bool { + !app.is_empty() + && app.len() <= 64 + && app + .chars() + .all(|c| c.is_ascii_alphanumeric() || c == '_' || c == '-') +} + /// Everything a caller needs to write code against one integration: its resource /// types, the provider knowledge the content repo authored, and facts derived from /// the shipped scripts. A hub older than the endpoint answers 404, which passes @@ -45,6 +56,11 @@ async fn get_hub_integration_meta( Path(app): Path, Extension(db): Extension, ) -> impl IntoResponse { + if !is_hub_integration_slug(&app) { + return Err(Error::BadRequest(format!( + "Not a valid integration name: {app}" + ))); + } let (status_code, headers, response) = query_elems_from_hub( &HTTP_CLIENT, &format!("{}/integrations/{}/meta", **HUB_BASE_URL.load(), app), @@ -54,3 +70,26 @@ async fn get_hub_integration_meta( .await?; Ok::<_, Error>((status_code, headers, response)) } + +#[cfg(test)] +mod tests { + use super::is_hub_integration_slug; + + #[test] + fn rejects_slugs_that_would_re_target_the_proxied_request() { + assert!(is_hub_integration_slug("confluence")); + assert!(is_hub_integration_slug("aws-ses")); + assert!(is_hub_integration_slug("bamboo_hr")); + assert!(is_hub_integration_slug("RSS")); + + for escape in [ + "../../scripts/top", + "confluence?foo=bar", + "confluence#frag", + "confluence/meta", + "", + ] { + assert!(!is_hub_integration_slug(escape), "accepted {escape}"); + } + } +} diff --git a/frontend/src/lib/components/copilot/chat/chatLoop.ts b/frontend/src/lib/components/copilot/chat/chatLoop.ts index df1789dc81..ad282cc1e9 100644 --- a/frontend/src/lib/components/copilot/chat/chatLoop.ts +++ b/frontend/src/lib/components/copilot/chat/chatLoop.ts @@ -70,10 +70,11 @@ export interface ChatLoopConfig { * lets the caller recover partial output if the loop throws or is aborted. */ addedMessages?: ChatCompletionMessageParam[] - /** Called before each request (e.g. to refresh tool schemas, or to record which - * model it is about to use), including again mid-iteration when a fallback - * changes `webSearch`. That argument is the effective value, and the system - * message is read after this returns, so a caller can resync its prompt in time. */ + /** Called before each iteration (e.g. to refresh tool schemas, or to record which + * model it is about to use), and again when the Completions fallback drops + * `webSearch`. That argument is the effective value, and the system message is read + * after this returns, so a caller can resync its prompt in time. A same-iteration + * retry does not re-enter this; `onWebSearchUnavailable` covers that path. */ onBeforeIteration?: ( tools: Tool[], helpers: any, diff --git a/frontend/src/lib/components/copilot/chat/shared.test.ts b/frontend/src/lib/components/copilot/chat/shared.test.ts index d1b81c66bb..291f8633ab 100644 --- a/frontend/src/lib/components/copilot/chat/shared.test.ts +++ b/frontend/src/lib/components/copilot/chat/shared.test.ts @@ -1671,13 +1671,38 @@ describe('getHubIntegrationTool', () => { expect(!!parsed.scripts_note).toBe(expected) }) + // A hub that times out has said nothing about whether the integration exists, and + // reporting it as absent would stick for the rest of the conversation. + it('does not report a transient hub failure as a missing integration', async () => { + const { IntegrationService } = await import('$lib/gen') + Object.assign(IntegrationService, { + getHubIntegrationMeta: vi.fn(async () => { + throw Object.assign(new Error('Service Unavailable'), { status: 503 }) + }), + listHubIntegrations: vi.fn(async () => [{ name: 'confluence' }]) + }) + + const { getHubIntegrationTool, clearHubIntegrationsCache } = await import('./shared') + clearHubIntegrationsCache() + const parsed = JSON.parse( + await getHubIntegrationTool.fn({ + args: { integration: 'confluence' }, + toolId: 't1', + toolCallbacks: { setToolStatus: vi.fn() } + } as any) + ) + + expect(parsed.error).toContain('Could not reach the hub') + expect(parsed.error).not.toContain('No hub metadata') + }) + // A hub with no such integration and one too old to serve the endpoint both 404; // neither may surface as a tool error, since the model can still read scripts. it('suggests real slugs instead of failing when the integration is unknown', async () => { const { IntegrationService } = await import('$lib/gen') Object.assign(IntegrationService, { getHubIntegrationMeta: vi.fn(async () => { - throw new Error('Not Found') + throw Object.assign(new Error('Not Found'), { status: 404 }) }), listHubIntegrations: vi.fn(async () => [{ name: 'stripe' }, { name: 'slack' }]) }) diff --git a/frontend/src/lib/components/copilot/chat/shared.ts b/frontend/src/lib/components/copilot/chat/shared.ts index fdd812caa8..8d4927e01f 100644 --- a/frontend/src/lib/components/copilot/chat/shared.ts +++ b/frontend/src/lib/components/copilot/chat/shared.ts @@ -1247,9 +1247,15 @@ export function isHubPath(path: string): boolean { const MAX_BROWSED_HUB_SCRIPTS = 20 const MAX_SUGGESTED_INTEGRATIONS = 5 -/** Common shape of the two hub listings. Both carry a description, but the hub - * has none for roughly a fifth of its scripts. */ -type HubScriptHit = { version_id: number; app: string; summary: string; description?: string } +/** Common shape of the two hub listings. Both carry a description, but the hub has + * none for roughly a fifth of its scripts, and says so as a null rather than by + * omitting the key. */ +type HubScriptHit = { + version_id: number + app: string + summary: string + description?: string | null +} /** The integration slugs are a large but static list, so one fetch per session * is enough. Only matched slugs ever reach the model, never the whole list. */ @@ -1281,12 +1287,10 @@ async function suggestHubIntegrations(query: string): Promise { .slice(0, MAX_SUGGESTED_INTEGRATIONS) } -/** Matches a query word against a slug on word boundaries rather than by bare - * substring. A substring test reads every three-letter English word as a hit — - * `for` in sales*for*ce, `the` in basis_*the*ory — so a request that names no - * integration still came back with five confident-looking ones. Short tokens must - * equal a slug or one of its parts, which is also what reaches the two-character - * slugs (`s3`, `wiz`) that a length filter alone hides. */ +/** Matches a query word against a slug on word boundaries. A bare substring test + * makes every three-letter English word a hit — `for` in sales*for*ce, `the` in + * basis_*the*ory. Short tokens must equal a slug or a part, which is also what + * reaches the two-character slugs (`s3`, `wiz`) a length floor would hide. */ function tokenMatchesSlug(token: string, slug: string): boolean { const parts = slug.split(/[_-]/).filter(Boolean) if (slug === token || parts.includes(token)) { @@ -1307,12 +1311,9 @@ function tokenMatchesSlug(token: string, slug: string): boolean { } /** The slug of an integration the query names outright, when it names exactly one. - * Semantic search ranks on the whole of a script's text, so a named vendor is weak - * signal against the task words: "create a jira ticket" ranks netlify, zendesk and - * intercom above every Jira script, and "look up an account in salesforce" puts - * Pinterest first, because its summaries say Salesforce. Narrowing to the named - * integration fixes both. The test is deliberately exact — a fuzzy one reads `send` - * in "send an invoice" as sendgrid and quietly searches the wrong integration. */ + * Semantic search ranks a named vendor weakly against the task words — "create a jira + * ticket" puts netlify and zendesk above every Jira script — so narrowing to it wins. + * Matching must stay exact: fuzzily, `send` in "send an invoice" is sendgrid. */ async function integrationNamedIn(query: string): Promise { const list = await loadHubIntegrations() const words = new Set( @@ -1373,14 +1374,20 @@ export const getHubIntegrationTool = { let doc: Awaited> try { doc = await IntegrationService.getHubIntegrationMeta({ app: integration }) - } catch { - // An unknown slug and a hub predating the endpoint both answer 404, and the - // response is the same either way: hand back real slugs so the model can - // retry or fall back to reading scripts. - toolCallbacks.setToolStatus(toolId, { content: `No hub integration named ${integration}` }) + } catch (err) { + // Only a 404 means the integration is absent — an unknown slug, or a hub + // predating the endpoint. Reporting a timeout the same way would teach the + // model that a real integration does not exist for the rest of the chat. + const absent = (err as { status?: number } | undefined)?.status === 404 + const label = absent + ? `No hub integration named ${integration}` + : `Could not reach the hub for ${integration}` + toolCallbacks.setToolStatus(toolId, { content: label }) const suggested = await suggestHubIntegrations(integration) return JSON.stringify({ - error: `No hub metadata for "${integration}".`, + error: absent + ? `No hub metadata for "${integration}".` + : `Could not reach the hub for "${integration}"; it may still exist. Read its scripts instead, or try again.`, suggested_integrations: suggested }) }