From 03d245993285166692a0cd32006c9f293a76e63a Mon Sep 17 00:00:00 2001 From: hugocasa Date: Thu, 8 Oct 2026 10:43:45 +0200 Subject: [PATCH] fix: report a failed resource type search and model a synced instance in evals Co-Authored-By: Claude Opus 5.5 --- ai_evals/adapters/frontend/mockBackend.ts | 133 +-- .../adapters/frontend/vitestAdapter.test.ts | 7 +- .../global/hub/synced_resource_types.json | 893 ++++++++++++++++++ .../components/copilot/chat/global/core.ts | 8 +- .../copilot/chat/script/core.test.ts | 23 +- .../components/copilot/chat/script/core.ts | 24 +- 6 files changed, 1006 insertions(+), 82 deletions(-) create mode 100644 ai_evals/fixtures/frontend/global/hub/synced_resource_types.json diff --git a/ai_evals/adapters/frontend/mockBackend.ts b/ai_evals/adapters/frontend/mockBackend.ts index 2e651e4631..f26f73e1f6 100644 --- a/ai_evals/adapters/frontend/mockBackend.ts +++ b/ai_evals/adapters/frontend/mockBackend.ts @@ -2,6 +2,7 @@ import { randomUUID } from 'node:crypto' import confluenceFixture from '../../fixtures/frontend/global/hub/confluence.json' import servicenowFixture from '../../fixtures/frontend/global/hub/servicenow.json' import outreachFixture from '../../fixtures/frontend/global/hub/outreach.json' +import syncedResourceTypesFixture from '../../fixtures/frontend/global/hub/synced_resource_types.json' import type { AppWithLastVersion, CompletedJob, @@ -99,51 +100,13 @@ export interface BenchmarkWorkspaceResource { description?: string } -/** A resource type the instance holds. Only a seeded one has a schema to look up by name. */ +/** A resource type the instance holds. Without a schema, a lookup by name misses it. */ export interface BenchmarkWorkspaceResourceType { name: string schema?: unknown description?: string } -/** Names a hub-synced instance holds, for cases that seed no resource types. Their schemas are - * not modelled, so a lookup by name misses them, as the similarity search does. */ -const SYNCED_RESOURCE_TYPE_NAMES = [ - 'ai_instruction', - 'ai_skill', - 'airtable', - 'asana', - 'aws', - 'azure_blob', - 'clickup', - 'discord_webhook', - 'gcal', - 'gcp_service_account', - 'github', - 'gitlab', - 'gmail', - 'gsheets', - 'hubspot', - 'jira', - 'linear', - 'mailchimp', - 'mongodb', - 'mysql', - 'notion', - 'openai', - 'postgresql', - 's3', - 'salesforce', - 'sendgrid', - 'shopify', - 'slack', - 'smtp', - 'stripe', - 'supabase', - 'twilio', - 'zendesk' -] - export interface BenchmarkWorkspaceJob { /** Stable id so a case prompt can reference a specific run (e.g. for get_run). */ id?: string @@ -376,26 +339,50 @@ export function listBenchmarkPlainResources(workspace: string): ListableResource })) } +function benchmarkResourceTypes(workspace: string): BenchmarkWorkspaceResourceType[] { + return benchmarkWorkspaceRunnables.get(workspace)?.resourceTypes ?? syncedResourceTypes() +} + /** Resource type names of a benchmark workspace, as `ResourceService.listResourceTypeNames` * returns them. Null when the workspace is not a benchmark one. */ export function listBenchmarkResourceTypeNames(workspace: string): string[] | null { if (!benchmarkWorkspaces.has(workspace)) { return null } - const seeded = benchmarkWorkspaceRunnables.get(workspace)?.resourceTypes - return seeded ? seeded.map((rt) => rt.name) : SYNCED_RESOURCE_TYPE_NAMES + return benchmarkResourceTypes(workspace).map((rt) => rt.name) } -/** A seeded resource type, as `ResourceService.getResourceType` returns it. */ +/** A resource type of a benchmark workspace, as `ResourceService.getResourceType` returns it. */ export function getBenchmarkResourceType(workspace: string, name: string): ResourceType | null { - const seed = benchmarkWorkspaceRunnables - .get(workspace) - ?.resourceTypes?.find((rt) => rt.name === name && rt.schema !== undefined) + const seed = benchmarkResourceTypes(workspace).find( + (rt) => rt.name === name && rt.schema !== undefined + ) return seed ? { workspace_id: 'admins', name: seed.name, schema: seed.schema, description: seed.description } : null } +/** Stands in for the embedding search: a type matches when a word of the query appears in its + * name or description, ranked by how many do. */ +export function queryBenchmarkResourceTypes( + workspace: string, + text: string, + limit = 10 +): { name: string; schema?: unknown; score: number }[] { + const words = text + .toLowerCase() + .split(/[^a-z0-9]+/) + .filter((word) => word.length >= 3) + return benchmarkResourceTypes(workspace) + .map((rt) => { + const haystack = `${rt.name} ${rt.description ?? ''}`.toLowerCase() + return { name: rt.name, schema: rt.schema, score: words.filter((w) => haystack.includes(w)).length } + }) + .filter((rt) => rt.score > 0) + .sort((a, b) => b.score - a.score) + .slice(0, limit) +} + /** A seeded resource with its value, as `ResourceService.getResource` returns it. Covers both * seed kinds, so it agrees with `existsResource` and `listResource` — both of those report AI * providers too, and a case that lists resources and then reads one by path would otherwise get @@ -1408,6 +1395,42 @@ const BENCHMARK_HUB_INTEGRATION_META: Record< } } +/** The resource type a mocked hub integration takes, as its metadata serves it. */ +function benchmarkHubIntegrationResourceType(app: string): BenchmarkWorkspaceResourceType | null { + const real = REAL_HUB_INTEGRATIONS.find((integration) => integration.app === app) + if (real) { + return real.resource_type + } + const entry = BENCHMARK_HUB_INTEGRATION_META[app] + if (!entry) { + return null + } + return { + name: app, + description: `${entry.display_name} credentials`, + schema: { + type: 'object', + required: ['apiKey'], + properties: { + apiKey: { type: 'string', description: `${entry.display_name} API key` } + } + } + } +} + +/** What a hub-synced instance holds: common hub types, plus the one each mocked hub + * integration takes, so the hub and the instance agree on them. */ +function syncedResourceTypes(): BenchmarkWorkspaceResourceType[] { + const integrations = [ + ...REAL_HUB_INTEGRATIONS.map((integration) => integration.app), + ...Object.keys(BENCHMARK_HUB_INTEGRATION_META) + ] + return [ + ...syncedResourceTypesFixture, + ...integrations.flatMap((app) => benchmarkHubIntegrationResourceType(app) ?? []) + ] +} + /** Mirrors the hub's own derivation: hosts seen in the shipped scripts, whether they * call the provider directly, and how many there are of each kind. */ function benchmarkDerivedFacts(app: string) { @@ -1542,7 +1565,7 @@ export function handleBenchmarkApiFetch(url: string, init?: RequestInit): Respon meta: real.meta, meta_updated_at: null, derived: benchmarkDerivedFacts(app), - resource_types: [{ id: 1, ...real.resource_type }] + resource_types: [{ id: 1, ...benchmarkHubIntegrationResourceType(app) }] }) } const entry = BENCHMARK_HUB_INTEGRATION_META[app] @@ -1559,23 +1582,7 @@ export function handleBenchmarkApiFetch(url: string, init?: RequestInit): Respon meta: entry.meta, meta_updated_at: null, derived: benchmarkDerivedFacts(app), - resource_types: [ - { - id: 1, - name: app, - description: `${entry.display_name} credentials`, - schema: { - type: 'object', - required: ['apiKey'], - properties: { - apiKey: { - type: 'string', - description: `${entry.display_name} API key` - } - } - } - } - ] + resource_types: [{ id: 1, ...benchmarkHubIntegrationResourceType(app) }] }) } if (path === '/api/integrations/hub/list') { diff --git a/ai_evals/adapters/frontend/vitestAdapter.test.ts b/ai_evals/adapters/frontend/vitestAdapter.test.ts index 79fb4cb3a9..e86dd92f79 100644 --- a/ai_evals/adapters/frontend/vitestAdapter.test.ts +++ b/ai_evals/adapters/frontend/vitestAdapter.test.ts @@ -108,6 +108,7 @@ vi.mock('$lib/gen', async () => { listBenchmarkAiProviderResources, listBenchmarkPlainResources, listBenchmarkResourceTypeNames, + queryBenchmarkResourceTypes, listBenchmarkApps, listBenchmarkDatatables, listBenchmarkDataMetrics, @@ -464,8 +465,10 @@ vi.mock('$lib/gen', async () => { } return value }, - queryResourceTypes: async (data: { workspace: string }) => - hasBenchmarkWorkspace(data.workspace) ? [] : actual.ResourceService.queryResourceTypes(data), + queryResourceTypes: async (data: { workspace: string; text: string; limit?: number }) => + hasBenchmarkWorkspace(data.workspace) + ? queryBenchmarkResourceTypes(data.workspace, data.text, data.limit) + : actual.ResourceService.queryResourceTypes(data), listResourceTypeNames: async (data: { workspace: string }) => listBenchmarkResourceTypeNames(data.workspace) ?? actual.ResourceService.listResourceTypeNames(data), diff --git a/ai_evals/fixtures/frontend/global/hub/synced_resource_types.json b/ai_evals/fixtures/frontend/global/hub/synced_resource_types.json new file mode 100644 index 0000000000..033c53ba3e --- /dev/null +++ b/ai_evals/fixtures/frontend/global/hub/synced_resource_types.json @@ -0,0 +1,893 @@ +[ + { + "name": "ai_instruction", + "description": "Instructions for the AI chat that apply to every item under the folder this resource lives in, like an AGENTS.md: an ai_instruction at f/billing/AGENTS is given to the assistant whenever it works on something under f/billing/. The file body is the instructions it follows.", + "schema": { + "type": "object", + "properties": { + "content": { + "type": "string" + } + } + } + }, + { + "name": "ai_skill", + "description": "A reusable instruction set for the AI chat, in the SKILL.md format. The resource description is what the assistant sees when deciding whether the skill applies; the file body is the instructions it follows.", + "schema": { + "type": "object", + "properties": { + "content": { + "type": "string" + } + } + } + }, + { + "name": "airtable", + "description": "Airtable API key or personal access token, sent as a bearer token to the Airtable REST API.", + "schema": { + "type": "object", + "$schema": "https://json-schema.org/draft/2020-12/schema", + "required": [], + "properties": { + "apiKey": { + "type": "string", + "default": "keyXXXXXXXXXXXXXX", + "description": "" + } + } + } + }, + { + "name": "asana", + "description": "Credentials for the Asana work management API using a personal access token.", + "schema": { + "type": "object", + "$schema": "https://json-schema.org/draft/2020-12/schema", + "required": [ + "token" + ], + "properties": { + "token": { + "type": "string", + "default": "", + "description": "" + } + } + } + }, + { + "name": "aws", + "description": "Amazon Web Services (AWS) credentials: access key ID, secret access key and default region.", + "schema": { + "type": "object", + "$schema": "https://json-schema.org/draft/2020-12/schema", + "required": [ + "awsAccessKeyId", + "awsSecretAccessKey" + ], + "properties": { + "region": { + "type": "string", + "default": "", + "description": "" + }, + "awsAccessKeyId": { + "type": "string", + "default": "", + "description": "" + }, + "awsSecretAccessKey": { + "type": "string", + "default": "", + "description": "" + } + } + } + }, + { + "name": "azure_blob", + "description": "Connection settings for a Microsoft Azure Blob Storage container: account name, container name, access key and endpoint.", + "schema": { + "type": "object", + "order": [ + "accountName", + "containerName", + "accessKey", + "useSSL", + "endpoint" + ], + "$schema": "https://json-schema.org/draft/2020-12/schema", + "required": [ + "accountName", + "containerName", + "useSSL" + ], + "properties": { + "useSSL": { + "type": "boolean", + "default": true, + "description": "" + }, + "endpoint": { + "type": "string", + "default": "", + "description": "" + }, + "accessKey": { + "type": "string", + "default": "", + "description": "" + }, + "accountName": { + "type": "string", + "default": "", + "description": "" + }, + "containerName": { + "type": "string", + "default": "", + "description": "" + } + } + } + }, + { + "name": "clickup", + "description": "Credentials for the ClickUp API v2 using a personal API token.", + "schema": { + "type": "object", + "$schema": "https://json-schema.org/draft/2020-12/schema", + "required": [ + "token" + ], + "properties": { + "token": { + "type": "string", + "description": "API token retrieved from the dashboard." + } + } + } + }, + { + "name": "discord_webhook", + "description": "Discord incoming webhook URL for posting messages to a Discord channel.\n\nSee: ", + "schema": { + "type": "object", + "$schema": "https://json-schema.org/draft/2020-12/schema", + "required": [], + "properties": { + "webhook_url": { + "type": "string", + "description": "The webhook url" + } + } + } + }, + { + "name": "gcal", + "description": "Google OAuth token authorizing the Google Calendar API.", + "schema": { + "type": "object", + "$schema": "https://json-schema.org/draft/2020-12/schema", + "required": [ + "token" + ], + "properties": { + "token": { + "type": "string", + "description": "The OAuth token, refreshed if necessary" + } + } + } + }, + { + "name": "gcp_service_account", + "description": "Google Cloud service account key (JSON) used to authenticate to Google Cloud APIs.", + "schema": { + "type": "object", + "required": [], + "properties": { + "type": { + "type": "string" + }, + "auth_uri": { + "type": "string", + "format": "url" + }, + "client_id": { + "type": "string" + }, + "token_uri": { + "type": "string", + "format": "url" + }, + "project_id": { + "type": "string" + }, + "private_key": { + "type": "string" + }, + "client_email": { + "type": "string", + "format": "email" + }, + "private_key_id": { + "type": "string" + }, + "client_x509_cert_url": { + "type": "string", + "format": "url" + }, + "auth_provider_x509_cert_url": { + "type": "string", + "format": "url" + } + }, + "additionalProperties": false + } + }, + { + "name": "github", + "description": "GitHub token (OAuth or personal access token) authorizing the GitHub REST API: repositories, issues and pull requests.", + "schema": { + "type": "object", + "$schema": "https://json-schema.org/draft/2020-12/schema", + "required": [ + "token" + ], + "properties": { + "token": { + "type": "string", + "description": "" + } + } + } + }, + { + "name": "gitlab", + "description": "Credentials for the GitLab API v4 using a personal access token, with the base URL of the GitLab instance (self-hosted or gitlab.com).", + "schema": { + "type": "object", + "$schema": "https://json-schema.org/draft/2020-12/schema", + "required": [ + "token" + ], + "properties": { + "token": { + "type": "string", + "default": "", + "description": "" + }, + "baseUrl": { + "type": "string", + "default": "https://gitlab.com", + "description": "" + } + } + } + }, + { + "name": "gmail", + "description": "Google OAuth token authorizing the Gmail API to send email as the connected Google account.", + "schema": { + "type": "object", + "$schema": "https://json-schema.org/draft/2020-12/schema", + "required": [ + "token" + ], + "properties": { + "token": { + "type": "string", + "description": "" + } + } + } + }, + { + "name": "gsheets", + "description": "Google OAuth token authorizing the Google Sheets API for reading and writing spreadsheets.", + "schema": { + "type": "object", + "$schema": "https://json-schema.org/draft/2020-12/schema", + "required": [ + "token" + ], + "properties": { + "token": { + "type": "string", + "description": "The OAuth token, refreshed if necessary" + } + } + } + }, + { + "name": "hubspot", + "description": "Credentials for the HubSpot CRM API using a private app access token.", + "schema": { + "type": "object", + "$schema": "https://json-schema.org/draft/2020-12/schema", + "required": [], + "properties": { + "token": { + "type": "string", + "default": "", + "description": "Access token of a private app" + } + } + } + }, + { + "name": "jira", + "description": "Credentials for the Jira Cloud REST API using basic authentication with an Atlassian site domain.", + "schema": { + "type": "object", + "$schema": "https://json-schema.org/draft/2020-12/schema", + "required": [ + "username", + "password", + "domain" + ], + "properties": { + "domain": { + "type": "string", + "default": "", + "description": "" + }, + "password": { + "type": "string", + "default": "", + "description": "" + }, + "username": { + "type": "string", + "default": "", + "description": "" + } + } + } + }, + { + "name": "linear", + "description": "Credentials for the Linear issue tracking API using a personal API key.", + "schema": { + "type": "object", + "$schema": "https://json-schema.org/draft/2020-12/schema", + "required": [ + "apiKey" + ], + "properties": { + "apiKey": { + "type": "string", + "description": "A personal API key (lin_api_\u2026) created under Settings \u2192 Security & access \u2192 Personal API keys (https://linear.app/settings/account/security)." + } + } + } + }, + { + "name": "mailchimp", + "description": "API key and account data center prefix (e.g. us12) for the Mailchimp Marketing API.", + "schema": { + "type": "object", + "$schema": "https://json-schema.org/draft/2020-12/schema", + "required": [], + "properties": { + "server": { + "type": "string", + "default": "", + "description": "The data center for your Mailchimp account. (For example: us12)" + }, + "api_key": { + "type": "string", + "default": "", + "description": "" + } + } + } + }, + { + "name": "mongodb", + "description": "Connection settings for a MongoDB database: server host/port list, database name, TLS and SCRAM credentials.", + "schema": { + "type": "object", + "$schema": "https://json-schema.org/draft/2020-12/schema", + "required": [ + "db", + "tls", + "servers", + "credential" + ], + "properties": { + "db": { + "type": "string", + "description": "" + }, + "tls": { + "type": "boolean", + "default": true, + "description": "" + }, + "servers": [ + { + "host": { + "type": "string", + "description": "" + }, + "port": { + "type": "integer", + "default": 27017, + "description": "" + } + } + ], + "credential": { + "db": { + "type": "string", + "description": "" + }, + "password": { + "type": "string", + "description": "" + }, + "username": { + "type": "string", + "description": "" + }, + "mechanism": { + "type": "string", + "default": "SCRAM-SHA-1", + "description": "" + } + } + } + } + }, + { + "name": "mysql", + "description": "Connection settings for a MySQL database, with optional SSL and a root CA certificate.", + "schema": { + "type": "object", + "order": [ + "host", + "port", + "user", + "database", + "password", + "ssl", + "root_certificate_pem" + ], + "$schema": "https://json-schema.org/draft/2020-12/schema", + "required": [ + "user", + "database", + "password" + ], + "properties": { + "ssl": { + "type": "boolean", + "default": false, + "description": "use ssl" + }, + "host": { + "type": "string", + "default": "", + "description": "instance host" + }, + "port": { + "type": "number", + "default": 3306, + "description": "instance port" + }, + "user": { + "type": "string", + "default": "", + "description": "username" + }, + "database": { + "type": "string", + "default": "", + "description": "database name" + }, + "password": { + "type": "string", + "default": "", + "description": "user's password" + }, + "root_certificate_pem": { + "type": "string", + "default": "", + "description": "" + } + } + } + }, + { + "name": "notion", + "description": "Notion API token: filled by Windmill's Notion OAuth connect, or an internal connection's API token / personal access token pasted in.", + "schema": { + "type": "object", + "$schema": "https://json-schema.org/draft/2020-12/schema", + "required": [ + "token" + ], + "properties": { + "token": { + "type": "string", + "default": "", + "description": "Filled automatically when you connect with OAuth. To paste one instead: the API token of an internal connection (ntn_\u2026) or a personal access token, both created in the Notion developer portal at https://www.notion.so/developers. A connection token only reaches the pages granted to it under Content access." + } + } + } + }, + { + "name": "openai", + "description": "OpenAI secret API key, with optional organization ID and base URL override, for the OpenAI API.", + "schema": { + "type": "object", + "order": [ + "api_key", + "organization_id", + "base_url" + ], + "$schema": "https://json-schema.org/draft/2020-12/schema", + "required": [ + "api_key" + ], + "properties": { + "api_key": { + "type": "string", + "default": "", + "description": "" + }, + "base_url": { + "type": "string", + "default": "", + "description": "Optional. Only needed to overwrite the default base URL." + }, + "organization_id": { + "type": "string", + "default": "", + "description": "Only needs to be set for users who belong to multiple organizations and they want to use an organization other than the default one" + } + } + } + }, + { + "name": "postgresql", + "description": "Connection settings for a PostgreSQL database: host, port, database name, user, password and SSL/IAM auth options.", + "schema": { + "type": "object", + "$schema": "https://json-schema.org/draft/2020-12/schema", + "required": [ + "dbname", + "user", + "password" + ], + "properties": { + "host": { + "type": "string", + "description": "The instance host" + }, + "port": { + "type": "integer", + "description": "The instance port" + }, + "user": { + "type": "string", + "description": "The postgres username" + }, + "dbname": { + "type": "string", + "description": "The database name" + }, + "region": { + "type": "string", + "description": "AWS region for IAM auth (e.g. us-east-1). Falls back to AWS_REGION env var if not set." + }, + "options": { + "type": "string", + "description": "Startup options sent to the server (the libpq options parameter), such as endpoint= for a Neon pooled or private connection, or -c search_path=myschema" + }, + "sslmode": { + "enum": [ + "disable", + "allow", + "prefer", + "require", + "verify-ca", + "verify-full" + ], + "type": "string", + "description": "Most cloud databases (Heroku, AWS, Supabase, etc.) require SSL. Self-hosted databases typically don't." + }, + "password": { + "type": "string", + "description": "The postgres users password" + }, + "use_iam_auth": { + "type": "boolean", + "default": false, + "description": "Use IAM authentication (AWS RDS/Aurora). When enabled, the password field is ignored and a short-lived IAM token is generated instead." + }, + "accept_invalid_certs": { + "type": "boolean", + "default": false, + "showExpr": "fields.sslmode == 'verify-ca' || fields.sslmode == 'verify-full'", + "description": "Accept any certificate presented by the server, which disables TLS certificate verification and is not recommended. Only applies to verify-ca/verify-full." + }, + "root_certificate_pem": { + "type": "string", + "default": "", + "description": "The root certificate in pe format" + } + } + } + }, + { + "name": "s3", + "description": "Connection settings for an Amazon S3 (AWS) bucket: endpoint, region, bucket name and access keys.", + "schema": { + "type": "object", + "order": [ + "endPoint", + "region", + "pathStyle", + "bucket", + "accessKey", + "secretKey", + "port", + "useSSL" + ], + "$schema": "https://json-schema.org/draft/2020-12/schema", + "required": [ + "bucket" + ], + "properties": { + "port": { + "type": "number", + "description": "Deprecated: pass in a complete URL to `endPoint` instead, including a port number if needed" + }, + "bucket": { + "type": "string", + "default": "", + "description": "" + }, + "region": { + "type": "string", + "default": "", + "description": "AWS region (e.g. us-east-1). Used to auto-generate the S3 endpoint URL when endPoint is not specified" + }, + "useSSL": { + "type": "boolean", + "default": true, + "description": "Deprecated: pass in a complete URL to `endPoint` instead, including 'https://' or 'http://'" + }, + "endPoint": { + "type": "string", + "default": "", + "description": "The full URL of the S3 endpoint (e.g. https://s3.us-east-1.amazonaws.com)" + }, + "accessKey": { + "type": "string", + "default": "", + "description": "" + }, + "pathStyle": { + "type": "boolean", + "default": true, + "description": "// instead of ./" + }, + "secretKey": { + "type": "string", + "default": "", + "description": "" + } + } + } + }, + { + "name": "salesforce", + "description": "Salesforce OAuth Authorization Code credential. The token is populated by Windmill's OAuth flow; instance_url pins every REST call to the org's data host, api_version selects the REST API version.", + "schema": { + "type": "object", + "$schema": "https://json-schema.org/draft/2020-12/schema", + "required": [ + "token", + "instance_url" + ], + "properties": { + "token": { + "type": "string", + "description": "OAuth access token. Populated by Windmill's OAuth flow." + }, + "api_version": { + "type": "string", + "description": "Salesforce REST API version path segment, e.g. v60.0. Optional \u2014 scripts default to v60.0 when left blank." + }, + "instance_url": { + "type": "string", + "description": "Org data host returned in the OAuth token response, e.g. https://acme.my.salesforce.com (production) or https://acme--sandbox.sandbox.my.salesforce.com (sandbox). Every REST call is made against this host, not login.salesforce.com." + } + } + } + }, + { + "name": "sendgrid", + "description": "Twilio SendGrid API key for the SendGrid v3 API: transactional email, contacts, marketing lists and bounces.", + "schema": { + "type": "object", + "$schema": "https://json-schema.org/draft/2020-12/schema", + "required": [ + "token" + ], + "properties": { + "token": { + "type": "string", + "default": "", + "description": "API token" + } + } + } + }, + { + "name": "shopify", + "description": "Credentials for the Shopify Admin API using a store name and access token.", + "schema": { + "type": "object", + "$schema": "https://json-schema.org/draft/2020-12/schema", + "required": [ + "store_name", + "token" + ], + "properties": { + "token": { + "type": "string", + "default": "", + "description": "" + }, + "store_name": { + "type": "string", + "default": "", + "description": "" + } + } + } + }, + { + "name": "slack", + "description": "Slack bot or user OAuth token authorizing the Slack Web API (messages, channels, users and files).", + "schema": { + "type": "object", + "$schema": "https://json-schema.org/draft/2020-12/schema", + "required": [], + "properties": { + "token": { + "type": "string", + "description": "The slack token" + } + } + } + }, + { + "name": "smtp", + "description": "Connection settings for an SMTP mail server used to send email.", + "schema": { + "type": "object", + "$schema": "https://json-schema.org/draft/2020-12/schema", + "required": [ + "host" + ], + "properties": { + "host": { + "type": "string", + "default": "mail.smtpbucket.com", + "description": "SMTP host address" + }, + "port": { + "type": "integer", + "default": 8025, + "description": "port number on which to connect" + }, + "user": { + "type": "string", + "description": "" + }, + "password": { + "type": "string", + "description": "" + } + } + } + }, + { + "name": "stripe", + "description": "Stripe secret API key for the Stripe payments API.", + "schema": { + "type": "object", + "$schema": "https://json-schema.org/draft/2020-12/schema", + "required": [], + "properties": { + "token": { + "type": "string", + "description": "" + } + } + } + }, + { + "name": "supabase", + "description": "Supabase project URL and API key for the Supabase REST, auth and storage APIs.\n\nRetrieve both values in \"Project Settings -> API\".", + "schema": { + "type": "object", + "$schema": "https://json-schema.org/draft/2020-12/schema", + "required": [], + "properties": { + "key": { + "type": "string", + "default": "", + "description": "The unique Supabase Key which is supplied when you create a new project in your project dashboard." + }, + "url": { + "type": "string", + "default": "", + "description": "The unique Supabase URL which is supplied when you create a new project in your project dashboard." + } + } + } + }, + { + "name": "twilio", + "description": "Credentials for the Twilio messaging and voice API using an Account SID and auth token.", + "schema": { + "type": "object", + "$schema": "https://json-schema.org/draft/2020-12/schema", + "required": [ + "token", + "accountSid" + ], + "properties": { + "token": { + "type": "string", + "default": "", + "description": "" + }, + "accountSid": { + "type": "string", + "default": "", + "description": "" + } + } + } + }, + { + "name": "zendesk", + "description": "Zendesk API credentials: account subdomain plus email and password or API token, used with HTTP Basic auth.", + "schema": { + "type": "object", + "$schema": "https://json-schema.org/draft/2020-12/schema", + "required": [ + "username", + "subdomain", + "password" + ], + "properties": { + "password": { + "type": "string", + "default": "", + "description": "password or api token" + }, + "username": { + "type": "string", + "default": "", + "description": "email or \"{email}/token\" if using an api token" + }, + "subdomain": { + "type": "string", + "default": "", + "description": "" + } + } + } + } +] diff --git a/frontend/src/lib/components/copilot/chat/global/core.ts b/frontend/src/lib/components/copilot/chat/global/core.ts index 1fd6feda16..be9fabd211 100644 --- a/frontend/src/lib/components/copilot/chat/global/core.ts +++ b/frontend/src/lib/components/copilot/chat/global/core.ts @@ -4283,7 +4283,7 @@ export const globalTools: SessionTool<{}>[] = [ toolCallbacks.setToolStatus(toolId, { content: `Searching resource types for "${parsed.query}"...` }) - const { resourceTypes, hubSyncNote } = await searchResourceTypes( + const { resourceTypes, note } = await searchResourceTypes( parsed.query, workspace, parsed.limit ?? 5 @@ -4291,11 +4291,7 @@ export const globalTools: SessionTool<{}>[] = [ toolCallbacks.setToolStatus(toolId, { content: `Found ${resourceTypes.length} resource type(s) for "${parsed.query}"` }) - return JSON.stringify( - hubSyncNote ? { resource_types: resourceTypes, note: hubSyncNote } : resourceTypes, - null, - 2 - ) + return JSON.stringify(note ? { resource_types: resourceTypes, note } : resourceTypes, null, 2) } }, createDbSchemaTool<{}>({ diff --git a/frontend/src/lib/components/copilot/chat/script/core.test.ts b/frontend/src/lib/components/copilot/chat/script/core.test.ts index 414c42d758..627fe8c0dd 100644 --- a/frontend/src/lib/components/copilot/chat/script/core.test.ts +++ b/frontend/src/lib/components/copilot/chat/script/core.test.ts @@ -30,10 +30,25 @@ describe('searchResourceTypes', () => { schema: { type: 'object' } }) - const { resourceTypes, hubSyncNote } = await searchResourceTypes('Stripe payments', 'ws', 5) + const { resourceTypes, note } = await searchResourceTypes('Stripe payments', 'ws', 5) expect(resourceTypes.map((rt) => rt.name)).toEqual(['stripe']) - expect(hubSyncNote).toBeUndefined() + expect(note).toBeUndefined() + }) + + // An empty result from a failed search says nothing about whether a type exists: read as + // "none", it sends the model to define a type the instance already has. + it('hands back the type names when the similarity search fails', async () => { + vi.spyOn(ResourceService, 'queryResourceTypes').mockRejectedValue(new Error('500')) + vi.spyOn(ResourceService, 'listResourceTypeNames').mockResolvedValue([ + ...syncedNames, + 'postgresql' + ]) + + const { resourceTypes, note } = await searchResourceTypes('postgres database', 'ws', 5) + + expect(resourceTypes).toEqual([]) + expect(note).toContain('postgresql') }) it('notes an instance that never synced with the hub', async () => { @@ -43,9 +58,9 @@ describe('searchResourceTypes', () => { 'ai_instruction' ]) - const { hubSyncNote } = await searchResourceTypes('stripe', 'ws', 5) + const { note } = await searchResourceTypes('stripe', 'ws', 5) - expect(hubSyncNote).toContain('never synced') + expect(note).toContain('never synced') }) }) diff --git a/frontend/src/lib/components/copilot/chat/script/core.ts b/frontend/src/lib/components/copilot/chat/script/core.ts index 1437f2f227..5e7d24c616 100644 --- a/frontend/src/lib/components/copilot/chat/script/core.ts +++ b/frontend/src/lib/components/copilot/chat/script/core.ts @@ -80,9 +80,13 @@ export async function searchResourceTypes( query: string, workspace: string, limit: number -): Promise<{ resourceTypes: Pick[]; hubSyncNote?: string }> { +): Promise<{ resourceTypes: Pick[]; note?: string }> { + let similarityFailed = false const [similar, names] = await Promise.all([ - ResourceService.queryResourceTypes({ workspace, text: query, limit }).catch(() => []), + ResourceService.queryResourceTypes({ workspace, text: query, limit }).catch(() => { + similarityFailed = true + return [] + }), ResourceService.listResourceTypeNames({ workspace }) .then((listed) => new Set(listed)) .catch(() => undefined) @@ -108,12 +112,18 @@ export async function searchResourceTypes( ...similar ].slice(0, limit) - const hubSyncNote = + // Without the similarity search, an empty result says nothing about whether a type exists, + // so the model gets the names to search again with rather than concluding there is none. + const notes = [ + similarityFailed && names + ? `The similarity search is unavailable on this instance, so only a type named in the query can be found. Search again with one of the instance's resource types: ${[...names].join(', ')}.` + : undefined, names && names.size < UNSYNCED_RESOURCE_TYPE_COUNT ? `Only ${names.size} resource types exist on this instance, so it has most likely never synced with the Windmill Hub, which has types for most services. ${HUB_SYNC_INSTRUCTIONS}` : undefined + ].filter((note) => note !== undefined) - return { resourceTypes, hubSyncNote } + return { resourceTypes, note: notes.length > 0 ? notes.join('\n\n') : undefined } } export const SUPPORTED_CHAT_SCRIPT_LANGUAGES = [ @@ -188,14 +198,14 @@ export async function getFormattedResourceTypes( case 'bunnative': case 'python3': case 'php': { - const { resourceTypes, hubSyncNote } = await searchResourceTypes(prompt, workspace, 5) + const { resourceTypes, note } = await searchResourceTypes(prompt, workspace, 5) const found = resourceTypes.length > 0 ? `RESOURCE_TYPES:\n${formatResourceTypes(resourceTypes, lang)}` - : `No resource type matches "${prompt}".` + : `No resource type found for "${prompt}".` - return hubSyncNote ? `${found}\n\n${hubSyncNote}` : found + return note ? `${found}\n\n${note}` : found } default: return ''