mirror of
https://github.com/windmill-labs/windmill.git
synced 2026-10-12 16:02:56 +00:00
fix: report a failed resource type search and model a synced instance in evals
Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
hugocasa
co-authored by
Claude Opus 5.5
Claude Opus 5.5
parent
bdb5d43425
commit
03d2459932
@@ -2,6 +2,7 @@ import { randomUUID } from 'node:crypto'
|
||||
import confluenceFixture from '../../fixtures/frontend/global/hub/confluence.json'
|
||||
import servicenowFixture from '../../fixtures/frontend/global/hub/servicenow.json'
|
||||
import outreachFixture from '../../fixtures/frontend/global/hub/outreach.json'
|
||||
import syncedResourceTypesFixture from '../../fixtures/frontend/global/hub/synced_resource_types.json'
|
||||
import type {
|
||||
AppWithLastVersion,
|
||||
CompletedJob,
|
||||
@@ -99,51 +100,13 @@ export interface BenchmarkWorkspaceResource {
|
||||
description?: string
|
||||
}
|
||||
|
||||
/** A resource type the instance holds. Only a seeded one has a schema to look up by name. */
|
||||
/** A resource type the instance holds. Without a schema, a lookup by name misses it. */
|
||||
export interface BenchmarkWorkspaceResourceType {
|
||||
name: string
|
||||
schema?: unknown
|
||||
description?: string
|
||||
}
|
||||
|
||||
/** Names a hub-synced instance holds, for cases that seed no resource types. Their schemas are
|
||||
* not modelled, so a lookup by name misses them, as the similarity search does. */
|
||||
const SYNCED_RESOURCE_TYPE_NAMES = [
|
||||
'ai_instruction',
|
||||
'ai_skill',
|
||||
'airtable',
|
||||
'asana',
|
||||
'aws',
|
||||
'azure_blob',
|
||||
'clickup',
|
||||
'discord_webhook',
|
||||
'gcal',
|
||||
'gcp_service_account',
|
||||
'github',
|
||||
'gitlab',
|
||||
'gmail',
|
||||
'gsheets',
|
||||
'hubspot',
|
||||
'jira',
|
||||
'linear',
|
||||
'mailchimp',
|
||||
'mongodb',
|
||||
'mysql',
|
||||
'notion',
|
||||
'openai',
|
||||
'postgresql',
|
||||
's3',
|
||||
'salesforce',
|
||||
'sendgrid',
|
||||
'shopify',
|
||||
'slack',
|
||||
'smtp',
|
||||
'stripe',
|
||||
'supabase',
|
||||
'twilio',
|
||||
'zendesk'
|
||||
]
|
||||
|
||||
export interface BenchmarkWorkspaceJob {
|
||||
/** Stable id so a case prompt can reference a specific run (e.g. for get_run). */
|
||||
id?: string
|
||||
@@ -376,26 +339,50 @@ export function listBenchmarkPlainResources(workspace: string): ListableResource
|
||||
}))
|
||||
}
|
||||
|
||||
function benchmarkResourceTypes(workspace: string): BenchmarkWorkspaceResourceType[] {
|
||||
return benchmarkWorkspaceRunnables.get(workspace)?.resourceTypes ?? syncedResourceTypes()
|
||||
}
|
||||
|
||||
/** Resource type names of a benchmark workspace, as `ResourceService.listResourceTypeNames`
|
||||
* returns them. Null when the workspace is not a benchmark one. */
|
||||
export function listBenchmarkResourceTypeNames(workspace: string): string[] | null {
|
||||
if (!benchmarkWorkspaces.has(workspace)) {
|
||||
return null
|
||||
}
|
||||
const seeded = benchmarkWorkspaceRunnables.get(workspace)?.resourceTypes
|
||||
return seeded ? seeded.map((rt) => rt.name) : SYNCED_RESOURCE_TYPE_NAMES
|
||||
return benchmarkResourceTypes(workspace).map((rt) => rt.name)
|
||||
}
|
||||
|
||||
/** A seeded resource type, as `ResourceService.getResourceType` returns it. */
|
||||
/** A resource type of a benchmark workspace, as `ResourceService.getResourceType` returns it. */
|
||||
export function getBenchmarkResourceType(workspace: string, name: string): ResourceType | null {
|
||||
const seed = benchmarkWorkspaceRunnables
|
||||
.get(workspace)
|
||||
?.resourceTypes?.find((rt) => rt.name === name && rt.schema !== undefined)
|
||||
const seed = benchmarkResourceTypes(workspace).find(
|
||||
(rt) => rt.name === name && rt.schema !== undefined
|
||||
)
|
||||
return seed
|
||||
? { workspace_id: 'admins', name: seed.name, schema: seed.schema, description: seed.description }
|
||||
: null
|
||||
}
|
||||
|
||||
/** Stands in for the embedding search: a type matches when a word of the query appears in its
|
||||
* name or description, ranked by how many do. */
|
||||
export function queryBenchmarkResourceTypes(
|
||||
workspace: string,
|
||||
text: string,
|
||||
limit = 10
|
||||
): { name: string; schema?: unknown; score: number }[] {
|
||||
const words = text
|
||||
.toLowerCase()
|
||||
.split(/[^a-z0-9]+/)
|
||||
.filter((word) => word.length >= 3)
|
||||
return benchmarkResourceTypes(workspace)
|
||||
.map((rt) => {
|
||||
const haystack = `${rt.name} ${rt.description ?? ''}`.toLowerCase()
|
||||
return { name: rt.name, schema: rt.schema, score: words.filter((w) => haystack.includes(w)).length }
|
||||
})
|
||||
.filter((rt) => rt.score > 0)
|
||||
.sort((a, b) => b.score - a.score)
|
||||
.slice(0, limit)
|
||||
}
|
||||
|
||||
/** A seeded resource with its value, as `ResourceService.getResource` returns it. Covers both
|
||||
* seed kinds, so it agrees with `existsResource` and `listResource` — both of those report AI
|
||||
* providers too, and a case that lists resources and then reads one by path would otherwise get
|
||||
@@ -1408,6 +1395,42 @@ const BENCHMARK_HUB_INTEGRATION_META: Record<
|
||||
}
|
||||
}
|
||||
|
||||
/** The resource type a mocked hub integration takes, as its metadata serves it. */
|
||||
function benchmarkHubIntegrationResourceType(app: string): BenchmarkWorkspaceResourceType | null {
|
||||
const real = REAL_HUB_INTEGRATIONS.find((integration) => integration.app === app)
|
||||
if (real) {
|
||||
return real.resource_type
|
||||
}
|
||||
const entry = BENCHMARK_HUB_INTEGRATION_META[app]
|
||||
if (!entry) {
|
||||
return null
|
||||
}
|
||||
return {
|
||||
name: app,
|
||||
description: `${entry.display_name} credentials`,
|
||||
schema: {
|
||||
type: 'object',
|
||||
required: ['apiKey'],
|
||||
properties: {
|
||||
apiKey: { type: 'string', description: `${entry.display_name} API key` }
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/** What a hub-synced instance holds: common hub types, plus the one each mocked hub
|
||||
* integration takes, so the hub and the instance agree on them. */
|
||||
function syncedResourceTypes(): BenchmarkWorkspaceResourceType[] {
|
||||
const integrations = [
|
||||
...REAL_HUB_INTEGRATIONS.map((integration) => integration.app),
|
||||
...Object.keys(BENCHMARK_HUB_INTEGRATION_META)
|
||||
]
|
||||
return [
|
||||
...syncedResourceTypesFixture,
|
||||
...integrations.flatMap((app) => benchmarkHubIntegrationResourceType(app) ?? [])
|
||||
]
|
||||
}
|
||||
|
||||
/** Mirrors the hub's own derivation: hosts seen in the shipped scripts, whether they
|
||||
* call the provider directly, and how many there are of each kind. */
|
||||
function benchmarkDerivedFacts(app: string) {
|
||||
@@ -1542,7 +1565,7 @@ export function handleBenchmarkApiFetch(url: string, init?: RequestInit): Respon
|
||||
meta: real.meta,
|
||||
meta_updated_at: null,
|
||||
derived: benchmarkDerivedFacts(app),
|
||||
resource_types: [{ id: 1, ...real.resource_type }]
|
||||
resource_types: [{ id: 1, ...benchmarkHubIntegrationResourceType(app) }]
|
||||
})
|
||||
}
|
||||
const entry = BENCHMARK_HUB_INTEGRATION_META[app]
|
||||
@@ -1559,23 +1582,7 @@ export function handleBenchmarkApiFetch(url: string, init?: RequestInit): Respon
|
||||
meta: entry.meta,
|
||||
meta_updated_at: null,
|
||||
derived: benchmarkDerivedFacts(app),
|
||||
resource_types: [
|
||||
{
|
||||
id: 1,
|
||||
name: app,
|
||||
description: `${entry.display_name} credentials`,
|
||||
schema: {
|
||||
type: 'object',
|
||||
required: ['apiKey'],
|
||||
properties: {
|
||||
apiKey: {
|
||||
type: 'string',
|
||||
description: `${entry.display_name} API key`
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
]
|
||||
resource_types: [{ id: 1, ...benchmarkHubIntegrationResourceType(app) }]
|
||||
})
|
||||
}
|
||||
if (path === '/api/integrations/hub/list') {
|
||||
|
||||
@@ -108,6 +108,7 @@ vi.mock('$lib/gen', async () => {
|
||||
listBenchmarkAiProviderResources,
|
||||
listBenchmarkPlainResources,
|
||||
listBenchmarkResourceTypeNames,
|
||||
queryBenchmarkResourceTypes,
|
||||
listBenchmarkApps,
|
||||
listBenchmarkDatatables,
|
||||
listBenchmarkDataMetrics,
|
||||
@@ -464,8 +465,10 @@ vi.mock('$lib/gen', async () => {
|
||||
}
|
||||
return value
|
||||
},
|
||||
queryResourceTypes: async (data: { workspace: string }) =>
|
||||
hasBenchmarkWorkspace(data.workspace) ? [] : actual.ResourceService.queryResourceTypes(data),
|
||||
queryResourceTypes: async (data: { workspace: string; text: string; limit?: number }) =>
|
||||
hasBenchmarkWorkspace(data.workspace)
|
||||
? queryBenchmarkResourceTypes(data.workspace, data.text, data.limit)
|
||||
: actual.ResourceService.queryResourceTypes(data),
|
||||
listResourceTypeNames: async (data: { workspace: string }) =>
|
||||
listBenchmarkResourceTypeNames(data.workspace) ??
|
||||
actual.ResourceService.listResourceTypeNames(data),
|
||||
|
||||
@@ -0,0 +1,893 @@
|
||||
[
|
||||
{
|
||||
"name": "ai_instruction",
|
||||
"description": "Instructions for the AI chat that apply to every item under the folder this resource lives in, like an AGENTS.md: an ai_instruction at f/billing/AGENTS is given to the assistant whenever it works on something under f/billing/. The file body is the instructions it follows.",
|
||||
"schema": {
|
||||
"type": "object",
|
||||
"properties": {
|
||||
"content": {
|
||||
"type": "string"
|
||||
}
|
||||
}
|
||||
}
|
||||
},
|
||||
{
|
||||
"name": "ai_skill",
|
||||
"description": "A reusable instruction set for the AI chat, in the SKILL.md format. The resource description is what the assistant sees when deciding whether the skill applies; the file body is the instructions it follows.",
|
||||
"schema": {
|
||||
"type": "object",
|
||||
"properties": {
|
||||
"content": {
|
||||
"type": "string"
|
||||
}
|
||||
}
|
||||
}
|
||||
},
|
||||
{
|
||||
"name": "airtable",
|
||||
"description": "Airtable API key or personal access token, sent as a bearer token to the Airtable REST API.",
|
||||
"schema": {
|
||||
"type": "object",
|
||||
"$schema": "https://json-schema.org/draft/2020-12/schema",
|
||||
"required": [],
|
||||
"properties": {
|
||||
"apiKey": {
|
||||
"type": "string",
|
||||
"default": "keyXXXXXXXXXXXXXX",
|
||||
"description": ""
|
||||
}
|
||||
}
|
||||
}
|
||||
},
|
||||
{
|
||||
"name": "asana",
|
||||
"description": "Credentials for the Asana work management API using a personal access token.",
|
||||
"schema": {
|
||||
"type": "object",
|
||||
"$schema": "https://json-schema.org/draft/2020-12/schema",
|
||||
"required": [
|
||||
"token"
|
||||
],
|
||||
"properties": {
|
||||
"token": {
|
||||
"type": "string",
|
||||
"default": "",
|
||||
"description": ""
|
||||
}
|
||||
}
|
||||
}
|
||||
},
|
||||
{
|
||||
"name": "aws",
|
||||
"description": "Amazon Web Services (AWS) credentials: access key ID, secret access key and default region.",
|
||||
"schema": {
|
||||
"type": "object",
|
||||
"$schema": "https://json-schema.org/draft/2020-12/schema",
|
||||
"required": [
|
||||
"awsAccessKeyId",
|
||||
"awsSecretAccessKey"
|
||||
],
|
||||
"properties": {
|
||||
"region": {
|
||||
"type": "string",
|
||||
"default": "",
|
||||
"description": ""
|
||||
},
|
||||
"awsAccessKeyId": {
|
||||
"type": "string",
|
||||
"default": "",
|
||||
"description": ""
|
||||
},
|
||||
"awsSecretAccessKey": {
|
||||
"type": "string",
|
||||
"default": "",
|
||||
"description": ""
|
||||
}
|
||||
}
|
||||
}
|
||||
},
|
||||
{
|
||||
"name": "azure_blob",
|
||||
"description": "Connection settings for a Microsoft Azure Blob Storage container: account name, container name, access key and endpoint.",
|
||||
"schema": {
|
||||
"type": "object",
|
||||
"order": [
|
||||
"accountName",
|
||||
"containerName",
|
||||
"accessKey",
|
||||
"useSSL",
|
||||
"endpoint"
|
||||
],
|
||||
"$schema": "https://json-schema.org/draft/2020-12/schema",
|
||||
"required": [
|
||||
"accountName",
|
||||
"containerName",
|
||||
"useSSL"
|
||||
],
|
||||
"properties": {
|
||||
"useSSL": {
|
||||
"type": "boolean",
|
||||
"default": true,
|
||||
"description": ""
|
||||
},
|
||||
"endpoint": {
|
||||
"type": "string",
|
||||
"default": "",
|
||||
"description": ""
|
||||
},
|
||||
"accessKey": {
|
||||
"type": "string",
|
||||
"default": "",
|
||||
"description": ""
|
||||
},
|
||||
"accountName": {
|
||||
"type": "string",
|
||||
"default": "",
|
||||
"description": ""
|
||||
},
|
||||
"containerName": {
|
||||
"type": "string",
|
||||
"default": "",
|
||||
"description": ""
|
||||
}
|
||||
}
|
||||
}
|
||||
},
|
||||
{
|
||||
"name": "clickup",
|
||||
"description": "Credentials for the ClickUp API v2 using a personal API token.",
|
||||
"schema": {
|
||||
"type": "object",
|
||||
"$schema": "https://json-schema.org/draft/2020-12/schema",
|
||||
"required": [
|
||||
"token"
|
||||
],
|
||||
"properties": {
|
||||
"token": {
|
||||
"type": "string",
|
||||
"description": "API token retrieved from the dashboard."
|
||||
}
|
||||
}
|
||||
}
|
||||
},
|
||||
{
|
||||
"name": "discord_webhook",
|
||||
"description": "Discord incoming webhook URL for posting messages to a Discord channel.\n\nSee: <https://support.discord.com/hc/en-us/articles/228383668-Intro-to-Webhooks>",
|
||||
"schema": {
|
||||
"type": "object",
|
||||
"$schema": "https://json-schema.org/draft/2020-12/schema",
|
||||
"required": [],
|
||||
"properties": {
|
||||
"webhook_url": {
|
||||
"type": "string",
|
||||
"description": "The webhook url"
|
||||
}
|
||||
}
|
||||
}
|
||||
},
|
||||
{
|
||||
"name": "gcal",
|
||||
"description": "Google OAuth token authorizing the Google Calendar API.",
|
||||
"schema": {
|
||||
"type": "object",
|
||||
"$schema": "https://json-schema.org/draft/2020-12/schema",
|
||||
"required": [
|
||||
"token"
|
||||
],
|
||||
"properties": {
|
||||
"token": {
|
||||
"type": "string",
|
||||
"description": "The OAuth token, refreshed if necessary"
|
||||
}
|
||||
}
|
||||
}
|
||||
},
|
||||
{
|
||||
"name": "gcp_service_account",
|
||||
"description": "Google Cloud service account key (JSON) used to authenticate to Google Cloud APIs.",
|
||||
"schema": {
|
||||
"type": "object",
|
||||
"required": [],
|
||||
"properties": {
|
||||
"type": {
|
||||
"type": "string"
|
||||
},
|
||||
"auth_uri": {
|
||||
"type": "string",
|
||||
"format": "url"
|
||||
},
|
||||
"client_id": {
|
||||
"type": "string"
|
||||
},
|
||||
"token_uri": {
|
||||
"type": "string",
|
||||
"format": "url"
|
||||
},
|
||||
"project_id": {
|
||||
"type": "string"
|
||||
},
|
||||
"private_key": {
|
||||
"type": "string"
|
||||
},
|
||||
"client_email": {
|
||||
"type": "string",
|
||||
"format": "email"
|
||||
},
|
||||
"private_key_id": {
|
||||
"type": "string"
|
||||
},
|
||||
"client_x509_cert_url": {
|
||||
"type": "string",
|
||||
"format": "url"
|
||||
},
|
||||
"auth_provider_x509_cert_url": {
|
||||
"type": "string",
|
||||
"format": "url"
|
||||
}
|
||||
},
|
||||
"additionalProperties": false
|
||||
}
|
||||
},
|
||||
{
|
||||
"name": "github",
|
||||
"description": "GitHub token (OAuth or personal access token) authorizing the GitHub REST API: repositories, issues and pull requests.",
|
||||
"schema": {
|
||||
"type": "object",
|
||||
"$schema": "https://json-schema.org/draft/2020-12/schema",
|
||||
"required": [
|
||||
"token"
|
||||
],
|
||||
"properties": {
|
||||
"token": {
|
||||
"type": "string",
|
||||
"description": ""
|
||||
}
|
||||
}
|
||||
}
|
||||
},
|
||||
{
|
||||
"name": "gitlab",
|
||||
"description": "Credentials for the GitLab API v4 using a personal access token, with the base URL of the GitLab instance (self-hosted or gitlab.com).",
|
||||
"schema": {
|
||||
"type": "object",
|
||||
"$schema": "https://json-schema.org/draft/2020-12/schema",
|
||||
"required": [
|
||||
"token"
|
||||
],
|
||||
"properties": {
|
||||
"token": {
|
||||
"type": "string",
|
||||
"default": "",
|
||||
"description": ""
|
||||
},
|
||||
"baseUrl": {
|
||||
"type": "string",
|
||||
"default": "https://gitlab.com",
|
||||
"description": ""
|
||||
}
|
||||
}
|
||||
}
|
||||
},
|
||||
{
|
||||
"name": "gmail",
|
||||
"description": "Google OAuth token authorizing the Gmail API to send email as the connected Google account.",
|
||||
"schema": {
|
||||
"type": "object",
|
||||
"$schema": "https://json-schema.org/draft/2020-12/schema",
|
||||
"required": [
|
||||
"token"
|
||||
],
|
||||
"properties": {
|
||||
"token": {
|
||||
"type": "string",
|
||||
"description": ""
|
||||
}
|
||||
}
|
||||
}
|
||||
},
|
||||
{
|
||||
"name": "gsheets",
|
||||
"description": "Google OAuth token authorizing the Google Sheets API for reading and writing spreadsheets.",
|
||||
"schema": {
|
||||
"type": "object",
|
||||
"$schema": "https://json-schema.org/draft/2020-12/schema",
|
||||
"required": [
|
||||
"token"
|
||||
],
|
||||
"properties": {
|
||||
"token": {
|
||||
"type": "string",
|
||||
"description": "The OAuth token, refreshed if necessary"
|
||||
}
|
||||
}
|
||||
}
|
||||
},
|
||||
{
|
||||
"name": "hubspot",
|
||||
"description": "Credentials for the HubSpot CRM API using a private app access token.",
|
||||
"schema": {
|
||||
"type": "object",
|
||||
"$schema": "https://json-schema.org/draft/2020-12/schema",
|
||||
"required": [],
|
||||
"properties": {
|
||||
"token": {
|
||||
"type": "string",
|
||||
"default": "",
|
||||
"description": "Access token of a private app"
|
||||
}
|
||||
}
|
||||
}
|
||||
},
|
||||
{
|
||||
"name": "jira",
|
||||
"description": "Credentials for the Jira Cloud REST API using basic authentication with an Atlassian site domain.",
|
||||
"schema": {
|
||||
"type": "object",
|
||||
"$schema": "https://json-schema.org/draft/2020-12/schema",
|
||||
"required": [
|
||||
"username",
|
||||
"password",
|
||||
"domain"
|
||||
],
|
||||
"properties": {
|
||||
"domain": {
|
||||
"type": "string",
|
||||
"default": "",
|
||||
"description": ""
|
||||
},
|
||||
"password": {
|
||||
"type": "string",
|
||||
"default": "",
|
||||
"description": ""
|
||||
},
|
||||
"username": {
|
||||
"type": "string",
|
||||
"default": "",
|
||||
"description": ""
|
||||
}
|
||||
}
|
||||
}
|
||||
},
|
||||
{
|
||||
"name": "linear",
|
||||
"description": "Credentials for the Linear issue tracking API using a personal API key.",
|
||||
"schema": {
|
||||
"type": "object",
|
||||
"$schema": "https://json-schema.org/draft/2020-12/schema",
|
||||
"required": [
|
||||
"apiKey"
|
||||
],
|
||||
"properties": {
|
||||
"apiKey": {
|
||||
"type": "string",
|
||||
"description": "A personal API key (lin_api_\u2026) created under Settings \u2192 Security & access \u2192 Personal API keys (https://linear.app/settings/account/security)."
|
||||
}
|
||||
}
|
||||
}
|
||||
},
|
||||
{
|
||||
"name": "mailchimp",
|
||||
"description": "API key and account data center prefix (e.g. us12) for the Mailchimp Marketing API.",
|
||||
"schema": {
|
||||
"type": "object",
|
||||
"$schema": "https://json-schema.org/draft/2020-12/schema",
|
||||
"required": [],
|
||||
"properties": {
|
||||
"server": {
|
||||
"type": "string",
|
||||
"default": "",
|
||||
"description": "The data center for your Mailchimp account. (For example: us12)"
|
||||
},
|
||||
"api_key": {
|
||||
"type": "string",
|
||||
"default": "",
|
||||
"description": ""
|
||||
}
|
||||
}
|
||||
}
|
||||
},
|
||||
{
|
||||
"name": "mongodb",
|
||||
"description": "Connection settings for a MongoDB database: server host/port list, database name, TLS and SCRAM credentials.",
|
||||
"schema": {
|
||||
"type": "object",
|
||||
"$schema": "https://json-schema.org/draft/2020-12/schema",
|
||||
"required": [
|
||||
"db",
|
||||
"tls",
|
||||
"servers",
|
||||
"credential"
|
||||
],
|
||||
"properties": {
|
||||
"db": {
|
||||
"type": "string",
|
||||
"description": ""
|
||||
},
|
||||
"tls": {
|
||||
"type": "boolean",
|
||||
"default": true,
|
||||
"description": ""
|
||||
},
|
||||
"servers": [
|
||||
{
|
||||
"host": {
|
||||
"type": "string",
|
||||
"description": ""
|
||||
},
|
||||
"port": {
|
||||
"type": "integer",
|
||||
"default": 27017,
|
||||
"description": ""
|
||||
}
|
||||
}
|
||||
],
|
||||
"credential": {
|
||||
"db": {
|
||||
"type": "string",
|
||||
"description": ""
|
||||
},
|
||||
"password": {
|
||||
"type": "string",
|
||||
"description": ""
|
||||
},
|
||||
"username": {
|
||||
"type": "string",
|
||||
"description": ""
|
||||
},
|
||||
"mechanism": {
|
||||
"type": "string",
|
||||
"default": "SCRAM-SHA-1",
|
||||
"description": ""
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
},
|
||||
{
|
||||
"name": "mysql",
|
||||
"description": "Connection settings for a MySQL database, with optional SSL and a root CA certificate.",
|
||||
"schema": {
|
||||
"type": "object",
|
||||
"order": [
|
||||
"host",
|
||||
"port",
|
||||
"user",
|
||||
"database",
|
||||
"password",
|
||||
"ssl",
|
||||
"root_certificate_pem"
|
||||
],
|
||||
"$schema": "https://json-schema.org/draft/2020-12/schema",
|
||||
"required": [
|
||||
"user",
|
||||
"database",
|
||||
"password"
|
||||
],
|
||||
"properties": {
|
||||
"ssl": {
|
||||
"type": "boolean",
|
||||
"default": false,
|
||||
"description": "use ssl"
|
||||
},
|
||||
"host": {
|
||||
"type": "string",
|
||||
"default": "",
|
||||
"description": "instance host"
|
||||
},
|
||||
"port": {
|
||||
"type": "number",
|
||||
"default": 3306,
|
||||
"description": "instance port"
|
||||
},
|
||||
"user": {
|
||||
"type": "string",
|
||||
"default": "",
|
||||
"description": "username"
|
||||
},
|
||||
"database": {
|
||||
"type": "string",
|
||||
"default": "",
|
||||
"description": "database name"
|
||||
},
|
||||
"password": {
|
||||
"type": "string",
|
||||
"default": "",
|
||||
"description": "user's password"
|
||||
},
|
||||
"root_certificate_pem": {
|
||||
"type": "string",
|
||||
"default": "",
|
||||
"description": ""
|
||||
}
|
||||
}
|
||||
}
|
||||
},
|
||||
{
|
||||
"name": "notion",
|
||||
"description": "Notion API token: filled by Windmill's Notion OAuth connect, or an internal connection's API token / personal access token pasted in.",
|
||||
"schema": {
|
||||
"type": "object",
|
||||
"$schema": "https://json-schema.org/draft/2020-12/schema",
|
||||
"required": [
|
||||
"token"
|
||||
],
|
||||
"properties": {
|
||||
"token": {
|
||||
"type": "string",
|
||||
"default": "",
|
||||
"description": "Filled automatically when you connect with OAuth. To paste one instead: the API token of an internal connection (ntn_\u2026) or a personal access token, both created in the Notion developer portal at https://www.notion.so/developers. A connection token only reaches the pages granted to it under Content access."
|
||||
}
|
||||
}
|
||||
}
|
||||
},
|
||||
{
|
||||
"name": "openai",
|
||||
"description": "OpenAI secret API key, with optional organization ID and base URL override, for the OpenAI API.",
|
||||
"schema": {
|
||||
"type": "object",
|
||||
"order": [
|
||||
"api_key",
|
||||
"organization_id",
|
||||
"base_url"
|
||||
],
|
||||
"$schema": "https://json-schema.org/draft/2020-12/schema",
|
||||
"required": [
|
||||
"api_key"
|
||||
],
|
||||
"properties": {
|
||||
"api_key": {
|
||||
"type": "string",
|
||||
"default": "",
|
||||
"description": ""
|
||||
},
|
||||
"base_url": {
|
||||
"type": "string",
|
||||
"default": "",
|
||||
"description": "Optional. Only needed to overwrite the default base URL."
|
||||
},
|
||||
"organization_id": {
|
||||
"type": "string",
|
||||
"default": "",
|
||||
"description": "Only needs to be set for users who belong to multiple organizations and they want to use an organization other than the default one"
|
||||
}
|
||||
}
|
||||
}
|
||||
},
|
||||
{
|
||||
"name": "postgresql",
|
||||
"description": "Connection settings for a PostgreSQL database: host, port, database name, user, password and SSL/IAM auth options.",
|
||||
"schema": {
|
||||
"type": "object",
|
||||
"$schema": "https://json-schema.org/draft/2020-12/schema",
|
||||
"required": [
|
||||
"dbname",
|
||||
"user",
|
||||
"password"
|
||||
],
|
||||
"properties": {
|
||||
"host": {
|
||||
"type": "string",
|
||||
"description": "The instance host"
|
||||
},
|
||||
"port": {
|
||||
"type": "integer",
|
||||
"description": "The instance port"
|
||||
},
|
||||
"user": {
|
||||
"type": "string",
|
||||
"description": "The postgres username"
|
||||
},
|
||||
"dbname": {
|
||||
"type": "string",
|
||||
"description": "The database name"
|
||||
},
|
||||
"region": {
|
||||
"type": "string",
|
||||
"description": "AWS region for IAM auth (e.g. us-east-1). Falls back to AWS_REGION env var if not set."
|
||||
},
|
||||
"options": {
|
||||
"type": "string",
|
||||
"description": "Startup options sent to the server (the libpq options parameter), such as endpoint=<endpoint-id> for a Neon pooled or private connection, or -c search_path=myschema"
|
||||
},
|
||||
"sslmode": {
|
||||
"enum": [
|
||||
"disable",
|
||||
"allow",
|
||||
"prefer",
|
||||
"require",
|
||||
"verify-ca",
|
||||
"verify-full"
|
||||
],
|
||||
"type": "string",
|
||||
"description": "Most cloud databases (Heroku, AWS, Supabase, etc.) require SSL. Self-hosted databases typically don't."
|
||||
},
|
||||
"password": {
|
||||
"type": "string",
|
||||
"description": "The postgres users password"
|
||||
},
|
||||
"use_iam_auth": {
|
||||
"type": "boolean",
|
||||
"default": false,
|
||||
"description": "Use IAM authentication (AWS RDS/Aurora). When enabled, the password field is ignored and a short-lived IAM token is generated instead."
|
||||
},
|
||||
"accept_invalid_certs": {
|
||||
"type": "boolean",
|
||||
"default": false,
|
||||
"showExpr": "fields.sslmode == 'verify-ca' || fields.sslmode == 'verify-full'",
|
||||
"description": "Accept any certificate presented by the server, which disables TLS certificate verification and is not recommended. Only applies to verify-ca/verify-full."
|
||||
},
|
||||
"root_certificate_pem": {
|
||||
"type": "string",
|
||||
"default": "",
|
||||
"description": "The root certificate in pe format"
|
||||
}
|
||||
}
|
||||
}
|
||||
},
|
||||
{
|
||||
"name": "s3",
|
||||
"description": "Connection settings for an Amazon S3 (AWS) bucket: endpoint, region, bucket name and access keys.",
|
||||
"schema": {
|
||||
"type": "object",
|
||||
"order": [
|
||||
"endPoint",
|
||||
"region",
|
||||
"pathStyle",
|
||||
"bucket",
|
||||
"accessKey",
|
||||
"secretKey",
|
||||
"port",
|
||||
"useSSL"
|
||||
],
|
||||
"$schema": "https://json-schema.org/draft/2020-12/schema",
|
||||
"required": [
|
||||
"bucket"
|
||||
],
|
||||
"properties": {
|
||||
"port": {
|
||||
"type": "number",
|
||||
"description": "Deprecated: pass in a complete URL to `endPoint` instead, including a port number if needed"
|
||||
},
|
||||
"bucket": {
|
||||
"type": "string",
|
||||
"default": "",
|
||||
"description": ""
|
||||
},
|
||||
"region": {
|
||||
"type": "string",
|
||||
"default": "",
|
||||
"description": "AWS region (e.g. us-east-1). Used to auto-generate the S3 endpoint URL when endPoint is not specified"
|
||||
},
|
||||
"useSSL": {
|
||||
"type": "boolean",
|
||||
"default": true,
|
||||
"description": "Deprecated: pass in a complete URL to `endPoint` instead, including 'https://' or 'http://'"
|
||||
},
|
||||
"endPoint": {
|
||||
"type": "string",
|
||||
"default": "",
|
||||
"description": "The full URL of the S3 endpoint (e.g. https://s3.us-east-1.amazonaws.com)"
|
||||
},
|
||||
"accessKey": {
|
||||
"type": "string",
|
||||
"default": "",
|
||||
"description": ""
|
||||
},
|
||||
"pathStyle": {
|
||||
"type": "boolean",
|
||||
"default": true,
|
||||
"description": "<host>/<bucket>/<path> instead of <bucket>.<host>/<path>"
|
||||
},
|
||||
"secretKey": {
|
||||
"type": "string",
|
||||
"default": "",
|
||||
"description": ""
|
||||
}
|
||||
}
|
||||
}
|
||||
},
|
||||
{
|
||||
"name": "salesforce",
|
||||
"description": "Salesforce OAuth Authorization Code credential. The token is populated by Windmill's OAuth flow; instance_url pins every REST call to the org's data host, api_version selects the REST API version.",
|
||||
"schema": {
|
||||
"type": "object",
|
||||
"$schema": "https://json-schema.org/draft/2020-12/schema",
|
||||
"required": [
|
||||
"token",
|
||||
"instance_url"
|
||||
],
|
||||
"properties": {
|
||||
"token": {
|
||||
"type": "string",
|
||||
"description": "OAuth access token. Populated by Windmill's OAuth flow."
|
||||
},
|
||||
"api_version": {
|
||||
"type": "string",
|
||||
"description": "Salesforce REST API version path segment, e.g. v60.0. Optional \u2014 scripts default to v60.0 when left blank."
|
||||
},
|
||||
"instance_url": {
|
||||
"type": "string",
|
||||
"description": "Org data host returned in the OAuth token response, e.g. https://acme.my.salesforce.com (production) or https://acme--sandbox.sandbox.my.salesforce.com (sandbox). Every REST call is made against this host, not login.salesforce.com."
|
||||
}
|
||||
}
|
||||
}
|
||||
},
|
||||
{
|
||||
"name": "sendgrid",
|
||||
"description": "Twilio SendGrid API key for the SendGrid v3 API: transactional email, contacts, marketing lists and bounces.",
|
||||
"schema": {
|
||||
"type": "object",
|
||||
"$schema": "https://json-schema.org/draft/2020-12/schema",
|
||||
"required": [
|
||||
"token"
|
||||
],
|
||||
"properties": {
|
||||
"token": {
|
||||
"type": "string",
|
||||
"default": "",
|
||||
"description": "API token"
|
||||
}
|
||||
}
|
||||
}
|
||||
},
|
||||
{
|
||||
"name": "shopify",
|
||||
"description": "Credentials for the Shopify Admin API using a store name and access token.",
|
||||
"schema": {
|
||||
"type": "object",
|
||||
"$schema": "https://json-schema.org/draft/2020-12/schema",
|
||||
"required": [
|
||||
"store_name",
|
||||
"token"
|
||||
],
|
||||
"properties": {
|
||||
"token": {
|
||||
"type": "string",
|
||||
"default": "",
|
||||
"description": ""
|
||||
},
|
||||
"store_name": {
|
||||
"type": "string",
|
||||
"default": "",
|
||||
"description": ""
|
||||
}
|
||||
}
|
||||
}
|
||||
},
|
||||
{
|
||||
"name": "slack",
|
||||
"description": "Slack bot or user OAuth token authorizing the Slack Web API (messages, channels, users and files).",
|
||||
"schema": {
|
||||
"type": "object",
|
||||
"$schema": "https://json-schema.org/draft/2020-12/schema",
|
||||
"required": [],
|
||||
"properties": {
|
||||
"token": {
|
||||
"type": "string",
|
||||
"description": "The slack token"
|
||||
}
|
||||
}
|
||||
}
|
||||
},
|
||||
{
|
||||
"name": "smtp",
|
||||
"description": "Connection settings for an SMTP mail server used to send email.",
|
||||
"schema": {
|
||||
"type": "object",
|
||||
"$schema": "https://json-schema.org/draft/2020-12/schema",
|
||||
"required": [
|
||||
"host"
|
||||
],
|
||||
"properties": {
|
||||
"host": {
|
||||
"type": "string",
|
||||
"default": "mail.smtpbucket.com",
|
||||
"description": "SMTP host address"
|
||||
},
|
||||
"port": {
|
||||
"type": "integer",
|
||||
"default": 8025,
|
||||
"description": "port number on which to connect"
|
||||
},
|
||||
"user": {
|
||||
"type": "string",
|
||||
"description": ""
|
||||
},
|
||||
"password": {
|
||||
"type": "string",
|
||||
"description": ""
|
||||
}
|
||||
}
|
||||
}
|
||||
},
|
||||
{
|
||||
"name": "stripe",
|
||||
"description": "Stripe secret API key for the Stripe payments API.",
|
||||
"schema": {
|
||||
"type": "object",
|
||||
"$schema": "https://json-schema.org/draft/2020-12/schema",
|
||||
"required": [],
|
||||
"properties": {
|
||||
"token": {
|
||||
"type": "string",
|
||||
"description": ""
|
||||
}
|
||||
}
|
||||
}
|
||||
},
|
||||
{
|
||||
"name": "supabase",
|
||||
"description": "Supabase project URL and API key for the Supabase REST, auth and storage APIs.\n\nRetrieve both values in \"Project Settings -> API\".",
|
||||
"schema": {
|
||||
"type": "object",
|
||||
"$schema": "https://json-schema.org/draft/2020-12/schema",
|
||||
"required": [],
|
||||
"properties": {
|
||||
"key": {
|
||||
"type": "string",
|
||||
"default": "",
|
||||
"description": "The unique Supabase Key which is supplied when you create a new project in your project dashboard."
|
||||
},
|
||||
"url": {
|
||||
"type": "string",
|
||||
"default": "",
|
||||
"description": "The unique Supabase URL which is supplied when you create a new project in your project dashboard."
|
||||
}
|
||||
}
|
||||
}
|
||||
},
|
||||
{
|
||||
"name": "twilio",
|
||||
"description": "Credentials for the Twilio messaging and voice API using an Account SID and auth token.",
|
||||
"schema": {
|
||||
"type": "object",
|
||||
"$schema": "https://json-schema.org/draft/2020-12/schema",
|
||||
"required": [
|
||||
"token",
|
||||
"accountSid"
|
||||
],
|
||||
"properties": {
|
||||
"token": {
|
||||
"type": "string",
|
||||
"default": "",
|
||||
"description": ""
|
||||
},
|
||||
"accountSid": {
|
||||
"type": "string",
|
||||
"default": "",
|
||||
"description": ""
|
||||
}
|
||||
}
|
||||
}
|
||||
},
|
||||
{
|
||||
"name": "zendesk",
|
||||
"description": "Zendesk API credentials: account subdomain plus email and password or API token, used with HTTP Basic auth.",
|
||||
"schema": {
|
||||
"type": "object",
|
||||
"$schema": "https://json-schema.org/draft/2020-12/schema",
|
||||
"required": [
|
||||
"username",
|
||||
"subdomain",
|
||||
"password"
|
||||
],
|
||||
"properties": {
|
||||
"password": {
|
||||
"type": "string",
|
||||
"default": "",
|
||||
"description": "password or api token"
|
||||
},
|
||||
"username": {
|
||||
"type": "string",
|
||||
"default": "",
|
||||
"description": "email or \"{email}/token\" if using an api token"
|
||||
},
|
||||
"subdomain": {
|
||||
"type": "string",
|
||||
"default": "",
|
||||
"description": ""
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
]
|
||||
@@ -4283,7 +4283,7 @@ export const globalTools: SessionTool<{}>[] = [
|
||||
toolCallbacks.setToolStatus(toolId, {
|
||||
content: `Searching resource types for "${parsed.query}"...`
|
||||
})
|
||||
const { resourceTypes, hubSyncNote } = await searchResourceTypes(
|
||||
const { resourceTypes, note } = await searchResourceTypes(
|
||||
parsed.query,
|
||||
workspace,
|
||||
parsed.limit ?? 5
|
||||
@@ -4291,11 +4291,7 @@ export const globalTools: SessionTool<{}>[] = [
|
||||
toolCallbacks.setToolStatus(toolId, {
|
||||
content: `Found ${resourceTypes.length} resource type(s) for "${parsed.query}"`
|
||||
})
|
||||
return JSON.stringify(
|
||||
hubSyncNote ? { resource_types: resourceTypes, note: hubSyncNote } : resourceTypes,
|
||||
null,
|
||||
2
|
||||
)
|
||||
return JSON.stringify(note ? { resource_types: resourceTypes, note } : resourceTypes, null, 2)
|
||||
}
|
||||
},
|
||||
createDbSchemaTool<{}>({
|
||||
|
||||
@@ -30,10 +30,25 @@ describe('searchResourceTypes', () => {
|
||||
schema: { type: 'object' }
|
||||
})
|
||||
|
||||
const { resourceTypes, hubSyncNote } = await searchResourceTypes('Stripe payments', 'ws', 5)
|
||||
const { resourceTypes, note } = await searchResourceTypes('Stripe payments', 'ws', 5)
|
||||
|
||||
expect(resourceTypes.map((rt) => rt.name)).toEqual(['stripe'])
|
||||
expect(hubSyncNote).toBeUndefined()
|
||||
expect(note).toBeUndefined()
|
||||
})
|
||||
|
||||
// An empty result from a failed search says nothing about whether a type exists: read as
|
||||
// "none", it sends the model to define a type the instance already has.
|
||||
it('hands back the type names when the similarity search fails', async () => {
|
||||
vi.spyOn(ResourceService, 'queryResourceTypes').mockRejectedValue(new Error('500'))
|
||||
vi.spyOn(ResourceService, 'listResourceTypeNames').mockResolvedValue([
|
||||
...syncedNames,
|
||||
'postgresql'
|
||||
])
|
||||
|
||||
const { resourceTypes, note } = await searchResourceTypes('postgres database', 'ws', 5)
|
||||
|
||||
expect(resourceTypes).toEqual([])
|
||||
expect(note).toContain('postgresql')
|
||||
})
|
||||
|
||||
it('notes an instance that never synced with the hub', async () => {
|
||||
@@ -43,9 +58,9 @@ describe('searchResourceTypes', () => {
|
||||
'ai_instruction'
|
||||
])
|
||||
|
||||
const { hubSyncNote } = await searchResourceTypes('stripe', 'ws', 5)
|
||||
const { note } = await searchResourceTypes('stripe', 'ws', 5)
|
||||
|
||||
expect(hubSyncNote).toContain('never synced')
|
||||
expect(note).toContain('never synced')
|
||||
})
|
||||
})
|
||||
|
||||
|
||||
@@ -80,9 +80,13 @@ export async function searchResourceTypes(
|
||||
query: string,
|
||||
workspace: string,
|
||||
limit: number
|
||||
): Promise<{ resourceTypes: Pick<ResourceType, 'name' | 'schema'>[]; hubSyncNote?: string }> {
|
||||
): Promise<{ resourceTypes: Pick<ResourceType, 'name' | 'schema'>[]; note?: string }> {
|
||||
let similarityFailed = false
|
||||
const [similar, names] = await Promise.all([
|
||||
ResourceService.queryResourceTypes({ workspace, text: query, limit }).catch(() => []),
|
||||
ResourceService.queryResourceTypes({ workspace, text: query, limit }).catch(() => {
|
||||
similarityFailed = true
|
||||
return []
|
||||
}),
|
||||
ResourceService.listResourceTypeNames({ workspace })
|
||||
.then((listed) => new Set(listed))
|
||||
.catch(() => undefined)
|
||||
@@ -108,12 +112,18 @@ export async function searchResourceTypes(
|
||||
...similar
|
||||
].slice(0, limit)
|
||||
|
||||
const hubSyncNote =
|
||||
// Without the similarity search, an empty result says nothing about whether a type exists,
|
||||
// so the model gets the names to search again with rather than concluding there is none.
|
||||
const notes = [
|
||||
similarityFailed && names
|
||||
? `The similarity search is unavailable on this instance, so only a type named in the query can be found. Search again with one of the instance's resource types: ${[...names].join(', ')}.`
|
||||
: undefined,
|
||||
names && names.size < UNSYNCED_RESOURCE_TYPE_COUNT
|
||||
? `Only ${names.size} resource types exist on this instance, so it has most likely never synced with the Windmill Hub, which has types for most services. ${HUB_SYNC_INSTRUCTIONS}`
|
||||
: undefined
|
||||
].filter((note) => note !== undefined)
|
||||
|
||||
return { resourceTypes, hubSyncNote }
|
||||
return { resourceTypes, note: notes.length > 0 ? notes.join('\n\n') : undefined }
|
||||
}
|
||||
|
||||
export const SUPPORTED_CHAT_SCRIPT_LANGUAGES = [
|
||||
@@ -188,14 +198,14 @@ export async function getFormattedResourceTypes(
|
||||
case 'bunnative':
|
||||
case 'python3':
|
||||
case 'php': {
|
||||
const { resourceTypes, hubSyncNote } = await searchResourceTypes(prompt, workspace, 5)
|
||||
const { resourceTypes, note } = await searchResourceTypes(prompt, workspace, 5)
|
||||
|
||||
const found =
|
||||
resourceTypes.length > 0
|
||||
? `RESOURCE_TYPES:\n${formatResourceTypes(resourceTypes, lang)}`
|
||||
: `No resource type matches "${prompt}".`
|
||||
: `No resource type found for "${prompt}".`
|
||||
|
||||
return hubSyncNote ? `${found}\n\n${hubSyncNote}` : found
|
||||
return note ? `${found}\n\n${note}` : found
|
||||
}
|
||||
default:
|
||||
return ''
|
||||
|
||||
Reference in New Issue
Block a user