diff --git a/ai_evals/adapters/frontend/core/global/globalEvalRunner.ts b/ai_evals/adapters/frontend/core/global/globalEvalRunner.ts index fe20d77d05..de0078d399 100644 --- a/ai_evals/adapters/frontend/core/global/globalEvalRunner.ts +++ b/ai_evals/adapters/frontend/core/global/globalEvalRunner.ts @@ -41,6 +41,7 @@ import type { TokenUsage, ToolCallDetail } from "../shared/types"; const MUTATING_GLOBAL_TOOLS = new Set([ "deploy_workspace_item", "delete_workspace_item", + "create_resource_type", ]); const DISABLE_ACTIVE_EDITOR_CONTEXT_ENV = "WMILL_AI_EVAL_DISABLE_ACTIVE_EDITOR_CONTEXT"; diff --git a/ai_evals/cases/global.yaml b/ai_evals/cases/global.yaml index 59ea364a77..d24e8a126f 100644 --- a/ai_evals/cases/global.yaml +++ b/ai_evals/cases/global.yaml @@ -761,7 +761,8 @@ # The instance never synced the hub's resource types, so the one Confluence takes is missing: # the script's connection picker stays empty until a superadmin syncs, which only the -# assistant's reply can tell the user. +# assistant's reply can tell the user. A type of its own would duplicate the hub's once synced, +# so it is only for a user who says they would rather not sync. - id: global-resource-type-hub-not-synced prompt: |- Write a script that creates a page in our Confluence space from a space key, a title and the page body. @@ -777,6 +778,7 @@ toolExpect: forbiddenToolsUsed: - write_variable + - create_resource_type - deploy_workspace_item assistantExpect: requiredMentionsAnyOf: @@ -786,7 +788,8 @@ - the script creates a Confluence page from a space key, a title and a body # An in-house API has no hub resource type, so its connection needs a type of the workspace's -# own: a token parameter or a variable read inside the code is the fallback the guidance rules out. +# own, created once the user confirms its schema: a token parameter or a variable read inside +# the code is the fallback the guidance rules out. - id: global-resource-type-custom-api prompt: |- Write a script that fetches an invoice by id from our internal billing API at https://billing.acme.internal/api/v1. It authenticates with a bearer token. @@ -801,6 +804,8 @@ - getVariable - get_variable toolExpect: + requiredToolsUsed: + - create_resource_type forbiddenToolsUsed: - write_variable - deploy_workspace_item diff --git a/frontend/src/lib/components/copilot/chat/global/core.ts b/frontend/src/lib/components/copilot/chat/global/core.ts index be9fabd211..d9be16a4f0 100644 --- a/frontend/src/lib/components/copilot/chat/global/core.ts +++ b/frontend/src/lib/components/copilot/chat/global/core.ts @@ -721,6 +721,20 @@ const searchResourceTypesSchema = z.object({ .describe('Max number of resource types to return. Defaults to 5.') }) +const createResourceTypeSchema = z.object({ + name: z + .string() + .describe( + 'Type name, 1 to 50 letters, digits, "_" or "-", e.g. acme_billing. Resources and resource-typed parameters refer to it by this name.' + ), + description: z.string().describe('What a resource of this type connects to.'), + schema: z + .record(z.string(), z.any()) + .describe( + 'JSON Schema of a resource value of this type: { "type": "object", "properties": {...}, "required": [...] }, with every connection field, secrets included (a resource fills those with "$var:" references).' + ) +}) + const getRunSchema = z.object({ id: z.string().describe('The UUID of the run (job) to inspect.'), step: z @@ -1404,8 +1418,10 @@ const buildGlobalSystemPrompt = ( // rule cannot outlive the tool it describes. An unresolved profile keeps every block. const canWriteDraft = !access || access.has('write_draft') const canRunPreview = !access || access.has('run_preview') - // The deploy tools take their kind as an argument, so `deploy` gates only the folder text. + // The deploy tools take their kind as an argument, so `deploy` gates only the folder and + // resource type text. const canCreateFolder = !access || access.has('deploy') + const canCreateResourceType = !access || access.has('deploy') // Each gated block carries its own leading newline, so dropping one leaves no blank // line behind and a full-access prompt is byte-for-byte the ungated text. const when = (cond: boolean, block: string) => (cond ? block : '') @@ -1488,7 +1504,7 @@ Rules:${when( - Hub scripts are prebuilt, vetted integrations for third-party services, hosted outside the workspace under \`hub///\` paths. Check search_hub_scripts before hand-writing code against a third-party API, even when the user never mentions the hub; read a result with read_workspace_item type "script" and its hub path to get its code, language, and input schema. Use what you find in whichever way fits: reference the hub path directly from a flow module or app runnable when a script already does the job, copy it into a workspace draft and adapt it when it is close (note the source hub path in a comment at the top of the code), or take it as a worked example and write your own. A script that does not do what the user asked is still worth reading when it is the only example of that integration: pass its \`integration\` back to search_hub_scripts to list that integration's other scripts with their descriptions, or use the \`suggested_integrations\` a search hands back when it finds nothing. - Before writing your own code against an integration the hub covers, call get_hub_integration with its slug: it returns the resource type to take, its auth fields and the integration's most-used scripts, which beats inferring them from script bodies. Call it for the integration you are about to write against, whichever it is. A search marks an integration \`documented\` when the hub additionally holds provider knowledge checked against the live API — pagination, enums, error codes and gotchas — so read that closely where it appears rather than trusting your own memory of the API. - If you have a web search tool and the hub does not cover a third-party API, search for the vendor's own API documentation rather than writing its endpoints and auth from memory, and link the page you relied on. Reserve it for external APIs: search_docs answers questions about Windmill itself. -${when(canRunPreview, '- Use get_db_schema with a database resource path to fetch its tables and columns before writing SQL (or a script querying that database).\n')}- Use get_instructions before writing scripts, flows, resources, or apps. For scripts, pass the target language. +${when(canCreateResourceType, '- create_resource_type creates a resource type the workspace lacks. The user confirms the schema in the call itself, so that is where you propose it rather than asking first in chat.\n')}${when(canRunPreview, '- Use get_db_schema with a database resource path to fetch its tables and columns before writing SQL (or a script querying that database).\n')}- Use get_instructions before writing scripts, flows, resources, or apps. For scripts, pass the target language. ${pipelineBullet}` )}${when( canRunPreview && canWriteDraft, @@ -4294,6 +4310,33 @@ export const globalTools: SessionTool<{}>[] = [ return JSON.stringify(note ? { resource_types: resourceTypes, note } : resourceTypes, null, 2) } }, + { + // Not a draft: resource types have no draft kind, so this writes the workspace once the + // user confirms, under the same right as a deploy. + requires: DEPLOY, + def: createToolDef( + createResourceTypeSchema, + 'create_resource_type', + 'Create a resource type in the workspace. Mutates the workspace once the user confirms the schema.', + { strict: false } + ), + showDetails: true, + requiresConfirmation: true, + confirmationMessage: 'Create resource type', + fn: async ({ args, workspace, toolId, toolCallbacks }) => { + const parsed = createResourceTypeSchema.parse(args) + await ResourceService.createResourceType({ + workspace, + requestBody: { + name: parsed.name, + description: parsed.description, + schema: parsed.schema + } + }) + toolCallbacks.setToolStatus(toolId, { content: `Created resource type ${parsed.name}` }) + return JSON.stringify({ success: true, message: `Created resource type ${parsed.name}` }) + } + }, createDbSchemaTool<{}>({ description: 'Fetch the schema (tables and columns) of a database resource by its path. Supports postgresql, mysql, ms_sql_server, snowflake and bigquery resources.', diff --git a/system_prompts/auto-generated/prompts.ts b/system_prompts/auto-generated/prompts.ts index c252683888..985d589ae8 100644 --- a/system_prompts/auto-generated/prompts.ts +++ b/system_prompts/auto-generated/prompts.ts @@ -834,7 +834,7 @@ The Windmill Hub publishes resource types for most services, and an instance get - If looking up resource types reports that the instance is missing the hub's types, mention that a superadmin can sync them: the hub most likely has one for the service. Create one of your own only if the user still wants it, since after a sync the workspace would hold both. - Otherwise, or when the user prefers it to syncing, propose a resource type holding the service's connection fields (base URL, account or region identifiers, credentials), as in "Custom Resource Types" below, and ask the user before creating it: every resource of that type and every script taking one depends on its fields. -- You cannot create a resource type yourself: give the user the schema to add with **Add resource type** on the Resources page. +- Where you have no way to create a resource type yourself, give the user the schema to add with **Add resource type** on the Resources page. - A plain variable instead of a resource is fine when the user asks for it; without that, default to the resource, its secret in a variable it references. ## Custom Resource Types diff --git a/system_prompts/base/resources.md b/system_prompts/base/resources.md index cc9733f412..2e8f497006 100644 --- a/system_prompts/base/resources.md +++ b/system_prompts/base/resources.md @@ -220,7 +220,7 @@ The Windmill Hub publishes resource types for most services, and an instance get - If looking up resource types reports that the instance is missing the hub's types, mention that a superadmin can sync them: the hub most likely has one for the service. Create one of your own only if the user still wants it, since after a sync the workspace would hold both. - Otherwise, or when the user prefers it to syncing, propose a resource type holding the service's connection fields (base URL, account or region identifiers, credentials), as in "Custom Resource Types" below, and ask the user before creating it: every resource of that type and every script taking one depends on its fields. -- You cannot create a resource type yourself: give the user the schema to add with **Add resource type** on the Resources page. +- Where you have no way to create a resource type yourself, give the user the schema to add with **Add resource type** on the Resources page. - A plain variable instead of a resource is fine when the user asks for it; without that, default to the resource, its secret in a variable it references.