From 589a06922810e3ea554c48fa85ff2839626f5121 Mon Sep 17 00:00:00 2001 From: hugocasa Date: Fri, 9 Oct 2026 10:09:59 +0200 Subject: [PATCH] test: require the c_ prefix in in-house API evals and a real sync mention Co-Authored-By: Claude Opus 5.5 --- ai_evals/cases/cli.yaml | 1 + ai_evals/cases/flow.yaml | 1 + ai_evals/cases/global.yaml | 7 ++++++- ai_evals/cases/script.yaml | 1 + 4 files changed, 9 insertions(+), 1 deletion(-) diff --git a/ai_evals/cases/cli.yaml b/ai_evals/cases/cli.yaml index 744c4124b0..fdf469130d 100644 --- a/ai_evals/cases/cli.yaml +++ b/ai_evals/cases/cli.yaml @@ -99,6 +99,7 @@ - creates the script at f/evals/get_invoice.ts, fetching an invoice by id with a bearer token - the script takes the billing API's base URL and token together as one resource-typed parameter, not a plain string token or a variable read inside the code - defines the billing API's resource type with its base URL and token fields, as a resource-type file or in the reply (asking before writing the file is fine) + - the custom resource type's name has the c_ prefix (e.g. `RT.CAcmeBilling` in TypeScript, `c_acme_billing.resource-type.json`) - id: flow-reuse-existing-script prompt: |- diff --git a/ai_evals/cases/flow.yaml b/ai_evals/cases/flow.yaml index b5e9490c1b..7454a85c9c 100644 --- a/ai_evals/cases/flow.yaml +++ b/ai_evals/cases/flow.yaml @@ -24,6 +24,7 @@ judgeChecklist: - the flow takes the billing API's base URL and token together as one resource-typed input, not a plain string token or a variable read inside the code - the flow fetches an invoice by id with a bearer token + - the custom resource type's name has the c_ prefix (e.g. `resource-c_acme_billing`) - id: flow-test1-reuse-existing-script prompt: |- diff --git a/ai_evals/cases/global.yaml b/ai_evals/cases/global.yaml index 31ca352100..dbd1323e5b 100644 --- a/ai_evals/cases/global.yaml +++ b/ai_evals/cases/global.yaml @@ -781,8 +781,12 @@ - create_resource_type - deploy_workspace_item assistantExpect: + # Plain substring test: a bare "sync" would also match "async". requiredMentionsAnyOf: - - - sync + - - " sync " + - sync resource types + - synced + - syncing judgeChecklist: - the script takes the Confluence connection as one resource-typed parameter rather than separate string credentials or a variable read inside the code - the script creates a Confluence page from a space key, a title and a body @@ -812,6 +816,7 @@ judgeChecklist: - the script takes the billing API's base URL and token together as one resource-typed parameter, not a plain string token or a variable read inside the code - the script fetches an invoice by id with a bearer token + - the custom resource type's name has the c_ prefix (e.g. `RT.CAcmeBilling` in TypeScript, `resource-c_acme_billing` in the schema) - id: global-test19-datatable-not-configured-asks-to-set-up prompt: |- diff --git a/ai_evals/cases/script.yaml b/ai_evals/cases/script.yaml index 91cf4ad257..c4ed76605b 100644 --- a/ai_evals/cases/script.yaml +++ b/ai_evals/cases/script.yaml @@ -22,6 +22,7 @@ judgeChecklist: - the script takes the billing API's base URL and token together as one resource-typed parameter, not a plain string token or a variable read inside the code - the script fetches an invoice by id with a bearer token + - the custom resource type's name has the c_ prefix (e.g. `RT.CAcmeBilling` in TypeScript) - id: script-test2-create-current-script-schedule prompt: |-