stabilize global ai eval smoke path (#9745)

This commit is contained in:
centdix
2026-06-24 00:31:14 +02:00
committed by GitHub
parent 9e4cf139b1
commit ae088fd032
5 changed files with 25 additions and 0 deletions
+1
View File
@@ -3,6 +3,7 @@
Create a draft Bun script at `f/evals/global/greet_user`.
It should take a string `name` input and return `Hello, ${name}!`.
Leave it as an AI draft only; do not deploy or save it.
initial: ai_evals/fixtures/frontend/global/initial/user_admin_evals_folder.json
runtime:
maxTurns: 10
validate:
+3
View File
@@ -212,6 +212,9 @@ describe("loadCases", () => {
},
],
});
expect(caseEntry?.initialPath).toContain(
"ai_evals/fixtures/frontend/global/initial/user_admin_evals_folder.json"
);
expect(caseEntry?.toolExpect).toMatchObject({
requiredToolsUsed: ["write_script"],
forbiddenToolsUsed: ["deploy_workspace_item", "delete_workspace_item"],
@@ -0,0 +1,8 @@
{
"user": {
"username": "admin",
"is_admin": true,
"folders": ["evals"],
"folders_read": ["evals"]
}
}
@@ -2757,6 +2757,18 @@ describe('prepareGlobalSystemMessage', () => {
expect(content).not.toContain('frontend AI draft store')
})
it('honors user-supplied shared folder paths without asking first', () => {
const content = prepareGlobalSystemMessage(undefined, {
user: { username: 'admin', is_admin: true, folders: ['evals'] }
}).content as string
expect(content).toContain(
'If the user supplies a fully qualified `f/<folder>/...` path, use that exact path'
)
expect(content).toContain('Do not ask for folder confirmation')
expect(content).toContain('substitute a `u/admin/...` path unless a tool rejects it')
})
describe('folder guidance', () => {
const guidanceOf = (user: {
username: string
@@ -748,6 +748,7 @@ Path conventions:
- A workspace path starts with one of two namespaces; its trailing <name> may itself contain "/", so a path has three or more segments:
- \`u/${username}/<name>\` — your personal scope. Default for ad-hoc, exploratory, or scratch work.
- \`f/<folder>/<name>\` — a shared folder scope; the <folder> must already exist (a bare \`f/<name>\` with no folder segment is INVALID and will fail).
- If the user supplies a fully qualified \`f/<folder>/...\` path, use that exact path; they have already chosen the folder. Do not ask for folder confirmation or substitute a \`u/${username}/...\` path unless a tool rejects it.
- Default a bare name with no namespace prefix (e.g. "create a flow called myflow") to \`u/${username}/<name>\`. Never invent an \`f/<folder>/...\` path for a folder that does not exist.${folderGuidanceBlock}
Rules: