diff --git a/ai_evals/adapters/frontend/mockBackend.ts b/ai_evals/adapters/frontend/mockBackend.ts index d27bb4d69a..950cc2e1b5 100644 --- a/ai_evals/adapters/frontend/mockBackend.ts +++ b/ai_evals/adapters/frontend/mockBackend.ts @@ -49,12 +49,16 @@ export interface BenchmarkWorkspaceFlow { export interface BenchmarkWorkspaceApp { path: string summary: string + /** Defaults to true. Set false for a drag-and-drop app, which the chat can list + * and read but has no tool to edit — its value is a grid, not files. */ + rawApp?: boolean value: { - files: Record - runnables: Record + files?: Record + runnables?: Record data?: unknown policy?: unknown custom_path?: unknown + [key: string]: unknown } } @@ -994,7 +998,7 @@ function buildBenchmarkListableApp(app: BenchmarkWorkspaceApp): ListableApp { extra_perms: {}, edited_at: BENCHMARK_TIMESTAMP, execution_mode: 'viewer', - raw_app: true + raw_app: app.rawApp ?? true } } @@ -1012,7 +1016,7 @@ function buildBenchmarkApp(app: BenchmarkWorkspaceApp): AppWithLastVersion { execution_mode: 'viewer', extra_perms: {}, custom_path: app.value.custom_path as string | undefined, - raw_app: true + raw_app: app.rawApp ?? true } } diff --git a/ai_evals/cases/global.yaml b/ai_evals/cases/global.yaml index 45316b2810..fdb2442729 100644 --- a/ai_evals/cases/global.yaml +++ b/ai_evals/cases/global.yaml @@ -2611,3 +2611,40 @@ judgeChecklist: - runs the existing script rather than rewriting it - passes the GitHub resource as the bare string $res:f/evals/global/github_main + +- id: global-drag-and-drop-app-not-editable + prompt: |- + Add a refresh button to the ops console app, and the same to the sales board app. + initial: ai_evals/fixtures/frontend/global/initial/apps_code_and_drag_and_drop.json + runtime: + maxTurns: 12 + validate: + # One request, two apps, only one of them editable: the code app must come back with a + # draft and the drag-and-drop one must not. Refusing both, or editing both, fails here — + # which is what makes this a test of the distinction rather than of caution. + draftCountExactly: 1 + requiredDrafts: + - type: app + path: f/evals/global/ops_console + forbiddenDrafts: + - type: app + path: f/evals/global/sales_board + toolExpect: + # Deliberately not constraining write_app_file/patch_app_file by argument: an entry there + # fails when its tool was never called, so naming both would fail on whichever the model + # did not pick. The draft assertions above cover the same ground, tool-agnostically. + forbiddenToolsUsed: + - init_app + - deploy_workspace_item + - delete_app_file + - delete_app_runnable + assistantExpect: + # A refusal leaves no draft for the judge to read, so the explanation is checked here. + # Only the app kind: substring tests cannot see paraphrase, and every wording of "I can't + # edit it" defeats a fixed list. + requiredMentionsAnyOf: + - - drag-and-drop + - drag and drop + - low-code + - no-code + skipJudge: true diff --git a/ai_evals/fixtures/frontend/global/initial/apps_code_and_drag_and_drop.json b/ai_evals/fixtures/frontend/global/initial/apps_code_and_drag_and_drop.json new file mode 100644 index 0000000000..fbef7b3e3d --- /dev/null +++ b/ai_evals/fixtures/frontend/global/initial/apps_code_and_drag_and_drop.json @@ -0,0 +1,30 @@ +{ + "user": { + "username": "admin", + "is_admin": true, + "folders": ["evals"], + "folders_read": ["evals"] + }, + "workspace": { + "apps": [ + { + "path": "f/evals/global/sales_board", + "summary": "Sales board", + "rawApp": false, + "value": { + "grid": [] + } + }, + { + "path": "f/evals/global/ops_console", + "summary": "Ops console", + "value": { + "files": { + "/App.tsx": "export default function App() {\n\treturn
Ops console
\n}\n" + }, + "runnables": {} + } + } + ] + } +} diff --git a/frontend/src/lib/components/copilot/chat/global/core.test.ts b/frontend/src/lib/components/copilot/chat/global/core.test.ts index 8dec692e16..876e178e31 100644 --- a/frontend/src/lib/components/copilot/chat/global/core.test.ts +++ b/frontend/src/lib/components/copilot/chat/global/core.test.ts @@ -2018,6 +2018,69 @@ describe('global AI tools', () => { }) }) + it('tells a code app from a drag-and-drop app', async () => { + vi.mocked(AppService.listApps).mockResolvedValueOnce([ + { path: 'f/apps/code', summary: 'Code app', raw_app: true }, + { path: 'f/apps/builder', summary: 'Builder app' } + ] as any) + // An app draft is always a code app: the chat cannot address a + // drag-and-drop app's draft kind at all. + seedBackendDraft( + 'raw_app', + 'u/admin/draft_listed', + { summary: 'Draft app' }, + { workspace: WORKSPACE } + ) + + const rows = JSON.parse(await callGlobalTool('list_workspace_items', { types: ['app'] })) + + expect(rows.map((r: any) => [r.path, r.rawApp])).toEqual([ + ['f/apps/code', true], + ['f/apps/builder', false], + ['u/admin/draft_listed', true] + ]) + }) + + it('still says which kind of app it is when the app is read directly', async () => { + // The flag decides whether the app tools are offered at all, and the model + // reads an app before it edits one — a listing that knows is not enough. + vi.mocked(AppService.getAppByPath).mockResolvedValueOnce({ + path: 'f/apps/builder', + summary: 'Builder app', + value: { grid: [] }, + raw_app: false + } as any) + + const read = JSON.parse( + await callGlobalTool('read_workspace_item', { type: 'app', path: 'f/apps/builder' }) + ) + expect(read.rawApp).toBe(false) + }) + + it('finds a staged app under the folder it was filed in, not its generated path', async () => { + // A never-deployed app the editor created lives at a generated path, so the + // folder the user filed it under exists only as its staged name. The server + // drops draft-only rows under any narrowing filter, leaving this pass the one + // that can answer a folder-scoped question about it. + seedBackendDraft( + 'raw_app', + 'u/admin/draft_7f21c9', + { summary: '', draft_path: 'f/team/invoice_tracker' }, + { workspace: WORKSPACE } + ) + + const matched = JSON.parse( + await callGlobalTool('list_workspace_items', { types: ['app'], path_prefix: 'f/team/' }) + ) + expect(matched).toHaveLength(1) + expect(matched[0].draftPath).toBe('f/team/invoice_tracker') + + const other = JSON.parse( + await callGlobalTool('list_workspace_items', { types: ['app'], path_prefix: 'f/other/' }) + ) + expect(other).toEqual([]) + }) + it('applies path_prefix to drafts before enforcing the result limit', async () => { await callGlobalTool('write_script', { path: 'f/other/outside', diff --git a/frontend/src/lib/components/copilot/chat/global/core.ts b/frontend/src/lib/components/copilot/chat/global/core.ts index ad077752c3..6efcb58f98 100644 --- a/frontend/src/lib/components/copilot/chat/global/core.ts +++ b/frontend/src/lib/components/copilot/chat/global/core.ts @@ -1421,6 +1421,7 @@ Flows: - Use patch_flow_json for structural flow edits and write_flow for full flow rewrites. Raw apps: +- The app tools below only work on raw (code) apps. \`rawApp\` says which: false is a drag-and-drop app, which you can list and read but not edit or deploy. Check it before offering to change an app. - read_workspace_item returns app metadata only. Use read_app_file for file and inline runnable contents. - Use write_app_file, patch_app_file, and delete_app_file for frontend files. - Use write_app_runnable and delete_app_runnable for backend runnables. @@ -1526,6 +1527,7 @@ function serializeWorkspaceItemForRead(item: WorkspaceItem): unknown { path: item.path, summary: item.summary, value: summarizeAppValue(item.value as AppDraftValue), + rawApp: item.rawApp, isDraft: item.isDraft } } @@ -1752,6 +1754,9 @@ function appToItem(app: ListableApp | AppWithLastVersion, includeValue: boolean) path: app.path, summary: app.summary, value: includeValue ? ((app as AppWithLastVersion).value as AppDraftValue) : undefined, + // The server omits this flag rather than sending false, so its absence in the + // response is a known false — not a value this listing failed to fetch. + rawApp: app.raw_app ?? false, isDraft: false } } @@ -2038,6 +2043,7 @@ async function readWorkspaceItem( path: app.path, summary: value.summary, value: metadata as unknown as AppDraftValue, + rawApp: app.raw_app, isDraft: false } } @@ -3408,9 +3414,16 @@ export const globalTools: Tool<{}>[] = [ // (it filters before the cap; query filters after). if ((parsed.page ?? 1) === 1) { const draftCountByType = new Map() + const prefix = parsed.path_prefix for (const draft of await listGlobalDrafts(workspace)) { if (!types.includes(draft.type)) continue - if (parsed.path_prefix && !draft.path.startsWith(parsed.path_prefix)) continue + // A draft's staged name is often not where it is stored: the editor parks + // a new script, flow or app at a generated `draft_` path, and a + // rename stages the new name over the old path. The server drops + // draft-only rows under any narrowing filter, leaving this pass their + // only source, so either name has to satisfy the prefix. + if (prefix && !draft.path.startsWith(prefix) && !draft.draftPath?.startsWith(prefix)) + continue const count = draftCountByType.get(draft.type) ?? 0 if (count >= limit) continue draftCountByType.set(draft.type, count + 1) diff --git a/frontend/src/lib/components/copilot/chat/global/userDraftAdapter.ts b/frontend/src/lib/components/copilot/chat/global/userDraftAdapter.ts index 7593546114..c53aa1b7d1 100644 --- a/frontend/src/lib/components/copilot/chat/global/userDraftAdapter.ts +++ b/frontend/src/lib/components/copilot/chat/global/userDraftAdapter.ts @@ -176,6 +176,9 @@ function appDraftToWorkspaceItem(path: string, draft: AppDraftValue): WorkspaceI summary: value.summary, parentVersionId: value.parent_version, value, + // The chat only ever addresses the `raw_app` draft kind (see itemKindFor), + // so every app draft it can see is a code app. + rawApp: true, isDraft: true } } @@ -541,6 +544,7 @@ function backendDraftRowToWorkspaceItem( value: undefined, isDraft: true, triggerKind, + rawApp: row.kind === 'raw_app' ? true : undefined, ...(isLiveDraft ? { isLiveDraft: true } : {}) } } diff --git a/frontend/src/lib/components/copilot/chat/global/workspaceItems.ts b/frontend/src/lib/components/copilot/chat/global/workspaceItems.ts index a67eb48ac3..61b2773ef8 100644 --- a/frontend/src/lib/components/copilot/chat/global/workspaceItems.ts +++ b/frontend/src/lib/components/copilot/chat/global/workspaceItems.ts @@ -139,6 +139,10 @@ export type WorkspaceItem = { * without it a reader cannot tell a secret from a plain variable, since the * value is always redacted. */ isSecret?: boolean + /** Apps only. True for a code app, false for one built in the drag-and-drop + * editor. The two are edited by disjoint tool sets, so the distinction has to + * reach the model before it picks one. */ + rawApp?: boolean isDraft: boolean isLiveDraft?: boolean }