feat(ai-chat): tell the chat which kind of app it is looking at (#11208)

* feat(ai-chat): tell the chat which kind of app it is looking at

Co-Authored-By: Claude Opus 5 (1M context) <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_018r5ppn8YmHsM1MaAwY7xgB

* refactor(ai-chat): drop a stale comment and hoist the prefix local

Co-Authored-By: Claude Opus 5 (1M context) <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_018r5ppn8YmHsM1MaAwY7xgB

* test(ai-chat): add a global eval for refusing to edit a drag-and-drop app

Co-Authored-By: Claude Opus 5 (1M context) <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_018r5ppn8YmHsM1MaAwY7xgB

* test(ai-chat): make the app-kind eval require editing the code app too

Co-Authored-By: Claude Opus 5 (1M context) <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_018r5ppn8YmHsM1MaAwY7xgB

---------

Co-authored-by: Claude Opus 5 (1M context) <noreply@anthropic.com>

ai_evals/adapters/frontend/mockBackend.ts

@@ -49,12 +49,16 @@ export interface BenchmarkWorkspaceFlow {
 export interface BenchmarkWorkspaceApp {
 	path: string
 	summary: string
+	/** Defaults to true. Set false for a drag-and-drop app, which the chat can list
+	 * and read but has no tool to edit — its value is a grid, not files. */
+	rawApp?: boolean
 	value: {
-		files: Record<string, string>
-		runnables: Record<string, unknown>
+		files?: Record<string, string>
+		runnables?: Record<string, unknown>
 		data?: unknown
 		policy?: unknown
 		custom_path?: unknown
+		[key: string]: unknown
 	}
 }
 
@@ -994,7 +998,7 @@ function buildBenchmarkListableApp(app: BenchmarkWorkspaceApp): ListableApp {
 		extra_perms: {},
 		edited_at: BENCHMARK_TIMESTAMP,
 		execution_mode: 'viewer',
-		raw_app: true
+		raw_app: app.rawApp ?? true
 	}
 }
 
@@ -1012,7 +1016,7 @@ function buildBenchmarkApp(app: BenchmarkWorkspaceApp): AppWithLastVersion {
 		execution_mode: 'viewer',
 		extra_perms: {},
 		custom_path: app.value.custom_path as string | undefined,
-		raw_app: true
+		raw_app: app.rawApp ?? true
 	}
 }
 

ai_evals/cases/global.yaml

@@ -2611,3 +2611,40 @@
   judgeChecklist:
     - runs the existing script rather than rewriting it
     - passes the GitHub resource as the bare string $res:f/evals/global/github_main
+
+- id: global-drag-and-drop-app-not-editable
+  prompt: |-
+    Add a refresh button to the ops console app, and the same to the sales board app.
+  initial: ai_evals/fixtures/frontend/global/initial/apps_code_and_drag_and_drop.json
+  runtime:
+    maxTurns: 12
+  validate:
+    # One request, two apps, only one of them editable: the code app must come back with a
+    # draft and the drag-and-drop one must not. Refusing both, or editing both, fails here —
+    # which is what makes this a test of the distinction rather than of caution.
+    draftCountExactly: 1
+    requiredDrafts:
+      - type: app
+        path: f/evals/global/ops_console
+    forbiddenDrafts:
+      - type: app
+        path: f/evals/global/sales_board
+  toolExpect:
+    # Deliberately not constraining write_app_file/patch_app_file by argument: an entry there
+    # fails when its tool was never called, so naming both would fail on whichever the model
+    # did not pick. The draft assertions above cover the same ground, tool-agnostically.
+    forbiddenToolsUsed:
+      - init_app
+      - deploy_workspace_item
+      - delete_app_file
+      - delete_app_runnable
+  assistantExpect:
+    # A refusal leaves no draft for the judge to read, so the explanation is checked here.
+    # Only the app kind: substring tests cannot see paraphrase, and every wording of "I can't
+    # edit it" defeats a fixed list.
+    requiredMentionsAnyOf:
+      - - drag-and-drop
+        - drag and drop
+        - low-code
+        - no-code
+  skipJudge: true

ai_evals/fixtures/frontend/global/initial/apps_code_and_drag_and_drop.json

@@ -0,0 +1,30 @@
+{
+  "user": {
+    "username": "admin",
+    "is_admin": true,
+    "folders": ["evals"],
+    "folders_read": ["evals"]
+  },
+  "workspace": {
+    "apps": [
+      {
+        "path": "f/evals/global/sales_board",
+        "summary": "Sales board",
+        "rawApp": false,
+        "value": {
+          "grid": []
+        }
+      },
+      {
+        "path": "f/evals/global/ops_console",
+        "summary": "Ops console",
+        "value": {
+          "files": {
+            "/App.tsx": "export default function App() {\n\treturn <div>Ops console</div>\n}\n"
+          },
+          "runnables": {}
+        }
+      }
+    ]
+  }
+}

frontend/src/lib/components/copilot/chat/global/core.test.ts

@@ -2018,6 +2018,69 @@ describe('global AI tools', () => {
 		})
 	})
 
+	it('tells a code app from a drag-and-drop app', async () => {
+		vi.mocked(AppService.listApps).mockResolvedValueOnce([
+			{ path: 'f/apps/code', summary: 'Code app', raw_app: true },
+			{ path: 'f/apps/builder', summary: 'Builder app' }
+		] as any)
+		// An app draft is always a code app: the chat cannot address a
+		// drag-and-drop app's draft kind at all.
+		seedBackendDraft(
+			'raw_app',
+			'u/admin/draft_listed',
+			{ summary: 'Draft app' },
+			{ workspace: WORKSPACE }
+		)
+
+		const rows = JSON.parse(await callGlobalTool('list_workspace_items', { types: ['app'] }))
+
+		expect(rows.map((r: any) => [r.path, r.rawApp])).toEqual([
+			['f/apps/code', true],
+			['f/apps/builder', false],
+			['u/admin/draft_listed', true]
+		])
+	})
+
+	it('still says which kind of app it is when the app is read directly', async () => {
+		// The flag decides whether the app tools are offered at all, and the model
+		// reads an app before it edits one — a listing that knows is not enough.
+		vi.mocked(AppService.getAppByPath).mockResolvedValueOnce({
+			path: 'f/apps/builder',
+			summary: 'Builder app',
+			value: { grid: [] },
+			raw_app: false
+		} as any)
+
+		const read = JSON.parse(
+			await callGlobalTool('read_workspace_item', { type: 'app', path: 'f/apps/builder' })
+		)
+		expect(read.rawApp).toBe(false)
+	})
+
+	it('finds a staged app under the folder it was filed in, not its generated path', async () => {
+		// A never-deployed app the editor created lives at a generated path, so the
+		// folder the user filed it under exists only as its staged name. The server
+		// drops draft-only rows under any narrowing filter, leaving this pass the one
+		// that can answer a folder-scoped question about it.
+		seedBackendDraft(
+			'raw_app',
+			'u/admin/draft_7f21c9',
+			{ summary: '', draft_path: 'f/team/invoice_tracker' },
+			{ workspace: WORKSPACE }
+		)
+
+		const matched = JSON.parse(
+			await callGlobalTool('list_workspace_items', { types: ['app'], path_prefix: 'f/team/' })
+		)
+		expect(matched).toHaveLength(1)
+		expect(matched[0].draftPath).toBe('f/team/invoice_tracker')
+
+		const other = JSON.parse(
+			await callGlobalTool('list_workspace_items', { types: ['app'], path_prefix: 'f/other/' })
+		)
+		expect(other).toEqual([])
+	})
+
 	it('applies path_prefix to drafts before enforcing the result limit', async () => {
 		await callGlobalTool('write_script', {
 			path: 'f/other/outside',

frontend/src/lib/components/copilot/chat/global/core.ts

@@ -1421,6 +1421,7 @@ Flows:
 - Use patch_flow_json for structural flow edits and write_flow for full flow rewrites.
 
 Raw apps:
+- The app tools below only work on raw (code) apps. \`rawApp\` says which: false is a drag-and-drop app, which you can list and read but not edit or deploy. Check it before offering to change an app.
 - read_workspace_item returns app metadata only. Use read_app_file for file and inline runnable contents.
 - Use write_app_file, patch_app_file, and delete_app_file for frontend files.
 - Use write_app_runnable and delete_app_runnable for backend runnables.
@@ -1526,6 +1527,7 @@ function serializeWorkspaceItemForRead(item: WorkspaceItem): unknown {
 			path: item.path,
 			summary: item.summary,
 			value: summarizeAppValue(item.value as AppDraftValue),
+			rawApp: item.rawApp,
 			isDraft: item.isDraft
 		}
 	}
@@ -1752,6 +1754,9 @@ function appToItem(app: ListableApp | AppWithLastVersion, includeValue: boolean)
 		path: app.path,
 		summary: app.summary,
 		value: includeValue ? ((app as AppWithLastVersion).value as AppDraftValue) : undefined,
+		// The server omits this flag rather than sending false, so its absence in the
+		// response is a known false — not a value this listing failed to fetch.
+		rawApp: app.raw_app ?? false,
 		isDraft: false
 	}
 }
@@ -2038,6 +2043,7 @@ async function readWorkspaceItem(
 				path: app.path,
 				summary: value.summary,
 				value: metadata as unknown as AppDraftValue,
+				rawApp: app.raw_app,
 				isDraft: false
 			}
 		}
@@ -3408,9 +3414,16 @@ export const globalTools: Tool<{}>[] = [
 			// (it filters before the cap; query filters after).
 			if ((parsed.page ?? 1) === 1) {
 				const draftCountByType = new Map<string, number>()
+				const prefix = parsed.path_prefix
 				for (const draft of await listGlobalDrafts(workspace)) {
 					if (!types.includes(draft.type)) continue
-					if (parsed.path_prefix && !draft.path.startsWith(parsed.path_prefix)) continue
+					// A draft's staged name is often not where it is stored: the editor parks
+					// a new script, flow or app at a generated `draft_<uuid>` path, and a
+					// rename stages the new name over the old path. The server drops
+					// draft-only rows under any narrowing filter, leaving this pass their
+					// only source, so either name has to satisfy the prefix.
+					if (prefix && !draft.path.startsWith(prefix) && !draft.draftPath?.startsWith(prefix))
+						continue
 					const count = draftCountByType.get(draft.type) ?? 0
 					if (count >= limit) continue
 					draftCountByType.set(draft.type, count + 1)

frontend/src/lib/components/copilot/chat/global/userDraftAdapter.ts

@@ -176,6 +176,9 @@ function appDraftToWorkspaceItem(path: string, draft: AppDraftValue): WorkspaceI
 		summary: value.summary,
 		parentVersionId: value.parent_version,
 		value,
+		// The chat only ever addresses the `raw_app` draft kind (see itemKindFor),
+		// so every app draft it can see is a code app.
+		rawApp: true,
 		isDraft: true
 	}
 }
@@ -541,6 +544,7 @@ function backendDraftRowToWorkspaceItem(
 		value: undefined,
 		isDraft: true,
 		triggerKind,
+		rawApp: row.kind === 'raw_app' ? true : undefined,
 		...(isLiveDraft ? { isLiveDraft: true } : {})
 	}
 }

frontend/src/lib/components/copilot/chat/global/workspaceItems.ts

@@ -139,6 +139,10 @@ export type WorkspaceItem = {
 	 * without it a reader cannot tell a secret from a plain variable, since the
 	 * value is always redacted. */
 	isSecret?: boolean
+	/** Apps only. True for a code app, false for one built in the drag-and-drop
+	 * editor. The two are edited by disjoint tool sets, so the distinction has to
+	 * reach the model before it picks one. */
+	rawApp?: boolean
 	isDraft: boolean
 	isLiveDraft?: boolean
 }