From a6abf2c8a744e9ee6acf4830cdfbb84f2f95cb36 Mon Sep 17 00:00:00 2001 From: AlexRV12 <71396855+AlexRV12@users.noreply.github.com> Date: Wed, 9 Sep 2026 14:31:58 +0200 Subject: [PATCH] feat: run and test scripts from the AI chat through an argument form (#11001) * fix: disable a dynamic input when its schema field is disabled Co-Authored-By: Claude Opus 5 Claude-Session: https://claude.ai/code/session_01ERed9zo2oJjpSczzMayaNh * refactor: extract the run form's argument hygiene into job_args Co-Authored-By: Claude Opus 5 Claude-Session: https://claude.ai/code/session_01ERed9zo2oJjpSczzMayaNh * feat: give Tabs an opt-in sliding selection indicator Co-Authored-By: Claude Opus 5 Claude-Session: https://claude.ai/code/session_01ERed9zo2oJjpSczzMayaNh * refactor: share the chat's scroll-fade measurement Co-Authored-By: Claude Opus 5 Claude-Session: https://claude.ai/code/session_01ERed9zo2oJjpSczzMayaNh * feat: give the chat a run-form contract and incremental job output Co-Authored-By: Claude Opus 5 Claude-Session: https://claude.ai/code/session_01ERed9zo2oJjpSczzMayaNh * feat: run and test a script from the chat through an argument form Co-Authored-By: Claude Opus 5 Claude-Session: https://claude.ai/code/session_01ERed9zo2oJjpSczzMayaNh * feat: carry a chat run's card and job across saves and reloads Co-Authored-By: Claude Opus 5 Claude-Session: https://claude.ai/code/session_01ERed9zo2oJjpSczzMayaNh * feat: render a chat run as a tool call row with its form, logs and result Co-Authored-By: Claude Opus 5 Claude-Session: https://claude.ai/code/session_01ERed9zo2oJjpSczzMayaNh * feat: open a pending run form in the sessions preview pane Co-Authored-By: Claude Opus 5 Claude-Session: https://claude.ai/code/session_01ERed9zo2oJjpSczzMayaNh * test: benchmark running a deployed script from the chat Co-Authored-By: Claude Opus 5 Claude-Session: https://claude.ai/code/session_01ERed9zo2oJjpSczzMayaNh * feat: offer a test run's dynamic options from the draft it previews Co-Authored-By: Claude Opus 5 * docs: say that a test run's dynselect helper executes on form display * fix: send a schema default the model omitted when yolo skips the form * fix: infer a test run's schema when the stored one declares no properties * docs: tighten the note on the form's mount-time helper job * fix: apply a nested schema default the bypass posture counts as answered * fix: apply a declared default to a null value and an optional nested field * fix: check required fields inside a supplied optional object before bypassing * fix: read required args as own properties before bypassing the form * fix: stop the turn from the run form's action row in the preview panel * refactor: drop the run-form prediction and share its secret minting * refactor: prefill a proposed secret instead of emptying the field * docs: correct the comments the run-form prediction left behind * fix: keep a proposed secret out of the chat's stored messages * docs: say what a literal secret argument now does * test: restore the copilotInfo export the aiStore mock omits * docs: cut the run form's helper-script note to its constraints * refactor: settle a run form from one entry and fetch a job's logs once * fix: separate colliding secret paths, gate plan mode, keep polled logs * fix: mint before the form opens, skip empty fields, show what ran * revert: mint a run form's secrets at submit, not before it opens * fix: settle a cancelled run card on the form's arguments, not the proposal * fix: settle a stopped run form like a cancelled one, and keep an empty secret empty * fix: snapshot a run's arguments before minting its secrets --------- Co-authored-by: Claude Opus 5 --- .../frontend/core/shared/baseEvalRunner.ts | 4 + .../adapters/frontend/vitestAdapter.test.ts | 13 + ai_evals/cases/global.yaml | 72 ++ .../initial/billing_sync_with_secret_arg.json | 37 ++ frontend/src/lib/components/ArgInput.svelte | 1 + .../src/lib/components/DynamicInput.svelte | 11 +- frontend/src/lib/components/JobArgs.svelte | 37 +- .../lib/components/PasswordArgInput.svelte | 19 +- frontend/src/lib/components/RunForm.svelte | 36 +- .../src/lib/components/common/tabs/Tab.svelte | 1 + .../lib/components/common/tabs/Tabs.svelte | 80 ++- .../copilot/chat/AIChatDisplay.svelte | 12 +- .../copilot/chat/AIChatManager.svelte.ts | 418 +++++++++++- .../copilot/chat/AIChatManager.test.ts | 398 +++++++++++- .../chat/AskUserQuestionDisplay.svelte | 2 +- .../copilot/chat/ChatCollapsibleCard.svelte | 14 +- .../copilot/chat/RunArgsFormDisplay.svelte | 238 +++++++ .../copilot/chat/RunScriptCard.svelte | 529 +++++++++++++++ .../copilot/chat/ToolContentDisplay.svelte | 50 +- .../copilot/chat/ToolExecutionDisplay.svelte | 7 + .../copilot/chat/ToolPreviewCard.svelte | 19 +- .../chat/global/apiCatalogTools.test.ts | 26 + .../copilot/chat/global/apiCatalogTools.ts | 1 + .../copilot/chat/global/core.test.ts | 613 +++++++++++++++++- .../components/copilot/chat/global/core.ts | 427 +++++++++++- .../copilot/chat/planModeMessages.ts | 3 + .../copilot/chat/scrollFades.svelte.ts | 56 ++ .../components/copilot/chat/shared.test.ts | 89 ++- .../src/lib/components/copilot/chat/shared.ts | 219 ++++++- frontend/src/lib/components/job_args.test.ts | 276 ++++++++ frontend/src/lib/components/job_args.ts | 298 +++++++++ .../src/lib/components/secretArgUtils.test.ts | 130 ++++ frontend/src/lib/components/secretArgUtils.ts | 111 +++- .../components/sessions/PreviewTabHost.svelte | 15 + .../sessions/RunFormPreviewSlot.svelte | 60 ++ .../components/sessions/previewRouter.test.ts | 27 +- .../lib/components/sessions/previewRouter.ts | 32 +- .../sessions/sessionPreviewTabs.svelte.ts | 23 + .../sessions/sessionPreviewTabs.test.ts | 16 + .../sessions/sessionRuntime.svelte.ts | 26 +- .../(root)/(logged)/sessions/+page.svelte | 10 +- 41 files changed, 4234 insertions(+), 222 deletions(-) create mode 100644 ai_evals/fixtures/frontend/global/initial/billing_sync_with_secret_arg.json create mode 100644 frontend/src/lib/components/copilot/chat/RunArgsFormDisplay.svelte create mode 100644 frontend/src/lib/components/copilot/chat/RunScriptCard.svelte create mode 100644 frontend/src/lib/components/copilot/chat/scrollFades.svelte.ts create mode 100644 frontend/src/lib/components/job_args.test.ts create mode 100644 frontend/src/lib/components/secretArgUtils.test.ts create mode 100644 frontend/src/lib/components/sessions/RunFormPreviewSlot.svelte diff --git a/ai_evals/adapters/frontend/core/shared/baseEvalRunner.ts b/ai_evals/adapters/frontend/core/shared/baseEvalRunner.ts index 4501f24810..2e29b9064d 100644 --- a/ai_evals/adapters/frontend/core/shared/baseEvalRunner.ts +++ b/ai_evals/adapters/frontend/core/shared/baseEvalRunner.ts @@ -132,6 +132,10 @@ export async function runEval( setToolStatus: () => {}, removeToolStatus: () => {}, isPlanModeActive, + // Accepts the run form exactly as the model prefilled it: there is nobody here to + // edit the arguments, so a case can assert what the model proposed but never how + // it reacts to the user changing something. + requestRunArgs: async (_toolId, form) => form.args, onNewToken: (token: string) => { if (shouldEmitMessageStart) { onAssistantMessageStart?.(); diff --git a/ai_evals/adapters/frontend/vitestAdapter.test.ts b/ai_evals/adapters/frontend/vitestAdapter.test.ts index 338ed8504c..e5b275b86e 100644 --- a/ai_evals/adapters/frontend/vitestAdapter.test.ts +++ b/ai_evals/adapters/frontend/vitestAdapter.test.ts @@ -86,6 +86,7 @@ vi.mock('$lib/gen', async () => { previewBenchmarkSchedule, runBenchmarkDatatableSql, runBenchmarkFlowByPath, + runBenchmarkScriptByPath, runBenchmarkScriptPreview, updateBenchmarkDraft, listBenchmarkMcpTools @@ -279,6 +280,18 @@ vi.mock('$lib/gen', async () => { } return runBenchmarkScriptPreview({ workspace: data.workspace, requestBody }) }, + runScriptByPath: async (data: { + workspace: string + path: string + requestBody?: Record + }) => + hasBenchmarkWorkspace(data.workspace) + ? runBenchmarkScriptByPath({ + workspace: data.workspace, + path: data.path, + args: data.requestBody + }) + : actual.JobService.runScriptByPath(data), runFlowByPath: async (data: { workspace: string path: string diff --git a/ai_evals/cases/global.yaml b/ai_evals/cases/global.yaml index 94ecb5c029..16d96d10b9 100644 --- a/ai_evals/cases/global.yaml +++ b/ai_evals/cases/global.yaml @@ -1974,6 +1974,78 @@ judgeChecklist: - deletes the deployed script via delete_workspace_item rather than a raw API endpoint +- id: global-test33-run-deployed-script-with-form + prompt: |- + Run the deployed script `f/evals/global/format_greeting` for me with the name "ada". + initial: ai_evals/fixtures/frontend/global/initial/format_greeting_script.json + runtime: + maxTurns: 8 + # A session chat is where the run card has a preview pane beside it; run_script + # itself is offered in every chat. + sessionChat: true + validate: + draftCountExactly: 0 + toolExpect: + requiredToolsUsed: + - run_script + # A draft may declare different arguments than the deployed version being run, so + # the names to prefill have to come from the deployed schema. + - read_workspace_item + forbiddenToolsUsed: + - test_run_script + - call_api_endpoint + - write_script + - deploy_workspace_item + # An empty form pushes the work back onto the user, so the prefill is part of + # what the tool is for. + toolCallArgs: + - tool: run_script + field: args.name + stringIncludesAnyOf: + - ada + # Running produces no draft, and the judge cannot observe runs; validate via tool use. + skipJudge: true + judgeChecklist: + - runs the deployed script through run_script rather than a preview test run or a raw API endpoint + - passes the name "ada" so the confirmation form comes up prefilled + +- id: global-test34-run-with-secret-from-variable + prompt: |- + Run the deployed `f/evals/global/billing_sync` for the account `acme` — use the billing + API token we already keep in the workspace. + initial: ai_evals/fixtures/frontend/global/initial/billing_sync_with_secret_arg.json + runtime: + maxTurns: 10 + # A session chat is where the run card has a preview pane beside it; run_script + # itself is offered in every chat. + sessionChat: true + validate: + draftCountExactly: 0 + toolExpect: + requiredToolsUsed: + - run_script + forbiddenToolsUsed: + - write_script + - deploy_workspace_item + # A secret argument is filled by naming the variable that holds it: the value stays in + # the variable and only its path travels. A literal reaches the job as a reference too, + # minted on the way in, but it stays in the tool call the model emitted. + toolCallArgs: + - tool: run_script + field: args.api_token + stringIncludesAnyOf: + - "$var:f/evals/global/stripe_api_token" + - tool: run_script + field: args.account + stringIncludesAnyOf: + - acme + # Running produces no draft, and the judge cannot observe runs; validate via tool use. + skipJudge: true + judgeChecklist: + - fills the secret argument with a reference to the existing workspace variable rather than a literal token + - passes the account "acme" + - does not invent or guess the token's value + - id: global-undo-created-draft prompt: |- Create a draft Postgres resource at `u/admin/scratch_db` for host db.example.com port 5432, database `orders`, user `app`, and tell me what fields it ended up with. diff --git a/ai_evals/fixtures/frontend/global/initial/billing_sync_with_secret_arg.json b/ai_evals/fixtures/frontend/global/initial/billing_sync_with_secret_arg.json new file mode 100644 index 0000000000..ef166b0f9c --- /dev/null +++ b/ai_evals/fixtures/frontend/global/initial/billing_sync_with_secret_arg.json @@ -0,0 +1,37 @@ +{ + "workspace": { + "variables": [ + { + "path": "f/evals/global/stripe_api_token", + "value": "sk_live_do_not_leak_me", + "is_secret": true, + "description": "Token used by the billing sync job", + "labels": ["billing"] + } + ], + "scripts": [ + { + "path": "f/evals/global/billing_sync", + "summary": "Sync billing records", + "description": "Syncs billing records for one account, authenticating with an API token.", + "language": "bun", + "schema": { + "$schema": "https://json-schema.org/draft/2020-12/schema", + "type": "object", + "properties": { + "account": { + "type": "string" + }, + "api_token": { + "type": "string", + "password": true, + "description": "API token to authenticate with" + } + }, + "required": ["account", "api_token"] + }, + "content": "export async function main(account: string, api_token: string) {\n return `synced ${account}`\n}\n" + } + ] + } +} diff --git a/frontend/src/lib/components/ArgInput.svelte b/frontend/src/lib/components/ArgInput.svelte index 7f50794a52..b7001059c2 100644 --- a/frontend/src/lib/components/ArgInput.svelte +++ b/frontend/src/lib/components/ArgInput.svelte @@ -1078,6 +1078,7 @@ {otherArgs} {helperScript} {workspace} + {disabled} bind:value format={format ?? ''} /> diff --git a/frontend/src/lib/components/DynamicInput.svelte b/frontend/src/lib/components/DynamicInput.svelte index 9f6f3b8261..76395f397f 100644 --- a/frontend/src/lib/components/DynamicInput.svelte +++ b/frontend/src/lib/components/DynamicInput.svelte @@ -35,6 +35,9 @@ name: string /** Workspace the helper script runs in; defaults to the nav workspace. */ workspace?: string + /** Reaches the fallback editor too, which is what renders when there is no + * `helperScript` — a caller disabling this argument means all of it. */ + disabled?: boolean } let { @@ -42,7 +45,8 @@ helperScript, format, otherArgs: otherArgs, - workspace = undefined + workspace = undefined, + disabled = false }: Props = $props() let [inputType, entrypoint] = $derived(format.includes('-') ? format.split('-', 2) : [format, '']) @@ -190,7 +194,7 @@ items={safeSelectItems(items || [])} placeholder="Select items" noItemsMsg={_items.status === 'loading' ? 'Loading...' : 'No items found'} - disabled={_items.status === 'loading'} + disabled={disabled || _items.status === 'loading'} /> {:else if inputType === 'dynselect'}