diff --git a/.agents/skills/ai-evals/SKILL.md b/.agents/skills/ai-evals/SKILL.md index ad334e2c6b..3acb0f7362 100644 --- a/.agents/skills/ai-evals/SKILL.md +++ b/.agents/skills/ai-evals/SKILL.md @@ -36,6 +36,10 @@ WMILL_AI_EVAL_BACKEND_URL=http://127.0.0.1: WMILL_AI_EVAL_BACKEND_WORKSPAC `WMILL_AI_EVAL_BACKEND_WORKSPACE=integration-tests` (or any existing workspace) to reuse one. The only side effect of a run is upserting an `f/evals/ai/` resource there. +- **Global mode creates its own workspace per attempt** and seeds the case's fixture in it, + because the session's tools run on the backend; it ignores + `WMILL_AI_EVAL_BACKEND_WORKSPACE` and needs a `postgresql` (native) worker for datatable + cases. `ai_evals/README.md` ("Global mode runs against a real workspace") lists the rest. - Provider keys live in `ai_evals/.env` and are auto-loaded by bun. The judge is a separate Anthropic call (default `claude-sonnet-4-6`) regardless of the model under test. diff --git a/.github/workflows/ai-evals-scheduled.yml b/.github/workflows/ai-evals-scheduled.yml index 2bd8cdb181..5907de5b76 100644 --- a/.github/workflows/ai-evals-scheduled.yml +++ b/.github/workflows/ai-evals-scheduled.yml @@ -105,16 +105,22 @@ jobs: SQLX_OFFLINE: true CARGO_BUILD_JOBS: 12 RUSTFLAGS: "" - run: cargo build --features quickjs + # The global suite's hub cases search the hub embeddings index; its trigger cases + # write HTTP triggers, whose routes the feature mounts. + run: cargo build --features quickjs,embedding,http_trigger - name: Start Windmill working-directory: ./backend env: DATABASE_URL: postgres://postgres:changeme@localhost:5432/windmill RUST_LOG: info + # Each global case seeds resource types and waits until the search indexes them. + RESOURCE_TYPE_EMBEDDINGS_CHECK_INTERVAL_SECS: "2" run: | mkdir -p ../ai_evals/logs ./target/debug/windmill > ../ai_evals/logs/windmill.log 2>&1 & + # The global suite's datatable cases run SQL on the `postgresql` tag, a native worker's. + MODE=worker WORKER_GROUP=native PORT=8001 ./target/debug/windmill > ../ai_evals/logs/windmill-native.log 2>&1 & for i in $(seq 1 60); do curl -sf http://localhost:8000/api/version > /dev/null 2>&1 && break sleep 2 @@ -142,6 +148,8 @@ jobs: MODEL: ${{ matrix.model }} WMILL_AI_EVAL_BACKEND_URL: http://localhost:8000 WMILL_AI_EVAL_BACKEND_WORKSPACE: integration-tests + # Global fixtures whose user is not the admin are created in the backend's database. + WMILL_AI_EVAL_DATABASE_URL: postgres://postgres:changeme@localhost:5432/windmill ANTHROPIC_API_KEY: ${{ secrets.ANTHROPIC_API_KEY }} OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }} GEMINI_API_KEY: ${{ secrets.GOOGLE_API_KEY }} diff --git a/.github/workflows/ai-evals-test.yml b/.github/workflows/ai-evals-test.yml index 4dd266c364..139e30d0ae 100644 --- a/.github/workflows/ai-evals-test.yml +++ b/.github/workflows/ai-evals-test.yml @@ -3,8 +3,9 @@ name: AI Evals (global mode) # Smoke-tests the production global AI chat proxy/frontend execution path via # the ai_evals harness, one case across one cheap model per provider. Runs only # when the eval harness or the global chat code change, since each run makes real -# (paid) LLM calls. The backend is built from source purely as the AI proxy the -# harness routes model calls through; the global tools/drafts run in-process in +# (paid) LLM calls. The backend is built from source as the AI proxy the harness +# routes model calls through and as the server the global tools run on, in a +# workspace the harness creates per attempt; the chat itself runs in-process in # the Vitest bridge against production frontend code. To avoid spending on every # commit, the PR side triggers only when a PR is marked ready for review (out of # draft) — not on `synchronize` — plus push to main and manual dispatch. @@ -15,6 +16,7 @@ on: paths: - "ai_evals/**" - "backend/windmill-api/src/ai.rs" + - "backend/windmill-api/src/ai_session_tools/**" - "backend/windmill-ai/**" - "frontend/src/lib/components/copilot/**" # The eval harness runs production frontend code in-process; these are the @@ -29,6 +31,7 @@ on: paths: - "ai_evals/**" - "backend/windmill-api/src/ai.rs" + - "backend/windmill-api/src/ai_session_tools/**" - "backend/windmill-ai/**" - "frontend/src/lib/components/copilot/**" # The eval harness runs production frontend code in-process; these are the @@ -137,6 +140,8 @@ jobs: env: WMILL_AI_EVAL_BACKEND_URL: http://localhost:8000 WMILL_AI_EVAL_BACKEND_WORKSPACE: integration-tests + # Global fixtures whose user is not the admin are created in the backend's database. + WMILL_AI_EVAL_DATABASE_URL: postgres://postgres:changeme@localhost:5432/windmill # Anthropic backs the haiku model. Google AI uses GEMINI_API_KEY. ANTHROPIC_API_KEY: ${{ secrets.ANTHROPIC_API_KEY }} OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }} diff --git a/AGENTS.md b/AGENTS.md index 4c99482d43..a1fb608291 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -69,7 +69,7 @@ Open-source platform for internal tools, workflows, API integrations, background - **Domain vocabulary**: `CONTEXT.md` — the words this codebase uses for its own concepts (step, step setting, trigger step, …). Name things the way it does. - **CLI commands**: when adding/modifying/removing a command, subcommand, option, or description in `cli/src/commands/`, run `python system_prompts/generate.py` to refresh `system_prompts/auto-generated/` and `cli/src/guidance/skills.gen.ts`. The CLI docs the agents use to operate `wmill` are derived from the source — stale generated files give agents the wrong flags. - **AI guidance** (chat prompts and CLI skills): what the AI knows about Windmill itself — how flows, scripts, apps, pipelines and resources work — is written once, in `system_prompts/base/` and `system_prompts/languages/`, and `system_prompts/generate.py` builds both the chat's `$system_prompts` exports and the CLI's skills from it. Add or change such guidance there, never inline in one consumer: a sentence only one side should see goes in a `` / `` block, and a new file reaches both through the `TOPICS` table. The chat prompt builders under `frontend/src/lib/components/copilot/chat/` add only tool plumbing and runtime values, and shared markdown names no chat tool, since it reaches every chat mode. Rerun `generate.py` after editing; `system_prompts/README.md` has the details. -- **Windmill MCP server**: `/api/mcp` gives a token with `mcp:all` the AI session's GLOBAL toolset (`backend/windmill-api/src/mcp/session/`), and a token scoped to a selection of scripts and flows the tools server's per-runnable tools, without endpoint tools. The tools server's own URLs, `/api/mcp/w/{workspace}/mcp` and `/api/mcp/gateway`, keep serving existing tokens, endpoint scopes included, except that they give the session toolset to an `mcp:all` token carrying `mcp:session` (added by the OAuth approve, kept across scope edits) or `mcp:read_only`/`mcp:read_run`. Every tool takes a `workspace_id`; the token bounds which (its bound workspace, the `mcp:workspaces:` list, or all of its owner's). With the `mcp_external_idp` instance setting, every MCP URL also accepts an access token of an external OIDC issuer (`mcp/external_idp.rs`), told apart from Windmill's own by its `iss`: the token's client is mapped to a workspace and to the same access a Windmill token takes (an access level, served the session toolset as `mcp:session`, or a selection of scripts and flows), and the request acts as the existing Windmill user with the token's email, under that access, as a workspace-bound token does. The protected-resource metadata then names that issuer. What a model reads there (tool names, descriptions, input schemas, instructions, `get_instructions` texts) is generated from the session's TypeScript into `session/gen/`: after changing a GLOBAL tool definition, the global system prompt or `system_prompts/`, run `npm run gen:mcp-session` from `frontend/` (`mcpSessionGen.test.ts` fails otherwise). What a tool *does* is written twice, in `chat/global/core.ts` and in a Rust handler: a behaviour change on one side must be mirrored on the other, and a new session tool is either implemented in Rust or added to `EDITOR_ONLY` (a `cargo test` in `session/mod.rs` fails otherwise). For MCP Apps clients the server adds tools of its own, defined in Rust (`session/views.rs`) rather than generated: `show_run`, `show_runs`, `show_deploy_review` and `show_item` draw a card in the `ui://windmill/card-v` view (`session/card.html`; bump `CARD_URI` when the `_meta` it reads changes shape, since hosts cache a view by URI) from their result's `_meta` (`windmill/run`, `windmill/runs`, `windmill/review`, `windmill/item`), which the model never reads. A host draws a card for every call of a tool that declares one, so only these show tools do: the tools the model reads and runs with draw nothing, and `get_run` builds no card data. The card acts only by calling the session's tools (`show_run` to refresh, `cancel_job`, `show_deploy_review` to re-read a draft, `deploy_workspace_item` from the review card's Deploy button, which the user presses), so it sees nothing the model could not. Every `DESTRUCTIVE` tool but those in `CARD_ACTIONS` is hidden from it. Every session tool carries a `title` and explicit read-only, destructive and open-world hints, which the Claude and ChatGPT directories require (`DESTRUCTIVE`, `OPEN_WORLD`). A client that declares capabilities without the MCP Apps extension is not offered the show tools. It exists only on the MCP side, so `core.ts` has nothing to mirror. +- **Windmill MCP server**: `/api/mcp` gives a token with `mcp:all` the AI session's GLOBAL toolset (`backend/windmill-api/src/ai_session_tools/`), and a token scoped to a selection of scripts and flows the tools server's per-runnable tools, without endpoint tools. The tools server's own URLs, `/api/mcp/w/{workspace}/mcp` and `/api/mcp/gateway`, keep serving existing tokens, endpoint scopes included, except that they give the session toolset to an `mcp:all` token carrying `mcp:session` (added by the OAuth approve, kept across scope edits) or `mcp:read_only`/`mcp:read_run`. Every tool takes a `workspace_id`; the token bounds which (its bound workspace, the `mcp:workspaces:` list, or all of its owner's). With the `mcp_external_idp` instance setting, every MCP URL also accepts an access token of an external OIDC issuer (`mcp/external_idp.rs`), told apart from Windmill's own by its `iss`: the token's client is mapped to a workspace and to the same access a Windmill token takes (an access level, served the session toolset as `mcp:session`, or a selection of scripts and flows), and the request acts as the existing Windmill user with the token's email, under that access, as a workspace-bound token does. The protected-resource metadata then names that issuer. What a model reads there (tool names, descriptions, input schemas, instructions, `get_instructions` texts) is generated from the session's TypeScript into `ai_session_tools/gen/`: after changing a GLOBAL tool definition, the global system prompt or `system_prompts/`, run `npm run gen:mcp-session` from `frontend/` (`mcpSessionGen.test.ts` fails otherwise). What a tool *does* is written once, in a Rust handler the chat also calls (`POST /w/{workspace}/ai_session_tools/{name}`, through `chat/global/backendSessionTools.ts`); the browser applies the effects it returns, saving drafts through its own draft store. A new session tool is either implemented in Rust or added to `EDITOR_ONLY` (a `cargo test` in `ai_session_tools/mod.rs` fails otherwise). Still written twice, so a behaviour change must be mirrored: `deploy_workspace_item` and `diff` (the chat uses the UI's shared deploy and diff code), and the run tools' form side (argument coercion, secrets, plan mode, the outcome text), where the chat resolves and starts the run on the server but owns the form in between; both sides test their argument handling against `ai_session_tools/run_args_cases.json`, so a change there comes with a case. For MCP Apps clients the server adds tools of its own, defined in Rust (`ai_session_tools/views.rs`) rather than generated, and not offered to the chat, which draws its own cards: `show_run`, `show_runs`, `show_deploy_review` and `show_item` draw a card in the `ui://windmill/card-v` view (`mcp/card.html`; bump `CARD_URI` in `mcp/session.rs` when the `_meta` it reads changes shape, since hosts cache a view by URI) from their result's `_meta` (`windmill/run`, `windmill/runs`, `windmill/review`, `windmill/item`), which the model never reads. A host draws a card for every call of a tool that declares one, so only these show tools do: the tools the model reads and runs with draw nothing, and `get_run` builds no card data. The card acts only by calling the session's tools (`show_run` to refresh, `cancel_job`, `show_deploy_review` to re-read a draft, `deploy_workspace_item` from the review card's Deploy button, which the user presses), so it sees nothing the model could not. Every `DESTRUCTIVE` tool but those in `CARD_ACTIONS` is hidden from it. Every session tool carries a `title` and explicit read-only, destructive and open-world hints, which the Claude and ChatGPT directories require (`DESTRUCTIVE`, `OPEN_WORLD`). A client that declares capabilities without the MCP Apps extension is not offered the show tools. - **Session recorder**: `frontend/src/lib/components/recording/` is also the recorder `wmill app dev --recording` serves, vendored into the CLI as `cli/src/commands/app/devRecorderBundle.gen.ts`. After changing `rawAppSnapshot.ts` or `rawAppRecording.svelte.ts`, run `bun run gen:dev-recorder` from `cli/` (`cli/test/dev_recorder_bundle_unit.test.ts` fails otherwise). - **Raw-app policy**: `frontend/src/lib/components/raw_apps/rawAppPolicy.ts` also derives the policy the server's raw-app deploy stores, vendored into the bundle job as `backend/windmill-api/src/apps_raw_policy.gen.js`. After changing it or anything it imports, run `bun run gen:app-policy` from `cli/` (`cli/test/app_policy_bundle_unit.test.ts` fails otherwise). It rides in the job rather than being read from the CLI the job runs because the images install `windmill-cli` unpinned, so an image can carry one older than its server. diff --git a/ai_evals/README.md b/ai_evals/README.md index 67592daee4..caa31a2873 100644 --- a/ai_evals/README.md +++ b/ai_evals/README.md @@ -170,13 +170,38 @@ chose in the artifact's version picker — which only `get_preview_status` repor Global initial fixtures can seed `workspace.variables` with `{ path, value, is_secret, description?, labels?, ws_specific? }` entries so cases can -read and edit variables that already exist in the workspace. The mock mirrors the real -`get_variable`, **decrypt-by-default included**: a secret's `value` is withheld only -when the caller explicitly passes `decryptSecret: false`, and omitting the flag returns -the decrypted value, exactly as against a real backend. The chat's read path passes -`decryptSecret: false`, so a case can verify it never invents a value it was not shown. -Seed a recognizable secret (the existing fixture uses `sk_live_do_not_leak_me`) and -assert it via `valueExcludes` to catch a leak. +read and edit variables that already exist in the workspace. Seed a recognizable secret +(the existing fixture uses `sk_live_do_not_leak_me`) and assert it via `valueExcludes` to +catch a leak. + +### Global mode runs against a real workspace + +The session's tools run on the backend, so each global attempt creates its own workspace on +the eval backend, seeds the fixture there through the API, and deletes it afterwards. What +that backend needs: + +- a worker serving the `postgresql` tag (the `native` group) for datatable fixtures and the + SQL tools, and `WMILL_AI_EVAL_DATATABLE_PG_URL` (default + `postgres://postgres:changeme@localhost:5432/postgres`) naming a Postgres the worker can + reach, where each datatable fixture gets a scratch database; +- a superadmin login (the default `admin@windmill.dev`), since it creates and deletes + workspaces. `WMILL_AI_EVAL_BACKEND_WORKSPACE` does not apply to global mode. A CE backend + allows two workspaces besides `admins`, so attempts past that wait for a free slot; +- `WMILL_AI_EVAL_DATABASE_URL`, the eval backend's own database, for fixtures whose `user` is + not the admin: the harness writes that user and a token of theirs there with `psql` (CE cannot create + users through the API), adds them to the workspace, and revokes the token afterwards; +- network access to the hub, which the hub tools reach through the backend, and the + `embedding` cargo feature, without which `search_hub_scripts` has no index to search; +- the `http_trigger` cargo feature, which mounts the routes HTTP trigger drafts are checked + against; +- `RESOURCE_TYPE_EMBEDDINGS_CHECK_INTERVAL_SECS` set low (CI uses 2): each attempt seeds the + hub's resource types into its workspace and waits until the resource type search sees them, + which takes up to that interval (60 s by default). + +A fixture's `jobs` become completed preview runs with the given id, path, args, logs and +outcome; `createdBy` is not reproduced. Its `resourceTypes` are created as workspace types, +alongside whatever types the instance already holds; a fixture that sets none gets the types +of an instance synced with the hub. `runtime.userAnswers` scripts the user's side of `askUserQuestion` in a global case: one answer per question, in the order they are asked, and a question past the last answer is @@ -213,24 +238,12 @@ recorded call to that tool may pass the field at all (an explicit `null` counts passing it). Use it for partial-update tools, where supplying a field the model could not have read is itself the failure — e.g. `write_variable.value` on a secret variable. -Global (and flow) initial fixtures can seed `workspace.datatables` so the -`list_datatables`, `get_datatable_table_schema`, and `exec_datatable_sql` tools -return seeded data during evals. Each entry is +Global initial fixtures can seed `workspace.datatables`. Each entry is `{ datatable_name, schemas: { : { : { columns, rows? } } } }`, plus `migrations_enabled: true` and `migrations: [{ timestamp, name, code_up, code_down?, applied? }]` -for a datatable that keeps its schema as migrations, which the migration tools then -read and write in memory. -SQL runs through a small in-memory engine (`datatableSqlEngine.ts`), not a real -database. Writes are **stateful within a case**: `CREATE`/`DROP`/`INSERT`/`UPDATE`/ -`DELETE` mutate the seeded datatable in place, so a later `list_datatables`, -`get_datatable_table_schema`, `SELECT`, or `information_schema` query reflects them -— this is what stops a model from looping when it re-queries to verify a write. -The engine is best-effort: `SELECT` returns all rows of the referenced (or first) -table with no WHERE filtering/projection/joins, `WHERE` on UPDATE/DELETE supports -`col = value` predicates joined by `AND`, and anything unparseable is a no-op -success. So validate datatable cases through tool-use and SQL-argument assertions -(`requiredToolsUsed`, `stringIncludesAnyOf`) — not through exact returned row -values. An empty/absent `datatables` seed makes `list_datatables` return `[]`, +for a datatable that keeps its schema as migrations. Each one becomes a real datatable of +the case's workspace, backed by its scratch database, so the datatable and SQL tools see +exactly what a query would. An absent `datatables` seed leaves the workspace with none, which is what the "no datatable configured" blocking cases rely on. Set `WMILL_AI_EVAL_DISABLE_ACTIVE_EDITOR_CONTEXT=1` to run those cases with diff --git a/ai_evals/adapters/frontend/aiProviderModels.vitest.ts b/ai_evals/adapters/frontend/aiProviderModels.vitest.ts deleted file mode 100644 index d5f097242a..0000000000 --- a/ai_evals/adapters/frontend/aiProviderModels.vitest.ts +++ /dev/null @@ -1,36 +0,0 @@ -import { describe, expect, it } from 'vitest' -import { - handleBenchmarkApiFetch, - hasBenchmarkApiHandler, - registerBenchmarkWorkspaceRunnables, - unregisterBenchmarkWorkspace -} from './mockBackend' - -// A global eval run registers its workspace under a mkdtemp path, so the workspace id the -// frontend interpolates into the models URL carries slashes. A handler that assumed a single -// path segment silently fell through to the network, and every run took the offline fallback. -const WORKSPACE = '/tmp/wmill-frontend-global-benchmark-abc123' -const RESOURCE = 'f/evals/global/anthropic_main' -const URL_FOR = (workspace: string) => - `http://benchmark.local/api/w/${workspace}/ai/proxy/models` - -describe('benchmark /ai/proxy/models', () => { - it('serves the seeded listing for a workspace id that is a path', async () => { - registerBenchmarkWorkspaceRunnables(WORKSPACE, { - aiProviders: [ - { path: RESOURCE, kind: 'anthropic', models: ['claude-sonnet-5', 'claude-opus-5'] } - ] - }) - try { - expect(hasBenchmarkApiHandler(URL_FOR(WORKSPACE))).toBe(true) - const response = handleBenchmarkApiFetch(URL_FOR(WORKSPACE), { - headers: { 'X-Resource-Path': RESOURCE, 'X-Provider': 'anthropic' } - }) - await expect(response.json()).resolves.toEqual({ - data: [{ id: 'claude-sonnet-5' }, { id: 'claude-opus-5' }] - }) - } finally { - unregisterBenchmarkWorkspace(WORKSPACE) - } - }) -}) diff --git a/ai_evals/adapters/frontend/core/global/globalEvalRunner.ts b/ai_evals/adapters/frontend/core/global/globalEvalRunner.ts index c648848387..16532b5f58 100644 --- a/ai_evals/adapters/frontend/core/global/globalEvalRunner.ts +++ b/ai_evals/adapters/frontend/core/global/globalEvalRunner.ts @@ -1,6 +1,3 @@ -import { mkdtemp, rm } from "fs/promises"; -import { tmpdir } from "os"; -import { join } from "path"; import type { ChatCompletionMessageParam } from "openai/resources/chat/completions.mjs"; import type { AIProvider, OperatorSettings } from "$lib/gen/types.gen"; import { @@ -29,6 +26,7 @@ import { appendPlanModeInstructions } from "../../../../../frontend/src/lib/comp import type { Tool as ProductionTool } from "../../../../../frontend/src/lib/components/copilot/chat/shared"; import { createEvalPlanTools } from "./planModeTools"; import { UserDraft } from "../../../../../frontend/src/lib/userDraft.svelte"; +import { DraftService } from "$lib/gen"; import { createEvalArtifactHelpers, type SeededArtifact, @@ -42,11 +40,11 @@ import type { EvalCaseProfile, ModeRunContext } from "../../../../core/types"; import type { GlobalDraftState } from "../../../../core/validators"; import type { WindmillBackendSettings } from "../../../../core/windmillBackendSettings"; import { - registerBenchmarkWorkspaceRunnables, - seedBenchmarkDraft, + registerBenchmarkAiModels, unregisterBenchmarkWorkspaceRunnables, type BenchmarkWorkspaceRunnables, } from "../../mockBackend"; +import { createRealWorkspace } from "./realWorkspace"; import { runEval } from "../shared"; import type { TokenUsage, ToolCallDetail } from "../shared/types"; @@ -140,7 +138,6 @@ export interface GlobalEvalOptions { userAnswers?: string[]; provider?: AIProvider; backend: WindmillBackendSettings; - workspaceRoot?: string; // Artifacts the session already holds when the run starts. artifacts?: SeededArtifact[]; /** Tabs already open in the side panel, including any artifact version the reader pinned. */ @@ -153,18 +150,22 @@ export async function runGlobalEval( apiKey: string, options: GlobalEvalOptions, ): Promise { - const workspaceRoot = - options.workspaceRoot ?? - (await mkdtemp(join(tmpdir(), "wmill-frontend-global-benchmark-"))); + const realWorkspace = await createRealWorkspace({ + settings: options.backend, + caseId: options.runContext?.caseId ?? "global", + attempt: options.runContext?.attempt ?? 1, + fixtures: options.workspaceFixtures ?? {}, + user: options.user, + profile: options.profile, + }); + const workspaceRoot = realWorkspace.id; clearGlobalDrafts(workspaceRoot); - registerBenchmarkWorkspaceRunnables( + registerBenchmarkAiModels( workspaceRoot, - options.workspaceFixtures ?? {}, + options.workspaceFixtures?.aiProviders ?? [], ); - seedLiveEditorDrafts(workspaceRoot, options.liveEditorDrafts ?? []); - // No deploy rules in the mock workspace, so they always pass. The role is read as a server - // that predates workspace roles describes it, from `operator_settings`. + // The eval workspace has no deploy rules, so they always pass. const roleAccess = options.profile && capabilitiesForRole({ @@ -187,6 +188,7 @@ export async function runGlobalEval( let panel: EvalPreviewPanel | undefined; try { + await seedLiveEditorDrafts(workspaceRoot, options.liveEditorDrafts ?? []); const evalArtifacts = createEvalArtifactHelpers(options.artifacts); // Only a session chat has a side panel, so only it gets one here. Seeded tabs would // otherwise vanish without a word, and the case would measure an empty panel. @@ -230,6 +232,9 @@ export async function runGlobalEval( }); const helpers = { ...evalArtifacts.helpers, + // A standalone chat has no session: the tools that behave differently in one (a pipeline + // node is built into the pipeline draft there, not written as a script) read this. + sessionId: options.sessionChat ? evalArtifacts.helpers.sessionId : undefined, access, ...(panel && { openArtifact: panel.openArtifact }), }; @@ -275,7 +280,7 @@ export async function runGlobalEval( model, workspace: workspaceRoot, provider: options.provider, - backend: options.backend, + backend: realWorkspace.backend, caseId: options.runContext?.caseId, attempt: options.runContext?.attempt, }, @@ -299,9 +304,7 @@ export async function runGlobalEval( clearLiveEditorDrafts(workspaceRoot, options.liveEditorDrafts ?? []); seedOperatorSettings(workspaceRoot, undefined); unregisterBenchmarkWorkspaceRunnables(workspaceRoot); - if (!options.workspaceRoot) { - await rm(workspaceRoot, { recursive: true, force: true }); - } + await realWorkspace.dispose(); } } @@ -331,17 +334,22 @@ async function collectGlobalDraftState( return { drafts: drafts as GlobalDraftState["drafts"] }; } -function seedLiveEditorDrafts( +async function seedLiveEditorDrafts( workspace: string, fixtures: GlobalLiveEditorDraftFixture[], -): void { +): Promise { for (const fixture of fixtures) { const itemKind = LIVE_EDITOR_ITEM_KINDS[fixture.type]; const storagePath = fixture.storagePath ?? fixture.effectivePath ?? ""; if (fixture.value !== undefined) { // Seed as a backend draft row, not an in-tab cell: a cell would shadow the // model's DB-backed edit when the output is read back via listGlobalDrafts. - seedBenchmarkDraft(workspace, itemKind, storagePath, fixture.value); + await DraftService.updateDraft({ + workspace, + kind: itemKind, + path: storagePath, + requestBody: { value: fixture.value, force: true }, + }); } UserDraft.setLiveEditorDraft({ workspace, @@ -382,6 +390,10 @@ function seedOperatorSettings( username: "", color: "", operator_settings: operatorSettings as OperatorSettings, + role: "operator", + capabilities: legacyOperatorCapabilities( + operatorSettings as OperatorSettings, + ), is_dev_workspace: false, disabled: false, }); diff --git a/ai_evals/adapters/frontend/core/global/realWorkspace.ts b/ai_evals/adapters/frontend/core/global/realWorkspace.ts new file mode 100644 index 0000000000..80b556b7f6 --- /dev/null +++ b/ai_evals/adapters/frontend/core/global/realWorkspace.ts @@ -0,0 +1,670 @@ +// A real Windmill workspace per global-mode attempt. The session's tools run on the backend +// (`/api/w//ai_session_tools/`), so a case's fixture has to exist there: this creates +// the workspace, seeds it through the public API, and deletes it afterwards. +import { execFile } from "node:child_process"; +import { randomUUID } from "node:crypto"; +import { promisify } from "node:util"; +import type { WindmillBackendSettings } from "../../../../core/windmillBackendSettings"; +import type { EvalCaseProfile } from "../../../../core/types"; +import { + syncedResourceTypes, + type BenchmarkDatatableSeed, + type BenchmarkWorkspaceRunnables, +} from "../../mockBackend"; +import { WindmillBackendClient } from "../../windmillBackend"; +import { buildWorkspaceId } from "../../workspaceId"; +import type { GlobalUserFixture } from "./globalEvalRunner"; + +/** Workspace id -> the token its case runs as. Read by the adapter's fetch stub. */ +const activeWorkspaces = new Map(); +let adminToken: string | undefined; +let baseUrl: string | undefined; + +export function realWorkspaceRoute( + url: string, +): { url: string; token: string } | undefined { + if (!baseUrl || !adminToken || !url.startsWith("/api/")) return undefined; + const workspace = /^\/api\/w\/([^/?]+)/.exec(url)?.[1]; + const token = workspace + ? activeWorkspaces.get(decodeURIComponent(workspace)) + : activeWorkspaces.size > 0 + ? adminToken + : undefined; + return token ? { url: `${baseUrl}${url}`, token } : undefined; +} + +// Without the `postgresql` tag (a `native` worker group) datatable fixtures and SQL tools hang. +const DATATABLE_PG_URL = + process.env.WMILL_AI_EVAL_DATATABLE_PG_URL ?? + "postgres://postgres:changeme@localhost:5432/postgres"; +const ROOT_PG_RESOURCE = "$res:u/admin/ai_evals_pg_root"; +const RESOURCE_TYPE_INDEX_WAIT_MS = 120_000; +const WORKSPACE_SLOT_WAIT_MS = 30 * 60_000; +/** The eval backend's own database, where a fixture user that is not the admin is created. */ +const DATABASE_URL = process.env.WMILL_AI_EVAL_DATABASE_URL; + +export interface RealWorkspace { + id: string; + /** Settings that make the model proxy reuse this workspace instead of creating another. */ + backend: WindmillBackendSettings; + dispose(): Promise; +} + +export async function createRealWorkspace(input: { + settings: WindmillBackendSettings; + caseId: string; + attempt: number; + fixtures: BenchmarkWorkspaceRunnables; + user?: GlobalUserFixture; + profile?: EvalCaseProfile; +}): Promise { + // Job ids are unique across the instance, and a case prompt can name a fixture job by its + // id: attempts that seed the same ids take turns, each holding them until it is deleted. + const releaseJobIds = await holdJobIds( + (input.fixtures.jobs ?? []).flatMap((job) => (job.id ? [job.id] : [])), + ); + const client = new WindmillBackendClient(input.settings); + baseUrl = input.settings.baseUrl; + const id = buildWorkspaceId(input.caseId, input.attempt); + const api = apiFor(client, id); + try { + adminToken = await client.getToken(); + await createWorkspace(api, id); + } catch (error) { + releaseJobIds(); + throw error; + } + let datatableDb: string | undefined; + let userToken: string | undefined; + const dispose = async () => { + activeWorkspaces.delete(id); + if (input.user && DATABASE_URL) { + await psql(`DELETE FROM token WHERE label = ${literal(tokenLabel(id))}`).catch( + () => undefined, + ); + } + const workers = (input.fixtures.workers ?? []).map((w) => literal(w.worker)); + if (workers.length) { + await psql(`DELETE FROM worker_ping WHERE worker IN (${workers.join(", ")})`).catch( + () => undefined, + ); + } + if (datatableDb) { + await sql( + api, + ROOT_PG_RESOURCE, + `DROP DATABASE IF EXISTS "${datatableDb}" WITH (FORCE)`, + ).catch(() => undefined); + } + await api("DELETE", `/workspaces/delete/${id}`).catch(() => undefined); + releaseJobIds(); + }; + try { + activeWorkspaces.set(id, adminToken); + userToken = await seedUser(api, id, input.user, input.profile); + datatableDb = await seed(api, id, input.fixtures); + activeWorkspaces.set(id, userToken ?? adminToken); + return { + id, + backend: { ...input.settings, workspaceOverride: id }, + dispose, + }; + } catch (error) { + await dispose(); + throw error; + } +} + +/** Job id -> the turn of the last attempt that asked for it. */ +const jobIdTurns = new Map>(); + +async function holdJobIds(ids: string[]): Promise<() => void> { + let release!: () => void; + const turn = new Promise((resolve) => (release = resolve)); + const previous = ids.map((id) => jobIdTurns.get(id) ?? Promise.resolve()); + for (const id of ids) jobIdTurns.set(id, turn); + await Promise.all(previous); + return release; +} + +type Api = (method: string, path: string, body?: unknown) => Promise; + +function apiFor(client: WindmillBackendClient, workspace: string): Api { + return async (method, path, body) => { + const scoped = path.startsWith("/") ? path : `/w/${workspace}/${path}`; + const response = await client.request(scoped, { + method, + ...(body instanceof FormData + ? { body } + : body === undefined + ? {} + : { + headers: { "Content-Type": "application/json" }, + body: JSON.stringify(body), + }), + }); + const text = await response.text(); + if (!response.ok) { + throw new Error(`${method} ${scoped}: ${response.status} ${text}`); + } + try { + return JSON.parse(text); + } catch { + return text; + } + }; +} + +// CE caps the number of workspaces, so a run with more concurrent attempts than slots waits. +async function createWorkspace(api: Api, id: string): Promise { + const deadline = Date.now() + WORKSPACE_SLOT_WAIT_MS; + for (;;) { + try { + await api("POST", "/workspaces/create", { id, name: id }); + return; + } catch (error) { + if ( + !String(error).includes("maximum number of workspaces") || + Date.now() > deadline + ) { + throw error; + } + await new Promise((resolve) => setTimeout(resolve, 5_000)); + } + } +} + +// The admin running the eval is `admin` in every workspace it creates. Any other identity is a +// real member, so the backend enforces its rights as it would for that user; CE cannot create +// users, so there the case runs as the admin, with the prompt alone describing the fixture user. +async function seedUser( + api: Api, + workspace: string, + user: GlobalUserFixture | undefined, + profile: EvalCaseProfile | undefined, +): Promise { + const username = user?.username ?? "admin"; + const writable = new Set(user?.folders ?? []); + const readable = new Set([...(user?.folders_read ?? []), ...writable]); + const asAdmin = username === "admin" && (user?.is_admin ?? true); + let token: string | undefined; + if (!asAdmin) { + // CE cannot create users through the API, so the user and a token of theirs are written + // straight into the instance database, as the backend's own tests do. The workspace + // membership still goes through the API. + const email = `${username}@ai-evals.windmill.dev`; + token = `aievals${randomUUID().replace(/-/g, "")}`; + await psql( + `INSERT INTO password (email, password_hash, login_type, super_admin, verified, name, username) + VALUES (${literal(email)}, 'ai-evals-no-login', 'password', false, true, ${literal(username)}, + ${literal(username)}) + ON CONFLICT (email) DO UPDATE SET username = EXCLUDED.username; + INSERT INTO token (token_hash, token_prefix, token, email, label, super_admin) + VALUES (encode(sha256(${literal(token)}::bytea), 'hex'), ${literal(token.slice(0, 10))}, + ${literal(token)}, ${literal(email)}, ${literal(tokenLabel(workspace))}, false);`, + ); + await api("POST", "workspaces/add_user", { + email, + is_admin: user?.is_admin ?? false, + operator: profile?.operator ?? false, + }); + } + if (profile?.operator && profile.operatorSettings) { + await api("POST", "workspaces/operator_settings", profile.operatorSettings); + } + for (const folder of readable) { + await api("POST", "folders/create", { + name: folder, + ...(asAdmin ? {} : { extra_perms: { [`u/${username}`]: writable.has(folder) } }), + }); + } + return token; +} + +/** Seeds every fixture kind; returns the scratch database a datatable fixture created. */ +async function seed( + api: Api, + workspace: string, + fixtures: BenchmarkWorkspaceRunnables, +): Promise { + // A fixture that sets no `resourceTypes` stands for an instance synced with the hub, + // which an eval backend usually is not; the types the instance already holds stay its own. + const held = new Set(await api("GET", "resources/type/listnames")); + const resourceTypes = (fixtures.resourceTypes ?? syncedResourceTypes()).filter( + (rt) => !held.has(rt.name), + ); + for (const rt of resourceTypes) { + await api("POST", "resources/type/create", { + name: rt.name, + schema: rt.schema ?? {}, + description: rt.description ?? "", + }); + } + // The resource type search reads an index the server rebuilds every + // RESOURCE_TYPE_EMBEDDINGS_CHECK_INTERVAL_SECS: a case must not start before it holds them. + const last = resourceTypes[resourceTypes.length - 1]?.name; + for (const deadline = Date.now() + RESOURCE_TYPE_INDEX_WAIT_MS; last; ) { + const found: { name: string }[] = await api( + "GET", + `embeddings/query_resource_types?text=${encodeURIComponent(last)}&limit=10`, + ); + if (found.some((rt) => rt.name === last)) break; + if (Date.now() > deadline) { + throw new Error(`the resource type search never found the seeded "${last}"`); + } + await new Promise((resolve) => setTimeout(resolve, 2_000)); + } + for (const group of fixtures.groups ?? []) { + await api("POST", "groups/create", { name: group.name, summary: group.summary }); + // A fixture names members who are not users of the instance; a membership row needs none. + for (const member of group.members ?? []) { + await psql( + `INSERT INTO usr_to_group (workspace_id, usr, group_) + VALUES (${literal(workspace)}, ${literal(member)}, ${literal(group.name)})`, + "This fixture seeds group members", + ); + } + } + for (const folder of fixtures.folders ?? []) { + await api("POST", "folders/create", { + name: folder.name, + summary: folder.summary, + owners: folder.owners ?? [], + extra_perms: folder.extra_perms ?? {}, + }); + } + for (const script of fixtures.scripts ?? []) { + await api("POST", "scripts/create", { + path: script.path, + summary: script.summary, + description: script.description ?? "", + content: script.content, + language: script.language, + schema: script.schema ?? {}, + }); + } + for (const flow of fixtures.flows ?? []) { + await api("POST", "flows/create", { + path: flow.path, + summary: flow.summary, + description: flow.description ?? "", + value: flow.value, + schema: flow.schema ?? {}, + }); + } + for (const app of fixtures.apps ?? []) { + await seedApp(api, app); + } + for (const variable of fixtures.variables ?? []) { + await api("POST", "variables/create", { + path: variable.path, + value: variable.value, + is_secret: variable.is_secret, + description: variable.description ?? "", + ...(variable.labels ? { labels: variable.labels } : {}), + }); + } + for (const resource of fixtures.resources ?? []) { + await api("POST", "resources/create", { + path: resource.path, + resource_type: resource.resource_type, + value: resource.value ?? {}, + description: resource.description ?? "", + }); + } + await seedAiProviders(api, fixtures.aiProviders ?? []); + for (const job of fixtures.jobs ?? []) { + await seedJob(api, workspace, job); + } + for (const schedule of fixtures.schedules ?? []) { + await api("POST", "schedules/create", { + path: schedule.path, + schedule: schedule.schedule, + timezone: "UTC", + script_path: schedule.script_path, + is_flow: schedule.is_flow ?? false, + args: {}, + enabled: schedule.enabled ?? true, + summary: schedule.summary, + }); + } + await seedWorkers(fixtures.workers ?? []); + if (fixtures.ducklakes?.length) { + await api("POST", "workspaces/edit_ducklake_config", { + settings: { + ducklakes: Object.fromEntries( + fixtures.ducklakes.map((name) => [ + name, + { + catalog: { resource_type: "instance", resource_path: `${workspace}_${name}` }, + storage: { storage: null, path: name }, + }, + ]), + ), + }, + }); + } + await seedDataMetrics(api, fixtures.dataMetrics ?? []); + const datatables = fixtures.datatables ?? []; + if (datatables.length > 1) { + throw new Error("A global fixture seeds at most one datatable."); + } + return datatables[0] ? await seedDatatable(api, workspace, datatables[0]) : undefined; +} + +async function seedApp( + api: Api, + app: NonNullable[number], +): Promise { + const policy = { execution_mode: "publisher", triggerables_v2: {} }; + if (app.rawApp === false) { + await api("POST", "apps/create", { + path: app.path, + summary: app.summary, + value: app.value, + policy, + }); + return; + } + const form = new FormData(); + form.append( + "app", + JSON.stringify({ + path: app.path, + summary: app.summary, + value: { ...app.value, data: app.value.data ?? { tables: [] } }, + policy, + }), + ); + form.append("js", ""); + form.append("css", ""); + await api("POST", "apps/create_raw", form); +} + +async function seedAiProviders( + api: Api, + providers: NonNullable, +): Promise { + if (!providers.length) return; + const config: Record = {}; + let defaultModel: { model: string; provider: string } | undefined; + for (const provider of providers) { + await api("POST", "resources/create", { + path: provider.path, + resource_type: provider.kind, + value: { + api_key: "ai-evals-placeholder", + ...(provider.base_url ? { base_url: provider.base_url } : {}), + }, + }); + const models = provider.configuredModels ?? provider.models ?? []; + config[provider.kind] = { resource_path: provider.path, models }; + if (provider.isDefault && models[0]) { + defaultModel = { model: models[0], provider: provider.kind }; + } + } + await api("POST", "workspaces/edit_copilot_config", { + providers: config, + ...(defaultModel ? { default_model: defaultModel } : {}), + }); +} + +// A completed run with the fixture's id, path, args, logs and outcome. It is a preview job run +// by the admin, so `createdBy` and a `script` job kind are not reproduced. +async function seedJob( + api: Api, + workspace: string, + job: NonNullable[number], +): Promise { + const args = job.args ?? {}; + const id = job.id ?? randomUUID(); + const state = job.state ?? "completed"; + if (state === "suspended") { + // A flow parked on an approval step: its first step suspends, as the fixture flow's does. + await api("POST", `jobs/run/f/${job.scriptPath}?job_id=${id}`, args); + await waitFor(`run ${id} to suspend`, async () => { + const run = await api("GET", `jobs_u/get/${id}?no_logs=true`); + return (run.flow_status?.modules ?? []).some( + (module: { type?: string }) => module.type === "WaitingForEvents", + ); + }); + } else if (state === "queued") { + // Stays queued only on a tag no worker of the eval backend serves, as the fixture's does. + await api("POST", `jobs/run/preview?job_id=${id}`, { + content: "export async function main() {}", + language: "bun", + path: job.scriptPath, + args, + tag: job.tag, + }); + } else if (state === "completed") { + const content = [ + `export async function main(${Object.keys(args).join(", ")}) {`, + ` console.log(${JSON.stringify(job.logs ?? "")});`, + job.success === false + ? ` throw new Error(${JSON.stringify(lastLine(job.logs) ?? "Job failed")});` + : ` return ${JSON.stringify(job.result ?? null)};`, + `}`, + ].join("\n"); + await api("POST", `jobs/run_wait_result/preview?job_id=${id}`, { + content, + language: "bun", + path: job.scriptPath, + args, + }).catch((error) => { + // A failing fixture run answers with its error, which is the point. + if (job.success !== false) throw error; + }); + } else { + throw new Error(`A ${state} fixture run cannot be seeded on a real backend.`); + } + // Who ran it, when and where cannot be asked of the API: the run is rewritten to match. + if ( + job.createdBy === undefined && + job.createdMinutesAgo === undefined && + job.durationMs === undefined && + job.tag === undefined && + job.schedulePath === undefined && + job.label === undefined + ) { + return; + } + const createdAt = + job.createdMinutesAgo === undefined + ? "created_at" + : `now() - interval '${Number(job.createdMinutesAgo)} minutes'`; + await psql( + `UPDATE v2_job SET created_at = ${createdAt}, + created_by = COALESCE(${literal(job.createdBy)}, created_by), + tag = COALESCE(${literal(job.tag)}, tag), + labels = COALESCE(${job.label === undefined ? "NULL" : `ARRAY[${literal(job.label)}]`}, labels) + ${job.schedulePath ? `, trigger = ${literal(job.schedulePath)}, trigger_kind = 'schedule'` : ""} + WHERE id = ${literal(id)} AND workspace_id = ${literal(workspace)}; + UPDATE v2_job_queue q SET created_at = j.created_at, scheduled_for = j.created_at + FROM v2_job j WHERE q.id = j.id AND q.id = ${literal(id)}; + UPDATE v2_job_completed c SET started_at = j.created_at, + duration_ms = COALESCE(${literal(job.durationMs)}, c.duration_ms), + completed_at = j.created_at + make_interval(secs => COALESCE(${literal(job.durationMs)}, c.duration_ms) / 1000.0) + FROM v2_job j WHERE c.id = j.id AND c.id = ${literal(id)};`, + "This fixture seeds who ran a run, when or where", + ); +} + +/** Fixture workers, listed beside the instance's own until the attempt ends. */ +async function seedWorkers( + workers: NonNullable, +): Promise { + // ponytail: rows are instance-wide, so concurrent attempts of one case share them. + for (const worker of workers) { + await psql( + `INSERT INTO worker_ping (worker, worker_instance, ping_at, started_at, ip, jobs_executed, + custom_tags, worker_group) + VALUES (${literal(worker.worker)}, ${literal(worker.worker)}, + now() - interval '${Number(worker.lastPingSecondsAgo ?? 0)} seconds', now(), 'ai-evals', + ${Number(worker.jobs_executed ?? 0)}, + ARRAY[${(worker.custom_tags ?? []).map(literal).join(", ")}]::varchar[], + ${literal(worker.worker_group)}) + ON CONFLICT (worker) DO UPDATE SET ping_at = EXCLUDED.ping_at`, + "This fixture seeds workers", + ); + } +} + +async function waitFor(what: string, done: () => Promise): Promise { + for (const deadline = Date.now() + 60_000; !(await done()); ) { + if (Date.now() > deadline) throw new Error(`timed out waiting for ${what}`); + await new Promise((resolve) => setTimeout(resolve, 500)); + } +} + +function lastLine(text: string | undefined): string | undefined { + return text?.trim().split("\n").pop(); +} + +// Metrics are catalogued from a DuckDB node's annotations when it is deployed. +async function seedDataMetrics( + api: Api, + metrics: NonNullable, +): Promise { + const byScript = new Map(); + for (const metric of metrics) { + const key = `${metric.script_path}\n${metric.table_path}`; + byScript.set(key, [...(byScript.get(key) ?? []), metric]); + } + for (const [key, entries] of byScript) { + const [scriptPath, tablePath] = key.split("\n"); + // `/.
`, as the catalog canonicalizes `ducklake:///
`. + const [lake, qualified] = tablePath.split("/"); + const table = qualified.split(".").pop(); + const lines = [`-- materialize ducklake://${lake}/${table}`]; + for (const metric of entries) { + lines.push( + metric.kind === "measure" + ? `-- measure ${metric.name} = ${metric.expr}${metric.filter ? ` where ${metric.filter}` : ""}` + : `-- dimension ${metric.name} = ${metric.expr}`, + ); + } + lines.push("SELECT 1"); + await api("POST", "scripts/create", { + path: scriptPath, + summary: "", + description: "", + content: lines.join("\n"), + language: "duckdb", + schema: {}, + }); + } +} + +async function seedDatatable( + api: Api, + workspace: string, + seed: BenchmarkDatatableSeed, +): Promise { + const root = new URL(DATATABLE_PG_URL); + const database = `${workspace.replace(/-/g, "_")}_dt`.slice(0, 63); + const resource = (dbname: string) => ({ + host: root.hostname, + port: Number(root.port || 5432), + user: decodeURIComponent(root.username), + password: decodeURIComponent(root.password), + dbname, + sslmode: root.searchParams.get("sslmode") ?? "disable", + }); + await api("POST", "resources/create", { + path: "u/admin/ai_evals_pg_root", + resource_type: "postgresql", + value: resource(root.pathname.slice(1) || "postgres"), + }); + await sql(api, ROOT_PG_RESOURCE, `CREATE DATABASE "${database}"`); + await api("POST", "resources/create", { + path: "u/admin/ai_evals_datatable", + resource_type: "postgresql", + value: resource(database), + }); + await api("POST", "workspaces/edit_datatable_config", { + settings: { + datatables: { + [seed.datatable_name]: { + database: { resource_type: "postgresql", resource_path: "u/admin/ai_evals_datatable" }, + }, + }, + }, + }); + // A new datatable keeps its schema as migrations; a fixture says when this one does not. + if (!seed.migrations_enabled) { + await api("POST", `workspaces/disable_datatable_migrations/${seed.datatable_name}`); + } + const target = `datatable://${seed.datatable_name}`; + for (const migration of seed.migrations ?? []) { + await api("POST", `workspaces/upsert_datatable_migration/${seed.datatable_name}`, { + timestamp: migration.timestamp, + name: migration.name, + code_up: migration.code_up, + ...(migration.code_down ? { code_down: migration.code_down } : {}), + }); + if (migration.applied) { + await api( + "POST", + `workspaces/run_datatable_migrations/${seed.datatable_name}?only=${migration.timestamp}`, + ); + } + } + const statements: string[] = []; + for (const [schema, tables] of Object.entries(seed.schemas)) { + statements.push(`CREATE SCHEMA IF NOT EXISTS "${schema}"`); + for (const [table, { columns, rows }] of Object.entries(tables)) { + const name = `"${schema}"."${table}"`; + statements.push( + `CREATE TABLE IF NOT EXISTS ${name} (${Object.entries(columns) + .map(([column, type]) => `"${column}" ${type}`) + .join(", ")})`, + ); + for (const row of rows ?? []) { + const keys = Object.keys(row); + statements.push( + `INSERT INTO ${name} (${keys.map((k) => `"${k}"`).join(", ")}) VALUES (${keys + .map((k) => literal(row[k])) + .join(", ")})`, + ); + } + } + } + await sql(api, target, statements.join(";\n")); + return database; +} + +function literal(value: unknown): string { + if (value === null || value === undefined) return "NULL"; + if (typeof value === "number" || typeof value === "boolean") return String(value); + const text = typeof value === "string" ? value : JSON.stringify(value); + return `'${text.replace(/'/g, "''")}'`; +} + +async function sql(api: Api, database: string, content: string): Promise { + const result = await api("POST", "jobs/run_wait_result/preview", { + content, + language: "postgresql", + args: { database }, + }); + if (result && typeof result === "object" && "error" in result) { + throw new Error(`seeding SQL failed: ${JSON.stringify(result.error)}`); + } +} + +/** Each attempt's user token carries its workspace, so the attempt revokes exactly its own. */ +function tokenLabel(workspace: string): string { + return `ai-evals ${workspace}`; +} + +async function psql( + sql: string, + why = "This fixture's user is not the admin", +): Promise { + if (!DATABASE_URL) { + throw new Error( + `${why}: set WMILL_AI_EVAL_DATABASE_URL to the eval backend's database URL so the harness can write it.`, + ); + } + await promisify(execFile)("psql", [DATABASE_URL, "-v", "ON_ERROR_STOP=1", "-qc", sql]); +} diff --git a/ai_evals/adapters/frontend/core/shared/baseEvalRunner.ts b/ai_evals/adapters/frontend/core/shared/baseEvalRunner.ts index 13aaa59806..13cb852229 100644 --- a/ai_evals/adapters/frontend/core/shared/baseEvalRunner.ts +++ b/ai_evals/adapters/frontend/core/shared/baseEvalRunner.ts @@ -138,7 +138,7 @@ export async function runEval( // Accepts the run form exactly as the model prefilled it: there is nobody here to // edit the arguments, so a case can assert what the model proposed but never how // it reacts to the user changing something. - requestRunArgs: async (_toolId, form) => form.args, + requestRunArgs: async (_toolId, form) => ({ kind: "args", args: form.args }), ...(pendingAnswers && { requestUserQuestion: async () => { const answer = pendingAnswers.shift(); diff --git a/ai_evals/adapters/frontend/datatableSqlEngine.test.ts b/ai_evals/adapters/frontend/datatableSqlEngine.test.ts deleted file mode 100644 index e0398b407b..0000000000 --- a/ai_evals/adapters/frontend/datatableSqlEngine.test.ts +++ /dev/null @@ -1,262 +0,0 @@ -import { describe, expect, it } from 'bun:test' -import { applyDatatableSql, type BenchmarkDatatableSeed } from './datatableSqlEngine' - -function makeDatatable(): BenchmarkDatatableSeed { - return { - datatable_name: 'main', - schemas: { - public: { - orders: { - columns: { id: 'int4', customer_id: 'int4', total: 'numeric', status: 'text' }, - rows: [ - { id: 1, customer_id: 1, total: 42.5, status: 'shipped' }, - { id: 2, customer_id: 2, total: 19.99, status: 'pending' }, - { id: 3, customer_id: 1, total: 88, status: 'shipped' } - ] - }, - customers: { - columns: { id: 'int4', name: 'text' }, - rows: [{ id: 1, name: 'Alice' }] - } - } - } - } -} - -describe('SELECT', () => { - it('returns the referenced table rows', () => { - const dt = makeDatatable() - expect(applyDatatableSql(dt, 'SELECT id, name FROM customers').rows).toEqual([ - { id: 1, name: 'Alice' } - ]) - }) - - it('falls back to the first table when no known table is referenced', () => { - const dt = makeDatatable() - expect(applyDatatableSql(dt, 'select 1').rows).toHaveLength(3) - }) - - it('resolves a schema-qualified table', () => { - const dt = makeDatatable() - expect(applyDatatableSql(dt, 'SELECT * FROM public.customers').rows).toEqual([ - { id: 1, name: 'Alice' } - ]) - }) -}) - -describe('CREATE TABLE', () => { - it('adds a table with parsed columns, skipping table constraints and FK clauses', () => { - const dt = makeDatatable() - const result = applyDatatableSql( - dt, - 'CREATE TABLE public.refunds (\n order_id int4 NOT NULL REFERENCES public.orders(id),\n amount numeric(10,2),\n PRIMARY KEY (order_id)\n)' - ) - expect(result.rows).toEqual([]) - expect(dt.schemas.public.refunds).toEqual({ - columns: { order_id: 'int4', amount: 'numeric(10,2)' }, - rows: [] - }) - }) - - it('defaults an unqualified table to the public schema', () => { - const dt = makeDatatable() - applyDatatableSql(dt, 'CREATE TABLE notes (id int4, body text)') - expect(dt.schemas.public.notes.columns).toEqual({ id: 'int4', body: 'text' }) - }) - - it('is a no-op for an existing table with IF NOT EXISTS', () => { - const dt = makeDatatable() - applyDatatableSql(dt, 'CREATE TABLE IF NOT EXISTS public.orders (x int4)') - expect(Object.keys(dt.schemas.public.orders.columns)).toContain('status') - }) -}) - -describe('DROP TABLE', () => { - it('removes the table', () => { - const dt = makeDatatable() - applyDatatableSql(dt, 'DROP TABLE IF EXISTS public.customers') - expect(dt.schemas.public.customers).toBeUndefined() - }) -}) - -describe('INSERT', () => { - it('appends a row using an explicit column list', () => { - const dt = makeDatatable() - applyDatatableSql(dt, "INSERT INTO customers (id, name) VALUES (2, 'Bob')") - expect(dt.schemas.public.customers.rows).toContainEqual({ id: 2, name: 'Bob' }) - }) - - it('infers columns from the table when none are given, and appends multiple tuples', () => { - const dt = makeDatatable() - applyDatatableSql(dt, "INSERT INTO customers VALUES (2, 'Bob'), (3, 'Carol')") - expect(dt.schemas.public.customers.rows).toHaveLength(3) - }) - - it('returns the inserted rows when RETURNING is present', () => { - const dt = makeDatatable() - const result = applyDatatableSql( - dt, - "INSERT INTO customers (id, name) VALUES (2, 'Bob') RETURNING *" - ) - expect(result.rows).toEqual([{ id: 2, name: 'Bob' }]) - }) -}) - -describe('UPDATE', () => { - it('updates only the rows matching an equality WHERE', () => { - const dt = makeDatatable() - const result = applyDatatableSql( - dt, - "UPDATE public.orders SET status = 'shipped' WHERE id = 2" - ) - expect(result.rows).toEqual([]) - expect(dt.schemas.public.orders.rows?.find((r) => r.id === 2)?.status).toBe('shipped') - expect(dt.schemas.public.orders.rows?.find((r) => r.id === 1)?.status).toBe('shipped') - }) - - it('strips a Postgres cast in the WHERE value', () => { - const dt = makeDatatable() - applyDatatableSql(dt, "UPDATE orders SET status = 'done' WHERE id = 2::int4") - expect(dt.schemas.public.orders.rows?.find((r) => r.id === 2)?.status).toBe('done') - }) - - it('matches multiple AND predicates including a numeric literal', () => { - const dt = makeDatatable() - applyDatatableSql( - dt, - "UPDATE orders SET status = 'done' WHERE customer_id = 2 AND total = 19.99" - ) - expect(dt.schemas.public.orders.rows?.find((r) => r.id === 2)?.status).toBe('done') - expect(dt.schemas.public.orders.rows?.find((r) => r.id === 1)?.status).toBe('shipped') - }) - - it('updates every row when there is no WHERE', () => { - const dt = makeDatatable() - applyDatatableSql(dt, "UPDATE orders SET status = 'archived'") - expect(dt.schemas.public.orders.rows?.every((r) => r.status === 'archived')).toBe(true) - }) - - it('returns the affected rows when RETURNING is present', () => { - const dt = makeDatatable() - const result = applyDatatableSql( - dt, - "UPDATE orders SET status = 'shipped' WHERE id = 2 RETURNING *" - ) - expect(result.rows).toHaveLength(1) - expect(result.rows[0]).toMatchObject({ id: 2, status: 'shipped' }) - }) - - it('affects no rows when the WHERE clause cannot be parsed', () => { - const dt = makeDatatable() - applyDatatableSql(dt, "UPDATE orders SET status = 'x' WHERE total > 20") - expect(dt.schemas.public.orders.rows?.some((r) => r.status === 'x')).toBe(false) - }) -}) - -describe('DELETE', () => { - it('removes only the matching rows', () => { - const dt = makeDatatable() - applyDatatableSql(dt, 'DELETE FROM orders WHERE id = 2') - expect(dt.schemas.public.orders.rows?.map((r) => r.id)).toEqual([1, 3]) - }) - - it('returns the removed rows when RETURNING is present', () => { - const dt = makeDatatable() - const result = applyDatatableSql(dt, 'DELETE FROM orders WHERE id = 2 RETURNING *') - expect(result.rows).toEqual([{ id: 2, customer_id: 2, total: 19.99, status: 'pending' }]) - }) -}) - -describe('writes are reflected by later reads', () => { - it('UPDATE then SELECT sees the new value (the verify-loop fix)', () => { - const dt = makeDatatable() - applyDatatableSql(dt, "UPDATE orders SET status = 'shipped' WHERE id = 2") - const seen = applyDatatableSql(dt, 'SELECT * FROM orders').rows - expect(seen.find((r) => r.id === 2)?.status).toBe('shipped') - }) - - it('INSERT then SELECT sees the new row', () => { - const dt = makeDatatable() - applyDatatableSql(dt, "INSERT INTO customers (id, name) VALUES (9, 'Zed')") - const seen = applyDatatableSql(dt, 'SELECT * FROM customers').rows - expect(seen).toContainEqual({ id: 9, name: 'Zed' }) - }) - - it('CREATE then SELECT on the new table returns its (empty) rows', () => { - const dt = makeDatatable() - applyDatatableSql(dt, 'CREATE TABLE public.refunds (order_id int4, amount numeric)') - expect(applyDatatableSql(dt, 'SELECT * FROM refunds').rows).toEqual([]) - }) -}) - -describe('system-catalog queries reflect the current tables/columns', () => { - it('lists current tables (including a freshly created one) via information_schema.tables', () => { - const dt = makeDatatable() - applyDatatableSql(dt, 'CREATE TABLE public.refunds (order_id int4)') - const rows = applyDatatableSql( - dt, - "SELECT table_name FROM information_schema.tables WHERE table_name = 'refunds'" - ).rows - expect(rows.map((r) => r.table_name)).toContain('refunds') - }) - - it('does not list a dropped table', () => { - const dt = makeDatatable() - applyDatatableSql(dt, 'DROP TABLE public.customers') - const rows = applyDatatableSql(dt, 'SELECT table_name FROM information_schema.tables').rows - expect(rows.map((r) => r.table_name)).not.toContain('customers') - }) - - it('reports columns via information_schema.columns', () => { - const dt = makeDatatable() - const rows = applyDatatableSql( - dt, - "SELECT column_name FROM information_schema.columns WHERE table_name = 'orders'" - ).rows - expect(rows.map((r) => r.column_name)).toContain('status') - }) -}) - -describe('parser robustness (string/paren-aware splitting)', () => { - it('does not treat the word "returning" inside a string value as a RETURNING clause', () => { - const dt = makeDatatable() - const result = applyDatatableSql( - dt, - "INSERT INTO customers (id, name) VALUES (5, 'is returning soon')" - ) - expect(result.rows).toEqual([]) - expect(dt.schemas.public.customers.rows).toContainEqual({ id: 5, name: 'is returning soon' }) - }) - - it('does not split on the word "where" inside a SET string value', () => { - const dt = makeDatatable() - applyDatatableSql(dt, "UPDATE orders SET status = 'ship where ordered' WHERE id = 2") - expect(dt.schemas.public.orders.rows?.find((r) => r.id === 2)?.status).toBe('ship where ordered') - expect(dt.schemas.public.orders.rows?.find((r) => r.id === 1)?.status).toBe('shipped') - }) - - it('keeps INSERT tuples intact when a value contains a function call', () => { - const dt = makeDatatable() - applyDatatableSql(dt, "INSERT INTO customers (id, name) VALUES (6, coalesce(NULL, 'x'))") - expect(dt.schemas.public.customers.rows).toHaveLength(2) - expect(dt.schemas.public.customers.rows?.[1]).toMatchObject({ id: 6 }) - }) - - it('CREATE TABLE ignores a trailing semicolon-separated statement', () => { - const dt = makeDatatable() - applyDatatableSql( - dt, - 'CREATE TABLE public.refunds (id int4, amount numeric); INSERT INTO refunds VALUES (1, 5)' - ) - expect(dt.schemas.public.refunds.columns).toEqual({ id: 'int4', amount: 'numeric' }) - expect(dt.schemas.public.refunds.rows).toEqual([]) - }) -}) - -describe('unparseable statements are a safe no-op', () => { - it('returns [] and does not throw', () => { - const dt = makeDatatable() - expect(applyDatatableSql(dt, 'VACUUM ANALYZE').rows).toEqual([]) - expect(applyDatatableSql(dt, 'GRANT SELECT ON orders TO someone').rows).toEqual([]) - }) -}) diff --git a/ai_evals/adapters/frontend/datatableSqlEngine.ts b/ai_evals/adapters/frontend/datatableSqlEngine.ts deleted file mode 100644 index 96a62f6804..0000000000 --- a/ai_evals/adapters/frontend/datatableSqlEngine.ts +++ /dev/null @@ -1,552 +0,0 @@ -/** - * A deliberately small, best-effort SQL engine for the benchmark datatable mock. - * - * This is NOT a real SQL implementation — it exists only so that writes a model - * issues during an eval (`CREATE TABLE`, `INSERT`, `UPDATE`, `DELETE`, `DROP`) - * become visible to its later reads (`list_datatables`, `get_datatable_table_schema`, - * `SELECT`). Without that, a model that re-queries to verify a write sees stale - * seed data, concludes the write failed, and loops until it exhausts its turns. - * - * It parses only the common statement shapes models produce. Anything it cannot - * parse is a no-op success (it never throws) — behavioral evals assert that the - * right statement was issued, not its exact data effects. Notable limits: - * - `SELECT` returns all rows of the referenced (or first) table — no WHERE - * filtering, projection, joins, or aggregation. - * - `WHERE` supports `col = value` predicates joined by `AND` only; an - * unparseable WHERE on UPDATE/DELETE affects zero rows (never the whole table). - */ - -/** One seeded datatable table: its columns (col -> compact_type) and optional rows. */ -export interface BenchmarkDatatableTableSeed { - columns: Record - rows?: Record[] -} - -/** A seeded datatable: `datatable_name` plus a `schema -> table -> seed` map. */ -export interface BenchmarkDatatableSeed { - datatable_name: string - schemas: { - [schema: string]: { - [table: string]: BenchmarkDatatableTableSeed - } - } - /** Whether the datatable keeps its schema as migrations. */ - migrations_enabled?: boolean - migrations?: BenchmarkDatatableMigrationSeed[] -} - -export interface BenchmarkDatatableMigrationSeed { - timestamp: number - name: string - code_up: string - code_down?: string - applied?: boolean -} - -export interface DatatableSqlResult { - rows: Record[] -} - -const DEFAULT_SCHEMA = 'public' - -type ParsedRef = { schema: string; table: string } -type Predicate = { column: string; value: unknown } - -/** - * Apply one SQL statement to `datatable` IN PLACE and return the result rows. - * SELECT returns the referenced/first table's rows; a mutation returns its - * affected rows when it has a RETURNING clause, otherwise `[]`. - */ -export function applyDatatableSql( - datatable: BenchmarkDatatableSeed, - sql: string -): DatatableSqlResult { - const statement = stripTrailingSemicolon(sql.trim()) - if (/^\s*(with|select)\b/i.test(statement)) { - return { rows: selectRows(datatable, statement) } - } - if (/^\s*create\s+table\b/i.test(statement)) { - return { rows: applyCreateTable(datatable, statement) } - } - if (/^\s*drop\s+table\b/i.test(statement)) { - return { rows: applyDropTable(datatable, statement) } - } - if (/^\s*insert\s+into\b/i.test(statement)) { - return { rows: applyInsert(datatable, statement) } - } - if (/^\s*update\b/i.test(statement)) { - return { rows: applyUpdate(datatable, statement) } - } - if (/^\s*delete\s+from\b/i.test(statement)) { - return { rows: applyDelete(datatable, statement) } - } - return { rows: [] } -} - -// ============= Reads ============= - -function selectRows( - datatable: BenchmarkDatatableSeed, - sql: string -): Record[] { - const fromRef = sql.match(/\bfrom\s+([a-zA-Z_"][\w."]*)/i)?.[1] - if (fromRef) { - const catalog = catalogRows(datatable, fromRef) - if (catalog) { - return catalog - } - } - const table = fromRef ? resolveTable(datatable, fromRef) : undefined - const seed = table ?? firstTable(datatable) - return seed?.rows ?? [] -} - -/** - * Synthesize rows for a system-catalog query so a model verifying a `CREATE`/`DROP` - * via `information_schema.tables` / `.columns` (or `pg_tables`) sees the current - * tables/columns instead of fallback data. WHERE is not applied, so the model gets - * the full set and finds (or no longer finds) the table it just changed. - * Returns `undefined` for non-catalog refs so normal table resolution proceeds. - */ -function catalogRows( - datatable: BenchmarkDatatableSeed, - ref: string -): Record[] | undefined { - const normalized = ref.toLowerCase().replace(/"/g, '') - const name = normalized.split('.').pop() - const isCatalog = normalized.includes('information_schema.') || normalized.startsWith('pg_') - if (!isCatalog) { - return undefined - } - const tables = allTables(datatable) - if (name === 'tables' || name === 'pg_tables') { - return tables.map(({ schema, table }) => ({ - table_schema: schema, - table_name: table, - schemaname: schema, - tablename: table - })) - } - if (name === 'columns') { - return tables.flatMap(({ schema, table, seed }) => - Object.entries(seed.columns).map(([column, type]) => ({ - table_schema: schema, - table_name: table, - column_name: column, - data_type: type - })) - ) - } - return undefined -} - -function allTables( - datatable: BenchmarkDatatableSeed -): { schema: string; table: string; seed: BenchmarkDatatableTableSeed }[] { - return Object.entries(datatable.schemas).flatMap(([schema, tables]) => - Object.entries(tables).map(([table, seed]) => ({ schema, table, seed })) - ) -} - -// ============= DDL ============= - -function applyCreateTable( - datatable: BenchmarkDatatableSeed, - sql: string -): Record[] { - const head = sql.match( - /^\s*create\s+table\s+(?:if\s+not\s+exists\s+)?([a-zA-Z_"][\w."]*)/i - ) - // The first top-level paren group is the column-definition list; using it (rather - // than a greedy `(...)` capture) ignores any trailing `;`-separated statement. - const columnText = extractParenGroups(sql)[0] - if (!head || columnText === undefined) { - return [] - } - const { schema, table } = parseRef(head[1]) - const existing = datatable.schemas[schema]?.[table] - if (existing) { - return [] - } - const columns: Record = {} - for (const rawDef of splitTopLevel(columnText)) { - const def = rawDef.trim() - if (!def || isTableConstraint(def)) { - continue - } - const tokens = def.split(/\s+/) - const column = unquoteIdentifier(tokens[0]) - if (!column) { - continue - } - columns[column] = tokens[1] ?? 'text' - } - if (!datatable.schemas[schema]) { - datatable.schemas[schema] = {} - } - datatable.schemas[schema][table] = { columns, rows: [] } - return [] -} - -function applyDropTable( - datatable: BenchmarkDatatableSeed, - sql: string -): Record[] { - const match = sql.match( - /^\s*drop\s+table\s+(?:if\s+exists\s+)?([a-zA-Z_"][\w."]*)/i - ) - if (!match) { - return [] - } - const { schema, table } = parseRef(match[1]) - if (datatable.schemas[schema]?.[table]) { - delete datatable.schemas[schema][table] - } - return [] -} - -// ============= DML ============= - -function applyInsert( - datatable: BenchmarkDatatableSeed, - sql: string -): Record[] { - const { body, returning } = splitOffReturning(sql) - const match = body.match( - /^\s*insert\s+into\s+([a-zA-Z_"][\w."]*)\s*(?:\(([^)]*)\))?\s*values\s*([\s\S]+)$/i - ) - if (!match) { - return [] - } - const table = resolveTable(datatable, match[1]) - if (!table) { - return [] - } - const columns = match[2] - ? splitTopLevel(match[2]).map((entry) => unquoteIdentifier(entry.trim())) - : Object.keys(table.columns) - const inserted: Record[] = [] - for (const tuple of extractParenGroups(match[3])) { - const values = splitTopLevel(tuple).map((entry) => parseValue(entry)) - const row: Record = {} - columns.forEach((column, index) => { - row[column] = values[index] - }) - inserted.push(row) - } - table.rows ??= [] - table.rows.push(...inserted) - return returning ? inserted : [] -} - -function applyUpdate( - datatable: BenchmarkDatatableSeed, - sql: string -): Record[] { - const { body, returning } = splitOffReturning(sql) - const match = body.match(/^\s*update\s+([a-zA-Z_"][\w."]*)\s+set\s+([\s\S]+)$/i) - if (!match) { - return [] - } - const table = resolveTable(datatable, match[1]) - if (!table) { - return [] - } - let assignmentText = match[2] - let whereText: string | undefined - const whereMatch = maskForClauseScan(assignmentText).match(/\swhere\s/i) - if (whereMatch && whereMatch.index !== undefined) { - whereText = assignmentText.slice(whereMatch.index + whereMatch[0].length) - assignmentText = assignmentText.slice(0, whereMatch.index) - } - const predicates = parsePredicates(whereText) - if (predicates === null) { - return [] - } - const assignments: Record = {} - for (const entry of splitTopLevel(assignmentText)) { - const pair = entry.match(/^\s*([a-zA-Z_"][\w."]*)\s*=\s*([\s\S]+?)\s*$/) - if (pair) { - assignments[lastIdentifier(pair[1])] = parseValue(pair[2]) - } - } - const affected = (table.rows ?? []).filter((row) => rowMatches(row, predicates)) - for (const row of affected) { - Object.assign(row, assignments) - } - return returning ? affected : [] -} - -function applyDelete( - datatable: BenchmarkDatatableSeed, - sql: string -): Record[] { - const { body, returning } = splitOffReturning(sql) - const match = body.match(/^\s*delete\s+from\s+([a-zA-Z_"][\w."]*)\s*([\s\S]*)$/i) - if (!match) { - return [] - } - const table = resolveTable(datatable, match[1]) - if (!table) { - return [] - } - const whereText = match[2].replace(/^\s*where\s+/i, '').trim() || undefined - const predicates = parsePredicates(whereText) - if (predicates === null) { - return [] - } - const rows = table.rows ?? [] - const removed = rows.filter((row) => rowMatches(row, predicates)) - table.rows = rows.filter((row) => !rowMatches(row, predicates)) - return returning ? removed : [] -} - -// ============= Parsing helpers ============= - -function resolveTable( - datatable: BenchmarkDatatableSeed, - ref: string -): BenchmarkDatatableTableSeed | undefined { - const { schema, table } = parseRef(ref) - const direct = datatable.schemas[schema]?.[table] - if (direct) { - return direct - } - // Bare table name: fall back to searching every schema for a matching table. - if (!ref.includes('.')) { - for (const tables of Object.values(datatable.schemas)) { - if (tables[table]) { - return tables[table] - } - } - } - return undefined -} - -function firstTable( - datatable: BenchmarkDatatableSeed -): BenchmarkDatatableTableSeed | undefined { - for (const tables of Object.values(datatable.schemas)) { - for (const seed of Object.values(tables)) { - return seed - } - } - return undefined -} - -function parseRef(ref: string): ParsedRef { - const parts = ref.split('.').map(unquoteIdentifier) - if (parts.length >= 2) { - return { schema: parts[parts.length - 2], table: parts[parts.length - 1] } - } - return { schema: DEFAULT_SCHEMA, table: parts[0] } -} - -/** A WHERE clause with no parseable form returns `null`; absent WHERE returns `[]` (match all). */ -function parsePredicates(whereText: string | undefined): Predicate[] | null { - if (whereText === undefined || whereText.trim() === '') { - return [] - } - const predicates: Predicate[] = [] - for (const part of whereText.split(/\s+and\s+/i)) { - const match = part.match(/^\s*([a-zA-Z_"][\w."]*)\s*=\s*([\s\S]+?)\s*$/) - if (!match) { - return null - } - predicates.push({ column: lastIdentifier(match[1]), value: parseValue(match[2]) }) - } - return predicates -} - -function rowMatches(row: Record, predicates: Predicate[]): boolean { - return predicates.every((predicate) => looseEquals(row[predicate.column], predicate.value)) -} - -function looseEquals(left: unknown, right: unknown): boolean { - if (left === null || left === undefined) { - return right === null || right === undefined - } - if (typeof left === 'number' && typeof right === 'number') { - return left === right - } - return String(left) === String(right) -} - -function parseValue(raw: string): unknown { - // Drop a trailing Postgres cast (e.g. `2::int4`) before interpreting the literal. - const token = raw.trim().replace(/::\s*[a-zA-Z_][\w]*(\([^)]*\))?\s*$/, '').trim() - const stringMatch = token.match(/^'([\s\S]*)'$/) - if (stringMatch) { - return stringMatch[1].replace(/''/g, "'") - } - if (/^-?\d+(\.\d+)?$/.test(token)) { - return Number(token) - } - if (/^true$/i.test(token)) { - return true - } - if (/^false$/i.test(token)) { - return false - } - if (/^null$/i.test(token)) { - return null - } - return token -} - -function splitOffReturning(sql: string): { body: string; returning: boolean } { - const match = maskForClauseScan(sql).match(/\sreturning\s/i) - if (!match || match.index === undefined) { - return { body: sql, returning: false } - } - return { body: sql.slice(0, match.index), returning: true } -} - -/** - * A same-length copy of `sql` with the contents of single-quoted strings and - * parenthesized groups blanked to spaces, so a top-level keyword scan - * (WHERE / RETURNING) cannot match inside a string literal or a subquery. Index - * positions in the result map 1:1 back onto the original. - */ -function maskForClauseScan(sql: string): string { - let masked = '' - let depth = 0 - let inString = false - for (let i = 0; i < sql.length; i++) { - const char = sql[i] - if (inString) { - if (char === "'") { - if (sql[i + 1] === "'") { - masked += ' ' - i++ - continue - } - inString = false - } - masked += ' ' - continue - } - if (char === "'") { - inString = true - masked += ' ' - } else if (char === '(') { - depth++ - masked += ' ' - } else if (char === ')') { - depth = Math.max(0, depth - 1) - masked += ' ' - } else { - masked += depth > 0 ? ' ' : char - } - } - return masked -} - -/** - * Inner text of each top-level `( ... )` group in `input`, honoring nested parens - * (e.g. `now()`, `numeric(10,2)`) and single-quoted strings. Used for the CREATE - * column-definition group and INSERT value tuples. - */ -function extractParenGroups(input: string): string[] { - const groups: string[] = [] - let depth = 0 - let inString = false - let current = '' - for (let i = 0; i < input.length; i++) { - const char = input[i] - if (inString) { - current += char - if (char === "'") { - if (input[i + 1] === "'") { - current += input[++i] - } else { - inString = false - } - } - continue - } - if (char === "'") { - inString = true - current += char - } else if (char === '(') { - depth++ - if (depth === 1) { - current = '' - } else { - current += char - } - } else if (char === ')') { - depth = Math.max(0, depth - 1) - if (depth === 0) { - groups.push(current) - current = '' - } else { - current += char - } - } else if (depth > 0) { - current += char - } - } - return groups -} - -/** Split on commas that are not inside parentheses or single-quoted strings. */ -function splitTopLevel(input: string): string[] { - const parts: string[] = [] - let depth = 0 - let inString = false - let current = '' - for (let i = 0; i < input.length; i++) { - const char = input[i] - if (inString) { - current += char - if (char === "'") { - if (input[i + 1] === "'") { - current += input[++i] - } else { - inString = false - } - } - continue - } - if (char === "'") { - inString = true - current += char - } else if (char === '(') { - depth++ - current += char - } else if (char === ')') { - depth = Math.max(0, depth - 1) - current += char - } else if (char === ',' && depth === 0) { - parts.push(current) - current = '' - } else { - current += char - } - } - if (current.trim() !== '') { - parts.push(current) - } - return parts -} - -function isTableConstraint(def: string): boolean { - return /^(primary\s+key|foreign\s+key|constraint|unique|check|exclude|like)\b/i.test(def) -} - -function unquoteIdentifier(identifier: string): string { - const trimmed = identifier.trim() - const quoted = trimmed.match(/^"([\s\S]*)"$/) - return quoted ? quoted[1] : trimmed -} - -/** For a qualified reference like `orders.id`, keep only the final identifier. */ -function lastIdentifier(reference: string): string { - const parts = reference.split('.') - return unquoteIdentifier(parts[parts.length - 1]) -} - -function stripTrailingSemicolon(sql: string): string { - return sql.replace(/;\s*$/, '') -} diff --git a/ai_evals/adapters/frontend/mockBackend.ts b/ai_evals/adapters/frontend/mockBackend.ts index 1de463690b..5e1de3bda8 100644 --- a/ai_evals/adapters/frontend/mockBackend.ts +++ b/ai_evals/adapters/frontend/mockBackend.ts @@ -3,37 +3,9 @@ import confluenceFixture from '../../fixtures/frontend/global/hub/confluence.jso import servicenowFixture from '../../fixtures/frontend/global/hub/servicenow.json' import outreachFixture from '../../fixtures/frontend/global/hub/outreach.json' import syncedResourceTypesFixture from '../../fixtures/frontend/global/hub/synced_resource_types.json' -import type { - AppWithLastVersion, - CompletedJob, - Flow, - Job, - ListableApp, - ListableResource, - ListableVariable, - Folder, - Group, - QueuedJob, - Resource, - Schedule, - Script, - WorkerPing -} from '../../../frontend/src/lib/gen' -import type { - DataMetric, - DataTableTables, - DataTableTableSchema, - GetDraftForUserResponse, - GetOwnDraftResponse, - ListDraftsResponse, - ScriptLang, - UpdateDraftResponse, - UserDraftItemKind -} from '../../../frontend/src/lib/gen/types.gen' +import type { CompletedJob, Flow, Script } from '../../../frontend/src/lib/gen' +import type { DataMetric, ScriptLang } from '../../../frontend/src/lib/gen/types.gen' import { buildScriptLintResult } from './core/script/preview' -import { applyDatatableSql, type BenchmarkDatatableSeed } from './datatableSqlEngine' - -export type { BenchmarkDatatableSeed, BenchmarkDatatableTableSeed } from './datatableSqlEngine' const BENCHMARK_TIMESTAMP = '1970-01-01T00:00:00.000Z' @@ -174,6 +146,33 @@ export interface BenchmarkWorkspaceSchedule { summary?: string } +/** One seeded datatable table: its columns (col -> compact_type) and optional rows. */ +export interface BenchmarkDatatableTableSeed { + columns: Record + rows?: Record[] +} + +/** A seeded datatable: `datatable_name` plus a `schema -> table -> seed` map. */ +export interface BenchmarkDatatableSeed { + datatable_name: string + schemas: { + [schema: string]: { + [table: string]: BenchmarkDatatableTableSeed + } + } + /** Whether the datatable keeps its schema as migrations. */ + migrations_enabled?: boolean + migrations?: BenchmarkDatatableMigrationSeed[] +} + +export interface BenchmarkDatatableMigrationSeed { + timestamp: number + name: string + code_up: string + code_down?: string + applied?: boolean +} + export interface BenchmarkWorkspaceRunnables { scripts?: BenchmarkWorkspaceScript[] flows?: BenchmarkWorkspaceFlow[] @@ -198,18 +197,11 @@ export interface BenchmarkWorkspaceRunnables { } type BenchmarkCompletedJob = CompletedJob & { type: 'CompletedJob' } -type BenchmarkQueuedJob = QueuedJob & { type: 'QueuedJob' } const benchmarkWorkspaces = new Set() const benchmarkWorkspaceRunnables = new Map() -// Keyed by `${workspace}::${jobId}` so concurrent attempts (or distinct cases) -// can seed the same fixed job id without clobbering each other's entry. -const benchmarkJobs = new Map< - string, - { workspace: string; job: BenchmarkCompletedJob | BenchmarkQueuedJob } ->() -// What set_workspace_settings applied, per workspace, so get_workspace_settings reads it back. -const benchmarkWorkspaceSettings = new Map>() +// Keyed by `${workspace}::${jobId}` so a job is only visible from the workspace that ran it. +const benchmarkJobs = new Map() function benchmarkJobKey(workspace: string, jobId: string): string { return `${workspace}::${jobId}` @@ -219,15 +211,6 @@ export function resetBenchmarkMockBackend(): void { benchmarkWorkspaces.clear() benchmarkWorkspaceRunnables.clear() benchmarkJobs.clear() - benchmarkDrafts.clear() - benchmarkWorkspaceSettings.clear() -} - -// Stand-in for FolderService.createFolder so the global create_folder tool runs in -// memory instead of mutating the real backend. It echoes the name without adding the -// folder to the fixture folders that `listBenchmarkFolders` reads. -export function createBenchmarkFolder(_workspace: string, name: string): string { - return name } export function registerBenchmarkWorkspace(workspace: string): void { @@ -239,53 +222,12 @@ export function registerBenchmarkWorkspaceRunnables( runnables: BenchmarkWorkspaceRunnables ): void { benchmarkWorkspaces.add(workspace) - // Fresh case: drop any drafts left from a prior run on this workspace id. - clearBenchmarkDrafts(workspace) - // Datatables are mutated in place by exec_datatable_sql (a write must be visible - // to later reads), so store an isolated deep copy — never mutate the caller's seed. - benchmarkWorkspaceRunnables.set(workspace, { - ...runnables, - datatables: runnables.datatables ? structuredClone(runnables.datatables) : undefined - }) - // Seed any fixture jobs so list_runs / get_run have data to return. - for (const seed of runnables.jobs ?? []) { - const createdAt = - seed.createdMinutesAgo === undefined - ? undefined - : new Date(Date.now() - seed.createdMinutesAgo * 60_000).toISOString() - if (seed.state && seed.state !== 'completed') { - createBenchmarkQueuedJob(workspace, seed, createdAt) - continue - } - createBenchmarkCompletedJob({ - workspace, - id: seed.id, - jobKind: seed.jobKind ?? 'script', - success: seed.success, - scriptPath: seed.scriptPath, - createdBy: seed.createdBy, - label: seed.label, - logs: seed.logs, - args: seed.args, - result: seed.result, - tag: seed.tag, - schedulePath: seed.schedulePath, - createdAt, - durationMs: seed.durationMs - }) - } - if (runnables.workspaceSettings) { - benchmarkWorkspaceSettings.set(workspace, structuredClone(runnables.workspaceSettings)) - } else { - benchmarkWorkspaceSettings.delete(workspace) - } + benchmarkWorkspaceRunnables.set(workspace, runnables) } export function unregisterBenchmarkWorkspace(workspace: string): void { benchmarkWorkspaces.delete(workspace) benchmarkWorkspaceRunnables.delete(workspace) - benchmarkWorkspaceSettings.delete(workspace) - clearBenchmarkDrafts(workspace) for (const [jobId, entry] of benchmarkJobs.entries()) { if (entry.workspace === workspace) { benchmarkJobs.delete(jobId) @@ -293,6 +235,15 @@ export function unregisterBenchmarkWorkspace(workspace: string): void { } } +/** Answer a real workspace's provider model listings from its fixture, which no eval run can + * reach the providers for. Leaves its other API calls to the real backend. */ +export function registerBenchmarkAiModels( + workspace: string, + aiProviders: BenchmarkWorkspaceAiProvider[] +): void { + benchmarkWorkspaceRunnables.set(workspace, { aiProviders }) +} + export function unregisterBenchmarkWorkspaceRunnables(workspace: string): void { unregisterBenchmarkWorkspace(workspace) } @@ -341,124 +292,18 @@ export function getBenchmarkFlowByPath(workspace: string, path: string): Flow | return flow ? buildBenchmarkFlow(flow) : null } -export function listBenchmarkApps(workspace: string): ListableApp[] | null { - const runnables = benchmarkWorkspaceRunnables.get(workspace) - if (!runnables) { - return null - } - return (runnables.apps ?? []).map(buildBenchmarkListableApp) -} - -export function getBenchmarkAppByPath(workspace: string, path: string): AppWithLastVersion | null { - const app = benchmarkWorkspaceRunnables.get(workspace)?.apps?.find((entry) => entry.path === path) - - return app ? buildBenchmarkApp(app) : null -} - -function buildBenchmarkVariable( - workspace: string, - seed: BenchmarkWorkspaceVariable, - decryptSecret: boolean -): ListableVariable { - return { - workspace_id: workspace, - path: seed.path, - // Mirror `get_variable`: a secret's value is withheld unless decryption was - // asked for, so a reader genuinely cannot see it. - value: seed.is_secret && !decryptSecret ? undefined : seed.value, - is_secret: seed.is_secret, - description: seed.description, - labels: seed.labels, - ws_specific: seed.ws_specific ?? false, - extra_perms: {}, - edited_at: BENCHMARK_TIMESTAMP - } -} - -export function listBenchmarkVariables(workspace: string): ListableVariable[] | null { - const runnables = benchmarkWorkspaceRunnables.get(workspace) - if (!runnables) { - return null - } - // The list route never decrypts. - return (runnables.variables ?? []).map((seed) => buildBenchmarkVariable(workspace, seed, false)) -} - -/** AI provider resources of a benchmark workspace, shaped like `ResourceService.listResource` - * rows (which carry no value). Null when the workspace is not a benchmark one. */ -export function listBenchmarkAiProviderResources(workspace: string): ListableResource[] | null { - const runnables = benchmarkWorkspaceRunnables.get(workspace) - if (!runnables) { - return null - } - return (runnables.aiProviders ?? []).map((seed) => ({ - workspace_id: workspace, - path: seed.path, - resource_type: seed.kind, - value: null, - is_oauth: false, - is_linked: false, - is_refreshed: false, - extra_perms: {}, - edited_at: BENCHMARK_TIMESTAMP - })) -} - -/** Plain seeded resources of a benchmark workspace, shaped like `ResourceService.listResource` - * rows. Null when the workspace is not a benchmark one. */ -export function listBenchmarkPlainResources(workspace: string): ListableResource[] | null { - const runnables = benchmarkWorkspaceRunnables.get(workspace) - if (!runnables) { - return null - } - return (runnables.resources ?? []).map((seed) => ({ - workspace_id: workspace, - path: seed.path, - resource_type: seed.resource_type, - description: seed.description, - value: null, - is_oauth: false, - is_linked: false, - is_refreshed: false, - extra_perms: {}, - edited_at: BENCHMARK_TIMESTAMP - })) -} - -function benchmarkResourceTypes(workspace: string): BenchmarkWorkspaceResourceType[] { - return benchmarkWorkspaceRunnables.get(workspace)?.resourceTypes ?? syncedResourceTypes() -} - /** Resource type names of a benchmark workspace, as `ResourceService.listResourceTypeNames` * returns them. Null when the workspace is not a benchmark one. */ export function listBenchmarkResourceTypeNames(workspace: string): string[] | null { if (!benchmarkWorkspaces.has(workspace)) { return null } - return benchmarkResourceTypes(workspace).map((rt) => rt.name) -} - -/** Adds a resource type to a benchmark workspace, as `ResourceService.createResourceType` does, - * so later searches in the same case find it. */ -export function createBenchmarkResourceType( - workspace: string, - resourceType: BenchmarkWorkspaceResourceType -): void { - const runnables = benchmarkWorkspaceRunnables.get(workspace) - if (!runnables) { - throw new Error(`Workspace "${workspace}" is not a benchmark workspace`) - } - const existing = benchmarkResourceTypes(workspace) - if (existing.some((rt) => rt.name === resourceType.name)) { - throw new Error(`Resource type ${resourceType.name} already exists`) - } - runnables.resourceTypes = [...existing, resourceType] + return syncedResourceTypes().map((rt) => rt.name) } /** Stands in for the resource type search: a type matches when a word of the query appears in * its name or description, ranked by how many do. */ export function queryBenchmarkResourceTypes( - workspace: string, text: string, limit = 10 ): { name: string; schema?: unknown; score: number }[] { @@ -466,7 +311,7 @@ export function queryBenchmarkResourceTypes( .toLowerCase() .split(/[^a-z0-9]+/) .filter((word) => word.length >= 3) - return benchmarkResourceTypes(workspace) + return syncedResourceTypes() .map((rt) => { const haystack = `${rt.name} ${rt.description ?? ''}`.toLowerCase() return { name: rt.name, schema: rt.schema, score: words.filter((w) => haystack.includes(w)).length } @@ -476,95 +321,6 @@ export function queryBenchmarkResourceTypes( .slice(0, limit) } -/** The tables a fixture seeds for a database resource; `null` when it seeds none. */ -export function getBenchmarkDbSchemaSeed( - workspace: string, - path: string -): { resourceType: string; schema: NonNullable } | null { - const seed = benchmarkWorkspaceRunnables.get(workspace)?.resources?.find((r) => r.path === path) - return seed?.dbSchema ? { resourceType: seed.resource_type, schema: seed.dbSchema } : null -} - -/** A seeded resource with its value, as `ResourceService.getResource` returns it. Covers both - * seed kinds, so it agrees with `existsResource` and `listResource` — both of those report AI - * providers too, and a case that lists resources and then reads one by path would otherwise get - * a row it cannot fetch. */ -export function getBenchmarkResource(workspace: string, path: string): Resource | null { - const runnables = benchmarkWorkspaceRunnables.get(workspace) - const seed = runnables?.resources?.find((entry) => entry.path === path) - if (seed) { - return { - workspace_id: workspace, - path: seed.path, - resource_type: seed.resource_type, - description: seed.description, - value: seed.value ?? {}, - is_oauth: false, - extra_perms: {} - } as Resource - } - const provider = runnables?.aiProviders?.find((entry) => entry.path === path) - if (!provider) { - return null - } - return { - workspace_id: workspace, - path: provider.path, - resource_type: provider.kind, - value: getBenchmarkResourceValue(workspace, path) ?? {}, - is_oauth: false, - extra_perms: {} - } as Resource -} - -/** The value of a seeded resource. For an AI provider only the endpoint fields are modelled — a - * key is never needed, because no eval run calls the provider through this resource. */ -export function getBenchmarkResourceValue( - workspace: string, - path: string -): Record | null { - const runnables = benchmarkWorkspaceRunnables.get(workspace) - const plain = runnables?.resources?.find((entry) => entry.path === path) - if (plain) { - return plain.value ?? {} - } - const seed = runnables?.aiProviders?.find((entry) => entry.path === path) - if (!seed) { - return null - } - return seed.base_url ? { base_url: seed.base_url } : {} -} - -/** The AI settings of a benchmark workspace, as `WorkspaceService.getCopilotInfo` returns them. */ -export function getBenchmarkAiConfig(workspace: string): Record | null { - const seeds = benchmarkWorkspaceRunnables.get(workspace)?.aiProviders - if (!seeds) { - return null - } - const providers: Record = {} - let defaultModel: { model: string; provider: string } | undefined - for (const seed of seeds) { - const models = seed.configuredModels ?? seed.models ?? [] - providers[seed.kind] = { resource_path: seed.path, models } - if (seed.isDefault && models[0]) { - defaultModel = { model: models[0], provider: seed.kind } - } - } - return { providers, ...(defaultModel ? { default_model: defaultModel } : {}) } -} - -export function getBenchmarkVariableByPath( - workspace: string, - path: string, - decryptSecret = true -): ListableVariable | null { - const seed = benchmarkWorkspaceRunnables - .get(workspace) - ?.variables?.find((entry) => entry.path === path) - - return seed ? buildBenchmarkVariable(workspace, seed, decryptSecret) : null -} - export function createBenchmarkCompletedJob(input: { workspace: string jobKind: CompletedJob['job_kind'] @@ -572,32 +328,21 @@ export function createBenchmarkCompletedJob(input: { result?: unknown logs?: string scriptPath?: string - scriptHash?: string args?: Record - id?: string - createdBy?: string - label?: string - tag?: string - schedulePath?: string - createdAt?: string - durationMs?: number }): string { - const jobId = input.id ?? `benchmark-job-${randomUUID()}` - const createdAt = input.createdAt ?? new Date().toISOString() - const durationMs = input.durationMs ?? 0 + const jobId = `benchmark-job-${randomUUID()}` + const now = new Date().toISOString() const job: BenchmarkCompletedJob = { type: 'CompletedJob', id: jobId, workspace_id: input.workspace, - created_by: input.createdBy ?? 'ai-evals', - created_at: createdAt, - started_at: createdAt, - completed_at: new Date(Date.parse(createdAt) + durationMs).toISOString(), - duration_ms: durationMs, - schedule_path: input.schedulePath, + created_by: 'ai-evals', + created_at: now, + started_at: now, + completed_at: now, + duration_ms: 0, success: input.success ?? true, script_path: input.scriptPath, - script_hash: input.scriptHash, args: input.args, result: input.result, logs: input.logs, @@ -608,8 +353,7 @@ export function createBenchmarkCompletedJob(input: { is_skipped: false, email: 'ai-evals@local', visible_to_owner: true, - tag: input.tag ?? 'benchmark', - labels: input.label ? [input.label] : undefined + tag: 'benchmark' } benchmarkJobs.set(benchmarkJobKey(input.workspace, jobId), { @@ -619,41 +363,6 @@ export function createBenchmarkCompletedJob(input: { return jobId } -/** A run still in the queue: waiting for a worker, running, or a flow parked on an - * approval step (running with a suspend outstanding, as the backend reports it). */ -function createBenchmarkQueuedJob( - workspace: string, - seed: BenchmarkWorkspaceJob, - createdAt = new Date().toISOString() -): string { - const jobId = seed.id ?? `benchmark-job-${randomUUID()}` - const started = seed.state === 'running' || seed.state === 'suspended' - const job: BenchmarkQueuedJob = { - type: 'QueuedJob', - id: jobId, - workspace_id: workspace, - created_by: seed.createdBy ?? 'ai-evals', - created_at: createdAt, - scheduled_for: createdAt, - started_at: started ? createdAt : undefined, - running: started, - suspend: seed.state === 'suspended' ? 1 : undefined, - script_path: seed.scriptPath, - schedule_path: seed.schedulePath, - args: seed.args, - logs: seed.logs, - canceled: false, - job_kind: seed.jobKind ?? 'script', - permissioned_as: 'u/ai-evals', - is_flow_step: false, - email: 'ai-evals@local', - visible_to_owner: true, - tag: seed.tag ?? 'benchmark' - } - benchmarkJobs.set(benchmarkJobKey(workspace, jobId), { workspace, job }) - return jobId -} - export function getBenchmarkCompletedJob( workspace: string, jobId: string @@ -665,614 +374,6 @@ export function getBenchmarkCompletedJob( return structuredClone(entry.job) } -/** Any seeded run, finished or not, as `JobService.getJob` returns it. */ -export function getBenchmarkJob(workspace: string, jobId: string): Job | null { - const entry = benchmarkJobs.get(benchmarkJobKey(workspace, jobId)) - return entry ? (structuredClone(entry.job) as Job) : null -} - -/** - * List seeded/recorded jobs for a benchmark workspace, most recent first — - * the shape `JobService.listJobs` returns. Returns `null` for a non-benchmark - * workspace so the caller can fall through to the real backend. Server-side - * filters (path/creator/status/limit) are intentionally not applied: global - * eval cases assert on the recorded `list_runs` tool call, not on filtering. - */ -export function listBenchmarkJobs(workspace: string): Job[] | null { - if (!hasBenchmarkWorkspace(workspace)) { - return null - } - return [...benchmarkJobs.values()] - .filter((entry) => entry.workspace === workspace) - .map((entry) => structuredClone(entry.job) as Job) - .sort((a, b) => (b.created_at ?? '').localeCompare(a.created_at ?? '')) -} - -/** - * Mirror `JobService.getJobLogs` (response is the raw log string). Throws a - * "not found" error for an unknown id, matching the backend 404. - */ -export function getBenchmarkJobLogs(workspace: string, jobId: string): string { - const job = getBenchmarkJob(workspace, jobId) - if (!job) { - throw new Error(`Job Logs not found for "${jobId}"`) - } - return job.logs ?? '' -} - -/** - * Mirror `JobService.getFlowAllResults`, which get_run calls for the execution - * tree. Fixture jobs are single runs with no steps, so only the root entry. - */ -export function getBenchmarkFlowAllResults(workspace: string, jobId: string) { - const job = getBenchmarkJob(workspace, jobId) - if (!job) { - throw new Error(`Job "${jobId}" not found in benchmark workspace`) - } - return { - entries: [ - { - job_id: jobId, - label: 'Flow', - kind: job.job_kind ?? 'script', - depth: 0, - sibling_index: 1, - sibling_count: 1, - ...('success' in job - ? { status: job.success ? 'success' : 'failure', success: job.success } - : { status: job.suspend ? 'suspended' : job.running ? 'running' : 'queued' }) - } - ], - truncated: false, - scope_filtered: false - } -} - -// ============= Workers, schedules ============= - -/** Mirror `WorkerService.listWorkers`; `null` when no case seeded workers. Workers name no - * workspace, so this answers from whichever case seeded them: run cases whose worker seeds - * differ one at a time. */ -export function listBenchmarkWorkers(): WorkerPing[] | null { - let workers: BenchmarkWorkspaceWorker[] | undefined - for (const workspace of benchmarkWorkspaces) { - workers ??= benchmarkWorkspaceRunnables.get(workspace)?.workers - } - if (!workers) return null - return workers.map((w) => ({ - worker: w.worker, - worker_instance: `${w.worker}-instance`, - worker_group: w.worker_group, - custom_tags: w.custom_tags, - last_ping: w.lastPingSecondsAgo ?? 2, - started_at: BENCHMARK_TIMESTAMP, - ip: '10.0.0.1', - jobs_executed: w.jobs_executed ?? 0, - wm_version: 'benchmark' - })) -} - -function buildBenchmarkSchedule(workspace: string, s: BenchmarkWorkspaceSchedule): Schedule { - return { - path: s.path, - edited_by: 'benchmark', - edited_at: BENCHMARK_TIMESTAMP, - schedule: s.schedule, - timezone: 'Etc/UTC', - enabled: s.enabled ?? true, - script_path: s.script_path, - is_flow: s.is_flow ?? false, - extra_perms: {}, - email: 'ai-evals@local', - summary: s.summary, - workspace_id: workspace - } as unknown as Schedule -} - -export function listBenchmarkSchedules(workspace: string): Schedule[] { - return (benchmarkWorkspaceRunnables.get(workspace)?.schedules ?? []).map((s) => - buildBenchmarkSchedule(workspace, s) - ) -} - -export function getBenchmarkSchedule(workspace: string, path: string): Schedule | null { - const s = benchmarkWorkspaceRunnables.get(workspace)?.schedules?.find((e) => e.path === path) - return s ? buildBenchmarkSchedule(workspace, s) : null -} - -// ============= Folders, groups, workspace settings ============= - -function buildBenchmarkFolder(workspace: string, f: BenchmarkWorkspaceFolder): Folder { - return { - workspace_id: workspace, - name: f.name, - summary: f.summary, - owners: f.owners ?? [], - extra_perms: f.extra_perms ?? {} - } as Folder -} - -export function listBenchmarkFolders(workspace: string): Folder[] { - return (benchmarkWorkspaceRunnables.get(workspace)?.folders ?? []).map((f) => - buildBenchmarkFolder(workspace, f) - ) -} - -export function getBenchmarkFolder(workspace: string, name: string): Folder | null { - const f = benchmarkWorkspaceRunnables.get(workspace)?.folders?.find((e) => e.name === name) - return f ? buildBenchmarkFolder(workspace, f) : null -} - -export function listBenchmarkGroups(workspace: string): Group[] { - return (benchmarkWorkspaceRunnables.get(workspace)?.groups ?? []).flatMap((g) => { - const group = getBenchmarkGroup(workspace, g.name) - return group ? [group] : [] - }) -} - -export function getBenchmarkGroup(workspace: string, name: string): Group | null { - const g = benchmarkWorkspaceRunnables.get(workspace)?.groups?.find((e) => e.name === name) - return g - ? ({ - workspace_id: workspace, - name: g.name, - summary: g.summary, - members: g.members ?? [], - extra_perms: {} - } as Group) - : null -} - -const DATA_TABLE_DATABASE_TYPES = ['postgresql', 'instance', 'external_instance'] - -/** Refuse a data table entry the backend's `DataTableDatabase` would not deserialize. */ -function assertBenchmarkDataTables(value: unknown): void { - if (value === undefined || value === null) return - const datatables = (value as { datatables?: Record }).datatables ?? {} - for (const [name, entry] of Object.entries(datatables)) { - const { database, reference } = (entry ?? {}) as { - database?: { resource_type?: unknown; resource_path?: unknown } - reference?: unknown - } - // A fork's pointer at its parent's entry owns no database. - if (reference && !database) continue - if (!DATA_TABLE_DATABASE_TYPES.includes(String(database?.resource_type))) { - throw new Error( - `datatable.datatables.${name}.database.resource_type must be one of ${DATA_TABLE_DATABASE_TYPES.join(', ')}` - ) - } - if (typeof database?.resource_path !== 'string' || database.resource_path === '') { - throw new Error(`datatable.datatables.${name}.database.resource_path is required`) - } - } -} - -/** Mirror `WorkspaceService.getWorkspaceSettingsDocument`; an unseeded workspace has an - * empty document. */ -export function getBenchmarkWorkspaceSettingsDocument(workspace: string): Record { - return structuredClone(benchmarkWorkspaceSettings.get(workspace) ?? {}) -} - -/** Mirror `WorkspaceService.applyWorkspaceSettingsDocument` for a partial document: each - * top-level key replaces the stored one, `null` clears it. */ -export function applyBenchmarkWorkspaceSettingsDocument(input: { - workspace: string - requestBody: { document: Record; dry_run?: boolean } -}): { - changes: Array<{ key: string; before: unknown; after: unknown }> - applied: boolean -} { - assertBenchmarkDataTables(input.requestBody.document.datatable) - const current = benchmarkWorkspaceSettings.get(input.workspace) ?? {} - const changes = Object.entries(input.requestBody.document) - .filter(([key, after]) => JSON.stringify(current[key] ?? null) !== JSON.stringify(after ?? null)) - .map(([key, after]) => ({ key, before: current[key] ?? null, after: after ?? null })) - if (!input.requestBody.dry_run) { - const next = { ...current } - for (const { key, after } of changes) { - if (after === null) delete next[key] - else next[key] = structuredClone(after) - } - benchmarkWorkspaceSettings.set(input.workspace, next) - } - return { changes, applied: !input.requestBody.dry_run } -} - -// ============= Drafts (per-user, DB-backed in production) ============= - -/** - * In-memory stand-in for the per-user draft backend (`DraftService`). The global - * AI chat now persists and reads drafts through the backend DB instead of an - * in-tab `UserDraft` cell, so the eval mocks the draft endpoints it exercises - * (`updateDraft` / `getOwnDraft` / `getDraftForUser` / `listDrafts`) and keeps the - * saved values here, keyed by workspace + draft kind + storage path. Mirrors the - * semantics of the production unit test's mock in - * `frontend/src/lib/components/copilot/chat/global/core.test.ts`. - */ -const benchmarkDrafts = new Map< - string, - { - workspace: string - kind: UserDraftItemKind - path: string - value: unknown - createdAt: string - } ->() - -// Counter-based timestamps: deterministic run-to-run (same event order → same -// values) but MONOTONIC per update, because production bumps a draft row's -// created_at on every upsert and the diff snapshot cache keys patch reuse on -// it — a fixed timestamp would serve stale patches after an edit. No eval -// simulates a concurrent writer, so every save is accepted and the conflict -// branch is never taken. -let benchmarkDraftClock = 0 -function nextBenchmarkDraftTimestamp(): string { - benchmarkDraftClock += 1 - return new Date(benchmarkDraftClock * 1000).toISOString() -} - -function benchmarkDraftKey(workspace: string, kind: string, path: string): string { - return `${workspace}::${kind}::${path}` -} - -export function clearBenchmarkDrafts(workspace: string): void { - for (const [key, entry] of benchmarkDrafts.entries()) { - if (entry.workspace === workspace) { - benchmarkDrafts.delete(key) - } - } -} - -/** - * Seed a draft straight into the store — used by the eval's live-editor draft - * fixtures, which model "the user already has this draft open/saved". Writing it - * here (instead of through `UserDraft.save`) keeps it a backend draft row with no - * shadowing in-tab cell, so a model edit that persists to the backend is what the - * output read-back captures — not the stale seed. - */ -export function seedBenchmarkDraft( - workspace: string, - kind: UserDraftItemKind, - path: string, - value: unknown -): void { - benchmarkDrafts.set(benchmarkDraftKey(workspace, kind, path), { - workspace, - kind, - path, - value, - createdAt: nextBenchmarkDraftTimestamp() - }) -} - -/** Mirror `DraftService.updateDraft`: a `null`/omitted value deletes the row. */ -export function updateBenchmarkDraft(input: { - workspace: string - kind: UserDraftItemKind - path: string - requestBody?: { value?: unknown } -}): UpdateDraftResponse { - const key = benchmarkDraftKey(input.workspace, input.kind, input.path) - const value = input.requestBody?.value - const createdAt = nextBenchmarkDraftTimestamp() - if (value == null) { - benchmarkDrafts.delete(key) - } else { - benchmarkDrafts.set(key, { - workspace: input.workspace, - kind: input.kind, - path: input.path, - value, - createdAt - }) - } - return { status: 'saved', current_timestamp: createdAt } -} - -/** Mirror `DraftService.getDraftForUser`: 404-shaped throw when absent so the - * adapter's narrowed catch treats it as "no draft" instead of re-throwing. */ -export function getBenchmarkDraftForUser(input: { - workspace: string - kind: UserDraftItemKind - path: string -}): GetDraftForUserResponse { - const entry = benchmarkDrafts.get(benchmarkDraftKey(input.workspace, input.kind, input.path)) - if (!entry) { - throw Object.assign(new Error(`no draft for "${input.path}"`), { - status: 404 - }) - } - return { value: entry.value, created_at: entry.createdAt } -} - -/** Mirror `DraftService.getOwnDraft`: `null` (200) when absent — unlike - * `getDraftForUser`, absence is not an error on this route. */ -export function getBenchmarkOwnDraft(input: { - workspace: string - kind: UserDraftItemKind - path: string -}): GetOwnDraftResponse { - const entry = benchmarkDrafts.get(benchmarkDraftKey(input.workspace, input.kind, input.path)) - if (!entry) { - return null - } - return { value: entry.value, created_at: entry.createdAt } -} - -/** Whether a deployed benchmark item exists for a draft row's kind+path — - * drives `draft_only`, which production computes against the deployed tables. */ -function benchmarkDeployedExists( - workspace: string, - kind: UserDraftItemKind, - path: string -): boolean { - if (kind === 'script') return Boolean(getBenchmarkScriptByPath(workspace, path)) - if (kind === 'flow') return Boolean(getBenchmarkFlowByPath(workspace, path)) - if (kind === 'app' || kind === 'raw_app') return Boolean(getBenchmarkAppByPath(workspace, path)) - if (kind === 'variable') return Boolean(getBenchmarkVariableByPath(workspace, path)) - // The remaining drawer kinds (resources/schedules/triggers) have no deployed - // benchmark stores today. - return false -} - -/** Mirror `DraftService.listDrafts`: metadata rows (no value) for a workspace. */ -export function listBenchmarkDrafts(workspace: string): ListDraftsResponse { - return [...benchmarkDrafts.values()] - .filter((entry) => entry.workspace === workspace) - .map((entry) => ({ - kind: entry.kind, - path: entry.path, - summary: (entry.value as { summary?: string } | null)?.summary, - draft_only: !benchmarkDeployedExists(workspace, entry.kind, entry.path), - legacy_draft: false, - created_at: entry.createdAt - })) -} - -// ============= Datatables (best-effort in-memory SQL) ============= - -/** - * Project the seeded datatables down to the `list_datatable_tables` response: - * `datatable_name` + `schema -> table_names`, with no column detail. - * Returns `null` for a non-benchmark workspace so callers can fall through to - * the real backend; an empty seed yields `[]`. - */ -export function listBenchmarkDatatables(workspace: string): DataTableTables[] | null { - const runnables = benchmarkWorkspaceRunnables.get(workspace) - if (!runnables) { - return null - } - return (runnables.datatables ?? []).map((datatable) => ({ - datatable_name: datatable.datatable_name, - schemas: Object.fromEntries( - Object.entries(datatable.schemas).map(([schema, tables]) => [schema, Object.keys(tables)]) - ), - ...(datatable.migrations_enabled && { migrations_enabled: true }) - })) -} - -// ============= Datatable migrations ============= - -function benchmarkDatatable(workspace: string, datatableName: string): BenchmarkDatatableSeed { - const datatable = (benchmarkWorkspaceRunnables.get(workspace)?.datatables ?? []).find( - (entry) => entry.datatable_name === datatableName - ) - if (!datatable) { - throw new Error(`datatable "${datatableName}" not found`) - } - return datatable -} - -/** Run a migration body through the in-memory engine, one statement at a time and without the - * transaction frame, so the tables it creates show up in later reads. */ -function applyBenchmarkMigrationSql(datatable: BenchmarkDatatableSeed, sql: string): void { - // Split on the semicolons outside single-quoted literals. - for (const statement of sql.match(/(?:'(?:[^']|'')*'|[^;'])+/g) ?? []) { - const trimmed = statement.replace(/^\s*--.*$/gm, '').trim() - if (trimmed !== '' && !/^(begin|end|commit)$/i.test(trimmed)) { - applyDatatableSql(datatable, trimmed) - } - } -} - -export function getBenchmarkDatatableMigrationsStatus(input: { - workspace: string - datatableName: string -}) { - const datatable = benchmarkDatatable(input.workspace, input.datatableName) - if (!datatable.migrations_enabled) { - return { enabled: false, migrations: [] } - } - return { - enabled: true, - migrations: (datatable.migrations ?? []).map(({ applied, ...migration }) => ({ - ...migration, - status: applied ? ('ran' as const) : ('not_run' as const) - })) - } -} - -export function createBenchmarkDatatableMigration(input: { - workspace: string - datatableName: string - requestBody: { name: string; code_up: string; code_down?: string } -}) { - const datatable = benchmarkDatatable(input.workspace, input.datatableName) - if (!datatable.migrations_enabled) { - throw new Error( - `Migrations are not enabled for data table '${input.datatableName}'. Enable them first.` - ) - } - const migrations = (datatable.migrations ??= []) - const timestamp = Math.max(20260101000000, ...migrations.map((m) => m.timestamp)) + 1 - migrations.push({ timestamp, ...input.requestBody }) - return { datatable: input.datatableName, timestamp, ...input.requestBody } -} - -export function upsertBenchmarkDatatableMigration(input: { - workspace: string - datatableName: string - requestBody: { timestamp: number; name: string; code_up: string; code_down?: string } -}): string { - const datatable = benchmarkDatatable(input.workspace, input.datatableName) - const migrations = (datatable.migrations ??= []) - const existing = migrations.find((m) => m.timestamp === input.requestBody.timestamp) - if (existing?.applied) { - throw new Error( - `Migration ${existing.timestamp} on data table '${input.datatableName}' has already been applied and cannot be modified` - ) - } - if (existing) { - // Replaced whole, as the server does: a down left out is stored as none. - delete existing.code_down - Object.assign(existing, input.requestBody) - } else { - migrations.push({ ...input.requestBody }) - migrations.sort((a, b) => a.timestamp - b.timestamp) - } - return `Upserted migration ${input.requestBody.timestamp} in ${input.datatableName}` -} - -export function runBenchmarkDatatableMigrations(input: { - workspace: string - datatableName: string - upTo?: number - only?: number -}) { - const datatable = benchmarkDatatable(input.workspace, input.datatableName) - const applied: { version: number; name: string }[] = [] - for (const migration of datatable.migrations ?? []) { - if ( - migration.applied || - (input.only !== undefined && migration.timestamp !== input.only) || - (input.only === undefined && input.upTo !== undefined && migration.timestamp > input.upTo) - ) { - continue - } - applyBenchmarkMigrationSql(datatable, migration.code_up) - migration.applied = true - applied.push({ version: migration.timestamp, name: migration.name }) - } - return { applied } -} - -export function rollbackBenchmarkDatatableMigration(input: { - workspace: string - datatableName: string - only?: number -}) { - const datatable = benchmarkDatatable(input.workspace, input.datatableName) - const last = (datatable.migrations ?? []) - .filter((m) => m.applied && (input.only === undefined || m.timestamp === input.only)) - .at(-1) - if (!last) { - return { rolled_back: [] } - } - if (last.code_down === undefined) { - throw new Error( - `Cannot roll back migration ${last.timestamp} (${last.name}): it has no down migration` - ) - } - applyBenchmarkMigrationSql(datatable, last.code_down) - last.applied = false - return { rolled_back: [{ version: last.timestamp, name: last.name }] } -} - -// ============= DuckLake catalogs and declared metrics ============= - -/** Seeded DuckLake names, or `null` for a non-benchmark workspace. */ -export function listBenchmarkDucklakes(workspace: string): string[] | null { - const runnables = benchmarkWorkspaceRunnables.get(workspace) - return runnables ? (runnables.ducklakes ?? []) : null -} - -/** - * Seeded metric declarations, or `null` for a non-benchmark workspace. - * - * The `table` / `path_prefix` filters are ignored: which rows a filter selects is - * `canonical_table_path`'s business and is pinned by `ducklakeTools.test.ts`. - * Re-deriving it here would give the eval its own copy of that spec to drift from, - * and the case this serves measures whether the model reaches for the tool at all. - */ -export function listBenchmarkDataMetrics(workspace: string): DataMetric[] | null { - const runnables = benchmarkWorkspaceRunnables.get(workspace) - return runnables ? (runnables.dataMetrics ?? []) : null -} - -export function getBenchmarkDatatableSchema(input: { - workspace: string - datatableName: string - schemaName: string - tableName: string -}): DataTableTableSchema { - const runnables = benchmarkWorkspaceRunnables.get(input.workspace) - const datatable = (runnables?.datatables ?? []).find( - (entry) => entry.datatable_name === input.datatableName - ) - if (!datatable) { - // Message MUST match the production `isDatatableNotConfiguredError` regex - // (/datatable\s+\S+\s+not found/i in datatableTools.ts) so the - // get_datatable_table_schema not-configured mapping is actually exercised. - throw new Error(`datatable "${input.datatableName}" not found`) - } - const table = datatable.schemas?.[input.schemaName]?.[input.tableName] - if (!table) { - throw new Error( - `table "${input.schemaName}.${input.tableName}" not found in datatable "${input.datatableName}"` - ) - } - return { - datatable_name: input.datatableName, - schema_name: input.schemaName, - table_name: input.tableName, - columns: table.columns - } -} - -/** - * Execute SQL against a seeded datatable through the best-effort in-memory engine - * (`applyDatatableSql`). Writes (CREATE/INSERT/UPDATE/DELETE/DROP) mutate the - * stored datatable in place so a later list/schema/SELECT reflects them; SELECT - * (and RETURNING) yield rows, other statements yield `[]`. Creates a benchmark - * completed job and returns its id, like `runBenchmarkScriptPreview`. - */ -export function runBenchmarkDatatableSql(input: { - workspace: string - datatableName: string - sql: string -}): string { - const runnables = benchmarkWorkspaceRunnables.get(input.workspace) - const datatable = (runnables?.datatables ?? []).find( - (entry) => entry.datatable_name === input.datatableName - ) - const rows = datatable ? applyDatatableSql(datatable, input.sql).rows : [] - return createBenchmarkCompletedJob({ - workspace: input.workspace, - jobKind: 'preview', - success: true, - args: { database: `datatable://${input.datatableName}` }, - result: rows - }) -} - -/** - * Mirror `JobService.getCompletedJobResultMaybe` for benchmark workspaces — the - * shape `pollJobResult` consumes. The job is created synchronously before - * polling, so it is always present and completed. - */ -export function getBenchmarkCompletedJobResultMaybe(input: { workspace: string; id: string }): { - success: boolean - completed: boolean - result: unknown -} { - const job = getBenchmarkCompletedJob(input.workspace, input.id) - if (!job) { - throw new Error(`Job "${input.id}" not found in benchmark workspace`) - } - return { success: job.success, completed: true, result: job.result } -} - export function runBenchmarkScriptPreview(input: { workspace: string requestBody: { @@ -1311,35 +412,6 @@ export function runBenchmarkScriptPreview(input: { }) } -export function runBenchmarkScriptByPath(input: { - workspace: string - path: string - args?: Record -}): string { - const script = getBenchmarkScriptByPath(input.workspace, input.path) - return createBenchmarkCompletedJob({ - workspace: input.workspace, - jobKind: 'script', - success: script !== null, - scriptPath: input.path, - args: input.args, - result: - script !== null - ? { - path: input.path, - args: input.args ?? {}, - mocked: true - } - : { - error: `Script "${input.path}" not found in benchmark workspace` - }, - logs: - script !== null - ? 'Mock benchmark script run completed successfully.' - : `Script "${input.path}" not found in benchmark workspace.` - }) -} - export function runBenchmarkFlowByPath(input: { workspace: string path: string @@ -1368,18 +440,10 @@ export function runBenchmarkFlowByPath(input: { }) } -/** - * Mirror `JobService.runFlowPreview` for benchmark workspaces, including the server's - * refusal of a chat-enabled flow run that names no conversation (`memory_id`). - */ export function runBenchmarkFlowPreview(input: { workspace: string - memoryId?: string - requestBody?: { path?: string; value?: { chat_input_enabled?: boolean }; args?: unknown } + requestBody?: { path?: string; args?: unknown } }): string { - if (input.requestBody?.value?.chat_input_enabled && !input.memoryId) { - throw new Error('Bad request: memory_id is required for chat-enabled flows') - } const args = (input.requestBody?.args ?? {}) as Record return createBenchmarkCompletedJob({ workspace: input.workspace, @@ -1491,44 +555,11 @@ function buildBenchmarkFlow(flow: BenchmarkWorkspaceFlow): Flow { } as Flow } -function buildBenchmarkListableApp(app: BenchmarkWorkspaceApp): ListableApp { - return { - id: 0, - workspace_id: 'benchmark', - path: app.path, - summary: app.summary, - version: 1, - extra_perms: {}, - edited_at: BENCHMARK_TIMESTAMP, - execution_mode: 'viewer', - raw_app: app.rawApp ?? true - } -} - -function buildBenchmarkApp(app: BenchmarkWorkspaceApp): AppWithLastVersion { - return { - id: 0, - workspace_id: 'benchmark', - path: app.path, - summary: app.summary, - versions: [1], - created_by: 'benchmark', - created_at: BENCHMARK_TIMESTAMP, - value: app.value, - policy: (app.value.policy ?? {}) as AppWithLastVersion['policy'], - execution_mode: 'viewer', - extra_perms: {}, - custom_path: app.value.custom_path as string | undefined, - raw_app: app.rawApp ?? true - } -} - -/** A stand-in Windmill hub. `search_hub_scripts` and a `hub/` read go out over - * relative `/api/...` fetches, which have no origin here, so without these the - * hub tools throw and no case can exercise hub reuse. Serving fixtures rather - * than the live hub also keeps assertions on script content stable as the real - * hub republishes new versions. */ -const BENCHMARK_INTEGRATION_META_PATH = /^\/api\/integrations\/hub\/([^/]+)\/meta$/ +// A stand-in Windmill hub. `search_hub_scripts` and a `hub/` read go out over +// relative `/api/...` fetches, which have no origin here, so without these the +// hub tools throw and no case can exercise hub reuse. Serving fixtures rather +// than the live hub also keeps assertions on script content stable as the real +// hub republishes new versions. /** One integration lifted verbatim from the content repo — its shipped scripts, its * authored meta.json and its resource type. Hand-written fixtures make both routes to @@ -1540,10 +571,6 @@ const REAL_HUB_INTEGRATIONS = [ outreachFixture ] as unknown as Array<{ app: string - display_name: string - description: string - docs_url: string - curated: boolean | null meta: unknown resource_type: { name: string; description: string; schema: unknown } scripts: Array<{ @@ -1791,56 +818,32 @@ function listBenchmarkHubScriptsByApp(app: string | null, kind: string | null) { })) } -/** What `/integrations/hub//meta` serves. Carries the integration's conventions - * and none of the endpoints a case asks the model to write, so the tool shortens the - * path to that knowledge without supplying answers. */ -/** The integrations a case writes against, none of which the content repo documents — - * the same majority shape as the live hub, where 18 of ~216 carry a meta.json. Their - * conventions live in their shipped scripts, which is what these cases exercise. */ -const BENCHMARK_HUB_INTEGRATION_META: Record< - string, - { - display_name: string - description: string - docs_url: string - curated: boolean | null - meta: unknown - } -> = { - baremetrics: { - display_name: 'Baremetrics', - description: 'Subscription analytics for recurring-revenue businesses.', - docs_url: 'https://developers.baremetrics.com/reference', - curated: null, - meta: null - }, - holded: { - display_name: 'Holded', - description: 'Invoicing, accounting and CRM for small businesses.', - docs_url: 'https://developers.holded.com/reference', - curated: null, - meta: null - } +/** Display names of the integrations a case writes against, none of which the content repo + * documents — the same majority shape as the live hub, where 18 of ~216 carry a meta.json. + * Their conventions live in their shipped scripts, which is what these cases exercise. */ +const BENCHMARK_HUB_INTEGRATION_NAMES: Record = { + baremetrics: 'Baremetrics', + holded: 'Holded' } -/** The resource type a mocked hub integration takes, as its metadata serves it. */ +/** The resource type a mocked hub integration takes. */ function benchmarkHubIntegrationResourceType(app: string): BenchmarkWorkspaceResourceType | null { const real = REAL_HUB_INTEGRATIONS.find((integration) => integration.app === app) if (real) { return real.resource_type } - const entry = BENCHMARK_HUB_INTEGRATION_META[app] - if (!entry) { + const displayName = BENCHMARK_HUB_INTEGRATION_NAMES[app] + if (!displayName) { return null } return { name: app, - description: `${entry.display_name} credentials`, + description: `${displayName} credentials`, schema: { type: 'object', required: ['apiKey'], properties: { - apiKey: { type: 'string', description: `${entry.display_name} API key` } + apiKey: { type: 'string', description: `${displayName} API key` } } } } @@ -1848,10 +851,12 @@ function benchmarkHubIntegrationResourceType(app: string): BenchmarkWorkspaceRes /** What a hub-synced instance holds: common hub types, plus the one each mocked hub * integration takes, so the hub and the instance agree on them. */ -function syncedResourceTypes(): BenchmarkWorkspaceResourceType[] { +/** The resource types of an instance synced with the hub, what a fixture that sets no + * `resourceTypes` stands for. */ +export function syncedResourceTypes(): BenchmarkWorkspaceResourceType[] { const integrations = [ ...REAL_HUB_INTEGRATIONS.map((integration) => integration.app), - ...Object.keys(BENCHMARK_HUB_INTEGRATION_META) + ...Object.keys(BENCHMARK_HUB_INTEGRATION_NAMES) ] return [ ...syncedResourceTypesFixture, @@ -1859,46 +864,6 @@ function syncedResourceTypes(): BenchmarkWorkspaceResourceType[] { ] } -/** Mirrors the hub's own derivation: hosts seen in the shipped scripts, whether they - * call the provider directly, and how many there are of each kind. */ -function benchmarkDerivedFacts(app: string) { - const scripts = BENCHMARK_HUB_SCRIPTS.filter((script) => script.app === app) - const hosts = new Map() - const languages: Record = {} - for (const script of scripts) { - languages[script.language] = (languages[script.language] ?? 0) + 1 - for (const match of script.content.matchAll(/https?:\/\/([a-zA-Z0-9._-]+)/g)) { - hosts.set(match[1], (hosts.get(match[1]) ?? 0) + 1) - } - } - return { - api_hosts: [...hosts.entries()] - .sort((a, b) => b[1] - a[1]) - .map(([host, count]) => ({ host, count })), - style: 'fetch', - languages, - script_counts: { - total: scripts.length, - by_kind: scripts.reduce>((acc, script) => { - const kind = script.kind ?? 'script' - acc[kind] = (acc[kind] ?? 0) + 1 - return acc - }, {}) - }, - top_scripts: scripts.map((script) => ({ - path: `hub/${script.version_id}/${script.app}/${script.summary.toLowerCase().replaceAll(/\s+/g, '_')}`, - ask_id: script.version_id, - version_id: script.version_id, - summary: script.summary, - description: script.description ?? null, - kind: script.kind ?? 'script', - language: script.language, - views: 0, - votes: 0 - })) - } -} - /** The hub keys a script by its version id; the app and slug segments that * follow are descriptive, so match on the id exactly as the real hub does. */ function getBenchmarkHubScript(path: string) { @@ -1906,29 +871,6 @@ function getBenchmarkHubScript(path: string) { return BENCHMARK_HUB_SCRIPTS.find((script) => script.version_id === versionId) } -const BENCHMARK_WORKERS = [ - { - worker: 'wk-benchmark-1', - worker_instance: 'benchmark-host', - last_ping: 2, - started_at: BENCHMARK_TIMESTAMP, - jobs_executed: 42, - custom_tags: null, - worker_group: 'default', - wm_version: 'benchmark' - }, - { - worker: 'wk-benchmark-2', - worker_instance: 'benchmark-host', - last_ping: 5, - started_at: BENCHMARK_TIMESTAMP, - jobs_executed: 17, - custom_tags: null, - worker_group: 'default', - wm_version: 'benchmark' - } -] - /** True when `handleBenchmarkApiFetch` has an answer for this `/api/...` url. * Any other relative fetch must keep its normal (non-benchmark) behavior — * intercepting it with a synthetic 404 sends the model into retry loops. */ @@ -1939,11 +881,9 @@ const BENCHMARK_AI_MODELS_PATH = /\/api\/w\/(.+)\/ai\/proxy\/models$/ export function hasBenchmarkApiHandler(url: string): boolean { const path = url.split('?')[0] return ( - path === '/api/workers/list' || path === '/api/embeddings/query_hub_scripts' || path === '/api/scripts/hub/top' || path === '/api/integrations/hub/list' || - BENCHMARK_INTEGRATION_META_PATH.test(path) || path.startsWith('/api/scripts/hub/get_full/') || BENCHMARK_AI_MODELS_PATH.test(path) ) @@ -1952,9 +892,6 @@ export function hasBenchmarkApiHandler(url: string): boolean { /** Answer the relative `/api/...` fetches no mocked service covers. */ export function handleBenchmarkApiFetch(url: string, init?: RequestInit): Response { const path = url.split('?')[0] - if (path === '/api/workers/list') { - return Response.json(BENCHMARK_WORKERS) - } // The provider's own model listing, which grounds an AI agent step's model id. Keyed by the // resource the caller names, so two seeded providers can serve different models. const aiModels = BENCHMARK_AI_MODELS_PATH.exec(path) @@ -1978,53 +915,13 @@ export function handleBenchmarkApiFetch(url: string, init?: RequestInit): Respon asks: listBenchmarkHubScriptsByApp(params.get('app'), params.get('kind')) }) } - const integrationMeta = BENCHMARK_INTEGRATION_META_PATH.exec(path) - if (integrationMeta) { - const app = decodeURIComponent(integrationMeta[1]) - const real = REAL_HUB_INTEGRATIONS.find((integration) => integration.app === app) - if (real) { - return Response.json({ - app, - display_name: real.display_name, - description: real.description, - docs_url: real.docs_url, - curated: real.curated, - metadata_source: real.meta ? 'curated' : 'derived', - meta: real.meta, - meta_updated_at: null, - derived: benchmarkDerivedFacts(app), - resource_types: [{ id: 1, ...benchmarkHubIntegrationResourceType(app) }] - }) - } - const entry = BENCHMARK_HUB_INTEGRATION_META[app] - if (!entry) { - return Response.json({ error: 'integration not found' }, { status: 404 }) - } - return Response.json({ - app, - display_name: entry.display_name, - description: entry.description, - docs_url: entry.docs_url, - curated: entry.curated, - metadata_source: entry.meta ? 'curated' : 'derived', - meta: entry.meta, - meta_updated_at: null, - derived: benchmarkDerivedFacts(app), - resource_types: [{ id: 1, ...benchmarkHubIntegrationResourceType(app) }] - }) - } if (path === '/api/integrations/hub/list') { const apps = [...new Set(BENCHMARK_HUB_SCRIPTS.map((script) => script.app))].sort() - // Read off both fixtures the metadata endpoint serves from, or the list would - // call an integration undocumented and then hand back its authored notes. - const documented = new Set([ - ...REAL_HUB_INTEGRATIONS.filter((integration) => integration.meta).map( + const documented = new Set( + REAL_HUB_INTEGRATIONS.filter((integration) => integration.meta).map( (integration) => integration.app - ), - ...Object.entries(BENCHMARK_HUB_INTEGRATION_META) - .filter(([, entry]) => entry.meta) - .map(([app]) => app) - ]) + ) + ) return Response.json(apps.map((name) => ({ name, documented: documented.has(name) }))) } if (path.startsWith('/api/scripts/hub/get_full/')) { diff --git a/ai_evals/adapters/frontend/mockBackendApi.test.ts b/ai_evals/adapters/frontend/mockBackendApi.test.ts deleted file mode 100644 index b3829e56da..0000000000 --- a/ai_evals/adapters/frontend/mockBackendApi.test.ts +++ /dev/null @@ -1,36 +0,0 @@ -import { describe, expect, it } from 'bun:test' -import { handleBenchmarkApiFetch, hasBenchmarkApiHandler } from './mockBackend' - -// `list_workers`'s client call is not faked, so its request really does leave as a relative -// `/api/...` url that node's fetch cannot resolve. Without this route the tool throws instead -// of answering. -describe('benchmark API fetch handlers', () => { - it('lists workers, the way list_workers reaches them', async () => { - const url = '/api/workers/list?per_page=100' - - expect(hasBenchmarkApiHandler(url)).toBe(true) - const body = await handleBenchmarkApiFetch(url).json() - - // The tool counts the list and maps over it, so it needs a bare array, not an envelope. - expect(Array.isArray(body)).toBe(true) - expect(body[0]).toMatchObject({ worker: expect.any(String), worker_group: expect.any(String) }) - }) -}) - -// A benchmark whose list calls an integration undocumented while its metadata endpoint -// hands back authored notes teaches the model the flag means nothing. -describe('benchmark hub integration list', () => { - it('flags exactly the integrations whose metadata carries authored notes', async () => { - const listed = (await handleBenchmarkApiFetch('/api/integrations/hub/list').json()) as Array<{ - name: string - documented: boolean - }> - expect(listed.length).toBeGreaterThan(0) - - for (const { name, documented } of listed) { - const res = handleBenchmarkApiFetch(`/api/integrations/hub/${name}/meta`) - const authored = res.status === 200 && !!((await res.json()) as { meta?: unknown }).meta - expect(authored).toBe(documented) - } - }) -}) diff --git a/ai_evals/adapters/frontend/mockBackendDatatables.test.ts b/ai_evals/adapters/frontend/mockBackendDatatables.test.ts deleted file mode 100644 index 5d12ebb661..0000000000 --- a/ai_evals/adapters/frontend/mockBackendDatatables.test.ts +++ /dev/null @@ -1,175 +0,0 @@ -import { afterEach, beforeEach, describe, expect, it } from 'bun:test' -import { - getBenchmarkCompletedJobResultMaybe, - getBenchmarkDatatableSchema, - listBenchmarkDatatables, - registerBenchmarkWorkspaceRunnables, - resetBenchmarkMockBackend, - runBenchmarkDatatableSql, - type BenchmarkWorkspaceRunnables -} from './mockBackend' - -const WORKSPACE = 'benchmark-datatable-ws' - -// Mirrors the production `isDatatableNotConfiguredError` regex in -// datatableTools.ts. The schema mock's "not configured" message MUST match it, -// otherwise the not-configured mapping in get_datatable_table_schema is silently -// untested. -const NOT_CONFIGURED_RE = /datatable\s+\S+\s+not found/i - -const SEED: BenchmarkWorkspaceRunnables = { - datatables: [ - { - datatable_name: 'main', - schemas: { - public: { - orders: { - columns: { id: 'int', total: 'numeric' }, - rows: [ - { id: 1, total: 10 }, - { id: 2, total: 20 } - ] - }, - customers: { - columns: { id: 'int', name: 'text' }, - rows: [{ id: 1, name: 'alice' }] - } - } - } - } - ] -} - -beforeEach(() => resetBenchmarkMockBackend()) -afterEach(() => resetBenchmarkMockBackend()) - -describe('listBenchmarkDatatables', () => { - it('returns null for a non-benchmark workspace (caller falls through to real backend)', () => { - expect(listBenchmarkDatatables('unregistered')).toBeNull() - }) - - it('returns [] for a registered workspace with no datatables seed', () => { - registerBenchmarkWorkspaceRunnables(WORKSPACE, {}) - expect(listBenchmarkDatatables(WORKSPACE)).toEqual([]) - }) - - it('projects seeded datatables to schema -> table names only (no columns)', () => { - registerBenchmarkWorkspaceRunnables(WORKSPACE, SEED) - expect(listBenchmarkDatatables(WORKSPACE)).toEqual([ - { datatable_name: 'main', schemas: { public: ['orders', 'customers'] } } - ]) - }) -}) - -describe('getBenchmarkDatatableSchema', () => { - beforeEach(() => registerBenchmarkWorkspaceRunnables(WORKSPACE, SEED)) - - it('returns the columns for a seeded table', () => { - expect( - getBenchmarkDatatableSchema({ - workspace: WORKSPACE, - datatableName: 'main', - schemaName: 'public', - tableName: 'orders' - }) - ).toEqual({ - datatable_name: 'main', - schema_name: 'public', - table_name: 'orders', - columns: { id: 'int', total: 'numeric' } - }) - }) - - it('throws a not-configured error matching the production regex for an unknown datatable', () => { - let error: Error | undefined - try { - getBenchmarkDatatableSchema({ - workspace: WORKSPACE, - datatableName: 'ghost', - schemaName: 'public', - tableName: 'orders' - }) - } catch (e) { - error = e as Error - } - expect(error).toBeDefined() - expect(error!.message).toMatch(NOT_CONFIGURED_RE) - }) - - it('throws a table-not-found error that does NOT match the datatable-not-configured regex', () => { - // The datatable IS configured; only the table is missing. Production maps - // this to a generic "error getting schema", not the blocking message. - let error: Error | undefined - try { - getBenchmarkDatatableSchema({ - workspace: WORKSPACE, - datatableName: 'main', - schemaName: 'public', - tableName: 'ghost' - }) - } catch (e) { - error = e as Error - } - expect(error).toBeDefined() - expect(error!.message).not.toMatch(NOT_CONFIGURED_RE) - }) -}) - -describe('runBenchmarkDatatableSql + getBenchmarkCompletedJobResultMaybe', () => { - beforeEach(() => registerBenchmarkWorkspaceRunnables(WORKSPACE, SEED)) - - function exec(sql: string): { success: boolean; completed: boolean; result: unknown } { - const jobId = runBenchmarkDatatableSql({ workspace: WORKSPACE, datatableName: 'main', sql }) - return getBenchmarkCompletedJobResultMaybe({ workspace: WORKSPACE, id: jobId }) - } - - it('returns the canned rows of the table named in a SELECT FROM clause', () => { - expect(exec('SELECT * FROM customers')).toEqual({ - success: true, - completed: true, - result: [{ id: 1, name: 'alice' }] - }) - }) - - it('falls back to the first seeded table when the SELECT references no known table', () => { - expect(exec('select 1').result).toEqual([ - { id: 1, total: 10 }, - { id: 2, total: 20 } - ]) - }) - - it('returns [] success for DDL and DML statements without RETURNING', () => { - expect(exec('CREATE TABLE foo (id int)').result).toEqual([]) - expect(exec('INSERT INTO orders VALUES (3, 30)').result).toEqual([]) - expect(exec('update orders set total = 0').result).toEqual([]) - }) - - it('reflects a write in a later SELECT, isolated from the shared seed', () => { - exec('UPDATE orders SET total = 999 WHERE id = 1') - expect((exec('SELECT * FROM orders').result as Record[])).toContainEqual({ - id: 1, - total: 999 - }) - // Registration deep-clones the seed, so the shared SEED const stays pristine. - expect(SEED.datatables![0].schemas.public.orders.rows).toContainEqual({ id: 1, total: 10 }) - }) - - it('reflects a CREATE in list_datatables and get_datatable_table_schema', () => { - exec('CREATE TABLE public.refunds (order_id int4, amount numeric)') - expect(listBenchmarkDatatables(WORKSPACE)?.[0].schemas.public).toContain('refunds') - expect( - getBenchmarkDatatableSchema({ - workspace: WORKSPACE, - datatableName: 'main', - schemaName: 'public', - tableName: 'refunds' - }).columns - ).toEqual({ order_id: 'int4', amount: 'numeric' }) - }) - - it('throws for an unknown job id', () => { - expect(() => - getBenchmarkCompletedJobResultMaybe({ workspace: WORKSPACE, id: 'does-not-exist' }) - ).toThrow() - }) -}) diff --git a/ai_evals/adapters/frontend/mockBackendDrafts.test.ts b/ai_evals/adapters/frontend/mockBackendDrafts.test.ts deleted file mode 100644 index 0ab79d216e..0000000000 --- a/ai_evals/adapters/frontend/mockBackendDrafts.test.ts +++ /dev/null @@ -1,116 +0,0 @@ -import { afterEach, beforeEach, describe, expect, it } from 'bun:test' -import { - clearBenchmarkDrafts, - getBenchmarkDraftForUser, - getBenchmarkOwnDraft, - listBenchmarkDrafts, - resetBenchmarkMockBackend, - seedBenchmarkDraft, - updateBenchmarkDraft -} from './mockBackend' - -const WORKSPACE = 'benchmark-drafts-ws' - -// Drives the in-memory stand-in for the per-user draft backend (`DraftService`) -// that the global AI-chat eval round-trips its drafts through. Mirrors the -// production-unit-test mock in -// `frontend/src/lib/components/copilot/chat/global/core.test.ts`. -describe('mockBackend drafts', () => { - beforeEach(() => resetBenchmarkMockBackend()) - afterEach(() => resetBenchmarkMockBackend()) - - it('round-trips a saved draft through update / get / list', () => { - const value = { summary: 'Greet a user', content: 'export async function main() {}' } - const res = updateBenchmarkDraft({ - workspace: WORKSPACE, - kind: 'script', - path: 'f/evals/greet', - requestBody: { value } - }) - expect(res.status).toBe('saved') - - expect(getBenchmarkDraftForUser({ workspace: WORKSPACE, kind: 'script', path: 'f/evals/greet' }).value).toEqual( - value - ) - - const rows = listBenchmarkDrafts(WORKSPACE) - expect(rows).toHaveLength(1) - expect(rows[0]).toMatchObject({ kind: 'script', path: 'f/evals/greet', summary: 'Greet a user', draft_only: true }) - }) - - it('treats a null value as a delete', () => { - updateBenchmarkDraft({ - workspace: WORKSPACE, - kind: 'variable', - path: 'f/evals/token', - requestBody: { value: { summary: 'token' } } - }) - updateBenchmarkDraft({ - workspace: WORKSPACE, - kind: 'variable', - path: 'f/evals/token', - requestBody: { value: null } - }) - - expect(listBenchmarkDrafts(WORKSPACE)).toHaveLength(0) - expect(() => getBenchmarkDraftForUser({ workspace: WORKSPACE, kind: 'variable', path: 'f/evals/token' })).toThrow() - }) - - it('returns null from getOwnDraft when no draft exists', () => { - expect( - getBenchmarkOwnDraft({ workspace: WORKSPACE, kind: 'trigger_schedule', path: 'u/evals/missing' }) - ).toBeNull() - }) - - // The global chat hydrates drawer-kind drafts (schedule/trigger/resource/variable) - // through getOwnDraft — getDraftForUser rejects those kinds as private. - it('hydrates a saved drawer-kind draft through getOwnDraft', () => { - const value = { path: 'u/evals/nightly', schedule: '0 0 9 * * *' } - updateBenchmarkDraft({ - workspace: WORKSPACE, - kind: 'trigger_schedule', - path: 'u/evals/nightly', - requestBody: { value } - }) - expect( - getBenchmarkOwnDraft({ workspace: WORKSPACE, kind: 'trigger_schedule', path: 'u/evals/nightly' })?.value - ).toEqual(value) - }) - - it('throws a 404-shaped error when no draft exists', () => { - try { - getBenchmarkDraftForUser({ workspace: WORKSPACE, kind: 'script', path: 'f/evals/missing' }) - throw new Error('expected a throw') - } catch (e) { - expect((e as { status?: number }).status).toBe(404) - } - }) - - it('seeds a draft as a backend row that a later edit overwrites', () => { - seedBenchmarkDraft(WORKSPACE, 'script', 'f/evals/current', { content: 'seed' }) - expect(getBenchmarkDraftForUser({ workspace: WORKSPACE, kind: 'script', path: 'f/evals/current' }).value).toEqual({ - content: 'seed' - }) - - // A model edit persists the same path and must win over the seed. - updateBenchmarkDraft({ - workspace: WORKSPACE, - kind: 'script', - path: 'f/evals/current', - requestBody: { value: { content: 'edited' } } - }) - expect(getBenchmarkDraftForUser({ workspace: WORKSPACE, kind: 'script', path: 'f/evals/current' }).value).toEqual({ - content: 'edited' - }) - }) - - it('clears only the targeted workspace', () => { - seedBenchmarkDraft(WORKSPACE, 'script', 'f/a', { content: 'a' }) - seedBenchmarkDraft('other-ws', 'script', 'f/b', { content: 'b' }) - - clearBenchmarkDrafts(WORKSPACE) - - expect(listBenchmarkDrafts(WORKSPACE)).toHaveLength(0) - expect(listBenchmarkDrafts('other-ws')).toHaveLength(1) - }) -}) diff --git a/ai_evals/adapters/frontend/vitestAdapter.test.ts b/ai_evals/adapters/frontend/vitestAdapter.test.ts index 754de224ac..b7529dc0d2 100644 --- a/ai_evals/adapters/frontend/vitestAdapter.test.ts +++ b/ai_evals/adapters/frontend/vitestAdapter.test.ts @@ -8,6 +8,7 @@ import { fileURLToPath } from 'url' import { handleBenchmarkApiFetch, hasBenchmarkApiHandler } from './mockBackend' import { resolveWindmillBackendSettings } from '../../core/windmillBackendSettings' import { WindmillBackendClient } from './windmillBackend' +import { realWorkspaceRoute } from './core/global/realWorkspace' const FRONTEND_DIR = fileURLToPath(new URL('../../../frontend/', import.meta.url)) @@ -30,6 +31,13 @@ globalThis.fetch = (async (input: unknown, init?: RequestInit) => { if (typeof url === 'string' && hasBenchmarkApiHandler(url)) { return handleBenchmarkApiFetch(url, init) } + // A global-mode case runs against a real workspace: send its calls there, as its user. + const real = typeof url === 'string' ? realWorkspaceRoute(url) : undefined + if (real) { + const headers = new Headers(init?.headers) + headers.set('Authorization', `Bearer ${real.token}`) + return ORIGINAL_FETCH(real.url, { ...init, headers }) + } // The docs tools read the documentation bundled into the backend: hand them to the // benchmark's real backend, logged in like the AI proxy calls. if (typeof url === 'string' && url.startsWith('/api/docs/')) { @@ -119,60 +127,21 @@ vi.mock('$lib/components/apps/components/display/dbtable/metadata', async () => vi.mock('$lib/gen', async () => { const actual = await vi.importActual('$lib/gen') const { - getBenchmarkAppByPath, - getBenchmarkJob, - getBenchmarkCompletedJobResultMaybe, - listBenchmarkWorkers, - listBenchmarkSchedules, - getBenchmarkSchedule, - listBenchmarkFolders, - getBenchmarkFolder, - getBenchmarkGroup, - listBenchmarkGroups, - getBenchmarkWorkspaceSettingsDocument, - applyBenchmarkWorkspaceSettingsDocument, - getBenchmarkDatatableSchema, - getBenchmarkDatatableMigrationsStatus, - createBenchmarkDatatableMigration, - upsertBenchmarkDatatableMigration, - runBenchmarkDatatableMigrations, - rollbackBenchmarkDatatableMigration, - getBenchmarkDraftForUser, + getBenchmarkCompletedJob, getBenchmarkFlowByPath, - getBenchmarkFlowAllResults, - getBenchmarkJobLogs, - getBenchmarkOwnDraft, getBenchmarkScriptByHash, getBenchmarkScriptByPath, - getBenchmarkAiConfig, - getBenchmarkResourceValue, - getBenchmarkVariableByPath, hasBenchmarkWorkspace, - getBenchmarkResource, - listBenchmarkAiProviderResources, - listBenchmarkPlainResources, listBenchmarkResourceTypeNames, queryBenchmarkResourceTypes, - createBenchmarkResourceType, - listBenchmarkApps, - listBenchmarkDatatables, - listBenchmarkDataMetrics, - listBenchmarkDrafts, - listBenchmarkDucklakes, listBenchmarkFlows, - listBenchmarkJobs, listBenchmarkScripts, - listBenchmarkVariables, - createBenchmarkFolder, createBenchmarkHttpTrigger, createBenchmarkSchedule, previewBenchmarkSchedule, - runBenchmarkDatatableSql, runBenchmarkFlowByPath, runBenchmarkFlowPreview, - runBenchmarkScriptByPath, - runBenchmarkScriptPreview, - updateBenchmarkDraft + runBenchmarkScriptPreview } = await import('./mockBackend') function wrapService(target: T, overrides: Record): T { @@ -188,104 +157,23 @@ vi.mock('$lib/gen', async () => { return { ...actual, - DraftService: wrapService(actual.DraftService, { - updateDraft: async (data: { - workspace: string - kind: any - path: string - requestBody?: { value?: unknown } - }) => - hasBenchmarkWorkspace(data.workspace) - ? updateBenchmarkDraft(data) - : actual.DraftService.updateDraft(data), - getDraftForUser: async (data: { workspace: string; kind: any; path: string }) => - hasBenchmarkWorkspace(data.workspace) - ? getBenchmarkDraftForUser(data) - : actual.DraftService.getDraftForUser(data), - getOwnDraft: async (data: { workspace: string; kind: any; path: string }) => - hasBenchmarkWorkspace(data.workspace) - ? getBenchmarkOwnDraft(data) - : actual.DraftService.getOwnDraft(data), - listDrafts: async (data: { workspace: string }) => - hasBenchmarkWorkspace(data.workspace) - ? listBenchmarkDrafts(data.workspace) - : actual.DraftService.listDrafts(data) - }), - FolderService: wrapService(actual.FolderService, { - createFolder: async (data: { workspace: string; requestBody: { name: string } }) => - hasBenchmarkWorkspace(data.workspace) - ? createBenchmarkFolder(data.workspace, data.requestBody.name) - : actual.FolderService.createFolder(data), - listFolders: async (data: { workspace: string }) => - hasBenchmarkWorkspace(data.workspace) - ? listBenchmarkFolders(data.workspace) - : actual.FolderService.listFolders(data), - getFolder: async (data: { workspace: string; name: string }) => { - if (!hasBenchmarkWorkspace(data.workspace)) return actual.FolderService.getFolder(data) - const folder = getBenchmarkFolder(data.workspace, data.name) - if (!folder) throw new Error(`Folder "${data.name}" not found in benchmark workspace`) - return folder - } - }), - GroupService: wrapService(actual.GroupService, { - listGroups: async (data: { workspace: string }) => - hasBenchmarkWorkspace(data.workspace) - ? listBenchmarkGroups(data.workspace) - : actual.GroupService.listGroups(data), - getGroup: async (data: { workspace: string; name: string }) => { - if (!hasBenchmarkWorkspace(data.workspace)) return actual.GroupService.getGroup(data) - const group = getBenchmarkGroup(data.workspace, data.name) - if (!group) throw new Error(`Group "${data.name}" not found in benchmark workspace`) - return group - } - }), ScriptService: wrapService(actual.ScriptService, { listScripts: async (data: { workspace: string }) => hasBenchmarkWorkspace(data.workspace) ? (listBenchmarkScripts(data.workspace) ?? []) : actual.ScriptService.listScripts(data), - existsScriptByPath: async (data: { workspace: string; path: string }) => - hasBenchmarkWorkspace(data.workspace) - ? Boolean(getBenchmarkScriptByPath(data.workspace, data.path)) - : actual.ScriptService.existsScriptByPath(data), - getScriptByPath: async (data: { workspace: string; path: string; getDraft?: boolean }) => { + getScriptByPath: async (data: { workspace: string; path: string }) => { if (hasBenchmarkWorkspace(data.workspace)) { const script = getBenchmarkScriptByPath(data.workspace, data.path) - // `getDraft` mirrors production's overlay: the row plus the caller's - // draft and a `no_deployed` marker (draft-only item). The diff tool - // reads through this shape — without it every draft looks absent. - const draft = data.getDraft - ? getBenchmarkOwnDraft({ workspace: data.workspace, kind: 'script', path: data.path }) - : null if (!script) { - if (data.getDraft && draft) { - return { - ...(draft.value as Record), - path: data.path, - draft: draft.value, - no_deployed: true - } - } throw Object.assign( new Error(`Script "${data.path}" not found in benchmark workspace`), { status: 404 } ) } - return data.getDraft - ? { ...script, draft: draft?.value ?? undefined, no_deployed: false } - : script - } - return actual.ScriptService.getScriptByPath(data) - }, - getScriptByPathWithDraft: async (data: { workspace: string; path: string }) => { - if (hasBenchmarkWorkspace(data.workspace)) { - const script = getBenchmarkScriptByPath(data.workspace, data.path) - if (!script) { - throw new Error(`Script "${data.path}" not found in benchmark workspace`) - } return script } - return actual.ScriptService.getScriptByPathWithDraft(data) + return actual.ScriptService.getScriptByPath(data) }, getScriptByHash: async (data: { workspace: string; hash: string }) => { if (hasBenchmarkWorkspace(data.workspace)) { @@ -303,56 +191,18 @@ vi.mock('$lib/gen', async () => { hasBenchmarkWorkspace(data.workspace) ? (listBenchmarkFlows(data.workspace) ?? []) : actual.FlowService.listFlows(data), - existsFlowByPath: async (data: { workspace: string; path: string }) => - hasBenchmarkWorkspace(data.workspace) - ? Boolean(getBenchmarkFlowByPath(data.workspace, data.path)) - : actual.FlowService.existsFlowByPath(data), - getFlowByPath: async (data: { workspace: string; path: string; getDraft?: boolean }) => { + getFlowByPath: async (data: { workspace: string; path: string }) => { if (hasBenchmarkWorkspace(data.workspace)) { const flow = getBenchmarkFlowByPath(data.workspace, data.path) - // Mirror production's `getDraft` overlay (see getScriptByPath above). - const draft = data.getDraft - ? getBenchmarkOwnDraft({ workspace: data.workspace, kind: 'flow', path: data.path }) - : null if (!flow) { - if (data.getDraft && draft) { - return { - ...(draft.value as Record), - path: data.path, - draft: draft.value, - no_deployed: true - } - } throw Object.assign( new Error(`Flow "${data.path}" not found in benchmark workspace`), { status: 404 } ) } - return data.getDraft - ? { ...flow, draft: draft?.value ?? undefined, no_deployed: false } - : flow - } - return actual.FlowService.getFlowByPath(data) - }, - getFlowByPathWithDraft: async (data: { workspace: string; path: string }) => { - if (hasBenchmarkWorkspace(data.workspace)) { - const flow = getBenchmarkFlowByPath(data.workspace, data.path) - if (!flow) { - throw new Error(`Flow "${data.path}" not found in benchmark workspace`) - } return flow } - return actual.FlowService.getFlowByPathWithDraft(data) - }, - getFlowLatestVersion: async (data: { workspace: string; path: string }) => { - if (hasBenchmarkWorkspace(data.workspace)) { - const flow = getBenchmarkFlowByPath(data.workspace, data.path) - if (!flow) { - throw new Error(`Flow "${data.path}" not found in benchmark workspace`) - } - return { id: 1 } - } - return actual.FlowService.getFlowLatestVersion(data) + return actual.FlowService.getFlowByPath(data) } }), JobService: wrapService(actual.JobService, { @@ -364,43 +214,16 @@ vi.mock('$lib/gen', async () => { args?: Record path?: string } - }) => { - if (!hasBenchmarkWorkspace(data.workspace)) { - return actual.JobService.runScriptPreview(data) - } - const requestBody = data.requestBody ?? {} - const database = requestBody.args?.database - // Datatable SQL runs as a `postgresql` preview against `datatable://`. - // Execute it through the canned-SQL mock instead of linting it as a script. - if ( - requestBody.language === 'postgresql' && - typeof database === 'string' && - database.startsWith('datatable://') - ) { - return runBenchmarkDatatableSql({ - workspace: data.workspace, - datatableName: database.slice('datatable://'.length), - sql: requestBody.content ?? '' - }) - } - return runBenchmarkScriptPreview({ workspace: data.workspace, requestBody }) - }, - runScriptByPath: async (data: { - workspace: string - path: string - requestBody?: Record }) => hasBenchmarkWorkspace(data.workspace) - ? runBenchmarkScriptByPath({ + ? runBenchmarkScriptPreview({ workspace: data.workspace, - path: data.path, - args: data.requestBody + requestBody: data.requestBody ?? {} }) - : actual.JobService.runScriptByPath(data), + : actual.JobService.runScriptPreview(data), runFlowPreview: async (data: { workspace: string - memoryId?: string - requestBody?: { path?: string; value?: { chat_input_enabled?: boolean }; args?: unknown } + requestBody?: { path?: string; args?: unknown } }) => hasBenchmarkWorkspace(data.workspace) ? runBenchmarkFlowPreview(data) @@ -426,24 +249,10 @@ vi.mock('$lib/gen', async () => { return job } return actual.JobService.getJob(data) - }, - getCompletedJobResultMaybe: async (data: { workspace: string; id: string }) => - hasBenchmarkWorkspace(data.workspace) - ? getBenchmarkCompletedJobResultMaybe({ workspace: data.workspace, id: data.id }) - : actual.JobService.getCompletedJobResultMaybe(data), - listJobs: async (data: { workspace: string }) => - hasBenchmarkWorkspace(data.workspace) - ? (listBenchmarkJobs(data.workspace) ?? []) - : actual.JobService.listJobs(data), - getJobLogs: async (data: { workspace: string; id: string }) => - hasBenchmarkWorkspace(data.workspace) - ? getBenchmarkJobLogs(data.workspace, data.id) - : actual.JobService.getJobLogs(data), - getFlowAllResults: async (data: { workspace: string; id: string }) => - hasBenchmarkWorkspace(data.workspace) - ? getBenchmarkFlowAllResults(data.workspace, data.id) - : actual.JobService.getFlowAllResults(data) + } }), + // A benchmark workspace holds no resources and no AI settings: answer as an empty + // workspace would, rather than send its temp-directory id to the real backend. WorkspaceService: wrapService(actual.WorkspaceService, { getWorkspaceSettingsDocument: async (data: { workspace: string }) => hasBenchmarkWorkspace(data.workspace) @@ -454,77 +263,13 @@ vi.mock('$lib/gen', async () => { ? applyBenchmarkWorkspaceSettingsDocument(data) : actual.WorkspaceService.applyWorkspaceSettingsDocument(data), getCopilotInfo: async (data: { workspace: string }) => - hasBenchmarkWorkspace(data.workspace) - ? (getBenchmarkAiConfig(data.workspace) ?? {}) - : actual.WorkspaceService.getCopilotInfo(data), - listDataTableTables: async (data: { workspace: string }) => - hasBenchmarkWorkspace(data.workspace) - ? (listBenchmarkDatatables(data.workspace) ?? []) - : actual.WorkspaceService.listDataTableTables(data), - listDucklakes: async (data: { workspace: string }) => - hasBenchmarkWorkspace(data.workspace) - ? (listBenchmarkDucklakes(data.workspace) ?? []) - : actual.WorkspaceService.listDucklakes(data), - getDataTableTableSchema: async (data: { - workspace: string - datatableName: string - schemaName: string - tableName: string - }) => - hasBenchmarkWorkspace(data.workspace) - ? getBenchmarkDatatableSchema({ - workspace: data.workspace, - datatableName: data.datatableName, - schemaName: data.schemaName, - tableName: data.tableName - }) - : actual.WorkspaceService.getDataTableTableSchema(data), - getDatatableMigrationsStatus: async (data: any) => - hasBenchmarkWorkspace(data.workspace) - ? getBenchmarkDatatableMigrationsStatus(data) - : actual.WorkspaceService.getDatatableMigrationsStatus(data), - createDatatableMigration: async (data: any) => - hasBenchmarkWorkspace(data.workspace) - ? createBenchmarkDatatableMigration(data) - : actual.WorkspaceService.createDatatableMigration(data), - upsertDatatableMigration: async (data: any) => - hasBenchmarkWorkspace(data.workspace) - ? upsertBenchmarkDatatableMigration(data) - : actual.WorkspaceService.upsertDatatableMigration(data), - runDatatableMigrations: async (data: any) => - hasBenchmarkWorkspace(data.workspace) - ? runBenchmarkDatatableMigrations(data) - : actual.WorkspaceService.runDatatableMigrations(data), - rollbackDatatableMigrations: async (data: any) => - hasBenchmarkWorkspace(data.workspace) - ? rollbackBenchmarkDatatableMigration(data) - : actual.WorkspaceService.rollbackDatatableMigrations(data) - }), - DataMetricService: wrapService(actual.DataMetricService, { - listDataMetrics: async (data: { workspace: string }) => - hasBenchmarkWorkspace(data.workspace) - ? { metrics: listBenchmarkDataMetrics(data.workspace) ?? [] } - : actual.DataMetricService.listDataMetrics(data) + hasBenchmarkWorkspace(data.workspace) ? {} : actual.WorkspaceService.getCopilotInfo(data) }), WorkerService: wrapService(actual.WorkerService, { listWorkers: async (data: Record) => listBenchmarkWorkers() ?? actual.WorkerService.listWorkers(data) }), ScheduleService: wrapService(actual.ScheduleService, { - existsSchedule: async (data: { workspace: string; path: string }) => - hasBenchmarkWorkspace(data.workspace) ? false : actual.ScheduleService.existsSchedule(data), - listSchedules: async (data: { workspace: string }) => - hasBenchmarkWorkspace(data.workspace) - ? listBenchmarkSchedules(data.workspace) - : actual.ScheduleService.listSchedules(data), - getSchedule: async (data: { workspace: string; path: string }) => { - if (hasBenchmarkWorkspace(data.workspace)) { - const schedule = getBenchmarkSchedule(data.workspace, data.path) - if (schedule) return schedule - throw new Error(`Schedule "${data.path}" not found in benchmark workspace`) - } - return actual.ScheduleService.getSchedule(data) - }, previewSchedule: async (data: { requestBody?: Record }) => previewBenchmarkSchedule(data), createSchedule: async (data: { workspace: string; requestBody: Record }) => @@ -533,254 +278,27 @@ vi.mock('$lib/gen', async () => { : actual.ScheduleService.createSchedule(data) }), ResourceService: wrapService(actual.ResourceService, { - existsResource: async (data: { workspace: string; path: string }) => - hasBenchmarkWorkspace(data.workspace) - ? Boolean(getBenchmarkResourceValue(data.workspace, data.path)) - : actual.ResourceService.existsResource(data), - listResource: async (data: { workspace: string; resourceType?: string }) => { - if (!hasBenchmarkWorkspace(data.workspace)) { - return actual.ResourceService.listResource(data) - } - const seeded = [ - ...(listBenchmarkAiProviderResources(data.workspace) ?? []), - ...(listBenchmarkPlainResources(data.workspace) ?? []) - ] - const wanted = data.resourceType?.split(',') - return wanted ? seeded.filter((r) => wanted.includes(r.resource_type)) : seeded - }, + listResource: async (data: { workspace: string }) => + hasBenchmarkWorkspace(data.workspace) ? [] : actual.ResourceService.listResource(data), getResource: async (data: { workspace: string; path: string }) => { if (hasBenchmarkWorkspace(data.workspace)) { - const resource = getBenchmarkResource(data.workspace, data.path) - if (!resource) { - throw new Error(`Resource "${data.path}" not found in benchmark workspace`) - } - return resource + throw new Error(`Resource "${data.path}" not found in benchmark workspace`) } return actual.ResourceService.getResource(data) }, - getResourceValue: async (data: { workspace: string; path: string }) => { - if (!hasBenchmarkWorkspace(data.workspace)) { - return actual.ResourceService.getResourceValue(data) - } - const value = getBenchmarkResourceValue(data.workspace, data.path) - if (!value) { - throw new Error(`Resource "${data.path}" not found in benchmark workspace`) - } - return value - }, queryResourceTypes: async (data: { workspace: string; text: string; limit?: number }) => hasBenchmarkWorkspace(data.workspace) - ? queryBenchmarkResourceTypes(data.workspace, data.text, data.limit) + ? queryBenchmarkResourceTypes(data.text, data.limit) : actual.ResourceService.queryResourceTypes(data), listResourceTypeNames: async (data: { workspace: string }) => listBenchmarkResourceTypeNames(data.workspace) ?? - actual.ResourceService.listResourceTypeNames(data), - createResourceType: async (data: { - workspace: string - requestBody: { name: string; schema?: unknown; description?: string } - }) => { - if (!hasBenchmarkWorkspace(data.workspace)) { - return actual.ResourceService.createResourceType(data) - } - createBenchmarkResourceType(data.workspace, data.requestBody) - return `resource_type ${data.requestBody.name} created` - } - }), - VariableService: wrapService(actual.VariableService, { - existsVariable: async (data: { workspace: string; path: string }) => - hasBenchmarkWorkspace(data.workspace) - ? Boolean(getBenchmarkVariableByPath(data.workspace, data.path)) - : actual.VariableService.existsVariable(data), - listVariable: async (data: { workspace: string }) => - hasBenchmarkWorkspace(data.workspace) - ? (listBenchmarkVariables(data.workspace) ?? []) - : actual.VariableService.listVariable(data), - getVariable: async (data: { - workspace: string - path: string - decryptSecret?: boolean - }) => { - if (hasBenchmarkWorkspace(data.workspace)) { - const variable = getBenchmarkVariableByPath( - data.workspace, - data.path, - data.decryptSecret ?? true - ) - if (!variable) { - throw new Error(`Variable "${data.path}" not found in benchmark workspace`) - } - return variable - } - return actual.VariableService.getVariable(data) - } - }), - AppService: wrapService(actual.AppService, { - existsApp: async (data: { workspace: string; path: string }) => - hasBenchmarkWorkspace(data.workspace) - ? Boolean(getBenchmarkAppByPath(data.workspace, data.path)) - : actual.AppService.existsApp(data), - listApps: async (data: { workspace: string }) => - hasBenchmarkWorkspace(data.workspace) - ? (listBenchmarkApps(data.workspace) ?? []) - : actual.AppService.listApps(data), - getAppByPath: async (data: { - workspace: string - path: string - getDraft?: boolean - rawApp?: boolean - }) => { - if (hasBenchmarkWorkspace(data.workspace)) { - const app = getBenchmarkAppByPath(data.workspace, data.path) - // Mirror production's `getDraft` overlay (see getScriptByPath above). - // Benchmark app drafts live under the raw_app kind. - const draft = data.getDraft - ? getBenchmarkOwnDraft({ workspace: data.workspace, kind: 'raw_app', path: data.path }) - : null - if (!app) { - if (data.getDraft && draft) { - return { - ...(draft.value as Record), - path: data.path, - raw_app: true, - draft: draft.value, - no_deployed: true - } - } - throw Object.assign( - new Error(`App "${data.path}" not found in benchmark workspace`), - { status: 404 } - ) - } - return data.getDraft - ? { ...app, draft: draft?.value ?? undefined, no_deployed: false } - : app - } - return actual.AppService.getAppByPath(data) - } + actual.ResourceService.listResourceTypeNames(data) }), HttpTriggerService: wrapService(actual.HttpTriggerService, { - existsHttpTrigger: async (data: { workspace: string; path: string }) => - hasBenchmarkWorkspace(data.workspace) ? false : actual.HttpTriggerService.existsHttpTrigger(data), - listHttpTriggers: async (data: { workspace: string }) => - hasBenchmarkWorkspace(data.workspace) ? [] : actual.HttpTriggerService.listHttpTriggers(data), - getHttpTrigger: async (data: { workspace: string; path: string }) => { - if (hasBenchmarkWorkspace(data.workspace)) { - throw new Error(`HTTP trigger "${data.path}" not found in benchmark workspace`) - } - return actual.HttpTriggerService.getHttpTrigger(data) - }, createHttpTrigger: async (data: { workspace: string; requestBody: Record }) => hasBenchmarkWorkspace(data.workspace) ? createBenchmarkHttpTrigger(data) : actual.HttpTriggerService.createHttpTrigger(data) - }), - WebsocketTriggerService: wrapService(actual.WebsocketTriggerService, { - existsWebsocketTrigger: async (data: { workspace: string; path: string }) => - hasBenchmarkWorkspace(data.workspace) - ? false - : actual.WebsocketTriggerService.existsWebsocketTrigger(data), - listWebsocketTriggers: async (data: { workspace: string }) => - hasBenchmarkWorkspace(data.workspace) - ? [] - : actual.WebsocketTriggerService.listWebsocketTriggers(data), - getWebsocketTrigger: async (data: { workspace: string; path: string }) => { - if (hasBenchmarkWorkspace(data.workspace)) { - throw new Error(`Websocket trigger "${data.path}" not found in benchmark workspace`) - } - return actual.WebsocketTriggerService.getWebsocketTrigger(data) - } - }), - KafkaTriggerService: wrapService(actual.KafkaTriggerService, { - existsKafkaTrigger: async (data: { workspace: string; path: string }) => - hasBenchmarkWorkspace(data.workspace) - ? false - : actual.KafkaTriggerService.existsKafkaTrigger(data), - listKafkaTriggers: async (data: { workspace: string }) => - hasBenchmarkWorkspace(data.workspace) ? [] : actual.KafkaTriggerService.listKafkaTriggers(data), - getKafkaTrigger: async (data: { workspace: string; path: string }) => { - if (hasBenchmarkWorkspace(data.workspace)) { - throw new Error(`Kafka trigger "${data.path}" not found in benchmark workspace`) - } - return actual.KafkaTriggerService.getKafkaTrigger(data) - } - }), - NatsTriggerService: wrapService(actual.NatsTriggerService, { - existsNatsTrigger: async (data: { workspace: string; path: string }) => - hasBenchmarkWorkspace(data.workspace) ? false : actual.NatsTriggerService.existsNatsTrigger(data), - listNatsTriggers: async (data: { workspace: string }) => - hasBenchmarkWorkspace(data.workspace) ? [] : actual.NatsTriggerService.listNatsTriggers(data), - getNatsTrigger: async (data: { workspace: string; path: string }) => { - if (hasBenchmarkWorkspace(data.workspace)) { - throw new Error(`NATS trigger "${data.path}" not found in benchmark workspace`) - } - return actual.NatsTriggerService.getNatsTrigger(data) - } - }), - PostgresTriggerService: wrapService(actual.PostgresTriggerService, { - existsPostgresTrigger: async (data: { workspace: string; path: string }) => - hasBenchmarkWorkspace(data.workspace) - ? false - : actual.PostgresTriggerService.existsPostgresTrigger(data), - listPostgresTriggers: async (data: { workspace: string }) => - hasBenchmarkWorkspace(data.workspace) - ? [] - : actual.PostgresTriggerService.listPostgresTriggers(data), - getPostgresTrigger: async (data: { workspace: string; path: string }) => { - if (hasBenchmarkWorkspace(data.workspace)) { - throw new Error(`Postgres trigger "${data.path}" not found in benchmark workspace`) - } - return actual.PostgresTriggerService.getPostgresTrigger(data) - } - }), - MqttTriggerService: wrapService(actual.MqttTriggerService, { - existsMqttTrigger: async (data: { workspace: string; path: string }) => - hasBenchmarkWorkspace(data.workspace) ? false : actual.MqttTriggerService.existsMqttTrigger(data), - listMqttTriggers: async (data: { workspace: string }) => - hasBenchmarkWorkspace(data.workspace) ? [] : actual.MqttTriggerService.listMqttTriggers(data), - getMqttTrigger: async (data: { workspace: string; path: string }) => { - if (hasBenchmarkWorkspace(data.workspace)) { - throw new Error(`MQTT trigger "${data.path}" not found in benchmark workspace`) - } - return actual.MqttTriggerService.getMqttTrigger(data) - } - }), - SqsTriggerService: wrapService(actual.SqsTriggerService, { - existsSqsTrigger: async (data: { workspace: string; path: string }) => - hasBenchmarkWorkspace(data.workspace) ? false : actual.SqsTriggerService.existsSqsTrigger(data), - listSqsTriggers: async (data: { workspace: string }) => - hasBenchmarkWorkspace(data.workspace) ? [] : actual.SqsTriggerService.listSqsTriggers(data), - getSqsTrigger: async (data: { workspace: string; path: string }) => { - if (hasBenchmarkWorkspace(data.workspace)) { - throw new Error(`SQS trigger "${data.path}" not found in benchmark workspace`) - } - return actual.SqsTriggerService.getSqsTrigger(data) - } - }), - GcpTriggerService: wrapService(actual.GcpTriggerService, { - existsGcpTrigger: async (data: { workspace: string; path: string }) => - hasBenchmarkWorkspace(data.workspace) ? false : actual.GcpTriggerService.existsGcpTrigger(data), - listGcpTriggers: async (data: { workspace: string }) => - hasBenchmarkWorkspace(data.workspace) ? [] : actual.GcpTriggerService.listGcpTriggers(data), - getGcpTrigger: async (data: { workspace: string; path: string }) => { - if (hasBenchmarkWorkspace(data.workspace)) { - throw new Error(`GCP trigger "${data.path}" not found in benchmark workspace`) - } - return actual.GcpTriggerService.getGcpTrigger(data) - } - }), - AzureTriggerService: wrapService(actual.AzureTriggerService, { - existsAzureTrigger: async (data: { workspace: string; path: string }) => - hasBenchmarkWorkspace(data.workspace) - ? false - : actual.AzureTriggerService.existsAzureTrigger(data), - listAzureTriggers: async (data: { workspace: string }) => - hasBenchmarkWorkspace(data.workspace) ? [] : actual.AzureTriggerService.listAzureTriggers(data), - getAzureTrigger: async (data: { workspace: string; path: string }) => { - if (hasBenchmarkWorkspace(data.workspace)) { - throw new Error(`Azure trigger "${data.path}" not found in benchmark workspace`) - } - return actual.AzureTriggerService.getAzureTrigger(data) - } }) } }) diff --git a/ai_evals/modes/global.ts b/ai_evals/modes/global.ts index bbff83dd8a..611989b486 100644 --- a/ai_evals/modes/global.ts +++ b/ai_evals/modes/global.ts @@ -33,7 +33,9 @@ export function createGlobalModeRunner( ): ModeRunner { return { mode: "global", - concurrency: 3, + // Each attempt holds a workspace of its own, and CE allows two besides `admins`; a third + // attempt would wait for a slot, and the wait would count toward its duration. + concurrency: 2, judgeThreshold: 80, async loadInitial(path) { return path ? await loadGlobalInitialFixture(path) : undefined; diff --git a/backend/windmill-api/openapi.yaml b/backend/windmill-api/openapi.yaml index a03385ce5f..eb9751ff06 100644 --- a/backend/windmill-api/openapi.yaml +++ b/backend/windmill-api/openapi.yaml @@ -14806,6 +14806,84 @@ paths: type: boolean description: more buckets matched than were returned, so summing them under-reports + /w/{workspace}/ai_session_tools/{name}: + post: + summary: run one of the AI session's tools as the calling user + description: The in-app chat's entry to the session toolset the MCP server also serves. Refused for job tokens and scoped tokens. + operationId: callAiSessionTool + tags: + - ai + parameters: + - $ref: "#/components/parameters/WorkspaceId" + - name: name + in: path + required: true + schema: + type: string + requestBody: + required: true + content: + application/json: + schema: + type: object + properties: + args: + description: the tool's arguments, as the model sent them + context: + type: object + description: browser-held state the tool reads + additionalProperties: true + properties: + live_editors: + type: array + items: + type: object + required: + - kind + - storage_path + properties: + kind: + type: string + storage_path: + type: string + effective_path: + type: string + cells: + type: array + items: + type: object + required: + - kind + - storage_path + - value + properties: + kind: + type: string + storage_path: + type: string + value: {} + responses: + "200": + description: what the model reads, and what the chat applies + content: + application/json: + schema: + type: object + required: + - text + - is_error + - effects + properties: + text: + type: string + is_error: + type: boolean + effects: + type: array + items: + type: object + additionalProperties: true + /w/{workspace}/ai/sessions/list: get: summary: list the calling user's AI session backups in the workspace object storage diff --git a/backend/windmill-api/src/ai_session_tools/ai_providers.rs b/backend/windmill-api/src/ai_session_tools/ai_providers.rs new file mode 100644 index 0000000000..940f511dca --- /dev/null +++ b/backend/windmill-api/src/ai_session_tools/ai_providers.rs @@ -0,0 +1,445 @@ +//! The provider config of AI agent and AI decision steps, checked against the workspace's +//! AI provider resources before a flow is written. +//! +//! The catalog of those resources and the models each serves is the chat's own +//! (`chat/flow/aiAgentProviderCatalog.ts`): its prompt needs it anyway, and it lists each +//! resource's models through the provider. The chat sends it when a call needs it. An MCP +//! client gets one built here from the resources and the AI settings alone, without model +//! listings, so it can rule out a malformed or unknown provider but never a model id. + +use serde::Deserialize; +use serde_json::{json, Value}; + +use super::chat::Effect; +use super::{array, truthy, Ctx, Surface}; + +#[derive(Deserialize, Clone, Default)] +#[serde(rename_all = "camelCase")] +pub(crate) struct Catalog { + #[serde(default)] + options: Vec, + /// `options` holds every AI provider resource of the workspace: one missing from it is + /// not a resource at all. + #[serde(default)] + resources_are_complete: bool, +} + +#[derive(Deserialize, Clone)] +#[serde(rename_all = "camelCase")] +struct ProviderOption { + kind: String, + resource_ref: String, + models: Models, + /// The models came from the endpoint's own listing, not a fallback. + models_are_live: bool, + /// The resource points at a gateway that may serve ids its listing omits. + custom_endpoint: bool, +} + +#[derive(Deserialize, Clone, Default)] +struct Models { + ids: Vec, + complete: bool, +} + +/// The kinds that serve decision models only, which an AI agent step cannot run on. +const DECISION_ONLY_KINDS: &[&str] = &["typesafe", "cloudflare"]; +/// The model of OpenAI's Decisions API, the one kind both step types run on. +const OPENAI_DECISION_MODEL: &str = "gpt-6-luna"; +const DECISION_PROVIDER_KINDS: &[&str] = &["typesafe", "cloudflare", "openai"]; +const MAX_PROMPTED_MODELS: usize = 25; +const PROVIDER_SHAPE: &str = r#"{ "type": "static", "value": { "kind": "", "resource": "$res:", "model": "" } }"#; + +/// Every AI agent and AI decision module of a module tree, agent tools included, in the +/// order the editor's `forEachAiAgentModule` visits them. +fn for_each_ai_module<'a>(modules: &'a Value, f: &mut impl FnMut(&'a Value, &'a Value)) { + for module in array(Some(modules)) { + let Some(value) = module.get("value") else { + continue; + }; + match value.get("type").and_then(Value::as_str) { + Some("aiagent") => { + f(module, value); + if let Some(tools) = value.get("tools") { + for_each_ai_module(tools, f); + } + } + Some("forloopflow" | "whileloopflow") => { + if let Some(inner) = value.get("modules") { + for_each_ai_module(inner, f); + } + } + Some("aidecision") => f(module, value), + Some(kind @ ("branchone" | "branchall")) => { + if kind == "branchone" { + if let Some(default) = value.get("default") { + for_each_ai_module(default, f); + } + } + for branch in array(value.get("branches")) { + if let Some(inner) = branch.get("modules") { + for_each_ai_module(inner, f); + } + } + } + _ => {} + } + } +} + +/// A step's own static provider: what the catalog checks. +fn static_provider(value: &Value) -> Option<&Value> { + if truthy(value.get("agent")) { + return None; + } + let transform = value.get("input_transforms")?.get("provider")?; + (transform.get("type").and_then(Value::as_str) == Some("static")).then_some(transform) +} + +/// The `$res:` references the steps' providers use, or `None` when no step states a provider +/// of its own and the catalog is not needed. +fn provider_refs(modules: &Value) -> Option> { + let mut needed = false; + let mut refs: Vec = Vec::new(); + for_each_ai_module(modules, &mut |_, value| { + let Some(transform) = static_provider(value) else { + return; + }; + needed = true; + if let Some(resource) = transform + .get("value") + .and_then(|v| v.get("resource")) + .and_then(Value::as_str) + { + if !refs.iter().any(|r| r == resource) { + refs.push(resource.to_string()); + } + } + }); + needed.then_some(refs) +} + +/// What the catalog for `modules` is: none needed, the one to check against, or none yet +/// because the chat has to send it first. +pub(super) enum CatalogFor { + NotNeeded, + Ready(Catalog), + Requested, +} + +pub(super) async fn catalog_for(ctx: &Ctx<'_>, modules: &Value) -> CatalogFor { + let Some(resource_refs) = provider_refs(modules) else { + return CatalogFor::NotNeeded; + }; + match ctx.surface { + Surface::Chat => match ctx.chat.and_then(|chat| chat.ai_provider_catalog.clone()) { + Some(catalog) => CatalogFor::Ready(catalog), + None => { + ctx.effect(Effect::NeedsAiProviderCatalog { resource_refs }); + CatalogFor::Requested + } + }, + Surface::Mcp => CatalogFor::Ready(listed_catalog(ctx).await), + } +} + +/// The resource types an AI agent or AI decision step's provider can be, as the editor lists +/// them (`AI_PROVIDERS` and `DECISION_AI_PROVIDERS` in `copilot/lib.ts`). +const AI_RESOURCE_TYPES: &[&str] = &[ + "openai", + "anthropic", + "googleai", + "azure_openai", + "azure_foundry", + "mistral", + "deepseek", + "groq", + "openrouter", + "togetherai", + "aws_bedrock", + "customai", + "typesafe", + "cloudflare", +]; +const MAX_PROVIDER_RESOURCES: usize = 8; + +/// The workspace's AI provider resources, without model listings: each one's models are the +/// ones its AI settings configure, which can never rule an id out. +async fn listed_catalog(ctx: &Ctx<'_>) -> Catalog { + let route = ctx.ws(&format!( + "resources/list?resource_type={}", + AI_RESOURCE_TYPES.join(",") + )); + let (resources, listed_all) = match ctx.get(&route).await { + Ok(resources) => (resources, true), + Err(_) => (json!([]), false), + }; + let config = ctx + .get(&ctx.ws("workspaces/get_copilot_info")) + .await + .unwrap_or(Value::Null); + let providers = config.get("providers").and_then(Value::as_object); + let configured = |path: &str| { + providers + .into_iter() + .flatten() + .find(|(_, p)| p.get("resource_path").and_then(Value::as_str) == Some(path)) + .map(|(kind, p)| (kind.clone(), p)) + }; + let default_kind = config + .get("default_model") + .and_then(|m| m.get("provider")) + .and_then(Value::as_str); + let mut candidates: Vec<(String, String)> = array(Some(&resources)) + .iter() + .filter_map(|r| { + let kind = r.get("resource_type")?.as_str()?; + let path = r.get("path")?.as_str()?; + AI_RESOURCE_TYPES + .contains(&kind) + .then(|| (kind.to_string(), path.to_string())) + }) + .collect(); + let rank = |(kind, path): &(String, String)| match configured(path) { + None => 2, + Some(_) if Some(kind.as_str()) == default_kind => 0, + Some(_) => 1, + }; + candidates.sort_by(|a, b| rank(a).cmp(&rank(b)).then_with(|| a.1.cmp(&b.1))); + let complete = listed_all && candidates.len() <= MAX_PROVIDER_RESOURCES; + let options = candidates + .into_iter() + .take(MAX_PROVIDER_RESOURCES) + .map(|(kind, path)| { + let ids = configured(&path) + .map(|(_, p)| { + array(p.get("models")) + .iter() + .filter_map(|m| m.as_str().map(String::from)) + .collect() + }) + .unwrap_or_default(); + ProviderOption { + kind, + resource_ref: format!("$res:{path}"), + models: Models { ids, complete: false }, + models_are_live: false, + custom_endpoint: true, + } + }) + .collect(); + Catalog { options, resources_are_complete: complete } +} + +struct Issue { + message: String, + blocking: bool, + /// The catalog established it rather than failing to rule it out. + settled: bool, +} + +fn blocking(message: String) -> Option { + Some(Issue { message, blocking: true, settled: false }) +} + +fn rules_out_other_models(option: &ProviderOption) -> bool { + option.models_are_live && option.models.complete && !option.custom_endpoint +} + +fn check_provider(value: Option<&Value>, catalog: &Catalog, step_type: &str) -> Option { + let provider = match value { + Some(Value::String(reference)) => { + return blocking(format!( + "provider must be an object {PROVIDER_SHAPE}, not the resource reference string {}", + json!(reference) + )) + } + Some(Value::Object(provider)) => provider, + _ => return blocking(format!("provider must be an object {PROVIDER_SHAPE}")), + }; + let text = |key: &str| { + provider + .get(key) + .and_then(Value::as_str) + .unwrap_or_default() + }; + let (kind, resource, model) = (text("kind"), text("resource"), text("model")); + if kind.is_empty() { + return blocking(format!( + "provider.kind is missing. Expected {PROVIDER_SHAPE}" + )); + } + if step_type == "aidecision" && !DECISION_PROVIDER_KINDS.contains(&kind) { + return blocking(format!("an AI decision step runs on a decision model: provider.kind must be \"typesafe\", \"cloudflare\" or \"openai\", not \"{kind}\"")); + } + if step_type != "aidecision" && DECISION_ONLY_KINDS.contains(&kind) { + return blocking(format!("\"{kind}\" serves decision models, which answer typed questions rather than messages: use an \"aidecision\" step rather than an AI agent")); + } + if !resource.starts_with("$res:") { + return blocking(format!( + "provider.resource must be a \"$res:\" reference. Expected {PROVIDER_SHAPE}" + )); + } + if model.is_empty() { + return blocking(format!( + "provider.model is missing. Expected {PROVIDER_SHAPE}" + )); + } + if catalog.options.is_empty() { + return catalog.resources_are_complete.then(|| Issue { + message: format!("this workspace has no AI provider resources, so \"{resource}\" cannot resolve and the step cannot run. Tell the user to create an AI provider resource (Anthropic, OpenAI, ...) and rewrite the step against it."), + blocking: false, + settled: true, + }); + } + let Some(found) = catalog.options.iter().find(|o| o.resource_ref == resource) else { + return Some(Issue { + message: format!( + "provider.resource \"{resource}\" is not one of this workspace's AI provider resources" + ), + blocking: catalog.resources_are_complete, + settled: false, + }); + }; + if found.kind != kind { + return blocking(format!( + "provider.kind \"{kind}\" does not match \"{resource}\", which is a `{}` resource", + found.kind + )); + } + // An OpenAI resource lists its chat models, which says nothing about its decision model. + if step_type == "aidecision" && kind == "openai" { + return (model != OPENAI_DECISION_MODEL).then(|| Issue { + message: format!("model \"{model}\" is not the model OpenAI's Decisions API is known to serve, \"{OPENAI_DECISION_MODEL}\""), + blocking: false, + settled: false, + }); + } + if found.models_are_live && !found.models.ids.iter().any(|id| id == model) { + return Some(Issue { + message: format!("model \"{model}\" is not in the model listing of \"{resource}\""), + blocking: rules_out_other_models(found), + settled: false, + }); + } + None +} + +fn describe_option(option: &ProviderOption) -> String { + let shown = &option.models.ids[..option.models.ids.len().min(MAX_PROMPTED_MODELS)]; + let more = if option.models.ids.len() > shown.len() { + format!(", ... ({} more)", option.models.ids.len() - shown.len()) + } else { + String::new() + }; + let caveat = if !option.models_are_live { + " (the endpoint would not list its models, so these are a guess — confirm the id with the user)" + } else if !rules_out_other_models(option) { + " (the endpoint may also serve model names this list does not show)" + } else { + "" + }; + let models = if shown.is_empty() { + "none listed".to_string() + } else { + format!("{}{more}{caveat}", shown.join(", ")) + }; + format!( + "- `{}` (kind `{}`) — models: {models}", + option.resource_ref, option.kind + ) +} + +/// Refuse a provider config that would fail at run time, and report in `warnings` what the +/// catalog could not establish, so an incomplete catalog never rejects a provider that works. +pub(super) fn validate( + modules: &Value, + catalog: Option<&Catalog>, + warnings: &mut Vec, +) -> Result<(), String> { + let empty = Catalog::default(); + let catalog = catalog.unwrap_or(&empty); + let mut errors = Vec::new(); + for_each_ai_module(modules, &mut |module, value| { + let Some(transform) = static_provider(value) else { + return; + }; + let step_type = value + .get("type") + .and_then(Value::as_str) + .unwrap_or_default(); + let Some(issue) = check_provider(transform.get("value"), catalog, step_type) else { + return; + }; + let id = module.get("id").and_then(Value::as_str).unwrap_or_default(); + if issue.blocking { + errors.push(format!("Step \"{id}\": {}", issue.message)); + } else if issue.settled { + warnings.push(format!("Step \"{id}\": {}", issue.message)); + } else { + warnings.push(format!("Step \"{id}\": {}, which could not be ruled out. Confirm it with the user if the step fails to run.", issue.message)); + } + }); + if errors.is_empty() { + return Ok(()); + } + let hint = if catalog.options.is_empty() { + String::new() + } else { + format!( + "\nAI provider resources in this workspace:\n{}", + catalog + .options + .iter() + .map(describe_option) + .collect::>() + .join("\n") + ) + }; + Err(format!( + "Invalid AI agent provider configuration:\n{}{hint}", + errors.join("\n") + )) +} + +/// The tool-result suffix for provider findings that did not block the write. +pub(super) fn format_warnings(warnings: &[String]) -> String { + if warnings.is_empty() { + return String::new(); + } + format!("\n\nAI agent provider warnings:\n{}", warnings.join("\n")) +} + +#[cfg(test)] +mod tests { + use super::*; + + fn agent(model: &str) -> Value { + json!([{ "id": "a", "value": { "type": "aiagent", "input_transforms": { "provider": { + "type": "static", + "value": { "kind": "openai", "resource": "$res:u/x/oai", "model": model } + } } } }]) + } + + fn catalog(models_are_live: bool) -> Catalog { + serde_json::from_value(json!({ + "resourcesAreComplete": true, + "options": [{ + "kind": "openai", "resourceRef": "$res:u/x/oai", "modelsAreLive": models_are_live, + "customEndpoint": false, "models": { "ids": ["gpt-a"], "complete": true } + }] + })) + .unwrap() + } + + #[test] + fn only_a_live_complete_listing_rules_out_a_model() { + let mut warnings = vec![]; + assert!(validate(&agent("gpt-z"), Some(&catalog(true)), &mut warnings).is_err()); + + assert!(validate(&agent("gpt-z"), Some(&catalog(false)), &mut warnings).is_ok()); + assert!(validate(&agent("gpt-z"), None, &mut warnings).is_ok()); + assert!(validate(&agent("gpt-a"), Some(&catalog(true)), &mut warnings).is_ok()); + assert!(warnings.is_empty()); + } +} diff --git a/backend/windmill-api/src/mcp/session/apps.rs b/backend/windmill-api/src/ai_session_tools/apps.rs similarity index 84% rename from backend/windmill-api/src/mcp/session/apps.rs rename to backend/windmill-api/src/ai_session_tools/apps.rs index 5e69fa348c..ff1af309c3 100644 --- a/backend/windmill-api/src/mcp/session/apps.rs +++ b/backend/windmill-api/src/ai_session_tools/apps.rs @@ -7,17 +7,16 @@ use std::sync::LazyLock; use serde_json::{json, Map, Value}; use sha2::{Digest, Sha256}; -use super::runs::finish_run; +use super::runs::{finish_run, run_tool, start_run, Resolved, RunTarget}; use super::workspace::{ api_message, app_source_to_draft_value, deployed_item_exists, find_and_replace, get_deployed, - get_draft, item_exists, normalize_app_draft, path_enc, persist_draft, Persist, + get_draft, item_exists, normalize_app_draft, persist_draft, Persist, }; use super::{ - bool_arg, js_head, js_len, matches_glob, opt_str_arg, pretty, str_arg, truthy, Ctx, ToolResult, + bool_arg, js_head, js_len, matches_glob, opt_str_arg, pretty, put, str_arg, truthy, Ctx, + ToolResult, }; -// `init_app` is dispatched below but stays out of this list until `gen/app_templates.json` -// is emitted by the generator. pub(super) const TOOLS: &[&str] = &[ "init_app", "read_app_file", @@ -214,23 +213,28 @@ async fn load_app(ctx: &Ctx<'_>, path: &str) -> Result { Ok(Loaded { value: app_source_to_draft_value(&app, Some(&app)), last_sync }) } -/// What the backend requires of a draft from an operator with builder rights: sandboxed, -/// never in viewer mode, and running as nobody but its builder. -async fn conform_builder_policy(ctx: &Ctx<'_>, value: &mut Value) { +/// Whether the caller's role composes apps out of deployed runnables. They can save no inline +/// runnable. +async fn builds_apps(ctx: &Ctx<'_>) -> bool { if !ctx.auth.is_operator { - return; + return false; } let Ok(memberships) = ctx.get("/workspaces/users").await else { - return; + return false; }; - let builder = memberships + memberships .get("workspaces") .and_then(Value::as_array) .and_then(|list| list.iter().find(|w| text_of(w, "id") == Some(ctx.w_id))) - .and_then(|w| w.get("operator_settings")) - .and_then(|s| s.get("builder_apps")) - == Some(&Value::Bool(true)); - if !builder { + .and_then(|w| w.get("capabilities")) + .and_then(|c| text_of(c, "apps")) + == Some("compose") +} + +/// What the backend requires of a draft from a role that composes apps: sandboxed, +/// never in viewer mode, and running as nobody but its builder. +async fn conform_builder_policy(ctx: &Ctx<'_>, value: &mut Value) { + if !builds_apps(ctx).await { return; } let policy = object_mut(value, "policy"); @@ -272,11 +276,13 @@ async fn save_app( let mut out = Map::new(); out.insert("success".into(), json!(true)); out.insert("message".into(), json!(message)); - out.insert( - "preview_url".into(), - json!(ctx.page_url(&format!( + put( + &mut out, + "preview_url", + ctx.link(&format!( "/apps_raw/edit/{path}?view=preview&nomenubar=true" - ))), + )) + .map(Value::from), ); if let Some(warning) = warning { out.insert("warning".into(), json!(warning)); @@ -390,9 +396,6 @@ fn app_table_ref(datatable: &str, schema: Option<&str>, reference: &str) -> Stri async fn init_app(ctx: &Ctx<'_>, args: &Value) -> ToolResult { let path = str_arg(args, "path")?; let framework = str_arg(args, "framework")?; - let template = APP_TEMPLATES["templates"] - .get(framework) - .ok_or_else(|| format!("Invalid framework \"{framework}\""))?; if get_draft(ctx, "raw_app", path).await?.is_some() { return Err(format!( @@ -405,13 +408,28 @@ async fn init_app(ctx: &Ctx<'_>, args: &Value) -> ToolResult { )); } + ctx.status(format!( + "Saving app \"{path}\" draft ({framework} template)…" + )); + // A builder can save no inline runnable, so they start from a static page instead of the tour. + let builder = builds_apps(ctx).await; + let template = APP_TEMPLATES[if builder { + "builder_templates" + } else { + "templates" + }] + .get(framework) + .ok_or_else(|| format!("Invalid framework \"{framework}\""))?; let starters = &APP_TEMPLATES["starter_runnables"]; let mut value = Map::new(); if let Some(summary) = opt_str_arg(args, "summary") { value.insert("summary".into(), json!(summary)); } value.insert("files".into(), template.clone()); - value.insert("runnables".into(), starters.clone()); + value.insert( + "runnables".into(), + if builder { json!({}) } else { starters.clone() }, + ); value.insert("policy".into(), json!({ "sandbox": true })); let data = args.get("data").filter(|d| truthy(Some(d))); let schema = data.and_then(|d| text_of(d, "schema")); @@ -449,9 +467,23 @@ async fn init_app(ctx: &Ctx<'_>, args: &Value) -> ToolResult { Some(schema) => format!(" Its data config names schema \"{schema}\" of data table \"{datatable}\" without creating it: if it does not exist yet, create it before the tables that go in it."), None => String::new(), }; + let start = if builder { + "The template is a static page with no runnables: replace its UI, and add only deployed scripts and flows, by path.".to_string() + } else { + format!("The template is a demo tour with starter runnables {}: replace its UI and delete the runnables the app does not need (delete_app_runnable).", starter_keys.join(", ")) + }; + // The chat opens the app in its own panel; an MCP client only has the link. + let open_preview = if ctx.link("/").is_some() { + " Open preview_url in a browser the user can see from this conversation if you have one, and otherwise give them the link: its live preview follows every later edit." + } else { + "" + }; let message = format!( - "Initialized a per-user draft app \"{path}\" from the {framework} template (saved server-side, not a deployed workspace item). The template is a demo tour with starter runnables {}: replace its UI and delete the runnables the app does not need (delete_app_runnable). Use write_app_file / write_app_runnable to evolve it.{data_note} Open preview_url in a browser the user can see from this conversation if you have one, and otherwise give them the link: its live preview follows every later edit.", - starter_keys.join(", ") + "Initialized a per-user draft app \"{path}\" from the {framework} template (saved server-side, not a deployed workspace item). {start} Use write_app_file / write_app_runnable to evolve it.{data_note}{open_preview}" + ); + ctx.status_after_save( + format!("Saved app \"{path}\" draft ({framework})"), + Some("Saved as draft"), ); save_app(ctx, path, value, None, message, None).await } @@ -507,6 +539,7 @@ async fn read_app_file(ctx: &Ctx<'_>, args: &Value) -> ToolResult { inline_runnable_content(&value, key, extension, path)?.0 } }; + ctx.status(format!("Read {}", target.file_path())); let int = |name: &str| args.get(name).and_then(Value::as_i64); Ok(format_file_read(content, int("offset"), int("limit"))) } @@ -565,7 +598,7 @@ fn search_files( query: &str, file_glob: Option<&str>, max_matches: i64, -) -> String { +) -> (String, usize, usize) { let needle = query.to_lowercase(); files.sort_by(|a, b| locale_compare(&a.0, &b.0)); if let Some(glob) = file_glob { @@ -601,7 +634,7 @@ fn search_files( } if total == 0 { - return format!( + let text = format!( "No matches. Try a broader or differently-spelled term{}.", if file_glob.is_some() { ", or drop the file_glob" @@ -609,6 +642,7 @@ fn search_files( "" } ); + return (text, 0, 0); } let plural = |n: usize, one: &str, many: &str| { @@ -665,7 +699,7 @@ fn search_files( .to_string(), ); } - out.join("\n") + (out.join("\n"), total, file_count) } async fn search_app(ctx: &Ctx<'_>, args: &Value) -> ToolResult { @@ -681,12 +715,18 @@ async fn search_app(ctx: &Ctx<'_>, args: &Value) -> ToolResult { .unwrap_or(SEARCH_DEFAULT_MAX_MATCHES) .min(SEARCH_MAX_MATCHES_CEILING); let file_glob = opt_str_arg(args, "file_glob").filter(|glob| !glob.is_empty()); - Ok(search_files( - searchable_files(&value), - query, - file_glob, - max_matches, - )) + let (text, total, files) = + search_files(searchable_files(&value), query, file_glob, max_matches); + ctx.status(if total == 0 { + format!("No matches for \"{query}\"") + } else { + format!( + "Found {total} match{} in {files} file{}", + if total == 1 { "" } else { "es" }, + if files == 1 { "" } else { "s" } + ) + }); + Ok(text) } // --------------------------------------------------------------------------- @@ -702,6 +742,16 @@ async fn write_app_file(ctx: &Ctx<'_>, args: &Value) -> ToolResult { assert_not_generated(&file_path)?; let Loaded { mut value, last_sync } = load_app(ctx, path).await?; + let before = value + .get("files") + .and_then(|files| text_of(files, &file_path)) + .unwrap_or_default() + .to_string(); + ctx.file_diff(&before, content, &file_path, &file_path); + ctx.status_after_save( + format!("Updated {file_path} in app \"{path}\""), + Some("Saved as draft"), + ); object_mut(&mut value, "files").insert(file_path.clone(), json!(content)); let message = format!("Updated draft app \"{path}\" with frontend file \"{file_path}\"."); save_app(ctx, path, value, last_sync.as_deref(), message, None).await @@ -715,14 +765,21 @@ async fn delete_app_file(ctx: &Ctx<'_>, args: &Value) -> ToolResult { assert_not_generated(&file_path)?; let Loaded { mut value, last_sync } = load_app(ctx, path).await?; - if object_mut(&mut value, "files") - .shift_remove(&file_path) - .is_none() - { + let Some(removed) = object_mut(&mut value, "files").shift_remove(&file_path) else { return Err(format!( "Frontend file \"{file_path}\" not found in app \"{path}\"." )); - } + }; + ctx.file_diff( + removed.as_str().unwrap_or_default(), + "", + &file_path, + &file_path, + ); + ctx.status_after_save( + format!("Removed {file_path} from app \"{path}\""), + Some("Saved as draft"), + ); let message = format!("Removed \"{file_path}\" from draft app \"{path}\"."); save_app(ctx, path, value, last_sync.as_deref(), message, None).await } @@ -748,11 +805,14 @@ async fn patch_app_file(ctx: &Ctx<'_>, args: &Value) -> ToolResult { format!("Frontend file \"{file_path}\" not found in app \"{path}\".") })?; let updated = find_and_replace(current, old, new, replace_all, &label)?; + ctx.file_diff(current, &updated, file_path, file_path); object_mut(&mut value, "files").insert(file_path.clone(), json!(updated)); } Target::Backend { key, extension, .. } => { let (current, runnable) = inline_runnable_content(&value, key, extension, path)?; let updated = find_and_replace(current, old, new, replace_all, &label)?; + let file_path = target.file_path(); + ctx.file_diff(current, &updated, &file_path, &file_path); let language = runnable .get("inlineScript") .and_then(|s| s.get("language")) @@ -764,6 +824,10 @@ async fn patch_app_file(ctx: &Ctx<'_>, args: &Value) -> ToolResult { object_mut(&mut value, "runnables").insert(key.clone(), runnable); } } + ctx.status_after_save( + format!("Patched {} in app \"{path}\"", target.file_path()), + Some("Saved as draft"), + ); let message = format!( "Patched \"{}\" in draft app \"{path}\".", target.file_path() @@ -782,7 +846,11 @@ fn build_persisted_runnable(input: &Value, existing: Option<&Value>) -> Result json!({}), Some(static_inputs) => Value::Object( static_inputs .iter() @@ -828,9 +896,16 @@ fn build_persisted_runnable(input: &Value, existing: Option<&Value>) -> Result) -> Result) -> Option<&str> { + let runnable = runnable?; + matches!(text_of(runnable, "type"), Some("inline" | "runnableByName")).then(|| { + runnable + .get("inlineScript") + .and_then(|s| text_of(s, "content")) + .unwrap_or_default() + }) +} + +/// The card's diff of a runnable's code. A runnable that references an item has no code +/// of its own to diff. +fn runnable_diff(ctx: &Ctx<'_>, key: &str, before: Option<&Value>, after: Option<&Value>) { + let (before_code, after_code) = (inline_code(before), inline_code(after)); + if (before_code.is_none() && after_code.is_none()) || (after.is_some() && after_code.is_none()) + { + return; + } + let file = |runnable: Option<&Value>| { + format!( + "backend/{key}/main.{}", + runnable.map_or("ts", inline_extension) + ) + }; + let after_file = file(after.or(before)); + let before_file = if before_code.is_none() { + after_file.clone() + } else { + file(before) + }; + ctx.file_diff( + before_code.unwrap_or_default(), + after_code.unwrap_or_default(), + &after_file, + &before_file, + ); +} + /// Whether the script or flow a path runnable names is deployed. None when it names /// neither, or the probe failed: the answer only annotates a write. async fn path_target_deployed(ctx: &Ctx<'_>, runnable: &Value) -> Option { @@ -871,6 +985,11 @@ async fn write_app_runnable(ctx: &Ctx<'_>, args: &Value) -> ToolResult { let Loaded { mut value, last_sync } = load_app(ctx, path).await?; let existing = value.get("runnables").and_then(|r| r.get(key)); let persisted = build_persisted_runnable(input, existing)?; + runnable_diff(ctx, key, existing, Some(&persisted)); + ctx.status_after_save( + format!("Updated runnable \"{key}\" in app \"{path}\""), + Some("Saved as draft"), + ); let warning = match path_target_deployed(ctx, &persisted).await { Some(false) => Some(format!( "This runnable points at an item that is NOT deployed (\"{key}\" → {} \"{}\"), so it fails at runtime — a path runnable runs the deployed item, never a draft. Offer to deploy just that item with deploy_workspace_item; the app itself does not need deploying, since the preview runs its draft.", @@ -889,14 +1008,16 @@ async fn delete_app_runnable(ctx: &Ctx<'_>, args: &Value) -> ToolResult { let path = str_arg(args, "path")?; let key = str_arg(args, "key")?; let Loaded { mut value, last_sync } = load_app(ctx, path).await?; - if object_mut(&mut value, "runnables") - .shift_remove(key) - .is_none() - { + let Some(removed) = object_mut(&mut value, "runnables").shift_remove(key) else { return Err(format!( "Backend runnable \"{key}\" not found in app \"{path}\"." )); - } + }; + runnable_diff(ctx, key, Some(&removed), None); + ctx.status_after_save( + format!("Removed runnable \"{key}\" from app \"{path}\""), + Some("Saved as draft"), + ); recompute_policy(&mut value); let message = format!("Removed runnable \"{key}\" from draft app \"{path}\"."); save_app(ctx, path, value, last_sync.as_deref(), message, None).await @@ -907,7 +1028,12 @@ async fn delete_app_runnable(ctx: &Ctx<'_>, args: &Value) -> ToolResult { // --------------------------------------------------------------------------- /// The `execute_component` body that runs a runnable the way the editor preview does. -fn execute_component_body(app_path: &str, key: &str, runnable: &Value, args: &Value) -> Value { +pub(super) fn execute_component_body( + app_path: &str, + key: &str, + runnable: &Value, + args: &Value, +) -> Value { let fields = entries(runnable, "fields"); let mut test_args = args.as_object().cloned().unwrap_or_default(); let mut static_fields = Map::new(); @@ -998,9 +1124,7 @@ fn execute_component_body(app_path: &str, key: &str, runnable: &Value, args: &Va Value::Object(body) } -async fn test_run_app_runnable(ctx: &Ctx<'_>, args: &Value) -> ToolResult { - let path = str_arg(args, "path")?; - let key = str_arg(args, "key")?; +async fn resolve_app_runnable(ctx: &Ctx<'_>, path: &str, key: &str) -> Result { let value = load_app(ctx, path).await?.value; let runnables = entries(&value, "runnables"); let Some(runnable) = runnables.get(key).filter(|r| truthy(Some(r))) else { @@ -1014,35 +1138,64 @@ async fn test_run_app_runnable(ctx: &Ctx<'_>, args: &Value) -> ToolResult { } )); }; - - let body = execute_component_body( - path, - key, - runnable, - args.get("args").unwrap_or(&Value::Null), - ); - let id = ctx - .request( - "POST", - &ctx.ws(&format!("apps_u/execute_component/{}", path_enc(path))), - Some(body), - ) - .await - .map_err(|e| { - let message = api_message(&e); - let message = if js_len(message) > MAX_TOOL_ERROR_LENGTH { - format!("{}... (truncated)", js_head(message, MAX_TOOL_ERROR_LENGTH)) - } else { - message.to_string() - }; - format!("Failed to execute test run: {message}") - })?; - let id = id.trim().trim_matches('"').to_string(); - // A path runnable aimed at a flow queues a flow job, whose steps are worth a look. let is_flow = text_of(runnable, "runType") == Some("flow"); - let label = format!("{path} / {key}"); - finish_run(ctx, Ok(id), &label, is_flow, "test", args).await + Ok(Resolved { + target: RunTarget::AppRunnable { + app_path: path.to_string(), + key: key.to_string(), + runnable: runnable.clone(), + }, + schema: None, + code: None, + lang: None, + summary: None, + runnable_kind: if is_flow { "flow" } else { "script" }, + start_message: format!("Running backend runnable \"{key}\" of app \"{path}\"..."), + conversational: false, + module: None, + }) +} + +async fn test_run_app_runnable(ctx: &Ctx<'_>, args: &Value) -> ToolResult { + let path = str_arg(args, "path")?; + let key = str_arg(args, "key")?; + run_tool( + ctx, + args, + resolve_app_runnable(ctx, path, key), + async |resolved: Resolved| { + let run_args = args + .get("args") + .and_then(Value::as_object) + .cloned() + .unwrap_or_default(); + let id = start_run(ctx, &resolved.target, run_args, &[]) + .await + .map_err(|message| { + let message = if js_len(&message) > MAX_TOOL_ERROR_LENGTH { + format!( + "{}... (truncated)", + js_head(&message, MAX_TOOL_ERROR_LENGTH) + ) + } else { + message + }; + format!("Failed to execute test run: {message}") + })?; + let label = format!("{path} / {key}"); + finish_run( + ctx, + Ok(id), + &label, + resolved.runnable_kind == "flow", + "test", + args, + ) + .await + }, + ) + .await } #[cfg(test)] @@ -1094,15 +1247,15 @@ mod tests { ("/App.tsx".to_string(), "foo".to_string()), ]; assert_eq!( - search_files(files.clone(), "foo", None, 100), + search_files(files.clone(), "foo", None, 100).0, "3 matches in 2 files\n/App.tsx\n 1: foo\n/b.ts\n 1: one\n 2: Foo()\n 3: foo()\n 4: four\n 5: five" ); assert_eq!( - search_files(files.clone(), "foo", Some("*.ts"), 1), + search_files(files.clone(), "foo", Some("*.ts"), 1).0, "2 matches in 1 file (showing the first 1; narrow with file_glob or a more specific query)\n/b.ts\n 1: one\n 2: Foo()\n 3: foo()\n 4: four" ); assert_eq!( - search_files(files, "nope", Some("*.ts"), 100), + search_files(files, "nope", Some("*.ts"), 100).0, "No matches. Try a broader or differently-spelled term, or drop the file_glob." ); } @@ -1137,6 +1290,19 @@ mod tests { .unwrap(); assert_eq!(retargeted["schema"], json!({})); + let agent = build_persisted_runnable( + &json!({ "name": "n", "type": "agent", "path": "u/a/agent" }), + Some(&existing), + ) + .unwrap(); + assert_eq!(agent["runType"], "agent"); + assert_eq!(agent["fields"], json!({})); + assert!(build_persisted_runnable( + &json!({ "name": "n", "type": "agent", "path": "u/a/agent", "staticInputs": { "m": 1 } }), + None, + ) + .is_err()); + let inline = build_persisted_runnable( &json!({ "name": "n", "type": "inline", "staticInputs": { "y": 2 }, diff --git a/backend/windmill-api/src/ai_session_tools/chat.rs b/backend/windmill-api/src/ai_session_tools/chat.rs new file mode 100644 index 0000000000..0aace14bdd --- /dev/null +++ b/backend/windmill-api/src/ai_session_tools/chat.rs @@ -0,0 +1,308 @@ +//! What the in-app chat sends beside a tool call, and what it gets back to apply in the +//! browser. MCP has neither: its tools read and write the server alone. + +use axum::{extract::Path, routing::post, Extension, Json, Router}; +use serde::{Deserialize, Serialize}; +use serde_json::{Map, Value}; +use windmill_api_auth::is_effectively_unscoped; +use windmill_common::error::{Error, JsonResult}; + +use super::{call_session_tool, CallRefused, Ctx, SessionAccess, Surface, ToolOutcome}; +use crate::db::ApiAuthed; + +#[derive(Clone)] +struct BaseInternalUrl(String); + +pub(crate) fn workspaced_service(base_internal_url: String) -> Router { + Router::new() + .route("/{name}", post(call_tool)) + .layer(Extension(BaseInternalUrl(base_internal_url))) +} + +#[derive(Deserialize)] +struct ToolCall { + #[serde(default)] + args: Value, + #[serde(default)] + context: ChatContext, +} + +#[derive(Serialize)] +struct ToolResponse { + #[serde(flatten)] + outcome: ToolOutcome, + effects: Vec, +} + +async fn call_tool( + authed: ApiAuthed, + Extension(BaseInternalUrl(base_internal_url)): Extension, + Path((w_id, name)): Path<(String, String)>, + Json(call): Json, +) -> JsonResult { + // A tool calls the API with a JWT carrying its caller's full rights, so a token + // minted for something narrower (a job, a scope) must not reach it. + if authed.job_id.is_some() || !is_effectively_unscoped(authed.scopes.as_deref()) { + return Err(Error::PermissionDenied( + "The AI session tools are only reachable with an unscoped user token".to_string(), + )); + } + let mut ctx = Ctx::new(&authed, &w_id, &base_internal_url, SessionAccess::Full); + ctx.surface = Surface::Chat; + ctx.chat = Some(&call.context); + let outcome = + call_session_tool(&ctx, &name, call.args) + .await + .map_err(|refused| match refused { + CallRefused::UnknownTool(message) + | CallRefused::NotAllowed(message) + | CallRefused::NotAvailable(message) => Error::BadRequest(message), + })?; + let effects = ctx.effects.into_inner().unwrap(); + Ok(Json(ToolResponse { outcome, effects })) +} + +/// Browser-held state a tool needs and the server cannot see. +#[derive(Deserialize, Default)] +pub(crate) struct ChatContext { + /// The editor open on each draft kind, at most one per kind. + #[serde(default)] + pub live_editors: Vec, + /// The draft cells the browser holds: an open editor's, and any whose latest edit is + /// not on the server yet. They are newer than the server's row. + #[serde(default)] + pub cells: Vec, + /// Run only the check that comes before the chat's confirmation card: what a + /// confirmed call would refuse, and what it would change. + #[serde(default)] + pub check_only: bool, + /// Whether the chat's user left the skill a call reads switched on. + #[serde(default)] + pub skill_enabled: Option, + /// The folders the session works in, which a listing defaults to. + #[serde(default)] + pub scope_roots: Option>, + /// The call comes from an AI session, which has a preview panel and builds pipeline + /// nodes in the folder's pipeline draft. + #[serde(default)] + pub session: bool, + /// The workspace's AI provider resources and the models each serves, sent once a call + /// asked for it with `needs_ai_provider_catalog`. + #[serde(default)] + pub ai_provider_catalog: Option, + /// A run tool called in two phases around the chat's own argument form: `prepare` + /// resolves what will run, `start` queues it with the arguments the form settled on. + #[serde(default)] + pub run: Option, +} + +#[derive(Deserialize)] +#[serde(tag = "phase", rename_all = "snake_case")] +pub(crate) enum RunPhase { + Prepare, + Start { + /// What `prepare` resolved, so the run is the version the form was shown for; + /// absent for a tool with no form, which resolves and starts in one call. + #[serde(default)] + target: Option, + #[serde(default)] + args: Map, + /// An agent step's form rows, each fed from the run's arguments. + #[serde(default)] + form_keys: Vec, + }, +} + +#[derive(Deserialize)] +pub(crate) struct LiveEditor { + /// The draft kind (`script`, `flow`, `raw_app`, `trigger_schedule`, …). + pub kind: String, + /// The key the draft is stored under: `u/{user}/draft_{uuid}` for a new item. + pub storage_path: String, + /// The path the editor shows, which a rename leaves the storage key behind. + pub effective_path: Option, +} + +#[derive(Deserialize)] +pub(crate) struct Cell { + pub kind: String, + pub storage_path: String, + pub value: Value, +} + +/// Something a call changed that the chat applies in the browser. +#[derive(Serialize)] +#[serde(tag = "type", rename_all = "snake_case")] +pub(crate) enum Effect { + /// Save `value` as the draft at `storage_path`, through the editor's own draft store. + /// `last_sync` is the server row the value was built on, when it was read from the + /// server rather than from a cell: the save detects a conflict against it. + DraftWrite { + kind: String, + storage_path: String, + value: Value, + last_sync: Option, + force: bool, + }, + /// Delete the draft at `storage_path`. `keep_live_editor` leaves an open editor's + /// copy in place, detached from the server row. + DraftDelete { + kind: String, + storage_path: String, + last_sync: Option, + keep_live_editor: bool, + }, + /// `get_run` read one job, which the card renders the run from. + InspectedRun { job_id: String }, + /// What the call found that only its card shows, not the model. + Facts(serde_json::Map), + /// A patch to the tool's card, in the order the call reached it. One carrying + /// `after_save` applies only once the chat saved the call's draft writes. + Status(serde_json::Map), + /// The call wrote nothing: it needs the AI provider catalog, which the chat builds, and + /// is to be made again with it. `resource_refs` are the ones the flow references. + NeedsAiProviderCatalog { resource_refs: Vec }, + /// The code a write changed, for the card's diff: of a script in `language`, or of an + /// app file at `file_path` (`before_file_path` when its language changed). + CodeDiff { + before: String, + after: String, + language: Option, + file_path: Option, + before_file_path: Option, + }, +} + +impl ChatContext { + fn live_editor(&self, kind: &str) -> Option<&LiveEditor> { + self.live_editors.iter().find(|e| e.kind == kind) + } + + /// The key a draft named `path` is stored under: an open editor's storage key when + /// `path` is either of its names, else `path` itself. + pub fn storage_path<'a>(&'a self, kind: &str, path: &'a str) -> &'a str { + match self.live_editor(kind) { + Some(editor) + if path == editor.storage_path + || editor.effective_path.as_deref() == Some(path) => + { + &editor.storage_path + } + _ => path, + } + } + + /// The path a draft stored at `storage_path` is shown under, and whether an editor is + /// open on it. + pub fn display_path<'a>(&'a self, kind: &str, storage_path: &'a str) -> (&'a str, bool) { + match self.live_editor(kind) { + Some(editor) if editor.storage_path == storage_path => ( + editor + .effective_path + .as_deref() + .filter(|p| !p.is_empty()) + .unwrap_or(storage_path), + true, + ), + _ => (storage_path, false), + } + } + + pub fn cell(&self, kind: &str, storage_path: &str) -> Option<&Value> { + self.cells + .iter() + .find(|c| c.kind == kind && c.storage_path == storage_path) + .map(|c| &c.value) + } +} + +impl super::Ctx<'_> { + /// Patch the chat's card for this call. + pub(super) fn status(&self, content: impl Into) { + let mut patch = serde_json::Map::new(); + patch.insert("content".into(), Value::String(content.into())); + self.effect(Effect::Status(patch)); + } + + /// The card's label while a draft write builds and saves. + pub(super) fn saving_draft(&self, item_type: &str, path: &str) { + self.status(format!("Saving {item_type} \"{path}\" as a draft…")); + } + + /// `status`, applied only once the chat saved the call's draft writes and deletes. + pub(super) fn status_after_save(&self, content: impl Into, result: Option<&str>) { + let mut patch = serde_json::Map::new(); + patch.insert("content".into(), Value::String(content.into())); + if let Some(result) = result { + patch.insert("result".into(), Value::String(result.to_string())); + } + patch.insert("after_save".into(), Value::Bool(true)); + self.effect(Effect::Status(patch)); + } + + pub(super) fn code_diff(&self, before: &str, after: &str, language: Option<&str>) { + self.effect(Effect::CodeDiff { + before: before.to_string(), + after: after.to_string(), + language: language.map(String::from), + file_path: None, + before_file_path: None, + }); + } + + /// `code_diff` of an app file, whose language its path gives. + pub(super) fn file_diff( + &self, + before: &str, + after: &str, + file_path: &str, + before_file_path: &str, + ) { + self.effect(Effect::CodeDiff { + before: before.to_string(), + after: after.to_string(), + language: None, + file_path: Some(file_path.to_string()), + before_file_path: (before_file_path != file_path).then(|| before_file_path.to_string()), + }); + } + + /// Whether the call comes from an AI session rather than the side-panel chat. + pub(super) fn in_session(&self) -> bool { + self.chat.is_some_and(|chat| chat.session) + } + + /// Hand the card facts the model does not read. + pub(super) fn facts(&self, facts: Value) { + if let Value::Object(facts) = facts { + self.effect(Effect::Facts(facts)); + } + } + + /// Whether the call is the chat's check before its confirmation card. + pub(super) fn check_only(&self) -> bool { + self.chat.is_some_and(|chat| chat.check_only) + } + + /// Mark the card failed with `message`, and answer it as the tool's result. + pub(super) fn failed(&self, message: String) -> String { + let mut patch = serde_json::Map::new(); + patch.insert("content".into(), Value::String(message.clone())); + patch.insert("error".into(), Value::String(message.clone())); + self.effect(Effect::Status(patch)); + message + } + + /// The key a draft named `path` is stored under; `path` itself outside the chat. + pub(super) fn storage_path<'s>(&'s self, kind: &str, path: &'s str) -> &'s str { + self.chat.map_or(path, |chat| chat.storage_path(kind, path)) + } + + /// The path a draft stored at `storage_path` is shown under, and whether an editor is + /// open on it. + pub(super) fn display_path<'s>(&'s self, kind: &str, storage_path: &'s str) -> (&'s str, bool) { + self.chat.map_or((storage_path, false), |chat| { + chat.display_path(kind, storage_path) + }) + } +} diff --git a/backend/windmill-api/src/mcp/session/data.rs b/backend/windmill-api/src/ai_session_tools/data.rs similarity index 90% rename from backend/windmill-api/src/mcp/session/data.rs rename to backend/windmill-api/src/ai_session_tools/data.rs index 665647c7ce..3f8bbff5e7 100644 --- a/backend/windmill-api/src/mcp/session/data.rs +++ b/backend/windmill-api/src/ai_session_tools/data.rs @@ -5,7 +5,10 @@ use std::sync::LazyLock; use regex::Regex; use serde_json::{json, Map, Value}; -use super::runs::{backgrounded_summary, run_script_preview, wait_budget, wait_job}; +use super::runs::{ + backgrounded_summary, run_script_preview, run_tool, start_run, wait_budget, wait_job, Resolved, + RunTarget, +}; use super::workspace::{api_message, path_enc}; use super::{array, enc, js_str, non_null, opt_str_arg, pretty, str_arg, truthy, Ctx, ToolResult}; @@ -128,6 +131,9 @@ async fn list_datatables(ctx: &Ctx<'_>, args: &Value) -> ToolResult { if truthy(d.get("error")) { out.insert("error".into(), d["error"].clone()); } + if truthy(d.get("migrations_enabled")) { + out.insert("migrations_enabled".into(), Value::Bool(true)); + } if truthy(d.get("permissioned")) { for key in ["usable_roles", "default_role"] { if let Some(v) = d.get(key) { @@ -221,30 +227,33 @@ async fn exec_datatable_sql(ctx: &Ctx<'_>, args: &Value) -> ToolResult { let name = str_arg(args, "datatable_name")?; let sql = str_arg(args, "sql")?; let role = role_arg(args)?; - let failed = |e: String| format!("Failed to execute test run: {e}"); - - let database = match role { - None => format!("datatable://{name}"), - // A stored name holding a `?` is read whole by the server, so a role cannot follow it. - Some(role) if name.contains('?') => { - return Err(failed(format!("Data table '{name}' has a '?' in its name, so it can only be used here as its default role. Rename it to connect as role '{role}'."))); - } - Some(role) => format!("datatable://{name}?role={role}"), + let resolved = async { + Ok(Resolved { + target: RunTarget::DatatableSql { + datatable: name.to_string(), + sql: sql.to_string(), + role: role.map(String::from), + }, + schema: None, + code: None, + lang: None, + summary: None, + runnable_kind: "script", + start_message: format!("Executing SQL on \"{name}\"..."), + conversational: false, + module: None, + }) }; - let id = run_script_preview( - ctx, - None, - "postgresql", - sql, - json!({ "database": database }), - ) + run_tool(ctx, args, resolved, async |resolved: Resolved| { + let id = start_run(ctx, &resolved.target, Map::new(), &[]) + .await + .map_err(|e| format!("Failed to execute test run: {e}"))?; + match wait_job(ctx, &id, wait_budget(args), false).await { + Some(job) => Ok(format_datatable_sql_completion(&job, name)), + None => Ok(backgrounded_summary(&id, &format!("SQL · {name}"))), + } + }) .await - .map_err(failed)?; - - match wait_job(ctx, &id, wait_budget(args), false).await { - Some(job) => Ok(format_datatable_sql_completion(&job, name)), - None => Ok(backgrounded_summary(&id, &format!("SQL · {name}"))), - } } fn format_datatable_sql_completion(job: &Value, datatable_name: &str) -> String { @@ -309,9 +318,13 @@ async fn migrations_status(ctx: &Ctx<'_>, datatable_name: &str) -> Result, path: &str) -> Result { let text = ctx - .request("POST", path, None) + .request_within("POST", path, None, MIGRATION_RUN_TIMEOUT) .await .map_err(|e| api_message(&e).to_string())?; serde_json::from_str(&text).map_err(|e| format!("unexpected response: {e}")) @@ -449,9 +462,12 @@ async fn list_datatable_migrations(ctx: &Ctx<'_>, args: &Value) -> ToolResult { let version = version_arg(args)?; let status = match migrations_status(ctx, datatable_name).await { Ok(status) => status, - Err(e) => return Ok(format!("Error listing migrations: {e}")), + Err(e) => return Ok(ctx.failed(format!("Error listing migrations: {}", api_message(&e)))), }; if !truthy(status.get("enabled")) { + ctx.status(format!( + "Migrations are not enabled on \"{datatable_name}\"" + )); return Ok(format!("Migrations are not enabled on datatable \"{datatable_name}\": its schema is changed with plain SQL and there is no history to list. Enabling them is a workspace admin's decision, made in Workspace settings → Data Tables.")); } let migrations = array(status.get("migrations")); @@ -474,16 +490,24 @@ async fn list_datatable_migrations(ctx: &Ctx<'_>, args: &Value) -> ToolResult { if truthy(status.get("error")) { out.insert("error".into(), status["error"].clone()); } + ctx.status(format!( + "Listed {} migration(s) of \"{datatable_name}\"", + migrations.len() + )); return Ok(pretty(&Value::Object(out))); }; let Some(migration) = migrations .iter() .find(|m| m["timestamp"].as_i64() == Some(version)) else { - return Ok(format!( + return Ok(ctx.failed(format!( "No migration {version} on datatable \"{datatable_name}\"" - )); + ))); }; + ctx.status(format!( + "Read migration {}", + migration_label(&version.into(), &migration["name"]) + )); let mut out = Map::new(); out.insert("version".into(), version.into()); out.insert("name".into(), migration["name"].clone()); @@ -605,11 +629,12 @@ async fn create_datatable_migration(ctx: &Ctx<'_>, args: &Value) -> ToolResult { } let version = match save_migration(ctx, datatable_name, version, definition).await { Ok(Ok(version)) => version, - Ok(Err(message)) => return Ok(message), - Err(e) => return Ok(format!("Error saving migration: {}", api_message(&e))), + Ok(Err(message)) => return Ok(ctx.failed(message)), + Err(e) => return Ok(ctx.failed(format!("Error saving migration: {}", api_message(&e)))), }; let label = format!("{version} ({name})"); if args.get("apply") == Some(&Value::Bool(false)) { + ctx.status(format!("Saved migration {label} as pending")); return Ok(format!( "Saved migration {label} as pending. It is not applied yet." )); @@ -619,19 +644,27 @@ async fn create_datatable_migration(ctx: &Ctx<'_>, args: &Value) -> ToolResult { let own_scope = (!scope.is_empty()).then_some(scope.as_slice()); let earlier = match earlier_pending(ctx, datatable_name, version, own_scope).await { Ok(earlier) => earlier, - Err(e) => return Ok(format!("Saved migration {label} as pending, but could not check what else is pending, so it was not applied: {e}")), + Err(e) => return Ok(ctx.failed(format!("Saved migration {label} as pending, but could not check what else is pending, so it was not applied: {}", api_message(&e)))), }; if !earlier.is_empty() { + ctx.status(format!( + "Saved migration {label}; {} earlier migration(s) still pending", + earlier.len() + )); return Ok(format!("Saved migration {label} as pending without applying it: earlier migrations are not applied yet ({}) and it may depend on them. Tell the user, then apply them all in order with run_datatable_migrations.", earlier.join(", "))); } + ctx.status(format!("Applying migration {label}...")); let run = format!( "{}?only={version}", migrations_path(ctx, "run_datatable_migrations", datatable_name) ); Ok(match post_migrations(ctx, &run).await { - Ok(_) => format!("Applied migration {label} on datatable \"{datatable_name}\"."), - Err(e) => format!("Migration {label} is saved but failed to apply: {e}. It stays pending and blocks later migrations. If nothing of it was applied, fix it by calling this tool again with version {version}; otherwise, or to drop it, tell the user, who can edit or delete it in Workspace settings → Data Tables."), + Ok(_) => { + ctx.status(format!("Applied migration {label}")); + format!("Applied migration {label} on datatable \"{datatable_name}\".") + } + Err(e) => ctx.failed(format!("Migration {label} is saved but failed to apply: {}. It stays pending and blocks later migrations. If nothing of it was applied, fix it by calling this tool again with version {version}; otherwise, or to drop it, tell the user, who can edit or delete it in Workspace settings → Data Tables.", api_message(&e))), }) } @@ -642,8 +675,13 @@ async fn run_datatable_migrations(ctx: &Ctx<'_>, args: &Value) -> ToolResult { Ok(result) => { let applied = array(result.get("applied")); if applied.is_empty() { + ctx.status("No pending migration to apply"); format!("No pending migration to apply on datatable \"{datatable_name}\".") } else { + ctx.status(format!( + "Applied {} migration(s) on \"{datatable_name}\"", + applied.len() + )); format!( "Applied {} migration(s) on datatable \"{datatable_name}\": {}.", applied.len(), @@ -651,7 +689,7 @@ async fn run_datatable_migrations(ctx: &Ctx<'_>, args: &Value) -> ToolResult { ) } } - Err(e) => format!("Error applying migrations: {e}. Migrations before the failing one may have been applied: list them to see."), + Err(e) => ctx.failed(format!("Error applying migrations: {}. Migrations before the failing one may have been applied: list them to see.", api_message(&e))), }) } @@ -663,12 +701,16 @@ async fn rollback_datatable_migration(ctx: &Ctx<'_>, args: &Value) -> ToolResult migrations_path(ctx, "rollback_datatable_migrations", datatable_name) ); Ok(match post_migrations(ctx, &path).await { - Ok(result) if array(result.get("rolled_back")).is_empty() => format!("Migration {version} is not applied on datatable \"{datatable_name}\", so there is nothing to roll back."), - Ok(result) => format!( - "Rolled back migration {} on datatable \"{datatable_name}\". It is pending again.", - labels(result.get("rolled_back")) - ), - Err(e) => format!("Error rolling back: {e}"), + Ok(result) if array(result.get("rolled_back")).is_empty() => { + ctx.status(format!("Migration {version} is not applied")); + format!("Migration {version} is not applied on datatable \"{datatable_name}\", so there is nothing to roll back.") + } + Ok(result) => { + let label = labels(result.get("rolled_back")); + ctx.status(format!("Rolled back migration {label}")); + format!("Rolled back migration {label} on datatable \"{datatable_name}\". It is pending again.") + } + Err(e) => ctx.failed(format!("Error rolling back: {}", api_message(&e))), }) } diff --git a/backend/windmill-api/src/mcp/session/deploy.rs b/backend/windmill-api/src/ai_session_tools/deploy.rs similarity index 97% rename from backend/windmill-api/src/mcp/session/deploy.rs rename to backend/windmill-api/src/ai_session_tools/deploy.rs index 4ddb924e25..770c166fbf 100644 --- a/backend/windmill-api/src/mcp/session/deploy.rs +++ b/backend/windmill-api/src/ai_session_tools/deploy.rs @@ -14,7 +14,7 @@ use super::workspace::{ }; use super::{ array, bool_arg, enc, is_not_found, js_str, matches_glob, non_null, opt_str_arg, pretty, - str_arg, truthy, Ctx, ToolResult, + str_arg, truthy, Ctx, Surface, ToolResult, }; pub(super) const TOOLS: &[&str] = &["deploy_workspace_item", "delete_workspace_item", "diff"]; @@ -299,7 +299,7 @@ async fn undeployed_runnable_targets(ctx: &Ctx<'_>, runnables: &Map, runnables: &Map, path: &str) -> Result { + match ctx + .get(&ctx.ws(&format!("resources/get/{}", path_enc(path)))) + .await + { + Ok(resource) => Ok(resource["resource_type"] == "ai_agent"), + Err(e) if is_not_found(&e) => Ok(false), + Err(e) => Err(e), + } +} + async fn guest_access_is_live(ctx: &Ctx<'_>) -> bool { let Ok(usage) = ctx.get(&ctx.ws("workspaces/guest_usage")).await else { return false; @@ -574,7 +591,7 @@ async fn deploy_item(ctx: &Ctx<'_>, args: &Value) -> ToolResult { } // The baseline keeps a draft saved since the read above from being deleted undeployed. - delete_draft(ctx, &kind, path, draft.created_at.as_deref()).await?; + delete_draft(ctx, &kind, path, draft.created_at.as_deref(), true).await?; let note = note.map(|note| format!(" {note}")).unwrap_or_default(); Ok(mutation_result( @@ -582,7 +599,7 @@ async fn deploy_item(ctx: &Ctx<'_>, args: &Value) -> ToolResult { item_type, path, trigger_kind, - deployed_page(item_type, path, value).map(|page| ctx.page_url(&page)), + deployed_page(item_type, path, value).and_then(|page| ctx.link(&page)), )) } @@ -597,17 +614,28 @@ async fn delete_item(ctx: &Ctx<'_>, args: &Value) -> ToolResult { if item_type == "trigger" && trigger_kind.is_none() { return Err("trigger_kind is required when deleting a trigger.".to_string()); } - // Undoing a draft is a discard, not a delete: say so before the delete route 404s. - if !deployed_item_exists(ctx, item_type, trigger_kind, path).await? { - let draft = get_draft_item(ctx, item_type, path, trigger_kind).await?; - return Err(if draft.is_some() { - format!( - "No deployed {item_type} at \"{path}\" — it only exists as a draft, so there is nothing to delete \ - from the workspace. Call discard_local_draft with the same arguments to remove the draft." - ) - } else { - format!("No {item_type} at \"{path}\": neither a deployed item nor a draft. Nothing to delete.") - }); + // Undoing a draft is a discard, not a delete: say so before the delete route 404s. The + // chat asks this before its confirmation card, which a refused delete never shows. + if ctx.surface == Surface::Mcp || ctx.check_only() { + if !deployed_item_exists(ctx, item_type, trigger_kind, path).await? { + let draft = get_draft_item(ctx, item_type, path, trigger_kind).await?; + let refusal = if draft.is_some() { + format!( + "No deployed {item_type} at \"{path}\" — it only exists as a draft, so there is nothing to delete \ + from the workspace. Call discard_local_draft with the same arguments to remove the draft." + ) + } else { + format!("No {item_type} at \"{path}\": neither a deployed item nor a draft. Nothing to delete.") + }; + if ctx.check_only() { + ctx.facts(json!({ "refused": refusal })); + return Ok(refusal); + } + return Err(refusal); + } + if ctx.check_only() { + return Ok(String::new()); + } } let path_in_url = path_enc(path); @@ -626,8 +654,9 @@ async fn delete_item(ctx: &Ctx<'_>, args: &Value) -> ToolResult { .await?; } if let Some(kind) = draft_kind(item_type, trigger_kind) { - delete_draft(ctx, &kind, path, None).await?; + delete_draft(ctx, &kind, path, None, false).await?; } + ctx.status_after_save(format!("Deleted {item_type} \"{path}\""), Some("Deleted")); Ok(mutation_result( format!( "Deleted {item_type} \"{path}\" from the workspace. Any matching draft was also cleared." diff --git a/backend/windmill-api/src/mcp/session/flows.rs b/backend/windmill-api/src/ai_session_tools/flows.rs similarity index 92% rename from backend/windmill-api/src/mcp/session/flows.rs rename to backend/windmill-api/src/ai_session_tools/flows.rs index 6911552c41..6a41f32bc3 100644 --- a/backend/windmill-api/src/mcp/session/flows.rs +++ b/backend/windmill-api/src/ai_session_tools/flows.rs @@ -9,6 +9,7 @@ use std::collections::{HashMap, HashSet}; use serde_json::{json, Map, Value}; use windmill_types::flows::FlowModule; +use super::ai_providers::{self, CatalogFor}; use super::workspace::{ compact_flow, empty_schema, find_and_replace, finish_draft_write, load_flow, persist_draft, write_base, Persist, FLOW_VALUE_SETTINGS_KEYS, @@ -69,7 +70,6 @@ const RESERVED_TOOL_NAMES: &[&str] = &[ "Trigger", "__wm_web_search", ]; -const PROVIDER_SHAPE: &str = r#"{ "type": "static", "value": { "kind": "", "resource": "$res:", "model": "" } }"#; fn module_id(module: &Value) -> &str { module.get("id").and_then(Value::as_str).unwrap_or_default() @@ -410,45 +410,6 @@ fn tool_name_error(name: &str, siblings: &[&str]) -> Option { (siblings.iter().filter(|n| **n == name).count() > 1).then(|| "Duplicate tool name".into()) } -/// A provider config that cannot run whatever the workspace holds. -fn provider_shape_error(provider: Option<&Value>, step_type: &str) -> Option { - let provider = match provider { - Some(Value::String(reference)) => { - return Some(format!( - "provider must be an object {PROVIDER_SHAPE}, not the resource reference string {}", - json!(reference) - )) - } - Some(Value::Object(provider)) => provider, - _ => return Some(format!("provider must be an object {PROVIDER_SHAPE}")), - }; - let text = |key: &str| { - provider - .get(key) - .and_then(Value::as_str) - .unwrap_or_default() - }; - let kind = text("kind"); - if kind.is_empty() { - return Some(format!( - "provider.kind is missing. Expected {PROVIDER_SHAPE}" - )); - } - let decision_only = ["typesafe", "cloudflare"].contains(&kind); - if step_type == "aidecision" && !decision_only && kind != "openai" { - return Some(format!("an AI decision step runs on a decision model: provider.kind must be \"typesafe\", \"cloudflare\" or \"openai\", not \"{kind}\"")); - } - if step_type != "aidecision" && decision_only { - return Some(format!("\"{kind}\" serves decision models, which answer typed questions rather than messages: use an \"aidecision\" step rather than an AI agent")); - } - if !text("resource").starts_with("$res:") { - return Some(format!("provider.resource must be a \"$res:\" reference. Expected {PROVIDER_SHAPE}")); - } - text("model") - .is_empty() - .then(|| format!("provider.model is missing. Expected {PROVIDER_SHAPE}")) -} - fn quoted_list(ids: &[String]) -> String { ids.iter() .map(|id| format!("\"{id}\"")) @@ -465,7 +426,6 @@ fn validate_modules(modules: &mut Value) -> Result, String> { let mut ids = Vec::new(); let mut providerless = Vec::new(); let mut tool_names = Vec::new(); - let mut providers = Vec::new(); walk(Some(modules), false, &mut |module, in_tool| { let id = module_id(module); // A tool is not a flow module: the agent that owns it checks its shape. @@ -517,13 +477,6 @@ fn validate_modules(modules: &mut Value) -> Result, String> { )); } } - - let is_static = |p: &&Value| p.get("type").and_then(Value::as_str) == Some("static"); - if let Some(provider) = provider.filter(|_| !linked).filter(is_static) { - if let Some(error) = provider_shape_error(provider.get("value"), kind) { - providers.push(format!("Step \"{id}\": {error}")); - } - } }); if !shape_errors.is_empty() { @@ -551,12 +504,6 @@ fn validate_modules(modules: &mut Value) -> Result, String> { tool_names.join("; ") )); } - if !providers.is_empty() { - return Err(format!( - "Invalid AI agent provider configuration:\n{}", - providers.join("\n") - )); - } Ok(unique) } @@ -722,8 +669,12 @@ fn validate_notes(raw: Option<&Value>, module_ids: &HashSet) -> Result Result, String> { +/// present, `null` when unset. Provider findings that do not block go to `warnings`. +fn validate_editable( + raw: Value, + catalog: Option<&ai_providers::Catalog>, + warnings: &mut Vec, +) -> Result, String> { let Value::Object(mut flow) = raw else { return Err("Flow JSON must be an object".into()); }; @@ -744,8 +695,9 @@ fn validate_editable(raw: Value) -> Result, String> { } validate_settings(&flow)?; - let mut modules = flow.remove("modules").unwrap_or(Value::Null); + let mut modules = flow.shift_remove("modules").unwrap_or(Value::Null); let module_ids = validate_modules(&mut modules)?; + ai_providers::validate(&modules, catalog, warnings)?; let schema = match flow.get("schema") { None | Some(Value::Null) => Value::Null, Some(schema @ Value::Object(_)) => schema.clone(), @@ -776,12 +728,12 @@ fn validate_editable(raw: Value) -> Result, String> { editable.insert("modules".into(), modules); editable.insert("schema".into(), schema); for (key, _) in SPECIAL_MODULES { - editable.insert(key.into(), flow.remove(key).unwrap_or(Value::Null)); + editable.insert(key.into(), flow.shift_remove(key).unwrap_or(Value::Null)); } editable.insert("groups".into(), groups); editable.insert("notes".into(), notes); for key in FLOW_VALUE_SETTINGS_KEYS { - if let Some(setting) = flow.remove(*key) { + if let Some(setting) = flow.shift_remove(*key) { editable.insert((*key).into(), setting); } } @@ -804,6 +756,7 @@ struct FlowWrite<'a> { async fn write_flow_draft(ctx: &Ctx<'_>, write: FlowWrite<'_>) -> ToolResult { let FlowWrite { path, mut value, .. } = write; + ctx.saving_draft("flow", path); let base = write_base(ctx, "flow", path, None).await?; let existed = base.base.is_some(); if let Some(groups) = write.groups { @@ -910,7 +863,13 @@ async fn write_flow(ctx: &Ctx<'_>, args: &Value) -> ToolResult { for key in STRUCTURAL_KEYS { raw.insert((*key).into(), json_arg(args, key)?); } - let mut editable = validate_editable(Value::Object(raw))?; + let catalog = match ai_providers::catalog_for(ctx, &raw["modules"]).await { + CatalogFor::Requested => return Ok(String::new()), + CatalogFor::Ready(catalog) => Some(catalog), + CatalogFor::NotNeeded => None, + }; + let mut warnings = Vec::new(); + let mut editable = validate_editable(Value::Object(raw), catalog.as_ref(), &mut warnings)?; // A placeholder keeps the body the current flow has for that module, so a rewrite // does not have to send code it is not changing. @@ -924,9 +883,9 @@ async fn write_flow(ctx: &Ctx<'_>, args: &Value) -> ToolResult { let schema = non_null(editable.get("schema")).cloned(); let groups = non_null(editable.get("groups")).cloned(); - editable.remove("schema"); + editable.shift_remove("schema"); editable.retain(|_, field| !field.is_null()); - write_flow_draft( + let written = write_flow_draft( ctx, FlowWrite { path, @@ -940,7 +899,18 @@ async fn write_flow(ctx: &Ctx<'_>, args: &Value) -> ToolResult { force: args.get("override").and_then(Value::as_bool) == Some(true), }, ) - .await + .await?; + Ok(with_provider_warnings(ctx, written, &warnings)) +} + +/// The provider findings that did not block the write, after the result. The chat appends +/// them itself when its own save fails and replaces this text. +fn with_provider_warnings(ctx: &Ctx<'_>, written: String, warnings: &[String]) -> String { + let suffix = ai_providers::format_warnings(warnings); + if !suffix.is_empty() { + ctx.facts(json!({ "text_suffix": suffix })); + } + format!("{written}{suffix}") } async fn patch_flow_json(ctx: &Ctx<'_>, args: &Value) -> ToolResult { @@ -959,10 +929,16 @@ async fn patch_flow_json(ctx: &Ctx<'_>, args: &Value) -> ToolResult { )?; let parsed: Value = serde_json::from_str(&updated) .map_err(|e| format!("Invalid JSON after replacement: {e}"))?; - let editable = validate_editable(parsed)?; + let catalog = match ai_providers::catalog_for(ctx, &parsed["modules"]).await { + CatalogFor::Requested => return Ok(String::new()), + CatalogFor::Ready(catalog) => Some(catalog), + CatalogFor::NotNeeded => None, + }; + let mut warnings = Vec::new(); + let editable = validate_editable(parsed, catalog.as_ref(), &mut warnings)?; let mut value = apply_editable(base.flow.get("value"), &editable, &scripts); finalize_inline_scripts(&mut value)?; - write_flow_draft( + let written = write_flow_draft( ctx, FlowWrite { path, @@ -976,14 +952,38 @@ async fn patch_flow_json(ctx: &Ctx<'_>, args: &Value) -> ToolResult { force: false, }, ) - .await + .await?; + Ok(with_provider_warnings(ctx, written, &warnings)) +} + +/// A step as the chat's card names it: its id, and its summary when it has one. +fn step_name(flow: Option<&Value>, module_id: &str) -> String { + let mut flow = flow.and_then(Value::as_object).cloned().unwrap_or_default(); + let mut summary = None; + walk_flow(&mut flow, &mut |module, _| { + if summary.is_none() && self::module_id(module) == module_id { + summary = module + .get("summary") + .and_then(Value::as_str) + .map(String::from); + } + }); + match summary.filter(|s| !s.is_empty()) { + Some(summary) => format!("{module_id} \"{summary}\""), + None => module_id.to_string(), + } } async fn read_flow_module_code(ctx: &Ctx<'_>, args: &Value) -> ToolResult { let (path, module_id) = (str_arg(args, "path")?, str_arg(args, "module_id")?); let base = load_flow(ctx, path).await?; let (_, mut scripts) = compact_flow(base.flow.get("value").unwrap_or(&Value::Null), None); - scripts.remove(module_id).ok_or_else(|| format!("Module \"{module_id}\" is not an inline rawscript in flow \"{path}\". (Path runnables, branches, and loops have no inline script content; use patch_flow_json to inspect them.)")) + let code = scripts.remove(module_id).ok_or_else(|| format!("Module \"{module_id}\" is not an inline rawscript in flow \"{path}\". (Path runnables, branches, and loops have no inline script content; use patch_flow_json to inspect them.)"))?; + ctx.status(format!( + "Read code of step {}", + step_name(base.flow.get("value"), module_id) + )); + Ok(code) } async fn set_flow_module_code(ctx: &Ctx<'_>, args: &Value) -> ToolResult { @@ -999,6 +999,14 @@ async fn set_flow_module_code(ctx: &Ctx<'_>, args: &Value) -> ToolResult { } scripts.insert(module_id.to_string(), code.to_string()); let editable = compact.as_object().cloned().unwrap_or_default(); + // Several code edits of one flow read as identical rows under the generic flow label. + ctx.status_after_save( + format!( + "Updated code of step {}", + step_name(base.flow.get("value"), module_id) + ), + None, + ); write_flow_draft( ctx, FlowWrite { @@ -1111,7 +1119,8 @@ mod tests { .to_string() .replace(",\"same_worker\":true", "") .replace("\"id\":\"b\"", "\"id\":\"c\""); - let editable = validate_editable(serde_json::from_str(&patched).unwrap()).unwrap(); + let editable = + validate_editable(serde_json::from_str(&patched).unwrap(), None, &mut vec![]).unwrap(); let mut applied = apply_editable(Some(&value), &editable, &scripts); finalize_inline_scripts(&mut applied).unwrap(); assert_eq!(content(&applied, "/0"), "A"); @@ -1124,7 +1133,7 @@ mod tests { #[test] fn validation_reports_what_the_editor_would_refuse() { - let check = |flow: Value| validate_editable(flow).unwrap_err(); + let check = |flow: Value| validate_editable(flow, None, &mut vec![]).unwrap_err(); assert_eq!( check(json!({ "modules": [raw("a", ""), raw("a", "")] })), "Duplicate module IDs found in flow" @@ -1159,7 +1168,7 @@ mod tests { assert!(check(agent(json!([tool("search docs")]))).starts_with( "Invalid AI agent tool name(s): agent \"agent\" tool \"t\" is named \"search docs\" - Tool name must only contain letters, numbers and underscores. " )); - validate_editable(agent(json!([tool("search_docs")]))).unwrap(); + validate_editable(agent(json!([tool("search_docs")])), None, &mut vec![]).unwrap(); } #[test] diff --git a/backend/windmill-api/src/mcp/session/gen/app_templates.json b/backend/windmill-api/src/ai_session_tools/gen/app_templates.json similarity index 57% rename from backend/windmill-api/src/mcp/session/gen/app_templates.json rename to backend/windmill-api/src/ai_session_tools/gen/app_templates.json index 02c1fa35b5..4aeabf1966 100644 --- a/backend/windmill-api/src/mcp/session/gen/app_templates.json +++ b/backend/windmill-api/src/ai_session_tools/gen/app_templates.json @@ -19,6 +19,26 @@ "/package.json": "{\n \"dependencies\": {\n \"svelte\": \"^5.56.8\"\n }\n}" } }, + "builder_templates": { + "react19": { + "/index.tsx": "\nimport React from 'react'\n\nimport { createRoot } from 'react-dom/client'\nimport App from './App'\n\nconst root = createRoot(document.getElementById('root')!);\nroot.render();\n", + "/App.tsx": "import React from 'react'\nimport './index.css'\n\n// Windmill injects the viewer (absent for anonymous visitors) and the workspace.\nconst ctx = (window as any).ctx ?? {}\nconst viewer: { name?: string; username?: string } | undefined = ctx.ctx\nconst firstName = (viewer?.name || viewer?.username || '').split(' ')[0]\n\ntype Step = { title: string; description: string; note: string; path?: string[]; code?: string }\n\nconst uses: { title: string; text: string }[] = [\n {\n \"title\": \"Forms for your scripts\",\n \"text\": \"Give a script a simple form, so anyone can run it without learning Windmill.\"\n },\n {\n \"title\": \"Dashboards\",\n \"text\": \"Show what a script or flow returns as a table or a chart, fresh every time the app opens.\"\n },\n {\n \"title\": \"Self-service tools\",\n \"text\": \"Share the app with your team or by link. It runs its scripts as you, so viewers need no access to them.\"\n }\n]\n\nconst steps: Step[] = [\n {\n \"title\": \"Pick what it runs\",\n \"description\": \"Choose a script or flow already deployed in this workspace.\",\n \"path\": [\n \"Backend panel\",\n \"+\",\n \"Select a script or flow\"\n ],\n \"note\": \"You only see the ones you can read. Each one gets an id, such as a.\"\n },\n {\n \"title\": \"Show it on the page\",\n \"description\": \"Each runnable becomes a function the page can call by its id.\",\n \"note\": \"Working with an AI assistant? Tell it the id and what to show. This is the line it writes.\",\n \"code\": \"import { backend } from './wmill'\\n\\nconst res = await backend.a({ name: 'world' })\"\n },\n {\n \"title\": \"Run long jobs in the background\",\n \"description\": \"backendAsync starts a job without making the page wait.\",\n \"note\": \"Check on it with getJob, or follow its output as it runs with streamJob.\",\n \"code\": \"import { backendAsync, getJob } from './wmill'\\n\\nconst id = await backendAsync.a({ name: 'world' })\\nconst job = await getJob(id)\"\n }\n]\n\nconst sectionTitle: React.CSSProperties = {\n margin: '32px 0 12px',\n fontSize: 12,\n fontWeight: 600,\n letterSpacing: '0.06em',\n textTransform: 'uppercase',\n color: 'var(--muted)'\n}\n\nfunction Card(props: { step: number; title: string; description: string; code?: string; children: React.ReactNode }) {\n return (\n
\n
\n {props.step}\n
\n

{props.title}

\n

{props.description}

\n
\n
\n
{props.children}
\n {props.code && (\n
\n          {props.code}\n        
\n )}\n
\n )\n}\n\nconst App = () => {\n return (\n
\n
\n Windmill app{ctx.workspace ? ' · ' + ctx.workspace : ''}\n

{firstName ? 'Welcome, ' + firstName : 'Welcome'}

\n

\n Turn the scripts and flows deployed in this workspace into a tool anyone can use. Edit\n App.tsx to make it yours.\n

\n
\n

What you can build

\n
\n {uses.map((use) => (\n
\n

{use.title}

\n

{use.text}

\n
\n ))}\n
\n

\n {\"Your app can only run scripts and flows already deployed in this workspace, and it runs isolated from each viewer’s session.\"}\n

\n

How it works

\n
\n {steps.map((step, i) => (\n \n {step.path && (\n
\n {step.path.map((label, j) => (\n \n {j > 0 && →}\n {label}\n \n ))}\n
\n )}\n {step.note}\n
\n ))}\n \n
\n Workspace \n {ctx.workspace}\n
\n
\n Viewer \n {viewer ? viewer.name || viewer.username : 'Anonymous visitor'}\n
\n \n
\n
\n )\n}\n\nexport default App\n", + "/index.css": "/* The app renders inside an iframe whose host page has its own theme the\n iframe cannot read, so paint an opaque surface here: without one the host's\n background shows through and dark text lands on it unreadable. */\n:root {\n color-scheme: light;\n --bg: #f4f5f8;\n --card: #ffffff;\n --sunken: #efeff4;\n --border: #e2e3eb;\n --fg: #3d4758;\n --fg-strong: #1d2430;\n --muted: #718096;\n --accent: #5074f6;\n --accent-hover: #2c5beb;\n --accent-soft: #e8ebfb;\n --success: #15803d;\n --success-soft: #dcfce7;\n --error: #b91c1c;\n --error-soft: #fef2f2;\n --warning: #a16207;\n --warning-soft: #fef9c3;\n}\n\n@media (prefers-color-scheme: dark) {\n :root {\n color-scheme: dark;\n --bg: #242832;\n --card: #2e3441;\n --sunken: #272c35;\n --border: #3b4252;\n --fg: #d4d7dd;\n --fg-strong: #eeeff2;\n --muted: #a9b0ba;\n --accent: #7085db;\n --accent-hover: #5670d5;\n --accent-soft: #343d5c;\n --success: #86efac;\n --success-soft: #1f3a2b;\n --error: #fca5a5;\n --error-soft: #431a1a;\n --warning: #fde68a;\n --warning-soft: #3d3419;\n }\n}\n\n* {\n box-sizing: border-box;\n}\n\nbody {\n margin: 0;\n min-height: 100vh;\n background: var(--bg);\n color: var(--fg);\n font-family: Inter, ui-sans-serif, system-ui, -apple-system, 'Segoe UI', sans-serif;\n font-size: 14px;\n line-height: 1.5;\n -webkit-font-smoothing: antialiased;\n}\n\nh1,\nh2 {\n margin: 0;\n color: var(--fg-strong);\n}\n\ncode,\npre {\n font-family: ui-monospace, SFMono-Regular, Menlo, Consolas, monospace;\n}\n\n.muted {\n color: var(--muted);\n}\n\n.error {\n color: var(--error);\n}\n\n.app {\n max-width: 1040px;\n margin: 0 auto;\n padding: 40px 20px 56px;\n}\n\n.hero {\n margin-bottom: 28px;\n}\n\n.hero h1 {\n font-size: 1.75rem;\n font-weight: 650;\n letter-spacing: -0.01em;\n margin: 8px 0 6px;\n}\n\n.hero p {\n max-width: 620px;\n margin: 0;\n}\n\n.eyebrow {\n display: inline-block;\n padding: 2px 10px;\n border-radius: 999px;\n background: var(--accent-soft);\n color: var(--accent);\n font-size: 12px;\n font-weight: 600;\n}\n\n.grid {\n display: grid;\n grid-template-columns: repeat(auto-fit, minmax(min(100%, 420px), 1fr));\n gap: 16px;\n}\n\n.card {\n display: flex;\n flex-direction: column;\n background: var(--card);\n border: 1px solid var(--border);\n border-radius: 10px;\n box-shadow: 0 1px 2px rgb(16 24 40 / 0.04);\n overflow: hidden;\n}\n\n.card-header {\n display: flex;\n gap: 12px;\n padding: 18px 18px 0;\n}\n\n.card-header h2 {\n font-size: 15px;\n font-weight: 600;\n}\n\n.card-header p {\n margin: 2px 0 0;\n font-size: 13px;\n}\n\n.card-step {\n flex: none;\n display: grid;\n place-items: center;\n width: 26px;\n height: 26px;\n border-radius: 7px;\n background: var(--accent-soft);\n color: var(--accent);\n font-size: 13px;\n font-weight: 700;\n}\n\n.card-body {\n flex: 1;\n display: flex;\n flex-direction: column;\n gap: 12px;\n padding: 16px 18px 18px;\n}\n\n.snippet {\n margin: 0;\n padding: 10px 18px;\n border-top: 1px solid var(--border);\n background: var(--sunken);\n color: var(--muted);\n font-size: 12px;\n white-space: pre-wrap;\n}\n\n.row {\n display: flex;\n gap: 8px;\n}\n\ninput,\ntextarea {\n flex: 1;\n min-width: 0;\n padding: 7px 10px;\n border: 1px solid var(--border);\n border-radius: 6px;\n background: var(--card);\n color: var(--fg-strong);\n font: inherit;\n resize: vertical;\n}\n\ninput:focus,\ntextarea:focus {\n outline: 2px solid var(--accent);\n outline-offset: -1px;\n}\n\nbutton {\n padding: 7px 14px;\n border: none;\n border-radius: 6px;\n background: var(--accent);\n color: #fff;\n font: inherit;\n font-weight: 500;\n cursor: pointer;\n transition: background 0.15s;\n}\n\nbutton:hover:not(:disabled) {\n background: var(--accent-hover);\n}\n\nbutton:disabled {\n opacity: 0.6;\n cursor: default;\n}\n\n.output {\n min-height: 44px;\n padding: 11px 12px;\n border-radius: 6px;\n background: var(--sunken);\n word-break: break-word;\n}\n\n.output.error {\n background: var(--error-soft);\n}\n\n.output.stream {\n min-height: 96px;\n white-space: pre-wrap;\n}\n\n.caret {\n display: inline-block;\n width: 7px;\n height: 1em;\n margin-left: 2px;\n vertical-align: text-bottom;\n background: var(--accent);\n animation: blink 1s steps(1) infinite;\n}\n\n@keyframes blink {\n 50% {\n opacity: 0;\n }\n}\n\n.stats {\n display: grid;\n grid-template-columns: repeat(3, 1fr);\n gap: 8px;\n}\n\n.stats > div {\n display: flex;\n flex-direction: column;\n padding: 10px 12px;\n border-radius: 6px;\n background: var(--sunken);\n}\n\n.stats strong {\n font-size: 20px;\n color: var(--fg-strong);\n}\n\n.stats span {\n font-size: 12px;\n}\n\n.stats .tags {\n grid-column: 1 / -1;\n flex-direction: row;\n flex-wrap: wrap;\n gap: 6px;\n padding: 0;\n background: none;\n}\n\n.tag {\n padding: 2px 8px;\n border-radius: 999px;\n background: var(--accent-soft);\n color: var(--accent);\n}\n\n.jobs {\n display: flex;\n flex-direction: column;\n gap: 6px;\n margin: 0;\n padding: 0;\n list-style: none;\n}\n\n.jobs li {\n display: flex;\n align-items: center;\n gap: 10px;\n padding: 8px 10px;\n border-radius: 6px;\n background: var(--sunken);\n font-size: 13px;\n}\n\n.pill {\n min-width: 64px;\n padding: 1px 8px;\n border-radius: 999px;\n font-size: 12px;\n font-weight: 600;\n text-align: center;\n}\n\n.pill.queued {\n background: var(--warning-soft);\n color: var(--warning);\n}\n\n.pill.running {\n background: var(--accent-soft);\n color: var(--accent);\n}\n\n.pill.success {\n background: var(--success-soft);\n color: var(--success);\n}\n\n.pill.failure {\n background: var(--error-soft);\n color: var(--error);\n}\n\n.bar {\n flex: 1;\n height: 4px;\n border-radius: 999px;\n background: var(--border);\n overflow: hidden;\n}\n\n.bar-fill {\n width: 0;\n height: 100%;\n border-radius: inherit;\n background: var(--accent);\n}\n\n.bar-fill.running {\n animation: fill linear forwards;\n}\n\n.bar-fill.success {\n width: 100%;\n background: var(--success);\n}\n\n.bar-fill.failure {\n width: 100%;\n background: var(--error);\n}\n\n@keyframes fill {\n to {\n width: 95%;\n }\n}\n", + "/package.json": "{\n \"dependencies\": {\n \"react\": \"19.0.0\",\n \"react-dom\": \"19.0.0\"\n },\n \"devDependencies\": {\n \"@types/react-dom\": \"^19.0.0\",\n \"@types/react\": \"^19.0.0\"\n }\n}" + }, + "react18": { + "/index.tsx": "\nimport React from 'react'\n\nimport { createRoot } from 'react-dom/client'\nimport App from './App'\n\nconst root = createRoot(document.getElementById('root')!);\nroot.render();\n", + "/App.tsx": "import React from 'react'\nimport './index.css'\n\n// Windmill injects the viewer (absent for anonymous visitors) and the workspace.\nconst ctx = (window as any).ctx ?? {}\nconst viewer: { name?: string; username?: string } | undefined = ctx.ctx\nconst firstName = (viewer?.name || viewer?.username || '').split(' ')[0]\n\ntype Step = { title: string; description: string; note: string; path?: string[]; code?: string }\n\nconst uses: { title: string; text: string }[] = [\n {\n \"title\": \"Forms for your scripts\",\n \"text\": \"Give a script a simple form, so anyone can run it without learning Windmill.\"\n },\n {\n \"title\": \"Dashboards\",\n \"text\": \"Show what a script or flow returns as a table or a chart, fresh every time the app opens.\"\n },\n {\n \"title\": \"Self-service tools\",\n \"text\": \"Share the app with your team or by link. It runs its scripts as you, so viewers need no access to them.\"\n }\n]\n\nconst steps: Step[] = [\n {\n \"title\": \"Pick what it runs\",\n \"description\": \"Choose a script or flow already deployed in this workspace.\",\n \"path\": [\n \"Backend panel\",\n \"+\",\n \"Select a script or flow\"\n ],\n \"note\": \"You only see the ones you can read. Each one gets an id, such as a.\"\n },\n {\n \"title\": \"Show it on the page\",\n \"description\": \"Each runnable becomes a function the page can call by its id.\",\n \"note\": \"Working with an AI assistant? Tell it the id and what to show. This is the line it writes.\",\n \"code\": \"import { backend } from './wmill'\\n\\nconst res = await backend.a({ name: 'world' })\"\n },\n {\n \"title\": \"Run long jobs in the background\",\n \"description\": \"backendAsync starts a job without making the page wait.\",\n \"note\": \"Check on it with getJob, or follow its output as it runs with streamJob.\",\n \"code\": \"import { backendAsync, getJob } from './wmill'\\n\\nconst id = await backendAsync.a({ name: 'world' })\\nconst job = await getJob(id)\"\n }\n]\n\nconst sectionTitle: React.CSSProperties = {\n margin: '32px 0 12px',\n fontSize: 12,\n fontWeight: 600,\n letterSpacing: '0.06em',\n textTransform: 'uppercase',\n color: 'var(--muted)'\n}\n\nfunction Card(props: { step: number; title: string; description: string; code?: string; children: React.ReactNode }) {\n return (\n
\n
\n {props.step}\n
\n

{props.title}

\n

{props.description}

\n
\n
\n
{props.children}
\n {props.code && (\n
\n          {props.code}\n        
\n )}\n
\n )\n}\n\nconst App = () => {\n return (\n
\n
\n Windmill app{ctx.workspace ? ' · ' + ctx.workspace : ''}\n

{firstName ? 'Welcome, ' + firstName : 'Welcome'}

\n

\n Turn the scripts and flows deployed in this workspace into a tool anyone can use. Edit\n App.tsx to make it yours.\n

\n
\n

What you can build

\n
\n {uses.map((use) => (\n
\n

{use.title}

\n

{use.text}

\n
\n ))}\n
\n

\n {\"Your app can only run scripts and flows already deployed in this workspace, and it runs isolated from each viewer’s session.\"}\n

\n

How it works

\n
\n {steps.map((step, i) => (\n \n {step.path && (\n
\n {step.path.map((label, j) => (\n \n {j > 0 && →}\n {label}\n \n ))}\n
\n )}\n {step.note}\n
\n ))}\n \n
\n Workspace \n {ctx.workspace}\n
\n
\n Viewer \n {viewer ? viewer.name || viewer.username : 'Anonymous visitor'}\n
\n \n
\n
\n )\n}\n\nexport default App\n", + "/index.css": "/* The app renders inside an iframe whose host page has its own theme the\n iframe cannot read, so paint an opaque surface here: without one the host's\n background shows through and dark text lands on it unreadable. */\n:root {\n color-scheme: light;\n --bg: #f4f5f8;\n --card: #ffffff;\n --sunken: #efeff4;\n --border: #e2e3eb;\n --fg: #3d4758;\n --fg-strong: #1d2430;\n --muted: #718096;\n --accent: #5074f6;\n --accent-hover: #2c5beb;\n --accent-soft: #e8ebfb;\n --success: #15803d;\n --success-soft: #dcfce7;\n --error: #b91c1c;\n --error-soft: #fef2f2;\n --warning: #a16207;\n --warning-soft: #fef9c3;\n}\n\n@media (prefers-color-scheme: dark) {\n :root {\n color-scheme: dark;\n --bg: #242832;\n --card: #2e3441;\n --sunken: #272c35;\n --border: #3b4252;\n --fg: #d4d7dd;\n --fg-strong: #eeeff2;\n --muted: #a9b0ba;\n --accent: #7085db;\n --accent-hover: #5670d5;\n --accent-soft: #343d5c;\n --success: #86efac;\n --success-soft: #1f3a2b;\n --error: #fca5a5;\n --error-soft: #431a1a;\n --warning: #fde68a;\n --warning-soft: #3d3419;\n }\n}\n\n* {\n box-sizing: border-box;\n}\n\nbody {\n margin: 0;\n min-height: 100vh;\n background: var(--bg);\n color: var(--fg);\n font-family: Inter, ui-sans-serif, system-ui, -apple-system, 'Segoe UI', sans-serif;\n font-size: 14px;\n line-height: 1.5;\n -webkit-font-smoothing: antialiased;\n}\n\nh1,\nh2 {\n margin: 0;\n color: var(--fg-strong);\n}\n\ncode,\npre {\n font-family: ui-monospace, SFMono-Regular, Menlo, Consolas, monospace;\n}\n\n.muted {\n color: var(--muted);\n}\n\n.error {\n color: var(--error);\n}\n\n.app {\n max-width: 1040px;\n margin: 0 auto;\n padding: 40px 20px 56px;\n}\n\n.hero {\n margin-bottom: 28px;\n}\n\n.hero h1 {\n font-size: 1.75rem;\n font-weight: 650;\n letter-spacing: -0.01em;\n margin: 8px 0 6px;\n}\n\n.hero p {\n max-width: 620px;\n margin: 0;\n}\n\n.eyebrow {\n display: inline-block;\n padding: 2px 10px;\n border-radius: 999px;\n background: var(--accent-soft);\n color: var(--accent);\n font-size: 12px;\n font-weight: 600;\n}\n\n.grid {\n display: grid;\n grid-template-columns: repeat(auto-fit, minmax(min(100%, 420px), 1fr));\n gap: 16px;\n}\n\n.card {\n display: flex;\n flex-direction: column;\n background: var(--card);\n border: 1px solid var(--border);\n border-radius: 10px;\n box-shadow: 0 1px 2px rgb(16 24 40 / 0.04);\n overflow: hidden;\n}\n\n.card-header {\n display: flex;\n gap: 12px;\n padding: 18px 18px 0;\n}\n\n.card-header h2 {\n font-size: 15px;\n font-weight: 600;\n}\n\n.card-header p {\n margin: 2px 0 0;\n font-size: 13px;\n}\n\n.card-step {\n flex: none;\n display: grid;\n place-items: center;\n width: 26px;\n height: 26px;\n border-radius: 7px;\n background: var(--accent-soft);\n color: var(--accent);\n font-size: 13px;\n font-weight: 700;\n}\n\n.card-body {\n flex: 1;\n display: flex;\n flex-direction: column;\n gap: 12px;\n padding: 16px 18px 18px;\n}\n\n.snippet {\n margin: 0;\n padding: 10px 18px;\n border-top: 1px solid var(--border);\n background: var(--sunken);\n color: var(--muted);\n font-size: 12px;\n white-space: pre-wrap;\n}\n\n.row {\n display: flex;\n gap: 8px;\n}\n\ninput,\ntextarea {\n flex: 1;\n min-width: 0;\n padding: 7px 10px;\n border: 1px solid var(--border);\n border-radius: 6px;\n background: var(--card);\n color: var(--fg-strong);\n font: inherit;\n resize: vertical;\n}\n\ninput:focus,\ntextarea:focus {\n outline: 2px solid var(--accent);\n outline-offset: -1px;\n}\n\nbutton {\n padding: 7px 14px;\n border: none;\n border-radius: 6px;\n background: var(--accent);\n color: #fff;\n font: inherit;\n font-weight: 500;\n cursor: pointer;\n transition: background 0.15s;\n}\n\nbutton:hover:not(:disabled) {\n background: var(--accent-hover);\n}\n\nbutton:disabled {\n opacity: 0.6;\n cursor: default;\n}\n\n.output {\n min-height: 44px;\n padding: 11px 12px;\n border-radius: 6px;\n background: var(--sunken);\n word-break: break-word;\n}\n\n.output.error {\n background: var(--error-soft);\n}\n\n.output.stream {\n min-height: 96px;\n white-space: pre-wrap;\n}\n\n.caret {\n display: inline-block;\n width: 7px;\n height: 1em;\n margin-left: 2px;\n vertical-align: text-bottom;\n background: var(--accent);\n animation: blink 1s steps(1) infinite;\n}\n\n@keyframes blink {\n 50% {\n opacity: 0;\n }\n}\n\n.stats {\n display: grid;\n grid-template-columns: repeat(3, 1fr);\n gap: 8px;\n}\n\n.stats > div {\n display: flex;\n flex-direction: column;\n padding: 10px 12px;\n border-radius: 6px;\n background: var(--sunken);\n}\n\n.stats strong {\n font-size: 20px;\n color: var(--fg-strong);\n}\n\n.stats span {\n font-size: 12px;\n}\n\n.stats .tags {\n grid-column: 1 / -1;\n flex-direction: row;\n flex-wrap: wrap;\n gap: 6px;\n padding: 0;\n background: none;\n}\n\n.tag {\n padding: 2px 8px;\n border-radius: 999px;\n background: var(--accent-soft);\n color: var(--accent);\n}\n\n.jobs {\n display: flex;\n flex-direction: column;\n gap: 6px;\n margin: 0;\n padding: 0;\n list-style: none;\n}\n\n.jobs li {\n display: flex;\n align-items: center;\n gap: 10px;\n padding: 8px 10px;\n border-radius: 6px;\n background: var(--sunken);\n font-size: 13px;\n}\n\n.pill {\n min-width: 64px;\n padding: 1px 8px;\n border-radius: 999px;\n font-size: 12px;\n font-weight: 600;\n text-align: center;\n}\n\n.pill.queued {\n background: var(--warning-soft);\n color: var(--warning);\n}\n\n.pill.running {\n background: var(--accent-soft);\n color: var(--accent);\n}\n\n.pill.success {\n background: var(--success-soft);\n color: var(--success);\n}\n\n.pill.failure {\n background: var(--error-soft);\n color: var(--error);\n}\n\n.bar {\n flex: 1;\n height: 4px;\n border-radius: 999px;\n background: var(--border);\n overflow: hidden;\n}\n\n.bar-fill {\n width: 0;\n height: 100%;\n border-radius: inherit;\n background: var(--accent);\n}\n\n.bar-fill.running {\n animation: fill linear forwards;\n}\n\n.bar-fill.success {\n width: 100%;\n background: var(--success);\n}\n\n.bar-fill.failure {\n width: 100%;\n background: var(--error);\n}\n\n@keyframes fill {\n to {\n width: 95%;\n }\n}\n", + "/package.json": "{\n \"dependencies\": {\n \"react\": \"18.3.1\",\n \"react-dom\": \"18.3.1\"\n },\n \"devDependencies\": {\n \"@types/react-dom\": \"^19.0.0\",\n \"@types/react\": \"^19.0.0\"\n }\n}" + }, + "svelte5": { + "/index.ts": "\nimport { mount } from 'svelte';\nimport App from './App.svelte'\nimport './index.css'\n\nconst app = mount(App, { target: document.getElementById(\"root\")! });\n\nexport default app;\n", + "/App.svelte": "\n\n{#snippet header(step: number, title: string, description: string)}\n
\n {step}\n
\n

{title}

\n

{description}

\n
\n
\n{/snippet}\n\n
\n
\n Windmill app{ctx.workspace ? ' · ' + ctx.workspace : ''}\n

{firstName ? 'Welcome, ' + firstName : 'Welcome'}

\n

\n Turn the scripts and flows deployed in this workspace into a tool anyone can use. Edit\n App.svelte to make it yours.\n

\n
\n\n

What you can build

\n
\n {#each uses as use (use.title)}\n
\n

{use.title}

\n

{use.text}

\n
\n {/each}\n
\n

{\"Your app can only run scripts and flows already deployed in this workspace, and it runs isolated from each viewer’s session.\"}

\n\n

How it works

\n
\n {#each steps as step, i (step.title)}\n
\n {@render header(i + 1, step.title, step.description)}\n
\n {#if step.path}\n
\n {#each step.path as label, j (label)}\n {#if j > 0}→{/if}\n {label}\n {/each}\n
\n {/if}\n {step.note}\n
\n {#if step.code}\n
{step.code}
\n {/if}\n
\n {/each}\n\n
\n {@render header(steps.length + 1, 'Know who is viewing', 'Windmill passes the workspace and the viewer to the app.')}\n
\n
Workspace {ctx.workspace}
\n
\n Viewer \n {viewer ? viewer.name || viewer.username : 'Anonymous visitor'}\n
\n
\n
{contextCode}
\n
\n
\n
\n\n\n", + "/index.css": "/* The app renders inside an iframe whose host page has its own theme the\n iframe cannot read, so paint an opaque surface here: without one the host's\n background shows through and dark text lands on it unreadable. */\n:root {\n color-scheme: light;\n --bg: #f4f5f8;\n --card: #ffffff;\n --sunken: #efeff4;\n --border: #e2e3eb;\n --fg: #3d4758;\n --fg-strong: #1d2430;\n --muted: #718096;\n --accent: #5074f6;\n --accent-hover: #2c5beb;\n --accent-soft: #e8ebfb;\n --success: #15803d;\n --success-soft: #dcfce7;\n --error: #b91c1c;\n --error-soft: #fef2f2;\n --warning: #a16207;\n --warning-soft: #fef9c3;\n}\n\n@media (prefers-color-scheme: dark) {\n :root {\n color-scheme: dark;\n --bg: #242832;\n --card: #2e3441;\n --sunken: #272c35;\n --border: #3b4252;\n --fg: #d4d7dd;\n --fg-strong: #eeeff2;\n --muted: #a9b0ba;\n --accent: #7085db;\n --accent-hover: #5670d5;\n --accent-soft: #343d5c;\n --success: #86efac;\n --success-soft: #1f3a2b;\n --error: #fca5a5;\n --error-soft: #431a1a;\n --warning: #fde68a;\n --warning-soft: #3d3419;\n }\n}\n\n* {\n box-sizing: border-box;\n}\n\nbody {\n margin: 0;\n min-height: 100vh;\n background: var(--bg);\n color: var(--fg);\n font-family: Inter, ui-sans-serif, system-ui, -apple-system, 'Segoe UI', sans-serif;\n font-size: 14px;\n line-height: 1.5;\n -webkit-font-smoothing: antialiased;\n}\n\nh1,\nh2 {\n margin: 0;\n color: var(--fg-strong);\n}\n\ncode,\npre {\n font-family: ui-monospace, SFMono-Regular, Menlo, Consolas, monospace;\n}\n\n.muted {\n color: var(--muted);\n}\n\n.error {\n color: var(--error);\n}\n\n.app {\n max-width: 1040px;\n margin: 0 auto;\n padding: 40px 20px 56px;\n}\n\n.hero {\n margin-bottom: 28px;\n}\n\n.hero h1 {\n font-size: 1.75rem;\n font-weight: 650;\n letter-spacing: -0.01em;\n margin: 8px 0 6px;\n}\n\n.hero p {\n max-width: 620px;\n margin: 0;\n}\n\n.eyebrow {\n display: inline-block;\n padding: 2px 10px;\n border-radius: 999px;\n background: var(--accent-soft);\n color: var(--accent);\n font-size: 12px;\n font-weight: 600;\n}\n\n.grid {\n display: grid;\n grid-template-columns: repeat(auto-fit, minmax(min(100%, 420px), 1fr));\n gap: 16px;\n}\n\n.card {\n display: flex;\n flex-direction: column;\n background: var(--card);\n border: 1px solid var(--border);\n border-radius: 10px;\n box-shadow: 0 1px 2px rgb(16 24 40 / 0.04);\n overflow: hidden;\n}\n\n.card-header {\n display: flex;\n gap: 12px;\n padding: 18px 18px 0;\n}\n\n.card-header h2 {\n font-size: 15px;\n font-weight: 600;\n}\n\n.card-header p {\n margin: 2px 0 0;\n font-size: 13px;\n}\n\n.card-step {\n flex: none;\n display: grid;\n place-items: center;\n width: 26px;\n height: 26px;\n border-radius: 7px;\n background: var(--accent-soft);\n color: var(--accent);\n font-size: 13px;\n font-weight: 700;\n}\n\n.card-body {\n flex: 1;\n display: flex;\n flex-direction: column;\n gap: 12px;\n padding: 16px 18px 18px;\n}\n\n.snippet {\n margin: 0;\n padding: 10px 18px;\n border-top: 1px solid var(--border);\n background: var(--sunken);\n color: var(--muted);\n font-size: 12px;\n white-space: pre-wrap;\n}\n\n.row {\n display: flex;\n gap: 8px;\n}\n\ninput,\ntextarea {\n flex: 1;\n min-width: 0;\n padding: 7px 10px;\n border: 1px solid var(--border);\n border-radius: 6px;\n background: var(--card);\n color: var(--fg-strong);\n font: inherit;\n resize: vertical;\n}\n\ninput:focus,\ntextarea:focus {\n outline: 2px solid var(--accent);\n outline-offset: -1px;\n}\n\nbutton {\n padding: 7px 14px;\n border: none;\n border-radius: 6px;\n background: var(--accent);\n color: #fff;\n font: inherit;\n font-weight: 500;\n cursor: pointer;\n transition: background 0.15s;\n}\n\nbutton:hover:not(:disabled) {\n background: var(--accent-hover);\n}\n\nbutton:disabled {\n opacity: 0.6;\n cursor: default;\n}\n\n.output {\n min-height: 44px;\n padding: 11px 12px;\n border-radius: 6px;\n background: var(--sunken);\n word-break: break-word;\n}\n\n.output.error {\n background: var(--error-soft);\n}\n\n.output.stream {\n min-height: 96px;\n white-space: pre-wrap;\n}\n\n.caret {\n display: inline-block;\n width: 7px;\n height: 1em;\n margin-left: 2px;\n vertical-align: text-bottom;\n background: var(--accent);\n animation: blink 1s steps(1) infinite;\n}\n\n@keyframes blink {\n 50% {\n opacity: 0;\n }\n}\n\n.stats {\n display: grid;\n grid-template-columns: repeat(3, 1fr);\n gap: 8px;\n}\n\n.stats > div {\n display: flex;\n flex-direction: column;\n padding: 10px 12px;\n border-radius: 6px;\n background: var(--sunken);\n}\n\n.stats strong {\n font-size: 20px;\n color: var(--fg-strong);\n}\n\n.stats span {\n font-size: 12px;\n}\n\n.stats .tags {\n grid-column: 1 / -1;\n flex-direction: row;\n flex-wrap: wrap;\n gap: 6px;\n padding: 0;\n background: none;\n}\n\n.tag {\n padding: 2px 8px;\n border-radius: 999px;\n background: var(--accent-soft);\n color: var(--accent);\n}\n\n.jobs {\n display: flex;\n flex-direction: column;\n gap: 6px;\n margin: 0;\n padding: 0;\n list-style: none;\n}\n\n.jobs li {\n display: flex;\n align-items: center;\n gap: 10px;\n padding: 8px 10px;\n border-radius: 6px;\n background: var(--sunken);\n font-size: 13px;\n}\n\n.pill {\n min-width: 64px;\n padding: 1px 8px;\n border-radius: 999px;\n font-size: 12px;\n font-weight: 600;\n text-align: center;\n}\n\n.pill.queued {\n background: var(--warning-soft);\n color: var(--warning);\n}\n\n.pill.running {\n background: var(--accent-soft);\n color: var(--accent);\n}\n\n.pill.success {\n background: var(--success-soft);\n color: var(--success);\n}\n\n.pill.failure {\n background: var(--error-soft);\n color: var(--error);\n}\n\n.bar {\n flex: 1;\n height: 4px;\n border-radius: 999px;\n background: var(--border);\n overflow: hidden;\n}\n\n.bar-fill {\n width: 0;\n height: 100%;\n border-radius: inherit;\n background: var(--accent);\n}\n\n.bar-fill.running {\n animation: fill linear forwards;\n}\n\n.bar-fill.success {\n width: 100%;\n background: var(--success);\n}\n\n.bar-fill.failure {\n width: 100%;\n background: var(--error);\n}\n\n@keyframes fill {\n to {\n width: 95%;\n }\n}\n", + "/package.json": "{\n \"dependencies\": {\n \"svelte\": \"^5.56.8\"\n }\n}" + } + }, "starter_runnables": { "greet": { "name": "greet", diff --git a/backend/windmill-api/src/ai_session_tools/gen/display_words.json b/backend/windmill-api/src/ai_session_tools/gen/display_words.json new file mode 100644 index 0000000000..acce6e4065 --- /dev/null +++ b/backend/windmill-api/src/ai_session_tools/gen/display_words.json @@ -0,0 +1,99 @@ +{ + "ai": "AI", + "ai21": "AI21", + "amqp": "AMQP", + "api": "API", + "aws": "AWS", + "cms": "CMS", + "crm": "CRM", + "db": "DB", + "ftp": "FTP", + "gcp": "GCP", + "gpg": "GPG", + "hr": "HR", + "http": "HTTP", + "id": "ID", + "ifs": "IFS", + "json": "JSON", + "jwt": "JWT", + "ldap": "LDAP", + "mcp": "MCP", + "mqtt": "MQTT", + "ms": "Microsoft", + "nats": "NATS", + "oauth": "OAuth", + "odk": "ODK", + "oidc": "OIDC", + "rss": "RSS", + "s3": "S3", + "sdk": "SDK", + "smtp": "SMTP", + "sql": "SQL", + "ssh": "SSH", + "url": "URL", + "ynab": "YNAB", + "abstractapi": "AbstractAPI", + "arcgis": "ArcGIS", + "assemblyai": "AssemblyAI", + "bigquery": "BigQuery", + "chromadb": "ChromaDB", + "circleci": "CircleCI", + "clickhouse": "ClickHouse", + "clickup": "ClickUp", + "cockroachdb": "CockroachDB", + "comapeo": "CoMapeo", + "convertkit": "ConvertKit", + "currencyapi": "CurrencyAPI", + "customai": "CustomAI", + "datocms": "DatoCMS", + "dbt": "dbt", + "deepl": "DeepL", + "deepseek": "DeepSeek", + "digitalocean": "DigitalOcean", + "docspring": "DocSpring", + "docusign": "DocuSign", + "edgedb": "EdgeDB", + "faunadb": "FaunaDB", + "gcloud": "Google Cloud", + "ghostcms": "Ghost CMS", + "gitbook": "GitBook", + "github": "GitHub", + "gitlab": "GitLab", + "googleai": "GoogleAI", + "graphql": "GraphQL", + "groqai": "GroqAI", + "hubspot": "HubSpot", + "ipinfo": "IPinfo", + "kobotoolbox": "KoboToolbox", + "leonardoai": "LeonardoAI", + "linkedin": "LinkedIn", + "lumaai": "LumaAI", + "mailerlite": "MailerLite", + "mongodb": "MongoDB", + "mysql": "MySQL", + "netbox": "NetBox", + "netsuite": "NetSuite", + "newsapi": "NewsAPI", + "neondb": "NeonDB", + "nocodb": "NocoDB", + "openai": "OpenAI", + "openrouter": "OpenRouter", + "oracledb": "OracleDB", + "pagerduty": "PagerDuty", + "pandadoc": "PandaDoc", + "paypal": "PayPal", + "planetscale": "PlanetScale", + "postgresql": "PostgreSQL", + "readme": "ReadMe", + "rest": "REST", + "quickbooks": "QuickBooks", + "sendgrid": "SendGrid", + "servicenow": "ServiceNow", + "signoz": "SigNoz", + "surrealdb": "SurrealDB", + "togetherai": "TogetherAI", + "webscrapingai": "WebScrapingAI", + "weatherapi": "WeatherAPI", + "whatsapp": "WhatsApp", + "woocommerce": "WooCommerce" +} diff --git a/backend/windmill-api/src/mcp/session/gen/instructions.json b/backend/windmill-api/src/ai_session_tools/gen/instructions.json similarity index 100% rename from backend/windmill-api/src/mcp/session/gen/instructions.json rename to backend/windmill-api/src/ai_session_tools/gen/instructions.json diff --git a/backend/windmill-api/src/mcp/session/gen/prompt.json b/backend/windmill-api/src/ai_session_tools/gen/prompt.json similarity index 100% rename from backend/windmill-api/src/mcp/session/gen/prompt.json rename to backend/windmill-api/src/ai_session_tools/gen/prompt.json diff --git a/backend/windmill-api/src/mcp/session/gen/schemas.json b/backend/windmill-api/src/ai_session_tools/gen/schemas.json similarity index 100% rename from backend/windmill-api/src/mcp/session/gen/schemas.json rename to backend/windmill-api/src/ai_session_tools/gen/schemas.json diff --git a/backend/windmill-api/src/mcp/session/gen/tools.json b/backend/windmill-api/src/ai_session_tools/gen/tools.json similarity index 100% rename from backend/windmill-api/src/mcp/session/gen/tools.json rename to backend/windmill-api/src/ai_session_tools/gen/tools.json diff --git a/backend/windmill-api/src/mcp/session/guidance.rs b/backend/windmill-api/src/ai_session_tools/guidance.rs similarity index 79% rename from backend/windmill-api/src/mcp/session/guidance.rs rename to backend/windmill-api/src/ai_session_tools/guidance.rs index 385aaa5c2a..669b02943e 100644 --- a/backend/windmill-api/src/mcp/session/guidance.rs +++ b/backend/windmill-api/src/ai_session_tools/guidance.rs @@ -6,7 +6,8 @@ use std::sync::LazyLock; use serde::Deserialize; use serde_json::Value; -use super::{opt_str_arg, str_arg, Ctx, ToolResult}; +use super::workspace::api_message; +use super::{opt_str_arg, str_arg, Ctx, Surface, ToolResult}; pub(super) const TOOLS: &[&str] = &["get_instructions", "read_skill"]; @@ -57,6 +58,8 @@ const TRUNCATED: &str = "… [truncated]"; pub(super) struct Skill { pub path: String, + /// Listed in the MCP instructions; the chat builds its own prompt. + #[cfg_attr(not(feature = "mcp"), allow(dead_code))] pub description: String, } @@ -103,11 +106,19 @@ async fn read_skill(ctx: &Ctx<'_>, args: &Value) -> ToolResult { // reads any resource holding a string `content`. let listed = match list_skills(ctx).await { Ok(skills) => skills.iter().any(|skill| skill.path == path), - Err(e) => return Ok(read_failed(path, &e)), + Err(e) => return Ok(read_failed(ctx, path, api_message(&e))), }; - if !listed { + // The chat's user can switch a skill off for that chat, a setting their browser holds. + let switched_off = ctx + .chat + .is_some_and(|chat| !chat.skill_enabled.unwrap_or(true)); + if !listed || switched_off { + let (available, listing) = match ctx.surface { + Surface::Chat => ("available to this chat", "the system prompt"), + Surface::Mcp => ("available here", "the instructions"), + }; return Ok(format!( - "\"{path}\" is not one of the skills available here. Only the paths listed under \"Skills\" in the instructions can be read." + "\"{path}\" is not one of the skills {available}. Only the paths listed under \"Skills\" in {listing} can be read." )); } let value = match ctx @@ -115,10 +126,11 @@ async fn read_skill(ctx: &Ctx<'_>, args: &Value) -> ToolResult { .await { Ok(value) => value, - Err(e) => return Ok(read_failed(path, &e)), + Err(e) => return Ok(read_failed(ctx, path, api_message(&e))), }; let Some(body) = value.get("content").and_then(Value::as_str) else { return Ok(read_failed( + ctx, path, &format!("resource {path} has no string \"content\" — is it an {SKILL_RESOURCE_TYPE}?"), )); @@ -135,8 +147,12 @@ async fn read_skill(ctx: &Ctx<'_>, args: &Value) -> ToolResult { Ok(format!("Skill: {path}\n\nInstructions:\n{body}")) } -fn read_failed(path: &str, message: &str) -> String { +fn read_failed(ctx: &Ctx<'_>, path: &str, message: &str) -> String { + let listing = match ctx.surface { + Surface::Chat => "the system prompt", + Surface::Mcp => "the instructions", + }; format!( - "Failed to read skill \"{path}\": {message}. Check the path against the Skills list in the instructions." + "Failed to read skill \"{path}\": {message}. Check the path against the Skills list in {listing}." ) } diff --git a/backend/windmill-api/src/mcp/session/instructions.rs b/backend/windmill-api/src/ai_session_tools/instructions.rs similarity index 96% rename from backend/windmill-api/src/mcp/session/instructions.rs rename to backend/windmill-api/src/ai_session_tools/instructions.rs index f5c15c6e17..9b12293506 100644 --- a/backend/windmill-api/src/mcp/session/instructions.rs +++ b/backend/windmill-api/src/ai_session_tools/instructions.rs @@ -121,7 +121,7 @@ async fn appendix(ctx: &Ctx<'_>) -> String { } /// The whole prompt, for a connection that reaches one workspace. -pub(in crate::mcp) async fn session_instructions(ctx: &Ctx<'_>) -> String { +pub(crate) async fn session_instructions(ctx: &Ctx<'_>) -> String { let mut prompt = PARTS .base .replace("{{folder_guidance}}", &folder_guidance(ctx.auth)) @@ -132,7 +132,7 @@ pub(in crate::mcp) async fn session_instructions(ctx: &Ctx<'_>) -> String { /// The prompt without anything specific to a workspace, for a connection that reaches /// several: usernames are per workspace, so paths read `u//...`. -pub(in crate::mcp) fn generic_session_instructions() -> String { +pub(crate) fn generic_session_instructions() -> String { PARTS .base .replace("{{folder_guidance}}", "") @@ -140,7 +140,7 @@ pub(in crate::mcp) fn generic_session_instructions() -> String { } /// What `generic_session_instructions` leaves out, for one workspace. -pub(in crate::mcp) async fn workspace_context(ctx: &Ctx<'_>) -> String { +pub(crate) async fn workspace_context(ctx: &Ctx<'_>) -> String { let username = ctx.auth.username(); format!( "Workspace `{}`. Your username here is \"{username}\", so your personal paths are \ diff --git a/backend/windmill-api/src/mcp/session/items.rs b/backend/windmill-api/src/ai_session_tools/items.rs similarity index 99% rename from backend/windmill-api/src/mcp/session/items.rs rename to backend/windmill-api/src/ai_session_tools/items.rs index c075586a40..7ca0d7f8e3 100644 --- a/backend/windmill-api/src/mcp/session/items.rs +++ b/backend/windmill-api/src/ai_session_tools/items.rs @@ -44,7 +44,7 @@ static SCHEDULE_SCHEMA: LazyLock = LazyLock::new(|| { let mut schema = tool_schema("write_schedule"); let advanced: Value = serde_json::from_str(&SCHEMAS.schedule).unwrap_or_default(); if let Some(properties) = schema.get_mut("properties").and_then(Value::as_object_mut) { - properties.remove("advanced"); + properties.shift_remove("advanced"); if let Some(advanced) = advanced.get("properties").and_then(Value::as_object) { properties.extend(advanced.clone()); } @@ -282,7 +282,7 @@ fn into_object(value: Option) -> Map { fn merge_config(base: Option, config: Value, path: &str) -> Map { let mut draft = into_object(base); for key in ["workspace_id", "edited_by", "edited_at", "email", "error"] { - draft.remove(key); + draft.shift_remove(key); } draft.extend(into_object(Some(config))); draft.insert("path".into(), json!(path)); @@ -454,6 +454,7 @@ async fn write_draft( from_deployed: impl FnOnce(Value) -> Value, build: impl FnOnce(Option) -> Result, ) -> ToolResult { + ctx.saving_draft(item_type, path); let base = write_base(ctx, item_type, path, trigger_kind).await?; let existed = base.base.is_some(); let from_draft = base.from_draft; @@ -480,7 +481,7 @@ async fn write_draft( fn take_override(parsed: &mut Value) -> bool { parsed .as_object_mut() - .and_then(|o| o.remove("override")) + .and_then(|o| o.shift_remove("override")) .and_then(|v| v.as_bool()) .unwrap_or(false) } diff --git a/backend/windmill-api/src/mcp/session/misc.rs b/backend/windmill-api/src/ai_session_tools/misc.rs similarity index 71% rename from backend/windmill-api/src/mcp/session/misc.rs rename to backend/windmill-api/src/ai_session_tools/misc.rs index d613b89f92..f45eb93d6d 100644 --- a/backend/windmill-api/src/mcp/session/misc.rs +++ b/backend/windmill-api/src/ai_session_tools/misc.rs @@ -1,13 +1,17 @@ //! Lookups that touch no draft: hub, docs, npm, resource types, workers, folders. -use std::collections::HashSet; +use std::collections::{HashMap, HashSet}; +use std::sync::LazyLock; use futures::future::join_all; use serde_json::{json, Map, Value}; use windmill_common::utils::HTTP_CLIENT; +use super::chat::Effect; +use super::workspace::api_message; use super::{ - array, enc, is_not_found, opt_str_arg, pretty, str_arg, truthy, Ctx, ToolResult, SCHEMAS, + array, enc, is_not_found, js_head, js_len, opt_str_arg, pretty, str_arg, truthy, Ctx, + ToolResult, SCHEMAS, }; pub(super) const TOOLS: &[&str] = &[ @@ -28,7 +32,7 @@ pub(super) async fn call(ctx: &Ctx<'_>, name: &str, args: &Value) -> ToolResult match name { "search_hub_scripts" => search_hub_scripts(ctx, args).await, "get_hub_integration" => get_hub_integration(ctx, args).await, - "search_npm_packages" => search_npm_packages(args).await, + "search_npm_packages" => search_npm_packages(ctx, args).await, "search_docs" => search_docs(ctx, args).await, "read_docs_page" => read_docs_page(ctx, args).await, "search_resource_types" => search_resource_types(ctx, args).await, @@ -90,12 +94,37 @@ async fn hub_integrations(ctx: &Ctx<'_>) -> Vec { } } +/// The frontend's `RESOURCE_TYPE_WORDS`: how a slug word reads where capitalizing gets it +/// wrong, some into several words (`gcloud` -> `Google Cloud`). +static DISPLAY_WORDS: LazyLock> = LazyLock::new(|| { + serde_json::from_str(include_str!("gen/display_words.json")) + .expect("gen/display_words.json is valid") +}); + +/// What the frontend's `integrationDisplayName` calls a slug the hub names nothing. +fn slug_display_name(slug: &str) -> String { + slug.split(['_', '-']) + .filter(|w| !w.is_empty()) + .map(|w| { + DISPLAY_WORDS + .get(&w.to_ascii_lowercase()) + .cloned() + .unwrap_or_else(|| w.to_string()) + }) + .collect::>() + .join(" ") +} + /// A token matches a whole word of the slug or display name, or extends one from four /// characters on: a bare substring test makes `for` hit sales*for*ce. fn token_matches_integration(token: &str, integration: &Value) -> bool { let name = integration.get("name").and_then(Value::as_str); - let display_name = integration.get("display_name").and_then(Value::as_str); - let words: Vec = [name, display_name] + let display_name = integration + .get("display_name") + .and_then(Value::as_str) + .map(str::to_string) + .or_else(|| name.map(slug_display_name)); + let words: Vec = [name, display_name.as_deref()] .into_iter() .flatten() .flat_map(words) @@ -254,8 +283,9 @@ async fn get_hub_integration(ctx: &Ctx<'_>, args: &Value) -> ToolResult { Ok(doc) => doc, Err(e) => { // Only a 404 means the integration is absent. Reporting a timeout the same - // way would teach the model that a real integration does not exist. - let error = if is_not_found(&e) { + // way would teach the model that a real integration does not exist. No slug holds + // a `/`, and the route refuses one as malformed rather than absent. + let error = if is_not_found(&e) || integration.contains('/') { format!("No hub metadata for \"{integration}\".") } else { format!("Could not reach the hub for \"{integration}\"; it may still exist. Read its scripts instead, or try again.") @@ -347,49 +377,183 @@ async fn get_hub_integration(ctx: &Ctx<'_>, args: &Value) -> ToolResult { // npm // --------------------------------------------------------------------------- -const NPM_SCORE_THRESHOLD: f64 = 1000.0; -/// The session caps a package's readme and types at 1% of its model's context window -/// each. The caller's model is unknown here, so this is 1% of the window the session -/// assumes for an unknown model (128000). +/// The registry's best match is kept, and the runner-up only when it scores this share of it: +/// the scores have no fixed scale (a descriptive query's right answer can score a few hundred, +/// a package's own name thousands), so a match is judged against the best one. +const NPM_RUNNER_UP_SHARE: f64 = 0.6; +/// A package's readme and types are each capped at 1% of the model's context window. An MCP +/// client's model is unknown, so this is 1% of the window assumed for an unknown model (128000). const NPM_TEXT_LIMIT: usize = 1280; -fn take_chars(text: &str, limit: usize) -> String { - text.chars().take(limit).collect() -} - async fn fetch(url: &str) -> Result { HTTP_CLIENT.get(url).send().await } -/// The package's declared entry declaration file. The session walks every `.d.ts` of the -/// package in the browser and keeps the same number of characters of the result. -async fn npm_package_types(name: &str, version: &str) -> Option { - let base = format!("https://unpkg.com/{name}@{version}"); - let manifest: Value = fetch(&format!("{base}/package.json")) - .await - .ok()? - .error_for_status() - .ok()? - .json() - .await - .ok()?; - let entry = ["types", "typings"] - .iter() - .find_map(|key| manifest.get(key)?.as_str()) - .unwrap_or("index.d.ts") - .trim_start_matches("./"); - let types = fetch(&format!("{base}/{entry}")) - .await - .ok()? - .error_for_status() - .ok()? - .text() - .await - .ok()?; - Some(format!("// /node_modules/{name}/{entry}\n{types}")) +/// Where a package's files come from: the public CDN, or the workspace's npm proxy, which +/// is the authoritative source once the instance names a registry of its own. Each falls +/// back to the other. +struct NpmSources<'a> { + ctx: &'a Ctx<'a>, + proxy_first: bool, } -async fn npm_search(query: &str) -> Result, String> { +impl NpmSources<'_> { + async fn new<'a>(ctx: &'a Ctx<'a>) -> NpmSources<'a> { + let proxy_first = ctx + .get(&ctx.ws("npm_proxy/config")) + .await + .is_ok_and(|config| truthy(config.get("registry_configured"))); + NpmSources { ctx, proxy_first } + } + + async fn json(&self, cdn: String, proxy: String) -> Option { + let via_cdn = || async { + let response = fetch(&cdn).await.ok()?.error_for_status().ok()?; + response.json::().await.ok() + }; + let via_proxy = || async { self.ctx.get(&self.ctx.ws(&proxy)).await.ok() }; + if self.proxy_first { + match via_proxy().await { + Some(v) => Some(v), + None => via_cdn().await, + } + } else { + match via_cdn().await { + Some(v) => Some(v), + None => via_proxy().await, + } + } + } + + async fn text(&self, cdn: String, proxy: String) -> Option { + let via_cdn = || async { + let response = fetch(&cdn).await.ok()?.error_for_status().ok()?; + response.text().await.ok() + }; + let via_proxy = || async { + self.ctx + .request("GET", &self.ctx.ws(&proxy), None) + .await + .ok() + }; + if self.proxy_first { + match via_proxy().await { + Some(v) => Some(v), + None => via_cdn().await, + } + } else { + match via_cdn().await { + Some(v) => Some(v), + None => via_proxy().await, + } + } + } + + /// `tag` as an exact version; one with two dots already is one. + async fn version(&self, module: &str, tag: &str) -> Option { + if tag.split('.').count() >= 3 { + return Some(tag.to_string()); + } + let resolved = self + .json( + format!("https://data.jsdelivr.com/v1/package/resolve/npm/{module}@{tag}"), + format!("npm_proxy/resolve/{}?tag={}", enc(module), enc(tag)), + ) + .await?; + resolved.get("version")?.as_str().map(String::from) + } + + /// The `.d.ts` files of `module` at `version`, in the order its file tree lists them. + async fn dts_files(&self, module: &str, version: &str) -> Option> { + let tree = self + .json( + format!("https://data.jsdelivr.com/v1/package/npm/{module}@{version}/flat"), + format!("npm_proxy/filetree/{}/{}", enc(module), enc(version)), + ) + .await?; + Some( + array(tree.get("files")) + .iter() + .filter_map(|f| f.get("name")?.as_str()) + .filter(|name| name.ends_with(".d.ts")) + .map(String::from) + .collect(), + ) + } +} + +/// `@bla/foo` → `bla__foo`, the DefinitelyTyped name of a scoped package. +fn dt_name(name: &str) -> String { + match name.strip_prefix('@') { + Some(scoped) if scoped.contains('/') => scoped.replacen('/', "__", 1), + _ => name.to_string(), + } +} + +/// Every declaration file of the package, or of its `@types` package when it ships none, +/// each headed by the path it would have under `node_modules`. +/// Files fetched at once, as the browser's type acquisition did. +const NPM_TYPES_CONCURRENCY: usize = 6; + +/// The package's declarations joined in order, fetched only until they reach `limit` +/// characters: only that head is returned, and `@types/lodash` alone has hundreds of files. +async fn npm_package_types( + sources: &NpmSources<'_>, + name: &str, + version: &str, + limit: usize, +) -> String { + let mut module = name.to_string(); + let mut prefix = format!("/node_modules/{name}"); + let mut version = match sources.version(name, version).await { + Some(version) => version, + None => return String::new(), + }; + let mut files = sources.dts_files(name, &version).await.unwrap_or_default(); + if files.is_empty() { + module = format!("@types/{}", dt_name(name)); + prefix = format!("/node_modules/@types/{}", dt_name(name)); + let Some(dt_version) = sources.version(&module, "latest").await else { + return String::new(); + }; + version = dt_version; + files = sources + .dts_files(&module, &version) + .await + .unwrap_or_default(); + } + let mut parts: Vec = Vec::new(); + for batch in files.chunks(NPM_TYPES_CONCURRENCY) { + let contents = join_all(batch.iter().map(|file| { + sources.text( + format!("https://cdn.jsdelivr.net/npm/{module}@{version}{file}"), + format!("npm_proxy/file/{}/{}{file}", enc(&module), enc(&version)), + ) + })) + .await; + parts.extend( + batch + .iter() + .zip(contents) + .filter_map(|(file, code)| Some(format!("// {prefix}{file}\n{}", code?))), + ); + if js_len(&parts.join("\n\n")) >= limit { + break; + } + } + parts.join("\n\n") +} + +fn confident_npm_matches(objects: &[Value]) -> Vec<&Value> { + let score = |o: &Value| o.get("searchScore").and_then(Value::as_f64).unwrap_or(0.0); + let best = objects.iter().map(score).fold(0.0, f64::max); + objects + .iter() + .filter(|o| score(o) >= best * NPM_RUNNER_UP_SHARE) + .collect() +} + +async fn npm_search(ctx: &Ctx<'_>, query: &str, limit: usize) -> Result, String> { let data: Value = fetch(&format!( "https://registry.npmjs.org/-/v1/search?text={}&size=2", enc(query) @@ -399,41 +563,36 @@ async fn npm_search(query: &str) -> Result, String> { .json() .await .map_err(|e| e.to_string())?; - let packages = data + let objects = data .get("objects") .and_then(Value::as_array) - .ok_or("no `objects` in the registry answer")? - .iter() - .filter(|o| { - o.get("searchScore").and_then(Value::as_f64).unwrap_or(0.0) >= NPM_SCORE_THRESHOLD - }) - .filter_map(|o| { - let package = o.get("package")?; - Some(( - package.get("name")?.as_str()?, - package.get("version").and_then(Value::as_str), - )) - }); + .ok_or("no `objects` in the registry answer")?; + let packages = confident_npm_matches(objects).into_iter().filter_map(|o| { + let package = o.get("package")?; + Some(( + package.get("name")?.as_str()?, + package.get("version").and_then(Value::as_str), + )) + }); + let sources = &NpmSources::new(ctx).await; Ok(join_all(packages.map(|(name, version)| async move { let documentation = match fetch(&format!("https://unpkg.com/{name}/readme.md")).await { Ok(response) => response.text().await.unwrap_or_default(), Err(_) => String::new(), }; - let types = npm_package_types(name, version.unwrap_or("latest")) - .await - .unwrap_or_default(); + let types = npm_package_types(sources, name, version.unwrap_or("latest"), limit).await; json!({ "package": name, - "documentation": take_chars(&documentation, NPM_TEXT_LIMIT), - "types": take_chars(&types, NPM_TEXT_LIMIT), + "documentation": js_head(&documentation, limit), + "types": js_head(&types, limit), }) })) .await) } -async fn search_npm_packages(args: &Value) -> ToolResult { +async fn search_npm_packages(ctx: &Ctx<'_>, args: &Value) -> ToolResult { let query = str_arg(args, "query")?; - Ok(match npm_search(query).await { + Ok(match npm_search(ctx, query, NPM_TEXT_LIMIT).await { Ok(results) => Value::Array(results).to_string(), Err(e) => { tracing::warn!("npm package search failed: {e}"); @@ -454,7 +613,10 @@ fn docs_text(result: Result, action: &str) -> String { .and_then(Value::as_str) .unwrap_or_default() .to_string(), - Err(e) => format!("Failed to {action}: {e}, pursuing with the user request..."), + Err(e) => format!( + "Failed to {action}: {}, pursuing with the user request...", + api_message(&e) + ), } } @@ -466,6 +628,14 @@ async fn search_docs(ctx: &Ctx<'_>, args: &Value) -> ToolResult { ); } let result = ctx.get(&format!("/docs/search?query={}", enc(query))).await; + if let Ok(response) = &result { + let mut facts = Map::new(); + facts.insert( + "result_count".into(), + array(response.get("results")).len().into(), + ); + ctx.effect(Effect::Facts(facts)); + } Ok(docs_text(result, "search documentation")) } @@ -661,7 +831,7 @@ async fn create_folder(ctx: &Ctx<'_>, args: &Value) -> ToolResult { "success": true, "message": format!("Created folder `f/{name}`. You can now write items to `f/{name}/`.") }), - Err(error) => json!({ "success": false, "error": error }), + Err(error) => json!({ "success": false, "error": api_message(&error) }), } .to_string(), ) @@ -671,6 +841,25 @@ async fn create_folder(ctx: &Ctx<'_>, args: &Value) -> ToolResult { mod tests { use super::*; + #[test] + fn npm_matches_are_judged_against_the_best_one() { + let names = |scores: &[f64]| { + let objects: Vec = scores + .iter() + .enumerate() + .map(|(i, s)| json!({ "searchScore": s, "package": { "name": i.to_string() } })) + .collect(); + confident_npm_matches(&objects) + .iter() + .map(|o| o["package"]["name"].as_str().unwrap().to_string()) + .collect::>() + }; + // A descriptive query's right answer scores a few hundred; a weak runner-up is dropped. + assert_eq!(names(&[815.0, 454.0]), ["0"]); + assert_eq!(names(&[419.0, 384.0]), ["0", "1"]); + assert!(names(&[]).is_empty()); + } + #[test] fn suggestions_rank_by_matched_words() { let integrations = vec![ @@ -690,6 +879,12 @@ mod tests { ["salesforce"] ); assert_eq!(suggest_hub_integrations(&integrations, "s3 bucket"), ["s3"]); + // With no display name from the hub, the slug reads as the frontend labels it. + let unnamed = vec![json!({ "name": "gcloud", "display_name": null })]; + assert_eq!( + suggest_hub_integrations(&unnamed, "deploy to google cloud"), + ["gcloud"] + ); } #[test] diff --git a/backend/windmill-api/src/mcp/session/mod.rs b/backend/windmill-api/src/ai_session_tools/mod.rs similarity index 72% rename from backend/windmill-api/src/mcp/session/mod.rs rename to backend/windmill-api/src/ai_session_tools/mod.rs index 6992c46221..6928545039 100644 --- a/backend/windmill-api/src/mcp/session/mod.rs +++ b/backend/windmill-api/src/ai_session_tools/mod.rs @@ -1,30 +1,46 @@ -//! The Windmill MCP server's toolset (`/api/mcp`, tokens with `mcp:all`): an AI session's toolset, for a client -//! outside the Windmill UI. +//! The AI session's GLOBAL toolset, run on the server for both of its callers: the +//! in-app chat (`POST /w/{workspace}/ai_session_tools/{name}`) and the Windmill MCP +//! server (`/api/mcp`, tokens with `mcp:all`). //! //! Everything a model reads here — tool names, descriptions, input schemas, the //! instructions, the `get_instructions` texts — is generated from the session's own //! definitions into `gen/` (`npm run gen:mcp-session` in `frontend/`). What a tool -//! *does* is written here, mirroring its session counterpart in -//! `frontend/src/lib/components/copilot/chat/global/core.ts`: a change to one side's -//! behaviour belongs on the other. +//! *does* is written here once; the chat keeps only what needs a browser (cards, +//! confirmations, the run form, the open editor). //! -//! A tool acts through the API as the caller, the way the session's does, so every -//! permission check is the API's own. It calls the server's router in-process rather -//! than over a socket. +//! A tool acts through the API as the caller, so every permission check is the API's +//! own. It calls the server's router in-process rather than over a socket. use std::collections::HashMap; use std::sync::LazyLock; -use serde::Deserialize; -use serde_json::{json, Map, Value}; -use windmill_mcp::server::{ - BackendResult, CallToolResult, ContentBlock, ErrorData, MetaObject, SessionAccess, Tool, - ToolAnnotations, -}; +use serde::{Deserialize, Serialize}; +use serde_json::{Map, Value}; +use windmill_common::auth::create_jwt_token; use crate::db::ApiAuthed; -use super::utils::{mint_caller_jwt, send_as_caller}; +pub(crate) mod chat; + +/// What a caller may do with the toolset. An MCP token sets it from its scopes; the chat +/// is always `Full`, its own gating running before a tool is called. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +#[cfg_attr(not(feature = "mcp"), allow(dead_code))] +pub(crate) enum SessionAccess { + ReadOnly, + ReadAndRun, + Full, +} + +/// Which caller a tool runs for, at the few spots where the two genuinely differ. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub(crate) enum Surface { + Mcp, + Chat, +} + +/// The largest tool result the MCP runner passes on whole; it truncates beyond it. +pub(crate) const MAX_TOOL_RESULT_BYTES: usize = 87_500; /// Each running server's fully layered router, by its `base_internal_url`. Keyed rather /// than global because the backend tests run one server per test in a single process, @@ -42,11 +58,13 @@ pub(crate) fn register_router(base_internal_url: &str, router: Option, read_only: bool, @@ -108,6 +128,7 @@ const EDITOR_ONLY: &[&str] = &[ /// live datatable, run code, cancel a job, delete work, or replace a draft's content, which /// discarding the draft does not restore. Stands in for the session's confirmation card. The /// ChatGPT and Claude directories reject such a tool without this hint. +#[cfg(any(test, feature = "mcp"))] const DESTRUCTIVE: &[&str] = &[ "write_script", "edit_script", @@ -144,15 +165,11 @@ const DESTRUCTIVE: &[&str] = &[ "delete_app_runnable", ]; -/// The destructive tools a card may call, each from a button the user presses. Every -/// other destructive tool is hidden from cards: a card acts as the user with no one -/// confirming. -const CARD_ACTIONS: &[&str] = &["deploy_workspace_item", "cancel_job"]; - /// Tools that reach a system outside Windmill on the workspace's behalf: running code can /// reach any system the workspace can, and `get_db_schema` connects to a resource's /// database, and `search_npm_packages` queries the npm registry, which Windmill does not /// control. Reading Windmill's own hub and docs does not count. +#[cfg(any(test, feature = "mcp"))] const OPEN_WORLD: &[&str] = &[ "search_npm_packages", "run_script", @@ -166,23 +183,6 @@ const OPEN_WORLD: &[&str] = &[ "get_db_schema", ]; -/// A tool's display name, which Claude shows in its tool list and approval prompts. -fn tool_title(name: &str) -> String { - let words: Vec = name - .split('_') - .map(|word| match word { - "db" | "sql" | "json" => word.to_uppercase(), - _ => word.to_string(), - }) - .collect(); - let title = words.join(" "); - let mut chars = title.chars(); - chars - .next() - .map(|first| first.to_uppercase().chain(chars).collect()) - .unwrap_or_default() -} - /// Tools the session counts as reads that still start a job. A read-only token runs /// nothing on the tools server, so it runs nothing here either. const RUNS_A_JOB: &[&str] = &["get_db_schema"]; @@ -202,6 +202,7 @@ fn allowed(def: &ToolDef, access: SessionAccess) -> bool { /// Each group's tools and its dispatcher, tried in order. macro_rules! groups { ($($group:ident),*) => { + #[cfg(any(test, feature = "mcp"))] fn implemented() -> impl Iterator { std::iter::empty()$(.chain($group::TOOLS.iter().copied()))* } @@ -216,92 +217,53 @@ macro_rules! groups { } groups!(guidance, workspace, items, misc, flows, apps, runs, deploy, data, settings); -/// Claude clients keep an MCP result in context only up to a token budget well under the -/// authoring references `get_instructions` returns, unless the tool raises it. Above the -/// runner's own cap on a result, so that cap is the one that applies. -fn result_size_meta() -> MetaObject { - let mut meta = Map::new(); - meta.insert("anthropic/maxResultSizeChars".to_string(), 100_000.into()); - MetaObject(meta) +/// A tool as a client lists it. +#[cfg(feature = "mcp")] +pub(crate) struct ToolInfo { + pub name: &'static str, + pub description: &'static str, + pub parameters: &'static Map, + pub read_only: bool, + pub destructive: bool, + pub open_world: bool, } -/// The MCP Apps view a client that renders them draws these tools' results in. It draws -/// what the tool put in its result's `_meta` (`Ctx::view`), and acts by calling tools. -/// Hosts cache a view by its URI: bump the version when a change breaks the `_meta` shape -/// the deployed card reads. -pub(super) const CARD_URI: &str = "ui://windmill/card-v1"; -const CARD_HTML: &str = include_str!("card.html"); - -/// The `ui://` views the session's tools render in, as `(uri, html)`. -pub(super) fn session_views() -> &'static [(&'static str, &'static str)] { - &[(CARD_URI, CARD_HTML)] -} - -pub(super) fn session_tools(access: SessionAccess) -> Vec { +/// The tools implemented here that `access` may call. +#[cfg(feature = "mcp")] +pub(crate) fn session_tools(access: SessionAccess) -> impl Iterator { TOOL_DEFS .iter() .filter(|def| implemented().any(|name| name == def.name)) - .filter(|def| allowed(def, access)) + .filter(move |def| allowed(def, access)) .map(|def| { let name = def.name.as_str(); - let mut meta = result_size_meta(); - let destructive = DESTRUCTIVE.contains(&name); - if destructive && !CARD_ACTIONS.contains(&name) { - meta.0 - .insert("ui".into(), json!({ "visibility": ["model"] })); + ToolInfo { + name, + description: &def.description, + parameters: &def.parameters, + read_only: def.read_only && !RUNS_A_JOB.contains(&name), + destructive: DESTRUCTIVE.contains(&name), + open_world: OPEN_WORLD.contains(&name), } - let title = tool_title(name); - Tool::new( - def.name.clone(), - def.description.clone(), - def.parameters.clone(), - ) - .with_title(title.clone()) - .with_annotations( - ToolAnnotations::with_title(title) - .read_only(def.read_only && !RUNS_A_JOB.contains(&name)) - .destructive(destructive) - .open_world(OPEN_WORLD.contains(&name)), - ) - .with_meta(meta) }) - .chain( - views::definitions() - .into_iter() - .filter(|(name, ..)| views::allowed(name, access)) - .map(|(name, description, schema)| { - let mut meta = result_size_meta(); - meta.0 - .insert("ui".into(), json!({ "resourceUri": CARD_URI })); - let Value::Object(schema) = schema else { - unreachable!("a view tool's schema is an object") - }; - let title = tool_title(name); - Tool::new(name, description, schema) - .with_title(title.clone()) - .with_annotations( - ToolAnnotations::with_title(title) - .read_only(true) - .destructive(false) - .open_world(false), - ) - .with_meta(meta) - }), - ) - .collect() } /// What a tool hands back: the text the model reads, or the message of a failed call. pub(super) type ToolResult = Result; /// The caller, and the API reached as them. -pub(super) struct Ctx<'a> { +pub(crate) struct Ctx<'a> { pub auth: &'a ApiAuthed, pub w_id: &'a str, pub base_internal_url: &'a str, pub access: SessionAccess, /// The scheme and host the MCP client reached, when known. pub origin: Option<&'a str>, + pub surface: Surface, + /// What the chat sends beside a call; `None` for MCP. + pub chat: Option<&'a chat::ChatContext>, + /// What the call changed that the chat has to apply: its editors, cards and drafts. + pub effects: std::sync::Mutex>, /// Minted once per tool call: every JWT the API sees is cached by the auth layer, and a /// tool can make dozens of calls, which would evict other users' cached sessions. pub jwt: tokio::sync::OnceCell, @@ -310,6 +272,38 @@ pub(super) struct Ctx<'a> { pub view: std::sync::Mutex>, } +impl<'a> Ctx<'a> { + pub(crate) fn new( + auth: &'a ApiAuthed, + w_id: &'a str, + base_internal_url: &'a str, + access: SessionAccess, + ) -> Self { + Ctx { + auth, + w_id, + base_internal_url, + access, + origin: None, + surface: Surface::Mcp, + chat: None, + effects: Default::default(), + jwt: Default::default(), + view: Default::default(), + } + } + + /// Hand the chat something to apply. A no-op for MCP, which has no editor. + pub(super) fn effect(&self, effect: chat::Effect) { + if self.surface == Surface::Chat { + self.effects.lock().unwrap().push(effect); + } + } +} + +/// The total timeout of the HTTP client the in-process router replaces. +const REQUEST_TIMEOUT: std::time::Duration = std::time::Duration::from_secs(30); + impl Ctx<'_> { /// Set what the card draws of this call, replacing what an inner step set. pub fn set_view(&self, key: &'static str, value: Value) { @@ -318,6 +312,20 @@ impl Ctx<'_> { /// Call the API as the caller. `path` follows `/api`; a non-2xx answer is the error. pub async fn request(&self, method: &str, path: &str, body: Option) -> ToolResult { + self.request_within(method, path, body, REQUEST_TIMEOUT) + .await + } + + /// `request` for a route that waits on jobs of its own, like the datatable migration + /// routes: each job is bounded server-side, so the default budget would cut a long one + /// off midway. + pub async fn request_within( + &self, + method: &str, + path: &str, + body: Option, + timeout: std::time::Duration, + ) -> ToolResult { let url = guarded_url(self.base_internal_url, self.w_id, method, path, self.access)?; // No route scope: this server only admits `mcp:all`, several routes (hub, embeddings) // map to none, and `guarded_url` is what keeps a call on its route. The tag fence @@ -337,17 +345,18 @@ impl Ctx<'_> { .get(self.base_internal_url) .cloned(); let (status, text) = match router { - Some(router) => self.call_router(router, method, &url, body, scopes).await?, - None => { - let response = send_as_caller(method, &url, self.w_id, self.auth, body, scopes) - .await - .map_err(|e| e.message.to_string())?; - let status = response.status(); - (status, response.text().await.map_err(|e| e.to_string())?) + Some(router) => { + self.call_router(router, method, &url, body, scopes, timeout) + .await? } + None => self.call_http(method, &url, body, scopes, timeout).await?, }; if status.is_success() { Ok(text) + } else if text.trim().is_empty() { + // What a browser's request reports for an answer with no body. + let reason = status.canonical_reason().unwrap_or_default(); + Err(format!("{}: {reason}", status.as_u16())) } else { Err(format!("{}: {}", status.as_u16(), text)) } @@ -362,6 +371,7 @@ impl Ctx<'_> { url: &str, body: Option, scopes: Option>, + timeout: std::time::Duration, ) -> Result<(http::StatusCode, String), String> { use tower::ServiceExt; let parsed = reqwest::Url::parse(url).map_err(|e| e.to_string())?; @@ -376,7 +386,7 @@ impl Ctx<'_> { .jwt .get_or_try_init(|| mint_caller_jwt(self.w_id, self.auth, scopes)) .await - .map_err(|e| e.message.to_string())?; + .map_err(|e| e.to_string())?; let mut request = http::Request::builder() .method(method) .uri(uri) @@ -389,8 +399,7 @@ impl Ctx<'_> { None => axum::body::Body::empty(), }; let request = request.body(body).map_err(|e| e.to_string())?; - // The total timeout of the HTTP client this replaces. - tokio::time::timeout(std::time::Duration::from_secs(30), async { + tokio::time::timeout(timeout, async { let response = router.oneshot(request).await.map_err(|e| e.to_string())?; let status = response.status(); let bytes = axum::body::to_bytes(response.into_body(), usize::MAX) @@ -402,6 +411,36 @@ impl Ctx<'_> { .map_err(|_| format!("Failed to execute request: {method} {url} timed out"))? } + /// `request` over HTTP, for a server whose router is not registered. + async fn call_http( + &self, + method: &str, + url: &str, + body: Option, + scopes: Option>, + timeout: std::time::Duration, + ) -> Result<(http::StatusCode, String), String> { + let method = http::Method::from_bytes(method.as_bytes()).map_err(|e| e.to_string())?; + let token = self + .jwt + .get_or_try_init(|| mint_caller_jwt(self.w_id, self.auth, scopes)) + .await + .map_err(|e| e.to_string())?; + let mut request = crate::HTTP_CLIENT + .request(method, url) + .header("Authorization", format!("Bearer {token}")) + .timeout(timeout); + if let Some(body) = body { + request = request.json(&body); + } + let response = request + .send() + .await + .map_err(|e| format!("Failed to execute request: {e}"))?; + let status = response.status(); + Ok((status, response.text().await.map_err(|e| e.to_string())?)) + } + pub async fn get(&self, path: &str) -> Result { let text = self.request("GET", path, None).await?; serde_json::from_str(&text).map_err(|e| format!("unexpected response from {path}: {e}")) @@ -424,6 +463,12 @@ impl Ctx<'_> { format!("/w/{}/{}", self.w_id, rest) } + /// `page_url` for an MCP client only: the chat shows what a tool made or ran in its own + /// cards and panel instead. + pub fn link(&self, page: &str) -> Option { + (self.surface == Surface::Mcp).then(|| self.page_url(page)) + } + /// A Windmill page in this workspace, as a link the user can open: an external client /// has no preview panel, so a link is how it shows what a tool made or ran. pub fn page_url(&self, page: &str) -> String { @@ -637,57 +682,92 @@ pub(super) fn matches_glob(subject: &str, glob: &str) -> bool { regex::Regex::new(&pattern).is_ok_and(|re| re.is_match(subject)) } -/// A generated session tool, refused when the token's access does not cover it. -async fn call_generated_tool(ctx: &Ctx<'_>, name: &str, args: &Value) -> BackendResult { +/// A JWT that acts as `api_authed`, carrying `scopes`; `None` is the caller's full rights. +pub(crate) async fn mint_caller_jwt( + workspace_id: &str, + api_authed: &ApiAuthed, + scopes: Option>, +) -> windmill_common::error::Result { + // Carry the caller's job provenance into the JWT: a job's WM_TOKEN is capped at + // workspace admin (GHSA-hfh4-cx4h-3fcr), and dropping `job_id` here would re-mint an + // uncapped token that satisfies require_super_admin / require_devops_role. + // A browser caller stays a session: runs, trigger history and schedules attribute a + // session's actions to the UI, and anything else to the API (`webhook`). + create_jwt_token( + windmill_common::db::Authed::from(api_authed.clone()), + workspace_id, + 3600, + api_authed.job_id, + api_authed.is_session_token.then(|| "session".to_string()), + None, + scopes, + ) + .await +} + +/// Why a call was refused before the tool ran. +pub(crate) enum CallRefused { + UnknownTool(String), + NotAllowed(String), + NotAvailable(String), +} + +/// What the model reads, and whether the call failed. +#[derive(Serialize)] +pub(crate) struct ToolOutcome { + pub text: String, + pub is_error: bool, +} + +/// The card tools only MCP Apps clients draw; the chat draws its own cards. +#[cfg(feature = "mcp")] +pub(crate) use views::{allowed as view_allowed, definitions as view_definitions}; + +pub(crate) async fn call_session_tool( + ctx: &Ctx<'_>, + name: &str, + args: Value, +) -> Result { + if ctx.surface == Surface::Mcp && views::TOOLS.contains(&name) { + return Ok(outcome(views::call(ctx, name, &args).await)); + } + let result = call_generated_tool(ctx, name, &args).await?; + Ok(outcome(result)) +} + +/// A generated session tool, refused when the caller's access does not cover it. +async fn call_generated_tool( + ctx: &Ctx<'_>, + name: &str, + args: &Value, +) -> Result { let Some(def) = TOOL_DEFS.iter().find(|def| def.name == name) else { - return Err(ErrorData::invalid_params( - format!("Unknown tool '{name}'"), - None, - )); + return Err(CallRefused::UnknownTool(format!("Unknown tool '{name}'"))); }; if !allowed(def, ctx.access) { let token = match ctx.access { SessionAccess::ReadAndRun => "can only read and run deployed scripts and flows", _ => "is read-only", }; - return Err(ErrorData::invalid_request( - format!("Tool '{name}' changes or runs something, and this token {token}"), - None, - )); + return Err(CallRefused::NotAllowed(format!( + "Tool '{name}' changes or runs something, and this token {token}" + ))); } - let Some(result) = dispatch(ctx, name, args).await else { - return Err(ErrorData::invalid_params( - format!("Tool '{name}' is not available on this server"), - None, - )); - }; - Ok(result) + dispatch(ctx, name, args).await.ok_or_else(|| { + CallRefused::NotAvailable(format!("Tool '{name}' is not available on this server")) + }) } -pub(super) async fn call_session_tool( - ctx: &Ctx<'_>, - name: &str, - args: Value, -) -> BackendResult { - let result = if views::TOOLS.contains(&name) { - views::call(ctx, name, &args).await - } else { - call_generated_tool(ctx, name, &args).await? - }; - windmill_common::feature_usage::log_feature_usage("mcp_workspace", "tool_call", name); - let mut result = match result { - Ok(text) => CallToolResult::success(vec![ContentBlock::text(text)]), +fn outcome(result: ToolResult) -> ToolOutcome { + match result { + Ok(text) => ToolOutcome { text, is_error: false }, // Without the status code `Ctx` prefixes an API failure with, as the session reads it. Err(message) => { let body = workspace::api_message(&message); let text = if body.is_empty() { &message } else { body }; - CallToolResult::error(vec![ContentBlock::text(text.to_string())]) + ToolOutcome { text: text.to_string(), is_error: true } } - }; - if let Some((key, view)) = ctx.view.lock().unwrap().take() { - result.meta = Some(MetaObject(Map::from_iter([(key.to_string(), view)]))); } - Ok(result) } #[cfg(test)] @@ -728,38 +808,6 @@ mod tests { assert!(url("POST", "/w/ws/variables/create", ReadOnly).is_err()); } - #[test] - fn no_card_can_call_a_destructive_tool_but_deploy_and_cancel() { - for tool in session_tools(SessionAccess::Full) { - let name = tool.name.as_ref(); - if DESTRUCTIVE.contains(&name) { - let ui = tool.meta.as_ref().and_then(|m| m.0.get("ui")); - let model_only = ui == Some(&json!({ "visibility": ["model"] })); - assert_eq!( - model_only, - !matches!(name, "deploy_workspace_item" | "cancel_job"), - "{name}" - ); - } - } - } - - /// The ChatGPT directory wants every hint explicit, and Claude's a title on every tool. - #[test] - fn every_session_tool_has_a_title_and_explicit_hints() { - for tool in session_tools(SessionAccess::Full) { - let a = tool.annotations.as_ref().expect("annotations"); - assert!(tool.title.is_some() && a.title.is_some(), "{}", tool.name); - assert!( - a.read_only_hint.is_some() - && a.destructive_hint.is_some() - && a.open_world_hint.is_some(), - "{}", - tool.name - ); - } - } - #[test] fn every_session_tool_is_implemented_or_editor_only() { let implemented: Vec<&str> = implemented().collect(); diff --git a/backend/windmill-api/src/ai_session_tools/run_args_cases.json b/backend/windmill-api/src/ai_session_tools/run_args_cases.json new file mode 100644 index 0000000000..35a850b4b7 --- /dev/null +++ b/backend/windmill-api/src/ai_session_tools/run_args_cases.json @@ -0,0 +1,120 @@ +{ + "//": "How a run's arguments are settled, which the chat does in the browser ($lib/components/job_args.ts, secretArgUtils.ts) and MCP on the server (runs.rs). Both test suites read this file, so a change on one side only fails. Each expectation applies its step to `args` alone. A minted variable is named .", + "cases": [ + { + "name": "undeclared arguments are dropped and named", + "schema": { "properties": { "a": { "type": "string" } } }, + "args": { "a": "x", "b": 1, "constructor": 2 }, + "declared": { "args": { "a": "x" }, "undeclared": ["b", "constructor"] } + }, + { + "name": "a schema without properties declares nothing", + "schema": {}, + "args": { "a": "x" }, + "declared": { "args": {}, "undeclared": ["a"] } + }, + { + "name": "a disabled field takes its default, reported only when a different value was proposed", + "schema": { + "properties": { + "locked": { "type": "string", "disabled": true, "default": "fixed" }, + "same": { "type": "object", "disabled": true, "default": { "k": [1, 2] } }, + "absent": { "type": "number", "disabled": true, "default": 3 }, + "no_default": { "type": "string", "disabled": true }, + "free": { "type": "string", "default": "d" } + } + }, + "args": { "locked": "mine", "same": { "k": [1, 2] }, "no_default": "kept", "free": "mine" }, + "disabled_defaults": { + "args": { + "locked": "fixed", + "same": { "k": [1, 2] }, + "absent": 3, + "no_default": "kept", + "free": "mine" + }, + "reset": ["locked"] + } + }, + { + "name": "secrets are minted at every depth, and references and empty fields are left alone", + "schema": { + "properties": { + "token": { "type": "string", "password": true }, + "creds": { "type": "object", "password": true, "properties": { "user": { "type": "string" } } }, + "nested": { "type": "object", "properties": { "inner": { "type": "string", "password": true } } }, + "rows": { + "type": "array", + "items": { "type": "object", "properties": { "key": { "type": "string", "password": true } } } + }, + "variant": { + "type": "object", + "oneOf": [ + { "properties": { "label": { "type": "string" } } }, + { "properties": { "pin": { "type": "string", "password": true } } } + ] + }, + "named": { "type": "string", "password": true }, + "named_object": { "type": "object", "password": true }, + "blank": { "type": "string", "password": true }, + "nothing": { "type": "string", "password": true }, + "plain": { "type": "string" } + } + }, + "args": { + "token": "hunter2", + "creds": { "user": "ada" }, + "nested": { "inner": "deep" }, + "rows": [{ "key": "k1" }, { "key": "$var:f/team/k2" }], + "variant": { "pin": "1234" }, + "named": "$var:f/team/api_token", + "named_object": "$var:f/team/creds", + "blank": "", + "nothing": null, + "plain": "kept" + }, + "secrets": { + "args": { + "token": "$var:", + "creds": "$jsonvar:", + "nested": { "inner": "$var:" }, + "rows": [{ "key": "$var:" }, { "key": "$var:f/team/k2" }], + "variant": { "pin": "$var:" }, + "named": "$var:f/team/api_token", + "named_object": "$jsonvar:f/team/creds", + "blank": "", + "nothing": null, + "plain": "kept" + }, + "stored": ["1234", "deep", "hunter2", "k1", "{\"user\":\"ada\"}"] + }, + "redacted": { + "token": "", + "creds": "", + "nested": { "inner": "" }, + "rows": [{ "key": "" }, { "key": "$var:f/team/k2" }], + "variant": { "pin": "" }, + "named": "$var:f/team/api_token", + "named_object": "$var:f/team/creds", + "blank": "", + "plain": "kept" + } + }, + { + "name": "file bytes are replaced by their size", + "schema": { + "properties": { + "doc": { "type": "string", "contentEncoding": "base64" }, + "docs": { "type": "array", "items": { "type": "string", "contentEncoding": "base64" } }, + "name": { "type": "string" } + } + }, + "args": { + "doc": "QUJDREVGR0hJSktMTU5PUFFSU1RVVldYWVo=", + "docs": ["QUJD", "QUJDREVGR0hJSktMTU5PUFFSU1RVVldYWVpBQkNERUZHSElKS0xNTk9QUVJTVFVWV1hZWg=="], + "name": "report.pdf" + }, + "redacted": { "doc": "", "docs": ["", ""], "name": "report.pdf" } + } + ] +} diff --git a/backend/windmill-api/src/mcp/session/runs.rs b/backend/windmill-api/src/ai_session_tools/runs.rs similarity index 78% rename from backend/windmill-api/src/mcp/session/runs.rs rename to backend/windmill-api/src/ai_session_tools/runs.rs index 273a273199..99b6a653ec 100644 --- a/backend/windmill-api/src/mcp/session/runs.rs +++ b/backend/windmill-api/src/ai_session_tools/runs.rs @@ -3,9 +3,10 @@ use std::collections::HashMap; use std::time::Duration; -use serde::Serialize; +use serde::{Deserialize, Serialize}; use serde_json::{json, Map, Value}; +use super::chat::{Effect, RunPhase}; use super::workspace::{ api_message, get_draft, infer_script_schema, load_flow, load_script, path_enc, }; @@ -200,7 +201,10 @@ pub(super) async fn finish_run( tool_args: &Value, ) -> ToolResult { let id = started.map_err(|e| format!("Failed to execute {action}: {}", api_message(&e)))?; - let run_page = format!("\n\nRun page: {}", ctx.page_url(&format!("/run/{id}"))); + let run_page = ctx + .link(&format!("/run/{id}")) + .map(|url| format!("\n\nRun page: {url}")) + .unwrap_or_default(); let Some(job) = wait_job(ctx, &id, wait_budget(tool_args), true).await else { return Ok(format!("{}{run_page}", backgrounded_summary(&id, label))); }; @@ -346,8 +350,37 @@ async fn process_secret_args( return Ok(args); } - let mut pending: Vec<(String, &'static str, Value)> = Vec::new(); - map_arg_leaves(&args, schema, is_secret_prop, &mut |value, prop, path| { + let mut minted: HashMap = HashMap::new(); + for (key, prefix, value) in secrets_to_mint(&args, schema) { + let variable = format!( + "u/{username}/secret_arg/{}", + windmill_common::utils::rd_string(12) + ); + let expires_at = (chrono::Utc::now() + chrono::Duration::days(7)) + .to_rfc3339_opts(chrono::SecondsFormat::Millis, true); + ctx.post( + &ctx.ws("variables/create"), + json!({ + "value": secret_text(prefix, &value), + "is_secret": true, + "path": variable, + "description": "Ephemeral secret variable", + "expires_at": expires_at, + }), + ) + .await?; + minted.insert(key, format!("{prefix}{variable}")); + } + Ok(substitute_secrets(&args, schema, &minted)) +} + +/// The sensitive literals to mint, each keyed by its path, with the reference it becomes. +fn secrets_to_mint( + args: &Map, + schema: &Value, +) -> Vec<(String, &'static str, Value)> { + let mut pending = Vec::new(); + map_arg_leaves(args, schema, is_secret_prop, &mut |value, prop, path| { let holds_secret = !value.is_null() && value != "" && !is_reference(value); if holds_secret { pending.push(( @@ -358,51 +391,37 @@ async fn process_secret_args( } Some(value.clone()) }); + pending +} - let mut minted: HashMap = HashMap::new(); - for (key, prefix, value) in pending { - let variable = format!( - "u/{username}/secret_arg/{}", - windmill_common::utils::rd_string(12) - ); - let secret = match &value { - Value::String(text) if prefix == "$var:" => text.clone(), - other => other.to_string(), - }; - let expires_at = (chrono::Utc::now() + chrono::Duration::days(7)) - .to_rfc3339_opts(chrono::SecondsFormat::Millis, true); - ctx.post( - &ctx.ws("variables/create"), - json!({ - "value": secret, - "is_secret": true, - "path": variable, - "description": "Ephemeral secret variable", - "expires_at": expires_at, - }), - ) - .await?; - minted.insert(key, format!("{prefix}{variable}")); +/// What the secret variable stores: the text for `$var:`, the JSON for `$jsonvar:`. +fn secret_text(prefix: &str, value: &Value) -> String { + match value { + Value::String(text) if prefix == "$var:" => text.clone(), + other => other.to_string(), } +} - Ok(map_arg_leaves( - &args, - schema, - is_secret_prop, - &mut |value, prop, path| { - if let Some(reference) = minted.get(&Value::from(path).to_string()) { - return Some(reference.as_str().into()); +/// The arguments with each minted literal replaced by its reference, keyed as +/// `secrets_to_mint` keys them. +fn substitute_secrets( + args: &Map, + schema: &Value, + minted: &HashMap, +) -> Map { + map_arg_leaves(args, schema, is_secret_prop, &mut |value, prop, path| { + if let Some(reference) = minted.get(&Value::from(path).to_string()) { + return Some(reference.as_str().into()); + } + // A plain variable named for a field that cannot hold a string: its contents + // are meant, read the way the field needs them. + match value.as_str().and_then(|s| s.strip_prefix("$var:")) { + Some(variable) if reference_prefix(prop, value) == "$jsonvar:" => { + Some(format!("$jsonvar:{variable}").into()) } - // A plain variable named for a field that cannot hold a string: its contents - // are meant, read the way the field needs them. - match value.as_str().and_then(|s| s.strip_prefix("$var:")) { - Some(variable) if reference_prefix(prop, value) == "$jsonvar:" => { - Some(format!("$jsonvar:{variable}").into()) - } - _ => Some(value.clone()), - } - }, - )) + _ => Some(value.clone()), + } + }) } fn file_marker(base64: &str) -> String { @@ -602,66 +621,412 @@ async fn conclude( // Scripts and flows // ============================================================================ -async fn test_run_script(ctx: &Ctx<'_>, args: &Value) -> ToolResult { - let path = str_arg(args, "path")?; - let script = load_script(ctx, path).await?; - let schema = schema_for_test_run(&script.language, &script.content, script.schema, None); - let prepared = - match prepare_args(ctx, path, proposed_args(args), schema.as_ref(), "script").await { - Ok(prepared) => prepared, - Err(message) => return Ok(message), - }; - let started = run_script_preview( +/// What a run tool runs, resolved once and queued later with the final arguments: the +/// chat shows its argument form in between, and the run has to be the version the form +/// was shown for. +#[derive(Serialize, Deserialize, Clone)] +#[serde(tag = "kind", rename_all = "snake_case")] +pub(crate) enum RunTarget { + ScriptPreview { + path: Option, + language: String, + content: String, + /// A flow's preprocessor, which takes the arguments of its own entrypoint. + preprocessor: bool, + }, + ScriptByPath { + path: String, + }, + FlowByPath { + path: String, + /// A chat-enabled flow is refused without a conversation, and a run is its own. + chat_enabled: bool, + }, + FlowPreview { + path: String, + value: Value, + /// The conversation the caller named. + memory_id: Option, + /// Without one, a chat-enabled flow's test run is a conversation of its own. + fresh_conversation: bool, + }, + /// A saved AI agent, as its own page runs it. + Agent { + path: String, + memory_id: Option, + }, + /// An AI agent or AI decision step, run alone as a one-step flow preview of `flow_path`. + Step { + flow_path: String, + step: Value, + }, + AppRunnable { + app_path: String, + key: String, + runnable: Value, + }, + DatatableSql { + datatable: String, + sql: String, + role: Option, + }, +} + +/// Queue `target` with `args` and answer the job id, or the server's own message. +pub(super) async fn start_run( + ctx: &Ctx<'_>, + target: &RunTarget, + args: Map, + form_keys: &[String], +) -> Result { + let message = |e: String| api_message(&e).to_string(); + match target { + RunTarget::ScriptPreview { path, language, content, preprocessor } => { + let mut with_entrypoint = Map::new(); + if *preprocessor { + with_entrypoint.insert("_ENTRYPOINT_OVERRIDE".into(), "preprocessor".into()); + } + with_entrypoint.extend(args); + run_script_preview( + ctx, + path.as_deref(), + language, + content, + Value::Object(with_entrypoint), + ) + .await + .map_err(message) + } + // The arguments fill the main input schema; a preprocessor would take them for a + // trigger event. + RunTarget::ScriptByPath { path } => ctx + .post( + &ctx.ws(&format!( + "jobs/run/p/{}?skip_preprocessor=true", + path_enc(path) + )), + Value::Object(args), + ) + .await + .map(job_id) + .map_err(message), + RunTarget::FlowByPath { path, chat_enabled } => { + let memory_id = if *chat_enabled { + format!("&memory_id={}", uuid::Uuid::new_v4()) + } else { + String::new() + }; + ctx.post( + &ctx.ws(&format!( + "jobs/run/f/{}?skip_preprocessor=true{memory_id}", + path_enc(path) + )), + Value::Object(args), + ) + .await + .map(job_id) + .map_err(message) + } + RunTarget::FlowPreview { path, value, memory_id, fresh_conversation } => { + let memory_id = memory_id.clone().or_else(|| { + (*fresh_conversation && truthy(value.get("chat_input_enabled"))) + .then(|| uuid::Uuid::new_v4().to_string()) + }); + run_flow_preview( + ctx, + Some(path), + value, + Value::Object(args), + memory_id.as_deref(), + ) + .await + .map_err(message) + } + RunTarget::Agent { path, memory_id } => { + let query = memory_id + .as_ref() + .map(|id| format!("?memory_id={}", enc(id))) + .unwrap_or_default(); + ctx.post( + &ctx.ws(&format!("jobs/run/agent/{}{query}", path_enc(path))), + Value::Object(args), + ) + .await + .map(job_id) + .map_err(message) + } + RunTarget::Step { flow_path, step } => { + let mut step = step.clone(); + let value = &mut step["value"]; + if value["type"] == "aiagent" { + *value = migrated_agent_value(value); + value["input_transforms"] = Value::Object(agent_test_transforms( + value.get("input_transforms"), + &args, + form_keys, + )); + // After the inputs are wired: a linked agent's draft brings its own brain + // and tools, and the step's run inputs stay on top. + with_agent_drafts(ctx, &mut step).await?; + } else { + // The inputs given here override the step's own, so a state that reads an + // earlier step can be passed directly. + let mut transforms = value["input_transforms"] + .as_object() + .cloned() + .unwrap_or_default(); + for key in args.keys() { + transforms.insert( + key.clone(), + json!({ "type": "javascript", "expr": format!("flow_input.{key}") }), + ); + } + value["input_transforms"] = Value::Object(transforms); + } + let preview = json!({ "modules": [step] }); + run_flow_preview(ctx, Some(flow_path), &preview, Value::Object(args), None) + .await + .map_err(message) + } + RunTarget::AppRunnable { app_path, key, runnable } => { + let body = + super::apps::execute_component_body(app_path, key, runnable, &Value::Object(args)); + ctx.request( + "POST", + &ctx.ws(&format!("apps_u/execute_component/{}", path_enc(app_path))), + Some(body), + ) + .await + .map(|id| id.trim().trim_matches('"').to_string()) + .map_err(message) + } + RunTarget::DatatableSql { datatable, sql, role } => { + let database = match role { + None => format!("datatable://{datatable}"), + // A stored name holding a `?` is read whole by the server, so a role + // cannot follow it. + Some(role) if datatable.contains('?') => { + return Err(format!("Data table '{datatable}' has a '?' in its name, so it can only be used here as its default role. Rename it to connect as role '{role}'.")); + } + Some(role) => format!("datatable://{datatable}?role={role}"), + }; + run_script_preview( + ctx, + None, + "postgresql", + sql, + json!({ "database": database }), + ) + .await + .map_err(message) + } + } +} + +/// A run tool's target, and what its argument form shows for it. +#[derive(Serialize)] +pub(super) struct Resolved { + pub target: RunTarget, + /// The input schema stored with what will run; `None` when it has none. The chat + /// infers a missing one with the editor's parsers, MCP with the backend's. + pub schema: Option, + pub code: Option, + pub lang: Option, + pub summary: Option, + /// `script` or `flow`: the kind of job the run queues. + pub runnable_kind: &'static str, + pub start_message: String, + /// The item's deployed page is a conversation rather than a run form. + pub conversational: bool, + /// For a step: the module, whose form an agent step builds from its own inputs. + pub module: Option, +} + +/// A run tool called by the chat, in its phases; by MCP, resolved and run here. +pub(super) async fn run_tool( + ctx: &Ctx<'_>, + tool_args: &Value, + resolved: impl std::future::Future>, + mcp: impl AsyncFnOnce(Resolved) -> ToolResult, +) -> ToolResult { + match ctx.chat.and_then(|chat| chat.run.as_ref()) { + Some(RunPhase::Prepare) => Ok(serde_json::to_string(&resolved.await?).unwrap_or_default()), + Some(RunPhase::Start { target, args, form_keys }) => { + let target = match target { + Some(target) => target.clone(), + None => resolved.await?.target, + }; + let args = if target_takes_tool_args(&target) && args.is_empty() { + proposed_args(tool_args) + } else { + args.clone() + }; + start_run(ctx, &target, args, form_keys).await + } + None => mcp(resolved.await?).await, + } +} + +/// A run with no argument form takes the tool's own `args`. +fn target_takes_tool_args(target: &RunTarget) -> bool { + matches!( + target, + RunTarget::AppRunnable { .. } | RunTarget::DatatableSql { .. } + ) +} + +/// MCP's run of a resolved target: arguments conformed and secrets minted here, as no +/// form stands in front of them, then the job awaited within the tool's budget. +async fn run_resolved( + ctx: &Ctx<'_>, + tool_args: &Value, + path: &str, + resolved: Resolved, + schema_noun: &str, + label: &str, + action: &str, +) -> ToolResult { + let proposed = proposed_args(tool_args); + run_resolved_with( ctx, - Some(path), - &script.language, - &script.content, - Value::Object(prepared.args), + tool_args, + path, + resolved, + proposed, + schema_noun, + label, + action, + ) + .await +} + +#[allow(clippy::too_many_arguments)] +async fn run_resolved_with( + ctx: &Ctx<'_>, + tool_args: &Value, + path: &str, + resolved: Resolved, + proposed: Map, + schema_noun: &str, + label: &str, + action: &str, +) -> ToolResult { + let schema = resolved.schema; + let prepared = match prepare_args(ctx, path, proposed, schema.as_ref(), schema_noun).await { + Ok(prepared) => prepared, + Err(message) => return Ok(message), + }; + let started = start_run( + ctx, + &resolved.target, + prepared.args, + &form_keys(schema.as_ref()), ) .await; - conclude(ctx, args, prepared.notes, started, path, false, "test").await + conclude( + ctx, + tool_args, + prepared.notes, + started, + label, + resolved.runnable_kind == "flow", + action, + ) + .await +} + +fn form_keys(schema: Option<&Value>) -> Vec { + schema + .and_then(|s| s["properties"].as_object()) + .map(|p| p.keys().cloned().collect()) + .unwrap_or_default() +} + +async fn resolve_test_script(ctx: &Ctx<'_>, path: &str) -> Result { + let script = load_script(ctx, path).await?; + Ok(Resolved { + schema: script.schema, + code: Some(script.content.clone()), + lang: Some(script.language.clone()), + summary: script.summary, + runnable_kind: "script", + start_message: format!("Running test for script \"{path}\"..."), + conversational: false, + module: None, + target: RunTarget::ScriptPreview { + path: Some(path.to_string()), + language: script.language, + content: script.content, + preprocessor: false, + }, + }) +} + +async fn test_run_script(ctx: &Ctx<'_>, args: &Value) -> ToolResult { + let path = str_arg(args, "path")?; + run_tool( + ctx, + args, + resolve_test_script(ctx, path), + async |mut resolved: Resolved| { + let (code, lang) = (resolved.code.clone(), resolved.lang.clone()); + resolved.schema = schema_for_test_run( + &lang.unwrap_or_default(), + &code.unwrap_or_default(), + resolved.schema, + None, + ); + run_resolved(ctx, args, path, resolved, "script", path, "test").await + }, + ) + .await } /// `run_script` and `run_flow`: the deployed item, whatever draft sits on top of it. -async fn run_deployed(ctx: &Ctx<'_>, args: &Value, is_flow: bool) -> ToolResult { - let path = str_arg(args, "path")?; - let (get, run) = if is_flow { - ("flows/get", "jobs/run/f") +async fn resolve_deployed(ctx: &Ctx<'_>, path: &str, is_flow: bool) -> Result { + let get = if is_flow { + "flows/get" } else { - ("scripts/get/p", "jobs/run/p") + "scripts/get/p" }; let item = ctx .get(&ctx.ws(&format!("{get}/{}", path_enc(path)))) .await?; - let schema = item - .get("schema") - .filter(|s| !s.is_null()) - .cloned() - .unwrap_or_else(|| json!({})); - let prepared = - match prepare_args(ctx, path, proposed_args(args), Some(&schema), "deployed").await { - Ok(prepared) => prepared, - Err(message) => return Ok(message), - }; - // A chat-enabled flow is refused without a conversation, and this run is one of its own. - let memory_id = if is_flow && truthy(item["value"].get("chat_input_enabled")) { - format!("&memory_id={}", uuid::Uuid::new_v4()) - } else { - String::new() - }; - // The arguments fill the main input schema; a preprocessor would take them for a - // trigger event. - let started = ctx - .post( - &ctx.ws(&format!( - "{run}/{}?skip_preprocessor=true{memory_id}", - path_enc(path) - )), - Value::Object(prepared.args), - ) - .await - .map(job_id); - conclude(ctx, args, prepared.notes, started, path, is_flow, "run").await + let chat_enabled = is_flow && truthy(item["value"].get("chat_input_enabled")); + Ok(Resolved { + schema: Some( + item.get("schema") + .filter(|s| !s.is_null()) + .cloned() + .unwrap_or_else(|| json!({})), + ), + code: None, + lang: None, + summary: item + .get("summary") + .and_then(Value::as_str) + .map(String::from), + runnable_kind: if is_flow { "flow" } else { "script" }, + start_message: format!("Running \"{path}\"..."), + conversational: chat_enabled, + module: None, + target: if is_flow { + RunTarget::FlowByPath { path: path.to_string(), chat_enabled } + } else { + RunTarget::ScriptByPath { path: path.to_string() } + }, + }) +} + +async fn run_deployed(ctx: &Ctx<'_>, args: &Value, is_flow: bool) -> ToolResult { + let path = str_arg(args, "path")?; + run_tool( + ctx, + args, + resolve_deployed(ctx, path, is_flow), + async |resolved| run_resolved(ctx, args, path, resolved, "deployed", path, "run").await, + ) + .await } /// The flow value a preview runs, from a loaded `{value, schema, groups}`. @@ -669,7 +1034,7 @@ fn flow_preview_value(flow: &Value) -> Value { let mut value = flow.get("value").cloned().unwrap_or_else(|| json!({})); if let (Some(groups), Some(map)) = (flow.get("groups"), value.as_object_mut()) { if groups.is_null() { - map.remove("groups"); + map.shift_remove("groups"); } else { map.insert("groups".into(), groups.clone()); } @@ -677,38 +1042,51 @@ fn flow_preview_value(flow: &Value) -> Value { value } -async fn test_run_flow(ctx: &Ctx<'_>, args: &Value) -> ToolResult { - let path = str_arg(args, "path")?; - let memory_id = opt_str_arg(args, "memory_id"); +async fn resolve_test_flow( + ctx: &Ctx<'_>, + path: &str, + memory_id: Option<&str>, +) -> Result { if memory_id.is_some_and(|id| id.len() != 36 || uuid::Uuid::parse_str(id).is_err()) { return Err("memory_id must be a UUID".to_string()); } let flow = load_flow(ctx, path).await?; - let schema = flow - .flow - .get("schema") - .filter(|s| !s.is_null()) - .cloned() - .unwrap_or_else(|| json!({})); - let prepared = match prepare_args(ctx, path, proposed_args(args), Some(&schema), "flow").await { - Ok(prepared) => prepared, - Err(message) => return Ok(message), - }; - let value = flow_preview_value(&flow.flow); - // The server refuses a chat-enabled flow's run without a conversation; a test run - // is one of its own unless the caller names another. - let memory_id = memory_id.map(String::from).or_else(|| { - truthy(value.get("chat_input_enabled")).then(|| uuid::Uuid::new_v4().to_string()) - }); - let started = run_flow_preview( + Ok(Resolved { + schema: Some( + flow.flow + .get("schema") + .filter(|s| !s.is_null()) + .cloned() + .unwrap_or_else(|| json!({})), + ), + code: None, + lang: None, + summary: flow.summary.clone(), + runnable_kind: "flow", + start_message: format!("Starting flow test run for \"{path}\"..."), + conversational: false, + module: None, + // The server refuses a chat-enabled flow's run without a conversation; a test run + // is one of its own unless the caller names another. + target: RunTarget::FlowPreview { + path: path.to_string(), + value: flow_preview_value(&flow.flow), + memory_id: memory_id.map(String::from), + fresh_conversation: true, + }, + }) +} + +async fn test_run_flow(ctx: &Ctx<'_>, args: &Value) -> ToolResult { + let path = str_arg(args, "path")?; + let memory_id = opt_str_arg(args, "memory_id"); + run_tool( ctx, - Some(path), - &value, - Value::Object(prepared.args), - memory_id.as_deref(), + args, + resolve_test_flow(ctx, path, memory_id), + async |resolved| run_resolved(ctx, args, path, resolved, "flow", path, "test").await, ) - .await; - conclude(ctx, args, prepared.notes, started, path, true, "test").await + .await } // ============================================================================ @@ -739,24 +1117,26 @@ fn agent_memory_id(args: &Value) -> Result, String> { Ok(memory_id) } -/// The agent's configuration a test runs: the user's draft when there is one, else the deployed -/// resource. Refuses a path holding some other kind of resource. -async fn load_agent_for_test(ctx: &Ctx<'_>, path: &str) -> Result { +/// The agent a test runs, as `(configuration, description)`: the user's draft when there is +/// one, else the deployed resource. Refuses a path holding some other kind of resource. +async fn load_agent_for_test(ctx: &Ctx<'_>, path: &str) -> Result<(Value, Value), String> { let draft = get_draft(ctx, "resource", path).await?.map(|d| d.value); // A draft records its type only when nothing is deployed at its path. - let (resource_type, config) = match draft { - Some(draft) if draft.get("resource_type").is_some() => { - (draft["resource_type"].clone(), draft["args"].clone()) - } + let (resource_type, config, description) = match draft { + Some(draft) if draft.get("resource_type").is_some() => ( + draft["resource_type"].clone(), + draft["args"].clone(), + draft["description"].clone(), + ), draft => { let deployed = ctx .get(&ctx.ws(&format!("resources/get/{}", path_enc(path)))) .await?; - let config = match draft { - Some(draft) => draft["args"].clone(), - None => deployed["value"].clone(), + let (config, description) = match draft { + Some(draft) => (draft["args"].clone(), draft["description"].clone()), + None => (deployed["value"].clone(), deployed["description"].clone()), }; - (deployed["resource_type"].clone(), config) + (deployed["resource_type"].clone(), config, description) } }; let resource_type = resource_type.as_str().unwrap_or_default(); @@ -765,13 +1145,34 @@ async fn load_agent_for_test(ctx: &Ctx<'_>, path: &str) -> Result "\"{path}\" is a {resource_type} resource, not an AI agent." )); } - Ok(config) + Ok((config, description)) } -async fn test_run_agent(ctx: &Ctx<'_>, args: &Value) -> ToolResult { - let path = str_arg(args, "path")?; - let memory_id = agent_memory_id(args)?; - let mut config = load_agent_for_test(ctx, path).await?; +fn agent_resolved( + target: RunTarget, + description: &Value, + start_message: String, + conversational: bool, +) -> Resolved { + Resolved { + target, + schema: Some(agent_run_schema()), + code: None, + lang: None, + summary: description.as_str().map(String::from), + runnable_kind: "flow", + start_message, + conversational, + module: None, + } +} + +async fn resolve_test_agent( + ctx: &Ctx<'_>, + path: &str, + memory_id: Option<&str>, +) -> Result { + let (mut config, description) = load_agent_for_test(ctx, path).await?; // Who the agent runs as and who may ask it in Slack, not inputs: a test runs as its caller. if let Some(map) = config.as_object_mut() { for key in ["on_behalf_of", "run_on_behalf_of", "slack_anyone"] { @@ -790,48 +1191,72 @@ async fn test_run_agent(ctx: &Ctx<'_>, args: &Value) -> ToolResult { ) }) .map_err(|e| e.to_string())?; - let flow = serde_json::to_value(flow).map_err(|e| e.to_string())?; - let schema = agent_run_schema(); - let prepared = match prepare_args(ctx, path, proposed_args(args), Some(&schema), "agent").await - { - Ok(prepared) => prepared, - Err(message) => return Ok(message), + let target = RunTarget::FlowPreview { + // Filed where the agent editor's chat files its conversations. + path: format!("{path}.chat"), + value: serde_json::to_value(flow).map_err(|e| e.to_string())?, + memory_id: Some( + memory_id + .map(String::from) + .unwrap_or_else(|| uuid::Uuid::new_v4().to_string()), + ), + fresh_conversation: false, }; - let memory_id = memory_id - .map(String::from) - .unwrap_or_else(|| uuid::Uuid::new_v4().to_string()); - // Filed where the agent editor's chat files its conversations. - let started = run_flow_preview( + Ok(agent_resolved( + target, + &description, + format!("Starting agent test run for \"{path}\"..."), + false, + )) +} + +async fn test_run_agent(ctx: &Ctx<'_>, args: &Value) -> ToolResult { + let path = str_arg(args, "path")?; + let memory_id = agent_memory_id(args)?; + run_tool( ctx, - Some(&format!("{path}.chat")), - &flow, - Value::Object(prepared.args), - Some(&memory_id), + args, + resolve_test_agent(ctx, path, memory_id), + async |resolved| run_resolved(ctx, args, path, resolved, "agent", path, "test").await, ) - .await; - conclude(ctx, args, prepared.notes, started, path, true, "test").await + .await +} + +async fn resolve_agent( + ctx: &Ctx<'_>, + path: &str, + memory_id: Option<&str>, +) -> Result { + let agent = ctx + .get(&ctx.ws(&format!("resources/get/{}", path_enc(path)))) + .await?; + let resource_type = agent["resource_type"].as_str().unwrap_or_default(); + if resource_type != "ai_agent" { + return Err(format!( + "\"{path}\" is a {resource_type} resource, not an AI agent." + )); + } + let target = + RunTarget::Agent { path: path.to_string(), memory_id: memory_id.map(String::from) }; + // A turn of a conversation stays on the card: the agent's page starts one of its own. + Ok(agent_resolved( + target, + &agent["description"], + format!("Running agent \"{path}\"..."), + memory_id.is_some(), + )) } async fn run_agent(ctx: &Ctx<'_>, args: &Value) -> ToolResult { let path = str_arg(args, "path")?; let memory_id = agent_memory_id(args)?; - let schema = agent_run_schema(); - let prepared = match prepare_args(ctx, path, proposed_args(args), Some(&schema), "agent").await - { - Ok(prepared) => prepared, - Err(message) => return Ok(message), - }; - let query = memory_id - .map(|id| format!("?memory_id={}", enc(id))) - .unwrap_or_default(); - let started = ctx - .post( - &ctx.ws(&format!("jobs/run/agent/{}{query}", path_enc(path))), - Value::Object(prepared.args), - ) - .await - .map(job_id); - conclude(ctx, args, prepared.notes, started, path, true, "run").await + run_tool( + ctx, + args, + resolve_agent(ctx, path, memory_id), + async |resolved| run_resolved(ctx, args, path, resolved, "agent", path, "run").await, + ) + .await } // ============================================================================ @@ -937,7 +1362,7 @@ const AGENT_OVERRIDE_KEYS: [&str; 2] = ["model", "reasoning_effort"]; /// Older spellings of two agent inputs, read as the current ones. fn migrate_agent_transforms(transforms: &mut Map) { if !transforms.contains_key("user_attachments") { - if let Some(images) = transforms.remove("user_images") { + if let Some(images) = transforms.shift_remove("user_images") { transforms.insert("user_attachments".into(), images); } } @@ -967,7 +1392,7 @@ fn migrate_agent_transforms(transforms: &mut Map) { } _ => {} } - transforms.remove("messages_context_length"); + transforms.shift_remove("messages_context_length"); } /// The inputs a test run of an agent step takes, in the agent form's order, and the @@ -1090,7 +1515,7 @@ fn inline_agent_draft(module: &mut Value, drafts: &HashMap) { .as_str() .and_then(|agent| drafts.get(normalize_agent_ref(agent))); if let (Some(draft), Some(map)) = (draft, value.as_object_mut()) { - map.remove("agent"); + map.shift_remove("agent"); let mut transforms = Map::new(); for (key, arg) in draft.as_object().into_iter().flatten() { if key != "tools" && !AGENT_HISTORY_KEYS.contains(&key.as_str()) { @@ -1150,11 +1575,21 @@ async fn with_agent_drafts(ctx: &Ctx<'_>, module: &mut Value) -> Result<(), Stri Ok(()) } -async fn test_run_step(ctx: &Ctx<'_>, args: &Value) -> ToolResult { - let path = str_arg(args, "path")?; - let step_id = str_arg(args, "stepId")?; +async fn resolve_step(ctx: &Ctx<'_>, path: &str, step_id: &str) -> Result { let flow_value = flow_preview_value(&load_flow(ctx, path).await?.flow); - let Some(module) = find_module(&flow_value, step_id) else { + let Some(module) = find_module(&flow_value, step_id).cloned() else { + let mut patch = Map::new(); + patch.insert( + "content".into(), + json!(format!("Step \"{step_id}\" not found in flow")), + ); + patch.insert( + "error".into(), + json!(format!( + "Step with id \"{step_id}\" does not exist in the current flow" + )), + ); + ctx.effect(Effect::Status(patch)); return Err(format!( "Step with id \"{step_id}\" not found in flow. Available steps: {}", available_step_ids(&flow_value) @@ -1163,39 +1598,37 @@ async fn test_run_step(ctx: &Ctx<'_>, args: &Value) -> ToolResult { let value = &module["value"]; let module_type = value["type"].as_str().unwrap_or_default(); let text = |key: &str| value[key].as_str().unwrap_or_default().to_string(); - // A preprocessor takes the arguments of its own entrypoint, which no schema declares. - let entrypoint = (module["id"] == "preprocessor").then_some("preprocessor"); - let label = format!("step {step_id}"); - let proposed = proposed_args(args); - - let (schema, proposed) = match module_type { - "rawscript" => ( - schema_for_test_run(&text("language"), &text("content"), None, entrypoint), - proposed, - ), - "script" | "flow" | "aidecision" => (None, proposed), - "aiagent" => { - let (keys, args) = agent_step_form(&migrated_agent_value(value), &proposed); - let properties: Map = - keys.into_iter().map(|key| (key, json!({}))).collect(); - (Some(json!({ "properties": properties })), args) - } - other => { - return Err(format!( - "Cannot test step of type \"{other}\". Supported types: rawscript, script, flow, aiagent, aidecision" - )); - } + let preprocessor = module["id"] == "preprocessor"; + let step = |runnable_kind, start_message: String, schema, code, lang, target| Resolved { + schema, + code, + lang, + summary: None, + runnable_kind, + start_message, + conversational: false, + module: Some(module.clone()), + target, }; - - // The targets read from the workspace, each with the schema of what will run. - let mut script = None; - let mut subflow_preview = None; - let schema = match module_type { + Ok(match module_type { + "rawscript" => step( + "script", + format!("Starting test run of step \"{step_id}\"..."), + None, + Some(text("content")), + Some(text("language")), + RunTarget::ScriptPreview { + path: None, + language: text("language"), + content: text("content"), + preprocessor, + }, + ), "script" => { let target = text("path"); let hash = value["hash"].as_str().filter(|h| !h.is_empty()); // A draft of the script runs even where the step pins a hash. - let loaded = match hash { + let (content, language, schema) = match hash { Some(hash) if get_draft(ctx, "script", &target).await?.is_none() => { let deployed = ctx .get(&ctx.ws(&format!("scripts/get/h/{}", enc(hash)))) @@ -1214,143 +1647,120 @@ async fn test_run_step(ctx: &Ctx<'_>, args: &Value) -> ToolResult { (loaded.content, loaded.language, loaded.schema) } }; - let schema = schema_for_test_run(&loaded.1, &loaded.0, loaded.2, entrypoint); - script = Some((target, loaded.0, loaded.1)); - schema + step( + "script", + format!("Starting test run of script step \"{step_id}\"..."), + schema, + Some(content.clone()), + Some(language.clone()), + RunTarget::ScriptPreview { path: Some(target), language, content, preprocessor }, + ) } "flow" => { // Only a draft is previewed; a deployed subflow runs by path, as its parent // flow would run it. - let subflow = load_flow(ctx, &text("path")).await?; - if subflow.is_draft { - subflow_preview = Some(flow_preview_value(&subflow.flow)); - } - Some( - subflow - .flow - .get("schema") - .filter(|s| !s.is_null()) - .cloned() - .unwrap_or_else(|| json!({})), - ) - } - // What a run supplies is the state; the provider and questions are the step's own. - "aidecision" => Some(json!({ "properties": { "state": {} } })), - _ => schema, - }; - - let prepared = match prepare_args(ctx, path, proposed, schema.as_ref(), "step").await { - Ok(prepared) => prepared, - Err(message) => return Ok(message), - }; - let mut run_args = prepared.args; - let with_entrypoint = |args: Map| { - let mut out = Map::new(); - if entrypoint.is_some() { - out.insert("_ENTRYPOINT_OVERRIDE".into(), "preprocessor".into()); - } - out.extend(args); - Value::Object(out) - }; - // Run alone, stripped of the settings around it: a mock would answer for the step - // and a suspend would hold the run for an approval. - let single_step = - |step_value: Value| json!({ "modules": [{ "id": module["id"], "value": step_value }] }); - - let (started, is_flow) = match module_type { - "rawscript" => ( - run_script_preview( - ctx, - None, - &text("language"), - &text("content"), - with_entrypoint(run_args), - ) - .await, - false, - ), - "script" => { - let (target, content, language) = script.unwrap_or_default(); - ( - run_script_preview( - ctx, - Some(&target), - &language, - &content, - with_entrypoint(run_args), - ) - .await, - false, - ) - } - "flow" => { let target = text("path"); - let started = match subflow_preview { - Some(preview) => { - run_flow_preview(ctx, Some(&target), &preview, Value::Object(run_args), None) - .await - } - None => ctx - .post( - &ctx.ws(&format!( - "jobs/run/f/{}?skip_preprocessor=true", - path_enc(&target) - )), - Value::Object(run_args), - ) - .await - .map(job_id), - }; - (started, true) - } - "aiagent" => { - let mut step_value = migrated_agent_value(value); - let form_keys: Vec = schema - .as_ref() - .and_then(|s| s["properties"].as_object()) - .map(|p| p.keys().cloned().collect()) - .unwrap_or_default(); - step_value["input_transforms"] = Value::Object(agent_test_transforms( - step_value.get("input_transforms"), - &run_args, - &form_keys, - )); - let mut step = json!({ "id": module["id"], "value": step_value }); - with_agent_drafts(ctx, &mut step).await?; - let preview = json!({ "modules": [step] }); - ( - run_flow_preview(ctx, Some(path), &preview, Value::Object(run_args), None).await, - true, - ) - } - _ => { - let mut step_value = value.clone(); - let mut transforms = value["input_transforms"] - .as_object() - .cloned() - .unwrap_or_default(); - for key in run_args.keys() { - transforms.insert( - key.clone(), - json!({ "type": "javascript", "expr": format!("flow_input.{key}") }), - ); - } - step_value["input_transforms"] = Value::Object(transforms); - let preview = single_step(step_value); - ( - run_flow_preview( - ctx, - Some(path), - &preview, - Value::Object(std::mem::take(&mut run_args)), + let subflow = load_flow(ctx, &target).await?; + let schema = subflow.flow.get("schema").filter(|s| !s.is_null()).cloned(); + if subflow.is_draft { + step( + "flow", + format!("Starting test run of draft flow step \"{step_id}\"..."), + schema, None, + None, + RunTarget::FlowPreview { + path: target, + value: flow_preview_value(&subflow.flow), + memory_id: None, + fresh_conversation: false, + }, ) - .await, - true, - ) + } else { + step( + "flow", + format!("Starting test run of flow step \"{step_id}\"..."), + schema, + None, + None, + RunTarget::FlowByPath { path: target, chat_enabled: false }, + ) + } } - }; - conclude(ctx, args, prepared.notes, started, &label, is_flow, "test").await + "aiagent" | "aidecision" => step( + "flow", + if module_type == "aiagent" { + format!("Starting test run of agent step \"{step_id}\"...") + } else { + format!("Starting test run of AI decision step \"{step_id}\"...") + }, + None, + None, + None, + RunTarget::Step { + flow_path: path.to_string(), + step: json!({ "id": module["id"], "value": value }), + }, + ), + other => { + let mut patch = Map::new(); + patch.insert( + "content".into(), + json!(format!("Step type \"{other}\" not supported for testing")), + ); + patch.insert( + "error".into(), + json!(format!("Cannot test step of type \"{other}\"")), + ); + ctx.effect(Effect::Status(patch)); + return Err(format!( + "Cannot test step of type \"{other}\". Supported types: rawscript, script, flow, aiagent, aidecision" + )); + } + }) +} + +async fn test_run_step(ctx: &Ctx<'_>, args: &Value) -> ToolResult { + let path = str_arg(args, "path")?; + let step_id = str_arg(args, "stepId")?; + run_tool( + ctx, + args, + resolve_step(ctx, path, step_id), + async |mut resolved: Resolved| { + let module = resolved.module.clone().unwrap_or_default(); + let value = &module["value"]; + // A preprocessor takes the arguments of its own entrypoint, which no schema declares. + let entrypoint = (module["id"] == "preprocessor").then_some("preprocessor"); + let mut proposed = proposed_args(args); + match value["type"].as_str().unwrap_or_default() { + "rawscript" | "script" => { + let (code, lang) = (resolved.code.clone(), resolved.lang.clone()); + resolved.schema = schema_for_test_run( + &lang.unwrap_or_default(), + &code.unwrap_or_default(), + resolved.schema.filter(|_| value["type"] == "script"), + entrypoint, + ); + } + "aiagent" => { + let (keys, form_args) = + agent_step_form(&migrated_agent_value(value), &proposed); + let properties: Map = + keys.into_iter().map(|key| (key, json!({}))).collect(); + resolved.schema = Some(json!({ "properties": properties })); + proposed = form_args; + } + // What a run supplies is the state; the provider and questions are the step's own. + "aidecision" => resolved.schema = Some(json!({ "properties": { "state": {} } })), + // The subflow's own schema, which a deployed one may lack: then nothing conforms. + _ => {} + } + let label = format!("step {step_id}"); + run_resolved_with(ctx, args, path, resolved, proposed, "step", &label, "test").await + }, + ) + .await } fn migrated_agent_value(value: &Value) -> Value { @@ -1721,7 +2131,7 @@ impl<'a> RunTree<'a> { run_overrides: &Map, ) -> Value { let mut run = self.shape_step(0, heads, None); - let steps = run.remove("steps"); + let steps = run.shift_remove("steps"); let kind = self.entries[0]["kind"].as_str().unwrap_or_default(); // The server labels every depth-0 job "Flow", a plain script run included. if steps.is_none() && kind != "flow" && kind != "flowpreview" { @@ -1730,8 +2140,8 @@ impl<'a> RunTree<'a> { // A replacement result supersedes the tree entry's head. Without one the head // stays: it is a real prefix of a payload too big to be read whole. if run_overrides.contains_key("result") { - run.remove("result"); - run.remove("result_total_chars"); + run.shift_remove("result"); + run.shift_remove("result_total_chars"); } for (key, value) in run_overrides { run.insert(key.clone(), value.clone()); @@ -1839,7 +2249,11 @@ async fn read_run(ctx: &Ctx<'_>, args: &Value, card: bool) -> ToolResult { results_route(STEP_RESULT_MAX_CHARS), enc(step) ); - return Ok(step_result(&ctx.get(&route).await?, step)); + let response = ctx.get(&route).await?; + if let Some(job_id) = finished_step_job(&response) { + ctx.effect(Effect::InspectedRun { job_id: job_id.to_string() }); + } + return Ok(step_result(&response, step)); } // Only the job read is load-bearing: logs and the step tree each answer part of @@ -1913,11 +2327,12 @@ async fn read_run(ctx: &Ctx<'_>, args: &Value, card: bool) -> ToolResult { None => report_run(ctx, id, Some(job), details), } } - Ok(format!( - "{}\n\nRun page: {}", - shape_run(id, &job?, logs, &results), - ctx.page_url(&format!("/run/{}", enc(id))) - )) + let run = shape_run(id, &job?, logs, &results); + ctx.effect(Effect::InspectedRun { job_id: id.to_string() }); + Ok(match ctx.link(&format!("/run/{}", enc(id))) { + Some(url) => format!("{run}\n\nRun page: {url}"), + None => run, + }) } /// The job read swaps a result past ~90 KB for a marker, so an image under the card's cap @@ -2478,7 +2893,7 @@ fn run_card(id: &str, job: Option<&Value>) -> Map { /// empty log. fn shape_run(id: &str, job: &Value, logs: Result, results: &Value) -> String { let mut run = summarize_run(job); - run.remove("id"); + run.shift_remove("id"); // Why a run ended badly. for (key, source) in [ ("canceled_by", "canceled_by"), @@ -2524,6 +2939,22 @@ fn shape_run(id: &str, job: &Value, logs: Result, results: &Valu shape_flow_run_tree(results, &run) } +/// The job a step address resolved to, once it finished: what the card renders. +fn finished_step_job(response: &Value) -> Option<&str> { + if response["step_error"] + .as_str() + .is_some_and(|e| !e.is_empty()) + { + return None; + } + let entry = response["entries"].get(0)?; + let status = entry["status"].as_str().unwrap_or_default(); + if matches!(status, "running" | "queued" | "suspended") { + return None; + } + entry["job_id"].as_str() +} + /// One step's result in full, from the single entry the server resolved its address to. fn step_result(response: &Value, step: &str) -> String { if let Some(error) = response["step_error"].as_str().filter(|e| !e.is_empty()) { @@ -2559,6 +2990,51 @@ fn step_result(response: &Value, step: &str) -> String { mod tests { use super::*; + // The chat settles a run's arguments in the browser with `job_args.ts`, read against the + // same cases by `runArgsCases.test.ts`. + #[test] + fn run_args_match_the_shared_cases() { + let cases: Value = serde_json::from_str(include_str!("run_args_cases.json")).unwrap(); + for case in cases["cases"].as_array().unwrap() { + let name = case["name"].as_str().unwrap(); + let schema = &case["schema"]; + let args = case["args"].as_object().cloned().unwrap(); + if let Some(expected) = case.get("declared") { + let (kept, undeclared) = drop_undeclared_args(args.clone(), schema); + assert_eq!(Value::Object(kept), expected["args"], "{name}"); + assert_eq!(json!(undeclared), expected["undeclared"], "{name}"); + } + if let Some(expected) = case.get("disabled_defaults") { + let mut settled = args.clone(); + let reset = enforce_disabled_defaults(&mut settled, schema); + assert_eq!(Value::Object(settled), expected["args"], "{name}"); + assert_eq!(json!(reset), expected["reset"], "{name}"); + } + if let Some(expected) = case.get("secrets") { + let pending = secrets_to_mint(&args, schema); + let mut stored: Vec = pending + .iter() + .map(|(_, prefix, value)| secret_text(prefix, value)) + .collect(); + stored.sort(); + assert_eq!(json!(stored), expected["stored"], "{name}"); + let minted = pending + .into_iter() + .map(|(key, prefix, _)| (key, format!("{prefix}"))) + .collect(); + let out = substitute_secrets(&args, schema, &minted); + assert_eq!(Value::Object(out), expected["args"], "{name}"); + } + if let Some(expected) = case.get("redacted") { + assert_eq!( + Value::Object(redact_args(&args, schema)), + *expected, + "{name}" + ); + } + } + } + fn entry(overrides: Value) -> Value { let mut entry = json!({ "job_id": "j", diff --git a/backend/windmill-api/src/mcp/session/settings.rs b/backend/windmill-api/src/ai_session_tools/settings.rs similarity index 58% rename from backend/windmill-api/src/mcp/session/settings.rs rename to backend/windmill-api/src/ai_session_tools/settings.rs index 3548dada4b..6e3e70d1f1 100644 --- a/backend/windmill-api/src/mcp/session/settings.rs +++ b/backend/windmill-api/src/ai_session_tools/settings.rs @@ -2,8 +2,9 @@ use serde_json::{json, Map, Value}; +use super::chat::Effect; use super::workspace::api_message; -use super::{array, js_head, js_len, js_str, str_arg, Ctx, ToolResult}; +use super::{array, js_head, js_len, js_str, str_arg, Ctx, Surface, ToolResult}; pub(super) const TOOLS: &[&str] = &["get_workspace_settings", "set_workspace_settings"]; @@ -65,37 +66,73 @@ fn parse_settings_yaml(text: &str) -> Result { } } -/// The dry run stands in for the session's confirmation card: it reports a no-op or an -/// invalid document before anything is written. +/// The dry run reports a no-op or an invalid document before anything is written: for MCP +/// it stands in for the session's confirmation card, and the chat runs it alone, as a +/// check-only call, before showing that card. async fn set_workspace_settings(ctx: &Ctx<'_>, args: &Value) -> ToolResult { let route = ctx.ws("workspaces/settings_document"); + let refuse = |label: String, text: String| { + ctx.facts(json!({ "refused": label })); + Ok(text) + }; let document = match parse_settings_yaml(str_arg(args, "yaml")?) { Ok(document) => document, - Err(e) => return Ok(format!("Nothing was changed: {e}")), + Err(e) if ctx.surface == Surface::Chat && !ctx.check_only() => return Err(e), + Err(e) => { + return refuse( + format!("Invalid settings: {e}"), + format!("Nothing was changed: {e}"), + ) + } }; - let dry_run = json!({ "document": document, "replace": false, "dry_run": true }); - let dry_run = match ctx.post(&route, dry_run).await { - Ok(result) => result, - Err(e) => return Ok(format!("Nothing was changed: {}", api_message(&e))), - }; - if array(dry_run.get("changes")).is_empty() { - let warnings: Vec = array(dry_run.get("warnings")).iter().map(js_str).collect(); - let warnings = if warnings.is_empty() { - String::new() - } else { - format!(" Warnings: {}", warnings.join(" ")) + // The chat's own check ran before its card, and the user confirmed what it showed. + if ctx.surface == Surface::Mcp || ctx.check_only() { + let dry_run = json!({ "document": document, "replace": false, "dry_run": true }); + let dry_run = match ctx.post(&route, dry_run).await { + Ok(result) => result, + Err(e) => { + let e = api_message(&e); + return refuse( + format!("Invalid settings: {e}"), + format!("Nothing was changed: {e}"), + ); + } }; - return Ok(format!( - "The settings already hold these values: nothing to change.{warnings}" - )); + if array(dry_run.get("changes")).is_empty() { + let warnings: Vec = array(dry_run.get("warnings")).iter().map(js_str).collect(); + let warnings = if warnings.is_empty() { + String::new() + } else { + format!(" Warnings: {}", warnings.join(" ")) + }; + return refuse( + "No settings change".to_string(), + format!("The settings already hold these values: nothing to change.{warnings}"), + ); + } + if ctx.check_only() { + ctx.facts(json!({ "changes": dry_run["changes"] })); + return Ok(String::new()); + } } let applied = match ctx .post(&route, json!({ "document": document, "replace": false })) .await { Ok(result) => result, - Err(e) => return Ok(json!({ "success": false, "error": api_message(&e) }).to_string()), + Err(e) => { + let error = api_message(&e); + let mut failed = Map::new(); + failed.insert( + "content".into(), + json!("Failed to change workspace settings"), + ); + failed.insert("error".into(), json!(error)); + ctx.effect(Effect::Status(failed)); + return Ok(json!({ "success": false, "error": error }).to_string()); + } }; + ctx.facts(json!({ "changes": applied["changes"] })); let keys: Vec = array(applied.get("changes")) .iter() .map(|c| js_str(c.get("key").unwrap_or(&Value::Null))) @@ -105,7 +142,11 @@ async fn set_workspace_settings(ctx: &Ctx<'_>, args: &Value) -> ToolResult { out.insert( "message".into(), json!(format!( - "These settings are now applied: {}.", + "{} now applied: {}.", + match ctx.surface { + Surface::Chat => "The user approved and these settings are", + Surface::Mcp => "These settings are", + }, keys.join(", ") )), ); diff --git a/backend/windmill-api/src/mcp/session/views.rs b/backend/windmill-api/src/ai_session_tools/views.rs similarity index 96% rename from backend/windmill-api/src/mcp/session/views.rs rename to backend/windmill-api/src/ai_session_tools/views.rs index 474171c2ee..9ecf77d493 100644 --- a/backend/windmill-api/src/mcp/session/views.rs +++ b/backend/windmill-api/src/ai_session_tools/views.rs @@ -3,7 +3,9 @@ //! the other tools, which draw nothing, and calls one of these when it means the user to //! see a result: a host draws a card for every call of a tool that has one. -use serde_json::{json, Value}; +#[cfg(feature = "mcp")] +use serde_json::json; +use serde_json::Value; use super::{deploy, runs, workspace, Ctx, SessionAccess, ToolResult}; @@ -11,7 +13,7 @@ pub(super) const TOOLS: &[&str] = &["show_run", "show_runs", "show_deploy_review /// The review card's one action deploys, which only a full token may do: such a token is /// neither offered the tool nor answered when it calls it by name. -pub(super) fn allowed(name: &str, access: SessionAccess) -> bool { +pub(crate) fn allowed(name: &str, access: SessionAccess) -> bool { name != "show_deploy_review" || access == SessionAccess::Full } @@ -32,7 +34,8 @@ pub(super) async fn call(ctx: &Ctx<'_>, name: &str, args: &Value) -> ToolResult /// Each tool as `(name, description, input schema)`. All only read: the review card's /// Deploy button calls `deploy_workspace_item`, which the token's own access gates. -pub(super) fn definitions() -> [(&'static str, &'static str, Value); 4] { +#[cfg(feature = "mcp")] +pub(crate) fn definitions() -> [(&'static str, &'static str, Value); 4] { [ ( "show_run", diff --git a/backend/windmill-api/src/mcp/session/workspace.rs b/backend/windmill-api/src/ai_session_tools/workspace.rs similarity index 87% rename from backend/windmill-api/src/mcp/session/workspace.rs rename to backend/windmill-api/src/ai_session_tools/workspace.rs index 2eed15c464..80e5da2041 100644 --- a/backend/windmill-api/src/mcp/session/workspace.rs +++ b/backend/windmill-api/src/ai_session_tools/workspace.rs @@ -4,9 +4,10 @@ use std::collections::HashMap; use serde_json::{json, Map, Value}; +use super::chat::Effect; use super::{ array, bool_arg, enc, is_not_found, js_len, js_str, non_null, opt_str_arg, pretty, put, - str_arg, truthy, Ctx, ToolResult, + str_arg, truthy, Ctx, Surface, ToolResult, }; pub(super) const TOOLS: &[&str] = &[ @@ -294,11 +295,17 @@ pub(super) struct Draft { pub created_at: Option, } +/// The caller's own draft named `path`. For the chat, a cell the browser holds is newer +/// than the server's row and wins, the way the editor reads it. pub(super) async fn get_draft( ctx: &Ctx<'_>, kind: &str, path: &str, ) -> Result, String> { + let path = ctx.storage_path(kind, path); + if let Some(value) = ctx.chat.and_then(|chat| chat.cell(kind, path)) { + return Ok(Some(Draft { value: value.clone(), created_at: None })); + } let row = ctx .get(&ctx.ws(&format!("drafts/get_own/{kind}/{}", path_enc(path)))) .await?; @@ -312,6 +319,9 @@ pub(super) async fn get_draft( Ok(Some(Draft { value, created_at })) } +/// Save the caller's draft, or delete it with `None`. False on a conflict with a newer +/// row. The chat saves through its editor's own draft store instead, after the call: it +/// gets the write as an effect, and reports a conflict or failure itself. pub(super) async fn save_draft( ctx: &Ctx<'_>, kind: &str, @@ -320,6 +330,25 @@ pub(super) async fn save_draft( last_sync: Option<&str>, force: bool, ) -> Result { + if ctx.surface == Surface::Chat { + let storage_path = ctx.storage_path(kind, path).to_string(); + ctx.effect(match value { + Some(value) => Effect::DraftWrite { + kind: kind.to_string(), + storage_path, + value: value.clone(), + last_sync: last_sync.map(String::from), + force, + }, + None => Effect::DraftDelete { + kind: kind.to_string(), + storage_path, + last_sync: last_sync.map(String::from), + keep_live_editor: false, + }, + }); + return Ok(true); + } let mut body = Map::new(); body.insert("value".into(), value.cloned().unwrap_or(Value::Null)); if !force { @@ -335,12 +364,24 @@ pub(super) async fn save_draft( Ok(response.get("status").and_then(Value::as_str) != Some("conflict")) } +/// Delete the caller's draft. `keep_live_editor` leaves an editor open on it with its +/// copy, detached from the row, as a deploy does. pub(super) async fn delete_draft( ctx: &Ctx<'_>, kind: &str, path: &str, last_sync: Option<&str>, + keep_live_editor: bool, ) -> Result<(), String> { + if ctx.surface == Surface::Chat { + ctx.effect(Effect::DraftDelete { + kind: kind.to_string(), + storage_path: ctx.storage_path(kind, path).to_string(), + last_sync: last_sync.map(String::from), + keep_live_editor, + }); + return Ok(()); + } match save_draft(ctx, kind, path, None, last_sync, false).await { Ok(true) => Ok(()), Ok(false) => Err(format!( @@ -364,6 +405,7 @@ pub(super) async fn list_drafts(ctx: &Ctx<'_>) -> Result, String> { let Some((item_type, trigger_kind)) = item_type_of_kind(kind) else { continue; }; + let (path, live) = ctx.display_path(kind, path); let mut item = Map::new(); item.insert("type".into(), json!(item_type)); item.insert("path".into(), json!(path)); @@ -380,8 +422,31 @@ pub(super) async fn list_drafts(ctx: &Ctx<'_>) -> Result, String> { if kind == "raw_app" { item.insert("rawApp".into(), json!(true)); } + if live { + item.insert("isLiveDraft".into(), json!(true)); + } drafts.insert(item_key(item_type, path, trigger_kind), Value::Object(item)); } + // The browser's cells carry edits the rows may not have yet, and win. + for cell in ctx + .chat + .map(|chat| chat.cells.as_slice()) + .unwrap_or_default() + { + let Some(Value::Object(mut item)) = + shown_draft_item(ctx, &cell.kind, &cell.storage_path, &cell.value) + else { + continue; + }; + item.shift_remove("value"); + item.shift_remove("schema"); + let text = |key: &str| item.get(key).and_then(Value::as_str).map(String::from); + let (Some(item_type), Some(path)) = (text("type"), text("path")) else { + continue; + }; + let key = item_key(&item_type, &path, text("triggerKind").as_deref()); + drafts.insert(key, Value::Object(item)); + } Ok(drafts.into_iter().map(|(_, item)| item).collect()) } @@ -540,7 +605,24 @@ pub(super) async fn get_draft_item( let Some(draft) = get_draft(ctx, &kind, path).await? else { return Ok(None); }; - Ok(draft_item(&kind, path, &draft.value).map(|item| (item, draft))) + Ok(shown_draft_item(ctx, &kind, path, &draft.value).map(|item| (item, draft))) +} + +/// `draft_item` under the path the user sees: an open editor's own name for its draft, +/// flagged as live. +pub(super) fn shown_draft_item( + ctx: &Ctx<'_>, + kind: &str, + path: &str, + draft: &Value, +) -> Option { + let (shown, live) = ctx.display_path(kind, ctx.storage_path(kind, path)); + let mut item = draft_item(kind, shown, draft)?; + if live { + item.as_object_mut()? + .insert("isLiveDraft".into(), json!(true)); + } + Some(item) } fn stash_inline_script( @@ -766,6 +848,7 @@ pub(super) fn summarize_app_value(value: &Value) -> Value { _ if !by_path => "inline", Some("flow") => "flow", Some("hubscript") => "hubscript", + Some("agent") => "agent", _ => "script", }; out.insert("type".into(), json!(run_type)); @@ -869,7 +952,7 @@ pub(super) fn serialize_for_read(item: &Value, execution_mode: Option) -> let mut value = flow.get("value").cloned().unwrap_or_else(|| json!({})); if let (Some(groups), Some(value)) = (flow.get("groups"), value.as_object_mut()) { if groups.is_null() { - value.remove("groups"); + value.shift_remove("groups"); } else { value.insert("groups".into(), groups.clone()); } @@ -1013,7 +1096,7 @@ pub(super) async fn persist_draft( ) -> Result { let kind = draft_kind(item_type, trigger_kind) .ok_or_else(|| format!("Unsupported draft type \"{item_type}\"."))?; - let item = draft_item(&kind, path, value) + let item = shown_draft_item(ctx, &kind, path, value) .ok_or_else(|| format!("Could not synthesize {item_type} draft \"{path}\"."))?; let failure = match save_draft(ctx, &kind, path, Some(value), last_sync, force).await { Ok(true) => return Ok(Persist::Saved(item)), @@ -1057,8 +1140,9 @@ pub(super) fn finish_draft_write(ctx: &Ctx<'_>, item: &Value, existed: bool) -> json!(format!("{verb} {item_type} \"{path}\" as a per-user draft (saved server-side, visible only to this user — not a deployed workspace item). It was not deployed.")), ); let value = item.get("value").unwrap_or(&Value::Null); + // The chat opens the draft in its own panel; an MCP client only has the link. if let Some(page) = draft_page(item_type, path, value) { - out.insert("url".into(), json!(ctx.page_url(&page))); + put(&mut out, "url", ctx.link(&page).map(Value::from)); } // A flow is not echoed back: the model just sent it. A variable comes back redacted. match item_type { @@ -1246,7 +1330,13 @@ pub(super) fn unified_patch( fn deployed_item(item_type: &str, trigger_kind: Option<&str>, row: &Value) -> Value { let mut item = Map::new(); item.insert("type".into(), json!(item_type)); - put(&mut item, "triggerKind", trigger_kind.map(Value::from)); + put( + &mut item, + "triggerKind", + trigger_kind + .filter(|_| item_type == "trigger") + .map(Value::from), + ); put(&mut item, "path", row.get("path").cloned()); let mut is_draft = false; match item_type { @@ -1313,8 +1403,29 @@ async fn list_workspace_items(ctx: &Ctx<'_>, args: &Value) -> ToolResult { let limit = args.get("limit").and_then(Value::as_u64).unwrap_or(50); let page = args.get("page").and_then(Value::as_u64); let prefix = opt_str_arg(args, "path_prefix"); + // Scoped, the session's folders are the default, one listing each; an explicit + // path_prefix, "" included, overrides them. + let scope_roots: &[String] = match prefix { + None => ctx + .chat + .and_then(|chat| chat.scope_roots.as_deref()) + .unwrap_or_default(), + Some(_) => &[], + }; + let prefixes: Vec> = if scope_roots.is_empty() { + vec![prefix.filter(|p| !p.is_empty()).map(str::to_string)] + } else { + scope_roots.iter().map(|root| Some(format!("{root}/"))).collect() + }; + let per_page = limit.min(MAX_LIST_LIMIT); + let page_number = page.unwrap_or(1).max(1); + // Several folders read as one listing, folder after folder, so `limit` stays one cap per + // type (per trigger kind for triggers): page N is rows (N-1)*limit..N*limit of each + // concatenation, which takes every folder's first N*limit rows. + let window = (prefixes.len() > 1) + .then(|| (page_number - 1) * per_page..page_number * per_page); - let mut query = format!("per_page={}", limit.min(MAX_LIST_LIMIT)); + let mut query = format!("per_page={per_page}"); if let Some(page) = page { query.push_str(&format!("&page={page}")); } @@ -1338,40 +1449,54 @@ async fn list_workspace_items(ctx: &Ctx<'_>, args: &Value) -> ToolResult { } } - if let Some(prefix) = prefix { - query.push_str(&format!("&path_start={}", enc(prefix))); - } - let mut by_key = Map::new(); - for item_type in ITEM_TYPES.iter().filter(|t| types.contains(t)) { - let kinds: Vec> = if *item_type == "trigger" { - TRIGGER_KINDS.iter().map(|kind| Some(*kind)).collect() - } else { - vec![None] - }; - let extra = if matches!(*item_type, "script" | "flow") { - "&include_draft_only=true&without_description=true" - } else { - "" - }; - for trigger_kind in kinds { - let route = ctx.ws(&format!( - "{}/list?{query}{extra}", - service(item_type, trigger_kind)? - )); - let rows = match ctx.get(&route).await { - Ok(rows) => rows, - // A trigger kind this build does not serve answers 404: skip that kind - // alone. Any other failure is real. - Err(e) if trigger_kind.is_some() && is_not_found(&e) => continue, - Err(e) => return Err(e), + // Each type's index into its concatenation across folders, per trigger kind for triggers. + let mut index_by_kind: HashMap = HashMap::new(); + for prefix in &prefixes { + for item_type in ITEM_TYPES.iter().filter(|t| types.contains(t)) { + let kinds: Vec> = if *item_type == "trigger" { + TRIGGER_KINDS.iter().map(|kind| Some(*kind)).collect() + } else { + vec![None] }; - for row in array(Some(&rows)) { - let path = row.get("path").and_then(Value::as_str).unwrap_or_default(); - by_key.insert( - item_key(item_type, path, trigger_kind), - deployed_item(item_type, trigger_kind, row), - ); + let extra = if matches!(*item_type, "script" | "flow") { + "&include_draft_only=true&without_description=true" + } else { + "" + }; + for trigger_kind in kinds { + let mut query = match &window { + Some(window) => format!("per_page={}", window.end), + None => query.clone(), + }; + if let Some(prefix) = prefix { + query.push_str(&format!("&path_start={}", enc(prefix))); + } + let route = ctx.ws(&format!( + "{}/list?{query}{extra}", + service(item_type, trigger_kind)? + )); + let rows = match ctx.get(&route).await { + Ok(rows) => rows, + // A trigger kind this build does not serve answers 404: skip that kind + // alone. Any other failure is real. + Err(e) if trigger_kind.is_some() && is_not_found(&e) => continue, + Err(e) => return Err(e), + }; + let index = index_by_kind + .entry(format!("{item_type}:{}", trigger_kind.unwrap_or_default())) + .or_default(); + for row in array(Some(&rows)) { + *index += 1; + if window.as_ref().is_some_and(|w| !w.contains(&(*index - 1))) { + continue; + } + let path = row.get("path").and_then(Value::as_str).unwrap_or_default(); + by_key.insert( + item_key(item_type, path, trigger_kind), + deployed_item(item_type, trigger_kind, row), + ); + } } } } @@ -1380,6 +1505,7 @@ async fn list_workspace_items(ctx: &Ctx<'_>, args: &Value) -> ToolResult { // pages never repeat a page-1 item as its draft twin. if page.unwrap_or(1) == 1 { let mut count_by_type: HashMap = HashMap::new(); + let narrowing: Vec<&str> = prefixes.iter().flatten().map(String::as_str).collect(); for draft in list_drafts(ctx).await? { let text = |key: &str| draft.get(key).and_then(Value::as_str); let (item_type, path) = ( @@ -1391,11 +1517,13 @@ async fn list_workspace_items(ctx: &Ctx<'_>, args: &Value) -> ToolResult { } // A draft's staged name is often not where it is stored, and this pass is // the only source of draft-only rows under a prefix: either name may match. - if let Some(prefix) = prefix { - let staged = text("draftPath").is_some_and(|p| p.starts_with(prefix)); - if !path.starts_with(prefix) && !staged { - continue; - } + let staged = text("draftPath"); + if !narrowing.is_empty() + && !narrowing.iter().any(|prefix| { + path.starts_with(prefix) || staged.is_some_and(|p| p.starts_with(prefix)) + }) + { + continue; } let count = count_by_type.entry(item_type.to_string()).or_default(); if *count >= limit { @@ -1425,7 +1553,15 @@ async fn list_workspace_items(ctx: &Ctx<'_>, args: &Value) -> ToolResult { }) }) .collect(); - Ok(pretty(&Value::Array(results))) + let listed = pretty(&Value::Array(results)); + if scope_roots.is_empty() { + return Ok(listed); + } + let folders: Vec<&str> = prefixes.iter().flatten().map(String::as_str).collect(); + Ok(format!( + "Items under {}, where this session works. Pass path_prefix \"\" for the whole workspace.\n{listed}", + folders.join(", ") + )) } /// The card of one workspace item: what `read_workspace_item` returns, under a one-line @@ -1663,8 +1799,28 @@ async fn write_script_draft( content: &str, force: bool, ) -> ToolResult { + // A session builds pipeline nodes into the folder's pipeline draft; a script draft + // written here never reaches the pipeline graph, and saving the pipeline drops it. + if ctx.in_session() && windmill_common::assets::parse_pipeline_annotations(content).in_pipeline + { + let folder = path + .strip_prefix("f/") + .and_then(|rest| rest.split_once('/')) + .map_or("", |(folder, _)| folder); + return Err(format!( + "\"{path}\" is a data pipeline node (its source starts with the pipeline annotation). In a session, pipeline nodes are not script drafts: call open_preview(kind=\"pipeline\", path=\"{folder}\") if that folder's pipeline editor is not open yet, then use build_pipeline_node (new node) or edit_pipeline_node (existing node). Nothing was written." + )); + } + ctx.saving_draft("script", path); let base = write_base(ctx, "script", path, None).await?; let existed = base.base.is_some(); + let before = base + .base + .as_ref() + .and_then(|b| b.get("content")) + .and_then(Value::as_str) + .unwrap_or_default() + .to_string(); let mut draft = match base.base { Some(Value::Object(mut script)) => { if !base.from_draft { @@ -1697,17 +1853,21 @@ async fn write_script_draft( }; // Inferred into the schema the base carries: its properties are the only copy of // what no parser emits (`password`, enums, formats, titles). Code that does not - // parse keeps the schema as it was. - let previous = draft - .get("schema") - .filter(|schema| truthy(schema.get("properties"))) - .cloned() - .unwrap_or_else(empty_schema); - let code = if content.is_empty() { " " } else { content }; - // A language this build cannot parse, or code that does not parse yet, keeps the - // schema the draft had. - let schema = infer_script_schema(language, code, Some(&previous), None).unwrap_or(previous); - draft.insert("schema".into(), schema); + // parse keeps the schema as it was. The chat infers with the editor's own parsers + // before saving, so its schema matches what the editor shows. + if ctx.surface == Surface::Mcp { + let previous = draft + .get("schema") + .filter(|schema| truthy(schema.get("properties"))) + .cloned() + .unwrap_or_else(empty_schema); + let code = if content.is_empty() { " " } else { content }; + // A language this build cannot parse, or code that does not parse yet, keeps + // the schema the draft had. + let schema = infer_script_schema(language, code, Some(&previous), None).unwrap_or(previous); + draft.insert("schema".into(), schema); + } + ctx.code_diff(&before, content, Some(language)); let persisted = persist_draft( ctx, "script", @@ -1755,7 +1915,11 @@ async fn discard_local_draft(ctx: &Ctx<'_>, args: &Value) -> ToolResult { let not_found = || format!("No draft found for {item_type} \"{path}\"."); let kind = draft_kind(item_type, trigger_kind).ok_or_else(not_found)?; let draft = get_draft(ctx, &kind, path).await?.ok_or_else(not_found)?; - delete_draft(ctx, &kind, path, draft.created_at.as_deref()).await?; + delete_draft(ctx, &kind, path, draft.created_at.as_deref(), false).await?; + ctx.status_after_save( + format!("Discarded {item_type} \"{path}\" draft"), + Some("Draft discarded"), + ); let mut out = Map::new(); out.insert("success".into(), json!(true)); out.insert( @@ -1826,18 +1990,18 @@ async fn rebase_draft(ctx: &Ctx<'_>, args: &Value) -> ToolResult { .as_ref() .map_or_else(|| "undefined".to_string(), js_str); if base.is_some() && base == head { + let message = format!( + "Draft \"{path}\" is already based on the latest deployed version ({head_text})." + ); + ctx.status(message.clone()); let mut out = Map::new(); out.insert("success".into(), json!(true)); out.insert("alreadyLatest".into(), json!(true)); put(&mut out, head_key, head); - out.insert( - "message".into(), - json!(format!( - "Draft \"{path}\" is already based on the latest deployed version ({head_text})." - )), - ); + out.insert("message".into(), json!(message)); return Ok(Value::Object(out).to_string()); } + ctx.status(format!("Rebasing draft \"{path}\" onto latest...")); // Captured before the discard: this diff is the only record of what to replay. A // base version that can no longer be fetched diffs against nothing, so the whole @@ -1928,7 +2092,7 @@ async fn rebase_draft(ctx: &Ctx<'_>, args: &Value) -> ToolResult { let result = pretty(&Value::Object(out)); // The diff is the only copy of the draft once it is discarded: a result the server // would truncate must keep the draft instead. - if result.len() > windmill_mcp::server::MAX_TOOL_RESULT_BYTES { + if ctx.surface == Surface::Mcp && result.len() > super::MAX_TOOL_RESULT_BYTES { let editor = match item_type { "script" => "/scripts/edit", "flow" => "/flows/edit", @@ -1940,7 +2104,8 @@ async fn rebase_draft(ctx: &Ctx<'_>, args: &Value) -> ToolResult { )); } let kind = draft_kind(item_type, None).ok_or_else(no_draft)?; - delete_draft(ctx, &kind, path, draft.created_at.as_deref()).await?; + delete_draft(ctx, &kind, path, draft.created_at.as_deref(), false).await?; + ctx.status_after_save(format!("Discarded stale draft \"{path}\""), Some("Rebased")); Ok(result) } diff --git a/backend/windmill-api/src/lib.rs b/backend/windmill-api/src/lib.rs index dc78b5c991..73a3451b59 100644 --- a/backend/windmill-api/src/lib.rs +++ b/backend/windmill-api/src/lib.rs @@ -208,12 +208,13 @@ mod variables; pub mod volumes_ee; mod volumes_oss; pub mod webhook_util; +mod workspace_settings_document; mod workspaces; #[cfg(feature = "private")] pub mod workspaces_ee; mod workspaces_export; -mod workspace_settings_document; +mod ai_session_tools; #[cfg(feature = "mcp")] mod mcp_tools; @@ -780,6 +781,10 @@ pub async fn run_server( Router::new() }) .nest("/ai", ai::workspaced_service()) + .nest( + "/ai_session_tools", + ai_session_tools::chat::workspaced_service(_base_internal_url.clone()), + ) .nest("/npm_proxy", windmill_api_npm_proxy::workspaced_service()) .nest( "/path_autocomplete", @@ -1287,8 +1292,7 @@ pub async fn run_server( if let Some(name) = name.as_ref() { tracing::info!("server starting for name={name}"); } - #[cfg(feature = "mcp")] - mcp::register_router(&_base_internal_url, Some(app.clone())); + ai_session_tools::register_router(&_base_internal_url, Some(app.clone())); let listener = listener.tap_io(move |tcp_stream| { let _ = tcp_stream.set_nodelay(!server_mode); @@ -1328,8 +1332,7 @@ pub async fn run_server( }); let served = server.await; - #[cfg(feature = "mcp")] - mcp::register_router(&_base_internal_url, None); + ai_session_tools::register_router(&_base_internal_url, None); served?; let forks = &windmill_api_workspaces::workspaces::BACKGROUND_FORKS; forks.close(); diff --git a/backend/windmill-api/src/mcp/session/card.html b/backend/windmill-api/src/mcp/card.html similarity index 100% rename from backend/windmill-api/src/mcp/session/card.html rename to backend/windmill-api/src/mcp/card.html diff --git a/backend/windmill-api/src/mcp/core.rs b/backend/windmill-api/src/mcp/core.rs index a67ce3d331..7055aada3d 100644 --- a/backend/windmill-api/src/mcp/core.rs +++ b/backend/windmill-api/src/mcp/core.rs @@ -83,15 +83,7 @@ impl WindmillBackend { w_id: &'a str, access: SessionAccess, ) -> session::Ctx<'a> { - session::Ctx { - auth, - w_id, - base_internal_url: &self.base_internal_url, - access, - origin: None, - jwt: Default::default(), - view: Default::default(), - } + session::Ctx::new(auth, w_id, &self.base_internal_url, session::access(access)) } pub fn new( @@ -507,7 +499,8 @@ impl McpBackend for WindmillBackend { access: SessionAccess, origin: Option<&str>, ) -> BackendResult { - let ctx = session::Ctx { origin, ..self.session_ctx_with(auth, workspace_id, access) }; + let mut ctx = self.session_ctx_with(auth, workspace_id, access); + ctx.origin = origin; session::call_session_tool(&ctx, name, args).await } } diff --git a/backend/windmill-api/src/mcp/mod.rs b/backend/windmill-api/src/mcp/mod.rs index 05449d1b93..6052e4d60f 100644 --- a/backend/windmill-api/src/mcp/mod.rs +++ b/backend/windmill-api/src/mcp/mod.rs @@ -16,4 +16,3 @@ pub use core::{ extract_and_store_workspace_id, extract_workspace_from_token, list_tools_service, reject_token_query_param, setup_mcp_server, toolset_service, McpEndpoint, }; -pub(crate) use session::register_router; diff --git a/backend/windmill-api/src/mcp/session.rs b/backend/windmill-api/src/mcp/session.rs new file mode 100644 index 0000000000..0d1ee40b9f --- /dev/null +++ b/backend/windmill-api/src/mcp/session.rs @@ -0,0 +1,175 @@ +//! The AI session's toolset (`crate::ai_session_tools`) as MCP serves it. + +use serde_json::{json, Map, Value}; +use windmill_mcp::server::{ + BackendResult, CallToolResult, ContentBlock, ErrorData, MetaObject, SessionAccess, Tool, + ToolAnnotations, +}; + +use crate::ai_session_tools::{self as tools, CallRefused}; + +pub(super) use tools::{ + generic_session_instructions, session_instructions, workspace_context, Ctx, +}; + +pub(super) fn access(access: SessionAccess) -> tools::SessionAccess { + match access { + SessionAccess::ReadOnly => tools::SessionAccess::ReadOnly, + SessionAccess::ReadAndRun => tools::SessionAccess::ReadAndRun, + SessionAccess::Full => tools::SessionAccess::Full, + } +} + +/// Claude clients keep an MCP result in context only up to a token budget well under the +/// authoring references `get_instructions` returns, unless the tool raises it. Above the +/// runner's own cap on a result, so that cap is the one that applies. +fn result_size_meta() -> MetaObject { + let mut meta = Map::new(); + meta.insert("anthropic/maxResultSizeChars".to_string(), 100_000.into()); + MetaObject(meta) +} + +/// The destructive tools a card may call, each from a button the user presses. Every +/// other destructive tool is hidden from cards: a card acts as the user with no one +/// confirming. +const CARD_ACTIONS: &[&str] = &["deploy_workspace_item", "cancel_job"]; + +/// A tool's display name, which Claude shows in its tool list and approval prompts. +fn tool_title(name: &str) -> String { + let words: Vec = name + .split('_') + .map(|word| match word { + "db" | "sql" | "json" => word.to_uppercase(), + _ => word.to_string(), + }) + .collect(); + let title = words.join(" "); + let mut chars = title.chars(); + chars + .next() + .map(|first| first.to_uppercase().chain(chars).collect()) + .unwrap_or_default() +} + +/// The MCP Apps view a client that renders them draws these tools' results in. It draws +/// what the tool put in its result's `_meta` (`Ctx::view`), and acts by calling tools. +/// Hosts cache a view by its URI: bump the version when a change breaks the `_meta` shape +/// the deployed card reads. +pub(super) const CARD_URI: &str = "ui://windmill/card-v1"; +const CARD_HTML: &str = include_str!("card.html"); + +/// The `ui://` views the session's tools render in, as `(uri, html)`. +pub(super) fn session_views() -> &'static [(&'static str, &'static str)] { + &[(CARD_URI, CARD_HTML)] +} + +pub(super) fn session_tools(session_access: SessionAccess) -> Vec { + let access = access(session_access); + tools::session_tools(access) + .map(|tool| { + let mut meta = result_size_meta(); + if tool.destructive && !CARD_ACTIONS.contains(&tool.name) { + meta.0 + .insert("ui".into(), json!({ "visibility": ["model"] })); + } + let title = tool_title(tool.name); + Tool::new(tool.name, tool.description, tool.parameters.clone()) + .with_title(title.clone()) + .with_annotations( + ToolAnnotations::with_title(title) + .read_only(tool.read_only) + .destructive(tool.destructive) + .open_world(tool.open_world), + ) + .with_meta(meta) + }) + .chain( + tools::view_definitions() + .into_iter() + .filter(|(name, ..)| tools::view_allowed(name, access)) + .map(|(name, description, schema)| { + let mut meta = result_size_meta(); + meta.0 + .insert("ui".into(), json!({ "resourceUri": CARD_URI })); + let Value::Object(schema) = schema else { + unreachable!("a view tool's schema is an object") + }; + let title = tool_title(name); + Tool::new(name, description, schema) + .with_title(title.clone()) + .with_annotations( + ToolAnnotations::with_title(title) + .read_only(true) + .destructive(false) + .open_world(false), + ) + .with_meta(meta) + }), + ) + .collect() +} + +pub(super) async fn call_session_tool( + ctx: &Ctx<'_>, + name: &str, + args: Value, +) -> BackendResult { + let outcome = + tools::call_session_tool(ctx, name, args) + .await + .map_err(|refused| match refused { + CallRefused::NotAllowed(message) => ErrorData::invalid_request(message, None), + CallRefused::UnknownTool(message) | CallRefused::NotAvailable(message) => { + ErrorData::invalid_params(message, None) + } + })?; + windmill_common::feature_usage::log_feature_usage("mcp_workspace", "tool_call", name); + let content = vec![ContentBlock::text(outcome.text)]; + let mut result = if outcome.is_error { + CallToolResult::error(content) + } else { + CallToolResult::success(content) + }; + if let Some((key, view)) = ctx.view.lock().unwrap().take() { + result.meta = Some(MetaObject(Map::from_iter([(key.to_string(), view)]))); + } + Ok(result) +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn no_card_can_call_a_destructive_tool_but_deploy_and_cancel() { + for tool in session_tools(SessionAccess::Full) { + let name = tool.name.as_ref(); + let destructive = tool.annotations.as_ref().and_then(|a| a.destructive_hint); + if destructive == Some(true) { + let ui = tool.meta.as_ref().and_then(|m| m.0.get("ui")); + let model_only = ui == Some(&json!({ "visibility": ["model"] })); + assert_eq!( + model_only, + !matches!(name, "deploy_workspace_item" | "cancel_job"), + "{name}" + ); + } + } + } + + /// The ChatGPT directory wants every hint explicit, and Claude's a title on every tool. + #[test] + fn every_session_tool_has_a_title_and_explicit_hints() { + for tool in session_tools(SessionAccess::Full) { + let a = tool.annotations.as_ref().expect("annotations"); + assert!(tool.title.is_some() && a.title.is_some(), "{}", tool.name); + assert!( + a.read_only_hint.is_some() + && a.destructive_hint.is_some() + && a.open_world_hint.is_some(), + "{}", + tool.name + ); + } + } +} diff --git a/backend/windmill-api/src/mcp/utils.rs b/backend/windmill-api/src/mcp/utils.rs index 04963c8bd7..ba13efdb70 100644 --- a/backend/windmill-api/src/mcp/utils.rs +++ b/backend/windmill-api/src/mcp/utils.rs @@ -9,8 +9,7 @@ use axum::body::{to_bytes, Body}; use axum::response::Response; use serde_json::Value; use sql_builder::prelude::*; -use windmill_common::auth::create_jwt_token; -use windmill_common::db::{Authed, UserDB}; +use windmill_common::db::UserDB; use windmill_common::error::Error; use windmill_common::runnable_ref::RunnableRef; use windmill_common::scripts::{get_full_hub_script_by_path, Schema}; @@ -214,9 +213,7 @@ fn items_query( } else { let conds: Vec = segments .iter() - .map(|s| { - "o.path LIKE ? ESCAPE '\\'".bind(&format!("%/{}", escape_like(s))) - }) + .map(|s| "o.path LIKE ? ESCAPE '\\'".bind(&format!("%/{}", escape_like(s)))) .collect(); sqlb.and_where(format!("({})", conds.join(" OR "))); } @@ -711,28 +708,6 @@ pub async fn create_http_request( send_as_caller(method, url, workspace_id, api_authed, body_json, scopes).await } -/// A JWT that acts as `api_authed`, carrying `scopes`; `None` is the caller's full rights. -pub(crate) async fn mint_caller_jwt( - workspace_id: &str, - api_authed: &ApiAuthed, - scopes: Option>, -) -> BackendResult { - // Carry the caller's job provenance into the JWT: a job's WM_TOKEN is capped at - // workspace admin (GHSA-hfh4-cx4h-3fcr), and dropping `job_id` here would re-mint an - // uncapped token that satisfies require_super_admin / require_devops_role. - create_jwt_token( - Authed::from(api_authed.clone()), - workspace_id, - 3600, - api_authed.job_id, - None, - None, - scopes, - ) - .await - .map_err(|e| ErrorData::internal_error(e.to_string(), None)) -} - /// Call the API as `api_authed` with a JWT carrying `scopes`; `None` is the caller's /// full rights, for a server whose own code picks every route it calls. pub async fn send_as_caller( @@ -758,7 +733,9 @@ pub async fn send_as_caller( } }; - let token = mint_caller_jwt(workspace_id, api_authed, scopes).await?; + let token = crate::ai_session_tools::mint_caller_jwt(workspace_id, api_authed, scopes) + .await + .map_err(|e| ErrorData::internal_error(e.to_string(), None))?; request_builder = request_builder.header("Authorization", format!("Bearer {}", token)); // Add body if present diff --git a/frontend/src/lib/components/copilot/chat/datatableMigrationTools.test.ts b/frontend/src/lib/components/copilot/chat/datatableMigrationTools.test.ts deleted file mode 100644 index 086adce9e9..0000000000 --- a/frontend/src/lib/components/copilot/chat/datatableMigrationTools.test.ts +++ /dev/null @@ -1,141 +0,0 @@ -import { beforeEach, describe, expect, it, vi } from 'vitest' - -const { statusMock, createMock, upsertMock, runMock } = vi.hoisted(() => ({ - upsertMock: vi.fn(), - statusMock: vi.fn(), - createMock: vi.fn(), - runMock: vi.fn() -})) - -vi.mock('./shared', () => ({ - createToolDef: (_schema: unknown, name: string, description: string) => ({ - type: 'function', - function: { name, description, parameters: {} } - }) -})) - -vi.mock('$lib/gen', () => ({ - WorkspaceService: { - getDatatableMigrationsStatus: statusMock, - createDatatableMigration: createMock, - upsertDatatableMigration: upsertMock, - runDatatableMigrations: runMock - } -})) - -import { getDatatableMigrationTools } from './datatableMigrationTools' - -function create(args: Record) { - const tool = getDatatableMigrationTools().find( - (entry) => entry.def.function.name === 'create_datatable_migration' - )! - return tool.fn({ - args: { - datatable_name: 'main', - name: 'add_refunds', - code_up: 'BEGIN;\nCREATE TABLE refunds ();\nEND;', - ...args - }, - workspace: 'test-workspace', - helpers: {}, - toolCallbacks: { setToolStatus: vi.fn(), removeToolStatus: vi.fn() } as any, - toolId: 'tool-1' - }) -} - -const applied = { timestamp: 10, name: 'init', code_up: '', status: 'ran' } -const created = { timestamp: 30, name: 'add_refunds', code_up: '', status: 'not_run' } - -beforeEach(() => { - for (const mock of [statusMock, createMock, upsertMock, runMock]) mock.mockReset() - createMock.mockResolvedValue({ timestamp: 30 }) - statusMock.mockResolvedValue({ enabled: true, migrations: [applied, created] }) - runMock.mockResolvedValue({ applied: [{ version: 30, name: 'add_refunds' }] }) -}) - -describe('create_datatable_migration', () => { - it('applies only the migration it created', async () => { - const result = await create({}) - expect(runMock).toHaveBeenCalledWith({ - workspace: 'test-workspace', - datatableName: 'main', - only: 30 - }) - expect(result).toContain('Applied migration 30 (add_refunds)') - }) - - it('leaves it pending when an earlier migration is not applied', async () => { - statusMock.mockResolvedValue({ - enabled: true, - migrations: [ - applied, - { timestamp: 20, name: 'earlier', code_up: '', status: 'not_run' }, - created - ] - }) - const result = await create({}) - expect(runMock).not.toHaveBeenCalled() - expect(result).toContain('20 (earlier)') - }) - - it('keeps a migration that fails to apply and rewrites it on request', async () => { - runMock.mockRejectedValueOnce({ body: 'relation "refunds" already exists' }) - const failed = await create({}) - expect(failed).toContain('relation "refunds" already exists') - expect(failed).toContain('with version 30') - - statusMock.mockResolvedValue({ - enabled: true, - migrations: [applied, { ...created, code_down: 'DROP TABLE refunds;' }] - }) - await create({ version: 30, code_up: 'BEGIN;\nCREATE TABLE refunds2 ();\nEND;' }) - expect(createMock).toHaveBeenCalledTimes(1) - expect(upsertMock).toHaveBeenCalledWith({ - workspace: 'test-workspace', - datatableName: 'main', - requestBody: { - timestamp: 30, - code_down: 'DROP TABLE refunds;', - name: 'add_refunds', - code_up: 'BEGIN;\nCREATE TABLE refunds2 ();\nEND;' - } - }) - expect(runMock).toHaveBeenLastCalledWith({ - workspace: 'test-workspace', - datatableName: 'main', - only: 30 - }) - }) - - it('refuses to rewrite a version that is not pending', async () => { - const result = await create({ version: 10 }) - expect(upsertMock).not.toHaveBeenCalled() - expect(runMock).not.toHaveBeenCalled() - expect(result).toContain('cannot be rewritten') - }) - - it('does not call unreadable earlier migrations pending', async () => { - statusMock.mockResolvedValue({ - enabled: true, - migrations: [ - { ...applied, status: 'unknown' }, - { ...created, status: 'unknown' } - ], - error: 'connection refused' - }) - const result = await create({}) - expect(runMock).not.toHaveBeenCalled() - expect(result).toContain('connection refused') - expect(result).not.toContain('10 (init)') - }) - - it('declares the role above the SQL of both directions', async () => { - await create({ role: 'app_rw', code_down: 'DROP TABLE refunds;', apply: false }) - expect(createMock.mock.calls[0][0].requestBody).toEqual({ - name: 'add_refunds', - code_up: '-- role app_rw\nBEGIN;\nCREATE TABLE refunds ();\nEND;', - code_down: '-- role app_rw\nDROP TABLE refunds;' - }) - expect(runMock).not.toHaveBeenCalled() - }) -}) diff --git a/frontend/src/lib/components/copilot/chat/datatableMigrationTools.ts b/frontend/src/lib/components/copilot/chat/datatableMigrationTools.ts index 6e5ec4f8a7..8a96c05423 100644 --- a/frontend/src/lib/components/copilot/chat/datatableMigrationTools.ts +++ b/frontend/src/lib/components/copilot/chat/datatableMigrationTools.ts @@ -1,14 +1,7 @@ import { z } from 'zod' -import { WorkspaceService } from '$lib/gen' -import { withMigrationRole } from '$lib/components/datatableMigrationRole' -import { - migrationScope, - parseMigrationSchemas, - scopesOverlap, - withMigrationSchemas -} from '$lib/components/datatableMigrationSchemas' -import { createToolDef, type ToolCallbacks } from './shared' +import { createToolDef } from './shared' import { NONE, RUN_PREVIEW, type SessionTool } from './sessionCapabilities' +import { runServerTool } from './global/backendSessionTools' /** * Tools over a data table's versioned SQL migrations (`datatable_migrations.rs`). @@ -23,21 +16,6 @@ const memo = (factory: () => T): (() => T) => { return () => (cached ??= factory()) } -function apiErrorMessage(e: unknown): string { - const err = e as { body?: unknown; message?: unknown } | undefined - const body = err?.body - if (typeof body === 'string' && body !== '') return body - if (typeof err?.message === 'string') return err.message - return String(e) -} - -function fail(toolCallbacks: ToolCallbacks, toolId: string, message: string): string { - toolCallbacks.setToolStatus(toolId, { content: message, error: message }) - return message -} - -const migrationLabel = (m: { version: number; name: string }) => `${m.version} (${m.name})` - const datatableNameSchema = () => z.string().describe('The datatable the migrations belong to, e.g. "main".') @@ -146,66 +124,12 @@ export function getDatatableMigrationTools(): SessionTool<{}>[] { requires: NONE, def: getListMigrationsToolDef(), planModeSafe: true, - fn: async ({ args, workspace, toolId, toolCallbacks }) => { - const { datatable_name, version } = getListMigrationsSchema().parse(args) - toolCallbacks.setToolStatus(toolId, { + fn: async (ctx) => { + const { datatable_name } = getListMigrationsSchema().parse(ctx.args) + ctx.toolCallbacks.setToolStatus(ctx.toolId, { content: `Listing migrations of "${datatable_name}"...` }) - try { - const status = await WorkspaceService.getDatatableMigrationsStatus({ - workspace, - datatableName: datatable_name - }) - if (!status.enabled) { - toolCallbacks.setToolStatus(toolId, { - content: `Migrations are not enabled on "${datatable_name}"` - }) - return `Migrations are not enabled on datatable "${datatable_name}": its schema is changed with plain SQL and there is no history to list. Enabling them is a workspace admin's decision, made in Workspace settings → Data Tables.` - } - if (version !== undefined) { - const migration = status.migrations.find((m) => m.timestamp === version) - if (!migration) { - return fail( - toolCallbacks, - toolId, - `No migration ${version} on datatable "${datatable_name}"` - ) - } - toolCallbacks.setToolStatus(toolId, { - content: `Read migration ${migrationLabel({ version, name: migration.name })}` - }) - return JSON.stringify( - { - version, - name: migration.name, - status: migrationStatus(migration.status), - code_up: migration.code_up, - ...(migration.code_down != null && { code_down: migration.code_down }) - }, - null, - 2 - ) - } - toolCallbacks.setToolStatus(toolId, { - content: `Listed ${status.migrations.length} migration(s) of "${datatable_name}"` - }) - return JSON.stringify( - { - migrations: status.migrations.map((m) => ({ - version: m.timestamp, - name: m.name, - status: migrationStatus(m.status), - schemas: m.schemas ?? 'whole database' - })), - // Set when the database could not be read, which is why every status is unknown. - ...(status.error && { error: status.error }) - }, - null, - 2 - ) - } catch (e) { - return fail(toolCallbacks, toolId, `Error listing migrations: ${apiErrorMessage(e)}`) - } + return (await runServerTool('list_datatable_migrations', ctx)).text } }, { @@ -218,124 +142,12 @@ export function getDatatableMigrationTools(): SessionTool<{}>[] { `${args?.version === undefined ? 'Create' : 'Rewrite'}${args?.apply === false ? '' : ' and apply'} data table migration`, queuedLabel: (args) => `Write migration "${args?.name ?? ''}"`, showDetails: true, - fn: async ({ args, workspace, toolId, toolCallbacks }) => { - const parsed = getCreateMigrationSchema().parse(args) - const datatableName = parsed.datatable_name - toolCallbacks.setToolStatus(toolId, { - content: `Saving migration "${parsed.name}" on "${datatableName}"...` + fn: async (ctx) => { + const parsed = getCreateMigrationSchema().parse(ctx.args) + ctx.toolCallbacks.setToolStatus(ctx.toolId, { + content: `Saving migration "${parsed.name}" on "${parsed.datatable_name}"...` }) - // The role is an annotation in the SQL, read from above everything else, and a rollback - // has to run as the role that created what it drops. - // No inference here: the model reads the SQL it wrote better than a pattern does, so - // the scope is what it declares, and the whole database when it declares none. - const declared = parseMigrationSchemas(parsed.code_up) - const scope = parsed.schemas ?? (declared.kind === 'schemas' ? declared.schemas : []) - const definition = { - name: parsed.name, - code_up: withMigrationSchemas( - withRole(parsed.code_up, parsed.role), - scope.map((s) => s.toLowerCase()) - ), - ...(parsed.code_down !== undefined && { - code_down: withRole(parsed.code_down, parsed.role) - }) - } - let version: number - try { - if (parsed.version === undefined) { - const created = await WorkspaceService.createDatatableMigration({ - workspace, - datatableName, - requestBody: definition - }) - version = created.timestamp - } else { - version = parsed.version - // The server inserts at a version it does not know, which would wedge a new - // migration into the middle of the history, and stores a down left out as none. - const { migrations } = await WorkspaceService.getDatatableMigrationsStatus({ - workspace, - datatableName - }) - const existing = migrations.find((m) => m.timestamp === version) - if (existing?.status !== 'not_run') { - return fail( - toolCallbacks, - toolId, - `Migration ${version} is not a pending migration of datatable "${datatableName}", so it cannot be rewritten. Omit \`version\` to add a new migration.` - ) - } - await WorkspaceService.upsertDatatableMigration({ - workspace, - datatableName, - requestBody: { - timestamp: version, - ...(existing.code_down != null && { code_down: existing.code_down }), - ...definition - } - }) - } - } catch (e) { - return fail(toolCallbacks, toolId, `Error saving migration: ${apiErrorMessage(e)}`) - } - const label = migrationLabel({ version, name: parsed.name }) - if (parsed.apply === false) { - toolCallbacks.setToolStatus(toolId, { content: `Saved migration ${label} as pending` }) - return `Saved migration ${label} as pending. It is not applied yet.` - } - - // Applying it alone would run it ahead of migrations it may depend on, and applying - // those with it would run SQL the user was never shown. - let earlierPending: string[] - try { - const status = await WorkspaceService.getDatatableMigrationsStatus({ - workspace, - datatableName - }) - const ownScope = migrationScope(definition.code_up) - const earlier = status.migrations.filter( - (m) => m.timestamp < version && scopesOverlap(m.schemas, ownScope) - ) - // Unreadable is not pending: reporting it so would send the user to apply - // migrations that already ran. - if (status.error || earlier.some((m) => m.status === 'unknown')) { - throw new Error(status.error ?? 'the applied migrations could not be read') - } - earlierPending = earlier - .filter((m) => m.status === 'not_run') - .map((m) => migrationLabel({ version: m.timestamp, name: m.name })) - } catch (e) { - return fail( - toolCallbacks, - toolId, - `Saved migration ${label} as pending, but could not check what else is pending, so it was not applied: ${apiErrorMessage(e)}` - ) - } - if (earlierPending.length > 0) { - toolCallbacks.setToolStatus(toolId, { - content: `Saved migration ${label}; ${earlierPending.length} earlier migration(s) still pending` - }) - return `Saved migration ${label} as pending without applying it: earlier migrations are not applied yet (${earlierPending.join(', ')}) and it may depend on them. Tell the user, then apply them all in order with run_datatable_migrations.` - } - - toolCallbacks.setToolStatus(toolId, { content: `Applying migration ${label}...` }) - try { - await WorkspaceService.runDatatableMigrations({ - workspace, - datatableName, - only: version - }) - } catch (e) { - // Kept whatever failed: part of it may be applied, or all of it with only the record - // missing, and the definition with its down is then the one trace of that. - return fail( - toolCallbacks, - toolId, - `Migration ${label} is saved but failed to apply: ${apiErrorMessage(e)}. It stays pending and blocks later migrations. If nothing of it was applied, fix it by calling this tool again with version ${version}; otherwise, or to drop it, tell the user, who can edit or delete it in Workspace settings → Data Tables.` - ) - } - toolCallbacks.setToolStatus(toolId, { content: `Applied migration ${label}` }) - return `Applied migration ${label} on datatable "${datatableName}".` + return (await runServerTool('create_datatable_migration', ctx)).text } }, { @@ -346,32 +158,12 @@ export function getDatatableMigrationTools(): SessionTool<{}>[] { confirmationMessage: 'Apply pending data table migrations', queuedLabel: (args) => `Apply migrations on "${args?.datatable_name ?? 'datatable'}"`, showDetails: true, - fn: async ({ args, workspace, toolId, toolCallbacks }) => { - const { datatable_name } = getRunMigrationsSchema().parse(args) - toolCallbacks.setToolStatus(toolId, { + fn: async (ctx) => { + const { datatable_name } = getRunMigrationsSchema().parse(ctx.args) + ctx.toolCallbacks.setToolStatus(ctx.toolId, { content: `Applying migrations on "${datatable_name}"...` }) - try { - const { applied } = await WorkspaceService.runDatatableMigrations({ - workspace, - datatableName: datatable_name - }) - if (applied.length === 0) { - toolCallbacks.setToolStatus(toolId, { content: 'No pending migration to apply' }) - return `No pending migration to apply on datatable "${datatable_name}".` - } - toolCallbacks.setToolStatus(toolId, { - content: `Applied ${applied.length} migration(s) on "${datatable_name}"` - }) - return `Applied ${applied.length} migration(s) on datatable "${datatable_name}": ${applied.map(migrationLabel).join(', ')}.` - } catch (e) { - // Those before the failing one are applied: the list is what says where it stopped. - return fail( - toolCallbacks, - toolId, - `Error applying migrations: ${apiErrorMessage(e)}. Migrations before the failing one may have been applied: list them to see.` - ) - } + return (await runServerTool('run_datatable_migrations', ctx)).text } }, { @@ -382,36 +174,13 @@ export function getDatatableMigrationTools(): SessionTool<{}>[] { confirmationMessage: (args) => `Roll back data table migration ${args?.version ?? ''}`, queuedLabel: (args) => `Roll back migration ${args?.version ?? ''}`, showDetails: true, - fn: async ({ args, workspace, toolId, toolCallbacks }) => { - const { datatable_name, version } = getRollbackMigrationSchema().parse(args) - toolCallbacks.setToolStatus(toolId, { + fn: async (ctx) => { + const { datatable_name, version } = getRollbackMigrationSchema().parse(ctx.args) + ctx.toolCallbacks.setToolStatus(ctx.toolId, { content: `Rolling back migration ${version} on "${datatable_name}"...` }) - try { - const { rolled_back } = await WorkspaceService.rollbackDatatableMigrations({ - workspace, - datatableName: datatable_name, - only: version - }) - if (rolled_back.length === 0) { - toolCallbacks.setToolStatus(toolId, { content: `Migration ${version} is not applied` }) - return `Migration ${version} is not applied on datatable "${datatable_name}", so there is nothing to roll back.` - } - const label = rolled_back.map(migrationLabel).join(', ') - toolCallbacks.setToolStatus(toolId, { content: `Rolled back migration ${label}` }) - return `Rolled back migration ${label} on datatable "${datatable_name}". It is pending again.` - } catch (e) { - return fail(toolCallbacks, toolId, `Error rolling back: ${apiErrorMessage(e)}`) - } + return (await runServerTool('rollback_datatable_migration', ctx)).text } } ] } - -function migrationStatus(status: 'ran' | 'not_run' | 'unknown'): 'applied' | 'pending' | 'unknown' { - return status === 'ran' ? 'applied' : status === 'not_run' ? 'pending' : 'unknown' -} - -function withRole(sql: string, role: string | undefined): string { - return role === undefined ? sql : withMigrationRole(sql, role) -} diff --git a/frontend/src/lib/components/copilot/chat/datatableTools.test.ts b/frontend/src/lib/components/copilot/chat/datatableTools.test.ts deleted file mode 100644 index 391510ca52..0000000000 --- a/frontend/src/lib/components/copilot/chat/datatableTools.test.ts +++ /dev/null @@ -1,226 +0,0 @@ -import { beforeEach, describe, expect, it, vi } from 'vitest' - -const { listMock, schemaMock, runScriptMock, executeTestRunMock } = vi.hoisted(() => ({ - listMock: vi.fn(), - schemaMock: vi.fn(), - runScriptMock: vi.fn(), - executeTestRunMock: vi.fn() -})) - -vi.mock('./shared', () => ({ - createToolDef: (_schema: unknown, name: string, description: string) => ({ - type: 'function', - function: { name, description, parameters: {} } - }), - executeTestRun: executeTestRunMock -})) - -vi.mock('$lib/gen', () => ({ - WorkspaceService: { - listDataTableTables: listMock, - getDataTableTableSchema: schemaMock - } -})) - -vi.mock('$lib/components/jobs/utils', () => ({ - runScript: runScriptMock -})) - -// exec_datatable_sql now routes through executeTestRun; stub it to exercise the -// jobStarter (so runScript args can be asserted) and then hand a completed job to -// the tool's formatCompletion — which owns the datatable-specific result shaping. -// The detach/tray orchestration itself is covered by shared.test.ts. -function stubExecuteTestRunWithJob(job: { success: boolean; result: unknown }) { - runScriptMock.mockResolvedValue('job-123') - executeTestRunMock.mockImplementation(async (config: any) => { - await config.jobStarter() - const { llmText, card } = config.formatCompletion(job) - config.toolCallbacks.setToolStatus(config.toolId, card) - return llmText - }) -} - -import { getDatatableTools } from './datatableTools' - -function createToolCallbacks() { - return { - setToolStatus: vi.fn(), - removeToolStatus: vi.fn() - } -} - -function getTool(name: string) { - const tool = getDatatableTools().find((entry) => entry.def.function.name === name) - if (!tool) { - throw new Error(`${name} tool not found`) - } - return tool -} - -function run(name: string, args: Record = {}) { - return getTool(name).fn({ - args, - workspace: 'test-workspace', - helpers: {}, - toolCallbacks: createToolCallbacks(), - toolId: `tool-${name}` - }) -} - -beforeEach(() => { - listMock.mockReset() - schemaMock.mockReset() - runScriptMock.mockReset() - executeTestRunMock.mockReset() -}) - -describe('list_datatables', () => { - it('aggregates the table count across schemas and returns metadata verbatim', async () => { - const metadata = [ - { datatable_name: 'main', schemas: { public: ['users', 'orders'], analytics: ['events'] } }, - { datatable_name: 'warehouse', schemas: { public: ['facts'] } } - ] - listMock.mockResolvedValue(metadata) - - const tool = getTool('list_datatables') - const callbacks = createToolCallbacks() - const result = await tool.fn({ - args: {}, - workspace: 'test-workspace', - helpers: {}, - toolCallbacks: callbacks, - toolId: 'tool-list' - }) - - expect(listMock).toHaveBeenCalledWith({ workspace: 'test-workspace' }) - expect(JSON.parse(result)).toEqual(metadata) - // 2 + 1 + 1 = 4 tables across 2 datatables - expect(callbacks.setToolStatus).toHaveBeenCalledWith('tool-list', { - content: 'Listed 2 datatable(s) with 4 table(s)' - }) - }) - - it('explains that configuring a datatable is a blocking prerequisite when none exist', async () => { - listMock.mockResolvedValue([]) - const result = await run('list_datatables') - expect(result).toContain('No datatables are configured in this workspace') - expect(result).toContain('Workspace settings → Data Tables') - expect(result).toContain('blocked') - expect(result).toContain('Do not call exec_datatable_sql') - }) - - it('surfaces backend errors as a readable message', async () => { - listMock.mockRejectedValue(new Error('boom')) - const result = await run('list_datatables') - expect(result).toContain('Error listing datatables: boom') - }) -}) - -describe('get_datatable_table_schema', () => { - it('returns the columns for one table', async () => { - schemaMock.mockResolvedValue({ - datatable_name: 'main', - schema_name: 'public', - table_name: 'users', - columns: { id: 'int4', email: 'text' } - }) - - const result = await run('get_datatable_table_schema', { - datatable_name: 'main', - schema_name: 'public', - table_name: 'users' - }) - - expect(schemaMock).toHaveBeenCalledWith({ - workspace: 'test-workspace', - datatableName: 'main', - schemaName: 'public', - tableName: 'users' - }) - expect(JSON.parse(result)).toEqual({ - datatable_name: 'main', - schema_name: 'public', - table_name: 'users', - columns: { id: 'int4', email: 'text' } - }) - }) -}) - -describe('exec_datatable_sql', () => { - it('requires confirmation', () => { - expect(getTool('exec_datatable_sql').requiresConfirmation).toBe(true) - }) - - it('starts the query as a postgresql job scoped to the datatable', async () => { - stubExecuteTestRunWithJob({ success: true, result: [] }) - await run('exec_datatable_sql', { datatable_name: 'main', sql: 'SELECT * FROM t' }) - expect(runScriptMock).toHaveBeenCalledWith({ - workspace: 'test-workspace', - requestBody: { - language: 'postgresql', - content: 'SELECT * FROM t', - args: { database: 'datatable://main' } - } - }) - }) - - it('returns all rows when the result is at or below the cap', async () => { - const rows = Array.from({ length: 100 }, (_, i) => ({ id: i })) - stubExecuteTestRunWithJob({ success: true, result: rows }) - - const parsed = JSON.parse( - await run('exec_datatable_sql', { datatable_name: 'main', sql: 'SELECT * FROM t' }) - ) - expect(parsed.success).toBe(true) - expect(parsed.rowCount).toBe(100) - expect(parsed.result).toHaveLength(100) - expect(parsed.note).toBeUndefined() - }) - - it('truncates results above the cap and reports the full count', async () => { - const rows = Array.from({ length: 150 }, (_, i) => ({ id: i })) - stubExecuteTestRunWithJob({ success: true, result: rows }) - - const parsed = JSON.parse( - await run('exec_datatable_sql', { datatable_name: 'main', sql: 'SELECT 1' }) - ) - expect(parsed.rowCount).toBe(150) - expect(parsed.result).toHaveLength(100) - expect(parsed.note).toBe('Showing first 100 of 150 rows') - }) - - it('treats a non-array result (e.g. DDL) as zero rows', async () => { - stubExecuteTestRunWithJob({ success: true, result: undefined }) - const parsed = JSON.parse( - await run('exec_datatable_sql', { - datatable_name: 'main', - sql: 'CREATE TABLE t (id serial primary key)' - }) - ) - expect(parsed).toEqual({ success: true, rowCount: 0, result: [] }) - }) - - it('returns a failure object when the SQL job errors', async () => { - stubExecuteTestRunWithJob({ success: false, result: { error: { message: 'syntax error' } } }) - const parsed = JSON.parse( - await run('exec_datatable_sql', { datatable_name: 'main', sql: 'SLECT' }) - ) - expect(parsed).toEqual({ success: false, error: 'syntax error' }) - }) - - it('turns the backend "datatable not found" error into an actionable, blocking message', async () => { - stubExecuteTestRunWithJob({ - success: false, - result: { error: { message: 'Internal: datatable main not found @workspaces.rs:565:20' } } - }) - const parsed = JSON.parse( - await run('exec_datatable_sql', { datatable_name: 'main', sql: 'CREATE TABLE t (id int)' }) - ) - expect(parsed.success).toBe(false) - expect(parsed.error).toContain('not configured in this workspace') - expect(parsed.error).toContain('Workspace settings → Data Tables') - expect(parsed.error).toContain('do not retry') - // The raw internal error is not surfaced to the model. - expect(parsed.error).not.toContain('workspaces.rs') - }) -}) diff --git a/frontend/src/lib/components/copilot/chat/datatableTools.ts b/frontend/src/lib/components/copilot/chat/datatableTools.ts index 097d64d06e..97b1524fb0 100644 --- a/frontend/src/lib/components/copilot/chat/datatableTools.ts +++ b/frontend/src/lib/components/copilot/chat/datatableTools.ts @@ -1,8 +1,6 @@ import { z } from 'zod' -import { WorkspaceService, type CompletedJob } from '$lib/gen' +import type { CompletedJob } from '$lib/gen' import type { DataTableTables } from '$lib/gen/types.gen' -import { runScript } from '$lib/components/jobs/utils' -import { datatableReference } from '$lib/components/dbTypes' import { createToolDef, executeTestRun, @@ -10,6 +8,7 @@ import { type ChatJobResultFormat } from './shared' import { NONE, RUN_PREVIEW, type SessionTool } from './sessionCapabilities' +import { callSessionTool, runOnServer } from './global/backendSessionTools' /** * Workspace-scoped datatable tools, with no app whitelist and no creation policy. @@ -30,41 +29,6 @@ const memo = (factory: () => T): (() => T) => { return () => (cached ??= factory()) } -// ============= Pure workspace-scoped operations ============= - -/** List the datatables configured in the workspace, with their schema/table names: all of them as - * their default role, or only `datatableName`, as `role` when one is given. */ -export async function listDatatables( - workspace: string, - datatableName?: string, - role?: string -): Promise { - if (datatableName === undefined) return await WorkspaceService.listDataTableTables({ workspace }) - return await WorkspaceService.listDataTableTables({ - workspace, - datatableName, - ...(role !== undefined && { roleFor: datatableName, role }) - }) -} - -/** Get the columns (column_name -> compact_type) of one datatable table. */ -export async function getDatatableColumns( - workspace: string, - datatableName: string, - schemaName: string, - tableName: string, - role?: string -): Promise> { - const schema = await WorkspaceService.getDataTableTableSchema({ - workspace, - datatableName, - schemaName, - tableName, - role - }) - return schema.columns -} - // ============= Error helpers ============= /** @@ -254,22 +218,16 @@ export function getDatatableTools(): SessionTool<{}>[] { planModeSafe: true, fn: async ({ args, workspace, toolId, toolCallbacks }) => { toolCallbacks.setToolStatus(toolId, { content: 'Listing datatables...' }) - try { - const parsedArgs = getListDatatablesSchema().parse(args ?? {}) - if (parsedArgs.role !== undefined && parsedArgs.datatable_name === undefined) { - throw new Error('`role` needs `datatable_name`, the datatable it is a role of') - } - const metadata = await listDatatables( - workspace, - parsedArgs.datatable_name, - parsedArgs.role - ) - if (metadata.length === 0) { - toolCallbacks.setToolStatus(toolId, { - content: 'No datatables configured — set one up in workspace settings' - }) - return NO_DATATABLES_CONFIGURED_MESSAGE - } + getListDatatablesSchema().parse(args ?? {}) + const { text } = await runOnServer('list_datatables', workspace, args ?? {}) + if (text === NO_DATATABLES_CONFIGURED_MESSAGE) { + toolCallbacks.setToolStatus(toolId, { + content: 'No datatables configured — set one up in workspace settings' + }) + } else if (text.startsWith('Error listing datatables: ')) { + toolCallbacks.setToolStatus(toolId, { content: text, error: text }) + } else { + const metadata: DataTableTables[] = JSON.parse(text) const totalTables = metadata.reduce( (acc, datatable) => acc + @@ -279,24 +237,8 @@ export function getDatatableTools(): SessionTool<{}>[] { toolCallbacks.setToolStatus(toolId, { content: `Listed ${metadata.length} datatable(s) with ${totalTables} table(s)` }) - // Only what the model acts on: the roles it may pass, not the creation privileges - // the manager's UI gates on. - return JSON.stringify( - metadata.map((d) => ({ - datatable_name: d.datatable_name, - schemas: d.schemas, - ...(d.error && { error: d.error }), - ...(d.migrations_enabled && { migrations_enabled: true }), - ...(d.permissioned && { usable_roles: d.usable_roles, default_role: d.default_role }) - })), - null, - 2 - ) - } catch (e) { - const errorMsg = `Error listing datatables: ${e instanceof Error ? e.message : String(e)}` - toolCallbacks.setToolStatus(toolId, { content: errorMsg, error: errorMsg }) - return errorMsg } + return text } }, { @@ -308,39 +250,21 @@ export function getDatatableTools(): SessionTool<{}>[] { toolCallbacks.setToolStatus(toolId, { content: `Getting schema for ${parsedArgs.datatable_name}.${parsedArgs.schema_name}.${parsedArgs.table_name}...` }) - try { - const columns = await getDatatableColumns( - workspace, - parsedArgs.datatable_name, - parsedArgs.schema_name, - parsedArgs.table_name, - parsedArgs.role - ) - toolCallbacks.setToolStatus(toolId, { - content: `Retrieved schema for ${parsedArgs.schema_name}.${parsedArgs.table_name}` - }) - return JSON.stringify( - { - datatable_name: parsedArgs.datatable_name, - schema_name: parsedArgs.schema_name, - table_name: parsedArgs.table_name, - columns - }, - null, - 2 - ) - } catch (e) { - const raw = e instanceof Error ? e.message : String(e) - const errorMsg = isDatatableNotConfiguredError(raw) - ? datatableNotConfiguredMessage(parsedArgs.datatable_name) - : `Error getting table schema: ${raw}` - toolCallbacks.setToolStatus(toolId, { content: errorMsg, error: errorMsg }) - return errorMsg - } + const { text } = await runOnServer('get_datatable_table_schema', workspace, args) + const failed = + text.startsWith('Error getting table schema: ') || + text === datatableNotConfiguredMessage(parsedArgs.datatable_name) + toolCallbacks.setToolStatus( + toolId, + failed + ? { content: text, error: text } + : { content: `Retrieved schema for ${parsedArgs.schema_name}.${parsedArgs.table_name}` } + ) + return text } }, { - // `runScript` below goes through /jobs/run/preview, which jobs.rs refuses operators. + // The query runs as a /jobs/run/preview job, which jobs.rs refuses operators. requires: RUN_PREVIEW, def: getExecDatatableSqlToolDef(), requiresConfirmation: true, @@ -355,15 +279,13 @@ export function getDatatableTools(): SessionTool<{}>[] { // tray (and honors `background`) instead of blocking the chat turn, // while keeping the datatable-specific result/error shaping. return executeTestRun({ - jobStarter: () => - runScript({ - workspace, - requestBody: { - language: 'postgresql', - content: parsedArgs.sql, - args: { database: datatableReference(name, parsedArgs.role) } - } - }), + jobStarter: async () => { + const response = await callSessionTool(workspace, 'exec_datatable_sql', args, { + run: { phase: 'start' } + }) + if (response.is_error) throw new Error(response.text) + return response.text + }, workspace, toolCallbacks, toolId, diff --git a/frontend/src/lib/components/copilot/chat/ducklakeTools.test.ts b/frontend/src/lib/components/copilot/chat/ducklakeTools.test.ts deleted file mode 100644 index d02d1b9497..0000000000 --- a/frontend/src/lib/components/copilot/chat/ducklakeTools.test.ts +++ /dev/null @@ -1,122 +0,0 @@ -import { beforeEach, describe, expect, it, vi } from 'vitest' - -const { listMock, listMetricsMock } = vi.hoisted(() => ({ - listMock: vi.fn(), - listMetricsMock: vi.fn() -})) - -vi.mock('./shared', () => ({ - createToolDef: (_schema: unknown, name: string, description: string) => ({ - type: 'function', - function: { name, description, parameters: {} } - }) -})) - -vi.mock('$lib/gen', () => ({ - WorkspaceService: { listDucklakes: listMock }, - DataMetricService: { listDataMetrics: listMetricsMock } -})) - -import { getDucklakeTools } from './ducklakeTools' - -function createToolCallbacks() { - return { setToolStatus: vi.fn(), removeToolStatus: vi.fn() } -} - -function run(name: string, args: Record = {}) { - const tool = getDucklakeTools().find((entry) => entry.def.function.name === name) - if (!tool) throw new Error(`${name} tool not found`) - return tool.fn({ - args, - workspace: 'test-workspace', - helpers: {}, - toolCallbacks: createToolCallbacks(), - toolId: `tool-${name}` - }) -} - -beforeEach(() => { - listMock.mockReset() - listMetricsMock.mockReset() -}) - -describe('list_ducklakes', () => { - it('returns the configured catalog names', async () => { - listMock.mockResolvedValue(['main', 'analytics']) - const result = await run('list_ducklakes') - expect(listMock).toHaveBeenCalledWith({ workspace: 'test-workspace' }) - expect(JSON.parse(result)).toEqual({ ducklakes: ['main', 'analytics'] }) - }) - - it('explains the storage prerequisite with role-appropriate steps when none exist', async () => { - listMock.mockResolvedValue([]) - const result = await run('list_ducklakes') - expect(result).toContain('No DuckLake catalogs are configured') - expect(result).toContain('Workspace settings → Object Storage') - expect(result).toContain('ask a workspace admin') - // Drafting is not blocked: the message must say the scripts can still be drafted. - expect(result).toContain('still draft the pipeline scripts') - }) -}) - -describe('list_data_metrics', () => { - it('forwards the filters and returns the declarations', async () => { - listMetricsMock.mockResolvedValue({ - metrics: [ - { - script_path: 'f/analytics/rev', - table_path: 'main/main.orders', - kind: 'measure', - name: 'revenue', - expr: 'sum(amount)', - filter: 'not is_test' - } - ] - }) - const result = await run('list_data_metrics', { - table: 'ducklake://main/main.orders', - path_prefix: 'f/analytics', - limit: 50 - }) - expect(listMetricsMock).toHaveBeenCalledWith({ - workspace: 'test-workspace', - table: 'ducklake://main/main.orders', - pathPrefix: 'f/analytics', - perPage: 50 - }) - expect(JSON.parse(result).metrics[0]).toMatchObject({ name: 'revenue', expr: 'sum(amount)' }) - }) - - it('never reports an empty result as proof that nothing is declared', async () => { - listMetricsMock.mockResolvedValue({ metrics: [] }) - const result = await run('list_data_metrics', {}) - // Unreadable declarations are omitted, not flagged, so absence is unprovable. - expect(result).toContain('does not establish') - expect(result).toContain('cannot read') - }) - - // The server matches a lake-less name against nothing, so the readability hedge - // would confirm "nothing is declared" for a filter worth retrying. The scheme is - // optional on the way in, so the retry it names must not carry it back. - it.each(['orders', 'ducklake://orders'])( - 'blames the missing lake, not readability, for table %s', - async (table) => { - listMetricsMock.mockResolvedValue({ metrics: [] }) - const result = await run('list_data_metrics', { table }) - expect(result).toContain('`/orders`') - expect(result).not.toContain('cannot read') - } - ) - - it('warns that more declarations exist when the page is cut short', async () => { - listMetricsMock.mockResolvedValue({ - // A cursor only comes back on a full page, never with an empty one. - metrics: [{ table_path: 't', kind: 'measure', name: 'n', script_path: 's' }], - next_cursor: { table_path: 't', kind: 'measure', name: 'n', script_path: 's' } - }) - const result = await run('list_data_metrics', {}) - // Without this the model reads a partial page as "no such measure" and - // re-derives a number that disagrees with the declared one. - expect(result).toContain('rather than concluding a measure is undeclared') - }) -}) diff --git a/frontend/src/lib/components/copilot/chat/ducklakeTools.ts b/frontend/src/lib/components/copilot/chat/ducklakeTools.ts index 3e19789e96..ebe193778f 100644 --- a/frontend/src/lib/components/copilot/chat/ducklakeTools.ts +++ b/frontend/src/lib/components/copilot/chat/ducklakeTools.ts @@ -1,7 +1,7 @@ import { z } from 'zod' -import { DataMetricService, WorkspaceService } from '$lib/gen' import { createToolDef } from './shared' import { NONE, type SessionTool } from './sessionCapabilities' +import { runOnServer } from './global/backendSessionTools' /** * Workspace-scoped DuckLake tools, the pipeline counterpart to `list_datatables` @@ -17,11 +17,6 @@ import { NONE, type SessionTool } from './sessionCapabilities' * set. */ -/** List the names of the DuckLake catalogs configured in the workspace. */ -export async function listDucklakes(workspace: string): Promise { - return await WorkspaceService.listDucklakes({ workspace }) -} - const NO_DUCKLAKES_CONFIGURED_MESSAGE = 'No DuckLake catalogs are configured in this workspace. A data pipeline that materializes DuckLake tables or reads/writes S3 assets cannot run until object storage and a DuckLake catalog are set up. ' + 'You can still draft the pipeline scripts, but tell the user how to enable it by role: a workspace admin adds object storage under Workspace settings → Object Storage (S3/Azure/GCS), then a DuckLake catalog on top of it; a user without admin rights should ask a workspace admin. ' + @@ -59,16 +54,6 @@ const listDataMetricsToolDef = createToolDef( 'List the measures and dimensions declared on DuckLake tables (from `// measure` / `// dimension` annotations in deployed scripts). Call this before writing any aggregate query over a DuckLake table: a declared measure is the canonical definition of that number, and reproducing it yourself silently disagrees with it (a `revenue` measure typically excludes refunds or test rows). Use each returned `expr` verbatim, and when a measure has a `filter` write it as `expr FILTER (WHERE filter)` so measures with different predicates share one GROUP BY. Only declarations whose producing script you can read are returned, so what comes back is never proof of what exists: if the number you need is not here you may still write your own aggregate, but say that you found no declared measure for it rather than implying none exists.' ) -// The endpoint drops declarations whose producing script the caller cannot read -// (token scope + RLS on `script`), so an empty result means "none declared" or -// "none readable by you" and the tool cannot tell which. -const NO_DATA_METRICS_NOTE = - 'Nothing matched. That does not establish that nothing is declared: declarations whose producing script you cannot read are omitted from this list, not flagged. You may write your own aggregate, but tell the user you found no declared measure you can read rather than stating none is declared.' - -// Well under the endpoint's 1000 cap: a full page is pretty-printed into the -// chat context, and 1000 declarations would cost tens of thousands of tokens. -const DEFAULT_DATA_METRICS_LIMIT = 200 - /** The workspace DuckLake tools, for registration in global mode. */ export function getDucklakeTools(): SessionTool<{}>[] { return [ @@ -78,59 +63,35 @@ export function getDucklakeTools(): SessionTool<{}>[] { planModeSafe: true, showDetails: true, fn: async ({ args, workspace, toolId, toolCallbacks }) => { - const parsed = listDataMetricsSchema.parse(args) + listDataMetricsSchema.parse(args) toolCallbacks.setToolStatus(toolId, { content: 'Listing declared measures...' }) - const limit = parsed.limit ?? DEFAULT_DATA_METRICS_LIMIT - const { metrics, next_cursor } = await DataMetricService.listDataMetrics({ - workspace, - table: parsed.table, - pathPrefix: parsed.path_prefix, - perPage: limit - }) - // `canonical_table_path` expands a value only once it contains a `/`, so a bare - // table name matches nothing — a retryable filter, not a permissions outcome. - const bareTable = parsed.table?.replace('ducklake://', '') - const emptyNote = - bareTable && !bareTable.includes('/') - ? `Nothing matched \`${parsed.table}\`: a table filter must name its lake, as \`/${bareTable}\`. Re-call with the lake before concluding anything about what is declared.` - : NO_DATA_METRICS_NOTE - const note = next_cursor - ? `More declarations exist beyond the first ${limit}. Re-call with table/path_prefix to target what you are looking for, or with a higher limit (max 1000) for the rest of the list, rather than concluding a measure is undeclared.` - : metrics.length === 0 - ? emptyNote - : undefined - const result = JSON.stringify({ metrics, ...(note ? { note } : {}) }, null, 2) + const { text } = await runOnServer('list_data_metrics', workspace, args) toolCallbacks.setToolStatus(toolId, { - content: `Listed ${metrics.length} declared measure(s)/dimension(s)`, - result + content: `Listed ${JSON.parse(text).metrics.length} declared measure(s)/dimension(s)`, + result: text }) - return result + return text } }, { requires: NONE, def: listDucklakesToolDef, planModeSafe: true, - fn: async ({ workspace, toolId, toolCallbacks }) => { + fn: async ({ args, workspace, toolId, toolCallbacks }) => { toolCallbacks.setToolStatus(toolId, { content: 'Listing DuckLake catalogs...' }) - try { - const ducklakes = await listDucklakes(workspace) - if (ducklakes.length === 0) { - toolCallbacks.setToolStatus(toolId, { - content: - 'No DuckLake configured — set up object storage + DuckLake in workspace settings' - }) - return NO_DUCKLAKES_CONFIGURED_MESSAGE - } + const { text } = await runOnServer('list_ducklakes', workspace, args) + if (text === NO_DUCKLAKES_CONFIGURED_MESSAGE) { toolCallbacks.setToolStatus(toolId, { - content: `Listed ${ducklakes.length} DuckLake catalog(s)` + content: 'No DuckLake configured — set up object storage + DuckLake in workspace settings' + }) + } else if (text.startsWith('Error listing DuckLake catalogs: ')) { + toolCallbacks.setToolStatus(toolId, { content: text, error: text }) + } else { + toolCallbacks.setToolStatus(toolId, { + content: `Listed ${JSON.parse(text).ducklakes.length} DuckLake catalog(s)` }) - return JSON.stringify({ ducklakes }, null, 2) - } catch (e) { - const errorMsg = `Error listing DuckLake catalogs: ${e instanceof Error ? e.message : String(e)}` - toolCallbacks.setToolStatus(toolId, { content: errorMsg, error: errorMsg }) - return errorMsg } + return text } } ] diff --git a/frontend/src/lib/components/copilot/chat/flow/aiAgentProviderCatalog.ts b/frontend/src/lib/components/copilot/chat/flow/aiAgentProviderCatalog.ts index 37bdec30ce..36c0fd1de5 100644 --- a/frontend/src/lib/components/copilot/chat/flow/aiAgentProviderCatalog.ts +++ b/frontend/src/lib/components/copilot/chat/flow/aiAgentProviderCatalog.ts @@ -118,6 +118,15 @@ export async function getAiAgentProviderCatalogFor( if (!needsCatalog) { return undefined } + return getAiAgentProviderCatalogForRefs(workspace, resourceRefs) +} + +/** `getAiAgentProviderCatalogFor` for modules whose steps state a provider and reference + * `resourceRefs`, as the server reports them when a flow write needs the catalog. */ +export async function getAiAgentProviderCatalogForRefs( + workspace: string | undefined, + resourceRefs: string[] +): Promise { const catalog = await getAiAgentProviderCatalog(workspace) const unknown = resourceRefs.some( (ref) => !catalog.options.some((option) => option.resourceRef === ref) diff --git a/frontend/src/lib/components/copilot/chat/flow/editableFlowJson.ts b/frontend/src/lib/components/copilot/chat/flow/editableFlowJson.ts index 45f7841fdf..d7dac2ec40 100644 --- a/frontend/src/lib/components/copilot/chat/flow/editableFlowJson.ts +++ b/frontend/src/lib/components/copilot/chat/flow/editableFlowJson.ts @@ -307,7 +307,7 @@ export function validateFlowSchema(rawSchema: unknown): Record | nu return rawSchema as Record } -function validateOptionalFlowModule(rawModule: unknown, fieldName: string): FlowModule | null { +export function validateOptionalFlowModule(rawModule: unknown, fieldName: string): FlowModule | null { if (rawModule == null) return null const result = flowModuleSchema.safeParse(rawModule) diff --git a/frontend/src/lib/components/copilot/chat/global/backendSessionTools.ts b/frontend/src/lib/components/copilot/chat/global/backendSessionTools.ts new file mode 100644 index 0000000000..81bc315dc7 --- /dev/null +++ b/frontend/src/lib/components/copilot/chat/global/backendSessionTools.ts @@ -0,0 +1,170 @@ +/** + * The session tools that run on the server (`backend/windmill-api/src/ai_session_tools`), + * the same code the Windmill MCP server serves. The chat sends what only the browser + * holds (its open editors and unsaved draft cells) and applies what the call changed + * through its own draft store, so an open editor, its auto-save toggle and its conflict + * dialog behave as when the browser wrote the draft itself. + */ +import { AiService } from '$lib/gen' +import type { ToolCallbacks, ToolDisplayMessage } from '../shared' +import { UserDraft, type UserDraftItemKind } from '$lib/userDraft.svelte' +import { + deleteDraftAt, + GLOBAL_DRAFT_KINDS, + persistDraftAt, + type DraftPersistResult +} from './userDraftAdapter' + +export type SessionToolEffect = + | { + type: 'draft_write' + kind: UserDraftItemKind + storage_path: string + value: unknown + last_sync?: string | null + force: boolean + } + | { + type: 'draft_delete' + kind: UserDraftItemKind + storage_path: string + last_sync?: string | null + keep_live_editor: boolean + } + | { type: 'inspected_run'; job_id: string } + | ({ type: 'facts' } & Record) + | ({ type: 'status'; after_save?: boolean } & Partial) + | { + type: 'code_diff' + before: string + after: string + language?: string | null + file_path?: string | null + before_file_path?: string | null + } + | { type: 'needs_ai_provider_catalog'; resource_refs: string[] } + +export type SessionToolResponse = { + text: string + is_error: boolean + effects: SessionToolEffect[] +} + +/** What the browser holds that a tool reads: the editor open on each draft kind, and + * every draft cell, which is newer than the server's row. */ +export function sessionToolContext(workspace: string) { + const live_editors = GLOBAL_DRAFT_KINDS.flatMap((kind) => { + const editor = UserDraft.getLiveEditorDraft(kind, { workspace }) + return editor + ? [{ kind, storage_path: editor.storagePath, effective_path: editor.effectivePath }] + : [] + }) + const cells = UserDraft.list({ workspace, itemKinds: [...GLOBAL_DRAFT_KINDS] }).map((entry) => ({ + kind: entry.itemKind, + storage_path: entry.path, + value: entry.value + })) + return { live_editors, cells } +} + +export async function callSessionTool( + workspace: string, + name: string, + args: unknown, + context: Record = {} +): Promise { + const response = await AiService.callAiSessionTool({ + workspace, + name, + requestBody: { args: args ?? {}, context: { ...sessionToolContext(workspace), ...context } } + }) + return response as SessionToolResponse +} + +/** Run a session tool on the server. A failed call throws its message, as the browser's own + * request did. */ +export async function runOnServer( + name: string, + workspace: string, + args: unknown, + context?: Record +): Promise { + const response = await callSessionTool(workspace, name, args, context) + if (response.is_error) throw new Error(response.text) + return response +} + +/** `runOnServer` for a tool whose card the server narrates: its status patches are applied + * in the order the call reached them. */ +export async function runServerTool( + name: string, + ctx: { + args: unknown + workspace: string + toolId: string + toolCallbacks: Pick + }, + context?: Record +): Promise { + const response = await callSessionTool(ctx.workspace, name, ctx.args, context) + applyStatusEffects(response.effects, ctx) + if (response.is_error) throw new Error(response.text) + return response +} + +export function applyStatusEffects( + effects: SessionToolEffect[], + ctx: { toolId: string; toolCallbacks: Pick } +): void { + for (const effect of effects) { + if (effect.type === 'status') { + const { type: _, after_save: __, ...patch } = effect + ctx.toolCallbacks.setToolStatus(ctx.toolId, patch) + } + } +} + +/** A failed delete, as the browser's own delete would have thrown it. */ +export type DraftDeleteFailure = { storagePath: string; message: string } + +export type AppliedEffects = { + /** One per `draft_write`, in order. */ + writes: DraftPersistResult[] + deleteFailure?: DraftDeleteFailure +} + +/** Apply a call's effects in order. A failed delete stops there, as a throw would have. */ +export async function applySessionToolEffects( + workspace: string, + effects: SessionToolEffect[], + deletePath: (storagePath: string) => string = (storagePath) => storagePath +): Promise { + const applied: AppliedEffects = { writes: [] } + for (const effect of effects) { + if (effect.type === 'draft_write') { + applied.writes.push( + await persistDraftAt(workspace, effect.kind, effect.storage_path, effect.value, { + force: effect.force, + lastSync: effect.last_sync + }) + ) + } else if (effect.type === 'draft_delete') { + try { + await deleteDraftAt( + workspace, + effect.kind, + effect.storage_path, + deletePath(effect.storage_path), + { preserveLiveDraft: effect.keep_live_editor, lastSync: effect.last_sync } + ) + } catch (e) { + applied.deleteFailure = { + storagePath: effect.storage_path, + message: e instanceof Error ? e.message : String(e) + } + return applied + } + } + } + return applied +} diff --git a/frontend/src/lib/components/copilot/chat/global/core.test.ts b/frontend/src/lib/components/copilot/chat/global/core.test.ts index ab7cf3f285..0af38c9b77 100644 --- a/frontend/src/lib/components/copilot/chat/global/core.test.ts +++ b/frontend/src/lib/components/copilot/chat/global/core.test.ts @@ -62,6 +62,12 @@ vi.mock('$lib/gen', async () => { return { ...actual, + // The server's session tools, stubbed per test by `stubSessionTools`. + AiService: wrapService(actual.AiService, { + callAiSessionTool: vi.fn(async ({ name }: any) => { + throw new Error(`session tool ${name} not stubbed`) + }) + }), ScriptService: wrapService(actual.ScriptService, { existsScriptByPath: vi.fn(async () => false), createScript: vi.fn(async () => 'created'), @@ -259,16 +265,10 @@ vi.mock('$lib/gen', async () => { createVariable: vi.fn(async () => 'created'), updateVariable: vi.fn(async () => 'updated') }), - WorkerService: wrapService(actual.WorkerService, { - listWorkers: vi.fn(async () => []) - }), // Guests unavailable by default, as on the shared cloud. WorkspaceService: wrapService(actual.WorkspaceService, { getGuestUsage: vi.fn(async () => ({ available: false })) }), - FolderService: wrapService(actual.FolderService, { - createFolder: vi.fn(async () => 'created') - }), UserService: wrapService(actual.UserService, { whoami: vi.fn(async ({ workspace }: any) => { const user = whoamiByWorkspace.get(workspace) @@ -372,6 +372,7 @@ vi.mock('$lib/infer', async () => ({ })) import { inferArgs } from '$lib/infer' +import { quickjsReady } from '$lib/utils/quickjsEval.svelte' import { buildRunsFilterSearchbarSchema } from '$lib/components/runs/runsFilter' import { buildOpenPageUrl, @@ -395,29 +396,25 @@ import { deleteGlobalDraft, listGlobalDrafts, persistGlobalDraft, - readGlobalDraftValue, saveGlobalAppDraft } from './userDraftAdapter' import { bundleRawAppDraft } from './rawAppBundlerBridge' +import { applySessionToolEffects } from './backendSessionTools' import { + AiService, AppService, - EmailTriggerService, FlowService, - FolderService, HttpTriggerService, - JobService, ResourceService, ScheduleService, ScriptService, UserService, VariableService, - WorkerService, WorkspaceService } from '$lib/gen' -import { devopsRole, superadmin, userStore, usersWorkspaceStore } from '$lib/stores' +import { superadmin, userStore, usersWorkspaceStore } from '$lib/stores' import { processSecretArgs } from '$lib/components/secretArgUtils' import { clearWorkspaceRoleCache } from '$lib/user' -import { resourceTypesStore } from '$lib/components/resourceTypesStore' import { get } from 'svelte/store' import type { Tool, ToolCallbacks } from '../shared' @@ -444,6 +441,51 @@ function stubInferredProperties(properties: Record): void { }) } +type SessionToolCall = { name: string; args: any; context: any } + +/** Answers the server's session tools: a string is a successful call's text. */ +function stubSessionTools(answer: (call: SessionToolCall) => unknown): void { + vi.mocked(AiService.callAiSessionTool).mockImplementation((async ({ name, requestBody }: any) => { + const response = await answer({ name, args: requestBody.args, context: requestBody.context }) + return typeof response === 'string' + ? { text: response, is_error: false, effects: [] } + : response + }) as any) +} + +/** A run tool whose prepare phase resolves to `prepared` and whose start answers `started`: + * a job id, or a whole response for a refused start. */ +function stubRun(prepared: Record = {}, started: unknown = 'job-started'): void { + stubSessionTools(({ name, args, context }) => + context.run?.phase === 'prepare' + ? JSON.stringify({ + target: { stub: name }, + schema: null, + code: null, + lang: null, + summary: null, + runnable_kind: 'script', + start_message: `Running "${args.path}"...`, + conversational: false, + module: null, + ...prepared + }) + : started + ) +} + +/** The `run` context the start phase was sent, or undefined when no run started. */ +function startedRun(): { args: Record; form_keys: string[] } | undefined { + const call = vi + .mocked(AiService.callAiSessionTool) + .mock.calls.find(([request]: any) => request.requestBody.context.run?.phase === 'start') + return (call?.[0] as any)?.requestBody.context.run +} + +beforeEach(() => { + vi.mocked(AiService.callAiSessionTool).mockReset() +}) + const toolCallbacks: ToolCallbacks = { setToolStatus: vi.fn(), removeToolStatus: vi.fn(), @@ -475,15 +517,6 @@ async function callGlobalTool( }) } -function localStorageSnapshot(): string { - const values: string[] = [] - for (let i = 0; i < localStorage.length; i++) { - const key = localStorage.key(i) - if (key) values.push(`${key}: ${localStorage.getItem(key)}`) - } - return values.join('\n') -} - async function withCompletedTestJob(run: () => Promise): Promise { vi.useFakeTimers() try { @@ -568,50 +601,6 @@ describe('global AI tools', () => { }) }) - // The rare schedule fields reach the draft through `advanced` instead of the tool - // definition, so the merge back into the persisted value is what has to hold. - it('folds advanced schedule options into the draft', async () => { - const advanced = JSON.parse(await callGlobalTool('get_schedule_schema', {})) - expect(Object.keys(advanced.properties)).toEqual( - expect.arrayContaining(['retry', 'paused_until', 'no_flow_overlap']) - ) - expect(advanced.properties).not.toHaveProperty('schedule') - - await callGlobalTool('write_schedule', { - path: 'f/schedules/adv', - schedule: '0 0 9 * * *', - timezone: 'UTC', - script_path: 'f/scripts/run', - is_flow: false, - args: {}, - advanced: { no_flow_overlap: true, tag: 'nightly' } - }) - - const draft = getBackendDraft('trigger_schedule', 'f/schedules/adv', { - workspace: WORKSPACE - }) - expect(draft).toMatchObject({ no_flow_overlap: true, tag: 'nightly' }) - expect(draft).not.toHaveProperty('advanced') - }) - - // A duplicate inside `advanced` must not quietly outrank the named argument. - it('lets a real schedule argument win over the same key inside advanced', async () => { - await callGlobalTool('write_schedule', { - path: 'f/schedules/prec', - schedule: '0 0 9 * * *', - timezone: 'UTC', - script_path: 'f/scripts/run', - is_flow: false, - args: {}, - advanced: { timezone: 'Europe/Paris', tag: 'nightly' } - }) - - const draft = getBackendDraft('trigger_schedule', 'f/schedules/prec', { - workspace: WORKSPACE - }) - expect(draft).toMatchObject({ timezone: 'UTC', tag: 'nightly' }) - }) - // Every sub-field of `retry` is optional, so a guessed shape validates clean and // strips to `{}`. Saving a schedule with no retry policy and reporting success is // the failure the on-demand schema makes reachable. @@ -695,25 +684,6 @@ describe('global AI tools', () => { expect(error).toMatch(/not schedule fields: not_a_field/) }) - // `args` is a real object-valued argument, not an advanced option: repeating it must - // not be reported as a dropped option just because the named one outranks the bag. - it('does not flag a duplicated object argument as dropped', async () => { - await callGlobalTool('write_schedule', { - path: 'f/schedules/dupargs', - schedule: '0 0 6 * * *', - timezone: 'UTC', - script_path: 'f/scripts/run', - is_flow: false, - args: { a: 1 }, - advanced: { args: { b: 2 }, tag: 'nightly' } - }) - - const draft = getBackendDraft('trigger_schedule', 'f/schedules/dupargs', { - workspace: WORKSPACE - }) - expect(draft).toMatchObject({ args: { a: 1 }, tag: 'nightly' }) - }) - it('rejects a trigger config that does not match the declared kind', async () => { const error = await callGlobalTool('write_trigger', { kind: 'kafka', @@ -732,893 +702,6 @@ describe('global AI tools', () => { expect((error as Error).message.slice(0, 2000)).toContain('get_trigger_schema') }) - it('lists recent runs with compact summaries and forwarded filters', async () => { - const result = await callGlobalTool('list_runs', { - path: 'f/team/runner', - created_by: 'alice', - success: true, - limit: 10 - }) - - expect(JobService.listJobs).toHaveBeenCalledWith({ - workspace: WORKSPACE, - scriptPathExact: 'f/team/runner', - createdBy: 'alice', - label: undefined, - success: true, - running: undefined, - perPage: 10 - }) - - const runs = JSON.parse(result) - expect(runs).toHaveLength(2) - expect(runs[0]).toMatchObject({ - id: 'completed-1', - status: 'success', - path: 'f/team/runner', - duration_ms: 1200 - }) - expect(runs[1]).toMatchObject({ id: 'queued-1', status: 'running' }) - // Heavy / sensitive fields must not leak into the summary. - expect(result).not.toContain('verbose logs') - expect(result).not.toContain('do-not-leak') - // The result must be surfaced to the tool display, otherwise the details - // panel shows "No result yet" even though the call succeeded. - expect(toolCallbacks.setToolStatus).toHaveBeenCalledWith( - 'test-list_runs', - expect.objectContaining({ result }) - ) - }) - - it('defaults list_runs to 30 results when no limit is given', async () => { - await callGlobalTool('list_runs', {}) - expect(JobService.listJobs).toHaveBeenCalledWith( - expect.objectContaining({ workspace: WORKSPACE, perPage: 30 }) - ) - }) - - it('lists workers with the diagnostic fields only', async () => { - vi.mocked(WorkerService.listWorkers).mockResolvedValueOnce([ - { - worker: 'wk-1', - worker_instance: 'host-1', - worker_group: 'gpu', - custom_tags: ['gpu'], - last_ping: 3, - jobs_executed: 12, - started_at: '2024-01-01T00:00:00Z', - ip: '10.0.0.1', - wm_version: 'v1', - memory: 123, - occupancy_rate: 0.5 - } - ]) - - const result = await callGlobalTool('list_workers', {}) - - expect(JSON.parse(result).workers).toEqual([ - { - worker: 'wk-1', - worker_group: 'gpu', - custom_tags: ['gpu'], - last_ping: 3, - jobs_executed: 12 - } - ]) - // Page telemetry must stay out of the model's context. - expect(result).not.toContain('occupancy_rate') - expect(result).not.toContain('10.0.0.1') - }) - - it('says so when the worker page is cut short', async () => { - vi.mocked(WorkerService.listWorkers).mockResolvedValueOnce( - Array(100).fill({ - worker: 'wk', - worker_group: 'default', - custom_tags: [], - last_ping: 1, - jobs_executed: 0 - }) as any - ) - - const result = await callGlobalTool('list_workers', {}) - - // A full page is indistinguishable from the whole fleet, and the model reasons - // about tag coverage from this list. - expect(JSON.parse(result).note).toContain('Only the first 100 workers') - }) - - describe('list_workers with nothing to show', () => { - afterEach(() => { - superadmin.set(undefined) - devopsRole.set(undefined) - }) - - it('never reports an empty list as an absence to a caller workers can be hidden from', async () => { - superadmin.set(false) - devopsRole.set(false) - vi.mocked(WorkerService.listWorkers).mockResolvedValueOnce([]) - - const result = await callGlobalTool('list_workers', {}) - - // An instance hiding workers from a non-devops caller answers with an empty - // list, so absence is unprovable here. - expect(result).toContain('does NOT establish that no workers are running') - expect(result).toContain('devops role') - expect(result).not.toContain('"workers"') - }) - - it('reports an empty list as an absence to a devops caller', async () => { - devopsRole.set('devops@windmill.dev') - vi.mocked(WorkerService.listWorkers).mockResolvedValueOnce([]) - - const result = await callGlobalTool('list_workers', {}) - - // Nothing is hidden from this caller, so hedging would withhold the answer a - // stuck queue is waiting on. - expect(result).toContain('No workers are connected') - expect(result).not.toContain('does NOT establish') - }) - - it('resolves the role before deciding, rather than reading unloaded stores as no role', async () => { - // Both stores start undefined; without the refresh a devops caller whose whoami - // has not landed yet is hedged at instead of answered. - expect(get(superadmin)).toBeUndefined() - expect(get(devopsRole)).toBeUndefined() - vi.mocked(WorkerService.listWorkers).mockResolvedValueOnce([]) - - const result = await callGlobalTool('list_workers', {}) - - expect(result).toContain('No workers are connected') - }) - }) - - it('returns args, result and logs of a run in one call', async () => { - const result = await callGlobalTool('get_run', { id: 'job-123' }) - - expect(JobService.getJobLogs).toHaveBeenCalledWith({ - workspace: WORKSPACE, - id: 'job-123', - // The backend's "to remove ansi colors, use: sed ..." hint is noise for - // the model, so it is always suppressed. - removeAnsiWarnings: true - }) - const parsed = JSON.parse(result) - expect(parsed.run).toMatchObject({ - status: 'success', - path: 'f/team/runner', - args: expect.stringContaining('"n": 3'), - result: expect.stringContaining('"ok": true'), - logs: 'job log line 1\njob log line 2' - }) - // The result must be surfaced as the tool result so the details panel shows - // it rather than "No result yet". - expect(toolCallbacks.setToolStatus).toHaveBeenCalledWith( - 'test-get_run', - expect.objectContaining({ result }) - ) - }) - - it('names the job the card renders, without changing what the model is handed', async () => { - const runResult = await callGlobalTool('get_run', { id: 'job-123' }) - expect(toolCallbacks.setToolStatus).toHaveBeenLastCalledWith( - 'test-get_run', - expect.objectContaining({ - result: runResult, - inspectedRun: { - jobId: 'job-123', - workspace: WORKSPACE, - runId: 'job-123', - step: undefined - } - }) - ) - - // A step is a job of its own, and the model's line of prose about it carries - // neither its arguments nor its logs — the card reads those from the job. The - // address travels with it, since a step job names neither the step nor its run. - vi.mocked(JobService.getFlowAllResults).mockResolvedValueOnce({ - entries: [ - { - job_id: 'step-job-1', - label: 'b', - kind: 'script', - depth: 1, - sibling_index: 1, - sibling_count: 1, - status: 'success', - success: true, - result_prefix: '{"ok":true}' - } - ] - } as any) - const stepResult = await callGlobalTool('get_run', { id: 'job-123', step: 'b' }) - expect(stepResult).toContain('(job step-job-1, success) result:') - expect(toolCallbacks.setToolStatus).toHaveBeenLastCalledWith( - 'test-get_run', - expect.objectContaining({ - result: stepResult, - inspectedRun: { - jobId: 'step-job-1', - workspace: WORKSPACE, - runId: 'job-123', - step: 'b' - } - }) - ) - - // An address naming several jobs resolves to none of them, so there is - // nothing for the card to bind to and the call renders as an ordinary row. - vi.mocked(JobService.getFlowAllResults).mockResolvedValueOnce({ - entries: [], - step_error: 'Step "b" ran 4 times (loop/branches) — pick one with "b[i]".' - } as any) - await callGlobalTool('get_run', { id: 'job-123', step: 'b' }) - expect(toolCallbacks.setToolStatus).toHaveBeenLastCalledWith( - 'test-get_run', - expect.not.objectContaining({ inspectedRun: expect.anything() }) - ) - }) - - it('reports when a run has no logs, and tells that apart from logs it could not read', async () => { - vi.mocked(JobService.getJobLogs).mockResolvedValueOnce(' ') - expect(JSON.parse(await callGlobalTool('get_run', { id: 'job-empty' })).run.logs).toBe( - 'No logs for this run.' - ) - - // A failed fetch must not read as "this run logged nothing" — the model - // would report that to the user as fact. - vi.mocked(JobService.getJobLogs).mockRejectedValueOnce(new Error('boom')) - expect(JSON.parse(await callGlobalTool('get_run', { id: 'job-123' })).run.logs).toBe( - 'Logs could not be read for this run.' - ) - - // Nor does every failed read reject: the generated client resolves undefined - // when it cannot read the body, which lands on the same "no logs" branch. - vi.mocked(JobService.getJobLogs).mockResolvedValueOnce(undefined as any) - expect(JSON.parse(await callGlobalTool('get_run', { id: 'job-123' })).run.logs).toBe( - 'Logs could not be read for this run.' - ) - }) - - it('keeps the end of a long log, and never opens it on half a surrogate pair', async () => { - // 12002 code points over 24003 UTF-16 units: the 12000-unit tail opens one - // unit into a unicorn, so the lone low surrogate has to be dropped. - vi.mocked(JobService.getJobLogs).mockResolvedValueOnce('🦄'.repeat(12001) + 'z') - - const [note, body] = JSON.parse( - await callGlobalTool('get_run', { id: 'job-123' }) - ).run.logs.split('\n') - - // The note goes first: the tail is what the model came for, and a note at - // the end would read as the last thing the run logged. - expect(note).toContain('12002 chars total') - expect(body).toHaveLength(11999) - expect(body.codePointAt(0)).toBe(0x1f984) - expect(body.endsWith('🦄z')).toBe(true) - }) - - it('keeps the step tree optional: a failed tree fetch still returns the run itself', async () => { - vi.mocked(JobService.getFlowAllResults).mockRejectedValueOnce(new Error('tree unavailable')) - - const parsed = JSON.parse(await callGlobalTool('get_run', { id: 'job-123' })) - - expect(parsed.run).toMatchObject({ args: expect.stringContaining('"n": 3') }) - expect(parsed.run.logs).toBe('job log line 1\njob log line 2') - // Silence here would read as "this flow ran no steps", which the model - // would then report to the user as fact. - expect(parsed.run.steps_unavailable).toBe(true) - // The tree's root entry normally names the run; without it nothing does. - expect(parsed.run.job_id).toBe('job-123') - - // And the tree read fails the same two ways the log read does: reading - // `.entries` off a resolved undefined throws, which would cost the model the - // job and logs already in hand rather than just the tree. - vi.mocked(JobService.getFlowAllResults).mockResolvedValueOnce(undefined as any) - const noTree = JSON.parse(await callGlobalTool('get_run', { id: 'job-123' })) - expect(noTree.run.steps_unavailable).toBe(true) - expect(noTree.run.logs).toBe('job log line 1\njob log line 2') - }) - - it('reports why a run was canceled or died, the fields getJob used to carry', async () => { - vi.mocked(JobService.getJob).mockResolvedValueOnce({ - type: 'CompletedJob', - id: 'job-oom', - job_kind: 'script', - success: false, - canceled: true, - canceled_by: 'alice', - canceled_reason: 'exceeded memory limit', - mem_peak: 2097152 - } as any) - - const run = JSON.parse(await callGlobalTool('get_run', { id: 'job-oom' })).run - - expect(run.status).toBe('canceled') - expect(run.canceled_by).toBe('alice') - expect(run.canceled_reason).toBe('exceeded memory limit') - expect(run.mem_peak_kb).toBe(2097152) - }) - - // Over ~90KB the job endpoint elides the payload, leaving the step tree as the - // only place a real (server-side truncated) head of the result survives. - function mockElidedJob() { - vi.mocked(JobService.getJob).mockResolvedValueOnce({ - type: 'CompletedJob', - id: 'job-big', - job_kind: 'script', - success: true, - canceled: false, - args: { reason: 'WINDMILL_TOO_BIG' }, - result: 'WINDMILL_TOO_BIG' - } as any) - vi.mocked(JobService.getFlowAllResults).mockResolvedValueOnce({ - entries: [ - { - job_id: 'job-big', - label: 'Flow', - kind: 'script', - depth: 0, - sibling_index: 1, - sibling_count: 1, - status: 'success', - success: true, - result_prefix: '{"blob":"real head"}', - result_length: 300018 - } - ], - truncated: false, - scope_filtered: false - } as any) - } - - it("reports getJob's WINDMILL_TOO_BIG placeholders instead of fetching around them", async () => { - mockElidedJob() - - const run = JSON.parse(await callGlobalTool('get_run', { id: 'job-big' })).run - - // The marker is never the run's own value, so it must not reach the model. - expect(JSON.stringify(run)).not.toContain('WINDMILL_TOO_BIG') - // The result falls to the step tree's real server-side head, flagged and - // sized so the model can't mistake the fragment for the whole payload. - expect(run.result).toBe('{"blob":"real head"}') - expect(run.result_total_chars).toBe(300018) - expect(run.result_truncated).toBe(true) - expect(run.args_truncated).toBe(true) - expect(run.args).toBeUndefined() - // The endpoints that return these whole take no length parameter, so - // reaching for one would pull the entire payload into the tab. - expect(JobService.getJobArgs).not.toHaveBeenCalled() - expect(JobService.getCompletedJobResultMaybe).not.toHaveBeenCalled() - }) - - it('keeps a payload that merely carries the marker string in a reason of its own', async () => { - // The backend elides a result to the bare string and args to exactly - // {reason: marker}. A payload with that reason plus fields of its own is the - // run's own value, and withholding it would report an elision that never was. - vi.mocked(JobService.getJob).mockResolvedValueOnce({ - type: 'CompletedJob', - id: 'job-reason', - job_kind: 'script', - success: false, - canceled: false, - args: { reason: 'WINDMILL_TOO_BIG', retries: 2 }, - result: { reason: 'WINDMILL_TOO_BIG', code: 42 } - } as any) - - const run = JSON.parse(await callGlobalTool('get_run', { id: 'job-reason' })).run - - expect(run.args_truncated).toBeUndefined() - expect(run.result_truncated).toBeUndefined() - expect(run.args).toContain('"retries": 2') - expect(run.result).toContain('"code": 42') - }) - - it('reports skipped and suspended runs as such rather than success or running', async () => { - // `success` is true for a skipped job, and a suspended job is `running`. - vi.mocked(JobService.getJob).mockResolvedValueOnce({ - type: 'CompletedJob', - id: 'job-skipped', - job_kind: 'script', - success: true, - canceled: false, - is_skipped: true - } as any) - expect(JSON.parse(await callGlobalTool('get_run', { id: 'job-skipped' })).run.status).toBe( - 'skipped' - ) - - vi.mocked(JobService.getJob).mockResolvedValueOnce({ - type: 'QueuedJob', - id: 'job-suspended', - job_kind: 'flow', - running: true, - suspend: 1 - } as any) - expect(JSON.parse(await callGlobalTool('get_run', { id: 'job-suspended' })).run.status).toBe( - 'suspended' - ) - }) - - it('searches hub scripts without fetching script contents', async () => { - vi.mocked(ScriptService.queryHubScripts).mockResolvedValueOnce([ - { - version_id: 7, - app: 'slack', - summary: 'Send Message' - } - ] as any) - - const raw = await callGlobalTool('search_hub_scripts', { - query: 'slack message' - }) - - expect(ScriptService.queryHubScripts).toHaveBeenCalledWith({ - text: 'slack message', - kind: 'script', - app: undefined - }) - expect(ScriptService.getHubScriptContentByPath).not.toHaveBeenCalled() - expect(JSON.parse(raw).results).toEqual([ - { - path: 'hub/7/slack/send_message', - summary: 'Send Message', - integration: 'slack' - } - ]) - }) - - it('reads a hub script path through the hub endpoint, not the workspace one', async () => { - vi.mocked(ScriptService.getHubScriptByPath).mockResolvedValueOnce({ - content: 'export async function main() {}', - language: 'bunnative', - summary: 'Send a message to discord using webhook', - schema: { type: 'object', properties: {} } - }) - - const raw = await callGlobalTool('read_workspace_item', { - type: 'script', - path: 'hub/28294/discord/send_a_message_to_discord_using_webhook' - }) - - expect(ScriptService.getHubScriptByPath).toHaveBeenCalledWith({ - path: 'hub/28294/discord/send_a_message_to_discord_using_webhook' - }) - expect(ScriptService.getScriptByPath).not.toHaveBeenCalled() - expect(JSON.parse(raw)).toMatchObject({ - language: 'bunnative', - value: 'export async function main() {}' - }) - }) - - it('reads the deployed state, skipping chat and DB drafts, with version: deployed', async () => { - await callGlobalTool('write_script', { - path: 'f/scripts/greet', - language: 'bun', - content: 'export async function main(renamed_input: string) {}' - }) - vi.mocked(ScriptService.getScriptByPath).mockResolvedValueOnce({ - hash: 1, - path: 'f/scripts/greet', - summary: 'Deployed greet', - content: 'export async function main(name: string) {}', - schema: { properties: { name: { type: 'string' } } }, - language: 'bun', - kind: 'script', - draft: { content: 'export async function main(db_draft_input: string) {}' } - } as any) - - const raw = await callGlobalTool('read_workspace_item', { - type: 'script', - path: 'f/scripts/greet', - version: 'deployed' - }) - const item = JSON.parse(raw) - - expect(ScriptService.getScriptByPath).toHaveBeenCalledWith({ - workspace: WORKSPACE, - path: 'f/scripts/greet', - getDraft: false - }) - expect(item.isDraft).toBe(false) - expect(item.schema).toEqual({ properties: { name: { type: 'string' } } }) - expect(raw).toContain('main(name: string)') - expect(raw).not.toContain('renamed_input') - expect(raw).not.toContain('db_draft_input') - }) - - // The flow read is rebuilt field by field for the model, so a field missing there is - // silently absent and reads as "shared with nobody". - it('reports who a deployed flow is shared with', async () => { - vi.mocked(FlowService.getFlowByPath).mockResolvedValueOnce({ - path: 'f/flows/shared', - summary: 'Shared flow', - value: { modules: [] }, - schema: {}, - extra_perms: { 'g/finance': false } - } as any) - - const item = JSON.parse( - await callGlobalTool('read_workspace_item', { - type: 'flow', - path: 'f/flows/shared', - version: 'deployed' - }) - ) - - expect(item.extraPerms).toEqual({ 'g/finance': false }) - }) - - it('redacts variable draft values when reading workspace items', async () => { - await callGlobalTool('write_variable', { - path: 'f/secrets/api_key', - value: 'super-secret-token', - is_secret: true, - description: 'API key' - }) - - const raw = await callGlobalTool('read_workspace_item', { - type: 'variable', - path: 'f/secrets/api_key' - }) - const item = JSON.parse(raw) - - expect(raw).not.toContain('super-secret-token') - expect(localStorageSnapshot()).not.toContain('super-secret-token') - expect(item).toEqual({ - type: 'variable', - path: 'f/secrets/api_key', - summary: 'API key', - isSecret: true, - isDraft: true - }) - }) - - it('writes resource drafts in the editor UserDraft shape', async () => { - vi.mocked(ResourceService.existsResource).mockResolvedValueOnce(true) - vi.mocked(ResourceService.getResource).mockResolvedValueOnce({ - path: 'f/resources/db', - description: 'existing database', - value: { host: 'old.example.com', port: 5432 }, - resource_type: 'postgresql', - labels: ['prod'], - ws_specific: true, - edited_at: '2026-05-22T09:30:00Z' - } as any) - - await callGlobalTool('write_resource', { - path: 'f/resources/db', - value: { host: 'new.example.com', port: 5432 }, - resource_type: 'postgresql' - }) - - expect(getBackendDraft('resource', 'f/resources/db', { workspace: WORKSPACE })).toEqual({ - path: 'f/resources/db', - description: 'existing database', - args: { host: 'new.example.com', port: 5432 }, - labels: ['prod'], - wsSpecific: true, - resource_type: 'postgresql' - }) - }) - - it('writes variable drafts in the editor UserDraft shape', async () => { - vi.mocked(VariableService.existsVariable).mockResolvedValueOnce(true) - vi.mocked(VariableService.getVariable).mockResolvedValueOnce({ - path: 'f/secrets/api_key', - value: undefined, - is_secret: true, - description: 'old description', - account: 123, - is_oauth: true, - expires_at: '2026-06-22T09:30:00Z', - labels: ['prod'], - ws_specific: true, - edited_at: '2026-05-22T09:30:00Z' - } as any) - - await callGlobalTool('write_variable', { - path: 'f/secrets/api_key', - value: 'new-secret-token', - is_secret: true, - description: 'new description' - }) - - expect(getBackendDraft('variable', 'f/secrets/api_key', { workspace: WORKSPACE })).toEqual( - { - path: 'f/secrets/api_key', - variable: { - value: 'new-secret-token', - is_secret: true, - description: 'new description' - }, - labels: ['prod'], - wsSpecific: true, - account: 123, - is_oauth: true, - expires_at: '2026-06-22T09:30:00Z' - } - ) - expect(localStorageSnapshot()).not.toContain('new-secret-token') - }) - - // The draft row is the only place a staged secret lives (the draft endpoint encrypts it - // at rest). It must never reach localStorage on the way there. - it('deploys a secret variable draft from the draft itself', async () => { - await callGlobalTool('write_variable', { - path: 'f/secrets/api_key', - value: 'new-secret-token', - is_secret: true, - description: 'new description' - }) - - expect( - getBackendDraft('variable', 'f/secrets/api_key', { workspace: WORKSPACE }) - ).toMatchObject({ - path: 'f/secrets/api_key', - variable: { - value: 'new-secret-token', - is_secret: true, - description: 'new description' - }, - wsSpecific: false - }) - expect(localStorageSnapshot()).not.toContain('new-secret-token') - - await callGlobalTool('deploy_workspace_item', { - type: 'variable', - path: 'f/secrets/api_key' - }) - - expect(VariableService.createVariable).toHaveBeenCalledWith({ - workspace: WORKSPACE, - requestBody: expect.objectContaining({ - path: 'f/secrets/api_key', - value: 'new-secret-token', - is_secret: true, - description: 'new description', - ws_specific: false - }) - }) - expect( - getBackendDraft('variable', 'f/secrets/api_key', { workspace: WORKSPACE }) - ).toBeUndefined() - expect(localStorageSnapshot()).not.toContain('new-secret-token') - }) - - // The draft has no null, so a cleared expiry must deploy as an explicit one. - it('clears a variable value expiry through to the deploy body', async () => { - vi.mocked(VariableService.existsVariable).mockResolvedValueOnce(true) // write probe - vi.mocked(VariableService.existsVariable).mockResolvedValueOnce(true) // deploy probe - vi.mocked(VariableService.getVariable).mockResolvedValueOnce({ - path: 'f/secrets/api_key', - value: 'tok', - is_secret: false, - description: 'd', - value_valid_until: '2027-03-15T08:30:00Z' - } as any) - - await callGlobalTool('write_variable', { - path: 'f/secrets/api_key', - value_valid_until: null - }) - - expect( - getBackendDraft('variable', 'f/secrets/api_key', { workspace: WORKSPACE }) - ?.value_valid_until - ).toBeUndefined() - - await callGlobalTool('deploy_workspace_item', { - type: 'variable', - path: 'f/secrets/api_key' - }) - - expect(VariableService.updateVariable).toHaveBeenCalledWith( - expect.objectContaining({ - requestBody: expect.objectContaining({ value_valid_until: null }) - }) - ) - }) - - // '' means "no value staged", so there is nothing to create the secret with. - it('does not create a secret variable draft that stages no value', async () => { - seedBackendDraft( - 'variable', - 'f/secrets/api_key', - { - path: 'f/secrets/api_key', - variable: { - value: '', - is_secret: true, - description: 'new description' - }, - labels: undefined, - wsSpecific: false - }, - { workspace: WORKSPACE } - ) - - await expect( - callGlobalTool('deploy_workspace_item', { - type: 'variable', - path: 'f/secrets/api_key' - }) - ).rejects.toThrow('stages no value') - expect(VariableService.createVariable).not.toHaveBeenCalled() - expect(VariableService.updateVariable).not.toHaveBeenCalled() - }) - - // A secret's value is unreadable, so an edit that does not set one must leave it - // alone end to end: keep is_secret, and omit `value` from the update body rather - // than deploying a placeholder over the stored secret. - it('keeps an existing secret intact when a write only changes the description', async () => { - vi.mocked(VariableService.existsVariable).mockResolvedValueOnce(true) // write probe - vi.mocked(VariableService.existsVariable).mockResolvedValueOnce(true) // deploy probe - vi.mocked(VariableService.getVariable).mockResolvedValueOnce({ - path: 'f/secrets/api_key', - // decryptSecret=false never returns a secret's value - value: undefined, - is_secret: true, - description: 'old description', - labels: ['prod'], - ws_specific: false - } as any) - - await callGlobalTool('write_variable', { - path: 'f/secrets/api_key', - description: 'new description' - }) - - expect( - getBackendDraft('variable', 'f/secrets/api_key', { workspace: WORKSPACE }) - ).toMatchObject({ - variable: { value: '', is_secret: true, description: 'new description' } - }) - - const deployed = JSON.parse( - await callGlobalTool('deploy_workspace_item', { - type: 'variable', - path: 'f/secrets/api_key' - }) - ) - - expect(VariableService.createVariable).not.toHaveBeenCalled() - const requestBody = vi.mocked(VariableService.updateVariable).mock.calls[0][0].requestBody - expect(requestBody).toMatchObject({ is_secret: true, description: 'new description' }) - expect(requestBody).not.toHaveProperty('value') - // Deploying without `value` must say so. A draft from a build that held secret values - // in memory looks identical to this one, so silence here would report a rotation that - // never happened. - expect(deployed.message).toContain('secret value was left unchanged') - }) - - // The drawer stages a secret in this same row as an `$encrypted:` marker (the draft - // endpoint encrypts at rest, the deploy endpoints decrypt). A metadata-only chat edit - // must leave it alone. - it('preserves an encrypted secret draft value through a metadata-only write', async () => { - seedBackendDraft( - 'variable', - 'f/secrets/api_key', - { - path: 'f/secrets/api_key', - variable: { - value: '$encrypted:Zm9vYmFy', - is_secret: true, - description: 'staged in the drawer' - }, - labels: undefined, - wsSpecific: false - }, - { workspace: WORKSPACE } - ) - vi.mocked(VariableService.existsVariable).mockResolvedValueOnce(true) // deploy probe - - await callGlobalTool('write_variable', { - path: 'f/secrets/api_key', - description: 'described by the chat' - }) - - expect( - getBackendDraft('variable', 'f/secrets/api_key', { workspace: WORKSPACE }) - ).toMatchObject({ - variable: { - value: '$encrypted:Zm9vYmFy', - is_secret: true, - description: 'described by the chat' - } - }) - - await callGlobalTool('deploy_workspace_item', { - type: 'variable', - path: 'f/secrets/api_key' - }) - - expect(vi.mocked(VariableService.updateVariable).mock.calls[0][0].requestBody).toMatchObject({ - value: '$encrypted:Zm9vYmFy', - is_secret: true, - description: 'described by the chat' - }) - }) - - // An `$encrypted:` marker is only decryptable while the variable stays secret, so - // un-securing still needs a real plaintext value — carrying the marker over would - // store the marker itself as the variable's value. - it('refuses to un-secret a variable with an encrypted draft value and no new value', async () => { - seedBackendDraft( - 'variable', - 'f/secrets/api_key', - { - path: 'f/secrets/api_key', - variable: { - value: '$encrypted:Zm9vYmFy', - is_secret: true, - description: 'staged in the drawer' - }, - labels: undefined, - wsSpecific: false - }, - { workspace: WORKSPACE } - ) - - await expect( - callGlobalTool('write_variable', { - path: 'f/secrets/api_key', - is_secret: false - }) - ).rejects.toThrow('without a value') - }) - - // Readable plaintext in a secret draft comes from the drawer's shared cell (stood in for - // by the draft store here, since only a Svelte context can hold a cell). It must survive - // a metadata-only chat edit — the drawer reads its own staged value back from there. - it('carries a locally staged plaintext secret through a metadata-only write', async () => { - seedBackendDraft( - 'variable', - 'f/secrets/api_key', - { - path: 'f/secrets/api_key', - variable: { - value: 'typed-in-the-drawer', - is_secret: true, - description: 'staged in the drawer' - }, - labels: undefined, - wsSpecific: false - }, - { workspace: WORKSPACE } - ) - vi.mocked(VariableService.existsVariable).mockResolvedValueOnce(true) // deploy probe - - await callGlobalTool('write_variable', { - path: 'f/secrets/api_key', - description: 'described by the chat' - }) - - // Left in the shared draft (the endpoint encrypts it at rest), not moved into the - // chat's memory-only map. - expect( - getBackendDraft('variable', 'f/secrets/api_key', { workspace: WORKSPACE }) - ).toMatchObject({ - variable: { - value: 'typed-in-the-drawer', - is_secret: true, - description: 'described by the chat' - } - }) - expect(localStorageSnapshot()).not.toContain('typed-in-the-drawer') - - await callGlobalTool('deploy_workspace_item', { - type: 'variable', - path: 'f/secrets/api_key' - }) - - expect(vi.mocked(VariableService.updateVariable).mock.calls[0][0].requestBody).toMatchObject({ - value: 'typed-in-the-drawer', - is_secret: true, - description: 'described by the chat' - }) - }) - // Deploying a drawer-staged secret with no chat write in between: the draft carries the // value, so it must be sent rather than omitted as if only metadata had changed. it('deploys a secret value staged outside the chat', async () => { @@ -1650,193 +733,22 @@ describe('global AI tools', () => { }) }) - // Making a readable variable secret keeps its value, as the drawer's is_secret toggle - // does — the endpoint encrypts it at rest once is_secret is set. - it('carries the old value when making a readable variable secret', async () => { - vi.mocked(VariableService.existsVariable).mockResolvedValueOnce(true) // write probe - vi.mocked(VariableService.existsVariable).mockResolvedValueOnce(true) // deploy probe - vi.mocked(VariableService.getVariable).mockResolvedValueOnce({ - path: 'u/admin/config', - value: 'was-readable', - is_secret: false, - description: 'plain config', - ws_specific: false - } as any) - - await callGlobalTool('write_variable', { - path: 'u/admin/config', - is_secret: true - }) - - expect( - getBackendDraft('variable', 'u/admin/config', { workspace: WORKSPACE }) - ).toMatchObject({ - variable: { value: 'was-readable', is_secret: true, description: 'plain config' } - }) - - await callGlobalTool('deploy_workspace_item', { type: 'variable', path: 'u/admin/config' }) - - expect(vi.mocked(VariableService.updateVariable).mock.calls[0][0].requestBody).toMatchObject({ - value: 'was-readable', - is_secret: true - }) - }) - - // One source of truth means no precedence rule to get wrong: whatever last landed in the - // shared row deploys, so a drawer edit after a chat write is not overwritten by a stale - // copy held elsewhere. - it('deploys the value most recently written to the draft', async () => { - vi.mocked(VariableService.existsVariable).mockResolvedValueOnce(true) // write probe - vi.mocked(VariableService.existsVariable).mockResolvedValueOnce(true) // deploy probe - vi.mocked(VariableService.getVariable).mockResolvedValueOnce({ - path: 'f/secrets/api_key', - value: undefined, - is_secret: true, - description: 'old description', - ws_specific: false - } as any) - - await callGlobalTool('write_variable', { - path: 'f/secrets/api_key', - value: 'chat-wrote-this-first' - }) - - // The drawer then types its own value into the same row. - const draft = getBackendDraft('variable', 'f/secrets/api_key', { workspace: WORKSPACE }) - seedBackendDraft( - 'variable', - 'f/secrets/api_key', - { ...draft, variable: { ...draft.variable, value: 'drawer-wrote-this-second' } }, - { workspace: WORKSPACE } - ) - - await callGlobalTool('deploy_workspace_item', { - type: 'variable', - path: 'f/secrets/api_key' - }) - - expect(vi.mocked(VariableService.updateVariable).mock.calls[0][0].requestBody).toMatchObject({ - value: 'drawer-wrote-this-second' - }) - }) - - // `get_variable` returns account/expires_at as explicit nulls. Carrying them into the - // draft adds `account: null` / `expires_at: null` to every diff the user reviews before - // deploying — noise that only shows up now that a metadata-only edit is possible. - it('does not add null account/expires_at when editing a variable that has neither', async () => { - vi.mocked(VariableService.existsVariable).mockResolvedValueOnce(true) // write probe - vi.mocked(VariableService.existsVariable).mockResolvedValueOnce(true) // deploy probe - vi.mocked(VariableService.getVariable).mockResolvedValueOnce({ - path: 'u/admin/plain_config', - value: 'readable', - is_secret: false, - description: 'old description', - account: null, - is_oauth: null, - expires_at: null, - ws_specific: false - } as any) - - await callGlobalTool('write_variable', { - path: 'u/admin/plain_config', - description: 'new description' - }) - - // Serialize as the real POST does: `undefined` only disappears on the wire, while the - // in-memory draft store keeps the key. - const onTheWire = (value: unknown) => JSON.parse(JSON.stringify(value)) - - const draft = onTheWire( - getBackendDraft('variable', 'u/admin/plain_config', { workspace: WORKSPACE }) - ) - expect(draft).not.toHaveProperty('account') - expect(draft).not.toHaveProperty('expires_at') - expect(draft).not.toHaveProperty('is_oauth') - expect(draft.variable).toMatchObject({ description: 'new description', value: 'readable' }) - - await callGlobalTool('deploy_workspace_item', { - type: 'variable', - path: 'u/admin/plain_config' - }) - - const requestBody = onTheWire( - vi.mocked(VariableService.updateVariable).mock.calls[0][0].requestBody - ) - expect(requestBody).not.toHaveProperty('account') - expect(requestBody).not.toHaveProperty('expires_at') - }) - - // '' is the draft's sentinel for "no value staged", so it cannot also mean "set the - // secret to empty" — accepting it would wipe the secret, which is the placeholder - // habit this schema change is meant to remove. - it('refuses an empty value for a secret variable', async () => { - vi.mocked(VariableService.existsVariable).mockResolvedValueOnce(true) - vi.mocked(VariableService.getVariable).mockResolvedValueOnce({ - path: 'f/secrets/api_key', - value: undefined, - is_secret: true, - description: 'old description', - ws_specific: false - } as any) - - await expect( - callGlobalTool('write_variable', { path: 'f/secrets/api_key', value: '' }) - ).rejects.toThrow('not a valid value for secret variable') - expect( - getBackendDraft('variable', 'f/secrets/api_key', { workspace: WORKSPACE }) - ).toBeUndefined() - }) - - // `get_variable` refreshes and returns a LIVE token for an expired OAuth variable even - // under decryptSecret: false, so the carry path must leave OAuth variables alone - // rather than pinning a rotating value to a static one. - it('never carries the value of an oauth-managed variable', async () => { - vi.mocked(VariableService.existsVariable).mockResolvedValueOnce(true) // write probe - vi.mocked(VariableService.existsVariable).mockResolvedValueOnce(true) // deploy probe - vi.mocked(VariableService.getVariable).mockResolvedValueOnce({ - path: 'u/admin/gh_token', - value: 'live-refreshed-access-token', - is_secret: true, - is_oauth: true, - account: 7, - description: 'github oauth', - ws_specific: false - } as any) - - await callGlobalTool('write_variable', { - path: 'u/admin/gh_token', - description: 'github oauth for the sync job' - }) - - const draft = getBackendDraft('variable', 'u/admin/gh_token', { workspace: WORKSPACE }) - expect(draft.variable.value).toBe('') - - await callGlobalTool('deploy_workspace_item', { type: 'variable', path: 'u/admin/gh_token' }) - - const requestBody = vi.mocked(VariableService.updateVariable).mock.calls[0][0].requestBody - expect(requestBody).not.toHaveProperty('value') - expect(JSON.stringify(requestBody)).not.toContain('live-refreshed-access-token') - }) - // Same carry rule, but the variable is readable: '' still means "not staged" for an // OAuth-managed value, so the deploy must omit it rather than blank the token. it('never deploys an empty value for a non-secret oauth-managed variable', async () => { - vi.mocked(VariableService.existsVariable).mockResolvedValueOnce(true) // write probe - vi.mocked(VariableService.existsVariable).mockResolvedValueOnce(true) // deploy probe - vi.mocked(VariableService.getVariable).mockResolvedValueOnce({ + seedBackendDraft('variable', 'u/admin/gh_token_readable', { path: 'u/admin/gh_token_readable', - value: 'live-refreshed-access-token', - is_secret: false, + variable: { + path: 'u/admin/gh_token_readable', + value: '', + is_secret: false, + description: 'github oauth for the sync job' + }, is_oauth: true, account: 7, - description: 'github oauth', - ws_specific: false - } as any) - - await callGlobalTool('write_variable', { - path: 'u/admin/gh_token_readable', - description: 'github oauth for the sync job' + wsSpecific: false }) + vi.mocked(VariableService.existsVariable).mockResolvedValueOnce(true) await callGlobalTool('deploy_workspace_item', { type: 'variable', @@ -1848,64 +760,22 @@ describe('global AI tools', () => { expect(requestBody).toMatchObject({ description: 'github oauth for the sync job' }) }) - // The backend refuses an is_secret change with no value, and a variable holding '' stages - // nothing to send — so the tool has to ask for one instead of letting that error surface. - it('refuses to make an empty-valued variable secret without a value', async () => { + // The draft has no null, so a cleared expiry must deploy as an explicit one. + it('clears a variable value expiry through to the deploy body', async () => { + seedBackendDraft('variable', 'f/secrets/api_key', { + path: 'f/secrets/api_key', + variable: { path: 'f/secrets/api_key', value: 'tok', is_secret: false, description: 'd' }, + wsSpecific: false + }) vi.mocked(VariableService.existsVariable).mockResolvedValueOnce(true) - vi.mocked(VariableService.getVariable).mockResolvedValueOnce({ - path: 'u/admin/blank', - value: '', - is_secret: false, - description: 'nothing yet', - ws_specific: false - } as any) - await expect( - callGlobalTool('write_variable', { - path: 'u/admin/blank', - is_secret: true - }) - ).rejects.toThrow('without a value') + await callGlobalTool('deploy_workspace_item', { type: 'variable', path: 'f/secrets/api_key' }) - expect(getBackendDraft('variable', 'u/admin/blank', { workspace: WORKSPACE })).toBeUndefined() - }) - - // A draft accumulates: omitting `value` means "this write does not touch the value", so a - // value set earlier stays set. Abandoning it needs discard_local_draft. - it('keeps a rotation staged across a later metadata-only write', async () => { - vi.mocked(VariableService.existsVariable).mockResolvedValueOnce(true) // write probe - vi.mocked(VariableService.existsVariable).mockResolvedValueOnce(true) // deploy probe - vi.mocked(VariableService.getVariable).mockResolvedValueOnce({ - path: 'f/secrets/api_key', - value: undefined, - is_secret: true, - description: 'old description', - ws_specific: false - } as any) - - await callGlobalTool('write_variable', { - path: 'f/secrets/api_key', - value: 'rotated-secret-99' - }) - await callGlobalTool('write_variable', { - path: 'f/secrets/api_key', - description: 'also fix the description' - }) - - const deployed = JSON.parse( - await callGlobalTool('deploy_workspace_item', { - type: 'variable', - path: 'f/secrets/api_key' + expect(VariableService.updateVariable).toHaveBeenCalledWith( + expect.objectContaining({ + requestBody: expect.objectContaining({ value_valid_until: null }) }) ) - - expect(vi.mocked(VariableService.updateVariable).mock.calls[0][0].requestBody).toMatchObject({ - value: 'rotated-secret-99', - is_secret: true, - description: 'also fix the description' - }) - // The value WAS deployed here, so the "left unchanged" note must stay away. - expect(deployed.message).not.toContain('left unchanged') }) it('deploys every field of a script draft (not just content/summary)', async () => { @@ -2080,777 +950,10 @@ describe('global AI tools', () => { conflictSpy.mockRestore() }) - it('writes script drafts into UserDraft', async () => { - const content = 'export async function main() {\n\treturn "hello"\n}' - - await callGlobalTool('write_script', { - path: 'f/scripts/hello', - summary: 'Hello script', - language: 'bun', - content - }) - - expect( - getBackendDraft('script', 'f/scripts/hello', { workspace: WORKSPACE }) - ).toMatchObject({ - path: 'f/scripts/hello', - summary: 'Hello script', - language: 'bun', - content - }) - }) - - it('records a creation diff for a new script draft', async () => { - const statuses: any[] = [] - - await callGlobalTool( - 'write_script', - { - path: 'f/scripts/new-script', - language: 'python3', - content: 'print("hello")' - }, - { ...toolCallbacks, setToolStatus: (_toolId, status) => statuses.push(status) } - ) - - expect(statuses).toContainEqual({ - codeDiff: { before: '', after: 'print("hello")', lang: 'python' } - }) - }) - - it('tells a code app from a drag-and-drop app', async () => { - vi.mocked(AppService.listApps).mockResolvedValueOnce([ - { path: 'f/apps/code', summary: 'Code app', raw_app: true }, - { path: 'f/apps/builder', summary: 'Builder app' } - ] as any) - // An app draft is always a code app: the chat cannot address a - // drag-and-drop app's draft kind at all. - seedBackendDraft( - 'raw_app', - 'u/admin/draft_listed', - { summary: 'Draft app' }, - { workspace: WORKSPACE } - ) - - const rows = JSON.parse(await callGlobalTool('list_workspace_items', { types: ['app'] })) - - expect(rows.map((r: any) => [r.path, r.rawApp])).toEqual([ - ['f/apps/code', true], - ['f/apps/builder', false], - ['u/admin/draft_listed', true] - ]) - }) - - it('still says which kind of app it is when the app is read directly', async () => { - // The flag decides whether the app tools are offered at all, and the model - // reads an app before it edits one — a listing that knows is not enough. - vi.mocked(AppService.getAppByPath).mockResolvedValueOnce({ - path: 'f/apps/builder', - summary: 'Builder app', - value: { grid: [] }, - raw_app: false - } as any) - - const read = JSON.parse( - await callGlobalTool('read_workspace_item', { type: 'app', path: 'f/apps/builder' }) - ) - expect(read.rawApp).toBe(false) - }) - - it('finds a staged app under the folder it was filed in, not its generated path', async () => { - // A never-deployed app the editor created lives at a generated path, so the - // folder the user filed it under exists only as its staged name. The server - // drops draft-only rows under any narrowing filter, leaving this pass the one - // that can answer a folder-scoped question about it. - seedBackendDraft( - 'raw_app', - 'u/admin/draft_7f21c9', - { summary: '', draft_path: 'f/team/invoice_tracker' }, - { workspace: WORKSPACE } - ) - - const matched = JSON.parse( - await callGlobalTool('list_workspace_items', { types: ['app'], path_prefix: 'f/team/' }) - ) - expect(matched).toHaveLength(1) - expect(matched[0].draftPath).toBe('f/team/invoice_tracker') - - const other = JSON.parse( - await callGlobalTool('list_workspace_items', { types: ['app'], path_prefix: 'f/other/' }) - ) - expect(other).toEqual([]) - }) - - it('applies path_prefix to drafts before enforcing the result limit', async () => { - await callGlobalTool('write_script', { - path: 'f/other/outside', - summary: 'Outside draft', - language: 'bun', - content: 'export async function main() { return "outside" }' - }) - await callGlobalTool('write_script', { - path: 'f/matching/inside', - summary: 'Inside draft', - language: 'bun', - content: 'export async function main() { return "inside" }' - }) - - const raw = await callGlobalTool('list_workspace_items', { - types: ['script'], - path_prefix: 'f/matching/', - limit: 1 - }) - - expect(JSON.parse(raw)).toEqual([ - expect.objectContaining({ - type: 'script', - path: 'f/matching/inside', - isDraft: true - }) - ]) - }) - - it('forwards page to the list calls, capping page-1 drafts at limit per type', async () => { - await callGlobalTool('write_script', { - path: 'f/scripts/draft_a', - language: 'bun', - content: 'export async function main() {}' - }) - await callGlobalTool('write_script', { - path: 'f/scripts/draft_b', - language: 'bun', - content: 'export async function main() {}' - }) - - const page1 = await callGlobalTool('list_workspace_items', { - types: ['script'], - limit: 1, - page: 1 - }) - const page2 = await callGlobalTool('list_workspace_items', { - types: ['script'], - limit: 1, - page: 2 - }) - - expect(ScriptService.listScripts).toHaveBeenCalledWith(expect.objectContaining({ page: 2 })) - // Bounded on page 1, no draft rows on later pages; the capped-out draft - // stays reachable through the query filter. - expect(JSON.parse(page1)).toHaveLength(1) - expect(JSON.parse(page2)).toEqual([]) - const byQuery = await callGlobalTool('list_workspace_items', { - types: ['script'], - query: 'draft_b' - }) - expect(JSON.parse(byQuery).map((i: any) => i.path)).toEqual(['f/scripts/draft_b']) - }) - - it('applies limit per item type so a full page of one type cannot hide another', async () => { - vi.mocked(ScriptService.listScripts).mockResolvedValueOnce([ - { path: 'f/scripts/s1', language: 'bun' }, - { path: 'f/scripts/s2', language: 'bun', draft_only: true } - ] as any) - vi.mocked(FlowService.listFlows).mockResolvedValueOnce([{ path: 'f/flows/f1' }] as any) - - const raw = await callGlobalTool('list_workspace_items', { - types: ['script', 'flow'], - limit: 2 - }) - - const items = JSON.parse(raw) - expect(items.map((i: any) => i.path)).toEqual(['f/scripts/s1', 'f/scripts/s2', 'f/flows/f1']) - // Server-synthesized draft-only rows must read as drafts, not deployed items. - expect(items.map((i: any) => i.isDraft)).toEqual([false, true, false]) - }) - - it('keeps limit one cap per type across the folders a session is scoped to', async () => { - vi.mocked(ScriptService.listScripts).mockImplementation((async ({ pathStart }: any) => - [1, 2].map((n) => ({ path: `${pathStart}s${n}`, language: 'bun' }))) as any) - const scoped: ToolCallbacks = { - ...toolCallbacks, - workingScope: { - scope: () => ({ roots: ['f/a', 'f/b'] }) as any, - requestDecision: async () => undefined, - add: () => {}, - recordWrite: () => {} - } - } - const list = async (page: number) => - JSON.parse( - ( - await callGlobalTool('list_workspace_items', { types: ['script'], limit: 3, page }, scoped) - ).replace(/^[^\n]*\n/, '') - ).map((i: any) => i.path) - - try { - expect(await list(1)).toEqual(['f/a/s1', 'f/a/s2', 'f/b/s1']) - expect(await list(2)).toEqual(['f/b/s2']) - } finally { - vi.mocked(ScriptService.listScripts).mockImplementation(async () => []) - } - }) - - it('lists and edits the live script editor draft through its effective path', async () => { - seedBackendDraft( - 'script', - '', - { - path: 'u/admin/amazed_script', - summary: 'Live script', - description: '', - content: 'export async function main(a: number, b: number) {\n\treturn a + b\n}', - schema: {}, - is_template: false, - language: 'bun', - kind: 'script' - }, - { workspace: WORKSPACE } - ) - UserDraft.setLiveEditorDraft({ - workspace: WORKSPACE, - itemKind: 'script', - storagePath: '', - effectivePath: 'u/admin/amazed_script' - }) - - const listRaw = await callGlobalTool('list_workspace_items', { types: ['script'] }) - expect(JSON.parse(listRaw)).toContainEqual( - expect.objectContaining({ - type: 'script', - path: 'u/admin/amazed_script', - isDraft: true, - isLiveDraft: true - }) - ) - - await callGlobalTool('edit_script', { - path: 'u/admin/amazed_script', - old_string: 'return a + b', - new_string: 'return a * b' - }) - - expect(getBackendDraft('script', '', { workspace: WORKSPACE })).toMatchObject({ - path: 'u/admin/amazed_script', - content: 'export async function main(a: number, b: number) {\n\treturn a * b\n}' - }) - expect( - getBackendDraft('script', 'u/admin/amazed_script', { workspace: WORKSPACE }) - ).toBeUndefined() - }) - - it('lists and writes the live flow editor draft through its effective path', async () => { - seedBackendDraft( - 'flow', - '', - { - path: '', - summary: 'Live flow', - value: { modules: [] }, - schema: {}, - edited_by: '', - edited_at: '', - archived: false, - extra_perms: {} - }, - { workspace: WORKSPACE } - ) - UserDraft.setLiveEditorDraft({ - workspace: WORKSPACE, - itemKind: 'flow', - storagePath: '', - effectivePath: 'u/admin/live_flow' - }) - - const listRaw = await callGlobalTool('list_workspace_items', { types: ['flow'] }) - expect(JSON.parse(listRaw)).toContainEqual( - expect.objectContaining({ - type: 'flow', - path: 'u/admin/live_flow', - isDraft: true, - isLiveDraft: true - }) - ) - - await callGlobalTool('write_flow', { - path: 'u/admin/live_flow', - summary: 'Updated live flow', - modules: JSON.stringify([{ id: 'step', value: { type: 'identity' } }]) - }) - - expect(getBackendDraft('flow', '', { workspace: WORKSPACE })).toMatchObject({ - path: 'u/admin/live_flow', - summary: 'Updated live flow', - value: { modules: [{ id: 'step', value: { type: 'identity' } }] } - }) - expect(getBackendDraft('flow', 'u/admin/live_flow', { workspace: WORKSPACE })).toBeUndefined() - }) - - it('writes the live raw app editor draft through its effective path', async () => { - seedBackendDraft( - 'raw_app', - '', - { - summary: 'Live app', - files: { '/src/App.tsx': 'export default function App() { return null }' }, - runnables: {}, - data: { tables: [] } - }, - { workspace: WORKSPACE } - ) - UserDraft.setLiveEditorDraft({ - workspace: WORKSPACE, - itemKind: 'raw_app', - storagePath: '', - effectivePath: 'u/admin/live_app' - }) - - await callGlobalTool('write_app_file', { - path: 'u/admin/live_app', - file_path: '/src/New.tsx', - content: 'export default function New() { return null }' - }) - - expect(getBackendDraft('raw_app', '', { workspace: WORKSPACE })).toMatchObject({ - files: { - '/src/App.tsx': 'export default function App() { return null }', - '/src/New.tsx': 'export default function New() { return null }' - } - }) - expect(getBackendDraft('raw_app', 'u/admin/live_app', { workspace: WORKSPACE })).toBeUndefined() - }) - - // A path runnable executes the DEPLOYED item, so pointing one at a draft-only flow - // produces an app that silently does nothing. The write still succeeds — flow and app - // are normally built together — but the model has to be told what is missing. - it('warns when a path runnable points at an item that is not deployed', async () => { - seedBackendDraft( - 'raw_app', - 'u/admin/wired_app', - { summary: 'Wired app', files: {}, runnables: {}, data: { tables: [] } }, - { workspace: WORKSPACE } - ) - - vi.mocked(FlowService.existsFlowByPath).mockResolvedValueOnce(false) - const undeployed = JSON.parse( - await callGlobalTool('write_app_runnable', { - path: 'u/admin/wired_app', - key: 'run_flow', - runnable: { name: 'Run the flow', type: 'flow', path: 'u/admin/hello_flow' } - }) - ) - expect(undeployed.success).toBe(true) - expect(undeployed.warning).toContain('u/admin/hello_flow') - expect(undeployed.warning).toContain('NOT deployed') - // The remedy is one item, not a release: the app runs its draft in the preview. - expect(undeployed.warning).toContain('deploy_workspace_item') - - vi.mocked(FlowService.existsFlowByPath).mockResolvedValueOnce(true) - const deployed = JSON.parse( - await callGlobalTool('write_app_runnable', { - path: 'u/admin/wired_app', - key: 'run_flow', - runnable: { name: 'Run the flow', type: 'flow', path: 'u/admin/hello_flow' } - }) - ) - expect(deployed.success).toBe(true) - expect(deployed.warning).toBeUndefined() - - // A script target resolves through existsScriptByPath, then the archived-inclusive - // getScriptByPath fallback — a 404 there is the only thing that means "not deployed". - vi.mocked(ScriptService.existsScriptByPath).mockResolvedValueOnce(false) - vi.mocked(ScriptService.getScriptByPath).mockRejectedValueOnce( - Object.assign(new Error('not found'), { status: 404 }) - ) - const scriptTarget = JSON.parse( - await callGlobalTool('write_app_runnable', { - path: 'u/admin/wired_app', - key: 'run_script', - runnable: { name: 'Run the script', type: 'script', path: 'u/admin/hello_script' } - }) - ) - expect(scriptTarget.warning).toContain('u/admin/hello_script') - - // A hub script lives outside the workspace and has no deployed/draft distinction, - // so it must never be probed or warned about. - const hub = JSON.parse( - await callGlobalTool('write_app_runnable', { - path: 'u/admin/wired_app', - key: 'run_hub', - runnable: { name: 'Hub', type: 'hubscript', path: 'hub/123/slack/send' } - }) - ) - expect(hub.warning).toBeUndefined() - }) - - // The execute_component payload is what makes this tool faithful to how the app - // really runs: force_viewer_static_fields is what selects preview mode server-side - // (apps.rs `is_preview`), and it must be sent even when there are no static fields. - it('runs an inline app runnable as a preview job with its draft code', async () => { - seedBackendDraft( - 'raw_app', - 'u/admin/tested_app', - { - summary: 'Tested app', - files: {}, - runnables: { - greet: { - name: 'Greet', - type: 'inline', - inlineScript: { language: 'bun', content: 'export async function main() { return 1 }' }, - fields: { - who: { type: 'ctx', ctx: 'email' }, - fixed: { type: 'static', value: 7 }, - api_key: { type: 'user', sensitive: true }, - plain: { type: 'user' } - } - } - }, - data: { tables: [] } - }, - { workspace: WORKSPACE } - ) - - await callGlobalTool('test_run_app_runnable', { - path: 'u/admin/tested_app', - key: 'greet', - args: { name: 'ada' } - }) - - const body = vi.mocked(AppService.executeComponent).mock.calls.at(-1)?.[0].requestBody as any - expect(body.force_viewer_static_fields).toEqual({ fixed: 7 }) - // A ctx-bound input is resolved server-side; sending it absent would fail the run - // for a reason unrelated to the runnable's code. - expect(body.args).toEqual({ name: 'ada', who: '$ctx:email' }) - expect(body.raw_code).toMatchObject({ language: 'bun' }) - expect(body.path).toBeUndefined() - // Only names listed here get encrypted before the args are queued, so a sensitive - // field left out of it is stored in plaintext for anyone with run access to read. - expect(body.force_viewer_sensitive_inputs).toEqual(['api_key']) - }) - - // The undeployed-flow 404 is the whole reason this tool exists, and the generated client - // leaves the server's message in `body` while `message` is the bare status text. - it("surfaces the server's message when a path runnable's target is not deployed", async () => { - seedBackendDraft( - 'raw_app', - 'u/admin/broken_app', - { - summary: 'Broken app', - files: {}, - runnables: { - go: { name: 'Go', type: 'path', runType: 'flow', path: 'u/admin/never_deployed' } - }, - data: { tables: [] } - }, - { workspace: WORKSPACE } - ) - vi.mocked(AppService.executeComponent).mockRejectedValueOnce({ - status: 404, - message: 'Not Found', - body: 'Not found: flow not found at name u/admin/never_deployed' - }) - - await expect( - callGlobalTool('test_run_app_runnable', { path: 'u/admin/broken_app', key: 'go' }) - ).rejects.toThrow(/flow not found at name u\/admin\/never_deployed/) - }) - - it('runs a path app runnable against the deployed item it names', async () => { - seedBackendDraft( - 'raw_app', - 'u/admin/wired_app2', - { - summary: 'Wired app', - files: {}, - runnables: { - run_flow: { name: 'Run', type: 'path', runType: 'flow', path: 'u/admin/hello_flow' } - }, - data: { tables: [] } - }, - { workspace: WORKSPACE } - ) - - await callGlobalTool('test_run_app_runnable', { path: 'u/admin/wired_app2', key: 'run_flow' }) - - const body = vi.mocked(AppService.executeComponent).mock.calls.at(-1)?.[0].requestBody as any - expect(body.path).toBe('flow/u/admin/hello_flow') - expect(body.raw_code).toBeUndefined() - // Absent rather than [], matching what the editor preview sends. - expect(body.force_viewer_sensitive_inputs).toBeUndefined() - }) - - // A hybrid runnable — inline code plus a leftover runType/path — contradicts its own - // kind, and convertPersistedToBackendRunnable is what reports it back to the model. - it("drops the other kind's fields when a runnable changes type", async () => { - seedBackendDraft( - 'raw_app', - 'u/admin/converted_app', - { summary: 'Converted', files: {}, runnables: {}, data: { tables: [] } }, - { workspace: WORKSPACE } - ) - - await callGlobalTool('write_app_runnable', { - path: 'u/admin/converted_app', - key: 'go', - runnable: { name: 'Run the flow', type: 'flow', path: 'u/admin/hello_flow' } - }) - await callGlobalTool('write_app_runnable', { - path: 'u/admin/converted_app', - key: 'go', - runnable: { - name: 'Now inline', - type: 'inline', - inlineScript: { language: 'bun', content: 'export async function main() { return 1 }' } - } - }) - - const asInline = getBackendDraft('raw_app', 'u/admin/converted_app', { - workspace: WORKSPACE - }).runnables.go - expect(asInline.type).toBe('inline') - expect(asInline.runType).toBeUndefined() - expect(asInline.path).toBeUndefined() - - await callGlobalTool('write_app_runnable', { - path: 'u/admin/converted_app', - key: 'go', - runnable: { name: 'Back to flow', type: 'flow', path: 'u/admin/hello_flow' } - }) - - const asPath = getBackendDraft('raw_app', 'u/admin/converted_app', { - workspace: WORKSPACE - }).runnables.go - expect(asPath.type).toBe('path') - expect(asPath.runType).toBe('flow') - expect(asPath.inlineScript).toBeUndefined() - }) - - // A static input is pinned by the deployed policy: carried into an agent runnable, it would - // replace every viewer's message. - it('drops inherited static inputs when a runnable becomes an agent', async () => { - seedBackendDraft( - 'raw_app', - 'u/admin/agent_conv_app', - { summary: 'Converted', files: {}, runnables: {}, data: { tables: [] } }, - { workspace: WORKSPACE } - ) - await callGlobalTool('write_app_runnable', { - path: 'u/admin/agent_conv_app', - key: 'chat', - runnable: { - name: 'Run the flow', - type: 'flow', - path: 'u/admin/hello_flow', - staticInputs: { user_message: 'pinned' } - } - }) - await callGlobalTool('write_app_runnable', { - path: 'u/admin/agent_conv_app', - key: 'chat', - runnable: { name: 'Chat', type: 'agent', path: 'u/admin/assistant' } - }) - - const asAgent = getBackendDraft('raw_app', 'u/admin/agent_conv_app', { - workspace: WORKSPACE - }).runnables.chat - expect(asAgent.runType).toBe('agent') - expect(asAgent.fields).toEqual({}) - }) - - it("resets a path runnable's schema when it is retargeted, and keeps it when it is not", async () => { - // The editor populates `schema` from the item the runnable points at, and - // genWmillTs types `backend.(args)` from it — so it must not outlive the target. - const flowSchema = { - type: 'object', - properties: { old_arg: { type: 'string' } } - } - seedBackendDraft( - 'raw_app', - 'u/admin/retargeted_app', - { - summary: 'Retargeted', - files: {}, - runnables: { - go: { - name: 'Run the flow', - type: 'path', - runType: 'flow', - path: 'u/admin/first_flow', - fields: {}, - schema: flowSchema - } - }, - data: { tables: [] } - }, - { workspace: WORKSPACE } - ) - - await callGlobalTool('write_app_runnable', { - path: 'u/admin/retargeted_app', - key: 'go', - runnable: { name: 'Run the flow', type: 'flow', path: 'u/admin/first_flow' } - }) - expect( - getBackendDraft('raw_app', 'u/admin/retargeted_app', { workspace: WORKSPACE }).runnables - .go.schema - ).toEqual(flowSchema) - - await callGlobalTool('write_app_runnable', { - path: 'u/admin/retargeted_app', - key: 'go', - runnable: { name: 'Run the other flow', type: 'flow', path: 'u/admin/second_flow' } - }) - expect( - getBackendDraft('raw_app', 'u/admin/retargeted_app', { workspace: WORKSPACE }).runnables - .go.schema - ).toEqual({}) - }) - - it('does not echo the app value back to the model on write', async () => { - const sentinel = 'SENTINEL_DO_NOT_ECHO_DEADBEEF' - seedBackendDraft( - 'raw_app', - '', - { - summary: 'Echo check', - files: { '/src/App.tsx': `export default function App() { return '${sentinel}' }` }, - runnables: {}, - data: { tables: [] } - }, - { workspace: WORKSPACE } - ) - UserDraft.setLiveEditorDraft({ - workspace: WORKSPACE, - itemKind: 'raw_app', - storagePath: '', - effectivePath: 'u/admin/echo_app' - }) - - const raw = await callGlobalTool('write_app_file', { - path: 'u/admin/echo_app', - file_path: '/src/New.tsx', - content: 'export default function New() { return null }' - }) - - const parsed = JSON.parse(raw) - expect(parsed.success).toBe(true) - expect(parsed.item).toBeUndefined() - // Neither the pre-existing file body nor the just-written one is resent. - expect(raw).not.toContain(sentinel) - expect(raw).not.toContain('function New') - }) - - it('records the data setup a new app is initialized with', async () => { - await callGlobalTool('init_app', { - path: 'u/admin/tickets', - framework: 'react19', - data: { - datatable: 'main', - schema: 'helpdesk', - tables: ['helpdesk:tickets', 'notes', 'main/customers'] - } - }) - - const draft = getBackendDraft('raw_app', 'u/admin/tickets', { workspace: WORKSPACE }) - expect(draft?.data).toEqual({ - datatable: 'main', - schema: 'helpdesk', - tables: ['main/helpdesk:tickets', 'main/helpdesk:notes', 'main/customers'] - }) - }) - - it('discards a draft without deleting the workspace item', async () => { - await callGlobalTool('write_script', { - path: 'f/scripts/discard-me', - summary: 'Temporary draft', - language: 'bun', - content: 'export async function main() { return 1 }' - }) - - expect( - getBackendDraft('script', 'f/scripts/discard-me', { workspace: WORKSPACE }) - ).toBeDefined() - - const raw = await callGlobalTool('discard_local_draft', { - type: 'script', - path: 'f/scripts/discard-me' - }) - - expect(JSON.parse(raw)).toMatchObject({ - success: true, - type: 'script', - path: 'f/scripts/discard-me' - }) - expect(raw).toContain('The deployed workspace item was not changed') - expect( - getBackendDraft('script', 'f/scripts/discard-me', { workspace: WORKSPACE }) - ).toBeUndefined() - }) - - // "Create a resource, then never mind": delete_workspace_item must reject a path - // that was never deployed, before the confirmation card — otherwise the user - // confirms a workspace mutation that 404s past the draft cleanup, leaving the - // draft they asked to be rid of. - it('rejects deleting a draft-only item and names discard_local_draft', async () => { - await callGlobalTool('write_resource', { - path: 'u/admin/never_mind_db', - value: { host: 'db.example.com', port: 5432 }, - resource_type: 'postgresql' - }) - - const error = await getGlobalTool('delete_workspace_item').validateBeforeConfirmation?.({ - args: { type: 'resource', path: 'u/admin/never_mind_db' }, - workspace: WORKSPACE, - helpers: {} - }) - - expect(error).toMatch(/only exists as a draft/) - expect(error).toMatch(/discard_local_draft/) - expect(ResourceService.deleteResource).not.toHaveBeenCalled() - expect( - getBackendDraft('resource', 'u/admin/never_mind_db', { workspace: WORKSPACE }) - ).toBeDefined() - }) - - it('lets delete_workspace_item through when the item is deployed', async () => { - vi.mocked(ResourceService.existsResource).mockResolvedValueOnce(true) - - await expect( - getGlobalTool('delete_workspace_item').validateBeforeConfirmation?.({ - args: { type: 'resource', path: 'u/admin/deployed_db' }, - workspace: WORKSPACE, - helpers: {} - }) - ).resolves.toBeUndefined() - }) - - // existsScriptByPath filters archived=false but deleteScriptByPath does not, so - // probing with it alone would make an archived script undeletable via the chat. - it('lets delete_workspace_item through for an archived script', async () => { - vi.mocked(ScriptService.existsScriptByPath).mockResolvedValueOnce(false) - vi.mocked(ScriptService.getScriptByPath).mockResolvedValueOnce({ - path: 'f/scripts/archived_one', - content: 'export async function main() {}', - language: 'bun', - archived: true - } as any) - - await expect( - getGlobalTool('delete_workspace_item').validateBeforeConfirmation?.({ - args: { type: 'script', path: 'f/scripts/archived_one' }, - workspace: WORKSPACE, - helpers: {} - }) - ).resolves.toBeUndefined() - }) - // Covers the conflict-on-save / override branch of `persistGlobalDraft` // directly: a non-force save whose recorded baseline is older than the // server row is rejected with `status:'conflict'`, and `override` (force) - // pushes our version through. NB: this targets persistGlobalDraft, not the - // write_* tools — those re-read the backend first (readGlobalDraftValue -> - // recordRemoteSync), which re-seeds the baseline and so can only surface a - // conflict when a live editor cell is mounted (not the case in unit tests). + // pushes our version through. it('persistGlobalDraft surfaces a conflict on a stale baseline and override forces it', async () => { const path = 'f/scripts/conflicted' const key = `script:${path}` @@ -2889,6 +992,32 @@ describe('global AI tools', () => { }) }) + // A server tool's write carries the row timestamp it read, so a row another writer advanced + // since conflicts in the browser's own store instead of being overwritten. + it('applies a draft_write effect through the draft store, conflict included', async () => { + const path = 'f/scripts/effect' + const v1 = { path, summary: 'v1', content: 'export function main() {}', language: 'bun' } + seedBackendDraft('script', path, v1) + serverTimestamps.set(`script:${path}`, '2026-06-15T00:01:00Z') + + const write = (force: boolean) => + applySessionToolEffects(WORKSPACE, [ + { + type: 'draft_write', + kind: 'script', + storage_path: path, + value: { ...v1, summary: 'v2' }, + last_sync: '2026-06-15T00:00:00Z', + force + } + ]) + + expect((await write(false)).writes[0].status).toBe('conflict') + expect(getBackendDraft('script', path)).toMatchObject({ summary: 'v1' }) + expect((await write(true)).writes[0].status).toBe('saved') + expect(getBackendDraft('script', path)).toMatchObject({ summary: 'v2' }) + }) + // A backend save failure (network/5xx) is recorded in the syncer's failure // map, not thrown — persistGlobalDraft must report 'error', never 'saved'. it('persistGlobalDraft reports an error (not saved) when the backend save fails', async () => { @@ -2908,14 +1037,6 @@ describe('global AI tools', () => { expect(getBackendDraft('script', path, { workspace: WORKSPACE })).toBeUndefined() }) - // A non-404 read failure must propagate, not collapse to "no draft" — else - // the write merge falls through to the deployed item, losing draft edits. - it('a non-404 backend read failure propagates instead of returning undefined', async () => { - const path = 'f/scripts/readfail' - failingReads.add(`script:${path}`) - await expect(readGlobalDraftValue(WORKSPACE, 'script', path)).rejects.toThrow() - }) - // Raw-app writes go through saveGlobalAppDraft, which must carry the conflict // status so write_app_* tools don't report a stale write as saved. it('saveGlobalAppDraft surfaces a conflict on a stale baseline', async () => { @@ -2949,31 +1070,11 @@ describe('global AI tools', () => { await expect(deleteGlobalDraft(WORKSPACE, 'script', path)).rejects.toThrow() }) - // `override` is a tool-only conflict flag and must not leak into the persisted - // schedule draft value. - it('does not persist the tool-only override flag into a schedule draft', async () => { - await callGlobalTool('write_schedule', { - path: 'f/schedules/ov', - schedule: '0 0 9 * * *', - timezone: 'UTC', - script_path: 'f/scripts/run', - is_flow: false, - args: {}, - override: true - }) - const draft = getBackendDraft('trigger_schedule', 'f/schedules/ov', { - workspace: WORKSPACE - }) - expect(draft).toBeTruthy() - expect(draft).not.toHaveProperty('override') - }) - // Schedule drafts (like all drawer kinds) are private to their owner, so the - // cross-user draft route 404s on them. Reading them back must go through the - // own-draft route, else a freshly written schedule draft is listed but can - // never be read or deployed. - it('reads and deploys a schedule draft written by the chat', async () => { - await callGlobalTool('write_schedule', { + // cross-user draft route 404s on them. The deploy must read them through the + // own-draft route, else a draft is listed but can never be deployed. + it('deploys a schedule draft read through the own-draft route', async () => { + seedBackendDraft('trigger_schedule', 'u/admin/test_schedule_greet', { path: 'u/admin/test_schedule_greet', schedule: '0 0 9 * * *', timezone: 'UTC', @@ -2982,16 +1083,6 @@ describe('global AI tools', () => { args: {} }) - const readRaw = await callGlobalTool('read_workspace_item', { - type: 'schedule', - path: 'u/admin/test_schedule_greet' - }) - expect(JSON.parse(readRaw)).toMatchObject({ - type: 'schedule', - path: 'u/admin/test_schedule_greet', - isDraft: true - }) - await callGlobalTool('deploy_workspace_item', { type: 'schedule', path: 'u/admin/test_schedule_greet' @@ -3005,38 +1096,19 @@ describe('global AI tools', () => { }) }) // The draft is consumed by the deploy. - expect( - getBackendDraft('trigger_schedule', 'u/admin/test_schedule_greet', { - workspace: WORKSPACE - }) - ).toBeUndefined() + expect(getBackendDraft('trigger_schedule', 'u/admin/test_schedule_greet')).toBeUndefined() }) // Same private-owner read path as schedules, for the trigger drawer kinds. - it('reads and deploys a trigger draft written by the chat', async () => { - await callGlobalTool('write_trigger', { - kind: 'http', - config: { - path: 'u/admin/fresh_route', - script_path: 'f/scripts/handler', - is_flow: false, - route_path: 'api/fresh', - http_method: 'get', - authentication_method: 'none', - is_static_website: false - } - }) - - const readRaw = await callGlobalTool('read_workspace_item', { - type: 'trigger', - trigger_kind: 'http', - path: 'u/admin/fresh_route' - }) - expect(JSON.parse(readRaw)).toMatchObject({ - type: 'trigger', - triggerKind: 'http', + it('deploys a trigger draft read through the own-draft route', async () => { + seedBackendDraft('trigger_http', 'u/admin/fresh_route', { path: 'u/admin/fresh_route', - isDraft: true + script_path: 'f/scripts/handler', + is_flow: false, + route_path: 'api/fresh', + http_method: 'get', + authentication_method: 'none', + is_static_website: false }) await callGlobalTool('deploy_workspace_item', { @@ -3052,128 +1124,16 @@ describe('global AI tools', () => { script_path: 'f/scripts/handler' }) }) - expect( - getBackendDraft('trigger_http', 'u/admin/fresh_route', { workspace: WORKSPACE }) - ).toBeUndefined() - }) - - // Email is the kind an implicit "run when an email is received" request maps to; - // guards the full draft->read->deploy wiring for it end to end. - it('reads and deploys an email trigger draft written by the chat', async () => { - await callGlobalTool('write_trigger', { - kind: 'email', - config: { - path: 'u/admin/fresh_inbox', - script_path: 'f/scripts/handler', - is_flow: false, - local_part: 'support' - } - }) - - const readRaw = await callGlobalTool('read_workspace_item', { - type: 'trigger', - trigger_kind: 'email', - path: 'u/admin/fresh_inbox' - }) - expect(JSON.parse(readRaw)).toMatchObject({ - type: 'trigger', - triggerKind: 'email', - path: 'u/admin/fresh_inbox', - isDraft: true - }) - - await callGlobalTool('deploy_workspace_item', { - type: 'trigger', - trigger_kind: 'email', - path: 'u/admin/fresh_inbox' - }) - expect(EmailTriggerService.createEmailTrigger).toHaveBeenCalledWith({ - workspace: WORKSPACE, - requestBody: expect.objectContaining({ - path: 'u/admin/fresh_inbox', - local_part: 'support', - script_path: 'f/scripts/handler', - // Omitted by the caller above; defaulted so the NOT NULL column is satisfied. - workspaced_local_part: false - }) - }) - expect( - getBackendDraft('trigger_email', 'u/admin/fresh_inbox', { workspace: WORKSPACE }) - ).toBeUndefined() - }) - - // Editing an existing email trigger whose workspaced_local_part is true while - // omitting the optional field must not reset it to false (that would change the - // receiving address). The default applies only to a genuinely new draft. - it('preserves workspaced_local_part when editing an existing email trigger', async () => { - vi.mocked(EmailTriggerService.existsEmailTrigger).mockResolvedValueOnce(true) - vi.mocked(EmailTriggerService.getEmailTrigger).mockResolvedValueOnce({ - path: 'u/admin/ws_inbox', - script_path: 'f/scripts/handler', - local_part: 'support', - is_flow: false, - workspaced_local_part: true - } as any) - - await callGlobalTool('write_trigger', { - kind: 'email', - config: { - path: 'u/admin/ws_inbox', - script_path: 'f/scripts/handler', - is_flow: false, - local_part: 'support' - } - }) - - const readRaw = await callGlobalTool('read_workspace_item', { - type: 'trigger', - trigger_kind: 'email', - path: 'u/admin/ws_inbox' - }) - expect(JSON.parse(readRaw).value).toMatchObject({ workspaced_local_part: true }) - }) - - // A trigger kind whose backend routes aren't compiled in (email without - // smtp+private, EE kinds on CE) 404s on list; that must not drop the whole listing. - it('skips unavailable trigger kinds when listing', async () => { - vi.mocked(HttpTriggerService.listHttpTriggers).mockResolvedValueOnce([ - { path: 'u/admin/live_route', script_path: 'f/scripts/handler', is_flow: false } - ] as any) - vi.mocked(EmailTriggerService.listEmailTriggers).mockRejectedValueOnce( - Object.assign(new Error('not found'), { status: 404 }) - ) - - const raw = await callGlobalTool('list_workspace_items', { types: ['trigger'] }) - const paths = JSON.parse(raw).map((i: any) => i.path) - expect(paths).toContain('u/admin/live_route') - }) - - // Only a 404 means "route not compiled in". A real failure (auth, 5xx) must not - // be swallowed into a successful-but-incomplete listing. - it('propagates a non-404 trigger-list failure', async () => { - vi.mocked(HttpTriggerService.listHttpTriggers).mockRejectedValueOnce( - Object.assign(new Error('server error'), { status: 500 }) - ) - await expect(callGlobalTool('list_workspace_items', { types: ['trigger'] })).rejects.toThrow() + expect(getBackendDraft('trigger_http', 'u/admin/fresh_route')).toBeUndefined() }) // Same private-owner read path as schedules, for the resource drawer kind. - it('reads and deploys a resource draft written by the chat', async () => { - await callGlobalTool('write_resource', { - path: 'u/admin/fresh_db', - value: { host: 'db.example.com', port: 5432 }, + it('deploys a resource draft read through the own-draft route', async () => { + seedBackendDraft('resource', 'u/admin/fresh_db', { + args: { host: 'db.example.com', port: 5432 }, resource_type: 'postgresql', - description: 'fresh database' - }) - - const readRaw = await callGlobalTool('read_workspace_item', { - type: 'resource', - path: 'u/admin/fresh_db' - }) - expect(JSON.parse(readRaw)).toMatchObject({ - type: 'resource', - path: 'u/admin/fresh_db', - isDraft: true + description: 'fresh database', + wsSpecific: false }) await callGlobalTool('deploy_workspace_item', { @@ -3188,29 +1148,20 @@ describe('global AI tools', () => { value: { host: 'db.example.com', port: 5432 } }) }) - expect( - getBackendDraft('resource', 'u/admin/fresh_db', { workspace: WORKSPACE }) - ).toBeUndefined() + expect(getBackendDraft('resource', 'u/admin/fresh_db')).toBeUndefined() }) - // Same private-owner read path as schedules, for the variable drawer kind. This pins the - // plain-value cycle; the secret cases are covered above. - it('reads and deploys a non-secret variable draft written by the chat', async () => { - await callGlobalTool('write_variable', { + // Same private-owner read path as schedules, for the variable drawer kind. + it('deploys a non-secret variable draft read through the own-draft route', async () => { + seedBackendDraft('variable', 'u/admin/fresh_config', { path: 'u/admin/fresh_config', - value: 'plain-value', - is_secret: false, - description: 'fresh config' - }) - - const readRaw = await callGlobalTool('read_workspace_item', { - type: 'variable', - path: 'u/admin/fresh_config' - }) - expect(JSON.parse(readRaw)).toMatchObject({ - type: 'variable', - path: 'u/admin/fresh_config', - isDraft: true + variable: { + path: 'u/admin/fresh_config', + value: 'plain-value', + is_secret: false, + description: 'fresh config' + }, + wsSpecific: false }) await callGlobalTool('deploy_workspace_item', { @@ -3226,57 +1177,7 @@ describe('global AI tools', () => { description: 'fresh config' }) }) - expect( - getBackendDraft('variable', 'u/admin/fresh_config', { workspace: WORKSPACE }) - ).toBeUndefined() - }) - - it('requires trigger_kind when discarding a trigger draft', async () => { - await expect( - callGlobalTool('discard_local_draft', { - type: 'trigger', - path: 'f/routes/missing-kind' - }) - ).rejects.toThrow('trigger_kind is required') - }) - - it('preserves existing script metadata and seeds freshness on first script write', async () => { - vi.mocked(ScriptService.existsScriptByPath).mockResolvedValueOnce(true) - vi.mocked(ScriptService.getScriptByPath).mockResolvedValueOnce({ - path: 'f/scripts/existing', - hash: 'deployed-hash', - summary: 'deployed summary', - description: 'deployed description', - content: 'old deployed content', - language: 'bun', - kind: 'script' - } as any) - - const statuses: any[] = [] - await callGlobalTool( - 'write_script', - { - path: 'f/scripts/existing', - summary: 'new summary', - language: 'bun', - content: 'new content' - }, - { ...toolCallbacks, setToolStatus: (_toolId, status) => statuses.push(status) } - ) - - expect( - getBackendDraft('script', 'f/scripts/existing', { workspace: WORKSPACE }) - ).toMatchObject({ - path: 'f/scripts/existing', - parent_hash: 'deployed-hash', - summary: 'new summary', - description: 'deployed description', - content: 'new content', - language: 'bun' - }) - expect(statuses).toContainEqual({ - codeDiff: { before: 'old deployed content', after: 'new content', lang: 'typescript' } - }) + expect(getBackendDraft('variable', 'u/admin/fresh_config')).toBeUndefined() }) describe('stale-draft deploy guard and rebase', () => { @@ -3368,45 +1269,6 @@ describe('global AI tools', () => { expect(ScriptService.createScript).toHaveBeenCalled() }) - it('rebase_draft discards the stale draft and surfaces the changes to replay', async () => { - seedStaleScriptDraft('f/scripts/stale', 'base-hash', 'base content\nmy added line\n') - mockDeployedScript('f/scripts/stale', 'new-hash', 'latest deployed content\n') - vi.mocked(ScriptService.getScriptByHash).mockResolvedValue({ - hash: 'base-hash', - content: 'base content\n', - language: 'bun' - } as any) - - const result = JSON.parse( - await callGlobalTool('rebase_draft', { type: 'script', path: 'f/scripts/stale' }) - ) - expect(result.success).toBe(true) - expect(result.latest_hash).toBe('new-hash') - // The diff surfaces the draft's own change over its fork base. - expect(result.your_changes).toContain('my added line') - - // The stale draft is discarded (not reset), so a premature deploy fails - // cleanly rather than silently shipping the latest unchanged. - expect(getBackendDraft('script', 'f/scripts/stale', { workspace: WORKSPACE })).toBeUndefined() - await expect( - callGlobalTool('deploy_workspace_item', { type: 'script', path: 'f/scripts/stale' }) - ).rejects.toThrow(/No .*draft/) - expect(ScriptService.createScript).not.toHaveBeenCalled() - - // Re-applying re-bases onto the current head; the deploy then passes. - await callGlobalTool('write_script', { - path: 'f/scripts/stale', - summary: 's', - language: 'bun', - content: 'latest deployed content\nmy added line\n' - }) - const deploy = JSON.parse( - await callGlobalTool('deploy_workspace_item', { type: 'script', path: 'f/scripts/stale' }) - ) - expect(deploy.success).toBe(true) - expect(ScriptService.createScript).toHaveBeenCalled() - }) - function seedStaleFlowDraft(path: string, versionId: number, modules: any[] = []) { seedBackendDraft('flow', path, { path, @@ -3440,28 +1302,6 @@ describe('global AI tools', () => { expect(FlowService.createFlow).not.toHaveBeenCalled() }) - it('rebase_draft discards the stale flow draft and surfaces the changes to replay', async () => { - seedStaleFlowDraft('f/flows/stale', 1, [{ id: 'a', value: { type: 'identity' } }]) - mockDeployedFlow('f/flows/stale', 2) - vi.mocked(FlowService.getFlowVersion).mockResolvedValue({ - value: { modules: [] } - } as any) - - const result = JSON.parse( - await callGlobalTool('rebase_draft', { type: 'flow', path: 'f/flows/stale' }) - ) - expect(result.success).toBe(true) - expect(result.latest_version).toBe(2) - expect(result.your_changes).toContain('identity') - - // The stale draft is discarded, so a premature deploy fails cleanly. - expect(getBackendDraft('flow', 'f/flows/stale', { workspace: WORKSPACE })).toBeUndefined() - await expect( - callGlobalTool('deploy_workspace_item', { type: 'flow', path: 'f/flows/stale' }) - ).rejects.toThrow(/No .*draft/) - expect(FlowService.updateFlow).not.toHaveBeenCalled() - }) - function seedStaleAppDraft(path: string, parentVersion: number, file = 'old') { seedBackendDraft('raw_app', path, { summary: 'a', @@ -3483,46 +1323,6 @@ describe('global AI tools', () => { } as any) } - it('grafts the fork-base version onto a new app draft and keeps it through the save whitelist', async () => { - // No draft yet: the first app edit projects the deployed app into a draft. - // This exercises the runtime path types can't catch — the graft in - // appSourceToDraftValue AND survival through normalizeAppDraftValue's whitelist. - mockDeployedApp('f/apps/fresh', 5) - - await callGlobalTool('write_app_file', { - path: 'f/apps/fresh', - file_path: '/src/New.tsx', - content: 'export default function New() { return null }' - }) - - const draft = getBackendDraft('raw_app', 'f/apps/fresh', { workspace: WORKSPACE }) - expect(draft.parent_version).toBe(5) - }) - - it('keeps a draft-only app friendly draft_path through the save whitelist on chat edits', async () => { - // A renamed draft-only app parks its typed name in the draft's - // `draft_path`. A chat edit round-trips the value through - // normalizeAppDraftValue — dropping the field there would rename the - // app back to its `draft_` storage key. - seedBackendDraft('raw_app', 'u/admin/draft_abc', { - summary: 'a', - files: { '/index.tsx': 'old' }, - runnables: {}, - data: { tables: [] }, - draft_path: 'u/admin/my_pretty_app' - }) - - await callGlobalTool('write_app_file', { - path: 'u/admin/draft_abc', - file_path: '/index.tsx', - content: 'new' - }) - - const draft = getBackendDraft('raw_app', 'u/admin/draft_abc', { workspace: WORKSPACE }) - expect(draft.files['/index.tsx']).toBe('new') - expect(draft.draft_path).toBe('u/admin/my_pretty_app') - }) - it('lists a live raw app staged rename as draftPath even when registered at the storage key', async () => { // Flow/raw-app renames live in the value's `draft_path` while `path` // stays the storage key; a live registration whose effectivePath is the @@ -3565,962 +1365,6 @@ describe('global AI tools', () => { expect(AppService.createAppRaw).not.toHaveBeenCalled() expect(AppService.updateAppRaw).not.toHaveBeenCalled() }) - - it('rebase_draft discards the stale app draft and surfaces the changes to replay', async () => { - seedStaleAppDraft('f/apps/stale', 1, 'my-change') - mockDeployedApp('f/apps/stale', 2, 'latest-deployed') - vi.mocked(AppService.getAppByVersion).mockResolvedValue({ - value: { files: { '/index.tsx': 'base' }, runnables: {}, data: { tables: [] } } - } as any) - - const result = JSON.parse( - await callGlobalTool('rebase_draft', { type: 'app', path: 'f/apps/stale' }) - ) - expect(result.success).toBe(true) - expect(result.latest_version).toBe(2) - expect(result.your_changes).toContain('my-change') - - // The stale draft is discarded, so a premature deploy fails cleanly. - expect(getBackendDraft('raw_app', 'f/apps/stale', { workspace: WORKSPACE })).toBeUndefined() - await expect( - callGlobalTool('deploy_workspace_item', { type: 'app', path: 'f/apps/stale' }) - ).rejects.toThrow(/No .*draft/) - expect(AppService.updateAppRaw).not.toHaveBeenCalled() - }) - }) - - it('preserves existing flow metadata and seeds freshness on first flow write', async () => { - vi.mocked(FlowService.existsFlowByPath).mockResolvedValueOnce(true) - vi.mocked(FlowService.getFlowLatestVersion).mockResolvedValueOnce({ id: 42 } as any) - vi.mocked(FlowService.getFlowByPath).mockResolvedValueOnce({ - path: 'f/flows/existing', - summary: 'deployed summary', - description: 'deployed description', - value: { modules: [], chat_input_enabled: true, same_worker: true }, - schema: { properties: { deployed: { type: 'boolean' } } }, - edited_by: 'admin', - edited_at: '2026-05-22T09:00:00Z', - archived: false, - extra_perms: {} - } as any) - - await callGlobalTool('write_flow', { - path: 'f/flows/existing', - summary: 'new summary', - modules: JSON.stringify([{ id: 'step', value: { type: 'identity' } }]) - }) - - expect( - getBackendDraft('flow', 'f/flows/existing', { workspace: WORKSPACE }) - ).toMatchObject({ - path: 'f/flows/existing', - summary: 'new summary', - description: 'deployed description', - value: { - modules: [{ id: 'step', value: { type: 'identity' } }], - chat_input_enabled: true, - same_worker: true - } - }) - }) - - it('preserves editor schedule fields when writing over an existing schedule', async () => { - vi.mocked(ScheduleService.existsSchedule).mockResolvedValueOnce(true) - vi.mocked(ScheduleService.getSchedule).mockResolvedValueOnce({ - path: 'f/schedules/nightly', - schedule: '0 0 0 * * *', - timezone: 'UTC', - enabled: true, - script_path: 'f/scripts/old', - is_flow: false, - args: {}, - extra_perms: { 'u/viewer': true }, - email: 'admin@windmill.dev', - permissioned_as: 'u/admin', - edited_by: 'admin', - edited_at: '2026-05-22T09:00:00Z', - summary: 'old summary', - description: 'keep this description', - no_flow_overlap: true, - cron_version: 'v2' - } as any) - - await callGlobalTool('write_schedule', { - path: 'f/schedules/nightly', - schedule: '0 15 0 * * *', - timezone: 'Europe/Paris', - script_path: 'f/flows/new', - is_flow: true, - args: { limit: 5 } - }) - - expect( - getBackendDraft('trigger_schedule', 'f/schedules/nightly', { workspace: WORKSPACE }) - ).toMatchObject({ - path: 'f/schedules/nightly', - schedule: '0 15 0 * * *', - timezone: 'Europe/Paris', - script_path: 'f/flows/new', - is_flow: true, - args: { limit: 5 }, - extra_perms: { 'u/viewer': true }, - permissioned_as: 'u/admin', - summary: 'old summary', - description: 'keep this description', - no_flow_overlap: true - }) - expect( - getBackendDraft('trigger_schedule', 'f/schedules/nightly', { workspace: WORKSPACE }) - ).not.toMatchObject({ - edited_by: expect.anything() - }) - }) - - it('preserves editor trigger fields when writing over an existing trigger', async () => { - vi.mocked(HttpTriggerService.existsHttpTrigger).mockResolvedValueOnce(true) - vi.mocked(HttpTriggerService.getHttpTrigger).mockResolvedValueOnce({ - path: 'f/routes/api', - script_path: 'f/scripts/old', - is_flow: false, - route_path: 'api/old', - http_method: 'post', - request_type: 'sync', - authentication_method: 'none', - is_static_website: false, - workspaced_route: false, - wrap_body: false, - raw_string: false, - mode: 'enabled', - extra_perms: { 'u/viewer': true }, - workspace_id: WORKSPACE, - edited_by: 'admin', - edited_at: '2026-05-22T09:00:00Z', - permissioned_as: 'u/admin', - summary: 'old route', - description: 'keep route description' - } as any) - - await callGlobalTool('write_trigger', { - kind: 'http', - config: { - path: 'f/routes/api', - script_path: 'f/flows/new', - is_flow: true, - route_path: 'api/new', - http_method: 'get', - authentication_method: 'windmill', - is_static_website: false - } - }) - - const draft = getBackendDraft('trigger_http', 'f/routes/api', { workspace: WORKSPACE }) - expect(draft).toMatchObject({ - path: 'f/routes/api', - script_path: 'f/flows/new', - is_flow: true, - route_path: 'api/new', - http_method: 'get', - authentication_method: 'windmill', - extra_perms: { 'u/viewer': true }, - permissioned_as: 'u/admin', - summary: 'old route', - description: 'keep route description' - }) - expect(draft).not.toMatchObject({ - workspace_id: expect.anything(), - edited_by: expect.anything() - }) - }) - - it('seeds raw app draft metadata on first app write', async () => { - vi.mocked(AppService.getAppByPath).mockResolvedValueOnce({ - path: 'f/apps/report', - summary: 'deployed app', - versions: [3, 4], - value: { - files: { '/src/App.tsx': 'deployed content' }, - runnables: { - main: { - type: 'inline', - inlineScript: { language: 'bun', content: 'export async function main() {}' } - } - }, - data: { tables: ['orders'], datatable: 'db', schema: 'public' } - }, - policy: { execution_mode: 'publisher' }, - custom_path: 'report' - } as any) - - await callGlobalTool('write_app_file', { - path: 'f/apps/report', - file_path: '/src/New.tsx', - content: 'export default function New() { return null }' - }) - - const draft = getBackendDraft('raw_app', 'f/apps/report', { workspace: WORKSPACE }) - expect(draft).toMatchObject({ - summary: 'deployed app', - files: { - '/src/App.tsx': 'deployed content', - '/src/New.tsx': 'export default function New() { return null }' - }, - runnables: { - main: { - type: 'inline', - inlineScript: { language: 'bun', content: 'export async function main() {}' } - } - }, - data: { tables: ['orders'], datatable: 'db', schema: 'public' }, - policy: { execution_mode: 'publisher' }, - custom_path: 'report' - }) - }) - - // Deploying the draft keeps the deployed app's mode, which the draft does not carry. - it('reports exposure for an app that has a draft over it', async () => { - seedBackendDraft( - 'raw_app', - 'f/apps/drafted', - { - files: { '/src/App.tsx': 'x' }, - runnables: {}, - policy: {} - } as any, - { workspace: WORKSPACE } - ) - vi.mocked(AppService.existsApp).mockResolvedValueOnce(true) - vi.mocked(AppService.getAppLiteByPath).mockResolvedValueOnce({ - policy: { execution_mode: 'anonymous' } - } as any) - - const read = await callGlobalTool('read_workspace_item', { - type: 'app', - path: 'f/apps/drafted' - }) - expect(JSON.parse(read)).toMatchObject({ isDraft: true, executionMode: 'anonymous' }) - }) - - it('summarizes local raw app drafts in read_workspace_item', async () => { - seedBackendDraft( - 'raw_app', - 'f/apps/local', - { - summary: 'local app', - files: { '/src/App.tsx': 'const frontendSecret = "do-not-dump"' }, - runnables: { - main: { - type: 'inline', - inlineScript: { - language: 'bun', - content: 'const backendSecret = "do-not-dump"' - } - } - }, - data: { tables: ['orders'] } - }, - { workspace: WORKSPACE } - ) - - const raw = await callGlobalTool('read_workspace_item', { - type: 'app', - path: 'f/apps/local' - }) - const item = JSON.parse(raw) - - expect(raw).not.toContain('frontendSecret') - expect(raw).not.toContain('backendSecret') - expect(item).toMatchObject({ - type: 'app', - path: 'f/apps/local', - summary: 'local app', - isDraft: true, - value: { - frontend: [{ path: '/src/App.tsx', size: 'const frontendSecret = "do-not-dump"'.length }], - backend: [ - expect.objectContaining({ - key: 'main', - name: 'main', - type: 'inline', - language: 'bun', - contentSize: 'const backendSecret = "do-not-dump"'.length - }) - ], - data: { tables: ['orders'] } - } - }) - expect(item.value.backend[0]).not.toHaveProperty('content') - }) - - it('summarizes backend raw apps from the same source as file reads', async () => { - const deployedApp = { - path: 'f/apps/report', - summary: 'deployed app', - versions: [5], - value: { - files: { - '/src/App.tsx': 'deployed content', - '/src/Helper.tsx': 'helper content' - }, - runnables: { - main: { - type: 'inline', - inlineScript: { - language: 'bun', - content: 'export async function main() { return "deployed" }' - } - } - }, - data: { tables: ['deployed'] } - } - } - vi.mocked(AppService.getAppByPath) - .mockResolvedValueOnce(deployedApp as any) - .mockResolvedValueOnce(deployedApp as any) - - const raw = await callGlobalTool('read_workspace_item', { - type: 'app', - path: 'f/apps/report' - }) - const item = JSON.parse(raw) - - expect(item).toMatchObject({ - type: 'app', - path: 'f/apps/report', - summary: 'deployed app', - value: { - frontend: [ - { path: '/src/App.tsx', size: 'deployed content'.length }, - { path: '/src/Helper.tsx', size: 'helper content'.length } - ], - backend: [ - expect.objectContaining({ - key: 'main', - name: 'main', - type: 'inline', - language: 'bun', - contentSize: 'export async function main() { return "deployed" }'.length - }) - ], - data: { tables: ['deployed'] } - }, - isDraft: false - }) - - await expect( - callGlobalTool('read_app_file', { - path: 'f/apps/report', - file_path: '/src/Helper.tsx' - }) - ).resolves.toBe('helper content') - expect(getBackendDraft('raw_app', 'f/apps/report', { workspace: WORKSPACE })).toBeUndefined() - }) - - // A low-code app has a grid, not files and runnables, so every app tool here would - // otherwise report it as empty rather than say it is the wrong kind of app. - it('reports a low-code app without its contents, and refuses to act on it', async () => { - const lowCode = { - path: 'f/apps/legacy', - summary: 'legacy app', - raw_app: false, - value: { grid: [{ id: 'a', data: { type: 'buttoncomponent' } }] } - } as any - // ...Once per call: a persistent implementation would outlive this test and - // disarm the factory's "mock not configured" guard for the rest of the file. - for (let i = 0; i < 3; i++) vi.mocked(AppService.getAppByPath).mockResolvedValueOnce(lowCode) - - // The read answers "this app is drag-and-drop" rather than throwing — but it must not - // summarize a grid as a file/runnable list, which reads as an empty app. - const read = JSON.parse( - await callGlobalTool('read_workspace_item', { type: 'app', path: 'f/apps/legacy' }) - ) - expect(read).toMatchObject({ type: 'app', path: 'f/apps/legacy', rawApp: false }) - expect(read).not.toHaveProperty('value') - - // The tools that would convert it to files and runnables still refuse: a staged draft - // would answer every later read in place of the app itself. - for (const [tool, args] of [ - ['read_app_file', { path: 'f/apps/legacy', file_path: '/index.tsx' }], - ['write_app_file', { path: 'f/apps/legacy', file_path: '/App.tsx', content: 'x' }] - ] as [string, any][]) { - const raw = await callGlobalTool(tool, args).catch((e) => String(e)) - expect(raw).toContain('low-code app') - } - expect(getBackendDraft('raw_app', 'f/apps/legacy', { workspace: WORKSPACE })).toBeUndefined() - }) - - // This read is the only way to ask who may open a deployed app. Both kinds answer: a - // drag-and-drop app can be anonymous too, and no other tool here can inspect one. - it('reports who may open an app, whichever kind it is', async () => { - const readMode = async (app: any) => { - vi.mocked(AppService.getAppByPath).mockResolvedValueOnce(app) - const read = await callGlobalTool('read_workspace_item', { type: 'app', path: app.path }) - return JSON.parse(read).executionMode - } - const guestApp = { - path: 'f/apps/code', - summary: 'Code app', - raw_app: true, - value: { files: {}, runnables: {} }, - policy: { execution_mode: 'guest' } - } - - vi.mocked(WorkspaceService.getGuestUsage).mockResolvedValueOnce({ - available: true - } as any) - expect(await readMode(guestApp)).toBe('guest') - - expect( - await readMode({ - path: 'f/apps/builder', - summary: 'Builder app', - raw_app: false, - value: { grid: [] }, - policy: { execution_mode: 'anonymous' } - }) - ).toBe('anonymous') - - // The same app where the deployment admits no guest: reporting the mode bare would - // name an exposure the server refuses. - expect(await readMode(guestApp)).toContain('inert') - }) - - it('reads raw app files without creating a draft', async () => { - vi.mocked(AppService.getAppByPath).mockResolvedValueOnce({ - path: 'f/apps/report', - summary: 'deployed app', - versions: [5], - value: { - files: { '/src/App.tsx': 'deployed content' }, - runnables: {}, - data: { tables: [] } - } - } as any) - - await expect( - callGlobalTool('read_app_file', { - path: 'f/apps/report', - file_path: '/src/App.tsx' - }) - ).resolves.toBe('deployed content') - expect(getBackendDraft('raw_app', 'f/apps/report', { workspace: WORKSPACE })).toBeUndefined() - }) - - const deployedAppWithFile = (filePath: string, content: string) => - ({ - path: 'f/apps/report', - summary: 'deployed app', - versions: [5], - value: { files: { [filePath]: content }, runnables: {}, data: {} } - }) as any - - it('truncates a large frontend file to a head slice with a paging annotation', async () => { - const lines = Array.from({ length: 2000 }, (_, i) => `line ${i + 1}`) - vi.mocked(AppService.getAppByPath).mockResolvedValueOnce( - deployedAppWithFile('/big.tsx', lines.join('\n')) - ) - - const result = await callGlobalTool('read_app_file', { - path: 'f/apps/report', - file_path: '/big.tsx' - }) - - expect(result).toContain('lines 1-1500 of 2000.') - expect(result).toContain('offset=1501') - expect(result).toContain('line 1500') - expect(result).not.toContain('line 1501') - }) - - it('returns the requested window when offset and limit are given', async () => { - const lines = Array.from({ length: 2000 }, (_, i) => `line ${i + 1}`) - vi.mocked(AppService.getAppByPath).mockResolvedValueOnce( - deployedAppWithFile('/big.tsx', lines.join('\n')) - ) - - const result = await callGlobalTool('read_app_file', { - path: 'f/apps/report', - file_path: '/big.tsx', - offset: 5, - limit: 3 - }) - - expect(result).toContain('lines 5-7 of 2000.') - expect(result).toContain('line 5\nline 6\nline 7') - expect(result).not.toContain('line 4') - expect(result).not.toContain('line 8') - }) - - it('truncates at the character budget for files with very long lines', async () => { - const bigLine = 'x'.repeat(30_000) - vi.mocked(AppService.getAppByPath).mockResolvedValueOnce( - deployedAppWithFile('/min.tsx', [bigLine, bigLine, bigLine].join('\n')) - ) - - const result = await callGlobalTool('read_app_file', { - path: 'f/apps/report', - file_path: '/min.tsx' - }) - - expect(result).toContain('lines 1-3 of 3, truncated to the first 50000 of 90002 chars.') - expect(result).toContain('the file is likely minified') - expect(result.split('\n\n')[1]).toHaveLength(50_000) - }) - - it('caps a single-line generated file at the character budget', async () => { - const bigLine = 'x'.repeat(60_000) - vi.mocked(AppService.getAppByPath).mockResolvedValueOnce( - deployedAppWithFile('/generated.js', bigLine) - ) - - const result = await callGlobalTool('read_app_file', { - path: 'f/apps/report', - file_path: '/generated.js' - }) - - expect(result).toContain('lines 1-1 of 1, truncated to the first 50000 of 60000 chars.') - expect(result).toContain('re-read with a smaller limit') - expect(result.split('\n\n')[1]).toBe('x'.repeat(50_000)) - }) - - it('reports an offset past the end of the file plainly', async () => { - const lines = Array.from({ length: 10 }, (_, i) => `line ${i + 1}`) - vi.mocked(AppService.getAppByPath).mockResolvedValueOnce( - deployedAppWithFile('/small.tsx', lines.join('\n')) - ) - - await expect( - callGlobalTool('read_app_file', { - path: 'f/apps/report', - file_path: '/small.tsx', - offset: 50 - }) - ).resolves.toBe('offset 50 is past the end of the file (10 lines).') - }) - - // Deterministic micro-benchmark: measures how much context the read_app_file cap - // saves over a realistic big-project read pattern, isolated from model - // nondeterminism. "Baseline" is the old behavior (whole file returned on every - // read); "actual" is the current line cap + char budget + paging. Asserting the - // ratio also guards against a future change silently weakening the savings. - it('micro-benchmark: the read cap cuts returned context for a realistic read pattern', async () => { - const bigContent = Array.from({ length: 5000 }, (_, i) => `const row${i} = ${i};`).join('\n') - const minified = 'a'.repeat(200_000) // single long line (e.g. a generated bundle) - const appValue = { - path: 'f/apps/report', - summary: 'big app', - versions: [5], - value: { files: { '/big.tsx': bigContent, '/min.js': minified }, runnables: {}, data: {} } - } as any - const fullSize: Record = { - '/big.tsx': bigContent.length, - '/min.js': minified.length - } - - // A plausible pass over a large app: read a big file head, page deeper into it, - // then hit a generated bundle (capped at the char budget). The old tool returned - // every file in full on every read. - const sequence = [ - { file_path: '/big.tsx' }, // 1. big file head (line cap) - { file_path: '/big.tsx', offset: 1501 }, // 2. next line chunk - { file_path: '/min.js' } // 3. minified bundle head (char budget) - ] - - let baselineChars = 0 - let actualChars = 0 - const perRead: number[] = [] - for (const read of sequence) { - baselineChars += fullSize[read.file_path] - vi.mocked(AppService.getAppByPath).mockResolvedValueOnce(appValue) - const out = await callGlobalTool('read_app_file', { path: 'f/apps/report', ...read }) - actualChars += out.length - perRead.push(out.length) - } - - const reductionPct = Math.round((1 - actualChars / baselineChars) * 100) - // Surfaced when the suite runs so the benchmark is readable, not just asserted. - // eslint-disable-next-line no-console - console.log( - `[read_app_file micro-benchmark] baseline=${baselineChars} chars, actual=${actualChars} chars ` + - `(per-read ${perRead.join(', ')}), reduction=${reductionPct}%` - ) - - // Each capped read is far smaller than the whole file it came from: - expect(perRead[0]).toBeLessThan(fullSize['/big.tsx']) // head slice < whole file - expect(perRead[1]).toBeLessThan(fullSize['/big.tsx']) // a paged line chunk too - expect(perRead[2]).toBeLessThan(51_000) // ~50k char budget + a short annotation - // Overall: well under half the bytes the old tool would have returned. - expect(actualChars).toBeLessThan(baselineChars * 0.5) - }) - - // A multi-file app: the revenue helper is referenced in three frontend files - // and one inline backend runnable; a generated file also mentions it (and must - // be excluded). Mirrors the analytics_dashboard fixture's "symbol spread". - const searchAppValue = () => - ({ - path: 'f/apps/report', - summary: 'search app', - versions: [5], - value: { - files: { - '/lib/aggregations.ts': 'export function computeRevenue(o) {\n return o.unitPrice\n}\n', - '/components/SummaryPanel.tsx': - 'import { computeRevenue } from "../lib/aggregations"\nconst total = computeRevenue(order)\n', - '/components/OrdersTable.tsx': 'const r = computeRevenue(row)\n// renders revenue\n', - '/styles.css': '.revenue { color: green }\n', - '/wmill.d.ts': 'declare function computeRevenue(o: any): number\n' - }, - runnables: { - computeSummary: { - type: 'inline', - inlineScript: { - language: 'bun', - content: 'export async function main() {\n return computeRevenue\n}\n' - } - } - }, - data: {} - } - }) as any - - it('greps across frontend files and inline runnables, returning file:line rows', async () => { - vi.mocked(AppService.getAppByPath).mockResolvedValueOnce(searchAppValue()) - - const result = await callGlobalTool('search_app', { - path: 'f/apps/report', - query: 'computeRevenue' - }) - - // header counts every match across the (non-generated) files, without echoing the query - expect(result).toMatch(/\d+ match(?:es)? in \d+ files?/) - // frontend rows use read_app_file's leading-slash addressing - expect(result).toContain('/lib/aggregations.ts') - expect(result).toContain('1: export function computeRevenue(o) {') - expect(result).toContain('/components/SummaryPanel.tsx') - // inline runnable rows use the backend//main. addressing - expect(result).toContain('backend/computeSummary/main.ts') - // generated files are never searched - expect(result).not.toContain('/wmill.d.ts') - }) - - it('matches case-insensitively', async () => { - vi.mocked(AppService.getAppByPath).mockResolvedValueOnce(searchAppValue()) - - const result = await callGlobalTool('search_app', { - path: 'f/apps/report', - query: 'COMPUTEREVENUE' // upper-case query still matches computeRevenue - }) - - expect(result).toContain('/lib/aggregations.ts') - expect(result).toContain('export function computeRevenue(o) {') - }) - - it('filters by a basename glob (matches nested files)', async () => { - vi.mocked(AppService.getAppByPath).mockResolvedValueOnce(searchAppValue()) - - const result = await callGlobalTool('search_app', { - path: 'f/apps/report', - query: 'computeRevenue', - file_glob: '*.tsx' - }) - - expect(result).toContain('/components/SummaryPanel.tsx') - expect(result).toContain('/components/OrdersTable.tsx') - expect(result).not.toContain('/lib/aggregations.ts') - expect(result).not.toContain('backend/computeSummary/main.ts') - }) - - it('filters by a path glob (e.g. backend/**)', async () => { - vi.mocked(AppService.getAppByPath).mockResolvedValueOnce(searchAppValue()) - - const result = await callGlobalTool('search_app', { - path: 'f/apps/report', - query: 'computeRevenue', - file_glob: 'backend/**' - }) - - expect(result).toContain('backend/computeSummary/main.ts') - expect(result).not.toContain('/lib/aggregations.ts') - }) - - it('reports zero matches with a hint instead of an empty result', async () => { - vi.mocked(AppService.getAppByPath).mockResolvedValueOnce(searchAppValue()) - - const result = await callGlobalTool('search_app', { - path: 'f/apps/report', - query: 'nonexistent_symbol_xyz' - }) - - expect(result).toContain('No matches') - expect(result).toContain('Try a broader') - }) - - it('truncates very long matching lines to keep results sparse', async () => { - const longLine = `const x = "${'q'.repeat(5000)} computeRevenue"` - vi.mocked(AppService.getAppByPath).mockResolvedValueOnce({ - path: 'f/apps/report', - summary: 'app', - versions: [5], - value: { files: { '/min.js': longLine }, runnables: {}, data: {} } - } as any) - - const result = await callGlobalTool('search_app', { - path: 'f/apps/report', - query: 'computeRevenue' - }) - - expect(result).toContain('[line truncated]') - expect(result.length).toBeLessThan(1000) - }) - - it('caps the number of match rows and says it truncated', async () => { - const manyLines = Array.from({ length: 500 }, (_, i) => `hit ${i}`).join('\n') - vi.mocked(AppService.getAppByPath).mockResolvedValueOnce({ - path: 'f/apps/report', - summary: 'app', - versions: [5], - value: { files: { '/big.tsx': manyLines }, runnables: {}, data: {} } - } as any) - - const result = await callGlobalTool('search_app', { - path: 'f/apps/report', - query: 'hit', - max_matches: 50 - }) - - expect(result).toContain('500 matches') - expect(result).toContain('showing the first 50') - // 50 capped match lines, each rendered with its fixed context window (deduped), - // so the body is bounded near max_matches and nowhere near the 500 total. - const rows = result.split('\n').filter((l) => /^\s+\d+: /.test(l)).length - expect(rows).toBeGreaterThanOrEqual(50) - expect(rows).toBeLessThan(60) - }) - - it('counts every file with a match, even matches past the render cap', async () => { - // The first (sorted) file exhausts max_matches; the later file's match falls - // past the cap but the symbol still lives there, so the header must count it. - vi.mocked(AppService.getAppByPath).mockResolvedValueOnce({ - path: 'f/apps/report', - summary: 'app', - versions: [5], - value: { - files: { '/a.tsx': 'hit\nhit\nhit\nhit\nhit', '/b.tsx': 'hit' }, - runnables: {}, - data: {} - } - } as any) - - const result = await callGlobalTool('search_app', { - path: 'f/apps/report', - query: 'hit', - max_matches: 3 - }) - - expect(result).toContain('6 matches in 2 files') - expect(result).toContain('showing the first 3') - }) - - // Deterministic micro-benchmark: how much context a single search_app call - // saves over locating a symbol by reading the candidate files whole. Baseline - // is the conservative "read only the files that actually contain the symbol" - // path (a model without search must read at least those in full); the real - // saving is larger because, lacking search, a model often reads non-matching - // files too. Isolated from model nondeterminism so it can gate regressions. - it('micro-benchmark: search_app locates a symbol far cheaper than reading files', async () => { - const fileBodies: Record = {} - // 8 component files, 3 of which reference the symbol, each ~120 lines. - for (let f = 0; f < 8; f++) { - const lines = Array.from({ length: 120 }, (_, i) => - f < 3 && i === 60 ? ` return computeRevenue(order${f})` : ` const v${i} = ${i} // padding` - ) - fileBodies[`/components/File${f}.tsx`] = lines.join('\n') - } - const appValue = { - path: 'f/apps/report', - summary: 'big app', - versions: [5], - value: { - files: fileBodies, - runnables: {}, - data: {} - } - } as any - - vi.mocked(AppService.getAppByPath).mockResolvedValueOnce(appValue) - const searchOut = await callGlobalTool('search_app', { - path: 'f/apps/report', - query: 'computeRevenue' - }) - - // Baseline: the bytes a model must pull to gather the same locations by - // reading each matching file in full. - const matchingFiles = Object.entries(fileBodies).filter(([, body]) => - body.includes('computeRevenue') - ) - const baselineChars = matchingFiles.reduce((sum, [, body]) => sum + body.length, 0) - const actualChars = searchOut.length - const reductionPct = Math.round((1 - actualChars / baselineChars) * 100) - // eslint-disable-next-line no-console - console.log( - `[search_app micro-benchmark] baseline=${baselineChars} chars (read ${matchingFiles.length} files whole), ` + - `actual=${actualChars} chars (one search), reduction=${reductionPct}%` - ) - - // The search surfaced exactly the 3 locations… - expect(matchingFiles.length).toBe(3) - expect((searchOut.match(/computeRevenue/g) ?? []).length).toBeGreaterThanOrEqual(3) - // …at a tiny fraction of reading those files whole. - expect(actualChars).toBeLessThan(baselineChars * 0.15) - }) - - it('does not persist a raw app draft when patch_app_file validation fails', async () => { - vi.mocked(AppService.getAppByPath).mockResolvedValueOnce({ - path: 'f/apps/report', - summary: 'deployed app', - versions: [5], - value: { - files: { '/src/App.tsx': 'deployed content' }, - runnables: {}, - data: { tables: [] } - } - } as any) - - await expect( - callGlobalTool('patch_app_file', { - path: 'f/apps/report', - file_path: '/src/App.tsx', - old_string: 'missing content', - new_string: 'replacement', - replace_all: false - }) - ).rejects.toThrow() - expect(getBackendDraft('raw_app', 'f/apps/report', { workspace: WORKSPACE })).toBeUndefined() - }) - - it('records the diff of the one raw app file a call changed', async () => { - seedBackendDraft( - 'raw_app', - 'u/admin/diffed_app', - { - summary: 'Diffed', - files: { '/index.tsx': 'untouched', '/styles.css': 'a { color: red; }\n' }, - runnables: { - go: { - name: 'Go', - type: 'inline', - inlineScript: { language: 'python3', content: 'x = 1' } - }, - greet: { - name: 'Greet', - type: 'inline', - inlineScript: { language: 'bun', content: 'export const main = () => 1' } - } - }, - data: { tables: [] } - }, - { workspace: WORKSPACE } - ) - const statuses: any[] = [] - const callbacks = { - ...toolCallbacks, - setToolStatus: (_toolId: string, status: any) => statuses.push(status) - } - - await callGlobalTool( - 'patch_app_file', - { - path: 'u/admin/diffed_app', - file_path: '/styles.css', - old_string: 'red', - new_string: 'blue', - replace_all: false - }, - callbacks - ) - await callGlobalTool( - 'delete_app_runnable', - { path: 'u/admin/diffed_app', key: 'go' }, - callbacks - ) - await callGlobalTool( - 'write_app_runnable', - { - path: 'u/admin/diffed_app', - key: 'ref', - runnable: { name: 'Ref', type: 'flow', path: 'u/admin/hello_flow' } - }, - callbacks - ) - await callGlobalTool( - 'write_app_runnable', - { - path: 'u/admin/diffed_app', - key: 'greet', - runnable: { - name: 'Greet', - type: 'inline', - inlineScript: { language: 'python3', content: 'def main(): return 1' } - } - }, - callbacks - ) - - expect(statuses.filter((status) => status.codeDiff).map((status) => status.codeDiff)).toEqual([ - { before: 'a { color: red; }\n', after: 'a { color: blue; }\n', lang: 'css' }, - { before: 'x = 1', after: '', lang: 'python' }, - { - before: 'export const main = () => 1', - after: 'def main(): return 1', - lang: 'python', - beforeLang: 'typescript' - } - ]) - }) - - it('does not persist a raw app draft when delete_app_file validation fails', async () => { - vi.mocked(AppService.getAppByPath).mockResolvedValueOnce({ - path: 'f/apps/report', - summary: 'deployed app', - versions: [5], - value: { - files: { '/src/App.tsx': 'deployed content' }, - runnables: {}, - data: { tables: [] } - } - } as any) - - await expect( - callGlobalTool('delete_app_file', { - path: 'f/apps/report', - file_path: '/src/Missing.tsx' - }) - ).rejects.toThrow('Frontend file "/src/Missing.tsx" not found in app "f/apps/report".') - expect(getBackendDraft('raw_app', 'f/apps/report', { workspace: WORKSPACE })).toBeUndefined() - }) - - it('does not persist a raw app draft when delete_app_runnable validation fails', async () => { - vi.mocked(AppService.getAppByPath).mockResolvedValueOnce({ - path: 'f/apps/report', - summary: 'deployed app', - versions: [5], - value: { - files: { '/src/App.tsx': 'deployed content' }, - runnables: { - main: { - type: 'inline', - inlineScript: { language: 'bun', content: 'export async function main() {}' } - } - }, - data: { tables: [] } - } - } as any) - - await expect( - callGlobalTool('delete_app_runnable', { - path: 'f/apps/report', - key: 'missing' - }) - ).rejects.toThrow('Backend runnable "missing" not found in app "f/apps/report".') - expect(getBackendDraft('raw_app', 'f/apps/report', { workspace: WORKSPACE })).toBeUndefined() }) it('deploys a new raw app draft by bundling files and creating a raw app', async () => { @@ -4868,432 +1712,18 @@ describe('global AI tools', () => { } }) - it('fills an empty rawscript module through set_flow_module_code', async () => { - await callGlobalTool('write_flow', { - path: 'f/flows/empty-module', - summary: 'Flow with empty module', - modules: JSON.stringify([ - { - id: 'empty_step', - value: { - type: 'rawscript', - language: 'bun', - content: '', - input_transforms: {} - } - } - ]) - }) - - const code = 'export async function main() {\n\treturn 42\n}' - - await expect( - callGlobalTool('set_flow_module_code', { - path: 'f/flows/empty-module', - module_id: 'empty_step', - code - }) - ).resolves.toContain('Updated flow') - - await expect( - callGlobalTool('read_flow_module_code', { - path: 'f/flows/empty-module', - module_id: 'empty_step' - }) - ).resolves.toBe(code) - }) - - // The worker reuses a non-empty step lock, so an edited step keeping its lock deploys - // against the old dependencies. - it('set_flow_module_code drops the lock of the edited step only', async () => { - const step = (id: string) => ({ - id, - value: { - type: 'rawscript', - language: 'python3', - content: 'import requests', - lock: 'requests==2.0', - input_transforms: {} - } - }) - seedBackendDraft('flow', 'f/flows/locked', { - path: 'f/flows/locked', - summary: 'f', - description: '', - value: { modules: [step('edited'), step('untouched')] }, - schema: {} - }) - - await callGlobalTool('set_flow_module_code', { - path: 'f/flows/locked', - module_id: 'edited', - code: 'import yaml' - }) - - const [edited, untouched] = getBackendDraft('flow', 'f/flows/locked').value.modules - expect(edited.value.content).toBe('import yaml') - expect(edited.value).not.toHaveProperty('lock') - expect(untouched.value.lock).toBe('requests==2.0') - }) - - it('warns about every empty rawscript body (top-level, nested, preprocessor, failure) and skips populated ones', async () => { - const result = JSON.parse( - await callGlobalTool('write_flow', { - path: 'f/flows/empty-bodies', - modules: JSON.stringify([ - { - id: 'empty_top', - value: { type: 'rawscript', language: 'bun', content: '', input_transforms: {} } - }, - { - id: 'filled', - value: { - type: 'rawscript', - language: 'bun', - content: 'export async function main() { return 1 }', - input_transforms: {} - } - }, - { - id: 'loop', - value: { - type: 'forloopflow', - iterator: { type: 'javascript', expr: 'results.filled' }, - skip_failures: false, - modules: [ - { - id: 'empty_nested', - value: { - type: 'rawscript', - language: 'bun', - content: '', - input_transforms: {} - } - } - ] - } - } - ]), - preprocessor_module: JSON.stringify({ - id: 'preprocessor', - value: { type: 'rawscript', language: 'bun', content: '', input_transforms: {} } - }), - failure_module: JSON.stringify({ - id: 'failure', - value: { type: 'rawscript', language: 'bun', content: '', input_transforms: {} } - }) - }) - ) - - expect(result.success).toBe(true) - expect(result.message).toContain('set_flow_module_code') - for (const id of ['empty_top', 'empty_nested', 'preprocessor', 'failure']) { - expect(result.message).toContain(`"${id}"`) - } - expect(result.message).not.toContain('"filled"') - }) - - it('does not append the empty-body warning when the flow was not saved', async () => { - const path = 'f/flows/write-fails' - failingWrites.add(`flow:${path}`) - - const result = JSON.parse( - await callGlobalTool('write_flow', { - path, - modules: JSON.stringify([ - { - id: 'empty_step', - value: { type: 'rawscript', language: 'bun', content: '', input_transforms: {} } - } - ]) - }) - ) - - expect(result.success).toBe(false) - expect(JSON.stringify(result)).not.toContain('set_flow_module_code') - }) - - it('hints at inline-code escaping when the modules JSON fails to parse', async () => { - await expect( - callGlobalTool('write_flow', { - path: 'f/flows/bad-json', - modules: '[{"id":"a","value":{"type":"rawscript","content":"oops"}]' - }) - ).rejects.toThrow(/Invalid JSON for modules.*set_flow_module_code/s) - }) - - it('warns when patch_flow_json adds a rawscript module left as an inline_script placeholder', async () => { - const path = 'f/flows/patch-new-module' - await callGlobalTool('write_flow', { - path, - modules: JSON.stringify([ - { - id: 'call_api', - value: { - type: 'rawscript', - language: 'bun', - content: 'export async function main() { return 1 }', - input_transforms: {} - } - } - ]) - }) - - // A structural patch that adds no module must not trigger the fill-code warning. - const benign = JSON.parse( - await callGlobalTool('patch_flow_json', { - path, - old_string: '"language":"bun"', - new_string: '"language":"deno"' - }) - ) - expect(benign.success).toBe(true) - expect(benign.message).not.toContain('set_flow_module_code') - - // Adding a new rawscript module in the compact view carries the - // inline_script. placeholder as its content; the result must flag it. - const result = JSON.parse( - await callGlobalTool('patch_flow_json', { - path, - old_string: '"input_transforms":{}}}]', - new_string: - '"input_transforms":{}}},{"id":"write_to_pg","value":{"type":"rawscript","language":"postgresql","content":"inline_script.write_to_pg","input_transforms":{}}}]' - }) - ) - expect(result.success).toBe(true) - expect(result.message).toContain('set_flow_module_code') - expect(result.message).toContain('"write_to_pg"') - expect(result.message).not.toContain('"call_api"') - - // The placeholder is blanked, never persisted as literal content, and the - // existing module's body survives the patch round-trip. - await expect( - callGlobalTool('read_flow_module_code', { path, module_id: 'write_to_pg' }) - ).resolves.toBe('') - await expect( - callGlobalTool('read_flow_module_code', { path, module_id: 'call_api' }) - ).resolves.toBe('export async function main() { return 1 }') - }) - - it('rejects a patch whose inline_script placeholder references no module', async () => { - const path = 'f/flows/patch-bad-ref' - await callGlobalTool('write_flow', { - path, - modules: JSON.stringify([ - { - id: 'call_api', - value: { - type: 'rawscript', - language: 'bun', - content: 'export async function main() { return 1 }', - input_transforms: {} - } - } - ]) - }) - - await expect( - callGlobalTool('patch_flow_json', { - path, - old_string: '"input_transforms":{}}}]', - new_string: - '"input_transforms":{}}},{"id":"write_to_pg","value":{"type":"rawscript","language":"postgresql","content":"inline_script.call_apy","input_transforms":{}}}]' - }) - ).rejects.toThrow(/Unresolved inline script reference/) - - // The rejected patch must not have touched the draft. - await expect( - callGlobalTool('read_flow_module_code', { path, module_id: 'call_api' }) - ).resolves.toBe('export async function main() { return 1 }') - }) - - it('write_flow resolves placeholders to existing module bodies on overwrite', async () => { - const path = 'f/flows/overwrite-keep-bodies' - const code = 'export async function main() { return 1 }' - await callGlobalTool('write_flow', { - path, - modules: JSON.stringify([ - { - id: 'call_api', - value: { type: 'rawscript', language: 'bun', content: code, input_transforms: {} } - } - ]) - }) - - const result = JSON.parse( - await callGlobalTool('write_flow', { - path, - summary: 'Reordered', - modules: JSON.stringify([ - { - id: 'call_api', - value: { - type: 'rawscript', - language: 'bun', - content: 'inline_script.call_api', - input_transforms: {} - } - }, - { - id: 'notify', - value: { type: 'rawscript', language: 'bun', content: '', input_transforms: {} } - } - ]) - }) - ) - expect(result.success).toBe(true) - expect(result.message).toContain('"notify"') - expect(result.message).not.toContain('"call_api"') - - await expect( - callGlobalTool('read_flow_module_code', { path, module_id: 'call_api' }) - ).resolves.toBe(code) - }) - - it('writes flows with flow-mode arguments and reads compact flow value', async () => { - const writeResult = JSON.parse( - await callGlobalTool('write_flow', { - path: 'f/flows/with-schema-and-groups', - summary: 'Flow with schema and groups', - modules: JSON.stringify([ - { - id: 'start', - summary: 'Start', - value: { - type: 'identity' - } - } - ]), - schema: JSON.stringify({ - type: 'object', - properties: { - name: { type: 'string' } - }, - required: ['name'] - }), - groups: JSON.stringify([{ summary: 'Main', start_id: 'start', end_id: 'start' }]) - }) - ) - - expect(writeResult.success).toBe(true) - // Write results must not echo the flow value back to the model. - expect(writeResult.item).toBeUndefined() - - const raw = await callGlobalTool('read_workspace_item', { - type: 'flow', - path: 'f/flows/with-schema-and-groups' - }) - const item = JSON.parse(raw) - - expect(item.value).toMatchObject({ - modules: [ - { - id: 'start', - summary: 'Start', - value: { type: 'identity' } - } - ], - schema: { - type: 'object', - properties: { - name: { type: 'string' } - }, - required: ['name'] - }, - preprocessor_module: null, - failure_module: null, - groups: [{ summary: 'Main', start_id: 'start', end_id: 'start' }] - }) - expect(item.value.value).toBeUndefined() - }) - - it('writes and reads back free-floating flow notes', async () => { - const writeResult = JSON.parse( - await callGlobalTool('write_flow', { - path: 'f/flows/with-notes', - summary: 'Flow with notes', - modules: JSON.stringify([ - { - id: 'start', - summary: 'Start', - value: { type: 'identity' } - } - ]), - notes: JSON.stringify([ - { id: 'n1', type: 'free', text: 'What this flow does', color: 'blue' } - ]) - }) - ) - - expect(writeResult.success).toBe(true) - - const item = JSON.parse( - await callGlobalTool('read_workspace_item', { - type: 'flow', - path: 'f/flows/with-notes' - }) - ) - - expect(item.value.notes).toHaveLength(1) - expect(item.value.notes[0]).toMatchObject({ - id: 'n1', - type: 'free', - text: 'What this flow does', - color: 'blue' - }) - // Free notes with no explicit geometry get auto-placed/sized by validation. - expect(item.value.notes[0].position).toBeDefined() - expect(item.value.notes[0].size).toBeDefined() - }) - - it('test_run_script previews draft script content by path', async () => { - const content = 'export async function main(name: string) {\n\treturn `hello ${name}`\n}' - await callGlobalTool('write_script', { - path: 'f/scripts/draft-test', - summary: 'Draft test script', - language: 'bun', - content - }) - - const result = await withCompletedTestJob(() => - callGlobalTool('test_run_script', { - path: 'f/scripts/draft-test' - }) - ) - - expect(JobService.runScriptPreview).toHaveBeenCalledWith({ - workspace: WORKSPACE, - requestBody: { - path: 'f/scripts/draft-test', - content, - args: {}, - language: 'bun' - } - }) - expect(ScriptService.getScriptByPath).not.toHaveBeenCalled() - expect(result).toContain('Result (SUCCESS)') - expect(result).toContain('test logs') - }) - - // No parser emits `password`, so the stored schema is the only thing carrying it: an edit - // that rewrites the draft's schema from scratch, or a draft read that drops it, unmarks - // the field — and the form then takes the secret as a plain literal into the job's args. + // No parser emits `password`, so the stored schema is the only thing carrying it: a form + // rebuilt from the content alone unmarks the field, and then takes the secret as a plain + // literal into the job's args. it('test_run_script keeps the password marking of the script it previews', async () => { - vi.mocked(ScriptService.existsScriptByPath).mockResolvedValueOnce(true) - vi.mocked(ScriptService.getScriptByPath).mockResolvedValueOnce({ - path: 'f/scripts/secretful', - language: 'bun', + stubRun({ + code: 'export async function main(token: string) { return 1 }', + lang: 'bun', schema: { type: 'object', properties: { token: { type: 'string', password: true } }, required: ['token'] } - } as any) - - await callGlobalTool('write_script', { - path: 'f/scripts/secretful', - language: 'bun', - content: 'export async function main(token: string) { return 1 }' }) let form: any @@ -5312,45 +1742,15 @@ describe('global AI tools', () => { expect(form?.schema?.properties).toMatchObject({ token: { password: true } }) }) - it('test_run_script previews deployed script content when no draft exists', async () => { - vi.mocked(ScriptService.getScriptByPath).mockResolvedValueOnce({ - path: 'f/scripts/deployed-test', - summary: 'Deployed test script', - content: 'def main(name):\n return name', - language: 'python3' - } as any) - - await withCompletedTestJob(() => - callGlobalTool('test_run_script', { - path: 'f/scripts/deployed-test' - }) - ) - - expect(ScriptService.getScriptByPath).toHaveBeenCalledWith({ - workspace: WORKSPACE, - path: 'f/scripts/deployed-test' - }) - expect(JobService.runScriptPreview).toHaveBeenCalledWith({ - workspace: WORKSPACE, - requestBody: { - path: 'f/scripts/deployed-test', - content: 'def main(name):\n return name', - args: {}, - language: 'python3' - } - }) - }) - // A test run meets the same card as a deployed one, so what previews is what the form // submitted — not what the model proposed. it('test_run_script previews the arguments the form submitted', async () => { - vi.mocked(ScriptService.getScriptByPath).mockResolvedValueOnce({ - path: 'f/scripts/formed-test', + stubRun({ summary: 'Formed test script', - content: 'export async function main(name: string) {}', - language: 'bun', + code: 'export async function main(name: string) {}', + lang: 'bun', schema: { properties: { name: { type: 'string' } } } - } as any) + }) let opened: Record | undefined let kind: string | undefined @@ -5380,15 +1780,7 @@ describe('global AI tools', () => { code: 'export async function main(name: string) {}', lang: 'bun' }) - expect(JobService.runScriptPreview).toHaveBeenCalledWith({ - workspace: WORKSPACE, - requestBody: { - path: 'f/scripts/formed-test', - content: 'export async function main(name: string) {}', - args: { name: 'Grace' }, - language: 'bun' - } - }) + expect(startedRun()?.args).toEqual({ name: 'Grace' }) }) // The bypass posture answers a run form as it answers any other confirmation, for a @@ -5396,14 +1788,11 @@ describe('global AI tools', () => { // will fill in is attached already settled, so no field is ever mounted, and the schema // it would have built them from never reaches the transcript. it('mounts no field on either run form under yolo', async () => { - const script = { - path: 'f/scripts/yolo', - content: 'export async function main(name: string) {}', - language: 'bun', + stubRun({ + code: 'export async function main(name: string) {}', + lang: 'bun', schema: { properties: { name: { type: 'string' } } } - } as any - vi.mocked(ScriptService.getScriptByPath).mockResolvedValueOnce(script) - vi.mocked(ScriptService.getScriptByPath).mockResolvedValueOnce(script) + }) const statuses: any[] = [] const requestRunArgs = vi.fn(async (_toolId: string, form: any) => ({ @@ -5427,11 +1816,10 @@ describe('global AI tools', () => { expect(testForm.schema).toBeUndefined() // Told the form is already answered, or the loop parks on a card with no fields. expect(requestRunArgs.mock.calls[0][2]).toEqual({ autoAccepted: true }) - expect(JobService.runScriptPreview).toHaveBeenCalledWith( - expect.objectContaining({ requestBody: expect.objectContaining({ args: { name: 'Ada' } }) }) - ) + expect(startedRun()?.args).toEqual({ name: 'Ada' }) statuses.length = 0 + vi.mocked(AiService.callAiSessionTool).mockClear() await withCompletedTestJob(() => callGlobalTool('run_script', { path: 'f/scripts/yolo', args: { name: 'Ada' } }, yolo) ) @@ -5439,11 +1827,7 @@ describe('global AI tools', () => { const deployedForm = statuses.find((s) => s.runForm)?.runForm expect(deployedForm.submitted).toBe(true) expect(deployedForm.schema).toBeUndefined() - // skipPreprocessor: the form fills the main input schema, so a preprocessor would take - // these arguments for a webhook body and run main on its output instead. - expect(JobService.runScriptByPath).toHaveBeenCalledWith( - expect.objectContaining({ requestBody: { name: 'Ada' }, skipPreprocessor: true }) - ) + expect(startedRun()?.args).toEqual({ name: 'Ada' }) }) // The posture is the user's standing answer to whether to ask, so a run it answers starts on @@ -5451,13 +1835,12 @@ describe('global AI tools', () => { // Predicting what a mounted field would hold starts runs the form itself would refuse; the // schema's own defaults are the worker's job, from the code's signature. it('sends the model arguments as proposed when the posture answers', async () => { - vi.mocked(ScriptService.getScriptByPath).mockResolvedValueOnce({ - path: 'f/scripts/defaulted', + stubRun({ schema: { properties: { name: { type: 'string' }, retries: { type: 'number', default: 3 } }, required: ['name', 'retries'] } - } as any) + }) const statuses: any[] = [] await withCompletedTestJob(() => callGlobalTool( @@ -5473,23 +1856,20 @@ describe('global AI tools', () => { ) expect(statuses.find((x) => x.runForm)?.runForm.submitted).toBe(true) - expect(JobService.runScriptByPath).toHaveBeenCalledWith( - expect.objectContaining({ requestBody: { name: 'Ada' } }) - ) + expect(startedRun()?.args).toEqual({ name: 'Ada' }) }) // A disabled field is declared as not the caller's to set, and the posture answering the // form does not make the model one of the callers it is kept from. it('holds a disabled default against the model when the posture answers', async () => { - vi.mocked(ScriptService.getScriptByPath).mockResolvedValueOnce({ - path: 'f/scripts/locked', + stubRun({ schema: { properties: { name: { type: 'string' }, mode: { type: 'string', disabled: true, default: 'safe' } } } - } as any) + }) const result = await withCompletedTestJob(() => callGlobalTool( @@ -5503,9 +1883,7 @@ describe('global AI tools', () => { ) ) - expect(JobService.runScriptByPath).toHaveBeenCalledWith( - expect.objectContaining({ requestBody: { name: 'Ada', mode: 'safe' } }) - ) + expect(startedRun()?.args).toEqual({ name: 'Ada', mode: 'safe' }) // Or the next call proposes the same override again. expect(result).toContain('disables mode') }) @@ -5514,10 +1892,9 @@ describe('global AI tools', () => { // a job's arguments are readable by everyone who can see its run. What starts the job must // be what came back from the minting, never the proposal. it('mints a proposed secret into a reference before starting a run the posture answers', async () => { - vi.mocked(ScriptService.getScriptByPath).mockResolvedValueOnce({ - path: 'f/scripts/secret', + stubRun({ schema: { properties: { token: { type: 'string', password: true } }, required: ['token'] } - } as any) + }) vi.mocked(processSecretArgs).mockImplementationOnce(async () => ({ token: '$var:u/ada/secret_arg/minted' })) @@ -5539,18 +1916,15 @@ describe('global AI tools', () => { expect.anything(), expect.anything() ) - expect(JobService.runScriptByPath).toHaveBeenCalledWith( - expect.objectContaining({ requestBody: { token: '$var:u/ada/secret_arg/minted' } }) - ) + expect(startedRun()?.args).toEqual({ token: '$var:u/ada/secret_arg/minted' }) }) // The bypass is the user's standing answer, not a licence for the host to skip asking: // a chat with nowhere to put a form still refuses the run under any other posture. it('run_script refuses a host with no form unless the posture answers for it', async () => { - vi.mocked(ScriptService.getScriptByPath).mockResolvedValue({ - path: 'f/scripts/noform', + stubRun({ schema: { properties: { name: { type: 'string' } } } - } as any) + }) const refused = await callGlobalTool( 'run_script', @@ -5558,7 +1932,7 @@ describe('global AI tools', () => { { ...toolCallbacks, requestRunArgs: undefined } ) - expect(JobService.runScriptByPath).not.toHaveBeenCalled() + expect(startedRun()).toBeUndefined() expect(refused).toContain('cannot show a run form') }) @@ -5566,11 +1940,10 @@ describe('global AI tools', () => { // start is the one end it cannot see for itself: the call is neither pending nor cancelled, // and nothing else tells it to stop offering a Run that would never reach a job. it('run_script says the form is over when the job is refused', async () => { - vi.mocked(ScriptService.getScriptByPath).mockResolvedValue({ - path: 'f/scripts/refused', - schema: { properties: { name: { type: 'string' } } } - } as any) - vi.mocked(JobService.runScriptByPath).mockRejectedValueOnce(new Error('not permitted')) + stubRun( + { schema: { properties: { name: { type: 'string' } } } }, + { text: 'not permitted', is_error: true, effects: [] } + ) const markRunFormEnded = vi.fn() await expect( @@ -5588,11 +1961,11 @@ describe('global AI tools', () => { // to look at and the card keeps the run. The card reads this off the form rather than // fetching the flow again. it('run_flow marks a chat-enabled flow as conversational on its form', async () => { - vi.mocked(FlowService.getFlowByPath).mockResolvedValue({ - path: 'f/flows/chat', + stubRun({ schema: { properties: { user_message: { type: 'string' } } }, - value: { modules: [], chat_input_enabled: true } - } as any) + runnable_kind: 'flow', + conversational: true + }) const statuses: any[] = [] await withCompletedTestJob(() => @@ -5610,10 +1983,9 @@ describe('global AI tools', () => { // never sees a submission: the card and the model would otherwise be left describing the // proposal as what ran, which is wrong as soon as the reader edits a field. it('run_script reports an adopted job with the arguments that page sent', async () => { - vi.mocked(ScriptService.getScriptByPath).mockResolvedValue({ - path: 'f/scripts/adopted', + stubRun({ schema: { properties: { name: { type: 'string' } } } - } as any) + }) const statuses: any[] = [] const result = await withCompletedTestJob(() => @@ -5634,7 +2006,7 @@ describe('global AI tools', () => { ) // Started by the page, so this tool must not start one of its own. - expect(JobService.runScriptByPath).not.toHaveBeenCalled() + expect(startedRun()).toBeUndefined() expect(statuses.filter((s) => s.parameters).at(-1)?.parameters).toEqual({ name: 'Grace' }) expect(result).toContain('Grace') expect(result).not.toContain('Ada') @@ -5645,10 +2017,9 @@ describe('global AI tools', () => { // schema never had. Redacting with ours would put the bytes in the persisted card and in // the model's context. it('run_script redacts an adopted job against the schema that page ran', async () => { - vi.mocked(ScriptService.getScriptByPath).mockResolvedValue({ - path: 'f/scripts/redeployed', + stubRun({ schema: { properties: { name: { type: 'string' } } } - } as any) + }) const statuses: any[] = [] const result = await withCompletedTestJob(() => @@ -5684,13 +2055,12 @@ describe('global AI tools', () => { // host without one has nothing left to ask. A secret still becomes a reference first, which // is the only thing the missing form would have done. it('run_script runs on a formless host under yolo', async () => { - vi.mocked(ScriptService.getScriptByPath).mockResolvedValue({ - path: 'f/scripts/noform-secret', + stubRun({ schema: { properties: { token: { type: 'string', password: true } }, required: ['token'] } - } as any) + }) vi.mocked(processSecretArgs).mockImplementationOnce(async () => ({ token: '$var:u/ada/secret_arg/minted' })) @@ -5707,21 +2077,18 @@ describe('global AI tools', () => { ) ) - expect(JobService.runScriptByPath).toHaveBeenCalledWith( - expect.objectContaining({ requestBody: { token: '$var:u/ada/secret_arg/minted' } }) - ) + expect(startedRun()?.args).toEqual({ token: '$var:u/ada/secret_arg/minted' }) }) // The transcript is re-cloned into IndexedDB on every save, and a form takes as much text // as the user pastes. What the card stores is bounded; what the job runs is not. it('run_script stores a marker for oversized arguments but runs them in full', async () => { const huge = 'x'.repeat(120_000) - vi.mocked(ScriptService.getScriptByPath).mockResolvedValue({ - path: 'f/scripts/big', - content: 'export async function main(blob: string) {}', - language: 'bun', + stubRun({ + code: 'export async function main(blob: string) {}', + lang: 'bun', schema: { properties: { blob: { type: 'string' } } } - } as any) + }) const statuses: any[] = [] await withCompletedTestJob(() => @@ -5738,9 +2105,7 @@ describe('global AI tools', () => { ) ) - expect(JobService.runScriptByPath).toHaveBeenCalledWith( - expect.objectContaining({ requestBody: { blob: huge } }) - ) + expect(startedRun()?.args).toEqual({ blob: huge }) const persisted = statuses.filter((s) => s.parameters !== undefined).at(-1)?.parameters expect(persisted).toEqual({ reason: 'WINDMILL_TOO_BIG' }) expect(JSON.stringify(statuses)).not.toContain(huge) @@ -5749,10 +2114,9 @@ describe('global AI tools', () => { // A dropped pass-through fails silently: the argument is accepted and ignored, and the // run just waits out the default budget. it('run_script detaches on background and on wait_seconds', async () => { - vi.mocked(ScriptService.getScriptByPath).mockResolvedValue({ - path: 'f/scripts/slow', + stubRun({ schema: { properties: { name: { type: 'string' } } } - } as any) + }) // wait_seconds 0 detaches the same way, so one run of each pins both lines. for (const detachArg of [{ background: true }, { wait_seconds: 0 }]) { @@ -5763,7 +2127,7 @@ describe('global AI tools', () => { { ...toolCallbacks, onJobStarted: vi.fn(), onJobDetached } ) - expect(onJobDetached).toHaveBeenCalledWith('job-script-by-path') + expect(onJobDetached).toHaveBeenCalledWith('job-started') } }) @@ -5771,12 +2135,11 @@ describe('global AI tools', () => { // that button is the whole confirmation this tool has. Skipping it because there is // nothing to fill in starts the script with no confirmation at all. it('test_run_script opens a form and starts no job when the schema declares no field', async () => { - vi.mocked(ScriptService.getScriptByPath).mockResolvedValue({ - path: 'f/scripts/noargs-test', - content: 'export async function main() {}', - language: 'bun', + stubRun({ + code: 'export async function main() {}', + lang: 'bun', schema: { properties: {} } - } as any) + }) const cancelled = await callGlobalTool( 'test_run_script', @@ -5784,7 +2147,7 @@ describe('global AI tools', () => { { ...toolCallbacks, requestRunArgs: async () => undefined } ) - expect(JobService.runScriptPreview).not.toHaveBeenCalled() + expect(startedRun()).toBeUndefined() expect(cancelled).toContain('The user cancelled the run form') // A schema declaring nothing is a form with no field to hold this, and the card says @@ -5800,19 +2163,16 @@ describe('global AI tools', () => { ) ) - expect(JobService.runScriptPreview).toHaveBeenCalledWith( - expect.objectContaining({ requestBody: expect.objectContaining({ args: {} }) }) - ) + expect(startedRun()?.args).toEqual({}) expect(answered).toContain('does not declare force_delete') }) // A host that answers the form without minting, as the eval harness does by returning the // proposal verbatim: the job's arguments are readable by everyone who can see its run. it('mints a secret the host handed back as a literal', async () => { - vi.mocked(ScriptService.getScriptByPath).mockResolvedValueOnce({ - path: 'f/scripts/host-literal', + stubRun({ schema: { properties: { token: { type: 'string', password: true } } } - } as any) + }) vi.mocked(processSecretArgs).mockImplementationOnce(async () => ({ token: '$var:u/ada/secret_arg/minted' })) @@ -5828,17 +2188,14 @@ describe('global AI tools', () => { ) ) - expect(JobService.runScriptByPath).toHaveBeenCalledWith( - expect.objectContaining({ requestBody: { token: '$var:u/ada/secret_arg/minted' } }) - ) + expect(startedRun()?.args).toEqual({ token: '$var:u/ada/secret_arg/minted' }) }) // The bypass has no form to have shown them either, so the rule holds there too. it('drops an undeclared argument when the posture answers too', async () => { - vi.mocked(ScriptService.getScriptByPath).mockResolvedValueOnce({ - path: 'f/scripts/noargs-yolo', + stubRun({ schema: { properties: { name: { type: 'string' } } } - } as any) + }) await withCompletedTestJob(() => callGlobalTool( @@ -5852,90 +2209,17 @@ describe('global AI tools', () => { ) ) - expect(JobService.runScriptByPath).toHaveBeenCalledWith( - expect.objectContaining({ requestBody: { name: 'Ada' } }) - ) + expect(startedRun()?.args).toEqual({ name: 'Ada' }) }) // The form offers the arguments the flow declares and no others, so a fixture flow that // takes one has to say so — as a real flow does, since nothing else could render a field. const FLOW_NAME_SCHEMA = { type: 'object', properties: { name: { type: 'string' } } } - it('test_run_flow previews draft flow content by path', async () => { - const modules = [{ id: 'start', value: { type: 'identity' } }] - await callGlobalTool('write_flow', { - path: 'f/flows/draft-test', - summary: 'Draft test flow', - modules: JSON.stringify(modules), - schema: JSON.stringify(FLOW_NAME_SCHEMA) - }) - - await withCompletedTestJob(() => - callGlobalTool('test_run_flow', { - path: 'f/flows/draft-test', - args: { name: 'Ada' } - }) - ) - - expect(FlowService.getFlowByPath).not.toHaveBeenCalled() - expect(JobService.runFlowPreview).toHaveBeenCalledWith({ - workspace: WORKSPACE, - requestBody: { - path: 'f/flows/draft-test', - value: { modules }, - args: { name: 'Ada' } - } - }) - }) - - it('test_run_flow previews deployed flow content when no draft exists', async () => { - const modules = [{ id: 'deployed_start', value: { type: 'identity' } }] - vi.mocked(FlowService.getFlowByPath).mockResolvedValueOnce({ - path: 'f/flows/deployed-test', - summary: 'Deployed test flow', - value: { modules }, - schema: FLOW_NAME_SCHEMA - } as any) - - await withCompletedTestJob(() => - callGlobalTool('test_run_flow', { - path: 'f/flows/deployed-test', - args: { name: 'Grace' } - }) - ) - - expect(FlowService.getFlowByPath).toHaveBeenCalledWith({ - workspace: WORKSPACE, - path: 'f/flows/deployed-test' - }) - expect(JobService.runFlowPreview).toHaveBeenCalledWith({ - workspace: WORKSPACE, - requestBody: { - path: 'f/flows/deployed-test', - value: { modules }, - args: { name: 'Grace' } - } - }) - }) - // The editor is driven by the key its draft is stored under, which reads and edits of the // path resolve through too: a staged rename leaves that key where it was. it('test_run_flow drives the live flow editor by its storage path', async () => { - seedBackendDraft( - 'flow', - 'u/admin/live_flow_storage', - { - path: 'u/admin/live_flow', - summary: 'Live flow', - value: { modules: [{ id: 'live_step', value: { type: 'identity' } }] }, - schema: FLOW_NAME_SCHEMA, - edited_by: '', - edited_at: '', - archived: false, - extra_perms: {} - }, - { workspace: WORKSPACE } - ) + stubRun({ schema: FLOW_NAME_SCHEMA, runnable_kind: 'flow' }) UserDraft.setLiveEditorDraft({ workspace: WORKSPACE, itemKind: 'flow', @@ -5967,110 +2251,12 @@ describe('global AI tools', () => { { name: 'Grace' }, undefined ) - expect(FlowService.getFlowByPath).not.toHaveBeenCalled() - expect(JobService.runFlowPreview).not.toHaveBeenCalled() + expect(startedRun()).toBeUndefined() expect(result).toContain('Result (SUCCESS)') }) - // A chat flow only shows its memory across turns, so the model has to be able to name - // the conversation it is continuing rather than getting a fresh one every call. - it('test_run_flow passes the memory id it was given to the live editor hook', async () => { - seedBackendDraft( - 'flow', - '', - { - path: 'u/admin/live_chat_flow', - summary: 'Live chat flow', - value: { modules: [{ id: 'live_step', value: { type: 'identity' } }] }, - schema: { type: 'object', properties: { user_message: { type: 'string' } } }, - edited_by: '', - edited_at: '', - archived: false, - extra_perms: {} - }, - { workspace: WORKSPACE } - ) - UserDraft.setLiveEditorDraft({ - workspace: WORKSPACE, - itemKind: 'flow', - storagePath: '', - effectivePath: 'u/admin/live_chat_flow' - }) - const testActiveFlow = vi.fn(async () => 'job-live-chat') - - await withCompletedTestJob(() => - callGlobalTool( - 'test_run_flow', - { - path: 'u/admin/live_chat_flow', - args: { user_message: 'hi' }, - memory_id: '550e8400-e29b-41d4-a716-446655440000' - }, - toolCallbacks, - { testActiveFlow } - ) - ) - - expect(testActiveFlow).toHaveBeenCalledWith( - '', - { user_message: 'hi' }, - '550e8400-e29b-41d4-a716-446655440000' - ) - }) - - it('test_run_flow gives a chat-enabled flow a conversation when none is named', async () => { - const value = { - modules: [{ id: 'chat_step', value: { type: 'identity' } }], - chat_input_enabled: true - } - seedBackendDraft( - 'flow', - 'u/admin/chat_preview', - { - path: 'u/admin/chat_preview', - summary: 'Chat preview', - value, - schema: { type: 'object', properties: { user_message: { type: 'string' } } }, - edited_by: '', - edited_at: '', - archived: false, - extra_perms: {} - }, - { workspace: WORKSPACE } - ) - - await withCompletedTestJob(() => - callGlobalTool('test_run_flow', { - path: 'u/admin/chat_preview', - args: { user_message: 'hi' } - }) - ) - - expect(JobService.runFlowPreview).toHaveBeenCalledWith({ - workspace: WORKSPACE, - memoryId: expect.stringMatching( - /^[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}$/ - ), - requestBody: { path: 'u/admin/chat_preview', value, args: { user_message: 'hi' } } - }) - }) - it('test_run_flow falls back to preview when the live flow editor test hook returns undefined', async () => { - seedBackendDraft( - 'flow', - 'u/admin/live_flow_fallback', - { - path: 'u/admin/live_flow_fallback', - summary: 'Live flow fallback', - value: { modules: [{ id: 'fallback_step', value: { type: 'identity' } }] }, - schema: FLOW_NAME_SCHEMA, - edited_by: '', - edited_at: '', - archived: false, - extra_perms: {} - }, - { workspace: WORKSPACE } - ) + stubRun({ schema: FLOW_NAME_SCHEMA, runnable_kind: 'flow' }) UserDraft.setLiveEditorDraft({ workspace: WORKSPACE, itemKind: 'flow', @@ -6096,30 +2282,19 @@ describe('global AI tools', () => { { name: 'Ada' }, undefined ) - expect(FlowService.getFlowByPath).not.toHaveBeenCalled() - expect(JobService.runFlowPreview).toHaveBeenCalledWith({ - workspace: WORKSPACE, - requestBody: { - path: 'u/admin/live_flow_fallback', - value: { modules: [{ id: 'fallback_step', value: { type: 'identity' } }] }, - args: { name: 'Ada' } - } - }) + expect(startedRun()?.args).toEqual({ name: 'Ada' }) }) // A flow reaches its run form the way a script does: opened on the flow's own input schema, // with the dynamic-option picker the schema carries, and the run takes what came back. it('test_run_flow opens the form on the flow schema and runs what it submitted', async () => { - const modules = [{ id: 'start', value: { type: 'identity' } }] - await callGlobalTool('write_flow', { - path: 'f/flows/formed-flow', - summary: 'Formed flow', - modules: JSON.stringify(modules), - schema: JSON.stringify({ + stubRun({ + runnable_kind: 'flow', + schema: { ...FLOW_NAME_SCHEMA, 'x-windmill-dyn-select-code': 'export function names() { return ["Ada"] }', 'x-windmill-dyn-select-lang': 'bun' - }) + } }) let form: any @@ -6145,36 +2320,14 @@ describe('global AI tools', () => { code: 'export function names() { return ["Ada"] }', lang: 'bun' }) - expect(JobService.runFlowPreview).toHaveBeenCalledWith({ - workspace: WORKSPACE, - requestBody: { - path: 'f/flows/formed-flow', - value: { modules }, - args: { name: 'Grace' } - } - }) + expect(startedRun()?.args).toEqual({ name: 'Grace' }) }) // The form waits as long as the user does, so which editor is on screen is only known when // they press Run — checking it when the card appeared would drive an editor they have since // moved away from. it('test_run_flow re-checks the editor on screen when the form is submitted', async () => { - const modules = [{ id: 'moved_step', value: { type: 'identity' } }] - seedBackendDraft( - 'flow', - 'u/admin/moved_flow', - { - path: 'u/admin/moved_flow', - summary: 'Moved flow', - value: { modules }, - schema: FLOW_NAME_SCHEMA, - edited_by: '', - edited_at: '', - archived: false, - extra_perms: {} - }, - { workspace: WORKSPACE } - ) + stubRun({ schema: FLOW_NAME_SCHEMA, runnable_kind: 'flow' }) UserDraft.setLiveEditorDraft({ workspace: WORKSPACE, itemKind: 'flow', @@ -6205,30 +2358,13 @@ describe('global AI tools', () => { ) expect(testActiveFlow).not.toHaveBeenCalled() - expect(JobService.runFlowPreview).toHaveBeenCalledWith({ - workspace: WORKSPACE, - requestBody: { path: 'u/admin/moved_flow', value: { modules }, args: { name: 'Ada' } } - }) + expect(startedRun()?.args).toEqual({ name: 'Ada' }) }) // The flow may be open in a session tab that isn't the one on screen: driving its editor // would paint the run into a tab the user is not looking at. it('test_run_flow previews rather than driving an editor the user is not looking at', async () => { - seedBackendDraft( - 'flow', - 'u/admin/background_flow', - { - path: 'u/admin/background_flow', - summary: 'Background flow', - value: { modules: [{ id: 'background_step', value: { type: 'identity' } }] }, - schema: FLOW_NAME_SCHEMA, - edited_by: '', - edited_at: '', - archived: false, - extra_perms: {} - }, - { workspace: WORKSPACE } - ) + stubRun({ schema: FLOW_NAME_SCHEMA, runnable_kind: 'flow' }) UserDraft.setLiveEditorDraft({ workspace: WORKSPACE, itemKind: 'flow', @@ -6247,48 +2383,20 @@ describe('global AI tools', () => { ) expect(testActiveFlow).not.toHaveBeenCalled() - expect(JobService.runFlowPreview).toHaveBeenCalledWith({ - workspace: WORKSPACE, - requestBody: { - path: 'u/admin/background_flow', - value: { modules: [{ id: 'background_step', value: { type: 'identity' } }] }, - args: { name: 'Ada' } - } - }) + expect(startedRun()?.args).toEqual({ name: 'Ada' }) }) - // What separates a deployed run from a test run of the same flow: the form is built from the - // deployed schema rather than the draft's, and the editor open on that path is left alone — - // it holds the draft, so a deployed run painted into its graph would show steps that are not - // the ones running. - it('run_flow forms on the deployed flow and leaves the live editor alone', async () => { - seedBackendDraft( - 'flow', - 'u/admin/deployed_and_drafted', - { - path: 'u/admin/deployed_and_drafted', - summary: 'Draft of the deployed flow', - value: { modules: [{ id: 'draft_step', value: { type: 'identity' } }] }, - schema: { type: 'object', properties: { draft_only: { type: 'string' } } }, - edited_by: '', - edited_at: '', - archived: false, - extra_perms: {} - }, - { workspace: WORKSPACE } - ) + // What separates a deployed run from a test run of the same flow: the editor open on that + // path is left alone — it holds the draft, so a deployed run painted into its graph would + // show steps that are not the ones running. + it('run_flow leaves the live editor alone', async () => { + stubRun({ schema: FLOW_NAME_SCHEMA, runnable_kind: 'flow' }) UserDraft.setLiveEditorDraft({ workspace: WORKSPACE, itemKind: 'flow', storagePath: 'u/admin/deployed_and_drafted', effectivePath: 'u/admin/deployed_and_drafted' }) - vi.mocked(FlowService.getFlowByPath).mockResolvedValueOnce({ - path: 'u/admin/deployed_and_drafted', - summary: 'Deployed flow', - value: { modules: [{ id: 'deployed_step', value: { type: 'identity' } }] }, - schema: FLOW_NAME_SCHEMA - } as any) const testActiveFlow = vi.fn(async () => 'job-live-flow') let form: any @@ -6310,177 +2418,30 @@ describe('global AI tools', () => { expect(form.runnableKind).toBe('flow') expect(form.schema?.properties).toEqual(FLOW_NAME_SCHEMA.properties) expect(testActiveFlow).not.toHaveBeenCalled() - expect(JobService.runFlowPreview).not.toHaveBeenCalled() - expect(JobService.runFlowByPath).toHaveBeenCalledWith({ - workspace: WORKSPACE, - path: 'u/admin/deployed_and_drafted', - requestBody: { name: 'Grace' }, - skipPreprocessor: true - }) + expect(startedRun()?.args).toEqual({ name: 'Grace' }) }) - it('test_run_step previews rawscript steps from the draft flow', async () => { - const content = 'export async function main(name: string) {\n\treturn name.toUpperCase()\n}' - // The form offers the fields the step's own code declares, so the step needs a schema - // for `name` to survive it. - stubInferredProperties({ name: { type: 'string' } }) - await callGlobalTool('write_flow', { - path: 'f/flows/rawscript-step', - summary: 'Flow with rawscript', - modules: JSON.stringify([ - { - id: 'format_name', - value: { - type: 'rawscript', - language: 'bun', - content, - input_transforms: {} - } - } - ]) + it('test_run_step refuses a linked agent whose unsaved edit is in conflict', async () => { + await quickjsReady() + stubRun({ + runnable_kind: 'flow', + module: { id: 'ask', value: { type: 'aiagent', agent: '$res:f/agents/support', tools: [] } } }) + const conflictSpy = vi + .spyOn(UserDraftDbSyncer, 'getConflict') + .mockReturnValue({ conflict: { serverTimestamp: '2026', localLastSync: null } } as any) - await withCompletedTestJob(() => + const raw = await withCompletedTestJob(() => callGlobalTool('test_run_step', { - path: 'f/flows/rawscript-step', - stepId: 'format_name', - args: { name: 'Ada' } - }) + path: 'f/flows/agent-step', + stepId: 'ask', + args: {} + }).catch((e) => String(e)) ) - expect(JobService.runScriptPreview).toHaveBeenCalledWith({ - workspace: WORKSPACE, - requestBody: { - content, - language: 'bun', - args: { name: 'Ada' } - } - }) - }) - - it('test_run_step prefers local script drafts for script steps', async () => { - const content = 'export async function main(name: string) {\n\treturn `draft ${name}`\n}' - await callGlobalTool('write_script', { - path: 'f/scripts/step-script', - summary: 'Step script', - language: 'bun', - content - }) - await callGlobalTool('write_flow', { - path: 'f/flows/script-step', - summary: 'Flow with script step', - modules: JSON.stringify([ - { - id: 'call_script', - value: { - type: 'script', - path: 'f/scripts/step-script', - input_transforms: {} - } - } - ]) - }) - - // A script draft carries no schema, so the form infers from the draft content — the - // version about to run. - stubInferredProperties({ name: { type: 'string' } }) - await withCompletedTestJob(() => - callGlobalTool('test_run_step', { - path: 'f/flows/script-step', - stepId: 'call_script', - args: { name: 'Ada' } - }) - ) - - expect(ScriptService.getScriptByPath).not.toHaveBeenCalled() - expect(JobService.runScriptPreview).toHaveBeenCalledWith({ - workspace: WORKSPACE, - requestBody: { - path: 'f/scripts/step-script', - content, - language: 'bun', - args: { name: 'Ada' } - } - }) - }) - - it('test_run_step previews draft subflows for flow steps', async () => { - const nestedModules = [{ id: 'nested_start', value: { type: 'identity' } }] - await callGlobalTool('write_flow', { - path: 'f/flows/nested-draft', - summary: 'Nested draft flow', - modules: JSON.stringify(nestedModules), - // A subflow step's form is the subflow's own inputs, so `name` needs declaring here - // for it to survive the form. - schema: JSON.stringify(FLOW_NAME_SCHEMA) - }) - await callGlobalTool('write_flow', { - path: 'f/flows/parent-flow', - summary: 'Parent flow', - modules: JSON.stringify([ - { - id: 'call_flow', - value: { - type: 'flow', - path: 'f/flows/nested-draft', - input_transforms: {} - } - } - ]) - }) - - await withCompletedTestJob(() => - callGlobalTool('test_run_step', { - path: 'f/flows/parent-flow', - stepId: 'call_flow', - args: { name: 'Ada' } - }) - ) - - expect(JobService.runFlowByPath).not.toHaveBeenCalled() - expect(JobService.runFlowPreview).toHaveBeenCalledWith({ - workspace: WORKSPACE, - requestBody: { - path: 'f/flows/nested-draft', - value: { modules: nestedModules }, - args: { name: 'Ada' } - } - }) - }) - - it('test_run_step runs a deployed subflow step past its preprocessor', async () => { - vi.mocked(FlowService.getFlowByPath).mockResolvedValueOnce({ - path: 'f/flows/deployed-sub', - summary: 'Deployed subflow', - value: { modules: [{ id: 'sub_start', value: { type: 'identity' } }] }, - schema: FLOW_NAME_SCHEMA - } as any) - await callGlobalTool('write_flow', { - path: 'f/flows/parent-of-deployed', - summary: 'Parent flow', - modules: JSON.stringify([ - { - id: 'call_deployed', - value: { type: 'flow', path: 'f/flows/deployed-sub', input_transforms: {} } - } - ]) - }) - - await withCompletedTestJob(() => - callGlobalTool('test_run_step', { - path: 'f/flows/parent-of-deployed', - stepId: 'call_deployed', - args: { name: 'Ada' } - }) - ) - - expect(JobService.runFlowPreview).not.toHaveBeenCalled() - expect(JobService.runFlowByPath).toHaveBeenCalledWith({ - workspace: WORKSPACE, - path: 'f/flows/deployed-sub', - requestBody: { name: 'Ada' }, - skipPreprocessor: true - }) + expect(raw).toMatch(/f\/agents\/support .*edited elsewhere/) + expect(startedRun()).toBeUndefined() + conflictSpy.mockRestore() }) // An agent step has no job of its own, so it runs as a flow preview of itself alone, opened on @@ -6488,36 +2449,35 @@ describe('global AI tools', () => { // leaves alone is not offered at all, so a mounted widget cannot invent a value for it: an // `enabled_tools` the form synthesized empty would advertise none of the agent's tools. it("test_run_step opens an aiagent step on the step's own configuration", async () => { + // Loaded outside the fake timers the run waits under, which its loader never sees fire. + await quickjsReady() const providerValue = { kind: 'openai', resource: '$res:u/admin/openai', model: 'gpt-4o' } let opened: RunFormDisplay | undefined - await callGlobalTool('write_flow', { - path: 'f/flows/agent-step', - summary: 'Flow with an agent step', - modules: JSON.stringify([ - { - id: 'ask', - value: { - type: 'aiagent', - tools: [ - { - id: 'b', - summary: 'get_code', - value: { - tool_type: 'flowmodule', - type: 'rawscript', - language: 'bun', - content: 'export async function main() { return 1 }', - input_transforms: {} - } + stubRun({ + runnable_kind: 'flow', + module: { + id: 'ask', + value: { + type: 'aiagent', + tools: [ + { + id: 'b', + summary: 'get_code', + value: { + tool_type: 'flowmodule', + type: 'rawscript', + language: 'bun', + content: 'export async function main() { return 1 }', + input_transforms: {} } - ], - input_transforms: { - provider: { type: 'static', value: providerValue }, - user_message: { type: 'static', value: 'authored question' } } + ], + input_transforms: { + provider: { type: 'static', value: providerValue }, + user_message: { type: 'static', value: 'authored question' } } } - ]) + } }) await withCompletedTestJob(() => @@ -6552,30 +2512,12 @@ describe('global AI tools', () => { 'system_prompt' ]) - expect(JobService.runFlowPreview).toHaveBeenCalledWith({ - workspace: WORKSPACE, - requestBody: { - value: { - modules: [ - { - id: 'ask', - value: expect.objectContaining({ - type: 'aiagent', - input_transforms: { - provider: { type: 'javascript', expr: 'flow_input.provider' }, - user_message: { type: 'javascript', expr: 'flow_input.user_message' }, - // Every row of the form runs from it, the blank ones too. - system_prompt: { type: 'javascript', expr: 'flow_input.system_prompt' } - } - }) - } - ] - }, - args: { provider: providerValue, user_message: 'What is the capital of France?' }, - // Filed under the flow, as the editor's step test does: a managed memory keyed on a - // named memory id is found under it. - path: 'f/flows/agent-step' - } + // Every row of the form runs from the flow input, the blank ones too. + expect(startedRun()).toEqual({ + phase: 'start', + target: { stub: 'test_run_step' }, + args: { provider: providerValue, user_message: 'What is the capital of France?' }, + form_keys: ['provider', 'user_message', 'system_prompt'] }) }) @@ -6608,35 +2550,23 @@ describe('global AI tools', () => { ])( 'test_run_step runs an aiagent field left blank by $answer as blank', async ({ callbacks }) => { - await callGlobalTool('write_flow', { - path: 'f/flows/agent-expr', - summary: 'Agent fed by an earlier step', - modules: JSON.stringify([ - { - id: 'first', - value: { - type: 'rawscript', - language: 'bun', - content: 'export async function main() { return { question: "hi" } }', - input_transforms: {} - } - }, - { - id: 'ask', - value: { - type: 'aiagent', - tools: [], - input_transforms: { - provider: { - type: 'static', - value: { kind: 'openai', resource: '$res:u/admin/openai', model: 'gpt-4o' } - }, - user_message: { type: 'javascript', expr: 'results.first.question' }, - enabled_tools: { type: 'javascript', expr: 'results.first.tools' } - } + stubRun({ + runnable_kind: 'flow', + module: { + id: 'ask', + value: { + type: 'aiagent', + tools: [], + input_transforms: { + provider: { + type: 'static', + value: { kind: 'openai', resource: '$res:u/admin/openai', model: 'gpt-4o' } + }, + user_message: { type: 'javascript', expr: 'results.first.question' }, + enabled_tools: { type: 'javascript', expr: 'results.first.tools' } } } - ]) + } }) await withCompletedTestJob(() => @@ -6647,12 +2577,11 @@ describe('global AI tools', () => { ) ) - const body = vi.mocked(JobService.runFlowPreview).mock.calls.at(-1)?.[0].requestBody as any - expect(body.value.modules[0].value.input_transforms.user_message).toEqual({ - type: 'javascript', - expr: 'flow_input.user_message' - }) - expect(body.args.enabled_tools).toEqual([]) + // A form key runs from the flow input, so a blank one runs blank rather than falling + // back to the step's expression. + const run = startedRun() + expect(run?.form_keys).toContain('user_message') + expect(run?.args.enabled_tools).toEqual([]) } ) @@ -6661,39 +2590,36 @@ describe('global AI tools', () => { // by hand that matches none silently narrows the run, so the roster is offered as in the editor. it("test_run_step offers an aiagent step the editor's choices", async () => { let opened: RunFormDisplay | undefined - await callGlobalTool('write_flow', { - path: 'f/flows/agent-legacy-memory', - summary: 'Agent with legacy memory', - modules: JSON.stringify([ - { - id: 'ask', - value: { - type: 'aiagent', - tools: [ - { - id: 'b', - summary: 'get_code', - value: { - tool_type: 'flowmodule', - type: 'rawscript', - language: 'bun', - content: 'export async function main() { return 1 }', - input_transforms: {} - } + stubRun({ + runnable_kind: 'flow', + module: { + id: 'ask', + value: { + type: 'aiagent', + tools: [ + { + id: 'b', + summary: 'get_code', + value: { + tool_type: 'flowmodule', + type: 'rawscript', + language: 'bun', + content: 'export async function main() { return 1 }', + input_transforms: {} } - ], - input_transforms: { - provider: { - type: 'static', - value: { kind: 'openai', resource: '$res:u/admin/openai', model: 'gpt-4o' } - }, - user_message: { type: 'static', value: 'hi' }, - memory: { type: 'static', value: { kind: 'auto', context_length: 5 } }, - enabled_tools: { type: 'static', value: ['get_code'] } } + ], + input_transforms: { + provider: { + type: 'static', + value: { kind: 'openai', resource: '$res:u/admin/openai', model: 'gpt-4o' } + }, + user_message: { type: 'static', value: 'hi' }, + memory: { type: 'static', value: { kind: 'auto', context_length: 5 } }, + enabled_tools: { type: 'static', value: ['get_code'] } } } - ]) + } }) await withCompletedTestJob(() => @@ -6722,17 +2648,14 @@ describe('global AI tools', () => { // fields this job ignores and drop the ones it takes. it('test_run_step opens the form on the step, not on the flow', async () => { const content = 'export async function main(name: string) {\n\treturn name.toUpperCase()\n}' - await callGlobalTool('write_flow', { - path: 'f/flows/step-form', - summary: 'Step form flow', - // The flow takes `customer`; the step takes `name`. Nothing links the two. - schema: JSON.stringify({ type: 'object', properties: { customer: { type: 'string' } } }), - modules: JSON.stringify([ - { - id: 'format_name', - value: { type: 'rawscript', language: 'bun', content, input_transforms: {} } - } - ]) + // The flow takes `customer`; the step takes `name`. Nothing links the two. + stubRun({ + code: content, + lang: 'bun', + module: { + id: 'format_name', + value: { type: 'rawscript', language: 'bun', content, input_transforms: {} } + } }) stubInferredProperties({ name: { type: 'string' } }) @@ -6760,106 +2683,21 @@ describe('global AI tools', () => { expect(form.summary).toBe('step "format_name"') // `customer` is the flow's argument, so the step's form never offered it. expect(form.args).toEqual({ name: 'Ada' }) - expect(JobService.runScriptPreview).toHaveBeenCalledWith({ - workspace: WORKSPACE, - requestBody: { content, language: 'bun', args: { name: 'Grace' } } - }) + expect(startedRun()?.args).toEqual({ name: 'Grace' }) }) - // The step runs the draft script's content, so a form built from the deployed schema - // would offer the arguments of code that is not the code about to run. - it('test_run_step opens a script step on the draft schema, not the deployed one', async () => { - const content = 'export async function main(name: string) {\n\treturn `draft ${name}`\n}' - seedBackendDraft('script', 'f/scripts/drifted', { - path: 'f/scripts/drifted', - summary: 'Drifted', - content, - language: 'bun' - }) - await callGlobalTool('write_flow', { - path: 'f/flows/drifted-step', - summary: 'Drifted step flow', - modules: JSON.stringify([ - { - id: 'call_script', - value: { type: 'script', path: 'f/scripts/drifted', input_transforms: {} } - } - ]) - }) - - // Inferred from the draft's content. Never fetching the deployed script is the point: - // its stored schema describes code this run is not about to execute. - stubInferredProperties({ name: { type: 'string' } }) - let form: any - await withCompletedTestJob(() => - callGlobalTool( - 'test_run_step', - { path: 'f/flows/drifted-step', stepId: 'call_script', args: { name: 'Ada' } }, - { - ...toolCallbacks, - requestRunArgs: async (_toolId, f) => ((form = f), { kind: 'args', args: f.args }) - } - ) - ) - - expect(ScriptService.getScriptByPath).not.toHaveBeenCalled() - expect(form.schema.properties).toEqual({ name: { type: 'string' } }) - }) - - // No parser emits `password`, so the draft's stored schema is the only thing carrying it. - // Rebuilding the form's fields from the content would offer the secret as a plain text - // box, and the literal typed into it would reach the job's arguments unminted. - it('test_run_step keeps the password marking of a drafted script step', async () => { - seedBackendDraft('script', 'f/scripts/secretful', { - path: 'f/scripts/secretful', - summary: 'Secretful', - content: 'export async function main(token: string) {\n\treturn 1\n}', - language: 'bun', - schema: { - type: 'object', - properties: { token: { type: 'string', password: true } }, - required: ['token'] - } - }) - await callGlobalTool('write_flow', { - path: 'f/flows/secretful-step', - summary: 'Secretful step flow', - modules: JSON.stringify([ - { - id: 'call_secretful', - value: { type: 'script', path: 'f/scripts/secretful', input_transforms: {} } - } - ]) - }) - - let form: any - await withCompletedTestJob(() => - callGlobalTool( - 'test_run_step', - { path: 'f/flows/secretful-step', stepId: 'call_secretful', args: {} }, - { - ...toolCallbacks, - requestRunArgs: async (_toolId, f) => ((form = f), { kind: 'args', args: f.args }) - } - ) - ) - - expect(form.schema.properties).toMatchObject({ token: { password: true } }) - }) - - // The entrypoint override is declared by no schema, so it has to be added after the form - // rather than proposed into it — anything that conforms arguments to a schema drops it, - // and the preprocessor then silently runs its `main`. - it('test_run_step keeps the preprocessor entrypoint out of the form and on the job', async () => { + // The entrypoint override is declared by no schema, so it is the server's to add after the + // form rather than proposed into it — anything that conforms arguments to a schema drops it. + // The form's fields are the preprocessor's own, not `main`'s. + it('test_run_step keeps the preprocessor entrypoint out of the form', async () => { const content = 'export async function preprocessor(event: string) {\n\treturn event\n}' - await callGlobalTool('write_flow', { - path: 'f/flows/preprocessed', - summary: 'Preprocessed flow', - modules: JSON.stringify([{ id: 'start', value: { type: 'identity' } }]), - preprocessor_module: JSON.stringify({ + stubRun({ + code: content, + lang: 'bun', + module: { id: 'preprocessor', value: { type: 'rawscript', language: 'bun', content, input_transforms: {} } - }) + } }) vi.mocked(inferArgs).mockClear() @@ -6880,23 +2718,15 @@ describe('global AI tools', () => { expect(vi.mocked(inferArgs).mock.calls[0][3]).toBe('preprocessor') expect(form.schema.properties).toEqual({ event: { type: 'string' } }) expect(form.args).toEqual({ event: 'signup' }) - expect(JobService.runScriptPreview).toHaveBeenCalledWith({ - workspace: WORKSPACE, - requestBody: { - content, - language: 'bun', - args: { _ENTRYPOINT_OVERRIDE: 'preprocessor', event: 'signup' } - } - }) + expect(startedRun()?.args).toEqual({ event: 'signup' }) }) // The form IS the consent, so a dismissed one must leave the script unrun. it('run_script starts no job when the user cancels the form', async () => { - vi.mocked(ScriptService.getScriptByPath).mockResolvedValueOnce({ - path: 'f/scripts/deployed', + stubRun({ summary: 'Deployed', schema: { properties: { name: { type: 'string' } } } - } as any) + }) const result = await callGlobalTool( 'run_script', @@ -6904,7 +2734,7 @@ describe('global AI tools', () => { { ...toolCallbacks, requestRunArgs: async () => undefined } ) - expect(JobService.runScriptByPath).not.toHaveBeenCalled() + expect(startedRun()).toBeUndefined() expect(result).toContain('The user cancelled the run form') expect(result).toContain('Do not call run_script again') }) @@ -6913,8 +2743,7 @@ describe('global AI tools', () => { // run_script still runs them. Delete a call from runThroughForm and every one of those // unit tests still passes, so one call has to cross all of them here. it('run_script puts the proposed arguments through the whole pipeline', async () => { - vi.mocked(ScriptService.getScriptByPath).mockResolvedValueOnce({ - path: 'f/scripts/everything', + stubRun({ schema: { properties: { count: { type: 'number' }, @@ -6925,7 +2754,7 @@ describe('global AI tools', () => { doc: { type: 'string', contentEncoding: 'base64' } } } - } as any) + }) const bytes = 'QUJD'.repeat(1024) const statuses: any[] = [] @@ -6980,17 +2809,12 @@ describe('global AI tools', () => { // The bytes belong in the job request and nowhere else: the card is persisted, and a // file small enough to survive truncation would otherwise reach the model whole. - expect(JobService.runScriptByPath).toHaveBeenCalledWith({ - workspace: WORKSPACE, - path: 'f/scripts/everything', - requestBody: { - count: 7, - size: '$var:u/admin/batch_size', - locked: 'fixed', - doc: bytes, - token: '$var:u/ada/prod_api_key' - }, - skipPreprocessor: true + expect(startedRun()?.args).toEqual({ + count: 7, + size: '$var:u/admin/batch_size', + locked: 'fixed', + doc: bytes, + token: '$var:u/ada/prod_api_key' }) expect(result).toContain('does not declare force_delete') expect(result).toContain('') @@ -7008,10 +2832,9 @@ describe('global AI tools', () => { // The form is its own confirmation, so it never reaches processToolCall's second gate. // Plan mode can be switched on while it sits open, and the job must not start. it('run_script starts no job when plan mode is entered while the form is open', async () => { - vi.mocked(ScriptService.getScriptByPath).mockResolvedValueOnce({ - path: 'f/scripts/noargs', + stubRun({ schema: { properties: {} } - } as any) + }) let planning = false const result = await callGlobalTool( @@ -7027,7 +2850,7 @@ describe('global AI tools', () => { } ) - expect(JobService.runScriptByPath).not.toHaveBeenCalled() + expect(startedRun()).toBeUndefined() expect(result).toContain('plan mode is active') }) @@ -7035,13 +2858,12 @@ describe('global AI tools', () => { // the user answers is too late to unmake — so plan mode arriving during the fetch counts. it('run_script opens no form when plan mode is entered during the schema fetch', async () => { let planning = false - vi.mocked(ScriptService.getScriptByPath).mockImplementationOnce(async () => { + stubRun({ schema: { properties: { token: { type: 'string', password: true } } } }) + const prepare = vi.mocked(AiService.callAiSessionTool).getMockImplementation()! + vi.mocked(AiService.callAiSessionTool).mockImplementationOnce(((request: any) => { planning = true - return { - path: 'f/scripts/pw', - schema: { properties: { token: { type: 'string', password: true } } } - } as any - }) + return prepare(request) + }) as any) let formOpened = false const result = await callGlobalTool( @@ -7058,15 +2880,14 @@ describe('global AI tools', () => { ) expect(formOpened).toBe(false) - expect(JobService.runScriptByPath).not.toHaveBeenCalled() + expect(startedRun()).toBeUndefined() expect(result).toContain('plan mode is active') }) it('run_script runs the arguments the user submitted, not the ones proposed', async () => { - vi.mocked(ScriptService.getScriptByPath).mockResolvedValueOnce({ - path: 'f/scripts/greet', + stubRun({ schema: { properties: { name: { type: 'string' } } } - } as any) + }) const result = await withCompletedTestJob(() => callGlobalTool( @@ -7079,12 +2900,7 @@ describe('global AI tools', () => { ) ) - expect(JobService.runScriptByPath).toHaveBeenCalledWith({ - workspace: WORKSPACE, - path: 'f/scripts/greet', - requestBody: { name: 'Grace' }, - skipPreprocessor: true - }) + expect(startedRun()?.args).toEqual({ name: 'Grace' }) // The model must not assume its proposal is what ran. expect(result).toContain('Ran with arguments: {"name":"Grace"}') }) @@ -7093,10 +2909,9 @@ describe('global AI tools', () => { // own differ from the proposed ones has its own clause — so an untouched form has // nothing to name, and naming it anyway pays for the copy on every later iteration. it('run_script names the arguments only when the user changed them', async () => { - vi.mocked(ScriptService.getScriptByPath).mockResolvedValueOnce({ - path: 'f/scripts/greet', + stubRun({ schema: { properties: { name: { type: 'string' } } } - } as any) + }) const result = await withCompletedTestJob(() => callGlobalTool( @@ -7113,42 +2928,6 @@ describe('global AI tools', () => { expect(result).toContain('unedited') }) - it('test_run_step lists nested step ids when a step is not found', async () => { - await callGlobalTool('write_flow', { - path: 'f/flows/nested-step-error', - summary: 'Flow with nested step', - modules: JSON.stringify([ - { - id: 'loop_step', - value: { - type: 'forloopflow', - iterator: { type: 'static', value: [1] }, - skip_failures: false, - modules: [ - { - id: 'nested_script_step', - value: { - type: 'rawscript', - language: 'bun', - content: 'export async function main() { return 1 }', - input_transforms: {} - } - } - ] - } - } - ]) - }) - - await expect( - callGlobalTool('test_run_step', { - path: 'f/flows/nested-step-error', - stepId: 'missing_nested_step', - args: {} - }) - ).rejects.toThrow(/Available steps: loop_step, nested_script_step/) - }) - it('asks the user a question and returns the selected answer', async () => { const callbacks: ToolCallbacks = { setToolStatus: vi.fn(), @@ -7305,6 +3084,42 @@ describe('folder tools', () => { userStore.set(undefined) }) + it('refuses a flow step the openflow schema rejects, which the server lets through', async () => { + stubSessionTools(() => ({ + text: JSON.stringify({ success: true, message: 'Created flow "f/x/flow" as a draft' }), + is_error: false, + effects: [ + { + type: 'draft_write', + kind: 'flow', + storage_path: 'f/x/flow', + value: { + path: 'f/x/flow', + value: { + modules: [ + { + id: 'a', + value: { + type: 'rawscript', + language: 'bun', + content: 'export function main(x) { return x }', + input_transforms: { x: { type: 'javascript', value: 'results.b' } } + } + } + ] + } + }, + force: false + } + ] + })) + + await expect( + callGlobalTool('write_flow', { path: 'f/x/flow', summary: 's', modules: '[]' }) + ).rejects.toThrow(/Module "a" -> value\.input_transforms\.x\.expr/) + expect(getBackendDraft('flow', 'f/x/flow', { workspace: WORKSPACE })).toBeUndefined() + }) + it('create_folder requires confirmation', () => { const tool = getGlobalTool('create_folder') expect(tool.requiresConfirmation).toBe(true) @@ -7320,33 +3135,19 @@ describe('folder tools', () => { is_admin: false, folders: ['existing'] } as any) - const raw = await callGlobalTool('create_folder', { name: 'analytics', summary: 'team data' }) + stubSessionTools(() => JSON.stringify({ success: true, message: 'Created folder f/analytics' })) + await callGlobalTool('create_folder', { name: 'analytics', summary: 'team data' }) - expect(vi.mocked(FolderService.createFolder)).toHaveBeenCalledWith({ - workspace: WORKSPACE, - requestBody: { name: 'analytics', summary: 'team data' } - }) - const parsed = JSON.parse(raw) - expect(parsed.success).toBe(true) - expect(parsed.message).toContain('f/analytics') expect((get(userStore) as any)?.folders).toContain('analytics') }) it('create_folder rejects an invalid name without calling the API', async () => { const raw = await callGlobalTool('create_folder', { name: 'bad name!' }) - expect(vi.mocked(FolderService.createFolder)).not.toHaveBeenCalled() + expect(AiService.callAiSessionTool).not.toHaveBeenCalled() const parsed = JSON.parse(raw) expect(parsed.success).toBe(false) expect(parsed.error).toContain('alphanumeric') }) - - it('create_folder surfaces a backend error (e.g. name conflict)', async () => { - vi.mocked(FolderService.createFolder).mockRejectedValueOnce(new Error('Folder already exists')) - const raw = await callGlobalTool('create_folder', { name: 'taken' }) - const parsed = JSON.parse(raw) - expect(parsed.success).toBe(false) - expect(parsed.error).toContain('Folder already exists') - }) }) describe('session pipeline surface (alpha)', () => { @@ -7919,21 +3720,6 @@ describe('session-only preview tools gating', () => { }) }) -describe('read_skill', () => { - // Every path is enabled by default now, so the listing is what keeps the tool to - // skills: without it the model could name any resource holding a string `content` - // and have it read back. - it('refuses a path that is not a skill in the workspace, without reading it', async () => { - localStorage.clear() - userStore.set({ username: 'bob', email: 'bob@windmill.dev', workspace_id: WORKSPACE } as any) - - const res = await callGlobalTool('read_skill', { path: 'u/someone/private-notes' }) - - expect(res).toContain('not one of the skills available') - expect(vi.mocked(ResourceService.getResourceValue)).not.toHaveBeenCalled() - }) -}) - describe('update_user_instructions', () => { function makeHelpers(initial = '') { let value = initial @@ -8449,6 +4235,7 @@ describe('global prompt identity', () => { }) it('credits a created folder to the workspace it was created in', async () => { + stubSessionTools(() => JSON.stringify({ success: true, message: 'Created folder f/analytics' })) await getGlobalTool('create_folder').fn({ args: { name: 'analytics' }, workspace: SESSION, @@ -8463,27 +4250,3 @@ describe('global prompt identity', () => { expect(get(userStore)?.folders).toEqual(['alice_stuff']) }) }) - -describe('create_resource_type', () => { - // The Add resource type form prefixes custom types with c_ unless an admin turns it off, so a - // hub type synced later under the same name cannot clash with one the chat created. - it.each([ - ['acme_billing', 'c_acme_billing'], - ['c_acme_billing', 'c_acme_billing'] - ])('creates %s as %s', async (given, created) => { - vi.mocked(ResourceService.createResourceType).mockClear() - resourceTypesStore.set(['postgresql']) - - const result = await callGlobalTool('create_resource_type', { - name: given, - description: 'Acme billing', - schema: { type: 'object' } - }) - - expect(vi.mocked(ResourceService.createResourceType).mock.calls[0][0].requestBody.name).toBe( - created - ) - expect(result).toContain(created) - expect(get(resourceTypesStore)).toBeUndefined() - }) -}) diff --git a/frontend/src/lib/components/copilot/chat/global/core.ts b/frontend/src/lib/components/copilot/chat/global/core.ts index b999a443d1..9943367090 100644 --- a/frontend/src/lib/components/copilot/chat/global/core.ts +++ b/frontend/src/lib/components/copilot/chat/global/core.ts @@ -1,15 +1,11 @@ import { randomUUID } from '$lib/utils/uuid' import { AppService, - AssetService, AzureTriggerService, EmailTriggerService, FlowService, - FolderService, GcpTriggerService, - GroupService, HttpTriggerService, - JobService, KafkaTriggerService, MqttTriggerService, AmqpTriggerService, @@ -21,58 +17,33 @@ import { SqsTriggerService, VariableService, WebsocketTriggerService, - WorkerService, WorkspaceService } from '$lib/gen' -import { createTwoFilesPatch } from 'diff' import { deepEqual } from 'fast-equals' import { promptSafe, type ArtifactVersionTarget } from '$lib/components/sessions/previewRouter' import { canDraftItem, roleCanDraft } from '$lib/editRights' import { $ScriptLang } from '$lib/gen/schemas.gen' import type { - AppWithLastVersion, - CreateResource, CreateVariable, - Flow, FlowModule, FlowValue, - ListableApp, - ListableResource, - ListableVariable, - NewSchedule, NewScript, - Resource, - Schedule, - Script, ScriptLang } from '$lib/gen/types.gen' import { loadSchemaFromModule, memoryPropertyFor } from '$lib/components/flows/flowInfers' import { linkedAgentStepSchema } from '$lib/components/flows/agentResourceUtils' -import { AGENT_FIELDS, initialVisibleAgentFields } from '$lib/components/flows/agentFormFields' import { - fetchAgentWithDraft, - inlineAgentDraft, - withAgentDrafts + linkedAgentPaths, + loadLinkedAgentDrafts, + settleAgentDraft } from '$lib/components/flows/linkedAgentDrafts' -import { - AGENT_CHAT_SCHEMA, - AGENT_STEP_ID, - agentChatFlow, - agentChatPath -} from '$lib/components/flows/conversations/agentEditorChat' -import type { AIAgentConfig } from '$lib/components/flows/agentResourceUtils' +import { AGENT_FIELDS, initialVisibleAgentFields } from '$lib/components/flows/agentFormFields' import { resolveLinkedAgentTools } from '$lib/components/flows/flowState' import { enabledToolNames, type AgentTool } from '$lib/components/flows/agentToolUtils' import { evalValue } from '$lib/components/flows/utils.svelte' -import { dropStaleStepLocks } from '$lib/components/flows/staleStepLocks' import { quickjsReady } from '$lib/utils/quickjsEval.svelte' import { updateRawAppPolicy } from '$lib/components/raw_apps/rawAppPolicy' -import { - BUILDER_FRAMEWORK_TEMPLATES, - FRAMEWORK_TEMPLATES, - STARTER_RUNNABLES, - type FrameworkKey -} from '$lib/components/raw_apps/templates' +import { STARTER_RUNNABLES, type FrameworkKey } from '$lib/components/raw_apps/templates' import { coerceArgsToSchema, dropUndeclaredArgs, @@ -82,17 +53,10 @@ import { stripFileArgs } from '$lib/components/job_args' import { processSecretArgs } from '$lib/components/secretArgUtils' -import { parsePipelineAnnotations } from '$lib/components/assets/AssetGraph/parsePipelineAnnotations' import { PLAN_MODE_MESSAGES } from '../planModeMessages' +import { DEFAULT_DATA as DEFAULT_RAW_APP_DATA } from '$lib/components/raw_apps/dataTableRefUtils' import { USE_MODE_WITHDRAWN } from '../workMode' -import { - DEFAULT_DATA as DEFAULT_RAW_APP_DATA, - formatDataTableRef -} from '$lib/components/raw_apps/dataTableRefUtils' -import { appSourceToDraftValue } from '$lib/components/raw_apps/rawAppDraftValue' import { withoutSharedPolicySettings } from '$lib/components/apps/editor/appDraftSettings' -import { conformBuilderAppPolicy } from '$lib/components/raw_apps/builderAppPolicy' -import { capabilitiesInWorkspace } from '$lib/operatorWriteRights' import { capabilitiesIn, withoutBuilding } from '$lib/roleCapabilities' import type { RoleCapabilities } from '$lib/gen' import type { RawAppDomQuery } from '$lib/components/raw_apps/rawAppDom' @@ -104,45 +68,22 @@ import { isChromiumBrowser } from '$lib/utils' import { isCloudHosted } from '$lib/cloud' import { BROWSER } from 'esm-env' import { - applyEditableFlowJsonToFlow, - buildEditableFlowJson, - EDITABLE_FLOW_STRUCTURAL_KEYS, - type EditableFlowJson, FLOW_VALUE_SETTINGS_KEYS, - pickFlowValueSettings, - finalizeUnresolvedInlineScripts, - restoreSpecialRawscriptModule, - validateEditableFlowJson + validateFlowModules, + validateOptionalFlowModule } from '../flow/editableFlowJson' import { getAiAgentProviderCatalog, - getAiAgentProviderCatalogFor + getAiAgentProviderCatalogForRefs } from '../flow/aiAgentProviderCatalog' -import { - formatAiAgentProviderWarnings, - formatAiAgentProvidersPrompt -} from '../flow/aiAgentProviders' -import { - createInlineScriptSession, - findUnresolvedInlineScriptRefs -} from '../flow/inlineScriptsUtils' -import { - resourceTypeSyncBlock, - searchNpmPackagesTool, - searchResourceTypes, - type ResourceTypeSync -} from '../script/core' +import { formatAiAgentProvidersPrompt } from '../flow/aiAgentProviders' +import {} from '../flow/inlineScriptsUtils' +import { resourceTypeSyncBlock, searchNpmPackagesTool, type ResourceTypeSync } from '../script/core' import type { McpServer } from './mcpTools' import { logFeatureUsage } from '$lib/utils/featureUsage' import { isSkillEnabled } from '../skills/enabledSkills' -import { - listSkillResources, - readSkillBody, - skillNameFromPath, - truncateChars, - truncateForPrompt -} from '../skills/skillResources' -import { MAX_SKILL_DESCRIPTION_LENGTH, MAX_SKILL_INSTRUCTIONS_LENGTH } from '../skills/skillMd' +import { listSkillResources, skillNameFromPath, truncateChars } from '../skills/skillResources' +import { MAX_SKILL_DESCRIPTION_LENGTH } from '../skills/skillMd' import { getDatatablePrompt, getWorkspaceSettingsPrompt, @@ -167,25 +108,20 @@ import { import { createToolDef, droppedOptionKeys, - createSearchHubScriptsTool, + decisionStepRunSchema, getHubIntegrationTool, executeTestRun, findAndReplace, - isHubPath, - resolveFlowStepRun, SPECIAL_MODULE_IDS, type CreatedResourceTriggerKind, type PreviewCardKind, type RunFormAnswer, type RunFormDisplay, type ToolCallbacks, - type ToolCodeDiff, type ToolDisplayAction } from '../shared' import { scriptLangToEditorLang } from '$lib/scripts' import { appFileEditorLang } from '../appFileEditorLang' -import { searchDocsTool, readDocsPageTool } from '../docs/core' -import { createDbSchemaTool } from '../script/core' import { isMentionContext, isUserContextPath, type ContextElement } from '../context' import { getDatatableTools } from '../datatableTools' import { getDatatableMigrationTools } from '../datatableMigrationTools' @@ -194,7 +130,6 @@ import { fileTools } from '../files/fileTools' import type { AttachedFilesStore } from '../files/attachedFiles.svelte' import { artifactTools } from '../artifacts/artifactTools' import type { SessionArtifactsStore } from '../artifacts/artifactsState.svelte' -import type { PathRunType, Runnable } from '$lib/components/apps/inputType' import { UserDraft } from '$lib/userDraft.svelte' import { agentEditHref } from '../workspaceItems.svelte' import { markAgentWritten } from '$lib/components/flows/agentEditorStore.svelte' @@ -208,15 +143,9 @@ import { variableRequestSchema } from '../workspaceToolsZod.gen' import { - getWorkspaceItemKey, TRIGGER_KINDS, type AppDraftValue, - type FlowDraftValue, - type ResourceDraftState, type TriggerKind, - type TriggerRequestBody, - type VariableDraftState, - type WorkspaceItem, type WorkspaceItemType } from './workspaceItems' import { @@ -230,7 +159,6 @@ import { workspaceStore } from '$lib/stores' import { getWorkspaceRole, type RoleLookup } from '$lib/user' -import { refreshSuperadmin } from '$lib/refreshUser' import { resourceTypesStore } from '$lib/components/resourceTypesStore' import { get } from 'svelte/store' import { @@ -240,10 +168,9 @@ import { } from '$lib/utils_draft_deploy' import { changedLineIndices, draftDeployedPatch, windowPatch } from './draftDiff' import { workspaceSettingsTools } from './workspaceSettingsTools' -import { getRun, summarizeRun } from './flowRunTree' import { UserDraftDbSyncer } from '$lib/userDraftDbSyncer.svelte' import { invalidateWorkspaceComparison } from '$lib/workspaceComparison' -import type { UserDraftItemKind } from '$lib/gen' +import type { CreateResource, UserDraftItemKind } from '$lib/gen' import { bundleRawAppDraft } from './rawAppBundlerBridge' import { canOpenEditor, @@ -294,15 +221,24 @@ import { getGlobalDraft, getGlobalDraftStoragePath, itemKindFor, - listGlobalDrafts, - persistGlobalDraft, - readGlobalDraftValue, readLocalDraftCellByKind, resolveGlobalDraftStoragePathByKind, - saveGlobalAppDraft, type DraftPersistResult } from './userDraftAdapter' -import { findModuleInFlow } from '$lib/components/flows/flowTree' +import { + applySessionToolEffects, + applyStatusEffects, + callSessionTool, + runOnServer, + runServerTool, + type SessionToolEffect +} from './backendSessionTools' +import { + dbSchemaServerTool, + readDocsPageServerTool, + searchDocsServerTool, + searchHubScriptsServerTool +} from './serverTools' import { DRAFT_CONFLICT_RESULT, DRAFT_SAVE_FAILED_RESULT } from '../draftWriteResults' import { computeDiffParts, @@ -615,102 +551,6 @@ const writeFlowSchema = z.object({ override: draftOverrideField }) -// modules/preprocessor_module/failure_module can carry rawscript `content`, whose -// quotes and newlines are the usual reason the JSON string fails to parse. -const FLOW_CODE_BEARING_FIELDS = new Set(['modules', 'preprocessor_module', 'failure_module']) - -function parseOptionalJsonArg(value: unknown, field: string): unknown { - if (value === undefined || value === null) { - return value - } - - try { - return typeof value === 'string' ? JSON.parse(value) : value - } catch (error) { - const message = error instanceof Error ? error.message : String(error) - const hint = FLOW_CODE_BEARING_FIELDS.has(field) - ? ' A rawscript "content" string with multi-line code or quotes is the usual cause. Instead of inlining large code, create the rawscript module with empty content ("") and fill its body with set_flow_module_code afterwards.' - : '' - throw new Error(`Invalid JSON for ${field}: ${message}${hint}`) - } -} - -/** - * Rawscript bodies are fragile to embed inside the `modules` JSON string: the - * code's quotes and newlines have to survive three levels of escaping (tool-call - * arguments -> modules string -> content string) and the model routinely mangles - * them. So a module may be saved with empty (or `inline_script.` placeholder) - * content; return the ids that still need a body filled out-of-band with - * `set_flow_module_code`. - */ -function emptyInlineScriptModuleIds(editable: EditableFlowJson): string[] { - const value: FlowValue = { - modules: editable.modules, - preprocessor_module: editable.preprocessor_module ?? undefined, - failure_module: editable.failure_module ?? undefined - } - const session = createInlineScriptSession() - buildEditableFlowJson({ value, schema: editable.schema }, session) - return Object.entries(session.getAll()) - .filter(([, content]) => content.trim() === '' || /^inline_script\./.test(content)) - .map(([id]) => id) -} - -/** - * Fold the empty-body warning into a flow write tool's JSON result — but only - * when the save actually succeeded. `writeFlowDraft` reports - * conflicts/persistence errors as `{ success: false }` rather than throwing; - * telling the model to fill code on a flow that was never saved would send it - * after a stale or nonexistent draft. - */ -function appendEmptyInlineScriptWarning(result: string, editable: EditableFlowJson): string { - const emptyIds = emptyInlineScriptModuleIds(editable) - if (emptyIds.length === 0) { - return result - } - let parsed: { success?: unknown; message?: unknown } - try { - parsed = JSON.parse(result) - } catch { - return result - } - if (parsed.success !== true || typeof parsed.message !== 'string') { - return result - } - const list = emptyIds.map((id) => `"${id}"`).join(', ') - parsed.message += `\n\nWarning: inline scripts ${list} have no code yet. Fill each one with set_flow_module_code(path, module_id, code) — do not re-send the whole flow.` - return JSON.stringify(parsed, null, 2) -} - -function editableFlowToDraftValue(editable: EditableFlowJson): FlowDraftValue { - const value: FlowValue = { - ...pickFlowValueSettings(editable), - modules: editable.modules, - preprocessor_module: editable.preprocessor_module ?? undefined, - failure_module: editable.failure_module ?? undefined, - groups: editable.groups ?? undefined, - notes: editable.notes ?? undefined - } - return { - value, - schema: editable.schema, - groups: editable.groups - } -} - -function flowDraftAsEditableInput(flowDraft: FlowDraftValue): { - value: FlowValue - schema?: Record | null | undefined -} { - return { - value: - flowDraft.groups === undefined - ? flowDraft.value - : { ...flowDraft.value, groups: flowDraft.groups ?? undefined }, - schema: flowDraft.schema - } -} - const writeScheduleSchema = scheduleRequestSchema.extend({ override: draftOverrideField }) const writeScheduleToolSchema = buildScheduleToolSchema().extend({ override: draftOverrideField }) @@ -775,8 +615,6 @@ const writeVariableSchema = variableRequestSchema.extend({ override: draftOverrideField }) -type WriteVariableArgs = z.infer - const searchResourceTypesSchema = z.object({ query: z .string() @@ -847,7 +685,6 @@ const NO_WORKERS_VISIBLE_MESSAGE = 'Tell the user you cannot see any workers and that worker visibility may be restricted for your account, and suggest they check the Workers page themselves if they can open it, or ask a workspace admin otherwise. Never state that no workers are online or that the instance has none.' const NO_WORKERS_CONNECTED_MESSAGE = 'No workers are connected to this instance (none pinged in the last 5 minutes). Queued runs will stay queued until a worker starts.' -const WORKER_PAGE_SIZE = 100 const deleteWorkspaceItemSchema = z.object({ type: itemTypeSchema, @@ -1964,350 +1801,8 @@ ${mcpServers.map((s) => `- ${s.path}`).join('\n')} }` } -const DEFAULT_LIST_TYPES = ['script', 'flow'] as const satisfies readonly WorkspaceItemType[] - -function getRequestedTypes(types: WorkspaceItemType[] | undefined): WorkspaceItemType[] { - return types && types.length > 0 ? types : [...DEFAULT_LIST_TYPES] -} - -function itemMatches( - item: Pick, - query: string | undefined -): boolean { - const normalized = query?.trim().toLowerCase() - if (!normalized) return true - return ( - item.path.toLowerCase().includes(normalized) || - (item.summary?.toLowerCase().includes(normalized) ?? false) - ) -} - -function scriptToItem(script: Script | NewScript, includeValue: boolean): WorkspaceItem { - return { - type: 'script', - path: script.path, - summary: script.summary, - language: script.language, - value: includeValue ? script.content : undefined, - schema: includeValue ? (script as Script).schema : undefined, - // Listings with includeDraftOnly synthesize rows for editor drafts that - // have no deployed counterpart — label those honestly. - isDraft: (script as Script).draft_only ?? false - } -} - -function flowToItem(flow: Flow, includeValue: boolean): WorkspaceItem { - return { - type: 'flow', - path: flow.path, - summary: flow.summary, - value: includeValue - ? { value: flow.value, schema: flow.schema, groups: flow.value.groups ?? null } - : undefined, - isDraft: (flow as Flow & { draft_only?: boolean }).draft_only ?? false - } -} - -/** - * Turn a flow workspace item into the compact response we send to the model: - * rawscript content is replaced with `inline_script.` placeholders. - * The model retrieves real content via `read_flow_module_code` and edits it via - * `set_flow_module_code`. `patch_flow_json` operates on this compact view too, - * so structural edits never have to traverse inline script bodies. - */ -function serializeWorkspaceItemForRead(item: WorkspaceItem): unknown { - if (item.type === 'variable') { - return { - type: 'variable', - path: item.path, - summary: item.summary, - isSecret: item.isSecret, - extraPerms: item.extraPerms, - isDraft: item.isDraft - } - } - - if ( - item.type === 'app' && - item.value && - typeof item.value === 'object' && - 'files' in item.value - ) { - return { - type: 'app', - path: item.path, - summary: item.summary, - value: summarizeAppValue(item.value as AppDraftValue), - rawApp: item.rawApp, - executionMode: item.executionMode, - extraPerms: item.extraPerms, - isDraft: item.isDraft - } - } - - if (item.type !== 'flow' || !item.value) return item - const flowDraft = item.value as FlowDraftValue - const session = createInlineScriptSession() - const editable = buildEditableFlowJson(flowDraftAsEditableInput(flowDraft), session) - return { - type: 'flow', - path: item.path, - summary: item.summary, - value: editable, - extraPerms: item.extraPerms, - isDraft: item.isDraft - } -} - -function scheduleToItem(schedule: Schedule, includeValue: boolean): WorkspaceItem { - return { - type: 'schedule', - path: schedule.path, - summary: schedule.summary ?? undefined, - value: includeValue ? (schedule as unknown as WorkspaceItem['value']) : undefined, - isDraft: false - } -} - -function resourceToItem(resource: ListableResource, includeValue: boolean): WorkspaceItem { - return { - type: 'resource', - path: resource.path, - summary: resource.description, - value: includeValue ? (resource as unknown as WorkspaceItem['value']) : undefined, - isDraft: false - } -} - -function variableToItem(variable: ListableVariable): WorkspaceItem { - // Never exposes a value to the chat, secret or not — the user reveals one in the - // variable editor (audited). Metadata only. - return { - type: 'variable', - path: variable.path, - summary: variable.description, - isSecret: variable.is_secret, - isDraft: false - } -} - // ============= App helpers ============= -type BackendRunnableInput = z.infer - -type PersistedRunnable = { - name: string - type: 'inline' | 'path' | 'runnableByName' | 'runnableByPath' - runType?: PathRunType - path?: string - inlineScript?: { language: string; content: string } - fields?: Record - schema?: any - [key: string]: any -} - -function convertPersistedToBackendRunnable( - persisted: PersistedRunnable | undefined, - key: string -): BackendRunnableInput | undefined { - if (!persisted) return undefined - - const out: BackendRunnableInput = { - name: persisted.name ?? key, - type: 'inline' - } - - if (persisted.type === 'inline' || persisted.type === 'runnableByName') { - out.type = 'inline' - if (persisted.inlineScript) { - let language = persisted.inlineScript.language - if (language === 'nativets' || language === 'deno') language = 'bun' - out.inlineScript = { - language: language as 'bun' | 'python3', - content: persisted.inlineScript.content ?? '' - } - } - if (persisted.viewer_data_access === true) out.viewerDataAccess = true - } else if (persisted.type === 'path' || persisted.type === 'runnableByPath') { - if (persisted.runType === 'flow') out.type = 'flow' - else if (persisted.runType === 'hubscript') out.type = 'hubscript' - else if (persisted.runType === 'agent') out.type = 'agent' - else out.type = 'script' - out.path = persisted.path - } - - if (persisted.fields) { - const staticInputs: Record = {} - for (const [k, v] of Object.entries(persisted.fields)) { - if (v && typeof v === 'object' && (v as any).type === 'static') { - staticInputs[k] = (v as any).value - } - } - if (Object.keys(staticInputs).length > 0) out.staticInputs = staticInputs - } - - return out -} - -function buildPersistedRunnable( - input: BackendRunnableInput, - existing: PersistedRunnable | undefined -): PersistedRunnable { - // An agent runnable takes every input from the app: a static one inherited from the kind it was - // converted from would be pinned by the deployed policy over every viewer's message. - const fields = - input.type === 'agent' - ? {} - : input.staticInputs - ? Object.fromEntries( - Object.entries(input.staticInputs).map(([k, v]) => [ - k, - { type: 'static', value: v, fieldType: 'object' } - ]) - ) - : (existing?.fields ?? {}) - - // Converting between kinds must drop the other kind's fields. Every consumer dispatches on - // `type` (isRunnableByName / isRunnableByPath), so a leftover does not change which code - // runs — it contradicts it: a runnable reported back as inline still names a flow in - // `path`, and that is what the model reads on its next turn. - const { - runType: _runType, - path: _path, - inlineScript: _inlineScript, - viewer_data_access: carriedAsViewer, - ...carried - }: Partial = existing ?? {} - const viewerDataAccess = input.viewerDataAccess ?? carriedAsViewer === true - - // `schema` describes the item a path runnable points at, so it only survives while that - // target is unchanged. Carried across a retarget it types `backend.(args)` from the - // previous item's inputs, because hiddenRunnableToTsType reads it for the path branch. - const sameTarget = - (existing?.type === 'path' || existing?.type === 'runnableByPath') && - existing?.runType === input.type && - existing?.path === input.path - - if (input.type === 'inline') { - if (!input.inlineScript) { - throw new Error('inlineScript is required when runnable type is "inline".') - } - return { - ...carried, - name: input.name, - type: 'inline', - inlineScript: { - content: input.inlineScript.content, - language: input.inlineScript.language - }, - ...(viewerDataAccess && { viewer_data_access: true }), - fields - } - } - - if (!input.path) { - throw new Error( - 'path is required when runnable type is "script", "flow", "hubscript" or "agent".' - ) - } - // A static input is pinned by the deployed policy: every viewer's turn would send the same one. - if (input.type === 'agent' && Object.keys(input.staticInputs ?? {}).length > 0) { - throw new Error( - 'An "agent" runnable takes no staticInputs: each turn\'s inputs come from the app.' - ) - } - return { - ...carried, - name: input.name, - type: 'path', - runType: input.type, - path: input.path, - fields, - schema: sameTarget ? (existing?.schema ?? {}) : {} - } -} - -type AppFrontendFileMetadata = { - path: string - size: number -} - -type AppBackendRunnableMetadata = { - key: string - name: string - type: BackendRunnableInput['type'] - path?: string - language?: 'bun' | 'python3' - contentSize?: number - staticInputKeys?: string[] -} - -type AppMetadata = { - frontend: AppFrontendFileMetadata[] - backend: AppBackendRunnableMetadata[] - data?: any -} - -type LoadedAppDraftValue = { - value: AppDraftValue -} - -function summarizeAppValue(value: AppDraftValue): AppMetadata { - const frontend: AppFrontendFileMetadata[] = Object.entries(value.files).map( - ([path, content]) => ({ - path, - size: typeof content === 'string' ? content.length : 0 - }) - ) - const backend: AppBackendRunnableMetadata[] = Object.entries(value.runnables).map( - ([key, runnable]) => { - const converted = convertPersistedToBackendRunnable(runnable as PersistedRunnable, key) - const staticInputKeys = converted?.staticInputs - ? Object.keys(converted.staticInputs) - : undefined - return { - key, - name: converted?.name ?? key, - type: converted?.type ?? 'inline', - ...(converted?.path && { path: converted.path }), - ...(converted?.inlineScript && { - language: converted.inlineScript.language, - contentSize: converted.inlineScript.content.length - }), - ...(staticInputKeys && staticInputKeys.length > 0 && { staticInputKeys }) - } - } - ) - return { - frontend, - backend, - ...(value.data && { data: value.data }) - } -} - -function appToItem(app: ListableApp | AppWithLastVersion, includeValue: boolean): WorkspaceItem { - return { - type: 'app', - path: app.path, - summary: app.summary, - value: includeValue ? ((app as AppWithLastVersion).value as AppDraftValue) : undefined, - // The server omits this flag rather than sending false, so its absence in the - // response is a known false — not a value this listing failed to fetch. - rawApp: app.raw_app ?? false, - isDraft: false - } -} - -const GENERATED_APP_FILE_PATHS = new Set(['/wmill.d.ts']) - -function assertNotGeneratedAppFile(filePath: string): void { - if (GENERATED_APP_FILE_PATHS.has(filePath)) { - throw new Error( - `"${filePath}" is generated automatically from backend runnables and cannot be modified directly.` - ) - } -} - type AppFileTarget = | { kind: 'frontend'; filePath: string } | { kind: 'backend'; filePath: string; key: string; extension: 'ts' | 'py' } @@ -2329,103 +1824,8 @@ function resolveAppFileTarget(rawPath: string): AppFileTarget { } } -function getInlineScriptExtension(runnable: PersistedRunnable | undefined): 'ts' | 'py' { - return runnable?.inlineScript?.language === 'python3' ? 'py' : 'ts' -} - -/** - * Resolve a backend file target to its inline script body, validating that the - * runnable exists, is inline, and matches the requested file extension. Throws - * with a clear message otherwise. - */ -function getInlineRunnableContent( - value: AppDraftValue, - target: { kind: 'backend'; filePath: string; key: string; extension: 'ts' | 'py' }, - appPath: string -): { content: string; runnable: PersistedRunnable } { - const runnable = value.runnables[target.key] as PersistedRunnable | undefined - if (!runnable) { - throw new Error(`Backend runnable "${target.key}" not found in app "${appPath}".`) - } - if (runnable.type !== 'inline' && runnable.type !== 'runnableByName') { - throw new Error( - `Runnable "${target.key}" is not inline. Use read_workspace_item on the referenced ${runnable.runType ?? 'item'} instead.` - ) - } - const expected = getInlineScriptExtension(runnable) - if (target.extension !== expected) { - throw new Error( - `Runnable "${target.key}" language is ${expected}. Use backend/${target.key}/main.${expected}.` - ) - } - return { content: runnable.inlineScript?.content ?? '', runnable } -} - -// appSourceToDraftValue drops a low-code app's `grid`, so converting one here would stage a -// code-app draft at its path: a false picture of the app that every later read answers from, -// and a deploy the server then refuses. -async function getRawAppByPath(workspace: string, path: string): Promise { - const app = await AppService.getAppByPath({ workspace, path }) - // Only an explicit false: the draft-only branch of get_app carries no version, and so - // no `raw_app`, and must not read as low-code. - if (app.raw_app === false) { - throw new Error( - `"${path}" is a low-code app. This chat only edits code-based apps — open it in the app editor instead.` - ) - } - return app -} - -async function loadAppValueForRead(path: string, workspace: string): Promise { - const draft = await getGlobalDraft(workspace, 'app', path) - if (draft && draft.value && typeof draft.value === 'object' && 'files' in draft.value) { - return draft.value as AppDraftValue - } - - const app = await getRawAppByPath(workspace, path) - return appSourceToDraftValue(app, app) -} - -async function loadAppDraftValue(path: string, workspace: string): Promise { - const draft = await getGlobalDraft(workspace, 'app', path) - if (draft && draft.value && typeof draft.value === 'object' && 'files' in draft.value) { - return { value: draft.value as AppDraftValue } - } - - const app = await getRawAppByPath(workspace, path) - return { value: appSourceToDraftValue(app, app) } -} - -async function saveAppDraft( - workspace: string, - path: string, - value: AppDraftValue -): Promise { - if (capabilitiesInWorkspace(workspace).apps === 'compose') { - const role = await roleForWorkspace(workspace) - value.policy ??= {} - conformBuilderAppPolicy(value.policy, role.kind === 'resolved' ? role.user : undefined) - } - return saveGlobalAppDraft(workspace, path, value) -} - type TriggerLike = { path: string; summary?: string | null } -function triggerToItem( - kind: TriggerKind, - trigger: TriggerLike, - includeValue: boolean -): WorkspaceItem { - return { - type: 'trigger', - triggerKind: kind, - path: trigger.path, - summary: trigger.summary ?? undefined, - value: includeValue ? (trigger as unknown as WorkspaceItem['value']) : undefined, - isDraft: false - } -} - type TriggerService = { exists(args: { workspace: string; path: string }): Promise get(args: { workspace: string; path: string }): Promise @@ -2543,312 +1943,6 @@ async function guestAccessIsLive(workspace: string): Promise { - if (!(await AppService.existsApp({ workspace, path }))) return undefined - const row = await AppService.getAppLiteByPath({ workspace, path }) - return row.policy.execution_mode ?? undefined -} - -/** Who may open an app, from the mode stored on it. An app sits in `guest` mode whether - * or not anyone is admitted by it, so reporting the mode bare would say strangers can - * open an app that admits members only. Say it is inert rather than hide it, as the - * app's deploy settings do. */ -async function describeAppExposure( - workspace: string, - mode: string | undefined -): Promise { - if (mode !== 'guest' || (await guestAccessIsLive(workspace)) !== false) { - return mode - } - return 'guest, but inert: this deployment admits no guest, so this app admits members only' -} - -async function readWorkspaceItem( - type: WorkspaceItemType, - path: string, - workspace: string, - triggerKind?: TriggerKind, - deployedOnly = false -): Promise { - switch (type) { - case 'script': { - // Hub scripts are not workspace items: search_hub_scripts hands back - // `hub///` paths, which getScriptByPath cannot resolve. - if (isHubPath(path)) { - const hub = await ScriptService.getHubScriptByPath({ path }) - return { - type: 'script', - path, - summary: hub.summary, - language: hub.language as ScriptLang, - value: hub.content, - schema: hub.schema, - isDraft: false - } - } - // Prefer the DB draft (newer than the deployed version) when one exists, - // unless the caller explicitly asked for the deployed state. - const script = await ScriptService.getScriptByPath({ - workspace, - path, - getDraft: !deployedOnly - }) - const draft = deployedOnly ? undefined : (script.draft as Script | undefined) - return { ...scriptToItem(draft ?? script, true), extraPerms: script.extra_perms } - } - case 'flow': { - // Prefer the DB draft (newer than the deployed version) when one exists, - // unless the caller explicitly asked for the deployed state. - const flow = await FlowService.getFlowByPath({ workspace, path, getDraft: !deployedOnly }) - const draft = deployedOnly ? undefined : (flow.draft as Flow | undefined) - return { ...flowToItem(draft ?? flow, true), extraPerms: flow.extra_perms } - } - case 'schedule': { - const schedule = await ScheduleService.getSchedule({ workspace, path }) - return { ...scheduleToItem(schedule, true), extraPerms: schedule.extra_perms } - } - case 'trigger': { - if (!triggerKind) { - throw new Error('trigger_kind is required when type is trigger.') - } - const trigger = await triggerServices[triggerKind].get({ workspace, path }) - return { - ...triggerToItem(triggerKind, trigger, true), - extraPerms: (trigger as { extra_perms?: Record }).extra_perms - } - } - case 'resource': { - const resource = await ResourceService.getResource({ workspace, path }) - return { ...resourceToItem(resource, true), extraPerms: resource.extra_perms } - } - case 'variable': { - // Never expose the value, even when read directly. Pass decryptSecret=false - // to avoid materializing secret values server-side. - const variable = await VariableService.getVariable({ - workspace, - path, - decryptSecret: false - }) - return { ...variableToItem(variable), extraPerms: variable.extra_perms } - } - case 'app': { - // Returns lightweight metadata only — file/runnable contents come via read_app_file. - const app = await AppService.getAppByPath({ workspace, path }) - // A grid is not files and runnables: summarizing one reports an empty app. Name the - // kind instead. - const executionMode = await describeAppExposure(workspace, app.policy?.execution_mode) - if (app.raw_app === false) { - return { - type: 'app', - path: app.path, - summary: app.summary, - rawApp: false, - executionMode, - extraPerms: app.extra_perms, - isDraft: false - } - } - const value = appSourceToDraftValue(app) - const metadata = summarizeAppValue(value) - return { - type: 'app', - path: app.path, - summary: value.summary, - value: metadata as unknown as AppDraftValue, - rawApp: app.raw_app, - executionMode, - extraPerms: app.extra_perms, - isDraft: false - } - } - } -} - -// Accepts the `f/` / `g/` form the model sees in extra_perms keys and paths. -function accessEntityName(type: AccessType, path: string): string { - const prefix = type === 'folder' ? 'f/' : 'g/' - const name = path.startsWith(prefix) ? path.slice(prefix.length) : path - return name.replace(/\/+$/, '') -} - -async function readAccessEntity(type: AccessType, path: string, workspace: string) { - const name = accessEntityName(type, path) - if (type === 'folder') { - const folder = await FolderService.getFolder({ workspace, name }) - return { - type, - name: folder.name, - summary: folder.summary, - owners: folder.owners, - extraPerms: folder.extra_perms, - defaultPermissionedAs: folder.default_permissioned_as - } - } - const group = await GroupService.getGroup({ workspace, name }) - return { - type, - name: group.name, - summary: group.summary, - members: group.members, - extraPerms: group.extra_perms - } -} - -type AccessEntityRow = { - type: AccessType - path: string - summary?: string - owners?: string[] - extraPerms?: Record -} - -// Group rows omit members: the list endpoint does not return them. -async function listAccessEntities( - types: AccessType[], - workspace: string, - perPage: number, - page?: number -): Promise { - const rows: AccessEntityRow[] = [] - if (types.includes('folder')) { - for (const f of await FolderService.listFolders({ workspace, perPage, page })) { - rows.push({ - type: 'folder', - path: f.name, - summary: f.summary, - owners: f.owners, - extraPerms: f.extra_perms - }) - } - } - if (types.includes('group')) { - for (const g of await GroupService.listGroups({ workspace, perPage, page })) { - rows.push({ type: 'group', path: g.name, summary: g.summary, extraPerms: g.extra_perms }) - } - } - return rows -} - -// Each list call is capped per type, and per trigger kind for triggers, so the page is cut -// on the same key. -function pageAcrossPrefixes( - listings: WorkspaceItem[][], - perPage: number, - page: number -): WorkspaceItem[] { - const seenByKey = new Map() - const start = (page - 1) * perPage - return listings.flat().filter((item) => { - const key = item.type === 'trigger' ? `trigger:${item.triggerKind}` : item.type - const index = seenByKey.get(key) ?? 0 - seenByKey.set(key, index + 1) - return index >= start && index < start + perPage - }) -} - -async function listWorkspaceItems( - types: WorkspaceItemType[], - workspace: string, - pathPrefix: string | undefined, - perPage: number, - page?: number -): Promise { - const items: WorkspaceItem[] = [] - - if (types.includes('script')) { - const scripts = await ScriptService.listScripts({ - workspace, - pathStart: pathPrefix, - perPage, - page, - includeDraftOnly: true, - withoutDescription: true - }) - for (const script of scripts) items.push(scriptToItem(script, false)) - } - - if (types.includes('flow')) { - const flows = await FlowService.listFlows({ - workspace, - pathStart: pathPrefix, - perPage, - page, - includeDraftOnly: true, - withoutDescription: true - }) - for (const flow of flows) items.push(flowToItem(flow, false)) - } - - if (types.includes('schedule')) { - const schedules = await ScheduleService.listSchedules({ - workspace, - pathStart: pathPrefix, - perPage, - page - }) - for (const schedule of schedules) items.push(scheduleToItem(schedule, false)) - } - - if (types.includes('trigger')) { - for (const kind of TRIGGER_KINDS) { - let triggers: Awaited> - try { - triggers = await triggerServices[kind].list({ - workspace, - pathStart: pathPrefix, - perPage, - page - }) - } catch (err) { - // A trigger kind whose backend routes aren't compiled in (e.g. email - // without smtp+private, or an EE kind on CE) 404s here; skip only that - // so one unavailable kind doesn't drop the whole listing. Any other - // failure (auth, 5xx, network) is real and must surface. - if ((err as { status?: number } | undefined)?.status === 404) continue - throw err - } - for (const trigger of triggers) items.push(triggerToItem(kind, trigger, false)) - } - } - - if (types.includes('resource')) { - const resources = await ResourceService.listResource({ - workspace, - pathStart: pathPrefix, - perPage, - page - }) - for (const resource of resources) items.push(resourceToItem(resource, false)) - } - - if (types.includes('variable')) { - const variables = await VariableService.listVariable({ - workspace, - pathStart: pathPrefix, - perPage, - page - }) - for (const variable of variables) items.push(variableToItem(variable)) - } - - if (types.includes('app')) { - const apps = await AppService.listApps({ - workspace, - pathStart: pathPrefix, - perPage, - page - }) - for (const app of apps) items.push(appToItem(app, false)) - } - - return items -} - // The get_instructions builders below add global mode's tools and draft rules on top of the shared // reference each ends with. Windmill domain guidance belongs in that reference // (system_prompts/base, shared with the other chat modes and the CLI's skills), not here. @@ -3269,42 +2363,28 @@ export const readSkillTool: SessionTool<{}> = { const parsed = readSkillSchema.parse(args) const name = skillNameFromPath(parsed.path) toolCallbacks.setToolStatus(toolId, { content: `Reading skill "${name}"...` }) - try { - // The prompt lists the skills in play, but the tool takes a path the model - // composed, so what may be read is checked here rather than assumed. Against - // the listing, not just the off-switch: any other path is now enabled too, - // and without this the tool reads any resource holding a string `content` — - // the user's own access, but not what "load a skill" says it does. - const isSkill = (await listSkillResources(workspace)).skills.some( - (s) => s.path === parsed.path - ) - const suspended = toolCallbacks.workingScope?.scope()?.loadsSkill(parsed.path) === false - if (!isSkill || !isSkillEnabled(workspace, parsed.path) || suspended) { - toolCallbacks.setToolStatus(toolId, { content: `Skill "${name}" is not available` }) - return `"${parsed.path}" is not one of the skills available to this chat. Only the paths listed under "Skills" in the system prompt can be read.` - } - // Bounded here rather than in the reader: any `ai_skill` resource is in play, - // including ones written through git sync or the resource editor - // that never passed the authoring form's limits, and an unbounded body - // would exhaust the context on one tool call. The editor reads the same - // resource untruncated, so opening a long skill cannot rewrite it short. - const instructions = truncateForPrompt( - await readSkillBody(workspace, parsed.path), - MAX_SKILL_INSTRUCTIONS_LENGTH - ) + const { text } = await runOnServer('read_skill', workspace, args, { + skill_enabled: + isSkillEnabled(workspace, parsed.path) && + toolCallbacks.workingScope?.scope()?.loadsSkill(parsed.path) !== false + }) + const failed = text.match( + /^Failed to read skill ".*?": (.*)\. Check the path against the Skills list in the system prompt\.$/s + ) + if (failed) { + toolCallbacks.setToolStatus(toolId, { + content: `Error reading skill "${name}"`, + error: failed[1] + }) + } else if (text.startsWith('Skill: ')) { toolCallbacks.setToolStatus(toolId, { content: `Read skill "${name}"` }) // Whether a skill is actually reached for. No key: the path is // workspace-authored text. logFeatureUsage('ai_session', 'skill_read', { workspace }) - return `Skill: ${parsed.path}\n\nInstructions:\n${instructions}` - } catch (e) { - const msg = e instanceof Error ? e.message : String(e) - toolCallbacks.setToolStatus(toolId, { - content: `Error reading skill "${name}"`, - error: msg - }) - return `Failed to read skill "${parsed.path}": ${msg}. Check the path against the Skills list in the system prompt.` + } else { + toolCallbacks.setToolStatus(toolId, { content: `Skill "${name}" is not available` }) } + return text } } @@ -4118,11 +3198,11 @@ export const globalTools: SessionTool<{}>[] = [ ) } }, - createSearchHubScriptsTool(false), + searchHubScriptsServerTool, getHubIntegrationTool, searchNpmPackagesTool, - searchDocsTool, - readDocsPageTool, + searchDocsServerTool, + readDocsPageServerTool, { requires: NONE, def: createToolDef( @@ -4277,99 +3357,16 @@ export const globalTools: SessionTool<{}>[] = [ ), planModeSafe: true, fn: async ({ args, workspace, toolId, toolCallbacks }) => { - const parsed = listWorkspaceItemsSchema.parse(args) - const accessTypes = (parsed.types ?? []).filter(isAccessType) - const itemTypes = (parsed.types ?? []).filter((t): t is WorkspaceItemType => !isAccessType(t)) - // Asking only for folders or groups must not fall back to the default scripts and flows. - const types = - accessTypes.length > 0 && itemTypes.length === 0 ? [] : getRequestedTypes(itemTypes) - const limit = parsed.limit ?? 50 + listWorkspaceItemsSchema.parse(args) toolCallbacks.setToolStatus(toolId, { content: 'Listing workspace items...' }) - - const accessRows = ( - await listAccessEntities( - accessTypes, - workspace, - Math.min(limit, MAX_LIST_LIMIT), - parsed.page - ) - ).filter((row) => itemMatches(row, parsed.query)) - - const byKey = new Map() - // Scoped, its folders are the default, one listing each; an explicit path_prefix, "" - // included, overrides them. - const scopeRoots = toolCallbacks.workingScope?.scope()?.roots - const scopedByDefault = parsed.path_prefix === undefined && !!scopeRoots?.length - const prefixes: (string | undefined)[] = scopedByDefault - ? scopeRoots!.map((r) => `${r}/`) - : [parsed.path_prefix || undefined] - const perPage = Math.min(limit, MAX_LIST_LIMIT) - const page = parsed.page ?? 1 - // Several folders read as one listing, folder after folder, so `limit` stays one cap - // per type: page N is rows (N-1)*limit..N*limit of each type's concatenation, which - // takes every folder's first N*limit rows. - const workspaceItems = - prefixes.length === 1 - ? await listWorkspaceItems(types, workspace, prefixes[0], perPage, parsed.page) - : pageAcrossPrefixes( - await Promise.all( - prefixes.map((prefix) => - listWorkspaceItems(types, workspace, prefix, perPage * page, 1) - ) - ), - perPage, - page - ) - for (const item of workspaceItems) { - byKey.set(getWorkspaceItemKey(item.type, item.path, item.triggerKind), item) - } - - // Drafts are not paginated server-side; overlay them on page 1 only, - // capped at `limit` per type, so results stay bounded and later pages - // never repeat a page-1 item as its draft twin. Chat draft counts are - // small — past the cap, a narrower path_prefix still finds any draft - // (it filters before the cap; query filters after). - if ((parsed.page ?? 1) === 1) { - const draftCountByType = new Map() - const narrowing = prefixes.filter((p): p is string => !!p) - for (const draft of await listGlobalDrafts(workspace)) { - if (!types.includes(draft.type)) continue - // A draft's staged name is often not where it is stored: the editor parks - // a new script, flow or app at a generated `draft_` path, and a - // rename stages the new name over the old path. The server drops - // draft-only rows under any narrowing filter, leaving this pass their - // only source, so either name has to satisfy the prefix. - if ( - narrowing.length && - !narrowing.some((p) => draft.path.startsWith(p) || draft.draftPath?.startsWith(p)) - ) - continue - const count = draftCountByType.get(draft.type) ?? 0 - if (count >= limit) continue - draftCountByType.set(draft.type, count + 1) - byKey.set(getWorkspaceItemKey(draft.type, draft.path, draft.triggerKind), { - ...draft, - value: undefined, - schema: undefined - }) - } - } - - // No cross-type truncation: each type is already capped at `limit` rows by - // its own list call, and slicing the concatenation would silently drop the - // later types' rows while their next page skips past them. - const results = [ - ...Array.from(byKey.values()).filter((item) => itemMatches(item, parsed.query)), - ...accessRows - ] - - toolCallbacks.setToolStatus(toolId, { - content: `Listed ${results.length} workspace item(s)` + const { text } = await runOnServer('list_workspace_items', workspace, args, { + scope_roots: toolCallbacks.workingScope?.scope()?.roots }) - const listed = JSON.stringify(results, null, 2) - return scopedByDefault - ? `Items under ${prefixes.join(', ')}, where this session works. Pass path_prefix "" for the whole workspace.\n${listed}` - : listed + // A scoped listing opens with a line naming its folders. + toolCallbacks.setToolStatus(toolId, { + content: `Listed ${JSON.parse(text.slice(text.indexOf('['))).length} workspace item(s)` + }) + return text } }, { @@ -4380,13 +3377,13 @@ export const globalTools: SessionTool<{}>[] = [ 'List the data pipelines of the workspace: each folder holding at least one pipeline script, with its number of scripts. A pipeline is named by its folder, not by an item path.' ), planModeSafe: true, - fn: async ({ workspace, toolId, toolCallbacks }) => { + fn: async ({ args, workspace, toolId, toolCallbacks }) => { toolCallbacks.setToolStatus(toolId, { content: 'Listing data pipelines...' }) - const pipelines = await AssetService.listPipelineFolders({ workspace }) + const { text } = await runOnServer('list_pipelines', workspace, args) toolCallbacks.setToolStatus(toolId, { - content: `Listed ${pipelines.length} data pipeline(s)` + content: `Listed ${JSON.parse(text).length} data pipeline(s)` }) - return JSON.stringify(pipelines) + return text } }, { @@ -4400,50 +3397,21 @@ export const globalTools: SessionTool<{}>[] = [ fn: async ({ args, workspace, toolId, toolCallbacks }) => { const parsed = readWorkspaceItemSchema.parse(args) const { type } = parsed + toolCallbacks.setToolStatus(toolId, { content: `Reading ${type} "${parsed.path}"...` }) + const { text } = await runOnServer('read_workspace_item', workspace, args) + const read = JSON.parse(text) if (isAccessType(type)) { - toolCallbacks.setToolStatus(toolId, { content: `Reading ${type} "${parsed.path}"...` }) - const entity = await readAccessEntity(type, parsed.path, workspace) - toolCallbacks.setToolStatus(toolId, { content: `Read ${type} "${entity.name}"` }) - return JSON.stringify(entity, null, 2) - } - if (type === 'trigger' && !parsed.trigger_kind) { - const message = 'trigger_kind is required when type is trigger.' - toolCallbacks.setToolStatus(toolId, { content: message, error: message }) - return JSON.stringify({ success: false, error: message }) - } - const draft = - parsed.version === 'deployed' || isHubPath(parsed.path) - ? null - : await getGlobalDraft(workspace, type, parsed.path, parsed.trigger_kind) - if (draft) { + toolCallbacks.setToolStatus(toolId, { content: `Read ${type} "${read.name}"` }) + } else if (read.success === false) { + toolCallbacks.setToolStatus(toolId, { content: read.error, error: read.error }) + } else { toolCallbacks.setToolStatus(toolId, { - content: `Read draft ${type} "${parsed.path}"` + content: read.isDraft + ? `Read draft ${type} "${parsed.path}"` + : `Read ${type} "${parsed.path}"` }) - // The mode reported is the one deploying the draft leaves: a deployed app keeps its - // own, which the Share dialog sets, and a never-deployed one takes the draft's. - const executionMode = - type === 'app' - ? await describeAppExposure( - workspace, - (await deployedAppExecutionMode(workspace, parsed.path)) ?? - (draft.value as AppDraftValue)?.policy?.execution_mode - ) - : undefined - return JSON.stringify(serializeWorkspaceItemForRead({ ...draft, executionMode }), null, 2) } - - toolCallbacks.setToolStatus(toolId, { - content: `Reading ${type} "${parsed.path}"...` - }) - const item = await readWorkspaceItem( - type, - parsed.path, - workspace, - parsed.trigger_kind, - parsed.version === 'deployed' - ) - toolCallbacks.setToolStatus(toolId, { content: `Read ${type} "${parsed.path}"` }) - return JSON.stringify(serializeWorkspaceItemForRead(item), null, 2) + return text } }, { @@ -4467,28 +3435,26 @@ export const globalTools: SessionTool<{}>[] = [ return JSON.stringify({ success: false, error }) } toolCallbacks.setToolStatus(toolId, { content: `Creating folder \`f/${parsed.name}\`...` }) - try { - await FolderService.createFolder({ - workspace, - requestBody: { name: parsed.name, summary: parsed.summary } + const { text } = await runOnServer('create_folder', workspace, args) + const created = JSON.parse(text) + if (!created.success) { + toolCallbacks.setToolStatus(toolId, { + content: `Error: ${created.error}`, + error: created.error }) - // Reflect the new folder for the rest of the session without a refetch, crediting - // the workspace it was created in: crediting the ambient one instead would hide - // it from the prompt that needs it. - const role = await roleForWorkspace(workspace) - if (role.kind === 'resolved') { - const user = role.user - if (!user.folders) user.folders = [] - if (!user.folders.includes(parsed.name)) user.folders.push(parsed.name) - } - const message = `Created folder \`f/${parsed.name}\`. You can now write items to \`f/${parsed.name}/\`.` - toolCallbacks.setToolStatus(toolId, { content: message }) - return JSON.stringify({ success: true, message }) - } catch (e) { - const error = e instanceof Error ? e.message : String(e) - toolCallbacks.setToolStatus(toolId, { content: `Error: ${error}`, error }) - return JSON.stringify({ success: false, error }) + return text } + // Reflect the new folder for the rest of the session without a refetch, crediting + // the workspace it was created in: crediting the ambient one instead would hide + // it from the prompt that needs it. + const role = await roleForWorkspace(workspace) + if (role.kind === 'resolved') { + const user = role.user + if (!user.folders) user.folders = [] + if (!user.folders.includes(parsed.name)) user.folders.push(parsed.name) + } + toolCallbacks.setToolStatus(toolId, { content: created.message }) + return text } }, { @@ -4498,8 +3464,8 @@ export const globalTools: SessionTool<{}>[] = [ streamArguments: true, showFade: true, fn: async (ctx) => { - const parsed = writeScriptSchema.parse(ctx.args) - return writeScriptDraft(parsed, ctx) + writeScriptSchema.parse(ctx.args) + return runDraftWrite('write_script', ctx) } }, { @@ -4509,39 +3475,8 @@ export const globalTools: SessionTool<{}>[] = [ streamArguments: true, showFade: true, fn: async (ctx) => { - const parsed = writeFlowSchema.parse(ctx.args) - const modules = parseOptionalJsonArg(parsed.modules, 'modules') - const aiProviders = await getAiAgentProviderCatalogFor(ctx.workspace, modules) - const aiProviderWarnings: string[] = [] - const editable = validateEditableFlowJson( - { - modules, - schema: parseOptionalJsonArg(parsed.schema, 'schema'), - preprocessor_module: parseOptionalJsonArg( - parsed.preprocessor_module, - 'preprocessor_module' - ), - failure_module: parseOptionalJsonArg(parsed.failure_module, 'failure_module'), - groups: parseOptionalJsonArg(parsed.groups, 'groups'), - notes: parseOptionalJsonArg(parsed.notes, 'notes') - }, - { aiProviders, aiProviderWarnings } - ) - const resolved = await resolveWriteFlowInlineScripts(parsed.path, editable, ctx.workspace) - const result = await writeFlowDraft( - { - path: parsed.path, - summary: parsed.summary, - description: parsed.description, - flow: editableFlowToDraftValue(resolved), - preserveBaseValueSettings: true - }, - ctx - ) - return ( - appendEmptyInlineScriptWarning(result, resolved) + - formatAiAgentProviderWarnings(aiProviderWarnings) - ) + writeFlowSchema.parse(ctx.args) + return runDraftWrite('write_flow', ctx) } }, { @@ -4567,7 +3502,7 @@ export const globalTools: SessionTool<{}>[] = [ if (dropped.length) { throw new Error(describeDroppedScheduleOptions(dropped)) } - return writeScheduleDraft(parsed, ctx) + return runDraftWrite('write_schedule', ctx) } }, { @@ -4595,7 +3530,7 @@ export const globalTools: SessionTool<{}>[] = [ .join('; ')}` ) } - return writeTriggerDraft({ ...parsed, config: config.data }, ctx) + return runDraftWrite('write_trigger', ctx) } }, { @@ -4633,7 +3568,7 @@ export const globalTools: SessionTool<{}>[] = [ showFade: true, fn: async (ctx) => { const parsed = editScriptSchema.parse(ctx.args) - return editScript(parsed, ctx) + return runDraftWrite('edit_script', ctx, { pending: `Editing script "${parsed.path}"...` }) } }, { @@ -4648,7 +3583,7 @@ export const globalTools: SessionTool<{}>[] = [ showFade: true, fn: async (ctx) => { const parsed = patchFlowJsonSchema.parse(ctx.args) - return patchFlowJson(parsed, ctx) + return runDraftWrite('patch_flow_json', ctx, { pending: `Patching flow "${parsed.path}"...` }) } }, { @@ -4774,24 +3709,14 @@ export const globalTools: SessionTool<{}>[] = [ planModeSafe: true, showDetails: true, fn: async ({ args, workspace, toolId, toolCallbacks }) => { - const parsed = listRunsSchema.parse(args) + listRunsSchema.parse(args) toolCallbacks.setToolStatus(toolId, { content: 'Listing runs...' }) - const jobs = await JobService.listJobs({ - workspace, - scriptPathExact: parsed.path, - createdBy: parsed.created_by, - label: parsed.label, - success: parsed.success, - running: parsed.running, - perPage: parsed.limit ?? 30 - }) - const runs = jobs.map(summarizeRun) - const result = JSON.stringify(runs, null, 2) + const { text } = await runOnServer('list_runs', workspace, args) toolCallbacks.setToolStatus(toolId, { - content: `Listed ${runs.length} run(s)`, - result + content: `Listed ${JSON.parse(text).length} run(s)`, + result: text }) - return result + return text } }, { @@ -4803,39 +3728,20 @@ export const globalTools: SessionTool<{}>[] = [ ), planModeSafe: true, showDetails: true, - fn: async ({ toolId, toolCallbacks }) => { + fn: async ({ args, workspace, toolId, toolCallbacks }) => { toolCallbacks.setToolStatus(toolId, { content: 'Listing workers...' }) - const pings = await WorkerService.listWorkers({ perPage: WORKER_PAGE_SIZE }) - if (pings.length === 0) { - // Both role stores resolve asynchronously, and an unloaded one must not read as - // an absent role — that hedges the answer this branch exists to give plainly. - // No-ops once they hold a value. - await refreshSuperadmin() - const hiddenFromCaller = !get(superadmin) && !get(devopsRole) - const message = hiddenFromCaller ? NO_WORKERS_VISIBLE_MESSAGE : NO_WORKERS_CONNECTED_MESSAGE - toolCallbacks.setToolStatus(toolId, { - content: hiddenFromCaller ? 'No workers visible' : 'No workers connected', - result: message - }) - return message - } - const workers = pings.map((w) => ({ - worker: w.worker, - worker_group: w.worker_group, - custom_tags: w.custom_tags, - last_ping: w.last_ping, - jobs_executed: w.jobs_executed - })) - const note = - workers.length === WORKER_PAGE_SIZE - ? `Only the first ${WORKER_PAGE_SIZE} workers are listed; more may be connected.` - : undefined - const result = JSON.stringify({ workers, ...(note ? { note } : {}) }, null, 2) + const { text } = await runOnServer('list_workers', workspace, args) + const noneShown = + text === NO_WORKERS_VISIBLE_MESSAGE + ? 'No workers visible' + : text === NO_WORKERS_CONNECTED_MESSAGE + ? 'No workers connected' + : undefined toolCallbacks.setToolStatus(toolId, { - content: `Listed ${workers.length} worker(s)`, - result + content: noneShown ?? `Listed ${JSON.parse(text).workers.length} worker(s)`, + result: text }) - return result + return text } }, { @@ -4854,7 +3760,8 @@ export const globalTools: SessionTool<{}>[] = [ ? `Fetching result of step ${parsed.step} in run ${parsed.id}...` : `Inspecting run ${parsed.id}...` }) - const { text, jobId } = await getRun(workspace, parsed.id, parsed.step) + const { text, effects } = await runOnServer('get_run', workspace, args) + const jobId = effects.flatMap((e) => (e.type === 'inspected_run' ? [e.job_id] : []))[0] toolCallbacks.setToolStatus(toolId, { content: parsed.step ? `Fetched result of step ${parsed.step} in run ${parsed.id}` @@ -4881,21 +3788,20 @@ export const globalTools: SessionTool<{}>[] = [ fn: async ({ args, workspace, toolId, toolCallbacks }) => { const parsed = cancelJobSchema.parse(args) toolCallbacks.setToolStatus(toolId, { content: `Canceling job ${parsed.id}...` }) - try { - await JobService.cancelQueuedJob({ workspace, id: parsed.id, requestBody: {} }) - } catch (e) { - const msg = e instanceof Error ? e.message : 'Unknown error' - toolCallbacks.setToolStatus(toolId, { - content: `Could not cancel job ${parsed.id}`, - error: msg - }) - return `Failed to cancel job ${parsed.id}: ${msg}. It may have already finished.` - } + const { text } = await runOnServer('cancel_job', workspace, args) + const failed = text.match( + /^Failed to cancel job .*?: (.*)\. It may have already finished\.$/s + ) // The tray's background poller will pick up the canceled state (updating // its status + Job snapshot together); a bare status patch here would leave // the badge stale and stop the poller. Just report to the model. - toolCallbacks.setToolStatus(toolId, { content: `Canceled job ${parsed.id}` }) - return `Job ${parsed.id} was canceled.` + toolCallbacks.setToolStatus( + toolId, + failed + ? { content: `Could not cancel job ${parsed.id}`, error: failed[1] } + : { content: `Canceled job ${parsed.id}` } + ) + return text } }, { @@ -4956,7 +3862,7 @@ export const globalTools: SessionTool<{}>[] = [ showFade: true, fn: async (ctx) => { const parsed = rebaseDraftSchema.parse(ctx.args) - return rebaseDraft(parsed, ctx) + return runDraftWrite('rebase_draft', ctx, { draftLabel: parsed.path }) } }, { @@ -5012,7 +3918,25 @@ export const globalTools: SessionTool<{}>[] = [ validateBeforeConfirmation: validateDeleteWorkspaceItemTarget, fn: async (ctx) => { const parsed = deleteWorkspaceItemSchema.parse(ctx.args) - return deleteWorkspaceItem(parsed, ctx) + const { type, path } = parsed + return runDraftWrite('delete_workspace_item', ctx, { + // A trigger named without its kind is refused before anything starts. + pending: + type === 'trigger' && !parsed.trigger_kind ? undefined : `Deleting ${type} "${path}"...`, + draftLabel: path, + // Deployed state changed: cached fork comparisons involving this workspace are no + // longer trustworthy, whether or not the draft cleanup then succeeds. + afterServer: () => invalidateWorkspaceComparison(ctx.workspace), + // Recorded in the chat's modified-items mask: in a fork this leaves a reviewable + // "removed" diff vs the parent, scoped to this chat. + onSaved: (effects) => { + for (const effect of effects) { + if (effect.type === 'draft_delete') { + ctx.toolCallbacks.onItemModified?.(effect.kind, effect.storage_path) + } + } + } + }) } }, { @@ -5031,7 +3955,18 @@ export const globalTools: SessionTool<{}>[] = [ confirmationMessage: 'Discard draft', fn: async (ctx) => { const parsed = discardLocalDraftSchema.parse(ctx.args) - return discardLocalDraft(parsed, ctx) + return runDraftWrite('discard_local_draft', ctx, { + draftLabel: parsed.path, + // The chat's touch on the item is undone — drop it from the mask so a + // pre-existing deployed item doesn't keep reading as this chat's edit. + onSaved: (effects) => { + for (const effect of effects) { + if (effect.type === 'draft_delete') { + ctx.toolCallbacks.onItemDiscarded?.(effect.kind, effect.storage_path) + } + } + } + }) } }, { @@ -5046,8 +3981,8 @@ export const globalTools: SessionTool<{}>[] = [ streamArguments: true, showFade: true, fn: async (ctx) => { - const parsed = writeResourceSchema.parse(ctx.args) - return writeResourceDraft(parsed, ctx) + writeResourceSchema.parse(ctx.args) + return runDraftWrite('write_resource', ctx) } }, { @@ -5062,8 +3997,8 @@ export const globalTools: SessionTool<{}>[] = [ streamArguments: true, showFade: true, fn: async (ctx) => { - const parsed = writeVariableSchema.parse(ctx.args) - return writeVariableDraft(parsed, ctx) + writeVariableSchema.parse(ctx.args) + return runDraftWrite('write_variable', ctx) } }, { @@ -5079,36 +4014,13 @@ export const globalTools: SessionTool<{}>[] = [ toolCallbacks.setToolStatus(toolId, { content: `Searching resource types for "${parsed.query}"...` }) - const { resourceTypes, note } = await searchResourceTypes( - parsed.query, - workspace, - parsed.limit ?? 5 - ) + const { text } = await runOnServer('search_resource_types', workspace, args) + // The listing is pretty-printed JSON, which holds no blank line; a note follows one. + const found = JSON.parse(text.split('\n\n')[0]).length toolCallbacks.setToolStatus(toolId, { - content: `Found ${resourceTypes.length} resource type(s) for "${parsed.query}"` + content: `Found ${found} resource type(s) for "${parsed.query}"` }) - const listing = JSON.stringify( - resourceTypes.map((rt) => ({ - name: rt.name, - schema: rt.schema - })), - null, - 2 - ) - // On an unsynced instance the dedicated type is only missing until a sync, so its note - // replaces the "no dedicated type" one, which would have the model create a duplicate. - if (note) { - return `${listing}\n\n${note}` - } - // The search is semantic and always returns its closest types, so a miss reads - // like a hit unless the absence is spelled out. - const words = (parsed.query.toLowerCase().match(/[a-z0-9]+/g) ?? []).filter( - (w) => w.length >= 3 - ) - const named = resourceTypes.some((rt) => words.some((w) => rt.name.toLowerCase().includes(w))) - return named || words.length === 0 - ? listing - : `${listing}\n\nNo resource type name contains "${parsed.query}": these are only the nearest matches, so no dedicated type exists. Searching again with synonyms will not find one — use a fitting generic type above, or create a custom c_ type.` + return text } }, { @@ -5127,32 +4039,18 @@ export const globalTools: SessionTool<{}>[] = [ confirmationMessage: 'Create resource type', fn: async ({ args, workspace, toolId, toolCallbacks }) => { const parsed = createResourceTypeSchema.parse(args) - // The prefix the Add resource type form adds unless an admin turns it off: a hub type - // of the same name, synced later, would otherwise clash with this one. - const name = parsed.name.startsWith('c_') ? parsed.name : `c_${parsed.name}` - await ResourceService.createResourceType({ - workspace, - requestBody: { - name, - description: parsed.description, - schema: parsed.schema - } - }) + const { text } = await runOnServer('create_resource_type', workspace, args) // Forms show a resource picker only for a type in this cached list, so a script // written against the new type would otherwise get a plain object input. resourceTypesStore.set(undefined) + const name = parsed.name.startsWith('c_') ? parsed.name : `c_${parsed.name}` toolCallbacks.setToolStatus(toolId, { content: `Created resource type ${name}` }) - return JSON.stringify({ - success: true, - message: `Created resource type ${name}. Refer to it by this name.` - }) + return text } }, - createDbSchemaTool<{}>({ - description: - 'Fetch the schema (tables and columns) of a database resource by its path. Supports postgresql, mysql, ms_sql_server, snowflake and bigquery resources.', - updateEditorCache: false - }), + dbSchemaServerTool( + 'Fetch the schema (tables and columns) of a database resource by its path. Supports postgresql, mysql, ms_sql_server, snowflake and bigquery resources.' + ), { requires: NONE, def: createToolDef( @@ -5163,7 +4061,10 @@ export const globalTools: SessionTool<{}>[] = [ planModeSafe: true, fn: async (ctx) => { const parsed = readFlowModuleCodeSchema.parse(ctx.args) - return readFlowModuleCode(parsed, ctx) + ctx.toolCallbacks.setToolStatus(ctx.toolId, { + content: `Reading inline script for module "${parsed.module_id}" from flow "${parsed.path}"...` + }) + return (await runServerTool('read_flow_module_code', ctx)).text } }, { @@ -5178,7 +4079,9 @@ export const globalTools: SessionTool<{}>[] = [ showFade: true, fn: async (ctx) => { const parsed = setFlowModuleCodeSchema.parse(ctx.args) - return setFlowModuleCode(parsed, ctx) + return runDraftWrite('set_flow_module_code', ctx, { + pending: `Updating inline script for module "${parsed.module_id}" in flow "${parsed.path}"...` + }) } }, { @@ -5194,8 +4097,8 @@ export const globalTools: SessionTool<{}>[] = [ showDetails: true, showFade: true, fn: async (ctx) => { - const parsed = initAppSchema.parse(ctx.args) - return initApp(parsed, ctx) + initAppSchema.parse(ctx.args) + return runDraftWrite('init_app', ctx, { app: true }) } }, { @@ -5208,7 +4111,10 @@ export const globalTools: SessionTool<{}>[] = [ planModeSafe: true, fn: async (ctx) => { const parsed = readAppFileSchema.parse(ctx.args) - return readAppFile(parsed, ctx) + ctx.toolCallbacks.setToolStatus(ctx.toolId, { + content: `Reading ${resolveAppFileTarget(parsed.file_path).filePath} from app "${parsed.path}"...` + }) + return (await runServerTool('read_app_file', ctx)).text } }, { @@ -5221,7 +4127,12 @@ export const globalTools: SessionTool<{}>[] = [ planModeSafe: true, fn: async (ctx) => { const parsed = searchAppSchema.parse(ctx.args) - return searchApp(parsed, ctx) + if (parsed.query.length > 0) { + ctx.toolCallbacks.setToolStatus(ctx.toolId, { + content: `Searching app "${parsed.path}" for "${parsed.query}"...` + }) + } + return (await runServerTool('search_app', ctx)).text } }, { @@ -5236,7 +4147,10 @@ export const globalTools: SessionTool<{}>[] = [ showFade: true, fn: async (ctx) => { const parsed = writeAppFileSchema.parse(ctx.args) - return writeAppFile(parsed, ctx) + return runDraftWrite('write_app_file', ctx, { + app: true, + pending: `Writing ${resolveAppFileTarget(parsed.file_path).filePath} to app "${parsed.path}"...` + }) } }, { @@ -5248,7 +4162,10 @@ export const globalTools: SessionTool<{}>[] = [ ), fn: async (ctx) => { const parsed = deleteAppFileSchema.parse(ctx.args) - return deleteAppFile(parsed, ctx) + return runDraftWrite('delete_app_file', ctx, { + app: true, + pending: `Deleting ${resolveAppFileTarget(parsed.file_path).filePath} from app "${parsed.path}"...` + }) } }, { @@ -5263,7 +4180,10 @@ export const globalTools: SessionTool<{}>[] = [ showFade: true, fn: async (ctx) => { const parsed = patchAppFileSchema.parse(ctx.args) - return patchAppFile(parsed, ctx) + return runDraftWrite('patch_app_file', ctx, { + app: true, + pending: `Patching ${resolveAppFileTarget(parsed.file_path).filePath} in app "${parsed.path}"...` + }) } }, { @@ -5279,7 +4199,10 @@ export const globalTools: SessionTool<{}>[] = [ showFade: true, fn: async (ctx) => { const parsed = writeAppRunnableSchema.parse(ctx.args) - return writeAppRunnable(parsed, ctx) + return runDraftWrite('write_app_runnable', ctx, { + app: true, + pending: `Writing runnable "${parsed.key}" to app "${parsed.path}"...` + }) } }, { @@ -5291,7 +4214,10 @@ export const globalTools: SessionTool<{}>[] = [ ), fn: async (ctx) => { const parsed = deleteAppRunnableSchema.parse(ctx.args) - return deleteAppRunnable(parsed, ctx) + return runDraftWrite('delete_app_runnable', ctx, { + app: true, + pending: `Removing runnable "${parsed.key}" from app "${parsed.path}"...` + }) } }, { @@ -5956,159 +4882,6 @@ export function setDeployedInSessionHandler(handler: DeployedInSessionHandler | deployedInSessionHandler = handler } -type DraftConfig = Record -type ScheduleDraftConfig = NewSchedule & DraftConfig -type TriggerDraftConfig = TriggerRequestBody & DraftConfig & { path: string } - -function stripBackendMetadata(value: T): T { - const draft = structuredClone(value) - delete draft.workspace_id - delete draft.edited_by - delete draft.edited_at - delete draft.email - delete draft.error - return draft -} - -function mergeDraftConfig( - base: T | undefined, - overrides: DraftConfig, - path: string -): T { - return { - ...(base ? stripBackendMetadata(base) : {}), - ...structuredClone(overrides), - path - } as unknown as T -} - -function resourceToDraftState(resource: Resource): ResourceDraftState { - return { - path: resource.path, - description: resource.description ?? '', - args: structuredClone((resource.value ?? {}) as Record), - labels: resource.labels ?? undefined, - wsSpecific: resource.ws_specific ?? false, - resource_type: resource.resource_type - } -} - -function createResourceToDraftState( - args: CreateResource, - base?: ResourceDraftState -): ResourceDraftState { - return { - ...base, - path: args.path, - description: args.description ?? base?.description ?? '', - args: structuredClone((args.value ?? base?.args ?? {}) as Record), - labels: args.labels ?? base?.labels, - wsSpecific: args.ws_specific ?? base?.wsSpecific ?? false, - resource_type: args.resource_type ?? base?.resource_type - } -} - -function variableToDraftState(variable: ListableVariable): VariableDraftState { - // An OAuth variable's value is owned by the refresh flow, and `get_variable` hands back - // a freshly refreshed LIVE token for an expired one even under decryptSecret: false - // (variables.rs checks is_expired first). Drop it here, at the boundary: carrying it - // into a draft would pin a rotating value to a static one on the next deploy. - const oauthManaged = variable.is_oauth || variable.account != null - return { - path: variable.path, - variable: { - value: oauthManaged ? '' : (variable.value ?? ''), - is_secret: variable.is_secret, - description: variable.description ?? '' - }, - labels: variable.labels ?? undefined, - wsSpecific: variable.ws_specific ?? false, - // `get_variable` sends these as explicit nulls; carrying a null through would add - // `account: null` / `expires_at: null` to the draft and to every diff the user - // reviews. Undefined drops out of the JSON instead. - account: variable.account ?? undefined, - is_oauth: variable.is_oauth ?? undefined, - expires_at: variable.expires_at ?? undefined, - value_valid_until: variable.value_valid_until ?? undefined - } -} - -function resolveVariableWrite( - args: WriteVariableArgs, - base?: VariableDraftState -): { - is_secret: boolean - value: string - description: string -} { - if (base === undefined && (args.value === undefined || args.is_secret === undefined)) { - throw new Error( - `Variable "${args.path}" does not exist yet, so creating it requires both value and is_secret.` - ) - } - const is_secret = args.is_secret ?? base?.variable.is_secret ?? false - // '' is the sentinel for "nothing staged" in a secret draft, so it cannot also mean - // "set the secret to empty". Refusing it matters because a model reaching for a - // placeholder — the habit this schema change removes — would otherwise wipe the secret. - if (is_secret && args.value === '') { - throw new Error( - `An empty string is not a valid value for secret variable "${args.path}". Omit value to keep the stored secret, or pass the real new one.` - ) - } - // Securing one needs a value too when it holds none: the deploy would send no `value` - // (nothing is staged) and the backend refuses an is_secret change without one. Saying - // so here keeps the model from having to interpret that error. - if ( - is_secret && - base?.variable.is_secret === false && - (args.value ?? base.variable.value) === '' - ) { - throw new Error( - `Cannot make variable "${args.path}" secret without a value: it currently holds an empty one, so there would be nothing to encrypt. Pass the value it should hold.` - ) - } - // Un-securing always needs a new plaintext value. An `$encrypted:` marker is no - // help: the deploy endpoints only decrypt it while the target stays secret, so - // carrying it into a non-secret variable would store the marker as the value. - if (is_secret === false && base?.variable.is_secret === true && args.value === undefined) { - throw new Error( - `Cannot turn secret variable "${args.path}" into a non-secret one without a value: its stored value cannot be read, so it would be replaced by an empty one. Pass the new plaintext value, or leave is_secret unset to keep it secret.` - ) - } - return { - is_secret, - value: args.value ?? base?.variable.value ?? '', - description: args.description ?? base?.variable.description ?? '' - } -} - -function createVariableToDraftState( - args: WriteVariableArgs, - base?: VariableDraftState -): VariableDraftState { - const { is_secret, value, description } = resolveVariableWrite(args, base) - return { - ...base, - path: args.path, - variable: { - value, - is_secret, - description - }, - labels: args.labels ?? base?.labels, - wsSpecific: args.ws_specific ?? base?.wsSpecific ?? false, - account: args.account ?? base?.account, - is_oauth: args.is_oauth ?? base?.is_oauth, - expires_at: args.expires_at ?? base?.expires_at, - // `null` clears and omitted keeps the base's. The draft stores a cleared expiry as - // `undefined`, since a stored draft goes through a JSON round-trip. - value_valid_until: - args.value_valid_until === null - ? undefined - : (args.value_valid_until ?? base?.value_valid_until) - } -} - // The deploy body for a variable that does not exist yet. function buildVariableCreateRequestBody(draftValue: CreateVariable): CreateVariable { const requestBody = structuredClone(draftValue) @@ -6147,12 +4920,6 @@ function buildVariableUpdateRequestBody(draftValue: CreateVariable): Omit< return { ...rest, value } } -function startDraftWrite(ctx: WriteDraftCtx, type: WorkspaceItemType, path: string): void { - ctx.toolCallbacks.setToolStatus(ctx.toolId, { - content: `Saving ${type} "${path}" as a draft…` - }) -} - // Conflict / save-failure handling shared by the kind write tools and the app // write tools. Returns the JSON tool-result for a non-saved persist, or undefined // when the save succeeded (the caller then emits its own success payload). @@ -6219,35 +4986,106 @@ function maybeAttachPreviewCard( return true } -// App write tools build varied success messages but share the same conflict / -// save-failure handling; `onSaved` supplies the per-tool status + message. -function finishAppDraftWrite( - result: DraftPersistResult, - ctx: WriteDraftCtx, - onSaved: () => { content: string; message: string; warning?: string }, - codeDiff?: ToolCodeDiff -): string { +/** + * A draft write run on the server. The server builds the value from the draft the browser + * holds (or the server's), and the chat saves it through its own draft store, so an open + * editor, its auto-save toggle and its conflict dialog behave as when the browser built it. + * A conflict or failed save replaces the server's text, which assumed the save went through. + */ +async function runDraftWrite( + name: string, + ctx: WriteDraftCtx & { args: unknown }, + opts: { + /** The card's label while the call runs. */ + pending?: string + /** A write of one of the app's files and runnables, which names its own success. */ + app?: boolean + /** What a failed delete names the draft as, in the error the model reads. */ + draftLabel?: string + /** Runs once the call's writes and deletes are saved. */ + onSaved?: (effects: SessionToolEffect[]) => void + /** Runs once the server call succeeded, before its drafts are saved. */ + afterServer?: () => void + } = {} +): Promise { + const { workspace, toolId, toolCallbacks } = ctx + if (opts.pending) toolCallbacks.setToolStatus(toolId, { content: opts.pending }) + const context: Record = { session: !!(ctx.sessionId ?? sessionIdFromCtx(ctx)) } + let response = await callSessionTool(workspace, name, ctx.args, context) + // A flow whose AI steps name their provider is checked against the catalog the chat + // builds for its prompt; the server asks for it only then, and writes nothing until sent. + const needed = response.effects.flatMap((e) => + e.type === 'needs_ai_provider_catalog' ? [e.resource_refs] : [] + )[0] + if (needed) { + context.ai_provider_catalog = await getAiAgentProviderCatalogForRefs(workspace, needed) + response = await callSessionTool(workspace, name, ctx.args, context) + } + const afterSave = response.effects.flatMap((e) => + e.type === 'status' && e.after_save ? [e] : [] + ) + applyStatusEffects( + response.effects.filter((e) => !(e.type === 'status' && e.after_save)), + ctx + ) + if (response.is_error) throw new Error(response.text) + // The server only checks that a flow's steps deserialize, which fills in what the openflow + // schema requires (an input's `expr`, a step's `id`); a step missing one would save and fail + // at run time. + for (const effect of response.effects) { + if (effect.type === 'draft_write' && effect.kind === 'flow') { + const flow = (effect.value as { value?: Record }).value ?? {} + validateFlowModules(flow.modules) + validateOptionalFlowModule(flow.preprocessor_module, 'preprocessor_module') + validateOptionalFlowModule(flow.failure_module, 'failure_module') + } + } + opts.afterServer?.() + for (const effect of response.effects) { + if (effect.type === 'draft_write' && effect.kind === 'script') { + await inferScriptDraftSchema(effect.value as NewScript) + } + } + const applied = await applySessionToolEffects( + workspace, + response.effects, + (storagePath) => opts.draftLabel ?? storagePath + ) + if (applied.deleteFailure) throw new Error(applied.deleteFailure.message) + const savedStatuses = () => { + for (const { type: _, after_save: __, ...patch } of afterSave) { + toolCallbacks.setToolStatus(toolId, patch) + } + } + const result = applied.writes[0] + if (!result) { + opts.onSaved?.(response.effects) + savedStatuses() + return response.text + } const failure = draftWriteFailure(result, ctx) - if (failure) return failure - if (codeDiff) ctx.toolCallbacks.setToolStatus(ctx.toolId, { codeDiff }) - ctx.toolCallbacks.onItemModified?.(result.itemKind, result.storagePath) - maybeAttachPreviewCard(ctx, PREVIEW_CARD_KIND_BY_ITEM_KIND[result.itemKind], result.item.path) - const { content, message, warning } = onSaved() - ctx.toolCallbacks.setToolStatus(ctx.toolId, { content, result: 'Saved as draft' }) - return JSON.stringify({ success: true, message, warning }, null, 2) -} - -function finishDraftWrite( - result: DraftPersistResult, - existed: boolean, - ctx: WriteDraftCtx, - codeDiff?: ToolCodeDiff, - actions?: ToolDisplayAction[] -): string { - const failure = draftWriteFailure(result, ctx) - if (failure) return failure - if (codeDiff) ctx.toolCallbacks.setToolStatus(ctx.toolId, { codeDiff }) - ctx.toolCallbacks.onItemModified?.(result.itemKind, result.storagePath) + if (failure) { + const suffix = response.effects.flatMap((e) => + e.type === 'facts' && typeof e.text_suffix === 'string' ? [e.text_suffix] : [] + )[0] + return failure + (suffix ?? '') + } + const diff = response.effects.flatMap((e) => (e.type === 'code_diff' ? [e] : []))[0] + if (diff) { + const lang = diff.file_path + ? appFileEditorLang(diff.file_path) + : scriptLangToEditorLang((diff.language ?? undefined) as ScriptLang | undefined) + const beforeLang = diff.before_file_path ? appFileEditorLang(diff.before_file_path) : lang + toolCallbacks.setToolStatus(toolId, { + codeDiff: { + before: diff.before, + after: diff.after, + lang, + ...(beforeLang !== lang ? { beforeLang } : {}) + } + }) + } + toolCallbacks.onItemModified?.(result.itemKind, result.storagePath) const stored = result.item const isAgent = stored.type === 'resource' && @@ -6257,337 +5095,41 @@ function finishDraftWrite( isAgent ? 'agent' : PREVIEW_CARD_KIND_BY_ITEM_KIND[result.itemKind], stored.path ) - // Outside a session there is no preview chip, so the actions carry the way in. - if (actions && !card) ctx.toolCallbacks.setToolStatus(ctx.toolId, { actions }) - const verb = existed ? 'Updated' : 'Created' - // Don't echo the flow value back: the model just sent it in the write call, - // so reflecting the (large) compact flow JSON only burns tokens. Variables - // echo a redacted item; everything else round-trips its small payload. - const serializedItem = - stored.type === 'flow' - ? undefined - : stored.type === 'variable' - ? serializeWorkspaceItemForRead(stored) - : stored - - ctx.toolCallbacks.setToolStatus(ctx.toolId, { - content: `${verb} ${isAgent ? 'agent' : stored.type} "${stored.path}" as a draft`, + // The agent editor is where the user reviews and deploys the draft. Outside a session there is + // no preview chip, so an action carries the way in. + if (isAgent && !card) { + toolCallbacks.setToolStatus(toolId, { actions: [createOpenAgentAction(stored.path)] }) + } + opts.onSaved?.(response.effects) + if (opts.app) { + savedStatuses() + return response.text + } + const written = JSON.parse(response.text.split('\n\nAI agent provider warnings:\n')[0]) + toolCallbacks.setToolStatus(toolId, { + content: `${written.message.startsWith('Updated') ? 'Updated' : 'Created'} ${isAgent ? 'agent' : stored.type} "${stored.path}" as a draft`, result: `Saved as draft` }) - return JSON.stringify( - { - success: true, - message: `${verb} ${stored.type} "${stored.path}" as a per-user draft (saved server-side, visible only to this user — not a deployed workspace item). It was not deployed.`, - item: serializedItem - }, - null, - 2 - ) + savedStatuses() + // A script's schema is inferred here, after the server answered: its item is the saved one. + return stored.type === 'script' + ? JSON.stringify({ ...written, item: stored }, null, 2) + : response.text } -// Per-draft-kind knowledge for the shared write skeleton. `fetchDeployed` returns -// the deployed item already shaped as a draft value (e.g. script with parent_hash) -// so `buildDraft` treats a draft base and a deployed base identically; a `base` of -// undefined is the create-from-scratch case. -type WriteSpec = { - probe: (workspace: string, path: string) => Promise - fetchDeployed: (workspace: string, path: string) => Promise - buildDraft: (base: T | undefined, args: A, path: string) => T | Promise -} - -async function writeDraft( - spec: WriteSpec, - type: WorkspaceItemType, - path: string, - args: A, - ctx: WriteDraftCtx, - opts: { - triggerKind?: TriggerKind - override?: boolean - codeDiff?: ToolCodeDiff | ((base: T | undefined, draft: T) => ToolCodeDiff | undefined) - actions?: ToolDisplayAction[] - } = {} -): Promise { - const { workspace } = ctx - startDraftWrite(ctx, type, path) - - const existingDraft = await readGlobalDraftValue(workspace, type, path, opts.triggerKind) - let base = existingDraft - let existed = existingDraft !== undefined - if (base === undefined && (await spec.probe(workspace, path))) { - base = await spec.fetchDeployed(workspace, path) - existed = true - } - - const draft = await spec.buildDraft(base, args, path) - - const result = await persistGlobalDraft(workspace, type, path, draft, { - triggerKind: opts.triggerKind, - force: opts.override - }) - const codeDiff = typeof opts.codeDiff === 'function' ? opts.codeDiff(base, draft) : opts.codeDiff - return finishDraftWrite(result, existed, ctx, codeDiff, opts.actions) -} - -type ScriptDraftArgs = { - path: string - summary?: string - language: ScriptLang - content: string - override?: boolean -} - -const SCRIPT_SPEC: WriteSpec = { - probe: (workspace, path) => ScriptService.existsScriptByPath({ workspace, path }), - fetchDeployed: async (workspace, path) => { - // The deployed lock describes the deployed content, not the draft's; the deploy recomputes it. - const { - lock: _lock, - lock_error_logs: _lockErrorLogs, - ...existing - } = await ScriptService.getScriptByPath({ workspace, path }) - return { ...(existing as unknown as NewScript), parent_hash: existing.hash } - }, - buildDraft: async (base, args, path) => { - const draft: NewScript = base - ? { - ...structuredClone(base), - path, - summary: args.summary ?? base.summary, - content: args.content, - language: args.language - } - : { - path, - summary: args.summary ?? '', - description: '', - content: args.content, - schema: emptySchema(), - is_template: false, - language: args.language, - kind: 'script' - } - // Into the schema the base carries, as the editor does at save: `inferArgs` re-seeds - // each arg from the properties it is handed, and those are the only copy of - // `password`, enums, formats and titles — no parser emits them. A clone, so a parse - // failure leaves the previous schema rather than half of one. - try { - const schema = structuredClone( - draft.schema?.properties ? draft.schema : emptySchema() - ) as Schema - await inferArgs(draft.language, draft.content, schema) - draft.schema = schema - } catch (e) { - console.error('Failed to infer script schema before saving draft', e) - } - return draft - } -} - -function writeScriptDraft( - args: ScriptDraftArgs, - ctx: WriteDraftCtx, - codeDiff?: ToolCodeDiff -): Promise { - // A session builds pipeline nodes into the folder's pipeline draft; a script draft - // written here never reaches the pipeline graph, and saving the pipeline drops it. - if ( - (ctx.sessionId ?? sessionIdFromCtx(ctx)) && - parsePipelineAnnotations(args.content).inPipeline - ) { - const folder = args.path.match(/^f\/([^/]+)\//)?.[1] ?? '' - throw new Error( - `"${args.path}" is a data pipeline node (its source starts with the pipeline annotation). In a session, pipeline nodes are not script drafts: call open_preview(kind="pipeline", path="${folder}") if that folder's pipeline editor is not open yet, then use build_pipeline_node (new node) or edit_pipeline_node (existing node). Nothing was written.` - ) - } - return writeDraft(SCRIPT_SPEC, 'script', args.path, args, ctx, { - override: args.override, - codeDiff: - codeDiff ?? - ((base, draft) => ({ - before: base?.content ?? '', - after: draft.content, - lang: scriptLangToEditorLang(draft.language) - })) - }) -} - -type FlowDraftArgs = { - path: string - summary?: string - description?: string - flow: FlowDraftValue - override?: boolean - /** Carry over the base value's non-structural fields (chat_input_enabled, - * same_worker, ...) into the new value. Set by write_flow, whose arguments - * cannot express them; patch_flow_json passes the full value state instead. */ - preserveBaseValueSettings?: boolean -} - -const FLOW_STRUCTURAL_VALUE_KEYS = new Set(EDITABLE_FLOW_STRUCTURAL_KEYS) - -const FLOW_SPEC: WriteSpec = { - probe: (workspace, path) => FlowService.existsFlowByPath({ workspace, path }), - fetchDeployed: (workspace, path) => FlowService.getFlowByPath({ workspace, path }), - buildDraft: (base, args, path) => { - const value = structuredClone(args.flow.value) - if (args.flow.groups !== undefined && args.flow.groups !== null) { - value.groups = structuredClone(args.flow.groups) - } - if (args.preserveBaseValueSettings && base?.value) { - for (const [key, fieldValue] of Object.entries(base.value)) { - if ( - !FLOW_STRUCTURAL_VALUE_KEYS.has(key) && - (value as Record)[key] === undefined - ) { - ;(value as Record)[key] = structuredClone(fieldValue) - } - } - } - // Also checked at deploy; dropped here too because a test run of the draft reads the locks. - dropStaleStepLocks(value, base?.value) - return base - ? { - ...structuredClone(base), - path, - summary: args.summary ?? base.summary, - description: args.description ?? base.description, - value, - schema: args.flow.schema ?? base.schema - } - : { - path, - summary: args.summary ?? '', - description: args.description ?? '', - value, - schema: args.flow.schema ?? emptySchema(), - edited_by: '', - edited_at: '', - archived: false, - extra_perms: {} - } - } -} - -function writeFlowDraft(args: FlowDraftArgs, ctx: WriteDraftCtx): Promise { - return writeDraft(FLOW_SPEC, 'flow', args.path, args, ctx, { override: args.override }) -} - -const SCHEDULE_SPEC: WriteSpec = { - probe: (workspace, path) => ScheduleService.existsSchedule({ workspace, path }), - fetchDeployed: async (workspace, path) => - (await ScheduleService.getSchedule({ workspace, path })) as ScheduleDraftConfig, - buildDraft: (base, args, path) => { - // `override` is a tool-only conflict-resolution flag, not schedule config — - // strip it so mergeDraftConfig doesn't clone it into the persisted draft. - const { override: _override, ...config } = args - return mergeDraftConfig(base, config as DraftConfig, path) - } -} - -function writeScheduleDraft( - args: NewSchedule & { override?: boolean }, - ctx: WriteDraftCtx -): Promise { - return writeDraft(SCHEDULE_SPEC, 'schedule', args.path, args, ctx, { override: args.override }) -} - -function triggerWriteSpec(kind: TriggerKind): WriteSpec { - const service = triggerServices[kind] - return { - probe: (workspace, path) => service.exists({ workspace, path }), - fetchDeployed: async (workspace, path) => - (await service.get({ workspace, path })) as TriggerDraftConfig, - buildDraft: (base, config, path) => { - const draft = mergeDraftConfig(base, config, path) - if (kind === 'email') { - // workspaced_local_part maps to a NOT NULL column but is optional in the - // tool schema. Default on the merged draft (not the incoming config) so an - // omitted field keeps the existing trigger's value instead of resetting it. - const email = draft as TriggerDraftConfig & { workspaced_local_part?: boolean } - email.workspaced_local_part = email.workspaced_local_part ?? false - } - return draft - } - } -} - -function writeTriggerDraft( - args: { kind: TriggerKind; config: unknown; override?: boolean }, - ctx: WriteDraftCtx -): Promise { - const config = args.config as TriggerDraftConfig - return writeDraft(triggerWriteSpec(args.kind), 'trigger', config.path, config, ctx, { - triggerKind: args.kind, - override: args.override - }) -} - -const RESOURCE_SPEC: WriteSpec = { - probe: (workspace, path) => ResourceService.existsResource({ workspace, path }), - fetchDeployed: async (workspace, path) => - resourceToDraftState(await ResourceService.getResource({ workspace, path })), - buildDraft: (base, args) => createResourceToDraftState(args, base) -} - -function writeResourceDraft( - args: CreateResource & { override?: boolean }, - ctx: WriteDraftCtx -): Promise { - return writeDraft(RESOURCE_SPEC, 'resource', args.path, args, ctx, { - override: args.override, - // The agent editor is where the user reviews and deploys the draft. - actions: args.resource_type === 'ai_agent' ? [createOpenAgentAction(args.path)] : undefined - }) -} - -const VARIABLE_SPEC: WriteSpec = { - probe: (workspace, path) => VariableService.existsVariable({ workspace, path }), - fetchDeployed: async (workspace, path) => - variableToDraftState( - await VariableService.getVariable({ workspace, path, decryptSecret: false }) - ), - buildDraft: (base, args) => createVariableToDraftState(args, base) -} - -function writeVariableDraft(args: WriteVariableArgs, ctx: WriteDraftCtx): Promise { - // A variable's value is never interpolated, so "$var:" as the value - // is always the model echoing the reference syntax back instead of a real value. - if (args.value === `$var:${args.path}`) { - throw new Error( - `"${args.value}" is not a valid value for variable "${args.path}" — it is a self-reference. Omit value to keep the current one.` - ) - } - return writeDraft(VARIABLE_SPEC, 'variable', args.path, args, ctx, { override: args.override }) -} - -async function loadScriptForEdit( - path: string, - workspace: string -): Promise<{ - content: string - language: ScriptLang - summary?: string - schema?: Record -}> { - const draft = await getGlobalDraft(workspace, 'script', path) - if (draft) { - if (typeof draft.value !== 'string' || !draft.language) { - throw new Error(`Draft script "${path}" is missing content or language.`) - } - return { - content: draft.value, - language: draft.language, - summary: draft.summary, - schema: draft.schema as Record | undefined - } - } - const script = await ScriptService.getScriptByPath({ workspace, path }) - return { - content: script.content, - language: script.language, - summary: script.summary, - schema: script.schema as Record | undefined +/** Infer a script draft's schema with the editor's own parsers, into the schema its base + * carries, as the editor does at save: `inferArgs` re-seeds each arg from the properties it is + * handed, and those are the only copy of `password`, enums, formats and titles. A clone, so a + * parse failure leaves the previous schema rather than half of one. */ +async function inferScriptDraftSchema(draft: NewScript): Promise { + try { + const schema = structuredClone( + draft.schema?.properties ? draft.schema : emptySchema() + ) as Schema + await inferArgs(draft.language, draft.content, schema) + draft.schema = schema + } catch (e) { + console.error('Failed to infer script schema before saving draft', e) } } @@ -6616,224 +5158,10 @@ async function schemaForTestRun(script: { return schema as unknown as Record } -async function editScript( - args: { path: string; old_string: string; new_string: string; replace_all: boolean }, - ctx: WriteDraftCtx -): Promise { - const { path, old_string: oldString, new_string: newString, replace_all: replaceAll } = args - ctx.toolCallbacks.setToolStatus(ctx.toolId, { content: `Editing script "${path}"...` }) - - const base = await loadScriptForEdit(path, ctx.workspace) - const updated = findAndReplace(base.content, oldString, newString, replaceAll, 'script source') - return writeScriptDraft( - { - path, - summary: base.summary, - language: base.language, - content: updated - }, - ctx, - { before: base.content, after: updated, lang: scriptLangToEditorLang(base.language) } - ) -} - -async function loadFlowDraftValue( - path: string, - workspace: string - // `isDraft` says which of the two this came from. Reported here because the draft lookup - // is a request of its own: a caller that needs to know would otherwise repeat it. -): Promise<{ flow: FlowDraftValue; summary?: string; isDraft: boolean }> { - const draft = await getGlobalDraft(workspace, 'flow', path) - if (draft) { - if (draft.value === undefined || typeof draft.value === 'string') { - throw new Error(`Draft flow "${path}" has no value.`) - } - return { flow: draft.value as FlowDraftValue, summary: draft.summary, isDraft: true } - } - const flow = await FlowService.getFlowByPath({ workspace, path }) - return { - flow: { value: flow.value, schema: flow.schema, groups: flow.value.groups ?? null }, - summary: flow.summary, - isDraft: false - } -} - -/** - * `write_flow` accepts `inline_script.` placeholders so the model can - * overwrite a flow without re-sending (or even reading) unchanged rawscript - * bodies. Resolve them against the current draft/deployed flow: an id with a - * stored body keeps it, a new module's own-id placeholder becomes an empty body - * (to fill via set_flow_module_code), and anything else rejects the write. - */ -async function resolveWriteFlowInlineScripts( - path: string, - editable: EditableFlowJson, - workspace: string -): Promise { - const specials = [editable.preprocessor_module, editable.failure_module].filter( - (module): module is FlowModule => module != null - ) - const hasPlaceholders = - findUnresolvedInlineScriptRefs(editable.modules).length > 0 || - findUnresolvedInlineScriptRefs(specials).length > 0 - if (!hasPlaceholders) { - return editable - } - - const session = createInlineScriptSession() - if ( - (await getGlobalDraft(workspace, 'flow', path)) || - (await FlowService.existsFlowByPath({ workspace, path })) - ) { - const base = await loadFlowDraftValue(path, workspace) - buildEditableFlowJson(flowDraftAsEditableInput(base.flow), session) - } - const resolved: EditableFlowJson = { - ...editable, - modules: session.restoreInlineScriptReferences(editable.modules), - preprocessor_module: restoreSpecialRawscriptModule(editable.preprocessor_module, session), - failure_module: restoreSpecialRawscriptModule(editable.failure_module, session) - } - finalizeUnresolvedInlineScripts(resolved) - return resolved -} - -async function patchFlowJson( - args: { path: string; old_string: string; new_string: string; replace_all: boolean }, - ctx: WriteDraftCtx -): Promise { - const { path, old_string: oldString, new_string: newString, replace_all: replaceAll } = args - ctx.toolCallbacks.setToolStatus(ctx.toolId, { content: `Patching flow "${path}"...` }) - - // Operate on the compact (placeholders-for-rawscript) view so inline script - // bodies don't appear in the JSON the model sees or patches. Real script - // content is preserved through the patch via the InlineScriptSession; the - // model uses set_flow_module_code to change inline script bodies. - const base = await loadFlowDraftValue(path, ctx.workspace) - const session = createInlineScriptSession() - const editable = buildEditableFlowJson(flowDraftAsEditableInput(base.flow), session) - const currentJson = JSON.stringify(editable) - const updatedJson = findAndReplace( - currentJson, - oldString, - newString, - replaceAll, - 'compact flow JSON' - ) - let parsedValue: unknown - try { - parsedValue = JSON.parse(updatedJson) - } catch (error) { - const message = error instanceof Error ? error.message : String(error) - throw new Error(`Invalid JSON after replacement: ${message}`) - } - - const aiProviders = await getAiAgentProviderCatalogFor( - ctx.workspace, - (parsedValue as { modules?: unknown } | null)?.modules - ) - const aiProviderWarnings: string[] = [] - const patchedEditable = validateEditableFlowJson(parsedValue, { aiProviders, aiProviderWarnings }) - const newFlowValue = applyEditableFlowJsonToFlow(base.flow.value, patchedEditable, session) - finalizeUnresolvedInlineScripts(newFlowValue) - - const result = await writeFlowDraft( - { - path, - summary: base.summary, - flow: { - ...base.flow, - value: newFlowValue, - schema: patchedEditable.schema, - groups: patchedEditable.groups - } - }, - ctx - ) - // Warn from the restored value, not patchedEditable — the compact view holds - // placeholders for every rawscript, so only post-restore content shows which - // modules still need bodies filled via set_flow_module_code. - return ( - appendEmptyInlineScriptWarning(result, { - ...patchedEditable, - modules: newFlowValue.modules, - preprocessor_module: newFlowValue.preprocessor_module ?? null, - failure_module: newFlowValue.failure_module ?? null - }) + formatAiAgentProviderWarnings(aiProviderWarnings) - ) -} - -async function readFlowModuleCode( - args: { path: string; module_id: string }, - ctx: WriteDraftCtx -): Promise { - const { workspace, toolId, toolCallbacks } = ctx - toolCallbacks.setToolStatus(toolId, { - content: `Reading inline script for module "${args.module_id}" from flow "${args.path}"...` - }) - const base = await loadFlowDraftValue(args.path, workspace) - const session = createInlineScriptSession() - buildEditableFlowJson(flowDraftAsEditableInput(base.flow), session) - const content = session.get(args.module_id) - if (content === undefined) { - throw new Error( - `Module "${args.module_id}" is not an inline rawscript in flow "${args.path}". (Path runnables, branches, and loops have no inline script content; use patch_flow_json to inspect them.)` - ) - } - toolCallbacks.setToolStatus(toolId, { - content: `Read code of step ${flowStepName(base.flow.value, args.module_id)}` - }) - return content -} - -async function setFlowModuleCode( - args: { path: string; module_id: string; code: string }, - ctx: WriteDraftCtx -): Promise { - const { workspace, toolId, toolCallbacks } = ctx - toolCallbacks.setToolStatus(toolId, { - content: `Updating inline script for module "${args.module_id}" in flow "${args.path}"...` - }) - const base = await loadFlowDraftValue(args.path, workspace) - const session = createInlineScriptSession() - const editable = buildEditableFlowJson(flowDraftAsEditableInput(base.flow), session) - if (!session.has(args.module_id)) { - throw new Error( - `Module "${args.module_id}" is not an inline rawscript in flow "${args.path}". Use patch_flow_json or write_flow for structural changes.` - ) - } - session.set(args.module_id, args.code) - const newFlowValue = applyEditableFlowJsonToFlow(base.flow.value, editable, session) - const result = await writeFlowDraft( - { - path: args.path, - summary: base.summary, - flow: { ...base.flow, value: newFlowValue } - }, - ctx - ) - // Several code edits of one flow read as identical rows under the generic flow label. - if (JSON.parse(result).success) { - toolCallbacks.setToolStatus(toolId, { - content: `Updated code of step ${flowStepName(base.flow.value, args.module_id)}` - }) - } - return result -} - -function flowStepName(flow: FlowValue, moduleId: string): string { - const summary = findModuleInFlow(flow, moduleId)?.summary - return summary ? `${moduleId} "${summary}"` : moduleId -} - function normalizeTestRunArgs(args: Record | null | undefined): Record { return args ?? {} } -function flowDraftValueForPreview(flowDraft: FlowDraftValue): FlowValue { - return flowDraftAsEditableInput(flowDraft).value -} - /** * The conversation a test run of a chat-enabled flow belongs to. The server refuses such a * run without one, and it is a query parameter rather than a flow argument, so there is no @@ -6844,47 +5172,6 @@ export function chatMemoryId(value: FlowValue): string | undefined { return value.chat_input_enabled ? randomUUID() : undefined } -async function loadScriptForFlowStep( - moduleValue: { path: string; hash?: string }, - workspace: string -): Promise<{ content: string; language: ScriptLang; schema?: Record }> { - const draft = await getGlobalDraft(workspace, 'script', moduleValue.path) - if (draft) { - if (typeof draft.value !== 'string' || !draft.language) { - throw new Error(`Draft script "${moduleValue.path}" is missing content or language.`) - } - return { - content: draft.value, - language: draft.language, - // The draft's own: no parser emits `password`, so a schema rebuilt from the content - // would offer a secret argument as a plain field and take the literal into the job. - schema: draft.schema as Record | undefined - } - } - - const script = moduleValue.hash - ? await ScriptService.getScriptByHash({ workspace, hash: moduleValue.hash }) - : await ScriptService.getScriptByPath({ workspace, path: moduleValue.path }) - return { - content: script.content, - language: script.language, - schema: script.schema as Record | undefined - } -} - -async function loadSubflowForFlowStep( - path: string, - workspace: string -): Promise<{ previewValue?: FlowValue; schema?: Record }> { - const nestedFlow = await loadFlowDraftValue(path, workspace) - return { - // Only a draft is previewed; a deployed subflow is run by path, as its parent flow - // would run it. The schema describes whichever of the two that leaves. - previewValue: nestedFlow.isDraft ? flowDraftValueForPreview(nestedFlow.flow) : undefined, - schema: nestedFlow.flow.schema ?? undefined - } -} - /** * The run form for an agent step: the rows the flow editor's own step test shows, opened on the * values it opens them with, built from that form's helpers so the two cannot drift. @@ -6955,45 +5242,82 @@ function pathLeaf(path: unknown, fallback: string): string { return p.split('/').filter(Boolean).pop() || p || fallback } +/** What a run tool runs and what its form shows for it, as the server resolved it. */ +type PreparedRun = { + /** Handed back to `startPreparedRun`, so the run is the version the form was shown for. */ + target: unknown + schema: Record | null + code: string | null + lang: ScriptLang | null + summary: string | null + runnable_kind: 'script' | 'flow' + start_message: string + conversational: boolean + module: FlowModule | null +} + +async function prepareRun( + name: string, + ctx: WriteDraftCtx & { args: unknown } +): Promise { + const response = await callSessionTool(ctx.workspace, name, ctx.args, { + run: { phase: 'prepare' } + }) + applyStatusEffects(response.effects, ctx) + if (response.is_error) throw new Error(response.text) + return JSON.parse(response.text) +} + +/** Queue a run on the server and answer its job id. `target` absent resolves it there. */ +async function startPreparedRun( + name: string, + ctx: WriteDraftCtx & { args: unknown }, + target: unknown, + args: Record, + formKeys?: readonly string[] +): Promise { + const response = await callSessionTool(ctx.workspace, name, ctx.args, { + run: { phase: 'start', target, args, form_keys: formKeys ?? [] } + }) + if (response.is_error) throw new Error(response.text) + return response.text +} + async function testRunScriptByPath( args: z.infer, - ctx: WriteDraftCtx + ctx: WriteDraftCtx & { args: unknown } ): Promise { - const { workspace } = ctx - const script = await loadScriptForEdit(args.path, workspace) - const schema = await schemaForTestRun(script) + const prepared = await prepareRun('test_run_script', ctx) + const code = prepared.code ?? '' + const lang = prepared.lang! + const schema = await schemaForTestRun({ + content: code, + language: lang, + schema: prepared.schema ?? undefined + }) return runThroughForm( { path: args.path, schema, - summary: script.summary, + summary: prepared.summary ?? undefined, kind: 'test', - code: script.content, - lang: script.language, + code, + lang, // Never "deployed" here: the code about to run is the draft the model is still // writing, and a line telling it to re-read the deployed schema would send it // to the wrong version. schemaNoun: 'script', toolName: 'test_run_script', proposed: args.args, - startMessage: `Running test for script "${args.path}"...`, + startMessage: prepared.start_message, contextName: 'script', // Its own loop: the model is told to test and iterate, so the posture answers the // form with what it opened with rather than parking the loop on a card. autoAcceptable: true, background: args.background, detachAfterMs: waitSecondsToDetachMs(args.wait_seconds), - startJob: (submitted) => - JobService.runScriptPreview({ - workspace, - requestBody: { - path: args.path, - content: script.content, - args: submitted, - language: script.language - } - }) + startJob: (submitted) => startPreparedRun('test_run_script', ctx, prepared.target, submitted) }, ctx ) @@ -7316,38 +5640,27 @@ async function runThroughForm(spec: FormRunSpec, ctx: WriteDraftCtx): Promise, - ctx: WriteDraftCtx + ctx: WriteDraftCtx & { args: unknown } ): Promise { - const { workspace } = ctx - // No getDraft: this runs the script as it is live, so the form has to offer the - // inputs the live version accepts and not a draft's. - const script = await ScriptService.getScriptByPath({ workspace, path: args.path }) + // The deployed version, not a draft: the form has to offer the inputs the live version accepts. + const prepared = await prepareRun('run_script', ctx) return runThroughForm( { path: args.path, - schema: (script.schema as Record) ?? {}, - summary: script.summary, + schema: prepared.schema ?? {}, + summary: prepared.summary ?? undefined, kind: 'run', schemaNoun: 'deployed', toolName: 'run_script', proposed: args.args, - startMessage: `Running "${args.path}"...`, + startMessage: prepared.start_message, contextName: 'script', // Bypassable like a test run: the posture is the user's standing answer, and a form // it parks on is a card nobody is watching. autoAcceptable: true, background: args.background, detachAfterMs: waitSecondsToDetachMs(args.wait_seconds), - startJob: (submitted) => - JobService.runScriptByPath({ - workspace, - path: args.path, - requestBody: submitted, - // As the script's own run page does: the form fills the main input schema, and a - // preprocessor would take these arguments for a webhook body and hand the script - // its own output instead. - skipPreprocessor: true - }) + startJob: (submitted) => startPreparedRun('run_script', ctx, prepared.target, submitted) }, ctx ) @@ -7355,44 +5668,29 @@ async function runDeployedScript( async function runDeployedFlow( args: z.infer, - ctx: WriteDraftCtx + ctx: WriteDraftCtx & { args: unknown } ): Promise { - const { workspace } = ctx // No live editor is driven here as a test run drives one: that editor holds the draft, and a // deployed run painted into its graph would show steps that are not the ones running. - const flow = await FlowService.getFlowByPath({ workspace, path: args.path }) + const prepared = await prepareRun('run_flow', ctx) return runThroughForm( { path: args.path, - schema: (flow.schema as Record) ?? {}, - summary: flow.summary, + schema: prepared.schema ?? {}, + summary: prepared.summary ?? undefined, kind: 'run', // Its deployed page is a conversation with no run form, so the card keeps this run. - conversational: flow.value?.chat_input_enabled ?? false, + conversational: prepared.conversational, // No code/lang: the dynamic-option pickers come from the deployed flow, not an inline copy. schemaNoun: 'deployed', toolName: 'run_flow', proposed: args.args, - startMessage: `Running "${args.path}"...`, + startMessage: prepared.start_message, contextName: 'flow', autoAcceptable: true, background: args.background, detachAfterMs: waitSecondsToDetachMs(args.wait_seconds), - startJob: (submitted) => - JobService.runFlowByPath({ - workspace, - path: args.path, - requestBody: submitted, - // A chat-enabled flow is refused without one, and it names a conversation rather - // than being a flow argument, so nothing in `args` could carry it. A fresh id: - // this run is its own conversation on the flow's page, not a turn appended to - // one someone is reading there. - memoryId: chatMemoryId(flow.value), - // As the flow's own run page does: the form fills the main input schema, and a - // preprocessor would take these arguments for a webhook body and hand the flow - // its own output instead. - skipPreprocessor: true - }) + startJob: (submitted) => startPreparedRun('run_flow', ctx, prepared.target, submitted) }, ctx ) @@ -7400,20 +5698,18 @@ async function runDeployedFlow( async function testRunFlowByPath( args: z.infer, - ctx: WriteDraftCtx + ctx: WriteDraftCtx & { args: unknown } ): Promise { - const { workspace } = ctx - // The schema must be in hand before the form is built, and the value rides along from the - // same read so the fields and the previewed flow are one version. With an editor open on - // this path this reads its in-memory cell rather than the network. - const flow = await loadFlowDraftValue(args.path, workspace) - const schema = (flow.flow.schema as Record | null | undefined) ?? {} + // The schema must be in hand before the form is built, and the value rides along in the same + // target so the fields and the previewed flow are one version. + const prepared = await prepareRun('test_run_flow', ctx) + const schema = prepared.schema ?? {} return runThroughForm( { path: args.path, schema, - summary: flow.summary, + summary: prepared.summary ?? undefined, kind: 'test', // A flow's dynamic-option pickers are one script stored on the schema itself, which is // where the editor's own test form reads them from (FlowPreviewContent). @@ -7424,7 +5720,7 @@ async function testRunFlowByPath( schemaNoun: 'flow', toolName: 'test_run_flow', proposed: args.args, - startMessage: `Starting flow test run for "${args.path}"...`, + startMessage: prepared.start_message, contextName: 'flow', // The model is told to test and iterate, so the bypass posture answers the form. autoAcceptable: true, @@ -7438,79 +5734,38 @@ async function testRunFlowByPath( if (jobId) { return jobId } - const value = flowDraftValueForPreview(flow.flow) - return JobService.runFlowPreview({ - workspace, - memoryId: args.memory_id ?? chatMemoryId(value), - requestBody: { - path: args.path, - value, - args: submitted - } - }) + return startPreparedRun('test_run_flow', ctx, prepared.target, submitted) } }, ctx ) } -/** The agent at `path` as a run of it would use it: the user's draft when there is one, else the - * deployed resource. Refuses a path holding some other kind of resource. */ -async function loadAgentForTest( - path: string, - workspace: string -): Promise<{ config: AIAgentConfig; description?: string }> { - // Settles a save the agent editor still has pending, or refuses: a test of the stored row would - // otherwise run an older configuration than the one on screen. - const { response, draft } = await fetchAgentWithDraft(path, workspace) - if (response.resource_type !== 'ai_agent') { - throw new Error(`"${path}" is a ${response.resource_type} resource, not an AI agent.`) - } - return draft - ? { config: draft.args, description: draft.description } - : { config: response.value as AIAgentConfig, description: response.description } -} - async function testRunAgentByPath( args: z.infer, - ctx: WriteDraftCtx + ctx: WriteDraftCtx & { args: unknown } ): Promise { - const { workspace } = ctx - const { config, description } = await loadAgentForTest(args.path, workspace) - // Who the agent runs as and who may ask it in Slack, not inputs: a test runs as its editor. - const { - on_behalf_of: _obo, - run_on_behalf_of: _runObo, - slack_anyone: _slack, - ...brain - } = config as AIAgentConfig & Record - // The run the agent editor's chat makes of its draft. - const flow = agentChatFlow({ - id: AGENT_STEP_ID, - value: inlineAgentDraft({ type: 'aiagent', tools: [], input_transforms: {} } as any, brain) - } as FlowModule) + // The server reads the stored draft: an edit the agent editor still has to save would test an + // older configuration than the one on screen. + await settleAgentDraft(args.path, ctx.workspace) + const prepared = await prepareRun('test_run_agent', ctx) return runThroughForm( { path: args.path, - schema: AGENT_CHAT_SCHEMA, - summary: description, + schema: prepared.schema ?? {}, + summary: prepared.summary ?? undefined, kind: 'test', schemaNoun: 'agent', toolName: 'test_run_agent', proposed: args.args, - startMessage: `Starting agent test run for "${args.path}"...`, + startMessage: prepared.start_message, contextName: 'flow', noun: 'agent', cardNoun: 'agent', autoAcceptable: true, background: args.background, detachAfterMs: waitSecondsToDetachMs(args.wait_seconds), - startJob: (submitted) => - JobService.runFlowPreview({ - workspace, - memoryId: args.memory_id ?? randomUUID(), - requestBody: { path: agentChatPath(args.path), value: flow.value, args: submitted } - }) + startJob: (submitted) => startPreparedRun('test_run_agent', ctx, prepared.target, submitted) }, ctx ) @@ -7518,40 +5773,30 @@ async function testRunAgentByPath( async function runDeployedAgent( args: z.infer, - ctx: WriteDraftCtx + ctx: WriteDraftCtx & { args: unknown } ): Promise { - const { workspace } = ctx - const agent = await ResourceService.getResource({ workspace, path: args.path }) - if (agent.resource_type !== 'ai_agent') { - throw new Error(`"${args.path}" is a ${agent.resource_type} resource, not an AI agent.`) - } + const prepared = await prepareRun('run_agent', ctx) return runThroughForm( { path: args.path, - schema: AGENT_CHAT_SCHEMA, - summary: agent.description, + schema: prepared.schema ?? {}, + summary: prepared.summary ?? undefined, kind: 'run', // Its page is the agent's rather than a flow's, which is where the form moves. A turn of a // conversation stays on the card: the page's run starts one of its own, without the memory id. agentPage: true, - conversational: args.memory_id !== undefined, + conversational: prepared.conversational, schemaNoun: 'agent', toolName: 'run_agent', proposed: args.args, - startMessage: `Running agent "${args.path}"...`, + startMessage: prepared.start_message, contextName: 'flow', noun: 'agent', cardNoun: 'agent', autoAcceptable: true, background: args.background, detachAfterMs: waitSecondsToDetachMs(args.wait_seconds), - startJob: (submitted) => - JobService.runAgent({ - workspace, - path: args.path, - memoryId: args.memory_id, - requestBody: submitted as { user_message?: string } - }) + startJob: (submitted) => startPreparedRun('run_agent', ctx, prepared.target, submitted) }, ctx ) @@ -7559,44 +5804,36 @@ async function runDeployedAgent( async function testRunFlowStepByPath( args: z.infer, - ctx: WriteDraftCtx + ctx: WriteDraftCtx & { args: unknown } ): Promise { - const { workspace, toolId, toolCallbacks } = ctx - const flow = await loadFlowDraftValue(args.path, workspace) - const resolved = await resolveFlowStepRun({ - flowValue: flowDraftValueForPreview(flow.flow), - stepId: args.stepId, - workspace, - toolCallbacks, - toolId, - loadScript: loadScriptForFlowStep, - loadSubflow: loadSubflowForFlowStep, - flowPath: args.path, - withAgentDrafts: (value) => withAgentDrafts(value, workspace) - }) + const { workspace } = ctx + const prepared = await prepareRun('test_run_step', ctx) + const module = prepared.module! // The module resolution landed on, not the id that was asked for: the job's entrypoint // override reads the same value, and a form built for the other entrypoint offers fields // the run will not take. - const isPreprocessor = resolved.module.id === SPECIAL_MODULE_IDS.PREPROCESSOR + const isPreprocessor = module.id === SPECIAL_MODULE_IDS.PREPROCESSOR // The step's own inputs, never the flow's: a step is fed by its input transforms, so the // flow's schema names arguments this job would ignore and omits the ones it takes. const agentForm = - resolved.module.value.type === 'aiagent' - ? await agentStepRunForm(resolved.module, workspace, args.args) + module.value.type === 'aiagent' + ? await agentStepRunForm(module, workspace, args.args) : undefined const schema = agentForm?.schema ?? - (resolved.code != undefined && resolved.lang + (prepared.code != undefined && prepared.lang ? await schemaForTestRun({ - content: resolved.code, - language: resolved.lang, - schema: resolved.schema, + content: prepared.code, + language: prepared.lang, + schema: prepared.schema ?? undefined, entrypoint: isPreprocessor ? 'preprocessor' : undefined }) - : (resolved.schema ?? {})) + : module.value.type === 'aidecision' + ? decisionStepRunSchema() + : (prepared.schema ?? {})) - const stepSummary = resolved.module.summary + const stepSummary = module.summary return runThroughForm( { // The flow: a step has no path of its own, and this is what the status and cancel @@ -7606,235 +5843,45 @@ async function testRunFlowStepByPath( schema, summary: stepSummary ? `step "${args.stepId}": ${stepSummary}` : `step "${args.stepId}"`, kind: 'test', - code: resolved.code ?? schema['x-windmill-dyn-select-code'], - lang: resolved.lang ?? schema['x-windmill-dyn-select-lang'], + code: prepared.code ?? schema['x-windmill-dyn-select-code'], + lang: prepared.lang ?? schema['x-windmill-dyn-select-lang'], // Never "deployed": a step test previews the draft flow, and the step's target may // itself be a draft. schemaNoun: 'step', toolName: 'test_run_step', proposed: agentForm?.args ?? args.args, - startMessage: resolved.startMessage, - contextName: resolved.runnableKind, + startMessage: prepared.start_message, + contextName: prepared.runnable_kind, noun: 'step', label: `step ${args.stepId}`, // The model is told to test and iterate, so the bypass posture answers the form. autoAcceptable: true, background: args.background, detachAfterMs: waitSecondsToDetachMs(args.wait_seconds), - startJob: agentForm - ? (submitted) => resolved.startJob(submitted, agentForm.schema.order) - : resolved.startJob + startJob: async (submitted) => { + // The server merges each linked agent's stored draft into the step; an edit still + // parked in the browser, or one that failed to save, would leave it running the + // older configuration. + await loadLinkedAgentDrafts(linkedAgentPaths({ modules: [module] } as FlowValue), workspace) + return startPreparedRun( + 'test_run_step', + ctx, + prepared.target, + submitted, + agentForm?.schema.order + ) + } }, ctx ) } -/** The `data.tables` entry for a table named without its data table (`schema:table`, - * `schema.table` or `table`), which models write for a table of the app's own data table. - * A bare name is a table of the app's own schema. */ -function appTableRef(datatable: string, schema: string | undefined, ref: string): string { - if (ref.includes('/')) return ref - const sep = ref.includes(':') ? ':' : '.' - const at = ref.indexOf(sep) - return formatDataTableRef( - at === -1 - ? { datatable, schema, table: ref } - : { datatable, schema: ref.slice(0, at), table: ref.slice(at + 1) } - ) -} - -async function initApp( - args: { - path: string - summary?: string - framework: FrameworkKey - data?: { datatable: string; schema?: string; tables?: string[] } - }, - ctx: WriteDraftCtx -): Promise { - const { workspace, toolId, toolCallbacks } = ctx - const { path, summary, framework, data } = args - - if (await getGlobalDraft(workspace, 'app', path)) { - throw new Error( - `A draft for app "${path}" already exists. Use write_app_file / write_app_runnable to modify it, or delete the existing draft first.` - ) - } - if (await AppService.existsApp({ workspace, path })) { - throw new Error( - `An app already exists at "${path}". Use read_workspace_item + write_app_file / write_app_runnable to modify it.` - ) - } - - toolCallbacks.setToolStatus(toolId, { - content: `Saving app "${path}" draft (${framework} template)…` - }) - - // A builder can save no inline runnable, so they start from a static page instead of the tour. - const builder = capabilitiesInWorkspace(workspace).apps === 'compose' - const value: AppDraftValue = { - summary, - files: { ...(builder ? BUILDER_FRAMEWORK_TEMPLATES : FRAMEWORK_TEMPLATES)[framework] }, - runnables: builder ? {} : structuredClone(STARTER_RUNNABLES), - policy: { sandbox: true }, - ...(data && { - data: { - tables: (data.tables ?? []).map((ref) => appTableRef(data.datatable, data.schema, ref)), - datatable: data.datatable, - schema: data.schema - } - }) - } - await recomputeAppPolicy(value) - const result = await saveAppDraft(workspace, path, value) - return finishAppDraftWrite(result, ctx, () => ({ - content: `Saved app "${path}" draft (${framework})`, - message: `Initialized a per-user draft app "${path}" from the ${framework} template (saved server-side, not a deployed workspace item). ${builder ? 'The template is a static page with no runnables: replace its UI, and add only deployed scripts and flows, by path.' : `The template is a demo tour with starter runnables ${Object.keys(STARTER_RUNNABLES).join(', ')}: replace its UI and delete the runnables the app does not need (delete_app_runnable).`} Use write_app_file / write_app_runnable to evolve it.${data?.schema ? ` Its data config names schema "${data.schema}" of data table "${data.datatable}" without creating it: if it does not exist yet, create it before the tables that go in it.` : ''}` - })) -} - -// read_app_file caps: a large frontend file or inline runnable would otherwise -// enter context in full and persist for the rest of the session. Default to a -// head slice with a pointer to page further; the model widens with offset/limit. -// The char budget is a hard ceiling on a single read: a selected line window over -// it is truncated and the model is told to narrow the line range. There is no -// char-level paging, so a single line longer than the budget can't be read past — -// add paging here if minified/long-line files must be fully readable. -const READ_APP_FILE_DEFAULT_LINE_LIMIT = 1500 -const READ_APP_FILE_CHAR_BUDGET = 50_000 - -type AppFileSlice = { - body: string - startLine: number - endLine: number - requestedStartLine: number - totalLines: number - lineWindowChars: number - charTruncated: boolean - truncated: boolean -} - -function sliceAppFileForRead(content: string, offset?: number, limit?: number): AppFileSlice { - const lines = content.split('\n') - const totalLines = lines.length - const requestedStartLine = offset ?? 1 - const start = Math.min(Math.max(requestedStartLine - 1, 0), totalLines) - const lineLimit = limit ?? READ_APP_FILE_DEFAULT_LINE_LIMIT - const end = Math.min(start + lineLimit, totalLines) - const selectedBody = lines.slice(start, end).join('\n') - const lineWindowChars = selectedBody.length - const body = selectedBody.slice(0, READ_APP_FILE_CHAR_BUDGET) - const charTruncated = lineWindowChars > READ_APP_FILE_CHAR_BUDGET - - return { - body, - startLine: start + 1, - endLine: end, - requestedStartLine, - totalLines, - lineWindowChars, - charTruncated, - truncated: start > 0 || end < totalLines || charTruncated - } -} - -function formatAppFileReadRangeLabel(slice: AppFileSlice): string { - const lineRange = `lines ${slice.startLine}-${slice.endLine} of ${slice.totalLines}` - if (!slice.charTruncated) { - return lineRange - } - return `${lineRange}, truncated to the first ${READ_APP_FILE_CHAR_BUDGET} of ${slice.lineWindowChars} chars` -} - -// Small files (returned whole, starting at line 1) keep the raw body so -// patch_app_file's exact-match stays trivial; truncated/windowed reads get a -// one-line annotation describing the range (not part of the file). -function formatAppFileReadResult(slice: AppFileSlice): string { - // offset past the last line: report it plainly instead of a backwards range. - if (slice.requestedStartLine > slice.totalLines) { - return `offset ${slice.requestedStartLine} is past the end of the file (${slice.totalLines} lines).` - } - if (!slice.truncated && slice.startLine === 1) { - return slice.body - } - - let more = '' - if (slice.charTruncated) { - more = ` Reached the ${READ_APP_FILE_CHAR_BUDGET}-char limit; re-read with a smaller limit (fewer lines). If a single line exceeds the limit the file is likely minified and not readable this way.` - } else if (slice.endLine < slice.totalLines) { - more = ` Call read_app_file again with offset=${slice.endLine + 1} to continue.` - } - // No tool-name/path prefix: the model already has them from the call args. The - // range line orients it; the body follows after a blank line. - return `${formatAppFileReadRangeLabel(slice)}.${more}\n\n${slice.body}` -} - -async function readAppFile( - args: { - path: string - file_path: string - offset?: number - limit?: number - }, - ctx: WriteDraftCtx -): Promise { - const { workspace, toolId, toolCallbacks } = ctx - const target = resolveAppFileTarget(args.file_path) - toolCallbacks.setToolStatus(toolId, { - content: `Reading ${target.filePath} from app "${args.path}"...` - }) - - const value = await loadAppValueForRead(args.path, workspace) - - let content: string - if (target.kind === 'frontend') { - const frontend = value.files[target.filePath] - if (frontend === undefined) { - throw new Error(`Frontend file "${target.filePath}" not found in app "${args.path}".`) - } - content = frontend - } else { - content = getInlineRunnableContent(value, target, args.path).content - } - - const slice = sliceAppFileForRead(content, args.offset, args.limit) - - toolCallbacks.setToolStatus(toolId, { content: `Read ${target.filePath}` }) - return formatAppFileReadResult(slice) -} - -// search_app caps: a single query must stay sparse and cheap even when it hits a -// minified bundle or a 5k-line data module. Per-line and total-output caps bound -// the result the same way read_app_file's char budget bounds one file read; the -// match cap keeps a broad query from flooding context instead of locating it. -const SEARCH_APP_DEFAULT_MAX_MATCHES = 100 -const SEARCH_APP_MAX_MATCHES_CEILING = 200 const SEARCH_APP_MAX_LINE_CHARS = 200 const SEARCH_APP_TOTAL_CHAR_BUDGET = 12_000 // Fixed surrounding-context window per match, kept off the tool schema to keep it // lean. Bump if matches need more context than the line ± this. const SEARCH_APP_CONTEXT_LINES = 2 -type AppSearchableFile = { filePath: string; content: string } - -// The files search_app scans: frontend files (minus generated ones) plus inline -// backend runnables, each addressed exactly as read_app_file expects so a match -// row's path can be passed straight back to read_app_file. -function collectSearchableAppFiles(value: AppDraftValue): AppSearchableFile[] { - const files: AppSearchableFile[] = [] - for (const [filePath, content] of Object.entries(value.files)) { - if (GENERATED_APP_FILE_PATHS.has(filePath)) continue - if (typeof content === 'string') files.push({ filePath, content }) - } - for (const [key, runnable] of Object.entries(value.runnables)) { - const persisted = runnable as PersistedRunnable | undefined - const content = persisted?.inlineScript?.content - if (typeof content !== 'string') continue - files.push({ filePath: `backend/${key}/main.${getInlineScriptExtension(persisted)}`, content }) - } - return files -} - // Minimal glob: * = any chars except '/', ** = any chars, ? = single non-slash. // A pattern without '/' matches the file name only (ripgrep-style), so "*.tsx" // finds nested files; a pattern with '/' matches the full path. @@ -7854,270 +5901,6 @@ function appFileMatchesGlob(filePath: string, glob: string): boolean { } } -type AppSearchMatch = { filePath: string; line: number; text: string } - -async function searchApp( - args: { - path: string - query: string - file_glob?: string - max_matches?: number - }, - ctx: WriteDraftCtx -): Promise { - const { workspace, toolId, toolCallbacks } = ctx - const query = args.query - if (query.length === 0) { - throw new Error('search_app requires a non-empty query.') - } - toolCallbacks.setToolStatus(toolId, { - content: `Searching app "${args.path}" for "${query}"...` - }) - - const value = await loadAppValueForRead(args.path, workspace) - const maxMatches = Math.min( - args.max_matches ?? SEARCH_APP_DEFAULT_MAX_MATCHES, - SEARCH_APP_MAX_MATCHES_CEILING - ) - const contextLines = SEARCH_APP_CONTEXT_LINES - const needle = query.toLowerCase() - - let files = collectSearchableAppFiles(value).sort((a, b) => a.filePath.localeCompare(b.filePath)) - if (args.file_glob) { - files = files.filter((f) => appFileMatchesGlob(f.filePath, args.file_glob as string)) - } - - const matches: AppSearchMatch[] = [] - let totalMatchCount = 0 - // Cap on matching LINES, not pushed rows — each match expands to its context - // window, so counting rows would make `max_matches`/"showing the first N" wrong. - let renderedMatchCount = 0 - let fileCount = 0 - let truncated = false - for (const file of files) { - const lines = file.content.split('\n') - let fileHadMatch = false - for (let i = 0; i < lines.length; i++) { - if (!lines[i].toLowerCase().includes(needle)) continue - totalMatchCount++ - // Count the file on its first match, before the render cap, so the - // "N matches in M files" header counts every file with the symbol — not - // only the ones whose matches landed in the rendered slice (find-all-usages). - fileHadMatch = true - if (renderedMatchCount >= maxMatches) { - truncated = true - continue - } - renderedMatchCount++ - const lo = Math.max(0, i - contextLines) - const hi = Math.min(lines.length - 1, i + contextLines) - for (let j = lo; j <= hi; j++) { - matches.push({ filePath: file.filePath, line: j + 1, text: lines[j] }) - } - } - if (fileHadMatch) fileCount++ - } - - if (totalMatchCount === 0) { - toolCallbacks.setToolStatus(toolId, { content: `No matches for "${query}"` }) - return `No matches. Try a broader or differently-spelled term${ - args.file_glob ? ', or drop the file_glob' : '' - }.` - } - - // No tool-name/query prefix: the model already has them from the call args. - const header = `${totalMatchCount} match${ - totalMatchCount === 1 ? '' : 'es' - } in ${fileCount} file${fileCount === 1 ? '' : 's'}${ - truncated - ? ` (showing the first ${maxMatches}; narrow with file_glob or a more specific query)` - : '' - }` - - const out: string[] = [header] - let currentFile = '' - let budgetSpent = header.length - let budgetHit = false - const seen = new Set() - for (const m of matches) { - // context windows of adjacent matches overlap — show each source line once. - const dedupeKey = `${m.filePath}:${m.line}` - if (seen.has(dedupeKey)) continue - seen.add(dedupeKey) - const text = - m.text.length > SEARCH_APP_MAX_LINE_CHARS - ? `${m.text.slice(0, SEARCH_APP_MAX_LINE_CHARS)}… [line truncated]` - : m.text - const fileHeader = m.filePath === currentFile ? '' : `${m.filePath}\n` - const row = `${fileHeader} ${m.line}: ${text}` - if (budgetSpent + row.length + 1 > SEARCH_APP_TOTAL_CHAR_BUDGET) { - budgetHit = true - break - } - if (fileHeader) currentFile = m.filePath - out.push(row) - budgetSpent += row.length + 1 - } - if (budgetHit) { - out.push( - `… output truncated at the context budget — narrow with file_glob or a more specific query.` - ) - } - - toolCallbacks.setToolStatus(toolId, { - content: `Found ${totalMatchCount} match${totalMatchCount === 1 ? '' : 'es'} in ${fileCount} file${ - fileCount === 1 ? '' : 's' - }` - }) - return out.join('\n') -} - -async function writeAppFile( - args: { path: string; file_path: string; content: string }, - ctx: WriteDraftCtx -): Promise { - const { workspace, toolId, toolCallbacks } = ctx - const target = resolveAppFileTarget(args.file_path) - if (target.kind !== 'frontend') { - throw new Error( - `write_app_file only writes frontend files. Use write_app_runnable to set inline backend script content.` - ) - } - assertNotGeneratedAppFile(target.filePath) - - toolCallbacks.setToolStatus(toolId, { - content: `Writing ${target.filePath} to app "${args.path}"...` - }) - - const { value } = await loadAppDraftValue(args.path, workspace) - const before = value.files[target.filePath] ?? '' - value.files = { ...value.files, [target.filePath]: args.content } - const result = await saveAppDraft(workspace, args.path, value) - return finishAppDraftWrite( - result, - ctx, - () => ({ - content: `Updated ${target.filePath} in app "${args.path}"`, - message: `Updated draft app "${args.path}" with frontend file "${target.filePath}".` - }), - { before, after: args.content, lang: appFileEditorLang(target.filePath) } - ) -} - -async function deleteAppFile( - args: { path: string; file_path: string }, - ctx: WriteDraftCtx -): Promise { - const { workspace, toolId, toolCallbacks } = ctx - const target = resolveAppFileTarget(args.file_path) - if (target.kind !== 'frontend') { - throw new Error( - `delete_app_file only deletes frontend files. Use delete_app_runnable for backend runnables.` - ) - } - assertNotGeneratedAppFile(target.filePath) - - toolCallbacks.setToolStatus(toolId, { - content: `Deleting ${target.filePath} from app "${args.path}"...` - }) - - const { value } = await loadAppDraftValue(args.path, workspace) - if (!(target.filePath in value.files)) { - throw new Error(`Frontend file "${target.filePath}" not found in app "${args.path}".`) - } - const { [target.filePath]: removed, ...remaining } = value.files - value.files = remaining - const result = await saveAppDraft(workspace, args.path, value) - return finishAppDraftWrite( - result, - ctx, - () => ({ - content: `Removed ${target.filePath} from app "${args.path}"`, - message: `Removed "${target.filePath}" from draft app "${args.path}".` - }), - { before: removed, after: '', lang: appFileEditorLang(target.filePath) } - ) -} - -async function patchAppFile( - args: { - path: string - file_path: string - old_string: string - new_string: string - replace_all: boolean - }, - ctx: WriteDraftCtx -): Promise { - const { workspace, toolId, toolCallbacks } = ctx - const { - path, - file_path: filePath, - old_string: oldString, - new_string: newString, - replace_all: replaceAll - } = args - const target = resolveAppFileTarget(filePath) - if (target.kind === 'frontend') { - assertNotGeneratedAppFile(target.filePath) - } - - toolCallbacks.setToolStatus(toolId, { - content: `Patching ${target.filePath} in app "${path}"...` - }) - - const { value } = await loadAppDraftValue(path, workspace) - let currentContent: string - let runnable: PersistedRunnable | undefined - - if (target.kind === 'frontend') { - const existing = value.files[target.filePath] - if (existing === undefined) { - throw new Error(`Frontend file "${target.filePath}" not found in app "${path}".`) - } - currentContent = existing - } else { - const resolved = getInlineRunnableContent(value, target, path) - currentContent = resolved.content - runnable = resolved.runnable - } - - const updated = findAndReplace( - currentContent, - oldString, - newString, - replaceAll, - `${target.filePath} content` - ) - - if (target.kind === 'frontend') { - value.files = { ...value.files, [target.filePath]: updated } - } else { - value.runnables = { - ...value.runnables, - [target.key]: { - ...runnable!, - inlineScript: { - language: - runnable!.inlineScript?.language ?? (target.extension === 'py' ? 'python3' : 'bun'), - content: updated - } - } - } - } - - const result = await saveAppDraft(workspace, path, value) - return finishAppDraftWrite( - result, - ctx, - () => ({ - content: `Patched ${target.filePath} in app "${path}"`, - message: `Patched "${target.filePath}" in draft app "${path}".` - }), - { before: currentContent, after: updated, lang: appFileEditorLang(target.filePath) } - ) -} - async function recomputeAppPolicy(value: AppDraftValue): Promise { const policy = (await updateRawAppPolicy( value.runnables as any, @@ -8129,67 +5912,6 @@ async function recomputeAppPolicy(value: AppDraftValue): Promise { value.policy = policy } -async function writeAppRunnable( - args: { path: string; key: string; runnable: BackendRunnableInput }, - ctx: WriteDraftCtx -): Promise { - const { workspace, toolId, toolCallbacks } = ctx - const { path, key, runnable: input } = args - toolCallbacks.setToolStatus(toolId, { - content: `Writing runnable "${key}" to app "${path}"...` - }) - - const { value } = await loadAppDraftValue(path, workspace) - const existing = value.runnables[key] as PersistedRunnable | undefined - const persisted = buildPersistedRunnable(input, existing) - value.runnables = { ...value.runnables, [key]: persisted } - await recomputeAppPolicy(value) - const undeployed = await undeployedRunnableTargets(workspace, { [key]: persisted }) - const result = await saveAppDraft(workspace, path, value) - return finishAppDraftWrite( - result, - ctx, - () => ({ - content: `Updated runnable "${key}" in app "${path}"`, - message: `Updated draft app "${path}" with runnable "${key}".`, - warning: undeployed.length - ? `This runnable points at an item that is NOT deployed (${undeployed[0]}), so it fails at runtime — ` + - `a path runnable runs the deployed item, never a draft. Offer to deploy just that item with ` + - `deploy_workspace_item; the app itself does not need deploying, since the preview runs its draft.` - : undefined - }), - inlineRunnableDiff(key, existing, persisted) - ) -} - -function inlineRunnableCode(runnable: PersistedRunnable | undefined): string | undefined { - return runnable?.type === 'inline' || runnable?.type === 'runnableByName' - ? (runnable.inlineScript?.content ?? '') - : undefined -} - -// A runnable that references a workspace or hub item has no code of its own to diff. -function inlineRunnableDiff( - key: string, - before: PersistedRunnable | undefined, - after: PersistedRunnable | undefined -): ToolCodeDiff | undefined { - const beforeCode = inlineRunnableCode(before) - const afterCode = inlineRunnableCode(after) - if (beforeCode === undefined && afterCode === undefined) return undefined - if (after !== undefined && afterCode === undefined) return undefined - const lang = (runnable: PersistedRunnable | undefined) => - appFileEditorLang(`backend/${key}/main.${getInlineScriptExtension(runnable)}`) - const afterLang = lang(after ?? before) - const beforeLang = beforeCode === undefined ? afterLang : lang(before) - return { - before: beforeCode ?? '', - after: afterCode ?? '', - lang: afterLang, - ...(beforeLang !== afterLang ? { beforeLang } : {}) - } -} - /** * Runs one backend runnable through `execute_component` in preview mode, the way the editor * preview does: inline executes draft code, a path runnable the deployed item it names. @@ -8197,75 +5919,26 @@ function inlineRunnableDiff( */ async function testRunAppRunnable( args: z.infer, - ctx: WriteDraftCtx + ctx: WriteDraftCtx & { args: unknown } ): Promise { const { workspace, toolId, toolCallbacks } = ctx const { path, key } = args - - const { value } = await loadAppDraftValue(path, workspace) - const runnable = value.runnables?.[key] as PersistedRunnable | undefined - if (!runnable) { - const known = Object.keys(value.runnables ?? {}) - throw new Error( - `App "${path}" has no backend runnable "${key}".` + - (known.length ? ` Available runnables: ${known.join(', ')}.` : '') - ) - } - - // Copied, not aliased: the ctx pass below writes into it. - const testArgs = { ...normalizeTestRunArgs(args.args) } - // Setting force_viewer_static_fields is what puts execute_component in preview - // mode (apps.rs `is_preview`), which is what makes inline draft code run at all. - // It must be sent even when the runnable has no static fields. - const staticFields = Object.fromEntries( - Object.entries(runnable.fields ?? {}) - .filter(([, field]) => field?.type === 'static') - .map(([name, field]) => [name, field?.value]) - ) - // A ctx-bound input is filled by the server from `$ctx:`, exactly as - // RawAppBackgroundRunner does before executing. Without this the argument arrives - // missing and the runnable fails for a reason that has nothing to do with its code. - for (const [name, field] of Object.entries(runnable.fields ?? {})) { - if (field?.type === 'ctx' && field?.ctx) testArgs[name] = `$ctx:${field.ctx}` - } - // The server encrypts a queued argument only when its name is in this list - // (apps.rs wraps it as `$encrypted:` with a job-scoped key). Omitting it writes a - // `sensitive` field's real value into job args as plaintext, readable by anyone - // with run access. Same filter the editor preview applies. - const sensitiveInputs = Object.entries(runnable.fields ?? {}) - .filter(([, field]) => field?.type === 'user' && field?.sensitive) - .map(([name]) => name) - - // Imported lazily: statically pulling the apps module graph into the chat's - // import chain drags the whole app-editor runtime in behind it. - const { executeRunnable } = await import( - '$lib/components/apps/components/helpers/executeRunnable' - ) - + const prepared = await prepareRun('test_run_app_runnable', ctx) return executeTestRun({ jobStarter: () => - executeRunnable( - runnable as unknown as Runnable, - workspace, - undefined, - get(userStore)?.username, - path, - key, - { - component: key, - args: testArgs, - force_viewer_static_fields: staticFields, - force_viewer_sensitive_inputs: sensitiveInputs.length ? sensitiveInputs : undefined - }, - undefined + startPreparedRun( + 'test_run_app_runnable', + ctx, + prepared.target, + normalizeTestRunArgs(args.args) ), workspace, toolCallbacks, toolId, - startMessage: `Running backend runnable "${key}" of app "${path}"...`, + startMessage: prepared.start_message, // A path runnable pointing at a flow really does queue a flow job, so the // failure path can offer get_run's step tree; everything else is a script job. - contextName: runnable.runType === 'flow' ? 'flow' : 'script', + contextName: prepared.runnable_kind, completionName: 'backend runnable', background: args.background, detachAfterMs: waitSecondsToDetachMs(args.wait_seconds), @@ -8273,35 +5946,6 @@ async function testRunAppRunnable( }) } -async function deleteAppRunnable( - args: { path: string; key: string }, - ctx: WriteDraftCtx -): Promise { - const { workspace, toolId, toolCallbacks } = ctx - const { path, key } = args - toolCallbacks.setToolStatus(toolId, { - content: `Removing runnable "${key}" from app "${path}"...` - }) - - const { value } = await loadAppDraftValue(path, workspace) - if (!(key in value.runnables)) { - throw new Error(`Backend runnable "${key}" not found in app "${path}".`) - } - const { [key]: removed, ...remaining } = value.runnables - value.runnables = remaining - await recomputeAppPolicy(value) - const result = await saveAppDraft(workspace, path, value) - return finishAppDraftWrite( - result, - ctx, - () => ({ - content: `Removed runnable "${key}" from app "${path}"`, - message: `Removed runnable "${key}" from draft app "${path}".` - }), - inlineRunnableDiff(key, removed as PersistedRunnable, undefined) - ) -} - const triggerLabels: Record = { http: 'HTTP trigger', websocket: 'WebSocket trigger', @@ -8373,51 +6017,6 @@ function createOpenVariableAction(path: string): ToolDisplayAction { } } -async function discardLocalDraft( - args: { type: WorkspaceItemType; path: string; trigger_kind?: TriggerKind }, - ctx: WriteDraftCtx -): Promise { - const { workspace, toolId, toolCallbacks } = ctx - const { type, path, trigger_kind: triggerKind } = args - - if (type === 'trigger' && !triggerKind) { - throw new Error('trigger_kind is required when discarding a trigger draft.') - } - - const draft = await getGlobalDraft(workspace, type, path, triggerKind) - if (!draft) { - throw new Error(`No draft found for ${type} "${path}".`) - } - - await deleteGlobalDraft(workspace, type, path, triggerKind) - - // The chat's touch on the item is undone — drop it from the mask so a - // pre-existing deployed item doesn't keep reading as this chat's edit. - const discardedKind = itemKindFor(type, triggerKind) - if (discardedKind) { - toolCallbacks.onItemDiscarded?.( - discardedKind, - getGlobalDraftStoragePath(workspace, type, path, triggerKind) - ) - } - - toolCallbacks.setToolStatus(toolId, { - content: `Discarded ${type} "${path}" draft`, - result: 'Draft discarded' - }) - return JSON.stringify( - { - success: true, - message: `Discarded the draft for ${type} "${path}". The deployed workspace item was not changed.`, - type, - path, - triggerKind - }, - null, - 2 - ) -} - // A draft started from an older deploy would silently overwrite whatever was // deployed since. Block the deploy and point the model at rebase_draft, unless it // explicitly forces the overwrite. `base`/`head` undefined ⇒ can't tell ⇒ allow. @@ -8438,249 +6037,6 @@ function assertDraftBasedOnLatest( ) } -// Discard a stale draft and return its own changes (vs the fork base) as a diff so -// the model can re-apply them on the latest deploy. Discarding rather than resetting -// keeps the base pointer honest (the next write re-bases on the current head) and -// fails safe: a premature deploy hits "no draft" instead of silently shipping the -// latest unchanged. -async function rebaseDraft( - args: { type: WorkspaceItemType; path: string }, - ctx: WriteDraftCtx -): Promise { - switch (args.type) { - case 'script': - return rebaseScriptDraft(args.path, ctx) - case 'flow': - return rebaseFlowDraft(args.path, ctx) - case 'app': - return rebaseAppDraft(args.path, ctx) - default: - throw new Error('rebase_draft currently supports scripts, flows, and apps.') - } -} - -async function rebaseScriptDraft(path: string, ctx: WriteDraftCtx): Promise { - const { workspace, toolId, toolCallbacks } = ctx - - const draft = await getGlobalDraft(workspace, 'script', path) - if (!draft || typeof draft.value !== 'string' || !draft.language) { - throw new Error(`No script draft found for "${path}".`) - } - if (!(await ScriptService.existsScriptByPath({ workspace, path }))) { - throw new Error(`Script "${path}" is not deployed; there is no newer version to rebase onto.`) - } - - const latest = await ScriptService.getScriptByPath({ workspace, path }) - const baseHash = draft.parentHash - if (baseHash && baseHash === latest.hash) { - const message = `Draft "${path}" is already based on the latest deployed version (${latest.hash}).` - toolCallbacks.setToolStatus(toolId, { content: message }) - return JSON.stringify({ success: true, alreadyLatest: true, latest_hash: latest.hash, message }) - } - - toolCallbacks.setToolStatus(toolId, { content: `Rebasing draft "${path}" onto latest...` }) - - // Capture the draft's own changes (vs its fork base) BEFORE discarding — this - // diff is the only clean record of what to replay. Best-effort: if the base - // version is gone, diff against empty so the full draft is surfaced. - let baseContent = '' - if (baseHash) { - try { - baseContent = (await ScriptService.getScriptByHash({ workspace, hash: baseHash })).content - } catch (e) { - console.error(`rebase_draft: could not fetch base version ${baseHash} for "${path}"`, e) - } - } - const yourChanges = createTwoFilesPatch( - 'fork-base', - 'your-draft', - baseContent, - draft.value, - '', - '' - ) - - // Discard the stale draft rather than resetting it to latest: the next write - // re-bases on the current head, and a premature deploy fails cleanly ("no - // draft") instead of silently shipping the latest unchanged and losing the work. - await deleteGlobalDraft(workspace, 'script', path) - - toolCallbacks.setToolStatus(toolId, { - content: `Discarded stale draft "${path}"`, - result: 'Rebased' - }) - return JSON.stringify( - { - success: true, - message: - `Discarded the stale draft for "${path}". Your changes are in "your_changes" (a diff against the ` + - `version you forked from). Re-apply them with edit_script / write_script — the new draft will be ` + - `based on the latest deployed version (hash ${latest.hash}) — then deploy.`, - latest_hash: latest.hash, - your_changes: yourChanges - }, - null, - 2 - ) -} - -async function rebaseFlowDraft(path: string, ctx: WriteDraftCtx): Promise { - const { workspace, toolId, toolCallbacks } = ctx - - const draft = await getGlobalDraft(workspace, 'flow', path) - if (!draft || draft.value === undefined || typeof draft.value === 'string') { - throw new Error(`No flow draft found for "${path}".`) - } - if (!(await FlowService.existsFlowByPath({ workspace, path }))) { - throw new Error(`Flow "${path}" is not deployed; there is no newer version to rebase onto.`) - } - - const latest = await FlowService.getFlowByPath({ workspace, path }) - const baseVersion = draft.parentVersionId - if (baseVersion != null && baseVersion === latest.version_id) { - const message = `Draft "${path}" is already based on the latest deployed version (${latest.version_id}).` - toolCallbacks.setToolStatus(toolId, { content: message }) - return JSON.stringify({ - success: true, - alreadyLatest: true, - latest_version: latest.version_id, - message - }) - } - - toolCallbacks.setToolStatus(toolId, { content: `Rebasing draft "${path}" onto latest...` }) - - // The draft's own changes vs its fork-base flow value, as a JSON diff for the - // model to replay. Best-effort: skip if the base version can't be fetched. - let baseValue: unknown = {} - if (baseVersion != null) { - try { - baseValue = (await FlowService.getFlowVersion({ workspace, version: baseVersion })).value - } catch (e) { - console.error( - `rebase_draft: could not fetch base flow version ${baseVersion} for "${path}"`, - e - ) - } - } - const oursValue = (draft.value as FlowDraftValue).value - const yourChanges = createTwoFilesPatch( - 'fork-base', - 'your-draft', - JSON.stringify(baseValue, null, 2), - JSON.stringify(oursValue, null, 2), - '', - '' - ) - - // Discard the stale draft (see rebaseScriptDraft): the next write re-bases on - // the current head, and a premature deploy fails cleanly instead of shipping - // the latest unchanged. - await deleteGlobalDraft(workspace, 'flow', path) - - toolCallbacks.setToolStatus(toolId, { - content: `Discarded stale draft "${path}"`, - result: 'Rebased' - }) - return JSON.stringify( - { - success: true, - message: - `Discarded the stale draft for "${path}". Your changes are in "your_changes" (a JSON diff against ` + - `the version you forked from). Re-apply them with the flow edit tools — the new draft will be ` + - `based on the latest deployed version (version ${latest.version_id}) — then deploy.`, - latest_version: latest.version_id, - your_changes: yourChanges - }, - null, - 2 - ) -} - -async function rebaseAppDraft(path: string, ctx: WriteDraftCtx): Promise { - const { workspace, toolId, toolCallbacks } = ctx - - const draft = await getGlobalDraft(workspace, 'app', path) - if (!draft || !draft.value || typeof draft.value === 'string' || !('files' in draft.value)) { - throw new Error(`No app draft found for "${path}".`) - } - if (!(await AppService.existsApp({ workspace, path }))) { - throw new Error(`App "${path}" is not deployed; there is no newer version to rebase onto.`) - } - - const deployed = await AppService.getAppByPath({ workspace, path }) - const headVersion = deployed.versions?.[deployed.versions.length - 1] - const baseVersion = draft.parentVersionId - if (baseVersion != null && baseVersion === headVersion) { - const message = `Draft "${path}" is already based on the latest deployed version (${headVersion}).` - toolCallbacks.setToolStatus(toolId, { content: message }) - return JSON.stringify({ - success: true, - alreadyLatest: true, - latest_version: headVersion, - message - }) - } - - toolCallbacks.setToolStatus(toolId, { content: `Rebasing draft "${path}" onto latest...` }) - - // The draft's own changes vs its fork-base app source, as a JSON diff for the - // model to replay. Best-effort: skip if the base version can't be fetched. - const oursValue = draft.value as AppDraftValue - let baseSource: Pick = { - files: {}, - runnables: {}, - data: undefined - } - if (baseVersion != null) { - try { - const baseApp = await AppService.getAppByVersion({ workspace, id: baseVersion }) - const base = appSourceToDraftValue(baseApp, baseApp) - baseSource = { files: base.files, runnables: base.runnables, data: base.data } - } catch (e) { - console.error( - `rebase_draft: could not fetch base app version ${baseVersion} for "${path}"`, - e - ) - } - } - const yourChanges = createTwoFilesPatch( - 'fork-base', - 'your-draft', - JSON.stringify(baseSource, null, 2), - JSON.stringify( - { files: oursValue.files, runnables: oursValue.runnables, data: oursValue.data }, - null, - 2 - ), - '', - '' - ) - - // Discard the stale draft (see rebaseScriptDraft): the next write re-projects - // the deployed app into a fresh draft (re-pinning parent_version to the head), - // and a premature deploy fails cleanly instead of shipping the latest unchanged. - await deleteGlobalDraft(workspace, 'app', path) - - toolCallbacks.setToolStatus(toolId, { - content: `Discarded stale draft "${path}"`, - result: 'Rebased' - }) - return JSON.stringify( - { - success: true, - message: - `Discarded the stale draft for "${path}". Your changes are in "your_changes" (a JSON diff against ` + - `the version you forked from). Re-apply them with the app edit tools — the new draft will be based ` + - `on the latest deployed version (version ${headVersion}) — then deploy.`, - latest_version: headVersion, - your_changes: yourChanges - }, - null, - 2 - ) -} - const MAX_DIFF_PATCH_CHARS = 50_000 function windowPatchBody(patch: string, offset: number, limit: number): string { @@ -9888,17 +7244,13 @@ async function validateDeleteWorkspaceItemTarget(args: { // Wave a malformed call through so it still fails with the canonical schema error. const parsed = deleteWorkspaceItemSchema.safeParse(args.args) if (!parsed.success) return undefined - const { type, path, trigger_kind: triggerKind } = parsed.data + const { type, trigger_kind: triggerKind } = parsed.data if (type === 'trigger' && !triggerKind) return undefined - const { workspace } = args - if (await deployedItemExists(workspace, type, path, triggerKind)) return undefined - - const draft = await getGlobalDraft(workspace, type, path, triggerKind) - return draft - ? `No deployed ${type} at "${path}" — it only exists as a draft, so there is nothing to delete ` + - `from the workspace. Call discard_local_draft with the same arguments to remove the draft.` - : `No ${type} at "${path}": neither a deployed item nor a draft. Nothing to delete.` + const { text } = await runOnServer('delete_workspace_item', args.workspace, args.args, { + check_only: true + }) + return text || undefined } /** @@ -9976,79 +7328,6 @@ async function deployedItemExists( } } -async function deleteWorkspaceItem( - args: { type: WorkspaceItemType; path: string; trigger_kind?: TriggerKind }, - ctx: WriteDraftCtx -): Promise { - const { workspace, toolId, toolCallbacks } = ctx - const { type, path, trigger_kind: triggerKind } = args - - if (type === 'trigger' && !triggerKind) { - throw new Error('trigger_kind is required when deleting a trigger.') - } - - toolCallbacks.setToolStatus(toolId, { - content: `Deleting ${type} "${path}"...` - }) - - switch (type) { - case 'script': - await ScriptService.deleteScriptByPath({ workspace, path }) - break - case 'flow': - await FlowService.deleteFlowByPath({ workspace, path }) - break - case 'schedule': - await ScheduleService.deleteSchedule({ workspace, path }) - break - case 'trigger': - await triggerServices[triggerKind!].delete({ workspace, path }) - break - case 'resource': - await ResourceService.deleteResource({ workspace, path }) - break - case 'variable': - await VariableService.deleteVariable({ workspace, path }) - break - case 'app': - await AppService.deleteApp({ workspace, path }) - break - } - - // Deployed state changed — cached fork comparisons involving this workspace - // are no longer trustworthy (same rule as deploy success). Before the - // draft cleanup: a cleanup failure must not leave stale comparisons. - invalidateWorkspaceComparison(workspace) - await deleteGlobalDraft(workspace, type, path, triggerKind) - - // Record the deletion in the chat's modified-items mask. In a fork this leaves a - // reviewable "removed" diff vs the parent that stays scoped to this chat. Keyed - // by the same (itemKind, storagePath) as writes so it joins the draft/fork lists. - const deletedKind = itemKindFor(type, triggerKind) - if (deletedKind) { - toolCallbacks.onItemModified?.( - deletedKind, - getGlobalDraftStoragePath(workspace, type, path, triggerKind) - ) - } - - toolCallbacks.setToolStatus(toolId, { - content: `Deleted ${type} "${path}"`, - result: 'Deleted' - }) - return JSON.stringify( - { - success: true, - message: `Deleted ${type} "${path}" from the workspace. Any matching draft was also cleared.`, - type, - path, - triggerKind - }, - null, - 2 - ) -} - export function prepareGlobalSystemMessage( instructions?: { workspace?: string; user?: string }, opts?: { diff --git a/frontend/src/lib/components/copilot/chat/global/flowRunTree.test.ts b/frontend/src/lib/components/copilot/chat/global/flowRunTree.test.ts deleted file mode 100644 index 7b2ddeb5fc..0000000000 --- a/frontend/src/lib/components/copilot/chat/global/flowRunTree.test.ts +++ /dev/null @@ -1,218 +0,0 @@ -import { describe, expect, it } from 'vitest' -import { buildFlowTree, shapeFlowRunTree, type FlowResultEntry } from './flowRunTree' - -let uuidCounter = 0 -function entry(partial: Partial): FlowResultEntry { - uuidCounter++ - return { - job_id: `00000000-0000-0000-0000-${String(uuidCounter).padStart(12, '0')}`, - label: partial.step_path ? `Step ${partial.step_path}` : 'Flow', - kind: 'script', - depth: 1, - sibling_index: 1, - sibling_count: 1, - status: 'success', - success: true, - result_prefix: '"ok"', - result_length: 4, - ...partial - } -} - -function root(partial: Partial = {}): FlowResultEntry { - return entry({ kind: 'flow', depth: 0, step_path: null, label: 'Flow', ...partial }) -} - -/** Root flow with steps a, b where b is a subflow containing c. */ -function nestedEntries(): FlowResultEntry[] { - return [ - root(), - entry({ step_path: 'a', flow_step_id: 'a' }), - entry({ step_path: 'b', flow_step_id: 'b', kind: 'flow', label: 'Step b (subflow)' }), - entry({ - step_path: 'b/c', - flow_step_id: 'c', - depth: 2, - status: 'failure', - success: false, - result_prefix: '{"error":"boom"}', - result_length: 16 - }) - ] -} - -/** Root flow with loop step l of `n` iterations; `failed` are 1-based indices. */ -function loopEntries(n: number, failed: number[]): FlowResultEntry[] { - const entries = [root()] - for (let i = 1; i <= n; i++) { - entries.push( - entry({ - step_path: 'l', - flow_step_id: 'l', - kind: 'flow', - label: `Step l forloop (iteration ${i}/${n})`, - parent_module_type: 'forloopflow', - sibling_index: i, - sibling_count: n, - status: failed.includes(i) ? 'failure' : 'success', - success: !failed.includes(i) - }) - ) - } - return entries -} - -describe('buildFlowTree', () => { - it('rebuilds nesting from the depth-first flat list', () => { - const tree = buildFlowTree(nestedEntries())! - expect(tree.children.map((c) => c.entry.flow_step_id)).toEqual(['a', 'b']) - expect(tree.children[1].children.map((c) => c.entry.flow_step_id)).toEqual(['c']) - }) -}) - -describe('shapeFlowRunTree', () => { - it('renders subflow steps nested under their parent step', () => { - const rendered = JSON.parse(shapeFlowRunTree({ entries: nestedEntries() })) - expect(rendered.run.status).toBe('success') - const stepB = rendered.steps[1] - expect(stepB.step).toBe('b') - expect(stepB.steps[0].step).toBe('b/c') - expect(stepB.steps[0].status).toBe('failure') - expect(stepB.steps[0].result).toBe('{"error":"boom"}') - }) - - it('collapses loop iterations to a tally with capped failed iterations plus the last', () => { - const rendered = JSON.parse(shapeFlowRunTree({ entries: loopEntries(50, [3, 7, 21, 30]) })) - const loop = rendered.steps[0] - expect(loop.type).toBe('forloopflow') - expect(loop.iterations).toBe(50) - expect(loop.ok).toBe(46) - expect(loop.failed_iterations).toEqual([3, 7, 21, 30]) - // 3 failed shown (cap) + the last iteration - expect(loop.iterations_shown.map((i: any) => i.iteration)).toEqual([3, 7, 21, 50]) - expect(loop.iterations_elided).toBe(46) - }) - - it('renders retried steps as attempts of one step, not loop iterations', () => { - const entries = [root()] - for (let i = 1; i <= 3; i++) { - entries.push( - entry({ - step_path: 'a', - flow_step_id: 'a', - label: `Step a (attempt ${i}/3)`, - parent_module_type: 'rawscript', - sibling_index: i, - sibling_count: 3, - status: i < 3 ? 'failure' : 'success', - success: i === 3 - }) - ) - } - const step = JSON.parse(shapeFlowRunTree({ entries })).steps[0] - expect(step.iterations).toBeUndefined() - expect(step.attempts).toBe(3) - expect(step.status).toBe('success') - expect(step.label).toBe('Step a (attempt 3/3)') - expect(step.previous_attempts.map((a: any) => a.status)).toEqual(['failure', 'failure']) - }) - - it('counts running iterations as unfinished and skipped ones as skipped', () => { - const entries = loopEntries(4, []) - entries[2].status = 'running' - entries[2].success = undefined - entries[2].result_prefix = undefined - entries[2].result_length = undefined - entries[3].status = 'skipped' - const loop = JSON.parse(shapeFlowRunTree({ entries })).steps[0] - expect(loop.ok).toBe(2) - expect(loop.unfinished).toBe(1) - expect(loop.skipped).toBe(1) - expect(loop.failed_iterations).toBeUndefined() - }) - - it('reports the truncated total size and surfaces the enclosing flow of a step job', () => { - const entries = nestedEntries() - entries[1].result_prefix = 'x'.repeat(700) - entries[1].result_length = 5000 - const rendered = JSON.parse(shapeFlowRunTree({ enclosing_job: 'enclosing-uuid', entries })) - expect(rendered.note).toContain('enclosing-uuid') - expect(rendered.steps[0].result_total_chars).toBe(5000) - }) - - it('notes when the server truncated the tree', () => { - const rendered = JSON.parse(shapeFlowRunTree({ entries: nestedEntries(), truncated: true })) - expect(rendered.note).toContain('more jobs') - }) - - it('compares result sizes in code points and never splits a surrogate pair', () => { - const entries = [root()] - // 400 astral chars: 800 UTF-16 units but 400 code points. The 700-unit - // head budget keeps only 350 of them — comparing in UTF-16 units - // (400 < 700) used to hide that truncation entirely. - entries.push( - entry({ - step_path: 'a', - flow_step_id: 'a', - status: 'failure', - success: false, - result_prefix: '🦄'.repeat(400), - result_length: 400 - }) - ) - const step = JSON.parse(shapeFlowRunTree({ entries })).steps[0] - expect(step.result_total_chars).toBe(400) - // the cut must never leave a lone high surrogate at the tail - expect(step.result.charCodeAt(step.result.length - 1)).toBeGreaterThan(0xdbff) - }) - - it('shrinks result heads to fit the total budget instead of overflowing', () => { - const entries = [root()] - for (let i = 1; i <= 60; i++) { - entries.push( - entry({ - step_path: `s${i}`, - flow_step_id: `s${i}`, - status: i === 1 ? 'failure' : 'success', - success: i !== 1, - result_prefix: 'y'.repeat(700), - result_length: 700 - }) - ) - } - const rendered = shapeFlowRunTree({ entries }) - expect(rendered.length).toBeLessThanOrEqual(21000) - const parsed = JSON.parse(rendered) - // failure detail outlives success detail: the failed step keeps a result - // head while succeeded steps are reduced to their size - expect(parsed.steps[0].status).toBe('failure') - expect(parsed.steps[0].result.length).toBeGreaterThan(0) - expect(parsed.steps[1].result).toBeUndefined() - expect(parsed.steps[1].result_total_chars).toBe(700) - }) - - it('spends the tree budget on the tree, not on the run payloads carried with it', () => { - const entries = [root()] - for (let i = 1; i <= 12; i++) { - entries.push( - entry({ - step_path: `s${i}`, - flow_step_id: `s${i}`, - status: 'failure', - success: false, - result_prefix: 'y'.repeat(700), - result_length: 700 - }) - ) - } - const logs = 'L'.repeat(12000) - const rendered = shapeFlowRunTree({ entries }, { status: 'failure', logs }) - - // Counting the overrides against the budget would shrink this tree away - // even though the tree itself fits: the model asked for the payloads. - expect(rendered.length).toBeGreaterThan(20000) - const parsed = JSON.parse(rendered) - expect(parsed.run.logs).toBe(logs) - expect(parsed.steps[0].result.length).toBe(700) - }) -}) diff --git a/frontend/src/lib/components/copilot/chat/global/flowRunTree.ts b/frontend/src/lib/components/copilot/chat/global/flowRunTree.ts deleted file mode 100644 index af203e2c8c..0000000000 --- a/frontend/src/lib/components/copilot/chat/global/flowRunTree.ts +++ /dev/null @@ -1,519 +0,0 @@ -import { JobService, type Job, type GetFlowAllResultsResponse } from '$lib/gen' -import { isWindmillTooBigObject } from '$lib/components/job_args' - -/** - * Model-facing view of a run for the global chat's get_run tool: the job's own - * summary, args, result and logs, plus — when the run has steps — its execution - * tree. The backend endpoint (get_flow_all_results) enumerates every job of the - * tree with per-entry truncated results; this module shapes that flat list into - * a compact per-step tree the model can read in one tool result. Step addresses - * ('b/c', 'b[12]/c') are resolved server-side by the same endpoint for - * full-result drill-down. - * - * `summarizeRun` also backs list_runs' per-job summary, so it lives here rather - * than in core.ts: get_run needs it, and importing it back would be circular. - */ - -export type FlowResultEntry = GetFlowAllResultsResponse['entries'][number] - -/** Per-entry result budget requested from the server for the tree view. */ -export const TREE_RESULT_HEAD_CHARS = 700 -/** Cap on a full payload handed to the model: a drilled step result, and the - * run's own args, result and logs. */ -export const STEP_RESULT_MAX_CHARS = 12000 -/** Cap on the whole rendered tree; heads shrink progressively to fit. */ -const TREE_TOTAL_BUDGET_CHARS = 20000 -/** Failed loop iterations shown with a result head (the rest are indices only). */ -const MAX_FAILED_ITERATIONS_SHOWN = 3 -/** Head-shrink ladder tried in order until the rendered tree fits the budget. - * `successHead` caps results of succeeded steps separately so failure detail - * survives longest. */ -const SHRINK_LADDER = [ - { head: TREE_RESULT_HEAD_CHARS, successHead: TREE_RESULT_HEAD_CHARS }, - { head: 250, successHead: 250 }, - { head: 250, successHead: 0 }, - { head: 100, successHead: 0 } -] - -export interface FlowTreeNode { - entry: FlowResultEntry - children: FlowTreeNode[] -} - -/** Rebuild the parent/child tree from the server's depth-first flat list. */ -export function buildFlowTree(entries: FlowResultEntry[]): FlowTreeNode | undefined { - let root: FlowTreeNode | undefined - const stack: FlowTreeNode[] = [] - for (const entry of entries) { - const node: FlowTreeNode = { entry, children: [] } - while (stack.length && stack[stack.length - 1].entry.depth >= entry.depth) { - stack.pop() - } - const parent = stack[stack.length - 1] - if (parent) { - parent.children.push(node) - } else if (!root) { - root = node - } - stack.push(node) - } - return root -} - -interface ChildGroup { - stepId: string - nodes: FlowTreeNode[] -} - -/** Group sibling jobs by flow step id (loop iterations / branches of one step - * form one group). Keyed rather than run-length so ordering quirks between - * parallel iterations can't split a step into two groups. */ -function groupChildren(children: FlowTreeNode[]): ChildGroup[] { - const groups = new Map() - for (const child of children) { - const stepId = child.entry.flow_step_id ?? `job:${child.entry.job_id}` - let group = groups.get(stepId) - if (!group) { - group = { stepId, nodes: [] } - groups.set(stepId, group) - } - group.nodes.push(child) - } - return Array.from(groups.values()) -} - -interface ShapeOpts { - head: number - successHead: number -} - -/** Postgres LEFT()/length() count code points while JS .length/.slice count - * UTF-16 units — compare and cut in code points so astral characters neither - * hide a truncation nor get split through a surrogate pair. */ -function countCodePoints(s: string): number { - let n = 0 - for (const _ of s) n++ - return n -} - -function sliceCodePointSafe(s: string, maxUnits: number): string { - const cut = s.slice(0, maxUnits) - const last = cut.charCodeAt(cut.length - 1) - // drop a trailing lone high surrogate - return last >= 0xd800 && last <= 0xdbff ? cut.slice(0, -1) : cut -} - -function sliceCodePointSafeEnd(s: string, maxUnits: number): string { - const cut = s.slice(-maxUnits) - const first = cut.charCodeAt(0) - // drop a leading lone low surrogate - return first >= 0xdc00 && first <= 0xdfff ? cut.slice(1) : cut -} - -function shapeResult( - entry: FlowResultEntry, - opts: ShapeOpts -): { result?: string; result_total_chars?: number } { - if (entry.result_prefix === undefined || entry.result_prefix === null) return {} - const budget = entry.success ? opts.successHead : opts.head - if (budget <= 0) return { result_total_chars: entry.result_length ?? undefined } - const head = sliceCodePointSafe(entry.result_prefix, budget) - const total = entry.result_length ?? countCodePoints(entry.result_prefix) - return { - result: head, - ...(total > countCodePoints(head) ? { result_total_chars: total } : {}) - } -} - -function shapeStep(node: FlowTreeNode, opts: ShapeOpts, iteration?: number): Record { - const entry = node.entry - const shaped: Record = { - ...(entry.step_path ? { step: entry.step_path } : {}), - ...(iteration !== undefined ? { iteration } : {}), - label: entry.label, - job_id: entry.job_id, - status: entry.status, - ...(entry.duration_ms !== undefined && entry.duration_ms !== null - ? { duration_ms: entry.duration_ms } - : {}), - ...shapeResult(entry, opts) - } - const childSteps = shapeChildren(node.children, opts) - if (childSteps.length > 0) { - shaped.steps = childSteps - } - return shaped -} - -/** Module types whose sibling jobs are iterations/branches; sibling jobs of - * any other step are retry attempts of that step. */ -const FAN_OUT_MODULE_TYPES = new Set([ - 'forloopflow', - 'whileloopflow', - 'branchall', - 'branchone', - 'aiagent' -]) - -function shapeGroup(group: ChildGroup, opts: ShapeOpts): Record { - if (group.nodes.length === 1 && group.nodes[0].entry.sibling_count <= 1) { - return shapeStep(group.nodes[0], opts) - } - - const first = group.nodes[0].entry - if (first.parent_module_type === 'branchall') { - return { - step: first.step_path ?? group.stepId, - type: 'branchall', - branches: group.nodes.map((n) => shapeStep(n, opts)) - } - } - - const byIndex = [...group.nodes].sort((a, b) => a.entry.sibling_index - b.entry.sibling_index) - - if (first.parent_module_type && !FAN_OUT_MODULE_TYPES.has(first.parent_module_type)) { - // Retried step: siblings are attempts of the same step, the last one is - // the final outcome — render it as the step, keeping earlier attempts as - // status-only references. - const shaped = shapeStep(byIndex[byIndex.length - 1], opts) - shaped.attempts = byIndex.length - shaped.previous_attempts = byIndex.slice(0, -1).map((n) => ({ - attempt: n.entry.sibling_index, - status: n.entry.status, - job_id: n.entry.job_id - })) - return shaped - } - - // Loop-like fan-out (forloopflow, whileloopflow, aiagent actions, or any - // other multi-job step): tally statuses, show failed iterations (capped) and - // the latest one, elide the rest. - const ok = byIndex.filter((n) => n.entry.status === 'success').length - const failedNodes = byIndex.filter( - (n) => n.entry.status === 'failure' || n.entry.status === 'canceled' - ) - const skipped = byIndex.filter((n) => n.entry.status === 'skipped').length - // skipped is terminal — only running/queued/suspended count as unfinished - const unfinished = byIndex.length - ok - failedNodes.length - skipped - - const shown = failedNodes.slice(0, MAX_FAILED_ITERATIONS_SHOWN) - const last = byIndex[byIndex.length - 1] - if (!shown.includes(last)) { - shown.push(last) - } - - return { - step: first.step_path ?? group.stepId, - type: first.parent_module_type || 'loop', - iterations: byIndex.length, - ok, - ...(failedNodes.length > 0 - ? { failed_iterations: failedNodes.map((n) => n.entry.sibling_index) } - : {}), - ...(skipped > 0 ? { skipped } : {}), - ...(unfinished > 0 ? { unfinished } : {}), - iterations_shown: shown.map((n) => shapeStep(n, opts, n.entry.sibling_index)), - ...(byIndex.length > shown.length ? { iterations_elided: byIndex.length - shown.length } : {}) - } -} - -function shapeChildren(children: FlowTreeNode[], opts: ShapeOpts): Record[] { - return groupChildren(children).map((g) => shapeGroup(g, opts)) -} - -function renderTree( - root: FlowTreeNode, - rootJobNote: string | undefined, - opts: ShapeOpts, - runOverrides?: Record -): Record { - const run = shapeStep(root, opts) - const steps = run.steps - delete run.steps - // The backend labels every depth-0 job "Flow"; correct that for the graceful - // non-flow case so the model doesn't mistake a plain script run for a flow. - if (!steps && root.entry.kind !== 'flow' && root.entry.kind !== 'flowpreview') { - run.label = `Job (${root.entry.kind})` - } - if (runOverrides) { - // A replacement result supersedes the tree entry's head, and the stale - // truncation marker must go with it. Without one the head stays: it is a - // real server-side prefix of a payload `getJob` would only hand back as a - // WINDMILL_TOO_BIG placeholder. - if ('result' in runOverrides) { - delete run.result - delete run.result_total_chars - } - Object.assign(run, runOverrides) - } - return { - ...(rootJobNote ? { note: rootJobNote } : {}), - run, - ...(steps - ? { - steps, - hint: `Step results are truncated. Call get_run again with step="" (e.g. "b/c", or "b[12]" for one loop iteration) for a step's result in full (up to ${STEP_RESULT_MAX_CHARS} chars), or with id set to a step's job_id for that step's own args, result and logs.` - } - : {}) - } -} - -/** Render the whole tree, shrinking result heads until it fits the budget. */ -export function shapeFlowRunTree( - response: GetFlowAllResultsResponse, - runOverrides?: Record -): string { - const root = buildFlowTree(response.entries) - if (!root) { - // The tree is the optional half: what the caller already read of the job - // still answers the question, so don't drop it with the missing tree. - return runOverrides - ? JSON.stringify({ run: runOverrides }, null, 1) - : 'No jobs found for this run.' - } - const notes = [ - ...(response.enclosing_job - ? [ - `This job is a step of a larger flow run — its enclosing run is ${response.enclosing_job}; pass that id to see more of the tree.` - ] - : []), - ...(response.truncated - ? [ - `The run has more jobs than the server returns — this tree only covers the first ${response.entries.length} (depth-first), so tallies may undercount.` - ] - : []), - ...(response.scope_filtered - ? [`Your token is tag-scoped: steps running on other tags are omitted from this tree.`] - : []) - ] - const rootJobNote = notes.length > 0 ? notes.join(' ') : undefined - - // The budget caps the tree, not the run's own args/result/logs — those are - // capped on their own and the model asked for them, so they don't count here - // and the last-resort slice keeps room for them (`run` precedes `steps`). - const overrideChars = runOverrides ? JSON.stringify(runOverrides, null, 1).length : 0 - let rendered = '' - for (const opts of SHRINK_LADDER) { - rendered = JSON.stringify(renderTree(root, rootJobNote, opts, runOverrides), null, 1) - if (rendered.length - overrideChars <= TREE_TOTAL_BUDGET_CHARS) { - return rendered - } - } - return ( - rendered.slice(0, TREE_TOTAL_BUDGET_CHARS + overrideChars) + - `\n… (tree truncated at ${TREE_TOTAL_BUDGET_CHARS} chars — drill into specific steps with the step parameter)` - ) -} - -// Compact metadata for one run. The raw Job carries args/result/logs/raw_code -// which can be huge — this is only what's needed to identify a run, so list_runs -// can return one entry per job. -export function summarizeRun(job: Job): Record { - const base = { - id: job.id, - job_kind: job.job_kind, - path: job.script_path, - created_by: job.created_by, - created_at: job.created_at, - started_at: job.started_at, - schedule_path: job.schedule_path, - is_flow_step: job.is_flow_step, - tag: job.tag, - worker: job.worker - } - if ('success' in job) { - // CompletedJob. `success` is true for a skipped job too, so is_skipped has - // to be read first or a skipped step reports as a successful one. - return { - ...base, - status: job.canceled - ? 'canceled' - : job.is_skipped - ? 'skipped' - : job.success - ? 'success' - : 'failure', - duration_ms: job.duration_ms - } - } - // QueuedJob. A running job with suspends outstanding is parked — on an approval - // step or on a parallelism slot — not working, and `running` alone hides that. - return { - ...base, - status: job.running ? (job.suspend ? 'suspended' : 'running') : 'queued' - } -} - -/** Cap a payload to STEP_RESULT_MAX_CHARS. `tail` keeps the end instead of the - * start — for logs, where the failure is at the bottom. - * - * Counts and cuts without materialising the string: logs arrive whole and - * unbounded, and spreading one into an array of code points costs ~9x its size - * (a 10MB log measured +90MB). Cutting in UTF-16 units keeps at most the budget - * in code points, never more, so an astral-heavy payload is trimmed slightly - * shorter than advertised rather than overshooting. */ -function cap(text: string, tail = false): string { - // UTF-16 length is never below the code-point count, so anything passing this - // is already under the cap and needs no counting pass at all. - if (text.length <= STEP_RESULT_MAX_CHARS) return text - const total = countCodePoints(text) - if (total <= STEP_RESULT_MAX_CHARS) return text - const note = `… (truncated: ${total} chars total)` - return tail - ? `${note}\n${sliceCodePointSafeEnd(text, STEP_RESULT_MAX_CHARS)}` - : `${sliceCodePointSafe(text, STEP_RESULT_MAX_CHARS)}\n${note}` -} - -/** Told apart from an empty log so the model doesn't report "no logs" for a run - * whose logs it simply failed to read. */ -const LOGS_UNREADABLE = 'Logs could not be read for this run.' - -/** `getJob` swaps a payload over ~90KB for a marker rather than sending it - * (`get_job_query!` in backend/windmill-api/src/jobs.rs), and the two fields use - * different ones: a result becomes the bare string, args become exactly - * `{reason: }`. Each field matches only its own form, so a payload that - * merely carries that string in a `reason` of its own stays the run's value. */ -const TOO_BIG_RESULT = 'WINDMILL_TOO_BIG' - -function stringify(value: unknown): string { - return typeof value === 'string' ? value : JSON.stringify(value, null, 1) -} - -/** An elided args/result payload is reported, never fetched around: the - * endpoints that return those whole take no length parameter, so recovering a - * usable head would mean pulling the entire payload (up to MAX_RESULT_SIZE_MB, - * 500 by default) into the tab to keep 12k of it. The flag is the whole report — - * the result keeps the head the step tree already carries beside it. */ -function shapeRunArgs(job: Job): Record { - if (!job.args) return {} - return isWindmillTooBigObject(job.args) - ? { args_truncated: true } - : { args: cap(stringify(job.args)) } -} - -function shapeRunResult(job: Job): Record { - if (!('result' in job) || job.result === undefined) return {} - // No `result` key when elided, so the tree's `result_prefix` — a real - // server-side head of the same payload — stands in its place unoverridden. - return job.result === TOO_BIG_RESULT - ? { result_truncated: true } - : { result: cap(stringify(job.result)) } -} - -/** Why a run ended badly. Kept out of summarizeRun, which list_runs pays per job. */ -function diagnoseRun(job: Job): Record { - return { - ...(job.canceled_by ? { canceled_by: job.canceled_by } : {}), - ...(job.canceled_reason ? { canceled_reason: job.canceled_reason } : {}), - ...(job.mem_peak ? { mem_peak_kb: job.mem_peak } : {}) - } -} - -/** What get_run answers with: the model's payload, and — when the call addresses one - * job — that job's id, which the card renders the run from. The two are separate - * audiences: `text` is capped for the model, `jobId` is how the user gets the whole - * thing. An address resolving to several jobs (a loop's `b`), an unfinished step or an - * unknown one carries no id, and the call renders as an ordinary tool row. */ -export type RunInspection = { text: string; jobId?: string } - -/** Entry point of the get_run tool. Without `step`: the run's summary, args, - * result and logs, plus the per-step tree when the run has steps. With `step`: - * that step's full (capped) result, resolved server-side. */ -export async function getRun(workspace: string, id: string, step?: string): Promise { - if (!step) { - // Only the job read is load-bearing: logs and the step tree each answer - // part of the question, so neither failing should cost the model the rest. - const [job, logs, results] = await Promise.all([ - JobService.getJob({ workspace, id, noLogs: true, noCode: true }), - // The dedicated endpoint rather than the job's own `logs` field, which holds too - // little to serve a tail — RunScriptCard's fetch has the mechanism. Only this - // job's own logs either way: a flow's are its orchestration lines, not its steps'. - JobService.getJobLogs({ - workspace, - id, - // Suppress the "to remove ansi colors, use: sed ..." hint the backend - // otherwise prepends — noise for the model, and not actual stripping. - removeAnsiWarnings: true - }).catch(() => LOGS_UNREADABLE), - // `.catch` alone would leave the tree half-guarded: like the log read, this - // one can fail by resolving `undefined` rather than rejecting, and reading - // `.entries` off that throws — losing the job and logs already in hand. - JobService.getFlowAllResults({ workspace, id, maxResultLen: TREE_RESULT_HEAD_CHARS }) - .catch(() => undefined) - .then((r) => r ?? ({ entries: [] } as GetFlowAllResultsResponse)) - ]) - const { id: _id, ...summary } = summarizeRun(job) - const payloads = { ...shapeRunArgs(job), ...shapeRunResult(job) } - // A read that fails without rejecting still arrives here: the generated client - // resolves `undefined` when it cannot read the body (a truncated response, a - // dropped connection). Logs are the one payload where empty is a real answer, - // so that has to be told apart from an empty log rather than reported as one. - const shapedLogs = - typeof logs !== 'string' || logs === LOGS_UNREADABLE - ? LOGS_UNREADABLE - : logs.trim() - ? cap(logs, true) - : 'No logs for this run.' - return { - text: shapeFlowRunTree(results, { - ...summary, - ...diagnoseRun(job), - // A successful read always carries the job itself as the root entry, so - // no entries means the read failed — and nothing else would name the run. - ...(results.entries.length === 0 ? { job_id: id, steps_unavailable: true } : {}), - ...payloads, - logs: shapedLogs - }), - jobId: id - } - } - - return getStepResult(workspace, id, step) -} - -/** One step's result in full, addressed by step path. The server resolves the - * address directly (a few indexed lookups, no tree enumeration) and returns the - * single job as an entry. */ -async function getStepResult(workspace: string, id: string, step: string): Promise { - const response = await JobService.getFlowAllResults({ - workspace, - id, - maxResultLen: STEP_RESULT_MAX_CHARS, - step - }) - if (response.step_error) { - return { - text: - response.step_error + - (response.scope_filtered - ? ' (Steps running on tags outside your token’s scope are hidden.)' - : '') - } - } - const entry = response.entries[0] - if (!entry) { - return { text: 'No jobs found for this run.' } - } - if (entry.status === 'running' || entry.status === 'queued' || entry.status === 'suspended') { - return { - text: `Step "${step}" (job ${entry.job_id}) has not completed yet — status: ${entry.status}.` - } - } - // Every completed step is a job of its own, so the card renders it from source — - // including a skipped one, whose inputs and logs are all there is to see. - if (entry.result_prefix === undefined || entry.result_prefix === null) { - return { - text: `Step "${step}" (job ${entry.job_id}, ${entry.status}) has no recorded result.`, - jobId: entry.job_id - } - } - const total = entry.result_length ?? countCodePoints(entry.result_prefix) - const capped = - total > countCodePoints(entry.result_prefix) - ? entry.result_prefix + `\n… (result truncated: ${total} chars total)` - : entry.result_prefix - return { - text: `Step "${step}" (job ${entry.job_id}, ${entry.status}) result:\n${capped}`, - jobId: entry.job_id - } -} diff --git a/frontend/src/lib/components/copilot/chat/global/mcpSessionGen.test.ts b/frontend/src/lib/components/copilot/chat/global/mcpSessionGen.test.ts index 0a38686399..c712a73109 100644 --- a/frontend/src/lib/components/copilot/chat/global/mcpSessionGen.test.ts +++ b/frontend/src/lib/components/copilot/chat/global/mcpSessionGen.test.ts @@ -1,5 +1,5 @@ /** - * Generates what the Windmill MCP server's toolset (backend/windmill-api/src/mcp/session) + * Generates what the Windmill MCP server's toolset (backend/windmill-api/src/ai_session_tools) * shows a model, from the same definitions an AI session uses: tool names, descriptions and * input schemas, the system prompt, and the `get_instructions` texts. The server implements * the tools in Rust; everything the model reads comes from here, so the two cannot drift @@ -19,10 +19,15 @@ import { prepareGlobalSystemMessage } from './core' import type { ScriptLang } from '$lib/gen' -import { FRAMEWORK_TEMPLATES, STARTER_RUNNABLES } from '$lib/components/raw_apps/templates' +import { + BUILDER_FRAMEWORK_TEMPLATES, + FRAMEWORK_TEMPLATES, + STARTER_RUNNABLES +} from '$lib/components/raw_apps/templates' +import { RESOURCE_TYPE_WORDS } from '$lib/components/resourceTypeDisplay' const OUT_DIR = fileURLToPath( - new URL('../../../../../../../backend/windmill-api/src/mcp/session/gen/', import.meta.url) + new URL('../../../../../../../backend/windmill-api/src/ai_session_tools/gen/', import.meta.url) ) // Sentences that describe the chat's run form, which an MCP client does not have. Each @@ -260,7 +265,10 @@ async function generate(): Promise> { const tools = toolDefs() const files: Record = { 'tools.json': JSON.stringify(tools, null, '\t') + '\n', - 'prompt.json': JSON.stringify(promptParts(), null, '\t') + '\n' + 'prompt.json': JSON.stringify(promptParts(), null, '\t') + '\n', + // How a slug's words read, for matching a query against a hub integration the hub + // gives no display name. + 'display_words.json': JSON.stringify(RESOURCE_TYPE_WORDS, null, '\t') + '\n' } const languages: (ScriptLang | undefined)[] = [ undefined, @@ -284,10 +292,15 @@ async function generate(): Promise> { index[subject][language ?? ''] = id } } - // What `init_app` seeds a new raw app with. + // What `init_app` seeds a new raw app with; an operator with builder rights starts from + // a static page, since they can save no inline runnable. files['app_templates.json'] = JSON.stringify( - { templates: FRAMEWORK_TEMPLATES, starter_runnables: STARTER_RUNNABLES }, + { + templates: FRAMEWORK_TEMPLATES, + builder_templates: BUILDER_FRAMEWORK_TEMPLATES, + starter_runnables: STARTER_RUNNABLES + }, null, '\t' ) + '\n' diff --git a/frontend/src/lib/components/copilot/chat/global/serverTools.ts b/frontend/src/lib/components/copilot/chat/global/serverTools.ts new file mode 100644 index 0000000000..377975624f --- /dev/null +++ b/frontend/src/lib/components/copilot/chat/global/serverTools.ts @@ -0,0 +1,97 @@ +/** + * The GLOBAL toolset's versions of tools the other chat modes share: the same definition, + * run on the server like every other GLOBAL tool. + */ +import type { SessionTool } from '../sessionCapabilities' +import { createSearchHubScriptsTool, searchHubScriptsSchema } from '../shared' +import { createDbSchemaTool } from '../script/core' +import { readDocsPageTool, searchDocsTool } from '../docs/core' +import { runOnServer } from './backendSessionTools' + +export const searchHubScriptsServerTool: SessionTool<{}> = { + ...createSearchHubScriptsTool(false), + fn: async ({ args, workspace, toolId, toolCallbacks }) => { + const { query, integration } = searchHubScriptsSchema.parse(args) + const subject = query ? `"${query}"` : `the ${integration} integration` + if (query || integration) { + toolCallbacks.setToolStatus(toolId, { content: `Searching hub scripts for ${subject}...` }) + } + const { text } = await runOnServer('search_hub_scripts', workspace, args) + if (query || integration) { + const found = JSON.parse(text).results.length + toolCallbacks.setToolStatus(toolId, { + content: + found === 0 + ? `No hub script found for ${subject}` + : `Found ${found} hub script${found === 1 ? '' : 's'} for ${subject}` + }) + } + return text + } +} + +export const searchDocsServerTool: SessionTool<{}> = { + ...searchDocsTool, + fn: async ({ args, workspace, toolId, toolCallbacks }) => { + const query = typeof args?.query === 'string' ? args.query.trim() : '' + toolCallbacks.setToolStatus(toolId, { + content: query ? `Searching documentation for "${query}"...` : 'Searching documentation...' + }) + const { text, effects } = await runOnServer('search_docs', workspace, args) + if (text.startsWith('Failed to search documentation:')) { + toolCallbacks.setToolStatus(toolId, { + content: 'Error searching documentation', + error: 'Error searching documentation' + }) + } else if (query) { + const count = Number( + effects.flatMap((e) => (e.type === 'facts' ? [e.result_count] : []))[0] ?? 0 + ) + toolCallbacks.setToolStatus(toolId, { + content: count > 0 ? `Found ${count} matching page(s)` : 'No matching pages found' + }) + } + return text + } +} + +export const readDocsPageServerTool: SessionTool<{}> = { + ...readDocsPageTool, + fn: async ({ args, workspace, toolId, toolCallbacks }) => { + const url = typeof args?.url === 'string' ? args.url : '' + const section = + typeof args?.section === 'string' && args.section.trim() ? args.section : undefined + toolCallbacks.setToolStatus(toolId, { + content: section ? `Reading docs section "${section}"...` : 'Reading documentation page...' + }) + const { text } = await runOnServer('read_docs_page', workspace, args) + if (text.startsWith('Failed to read documentation page:')) { + toolCallbacks.setToolStatus(toolId, { + content: 'Error reading documentation page', + error: 'Error reading documentation page' + }) + } else if (url.trim()) { + toolCallbacks.setToolStatus(toolId, { content: 'Read documentation page' }) + } + return text + } +} + +export function dbSchemaServerTool(description: string): SessionTool<{}> { + return { + ...createDbSchemaTool<{}>({ description, updateEditorCache: false }), + fn: async ({ args, workspace, toolId, toolCallbacks }) => { + if (!args.resourcePath) { + throw new Error('Database path not provided') + } + toolCallbacks.setToolStatus(toolId, { + content: 'Getting database schema for ' + args.resourcePath + '...' + }) + const { text } = await runOnServer('get_db_schema', workspace, args) + toolCallbacks.setToolStatus(toolId, { + content: 'Retrieved database schema for ' + args.resourcePath + }) + return text + } + } +} diff --git a/frontend/src/lib/components/copilot/chat/global/userDraftAdapter.ts b/frontend/src/lib/components/copilot/chat/global/userDraftAdapter.ts index 1a214312f4..934b8d357f 100644 --- a/frontend/src/lib/components/copilot/chat/global/userDraftAdapter.ts +++ b/frontend/src/lib/components/copilot/chat/global/userDraftAdapter.ts @@ -36,7 +36,7 @@ const TRIGGER_KIND_BY_DRAFT_KIND = Object.fromEntries( ]) ) as Partial> -const GLOBAL_DRAFT_KINDS = [ +export const GLOBAL_DRAFT_KINDS = [ 'script', 'flow', 'raw_app', @@ -377,22 +377,6 @@ async function fetchBackendDraftValue( return resp.value ?? undefined } -// Draft VALUE for a write merge: cell-if-present (the user's freshest in-tab -// edits) else the current user's backend draft. -export async function readGlobalDraftValue( - workspace: string, - type: WorkspaceItemType, - path: string, - triggerKind?: TriggerKind -): Promise { - const itemKind = itemKindFor(type, triggerKind) - if (!itemKind) return undefined - const storagePath = resolveDraftStoragePath(workspace, itemKind, path) - const cell = UserDraft.get(itemKind, storagePath, { workspace }) - if (cell !== undefined) return cell - return (await fetchBackendDraftValue(workspace, itemKind, storagePath)) as V | undefined -} - // `itemKind` + `storagePath` are the canonical identity of the persisted draft // (NOT item.path, which is the friendly display path). Callers use them to record // the chat's modified-items mask. @@ -427,6 +411,22 @@ export async function persistGlobalDraft( const itemKind = itemKindFor(type, opts.triggerKind) if (!itemKind) throw new Error(`Unsupported draft type "${type}".`) const storagePath = resolveDraftStoragePath(workspace, itemKind, path) + return persistDraftAt(workspace, itemKind, storagePath, value, opts) +} + +/** `persistGlobalDraft` at a resolved storage key. `lastSync` is the server row the value + * was built on when it was read from the server rather than from a cell: the save detects + * a conflict against it, as a read through `fetchBackendDraftValue` records. */ +export async function persistDraftAt( + workspace: string, + itemKind: UserDraftItemKind, + storagePath: string, + value: unknown, + opts: { force?: boolean; lastSync?: string | null } = {} +): Promise { + if (opts.lastSync) { + UserDraftDbSyncer.recordRemoteSync({ workspace, itemKind, path: storagePath }, opts.lastSync) + } UserDraft.seed(itemKind, storagePath, value, { workspace }) await UserDraftDbSyncer.save({ workspace, @@ -442,7 +442,7 @@ export async function persistGlobalDraft( displayPath, isLiveDraft ) - if (!item) throw new Error(`Could not synthesize ${type} draft "${path}".`) + if (!item) throw new Error(`Could not synthesize ${itemKind} draft "${storagePath}".`) // A failed save (network/5xx) is recorded in the syncer's failure map, not // thrown — so check it before reporting success, else a write tool would tell // the chat "saved" while the DB-backed source of truth was never updated. @@ -593,6 +593,20 @@ export async function deleteGlobalDraft( const itemKind = itemKindFor(type, triggerKind) if (!itemKind) return const storagePath = resolveDraftStoragePath(workspace, itemKind, path) + await deleteDraftAt(workspace, itemKind, storagePath, path, options) +} + +/** `deleteGlobalDraft` at a resolved storage key; `path` names the draft in its errors. */ +export async function deleteDraftAt( + workspace: string, + itemKind: UserDraftItemKind, + storagePath: string, + path: string, + options: DeleteGlobalDraftOptions & { lastSync?: string | null } = {} +): Promise { + if (options.lastSync) { + UserDraftDbSyncer.recordRemoteSync({ workspace, itemKind, path: storagePath }, options.lastSync) + } const liveDraft = UserDraft.getLiveEditorDraft(itemKind, { workspace }) if (options.preserveLiveDraft && liveDraft?.storagePath === storagePath) { UserDraft.remove(itemKind, storagePath, { workspace }) diff --git a/frontend/src/lib/components/copilot/chat/global/workspaceSettingsTools.ts b/frontend/src/lib/components/copilot/chat/global/workspaceSettingsTools.ts index 1f37eb9300..eb7175b20f 100644 --- a/frontend/src/lib/components/copilot/chat/global/workspaceSettingsTools.ts +++ b/frontend/src/lib/components/copilot/chat/global/workspaceSettingsTools.ts @@ -1,14 +1,13 @@ import { z } from 'zod' -import YAML from 'yaml' -import { ApiError, WorkspaceService, type ApplyWorkspaceSettingsDocumentResponse } from '$lib/gen' +import type { ApplyWorkspaceSettingsDocumentResponse } from '$lib/gen' import { createToolDef, type ToolCodeDiff } from '../shared' import type { SessionTool, SessionToolPolicy } from '../sessionCapabilities' import { toStableYaml } from './draftDiff' +import { runOnServer, runServerTool, type SessionToolEffect } from './backendSessionTools' // Both endpoints call `require_admin`. const ADMIN: SessionToolPolicy = ['admin'] -const MAX_READ_CHARS = 20_000 type SettingsChange = ApplyWorkspaceSettingsDocumentResponse['changes'][number] @@ -27,21 +26,6 @@ const setWorkspaceSettingsSchema = z.object({ ) }) -function apiErrorText(e: unknown): string { - if (e instanceof ApiError) { - return typeof e.body === 'string' ? e.body : JSON.stringify(e.body) - } - return e instanceof Error ? e.message : String(e) -} - -function parseSettingsYaml(text: string): Record { - const doc = YAML.parse(text) - if (doc === null || typeof doc !== 'object' || Array.isArray(doc)) { - throw new Error('The yaml must be a mapping of setting keys to values.') - } - return doc -} - // An unset value is left out rather than written as null, so a side with nothing set is empty. function changesSide(changes: SettingsChange[], side: 'before' | 'after'): string { const set = changes.filter((c) => c[side] != null) @@ -72,23 +56,7 @@ export const workspaceSettingsTools: SessionTool<{}>[] = [ showDetails: true, fn: async ({ args, workspace, toolId, toolCallbacks }) => { const parsed = getWorkspaceSettingsSchema.parse(args) - const doc = (await WorkspaceService.getWorkspaceSettingsDocument({ - workspace, - secrets: 'redacted' - })) as Record - let picked = doc - let note = '' - if (parsed.keys?.length) { - picked = Object.fromEntries(parsed.keys.filter((k) => k in doc).map((k) => [k, doc[k]])) - const unset = parsed.keys.filter((k) => !(k in doc)) - if (unset.length) { - note = `\n# Not in the document (unset, or not a setting): ${unset.join(', ')}` - } - } - let text = YAML.stringify(picked, { aliasDuplicateObjects: false, lineWidth: 0 }) + note - if (text.length > MAX_READ_CHARS) { - text = `${text.slice(0, MAX_READ_CHARS)}\n… truncated at ${MAX_READ_CHARS} chars; read the keys you need with \`keys\`.` - } + const { text } = await runOnServer('get_workspace_settings', workspace, args) toolCallbacks.setToolStatus(toolId, { content: parsed.keys?.length ? `Read workspace settings: ${parsed.keys.join(', ')}` @@ -110,27 +78,15 @@ export const workspaceSettingsTools: SessionTool<{}>[] = [ cancellationMessage: 'The user declined: no setting was changed.', showDetails: true, validateBeforeConfirmation: async ({ args, workspace }) => { - let changes: SettingsChange[] - let warnings: string[] | undefined - try { - const document = parseSettingsYaml(setWorkspaceSettingsSchema.parse(args).yaml) - ;({ changes, warnings } = await WorkspaceService.applyWorkspaceSettingsDocument({ - workspace, - requestBody: { document, replace: false, dry_run: true } - })) - } catch (e) { - const error = apiErrorText(e) - return { label: `Invalid settings: ${error}`, result: `Nothing was changed: ${error}` } + setWorkspaceSettingsSchema.parse(args) + const { text, effects } = await runOnServer('set_workspace_settings', workspace, args, { + check_only: true + }) + const facts = settingsFacts(effects) + if (facts.refused) { + return { label: facts.refused, result: text } } - if (changes.length === 0) { - return { - label: 'No settings change', - result: `The settings already hold these values: nothing to change.${ - warnings?.length ? ` Warnings: ${warnings.join(' ')}` : '' - }` - } - } - dryRuns.set(args, changes) + dryRuns.set(args, facts.changes ?? []) return undefined }, onConfirmationRequested: ({ args, toolCallbacks, toolId }) => { @@ -140,31 +96,33 @@ export const workspaceSettingsTools: SessionTool<{}>[] = [ } }, fn: async ({ args, workspace, toolId, toolCallbacks }) => { - const document = parseSettingsYaml(setWorkspaceSettingsSchema.parse(args).yaml) + setWorkspaceSettingsSchema.parse(args) toolCallbacks.setToolStatus(toolId, { content: 'Changing workspace settings...' }) - try { - const { changes, warnings } = await WorkspaceService.applyWorkspaceSettingsDocument({ - workspace, - requestBody: { document, replace: false } - }) - const keys = changes.map((c) => c.key) + const { text, effects } = await runServerTool('set_workspace_settings', { + args, + workspace, + toolId, + toolCallbacks + }) + const { changes } = settingsFacts(effects) + if (changes) { toolCallbacks.setToolStatus(toolId, { - content: `Changed workspace settings: ${keys.join(', ')}`, + content: `Changed workspace settings: ${changes.map((c) => c.key).join(', ')}`, codeDiff: changesDiff(changes) }) - return JSON.stringify({ - success: true, - message: `The user approved and these settings are now applied: ${keys.join(', ')}.`, - ...(warnings?.length ? { warnings } : {}) - }) - } catch (e) { - const error = apiErrorText(e) - toolCallbacks.setToolStatus(toolId, { - content: 'Failed to change workspace settings', - error - }) - return JSON.stringify({ success: false, error }) } + return text } } ] + +function settingsFacts(effects: SessionToolEffect[]): { + refused?: string + changes?: SettingsChange[] +} { + const facts = effects.find((e) => e.type === 'facts') as Record | undefined + return { + refused: facts?.refused as string | undefined, + changes: facts?.changes as SettingsChange[] | undefined + } +} diff --git a/frontend/src/lib/components/copilot/chat/script/core.ts b/frontend/src/lib/components/copilot/chat/script/core.ts index 93ad12ff67..e7053ce141 100644 --- a/frontend/src/lib/components/copilot/chat/script/core.ts +++ b/frontend/src/lib/components/copilot/chat/script/core.ts @@ -32,8 +32,10 @@ import { getDbSchemas } from '$lib/components/apps/components/display/dbtable/me import { scriptLangToEditorLang } from '$lib/scripts' import { getScriptPrompt, getWorkflowAsCodePrompt } from '$system_prompts' -// Score threshold for npm packages search filtering -const SCORE_THRESHOLD = 1000 +// The registry's best match is kept, and the runner-up only when it scores this share of it: +// the scores have no fixed scale (a descriptive query's right answer can score a few hundred, a +// package's own name thousands), so a match is judged against the best one. +const NPM_RUNNER_UP_SHARE = 0.6 // percentage of the context window for documentation of npm packages const DOCS_CONTEXT_PERCENTAGE = 1 // percentage of the context window for types of npm packages @@ -612,8 +614,9 @@ export async function searchExternalIntegrationResources(args: { query: string } `https://registry.npmjs.org/-/v1/search?text=${encodeURIComponent(args.query)}&size=2` ) const data = await result.json() + const best = Math.max(0, ...data.objects.map((r: PackageSearchQuery) => r.searchScore)) const filtered = data.objects.filter( - (r: PackageSearchQuery) => r.searchScore >= SCORE_THRESHOLD + (r: PackageSearchQuery) => r.searchScore >= best * NPM_RUNNER_UP_SHARE ) const model = getCurrentModel() diff --git a/frontend/src/lib/components/copilot/chat/shared.test.ts b/frontend/src/lib/components/copilot/chat/shared.test.ts index 88a5efb7c5..12b0912796 100644 --- a/frontend/src/lib/components/copilot/chat/shared.test.ts +++ b/frontend/src/lib/components/copilot/chat/shared.test.ts @@ -2251,152 +2251,6 @@ describe('getHubIntegrationTool', () => { expect(getHubIntegrationTool.planModeSafe).toBe(true) expect(createSearchHubScriptsTool().planModeSafe).toBe(true) }) - - const doc = { - app: 'confluence', - display_name: 'Confluence', - // Authored knowledge, but nobody has asserted whether the script set was pruned. - curated: null, - metadata_source: 'curated', - meta: { gotchas: ['Auth is Basic with an API token, not the password'] }, - derived: { - api_hosts: [{ host: 'api.atlassian.com', count: 12 }], - style: 'fetch', - languages: { bun: 14 }, - script_counts: { total: 14, by_kind: { script: 13 } }, - top_scripts: [ - { path: 'hub/1/confluence/create_page', summary: 'Create page', language: 'bun' } - ] - }, - resource_types: [{ name: 'confluence', schema: { type: 'object' } }] - } - - // Hand-validated provider knowledge and facts inferred from script bodies must - // stay under separate keys, or the model will report guesses as verified. - it('keeps authored notes apart from what was inferred from the scripts', async () => { - const { IntegrationService } = await import('$lib/gen') - Object.assign(IntegrationService, { getHubIntegrationMeta: vi.fn(async () => doc) }) - - const { getHubIntegrationTool } = await import('./shared') - const parsed = JSON.parse( - await getHubIntegrationTool.fn({ - args: { integration: 'confluence' }, - toolId: 't1', - toolCallbacks: { setToolStatus: vi.fn() } - } as any) - ) - - expect(parsed.verified_provider_notes).toEqual(doc.meta) - expect(parsed.scripts_note).toBeUndefined() - expect(parsed.observed_from_scripts).toEqual({ - api_hosts: ['api.atlassian.com'], - style: 'fetch', - languages: ['bun'], - script_counts: { total: 14, by_kind: { script: 13 } } - }) - }) - - // The hub repo owns this payload, so a hub older than this client can send a - // subset. Reading it must degrade to less content, never to a tool error. - it('returns what an older hub sent instead of failing on the missing sections', async () => { - const { IntegrationService } = await import('$lib/gen') - Object.assign(IntegrationService, { - getHubIntegrationMeta: vi.fn(async () => ({ - app: 'stripe', - display_name: 'Stripe', - curated: false - })) - }) - - const { getHubIntegrationTool } = await import('./shared') - const parsed = JSON.parse( - await getHubIntegrationTool.fn({ - args: { integration: 'stripe' }, - toolId: 't1', - toolCallbacks: { setToolStatus: vi.fn() } - } as any) - ) - - expect(parsed.integration).toBe('stripe') - expect(parsed.observed_from_scripts).toBeUndefined() - expect(parsed.resource_types).toEqual([]) - expect(parsed.example_scripts).toEqual([]) - }) - - // `curated` has three states and only `true` is a licence to call the scripts good - // examples. `false` and the unassessed `null` must both stay silent, or the 212 - // integrations nobody has looked at get characterised anyway. - it.each([ - [true, true], - [false, false], - [null, false] - ])('speaks about the script set only when curated is true (%s)', async (curated, expected) => { - const { IntegrationService } = await import('$lib/gen') - Object.assign(IntegrationService, { - getHubIntegrationMeta: vi.fn(async () => ({ ...doc, curated })) - }) - - const { getHubIntegrationTool } = await import('./shared') - const parsed = JSON.parse( - await getHubIntegrationTool.fn({ - args: { integration: 'confluence' }, - toolId: 't1', - toolCallbacks: { setToolStatus: vi.fn() } - } as any) - ) - - expect(!!parsed.scripts_note).toBe(expected) - }) - - // A hub that times out has said nothing about whether the integration exists, and - // reporting it as absent would stick for the rest of the conversation. - it('does not report a transient hub failure as a missing integration', async () => { - const { IntegrationService } = await import('$lib/gen') - Object.assign(IntegrationService, { - getHubIntegrationMeta: vi.fn(async () => { - throw Object.assign(new Error('Service Unavailable'), { status: 503 }) - }), - listHubIntegrations: vi.fn(async () => [{ name: 'confluence' }]) - }) - - const { getHubIntegrationTool, clearHubIntegrationsCache } = await import('./shared') - clearHubIntegrationsCache() - const parsed = JSON.parse( - await getHubIntegrationTool.fn({ - args: { integration: 'confluence' }, - toolId: 't1', - toolCallbacks: { setToolStatus: vi.fn() } - } as any) - ) - - expect(parsed.error).toContain('Could not reach the hub') - expect(parsed.error).not.toContain('No hub metadata') - }) - - // A hub with no such integration and one too old to serve the endpoint both 404; - // neither may surface as a tool error, since the model can still read scripts. - it('suggests real slugs instead of failing when the integration is unknown', async () => { - const { IntegrationService } = await import('$lib/gen') - Object.assign(IntegrationService, { - getHubIntegrationMeta: vi.fn(async () => { - throw Object.assign(new Error('Not Found'), { status: 404 }) - }), - listHubIntegrations: vi.fn(async () => [{ name: 'stripe' }, { name: 'slack' }]) - }) - - const { getHubIntegrationTool, clearHubIntegrationsCache } = await import('./shared') - clearHubIntegrationsCache() - const parsed = JSON.parse( - await getHubIntegrationTool.fn({ - args: { integration: 'stripe_billing' }, - toolId: 't1', - toolCallbacks: { setToolStatus: vi.fn() } - } as any) - ) - - expect(parsed.error).toContain('stripe_billing') - expect(parsed.suggested_integrations).toEqual(['stripe']) - }) }) describe('processToolCall confirmation hooks', () => { diff --git a/frontend/src/lib/components/copilot/chat/shared.ts b/frontend/src/lib/components/copilot/chat/shared.ts index 71f191c020..0e9fd516a6 100644 --- a/frontend/src/lib/components/copilot/chat/shared.ts +++ b/frontend/src/lib/components/copilot/chat/shared.ts @@ -78,6 +78,7 @@ import { import type { editor as meditor } from 'monaco-editor' import { pendingFolderInstructions, type FolderInstructionsContext } from './folderInstructions' import type { WebSearchSource } from './webSearchResult' +import { runOnServer } from './global/backendSessionTools' // Prettify function for code arguments - extracts and formats code from JSON function prettifyCodeArguments(content: string): string { @@ -1704,7 +1705,7 @@ function hasOptionalProperties(schema: Record | undefined): boolean return false } -const searchHubScriptsSchema = z.object({ +export const searchHubScriptsSchema = z.object({ query: z .string() .optional() @@ -1861,41 +1862,6 @@ const getHubIntegrationToolDef = createToolDef( 'Read how one integration works before writing code against it: the resource type it takes, its auth fields, and its most-used scripts as examples. For an integration a search marked `documented` it also returns provider knowledge checked against the live API: pagination, enums, error codes and gotchas.' ) -/** Enough to show the integration's idiom; search_hub_scripts is the way to find - * a specific one. */ -const MAX_INTEGRATION_EXAMPLES = 5 - -/** How the authored notes were checked — method, per-surface confidence, the instance - * used — is provenance for a human reviewing the hub, and up to a third of the - * document. The verdict and any fields it flags as shaky are the parts a caller can - * act on, so those stay. */ -function withoutProvenance(meta: unknown): unknown { - if (typeof meta !== 'object' || meta === null || Array.isArray(meta)) { - return meta - } - const { validation, ...rest } = meta as Record - const kept = Object.fromEntries( - Object.entries((validation as Record | undefined) ?? {}).filter( - ([key]) => key === 'status' || key === 'low_confidence_fields' - ) - ) - return Object.keys(kept).length ? { ...rest, validation: kept } : rest -} - -/** The hub keeps a resource type's schema in a text column and hands it back as a - * JSON string, so parse it rather than passing an escaped blob to the model. - * Anything already structured goes through untouched. */ -function parseResourceTypeSchema(schema: unknown): unknown { - if (typeof schema !== 'string') { - return schema - } - try { - return JSON.parse(schema) - } catch { - return schema - } -} - export const HUB_SYNC_INSTRUCTIONS = 'Tell the user: no resource of a missing type can be created until a superadmin syncs resource types from the hub, with "Sync resource types with Hub" in the Add resource dialog of the Resources page.' @@ -1904,76 +1870,20 @@ export const getHubIntegrationTool = { def: getHubIntegrationToolDef, // Reads one hub document over a GET and writes nothing, so planning may use it. planModeSafe: true, - fn: async ({ args, toolId, toolCallbacks }) => { + fn: async ({ args, workspace, toolId, toolCallbacks }) => { const { integration } = getHubIntegrationSchema.parse(args) toolCallbacks.setToolStatus(toolId, { content: `Reading the ${integration} integration...` }) - - let doc: Awaited> - try { - doc = await IntegrationService.getHubIntegrationMeta({ app: integration }) - } catch (err) { - // Only a 404 means the integration is absent — an unknown slug, or a hub - // predating the endpoint. Reporting a timeout the same way would teach the - // model that a real integration does not exist for the rest of the chat. - const absent = (err as { status?: number } | undefined)?.status === 404 - const label = absent - ? `No hub integration named ${integration}` - : `Could not reach the hub for ${integration}` - toolCallbacks.setToolStatus(toolId, { content: label }) - const suggested = await suggestHubIntegrations(integration) - return JSON.stringify({ - error: absent - ? `No hub metadata for "${integration}".` - : `Could not reach the hub for "${integration}"; it may still exist. Read its scripts instead, or try again.`, - suggested_integrations: suggested - }) - } - - toolCallbacks.setToolStatus(toolId, { content: `Read the ${doc.display_name} integration` }) - // The hub owns this response's shape and can be older than this client, so - // every section is read as optional: a hub that sends less should return less, - // not fail the call. - const derived = doc.derived - return JSON.stringify({ - integration: doc.app, - display_name: doc.display_name, - ...(doc.description ? { description: doc.description } : {}), - ...(doc.docs_url ? { docs_url: doc.docs_url } : {}), - // Authored provider knowledge and facts inferred from the scripts stay - // separate: only the former was checked against the live API. - ...(doc.meta ? { verified_provider_notes: withoutProvenance(doc.meta) } : {}), - ...(derived - ? { - observed_from_scripts: { - api_hosts: derived.api_hosts?.map((h) => h.host) ?? [], - style: derived.style, - languages: Object.keys(derived.languages ?? {}), - script_counts: derived.script_counts - } - } - : {}), - // `curated` is three-state: only an integration's own meta.json asserts it, and - // anything else means nobody has assessed the script set. Speak only for a - // positive assertion, so an unassessed integration is never characterised - // either way; `script_counts` below is the factual signal for the rest. - ...(doc.curated === true - ? { - scripts_note: - 'These scripts were pruned to idiomatic actions rather than generated one per API endpoint, so they are worth following as style examples.' - } - : {}), - resource_types: (doc.resource_types ?? []).map((rt) => ({ - name: rt.name, - ...(rt.description ? { description: rt.description } : {}), - schema: parseResourceTypeSchema(rt.schema) - })), - example_scripts: (derived?.top_scripts ?? []).slice(0, MAX_INTEGRATION_EXAMPLES).map((s) => ({ - path: s.path, - summary: s.summary, - ...(s.description ? { description: s.description } : {}), - ...(s.language ? { language: s.language } : {}) - })) + const { text } = await runOnServer('get_hub_integration', workspace, args) + const doc = JSON.parse(text) + toolCallbacks.setToolStatus(toolId, { + content: + doc.error === undefined + ? `Read the ${doc.display_name} integration` + : doc.error.startsWith('No hub metadata') + ? `No hub integration named ${integration}` + : `Could not reach the hub for ${integration}` }) + return text } } satisfies SessionTool<{}> @@ -2692,6 +2602,18 @@ async function loadDeployedScriptForFlowStep( } } +/** What a test run of an AI decision step supplies: the state. The provider and questions + * are the step's own. */ +export function decisionStepRunSchema(): Record { + return { + $schema: 'https://json-schema.org/draft/2020-12/schema', + type: 'object', + properties: { state: AI_DECISION_SCHEMA.properties.state }, + required: ['state'], + order: ['state'] + } +} + export async function resolveFlowStepRun({ flowValue, stepId, @@ -2833,14 +2755,7 @@ export async function resolveFlowStepRun({ // Run alone as a one-step flow preview, as the editor's step test does: the inputs given here // override the step's own, so a state that reads an earlier step can be passed directly. runnableKind: 'flow', - // What a run supplies: the state. The provider and questions are the step's own. - schema: { - $schema: 'https://json-schema.org/draft/2020-12/schema', - type: 'object', - properties: { state: AI_DECISION_SCHEMA.properties.state }, - required: ['state'], - order: ['state'] - }, + schema: decisionStepRunSchema(), startMessage: `Starting test run of AI decision step "${stepId}"...`, startJob: async (args) => JobService.runFlowPreview({ diff --git a/frontend/src/lib/components/flows/linkedAgentDrafts.ts b/frontend/src/lib/components/flows/linkedAgentDrafts.ts index 1a5b7ac7f1..0031353098 100644 --- a/frontend/src/lib/components/flows/linkedAgentDrafts.ts +++ b/frontend/src/lib/components/flows/linkedAgentDrafts.ts @@ -78,17 +78,15 @@ export function agentDraftState( export class AgentDraftUnavailable extends Error {} /** - * An agent's resource together with the draft a run of it would use. + * Land the agent editor's pending save of `path`, or refuse: a read of the stored draft would + * otherwise be older than the edit still held in the browser. * - * The flush is what makes the answer current. Autosave is debounced by 1.5s (10s ceiling), and + * The flush is what makes the stored row current. Autosave is debounced by 1.5s (10s ceiling), and * closing the agent editor releases the in-memory cell without cancelling that pending POST — so * testing or deploying right after closing would otherwise read a row the last edits have not * reached yet. `flush` replays the parked save and is a no-op when there is none. */ -export async function fetchAgentWithDraft( - path: string, - workspace: string -): Promise<{ response: Resource; draft: AgentResourceState | undefined }> { +export async function settleAgentDraft(path: string, workspace: string): Promise { const query = { workspace, itemKind: 'resource' as const, path } await UserDraftDbSyncer.flush(query) // `flush` resolves whether or not the save actually landed: `postSave` catches network and @@ -106,6 +104,14 @@ export async function fetchAgentWithDraft( `The unsaved changes to ${path} could not be saved because it was edited elsewhere. Open the agent to resolve it.` ) } +} + +/** An agent's resource together with the draft a run of it would use, its pending save landed. */ +export async function fetchAgentWithDraft( + path: string, + workspace: string +): Promise<{ response: Resource; draft: AgentResourceState | undefined }> { + await settleAgentDraft(path, workspace) const response = await ResourceService.getResource({ workspace, path, getDraft: true }) return { response, draft: agentDraftState(response, path, workspace) } } diff --git a/frontend/src/lib/components/resourceTypeDisplay.ts b/frontend/src/lib/components/resourceTypeDisplay.ts index ad01c05bb9..5fdd47b1a1 100644 --- a/frontend/src/lib/components/resourceTypeDisplay.ts +++ b/frontend/src/lib/components/resourceTypeDisplay.ts @@ -81,7 +81,7 @@ export function sortResourceTypesByMatch( * that carry a capital inside the word. Anything absent is capitalized, which is right for * the great majority of types (`stripe` -> `Stripe`). */ -const RESOURCE_TYPE_WORDS: Record = { +export const RESOURCE_TYPE_WORDS: Record = { ai: 'AI', ai21: 'AI21', amqp: 'AMQP', diff --git a/frontend/src/lib/components/runArgsCases.test.ts b/frontend/src/lib/components/runArgsCases.test.ts new file mode 100644 index 0000000000..2cbd3aab4e --- /dev/null +++ b/frontend/src/lib/components/runArgsCases.test.ts @@ -0,0 +1,67 @@ +// The MCP server settles a run's arguments in Rust (`ai_session_tools/runs.rs`), read against +// the same cases by `run_args_match_the_shared_cases`. +import { readFileSync } from 'node:fs' +import { describe, expect, it, vi } from 'vitest' + +const stored: string[] = [] + +vi.mock('$lib/gen', () => ({ + VariableService: { + createVariable: vi.fn(async ({ requestBody }: any) => { + stored.push(requestBody.value) + return requestBody.path + }) + } +})) + +vi.mock('$lib/stores', async () => { + const { writable } = await import('svelte/store') + return { workspaceStore: writable('test-ws'), userStore: writable({ username: 'ada' }) } +}) + +import { + dropUndeclaredArgs, + enforceDisabledDefaults, + redactFileArgs, + redactSecretArgs +} from './job_args' +import { processSecretArgs } from './secretArgUtils' + +const { cases } = JSON.parse( + readFileSync( + new URL( + '../../../../backend/windmill-api/src/ai_session_tools/run_args_cases.json', + import.meta.url + ), + 'utf8' + ) +) + +describe('run arguments, against the cases the server shares', () => { + for (const c of cases) { + it(c.name, async () => { + if (c.declared) { + const { args, undeclaredKeys } = dropUndeclaredArgs(c.args, c.schema) + expect(args).toEqual(c.declared.args) + expect(undeclaredKeys).toEqual(c.declared.undeclared) + } + if (c.disabled_defaults) { + const { args, resetKeys } = enforceDisabledDefaults(c.args, c.schema) + expect(args).toEqual(c.disabled_defaults.args) + expect(resetKeys).toEqual(c.disabled_defaults.reset) + } + if (c.secrets) { + stored.length = 0 + const out = await processSecretArgs(c.args, c.schema, 'test-ws') + const named = JSON.parse( + JSON.stringify(out).replace(/u\/ada\/secret_arg\/[A-Za-z0-9]+/g, '') + ) + expect(named).toEqual(c.secrets.args) + expect([...stored].sort()).toEqual(c.secrets.stored) + } + if (c.redacted) { + expect(redactFileArgs(redactSecretArgs(c.args, c.schema), c.schema)).toEqual(c.redacted) + } + }) + } +}) diff --git a/system_prompts/README.md b/system_prompts/README.md index 32c0c521c4..14a940ea0f 100644 --- a/system_prompts/README.md +++ b/system_prompts/README.md @@ -10,7 +10,7 @@ are built from it: `cli/src/guidance/skills.gen.ts`, next to the `AGENTS.wmill.md` template in `cli/src/guidance/core.ts`. -The workspace MCP server (`backend/windmill-api/src/mcp/session/`) is a third reader, one step +The workspace MCP server (`backend/windmill-api/src/ai_session_tools/`) is a third reader, one step removed: it serves the global chat's own `get_instructions` texts and system prompt, rendered for an external client by `npm run gen:mcp-session` in `frontend/`. Rerun that after `generate.py`.