diff --git a/.github/workflows/sdk-tests.yml b/.github/workflows/sdk-tests.yml new file mode 100644 index 0000000000..efcbcd6ec1 --- /dev/null +++ b/.github/workflows/sdk-tests.yml @@ -0,0 +1,61 @@ +# The python and typescript SDK unit suites, on release tags only: they guard +# what gets published to npm / PyPI / JSR, and a tag is the moment that decides +# it. +# +# This runs alongside the publish workflows rather than ahead of them, so it +# reports a broken SDK rather than holding one back. Gating would mean putting +# the job inside each publish workflow, since Actions cannot express `needs` +# across workflows. +name: SDK Tests + +on: + workflow_dispatch: + push: + tags: + - "v*" + +jobs: + typescript-client: + runs-on: ubuntu-latest + steps: + - name: Checkout code + uses: actions/checkout@v4 + + - name: Setup Bun + uses: oven-sh/setup-bun@v2 + with: + bun-version: latest + + # No build step: these suites are deliberately free of the generated API + # client, so they run against the sources as committed. + - name: Run tests + working-directory: ./typescript-client + run: bun test --timeout 120000 tests/ + + python-client: + runs-on: ubuntu-latest + steps: + - name: Checkout code + uses: actions/checkout@v4 + + - name: Setup uv + uses: astral-sh/setup-uv@v5 + + # The interpreter is named explicitly: on a clean checkout uv picks the + # runner's system python and stops with "not compatible with the locked + # Python requirement" rather than fetching one. Keep in step with + # `requires-python` in uv.lock. + # + # Note this is not the version a worker runs the SDK on — those are 3.12. + # `uv.lock` asks for >=3.14, so pinning lower means regenerating it, which + # is worth doing separately. + - name: Install the interpreter the lockfile requires + run: uv python install 3.14 + + # `--frozen` so a drifted lockfile fails here rather than quietly + # resolving to something nobody has run. + - name: Run tests + working-directory: ./python-client/wmill + env: + PYTHONPATH: . + run: uv run --frozen --python 3.14 pytest tests/ -q diff --git a/AGENTS.md b/AGENTS.md index d7ebf2452d..90c0060ae2 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -12,6 +12,7 @@ Open-source platform for internal tools, workflows, API integrations, background ## Documentation - **Validation**: `docs/validation.md` — what checks to run based on what you changed +- **Unreleased SDK changes**: `docs/wac-sdk-e2e.md` — exercising a client change on a real worker - **Enterprise**: `docs/enterprise.md` — EE file conventions and PR workflow - **Backend patterns**: use the `rust-backend` skill when writing Rust code - **Frontend patterns**: use the `svelte-frontend` skill when writing Svelte code. Do NOT edit svelte files unless you have read that skill. diff --git a/CHANGELOG.md b/CHANGELOG.md index 24b4b0cc25..a69d7b0389 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,67 @@ # Changelog +## [1.775.2](https://github.com/windmill-labs/windmill/compare/v1.775.1...v1.775.2) (2026-07-29) + + +### Bug Fixes + +* **cli:** compile runes in .svelte.ts / .svelte.js modules ([#10400](https://github.com/windmill-labs/windmill/issues/10400)) ([9c37b02](https://github.com/windmill-labs/windmill/commit/9c37b0217c79390271d9855abe135df02f35c41f)) +* distinguish waiting-on-user from streaming in ai sessions ([#10396](https://github.com/windmill-labs/windmill/issues/10396)) ([a17ccbd](https://github.com/windmill-labs/windmill/commit/a17ccbdd0d3c9f2f6234c4cc70d0917e83eeaceb)) +* surface the real reason git sync settings saves are rejected ([#10398](https://github.com/windmill-labs/windmill/issues/10398)) ([33d2845](https://github.com/windmill-labs/windmill/commit/33d28456cc8ef9f64301eb3dd6dad6d7325018e3)) + +## [1.775.1](https://github.com/windmill-labs/windmill/compare/v1.775.0...v1.775.1) (2026-07-28) + + +### Bug Fixes + +* **ai:** render pending parallel tool calls as faded queued cards ([#10208](https://github.com/windmill-labs/windmill/issues/10208)) ([d842f76](https://github.com/windmill-labs/windmill/commit/d842f765a457f0c80a2cd1eb06bc703167512160)) +* **frontend:** add the preprocessor node before the error handler markers ([#10395](https://github.com/windmill-labs/windmill/issues/10395)) ([39dd411](https://github.com/windmill-labs/windmill/commit/39dd411481d450b1cdb92ddba3a8b613ddd2218a)) + +## [1.775.0](https://github.com/windmill-labs/windmill/compare/v1.774.0...v1.775.0) (2026-07-28) + + +### Features + +* add github dark mode variant switchable in user settings ([#10002](https://github.com/windmill-labs/windmill/issues/10002)) ([bd24664](https://github.com/windmill-labs/windmill/commit/bd246644c962f754a8477353d49ed39ba9b87ffd)) +* validate and make explicit the hub project's resource type export ([#10388](https://github.com/windmill-labs/windmill/issues/10388)) ([91b8ce5](https://github.com/windmill-labs/windmill/commit/91b8ce581a15baf7fc9ea4e983b50f233cf5e023)) + + +### Bug Fixes + +* render the flow editor's error handler node as an inert run marker ([#10391](https://github.com/windmill-labs/windmill/issues/10391)) ([bff6545](https://github.com/windmill-labs/windmill/commit/bff654596fb9cf8b086919367614ba0915cd7b87)) +* truncate strings on char boundaries to avoid panics on multibyte input ([#10390](https://github.com/windmill-labs/windmill/issues/10390)) ([faa2aaf](https://github.com/windmill-labs/windmill/commit/faa2aaf214444f4fbb129f10a46060d4a8ab3ae8)) + +## [1.774.0](https://github.com/windmill-labs/windmill/compare/v1.773.0...v1.774.0) (2026-07-28) + + +### Features + +* **ai-chat:** add get_flow_run_details tool for per-step flow run results ([#10374](https://github.com/windmill-labs/windmill/issues/10374)) ([c12e7c3](https://github.com/windmill-labs/windmill/commit/c12e7c34310b417e89f94efc3c412b3c7278344b)) +* **ai-chat:** recall queued/last message into the composer ([#10191](https://github.com/windmill-labs/windmill/issues/10191)) ([ad0fef4](https://github.com/windmill-labs/windmill/commit/ad0fef4a2e0ea63ae00562ff10d9ebbae66c42d0)) +* bulk discard selected drafts on the compare & deploy page ([#10372](https://github.com/windmill-labs/windmill/issues/10372)) ([ac970ef](https://github.com/windmill-labs/windmill/commit/ac970efa1698e42ad1e7b3cea5f06f18a567269d)) +* reusable AI agent steps with rigid linking and edit/fork ([#9825](https://github.com/windmill-labs/windmill/issues/9825)) ([3b95a2d](https://github.com/windmill-labs/windmill/commit/3b95a2d0967544640feb53fe962a8215e049f4d2)) + + +### Bug Fixes + +* **ai-agent:** keep tool description through flow deployment ([#10373](https://github.com/windmill-labs/windmill/issues/10373)) ([8d684c0](https://github.com/windmill-labs/windmill/commit/8d684c0b2305798a781685ca022b1b9dc0731cf0)) +* **ai-chat:** make hub script paths readable from the global chat ([#10381](https://github.com/windmill-labs/windmill/issues/10381)) ([5feec9b](https://github.com/windmill-labs/windmill/commit/5feec9b4cd3cdd4eed3edf5be0fc42dabfeb13d2)) +* **ai:** resolve deployment-pinned Azure base URLs to the v1 surface ([#10362](https://github.com/windmill-labs/windmill/issues/10362)) ([621718b](https://github.com/windmill-labs/windmill/commit/621718b32fe5322638039d4a560b845e11b38008)) +* **apps:** stop cross-origin isolating the raw app viewer ([#10370](https://github.com/windmill-labs/windmill/issues/10370)) ([3c2dab9](https://github.com/windmill-labs/windmill/commit/3c2dab9f8f302e2f80ef800a4271bfe5aa152767)) +* **datatable:** provision the replication user on managed postgres ([#10375](https://github.com/windmill-labs/windmill/issues/10375)) ([1b6b2aa](https://github.com/windmill-labs/windmill/commit/1b6b2aa8598100b5327ecc3e16f1541db95f9e47)) +* **frontend:** preserve top-level flow settings in AI flow tools ([#10369](https://github.com/windmill-labs/windmill/issues/10369)) ([a544dfd](https://github.com/windmill-labs/windmill/commit/a544dfde9a5f409247e78ef1c1aac2b9699d9ac8)) +* home kind filter no longer resets a fork or reloads the page ([#10384](https://github.com/windmill-labs/windmill/issues/10384)) ([2a2ef41](https://github.com/windmill-labs/windmill/commit/2a2ef411525f032743af93f143c10ad373c20fc8)) +* loop "Test an iteration" progress bar, while-loop modules and schema ([#10357](https://github.com/windmill-labs/windmill/issues/10357)) ([350eb66](https://github.com/windmill-labs/windmill/commit/350eb66560d2626c4100a21edb2101cdfa4a0a72)) +* make same worker mutually exclusive with retries and sleeps in the flow editor ([#10379](https://github.com/windmill-labs/windmill/issues/10379)) ([ecde945](https://github.com/windmill-labs/windmill/commit/ecde94567c5ec43002a06d2d765927e0d89640a4)) +* mark Setup URL as required in self-managed GitHub App instructions ([#10380](https://github.com/windmill-labs/windmill/issues/10380)) ([c4e7568](https://github.com/windmill-labs/windmill/commit/c4e75683a8a346f4cb39038f3045c94f3a9621b2)) +* raw apps with no stylesheet were permanently un-deployable ([#10364](https://github.com/windmill-labs/windmill/issues/10364)) ([8a96e3a](https://github.com/windmill-labs/windmill/commit/8a96e3a4ec47c0a28dd422b31388f7f0cced8e67)) +* show draft badge and disable the toggle for draft-only triggers ([#10155](https://github.com/windmill-labs/windmill/issues/10155)) ([b0c7e09](https://github.com/windmill-labs/windmill/commit/b0c7e0917376248cda0f894b138075d82f19f620)) +* surface postgres publication errors as 400 instead of 500 ([#10376](https://github.com/windmill-labs/windmill/issues/10376)) ([5e52346](https://github.com/windmill-labs/windmill/commit/5e523462423f73419df25e4d7aa1cd2e77c0228a)) +* surface the real postgres error when data table migrations fail ([#10371](https://github.com/windmill-labs/windmill/issues/10371)) ([fbf9f04](https://github.com/windmill-labs/windmill/commit/fbf9f04e107c674474f61c1559a2f1e56dfeeaba)) +* **wac:** one failure record for tasks and steps, in every round ([#10368](https://github.com/windmill-labs/windmill/issues/10368)) ([aeaea57](https://github.com/windmill-labs/windmill/commit/aeaea57ca1a0fac02157035530e84c3588a7aec1)) +* **wac:** report a task failure the child round's body catches ([#10366](https://github.com/windmill-labs/windmill/issues/10366)) ([727d22b](https://github.com/windmill-labs/windmill/commit/727d22b9a1f9520788afb9b519be8fcd8fd64319)) +* **wac:** return the checkpointed value from step(), not the live object ([#10367](https://github.com/windmill-labs/windmill/issues/10367)) ([044ce39](https://github.com/windmill-labs/windmill/commit/044ce39e5f9c8207287e19a0d2eefa22b79188f2)) + ## [1.773.0](https://github.com/windmill-labs/windmill/compare/v1.772.0...v1.773.0) (2026-07-27) diff --git a/ai_evals/adapters/frontend/mockBackend.ts b/ai_evals/adapters/frontend/mockBackend.ts index 64cacabd58..40a6dfa4ad 100644 --- a/ai_evals/adapters/frontend/mockBackend.ts +++ b/ai_evals/adapters/frontend/mockBackend.ts @@ -809,6 +809,142 @@ export function listBenchmarkMcpTools(): EndpointTool[] { return BENCHMARK_MCP_TOOLS } +/** A stand-in Windmill hub. `search_hub_scripts` and a `hub/` read go out over + * relative `/api/...` fetches, which have no origin here, so without these the + * hub tools throw and no case can exercise hub reuse. Serving fixtures rather + * than the live hub also keeps assertions on script content stable as the real + * hub republishes new versions. */ +const BENCHMARK_HUB_SCRIPTS = [ + { + version_id: 22235, + app: 'holded', + summary: 'Send Document', + terms: 'holded invoice document send email mail', + language: 'bun', + content: `//native +type Holded = { + apiKey: string; +}; +/** + * Send Document + * Send a specific document by email. + */ +export async function main( + auth: Holded, + docType: string, + documentId: string, + body: { + mailTemplateId?: string; + emails: string; + subject?: string; + message?: string; + docIds?: string; + }, +) { + const url = new URL( + \`https://api.holded.com/api/invoicing/v1/documents/\${docType}/\${documentId}/send\`, + ); + + const response = await fetch(url, { + method: "POST", + headers: { + "Content-Type": "application/json", + key: auth.apiKey, + }, + body: JSON.stringify(body), + }); + if (!response.ok) { + const text = await response.text(); + throw new Error(\`\${response.status} \${text}\`); + } + return await response.json(); +} +`, + schema: { + type: 'object', + required: ['auth', 'docType', 'documentId', 'body'], + properties: { + auth: { type: 'object', format: 'resource-holded' }, + docType: { type: 'string' }, + documentId: { type: 'string' }, + body: { type: 'object' } + } + } + }, + { + version_id: 28294, + app: 'discord', + summary: 'Send a message to Discord using Webhook', + terms: 'discord webhook message send chat channel', + language: 'bunnative', + content: `//native + +type DiscordWebhook = { + webhook_url: string; +}; +export async function main(discord_webhook: DiscordWebhook, message: string) { + const response = await fetch(\`\${discord_webhook.webhook_url}?wait=true\`, { + method: "POST", + headers: { "Content-Type": "application/json" }, + body: JSON.stringify({ content: message }), + }); + if (!response.ok) { + throw new Error(\`\${response.status} \${await response.text()}\`); + } + return await response.json(); +} +`, + schema: { + type: 'object', + required: ['discord_webhook', 'message'], + properties: { + discord_webhook: { type: 'object', format: 'resource-discord_webhook' }, + message: { type: 'string' } + } + } + } +] + +/** Naive whole-word overlap — enough to rank a handful of fixtures for a natural + * query without pulling an embedding model into the benchmark. Every frontend eval + * shares this handler, so the bar to match is deliberately high: naming the + * integration, or overlapping on three meaningful words. A looser bar answers + * "send a Slack message" with the Discord fixture, handing an unrelated case a + * plausible-looking wrong integration. */ +function searchBenchmarkHubScripts(text: string) { + const tokens = new Set( + text + .toLowerCase() + .split(/[^a-z0-9]+/) + .filter((token) => token.length > 2) + ) + return BENCHMARK_HUB_SCRIPTS.map((script) => { + const words = new Set( + `${script.app} ${script.summary} ${script.terms}`.toLowerCase().split(/[^a-z0-9]+/) + ) + const score = [...tokens].filter((token) => words.has(token)).length + return { script, score, namesApp: tokens.has(script.app) } + }) + .filter((entry) => entry.namesApp || entry.score >= 3) + .sort((a, b) => b.score - a.score) + .map(({ script }, index) => ({ + ask_id: script.version_id, + id: script.version_id, + version_id: script.version_id, + summary: script.summary, + app: script.app, + kind: 'script', + score: 1 - index * 0.01 + })) +} + +/** The hub keys a script by its version id; the app and slug segments that + * follow are descriptive, so match on the id exactly as the real hub does. */ +function getBenchmarkHubScript(path: string) { + const versionId = Number(path.replace(/^\/api\/scripts\/hub\/get_full\/hub\//, '').split('/')[0]) + return BENCHMARK_HUB_SCRIPTS.find((script) => script.version_id === versionId) +} + const BENCHMARK_WORKERS = [ { worker: 'wk-benchmark-1', @@ -837,10 +973,16 @@ const BENCHMARK_WORKERS = [ * intercepting it with a synthetic 404 sends the model into retry loops. */ export function hasBenchmarkApiHandler(url: string): boolean { const path = url.split('?')[0] - return path === '/api/workers/list' || /^\/api\/w\/[^/]+\/jobs\/queue\/list$/.test(path) + return ( + path === '/api/workers/list' || + /^\/api\/w\/[^/]+\/jobs\/queue\/list$/.test(path) || + path === '/api/embeddings/query_hub_scripts' || + path.startsWith('/api/scripts/hub/get_full/') + ) } -/** Answer a relative `/api/...` fetch issued by the API catalog executor. */ +/** Answer a relative `/api/...` fetch — from the API catalog executor, or from the + * chat's hub tools. */ export function handleBenchmarkApiFetch(url: string): Response { const path = url.split('?')[0] if (path === '/api/workers/list') { @@ -849,5 +991,21 @@ export function handleBenchmarkApiFetch(url: string): Response { if (/^\/api\/w\/[^/]+\/jobs\/queue\/list$/.test(path)) { return Response.json([]) } + if (path === '/api/embeddings/query_hub_scripts') { + const text = new URLSearchParams(url.split('?')[1] ?? '').get('text') ?? '' + return Response.json(searchBenchmarkHubScripts(text)) + } + if (path.startsWith('/api/scripts/hub/get_full/')) { + const script = getBenchmarkHubScript(path) + if (!script) { + return Response.json({ error: 'hub script not found' }, { status: 404 }) + } + return Response.json({ + content: script.content, + language: script.language, + schema: script.schema, + summary: script.summary + }) + } return Response.json({ error: `no benchmark handler for ${path}` }, { status: 404 }) } diff --git a/ai_evals/cases/global.yaml b/ai_evals/cases/global.yaml index 72739d2778..005c677e79 100644 --- a/ai_evals/cases/global.yaml +++ b/ai_evals/cases/global.yaml @@ -1877,3 +1877,49 @@ stringIncludesAnyOf: - email skipJudge: true + +# --- Windmill Hub reuse (search_hub_scripts + read_workspace_item on a hub/ path) --- +# Holded's API is obscure enough that a model writing from memory cannot reproduce +# its endpoint and `key` auth header — so the draft's fidelity to the published +# script is what proves the hub content was actually fetched, not guessed. +- id: global-hub1-reuse-hub-script + prompt: |- + I want to email one of my Holded invoices to a customer from Windmill. + There is already a script for that on the Windmill hub — reuse it instead of writing + your own, and save it as a draft script at `f/evals/global/holded_send_document`. + Leave it as an AI draft; do not deploy it. + initial: ai_evals/fixtures/frontend/global/initial/user_admin_evals_folder.json + runtime: + maxTurns: 12 + validate: + draftCountExactly: 1 + requiredDrafts: + - type: script + path: f/evals/global/holded_send_document + language: bun + valueIncludes: + - api.holded.com/api/invoicing/v1/documents + # `mailTemplateId` is an optional field of the published script's body + # that a model writing from memory does not invent, so it is what + # separates reusing the hub script from re-deriving one that merely + # hits the same endpoint. + - mailTemplateId + toolExpect: + requiredToolsUsed: + - search_hub_scripts + - read_workspace_item + - write_script + forbiddenToolsUsed: + - deploy_workspace_item + - delete_workspace_item + toolCallArgs: + - tool: read_workspace_item + field: path + stringIncludesAnyOf: + - hub/ + judgeChecklist: + - the draft sends an existing Holded document by email rather than creating one + - the request targets Holded's document send endpoint, not an invented URL + - authentication uses Holded's own key header rather than a bearer token + - the document type, document id, and recipient emails are inputs to the script + - the result stays an AI draft and is not deployed diff --git a/backend/.sqlx/query-2ef832a6d8e9feb83de6706b0ffe8e849e56f8db795df2ad4d907cdb3c9fce60.json b/backend/.sqlx/query-2ef832a6d8e9feb83de6706b0ffe8e849e56f8db795df2ad4d907cdb3c9fce60.json new file mode 100644 index 0000000000..183c620240 --- /dev/null +++ b/backend/.sqlx/query-2ef832a6d8e9feb83de6706b0ffe8e849e56f8db795df2ad4d907cdb3c9fce60.json @@ -0,0 +1,24 @@ +{ + "db_name": "PostgreSQL", + "query": "SELECT DISTINCT flow_step_id as \"flow_step_id!\" FROM v2_job\n WHERE parent_job = $1 AND workspace_id = $2 AND flow_step_id IS NOT NULL\n AND ($3::text[] IS NULL OR tag = ANY($3))\n ORDER BY flow_step_id", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "flow_step_id!", + "type_info": "Varchar" + } + ], + "parameters": { + "Left": [ + "Uuid", + "Text", + "TextArray" + ] + }, + "nullable": [ + true + ] + }, + "hash": "2ef832a6d8e9feb83de6706b0ffe8e849e56f8db795df2ad4d907cdb3c9fce60" +} diff --git a/backend/.sqlx/query-48c2eeca50aedc1895256f615aecc8b68627c8ed9a6da969674734854ef2742a.json b/backend/.sqlx/query-48c2eeca50aedc1895256f615aecc8b68627c8ed9a6da969674734854ef2742a.json new file mode 100644 index 0000000000..93b908a567 --- /dev/null +++ b/backend/.sqlx/query-48c2eeca50aedc1895256f615aecc8b68627c8ed9a6da969674734854ef2742a.json @@ -0,0 +1,37 @@ +{ + "db_name": "PostgreSQL", + "query": "SELECT j.id,\n COALESCE(c.status::text,\n CASE WHEN q.running AND q.suspend > 0 THEN 'suspended'\n WHEN q.running THEN 'running'\n ELSE 'queued' END) as \"status!\",\n COALESCE((\n SELECT m->'value'->>'type'\n FROM v2_job parent_j\n LEFT JOIN flow_version fv ON fv.id = parent_j.runnable_id\n AND parent_j.kind::text = 'flow'\n LEFT JOIN flow f ON f.path = parent_j.runnable_path\n AND f.workspace_id = parent_j.workspace_id\n LEFT JOIN flow_node fn ON fn.id = parent_j.runnable_id\n CROSS JOIN LATERAL jsonb_array_elements(\n COALESCE(parent_j.raw_flow, fv.value, f.value, fn.flow)->'modules'\n ) m\n WHERE parent_j.id = $1\n AND m->>'id' = $3\n LIMIT 1\n ), '')::text as \"parent_module_type!\"\n FROM v2_job j\n LEFT JOIN v2_job_completed c ON c.id = j.id\n LEFT JOIN v2_job_queue q ON q.id = j.id\n LEFT JOIN (\n SELECT fj.jid, fj.ord\n FROM (SELECT COALESCE(\n (SELECT flow_status FROM v2_job_completed WHERE id = $1),\n (SELECT flow_status FROM v2_job_status WHERE id = $1)\n ) AS fs) pf\n CROSS JOIN LATERAL (\n SELECT m FROM jsonb_array_elements(pf.fs->'modules') m\n WHERE m->>'id' = $3\n LIMIT 1\n ) md\n CROSS JOIN LATERAL jsonb_array_elements_text(md.m->'flow_jobs')\n WITH ORDINALITY fj(jid, ord)\n ) pos ON pos.jid = j.id::text\n WHERE j.parent_job = $1 AND j.workspace_id = $2 AND j.flow_step_id = $3\n AND ($4::text[] IS NULL OR j.tag = ANY($4))\n ORDER BY pos.ord NULLS LAST, j.id", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "id", + "type_info": "Uuid" + }, + { + "ordinal": 1, + "name": "status!", + "type_info": "Text" + }, + { + "ordinal": 2, + "name": "parent_module_type!", + "type_info": "Text" + } + ], + "parameters": { + "Left": [ + "Uuid", + "Text", + "Text", + "TextArray" + ] + }, + "nullable": [ + false, + null, + null + ] + }, + "hash": "48c2eeca50aedc1895256f615aecc8b68627c8ed9a6da969674734854ef2742a" +} diff --git a/backend/.sqlx/query-5b95f464d492227ffe8a449844da00a6539788f9be647d6b52f01cbb2967473e.json b/backend/.sqlx/query-5b95f464d492227ffe8a449844da00a6539788f9be647d6b52f01cbb2967473e.json deleted file mode 100644 index 3b1d7fae8c..0000000000 --- a/backend/.sqlx/query-5b95f464d492227ffe8a449844da00a6539788f9be647d6b52f01cbb2967473e.json +++ /dev/null @@ -1,83 +0,0 @@ -{ - "db_name": "PostgreSQL", - "query": "WITH RECURSIVE job_tree AS (\n SELECT j.id, j.kind::text, j.flow_step_id, j.parent_job,\n '' as path_label, 0 as depth,\n j.id::text as id_path,\n ''::text as parent_module_type\n FROM v2_job j\n WHERE j.id = $2 AND j.workspace_id = $1\n UNION ALL\n SELECT j.id, j.kind::text, j.flow_step_id, j.parent_job,\n CASE\n WHEN jt.path_label = '' THEN COALESCE(j.flow_step_id, '')\n ELSE jt.path_label || '/' || COALESCE(j.flow_step_id, '')\n END,\n jt.depth + 1,\n jt.id_path || '/' || j.id::text,\n COALESCE((\n SELECT m->'value'->>'type'\n FROM v2_job parent_j\n LEFT JOIN flow f ON f.path = parent_j.runnable_path\n AND f.workspace_id = parent_j.workspace_id\n LEFT JOIN flow_node fn ON fn.id = parent_j.runnable_id\n CROSS JOIN LATERAL jsonb_array_elements(\n COALESCE(parent_j.raw_flow, f.value, fn.flow)->'modules'\n ) m\n WHERE parent_j.id = jt.id\n AND m->>'id' = j.flow_step_id\n LIMIT 1\n ), '')::text\n FROM v2_job j\n JOIN job_tree jt ON j.parent_job = jt.id\n WHERE j.workspace_id = $1\n ),\n with_sibling_index AS (\n SELECT jt.*,\n ROW_NUMBER() OVER (\n PARTITION BY jt.parent_job, jt.flow_step_id\n ORDER BY jt.id\n ) as sibling_index,\n COUNT(*) OVER (\n PARTITION BY jt.parent_job, jt.flow_step_id\n ) as sibling_count\n FROM job_tree jt\n )\n SELECT w.id, w.kind, w.flow_step_id, w.path_label,\n w.sibling_index::int as sibling_index,\n w.sibling_count::int as sibling_count,\n w.depth::int as depth,\n w.parent_module_type,\n coalesce(job_logs.logs, '') as logs,\n COALESCE(job_logs.log_offset, 0) as log_offset,\n job_logs.log_file_index\n FROM with_sibling_index w\n LEFT JOIN job_logs ON job_logs.job_id = w.id\n ORDER BY w.id_path ASC", - "describe": { - "columns": [ - { - "ordinal": 0, - "name": "id", - "type_info": "Uuid" - }, - { - "ordinal": 1, - "name": "kind", - "type_info": "Text" - }, - { - "ordinal": 2, - "name": "flow_step_id", - "type_info": "Varchar" - }, - { - "ordinal": 3, - "name": "path_label", - "type_info": "Text" - }, - { - "ordinal": 4, - "name": "sibling_index", - "type_info": "Int4" - }, - { - "ordinal": 5, - "name": "sibling_count", - "type_info": "Int4" - }, - { - "ordinal": 6, - "name": "depth", - "type_info": "Int4" - }, - { - "ordinal": 7, - "name": "parent_module_type", - "type_info": "Text" - }, - { - "ordinal": 8, - "name": "logs", - "type_info": "Text" - }, - { - "ordinal": 9, - "name": "log_offset", - "type_info": "Int4" - }, - { - "ordinal": 10, - "name": "log_file_index", - "type_info": "TextArray" - } - ], - "parameters": { - "Left": [ - "Text", - "Uuid" - ] - }, - "nullable": [ - null, - null, - null, - null, - null, - null, - null, - null, - null, - null, - true - ] - }, - "hash": "5b95f464d492227ffe8a449844da00a6539788f9be647d6b52f01cbb2967473e" -} diff --git a/backend/.sqlx/query-6981553b904f9ae38d6eaeae05716ac71390028dfe8c54798b1701b01591fff7.json b/backend/.sqlx/query-6981553b904f9ae38d6eaeae05716ac71390028dfe8c54798b1701b01591fff7.json new file mode 100644 index 0000000000..1a965a6dc3 --- /dev/null +++ b/backend/.sqlx/query-6981553b904f9ae38d6eaeae05716ac71390028dfe8c54798b1701b01591fff7.json @@ -0,0 +1,66 @@ +{ + "db_name": "PostgreSQL", + "query": "SELECT j.kind::text as kind,\n c.status::text as completed_status,\n c.duration_ms as \"duration_ms?\",\n COALESCE(c.started_at, q.started_at) as started_at,\n LEFT(c.result::text, $3) as result_prefix,\n length(c.result::text) as result_length,\n q.running as \"q_running?\",\n q.suspend as \"q_suspend?\"\n FROM v2_job j\n LEFT JOIN v2_job_completed c ON c.id = j.id\n LEFT JOIN v2_job_queue q ON q.id = j.id\n WHERE j.id = $1 AND j.workspace_id = $2", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "kind", + "type_info": "Text" + }, + { + "ordinal": 1, + "name": "completed_status", + "type_info": "Text" + }, + { + "ordinal": 2, + "name": "duration_ms?", + "type_info": "Int8" + }, + { + "ordinal": 3, + "name": "started_at", + "type_info": "Timestamptz" + }, + { + "ordinal": 4, + "name": "result_prefix", + "type_info": "Text" + }, + { + "ordinal": 5, + "name": "result_length", + "type_info": "Int4" + }, + { + "ordinal": 6, + "name": "q_running?", + "type_info": "Bool" + }, + { + "ordinal": 7, + "name": "q_suspend?", + "type_info": "Int4" + } + ], + "parameters": { + "Left": [ + "Uuid", + "Text", + "Int4" + ] + }, + "nullable": [ + null, + null, + false, + null, + null, + null, + false, + false + ] + }, + "hash": "6981553b904f9ae38d6eaeae05716ac71390028dfe8c54798b1701b01591fff7" +} diff --git a/backend/.sqlx/query-94f0e271a67179c78de217ae1f4d7190324533119792b650c6db734a3b99575c.json b/backend/.sqlx/query-94f0e271a67179c78de217ae1f4d7190324533119792b650c6db734a3b99575c.json new file mode 100644 index 0000000000..c284efca75 --- /dev/null +++ b/backend/.sqlx/query-94f0e271a67179c78de217ae1f4d7190324533119792b650c6db734a3b99575c.json @@ -0,0 +1,84 @@ +{ + "db_name": "PostgreSQL", + "query": "WITH RECURSIVE job_tree AS (\n SELECT j.id, j.kind::text, j.flow_step_id, j.parent_job,\n '' as path_label, 0 as depth,\n j.id::text as id_path,\n ''::text as parent_module_type\n FROM v2_job j\n WHERE j.id = $2 AND j.workspace_id = $1\n UNION ALL\n SELECT j.id, j.kind::text, j.flow_step_id, j.parent_job,\n CASE\n WHEN jt.path_label = '' THEN COALESCE(j.flow_step_id, '')\n ELSE jt.path_label || '/' || COALESCE(j.flow_step_id, '')\n END,\n jt.depth + 1,\n jt.id_path || '/' || j.id::text,\n COALESCE((\n SELECT m->'value'->>'type'\n FROM v2_job parent_j\n LEFT JOIN flow_version fv ON fv.id = parent_j.runnable_id\n AND parent_j.kind::text = 'flow'\n LEFT JOIN flow f ON f.path = parent_j.runnable_path\n AND f.workspace_id = parent_j.workspace_id\n LEFT JOIN flow_node fn ON fn.id = parent_j.runnable_id\n CROSS JOIN LATERAL jsonb_array_elements(\n COALESCE(parent_j.raw_flow, fv.value, f.value, fn.flow)->'modules'\n ) m\n WHERE parent_j.id = jt.id\n AND m->>'id' = j.flow_step_id\n LIMIT 1\n ), '')::text\n FROM v2_job j\n JOIN job_tree jt ON j.parent_job = jt.id\n WHERE j.workspace_id = $1\n AND ($3::text[] IS NULL OR j.tag = ANY($3))\n ),\n positions AS (\n SELECT g.parent_job, g.flow_step_id, fj.jid, fj.ord\n FROM (SELECT DISTINCT parent_job, flow_step_id FROM job_tree\n WHERE parent_job IS NOT NULL AND flow_step_id IS NOT NULL) g\n CROSS JOIN LATERAL (\n SELECT COALESCE(\n (SELECT flow_status FROM v2_job_completed WHERE id = g.parent_job),\n (SELECT flow_status FROM v2_job_status WHERE id = g.parent_job)\n ) AS fs\n ) pf\n CROSS JOIN LATERAL (\n SELECT m FROM jsonb_array_elements(pf.fs->'modules') m\n WHERE m->>'id' = g.flow_step_id\n LIMIT 1\n ) md\n CROSS JOIN LATERAL jsonb_array_elements_text(md.m->'flow_jobs')\n WITH ORDINALITY fj(jid, ord)\n ),\n with_sibling_index AS (\n SELECT jt.*,\n ROW_NUMBER() OVER (\n PARTITION BY jt.parent_job, jt.flow_step_id\n ORDER BY pos.ord NULLS LAST, jt.id\n ) as sibling_index,\n COUNT(*) OVER (\n PARTITION BY jt.parent_job, jt.flow_step_id\n ) as sibling_count\n FROM job_tree jt\n LEFT JOIN positions pos ON pos.parent_job = jt.parent_job\n AND pos.flow_step_id = jt.flow_step_id\n AND pos.jid = jt.id::text\n )\n SELECT w.id, w.kind, w.flow_step_id, w.path_label,\n w.sibling_index::int as sibling_index,\n w.sibling_count::int as sibling_count,\n w.depth::int as depth,\n w.parent_module_type,\n coalesce(job_logs.logs, '') as logs,\n COALESCE(job_logs.log_offset, 0) as log_offset,\n job_logs.log_file_index\n FROM with_sibling_index w\n LEFT JOIN job_logs ON job_logs.job_id = w.id\n ORDER BY w.id_path ASC", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "id", + "type_info": "Uuid" + }, + { + "ordinal": 1, + "name": "kind", + "type_info": "Text" + }, + { + "ordinal": 2, + "name": "flow_step_id", + "type_info": "Varchar" + }, + { + "ordinal": 3, + "name": "path_label", + "type_info": "Text" + }, + { + "ordinal": 4, + "name": "sibling_index", + "type_info": "Int4" + }, + { + "ordinal": 5, + "name": "sibling_count", + "type_info": "Int4" + }, + { + "ordinal": 6, + "name": "depth", + "type_info": "Int4" + }, + { + "ordinal": 7, + "name": "parent_module_type", + "type_info": "Text" + }, + { + "ordinal": 8, + "name": "logs", + "type_info": "Text" + }, + { + "ordinal": 9, + "name": "log_offset", + "type_info": "Int4" + }, + { + "ordinal": 10, + "name": "log_file_index", + "type_info": "TextArray" + } + ], + "parameters": { + "Left": [ + "Text", + "Uuid", + "TextArray" + ] + }, + "nullable": [ + null, + null, + null, + null, + null, + null, + null, + null, + null, + null, + true + ] + }, + "hash": "94f0e271a67179c78de217ae1f4d7190324533119792b650c6db734a3b99575c" +} diff --git a/backend/.sqlx/query-b8e732a03969666444f73397ac153d23ec2af084b2f93da24c920532c1916384.json b/backend/.sqlx/query-b8e732a03969666444f73397ac153d23ec2af084b2f93da24c920532c1916384.json new file mode 100644 index 0000000000..b892061f56 --- /dev/null +++ b/backend/.sqlx/query-b8e732a03969666444f73397ac153d23ec2af084b2f93da24c920532c1916384.json @@ -0,0 +1,110 @@ +{ + "db_name": "PostgreSQL", + "query": "WITH RECURSIVE job_tree AS (\n SELECT j.id, j.kind::text, j.flow_step_id, j.parent_job,\n '' as path_label, 0 as depth,\n j.id::text as id_path,\n ''::text as parent_module_type\n FROM v2_job j\n WHERE j.id = $2 AND j.workspace_id = $1\n UNION ALL\n SELECT j.id, j.kind::text, j.flow_step_id, j.parent_job,\n CASE\n WHEN jt.path_label = '' THEN COALESCE(j.flow_step_id, '')\n ELSE jt.path_label || '/' || COALESCE(j.flow_step_id, '')\n END,\n jt.depth + 1,\n jt.id_path || '/' || j.id::text,\n COALESCE((\n SELECT m->'value'->>'type'\n FROM v2_job parent_j\n LEFT JOIN flow_version fv ON fv.id = parent_j.runnable_id\n AND parent_j.kind::text = 'flow'\n LEFT JOIN flow f ON f.path = parent_j.runnable_path\n AND f.workspace_id = parent_j.workspace_id\n LEFT JOIN flow_node fn ON fn.id = parent_j.runnable_id\n CROSS JOIN LATERAL jsonb_array_elements(\n COALESCE(parent_j.raw_flow, fv.value, f.value, fn.flow)->'modules'\n ) m\n WHERE parent_j.id = jt.id\n AND m->>'id' = j.flow_step_id\n LIMIT 1\n ), '')::text\n FROM v2_job j\n JOIN job_tree jt ON j.parent_job = jt.id\n WHERE j.workspace_id = $1\n AND ($4::text[] IS NULL OR j.tag = ANY($4))\n ),\n positions AS (\n SELECT g.parent_job, g.flow_step_id, fj.jid, fj.ord\n FROM (SELECT DISTINCT parent_job, flow_step_id FROM job_tree\n WHERE parent_job IS NOT NULL AND flow_step_id IS NOT NULL) g\n CROSS JOIN LATERAL (\n SELECT COALESCE(\n (SELECT flow_status FROM v2_job_completed WHERE id = g.parent_job),\n (SELECT flow_status FROM v2_job_status WHERE id = g.parent_job)\n ) AS fs\n ) pf\n CROSS JOIN LATERAL (\n SELECT m FROM jsonb_array_elements(pf.fs->'modules') m\n WHERE m->>'id' = g.flow_step_id\n LIMIT 1\n ) md\n CROSS JOIN LATERAL jsonb_array_elements_text(md.m->'flow_jobs')\n WITH ORDINALITY fj(jid, ord)\n ),\n with_sibling_index AS (\n SELECT jt.*,\n ROW_NUMBER() OVER (\n PARTITION BY jt.parent_job, jt.flow_step_id\n ORDER BY pos.ord NULLS LAST, jt.id\n ) as sibling_index,\n COUNT(*) OVER (\n PARTITION BY jt.parent_job, jt.flow_step_id\n ) as sibling_count\n FROM job_tree jt\n LEFT JOIN positions pos ON pos.parent_job = jt.parent_job\n AND pos.flow_step_id = jt.flow_step_id\n AND pos.jid = jt.id::text\n ),\n limited AS (\n SELECT * FROM with_sibling_index ORDER BY id_path ASC LIMIT $5\n )\n SELECT w.id, w.kind, w.flow_step_id, w.path_label,\n w.sibling_index::int as sibling_index,\n w.sibling_count::int as sibling_count,\n w.depth::int as depth,\n w.parent_module_type,\n c.status::text as completed_status,\n c.duration_ms as \"duration_ms?\",\n COALESCE(c.started_at, q.started_at) as started_at,\n LEFT(c.result::text, $3) as result_prefix,\n length(c.result::text) as result_length,\n q.running as \"q_running?\",\n q.suspend as \"q_suspend?\"\n FROM limited w\n LEFT JOIN v2_job_completed c ON c.id = w.id\n LEFT JOIN v2_job_queue q ON q.id = w.id\n ORDER BY w.id_path ASC", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "id", + "type_info": "Uuid" + }, + { + "ordinal": 1, + "name": "kind", + "type_info": "Text" + }, + { + "ordinal": 2, + "name": "flow_step_id", + "type_info": "Varchar" + }, + { + "ordinal": 3, + "name": "path_label", + "type_info": "Text" + }, + { + "ordinal": 4, + "name": "sibling_index", + "type_info": "Int4" + }, + { + "ordinal": 5, + "name": "sibling_count", + "type_info": "Int4" + }, + { + "ordinal": 6, + "name": "depth", + "type_info": "Int4" + }, + { + "ordinal": 7, + "name": "parent_module_type", + "type_info": "Text" + }, + { + "ordinal": 8, + "name": "completed_status", + "type_info": "Text" + }, + { + "ordinal": 9, + "name": "duration_ms?", + "type_info": "Int8" + }, + { + "ordinal": 10, + "name": "started_at", + "type_info": "Timestamptz" + }, + { + "ordinal": 11, + "name": "result_prefix", + "type_info": "Text" + }, + { + "ordinal": 12, + "name": "result_length", + "type_info": "Int4" + }, + { + "ordinal": 13, + "name": "q_running?", + "type_info": "Bool" + }, + { + "ordinal": 14, + "name": "q_suspend?", + "type_info": "Int4" + } + ], + "parameters": { + "Left": [ + "Text", + "Uuid", + "Int4", + "TextArray", + "Int8" + ] + }, + "nullable": [ + null, + null, + null, + null, + null, + null, + null, + null, + null, + false, + null, + null, + null, + false, + false + ] + }, + "hash": "b8e732a03969666444f73397ac153d23ec2af084b2f93da24c920532c1916384" +} diff --git a/backend/.sqlx/query-bd594f06413dd21d7e6923e6754b58fbce8c09e6ec8172d9ed94716958a29b40.json b/backend/.sqlx/query-bd594f06413dd21d7e6923e6754b58fbce8c09e6ec8172d9ed94716958a29b40.json new file mode 100644 index 0000000000..5e9e4bfa6e --- /dev/null +++ b/backend/.sqlx/query-bd594f06413dd21d7e6923e6754b58fbce8c09e6ec8172d9ed94716958a29b40.json @@ -0,0 +1,30 @@ +{ + "db_name": "PostgreSQL", + "query": "SELECT created_by, COALESCE(flow_innermost_root_job, parent_job) as enclosing_job FROM v2_job WHERE id = $1 AND workspace_id = $2 AND ($3::text[] IS NULL OR tag = ANY($3))", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "created_by", + "type_info": "Varchar" + }, + { + "ordinal": 1, + "name": "enclosing_job", + "type_info": "Uuid" + } + ], + "parameters": { + "Left": [ + "Uuid", + "Text", + "TextArray" + ] + }, + "nullable": [ + false, + null + ] + }, + "hash": "bd594f06413dd21d7e6923e6754b58fbce8c09e6ec8172d9ed94716958a29b40" +} diff --git a/backend/Cargo.lock b/backend/Cargo.lock index 1c2e232021..8dc76890c2 100644 --- a/backend/Cargo.lock +++ b/backend/Cargo.lock @@ -4482,13 +4482,13 @@ dependencies = [ [[package]] name = "displaydoc" -version = "0.2.6" +version = "0.2.7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1ac70aa55017e108007fbaf5aa0f54b021c98f92ff8af59d42eda9da96e3dd4f" +checksum = "c6232dd377dcc64799954cbd3a9bb882e9cdc1308ccd87b1c098f1fb2eaf82a8" dependencies = [ "proc-macro2", "quote", - "syn 2.0.119", + "syn 3.0.3", ] [[package]] @@ -7073,7 +7073,7 @@ dependencies = [ "bitflags 2.13.1", "libc", "plain", - "redox_syscall 0.9.0", + "redox_syscall 0.9.1", ] [[package]] @@ -9926,9 +9926,9 @@ dependencies = [ [[package]] name = "redox_syscall" -version = "0.9.0" +version = "0.9.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c5102a6aaa05aa011a238e178e6bca86d2cb56fc9f586d37cb80f5bca6e07759" +checksum = "07507be7b4a5f9f26eeb41eeaebb1f5a7ff29dfb29739facc21d35bf8b11c21e" dependencies = [ "bitflags 2.13.1", ] @@ -14495,7 +14495,7 @@ dependencies = [ [[package]] name = "windmill" -version = "1.773.0" +version = "1.775.2" dependencies = [ "anyhow", "async-nats", @@ -14580,7 +14580,7 @@ dependencies = [ [[package]] name = "windmill-ai" -version = "1.773.0" +version = "1.775.2" dependencies = [ "async-stream", "async-trait", @@ -14613,7 +14613,7 @@ dependencies = [ [[package]] name = "windmill-alerting" -version = "1.773.0" +version = "1.775.2" dependencies = [ "axum 0.8.9", "chrono", @@ -14626,7 +14626,7 @@ dependencies = [ [[package]] name = "windmill-api" -version = "1.773.0" +version = "1.775.2" dependencies = [ "anyhow", "argon2", @@ -14765,7 +14765,7 @@ dependencies = [ [[package]] name = "windmill-api-agent-workers" -version = "1.773.0" +version = "1.775.2" dependencies = [ "axum 0.8.9", "chrono", @@ -14788,7 +14788,7 @@ dependencies = [ [[package]] name = "windmill-api-assets" -version = "1.773.0" +version = "1.775.2" dependencies = [ "axum 0.8.9", "chrono", @@ -14803,7 +14803,7 @@ dependencies = [ [[package]] name = "windmill-api-auth" -version = "1.773.0" +version = "1.775.2" dependencies = [ "anyhow", "axum 0.8.9", @@ -14829,7 +14829,7 @@ dependencies = [ [[package]] name = "windmill-api-client" -version = "1.773.0" +version = "1.775.2" dependencies = [ "reqwest 0.12.28", "serde", @@ -14839,7 +14839,7 @@ dependencies = [ [[package]] name = "windmill-api-configs" -version = "1.773.0" +version = "1.775.2" dependencies = [ "axum 0.8.9", "chrono", @@ -14856,7 +14856,7 @@ dependencies = [ [[package]] name = "windmill-api-debug" -version = "1.773.0" +version = "1.775.2" dependencies = [ "axum 0.8.9", "base64 0.22.1", @@ -14878,7 +14878,7 @@ dependencies = [ [[package]] name = "windmill-api-embeddings" -version = "1.773.0" +version = "1.775.2" dependencies = [ "anyhow", "axum 0.8.9", @@ -14901,7 +14901,7 @@ dependencies = [ [[package]] name = "windmill-api-flow-conversations" -version = "1.773.0" +version = "1.775.2" dependencies = [ "axum 0.8.9", "chrono", @@ -14917,7 +14917,7 @@ dependencies = [ [[package]] name = "windmill-api-flows" -version = "1.773.0" +version = "1.775.2" dependencies = [ "axum 0.8.9", "chrono", @@ -14938,7 +14938,7 @@ dependencies = [ [[package]] name = "windmill-api-groups" -version = "1.773.0" +version = "1.775.2" dependencies = [ "axum 0.8.9", "chrono", @@ -14959,7 +14959,7 @@ dependencies = [ [[package]] name = "windmill-api-inputs" -version = "1.773.0" +version = "1.775.2" dependencies = [ "axum 0.8.9", "chrono", @@ -14973,7 +14973,7 @@ dependencies = [ [[package]] name = "windmill-api-integration-tests" -version = "1.773.0" +version = "1.775.2" dependencies = [ "anyhow", "async-nats", @@ -15008,7 +15008,7 @@ dependencies = [ [[package]] name = "windmill-api-jobs" -version = "1.773.0" +version = "1.775.2" dependencies = [ "anyhow", "axum 0.8.9", @@ -15033,7 +15033,7 @@ dependencies = [ [[package]] name = "windmill-api-npm-proxy" -version = "1.773.0" +version = "1.775.2" dependencies = [ "axum 0.8.9", "flate2", @@ -15051,7 +15051,7 @@ dependencies = [ [[package]] name = "windmill-api-openapi" -version = "1.773.0" +version = "1.775.2" dependencies = [ "anyhow", "axum 0.8.9", @@ -15073,7 +15073,7 @@ dependencies = [ [[package]] name = "windmill-api-schedule" -version = "1.773.0" +version = "1.775.2" dependencies = [ "axum 0.8.9", "chrono", @@ -15093,7 +15093,7 @@ dependencies = [ [[package]] name = "windmill-api-scripts" -version = "1.773.0" +version = "1.775.2" dependencies = [ "axum 0.8.9", "chrono", @@ -15130,7 +15130,7 @@ dependencies = [ [[package]] name = "windmill-api-settings" -version = "1.773.0" +version = "1.775.2" dependencies = [ "anyhow", "axum 0.8.9", @@ -15158,7 +15158,7 @@ dependencies = [ [[package]] name = "windmill-api-sse" -version = "1.773.0" +version = "1.775.2" dependencies = [ "lazy_static", "serde", @@ -15170,7 +15170,7 @@ dependencies = [ [[package]] name = "windmill-api-users" -version = "1.773.0" +version = "1.775.2" dependencies = [ "argon2", "axum 0.8.9", @@ -15195,7 +15195,7 @@ dependencies = [ [[package]] name = "windmill-api-workers" -version = "1.773.0" +version = "1.775.2" dependencies = [ "axum 0.8.9", "chrono", @@ -15209,7 +15209,7 @@ dependencies = [ [[package]] name = "windmill-api-workspaces" -version = "1.773.0" +version = "1.775.2" dependencies = [ "axum 0.8.9", "chrono", @@ -15244,7 +15244,7 @@ dependencies = [ [[package]] name = "windmill-audit" -version = "1.773.0" +version = "1.775.2" dependencies = [ "chrono", "lazy_static", @@ -15258,7 +15258,7 @@ dependencies = [ [[package]] name = "windmill-autoscaling" -version = "1.773.0" +version = "1.775.2" dependencies = [ "anyhow", "axum 0.8.9", @@ -15277,7 +15277,7 @@ dependencies = [ [[package]] name = "windmill-common" -version = "1.773.0" +version = "1.775.2" dependencies = [ "aes-gcm", "aho-corasick", @@ -15381,7 +15381,7 @@ dependencies = [ [[package]] name = "windmill-dep-map" -version = "1.773.0" +version = "1.775.2" dependencies = [ "chrono", "itertools 0.14.0", @@ -15400,7 +15400,7 @@ dependencies = [ [[package]] name = "windmill-git-sync" -version = "1.773.0" +version = "1.775.2" dependencies = [ "regex", "serde", @@ -15415,7 +15415,7 @@ dependencies = [ [[package]] name = "windmill-indexer" -version = "1.773.0" +version = "1.775.2" dependencies = [ "anyhow", "astral-tokio-tar", @@ -15439,7 +15439,7 @@ dependencies = [ [[package]] name = "windmill-jseval" -version = "1.773.0" +version = "1.775.2" dependencies = [ "anyhow", "futures", @@ -15456,7 +15456,7 @@ dependencies = [ [[package]] name = "windmill-macros" -version = "1.773.0" +version = "1.775.2" dependencies = [ "itertools 0.14.0", "lazy_static", @@ -15472,7 +15472,7 @@ dependencies = [ [[package]] name = "windmill-mcp" -version = "1.773.0" +version = "1.775.2" dependencies = [ "anyhow", "async-trait", @@ -15493,7 +15493,7 @@ dependencies = [ [[package]] name = "windmill-native-triggers" -version = "1.773.0" +version = "1.775.2" dependencies = [ "anyhow", "async-trait", @@ -15524,7 +15524,7 @@ dependencies = [ [[package]] name = "windmill-oauth" -version = "1.773.0" +version = "1.775.2" dependencies = [ "anyhow", "arc-swap", @@ -15549,7 +15549,7 @@ dependencies = [ [[package]] name = "windmill-object-store" -version = "1.773.0" +version = "1.775.2" dependencies = [ "anyhow", "async-stream", @@ -15583,7 +15583,7 @@ dependencies = [ [[package]] name = "windmill-operator" -version = "1.773.0" +version = "1.775.2" dependencies = [ "anyhow", "futures", @@ -15601,7 +15601,7 @@ dependencies = [ [[package]] name = "windmill-parser" -version = "1.773.0" +version = "1.775.2" dependencies = [ "convert_case 0.6.0", "serde", @@ -15610,7 +15610,7 @@ dependencies = [ [[package]] name = "windmill-parser-bash" -version = "1.773.0" +version = "1.775.2" dependencies = [ "anyhow", "lazy_static", @@ -15622,7 +15622,7 @@ dependencies = [ [[package]] name = "windmill-parser-csharp" -version = "1.773.0" +version = "1.775.2" dependencies = [ "anyhow", "serde_json", @@ -15634,7 +15634,7 @@ dependencies = [ [[package]] name = "windmill-parser-go" -version = "1.773.0" +version = "1.775.2" dependencies = [ "anyhow", "gosyn", @@ -15646,7 +15646,7 @@ dependencies = [ [[package]] name = "windmill-parser-graphql" -version = "1.773.0" +version = "1.775.2" dependencies = [ "anyhow", "lazy_static", @@ -15658,7 +15658,7 @@ dependencies = [ [[package]] name = "windmill-parser-java" -version = "1.773.0" +version = "1.775.2" dependencies = [ "anyhow", "serde_json", @@ -15670,7 +15670,7 @@ dependencies = [ [[package]] name = "windmill-parser-nu" -version = "1.773.0" +version = "1.775.2" dependencies = [ "anyhow", "nu-parser", @@ -15681,7 +15681,7 @@ dependencies = [ [[package]] name = "windmill-parser-php" -version = "1.773.0" +version = "1.775.2" dependencies = [ "anyhow", "itertools 0.14.0", @@ -15692,7 +15692,7 @@ dependencies = [ [[package]] name = "windmill-parser-py" -version = "1.773.0" +version = "1.775.2" dependencies = [ "anyhow", "itertools 0.14.0", @@ -15704,7 +15704,7 @@ dependencies = [ [[package]] name = "windmill-parser-py-asset" -version = "1.773.0" +version = "1.775.2" dependencies = [ "anyhow", "rustpython-ast", @@ -15715,7 +15715,7 @@ dependencies = [ [[package]] name = "windmill-parser-py-imports" -version = "1.773.0" +version = "1.775.2" dependencies = [ "anyhow", "async-recursion", @@ -15737,7 +15737,7 @@ dependencies = [ [[package]] name = "windmill-parser-r" -version = "1.773.0" +version = "1.775.2" dependencies = [ "anyhow", "serde_json", @@ -15749,7 +15749,7 @@ dependencies = [ [[package]] name = "windmill-parser-ruby" -version = "1.773.0" +version = "1.775.2" dependencies = [ "anyhow", "lazy_static", @@ -15763,7 +15763,7 @@ dependencies = [ [[package]] name = "windmill-parser-rust" -version = "1.773.0" +version = "1.775.2" dependencies = [ "anyhow", "convert_case 0.6.0", @@ -15780,7 +15780,7 @@ dependencies = [ [[package]] name = "windmill-parser-sql" -version = "1.773.0" +version = "1.775.2" dependencies = [ "anyhow", "lazy_static", @@ -15793,7 +15793,7 @@ dependencies = [ [[package]] name = "windmill-parser-sql-asset" -version = "1.773.0" +version = "1.775.2" dependencies = [ "anyhow", "serde", @@ -15805,7 +15805,7 @@ dependencies = [ [[package]] name = "windmill-parser-ts" -version = "1.773.0" +version = "1.775.2" dependencies = [ "anyhow", "lazy_static", @@ -15823,7 +15823,7 @@ dependencies = [ [[package]] name = "windmill-parser-ts-asset" -version = "1.773.0" +version = "1.775.2" dependencies = [ "anyhow", "serde-wasm-bindgen", @@ -15839,7 +15839,7 @@ dependencies = [ [[package]] name = "windmill-parser-wac" -version = "1.773.0" +version = "1.775.2" dependencies = [ "anyhow", "rustpython-ast", @@ -15855,7 +15855,7 @@ dependencies = [ [[package]] name = "windmill-parser-yaml" -version = "1.773.0" +version = "1.775.2" dependencies = [ "anyhow", "serde", @@ -15866,7 +15866,7 @@ dependencies = [ [[package]] name = "windmill-queue" -version = "1.773.0" +version = "1.775.2" dependencies = [ "anyhow", "async-recursion", @@ -15905,7 +15905,7 @@ dependencies = [ [[package]] name = "windmill-runtime-nativets" -version = "1.773.0" +version = "1.775.2" dependencies = [ "anyhow", "const_format", @@ -15945,7 +15945,7 @@ dependencies = [ [[package]] name = "windmill-sql-datatype-parser-wasm" -version = "1.773.0" +version = "1.775.2" dependencies = [ "getrandom 0.3.4", "wasm-bindgen", @@ -15956,7 +15956,7 @@ dependencies = [ [[package]] name = "windmill-store" -version = "1.773.0" +version = "1.775.2" dependencies = [ "anyhow", "async-recursion", @@ -15990,7 +15990,7 @@ dependencies = [ [[package]] name = "windmill-test-utils" -version = "1.773.0" +version = "1.775.2" dependencies = [ "anyhow", "async-trait", @@ -16014,7 +16014,7 @@ dependencies = [ [[package]] name = "windmill-trigger" -version = "1.773.0" +version = "1.775.2" dependencies = [ "anyhow", "async-trait", @@ -16047,7 +16047,7 @@ dependencies = [ [[package]] name = "windmill-trigger-amqp" -version = "1.773.0" +version = "1.775.2" dependencies = [ "anyhow", "async-trait", @@ -16074,7 +16074,7 @@ dependencies = [ [[package]] name = "windmill-trigger-azure" -version = "1.773.0" +version = "1.775.2" dependencies = [ "anyhow", "async-trait", @@ -16107,7 +16107,7 @@ dependencies = [ [[package]] name = "windmill-trigger-email" -version = "1.773.0" +version = "1.775.2" dependencies = [ "anyhow", "async-trait", @@ -16127,7 +16127,7 @@ dependencies = [ [[package]] name = "windmill-trigger-gcp" -version = "1.773.0" +version = "1.775.2" dependencies = [ "anyhow", "async-trait", @@ -16161,7 +16161,7 @@ dependencies = [ [[package]] name = "windmill-trigger-http" -version = "1.773.0" +version = "1.775.2" dependencies = [ "anyhow", "async-trait", @@ -16197,7 +16197,7 @@ dependencies = [ [[package]] name = "windmill-trigger-kafka" -version = "1.773.0" +version = "1.775.2" dependencies = [ "anyhow", "async-trait", @@ -16220,7 +16220,7 @@ dependencies = [ [[package]] name = "windmill-trigger-mqtt" -version = "1.773.0" +version = "1.775.2" dependencies = [ "anyhow", "async-trait", @@ -16244,7 +16244,7 @@ dependencies = [ [[package]] name = "windmill-trigger-nats" -version = "1.773.0" +version = "1.775.2" dependencies = [ "anyhow", "async-nats", @@ -16268,7 +16268,7 @@ dependencies = [ [[package]] name = "windmill-trigger-postgres" -version = "1.773.0" +version = "1.775.2" dependencies = [ "anyhow", "async-trait", @@ -16303,7 +16303,7 @@ dependencies = [ [[package]] name = "windmill-trigger-sqs" -version = "1.773.0" +version = "1.775.2" dependencies = [ "anyhow", "async-trait", @@ -16331,7 +16331,7 @@ dependencies = [ [[package]] name = "windmill-trigger-websocket" -version = "1.773.0" +version = "1.775.2" dependencies = [ "anyhow", "async-trait", @@ -16356,7 +16356,7 @@ dependencies = [ [[package]] name = "windmill-types" -version = "1.773.0" +version = "1.775.2" dependencies = [ "anyhow", "bitflags 2.13.1", @@ -16375,7 +16375,7 @@ dependencies = [ [[package]] name = "windmill-worker" -version = "1.773.0" +version = "1.775.2" dependencies = [ "anyhow", "async-once-cell", @@ -16490,7 +16490,7 @@ dependencies = [ [[package]] name = "windmill-worker-volumes" -version = "1.773.0" +version = "1.775.2" dependencies = [ "bytes", "futures", diff --git a/backend/Cargo.toml b/backend/Cargo.toml index 71ddc9e998..126c9da88b 100644 --- a/backend/Cargo.toml +++ b/backend/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "windmill" -version = "1.773.0" +version = "1.775.2" authors.workspace = true edition.workspace = true @@ -89,7 +89,7 @@ members = [ exclude = ["./windmill-duckdb-ffi-internal", "./parsers/windmill-parser-wasm"] [workspace.package] -version = "1.773.0" +version = "1.775.2" authors = ["Ruben Fiszel "] edition = "2021" diff --git a/backend/ee-repo-ref.txt b/backend/ee-repo-ref.txt index d0da94f9a3..fa769ea50f 100644 --- a/backend/ee-repo-ref.txt +++ b/backend/ee-repo-ref.txt @@ -1 +1 @@ -d884daca8d297faffe7c74aaa1b76cfc589be15d +06399edec0fad7631d96f384db3d3c6673fa6797 diff --git a/backend/parsers/windmill-parser-wasm/Cargo.lock b/backend/parsers/windmill-parser-wasm/Cargo.lock index 193a97ed0e..0e4e12eb89 100644 --- a/backend/parsers/windmill-parser-wasm/Cargo.lock +++ b/backend/parsers/windmill-parser-wasm/Cargo.lock @@ -6191,7 +6191,7 @@ checksum = "712e227841d057c1ee1cd2fb22fa7e5a5461ae8e48fa2ca79ec42cfc1931183f" [[package]] name = "windmill-common" -version = "1.773.0" +version = "1.775.2" dependencies = [ "aho-corasick", "anyhow", @@ -6272,7 +6272,7 @@ dependencies = [ [[package]] name = "windmill-macros" -version = "1.773.0" +version = "1.775.2" dependencies = [ "proc-macro2", "quote", @@ -6284,7 +6284,7 @@ dependencies = [ [[package]] name = "windmill-parser" -version = "1.773.0" +version = "1.775.2" dependencies = [ "convert_case", "serde", @@ -6293,7 +6293,7 @@ dependencies = [ [[package]] name = "windmill-parser-bash" -version = "1.773.0" +version = "1.775.2" dependencies = [ "anyhow", "lazy_static", @@ -6305,7 +6305,7 @@ dependencies = [ [[package]] name = "windmill-parser-csharp" -version = "1.773.0" +version = "1.775.2" dependencies = [ "anyhow", "serde_json", @@ -6317,7 +6317,7 @@ dependencies = [ [[package]] name = "windmill-parser-go" -version = "1.773.0" +version = "1.775.2" dependencies = [ "anyhow", "gosyn", @@ -6329,7 +6329,7 @@ dependencies = [ [[package]] name = "windmill-parser-graphql" -version = "1.773.0" +version = "1.775.2" dependencies = [ "anyhow", "lazy_static", @@ -6341,7 +6341,7 @@ dependencies = [ [[package]] name = "windmill-parser-java" -version = "1.773.0" +version = "1.775.2" dependencies = [ "anyhow", "serde_json", @@ -6353,7 +6353,7 @@ dependencies = [ [[package]] name = "windmill-parser-nu" -version = "1.773.0" +version = "1.775.2" dependencies = [ "anyhow", "nu-parser", @@ -6364,7 +6364,7 @@ dependencies = [ [[package]] name = "windmill-parser-php" -version = "1.773.0" +version = "1.775.2" dependencies = [ "anyhow", "itertools 0.14.0", @@ -6375,7 +6375,7 @@ dependencies = [ [[package]] name = "windmill-parser-py" -version = "1.773.0" +version = "1.775.2" dependencies = [ "anyhow", "itertools 0.14.0", @@ -6387,7 +6387,7 @@ dependencies = [ [[package]] name = "windmill-parser-py-asset" -version = "1.773.0" +version = "1.775.2" dependencies = [ "anyhow", "rustpython-ast", @@ -6398,7 +6398,7 @@ dependencies = [ [[package]] name = "windmill-parser-py-imports" -version = "1.773.0" +version = "1.775.2" dependencies = [ "anyhow", "async-recursion", @@ -6420,7 +6420,7 @@ dependencies = [ [[package]] name = "windmill-parser-r" -version = "1.773.0" +version = "1.775.2" dependencies = [ "anyhow", "serde_json", @@ -6432,7 +6432,7 @@ dependencies = [ [[package]] name = "windmill-parser-ruby" -version = "1.773.0" +version = "1.775.2" dependencies = [ "anyhow", "lazy_static", @@ -6446,7 +6446,7 @@ dependencies = [ [[package]] name = "windmill-parser-rust" -version = "1.773.0" +version = "1.775.2" dependencies = [ "anyhow", "convert_case", @@ -6463,7 +6463,7 @@ dependencies = [ [[package]] name = "windmill-parser-sql" -version = "1.773.0" +version = "1.775.2" dependencies = [ "anyhow", "lazy_static", @@ -6476,7 +6476,7 @@ dependencies = [ [[package]] name = "windmill-parser-sql-asset" -version = "1.773.0" +version = "1.775.2" dependencies = [ "anyhow", "serde", @@ -6488,7 +6488,7 @@ dependencies = [ [[package]] name = "windmill-parser-ts" -version = "1.773.0" +version = "1.775.2" dependencies = [ "anyhow", "lazy_static", @@ -6506,7 +6506,7 @@ dependencies = [ [[package]] name = "windmill-parser-ts-asset" -version = "1.773.0" +version = "1.775.2" dependencies = [ "anyhow", "serde-wasm-bindgen", @@ -6522,7 +6522,7 @@ dependencies = [ [[package]] name = "windmill-parser-wac" -version = "1.773.0" +version = "1.775.2" dependencies = [ "anyhow", "rustpython-ast", @@ -6538,7 +6538,7 @@ dependencies = [ [[package]] name = "windmill-parser-wasm" -version = "1.773.0" +version = "1.775.2" dependencies = [ "anyhow", "getrandom 0.2.17", @@ -6570,7 +6570,7 @@ dependencies = [ [[package]] name = "windmill-parser-yaml" -version = "1.773.0" +version = "1.775.2" dependencies = [ "anyhow", "serde", @@ -6581,7 +6581,7 @@ dependencies = [ [[package]] name = "windmill-types" -version = "1.773.0" +version = "1.775.2" dependencies = [ "anyhow", "bitflags", diff --git a/backend/parsers/windmill-parser-wasm/Cargo.toml b/backend/parsers/windmill-parser-wasm/Cargo.toml index 1866ea19d5..1cb5004022 100644 --- a/backend/parsers/windmill-parser-wasm/Cargo.toml +++ b/backend/parsers/windmill-parser-wasm/Cargo.toml @@ -12,7 +12,7 @@ resolver = "2" members = ["."] [workspace.package] -version = "1.773.0" +version = "1.775.2" edition = "2021" authors = ["Ruben Fiszel "] diff --git a/backend/tests/datatable_migrations_grants.rs b/backend/tests/datatable_migrations_grants.rs new file mode 100644 index 0000000000..ebe9aeba4d --- /dev/null +++ b/backend/tests/datatable_migrations_grants.rs @@ -0,0 +1,300 @@ +//! Regression test for running data table migrations against a database whose +//! role only holds DML grants. +//! +//! Two failure modes are pinned here: +//! - the Postgres message must reach the caller. `tokio_postgres::Error`'s +//! `Display` renders only the error kind, so interpolating it with `{}` +//! produced a bare `Failed to ensure _wm_migrations table: db error`. +//! - `CREATE TABLE IF NOT EXISTS` checks CREATE on the schema *before* it +//! checks existence, so the run must probe for `_wm_migrations` first or an +//! unprivileged role can never migrate, even against a pre-created table. +//! +//! Plus the privilege report that surfaces the same state from workspace +//! settings before anyone reaches a migration. + +use serde_json::{json, Value}; +use sqlx::{Pool, Postgres}; + +use windmill_test_utils::*; + +const ROLE: &str = "wm_dtmig_test_role"; +const ROLE_PASSWORD: &str = "wm_dtmig_test_pwd"; +/// Deliberately hyphenated: it only parses inside double quotes, so it pins that +/// the emitted recovery statement quotes the role rather than interpolating it. +const NOSCHEMA_ROLE: &str = "wm-dtmig-noschema"; + +fn authed(b: reqwest::RequestBuilder) -> reqwest::RequestBuilder { + b.header("Authorization", "Bearer DTMIG_ADMIN_TOKEN") +} + +/// Point the fixture's data table at this test's own database, connecting as a +/// role that may read and write but not create: `GRANT USAGE` without `CREATE`, +/// and the schema's own CREATE revoked from PUBLIC so the outcome does not +/// depend on the server's default `public` grants (relaxed before Postgres 15). +async fn setup_unprivileged_datatable_role(db: &Pool) -> anyhow::Result<()> { + let opts = (*db.connect_options()).clone(); + let dbname = opts.get_database().expect("test database name").to_string(); + + sqlx::query(&format!( + // Roles are cluster objects, not per-test-database ones. A previous run + // leaving the role behind raises duplicate_object; the tests in this + // binary run in parallel, so two sessions can also clear that check + // together and collide on pg_authid's unique index instead. + "DO $$ BEGIN \ + CREATE ROLE {ROLE} LOGIN PASSWORD '{ROLE_PASSWORD}'; \ + EXCEPTION WHEN duplicate_object OR unique_violation THEN NULL; \ + END $$" + )) + .execute(db) + .await?; + sqlx::raw_sql(&format!( + "REVOKE CREATE ON SCHEMA public FROM PUBLIC; \ + GRANT CONNECT ON DATABASE \"{dbname}\" TO {ROLE}; \ + GRANT USAGE ON SCHEMA public TO {ROLE};" + )) + .execute(db) + .await?; + + sqlx::query( + "INSERT INTO resource (workspace_id, path, value, resource_type, created_by) \ + VALUES ('dtmig-ws', 'u/dtmig-admin/pg', $1, 'postgresql', 'dtmig-admin')", + ) + .bind(json!({ + "host": opts.get_host(), + "port": opts.get_port(), + "dbname": dbname, + "user": ROLE, + "password": ROLE_PASSWORD, + "sslmode": "disable", + })) + .execute(db) + .await?; + + Ok(()) +} + +#[sqlx::test(fixtures("datatable_migrations_grants"))] +async fn test_run_migrations_without_create_privilege(db: Pool) -> anyhow::Result<()> { + initialize_tracing().await; + setup_unprivileged_datatable_role(&db).await?; + + let server = ApiServer::start(db.clone()).await?; + let port = server.addr.port(); + let url = + format!("http://localhost:{port}/api/w/dtmig-ws/workspaces/run_datatable_migrations/main"); + + // No `_wm_migrations` yet and no way to create one: the caller must be told + // what Postgres actually refused, not "db error". + let resp = authed(reqwest::Client::new().post(&url)).send().await?; + assert_eq!(resp.status(), 500); + let body = resp.text().await?; + assert!( + body.contains("permission denied for schema"), + "the Postgres message should reach the caller, got: {body}" + ); + // The suggested statement must be complete and quoted, not a placeholder. + assert!( + body.contains(&format!("GRANT CREATE ON SCHEMA \"public\" TO \"{ROLE}\"")), + "the hint should name the actual role and schema, got: {body}" + ); + + // Once an operator has created the bookkeeping table and granted DML on it, + // migrations run even though the role still cannot create tables. + sqlx::raw_sql(&format!( + "CREATE TABLE _wm_migrations ( \ + datatable TEXT NOT NULL, \ + version BIGINT NOT NULL, \ + installed_at TIMESTAMPTZ NOT NULL DEFAULT now(), \ + PRIMARY KEY (datatable, version)); \ + GRANT SELECT, INSERT, UPDATE, DELETE ON _wm_migrations TO {ROLE};" + )) + .execute(&db) + .await?; + + let resp = authed(reqwest::Client::new().post(&url)).send().await?; + let status = resp.status(); + let body = resp.text().await?; + assert_eq!( + status, 200, + "run should succeed on a pre-created table: {body}" + ); + + Ok(()) +} + +#[sqlx::test(fixtures("datatable_migrations_grants"))] +async fn test_datatable_connection_report(db: Pool) -> anyhow::Result<()> { + initialize_tracing().await; + setup_unprivileged_datatable_role(&db).await?; + + let server = ApiServer::start(db.clone()).await?; + let port = server.addr.port(); + let url = + format!("http://localhost:{port}/api/w/dtmig-ws/workspaces/test_datatable_connection/main"); + + // The report is a privilege disclosure about the data table's database, so + // it stays behind the same bar as editing the data table config. + let resp = reqwest::Client::new() + .get(&url) + .header("Authorization", "Bearer DTMIG_USER_TOKEN") + .send() + .await?; + assert_eq!(resp.status(), 403, "non-admins must not get the report"); + + let report: Value = authed(reqwest::Client::new().get(&url)) + .send() + .await? + .json() + .await?; + assert_eq!(report["user"], ROLE); + assert_eq!(report["schema"], "public"); + assert_eq!(report["can_create_table"], false); + assert_eq!(report["can_create_schema"], false); + let grants = report["suggested_grants"].as_array().unwrap(); + assert!( + grants + .iter() + .any(|g| g.as_str().unwrap() + == format!("GRANT CREATE ON SCHEMA \"public\" TO \"{ROLE}\"")), + "missing schema grant: {report}" + ); + // Pin the name, not just the shape: the endpoint reads it from + // `current_database()` rather than the resource, and a prefix assertion + // would pass either way. + let dbname = (*db.connect_options()) + .clone() + .get_database() + .expect("test database name") + .to_string(); + assert!( + grants.iter().any(|g| g.as_str().unwrap() + == format!("GRANT CREATE ON DATABASE \"{dbname}\" TO \"{ROLE}\"")), + "missing database grant for {dbname}: {report}" + ); + + // A pre-created bookkeeping table lets migration *tracking* work, but the + // role still cannot create anything: the report must keep saying so rather + // than falling silent because nothing needs creating right now. + sqlx::raw_sql(&format!( + "CREATE TABLE _wm_migrations ( \ + datatable TEXT NOT NULL, \ + version BIGINT NOT NULL, \ + installed_at TIMESTAMPTZ NOT NULL DEFAULT now(), \ + PRIMARY KEY (datatable, version)); \ + GRANT SELECT, INSERT, UPDATE, DELETE ON _wm_migrations TO {ROLE};" + )) + .execute(&db) + .await?; + + let report: Value = authed(reqwest::Client::new().get(&url)) + .send() + .await? + .json() + .await?; + assert_eq!(report["migrations_table_exists"], true); + assert_eq!(report["can_create_table"], false); + assert!( + report["suggested_grants"] + .as_array() + .unwrap() + .iter() + .any(|g| g.as_str().unwrap().contains("ON SCHEMA")), + "an existing bookkeeping table must not suppress the schema grant: {report}" + ); + + // Granting the privileges clears the suggestions. + sqlx::raw_sql(&format!( + "GRANT CREATE ON SCHEMA public TO {ROLE}; \ + GRANT CREATE ON DATABASE \"{dbname}\" TO {ROLE};" + )) + .execute(&db) + .await?; + + let report: Value = authed(reqwest::Client::new().get(&url)) + .send() + .await? + .json() + .await?; + assert_eq!(report["can_create_table"], true); + assert_eq!(report["can_create_schema"], true); + assert_eq!(report["suggested_grants"].as_array().unwrap().len(), 0); + + Ok(()) +} + +/// Point the fixture's second data table at a role whose `search_path` resolves +/// to nothing, the one state where no grant helps. +async fn setup_schemaless_datatable_role(db: &Pool) -> anyhow::Result<()> { + let opts = (*db.connect_options()).clone(); + let dbname = opts.get_database().expect("test database name").to_string(); + + sqlx::query(&format!( + "DO $$ BEGIN \ + CREATE ROLE \"{NOSCHEMA_ROLE}\" LOGIN PASSWORD '{ROLE_PASSWORD}'; \ + EXCEPTION WHEN duplicate_object OR unique_violation THEN NULL; \ + END $$" + )) + .execute(db) + .await?; + // Cluster-wide for this role, which is why it gets one of its own rather + // than sharing the role the other assertions connect with. + sqlx::raw_sql(&format!( + "ALTER ROLE \"{NOSCHEMA_ROLE}\" SET search_path = wm_dtmig_absent_schema; \ + GRANT CONNECT ON DATABASE \"{dbname}\" TO \"{NOSCHEMA_ROLE}\";" + )) + .execute(db) + .await?; + + sqlx::query( + "INSERT INTO resource (workspace_id, path, value, resource_type, created_by) \ + VALUES ('dtmig-ws', 'u/dtmig-admin/pg_noschema', $1, 'postgresql', 'dtmig-admin')", + ) + .bind(json!({ + "host": opts.get_host(), + "port": opts.get_port(), + "dbname": dbname, + "user": NOSCHEMA_ROLE, + "password": ROLE_PASSWORD, + "sslmode": "disable", + })) + .execute(db) + .await?; + + Ok(()) +} + +#[sqlx::test(fixtures("datatable_migrations_grants"))] +async fn test_datatable_connection_without_a_resolvable_schema( + db: Pool, +) -> anyhow::Result<()> { + initialize_tracing().await; + setup_schemaless_datatable_role(&db).await?; + + let server = ApiServer::start(db.clone()).await?; + let port = server.addr.port(); + let report: Value = authed(reqwest::Client::new().get(format!( + "http://localhost:{port}/api/w/dtmig-ws/workspaces/test_datatable_connection/noschema" + ))) + .send() + .await? + .json() + .await?; + + assert!(report["schema"].is_null(), "expected no schema: {report}"); + // No grant fixes an empty search_path, so suggesting one would send the + // reader after a statement that changes nothing. + assert!( + !report["suggested_grants"] + .as_array() + .unwrap() + .iter() + .any(|g| g.as_str().unwrap().contains("ON SCHEMA")), + "an empty search_path must not yield a schema grant: {report}" + ); + assert_eq!( + report["suggested_search_path"], + format!("ALTER ROLE \"{NOSCHEMA_ROLE}\" SET search_path = public") + ); + + Ok(()) +} diff --git a/backend/tests/fixtures/datatable_migrations_grants.sql b/backend/tests/fixtures/datatable_migrations_grants.sql new file mode 100644 index 0000000000..81feb9ce91 --- /dev/null +++ b/backend/tests/fixtures/datatable_migrations_grants.sql @@ -0,0 +1,35 @@ +-- Fixture for the data table migration bookkeeping-grants regression test. +-- Workspace + admin token + a data table pointing at a postgres resource; the +-- test fills that resource in with credentials for a deliberately unprivileged +-- role, since the database name is allocated per test run. + +INSERT INTO workspace (id, name, owner) VALUES + ('dtmig-ws', 'DTMIG WS', 'dtmig-admin'); + +INSERT INTO workspace_key (workspace_id, kind, key) VALUES + ('dtmig-ws', 'cloud', 'dtmig-key'); + +INSERT INTO workspace_settings (workspace_id, datatable) VALUES + ('dtmig-ws', '{"datatables": {"main": {"database": {"resource_type": "postgresql", "resource_path": "u/dtmig-admin/pg"}, "migrations_enabled": true}, "noschema": {"database": {"resource_type": "postgresql", "resource_path": "u/dtmig-admin/pg_noschema"}, "migrations_enabled": true}}}'); + +INSERT INTO group_ (workspace_id, name, summary, extra_perms) VALUES + ('dtmig-ws', 'all', 'All users', '{}'); + +INSERT INTO password(email, password_hash, login_type, super_admin, verified, name, username) + VALUES ('dtmig-admin@windmill.dev', 'x', 'password', true, true, 'DTMIG Admin', 'dtmig-admin'); + +INSERT INTO usr(workspace_id, email, username, is_admin, role) VALUES + ('dtmig-ws', 'dtmig-admin@windmill.dev', 'dtmig-admin', true, 'Admin'); + +INSERT INTO token(token_hash, token_prefix, token, email, label, super_admin) + VALUES (encode(sha256('DTMIG_ADMIN_TOKEN'::bytea), 'hex'), 'DTMIG_ADM', 'DTMIG_ADMIN_TOKEN', 'dtmig-admin@windmill.dev', 't', true); + +-- Non-admin member, to pin that the privilege report stays admin-only. +INSERT INTO password(email, password_hash, login_type, super_admin, verified, name, username) + VALUES ('dtmig-user@windmill.dev', 'x', 'password', false, true, 'DTMIG User', 'dtmig-user'); + +INSERT INTO usr(workspace_id, email, username, is_admin, role) VALUES + ('dtmig-ws', 'dtmig-user@windmill.dev', 'dtmig-user', false, 'User'); + +INSERT INTO token(token_hash, token_prefix, token, email, label) + VALUES (encode(sha256('DTMIG_USER_TOKEN'::bytea), 'hex'), 'DTMIG_USR', 'DTMIG_USER_TOKEN', 'dtmig-user@windmill.dev', 't'); diff --git a/backend/windmill-api-debug/src/lib.rs b/backend/windmill-api-debug/src/lib.rs index 61ec59be75..3b43a04ad1 100644 --- a/backend/windmill-api-debug/src/lib.rs +++ b/backend/windmill-api-debug/src/lib.rs @@ -422,11 +422,7 @@ async fn sign_expression( let mut tx = user_db.begin(&authed).await?; // Truncate expression for resource field if too long (max 255 chars) - let resource = if request.expression.len() > 200 { - format!("{}...", &request.expression[..200]) - } else { - request.expression.clone() - }; + let resource = windmill_common::utils::truncate_with_ellipsis(&request.expression, 200); audit_log( &mut *tx, diff --git a/backend/windmill-api-settings/src/lib.rs b/backend/windmill-api-settings/src/lib.rs index bd34ca35ef..eca0001eab 100644 --- a/backend/windmill-api-settings/src/lib.rs +++ b/backend/windmill-api-settings/src/lib.rs @@ -53,7 +53,7 @@ use windmill_common::{ auth::is_super_admin_email, ee_oss::{get_license_plan, LicensePlan}, email_oss::send_email_plain_text, - error::{self, JsonResult, Result}, + error::{self, pg_error_message, JsonResult, Result}, get_database_url, global_settings::{ AI_CONFIG_SETTING, APP_WORKSPACED_ROUTE_SETTING, AUTOMATE_USERNAME_CREATION_SETTING, @@ -1705,7 +1705,7 @@ async fn setup_custom_instance_pg_database_inner( .map_err(|e| { error::Error::ExecutionErr(format!( "Failed to grant permissions to custom_instance_user: {}", - e.to_string(), + pg_error_message(&e), )) })?; diff --git a/backend/windmill-api-workspaces/src/datatable_migrations.rs b/backend/windmill-api-workspaces/src/datatable_migrations.rs index fef9963933..73e2fb85cc 100644 --- a/backend/windmill-api-workspaces/src/datatable_migrations.rs +++ b/backend/windmill-api-workspaces/src/datatable_migrations.rs @@ -21,14 +21,16 @@ use chrono::Utc; use serde::{Deserialize, Serialize}; use sqlx::{Postgres, Transaction}; use std::collections::{HashMap, HashSet}; +use tokio_postgres::error::SqlState; use windmill_api_auth::{require_super_admin, ApiAuthed}; use windmill_api_jobs::run_wait_result_internal; use windmill_audit::audit_oss::audit_log; use windmill_audit::ActionKind; use windmill_common::db::UserDB; -use windmill_common::error::{Error, JsonResult, Result}; +use windmill_common::error::{pg_error_message, Error, JsonResult, Result}; use windmill_common::jobs::{JobPayload, RawCode}; +use windmill_common::query_builders::{render_db_quoted_identifier, DbType}; use windmill_common::runnable_settings::{ConcurrencySettingsWithCustom, DebouncingSettings}; use windmill_common::scripts::ScriptLang; use windmill_common::users::username_to_permissioned_as; @@ -222,7 +224,31 @@ async fn run_datatable_migration_job( /// key would let one data table's migration mark another's same-version /// migration as already applied (and rollback could touch the wrong row). async fn ensure_wm_migrations_schema(client: &tokio_postgres::Client) -> Result<()> { - client + // `CREATE TABLE IF NOT EXISTS` checks CREATE on the schema before it checks + // existence, so probing first is what lets a data table whose role only holds + // DML grants keep migrating against an already-created bookkeeping table. + // `to_regclass` resolves through search_path, like the unqualified statements + // the rest of this module runs against it. Takes no parameters, so it goes + // through the simple protocol: a named prepared statement is what stalls + // behind a transaction-pooling proxy (see `pg_get_full_schema`). + let rows = client + .simple_query("SELECT to_regclass('_wm_migrations') IS NOT NULL AS present") + .await + .map_err(|e| { + Error::internal_err(format!( + "Failed to look up _wm_migrations table: {}", + pg_error_message(&e) + )) + })?; + let exists = rows.iter().any(|msg| match msg { + tokio_postgres::SimpleQueryMessage::Row(row) => row.get("present") == Some("t"), + _ => false, + }); + if exists { + return Ok(()); + } + + let Err(e) = client .batch_execute( "CREATE TABLE IF NOT EXISTS _wm_migrations (\ datatable TEXT NOT NULL, \ @@ -231,10 +257,53 @@ async fn ensure_wm_migrations_schema(client: &tokio_postgres::Client) -> Result< PRIMARY KEY (datatable, version))", ) .await - .map_err(|e| { - Error::internal_err(format!("Failed to ensure _wm_migrations table: {}", e)) - })?; - Ok(()) + else { + return Ok(()); + }; + + let mut msg = format!( + "Failed to ensure _wm_migrations table: {}", + pg_error_message(&e) + ); + // A role with only table-level grants cannot create it: since Postgres 15 the + // `public` schema no longer grants CREATE to PUBLIC, so this is the usual + // failure on a bring-your-own database. + if e.code() == Some(&SqlState::INSUFFICIENT_PRIVILEGE) { + // Windmill connects as the role that lacks the privilege, so it cannot + // grant it: hand over the statement a schema owner has to run instead. + // Keep it ahead of the explanation below — the UI collapses everything + // past the first couple of lines behind a "Show more". + if let Some((user, schema)) = connection_identity(client).await { + // Both come back unquoted, so a mixed-case or hyphenated name would + // otherwise render a statement that targets a different schema. + msg.push_str(&format!( + ". Run: GRANT CREATE ON SCHEMA {} TO {}", + render_db_quoted_identifier(&schema, DbType::Postgresql), + render_db_quoted_identifier(&user, DbType::Postgresql), + )); + } + msg.push_str( + ". Applied migrations are recorded in a `_wm_migrations` table in the data \ + table's own database, so its user needs to be able to create it", + ); + } + Err(Error::internal_err(msg)) +} + +/// The role and default schema of a data table connection, for grant hints. +/// Both come from the server so the statement we suggest names what the +/// connection actually resolves to, not what the resource happens to say. +async fn connection_identity(client: &tokio_postgres::Client) -> Option<(String, String)> { + let rows = client + .simple_query("SELECT current_user AS usr, current_schema() AS sch") + .await + .ok()?; + rows.iter().find_map(|msg| match msg { + tokio_postgres::SimpleQueryMessage::Row(row) => { + Some((row.get("usr")?.to_string(), row.get("sch")?.to_string())) + } + _ => None, + }) } /// Open a connection to a data table's own database and hold the session-level @@ -265,7 +334,12 @@ async fn lock_datatable_migration_runs( client .batch_execute("SELECT pg_advisory_lock(hashtext('windmill_datatable_migrations')::int8)") .await - .map_err(|e| Error::internal_err(format!("Failed to acquire migration lock: {}", e)))?; + .map_err(|e| { + Error::internal_err(format!( + "Failed to acquire migration lock: {}", + pg_error_message(&e) + )) + })?; Ok(client) } @@ -287,7 +361,7 @@ async fn read_applied_versions_on_client( Err(e) if e.as_db_error().map(|d| d.code().code()) == Some("42P01") => Ok(HashSet::new()), Err(e) => Err(Error::internal_err(format!( "Failed to read _wm_migrations: {}", - e + pg_error_message(&e) ))), } } @@ -366,7 +440,12 @@ async fn run_datatable_migrations( &[&datatable_name, &m.timestamp], ) .await - .map_err(|e| Error::internal_err(format!("Failed to record migration: {}", e)))?; + .map_err(|e| { + Error::internal_err(format!( + "Failed to record migration: {}", + pg_error_message(&e) + )) + })?; applied.push(AppliedMigration { version: m.timestamp, name: m.name }); } @@ -433,7 +512,12 @@ async fn rollback_datatable_migrations( &[&datatable_name, &only], ) .await - .map_err(|e| Error::internal_err(format!("Failed to read _wm_migrations: {}", e)))?, + .map_err(|e| { + Error::internal_err(format!( + "Failed to read _wm_migrations: {}", + pg_error_message(&e) + )) + })?, None => client .query_opt( "SELECT version FROM _wm_migrations WHERE datatable = $1 \ @@ -441,7 +525,12 @@ async fn rollback_datatable_migrations( &[&datatable_name], ) .await - .map_err(|e| Error::internal_err(format!("Failed to read _wm_migrations: {}", e)))?, + .map_err(|e| { + Error::internal_err(format!( + "Failed to read _wm_migrations: {}", + pg_error_message(&e) + )) + })?, }; let version: i64 = match target { @@ -492,7 +581,12 @@ async fn rollback_datatable_migrations( &[&datatable_name, &version], ) .await - .map_err(|e| Error::internal_err(format!("Failed to drop migration record: {}", e)))?; + .map_err(|e| { + Error::internal_err(format!( + "Failed to drop migration record: {}", + pg_error_message(&e) + )) + })?; Ok(Json(RollbackDatatableMigrationsResult { rolled_back: vec![RolledBackMigration { version, name: definition.name }], @@ -940,7 +1034,10 @@ async fn mark_datatable_version_installed( ) .await .map_err(|e| { - Error::internal_err(format!("Failed to mark initial migration installed: {}", e)) + Error::internal_err(format!( + "Failed to mark initial migration installed: {}", + pg_error_message(&e) + )) })?; Ok(()) } @@ -1378,7 +1475,7 @@ fn ignore_missing_wm_migrations(e: tokio_postgres::Error) -> Result<()> { Some("42P01") => Ok(()), _ => Err(Error::internal_err(format!( "Failed to update _wm_migrations: {}", - e + pg_error_message(&e) ))), } } diff --git a/backend/windmill-api-workspaces/src/workspaces.rs b/backend/windmill-api-workspaces/src/workspaces.rs index c8f67c9d2e..9501ed96f9 100644 --- a/backend/windmill-api-workspaces/src/workspaces.rs +++ b/backend/windmill-api-workspaces/src/workspaces.rs @@ -34,6 +34,7 @@ use windmill_audit::audit_oss::{audit_log, AuditAuthorable}; use windmill_audit::ActionKind; use windmill_common::db::UserDB; use windmill_common::global_settings::HTTP_ROUTE_WORKSPACED_ROUTE; +use windmill_common::query_builders::{render_db_quoted_identifier, DbType}; use windmill_common::users::username_to_permissioned_as; use windmill_common::variables::{ build_crypt, decrypt, encrypt, SECRET_SALT, WORKSPACE_CRYPT_CACHE, @@ -51,7 +52,7 @@ use windmill_common::workspaces::{ use windmill_common::workspaces::{Ducklake, DucklakeCatalogResourceType}; use windmill_common::PgDatabase; use windmill_common::{ - error::{Error, JsonResult, Result}, + error::{pg_error_message, Error, JsonResult, Result}, global_settings::{ AUTOMATE_USERNAME_CREATION_SETTING, DISABLE_WORKSPACE_INVITE_EMAILS_SETTING, }, @@ -133,6 +134,10 @@ pub fn workspaced_service() -> Router { get(get_datatable_table_schema), ) .route("/edit_datatable_config", post(edit_datatable_config)) + .route( + "/test_datatable_connection/{datatable_name}", + get(test_datatable_connection), + ) .merge(crate::datatable_migrations::routes()) .route("/git_sync_enabled", get(get_git_sync_enabled)) .route("/git_sync_deploy_mode", get(get_git_sync_deploy_mode)) @@ -2023,6 +2028,127 @@ struct DataTableTableSchema { columns: ColumnMap, } +#[derive(Serialize, Debug)] +struct DataTableConnectionCheck { + /// The role the data table actually connects as, and the schema its + /// unqualified statements resolve to. Both are read from the server rather + /// than the resource, which need not spell either of them out. + user: String, + schema: Option, + /// Whether that role can create tables in `schema` / schemas in the database. + can_create_table: bool, + can_create_schema: bool, + /// Whether the migration bookkeeping table is already present. Informative + /// only: it explains why migration *tracking* can work without CREATE, and + /// grants nothing beyond that. + migrations_table_exists: bool, + /// Statements to run for the privileges that are missing, empty when there + /// are none. Windmill connects as the role that lacks them, so it can only + /// name them for a schema owner to run. + suggested_grants: Vec, + /// Statement that gives the session a schema to work in, when `search_path` + /// resolves to none. Rendered here rather than by the caller so identifier + /// quoting stays in one place. + #[serde(skip_serializing_if = "Option::is_none")] + suggested_search_path: Option, +} + +/// Report what the data table's own database lets its role do. Surfacing this +/// from the settings page is the difference between finding out here and finding +/// out on a first schema change, when the failure reads as a Postgres refusal +/// deep inside a migration. +async fn test_datatable_connection( + authed: ApiAuthed, + Extension(db): Extension, + Path((w_id, datatable_name)): Path<(String, String)>, +) -> JsonResult { + require_admin(authed.is_admin, &authed.username)?; + + let db_resource = get_datatable_resource_from_db_unchecked(&db, &w_id, &datatable_name).await?; + let pg_db: PgDatabase = serde_json::from_value(db_resource) + .map_err(|e| Error::internal_err(format!("Failed to parse database credentials: {}", e)))?; + let (client, connection) = pg_db.connect(Some(&db)).await?; + let join_handle = tokio::spawn(async move { connection.await }); + + // One round trip, no side effects: `has_*_privilege` answers for the + // connected role without attempting the operation. + let rows = client + .simple_query( + "SELECT current_user AS usr, \ + current_schema() AS sch, \ + current_database() AS db, \ + has_schema_privilege(current_schema(), 'CREATE') AS can_create_table, \ + has_database_privilege(current_database(), 'CREATE') AS can_create_schema, \ + to_regclass('_wm_migrations') IS NOT NULL AS has_migrations_table", + ) + .await + .map_err(|e| { + Error::internal_err(format!( + "Failed to inspect data table privileges: {}", + pg_error_message(&e) + )) + }); + + drop(client); + let _ = windmill_common::shutdown_pg_connection(join_handle).await; + + let row = rows? + .into_iter() + .find_map(|msg| match msg { + tokio_postgres::SimpleQueryMessage::Row(row) => Some(row), + _ => None, + }) + .ok_or_else(|| Error::internal_err("Privilege query returned no row".to_string()))?; + + let user = row.get("usr").unwrap_or_default().to_string(); + let schema = row.get("sch").map(str::to_string); + let can_create_table = row.get("can_create_table") == Some("t"); + let can_create_schema = row.get("can_create_schema") == Some("t"); + let migrations_table_exists = row.get("has_migrations_table") == Some("t"); + + let quoted_user = render_db_quoted_identifier(&user, DbType::Postgresql); + let mut suggested_grants = Vec::new(); + // Suggest on the capability alone: an existing `_wm_migrations` spares only + // that one table, and says nothing about the tables a migration will create. + // A NULL `current_schema()` means search_path resolves to nothing, and no + // grant fixes that — an unqualified CREATE fails with `no schema has been + // selected to create in` whoever holds the privilege — so suggest nothing + // and let `schema: null` carry the diagnosis. + if let (false, Some(target)) = (can_create_table, schema.as_deref()) { + suggested_grants.push(format!( + "GRANT CREATE ON SCHEMA {} TO {}", + render_db_quoted_identifier(target, DbType::Postgresql), + quoted_user + )); + } + if !can_create_schema { + // Named from the server like every other identifier here: behind a + // pooler the resource's dbname can be an alias for another database. + let dbname = row.get("db").unwrap_or(pg_db.dbname.as_str()); + suggested_grants.push(format!( + "GRANT CREATE ON DATABASE {} TO {}", + render_db_quoted_identifier(dbname, DbType::Postgresql), + quoted_user + )); + } + + // An empty search_path is not a privilege problem, so it gets a statement of + // its own rather than a grant. + let suggested_search_path = schema + .is_none() + .then(|| format!("ALTER ROLE {quoted_user} SET search_path = public")); + + Ok(Json(DataTableConnectionCheck { + user, + schema, + can_create_table, + can_create_schema, + migrations_table_exists, + suggested_grants, + suggested_search_path, + })) +} + async fn list_datatable_schemas( _authed: ApiAuthed, Extension(db): Extension, @@ -2139,7 +2265,9 @@ async fn get_datatable_schema(db: &DB, w_id: &str, datatable_name: &str) -> Resu &[], ) .await - .map_err(|e| Error::internal_err(format!("Failed to query schemas: {}", e)))?; + .map_err(|e| { + Error::internal_err(format!("Failed to query schemas: {}", pg_error_message(&e))) + })?; // Build hierarchical structure: schema -> table -> column -> compact_type let mut schema_map: SchemaMap = HashMap::new(); @@ -2173,7 +2301,9 @@ async fn get_datatable_schema(db: &DB, w_id: &str, datatable_name: &str) -> Resu &[&schema_names], ) .await - .map_err(|e| Error::internal_err(format!("Failed to query columns: {}", e)))?; + .map_err(|e| { + Error::internal_err(format!("Failed to query columns: {}", pg_error_message(&e))) + })?; for row in rows { let table_schema: String = row.get(0); @@ -2221,7 +2351,9 @@ async fn get_datatable_tables(db: &DB, w_id: &str, datatable_name: &str) -> Resu &[], ) .await - .map_err(|e| Error::internal_err(format!("Failed to query schemas: {}", e)))?; + .map_err(|e| { + Error::internal_err(format!("Failed to query schemas: {}", pg_error_message(&e))) + })?; let mut table_map: TableListMap = HashMap::new(); let schema_names: Vec = schema_rows @@ -2247,7 +2379,9 @@ async fn get_datatable_tables(db: &DB, w_id: &str, datatable_name: &str) -> Resu &[&schema_names], ) .await - .map_err(|e| Error::internal_err(format!("Failed to query tables: {}", e)))?; + .map_err(|e| { + Error::internal_err(format!("Failed to query tables: {}", pg_error_message(&e))) + })?; for row in rows { let table_schema: String = row.get(0); @@ -2299,7 +2433,9 @@ async fn get_datatable_table_columns( &[&schema_name, &table_name], ) .await - .map_err(|e| Error::internal_err(format!("Failed to query columns: {}", e)))?; + .map_err(|e| { + Error::internal_err(format!("Failed to query columns: {}", pg_error_message(&e))) + })?; if rows.is_empty() { return Err(Error::NotFound(format!( @@ -2671,7 +2807,10 @@ async fn create_pg_database( ) .await .map_err(|e| { - Error::internal_err(format!("Failed to check database existence: {}", e)) + Error::internal_err(format!( + "Failed to check database existence: {}", + pg_error_message(&e) + )) })?; let db_exists: bool = row.get(0); @@ -2690,7 +2829,8 @@ async fn create_pg_database( .map_err(|e| { Error::internal_err(format!( "Failed to create database '{}': {}", - req.target_dbname, e + req.target_dbname, + pg_error_message(&e) )) })?; @@ -3209,20 +3349,60 @@ async fn check_dev_promotion_targets_parent_repo<'a>( repos: impl Iterator, ) -> Result<()> { for r in repos.filter(|r| r.use_individual_branch.unwrap_or(false)) { - if !windmill_common::git_sync_ee::dev_promotion_target_matches_parent( + let Some(m) = windmill_common::git_sync_ee::dev_promotion_target_mismatch( db, w_id, &r.git_repo_resource_path, ) .await? - { - return Err(Error::BadRequest( - "Promotion mode on a dev workspace must reuse the parent workspace's git repository \ - (same URL and branch), but this repository is not one the parent tracks — promotion \ - would target a repository the parent does not sync with." - .to_string(), - )); - } + else { + continue; + }; + // A resource with no `branch` field normalizes to an empty string. + let named = |b: &str| { + if b.is_empty() { + "".to_string() + } else { + b.to_string() + } + }; + let branch = named(&m.branch); + let hint = if m.parent_branches.is_empty() { + format!( + "the parent workspace '{}' does not track that repository. Point this repository at \ + the one the parent syncs with, or add it to the parent's git sync settings.", + m.parent_workspace + ) + } else { + // With no branch of its own there is nothing for the parent to add, + // so only offer the side of the advice that can be acted on. + let other_way = if m.branch.is_empty() { + String::new() + } else { + format!(", or add branch '{branch}' to the parent's git sync settings") + }; + format!( + "the parent workspace '{}' tracks it on {} '{}'. Set this repository's branch to \ + match{other_way}.", + m.parent_workspace, + if m.parent_branches.len() > 1 { + "branches" + } else { + "branch" + }, + m.parent_branches + .iter() + .map(|b| named(b)) + .collect::>() + .join("', '") + ) + }; + return Err(Error::BadRequest(format!( + "Promotion mode on a dev workspace must reuse the parent workspace's git repository \ + (same URL and branch). Repository '{}' targets '{}' on branch '{branch}', but {hint}", + r.git_repo_resource_path.trim_start_matches("$res:"), + m.repo, + ))); } Ok(()) } diff --git a/backend/windmill-api/openapi.yaml b/backend/windmill-api/openapi.yaml index 758db99220..8ca3ceebd9 100644 --- a/backend/windmill-api/openapi.yaml +++ b/backend/windmill-api/openapi.yaml @@ -1,7 +1,7 @@ openapi: "3.0.3" info: - version: 1.773.0 + version: 1.775.2 title: Windmill API contact: @@ -4633,6 +4633,52 @@ paths: items: $ref: "#/components/schemas/DataTableSchema" + /w/{workspace}/workspaces/test_datatable_connection/{datatable_name}: + get: + summary: check what the data table's database lets its role do + operationId: testDataTableConnection + tags: + - workspace + parameters: + - $ref: "#/components/parameters/WorkspaceId" + - name: datatable_name + in: path + required: true + schema: + type: string + responses: + "200": + description: connection and privilege report + content: + application/json: + schema: + type: object + properties: + user: + type: string + schema: + type: string + nullable: true + can_create_table: + type: boolean + can_create_schema: + type: boolean + migrations_table_exists: + type: boolean + suggested_grants: + type: array + items: + type: string + suggested_search_path: + type: string + required: + - user + - schema + - can_create_table + - can_create_schema + - migrations_table_exists + - suggested_grants + /w/{workspace}/workspaces/list_datatable_tables: get: summary: list tables of all connected Datatables @@ -14212,6 +14258,116 @@ paths: - sibling_count - logs + /w/{workspace}/jobs_u/get_flow_all_results/{id}: + get: + summary: get statuses and truncated results for all jobs of a flow job's execution tree + operationId: getFlowAllResults + tags: + - job + parameters: + - $ref: "#/components/parameters/WorkspaceId" + - $ref: "#/components/parameters/JobId" + - name: max_result_len + in: query + description: per-entry cap (in characters of JSON text) on result_prefix (default 2000, max 30000) + schema: + type: integer + - name: step + in: query + description: 'step address to resolve to a single job instead of enumerating the tree: "b", "b/c", "b[12]" (1-based iteration/branch), composable as "b[12]/c"' + schema: + type: string + responses: + "200": + description: per-job statuses and truncated results of the flow's execution tree, or the single resolved entry when step is provided + content: + application/json: + schema: + type: object + properties: + enclosing_job: + type: string + format: uuid + description: set when the requested job is itself a step of a larger flow run; id of the flow run directly enclosing it + entries: + type: array + items: + type: object + properties: + job_id: + type: string + label: + type: string + description: human-readable label describing the job's position in the flow tree + kind: + type: string + description: job kind (script, flow, forloopflow, ...) + flow_step_id: + type: string + nullable: true + step_path: + type: string + nullable: true + description: materialized step path (e.g. "a/b") + depth: + type: integer + description: depth in the flow tree (0 for the root flow job) + parent_module_type: + type: string + nullable: true + description: parent module type (forloopflow, branchall, ...) + sibling_index: + type: integer + description: 1-based index of this job among siblings sharing the same step + sibling_count: + type: integer + description: total number of siblings sharing the same step + status: + type: string + enum: + [ + success, + failure, + canceled, + skipped, + suspended, + running, + queued, + ] + success: + type: boolean + duration_ms: + type: integer + format: int64 + started_at: + type: string + format: date-time + result_prefix: + type: string + description: result JSON text truncated to the per-entry budget; absent until the job has completed + result_length: + type: integer + description: full length in characters of the result JSON text (greater than the prefix length when truncated) + required: + - job_id + - label + - kind + - depth + - sibling_index + - sibling_count + - status + truncated: + type: boolean + description: true when the tree has more jobs than the entry cap; entries then hold the depth-first prefix + scope_filtered: + type: boolean + description: true when the caller's token is tag-scoped; steps running on other tags are omitted + step_error: + type: string + description: set when step was provided but could not be resolved; a diagnostic listing available step ids or iteration statuses + required: + - entries + /w/{workspace}/jobs_u/get_completed_logs_tail/{id}: get: summary: get completed job logs tail diff --git a/backend/windmill-api/src/jobs.rs b/backend/windmill-api/src/jobs.rs index 663f65ca6f..3621ea0564 100644 --- a/backend/windmill-api/src/jobs.rs +++ b/backend/windmill-api/src/jobs.rs @@ -406,6 +406,7 @@ pub fn workspace_unauthed_service() -> Router { "/get_flow_all_logs_structured/{id}", get(get_flow_all_logs_structured), ) + .route("/get_flow_all_results/{id}", get(get_flow_all_results)) .route( "/get_completed_logs_tail/{id}", get(get_completed_job_logs_tail), @@ -2269,48 +2270,35 @@ async fn resolve_logs_to_string( logs.to_string() } -/// A single job in a flow's execution tree, with its resolved logs and a -/// human-readable label describing its position (iteration, branch, subflow…). -#[derive(Serialize)] -struct FlowLogEntry { - job_id: String, - /// Human-readable label, e.g. "Step a (iteration 2/3)" or "Flow". - label: String, - /// Job kind (script, flow, forloopflow, …). - kind: String, - /// The flow step id this job corresponds to, if any. - flow_step_id: Option, - /// Materialized step path (e.g. "a/b") used to locate the step in the flow. - step_path: Option, - /// Depth in the flow tree (0 for the root flow job). - depth: i32, - /// The parent module type (forloopflow, branchall, …), if any. - parent_module_type: Option, - /// 1-based index of this job among its siblings sharing the same step. - sibling_index: i32, - /// Total number of siblings sharing the same step. - sibling_count: i32, - /// Resolved logs for this job (pulled from disk/object store as needed). - logs: String, +struct FlowTreeReadAuth { + /// Enclosing flow job id when the requested job is itself a step of a + /// larger flow run (flow_innermost_root_job is NULL on subflow jobs — + /// parent_job covers them). + enclosing_job: Option, + /// Scope tags of the caller's token, to re-apply on every descendant query + /// (child jobs of a `preserve_step_tags` flow can run on other tags). + scope_tags: Option>, } -async fn collect_flow_log_entries( +/// Shared preamble of the flow-tree endpoints (`get_flow_all_logs*`, +/// `get_flow_all_results`): verifies the job exists (scope-tag filtered), +/// checks read access, and records the view. +async fn authorize_flow_tree_read( view_token: Option, - opt_authed: Option, - opt_tokened: OptTokened, + opt_authed: &Option, + opt_tokened: &OptTokened, db: &DB, user_db: &UserDB, w_id: &str, id: Uuid, -) -> error::Result> { +) -> error::Result { let tags = opt_authed .as_ref() .map(|authed| get_scope_tags(authed).map(|v| v.iter().map(|s| s.to_string()).collect_vec())) .flatten(); - // Verify the root job exists and check auth let root_job = sqlx::query!( - "SELECT created_by FROM v2_job WHERE id = $1 AND workspace_id = $2 AND ($3::text[] IS NULL OR tag = ANY($3))", + "SELECT created_by, COALESCE(flow_innermost_root_job, parent_job) as enclosing_job FROM v2_job WHERE id = $1 AND workspace_id = $2 AND ($3::text[] IS NULL OR tag = ANY($3))", id, w_id, tags.as_ref().map(|v| v.as_slice()) @@ -2346,6 +2334,128 @@ async fn collect_flow_log_entries( ) .await?; + Ok(FlowTreeReadAuth { + enclosing_job: root_job.enclosing_job.filter(|r| *r != id), + scope_tags: tags, + }) +} + +/// Human-readable label for a job's position in a flow's execution tree, +/// e.g. "Flow", "Step a (iteration 2/3)", "Step b (subflow)". +fn flow_tree_entry_label( + kind: &str, + path: &str, + parent_module_type: &str, + sibling_index: i32, + sibling_count: i32, + depth: i32, +) -> String { + let kind_label = match parent_module_type { + "branchall" => " branchall", + "branchone" => " branchone", + "forloopflow" => " forloop", + "whileloopflow" => " whileloop", + "aiagent" => " ai-agent", + _ => "", + }; + + // Only a known non-fan-out parent module marks siblings as retry attempts; + // an unresolved type ("" — e.g. the flow was edited since the run) keeps + // the generic iteration wording, matching resolve_flow_step_job and the + // frontend's FAN_OUT_MODULE_TYPES check. + let attempt_like = !parent_module_type.is_empty() && !is_fan_out_module(parent_module_type); + + if depth == 0 { + "Flow".to_string() + } else if matches!( + kind, + "flow" | "flowpreview" | "flownode" | "singlestepflow" | "aiagent" + ) { + // Intermediate flow job (loop iteration or branch) + if parent_module_type == "branchone" { + // sibling_index is a row number among sibling jobs, not the chosen + // branch (branchone only enqueues the branch it selected) — don't + // pretend to know which branch ran. + format!("Step {}{} (selected branch)", path, kind_label) + } else if parent_module_type == "branchall" { + format!("Step {}{} (branch {})", path, kind_label, sibling_index) + } else if parent_module_type == "forloopflow" || parent_module_type == "whileloopflow" { + format!( + "Step {}{} (iteration {}/{})", + path, kind_label, sibling_index, sibling_count + ) + } else if sibling_count > 1 { + if attempt_like { + format!( + "Step {} (attempt {}/{})", + path, sibling_index, sibling_count + ) + } else { + format!( + "Step {}{} (iteration {}/{})", + path, kind_label, sibling_index, sibling_count + ) + } + } else { + format!("Step {} (subflow)", path) + } + } else if sibling_count > 1 { + if attempt_like { + format!( + "Step {} (attempt {}/{})", + path, sibling_index, sibling_count + ) + } else { + // Simple module optimization: forloop/whileloop with single step + // runs iterations as direct script jobs instead of subflows + format!( + "Step {}{} (iteration {}/{})", + path, kind_label, sibling_index, sibling_count + ) + } + } else { + format!("Step {}", path) + } +} + +/// A single job in a flow's execution tree, with its resolved logs and a +/// human-readable label describing its position (iteration, branch, subflow…). +#[derive(Serialize)] +struct FlowLogEntry { + job_id: String, + /// Human-readable label, e.g. "Step a (iteration 2/3)" or "Flow". + label: String, + /// Job kind (script, flow, forloopflow, …). + kind: String, + /// The flow step id this job corresponds to, if any. + flow_step_id: Option, + /// Materialized step path (e.g. "a/b") used to locate the step in the flow. + step_path: Option, + /// Depth in the flow tree (0 for the root flow job). + depth: i32, + /// The parent module type (forloopflow, branchall, …), if any. + parent_module_type: Option, + /// 1-based index of this job among its siblings sharing the same step. + sibling_index: i32, + /// Total number of siblings sharing the same step. + sibling_count: i32, + /// Resolved logs for this job (pulled from disk/object store as needed). + logs: String, +} + +async fn collect_flow_log_entries( + view_token: Option, + opt_authed: Option, + opt_tokened: OptTokened, + db: &DB, + user_db: &UserDB, + w_id: &str, + id: Uuid, +) -> error::Result> { + let auth = + authorize_flow_tree_read(view_token, &opt_authed, &opt_tokened, db, user_db, w_id, id) + .await?; + // Fetch all jobs in the flow tree using recursive CTE. // Uses a materialized id_path for depth-first ordering so children // appear right after their parent (e.g. iteration 1 → its steps → iteration 2 → ...). @@ -2369,11 +2479,13 @@ async fn collect_flow_log_entries( COALESCE(( SELECT m->'value'->>'type' FROM v2_job parent_j + LEFT JOIN flow_version fv ON fv.id = parent_j.runnable_id + AND parent_j.kind::text = 'flow' LEFT JOIN flow f ON f.path = parent_j.runnable_path AND f.workspace_id = parent_j.workspace_id LEFT JOIN flow_node fn ON fn.id = parent_j.runnable_id CROSS JOIN LATERAL jsonb_array_elements( - COALESCE(parent_j.raw_flow, f.value, fn.flow)->'modules' + COALESCE(parent_j.raw_flow, fv.value, f.value, fn.flow)->'modules' ) m WHERE parent_j.id = jt.id AND m->>'id' = j.flow_step_id @@ -2382,17 +2494,39 @@ async fn collect_flow_log_entries( FROM v2_job j JOIN job_tree jt ON j.parent_job = jt.id WHERE j.workspace_id = $1 + AND ($3::text[] IS NULL OR j.tag = ANY($3)) + ), + positions AS ( + SELECT g.parent_job, g.flow_step_id, fj.jid, fj.ord + FROM (SELECT DISTINCT parent_job, flow_step_id FROM job_tree + WHERE parent_job IS NOT NULL AND flow_step_id IS NOT NULL) g + CROSS JOIN LATERAL ( + SELECT COALESCE( + (SELECT flow_status FROM v2_job_completed WHERE id = g.parent_job), + (SELECT flow_status FROM v2_job_status WHERE id = g.parent_job) + ) AS fs + ) pf + CROSS JOIN LATERAL ( + SELECT m FROM jsonb_array_elements(pf.fs->'modules') m + WHERE m->>'id' = g.flow_step_id + LIMIT 1 + ) md + CROSS JOIN LATERAL jsonb_array_elements_text(md.m->'flow_jobs') + WITH ORDINALITY fj(jid, ord) ), with_sibling_index AS ( SELECT jt.*, ROW_NUMBER() OVER ( PARTITION BY jt.parent_job, jt.flow_step_id - ORDER BY jt.id + ORDER BY pos.ord NULLS LAST, jt.id ) as sibling_index, COUNT(*) OVER ( PARTITION BY jt.parent_job, jt.flow_step_id ) as sibling_count FROM job_tree jt + LEFT JOIN positions pos ON pos.parent_job = jt.parent_job + AND pos.flow_step_id = jt.flow_step_id + AND pos.jid = jt.id::text ) SELECT w.id, w.kind, w.flow_step_id, w.path_label, w.sibling_index::int as sibling_index, @@ -2407,6 +2541,7 @@ async fn collect_flow_log_entries( ORDER BY w.id_path ASC", w_id, id, + auth.scope_tags.as_deref(), ) .fetch_all(db) .await?; @@ -2421,57 +2556,15 @@ async fn collect_flow_log_entries( let sibling_index = record.sibling_index.unwrap_or(0); let parent_module_type = record.parent_module_type.as_deref().unwrap_or(""); - // Build a descriptive label let path = record.path_label.as_deref().unwrap_or(step_id); - - let kind_label = match parent_module_type { - "branchall" => " branchall", - "branchone" => " branchone", - "forloopflow" => " forloop", - "whileloopflow" => " whileloop", - "aiagent" => " ai-agent", - _ => "", - }; - - let label = if depth == 0 { - "Flow".to_string() - } else if matches!( + let label = flow_tree_entry_label( kind, - "flow" | "flowpreview" | "flownode" | "singlestepflow" | "aiagent" - ) { - // Intermediate flow job (loop iteration or branch) - if parent_module_type == "branchone" { - let branch_label = if sibling_index == 1 { - "default".to_string() - } else { - format!("{}", sibling_index - 1) - }; - format!("Step {}{} (branch {})", path, kind_label, branch_label) - } else if parent_module_type == "branchall" { - format!("Step {}{} (branch {})", path, kind_label, sibling_index) - } else if parent_module_type == "forloopflow" || parent_module_type == "whileloopflow" { - format!( - "Step {}{} (iteration {}/{})", - path, kind_label, sibling_index, sibling_count - ) - } else if sibling_count > 1 { - format!( - "Step {} (iteration {}/{})", - path, sibling_index, sibling_count - ) - } else { - format!("Step {} (subflow)", path) - } - } else if sibling_count > 1 { - // Simple module optimization: forloop/whileloop with single step - // runs iterations as direct script jobs instead of subflows - format!( - "Step {}{} (iteration {}/{})", - path, kind_label, sibling_index, sibling_count - ) - } else { - format!("Step {}", path) - }; + path, + parent_module_type, + sibling_index, + sibling_count, + depth, + ); let job_id = record.id.map(|u| u.to_string()).unwrap_or_default(); let logs = record.logs.as_deref().unwrap_or(""); @@ -2557,6 +2650,636 @@ async fn get_flow_all_logs_structured( Ok(Json(entries)) } +/// A single job in a flow's execution tree with its status and (truncated) +/// result — the results twin of `FlowLogEntry`. +#[derive(Serialize)] +struct FlowResultEntry { + job_id: String, + /// Human-readable label, e.g. "Step a (iteration 2/3)" or "Flow". + label: String, + /// Job kind (script, flow, forloopflow, …). + kind: String, + /// The flow step id this job corresponds to, if any. + flow_step_id: Option, + /// Materialized step path (e.g. "a/b") used to locate the step in the flow. + step_path: Option, + /// Depth in the flow tree (0 for the root flow job). + depth: i32, + /// The parent module type (forloopflow, branchall, …), if any. + parent_module_type: Option, + /// 1-based index of this job among its siblings sharing the same step. + sibling_index: i32, + /// Total number of siblings sharing the same step. + sibling_count: i32, + /// success | failure | canceled | skipped | suspended | running | queued + status: String, + #[serde(skip_serializing_if = "Option::is_none")] + success: Option, + #[serde(skip_serializing_if = "Option::is_none")] + duration_ms: Option, + #[serde(skip_serializing_if = "Option::is_none")] + started_at: Option>, + /// Result JSON text truncated to the per-entry budget; absent until the job + /// has completed. + #[serde(skip_serializing_if = "Option::is_none")] + result_prefix: Option, + /// Full length in characters of the result JSON text (greater than the + /// prefix length when truncated). + #[serde(skip_serializing_if = "Option::is_none")] + result_length: Option, +} + +#[derive(Serialize)] +struct FlowAllResultsResponse { + /// Set when the requested job is itself a step of a larger flow run: the id + /// of the flow run directly enclosing it. + #[serde(skip_serializing_if = "Option::is_none")] + enclosing_job: Option, + entries: Vec, + /// True when the tree has more jobs than the entry cap; entries then hold + /// the depth-first prefix. + #[serde(skip_serializing_if = "std::ops::Not::not")] + truncated: bool, + /// True when the caller's token is tag-scoped: steps running on other tags + /// are omitted and indistinguishable from steps that never ran. + #[serde(skip_serializing_if = "std::ops::Not::not")] + scope_filtered: bool, + /// Set when `step` was provided but could not be resolved; a diagnostic the + /// caller can act on (available step ids, iteration statuses). + #[serde(skip_serializing_if = "Option::is_none")] + step_error: Option, +} + +#[derive(Deserialize)] +struct FlowAllResultsQuery { + /// Per-entry cap (in characters of JSON text) on `result_prefix`. + max_result_len: Option, + /// Step address to resolve instead of enumerating the tree: 'b', 'b/c', + /// 'b[12]' (1-based iteration/branch), composable as 'b[12]/c'. + step: Option, +} + +const FLOW_ALL_RESULTS_DEFAULT_MAX_LEN: i32 = 2000; +const FLOW_ALL_RESULTS_MAX_MAX_LEN: i32 = 30_000; +/// Hard cap on enumerated tree entries — bounds both the response size and the +/// per-row result::text materialization the query performs. +const FLOW_ALL_RESULTS_MAX_ENTRIES: i64 = 2000; + +struct ResolvedStepJob { + job_id: Uuid, + flow_step_id: String, + path: String, + sibling_index: i32, + sibling_count: i32, + depth: i32, + parent_module_type: String, +} + +/// Fan-out module types whose sibling jobs are iterations/branches; sibling +/// jobs of any other step are retry attempts. +fn is_fan_out_module(parent_module_type: &str) -> bool { + matches!( + parent_module_type, + "forloopflow" | "whileloopflow" | "branchall" | "branchone" | "aiagent" + ) +} + +/// 'b[12]' -> ("b", Some(12)); 'b' -> ("b", None). +fn parse_step_segment(segment: &str) -> (&str, Option) { + if let Some(rest) = segment.strip_suffix(']') { + if let Some((step_id, idx)) = rest.rsplit_once('[') { + if let Ok(idx) = idx.parse::() { + return (step_id, Some(idx)); + } + } + } + (segment, None) +} + +#[cfg(test)] +mod flow_tree_tests { + use super::{flow_tree_entry_label, parse_step_segment}; + + #[test] + fn label_classifies_iterations_branches_and_retry_attempts() { + assert_eq!(flow_tree_entry_label("flow", "", "", 1, 1, 0), "Flow"); + assert_eq!( + flow_tree_entry_label("script", "l", "forloopflow", 2, 3, 1), + "Step l forloop (iteration 2/3)" + ); + assert_eq!( + flow_tree_entry_label("flow", "b", "branchall", 2, 2, 1), + "Step b branchall (branch 2)" + ); + // branchone only enqueues its chosen branch — the label must not claim + // which branch that was + assert_eq!( + flow_tree_entry_label("flow", "b", "branchone", 1, 1, 1), + "Step b branchone (selected branch)" + ); + // siblings of a non-fan-out step are retry attempts, for both direct + // and subflow steps + assert_eq!( + flow_tree_entry_label("script", "a", "rawscript", 2, 3, 1), + "Step a (attempt 2/3)" + ); + assert_eq!( + flow_tree_entry_label("flow", "b", "flow", 2, 2, 1), + "Step b (attempt 2/2)" + ); + // unresolved parent module type ("", e.g. the flow was edited since the + // run) must NOT claim retries — keep the generic iteration wording + assert_eq!( + flow_tree_entry_label("script", "a", "", 2, 3, 1), + "Step a (iteration 2/3)" + ); + assert_eq!( + flow_tree_entry_label("flow", "b", "flow", 1, 1, 1), + "Step b (subflow)" + ); + assert_eq!( + flow_tree_entry_label("script", "a", "rawscript", 1, 1, 1), + "Step a" + ); + } + + #[test] + fn parses_step_segments() { + assert_eq!(parse_step_segment("b"), ("b", None)); + assert_eq!(parse_step_segment("b[12]"), ("b", Some(12))); + assert_eq!(parse_step_segment("b[0]"), ("b", Some(0))); + assert_eq!(parse_step_segment("b[x]"), ("b[x]", None)); + assert_eq!(parse_step_segment("b]"), ("b]", None)); + } +} + +/// Resolve a step address ('b/c', 'b[12]/c', '.' separators accepted) to a +/// single job of the flow tree rooted at `root`, walking one indexed +/// parent_job + flow_step_id lookup per segment — no tree enumeration. +/// Siblings are numbered by their position in the parent flow_status's +/// flow_jobs array — the authoritative iteration order — matching the tree +/// view's numbering; job-id order alone would shuffle parallel iterations +/// created in the same millisecond (ULID randomness). Retries have no +/// flow_jobs and fall back to id order. Err carries a diagnostic listing what +/// exists instead. +async fn resolve_flow_step_job( + db: &DB, + w_id: &str, + root: Uuid, + address: &str, + scope_tags: Option<&[String]>, +) -> error::Result> { + let segments = address + .replace('.', "/") + .split('/') + .filter(|s| !s.is_empty()) + .map(|s| s.to_string()) + .collect_vec(); + if segments.is_empty() { + return Ok(Err("Empty step address.".to_string())); + } + + let mut current = root; + let mut resolved: Option = None; + let mut path_segments: Vec<&str> = vec![]; + for (depth, segment) in segments.iter().enumerate() { + let (step_id, index) = parse_step_segment(segment); + path_segments.push(step_id); + + // parent_module_type is resolved the same way as in the tree CTE (the + // module named `step_id` in the parent's flow definition); it is + // constant across the sibling rows. + let siblings = sqlx::query!( + "SELECT j.id, + COALESCE(c.status::text, + CASE WHEN q.running AND q.suspend > 0 THEN 'suspended' + WHEN q.running THEN 'running' + ELSE 'queued' END) as \"status!\", + COALESCE(( + SELECT m->'value'->>'type' + FROM v2_job parent_j + LEFT JOIN flow_version fv ON fv.id = parent_j.runnable_id + AND parent_j.kind::text = 'flow' + LEFT JOIN flow f ON f.path = parent_j.runnable_path + AND f.workspace_id = parent_j.workspace_id + LEFT JOIN flow_node fn ON fn.id = parent_j.runnable_id + CROSS JOIN LATERAL jsonb_array_elements( + COALESCE(parent_j.raw_flow, fv.value, f.value, fn.flow)->'modules' + ) m + WHERE parent_j.id = $1 + AND m->>'id' = $3 + LIMIT 1 + ), '')::text as \"parent_module_type!\" + FROM v2_job j + LEFT JOIN v2_job_completed c ON c.id = j.id + LEFT JOIN v2_job_queue q ON q.id = j.id + LEFT JOIN ( + SELECT fj.jid, fj.ord + FROM (SELECT COALESCE( + (SELECT flow_status FROM v2_job_completed WHERE id = $1), + (SELECT flow_status FROM v2_job_status WHERE id = $1) + ) AS fs) pf + CROSS JOIN LATERAL ( + SELECT m FROM jsonb_array_elements(pf.fs->'modules') m + WHERE m->>'id' = $3 + LIMIT 1 + ) md + CROSS JOIN LATERAL jsonb_array_elements_text(md.m->'flow_jobs') + WITH ORDINALITY fj(jid, ord) + ) pos ON pos.jid = j.id::text + WHERE j.parent_job = $1 AND j.workspace_id = $2 AND j.flow_step_id = $3 + AND ($4::text[] IS NULL OR j.tag = ANY($4)) + ORDER BY pos.ord NULLS LAST, j.id", + current, + w_id, + step_id, + scope_tags, + ) + .fetch_all(db) + .await?; + + if siblings.is_empty() { + let available = sqlx::query_scalar!( + "SELECT DISTINCT flow_step_id as \"flow_step_id!\" FROM v2_job + WHERE parent_job = $1 AND workspace_id = $2 AND flow_step_id IS NOT NULL + AND ($3::text[] IS NULL OR tag = ANY($3)) + ORDER BY flow_step_id", + current, + w_id, + scope_tags, + ) + .fetch_all(db) + .await?; + return Ok(Err(format!( + "No step \"{}\" at this level. Available steps: {}.", + step_id, + if available.is_empty() { + "(none)".to_string() + } else { + available.join(", ") + } + ))); + } + + let parent_module_type = siblings[0].parent_module_type.clone(); + let node = if let Some(index) = index { + match index.checked_sub(1).and_then(|i| siblings.get(i)) { + Some(node) => node, + None => { + return Ok(Err(format!( + "Step \"{}\" has no iteration [{}]. Existing: 1..{}.", + step_id, + index, + siblings.len() + ))); + } + } + } else if siblings.len() > 1 { + // These diagnostics go verbatim into the model's context — cap the + // per-sibling listing so huge loops can't flood it. + const MAX_STATUSES_LISTED: usize = 20; + let mut statuses = siblings + .iter() + .take(MAX_STATUSES_LISTED) + .enumerate() + .map(|(i, s)| format!("[{}] {}", i + 1, s.status)) + .join(", "); + if siblings.len() > MAX_STATUSES_LISTED { + statuses.push_str(&format!( + ", … (+{} more)", + siblings.len() - MAX_STATUSES_LISTED + )); + } + return Ok(Err( + if is_fan_out_module(&parent_module_type) || parent_module_type.is_empty() { + format!( + "Step \"{}\" ran {} times (loop/branches) — pick one with \"{}[i]\". Iterations: {}.", + step_id, + siblings.len(), + step_id, + statuses + ) + } else { + format!( + "Step \"{}\" was retried — {} attempts, the last one is the final outcome. Pick one with \"{}[i]\". Attempts: {}.", + step_id, + siblings.len(), + step_id, + statuses + ) + }, + )); + } else { + &siblings[0] + }; + + let index_in_siblings = siblings.iter().position(|s| s.id == node.id).unwrap_or(0); + current = node.id; + resolved = Some(ResolvedStepJob { + job_id: node.id, + flow_step_id: step_id.to_string(), + path: path_segments.join("/"), + sibling_index: (index_in_siblings + 1) as i32, + sibling_count: siblings.len() as i32, + depth: (depth + 1) as i32, + parent_module_type, + }); + } + + Ok(Ok(resolved.expect("segments is non-empty"))) +} + +/// Results twin of `get_flow_all_logs_structured`: one entry per job of the +/// flow's execution tree (same recursive enumeration), carrying per-job status +/// and truncated result instead of logs. +async fn get_flow_all_results( + OptViewToken(view_token): OptViewToken, + OptAuthed(opt_authed): OptAuthed, + opt_tokened: OptTokened, + Extension(db): Extension, + Extension(user_db): Extension, + Path((w_id, id)): Path<(String, Uuid)>, + Query(query): Query, +) -> JsonResult { + let auth = authorize_flow_tree_read( + view_token, + &opt_authed, + &opt_tokened, + &db, + &user_db, + &w_id, + id, + ) + .await?; + let enclosing_job = auth.enclosing_job; + let scope_tags = auth.scope_tags; + + let max_result_len = query + .max_result_len + .unwrap_or(FLOW_ALL_RESULTS_DEFAULT_MAX_LEN) + .clamp(1, FLOW_ALL_RESULTS_MAX_MAX_LEN); + + // Step mode: resolve the address to one job directly (a few indexed + // lookups) instead of enumerating the whole tree. + if let Some(step) = query.step.as_deref().filter(|s| !s.trim().is_empty()) { + let resolved = + match resolve_flow_step_job(&db, &w_id, id, step, scope_tags.as_deref()).await? { + Ok(resolved) => resolved, + Err(step_error) => { + return Ok(Json(FlowAllResultsResponse { + enclosing_job, + entries: vec![], + truncated: false, + scope_filtered: scope_tags.is_some(), + step_error: Some(step_error), + })); + } + }; + + let record = sqlx::query!( + "SELECT j.kind::text as kind, + c.status::text as completed_status, + c.duration_ms as \"duration_ms?\", + COALESCE(c.started_at, q.started_at) as started_at, + LEFT(c.result::text, $3) as result_prefix, + length(c.result::text) as result_length, + q.running as \"q_running?\", + q.suspend as \"q_suspend?\" + FROM v2_job j + LEFT JOIN v2_job_completed c ON c.id = j.id + LEFT JOIN v2_job_queue q ON q.id = j.id + WHERE j.id = $1 AND j.workspace_id = $2", + resolved.job_id, + w_id, + max_result_len, + ) + .fetch_one(&db) + .await?; + + let kind = record.kind.as_deref().unwrap_or(""); + let status = match record.completed_status.as_deref() { + Some(s) => s.to_string(), + None => { + if record.q_running.unwrap_or(false) && record.q_suspend.unwrap_or(0) > 0 { + "suspended".to_string() + } else if record.q_running.unwrap_or(false) { + "running".to_string() + } else { + "queued".to_string() + } + } + }; + let success = record.completed_status.as_deref().map(|s| s == "success"); + let label = flow_tree_entry_label( + kind, + &resolved.path, + &resolved.parent_module_type, + resolved.sibling_index, + resolved.sibling_count, + resolved.depth, + ); + + return Ok(Json(FlowAllResultsResponse { + enclosing_job, + entries: vec![FlowResultEntry { + job_id: resolved.job_id.to_string(), + label, + kind: kind.to_string(), + flow_step_id: Some(resolved.flow_step_id), + step_path: Some(resolved.path), + depth: resolved.depth, + parent_module_type: if resolved.parent_module_type.is_empty() { + None + } else { + Some(resolved.parent_module_type) + }, + sibling_index: resolved.sibling_index, + sibling_count: resolved.sibling_count, + status, + success, + duration_ms: record.duration_ms, + started_at: record.started_at, + result_prefix: record.result_prefix, + result_length: record.result_length, + }], + truncated: false, + scope_filtered: scope_tags.is_some(), + step_error: None, + })); + } + + // Same recursive CTE as `collect_flow_log_entries` (kept textually in sync — + // sqlx macros cannot share the SQL string), joined against the completed and + // queue tables instead of `job_logs`. + let records = sqlx::query!( + "WITH RECURSIVE job_tree AS ( + SELECT j.id, j.kind::text, j.flow_step_id, j.parent_job, + '' as path_label, 0 as depth, + j.id::text as id_path, + ''::text as parent_module_type + FROM v2_job j + WHERE j.id = $2 AND j.workspace_id = $1 + UNION ALL + SELECT j.id, j.kind::text, j.flow_step_id, j.parent_job, + CASE + WHEN jt.path_label = '' THEN COALESCE(j.flow_step_id, '') + ELSE jt.path_label || '/' || COALESCE(j.flow_step_id, '') + END, + jt.depth + 1, + jt.id_path || '/' || j.id::text, + COALESCE(( + SELECT m->'value'->>'type' + FROM v2_job parent_j + LEFT JOIN flow_version fv ON fv.id = parent_j.runnable_id + AND parent_j.kind::text = 'flow' + LEFT JOIN flow f ON f.path = parent_j.runnable_path + AND f.workspace_id = parent_j.workspace_id + LEFT JOIN flow_node fn ON fn.id = parent_j.runnable_id + CROSS JOIN LATERAL jsonb_array_elements( + COALESCE(parent_j.raw_flow, fv.value, f.value, fn.flow)->'modules' + ) m + WHERE parent_j.id = jt.id + AND m->>'id' = j.flow_step_id + LIMIT 1 + ), '')::text + FROM v2_job j + JOIN job_tree jt ON j.parent_job = jt.id + WHERE j.workspace_id = $1 + AND ($4::text[] IS NULL OR j.tag = ANY($4)) + ), + positions AS ( + SELECT g.parent_job, g.flow_step_id, fj.jid, fj.ord + FROM (SELECT DISTINCT parent_job, flow_step_id FROM job_tree + WHERE parent_job IS NOT NULL AND flow_step_id IS NOT NULL) g + CROSS JOIN LATERAL ( + SELECT COALESCE( + (SELECT flow_status FROM v2_job_completed WHERE id = g.parent_job), + (SELECT flow_status FROM v2_job_status WHERE id = g.parent_job) + ) AS fs + ) pf + CROSS JOIN LATERAL ( + SELECT m FROM jsonb_array_elements(pf.fs->'modules') m + WHERE m->>'id' = g.flow_step_id + LIMIT 1 + ) md + CROSS JOIN LATERAL jsonb_array_elements_text(md.m->'flow_jobs') + WITH ORDINALITY fj(jid, ord) + ), + with_sibling_index AS ( + SELECT jt.*, + ROW_NUMBER() OVER ( + PARTITION BY jt.parent_job, jt.flow_step_id + ORDER BY pos.ord NULLS LAST, jt.id + ) as sibling_index, + COUNT(*) OVER ( + PARTITION BY jt.parent_job, jt.flow_step_id + ) as sibling_count + FROM job_tree jt + LEFT JOIN positions pos ON pos.parent_job = jt.parent_job + AND pos.flow_step_id = jt.flow_step_id + AND pos.jid = jt.id::text + ), + limited AS ( + SELECT * FROM with_sibling_index ORDER BY id_path ASC LIMIT $5 + ) + SELECT w.id, w.kind, w.flow_step_id, w.path_label, + w.sibling_index::int as sibling_index, + w.sibling_count::int as sibling_count, + w.depth::int as depth, + w.parent_module_type, + c.status::text as completed_status, + c.duration_ms as \"duration_ms?\", + COALESCE(c.started_at, q.started_at) as started_at, + LEFT(c.result::text, $3) as result_prefix, + length(c.result::text) as result_length, + q.running as \"q_running?\", + q.suspend as \"q_suspend?\" + FROM limited w + LEFT JOIN v2_job_completed c ON c.id = w.id + LEFT JOIN v2_job_queue q ON q.id = w.id + ORDER BY w.id_path ASC", + w_id, + id, + max_result_len, + scope_tags.as_deref(), + FLOW_ALL_RESULTS_MAX_ENTRIES + 1, + ) + .fetch_all(&db) + .await?; + + // One row past the cap fetched only to detect truncation. + let mut records = records; + let truncated = records.len() as i64 > FLOW_ALL_RESULTS_MAX_ENTRIES; + if truncated { + records.truncate(FLOW_ALL_RESULTS_MAX_ENTRIES as usize); + } + + let mut entries = Vec::with_capacity(records.len()); + + for record in records { + let kind = record.kind.as_deref().unwrap_or(""); + let step_id = record.flow_step_id.as_deref().unwrap_or(""); + let depth = record.depth.unwrap_or(0); + let sibling_count = record.sibling_count.unwrap_or(1) as i32; + let sibling_index = record.sibling_index.unwrap_or(0) as i32; + let parent_module_type = record.parent_module_type.as_deref().unwrap_or(""); + + let path = record.path_label.as_deref().unwrap_or(step_id); + let label = flow_tree_entry_label( + kind, + path, + parent_module_type, + sibling_index, + sibling_count, + depth, + ); + + let status = match record.completed_status.as_deref() { + Some(s) => s.to_string(), + None => { + if record.q_running.unwrap_or(false) && record.q_suspend.unwrap_or(0) > 0 { + "suspended".to_string() + } else if record.q_running.unwrap_or(false) { + "running".to_string() + } else { + "queued".to_string() + } + } + }; + let success = record.completed_status.as_deref().map(|s| s == "success"); + + entries.push(FlowResultEntry { + job_id: record.id.map(|u| u.to_string()).unwrap_or_default(), + label, + kind: kind.to_string(), + flow_step_id: record.flow_step_id.clone(), + step_path: record.path_label.clone(), + depth, + parent_module_type: if parent_module_type.is_empty() { + None + } else { + Some(parent_module_type.to_string()) + }, + sibling_index, + sibling_count, + status, + success, + duration_ms: record.duration_ms, + started_at: record.started_at, + result_prefix: record.result_prefix, + result_length: record.result_length, + }); + } + + Ok(Json(FlowAllResultsResponse { + enclosing_job, + entries, + truncated, + scope_filtered: scope_tags.is_some(), + step_error: None, + })) +} + async fn get_args( OptViewToken(view_token): OptViewToken, OptAuthed(opt_authed): OptAuthed, diff --git a/backend/windmill-common/src/error.rs b/backend/windmill-common/src/error.rs index f202bf27f9..0d5ebdc2c3 100644 --- a/backend/windmill-common/src/error.rs +++ b/backend/windmill-common/src/error.rs @@ -247,6 +247,41 @@ pub fn to_anyhow(e: T) -> anyhow:: From::from(e) } +/// Render a `tokio_postgres` error for a user-facing message. +/// +/// The pinned rust-postgres build prints only the error *kind* in its `Display` +/// impl, so `format!("{e}")` on one of these yields the useless `db error` and +/// drops the Postgres message. Interpolate errors from a `tokio_postgres::Client` +/// through this instead. +pub fn pg_error_message(e: &tokio_postgres::Error) -> String { + match e.as_db_error() { + Some(db_err) => format_db_error(db_err.message(), db_err.detail(), db_err.hint()), + // Non-database failures (io, tls, protocol) keep their message in the cause. + None => error_source_chain(e), + } +} + +fn format_db_error(message: &str, detail: Option<&str>, hint: Option<&str>) -> String { + let mut msg = message.to_string(); + if let Some(detail) = detail { + msg.push_str(&format!(" ({detail})")); + } + if let Some(hint) = hint { + msg.push_str(&format!(". Hint: {hint}")); + } + msg +} + +fn error_source_chain(e: &dyn std::error::Error) -> String { + let mut msg = e.to_string(); + let mut source = e.source(); + while let Some(cause) = source { + msg.push_str(&format!(": {cause}")); + source = cause.source(); + } + msg +} + impl IntoResponse for Error { fn into_response(self) -> axum::response::Response { let status = match self { @@ -419,4 +454,56 @@ mod tests { let rendered = Error::JsonErr(v).to_string(); assert_eq!(rendered, "[\n 1,\n 2,\n 3\n]"); } + + #[test] + fn db_error_renders_message_with_detail_and_hint() { + assert_eq!( + super::format_db_error("permission denied for schema public", None, None), + "permission denied for schema public" + ); + assert_eq!( + super::format_db_error("insert violates foreign key", Some("Key (id)=(1)"), None), + "insert violates foreign key (Key (id)=(1))" + ); + assert_eq!( + super::format_db_error( + "column does not exist", + None, + Some("Perhaps you meant \"b\"") + ), + "column does not exist. Hint: Perhaps you meant \"b\"" + ); + } + + #[test] + fn non_db_error_walks_the_source_chain() { + #[derive(Debug)] + struct Layer(&'static str, Option>); + impl std::fmt::Display for Layer { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + f.write_str(self.0) + } + } + impl std::error::Error for Layer { + fn source(&self) -> Option<&(dyn std::error::Error + 'static)> { + self.1 + .as_ref() + .map(|c| c.as_ref() as &(dyn std::error::Error + 'static)) + } + } + + // The pinned rust-postgres build renders only the kind, so everything + // actionable is in the causes: they must all reach the message. + let err = Layer( + "error connecting to server", + Some(Box::new(Layer( + "tcp connect error", + Some(Box::new(Layer("timed out", None))), + ))), + ); + assert_eq!( + super::error_source_chain(&err), + "error connecting to server: tcp connect error: timed out" + ); + } } diff --git a/backend/windmill-common/src/flow_conversations.rs b/backend/windmill-common/src/flow_conversations.rs index 33056f59fa..21b1f56389 100644 --- a/backend/windmill-common/src/flow_conversations.rs +++ b/backend/windmill-common/src/flow_conversations.rs @@ -5,6 +5,7 @@ use uuid::Uuid; use crate::db::DB; use crate::error::Result; +use crate::utils::truncate_with_ellipsis; #[derive(Debug, Clone, Copy, Serialize, Deserialize, PartialEq, Eq, sqlx::Type)] #[sqlx(type_name = "MESSAGE_TYPE", rename_all = "lowercase")] @@ -51,12 +52,8 @@ pub async fn get_or_create_conversation_with_id( return Ok(existing); } - // Truncate title to 25 char characters max - let title = if title.len() > 25 { - format!("{}...", &title[..25]) - } else { - title.to_string() - }; + // Truncate title to 25 characters max + let title = truncate_with_ellipsis(title, 25); // Create new conversation with provided ID let conversation = sqlx::query_as!( diff --git a/backend/windmill-common/src/instance_config.rs b/backend/windmill-common/src/instance_config.rs index abf1c1b662..0c8f8e4f4a 100644 --- a/backend/windmill-common/src/instance_config.rs +++ b/backend/windmill-common/src/instance_config.rs @@ -1077,12 +1077,7 @@ pub fn format_setting_value(key: &str, value: &serde_json::Value) -> String { }; } let value = mask_nested_sensitive(key, value); - let s = value.to_string(); - if s.len() > 200 { - format!("{}...", &s[..197]) - } else { - s - } + crate::utils::truncate_with_ellipsis(&value.to_string(), 197) } /// Extract the expiry timestamp from a license key JSON value. diff --git a/backend/windmill-common/src/lib.rs b/backend/windmill-common/src/lib.rs index 1eda7503af..30a5edad32 100644 --- a/backend/windmill-common/src/lib.rs +++ b/backend/windmill-common/src/lib.rs @@ -1257,7 +1257,7 @@ pub async fn create_custom_instance_database( tracing::warn!( "Failed to grant permissions on '{}': {}. Continuing.", dbname, - e + crate::error::pg_error_message(&e) ); } diff --git a/backend/windmill-common/src/query_builders.rs b/backend/windmill-common/src/query_builders.rs index c0eedadbc4..c315a564af 100644 --- a/backend/windmill-common/src/query_builders.rs +++ b/backend/windmill-common/src/query_builders.rs @@ -1,6 +1,8 @@ use serde::{Deserialize, Deserializer, Serialize}; use windmill_types::scripts::ScriptLang; +use crate::error::pg_error_message; + fn deserialize_bool_from_null<'de, D>(deserializer: D) -> Result where D: Deserializer<'de>, @@ -4848,7 +4850,7 @@ fn required_str<'a>( column: &str, ) -> Result<&'a str, String> { row.try_get(column) - .map_err(|e| format!("Failed to read column {}: {}", column, e))? + .map_err(|e| format!("Failed to read column {}: {}", column, pg_error_message(&e)))? .ok_or_else(|| format!("Unexpected NULL in column {}", column)) } @@ -4894,7 +4896,7 @@ pub async fn pg_get_full_schema( ) .await .map(simple_query_rows) - .map_err(|e| format!("Failed to query columns: {}", e))?; + .map_err(|e| format!("Failed to query columns: {}", pg_error_message(&e)))?; let fk_rows = client .simple_query( @@ -4922,7 +4924,7 @@ pub async fn pg_get_full_schema( ) .await .map(simple_query_rows) - .map_err(|e| format!("Failed to query foreign keys: {}", e))?; + .map_err(|e| format!("Failed to query foreign keys: {}", pg_error_message(&e)))?; let mut result: FullDatabaseSchema = std::collections::HashMap::new(); diff --git a/backend/windmill-common/src/utils.rs b/backend/windmill-common/src/utils.rs index 7b1f2a13f9..2bd6f1b204 100644 --- a/backend/windmill-common/src/utils.rs +++ b/backend/windmill-common/src/utils.rs @@ -1364,10 +1364,45 @@ pub fn strip_json_nul(serialized: &str) -> Cow<'_, str> { Cow::Owned(String::from_utf8(out).expect("removing a NUL escape preserves valid UTF-8")) } +/// Prefix of `s` holding at most `max_chars` characters. +/// Slicing by byte index (`&s[..n]`) panics when `n` lands inside a multibyte character. +pub fn truncate_chars(s: &str, max_chars: usize) -> &str { + match s.char_indices().nth(max_chars) { + Some((byte_idx, _)) => &s[..byte_idx], + None => s, + } +} + +/// Keep at most `max_chars` characters of `s`, appending `...` when anything was dropped — +/// so a truncated result is `max_chars + 3` characters long, not `max_chars`. +pub fn truncate_with_ellipsis(s: &str, max_chars: usize) -> String { + let truncated = truncate_chars(s, max_chars); + if truncated.len() < s.len() { + format!("{}...", truncated) + } else { + s.to_string() + } +} + #[cfg(test)] mod tests { use super::*; + #[test] + fn truncate_handles_multibyte_at_boundary() { + // Byte 25 of this string falls inside a 2-byte 'а'; naive `&s[..25]` would panic. + let cyrillic = "а".repeat(30); + assert_eq!(truncate_chars(&cyrillic, 25), "а".repeat(25)); + assert_eq!( + truncate_with_ellipsis(&cyrillic, 25), + format!("{}...", "а".repeat(25)) + ); + assert_eq!(truncate_with_ellipsis("ааааааааааааа", 25), "ааааааааааааа"); + assert_eq!(truncate_chars("abcd", 3), "abc"); + assert_eq!(truncate_with_ellipsis("abc", 3), "abc"); + assert_eq!(truncate_with_ellipsis("abcd", 3), "abc..."); + } + // The 6-char JSON escape for U+0000: backslash + "u0000". Written via an // escaped backslash so no literal NUL byte ever appears in this source. const NUL_ESC: &str = "\\u0000"; diff --git a/backend/windmill-common/src/wac_failure_corpus.json b/backend/windmill-common/src/wac_failure_corpus.json new file mode 100644 index 0000000000..e4498159cd --- /dev/null +++ b/backend/windmill-common/src/wac_failure_corpus.json @@ -0,0 +1,108 @@ +{ + "_readme": [ + "Cases the python and the typescript SDK must both satisfy when they serialize", + "a failed step() body into a `__wmill_error` marker.", + "", + "The two drifted apart twice while #10368 was in review: `name` was taken from", + "the constructor in one client and from `e.name` in the other, and `stack` was", + "written in two different formats. Each was caught by a reviewer reading the", + "diff, not by a test, because each suite only ever checked its own language.", + "A shared corpus turns the next divergence into a failure in both suites.", + "", + "It sits next to `wac_failure_record`, the function that decides the record's", + "final shape, because that is the contract these markers are the raw material", + "for. The SDK test suites read it by relative path.", + "", + "`thrown` describes a value each language constructs natively:", + " name the name the record must report", + " message the message the record must report", + " props own properties/attributes that must reach `extra`", + " circular_prop an own property holding a cycle, which must be dropped", + " without taking the rest of `extra` with it", + "`expect.stack` is only \"present\" or \"absent\": its text is language-specific", + "(each client matches its own executor's format) and is asserted there.", + "", + "Known divergence, deliberately not covered here: a non-finite float reaches", + "`extra` as the string \"NaN\" in python and as null in typescript, because", + "JSON.stringify has no hook for it. The executors differ the same way.", + "", + "Second known divergence, also not covered: an attribute named like a field", + "the executors report separately \u2014 `name`, `message`, `stack` \u2014 reaches", + "`extra` in python and not in typescript. Each client mirrors its own", + "executor, which differ the same way: the python one copies `__dict__`", + "wholesale, the bun/deno one filters that skip-list out." + ], + "cases": [ + { + "case": "a named error keeps its name and message", + "thrown": { + "name": "HttpError", + "message": "429 too many requests" + }, + "expect": { + "name": "HttpError", + "message": "429 too many requests", + "stack": "present", + "absent": [ + "extra" + ] + } + }, + { + "case": "custom properties reach extra", + "thrown": { + "name": "HttpError", + "message": "429", + "props": { + "code": 429, + "retry_after": 5, + "endpoint": "/v1/jobs" + } + }, + "expect": { + "name": "HttpError", + "message": "429", + "stack": "present", + "extra": { + "code": 429, + "retry_after": 5, + "endpoint": "/v1/jobs" + } + } + }, + { + "case": "a property that cannot be serialized is dropped on its own", + "thrown": { + "name": "RequestError", + "message": "socket hang up", + "props": { + "code": "ECONNRESET" + }, + "circular_prop": "request" + }, + "expect": { + "name": "RequestError", + "message": "socket hang up", + "stack": "present", + "extra": { + "code": "ECONNRESET" + } + } + }, + { + "case": "an error carrying nothing of its own has no extra", + "thrown": { + "name": "ValueError", + "message": "nope" + }, + "expect": { + "name": "ValueError", + "message": "nope", + "stack": "present", + "absent": [ + "extra" + ] + } + } + ] +} diff --git a/benchmarks/lib.ts b/benchmarks/lib.ts index be6fe5b529..57c877b441 100644 --- a/benchmarks/lib.ts +++ b/benchmarks/lib.ts @@ -2,7 +2,7 @@ import { sleep } from "https://deno.land/x/sleep@v1.2.1/mod.ts"; import * as windmill from "https://deno.land/x/windmill@v1.174.0/mod.ts"; import * as api from "https://deno.land/x/windmill@v1.174.0/windmill-api/index.ts"; -export const VERSION = "v1.773.0"; +export const VERSION = "v1.775.2"; export async function login(email: string, password: string): Promise { return await windmill.UserService.login({ diff --git a/cli/bun.lock b/cli/bun.lock index c783e411c7..c48e0f3f22 100644 --- a/cli/bun.lock +++ b/cli/bun.lock @@ -17,7 +17,7 @@ "minimatch": "^10.0.0", "open": "^10.0.0", "pg-gateway": "0.3.0-beta.4", - "svelte": "^5.45.2", + "svelte": "^5.56.8", "tar-stream": "^3.1.7", "windmill-parser-wasm-asset": "1.753.0", "windmill-parser-wasm-csharp": "1.510.1", @@ -156,7 +156,7 @@ "@stoplight/yaml-ast-parser": ["@stoplight/yaml-ast-parser@0.0.50", "", {}, "sha512-Pb6M8TDO9DtSVla9yXSTAxmo9GVEouq5P40DWXdOie69bXogZTkgvopCq+yEvTMA0F6PEvdJmbtTV3ccIp11VQ=="], - "@sveltejs/acorn-typescript": ["@sveltejs/acorn-typescript@1.0.9", "", { "peerDependencies": { "acorn": "^8.9.0" } }, "sha512-lVJX6qEgs/4DOcRTpo56tmKzVPtoWAaVbL4hfO7t7NVwl9AAXzQR6cihesW1BmNMPl+bK6dreu2sOKBP2Q9CIA=="], + "@sveltejs/acorn-typescript": ["@sveltejs/acorn-typescript@1.0.11", "", { "peerDependencies": { "acorn": "^8.9.0" } }, "sha512-LFuZUkjJ9iF7JZye/aG5XM0SFcQ5VyL0oVX4WJ9dc0Va3R3s0OauX1BESVCb+YN/ol8TAfqGDDAQsTG627Y5kw=="], "@types/bun": ["@types/bun@1.3.9", "", { "dependencies": { "bun-types": "1.3.9" } }, "sha512-KQ571yULOdWJiMH+RIWIOZ7B2RXQGpL1YQrBtLIV3FqDcCu6FsbFUBwhdKUlCKUpS3PJDsHlJ1QKlpxoVR+xtw=="], @@ -180,7 +180,7 @@ "ajv": ["ajv@8.18.0", "", { "dependencies": { "fast-deep-equal": "^3.1.3", "fast-uri": "^3.0.1", "json-schema-traverse": "^1.0.0", "require-from-string": "^2.0.2" } }, "sha512-PlXPeEWMXMZ7sPYOHqmDyCJzcfNrUr3fGNKtezX14ykXOEIvyK81d+qydx89KY5O71FKMPaQ2vBfBFI5NHR63A=="], - "aria-query": ["aria-query@5.3.2", "", {}, "sha512-COROpnaoap1E2F000S62r6A60uHZnmlvomhfyT2DlTcrY1OrBKn2UhH7qn5wTC9zMvD0AY7csdPSNwKP+7WiQw=="], + "aria-query": ["aria-query@5.3.1", "", {}, "sha512-Z/ZeOgVl7bcSYZ/u/rh0fOpvEpq//LZmdbkXyc7syVzjPAhfOa9ebsdTSjEBDU4vs5nC98Kfduj1uFo0qyET3g=="], "axobject-query": ["axobject-query@4.1.0", "", {}, "sha512-qIj0G9wZbMGNLjLmg1PT6v2mE9AH2zlnADJD/2tC6E00hgmhUOfEB6greHPAfLRSufHqROIUTkw6E+M3lH0PTQ=="], @@ -206,7 +206,7 @@ "define-lazy-prop": ["define-lazy-prop@3.0.0", "", {}, "sha512-N+MeXYoqr3pOgn8xfyRPREN7gHakLYjhsHhWGT3fWAiL4IkAt0iDw14QiiEm2bE30c5XX5q0FtAA3CK5f9/BUg=="], - "devalue": ["devalue@5.6.3", "", {}, "sha512-nc7XjUU/2Lb+SvEFVGcWLiKkzfw8+qHI7zn8WYXKkLMgfGSHbgCEaR6bJpev8Cm6Rmrb19Gfd/tZvGqx9is3wg=="], + "devalue": ["devalue@5.8.2", "", {}, "sha512-DObPPAfdtFbXjxLqK8s2Xk9ZuWz5+ZoFEhC7J76es4GU/rEiXwHTmbImoCdyoCOcBH1UF3+Cz6Z2sYD4hyl5TA=="], "diff": ["diff@5.2.2", "", {}, "sha512-vtcDfH3TOjP8UekytvnHH1o1P4FcUdt4eQ1Y+Abap1tk/OB2MWQvcwS2ClCd1zuIhc3JKOx6p3kod8Vfys3E+A=="], @@ -214,7 +214,7 @@ "esm-env": ["esm-env@1.2.2", "", {}, "sha512-Epxrv+Nr/CaL4ZcFGPJIYLWFom+YeV1DqMLHJoEd9SYRxNbaFruBwfEX/kkHUJf55j2+TUbmDcmuilbP1TmXHA=="], - "esrap": ["esrap@2.2.3", "", { "dependencies": { "@jridgewell/sourcemap-codec": "^1.4.15" } }, "sha512-8fOS+GIGCQZl/ZIlhl59htOlms6U8NvX6ZYgYHpRU/b6tVSh3uHkOHZikl3D4cMbYM0JlpBe+p/BkZEi8J9XIQ=="], + "esrap": ["esrap@2.3.0", "", { "dependencies": { "@jridgewell/sourcemap-codec": "^1.4.15" }, "peerDependencies": { "@typescript-eslint/types": "^8.2.0" }, "optionalPeers": ["@typescript-eslint/types"] }, "sha512-GQ/7RN8uOtEfNpzZzBMTzW9JBcX42oaSVtPzdF+6cEL8pqIL094iUpr9jzYGn4O4P/1S60dJ6izyT8F4LYARng=="], "events-universal": ["events-universal@1.0.1", "", { "dependencies": { "bare-events": "^2.7.0" } }, "sha512-LUd5euvbMLpwOF8m6ivPCbhQeSiYVNb8Vs0fQ8QjXo0JTkEHpz8pxdQf0gStltaPpw0Cca8b39KxvK9cfKRiAw=="], @@ -274,7 +274,7 @@ "string_decoder": ["string_decoder@1.1.1", "", { "dependencies": { "safe-buffer": "~5.1.0" } }, "sha512-n/ShnvDi6FHbbVfviro+WojiFzv+s8MPMHBczVePfUpDJLwoLT0ht1l4YwBCbi8pJAveEEdnkHyPyTP/mzRfwg=="], - "svelte": ["svelte@5.53.2", "", { "dependencies": { "@jridgewell/remapping": "^2.3.4", "@jridgewell/sourcemap-codec": "^1.5.0", "@sveltejs/acorn-typescript": "^1.0.5", "@types/estree": "^1.0.5", "@types/trusted-types": "^2.0.7", "acorn": "^8.12.1", "aria-query": "^5.3.1", "axobject-query": "^4.1.0", "clsx": "^2.1.1", "devalue": "^5.6.3", "esm-env": "^1.2.1", "esrap": "^2.2.2", "is-reference": "^3.0.3", "locate-character": "^3.0.0", "magic-string": "^0.30.11", "zimmerframe": "^1.1.2" } }, "sha512-yGONuIrcl/BMmqbm6/52Q/NYzfkta7uVlos5NSzGTfNJTTFtPPzra6rAQoQIwAqupeM3s9uuTf5PvioeiCdg9g=="], + "svelte": ["svelte@5.56.8", "", { "dependencies": { "@jridgewell/remapping": "^2.3.4", "@jridgewell/sourcemap-codec": "^1.5.0", "@sveltejs/acorn-typescript": "^1.0.10", "@types/estree": "^1.0.5", "@types/trusted-types": "^2.0.7", "acorn": "^8.12.1", "aria-query": "5.3.1", "axobject-query": "^4.1.0", "clsx": "^2.1.1", "devalue": "^5.8.1", "esm-env": "^1.2.1", "esrap": "^2.2.12", "is-reference": "^3.0.3", "locate-character": "^3.0.0", "magic-string": "^0.30.11", "zimmerframe": "^1.1.2" } }, "sha512-PY8LOw7xP6c8IOiVqdo0sbbZVYhXRSfklOQLAUyGBKqjTX0wx/z4l/9J+PmBpmlLnxzEb1NqltxQ5/wZme/Cmg=="], "tar-stream": ["tar-stream@3.1.7", "", { "dependencies": { "b4a": "^1.6.4", "fast-fifo": "^1.2.0", "streamx": "^2.15.0" } }, "sha512-qJj60CXt7IU1Ffyc3NJMjh6EkuCFej46zUqJ4J7pqYlThyd9bO0XBTmcOIhSzZJVWfsLks0+nle/j538YAW9RQ=="], diff --git a/cli/package.json b/cli/package.json index 548588a402..12f8e6158f 100644 --- a/cli/package.json +++ b/cli/package.json @@ -26,7 +26,7 @@ "minimatch": "^10.0.0", "open": "^10.0.0", "pg-gateway": "0.3.0-beta.4", - "svelte": "^5.45.2", + "svelte": "^5.56.8", "tar-stream": "^3.1.7", "windmill-parser-wasm-asset": "1.753.0", "windmill-parser-wasm-csharp": "1.510.1", diff --git a/cli/src/commands/app/bundle.ts b/cli/src/commands/app/bundle.ts index c515e1e86e..9987713e4f 100644 --- a/cli/src/commands/app/bundle.ts +++ b/cli/src/commands/app/bundle.ts @@ -2,6 +2,8 @@ import * as fs from "node:fs"; import * as path from "node:path"; import process from "node:process"; import { spawn } from "node:child_process"; +import { createRequire } from "node:module"; +import { pathToFileURL } from "node:url"; import * as log from "../../core/log.ts"; import { colors } from "@cliffy/ansi/colors"; import * as windmillUtils from "@windmill-labs/shared-utils"; @@ -78,38 +80,70 @@ export function detectFrameworks(appDir: string): { svelte: boolean; vue: boolea } } +/** + * Loads the Svelte compiler out of the *app's* node_modules. + * + * The app brings its own Svelte runtime via package.json, and compiler and + * runtime have to agree on internals: Svelte 5.52.0 moved delegated event + * handlers off `element.__click` onto a Symbol-keyed map, so an older + * compiler's `onclick` output is silently ignored by a newer runtime — the app + * builds and renders with every handler dead. A bare `import("svelte/compiler")` + * resolves against this CLI instead, whose own svelte floats independently of + * the app's, which is exactly how the two drift apart. + * + * Falls back to the CLI's own compiler when the app has none resolvable. + */ +export async function loadSvelteCompiler(appDir: string): Promise { + try { + const requireFromApp = createRequire( + path.join(path.resolve(appDir), "package.json") + ); + const entry = requireFromApp.resolve("svelte/compiler"); + return await import(pathToFileURL(entry).href); + } catch { + return await import("svelte/compiler"); + } +} + /** * Creates a Svelte esbuild plugin * Uses the svelte compiler from the project's node_modules */ function createSveltePlugin(appDir: string): any { + // Resolved once per build, not per file. + let compilerPromise: Promise | undefined; + const svelteCompiler = () => + (compilerPromise ??= loadSvelteCompiler(appDir)); + + // This converts a message in Svelte's format to esbuild's format + const messageConverter = + (source: string, filename: string) => + ({ message, start, end }: any) => { + let location; + if (start && end) { + const lineText = source.split(/\r\n|\r|\n/g)[start.line - 1]; + const lineEnd = start.line === end.line ? end.column : lineText.length; + location = { + file: filename, + line: start.line, + column: start.column, + length: lineEnd - start.column, + lineText, + }; + } + return { text: message, location }; + }; + return { name: "svelte", setup(build: any) { build.onLoad({ filter: /\.svelte$/ }, async (args: any) => { - // Import svelte compiler from the project's node_modules - const svelte = await import("svelte/compiler"); + const svelte = await svelteCompiler(); // Load the file from the file system const source = await readTextFile(args.path); const filename = path.relative(process.cwd(), args.path); - - // This converts a message in Svelte's format to esbuild's format - const convertMessage = ({ message, start, end }: any) => { - let location; - if (start && end) { - const lineText = source.split(/\r\n|\r|\n/g)[start.line - 1]; - const lineEnd = start.line === end.line ? end.column : lineText.length; - location = { - file: filename, - line: start.line, - column: start.column, - length: lineEnd - start.column, - lineText, - }; - } - return { text: message, location }; - }; + const convertMessage = messageConverter(source, filename); // Convert Svelte syntax to JavaScript try { @@ -120,6 +154,41 @@ function createSveltePlugin(appDir: string): any { return { errors: [convertMessage(e)] }; } }); + + // `lib.svelte.ts` / `lib.svelte.js` are plain modules that may use runes. + // They need `compileModule`, otherwise `$state`/`$derived` sail through + // esbuild untouched and the bundle throws "$state is not defined". + build.onLoad({ filter: /\.svelte\.[jt]s$/ }, async (args: any) => { + const svelte = await svelteCompiler(); + + const source = await readTextFile(args.path); + const filename = path.relative(process.cwd(), args.path); + const convertMessage = messageConverter(source, filename); + + try { + // `compileModule` parses with plain acorn and chokes on TypeScript, so + // types have to come off first (vite-plugin-svelte gets this for free + // by running after Vite's esbuild transform). + const code = filename.endsWith(".ts") + ? ( + await build.esbuild.transform(source, { + loader: "ts", + sourcefile: filename, + }) + ).code + : source; + + const { js, warnings } = svelte.compileModule(code, { filename }); + const contents = js.code + `//# sourceMappingURL=` + js.map.toUrl(); + return { + contents, + loader: "js", + warnings: warnings.map(convertMessage), + }; + } catch (e: any) { + return { errors: [convertMessage(e)] }; + } + }); }, }; } diff --git a/cli/src/core/constants.ts b/cli/src/core/constants.ts index 40aee3823e..59de21e2bf 100644 --- a/cli/src/core/constants.ts +++ b/cli/src/core/constants.ts @@ -10,4 +10,4 @@ export const WM_FORK_PREFIX = "wm-fork"; // (e.g. utils.ts) can read it without importing main.ts and creating a circular // dependency (main → workspace → utils → main) that triggers a TDZ. // Re-exported from main.ts for backwards compatibility. -export const VERSION = "1.773.0"; +export const VERSION = "1.775.2"; diff --git a/cli/test/raw_app_svelte_compiler_resolution_unit.test.ts b/cli/test/raw_app_svelte_compiler_resolution_unit.test.ts new file mode 100644 index 0000000000..5fbf05384e --- /dev/null +++ b/cli/test/raw_app_svelte_compiler_resolution_unit.test.ts @@ -0,0 +1,89 @@ +/** + * A raw app brings its own Svelte *runtime* via package.json, so it has to be + * compiled with its own *compiler* too — the two share internals that change + * between versions. Svelte 5.52.0 moved delegated event handlers off + * `element.__click` onto a Symbol-keyed map, so compiling with a version on the + * other side of that line produces an app that builds, renders, and has every + * `onclick` silently dead. + * + * `import("svelte/compiler")` resolves against the CLI, whose svelte floats + * independently of the app's. These cover the app-local resolution that avoids + * that drift. + */ + +import { afterEach, beforeEach, describe, expect, test } from "bun:test"; +import * as fs from "node:fs"; +import * as path from "node:path"; +import * as os from "node:os"; +import { loadSvelteCompiler } from "../src/commands/app/bundle.ts"; + +let tempDir: string; + +/** A stand-in svelte install, identifiable by a version the real one never has. */ +function installStubSvelte(dir: string, version: string) { + const pkgDir = path.join(dir, "node_modules", "svelte"); + fs.mkdirSync(pkgDir, { recursive: true }); + fs.writeFileSync( + path.join(pkgDir, "package.json"), + JSON.stringify({ + name: "svelte", + version, + type: "module", + exports: { "./compiler": "./compiler.js" }, + }), + "utf-8", + ); + fs.writeFileSync( + path.join(pkgDir, "compiler.js"), + `export const VERSION = ${JSON.stringify(version)};\n` + + `export function compile() { return { js: { code: "", map: null }, warnings: [] }; }\n` + + `export function compileModule() { return { js: { code: "", map: null }, warnings: [] }; }\n`, + "utf-8", + ); +} + +beforeEach(() => { + tempDir = fs.mkdtempSync(path.join(os.tmpdir(), "raw-app-svelte-compiler-")); + fs.writeFileSync( + path.join(tempDir, "package.json"), + JSON.stringify({ name: "app", private: true, dependencies: { svelte: "*" } }), + "utf-8", + ); +}); + +afterEach(() => { + fs.rmSync(tempDir, { recursive: true, force: true }); +}); + +describe("loadSvelteCompiler", () => { + test("uses the app's own compiler rather than the CLI's", async () => { + installStubSvelte(tempDir, "0.0.0-app-local"); + + const compiler = await loadSvelteCompiler(tempDir); + + expect(compiler.VERSION).toBe("0.0.0-app-local"); + }); + + test("resolves against the app even when given a relative dir", async () => { + installStubSvelte(tempDir, "0.0.0-relative"); + const cwd = process.cwd(); + try { + process.chdir(path.dirname(tempDir)); + const compiler = await loadSvelteCompiler(path.basename(tempDir)); + expect(compiler.VERSION).toBe("0.0.0-relative"); + } finally { + process.chdir(cwd); + } + }); + + test("falls back to the CLI's compiler when the app has none", async () => { + const compiler = await loadSvelteCompiler(tempDir); + + // The real thing, not a stub — and new enough to emit the Symbol-map form + // of delegated handlers that a modern runtime reads. + expect(typeof compiler.compile).toBe("function"); + expect(compiler.VERSION).not.toContain("app-local"); + const [major, minor] = compiler.VERSION.split(".").map(Number); + expect(major > 5 || (major === 5 && minor >= 52)).toBe(true); + }); +}); diff --git a/cli/test/raw_app_svelte_module_unit.test.ts b/cli/test/raw_app_svelte_module_unit.test.ts new file mode 100644 index 0000000000..736e048100 --- /dev/null +++ b/cli/test/raw_app_svelte_module_unit.test.ts @@ -0,0 +1,118 @@ +/** + * `lib.svelte.ts` / `lib.svelte.js` modules are plain modules that may use + * runes. They need `svelte.compileModule`; without it esbuild happily bundles + * `$state(...)` as an ordinary call and the app dies at runtime with + * "ReferenceError: $state is not defined". + */ + +import { afterAll, beforeAll, describe, expect, test } from "bun:test"; +import * as fs from "node:fs"; +import * as path from "node:path"; +import * as os from "node:os"; +import { createBundle } from "../src/commands/app/bundle.ts"; + +let tempDir: string; +let originalCwd: string; + +// Runes used as bare calls, ignoring the `$state(...)` mentions inside Svelte's +// own warning message template literals. +function bareRuneCalls(js: string): string[] { + return [...js.matchAll(/(^|[^.\w$`])\$(state|derived|effect|props)\s*\(/g)].map( + (m) => m[0] + ); +} + +function writeApp(files: Record) { + for (const [name, content] of Object.entries(files)) { + fs.writeFileSync(path.join(tempDir, name), content, "utf-8"); + } +} + +async function bundle(entry: string): Promise { + const { js } = await createBundle({ + entryPoint: path.join(tempDir, entry), + minify: false, + sourcemap: false, + }); + return js; +} + +beforeAll(() => { + originalCwd = process.cwd(); + tempDir = fs.mkdtempSync(path.join(os.tmpdir(), "raw-app-svelte-module-")); + fs.writeFileSync( + path.join(tempDir, "package.json"), + JSON.stringify({ name: "app", private: true, dependencies: { svelte: "*" } }), + "utf-8" + ); + // Reuse the CLI's own svelte install instead of paying for an npm install; + // `ensureNodeModules` only checks that the directory is there. + fs.symlinkSync( + path.join(originalCwd, "node_modules"), + path.join(tempDir, "node_modules") + ); + process.chdir(tempDir); +}); + +afterAll(() => { + process.chdir(originalCwd); + fs.rmSync(tempDir, { recursive: true, force: true }); +}); + +describe("svelte plugin: .svelte.ts modules", () => { + test("compiles runes in a TypeScript rune module and the bundle runs", async () => { + writeApp({ + "lib.svelte.ts": `export class Cycle { + #options: T[] = []; + #index = $state(0); + current = $derived(this.#options[this.#index]); + + constructor(options: T[]) { + this.#options = options; + } + + next() { + this.#index = (this.#index + 1) % this.#options.length; + } +} +`, + "ts_entry.ts": `import { Cycle } from './lib.svelte'; + +const cycle = new Cycle(['a', 'b', 'c']); +cycle.next(); +(globalThis as any).__cycleResult = cycle.current; +`, + }); + + const js = await bundle("ts_entry.ts"); + + expect(bareRuneCalls(js)).toEqual([]); + // Runtime is the real check: unfixed, this throws + // "ReferenceError: $state is not defined". + new Function(js)(); + expect((globalThis as any).__cycleResult).toBe("b"); + }); + + test("compiles runes in a JavaScript rune module", async () => { + writeApp({ + "counter.svelte.js": `export const counter = $state({ n: 0 }); + +export function bump() { + counter.n += 1; +} +`, + "js_entry.ts": `import { counter, bump } from './counter.svelte.js'; + +bump(); +bump(); +(globalThis as any).__counterResult = counter.n; +`, + }); + + const js = await bundle("js_entry.ts"); + + expect(bareRuneCalls(js)).toEqual([]); + new Function(js)(); + expect((globalThis as any).__counterResult).toBe(2); + }); +}); diff --git a/docs/wac-sdk-e2e.md b/docs/wac-sdk-e2e.md new file mode 100644 index 0000000000..5c6682d93d --- /dev/null +++ b/docs/wac-sdk-e2e.md @@ -0,0 +1,68 @@ +# Exercising an unreleased SDK change end to end + +A job installs the **published** `windmill-client` / `wmill`, so a change in this +repo is invisible to a real job until it is injected into the worker's dependency +cache. The SDK unit suites cover neither the worker nor the SDK the worker +installs, and that gap is where the Workflow-as-Code failure contract kept coming +apart: every finding behind #10366, #10367 and #10368 came from review or from a +run like the one below, never from a green suite. + +## Recipe + +Use a private `WINDMILL_DIR`. `/tmp/windmill/` is shared by every worktree's +backend, so patching it in place leaks a modified SDK into other people's jobs. + +```bash +# Run from the repository root. The backend holds a terminal of its own; every +# other command is a subshell, so nothing depends on where the last one left you. + +# 1. a backend of your own, with its own cache root — in its own terminal +(cd backend && DATABASE_URL=... PORT=8062 WINDMILL_DIR=/tmp/windmill-mytest \ + cargo run --features quickjs) # add ,python to run python jobs + +# 2. one job to populate the cache with the published SDK +# (any preview job importing the client will do) + +# 3. build the client and overwrite what the cache holds +C=$(echo /tmp/windmill-mytest/cache_nomount/bun/windmill-client@*@@@1) +(cd typescript-client && ./build.sh && npx tsdown --format esm --no-dts \ + && cp dist/index.mjs "$C/dist/index.mjs" \ + && cp dist/index.mjs "$C/dist/client.mjs") # code-split package: cover both + +# python instead: copy the source file straight over, then drop the bytecode +cp python-client/wmill/wmill/client.py \ + /tmp/windmill-mytest/cache/python_3_12/wmill==*/wmill/client.py +find /tmp/windmill-mytest -name __pycache__ -type d -exec rm -rf {} + + +# 4. RESTART the backend — see below +# 5. run your scenarios, and rm -rf /tmp/windmill-mytest when done +``` + +## Restart the workers after injecting + +A worker materializes the package once and keeps using its copy, so patching the +cache under a running backend leaves some workers on the old code. With more than +one worker the results then **alternate run to run** as jobs land on one worker or +the other, which reads like flakiness in the product rather than in the harness. +Restarting after the swap makes it deterministic. + +Symptom worth recognising: identical jobs returning two different answers in a +stable pattern, with each run internally consistent. + +## Run your scenarios twice + +Once against the published SDK, once against the injected one. A scenario that +behaves the same either way is not testing what you think it is, and it is easy +to write several of those without noticing. + +As a calibration: a spread of WAC scenarios written this way scored 10/17 (bun) +and 7/18 (python) against the published SDK and 17/17 and 18/18 against a client +carrying #10366, #10367 and #10368. The ones that did not move were covering +behaviour those PRs never touched — worth knowing before concluding that a green +run means anything. + +## Worth covering, and easy to miss + +The deno path, `taskScript` / `taskFlow`, `waitForApproval`, and failures +interleaved with parallelism. None of these were exercised while the failure +record was being unified. diff --git a/frontend/package-lock.json b/frontend/package-lock.json index 1cbd409013..4ab6621c30 100644 --- a/frontend/package-lock.json +++ b/frontend/package-lock.json @@ -1,12 +1,12 @@ { "name": "@windmill-labs/components", - "version": "1.773.0", + "version": "1.775.2", "lockfileVersion": 3, "requires": true, "packages": { "": { "name": "@windmill-labs/components", - "version": "1.773.0", + "version": "1.775.2", "hasInstallScript": true, "license": "AGPL-3.0", "dependencies": { diff --git a/frontend/package.json b/frontend/package.json index 9c300c94d1..01e68d2acf 100644 --- a/frontend/package.json +++ b/frontend/package.json @@ -1,6 +1,6 @@ { "name": "@windmill-labs/components", - "version": "1.773.0", + "version": "1.775.2", "scripts": { "dev": "vite dev", "dev:ui-builder": "mv static/ui_builder static/ui_builder.dev-disabled 2>/dev/null || true ; trap 'mv static/ui_builder.dev-disabled static/ui_builder 2>/dev/null || true' EXIT ; vite dev", diff --git a/frontend/scripts/ui_builder_artifact.json b/frontend/scripts/ui_builder_artifact.json index 4360794dbe..dcba3214de 100644 --- a/frontend/scripts/ui_builder_artifact.json +++ b/frontend/scripts/ui_builder_artifact.json @@ -1,5 +1,5 @@ { "baseUrl": "https://pub-06154ed168a24e73a86ab84db6bf15d8.r2.dev", - "version": "1f1fe4f", - "sha256": "4c20b2b51f324e93dda3b46d914ebf7bc7d0cb4dc7eb2d37068408191220d206" + "version": "2471791", + "sha256": "04048791fb6b2ebca5cc30447e163826cdb143d65cce07a629252e131ac83618" } diff --git a/frontend/src/lib/assets/tokens/githubDark.json b/frontend/src/lib/assets/tokens/githubDark.json new file mode 100644 index 0000000000..ea5b36145e --- /dev/null +++ b/frontend/src/lib/assets/tokens/githubDark.json @@ -0,0 +1,35 @@ +{ + "surface-accent-primary": "#506ab8", + "surface-accent-hover": "#5873c8", + "surface-accent-clicked": "#425797", + "text-primary": "#d5dbe2", + "text-secondary": "#8b949e", + "text-primary-inverse": "#24292f", + "text-secondary-inverse": "#57606a", + "text-tertiary-inverse": "#6e7781", + "surface-selected": "#212830", + "surface-disabled": "#151b23", + "surface-secondary": "#010409", + "surface-hover": "#6e768119", + "surface-primary": "#0d1117", + "border-light": "#242a32", + "border-normal": "#6e7681", + "border-accent": "#4a6ed6", + "surface-accent-selected": "#141a2e", + "surface-accent-secondary": "#e3e8ee", + "surface-tertiary": "#151b23", + "text-emphasis": "#dfe5eb", + "text-hint": "#6e7681", + "text-disabled": "#484f58", + "surface-accent-secondary-hover": "#d0d7de", + "surface-accent-secondary-clicked": "#afb8c1", + "component-button-accent-secondary": "#f0f6fc", + "text-emphasis-inverse": "#1f2328", + "reserved-ai": "#d2a8ff", + "component-virtual-node": "#151b23", + "text-accent": "#bdc8ee", + "border-selected": "#5a72c4", + "surface-sunken": "#010409", + "text-tertiary": "#6e7681", + "surface-input": "#0c0f14" +} diff --git a/frontend/src/lib/assets/tokens/tokens.json b/frontend/src/lib/assets/tokens/tokens.json index f4f35c81d9..349b4376b8 100644 --- a/frontend/src/lib/assets/tokens/tokens.json +++ b/frontend/src/lib/assets/tokens/tokens.json @@ -239,7 +239,12 @@ "magenta-950": "#6e2ba1" } }, - "guidelines": { "mode-1": { "blue": "#5e81ac", "demo-background": "#ffffff00" } }, + "guidelines": { + "mode-1": { + "blue": "#5e81ac", + "demo-background": "#ffffff00" + } + }, "tailwind-c-s-s-v-3-3-2": { "mode-1": { "black": "#000000", diff --git a/frontend/src/lib/components/CompareDrafts.svelte b/frontend/src/lib/components/CompareDrafts.svelte index 32be6caf35..65fe205046 100644 --- a/frontend/src/lib/components/CompareDrafts.svelte +++ b/frontend/src/lib/components/CompareDrafts.svelte @@ -22,7 +22,11 @@ draftBaseIsStale } from '$lib/utils_draft_deploy' import { checkDeployPermission, type DeployPermission } from '$lib/utils_workspace_deploy' - import { type DraftItem, useWorkspaceDrafts } from '$lib/workspaceDrafts.svelte' + import { + type DraftItem, + invalidateWorkspaceDrafts, + useWorkspaceDrafts + } from '$lib/workspaceDrafts.svelte' import type { Kind as LayoutKind } from '$lib/utils_deployable' import { userStore } from '$lib/stores' @@ -44,8 +48,8 @@ draftCount?: number /** When set (reached via a session's Review button), preselect only the * rows this chat modified — `${UserDraftItemKind}:${path}` keys, matching - * Row.key. Undefined → preselect all deployable rows (the default). All rows - * are still shown either way. */ + * Row.key. Undefined → preselect all actionable rows (the default). All + * rows are still shown either way. */ chatMask?: Set /** False while the (async) chatMask is still loading. The select-all default * waits for this so it doesn't race the mask and select everything. Defaults @@ -180,35 +184,45 @@ isFork && hideUnchanged ? items.filter((i) => i.unchanged_from_parent !== true) : items ) - // A row is actionable when it isn't already deployed this session, the user has - // write permission, AND it's their own draft (you can't deploy someone else's - // draft — those show view-only in the "all drafts" view). The server enforces - // the same; this keeps the UI honest. A data-pipeline bundle is never deployable - // from this page — its scripts deploy individually inside the pipeline view — so - // it's excluded from every selection path. - function isSelectable(item: Row): boolean { + // Selection gate: a row is actionable when it isn't already deployed this + // session AND it's the user's own draft (someone else's shows view-only in the + // "all drafts" view). A data-pipeline bundle is excluded — its scripts deploy + // individually inside the pipeline view. Every selectable row can at least be + // discarded: discarding your own email-scoped draft never needs write + // permission on the path — only legacy (ownerless) drafts stay write-gated, + // mirroring the server's discard check. + function isDiscardable(item: Row): boolean { return ( deploymentStatus[item.key]?.status !== 'deployed' && - item.can_write && item.mine && - item.draftKind !== 'data_pipeline' + item.draftKind !== 'data_pipeline' && + (!item.legacy_draft || item.can_write) ) } - // Why a row can't be deployed (drives the disabled-checkbox tooltip). - // `undefined` ⇒ actionable. + // Deploying additionally requires write permission on the path, so the Deploy + // count can be lower than the selection when a drafted path lost writability. + function isDeployable(item: Row): boolean { + return isDiscardable(item) && item.can_write + } + + // Why a row can't be selected (drives the disabled-checkbox tooltip). + // `undefined` ⇒ selectable. function blockedReason(item: Row): string | undefined { if (!item.mine) return 'This draft belongs to another user' - if (!item.can_write) return "You don't have write permission on this path" + if (item.legacy_draft && !item.can_write) + return 'Discarding a legacy draft requires write permission on the path' return undefined } // Why a row can't be discarded (drives the Discard button's title). - // Discarding only removes the caller's own draft row, which they always own, - // so — unlike deploy — it never requires write permission on the path. The - // only block is someone else's draft (view-only in the "all drafts" view). + // Discarding only removes the caller's own draft row, so it doesn't require + // write permission on the path — except for legacy (ownerless) drafts, which + // the server write-gates like a deploy. function discardBlockedReason(item: Row): string | undefined { if (!item.mine) return 'This draft belongs to another user' + if (item.legacy_draft && !item.can_write) + return 'Discarding a legacy draft requires write permission on the path' return undefined } @@ -290,8 +304,9 @@ if (ws === currentWorkspaceId) deployPerm = p }) }) - // Select all on the first non-empty load (deploy-all is the common intent); - // only once, so a refetch after a deploy doesn't re-select the leftovers. + // Select all on the first non-empty load (acting on everything is the common + // intent); only once, so a refetch after a deploy doesn't re-select the + // leftovers. let hasAutoSelected = $state(false) const deploymentStatus: Record< @@ -314,9 +329,9 @@ $effect(() => { if (!hasAutoSelected && chatMaskReady && visibleItems.length > 0) { - // Default intent is deploy-all; when reached from a session's Review + // Default intent is act-on-all; when reached from a session's Review // (chatMask set), preselect only that chat's items instead. - const selectable = visibleItems.filter(isSelectable) + const selectable = visibleItems.filter(isDiscardable) selectedItems = ( chatMask ? selectable.filter((i) => @@ -332,17 +347,24 @@ } }) - // Selected items still in the visible list and deployable. Derived (not a - // pruning effect) so the "Deploy N drafts" button stays reactive to the - // Workspace Drafts resource: deploy/discard drop items, and stale keys left in - // selectedItems are simply ignored here (and by deploySelected). - let selectedCount = $derived( - visibleItems.filter((i) => selectedItems.includes(i.key) && isSelectable(i)).length + // Selected items still in the visible list, per action. Derived (not a + // pruning effect) so the footer buttons stay reactive to the Workspace + // Drafts resource: deploy/discard drop items, and stale keys left in + // selectedItems are simply ignored here (and by the action handlers). + let deployableCount = $derived( + visibleItems.filter((i) => selectedItems.includes(i.key) && isDeployable(i)).length ) + let discardableCount = $derived( + visibleItems.filter((i) => selectedItems.includes(i.key) && isDiscardable(i)).length + ) + // Selected rows the user can discard but not deploy (own draft on a path + // without write permission) — surfaced under the footer so the diverging + // button counts are explained. + let undeployableSelectedCount = $derived(discardableCount - deployableCount) let allSelected = $derived( - visibleItems.filter(isSelectable).length > 0 && - visibleItems.filter(isSelectable).every((i) => selectedItems.includes(i.key)) + visibleItems.filter(isDiscardable).length > 0 && + visibleItems.filter(isDiscardable).every((i) => selectedItems.includes(i.key)) ) function toggleItem(item: { key: string }) { @@ -354,7 +376,7 @@ } function selectAll() { - selectedItems = visibleItems.filter(isSelectable).map((i) => i.key) + selectedItems = visibleItems.filter(isDiscardable).map((i) => i.key) } function deselectAll() { @@ -394,8 +416,8 @@ deploying = true // Snapshot the items to deploy: deployDraft invalidates the Workspace Drafts // resource, so `items` can change mid-loop — iterate a stable copy. Guard on - // isSelectable so a non-writable row can never be deployed via a stale key. - const toDeploy = visibleItems.filter((i) => selectedItems.includes(i.key) && isSelectable(i)) + // isDeployable so a non-writable row can never be deployed via a stale key. + const toDeploy = visibleItems.filter((i) => selectedItems.includes(i.key) && isDeployable(i)) let deployedAny = false for (const item of toDeploy) { deploymentStatus[item.key] = { status: 'loading' } @@ -470,6 +492,64 @@ if (item) void doDiscard(item) } + // --- Bulk discard --- + // One click can drop many drafts at once, and some of them (draft_only with + // no other drafter) are permanent deletions — always confirm, listing the + // permanent ones explicitly. + let bulkDiscardItems = $state(undefined) + let discarding = $state(false) + const bulkPermanent = $derived((bulkDiscardItems ?? []).filter(isDestructiveDiscard)) + // Third outcome the modal must cover: a draft-only item someone else also + // drafted isn't deleted — only this user's draft goes; the item survives via + // the other drafts. + const bulkSharedCount = $derived( + (bulkDiscardItems ?? []).filter((i) => i.draft_only && !isDestructiveDiscard(i)).length + ) + + function onDiscardSelectedClick() { + const toDiscard = visibleItems.filter((i) => selectedItems.includes(i.key) && isDiscardable(i)) + if (toDiscard.length > 0) bulkDiscardItems = toDiscard + } + + async function discardSelected(toDiscard: Row[]) { + discarding = true + let changed = false + for (const item of toDiscard) { + deploymentStatus[item.key] = { status: 'loading' } + // invalidate: false — one refetch after the whole batch (below), not + // one per row. + const res = await discardDraft( + item.draftKind, + item.path, + currentWorkspaceId, + item.draft_only, + item.legacy_draft, + false + ) + if (res.success) { + changed = true + delete deploymentStatus[item.key] + } else { + deploymentStatus[item.key] = { status: 'failed', error: res.error } + sendUserToast(`Failed to discard ${item.path}: ${res.error}`, true) + } + } + discarding = false + selectedItems = [] + if (changed) { + // Refetch the Draft list once for the batch, then refresh the fork + // comparison. + invalidateWorkspaceDrafts(currentWorkspaceId) + onChanged?.() + } + } + + function confirmBulkDiscard() { + const toDiscard = bulkDiscardItems + bulkDiscardItems = undefined + if (toDiscard) void discardSelected(toDiscard) + } + // Editor URL for a draft item, scoped to the current workspace. Raw apps live // under a different editor route, so map their kind accordingly. Kinds whose // editor is a drawer on a list page (variables, resources, schedules, @@ -559,7 +639,7 @@ {selectedItems} {deploymentStatus} {allSelected} - selectablePredicate={(item) => isSelectable(item as unknown as Row)} + selectablePredicate={(item) => isDiscardable(item as unknown as Row)} selectBlockedReason={(item) => blockedReason(item as unknown as Row)} onToggleItem={toggleItem} onSelectAll={selectAll} @@ -752,17 +832,34 @@ {#snippet footer()}
- +
+ + +
{#if !deployPerm.ok} {deployPerm.reason} + {:else if undeployableSelectedCount > 0} + + {undeployableSelectedCount} selected draft{undeployableSelectedCount !== 1 ? 's' : ''} + can't be deployed (no write permission on the path) but can still be discarded + {/if}
{/snippet} @@ -772,6 +869,39 @@ + (bulkDiscardItems = undefined)} +> +

+ This will discard {bulkDiscardItems?.length} draft{(bulkDiscardItems?.length ?? 0) !== 1 + ? 's' + : ''}. Items with a deployed version revert to it. +

+ {#if bulkSharedCount > 0} +

+ {bulkSharedCount} draft-only {bulkSharedCount === 1 ? 'item is' : 'items are'} also drafted by + other users: only your draft is removed and the {bulkSharedCount === 1 ? 'item' : 'items'} will + remain through theirs. +

+ {/if} + {#if bulkPermanent.length > 0} +

+ {bulkPermanent.length} + {bulkPermanent.length === 1 ? 'item exists' : 'items exist'} only as a draft and will be + permanently deleted: +

+
    + {#each bulkPermanent as item (item.key)} +
  • {item.draft_path ?? item.path}
    • + {/each} +
+ {/if} + + - import { isInitialized } from './vscode' + import { getEditorTheme, isInitialized } from './vscode' import { editor as meditor } from 'monaco-editor' @@ -8,11 +8,7 @@ function onThemeChange() { if (isInitialized) { - if (document.documentElement.classList.contains('dark')) { - meditor.setTheme('nord') - } else { - meditor.setTheme('myTheme') - } + meditor.setTheme(getEditorTheme()) } } diff --git a/frontend/src/lib/components/ScriptEditor.svelte b/frontend/src/lib/components/ScriptEditor.svelte index aaca5752c1..9a65fc9d2b 100644 --- a/frontend/src/lib/components/ScriptEditor.svelte +++ b/frontend/src/lib/components/ScriptEditor.svelte @@ -2505,7 +2505,7 @@ {/snippet} {#snippet editorContent()} -
+
{#if supportsModules}
(getDarkModeVariant()) const dispatch = createEventDispatcher() @@ -51,15 +59,39 @@ } - + (darkVariant = getDarkModeVariant())} + on:close={removeHash} + {disableChatOffset} +>
{#if scopes == undefined}
-
- Theme +
+
+ Theme +
+
+ Dark variant + { + darkVariant = v + setDarkModeVariant(v) + }} + > + {#snippet children({ item })} + + + {/snippet} + +
Windmill diff --git a/frontend/src/lib/components/apps/editor/PublicApp.svelte b/frontend/src/lib/components/apps/editor/PublicApp.svelte index 8995684e28..c0cf693e0f 100644 --- a/frontend/src/lib/components/apps/editor/PublicApp.svelte +++ b/frontend/src/lib/components/apps/editor/PublicApp.svelte @@ -1,5 +1,6 @@ - - - - - - - {label ? label + ' · ' : ''}{formatElapsed(loadingElapsedMs)} - + + Waiting for your input + +{:else} + + + + + + + {label ? label + ' · ' : ''}{formatElapsed(elapsedMs)} + +{/if} diff --git a/frontend/src/lib/components/copilot/chat/ToolExecutionDisplay.svelte b/frontend/src/lib/components/copilot/chat/ToolExecutionDisplay.svelte index 8360ed962f..a1ee9984e5 100644 --- a/frontend/src/lib/components/copilot/chat/ToolExecutionDisplay.svelte +++ b/frontend/src/lib/components/copilot/chat/ToolExecutionDisplay.svelte @@ -26,6 +26,7 @@ const isSuccessful = $derived( !message.isLoading && + !message.isQueued && !message.error && !message.needsConfirmation && !message.isStreamingArguments @@ -64,7 +65,14 @@ {#if activeUserQuestion} {:else} -
+ +
{#snippet headerButton()} + {#snippet text()} + Hide this marker until the next run. The error handler itself is kept. + {/snippet} + +
+
+ diff --git a/frontend/src/lib/components/home/ItemsList.svelte b/frontend/src/lib/components/home/ItemsList.svelte index 8eafad1824..041d8fc58e 100644 --- a/frontend/src/lib/components/home/ItemsList.svelte +++ b/frontend/src/lib/components/home/ItemsList.svelte @@ -1361,7 +1361,7 @@ if (itemKind != 'all') { subtab = v } - setQuery(page.url, 'kind', v) + setQuery('kind', v) }} > {#snippet children({ item })} diff --git a/frontend/src/lib/components/instanceSettings/GhesAppSettings.svelte b/frontend/src/lib/components/instanceSettings/GhesAppSettings.svelte index 86e3038fdf..ae5347d278 100644 --- a/frontend/src/lib/components/instanceSettings/GhesAppSettings.svelte +++ b/frontend/src/lib/components/instanceSettings/GhesAppSettings.svelte @@ -184,11 +184,15 @@ Homepage URL: your Windmill instance URL
  • - Callback URL: <your-windmill-url>/gh_success + Setup URL (under "Post installation"): + <your-windmill-url>/gh_success with "Redirect on update" checked. + Required for installing the app from workspace Git sync settings — GitHub redirects + here after installation to link it to the workspace. Without it, installations can + still be linked manually from the Workspace assignments table that appears below + once the app details are saved.
  • - Setup URL (optional): - <your-windmill-url>/gh_success with "Redirect on update" checked + Callback URL: <your-windmill-url>/gh_success
  • Uncheck Active under Webhook (not needed)
    • diff --git a/frontend/src/lib/components/raw_apps/RawAppEditor.svelte b/frontend/src/lib/components/raw_apps/RawAppEditor.svelte index 75f703aa17..31e99dbe89 100644 --- a/frontend/src/lib/components/raw_apps/RawAppEditor.svelte +++ b/frontend/src/lib/components/raw_apps/RawAppEditor.svelte @@ -29,6 +29,7 @@ import { runDomQueryOnHtml, type RawAppDomQuery, type RawAppDomRequester } from './rawAppDom' import InlineElementPrompt from './InlineElementPrompt.svelte' import DarkModeObserver from '../DarkModeObserver.svelte' + import { getAppliedDarkModeVariant, type DarkModeVariant } from '$lib/darkModeVariant' import RawAppSidebar from './RawAppSidebar.svelte' import type { Modules } from './RawAppModules.svelte' import { isRunnableByName, isRunnableByPath } from '../apps/inputType' @@ -1304,7 +1305,7 @@ // it always matches the editor's current state. Plain rebuilds use // `syncExternalPreview` alone (the theme hasn't changed). function feedExternalPreview() { - postToExternalPreview({ type: 'setDarkMode', dark: darkMode }) + postToExternalPreview({ type: 'setDarkMode', dark: darkMode, variant: darkVariant }) syncExternalPreview() } @@ -1697,6 +1698,17 @@ }) let darkMode: boolean = $state(false) + // Mirrors the `github-dark` class (the runtime source of truth); the + // DarkModeObserver below must keep it in sync or it goes stale. + let darkVariant: DarkModeVariant = $state(getAppliedDarkModeVariant()) + // Read the DOM classes, not reactive state, so the src stays constant after + // mount: a reactive src would reload the iframe on every theme toggle. Live + // theme changes travel through postMessage instead (see the $effect below). + function uiBuilderIframeSrc(): string { + const dark = document.documentElement.classList.contains('dark') + const variant = getAppliedDarkModeVariant() + return `/ui_builder/index.html?dark=${dark}&variant=${variant}` + } // Host's computed `text-xs` size in px. Windmill bumps :root to 18px at // ≥1760px viewports, so this re-evaluates on resize via the listener below. let editorFontSize = $state(12) @@ -1746,12 +1758,18 @@ // Push dark mode to both children. The UI Builder iframe and the // preview iframe each listen for `setDarkMode` separately. if (iframe && iframeLoaded) { - iframe.contentWindow?.postMessage({ type: 'setDarkMode', dark: darkMode }, '*') + iframe.contentWindow?.postMessage( + { type: 'setDarkMode', dark: darkMode, variant: darkVariant }, + '*' + ) } if (previewIframe && previewIframeLoaded) { - previewIframe.contentWindow?.postMessage({ type: 'setDarkMode', dark: darkMode }, '*') + previewIframe.contentWindow?.postMessage( + { type: 'setDarkMode', dark: darkMode, variant: darkVariant }, + '*' + ) } - postToExternalPreview({ type: 'setDarkMode', dark: darkMode }) + postToExternalPreview({ type: 'setDarkMode', dark: darkMode, variant: darkVariant }) }) $effect(() => { // Match VS Code's editor font size to Windmill's text-xs. @@ -2036,7 +2054,12 @@ - + { + darkVariant = getAppliedDarkModeVariant() + }} +/> diff --git a/frontend/src/lib/components/raw_apps/templates.ts b/frontend/src/lib/components/raw_apps/templates.ts index 335ae4d8e8..22fa773f85 100644 --- a/frontend/src/lib/components/raw_apps/templates.ts +++ b/frontend/src/lib/components/raw_apps/templates.ts @@ -136,7 +136,7 @@ export const svelte5Template = { '/index.css': indexCss, '/package.json': `{ "dependencies": { - "svelte": "^5.55.5", + "svelte": "^5.56.8", "windmill-client": "^1" } }` diff --git a/frontend/src/lib/components/sessions/SessionModeSwitch.svelte b/frontend/src/lib/components/sessions/SessionModeSwitch.svelte index c4d29c9791..d721b6f16f 100644 --- a/frontend/src/lib/components/sessions/SessionModeSwitch.svelte +++ b/frontend/src/lib/components/sessions/SessionModeSwitch.svelte @@ -1,8 +1,11 @@ + {:else if status === 'error'} {:else if isFork} diff --git a/frontend/src/lib/components/sessions/sessionRuntime.svelte.ts b/frontend/src/lib/components/sessions/sessionRuntime.svelte.ts index f95e4afa9b..83e6aae8dd 100644 --- a/frontend/src/lib/components/sessions/sessionRuntime.svelte.ts +++ b/frontend/src/lib/components/sessions/sessionRuntime.svelte.ts @@ -86,7 +86,7 @@ import type { RawAppDomResult } from '$lib/components/raw_apps/rawAppDom' import { getNonStreamingMetadataCompletion } from '$lib/components/copilot/lib' -import type { DisplayMessage } from '$lib/components/copilot/chat/shared' +import { pendingUserAction, type DisplayMessage } from '$lib/components/copilot/chat/shared' import type { ChatCompletionMessageParam } from 'openai/resources/index.mjs' // Per-kind load state for a session's editor target. Pure state container the @@ -967,7 +967,10 @@ export function resetSessionPreviewTabs(sessionId: string, url: string): void { export type SessionChatStatus = | 'idle' | 'streaming' + // The assistant's turn ended and the user has yet to reply — passive, unlike + // 'awaiting-answer'/'needs-confirmation', where a running loop is blocked. | 'awaiting-user' + | 'awaiting-answer' | 'needs-confirmation' | 'draft' | 'error' @@ -1243,10 +1246,15 @@ setScreenshotHandler(async ({ sessionId: callerSessionId }) => { export function getSessionChatStatus(runtime: SessionRuntime): SessionChatStatus { const m = runtime.manager + const last = m.displayMessages[m.displayMessages.length - 1] + // A loop parked on the user still reports `loading`, so these must be tested + // before `streaming` — otherwise "answer me" renders as "the AI is typing" + // and a session that needs the user looks like one that doesn't. + const pending = pendingUserAction(m.displayMessages) + if (pending === 'question') return 'awaiting-answer' + if (pending === 'confirmation') return 'needs-confirmation' if (m.loading) return 'streaming' if (m.instructions.trim().length > 0) return 'draft' - const last = m.displayMessages[m.displayMessages.length - 1] - if (last?.role === 'tool' && last.needsConfirmation) return 'needs-confirmation' if (last?.role === 'user' && last.error) return 'error' if (last && (last.role === 'assistant' || last.role === 'tool')) return 'awaiting-user' return 'idle' diff --git a/frontend/src/lib/components/sidebar/WorkspaceMenu.svelte b/frontend/src/lib/components/sidebar/WorkspaceMenu.svelte index 2631af792f..2e57cf88fa 100644 --- a/frontend/src/lib/components/sidebar/WorkspaceMenu.svelte +++ b/frontend/src/lib/components/sidebar/WorkspaceMenu.svelte @@ -179,6 +179,15 @@ return ambiguous }) + // Opening while a fork is active expands that fork's family, so the tick sits + // on the active fork's own row instead of on its collapsed root. + function seedExpandedFamilies() { + expandedFamilies.clear() + if (currentFamily && currentFamily.id !== $workspaceStore) { + expandedFamilies.add(currentFamily.id) + } + } + // The active workspace itself (fork included) — names the settings entry. const activeWorkspace = $derived($userWorkspaces?.find((w) => w.id === $workspaceStore)) const canManageWorkspace = $derived( @@ -193,13 +202,14 @@ - + expandedFamilies.clear()} > {#snippet triggr({ trigger })} diff --git a/frontend/src/lib/components/sidebar/sidebarChrome.ts b/frontend/src/lib/components/sidebar/sidebarChrome.ts index 110435c058..71ff703d27 100644 --- a/frontend/src/lib/components/sidebar/sidebarChrome.ts +++ b/frontend/src/lib/components/sidebar/sidebarChrome.ts @@ -1,5 +1,6 @@ -// Sidebar rail background. Deliberately raw hex — no theme token exists for -// the rail chrome — kept here as the single source for every host that renders -// or mimics the sidebar (app layout, kitchen-sink harness). +// Sidebar rail background. Kept here as the single source for every host that +// renders or mimics the sidebar (app layout, kitchen-sink harness). +// The dark value is a theme-driven CSS variable so it adapts across dark +// variants (default vs GitHub dark) — see tailwind.config.cjs `--sidebar-bg-dark`. export const SIDEBAR_BG = '#F3F3F7' -export const SIDEBAR_BG_DARK = '#1e232e' +export const SIDEBAR_BG_DARK = 'var(--sidebar-bg-dark)' diff --git a/frontend/src/lib/components/vscode.ts b/frontend/src/lib/components/vscode.ts index 1277ac1b4a..f30ffcd780 100644 --- a/frontend/src/lib/components/vscode.ts +++ b/frontend/src/lib/components/vscode.ts @@ -1,6 +1,7 @@ import '@codingame/monaco-vscode-standalone-typescript-language-features' import { editor as meditor, Uri as mUri } from 'monaco-editor' +import { getAppliedDarkModeVariant } from '$lib/darkModeVariant' export let isInitialized = false export let isInitializing = false @@ -12,6 +13,15 @@ import { import getLanguagesServiceOverride from '@codingame/monaco-vscode-languages-service-override' import { getCssColor } from '$lib/utils' +// Monaco theme name matching the current document theme (light / dark / GitHub dark variant). +export function getEditorTheme(): string { + const classes = document.documentElement.classList + if (!classes.contains('dark')) { + return 'myTheme' + } + return getAppliedDarkModeVariant() === 'github' ? 'github-dark' : 'nord' +} + export function buildWorkerDefinition() { const envEnhanced = getEnhancedMonacoEnvironment() const getWorker = (moduleId: string, label: string) => { @@ -248,6 +258,47 @@ export async function initializeVscode(caller?: string, htmlContainer?: HTMLElem } }) + meditor.defineTheme('github-dark', { + base: 'vs-dark', + inherit: true, + rules: [ + { background: '0D1117', token: '' }, + { foreground: '8b949e', token: 'comment' }, + { foreground: 'a5d6ff', token: 'string' }, + { foreground: '79c0ff', token: 'constant.numeric' }, + { foreground: '79c0ff', token: 'constant.language' }, + { foreground: 'ff7b72', token: 'keyword' }, + { foreground: 'ff7b72', token: 'storage' }, + { foreground: 'ff7b72', token: 'storage.type' }, + { foreground: 'ffa657', token: 'entity.name.class' }, + { foreground: 'ffa657', fontStyle: 'bold', token: 'entity.other.inherited-class' }, + { foreground: 'd2a8ff', token: 'entity.name.function' }, + { foreground: '7ee787', token: 'entity.name.tag' }, + { foreground: '79c0ff', token: 'entity.other.attribute-name' }, + { foreground: 'd2a8ff', token: 'support.function' }, + { foreground: 'f0f6fc', background: 'f85149', token: 'invalid' }, + { foreground: 'f0f6fc', background: 'bd561d', token: 'invalid.deprecated' }, + { foreground: '79c0ff', token: 'constant.color.other.rgb-value' }, + { foreground: '79c0ff', token: 'constant.character.escape' }, + { foreground: 'ffa657', token: 'variable.other.constant' }, + { token: 'string.value.json', foreground: 'a5d6ff' }, // string values in JSON + { token: 'keyword.json', foreground: '79c0ff' } // true, false, null in JSON + ], + colors: { + 'editor.foreground': '#e6edf3', + 'editor.background': getCssColor('surface-input', { format: 'hex-github-dark' }), + 'editor.selectionBackground': '#3392FF44', + 'editor.inactiveSelectionBackground': '#3392FF22', + 'editor.lineHighlightBackground': '#161b22', + 'editorCursor.foreground': '#e6edf3', + 'editorWhitespace.foreground': '#484f58', + 'editorIndentGuide.background1': '#21262d', + 'editorIndentGuide.activeBackground1': '#30363d', + 'editorLineNumber.foreground': '#6e7681', + 'editorLineNumber.activeForeground': '#e6edf3' + } + }) + meditor.defineTheme('myTheme', { base: 'vs', inherit: true, @@ -262,11 +313,7 @@ export async function initializeVscode(caller?: string, htmlContainer?: HTMLElem } }) - if (document.documentElement.classList.contains('dark')) { - meditor.setTheme('nord') - } else { - meditor.setTheme('myTheme') - } + meditor.setTheme(getEditorTheme()) } catch (e) { console.error('Failed to initialize monaco services', e) } finally { diff --git a/frontend/src/lib/components/workspaceSettings/DataTableSettings.svelte b/frontend/src/lib/components/workspaceSettings/DataTableSettings.svelte index f652ed8704..069ef6093d 100644 --- a/frontend/src/lib/components/workspaceSettings/DataTableSettings.svelte +++ b/frontend/src/lib/components/workspaceSettings/DataTableSettings.svelte @@ -51,7 +51,7 @@ {@render children?.()} diff --git a/frontend/src/routes/user/login_callback/[client_name]/+page.svelte b/frontend/src/routes/user/login_callback/[client_name]/+page.svelte index 737da14321..1aeb57f223 100644 --- a/frontend/src/routes/user/login_callback/[client_name]/+page.svelte +++ b/frontend/src/routes/user/login_callback/[client_name]/+page.svelte @@ -1,5 +1,6 @@