diff --git a/.release-please-manifest.json b/.release-please-manifest.json index 6046035ab7..78bc4dfc98 100644 --- a/.release-please-manifest.json +++ b/.release-please-manifest.json @@ -1,3 +1,3 @@ { - ".": "1.819.0" + ".": "1.820.0" } diff --git a/CHANGELOG.md b/CHANGELOG.md index 93b0e0dd9c..d1a8762b7f 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,36 @@ # Changelog +## [1.820.0](https://github.com/windmill-labs/windmill/compare/v1.819.0...v1.820.0) (2026-09-29) + + +### Features + +* add pull_batch to claim jobs for many waiting workers at once ([#11350](https://github.com/windmill-labs/windmill/issues/11350)) ([9f40cdc](https://github.com/windmill-labs/windmill/commit/9f40cdca6258a9218770ebe1e7b89179e7d6667f)) +* add tree view to the schedules page ([#11360](https://github.com/windmill-labs/windmill/issues/11360)) ([e68ff0d](https://github.com/windmill-labs/windmill/commit/e68ff0d969edaacafc0fc5e246ac3945ae28109b)) +* **ai-agent:** add compaction memory that summarizes older context ([#10928](https://github.com/windmill-labs/windmill/issues/10928)) ([c1b59f7](https://github.com/windmill-labs/windmill/commit/c1b59f70dd0e1f4a7597d31cff031ce6c0e7d0ab)) +* detect and alert when a schedule skips occurrences ([#10917](https://github.com/windmill-labs/windmill/issues/10917)) ([97fa55b](https://github.com/windmill-labs/windmill/commit/97fa55b719a874067d1a81a1d680a42d1f1a4c98)) +* external instance cluster for data tables and Ducklake catalogs ([#11197](https://github.com/windmill-labs/windmill/issues/11197)) ([8741843](https://github.com/windmill-labs/windmill/commit/8741843d2e884783eed1c00bbc2293e526f3e80d)) +* refuse OIDC tokens to previews that impersonate a path their user cannot write ([#11396](https://github.com/windmill-labs/windmill/issues/11396)) ([d1260d7](https://github.com/windmill-labs/windmill/commit/d1260d7c5a3d132d4427079b794452309e03e28e)) +* restart perpetual runs on the version a deploy makes runnable ([#11200](https://github.com/windmill-labs/windmill/issues/11200)) ([cf5c49c](https://github.com/windmill-labs/windmill/commit/cf5c49c3dca2201f739fb872d68bcd6c0c7665f7)) +* run turns in several flow chat conversations at once ([#11202](https://github.com/windmill-labs/windmill/issues/11202)) ([f367eaf](https://github.com/windmill-labs/windmill/commit/f367eaf6d0258649ca0b3d47b1270af9a3fcd484)) +* share AI guidance with the CLI skills and lint flow groups ([#11397](https://github.com/windmill-labs/windmill/issues/11397)) ([ede4103](https://github.com/windmill-labs/windmill/commit/ede4103b00c6ada3b5ea8ba3e98bf6b78a791d63)) +* show native web search in the assistant settings modal ([#11394](https://github.com/windmill-labs/windmill/issues/11394)) ([235410c](https://github.com/windmill-labs/windmill/commit/235410c1e417bb64b432c96efed870cab74cb364)) +* stop cross-origin isolating the raw app editor ([#11411](https://github.com/windmill-labs/windmill/issues/11411)) ([6064aec](https://github.com/windmill-labs/windmill/commit/6064aecdecc49facde3870446b570abd8e8124b2)) + + +### Bug Fixes + +* bound a resumed session fork's wait, keep its intent while in flight ([#11413](https://github.com/windmill-labs/windmill/issues/11413)) ([e6df8d7](https://github.com/windmill-labs/windmill/commit/e6df8d78d496cf9640bd5a1e9d7e8f4c2cc2adf7)) +* create workspace forks in the background, with progress ([#11406](https://github.com/windmill-labs/windmill/issues/11406)) ([054109d](https://github.com/windmill-labs/windmill/commit/054109d8559cc0a41ef361786c4d1d99eab9c502)) +* drop caller-supplied _MODULES at push for every job ([#11418](https://github.com/windmill-labs/windmill/issues/11418)) ([c5f59d6](https://github.com/windmill-labs/windmill/commit/c5f59d6ccc71d08fc34ede47e7a77b35a197b9db)) +* keep new-menu submenus open while crossing the gutter ([#11414](https://github.com/windmill-labs/windmill/issues/11414)) ([2f87bc6](https://github.com/windmill-labs/windmill/commit/2f87bc6a2d237382641ad3f0e7a8df36f0b74c15)) +* keep secret variable values hidden when toggling secret ([#11383](https://github.com/windmill-labs/windmill/issues/11383)) ([da5c58d](https://github.com/windmill-labs/windmill/commit/da5c58de2a0c297cfa81a601f9f543f46107e3f6)) +* refuse flow preview restarts from runs the caller cannot read ([#11407](https://github.com/windmill-labs/windmill/issues/11407)) ([c34abf7](https://github.com/windmill-labs/windmill/commit/c34abf7330e7f3fbbf5b68e325f0f0991c6c60d5)) +* report a worker's last job when it ran under one poll interval ([#11399](https://github.com/windmill-labs/windmill/issues/11399)) ([797147e](https://github.com/windmill-labs/windmill/commit/797147ea7a4a911d3c82e9ac9cdaf066ad2795d6)) +* run custom schedule handlers as the schedule ([#11398](https://github.com/windmill-labs/windmill/issues/11398)) ([bda00c6](https://github.com/windmill-labs/windmill/commit/bda00c6cad2de2ce0927651fe0e53653bdd20a96)) +* surface raw app build errors to the session AI ([#11415](https://github.com/windmill-labs/windmill/issues/11415)) ([50ffad5](https://github.com/windmill-labs/windmill/commit/50ffad52d765497fa3a561ebe8842b40fc8bd58e)) +* turn the default raw app into a feature tour ([#11417](https://github.com/windmill-labs/windmill/issues/11417)) ([83b8954](https://github.com/windmill-labs/windmill/commit/83b8954ae26f8d453d38e867e62366341cfe7729)) + ## [1.819.0](https://github.com/windmill-labs/windmill/compare/v1.818.0...v1.819.0) (2026-09-28) diff --git a/ai_evals/adapters/frontend/vitestAdapter.test.ts b/ai_evals/adapters/frontend/vitestAdapter.test.ts index 0c8a22a987..9adb4d05ac 100644 --- a/ai_evals/adapters/frontend/vitestAdapter.test.ts +++ b/ai_evals/adapters/frontend/vitestAdapter.test.ts @@ -1,10 +1,14 @@ import { expect, it, vi } from 'vitest' // @ts-ignore - Node.js fs/promises -import { mkdir, writeFile } from 'fs/promises' +import { mkdir, readFile, writeFile } from 'fs/promises' // @ts-ignore - Node.js path import { dirname, resolve } from 'path' +// @ts-ignore - Node.js url +import { fileURLToPath } from 'url' import { handleBenchmarkApiFetch, hasBenchmarkApiHandler } from './mockBackend' +const FRONTEND_DIR = fileURLToPath(new URL('../../../frontend/', import.meta.url)) + // Some tools reach the backend by relative fetch('/api/...'), which has no meaning // in the vitest environment — serve the ones the benchmark handles. // Every other relative fetch keeps its normal behavior (it fails the same way @@ -24,9 +28,30 @@ globalThis.fetch = (async (input: unknown, init?: RequestInit) => { if (typeof url === 'string' && hasBenchmarkApiHandler(url)) { return handleBenchmarkApiFetch(url, init) } + // The parsers behind inferArgs load their wasm from a vite `?url` path, which only a dev + // server serves. Unserved, every script schema infers as empty, and the tools then tell the + // model its arguments are undeclared, so it rewrites a correct script until it runs out of turns. + const wasmPath = typeof url === 'string' ? wasmFilePath(url) : undefined + if (wasmPath) { + return new Response(await readFile(wasmPath), { + headers: { 'Content-Type': 'application/wasm' } + }) + } return ORIGINAL_FETCH(input as Parameters[0], init) }) as typeof fetch +function wasmFilePath(url: string): string | undefined { + const raw = url.split(/[?#]/)[0] + if (!raw.endsWith('.wasm')) return undefined + const pathname = decodeURIComponent(raw) + // Located past whatever base path the config prefixes. + const fs = pathname.indexOf('/@fs/') + if (fs !== -1) return pathname.slice(fs + '/@fs'.length) + const modules = pathname.indexOf('/node_modules/') + if (modules !== -1) return resolve(FRONTEND_DIR, `.${pathname.slice(modules)}`) + return undefined +} + vi.mock('monaco-editor', () => ({ editor: {}, languages: {}, @@ -640,6 +665,13 @@ benchmarkIt( async () => { const { resetBenchmarkMockBackend } = await import('./mockBackend') resetBenchmarkMockBackend() + // The tools swallow inference failures, so a wasm the fetch stub stops serving would + // only show up as empty schemas the model chases, never as an error. + const { inferArgs } = await import('$lib/infer') + const { emptySchema } = await import('$lib/utils') + const probe = emptySchema() + await inferArgs('bun', 'export async function main(name: string) {}', probe) + expect(Object.keys(probe.properties)).toEqual(['name']) const { runFrontendBenchmarkFromEnv } = await import('./benchmarkRunner') try { const payload = await runFrontendBenchmarkFromEnv() diff --git a/backend/.sqlx/query-0c2bf6925de6dd4d9f8d47d7fdd9e7801129570106b5acd49c0f708398606771.json b/backend/.sqlx/query-0c2bf6925de6dd4d9f8d47d7fdd9e7801129570106b5acd49c0f708398606771.json new file mode 100644 index 0000000000..a7b61d7282 --- /dev/null +++ b/backend/.sqlx/query-0c2bf6925de6dd4d9f8d47d7fdd9e7801129570106b5acd49c0f708398606771.json @@ -0,0 +1,23 @@ +{ + "db_name": "PostgreSQL", + "query": "WITH RECURSIVE descendants AS (\n SELECT id FROM v2_job WHERE parent_job = $1 AND workspace_id = $2\n UNION ALL\n SELECT j.id FROM v2_job j JOIN descendants d ON j.parent_job = d.id\n WHERE j.workspace_id = $2\n ) SELECT d.id AS \"id!\" FROM descendants d JOIN v2_job_queue q ON q.id = d.id", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "id!", + "type_info": "Uuid" + } + ], + "parameters": { + "Left": [ + "Uuid", + "Text" + ] + }, + "nullable": [ + null + ] + }, + "hash": "0c2bf6925de6dd4d9f8d47d7fdd9e7801129570106b5acd49c0f708398606771" +} diff --git a/backend/.sqlx/query-201c4d014dd71da5a38ba832f836ca71b0fe6dbee8645449e97517aeb4ef9655.json b/backend/.sqlx/query-201c4d014dd71da5a38ba832f836ca71b0fe6dbee8645449e97517aeb4ef9655.json deleted file mode 100644 index 2a8d0ab26f..0000000000 --- a/backend/.sqlx/query-201c4d014dd71da5a38ba832f836ca71b0fe6dbee8645449e97517aeb4ef9655.json +++ /dev/null @@ -1,83 +0,0 @@ -{ - "db_name": "PostgreSQL", - "query": "WITH RECURSIVE lineage AS (\n SELECT id, parent_job, kind, runnable_path, runnable_id, trigger_kind, trigger,\n 0 AS depth\n FROM v2_job WHERE id = $1 AND workspace_id = $2\n UNION ALL\n SELECT p.id, p.parent_job, p.kind, p.runnable_path, p.runnable_id, p.trigger_kind,\n p.trigger, l.depth + 1\n FROM v2_job p JOIN lineage l ON p.id = l.parent_job\n WHERE p.workspace_id = $2 AND l.depth < 100\n )\n SELECT id AS \"id!\", parent_job, kind AS \"kind!: JobKind\", runnable_path,\n trigger_kind::text AS trigger_kind,\n -- A restart takes its flow version from the request, so a trusted flow path\n -- can carry another flow's code: the version must belong to that path.\n CASE kind\n WHEN 'flow' THEN EXISTS (SELECT 1 FROM flow_version fv\n WHERE fv.id = runnable_id AND fv.path = runnable_path AND fv.workspace_id = $2)\n WHEN 'script' THEN EXISTS (SELECT 1 FROM script s\n WHERE s.hash = runnable_id AND s.path = runnable_path AND s.workspace_id = $2)\n -- An app editor preview also runs an app script, at an app path it does not\n -- have to own; only deployed-app runs are stamped with their app.\n WHEN 'appscript' THEN COALESCE(trigger_kind = 'app'\n AND starts_with(runnable_path, trigger || '/'), false)\n ELSE true\n END AS \"origin_verified!\"\n FROM lineage ORDER BY depth", - "describe": { - "columns": [ - { - "ordinal": 0, - "name": "id!", - "type_info": "Uuid" - }, - { - "ordinal": 1, - "name": "parent_job", - "type_info": "Uuid" - }, - { - "ordinal": 2, - "name": "kind!: JobKind", - "type_info": { - "Custom": { - "name": "job_kind", - "kind": { - "Enum": [ - "script", - "preview", - "flow", - "dependencies", - "flowpreview", - "script_hub", - "identity", - "flowdependencies", - "http", - "graphql", - "postgresql", - "noop", - "appdependencies", - "deploymentcallback", - "singlestepflow", - "flowscript", - "flownode", - "appscript", - "aiagent", - "unassigned_script", - "unassigned_flow", - "unassigned_singlestepflow" - ] - } - } - } - }, - { - "ordinal": 3, - "name": "runnable_path", - "type_info": "Varchar" - }, - { - "ordinal": 4, - "name": "trigger_kind", - "type_info": "Text" - }, - { - "ordinal": 5, - "name": "origin_verified!", - "type_info": "Bool" - } - ], - "parameters": { - "Left": [ - "Uuid", - "Text" - ] - }, - "nullable": [ - null, - null, - null, - null, - null, - null - ] - }, - "hash": "201c4d014dd71da5a38ba832f836ca71b0fe6dbee8645449e97517aeb4ef9655" -} diff --git a/backend/.sqlx/query-451f303f4a24d848ba4ccdc2441ca7cb85bdfc66110b6534260f8642f91f006a.json b/backend/.sqlx/query-451f303f4a24d848ba4ccdc2441ca7cb85bdfc66110b6534260f8642f91f006a.json new file mode 100644 index 0000000000..8209f67c75 --- /dev/null +++ b/backend/.sqlx/query-451f303f4a24d848ba4ccdc2441ca7cb85bdfc66110b6534260f8642f91f006a.json @@ -0,0 +1,22 @@ +{ + "db_name": "PostgreSQL", + "query": "UPDATE v2_job_queue SET started_at = now() WHERE id = $1 RETURNING started_at", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "started_at", + "type_info": "Timestamptz" + } + ], + "parameters": { + "Left": [ + "Uuid" + ] + }, + "nullable": [ + true + ] + }, + "hash": "451f303f4a24d848ba4ccdc2441ca7cb85bdfc66110b6534260f8642f91f006a" +} diff --git a/backend/.sqlx/query-4a257470121ee6bffd9c68e88aabb7563c0ba31233854c8922f9449da7b43e9e.json b/backend/.sqlx/query-4a257470121ee6bffd9c68e88aabb7563c0ba31233854c8922f9449da7b43e9e.json new file mode 100644 index 0000000000..465eac3a37 --- /dev/null +++ b/backend/.sqlx/query-4a257470121ee6bffd9c68e88aabb7563c0ba31233854c8922f9449da7b43e9e.json @@ -0,0 +1,77 @@ +{ + "db_name": "PostgreSQL", + "query": "SELECT\n j.runnable_path as script_path, j.args AS \"args: sqlx::types::Json>>\",\n j.tag AS \"tag!\", j.priority, j.kind AS \"kind!: JobKind\"\n FROM v2_job j\n WHERE j.id = $1 and j.workspace_id = $2", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "script_path", + "type_info": "Varchar" + }, + { + "ordinal": 1, + "name": "args: sqlx::types::Json>>", + "type_info": "Jsonb" + }, + { + "ordinal": 2, + "name": "tag!", + "type_info": "Varchar" + }, + { + "ordinal": 3, + "name": "priority", + "type_info": "Int2" + }, + { + "ordinal": 4, + "name": "kind!: JobKind", + "type_info": { + "Custom": { + "name": "job_kind", + "kind": { + "Enum": [ + "script", + "preview", + "flow", + "dependencies", + "flowpreview", + "script_hub", + "identity", + "flowdependencies", + "http", + "graphql", + "postgresql", + "noop", + "appdependencies", + "deploymentcallback", + "singlestepflow", + "flowscript", + "flownode", + "appscript", + "aiagent", + "unassigned_script", + "unassigned_flow", + "unassigned_singlestepflow" + ] + } + } + } + } + ], + "parameters": { + "Left": [ + "Uuid", + "Text" + ] + }, + "nullable": [ + true, + true, + false, + true, + false + ] + }, + "hash": "4a257470121ee6bffd9c68e88aabb7563c0ba31233854c8922f9449da7b43e9e" +} diff --git a/backend/.sqlx/query-dc37f9ad685a2f2bd781be8678e0548a860815f30e866362344434f23aa7c83c.json b/backend/.sqlx/query-4afdc63af51e599b9d6fe6cedcd7980fd761a29df0f2a6820f3c8d7a29c3c20a.json similarity index 50% rename from backend/.sqlx/query-dc37f9ad685a2f2bd781be8678e0548a860815f30e866362344434f23aa7c83c.json rename to backend/.sqlx/query-4afdc63af51e599b9d6fe6cedcd7980fd761a29df0f2a6820f3c8d7a29c3c20a.json index 02d889c38b..02a2cb9973 100644 --- a/backend/.sqlx/query-dc37f9ad685a2f2bd781be8678e0548a860815f30e866362344434f23aa7c83c.json +++ b/backend/.sqlx/query-4afdc63af51e599b9d6fe6cedcd7980fd761a29df0f2a6820f3c8d7a29c3c20a.json @@ -1,6 +1,6 @@ { "db_name": "PostgreSQL", - "query": "WITH active_users as (SELECT distinct username as email FROM audit_partitioned WHERE timestamp > NOW() - INTERVAL '1 month' AND (operation = 'users.login' OR operation = 'oauth.login' OR operation = 'users.token.refresh')),\n active_authors as (SELECT distinct email FROM usr WHERE usr.operator IS false AND email IN (SELECT email FROM active_users)),\n active_authors_agg as (SELECT array_agg(email) as authors FROM active_authors),\n active_ops_agg as (SELECT array_agg(email) as operators from active_users WHERE email NOT IN (SELECT email FROM active_authors))\n SELECT active_authors_agg.authors, active_ops_agg.operators, array_length(active_authors_agg.authors, 1) as author_count, array_length(active_ops_agg.operators, 1) as operator_count FROM active_authors_agg, active_ops_agg", + "query": "WITH active_users as (SELECT distinct username as email FROM audit_partitioned WHERE timestamp > NOW() - INTERVAL '1 month' AND (operation = 'users.login' OR operation = 'oauth.login' OR operation = 'users.token.refresh') AND username NOT IN (SELECT email FROM usr WHERE is_service_account)),\n active_authors as (SELECT distinct email FROM usr WHERE usr.operator IS false AND email IN (SELECT email FROM active_users)),\n active_authors_agg as (SELECT array_agg(email) as authors FROM active_authors),\n active_ops_agg as (SELECT array_agg(email) as operators from active_users WHERE email NOT IN (SELECT email FROM active_authors))\n SELECT active_authors_agg.authors, active_ops_agg.operators, array_length(active_authors_agg.authors, 1) as author_count, array_length(active_ops_agg.operators, 1) as operator_count FROM active_authors_agg, active_ops_agg", "describe": { "columns": [ { @@ -34,5 +34,5 @@ null ] }, - "hash": "dc37f9ad685a2f2bd781be8678e0548a860815f30e866362344434f23aa7c83c" + "hash": "4afdc63af51e599b9d6fe6cedcd7980fd761a29df0f2a6820f3c8d7a29c3c20a" } diff --git a/backend/.sqlx/query-5176c354a8c14b8acfbcc7e32da2fcd5013543d600a07a36f3451ddb00cad6aa.json b/backend/.sqlx/query-5176c354a8c14b8acfbcc7e32da2fcd5013543d600a07a36f3451ddb00cad6aa.json new file mode 100644 index 0000000000..dda9bc9869 --- /dev/null +++ b/backend/.sqlx/query-5176c354a8c14b8acfbcc7e32da2fcd5013543d600a07a36f3451ddb00cad6aa.json @@ -0,0 +1,101 @@ +{ + "db_name": "PostgreSQL", + "query": "WITH RECURSIVE lineage AS (\n SELECT id, parent_job, kind, runnable_path, runnable_id, trigger_kind, trigger,\n permissioned_as, args, 0 AS depth\n FROM v2_job WHERE id = $1 AND workspace_id = $2\n UNION ALL\n SELECT p.id, p.parent_job, p.kind, p.runnable_path, p.runnable_id, p.trigger_kind,\n p.trigger, p.permissioned_as, p.args, l.depth + 1\n FROM v2_job p JOIN lineage l ON p.id = l.parent_job\n WHERE p.workspace_id = $2 AND l.depth < 100\n )\n SELECT id AS \"id!\", parent_job, kind AS \"kind!: JobKind\", runnable_path,\n trigger_kind::text AS trigger_kind, permissioned_as AS \"permissioned_as!\",\n -- A restart takes its flow version from the request, so a trusted flow path\n -- can carry another flow's code: the version must belong to that path.\n CASE kind\n WHEN 'flow' THEN EXISTS (SELECT 1 FROM flow_version fv\n WHERE fv.id = runnable_id AND fv.path = runnable_path AND fv.workspace_id = $2)\n WHEN 'script' THEN EXISTS (SELECT 1 FROM script s\n WHERE s.hash = runnable_id AND s.path = runnable_path AND s.workspace_id = $2)\n ELSE true\n END AS \"origin_verified!\",\n -- Only deployed-app runs are stamped with their app; an app editor preview\n -- runs app code at an app path it does not have to own.\n COALESCE(trigger_kind = 'app' AND starts_with(runnable_path, trigger || '/'), false)\n AS \"app_stamped!\",\n -- A preview's modules come from its args, which its parent may have taken from\n -- the caller.\n COALESCE(jsonb_typeof(args->'_MODULES') = 'object', false) AS \"args_modules!\"\n FROM lineage ORDER BY depth", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "id!", + "type_info": "Uuid" + }, + { + "ordinal": 1, + "name": "parent_job", + "type_info": "Uuid" + }, + { + "ordinal": 2, + "name": "kind!: JobKind", + "type_info": { + "Custom": { + "name": "job_kind", + "kind": { + "Enum": [ + "script", + "preview", + "flow", + "dependencies", + "flowpreview", + "script_hub", + "identity", + "flowdependencies", + "http", + "graphql", + "postgresql", + "noop", + "appdependencies", + "deploymentcallback", + "singlestepflow", + "flowscript", + "flownode", + "appscript", + "aiagent", + "unassigned_script", + "unassigned_flow", + "unassigned_singlestepflow" + ] + } + } + } + }, + { + "ordinal": 3, + "name": "runnable_path", + "type_info": "Varchar" + }, + { + "ordinal": 4, + "name": "trigger_kind", + "type_info": "Text" + }, + { + "ordinal": 5, + "name": "permissioned_as!", + "type_info": "Varchar" + }, + { + "ordinal": 6, + "name": "origin_verified!", + "type_info": "Bool" + }, + { + "ordinal": 7, + "name": "app_stamped!", + "type_info": "Bool" + }, + { + "ordinal": 8, + "name": "args_modules!", + "type_info": "Bool" + } + ], + "parameters": { + "Left": [ + "Uuid", + "Text" + ] + }, + "nullable": [ + null, + null, + null, + null, + null, + null, + null, + null, + null + ] + }, + "hash": "5176c354a8c14b8acfbcc7e32da2fcd5013543d600a07a36f3451ddb00cad6aa" +} diff --git a/backend/.sqlx/query-52f9582423689f098a337f88341239a136ee5132bc97f4b06cc1ee5beaa621e5.json b/backend/.sqlx/query-52f9582423689f098a337f88341239a136ee5132bc97f4b06cc1ee5beaa621e5.json deleted file mode 100644 index cc6a57c005..0000000000 --- a/backend/.sqlx/query-52f9582423689f098a337f88341239a136ee5132bc97f4b06cc1ee5beaa621e5.json +++ /dev/null @@ -1,41 +0,0 @@ -{ - "db_name": "PostgreSQL", - "query": "SELECT c.id IS NOT NULL AS \"completed!\",\n c.result AS \"result: sqlx::types::Json>\",\n c.status = 'success' AS \"success\",\n EXISTS(SELECT 1 FROM v2_job_queue WHERE id = $1) AS \"queued!\"\n FROM (SELECT 1) one\n LEFT JOIN v2_job_completed c ON c.id = $1 AND c.workspace_id = $2", - "describe": { - "columns": [ - { - "ordinal": 0, - "name": "completed!", - "type_info": "Bool" - }, - { - "ordinal": 1, - "name": "result: sqlx::types::Json>", - "type_info": "Jsonb" - }, - { - "ordinal": 2, - "name": "success", - "type_info": "Bool" - }, - { - "ordinal": 3, - "name": "queued!", - "type_info": "Bool" - } - ], - "parameters": { - "Left": [ - "Uuid", - "Text" - ] - }, - "nullable": [ - null, - true, - null, - null - ] - }, - "hash": "52f9582423689f098a337f88341239a136ee5132bc97f4b06cc1ee5beaa621e5" -} diff --git a/backend/.sqlx/query-789c3f6d29f46fcb17a22fe97405a1754ccb8ddeb08c23795dfe14e9f332e9dc.json b/backend/.sqlx/query-789c3f6d29f46fcb17a22fe97405a1754ccb8ddeb08c23795dfe14e9f332e9dc.json new file mode 100644 index 0000000000..9702cbf61d --- /dev/null +++ b/backend/.sqlx/query-789c3f6d29f46fcb17a22fe97405a1754ccb8ddeb08c23795dfe14e9f332e9dc.json @@ -0,0 +1,35 @@ +{ + "db_name": "PostgreSQL", + "query": "SELECT id, status = 'success' AS \"success!\", result AS \"result: Json>\"\n FROM v2_job_completed WHERE workspace_id = $1 AND id = ANY($2)", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "id", + "type_info": "Uuid" + }, + { + "ordinal": 1, + "name": "success!", + "type_info": "Bool" + }, + { + "ordinal": 2, + "name": "result: Json>", + "type_info": "Jsonb" + } + ], + "parameters": { + "Left": [ + "Text", + "UuidArray" + ] + }, + "nullable": [ + false, + null, + true + ] + }, + "hash": "789c3f6d29f46fcb17a22fe97405a1754ccb8ddeb08c23795dfe14e9f332e9dc" +} diff --git a/backend/.sqlx/query-be004608fc1de826803f266dfce96e09e634977478443b6372bca5e3dd0ce5ff.json b/backend/.sqlx/query-be004608fc1de826803f266dfce96e09e634977478443b6372bca5e3dd0ce5ff.json new file mode 100644 index 0000000000..dbd7e7c9ec --- /dev/null +++ b/backend/.sqlx/query-be004608fc1de826803f266dfce96e09e634977478443b6372bca5e3dd0ce5ff.json @@ -0,0 +1,22 @@ +{ + "db_name": "PostgreSQL", + "query": "SELECT NOT starts_with(COALESCE(label, ''), 'impersonation:') FROM token\n WHERE token_hash = $1 AND (expiration IS NULL OR expiration > now())", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "?column?", + "type_info": "Bool" + } + ], + "parameters": { + "Left": [ + "Text" + ] + }, + "nullable": [ + null + ] + }, + "hash": "be004608fc1de826803f266dfce96e09e634977478443b6372bca5e3dd0ce5ff" +} diff --git a/backend/.sqlx/query-f5c87b6fc1decd6dc53590fe4d16f59430530b8b7428e42d1777927e63395ff8.json b/backend/.sqlx/query-f5c87b6fc1decd6dc53590fe4d16f59430530b8b7428e42d1777927e63395ff8.json new file mode 100644 index 0000000000..197ecaf3d9 --- /dev/null +++ b/backend/.sqlx/query-f5c87b6fc1decd6dc53590fe4d16f59430530b8b7428e42d1777927e63395ff8.json @@ -0,0 +1,16 @@ +{ + "db_name": "PostgreSQL", + "query": "UPDATE v2_job_queue SET worker = $1, running = true, started_at = now()\n WHERE id = $2 AND tag = ANY($3)", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "Varchar", + "Uuid", + "TextArray" + ] + }, + "nullable": [] + }, + "hash": "f5c87b6fc1decd6dc53590fe4d16f59430530b8b7428e42d1777927e63395ff8" +} diff --git a/backend/.sqlx/query-f7753f8f7b40aebf66de607203f97681d06ac04f9d547b4c49c032d9e933a29e.json b/backend/.sqlx/query-f7753f8f7b40aebf66de607203f97681d06ac04f9d547b4c49c032d9e933a29e.json deleted file mode 100644 index 09ac527a72..0000000000 --- a/backend/.sqlx/query-f7753f8f7b40aebf66de607203f97681d06ac04f9d547b4c49c032d9e933a29e.json +++ /dev/null @@ -1,41 +0,0 @@ -{ - "db_name": "PostgreSQL", - "query": "SELECT\n j.runnable_path as script_path, j.args AS \"args: sqlx::types::Json>>\",\n j.tag AS \"tag!\", j.priority\n FROM v2_job j\n WHERE j.id = $1 and j.workspace_id = $2", - "describe": { - "columns": [ - { - "ordinal": 0, - "name": "script_path", - "type_info": "Varchar" - }, - { - "ordinal": 1, - "name": "args: sqlx::types::Json>>", - "type_info": "Jsonb" - }, - { - "ordinal": 2, - "name": "tag!", - "type_info": "Varchar" - }, - { - "ordinal": 3, - "name": "priority", - "type_info": "Int2" - } - ], - "parameters": { - "Left": [ - "Uuid", - "Text" - ] - }, - "nullable": [ - true, - true, - false, - true - ] - }, - "hash": "f7753f8f7b40aebf66de607203f97681d06ac04f9d547b4c49c032d9e933a29e" -} diff --git a/backend/Cargo.lock b/backend/Cargo.lock index 4d73a953b8..114fb76b67 100644 --- a/backend/Cargo.lock +++ b/backend/Cargo.lock @@ -14792,7 +14792,7 @@ dependencies = [ [[package]] name = "windmill" -version = "1.819.0" +version = "1.820.0" dependencies = [ "anyhow", "async-nats", @@ -14876,11 +14876,12 @@ dependencies = [ "windmill-worker-volumes", "windows-service", "windows-sys 0.52.0", + "winresource", ] [[package]] name = "windmill-ai" -version = "1.819.0" +version = "1.820.0" dependencies = [ "async-stream", "async-trait", @@ -14914,7 +14915,7 @@ dependencies = [ [[package]] name = "windmill-alerting" -version = "1.819.0" +version = "1.820.0" dependencies = [ "axum 0.8.9", "chrono", @@ -14927,7 +14928,7 @@ dependencies = [ [[package]] name = "windmill-api" -version = "1.819.0" +version = "1.820.0" dependencies = [ "anyhow", "argon2", @@ -15067,7 +15068,7 @@ dependencies = [ [[package]] name = "windmill-api-agent-workers" -version = "1.819.0" +version = "1.820.0" dependencies = [ "axum 0.8.9", "chrono", @@ -15090,7 +15091,7 @@ dependencies = [ [[package]] name = "windmill-api-assets" -version = "1.819.0" +version = "1.820.0" dependencies = [ "axum 0.8.9", "chrono", @@ -15107,7 +15108,7 @@ dependencies = [ [[package]] name = "windmill-api-auth" -version = "1.819.0" +version = "1.820.0" dependencies = [ "anyhow", "axum 0.8.9", @@ -15133,7 +15134,7 @@ dependencies = [ [[package]] name = "windmill-api-client" -version = "1.819.0" +version = "1.820.0" dependencies = [ "reqwest 0.12.28", "serde", @@ -15143,7 +15144,7 @@ dependencies = [ [[package]] name = "windmill-api-configs" -version = "1.819.0" +version = "1.820.0" dependencies = [ "axum 0.8.9", "chrono", @@ -15160,7 +15161,7 @@ dependencies = [ [[package]] name = "windmill-api-debug" -version = "1.819.0" +version = "1.820.0" dependencies = [ "axum 0.8.9", "base64 0.22.1", @@ -15182,7 +15183,7 @@ dependencies = [ [[package]] name = "windmill-api-embeddings" -version = "1.819.0" +version = "1.820.0" dependencies = [ "anyhow", "axum 0.8.9", @@ -15205,7 +15206,7 @@ dependencies = [ [[package]] name = "windmill-api-flow-conversations" -version = "1.819.0" +version = "1.820.0" dependencies = [ "axum 0.8.9", "chrono", @@ -15221,7 +15222,7 @@ dependencies = [ [[package]] name = "windmill-api-flows" -version = "1.819.0" +version = "1.820.0" dependencies = [ "axum 0.8.9", "chrono", @@ -15243,7 +15244,7 @@ dependencies = [ [[package]] name = "windmill-api-groups" -version = "1.819.0" +version = "1.820.0" dependencies = [ "axum 0.8.9", "chrono", @@ -15265,7 +15266,7 @@ dependencies = [ [[package]] name = "windmill-api-inputs" -version = "1.819.0" +version = "1.820.0" dependencies = [ "axum 0.8.9", "chrono", @@ -15279,7 +15280,7 @@ dependencies = [ [[package]] name = "windmill-api-integration-tests" -version = "1.819.0" +version = "1.820.0" dependencies = [ "anyhow", "async-nats", @@ -15315,7 +15316,7 @@ dependencies = [ [[package]] name = "windmill-api-jobs" -version = "1.819.0" +version = "1.820.0" dependencies = [ "anyhow", "axum 0.8.9", @@ -15340,7 +15341,7 @@ dependencies = [ [[package]] name = "windmill-api-npm-proxy" -version = "1.819.0" +version = "1.820.0" dependencies = [ "axum 0.8.9", "chrono", @@ -15368,7 +15369,7 @@ dependencies = [ [[package]] name = "windmill-api-openapi" -version = "1.819.0" +version = "1.820.0" dependencies = [ "anyhow", "axum 0.8.9", @@ -15390,7 +15391,7 @@ dependencies = [ [[package]] name = "windmill-api-schedule" -version = "1.819.0" +version = "1.820.0" dependencies = [ "axum 0.8.9", "chrono", @@ -15410,7 +15411,7 @@ dependencies = [ [[package]] name = "windmill-api-scripts" -version = "1.819.0" +version = "1.820.0" dependencies = [ "anyhow", "axum 0.8.9", @@ -15452,7 +15453,7 @@ dependencies = [ [[package]] name = "windmill-api-settings" -version = "1.819.0" +version = "1.820.0" dependencies = [ "anyhow", "axum 0.8.9", @@ -15481,7 +15482,7 @@ dependencies = [ [[package]] name = "windmill-api-sse" -version = "1.819.0" +version = "1.820.0" dependencies = [ "lazy_static", "serde", @@ -15493,7 +15494,7 @@ dependencies = [ [[package]] name = "windmill-api-users" -version = "1.819.0" +version = "1.820.0" dependencies = [ "argon2", "axum 0.8.9", @@ -15517,7 +15518,7 @@ dependencies = [ [[package]] name = "windmill-api-workers" -version = "1.819.0" +version = "1.820.0" dependencies = [ "axum 0.8.9", "chrono", @@ -15531,7 +15532,7 @@ dependencies = [ [[package]] name = "windmill-api-workspaces" -version = "1.819.0" +version = "1.820.0" dependencies = [ "axum 0.8.9", "chrono", @@ -15570,7 +15571,7 @@ dependencies = [ [[package]] name = "windmill-audit" -version = "1.819.0" +version = "1.820.0" dependencies = [ "chrono", "lazy_static", @@ -15584,7 +15585,7 @@ dependencies = [ [[package]] name = "windmill-autoscaling" -version = "1.819.0" +version = "1.820.0" dependencies = [ "anyhow", "axum 0.8.9", @@ -15603,7 +15604,7 @@ dependencies = [ [[package]] name = "windmill-common" -version = "1.819.0" +version = "1.820.0" dependencies = [ "aes-gcm", "aho-corasick", @@ -15716,7 +15717,7 @@ dependencies = [ [[package]] name = "windmill-dep-map" -version = "1.819.0" +version = "1.820.0" dependencies = [ "chrono", "futures", @@ -15736,7 +15737,7 @@ dependencies = [ [[package]] name = "windmill-git-sync" -version = "1.819.0" +version = "1.820.0" dependencies = [ "regex", "serde", @@ -15753,7 +15754,7 @@ dependencies = [ [[package]] name = "windmill-indexer" -version = "1.819.0" +version = "1.820.0" dependencies = [ "anyhow", "astral-tokio-tar", @@ -15780,7 +15781,7 @@ dependencies = [ [[package]] name = "windmill-jseval" -version = "1.819.0" +version = "1.820.0" dependencies = [ "anyhow", "futures", @@ -15797,7 +15798,7 @@ dependencies = [ [[package]] name = "windmill-macros" -version = "1.819.0" +version = "1.820.0" dependencies = [ "itertools 0.14.0", "lazy_static", @@ -15813,7 +15814,7 @@ dependencies = [ [[package]] name = "windmill-mcp" -version = "1.819.0" +version = "1.820.0" dependencies = [ "anyhow", "async-trait", @@ -15834,7 +15835,7 @@ dependencies = [ [[package]] name = "windmill-native-triggers" -version = "1.819.0" +version = "1.820.0" dependencies = [ "anyhow", "async-trait", @@ -15865,7 +15866,7 @@ dependencies = [ [[package]] name = "windmill-oauth" -version = "1.819.0" +version = "1.820.0" dependencies = [ "anyhow", "arc-swap", @@ -15890,7 +15891,7 @@ dependencies = [ [[package]] name = "windmill-object-store" -version = "1.819.0" +version = "1.820.0" dependencies = [ "anyhow", "async-stream", @@ -15926,7 +15927,7 @@ dependencies = [ [[package]] name = "windmill-operator" -version = "1.819.0" +version = "1.820.0" dependencies = [ "anyhow", "futures", @@ -15944,7 +15945,7 @@ dependencies = [ [[package]] name = "windmill-parser" -version = "1.819.0" +version = "1.820.0" dependencies = [ "convert_case 0.6.0", "serde", @@ -15953,7 +15954,7 @@ dependencies = [ [[package]] name = "windmill-parser-bash" -version = "1.819.0" +version = "1.820.0" dependencies = [ "anyhow", "lazy_static", @@ -15965,7 +15966,7 @@ dependencies = [ [[package]] name = "windmill-parser-csharp" -version = "1.819.0" +version = "1.820.0" dependencies = [ "anyhow", "serde_json", @@ -15977,7 +15978,7 @@ dependencies = [ [[package]] name = "windmill-parser-go" -version = "1.819.0" +version = "1.820.0" dependencies = [ "anyhow", "gosyn", @@ -15989,7 +15990,7 @@ dependencies = [ [[package]] name = "windmill-parser-graphql" -version = "1.819.0" +version = "1.820.0" dependencies = [ "anyhow", "lazy_static", @@ -16001,7 +16002,7 @@ dependencies = [ [[package]] name = "windmill-parser-java" -version = "1.819.0" +version = "1.820.0" dependencies = [ "anyhow", "serde_json", @@ -16013,7 +16014,7 @@ dependencies = [ [[package]] name = "windmill-parser-nu" -version = "1.819.0" +version = "1.820.0" dependencies = [ "anyhow", "nu-parser", @@ -16024,7 +16025,7 @@ dependencies = [ [[package]] name = "windmill-parser-php" -version = "1.819.0" +version = "1.820.0" dependencies = [ "anyhow", "itertools 0.14.0", @@ -16035,7 +16036,7 @@ dependencies = [ [[package]] name = "windmill-parser-py" -version = "1.819.0" +version = "1.820.0" dependencies = [ "anyhow", "itertools 0.14.0", @@ -16047,7 +16048,7 @@ dependencies = [ [[package]] name = "windmill-parser-py-asset" -version = "1.819.0" +version = "1.820.0" dependencies = [ "anyhow", "rustpython-ast", @@ -16058,7 +16059,7 @@ dependencies = [ [[package]] name = "windmill-parser-py-imports" -version = "1.819.0" +version = "1.820.0" dependencies = [ "anyhow", "async-recursion", @@ -16080,7 +16081,7 @@ dependencies = [ [[package]] name = "windmill-parser-r" -version = "1.819.0" +version = "1.820.0" dependencies = [ "anyhow", "serde_json", @@ -16092,7 +16093,7 @@ dependencies = [ [[package]] name = "windmill-parser-ruby" -version = "1.819.0" +version = "1.820.0" dependencies = [ "anyhow", "lazy_static", @@ -16106,7 +16107,7 @@ dependencies = [ [[package]] name = "windmill-parser-rust" -version = "1.819.0" +version = "1.820.0" dependencies = [ "anyhow", "convert_case 0.6.0", @@ -16123,7 +16124,7 @@ dependencies = [ [[package]] name = "windmill-parser-sql" -version = "1.819.0" +version = "1.820.0" dependencies = [ "anyhow", "lazy_static", @@ -16136,7 +16137,7 @@ dependencies = [ [[package]] name = "windmill-parser-sql-asset" -version = "1.819.0" +version = "1.820.0" dependencies = [ "anyhow", "serde", @@ -16148,7 +16149,7 @@ dependencies = [ [[package]] name = "windmill-parser-ts" -version = "1.819.0" +version = "1.820.0" dependencies = [ "anyhow", "lazy_static", @@ -16166,7 +16167,7 @@ dependencies = [ [[package]] name = "windmill-parser-ts-asset" -version = "1.819.0" +version = "1.820.0" dependencies = [ "anyhow", "serde-wasm-bindgen", @@ -16182,7 +16183,7 @@ dependencies = [ [[package]] name = "windmill-parser-wac" -version = "1.819.0" +version = "1.820.0" dependencies = [ "anyhow", "rustpython-ast", @@ -16198,7 +16199,7 @@ dependencies = [ [[package]] name = "windmill-parser-yaml" -version = "1.819.0" +version = "1.820.0" dependencies = [ "anyhow", "lazy_static", @@ -16212,7 +16213,7 @@ dependencies = [ [[package]] name = "windmill-queue" -version = "1.819.0" +version = "1.820.0" dependencies = [ "anyhow", "async-recursion", @@ -16251,7 +16252,7 @@ dependencies = [ [[package]] name = "windmill-runtime-nativets" -version = "1.819.0" +version = "1.820.0" dependencies = [ "anyhow", "const_format", @@ -16291,7 +16292,7 @@ dependencies = [ [[package]] name = "windmill-sql-datatype-parser-wasm" -version = "1.819.0" +version = "1.820.0" dependencies = [ "getrandom 0.3.4", "wasm-bindgen", @@ -16302,7 +16303,7 @@ dependencies = [ [[package]] name = "windmill-store" -version = "1.819.0" +version = "1.820.0" dependencies = [ "anyhow", "async-recursion", @@ -16337,7 +16338,7 @@ dependencies = [ [[package]] name = "windmill-test-utils" -version = "1.819.0" +version = "1.820.0" dependencies = [ "anyhow", "async-trait", @@ -16361,7 +16362,7 @@ dependencies = [ [[package]] name = "windmill-trigger" -version = "1.819.0" +version = "1.820.0" dependencies = [ "anyhow", "async-trait", @@ -16394,7 +16395,7 @@ dependencies = [ [[package]] name = "windmill-trigger-amqp" -version = "1.819.0" +version = "1.820.0" dependencies = [ "anyhow", "async-trait", @@ -16421,7 +16422,7 @@ dependencies = [ [[package]] name = "windmill-trigger-azure" -version = "1.819.0" +version = "1.820.0" dependencies = [ "anyhow", "async-trait", @@ -16454,7 +16455,7 @@ dependencies = [ [[package]] name = "windmill-trigger-email" -version = "1.819.0" +version = "1.820.0" dependencies = [ "anyhow", "async-trait", @@ -16474,7 +16475,7 @@ dependencies = [ [[package]] name = "windmill-trigger-gcp" -version = "1.819.0" +version = "1.820.0" dependencies = [ "anyhow", "async-trait", @@ -16508,7 +16509,7 @@ dependencies = [ [[package]] name = "windmill-trigger-http" -version = "1.819.0" +version = "1.820.0" dependencies = [ "anyhow", "async-trait", @@ -16545,7 +16546,7 @@ dependencies = [ [[package]] name = "windmill-trigger-kafka" -version = "1.819.0" +version = "1.820.0" dependencies = [ "anyhow", "async-trait", @@ -16568,7 +16569,7 @@ dependencies = [ [[package]] name = "windmill-trigger-mqtt" -version = "1.819.0" +version = "1.820.0" dependencies = [ "anyhow", "async-trait", @@ -16592,7 +16593,7 @@ dependencies = [ [[package]] name = "windmill-trigger-nats" -version = "1.819.0" +version = "1.820.0" dependencies = [ "anyhow", "async-nats", @@ -16616,7 +16617,7 @@ dependencies = [ [[package]] name = "windmill-trigger-postgres" -version = "1.819.0" +version = "1.820.0" dependencies = [ "anyhow", "async-trait", @@ -16651,7 +16652,7 @@ dependencies = [ [[package]] name = "windmill-trigger-sqs" -version = "1.819.0" +version = "1.820.0" dependencies = [ "anyhow", "async-trait", @@ -16679,7 +16680,7 @@ dependencies = [ [[package]] name = "windmill-trigger-websocket" -version = "1.819.0" +version = "1.820.0" dependencies = [ "anyhow", "async-trait", @@ -16704,7 +16705,7 @@ dependencies = [ [[package]] name = "windmill-types" -version = "1.819.0" +version = "1.820.0" dependencies = [ "anyhow", "bitflags 2.13.2", @@ -16723,7 +16724,7 @@ dependencies = [ [[package]] name = "windmill-worker" -version = "1.819.0" +version = "1.820.0" dependencies = [ "anyhow", "async-once-cell", @@ -16841,7 +16842,7 @@ dependencies = [ [[package]] name = "windmill-worker-volumes" -version = "1.819.0" +version = "1.820.0" dependencies = [ "bytes", "futures", @@ -17382,6 +17383,15 @@ dependencies = [ "memchr", ] +[[package]] +name = "winresource" +version = "0.1.31" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0986a8b1d586b7d3e4fe3d9ea39fb451ae22869dcea4aa109d287a374d866087" +dependencies = [ + "version_check", +] + [[package]] name = "winsafe" version = "0.0.19" diff --git a/backend/Cargo.toml b/backend/Cargo.toml index 27d8aeac8e..9c09265dbd 100644 --- a/backend/Cargo.toml +++ b/backend/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "windmill" -version = "1.819.0" +version = "1.820.0" authors.workspace = true edition.workspace = true @@ -88,7 +88,7 @@ members = [ exclude = ["./windmill-duckdb-ffi-internal", "./parsers/windmill-parser-wasm"] [workspace.package] -version = "1.819.0" +version = "1.820.0" authors = ["Ruben Fiszel "] edition = "2021" @@ -329,6 +329,9 @@ strum.workspace = true windows-service = "0.7" windows-sys = { version = "0.52", features = ["Win32_System_Services", "Win32_System_Console", "Win32_Foundation"] } +[target.'cfg(windows)'.build-dependencies] +winresource = { version = "0.1.31", default-features = false } + [target.'cfg(not(target_env = "msvc"))'.dependencies] tikv-jemallocator = { optional = true, workspace = true } tikv-jemalloc-sys = { optional = true, workspace = true } diff --git a/backend/build.rs b/backend/build.rs index d5068697c7..88917b0492 100644 --- a/backend/build.rs +++ b/backend/build.rs @@ -2,4 +2,19 @@ fn main() { // trigger recompilation when a new migration is added println!("cargo:rerun-if-changed=migrations"); + + // Embed a VERSIONINFO resource so the exe's file properties carry the release + // version (from Cargo.toml), which Windows worker update scripts read. + // `cfg(windows)` is the host, matching the Windows-only build-dependency, so + // cross builds to Windows get no version resource. + #[cfg(windows)] + if std::env::var("CARGO_CFG_TARGET_OS").as_deref() == Ok("windows") { + winresource::WindowsResource::new() + .set("ProductName", "Windmill") + .set("FileDescription", "Windmill") + .set("CompanyName", "Windmill Labs, Inc.") + .set("LegalCopyright", "Copyright Windmill Labs, Inc.") + .compile() + .expect("failed to embed the Windows version resource"); + } } diff --git a/backend/ee-repo-ref.txt b/backend/ee-repo-ref.txt index 4d77452901..4cfbcb8447 100644 --- a/backend/ee-repo-ref.txt +++ b/backend/ee-repo-ref.txt @@ -1 +1 @@ -12a83524b81368043bc939caeb3764fd27a32523 +5a2b6b8527250bd12cf856d8a667a9ef3106ec60 diff --git a/backend/parsers/windmill-parser-wasm/Cargo.lock b/backend/parsers/windmill-parser-wasm/Cargo.lock index 0ad36d8e81..40dcff21f8 100644 --- a/backend/parsers/windmill-parser-wasm/Cargo.lock +++ b/backend/parsers/windmill-parser-wasm/Cargo.lock @@ -6216,7 +6216,7 @@ checksum = "712e227841d057c1ee1cd2fb22fa7e5a5461ae8e48fa2ca79ec42cfc1931183f" [[package]] name = "windmill-common" -version = "1.819.0" +version = "1.820.0" dependencies = [ "aho-corasick", "anyhow", @@ -6302,7 +6302,7 @@ dependencies = [ [[package]] name = "windmill-macros" -version = "1.819.0" +version = "1.820.0" dependencies = [ "proc-macro2", "quote", @@ -6314,7 +6314,7 @@ dependencies = [ [[package]] name = "windmill-parser" -version = "1.819.0" +version = "1.820.0" dependencies = [ "convert_case", "serde", @@ -6323,7 +6323,7 @@ dependencies = [ [[package]] name = "windmill-parser-bash" -version = "1.819.0" +version = "1.820.0" dependencies = [ "anyhow", "lazy_static", @@ -6335,7 +6335,7 @@ dependencies = [ [[package]] name = "windmill-parser-csharp" -version = "1.819.0" +version = "1.820.0" dependencies = [ "anyhow", "serde_json", @@ -6347,7 +6347,7 @@ dependencies = [ [[package]] name = "windmill-parser-go" -version = "1.819.0" +version = "1.820.0" dependencies = [ "anyhow", "gosyn", @@ -6359,7 +6359,7 @@ dependencies = [ [[package]] name = "windmill-parser-graphql" -version = "1.819.0" +version = "1.820.0" dependencies = [ "anyhow", "lazy_static", @@ -6371,7 +6371,7 @@ dependencies = [ [[package]] name = "windmill-parser-java" -version = "1.819.0" +version = "1.820.0" dependencies = [ "anyhow", "serde_json", @@ -6383,7 +6383,7 @@ dependencies = [ [[package]] name = "windmill-parser-nu" -version = "1.819.0" +version = "1.820.0" dependencies = [ "anyhow", "nu-parser", @@ -6394,7 +6394,7 @@ dependencies = [ [[package]] name = "windmill-parser-php" -version = "1.819.0" +version = "1.820.0" dependencies = [ "anyhow", "itertools 0.14.0", @@ -6405,7 +6405,7 @@ dependencies = [ [[package]] name = "windmill-parser-py" -version = "1.819.0" +version = "1.820.0" dependencies = [ "anyhow", "itertools 0.14.0", @@ -6417,7 +6417,7 @@ dependencies = [ [[package]] name = "windmill-parser-py-asset" -version = "1.819.0" +version = "1.820.0" dependencies = [ "anyhow", "rustpython-ast", @@ -6428,7 +6428,7 @@ dependencies = [ [[package]] name = "windmill-parser-py-imports" -version = "1.819.0" +version = "1.820.0" dependencies = [ "anyhow", "async-recursion", @@ -6450,7 +6450,7 @@ dependencies = [ [[package]] name = "windmill-parser-r" -version = "1.819.0" +version = "1.820.0" dependencies = [ "anyhow", "serde_json", @@ -6462,7 +6462,7 @@ dependencies = [ [[package]] name = "windmill-parser-ruby" -version = "1.819.0" +version = "1.820.0" dependencies = [ "anyhow", "lazy_static", @@ -6476,7 +6476,7 @@ dependencies = [ [[package]] name = "windmill-parser-rust" -version = "1.819.0" +version = "1.820.0" dependencies = [ "anyhow", "convert_case", @@ -6493,7 +6493,7 @@ dependencies = [ [[package]] name = "windmill-parser-sql" -version = "1.819.0" +version = "1.820.0" dependencies = [ "anyhow", "lazy_static", @@ -6506,7 +6506,7 @@ dependencies = [ [[package]] name = "windmill-parser-sql-asset" -version = "1.819.0" +version = "1.820.0" dependencies = [ "anyhow", "serde", @@ -6518,7 +6518,7 @@ dependencies = [ [[package]] name = "windmill-parser-ts" -version = "1.819.0" +version = "1.820.0" dependencies = [ "anyhow", "lazy_static", @@ -6536,7 +6536,7 @@ dependencies = [ [[package]] name = "windmill-parser-ts-asset" -version = "1.819.0" +version = "1.820.0" dependencies = [ "anyhow", "serde-wasm-bindgen", @@ -6552,7 +6552,7 @@ dependencies = [ [[package]] name = "windmill-parser-wac" -version = "1.819.0" +version = "1.820.0" dependencies = [ "anyhow", "rustpython-ast", @@ -6568,7 +6568,7 @@ dependencies = [ [[package]] name = "windmill-parser-wasm" -version = "1.819.0" +version = "1.820.0" dependencies = [ "anyhow", "getrandom 0.2.17", @@ -6600,7 +6600,7 @@ dependencies = [ [[package]] name = "windmill-parser-yaml" -version = "1.819.0" +version = "1.820.0" dependencies = [ "anyhow", "lazy_static", @@ -6614,7 +6614,7 @@ dependencies = [ [[package]] name = "windmill-types" -version = "1.819.0" +version = "1.820.0" dependencies = [ "anyhow", "bitflags", diff --git a/backend/parsers/windmill-parser-wasm/Cargo.toml b/backend/parsers/windmill-parser-wasm/Cargo.toml index 597b7621c7..ce27a75bc0 100644 --- a/backend/parsers/windmill-parser-wasm/Cargo.toml +++ b/backend/parsers/windmill-parser-wasm/Cargo.toml @@ -12,7 +12,7 @@ resolver = "2" members = ["."] [workspace.package] -version = "1.819.0" +version = "1.820.0" edition = "2021" authors = ["Ruben Fiszel "] diff --git a/backend/tests/ai_tool_queue.rs b/backend/tests/ai_tool_queue.rs new file mode 100644 index 0000000000..4d39176cc9 --- /dev/null +++ b/backend/tests/ai_tool_queue.rs @@ -0,0 +1,220 @@ +use axum::{ + extract::State, + routing::{get, post}, + Json, Router, +}; +use serde_json::{json, Value}; +use sqlx::{Pool, Postgres}; +use std::{sync::Arc, time::Duration}; +use tokio::sync::Notify; +use windmill_common::{flows::FlowValue, jobs::JobPayload}; +use windmill_test_utils::{ + completed_job, in_test_worker, listen_for_completed_jobs, ApiServer, RunJob, StreamFind, +}; + +async fn model(Json(body): Json) -> ([(&'static str, &'static str); 1], String) { + let finished = body["messages"] + .as_array() + .unwrap() + .iter() + .any(|m| m["role"] == "tool"); + let delta = if finished { + json!({"role":"assistant", "content":"done"}) + } else { + json!({"role":"assistant", "tool_calls": (0..3).map(|i| json!({ + "index":i, "id":format!("call_{i}"), "type":"function", + "function":{"name":format!("tool_{i}"), "arguments":"{}"} + })).collect::>()}) + }; + let event = json!({"choices":[{"index":0, "delta":delta, "finish_reason":null}]}); + let end = json!({"choices":[{"index":0, "delta":{}, "finish_reason":if finished {"stop"} else {"tool_calls"}}]}); + ( + [("content-type", "text/event-stream")], + format!("data: {event}\n\ndata: {end}\n\ndata: [DONE]\n\n"), + ) +} + +async fn run_batch( + db: Pool, + parallel: bool, + limited: bool, + unsupported: bool, +) -> anyhow::Result<()> { + std::env::set_var("ALLOW_PRIVATE_AI_BASE_URLS", "true"); + let server = ApiServer::start(db.clone()).await?; + let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await?; + let base = format!("http://{}", listener.local_addr()?); + let gate = Arc::new(Notify::new()); + let router = Router::new() + .route("/v1/chat/completions", post(model)) + .route( + "/first", + get(|State(gate): State>| async move { + gate.notified().await; + "ok" + }), + ) + .route( + "/second", + get(|State(gate): State>| async move { + gate.notify_one(); + "ok" + }), + ) + .with_state(gate); + let stub = tokio::spawn(async move { + axum::serve(listener, router).await.unwrap(); + }); + let tools: Vec = (0..3).map(|i| { + let wait = if parallel && !limited && i < 2 { + format!("await fetch('{base}/{}');", if i == 0 {"first"} else {"second"}) + } else { String::new() }; + let finish = if i == 2 { "throw new Error('expected tool failure');".to_string() } + else { format!("return {i};") }; + json!({"id":format!("t{i}"),"summary":format!("tool_{i}"),"value":{ + "type":"rawscript", "language":"bun", "input_transforms":{}, + "tag": if unsupported && i != 1 { "unserved-tool-tag" } else { "bun" }, + "concurrent_limit": if limited { Some(1) } else { None }, + "custom_concurrency_key": if limited { Some("agent-tool-test") } else { None }, + "content":format!("export async function main() {{ {wait} await Bun.sleep({}); {finish} }}", if i == 0 {200} else {0}) + }}) + }).collect(); + let flow: FlowValue = serde_json::from_value( + json!({"modules":[{"id":"agent","timeout":if unsupported {Some(3)} else {None},"value":{ + "type":"aiagent", "tools":tools, "input_transforms":{ + "provider":{"type":"static","value":{"kind":"customai","model":"queue-test","resource":{"base_url":format!("{base}/v1")}}}, + "user_message":{"type":"static","value":"run the tools"}, + "max_iterations":{"type":"static","value":3} + } + }}]}), + )?; + let id = RunJob::from(JobPayload::RawFlow { + value: flow, + path: Some("u/test/agent_queue".into()), + restarted_from: None, + }) + .push(&db) + .await; + let notifications = listen_for_completed_jobs(&db).await; + let wait = async { + if parallel { + in_test_worker(&db, notifications.find(&id), server.addr.port()).await; + } else { + notifications.find(&id).await; + } + }; + tokio::time::timeout( + Duration::from_secs(45), + in_test_worker(&db, wait, server.addr.port()), + ) + .await?; + let result = completed_job(id, &db).await; + if unsupported { + assert!( + !result.success, + "agent must time out waiting for unsupported tools" + ); + let children: Vec<(String, bool, bool)> = sqlx::query_as( + "SELECT j.runnable_path, c.status = 'success', c.started_at IS NULL + FROM v2_job j JOIN v2_job_completed c USING(id) + WHERE j.parent_job IN (SELECT id FROM v2_job WHERE parent_job = $1) + ORDER BY j.runnable_path", + ) + .bind(id) + .fetch_all(&db) + .await?; + assert_eq!(children.len(), 3); + assert_eq!( + children.iter().map(|c| (c.1, c.2)).collect::>(), + vec![(false, true), (true, false), (false, true)], + "neither the first-tool reservation nor fallback may execute unsupported tags" + ); + stub.abort(); + server.close().await?; + return Ok(()); + } + assert!(result.success, "{:?}", result.result); + let result = result.json_result().expect("agent result"); + let messages: Vec<_> = result["messages"] + .as_array() + .unwrap() + .iter() + .filter(|m| m["role"] == "tool") + .collect(); + assert_eq!(messages.len(), 3); + for (index, message) in messages.iter().enumerate() { + assert_eq!(message["tool_call_id"], format!("call_{index}")); + } + assert_eq!(messages[0]["content"], "0"); + assert_eq!(messages[1]["content"], "1"); + assert!(messages[2]["content"] + .as_str() + .unwrap() + .contains("expected tool failure")); + let (parent_id, parent_worker): (uuid::Uuid, String) = sqlx::query_as( + "SELECT j.id, c.worker FROM v2_job j JOIN v2_job_completed c USING(id) WHERE j.parent_job = $1" + ).bind(id).fetch_one(&db).await?; + let children: Vec<(String, String, bool)> = sqlx::query_as( + "SELECT j.runnable_path, c.worker, c.status = 'success' FROM v2_job j JOIN v2_job_completed c USING(id) WHERE j.parent_job = $1 ORDER BY j.runnable_path" + ).bind(parent_id).fetch_all(&db).await?; + assert_eq!(children.len(), 3); + assert_eq!( + children[0].1, parent_worker, + "first tool must stay on the parent worker" + ); + assert_eq!( + children.iter().map(|c| c.2).collect::>(), + vec![true, true, false] + ); + if limited { + let overlapping: i64 = sqlx::query_scalar( + "SELECT count(*) FROM v2_job j1 JOIN v2_job_completed c1 ON c1.id = j1.id + JOIN v2_job j2 ON j2.parent_job = j1.parent_job AND j2.id > j1.id + JOIN v2_job_completed c2 ON c2.id = j2.id + WHERE j1.parent_job = $1 AND c1.started_at < c2.completed_at AND c2.started_at < c1.completed_at" + ).bind(parent_id).fetch_one(&db).await?; + assert_eq!( + overlapping, 0, + "the reserved first job must count toward the shared limit" + ); + } else if parallel { + assert_ne!( + children[1].1, parent_worker, + "second tool must unblock the first from another worker" + ); + } else { + assert!(children.iter().all(|child| child.1 == parent_worker)); + } + stub.abort(); + server.close().await?; + Ok(()) +} + +#[sqlx::test(fixtures("base"))] +#[serial_test::serial] +async fn parent_drains_tools_without_another_worker(db: Pool) -> anyhow::Result<()> { + run_batch(db, false, false, false).await +} + +#[sqlx::test(fixtures("base"))] +#[serial_test::serial] +async fn parent_keeps_first_tool_while_another_worker_runs_siblings( + db: Pool, +) -> anyhow::Result<()> { + run_batch(db, true, false, false).await +} + +#[cfg(all(feature = "enterprise", feature = "private"))] +#[sqlx::test(fixtures("base"))] +#[serial_test::serial] +async fn reserved_tool_obeys_shared_concurrency_limit(db: Pool) -> anyhow::Result<()> { + run_batch(db, true, true, false).await +} + +#[sqlx::test(fixtures("base"))] +#[serial_test::serial] +async fn parent_leaves_unsupported_first_and_remaining_tools_queued( + db: Pool, +) -> anyhow::Result<()> { + run_batch(db, false, false, true).await +} diff --git a/backend/tests/caller_modules_strip.rs b/backend/tests/caller_modules_strip.rs new file mode 100644 index 0000000000..e6f96c04ec --- /dev/null +++ b/backend/tests/caller_modules_strip.rs @@ -0,0 +1,110 @@ +//! A caller's `_MODULES` never reaches a job: the worker builds that arg in as module code, +//! so on a deployed runnable it would run caller code as the runnable (and its +//! `on_behalf_of`). `push` drops it from `args` and `extra` alike and only sets it from a +//! preview's own `RawCode::modules`. + +use std::collections::HashMap; + +use serde_json::{json, value::RawValue}; +use sqlx::{Pool, Postgres}; +use windmill_common::{ + jobs::{JobPayload, RawCode}, + runnable_settings::{ConcurrencySettings, DebouncingSettings}, + scripts::{ScriptHash, ScriptLang, ScriptModule}, +}; +use windmill_queue::{PushArgs, PushIsolationLevel}; + +fn modules(v: &str) -> serde_json::Value { + json!({ "helper.ts": { "content": format!("export const v = \"{v}\""), "language": "bun" } }) +} + +/// Pushes `payload` with a caller `_MODULES` both in `args` and in `extra` (where webhook +/// query and headers land), and returns the stored `_MODULES`. +async fn stored_modules(db: &Pool, payload: JobPayload) -> Option { + let caller: Box = serde_json::value::to_raw_value(&modules("caller")).unwrap(); + let args = HashMap::from([("_MODULES".to_string(), caller.clone())]); + let extra = HashMap::from([("_MODULES".to_string(), caller)]); + let (id, tx) = windmill_queue::push( + db, + PushIsolationLevel::IsolatedRoot(db.clone()), + "test-workspace", + payload, + PushArgs { args: &args, extra: Some(extra) }, + "test-user", + "test@windmill.dev", + "u/test-user".to_string(), + None, + None, + None, + None, + None, + None, + None, + None, + false, + false, + None, + true, + None, + None, + None, + None, + None, + false, + None, + None, + None, + ) + .await + .expect("push must succeed"); + tx.commit().await.unwrap(); + sqlx::query_scalar::<_, Option>( + "SELECT args->'_MODULES' FROM v2_job WHERE id = $1", + ) + .bind(id) + .fetch_one(db) + .await + .unwrap() +} + +#[sqlx::test(fixtures("base"))] +async fn caller_modules_never_reach_a_job(db: Pool) { + let deployed = JobPayload::ScriptHash { + hash: ScriptHash(123412), + path: "f/system/hello".to_string(), + cache_ttl: None, + cache_ignore_s3_path: None, + dedicated_worker: None, + language: ScriptLang::Bun, + priority: None, + apply_preprocessor: false, + concurrency_settings: ConcurrencySettings::default(), + debouncing_settings: DebouncingSettings::default(), + labels: None, + }; + assert_eq!(stored_modules(&db, deployed).await, None); + + let preview = |modules: Option>| { + JobPayload::Code(RawCode { + hash: None, + content: "import { v } from \"./helper\"; export function main() { return v }" + .to_string(), + path: None, + language: ScriptLang::Bun, + lock: None, + concurrency_settings: ConcurrencySettings::default().into(), + debouncing_settings: DebouncingSettings::default(), + cache_ttl: None, + cache_ignore_s3_path: None, + dedicated_worker: None, + modules, + tag: None, + }) + }; + assert_eq!(stored_modules(&db, preview(None)).await, None); + let own = serde_json::from_value(modules("preview")).unwrap(); + assert_eq!( + stored_modules(&db, preview(Some(own))).await, + Some(modules("preview")) + ); +} diff --git a/backend/tests/datatable_instance_pg_disabled.rs b/backend/tests/datatable_instance_pg_disabled.rs new file mode 100644 index 0000000000..8d424052e6 --- /dev/null +++ b/backend/tests/datatable_instance_pg_disabled.rs @@ -0,0 +1,40 @@ +//! Turning Windmill's own Postgres off as a data table substrate takes the data tables already on +//! it out of use, not only new ones: resolution is the one path every job, API call and trigger +//! reaches a data table through. + +use sqlx::{Pool, Postgres}; +use windmill_common::workspaces::get_datatable_resource_from_db_unchecked; + +#[sqlx::test] +async fn test_instance_datatable_refused_once_instance_pg_disabled( + db: Pool, +) -> anyhow::Result<()> { + sqlx::query("INSERT INTO workspace (id, name, owner) VALUES ('dtoff-ws', 'DTOFF', 'dtoff')") + .execute(&db) + .await?; + sqlx::query( + "INSERT INTO workspace_settings (workspace_id, datatable) VALUES ('dtoff-ws', \ + '{\"datatables\": {\"main\": {\"database\": {\"resource_type\": \"instance\", \ + \"resource_path\": \"dt_dtoff\"}}}}')", + ) + .execute(&db) + .await?; + + let resolved = get_datatable_resource_from_db_unchecked(&db, "dtoff-ws", "main").await?; + assert_eq!(resolved["dbname"], "dt_dtoff"); + + sqlx::query( + "INSERT INTO global_settings (name, value) VALUES ('instance_pg_disabled', 'true')", + ) + .execute(&db) + .await?; + + let refused = get_datatable_resource_from_db_unchecked(&db, "dtoff-ws", "main") + .await + .expect_err("an instance data table must not resolve while Windmill's database is off"); + assert!( + refused.to_string().contains("disabled on this instance"), + "unexpected error: {refused}" + ); + Ok(()) +} diff --git a/backend/tests/fixtures/job_provenance.sql b/backend/tests/fixtures/job_provenance.sql index 202c8f2df7..49eb4212d5 100644 --- a/backend/tests/fixtures/job_provenance.sql +++ b/backend/tests/fixtures/job_provenance.sql @@ -30,3 +30,34 @@ INSERT INTO v2_job (id, workspace_id, kind, runnable_path, trigger_kind, trigger VALUES ('3bb0c0de-0000-4000-8000-000000000007', 'test-workspace', 'appscript', 'f/t/app/comp', 'app', 'f/t/app', 'test-user', 'u/test-user', 'test@windmill.dev'), ('3bb0c0de-0000-4000-8000-000000000008', 'test-workspace', 'appscript', 'f/t/app/comp', NULL, NULL, 'test-user', 'u/test-user', 'test@windmill.dev'); + +-- Inline steps of the deployed flow f/t/agent run as previews (no flow node, or +-- DISABLE_FLOW_SCRIPT): a direct step, a step in a loop body, and a step whose path +-- leaves the flow's namespace. +INSERT INTO v2_job (id, workspace_id, kind, runnable_path, parent_job, created_by, permissioned_as, permissioned_as_email) +VALUES + ('3bb0c0de-0000-4000-8000-000000000009', 'test-workspace', 'preview', 'f/t/agent/a', '3bb0c0de-0000-4000-8000-000000000001', 'test-user', 'u/test-user', 'test@windmill.dev'), + ('3bb0c0de-0000-4000-8000-00000000000a', 'test-workspace', 'flowpreview', 'f/t/agent/loop-0', '3bb0c0de-0000-4000-8000-000000000001', 'test-user', 'u/test-user', 'test@windmill.dev'), + ('3bb0c0de-0000-4000-8000-00000000000b', 'test-workspace', 'preview', 'f/t/agent/loop-0/a', '3bb0c0de-0000-4000-8000-00000000000a', 'test-user', 'u/test-user', 'test@windmill.dev'), + ('3bb0c0de-0000-4000-8000-00000000000c', 'test-workspace', 'preview', 'f/t/agentx', '3bb0c0de-0000-4000-8000-000000000001', 'test-user', 'u/test-user', 'test@windmill.dev'); + +-- Steps of the flow preview claiming f/t/agent: one nested in a branch, and one whose +-- path names another item. +INSERT INTO v2_job (id, workspace_id, kind, runnable_path, parent_job, created_by, permissioned_as, permissioned_as_email) +VALUES + ('3bb0c0de-0000-4000-8000-00000000000d', 'test-workspace', 'flowpreview', 'f/t/agent/branchone-1', '3bb0c0de-0000-4000-8000-000000000003', 'test-user', 'u/test-user', 'test@windmill.dev'), + ('3bb0c0de-0000-4000-8000-00000000000e', 'test-workspace', 'preview', 'f/t/agent/branchone-1/a', '3bb0c0de-0000-4000-8000-00000000000d', 'test-user', 'u/test-user', 'test@windmill.dev'), + ('3bb0c0de-0000-4000-8000-00000000000f', 'test-workspace', 'preview', 'f/prod/deploy', '3bb0c0de-0000-4000-8000-000000000003', 'test-user', 'u/test-user', 'test@windmill.dev'); + +-- A deployed app without app_script entries runs its inline script as a stamped preview. +INSERT INTO v2_job (id, workspace_id, kind, runnable_path, trigger_kind, trigger, created_by, permissioned_as, permissioned_as_email) +VALUES + ('3bb0c0de-0000-4000-8000-000000000010', 'test-workspace', 'preview', 'f/t/app/comp', 'app', 'f/t/app', 'test-user', 'u/test-user', 'test@windmill.dev'); + +-- A step of the deployed flow whose modules came in its args, and a flow preview at the +-- bare `f` whose step names another item. +INSERT INTO v2_job (id, workspace_id, kind, runnable_path, parent_job, args, created_by, permissioned_as, permissioned_as_email) +VALUES + ('3bb0c0de-0000-4000-8000-000000000011', 'test-workspace', 'preview', 'f/t/agent/m', '3bb0c0de-0000-4000-8000-000000000001', '{"_MODULES": {}}', 'test-user', 'u/test-user', 'test@windmill.dev'), + ('3bb0c0de-0000-4000-8000-000000000012', 'test-workspace', 'flowpreview', 'f', NULL, NULL, 'test-user', 'u/test-user', 'test@windmill.dev'), + ('3bb0c0de-0000-4000-8000-000000000013', 'test-workspace', 'preview', 'f/prod/deploy', '3bb0c0de-0000-4000-8000-000000000012', NULL, 'test-user', 'u/test-user', 'test@windmill.dev'); diff --git a/backend/tests/job_provenance.rs b/backend/tests/job_provenance.rs index 41de4f6f77..af9bc19493 100644 --- a/backend/tests/job_provenance.rs +++ b/backend/tests/job_provenance.rs @@ -3,14 +3,32 @@ use sqlx::{Pool, Postgres}; use uuid::Uuid; -use windmill_common::job_provenance::job_provenance; +use windmill_common::job_provenance::{ + job_provenance, ClaimedItem, ClaimedPath, JobProvenance, +}; -async fn deployed(db: &Pool, job: &str) -> bool { +async fn provenance(db: &Pool, job: &str) -> JobProvenance { job_provenance(db, &Uuid::parse_str(job).unwrap(), "test-workspace") .await .unwrap() .unwrap() - .deployed +} + +async fn deployed(db: &Pool, job: &str) -> bool { + provenance(db, job).await.deployed +} + +async fn claims(db: &Pool, job: &str) -> Vec<(String, ClaimedItem)> { + provenance(db, job) + .await + .claimed_paths + .into_iter() + .map(|ClaimedPath { path, item, .. }| (path, item)) + .collect() +} + +fn claim(path: &str, item: ClaimedItem) -> (String, ClaimedItem) { + (path.to_string(), item) } #[sqlx::test(fixtures("base", "job_provenance"))] @@ -40,5 +58,46 @@ async fn flow_running_another_flows_version_is_not_deployed(db: Pool) #[sqlx::test(fixtures("base", "job_provenance"))] async fn app_script_is_deployed_only_from_a_deployed_app_run(db: Pool) { assert!(deployed(&db, "3bb0c0de-0000-4000-8000-000000000007").await); + assert!(deployed(&db, "3bb0c0de-0000-4000-8000-000000000010").await); assert!(!deployed(&db, "3bb0c0de-0000-4000-8000-000000000008").await); + assert_eq!( + claims(&db, "3bb0c0de-0000-4000-8000-000000000008").await, + [claim("f/t/app", ClaimedItem::App)] + ); +} + +/// Inline steps of a flow deployed before flow nodes run as previews, at paths the +/// worker derives under the flow's. +#[sqlx::test(fixtures("base", "job_provenance"))] +async fn preview_step_of_deployed_flow_is_deployed_within_its_path(db: Pool) { + for step in ["09", "0b"] { + let p = provenance(&db, &format!("3bb0c0de-0000-4000-8000-0000000000{step}")).await; + assert!(p.deployed && p.claimed_paths.is_empty(), "step {step}"); + } + assert!(!deployed(&db, "3bb0c0de-0000-4000-8000-00000000000c").await); + assert!(!deployed(&db, "3bb0c0de-0000-4000-8000-000000000011").await); + assert_eq!( + claims(&db, "3bb0c0de-0000-4000-8000-00000000000c").await, + [claim("f/t/agentx", ClaimedItem::Script)] + ); +} + +/// A token acts as every path its chain took from a request: the top-most preview, and +/// any step whose path leaves its parent's. +#[sqlx::test(fixtures("base", "job_provenance"))] +async fn claimed_paths_are_the_request_supplied_origins(db: Pool) { + let flow = || claim("f/t/agent", ClaimedItem::Flow); + assert_eq!(claims(&db, "3bb0c0de-0000-4000-8000-000000000004").await, [flow()]); + assert_eq!(claims(&db, "3bb0c0de-0000-4000-8000-00000000000e").await, [flow()]); + assert_eq!( + claims(&db, "3bb0c0de-0000-4000-8000-00000000000f").await, + [flow(), claim("f/prod/deploy", ClaimedItem::Script)] + ); + assert_eq!(claims(&db, "3bb0c0de-0000-4000-8000-000000000006").await, [flow()]); + assert!(claims(&db, "3bb0c0de-0000-4000-8000-000000000002").await.is_empty()); + // A parent whose path names no item cannot cover the claims below it. + assert_eq!( + claims(&db, "3bb0c0de-0000-4000-8000-000000000013").await, + [claim("f/prod/deploy", ClaimedItem::Script)] + ); } diff --git a/backend/windmill-ai/src/ai_google.rs b/backend/windmill-ai/src/ai_google.rs index b90bda9e1c..bb8ae83e13 100644 --- a/backend/windmill-ai/src/ai_google.rs +++ b/backend/windmill-ai/src/ai_google.rs @@ -303,7 +303,7 @@ pub struct GeminiUsageMetadata { /// 60 tool-use + 17 candidates + 52 thoughts against a `totalTokenCount` of 146, so /// leaving them out under-reports the input of every tool-using turn. Cached tokens /// are not added here, being already part of `promptTokenCount`. -fn gemini_prompt_tokens(usage: &GeminiUsageMetadata) -> i32 { +pub(crate) fn gemini_prompt_tokens(usage: &GeminiUsageMetadata) -> i32 { usage .prompt_token_count .unwrap_or(0) @@ -313,7 +313,7 @@ fn gemini_prompt_tokens(usage: &GeminiUsageMetadata) -> i32 { /// Output tokens as billed: Gemini counts thinking apart from `candidatesTokenCount` /// but charges it at the output rate, so a reply that thought would otherwise be /// reported as far cheaper than it was. -fn gemini_completion_tokens(usage: &GeminiUsageMetadata) -> i32 { +pub(crate) fn gemini_completion_tokens(usage: &GeminiUsageMetadata) -> i32 { usage .candidates_token_count .unwrap_or(0) @@ -1041,7 +1041,10 @@ mod tests { // through unchanged and the cached share is reported alongside it; thinking // is billed as output but counted apart from the candidates. assert_eq!(usage_chunk["usage"]["prompt_tokens"], 1000); - assert_eq!(usage_chunk["usage"]["prompt_tokens_details"]["cached_tokens"], 900); + assert_eq!( + usage_chunk["usage"]["prompt_tokens_details"]["cached_tokens"], + 900 + ); assert_eq!(usage_chunk["usage"]["completion_tokens"], 120); } diff --git a/backend/windmill-ai/src/lib.rs b/backend/windmill-ai/src/lib.rs index b6487c0ac5..e81d030ed3 100644 --- a/backend/windmill-ai/src/lib.rs +++ b/backend/windmill-ai/src/lib.rs @@ -6,6 +6,7 @@ pub mod ai_providers; pub mod ai_types; pub mod credentials; pub mod image_handler; +pub mod model_context; pub mod providers; pub mod proxy; pub mod query_builder; diff --git a/backend/windmill-ai/src/model_context.rs b/backend/windmill-ai/src/model_context.rs new file mode 100644 index 0000000000..4b7b8fd68d --- /dev/null +++ b/backend/windmill-ai/src/model_context.rs @@ -0,0 +1,165 @@ +//! What a model's context window holds, for the code that has to keep a conversation +//! inside it. + +/// Assumed window for a model that is not in the table. Conservative on purpose: a +/// guess that is too small only compacts earlier, which is recoverable, while one that +/// is too large overflows and the provider raises the context error itself. +pub const DEFAULT_CONTEXT_WINDOW: usize = 128_000; + +/// Context windows of the models we know, most specific entry first — the first name +/// found in the bare model id wins, so vendor-namespaced and date-suffixed ids +/// (`anthropic.claude-sonnet-4-6-…-v1:0`, `gpt-5.2-2026-01-01`) still resolve. +/// Conservative family fallbacks sit below the explicit entries. +/// +/// **Mirrors `MODEL_CONTEXT_WINDOWS` in +/// `frontend/src/lib/components/copilot/modelConfig.ts`**, which the AI session's own +/// compaction reads. Add a model to one and it must go in the other, or the same model +/// compacts at one size in a chat and another in an agent step. +const MODEL_CONTEXT_WINDOWS: &[(&str, usize)] = &[ + // Anthropic — Sonnet/Opus 4.6+ ship a 1M window at standard pricing (GA); + // Haiku, older Claude models (3.x, 4.0, 4.1, 4.5) and date-suffixed Claude 4 + // base ids (claude-sonnet-4-20250514) fall through to 200K + ("claude-fable-5", 1_000_000), + ("claude-mythos-5", 1_000_000), + ("claude-opus-5", 1_000_000), + ("claude-sonnet-5", 1_000_000), + ("claude-opus-4-8", 1_000_000), + ("claude-opus-4-7", 1_000_000), + ("claude-opus-4-6", 1_000_000), + ("claude-sonnet-4-6", 1_000_000), + ("claude", 200_000), + // OpenAI — gpt-5 covers the base family (-mini / -nano) and the 5.1/5.2 + // revisions, all 400K; 5.4/5.5 moved to 1M and 5.6 to 1.05M + ("gpt-5-6", 1_050_000), + ("gpt-5-5", 1_000_000), + ("gpt-5-4", 1_000_000), + ("gpt-5", 400_000), + ("gpt-4-1", 1_000_000), + ("gpt-4o", 128_000), + ("o4-mini", 200_000), + ("o3", 200_000), + // Google — the 2.5 / 3 / 3.1 Gemini families are all 1M + ("gemini-3-1", 1_000_000), + ("gemini-3", 1_000_000), + ("gemini-2-5", 1_000_000), + // DeepSeek — the V4 family (pro / flash) is 1M. The deepseek-chat / + // deepseek-reasoner aliases were retired 2026-07-24 but can still sit in a + // saved selection, so they keep resolving to the window they had. + ("deepseek-v4", 1_000_000), + ("deepseek-chat", 1_000_000), + ("deepseek-reasoner", 1_000_000), + ("deepseek", 128_000), + // Alibaba — Qwen3-Max is 256K. No qwen family fallback: variant windows range + // from 8K (character models) to 1M, too wide for even a conservative guess + ("qwen3-max", 256_000), + // Others — Mistral Medium 3.5 is 256K, reachable under both its version and + // the `-latest` alias. There is deliberately no `mistral-medium` family row: + // pinned older snapshots are 128K, and over-claiming a window overflows it. + ("mistral-medium-3-5", 256_000), + ("mistral-medium-latest", 256_000), + ("llama", 128_000), + ("codestral", 32_000), +]; + +/// The bare model id an entry is matched against: lowercased, the leading `~` and any +/// vendor path segment dropped, the `:`-suffixed route removed, and `.` collapsed to +/// `-`. Version separators differ by route to the same model — Anthropic writes +/// `claude-opus-4-8`, OpenRouter `anthropic/claude-opus-4.8` — so collapsing both sides +/// keeps one entry covering every route. +fn bare_model_id(model: &str) -> String { + let normalized = model.trim().to_lowercase(); + let normalized = normalized.strip_prefix('~').unwrap_or(&normalized); + let last = normalized.rsplit('/').next().unwrap_or(normalized); + let base = match last.find(':') { + Some(0) | None => last, + Some(colon) => &last[..colon], + }; + base.replace('.', "-") +} + +/// Whether this is one of OpenAI's reasoning models (gpt-5 and later, the o-series), +/// which think by default. Mirrors `requiresMaxCompletionTokens` in the frontend's +/// `modelConfig.ts`: the o-series match wants a digit after the `o` so that ids such +/// as `open-mistral-*` stay out. +pub fn is_openai_reasoning_model(model: &str) -> bool { + let id = bare_model_id(model); + id.starts_with("gpt-5") + || (id.starts_with('o') && id[1..].starts_with(|c: char| c.is_ascii_digit())) +} + +/// The window this model is known to hold, `None` for one not in the table. +fn known_model_context_window(model: &str) -> Option { + let id = bare_model_id(model); + MODEL_CONTEXT_WINDOWS + .iter() + .find(|(name, _)| matches_entry(&id, name)) + .map(|(_, window)| *window) +} + +/// The window to plan against: the model's where it is known, the conservative +/// assumption otherwise. +pub fn model_context_window(model: &str) -> usize { + known_model_context_window(model).unwrap_or(DEFAULT_CONTEXT_WINDOW) +} + +/// An entry matches anywhere in the id, so a vendor prefix or a date suffix does not +/// hide it. One ending on a version digit must not run into a longer version: +/// `gpt-4-1` would otherwise claim `gpt-4-1106-preview`. A family fallback ending on a +/// letter gets no such guard — a version welded straight onto the name (`llama3-1`) is +/// what it exists to catch. +fn matches_entry(id: &str, name: &str) -> bool { + let guard_digits = name.ends_with(|c: char| c.is_ascii_digit()); + id.match_indices(name).any(|(at, _)| { + !guard_digits + || !id[at + name.len()..] + .chars() + .next() + .is_some_and(|c| c.is_ascii_digit()) + }) +} + +#[cfg(test)] +mod tests { + use super::*; + + /// One entry has to cover every route to the same model: the vendor path and dot + /// versions OpenRouter uses, Bedrock's prefixed and `-v1:0`-suffixed ids, and the + /// date suffixes the providers' own ids carry. + #[test] + fn every_route_to_a_model_resolves_to_one_window() { + for id in [ + "claude-opus-4-8", + "anthropic/claude-opus-4.8", + "~anthropic/claude-opus-4.8", + "anthropic.claude-opus-4-8-20260101-v1:0", + ] { + assert_eq!(known_model_context_window(id), Some(1_000_000), "{id}"); + } + // Falls through the explicit rows to the family fallback. + assert_eq!( + known_model_context_window("claude-3-5-haiku"), + Some(200_000) + ); + } + + /// The guard that keeps an entry ending on a version digit from claiming a longer + /// version of the same family. + #[test] + fn a_version_entry_does_not_claim_a_longer_version() { + assert_eq!(known_model_context_window("gpt-4.1"), Some(1_000_000)); + assert_eq!(known_model_context_window("gpt-4-1106-preview"), None); + assert_eq!(known_model_context_window("gpt-5-mini"), Some(400_000)); + assert_eq!(known_model_context_window("gpt-5.6"), Some(1_050_000)); + } + + /// An unknown model still gets a number, so nothing downstream has to carry a + /// "no limit" case that would let a conversation grow unbounded. + #[test] + fn an_unknown_model_falls_back_to_the_assumed_window() { + assert_eq!(known_model_context_window("some-local-llm"), None); + assert_eq!( + model_context_window("some-local-llm"), + DEFAULT_CONTEXT_WINDOW + ); + } +} diff --git a/backend/windmill-ai/src/providers/anthropic.rs b/backend/windmill-ai/src/providers/anthropic.rs index d274d91fa2..7acd2db939 100644 --- a/backend/windmill-ai/src/providers/anthropic.rs +++ b/backend/windmill-ai/src/providers/anthropic.rs @@ -782,7 +782,7 @@ impl QueryBuilder for AnthropicQueryBuilder { // Convert Anthropic usage to TokenUsage let usage = anthropic_usage.map(|u| { TokenUsage::from_input_output(u.input_tokens, u.output_tokens) - .with_cache(u.cache_read_input_tokens, u.cache_creation_input_tokens) + .with_cache_beside_input(u.cache_read_input_tokens, u.cache_creation_input_tokens) }); Ok(ParsedResponse::Text { diff --git a/backend/windmill-ai/src/providers/bedrock.rs b/backend/windmill-ai/src/providers/bedrock.rs index 412e057396..836244a666 100644 --- a/backend/windmill-ai/src/providers/bedrock.rs +++ b/backend/windmill-ai/src/providers/bedrock.rs @@ -1148,7 +1148,7 @@ impl BedrockQueryBuilder { Some(token_usage.output_tokens()), Some(token_usage.total_tokens()), ) - .with_cache( + .with_cache_beside_input( token_usage .cache_read_input_tokens() .map(|v| i32::try_from(v).unwrap_or(i32::MAX)), diff --git a/backend/windmill-ai/src/providers/google_ai.rs b/backend/windmill-ai/src/providers/google_ai.rs index ce37e45df7..97d0383ba4 100644 --- a/backend/windmill-ai/src/providers/google_ai.rs +++ b/backend/windmill-ai/src/providers/google_ai.rs @@ -1,10 +1,11 @@ use crate::{ ai_google::{ - gemini_event_to_openai_sse_chunks, gemini_response_to_openai, openai_messages_to_gemini, - openai_tools_to_gemini, parse_gemini_response, parse_gemini_sse_event, - sanitize_schema_for_google, GeminiFunctionDeclaration, GeminiGenerationConfig, - GeminiImageContent, GeminiImageRequest, GeminiImageResponse, GeminiInlineData, GeminiPart, - GeminiPredictContent, GeminiTextRequest, GeminiThinkingConfig, GeminiTool, + gemini_completion_tokens, gemini_event_to_openai_sse_chunks, gemini_prompt_tokens, + gemini_response_to_openai, openai_messages_to_gemini, openai_tools_to_gemini, + parse_gemini_response, parse_gemini_sse_event, sanitize_schema_for_google, + GeminiFunctionDeclaration, GeminiGenerationConfig, GeminiImageContent, GeminiImageRequest, + GeminiImageResponse, GeminiInlineData, GeminiPart, GeminiPredictContent, GeminiTextRequest, + GeminiThinkingConfig, GeminiTool, }, image_handler::{download_and_encode_s3_image, prepare_messages_for_api}, proxy::{ProxyBuildArgs, ProxyRequest}, @@ -685,12 +686,32 @@ impl QueryBuilder for GoogleAIQueryBuilder { stream_event_processor.send(event, &mut events_str).await?; } - let usage = gemini_usage.map(|u| { - TokenUsage::new( - u.prompt_token_count, - u.candidates_token_count, - u.total_token_count, - ) + // Through the shared helpers, which fold in the two counts Gemini reports beside + // the headline ones: tool-use prompt tokens, disjoint from `promptTokenCount` and + // present on every tool-using turn, and thinking tokens. Reading the headline + // fields alone under-reports the prompt of exactly the conversations compaction + // has to notice. + let usage = gemini_usage.and_then(|u| { + let has_prompt = + u.prompt_token_count.is_some() || u.tool_use_prompt_token_count.is_some(); + let has_completion = + u.candidates_token_count.is_some() || u.thoughts_token_count.is_some(); + if !has_prompt && !has_completion && u.total_token_count.is_none() { + // An endpoint that reported nothing has to stay `None`. A zeroed count + // reads downstream as an empty prompt, where absent means "fall back to + // estimating the conversation". + return None; + } + let prompt = has_prompt.then(|| gemini_prompt_tokens(&u)); + let completion = has_completion.then(|| gemini_completion_tokens(&u)); + Some(TokenUsage::new( + prompt, + completion, + u.total_token_count.or_else(|| match (prompt, completion) { + (Some(p), Some(c)) => Some(p.saturating_add(c)), + _ => None, + }), + )) }); Ok(ParsedResponse::Text { @@ -1014,4 +1035,22 @@ mod tests { .headers .contains(&("x-goog-api-key".to_string(), "api-key".to_string()))); } + + /// Gemini reports a tool-using turn's input across two disjoint fields. Reading only + /// the headline one hides the tool results from the agent step's usage — and from the + /// compaction trigger, which is what would notice them. + #[test] + fn gemini_agent_usage_counts_tool_use_and_thinking_tokens() { + let usage = crate::ai_google::GeminiUsageMetadata { + prompt_token_count: Some(17), + candidates_token_count: Some(17), + total_token_count: Some(146), + tool_use_prompt_token_count: Some(60), + thoughts_token_count: Some(52), + ..Default::default() + }; + + assert_eq!(gemini_prompt_tokens(&usage), 77); + assert_eq!(gemini_completion_tokens(&usage), 69); + } } diff --git a/backend/windmill-ai/src/query_builder.rs b/backend/windmill-ai/src/query_builder.rs index 3281b51cd0..986c433f42 100644 --- a/backend/windmill-ai/src/query_builder.rs +++ b/backend/windmill-ai/src/query_builder.rs @@ -6,6 +6,49 @@ use crate::ai_types::OpenAIToolCall; use crate::proxy::{ProxyBuildArgs, ProxyRequest}; use crate::types::*; +/// Only explicit input-context rejections warrant dropping conversation history. +/// Rate limits, output-token limits and generic validation errors must propagate. +pub fn is_context_length_error(message: &str) -> bool { + let message = message.to_ascii_lowercase(); + message.contains("context_length_exceeded") + || message.contains("maximum context length") + || message.contains("prompt is too long") + || message.contains("exceed context limit") + || message.contains("input is too long for requested model") + || (message.contains("input token count") && message.contains("exceeds the maximum")) + || message.contains("too many input tokens") +} + +#[cfg(test)] +mod context_error_tests { + use super::is_context_length_error; + + #[test] + fn recognizes_context_rejections_without_retrying_other_provider_errors() { + for message in [ + r#"{"error":{"code":"context_length_exceeded"}}"#, + "This model's maximum context length is 128000 tokens", + "prompt is too long: 210000 tokens > 200000 maximum", + "input length and `max_tokens` exceed context limit: 199000 + 8192 > 200000", + "The input token count (10000) exceeds the maximum number of tokens allowed (8192)", + "ValidationException: Input is too long for requested model.", + "ValidationException: Too many input tokens", + ] { + assert!(is_context_length_error(message), "{message}"); + } + for message in [ + "Rate limit exceeded: tokens per minute", + "max_tokens exceeds the maximum output tokens", + "Invalid tool schema", + "Additional properties are not allowed: stream_options", + "Request body too large", + "Internal server error", + ] { + assert!(!is_context_length_error(message), "{message}"); + } + } +} + /// Arguments for building an AI request pub struct BuildRequestArgs<'a> { pub messages: &'a [OpenAIMessage], diff --git a/backend/windmill-ai/src/sse.rs b/backend/windmill-ai/src/sse.rs index 57acd66738..ada50afca5 100644 --- a/backend/windmill-ai/src/sse.rs +++ b/backend/windmill-ai/src/sse.rs @@ -304,7 +304,16 @@ pub enum AnthropicDelta { Unknown, } -/// Anthropic usage information from message_delta event +/// The `message` envelope of a `message_start` event. Only its usage is read: the +/// prompt-side counts appear here and nowhere else in the stream. +#[derive(Deserialize, Debug)] +pub struct AnthropicStreamMessage { + #[serde(default)] + pub usage: Option, +} + +/// Anthropic usage information, reported across `message_start` (prompt side) and +/// `message_delta` (completion side) #[derive(Deserialize, Debug, Clone)] pub struct AnthropicUsage { #[serde(default)] @@ -322,7 +331,10 @@ pub struct AnthropicUsage { #[serde(tag = "type")] pub enum AnthropicSSEEvent { #[serde(rename = "message_start")] - MessageStart {}, + MessageStart { + #[serde(default)] + message: Option, + }, #[serde(rename = "content_block_start")] ContentBlockStart { index: usize, content_block: AnthropicContentBlockStart }, #[serde(rename = "content_block_delta")] @@ -371,7 +383,8 @@ pub struct AnthropicSSEParser { pub annotations: Vec, /// Whether web search was used in this response pub used_websearch: bool, - /// Token usage from message_delta event + /// Token usage, merged from the `message_start` (prompt side) and `message_delta` + /// (completion side) events pub usage: Option, /// Claude thinking block accumulated from `thinking`/`signature` deltas /// (or a redacted block). Attached to the first tool call of the turn so it @@ -574,14 +587,35 @@ impl SSEParser for AnthropicSSEParser { let error_msg = message.unwrap_or_else(|| "Unknown error".to_string()); tracing::error!("Anthropic streaming error: {}", error_msg); } + AnthropicSSEEvent::MessageStart { message } => { + // The only event carrying the prompt-side counts. `message_delta` + // reports the completion, so dropping this one leaves the request + // with no input token count at all. + if let Some(usage) = message.and_then(|message| message.usage) { + self.usage = Some(usage); + } + } AnthropicSSEEvent::MessageDelta { usage } => { if let Some(u) = usage { - self.usage = Some(u); + match &mut self.usage { + // Field by field, so the prompt counts from `message_start` + // survive a delta that only reports the completion. + Some(existing) => { + existing.input_tokens = u.input_tokens.or(existing.input_tokens); + existing.output_tokens = u.output_tokens.or(existing.output_tokens); + existing.cache_read_input_tokens = u + .cache_read_input_tokens + .or(existing.cache_read_input_tokens); + existing.cache_creation_input_tokens = u + .cache_creation_input_tokens + .or(existing.cache_creation_input_tokens); + } + None => self.usage = Some(u), + } } } // Ignore other events - AnthropicSSEEvent::MessageStart {} - | AnthropicSSEEvent::MessageStop {} + AnthropicSSEEvent::MessageStop {} | AnthropicSSEEvent::Ping {} | AnthropicSSEEvent::Unknown => {} } @@ -794,6 +828,11 @@ pub enum OpenAIResponsesSSEEvent { #[serde(rename = "response.completed")] Completed { response: OpenAIResponsesResponse }, + /// Response ended by `max_output_tokens` or a filter: the same object, with the + /// tokens spent so far, so it is billed the same + #[serde(rename = "response.incomplete")] + Incomplete { response: OpenAIResponsesResponse }, + /// Response created #[serde(rename = "response.created")] Created {}, @@ -969,8 +1008,8 @@ impl SSEParser for OpenAIResponsesSSEParser { }); } - OpenAIResponsesSSEEvent::Completed { response } => { - // Extract usage from response.completed event + OpenAIResponsesSSEEvent::Completed { response } + | OpenAIResponsesSSEEvent::Incomplete { response } => { if let Some(usage) = response.usage { self.usage = Some(usage); } @@ -1041,6 +1080,43 @@ mod tests { assert_eq!(token_usage.total_tokens, Some(4820)); } + struct NoopSink; + + #[async_trait::async_trait] + impl StreamEventSink for NoopSink { + async fn send( + &self, + _event: StreamingEvent, + _events_str: &mut String, + ) -> Result<(), Error> { + Ok(()) + } + } + + /// The prompt-side counts arrive only on `message_start` and the completion total + /// only on `message_delta`; a parser that keeps just the last one reports a request + /// with no input tokens at all. + #[tokio::test] + async fn anthropic_usage_merges_message_start_and_message_delta() { + let mut parser = AnthropicSSEParser::new(Box::new(NoopSink)); + parser + .parse_event_data( + r#"{"type":"message_start","message":{"id":"msg_1","role":"assistant","usage":{"input_tokens":4821,"output_tokens":1,"cache_read_input_tokens":4096,"cache_creation_input_tokens":128}}}"#, + ) + .await + .unwrap(); + parser + .parse_event_data(r#"{"type":"message_delta","usage":{"output_tokens":312}}"#) + .await + .unwrap(); + + let usage = parser.usage.expect("usage"); + assert_eq!(usage.input_tokens, Some(4821)); + assert_eq!(usage.output_tokens, Some(312)); + assert_eq!(usage.cache_read_input_tokens, Some(4096)); + assert_eq!(usage.cache_creation_input_tokens, Some(128)); + } + #[test] fn openai_responses_usage_maps_cached_input_tokens() { // Payload shape returned by the OpenAI Responses API. diff --git a/backend/windmill-ai/src/types.rs b/backend/windmill-ai/src/types.rs index ac65f637e2..03476240a3 100644 --- a/backend/windmill-ai/src/types.rs +++ b/backend/windmill-ai/src/types.rs @@ -82,6 +82,14 @@ pub enum Memory { #[serde(default, deserialize_with = "deserialize_null_as_zero")] context_length: usize, }, + Compaction { + /// Overrides the window looked up from the model. Only a step the lookup cannot + /// serve — a Custom AI deployment, a model id not in the table — needs one, and + /// a value larger than the model actually serves lets the conversation overflow + /// before the summary is ever taken. + #[serde(default, deserialize_with = "deserialize_null_as_zero")] + context_window: usize, + }, /// Written before `window`. Its `memory_id` stays a fallback behind the run's memory id. Auto { #[serde(default, deserialize_with = "deserialize_null_as_zero")] @@ -388,13 +396,28 @@ impl TokenUsage { Self::new(input, output, total) } - /// Add cache token information + /// Records a cached prefix the provider counted *inside* `input_tokens`, which the + /// OpenAI-shaped ones do. The counts are kept only for the cost split. pub fn with_cache(mut self, read: Option, write: Option) -> Self { self.cache_read_input_tokens = read; self.cache_write_input_tokens = write; self } + /// Records a cached prefix the provider reported *beside* `input_tokens` rather than + /// inside it, which Anthropic and Bedrock do. `input_tokens` is raised to the whole + /// prompt, so `input_tokens` means the same thing whatever served the request; the + /// cache counts stay as the subsets they have become, and the total follows. + pub fn with_cache_beside_input(mut self, read: Option, write: Option) -> Self { + let beside = read.unwrap_or(0).saturating_add(write.unwrap_or(0)); + self.input_tokens = self.input_tokens.map(|input| input.saturating_add(beside)); + self.total_tokens = match (self.input_tokens, self.output_tokens) { + (Some(input), Some(output)) => Some(input.saturating_add(output)), + _ => self.total_tokens.map(|total| total.saturating_add(beside)), + }; + self.with_cache(read, write) + } + pub fn is_empty(&self) -> bool { self.input_tokens.is_none() && self.output_tokens.is_none() @@ -939,6 +962,26 @@ mod tests { use super::*; use std::collections::HashMap; + /// Whichever provider served the request, `input_tokens` has to end up meaning the + /// whole prompt. Leaving the Anthropic shape as reported under-states it by the + /// entire cached prefix, which is exactly what compaction has to notice. + #[test] + fn both_provider_cache_shapes_report_the_whole_prompt() { + // OpenAI-shaped: cached_tokens is already inside input_tokens. + let openai = TokenUsage::new(Some(1000), Some(10), Some(1010)).with_cache(Some(800), None); + assert_eq!(openai.input_tokens, Some(1000)); + assert_eq!(openai.total_tokens, Some(1010)); + + // Anthropic-shaped: the cached prefix is reported beside input_tokens, and the + // total follows the prompt it is folded into. + let anthropic = TokenUsage::from_input_output(Some(200), Some(10)) + .with_cache_beside_input(Some(5000), Some(300)); + assert_eq!(anthropic.input_tokens, Some(5500)); + assert_eq!(anthropic.total_tokens, Some(5510)); + // Kept as the subsets they have become, for the cost split. + assert_eq!(anthropic.cache_read_input_tokens, Some(5000)); + } + /// Helper to create a simple string type schema fn string_schema() -> OpenAPISchema { OpenAPISchema { diff --git a/backend/windmill-api-integration-tests/tests/users.rs b/backend/windmill-api-integration-tests/tests/users.rs index 65d44509d7..56656e54dd 100644 --- a/backend/windmill-api-integration-tests/tests/users.rs +++ b/backend/windmill-api-integration-tests/tests/users.rs @@ -993,3 +993,56 @@ async fn test_drafts_follow_their_owner_without_a_fkey(db: Pool) -> an Ok(()) } + +#[sqlx::test(migrations = "../migrations", fixtures("base"))] +async fn test_impersonation_session_is_never_refreshed(db: Pool) -> anyhow::Result<()> { + initialize_tracing().await; + let server = ApiServer::start(db.clone()).await?; + let base = format!("http://localhost:{}/api/users", server.addr.port()); + let get = |token: &'static str, path: &str| { + client() + .get(format!("{base}/{path}")) + .header("Authorization", format!("Bearer {token}")) + .send() + }; + + sqlx::query( + "INSERT INTO token (token_hash, token_prefix, email, label, expiration) + VALUES (encode(sha256('IMPERSONATION_TOKEN'::bytea), 'hex'), 'IMPERSONAT', + 'test2@windmill.dev', 'impersonation:test@windmill.dev', now() + interval '1 day')", + ) + .execute(&db) + .await?; + let refreshed = get("IMPERSONATION_TOKEN", "refresh_token") + .await? + .text() + .await?; + assert_eq!(refreshed, "this session cannot be refreshed"); + + // Once the row is gone, as when the monitor sweeps an expired one, the auth cache still + // accepts the token for a while. + sqlx::query("DELETE FROM token WHERE label LIKE 'impersonation:%'") + .execute(&db) + .await?; + assert_eq!(get("IMPERSONATION_TOKEN", "whoami").await?.status(), 200); + let refreshed = get("IMPERSONATION_TOKEN", "refresh_token") + .await? + .text() + .await?; + assert_eq!(refreshed, "this session cannot be refreshed"); + + let sessions: i64 = sqlx::query_scalar( + "SELECT count(*) FROM token WHERE email = 'test2@windmill.dev' AND label = 'session'", + ) + .fetch_one(&db) + .await?; + assert_eq!(sessions, 0); + + let refreshed = get("SECRET_TOKEN_3", "refresh_token").await?.text().await?; + assert_eq!( + refreshed, "token refreshed", + "an ordinary token still refreshes" + ); + + Ok(()) +} diff --git a/backend/windmill-api-users/src/users.rs b/backend/windmill-api-users/src/users.rs index 8d1242befa..ac9defcd53 100644 --- a/backend/windmill-api-users/src/users.rs +++ b/backend/windmill-api-users/src/users.rs @@ -2866,8 +2866,26 @@ async fn refresh_token( .to_string(), )); } + let t_hash = windmill_common::auth::hash_token(&token); + // Only a live, non-impersonation row may be exchanged for a session: an impersonation must + // end at its 24h expiry, not become a renewable service-account session billed as active. + // Read the row rather than trust `authed`, which the auth cache keeps serving for up to + // 120s after the row expired or was swept. `jwt_` tokens have no row. + if !token.starts_with("jwt_") { + let refreshable = sqlx::query_scalar!( + "SELECT NOT starts_with(COALESCE(label, ''), 'impersonation:') FROM token + WHERE token_hash = $1 AND (expiration IS NULL OR expiration > now())", + &t_hash + ) + .fetch_optional(&db) + .await? + .flatten() + .unwrap_or(false); + if !refreshable { + return Ok("this session cannot be refreshed".to_string()); + } + } if let Some(thresh_s) = query.if_expiring_in_less_than_s { - let t_hash = windmill_common::auth::hash_token(&token); let not_expired = sqlx::query_scalar!("SELECT true FROM token WHERE token_hash = $1 and expiration IS NOT NULL and expiration > now() + $2::int * '1 sec'::interval", &t_hash, thresh_s) .fetch_optional(&db) .await? diff --git a/backend/windmill-api/openapi.yaml b/backend/windmill-api/openapi.yaml index e7afd78ceb..bfdbb05d65 100644 --- a/backend/windmill-api/openapi.yaml +++ b/backend/windmill-api/openapi.yaml @@ -1,7 +1,7 @@ openapi: "3.0.3" info: - version: 1.819.0 + version: 1.820.0 title: Windmill API contact: diff --git a/backend/windmill-api/src/apps.rs b/backend/windmill-api/src/apps.rs index ef48927676..f01275c953 100644 --- a/backend/windmill-api/src/apps.rs +++ b/backend/windmill-api/src/apps.rs @@ -4362,12 +4362,10 @@ async fn execute_component( let resolved_delete_secs = resolve_delete_after_secs(None, policy_triggerables.delete_after_secs); - // `_MODULES` and `_TEMP_SCRIPT_REFS` are server-injected control keys (into - // `extra`) that the worker reads back for a `Preview` job — which an inline run - // is. A caller supplying them in `args` would inject module content/locks or - // redirect relative-import resolution, unpinned, as the app identity. Drop them; - // legitimate values ride in `extra`, never the request `args`. - payload.args.remove("_MODULES"); + // `_TEMP_SCRIPT_REFS` is a server-injected control key (into `extra`) that the + // worker reads back for a `Preview` job, which an inline run is. A caller + // supplying it in `args` would redirect relative-import resolution, unpinned, as + // the app identity. Drop it; the legitimate value rides in `extra`. payload.args.remove("_TEMP_SCRIPT_REFS"); let (mut args, job_id) = build_args( diff --git a/backend/windmill-api/src/jobs.rs b/backend/windmill-api/src/jobs.rs index fe73e216bf..8b63289175 100644 --- a/backend/windmill-api/src/jobs.rs +++ b/backend/windmill-api/src/jobs.rs @@ -7152,7 +7152,7 @@ pub async fn restart_flow( let completed_job = sqlx::query!( "SELECT j.runnable_path as script_path, j.args AS \"args: sqlx::types::Json>>\", - j.tag AS \"tag!\", j.priority + j.tag AS \"tag!\", j.priority, j.kind AS \"kind!: JobKind\" FROM v2_job j WHERE j.id = $1 and j.workspace_id = $2", job_id, @@ -7169,6 +7169,13 @@ pub async fn restart_flow( check_scopes(&authed, || format!("jobs:run:flows:{flow_path}"))?; let mut run_query = run_query; drop_unclaimable_run_lineage(&db, &w_id, &mut run_query, &authed).await?; + // A restarted flow preview reruns the value its request supplied, while a flow preview + // under a parent is read (by job provenance) as that parent's own definition. + let (parent_job, root_job) = if completed_job.kind == JobKind::FlowPreview { + (None, None) + } else { + (run_query.parent_job, run_query.root_job) + }; let ehm = HashMap::new(); let push_args = completed_job @@ -7212,9 +7219,9 @@ pub async fn restart_flow( authed.username_override.as_deref(), scheduled_for, None, - run_query.parent_job, + parent_job, None, - run_query.root_job, + root_job, run_query.job_id, false, false, @@ -7414,6 +7421,20 @@ pub async fn run_workflow_as_code( ) .await?; + // A task re-runs a preview with the preview's modules: the ones `push` stored for it, never + // the task's args. Read on their own, as `fetch_queued` swaps oversized args for a placeholder. + let preview_modules = if job.job_kind == JobKind::Preview { + sqlx::query_scalar::<_, Option>>>( + "SELECT args->'_MODULES' FROM v2_job WHERE id = $1", + ) + .bind(job.id) + .fetch_one(&db) + .await? + .map(|modules| modules.0) + } else { + None + }; + let (job_payload, tag, _delete_after_use, _delete_after_secs, timeout, on_behalf_of) = match job.job_kind { JobKind::Preview => ( @@ -7437,7 +7458,7 @@ pub async fn run_workflow_as_code( dedicated_worker: None, // TODO(debouncing): enable for this mode debouncing_settings: DebouncingSettings::default(), - modules: None, + modules: preview_modules, tag: None, }), Some(job.tag.clone()), @@ -8623,9 +8644,6 @@ async fn run_preview_script( if let Some(fp) = &preview.flow_path { extra.insert("_FLOW_PATH".to_string(), to_raw_value(fp)); } - if let Some(ref modules) = preview.modules { - extra.insert("_MODULES".to_string(), to_raw_value(modules)); - } if let Some(ref temp_script_refs) = preview.temp_script_refs { extra.insert( "_TEMP_SCRIPT_REFS".to_string(), diff --git a/backend/windmill-common/src/job_provenance.rs b/backend/windmill-common/src/job_provenance.rs index 1e4130de8c..e3dfdbc205 100644 --- a/backend/windmill-common/src/job_provenance.rs +++ b/backend/windmill-common/src/job_provenance.rs @@ -7,6 +7,15 @@ pub struct JobProvenance { /// True only when the job and every ancestor run code at a path taken from a /// deployed version, never from the request that created the job. pub deployed: bool, + /// The chain reached a parent it cannot read (other workspace, depth cap), so + /// nothing above that point is known. + pub unproven_ancestry: bool, + /// The item paths the chain took from a request, top-most first: for the job to act as + /// them, its identity must be one that could have deployed there. A job whose code + /// and path both derive from a claimed parent adds no claim of its own. + pub claimed_paths: Vec, + /// The identity the job runs as. + pub permissioned_as: String, pub parent_path: Option, pub root_id: Uuid, pub root_path: Option, @@ -14,13 +23,36 @@ pub struct JobProvenance { pub root_trigger_kind: Option, } +#[derive(Debug, PartialEq, Eq)] +pub struct ClaimedPath { + /// The deployable item the claim stands for: an app script's claim is its app. + pub path: String, + pub item: ClaimedItem, + /// The identity of the job that made the claim: a deployed child can run as another + /// identity (`on_behalf_of`) than the preview it runs under. + pub permissioned_as: String, +} + +#[derive(Debug, PartialEq, Eq, Clone, Copy)] +pub enum ClaimedItem { + Script, + /// A preview without a parent at `/`: deploying the app runs its + /// inline script at that path with the same identity, so writing the app suffices. + ScriptOrAppComponent, + Flow, + App, +} + struct LineageJob { id: Uuid, parent_job: Option, kind: JobKind, runnable_path: Option, trigger_kind: Option, + permissioned_as: String, origin_verified: bool, + app_stamped: bool, + args_modules: bool, } /// Reads any job of `w_id` regardless of the caller: authorize access to `job_id` first. @@ -29,16 +61,16 @@ pub async fn job_provenance(db: &DB, job_id: &Uuid, w_id: &str) -> Result