diff --git a/.github/workflows/backend-test-windows.yml b/.github/workflows/backend-test-windows.yml index c43e4ece79..e2e7318ab6 100644 --- a/.github/workflows/backend-test-windows.yml +++ b/.github/workflows/backend-test-windows.yml @@ -179,15 +179,14 @@ jobs: # binary link spikes several hundred MB of transient I/O. Capping at # 8 trades ~25% wall time for headroom on the ~75GB runner disk. CARGO_BUILD_JOBS: 8 - # backend/Cargo.toml leaves profile.dev at the default debug = 2 for - # the (large) windmill workspace crates; that debuginfo is emitted - # into every object file and embedded in each test binary, and on - # windows-msvc also spawns the mspdbsrv.exe PDB type server. Across a - # full --all --features build it is the dominant consumer of the - # ~63GB free on the runner disk (LNK1180 / disk-full during linking). - # CI needs no debug info, so drop it entirely for the dev/test - # profiles here. debug = 0 supersedes the previous split-debuginfo=off - # knob (no debuginfo => no .pdb and no LNK1318 type-server limit). + # backend/Cargo.toml keeps line tables on profile.dev for the (large) + # windmill workspace crates; that debuginfo is emitted into every + # object file and embedded in each test binary, and on windows-msvc + # also spawns the mspdbsrv.exe PDB type server. Across the worker + # crates' test build it drives the peak on the ~63GB free of the + # runner disk (LNK1180 / disk-full during linking). CI reads no + # backtraces, so drop it entirely for the dev/test profiles here: + # debug = 0 means no .pdb and no LNK1318 type-server limit. CARGO_PROFILE_DEV_DEBUG: "0" CARGO_PROFILE_TEST_DEBUG: "0" # Tests' poll-time stack frames (deep nested async fn chains in diff --git a/.github/workflows/backend-test.yml b/.github/workflows/backend-test.yml index 5b6a461668..cdd0e2a212 100644 --- a/.github/workflows/backend-test.yml +++ b/.github/workflows/backend-test.yml @@ -268,13 +268,13 @@ jobs: # overhead and extra disk. Off here (kept on for local dev via # .cargo/config.toml). Matches backend-test-windows.yml. CARGO_INCREMENTAL: "0" - # backend/Cargo.toml leaves profile.dev at the default debug = 2 for - # the (large) windmill workspace crates; that debug info is emitted - # into every object file and embedded in each test binary. Across the - # full --all --features build it is the dominant memory/disk consumer - # when mold links the windmill-api-integration-tests binary, tipping - # the runner over (lost runner reported as a canceled step). CI needs - # no debug info, so drop it entirely for the dev/test profiles here. + # backend/Cargo.toml keeps line tables on profile.dev for the (large) + # windmill workspace crates; that debug info is emitted into every + # object file and embedded in each test binary. Across the full + # --all --features build it drives the memory/disk peak when mold + # links the windmill-api-integration-tests binary, tipping the runner + # over (lost runner reported as a canceled step). CI reads no + # backtraces, so drop it entirely for the dev/test profiles here. # (test profile inherits dev, but the workspace crates link in as # dev-profile deps, so both must be set.) CI-only; local dev builds # are unaffected. diff --git a/.release-please-manifest.json b/.release-please-manifest.json index e2684ac636..ae93300697 100644 --- a/.release-please-manifest.json +++ b/.release-please-manifest.json @@ -1,3 +1,3 @@ { - ".": "1.795.0" + ".": "1.796.0" } diff --git a/CHANGELOG.md b/CHANGELOG.md index c2866df6ad..947e63f5c6 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,25 @@ # Changelog +## [1.796.0](https://github.com/windmill-labs/windmill/compare/v1.795.0...v1.796.0) (2026-08-24) + + +### Features + +* add instance setting to mute zombie job restart alerts ([#10813](https://github.com/windmill-labs/windmill/issues/10813)) ([2906504](https://github.com/windmill-labs/windmill/commit/2906504125136afa280884b55e2f877a29a466a9)) +* AI agent evals: datasets, scored runs and comparison ([#10633](https://github.com/windmill-labs/windmill/issues/10633)) ([9c55785](https://github.com/windmill-labs/windmill/commit/9c557859c5ffede921690cd3d224239b9305c9b8)) +* **datatables:** add a down migration from the migration viewer ([#10812](https://github.com/windmill-labs/windmill/issues/10812)) ([3b2a6d7](https://github.com/windmill-labs/windmill/commit/3b2a6d76045cf5ae48ddfb76b871e24fd889298a)) +* **frontend:** warn when COEP blocks cross-origin resources in raw app editor preview ([#10328](https://github.com/windmill-labs/windmill/issues/10328)) ([7751d3e](https://github.com/windmill-labs/windmill/commit/7751d3e43ee1abbba9a6cca026be78504f0c5dff)) +* track token cost in AI sessions and chats ([#10688](https://github.com/windmill-labs/windmill/issues/10688)) ([b6e0591](https://github.com/windmill-labs/windmill/commit/b6e059116aa55fa5aa1226f5b3300bb2c8683f1a)) + + +### Bug Fixes + +* **ai-chat:** keep the composer usable while a question is pending ([#10816](https://github.com/windmill-labs/windmill/issues/10816)) ([25a3e6e](https://github.com/windmill-labs/windmill/commit/25a3e6ea7a7efb29ea3868ac0d1453d074325717)) +* **frontend:** mint string password secrets in the operating workspace ([#10815](https://github.com/windmill-labs/windmill/issues/10815)) ([93081e2](https://github.com/windmill-labs/windmill/commit/93081e255f06386c3e21a838a752e5302ad6e6fe)) +* keep ai chat messages when leaving the page mid-generation ([#10809](https://github.com/windmill-labs/windmill/issues/10809)) ([541b6c8](https://github.com/windmill-labs/windmill/commit/541b6c849657d13fed3580407a00a996e891ad9e)) +* patch sqlx so a cancelled BEGIN cannot poison a pooled connection ([#10823](https://github.com/windmill-labs/windmill/issues/10823)) ([8dbd12e](https://github.com/windmill-labs/windmill/commit/8dbd12ecc1a8d0c03ba32b797c5a8cd9ee2d57b4)) +* qualify foreign key targets in generated datatable migrations ([#10821](https://github.com/windmill-labs/windmill/issues/10821)) ([29c311a](https://github.com/windmill-labs/windmill/commit/29c311ab318f7e443d696baf871b8a0558fa8524)) + ## [1.795.0](https://github.com/windmill-labs/windmill/compare/v1.794.1...v1.795.0) (2026-08-22) diff --git a/backend/.sqlx/query-01bce88dd622f314d1a09c24cd12df5e7d3ff6a15a93e1c0c95e262f7b3d0ef1.json b/backend/.sqlx/query-01bce88dd622f314d1a09c24cd12df5e7d3ff6a15a93e1c0c95e262f7b3d0ef1.json new file mode 100644 index 0000000000..64bf73746f --- /dev/null +++ b/backend/.sqlx/query-01bce88dd622f314d1a09c24cd12df5e7d3ff6a15a93e1c0c95e262f7b3d0ef1.json @@ -0,0 +1,17 @@ +{ + "db_name": "PostgreSQL", + "query": "UPDATE eval_experiment\n SET subject = jsonb_set(\n jsonb_set(subject, '{kind}', '\"agent\"'),\n '{version}', to_jsonb($4::bigint))\n WHERE workspace_id = $1 AND dataset_path = $2 AND id = $3\n AND subject ->> 'kind' = 'agent_draft'", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "Text", + "Text", + "Uuid", + "Int8" + ] + }, + "nullable": [] + }, + "hash": "01bce88dd622f314d1a09c24cd12df5e7d3ff6a15a93e1c0c95e262f7b3d0ef1" +} diff --git a/backend/.sqlx/query-0276e6030abb2eb00a68c568a9cc60f3e7c2af0331388c4b358035de865a121a.json b/backend/.sqlx/query-0276e6030abb2eb00a68c568a9cc60f3e7c2af0331388c4b358035de865a121a.json new file mode 100644 index 0000000000..a7452b0602 --- /dev/null +++ b/backend/.sqlx/query-0276e6030abb2eb00a68c568a9cc60f3e7c2af0331388c4b358035de865a121a.json @@ -0,0 +1,35 @@ +{ + "db_name": "PostgreSQL", + "query": "SELECT path, value, version FROM resource_version WHERE workspace_id = $1 AND id = $2", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "path", + "type_info": "Varchar" + }, + { + "ordinal": 1, + "name": "value", + "type_info": "Jsonb" + }, + { + "ordinal": 2, + "name": "version", + "type_info": "Int8" + } + ], + "parameters": { + "Left": [ + "Text", + "Int8" + ] + }, + "nullable": [ + false, + true, + false + ] + }, + "hash": "0276e6030abb2eb00a68c568a9cc60f3e7c2af0331388c4b358035de865a121a" +} diff --git a/backend/.sqlx/query-0335de6713de6678b9bf266121af23abc46d5db95da095bb15726c5a2db7ad2f.json b/backend/.sqlx/query-0335de6713de6678b9bf266121af23abc46d5db95da095bb15726c5a2db7ad2f.json new file mode 100644 index 0000000000..f56ef7ead0 --- /dev/null +++ b/backend/.sqlx/query-0335de6713de6678b9bf266121af23abc46d5db95da095bb15726c5a2db7ad2f.json @@ -0,0 +1,24 @@ +{ + "db_name": "PostgreSQL", + "query": "SELECT coalesce(max(run_number), 0) + 1 FROM eval_experiment\n WHERE workspace_id = $1 AND dataset_path = $2 AND subject ->> 'path' = $3", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "?column?", + "type_info": "Int4" + } + ], + "parameters": { + "Left": [ + "Text", + "Text", + "Text" + ] + }, + "nullable": [ + null + ] + }, + "hash": "0335de6713de6678b9bf266121af23abc46d5db95da095bb15726c5a2db7ad2f" +} diff --git a/backend/.sqlx/query-0aae275d9196e742b5783df4e67c72459d45e275bfeafa2952349cae259ac9f0.json b/backend/.sqlx/query-0aae275d9196e742b5783df4e67c72459d45e275bfeafa2952349cae259ac9f0.json new file mode 100644 index 0000000000..25e389fd30 --- /dev/null +++ b/backend/.sqlx/query-0aae275d9196e742b5783df4e67c72459d45e275bfeafa2952349cae259ac9f0.json @@ -0,0 +1,16 @@ +{ + "db_name": "PostgreSQL", + "query": "UPDATE eval_dataset SET scorers = COALESCE((\n SELECT jsonb_agg(\n CASE WHEN elem->>'path' LIKE ('u/' || $2 || '/%')\n THEN jsonb_set(elem, '{path}', to_jsonb(REGEXP_REPLACE(elem->>'path', 'u/' || $2 || '/(.*)', $1 || '/\\1')))\n ELSE elem END)\n FROM jsonb_array_elements(scorers) elem), '[]'::jsonb)\n WHERE workspace_id = $3\n AND EXISTS (SELECT 1 FROM jsonb_array_elements(scorers) e WHERE e->>'path' LIKE ('u/' || $2 || '/%'))", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "Text", + "Text", + "Text" + ] + }, + "nullable": [] + }, + "hash": "0aae275d9196e742b5783df4e67c72459d45e275bfeafa2952349cae259ac9f0" +} diff --git a/backend/.sqlx/query-0d6700ccffb8179e365bbc1f03398e474f23b013f642ba29ad6f68e6f047c1e5.json b/backend/.sqlx/query-0d6700ccffb8179e365bbc1f03398e474f23b013f642ba29ad6f68e6f047c1e5.json new file mode 100644 index 0000000000..2ec3036fb0 --- /dev/null +++ b/backend/.sqlx/query-0d6700ccffb8179e365bbc1f03398e474f23b013f642ba29ad6f68e6f047c1e5.json @@ -0,0 +1,24 @@ +{ + "db_name": "PostgreSQL", + "query": "SELECT pg_advisory_xact_lock(hashtext('ai_eval_open:' || $1 || '/' || $2 || '/' || $3))", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "pg_advisory_xact_lock", + "type_info": "Void" + } + ], + "parameters": { + "Left": [ + "Text", + "Text", + "Text" + ] + }, + "nullable": [ + null + ] + }, + "hash": "0d6700ccffb8179e365bbc1f03398e474f23b013f642ba29ad6f68e6f047c1e5" +} diff --git a/backend/.sqlx/query-1815730982dcaf7239ddcb22f88ae5c79794213cf6278167f8afdbca30b1b15c.json b/backend/.sqlx/query-1815730982dcaf7239ddcb22f88ae5c79794213cf6278167f8afdbca30b1b15c.json new file mode 100644 index 0000000000..a8134921e5 --- /dev/null +++ b/backend/.sqlx/query-1815730982dcaf7239ddcb22f88ae5c79794213cf6278167f8afdbca30b1b15c.json @@ -0,0 +1,16 @@ +{ + "db_name": "PostgreSQL", + "query": "UPDATE eval_experiment SET subject = jsonb_set(subject, '{path}', to_jsonb(REGEXP_REPLACE(subject->>'path', 'u/' || $2 || '/(.*)', $1 || '/\\1'))) WHERE subject->>'path' LIKE ('u/' || $2 || '/%') AND workspace_id = $3", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "Text", + "Text", + "Text" + ] + }, + "nullable": [] + }, + "hash": "1815730982dcaf7239ddcb22f88ae5c79794213cf6278167f8afdbca30b1b15c" +} diff --git a/backend/.sqlx/query-186c663249ffada82abf61ce214f52e2730501774a0ca4dc855380e6c6487917.json b/backend/.sqlx/query-186c663249ffada82abf61ce214f52e2730501774a0ca4dc855380e6c6487917.json new file mode 100644 index 0000000000..5cead4df77 --- /dev/null +++ b/backend/.sqlx/query-186c663249ffada82abf61ce214f52e2730501774a0ca4dc855380e6c6487917.json @@ -0,0 +1,22 @@ +{ + "db_name": "PostgreSQL", + "query": "SELECT count(*) AS \"count!\" FROM eval_experiment_case\n WHERE experiment_id = $1 AND status IS NOT NULL", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "count!", + "type_info": "Int8" + } + ], + "parameters": { + "Left": [ + "Uuid" + ] + }, + "nullable": [ + null + ] + }, + "hash": "186c663249ffada82abf61ce214f52e2730501774a0ca4dc855380e6c6487917" +} diff --git a/backend/.sqlx/query-196939257a334f7d37aa6d66153b251446d7701a893cb7852b81cf842c0fa228.json b/backend/.sqlx/query-196939257a334f7d37aa6d66153b251446d7701a893cb7852b81cf842c0fa228.json new file mode 100644 index 0000000000..d9643875cf --- /dev/null +++ b/backend/.sqlx/query-196939257a334f7d37aa6d66153b251446d7701a893cb7852b81cf842c0fa228.json @@ -0,0 +1,59 @@ +{ + "db_name": "PostgreSQL", + "query": "SELECT path, summary, scorers, created_at, created_by,\n edited_at, edited_by\n FROM eval_dataset WHERE workspace_id = $1 AND path = $2", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "path", + "type_info": "Varchar" + }, + { + "ordinal": 1, + "name": "summary", + "type_info": "Varchar" + }, + { + "ordinal": 2, + "name": "scorers", + "type_info": "Jsonb" + }, + { + "ordinal": 3, + "name": "created_at", + "type_info": "Timestamptz" + }, + { + "ordinal": 4, + "name": "created_by", + "type_info": "Varchar" + }, + { + "ordinal": 5, + "name": "edited_at", + "type_info": "Timestamptz" + }, + { + "ordinal": 6, + "name": "edited_by", + "type_info": "Varchar" + } + ], + "parameters": { + "Left": [ + "Text", + "Text" + ] + }, + "nullable": [ + false, + true, + false, + false, + false, + false, + false + ] + }, + "hash": "196939257a334f7d37aa6d66153b251446d7701a893cb7852b81cf842c0fa228" +} diff --git a/backend/.sqlx/query-1b6e229545f6b877e72d21728257d1bddaba15ef0fbe72bb4f43b45140f184ce.json b/backend/.sqlx/query-1b6e229545f6b877e72d21728257d1bddaba15ef0fbe72bb4f43b45140f184ce.json new file mode 100644 index 0000000000..54e381be1a --- /dev/null +++ b/backend/.sqlx/query-1b6e229545f6b877e72d21728257d1bddaba15ef0fbe72bb4f43b45140f184ce.json @@ -0,0 +1,28 @@ +{ + "db_name": "PostgreSQL", + "query": "SELECT case_id, ordinal FROM eval_experiment_case WHERE experiment_id = $1", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "case_id", + "type_info": "Uuid" + }, + { + "ordinal": 1, + "name": "ordinal", + "type_info": "Int4" + } + ], + "parameters": { + "Left": [ + "Uuid" + ] + }, + "nullable": [ + false, + false + ] + }, + "hash": "1b6e229545f6b877e72d21728257d1bddaba15ef0fbe72bb4f43b45140f184ce" +} diff --git a/backend/.sqlx/query-1db80f3ba2c6c769a98424ebf9aaf168a4fa2c64e446a824038cf267236fe979.json b/backend/.sqlx/query-1db80f3ba2c6c769a98424ebf9aaf168a4fa2c64e446a824038cf267236fe979.json new file mode 100644 index 0000000000..01bae0f126 --- /dev/null +++ b/backend/.sqlx/query-1db80f3ba2c6c769a98424ebf9aaf168a4fa2c64e446a824038cf267236fe979.json @@ -0,0 +1,23 @@ +{ + "db_name": "PostgreSQL", + "query": "SELECT DISTINCT path FROM script\n WHERE workspace_id = $1 AND path = ANY($2)\n AND deleted = false AND lock IS NOT NULL AND lock_error_logs IS NULL", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "path", + "type_info": "Varchar" + } + ], + "parameters": { + "Left": [ + "Text", + "TextArray" + ] + }, + "nullable": [ + false + ] + }, + "hash": "1db80f3ba2c6c769a98424ebf9aaf168a4fa2c64e446a824038cf267236fe979" +} diff --git a/backend/.sqlx/query-242845c86084e010ab33c2197d44af9aeb181672a2f2330dbe65bfe586376450.json b/backend/.sqlx/query-242845c86084e010ab33c2197d44af9aeb181672a2f2330dbe65bfe586376450.json new file mode 100644 index 0000000000..38dcbadfb0 --- /dev/null +++ b/backend/.sqlx/query-242845c86084e010ab33c2197d44af9aeb181672a2f2330dbe65bfe586376450.json @@ -0,0 +1,15 @@ +{ + "db_name": "PostgreSQL", + "query": "UPDATE eval_score SET error = 'The case did not run'\n WHERE experiment_id = $1 AND ordinal = ANY($2)\n AND score IS NULL AND error IS NULL AND NOT not_applicable", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "Uuid", + "Int4Array" + ] + }, + "nullable": [] + }, + "hash": "242845c86084e010ab33c2197d44af9aeb181672a2f2330dbe65bfe586376450" +} diff --git a/backend/.sqlx/query-24fcc2b69f30953915f0cbf246e1c19b2310075a644d81c21784c991e52b4001.json b/backend/.sqlx/query-24fcc2b69f30953915f0cbf246e1c19b2310075a644d81c21784c991e52b4001.json new file mode 100644 index 0000000000..a2cd96857a --- /dev/null +++ b/backend/.sqlx/query-24fcc2b69f30953915f0cbf246e1c19b2310075a644d81c21784c991e52b4001.json @@ -0,0 +1,24 @@ +{ + "db_name": "PostgreSQL", + "query": "INSERT INTO ai_token_usage (workspace_id, email, provider, model, session_id, input_tokens, cache_read_tokens, cache_write_tokens, output_tokens, reported_cost_nano_usd, requests)\n SELECT $1, $2, * FROM UNNEST($3::text[], $4::text[], $5::text[], $6::bigint[], $7::bigint[], $8::bigint[], $9::bigint[], $10::bigint[], $11::bigint[])\n ON CONFLICT (workspace_id, day, email, provider, model, session_id)\n DO UPDATE SET\n input_tokens = ai_token_usage.input_tokens + EXCLUDED.input_tokens,\n cache_read_tokens = ai_token_usage.cache_read_tokens + EXCLUDED.cache_read_tokens,\n cache_write_tokens = ai_token_usage.cache_write_tokens + EXCLUDED.cache_write_tokens,\n output_tokens = ai_token_usage.output_tokens + EXCLUDED.output_tokens,\n reported_cost_nano_usd = CASE\n WHEN EXCLUDED.reported_cost_nano_usd IS NULL\n THEN ai_token_usage.reported_cost_nano_usd\n ELSE COALESCE(ai_token_usage.reported_cost_nano_usd, 0)\n + EXCLUDED.reported_cost_nano_usd\n END,\n requests = ai_token_usage.requests + EXCLUDED.requests,\n updated_at = now()", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "Varchar", + "Varchar", + "TextArray", + "TextArray", + "TextArray", + "Int8Array", + "Int8Array", + "Int8Array", + "Int8Array", + "Int8Array", + "Int8Array" + ] + }, + "nullable": [] + }, + "hash": "24fcc2b69f30953915f0cbf246e1c19b2310075a644d81c21784c991e52b4001" +} diff --git a/backend/.sqlx/query-2b41dc4d872af0e230c31bef1a496604c1562cdfb0bdb54bd3b0c6579d73e46d.json b/backend/.sqlx/query-2b41dc4d872af0e230c31bef1a496604c1562cdfb0bdb54bd3b0c6579d73e46d.json new file mode 100644 index 0000000000..4506a038fc --- /dev/null +++ b/backend/.sqlx/query-2b41dc4d872af0e230c31bef1a496604c1562cdfb0bdb54bd3b0c6579d73e46d.json @@ -0,0 +1,28 @@ +{ + "db_name": "PostgreSQL", + "query": "SELECT path, scorers FROM eval_dataset\n WHERE workspace_id = $1 ORDER BY edited_at DESC LIMIT 100", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "path", + "type_info": "Varchar" + }, + { + "ordinal": 1, + "name": "scorers", + "type_info": "Jsonb" + } + ], + "parameters": { + "Left": [ + "Text" + ] + }, + "nullable": [ + false, + false + ] + }, + "hash": "2b41dc4d872af0e230c31bef1a496604c1562cdfb0bdb54bd3b0c6579d73e46d" +} diff --git a/backend/.sqlx/query-c5639d48c92d6863f16f97de91eae590d0ed2f4b7831956bb429a47c478f7c1f.json b/backend/.sqlx/query-2b93351b9c2c91272c3f69c9c8eba192d2e4d087c7d7db8d3b257bb864c8265d.json similarity index 59% rename from backend/.sqlx/query-c5639d48c92d6863f16f97de91eae590d0ed2f4b7831956bb429a47c478f7c1f.json rename to backend/.sqlx/query-2b93351b9c2c91272c3f69c9c8eba192d2e4d087c7d7db8d3b257bb864c8265d.json index fa1396235f..b82fcf2e17 100644 --- a/backend/.sqlx/query-c5639d48c92d6863f16f97de91eae590d0ed2f4b7831956bb429a47c478f7c1f.json +++ b/backend/.sqlx/query-2b93351b9c2c91272c3f69c9c8eba192d2e4d087c7d7db8d3b257bb864c8265d.json @@ -1,6 +1,6 @@ { "db_name": "PostgreSQL", - "query": "INSERT INTO datatable_migrations (workspace_id, datatable, timestamp, name, code_up, code_down) VALUES ($1, $2, $3, $4, $5, $6) ON CONFLICT (workspace_id, datatable, timestamp) DO UPDATE SET name = EXCLUDED.name, code_up = EXCLUDED.code_up, code_down = EXCLUDED.code_down", + "query": "INSERT INTO datatable_migrations (workspace_id, datatable, timestamp, name, code_up, code_down) VALUES ($1, $2, $3, $4, $5, $6) ON CONFLICT (workspace_id, datatable, timestamp) DO UPDATE SET name = EXCLUDED.name, code_up = EXCLUDED.code_up, code_down = EXCLUDED.code_down WHERE NOT $7 OR (datatable_migrations.name = $8::text AND datatable_migrations.code_up = $9::text AND datatable_migrations.code_down IS NOT DISTINCT FROM $10::text)", "describe": { "columns": [], "parameters": { @@ -10,10 +10,14 @@ "Int8", "Varchar", "Text", + "Text", + "Bool", + "Text", + "Text", "Text" ] }, "nullable": [] }, - "hash": "c5639d48c92d6863f16f97de91eae590d0ed2f4b7831956bb429a47c478f7c1f" + "hash": "2b93351b9c2c91272c3f69c9c8eba192d2e4d087c7d7db8d3b257bb864c8265d" } diff --git a/backend/.sqlx/query-307d5b797e51122dbf087e5dc9f33235a349731828d500c74a28b577469e2624.json b/backend/.sqlx/query-307d5b797e51122dbf087e5dc9f33235a349731828d500c74a28b577469e2624.json new file mode 100644 index 0000000000..2cfec01e9b --- /dev/null +++ b/backend/.sqlx/query-307d5b797e51122dbf087e5dc9f33235a349731828d500c74a28b577469e2624.json @@ -0,0 +1,23 @@ +{ + "db_name": "PostgreSQL", + "query": "SELECT path FROM resource WHERE workspace_id = $1 AND path = ANY($2) AND resource_type = 'ai_agent'", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "path", + "type_info": "Varchar" + } + ], + "parameters": { + "Left": [ + "Text", + "TextArray" + ] + }, + "nullable": [ + false + ] + }, + "hash": "307d5b797e51122dbf087e5dc9f33235a349731828d500c74a28b577469e2624" +} diff --git a/backend/.sqlx/query-30b8590939bf3d6770cabca9f4ab3527e750d85d3b8cc0eadef55461de4a2687.json b/backend/.sqlx/query-30b8590939bf3d6770cabca9f4ab3527e750d85d3b8cc0eadef55461de4a2687.json new file mode 100644 index 0000000000..d3a443a0bf --- /dev/null +++ b/backend/.sqlx/query-30b8590939bf3d6770cabca9f4ab3527e750d85d3b8cc0eadef55461de4a2687.json @@ -0,0 +1,23 @@ +{ + "db_name": "PostgreSQL", + "query": "SELECT path FROM eval_dataset WHERE path LIKE $1 AND workspace_id = $2", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "path", + "type_info": "Varchar" + } + ], + "parameters": { + "Left": [ + "Text", + "Text" + ] + }, + "nullable": [ + false + ] + }, + "hash": "30b8590939bf3d6770cabca9f4ab3527e750d85d3b8cc0eadef55461de4a2687" +} diff --git a/backend/.sqlx/query-316e7c86082b6ee2864b88674cfac5b5b34c5de851f1c6f832b868101edcb052.json b/backend/.sqlx/query-316e7c86082b6ee2864b88674cfac5b5b34c5de851f1c6f832b868101edcb052.json new file mode 100644 index 0000000000..a3443236ea --- /dev/null +++ b/backend/.sqlx/query-316e7c86082b6ee2864b88674cfac5b5b34c5de851f1c6f832b868101edcb052.json @@ -0,0 +1,23 @@ +{ + "db_name": "PostgreSQL", + "query": "SELECT version FROM resource_version WHERE workspace_id = $1 AND path = $2\n ORDER BY version DESC LIMIT 1", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "version", + "type_info": "Int8" + } + ], + "parameters": { + "Left": [ + "Text", + "Text" + ] + }, + "nullable": [ + false + ] + }, + "hash": "316e7c86082b6ee2864b88674cfac5b5b34c5de851f1c6f832b868101edcb052" +} diff --git a/backend/.sqlx/query-34fbb2b141ad691e0cdc55bf2ebd6974e02a7c81d91217d3f74e03113edc9b0a.json b/backend/.sqlx/query-34fbb2b141ad691e0cdc55bf2ebd6974e02a7c81d91217d3f74e03113edc9b0a.json new file mode 100644 index 0000000000..c79145adaf --- /dev/null +++ b/backend/.sqlx/query-34fbb2b141ad691e0cdc55bf2ebd6974e02a7c81d91217d3f74e03113edc9b0a.json @@ -0,0 +1,16 @@ +{ + "db_name": "PostgreSQL", + "query": "DELETE FROM eval_case\n WHERE workspace_id = $1 AND dataset_path = $2 AND NOT (id = ANY($3))", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "Text", + "Text", + "UuidArray" + ] + }, + "nullable": [] + }, + "hash": "34fbb2b141ad691e0cdc55bf2ebd6974e02a7c81d91217d3f74e03113edc9b0a" +} diff --git a/backend/.sqlx/query-3ca2f72d2917f48644cb79daba9858f472be9f1bb8e504cc3dc2eb998e3f7b91.json b/backend/.sqlx/query-3ca2f72d2917f48644cb79daba9858f472be9f1bb8e504cc3dc2eb998e3f7b91.json new file mode 100644 index 0000000000..fd5e4325a0 --- /dev/null +++ b/backend/.sqlx/query-3ca2f72d2917f48644cb79daba9858f472be9f1bb8e504cc3dc2eb998e3f7b91.json @@ -0,0 +1,16 @@ +{ + "db_name": "PostgreSQL", + "query": "UPDATE eval_experiment SET created_by = $1 WHERE created_by = $2 AND workspace_id = $3", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "Varchar", + "Text", + "Text" + ] + }, + "nullable": [] + }, + "hash": "3ca2f72d2917f48644cb79daba9858f472be9f1bb8e504cc3dc2eb998e3f7b91" +} diff --git a/backend/.sqlx/query-44a39475ba202bd5852b666335308360569aae717eff3fe9cc9253a261ae2fa7.json b/backend/.sqlx/query-44a39475ba202bd5852b666335308360569aae717eff3fe9cc9253a261ae2fa7.json new file mode 100644 index 0000000000..cf0806757d --- /dev/null +++ b/backend/.sqlx/query-44a39475ba202bd5852b666335308360569aae717eff3fe9cc9253a261ae2fa7.json @@ -0,0 +1,23 @@ +{ + "db_name": "PostgreSQL", + "query": "SELECT status::text AS \"status!\" FROM v2_job_completed WHERE id = $1 AND workspace_id = $2", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "status!", + "type_info": "Text" + } + ], + "parameters": { + "Left": [ + "Uuid", + "Text" + ] + }, + "nullable": [ + null + ] + }, + "hash": "44a39475ba202bd5852b666335308360569aae717eff3fe9cc9253a261ae2fa7" +} diff --git a/backend/.sqlx/query-4550ae568abf23045259f95195bf1a35f5dc8f6828c2f3df51520683af48ebbd.json b/backend/.sqlx/query-4550ae568abf23045259f95195bf1a35f5dc8f6828c2f3df51520683af48ebbd.json new file mode 100644 index 0000000000..fd2ab8fbee --- /dev/null +++ b/backend/.sqlx/query-4550ae568abf23045259f95195bf1a35f5dc8f6828c2f3df51520683af48ebbd.json @@ -0,0 +1,16 @@ +{ + "db_name": "PostgreSQL", + "query": "UPDATE eval_dataset SET path = REGEXP_REPLACE(path,'u/' || $2 || '/(.*)','u/' || $1 || '/\\1') WHERE path LIKE ('u/' || $2 || '/%') AND workspace_id = $3", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "Text", + "Text", + "Text" + ] + }, + "nullable": [] + }, + "hash": "4550ae568abf23045259f95195bf1a35f5dc8f6828c2f3df51520683af48ebbd" +} diff --git a/backend/.sqlx/query-467ed4d282af003dd3b0d9542caec73feba3141f7345ffd1a6554fe44a6a9171.json b/backend/.sqlx/query-467ed4d282af003dd3b0d9542caec73feba3141f7345ffd1a6554fe44a6a9171.json new file mode 100644 index 0000000000..5399d071a9 --- /dev/null +++ b/backend/.sqlx/query-467ed4d282af003dd3b0d9542caec73feba3141f7345ffd1a6554fe44a6a9171.json @@ -0,0 +1,24 @@ +{ + "db_name": "PostgreSQL", + "query": "WITH updated AS (\n UPDATE eval_dataset SET path = REGEXP_REPLACE(path, 'u/' || $2 || '/(.*)', $1 || '/\\1')\n WHERE path LIKE ('u/' || $2 || '/%') AND workspace_id = $3\n RETURNING 1\n ) SELECT COUNT(*) FROM updated", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "count", + "type_info": "Int8" + } + ], + "parameters": { + "Left": [ + "Text", + "Text", + "Text" + ] + }, + "nullable": [ + null + ] + }, + "hash": "467ed4d282af003dd3b0d9542caec73feba3141f7345ffd1a6554fe44a6a9171" +} diff --git a/backend/.sqlx/query-479427dc09ebacb80cb20d553e93a577bb85b389ab056909a326212565a338bc.json b/backend/.sqlx/query-479427dc09ebacb80cb20d553e93a577bb85b389ab056909a326212565a338bc.json new file mode 100644 index 0000000000..8dd3b1ec7d --- /dev/null +++ b/backend/.sqlx/query-479427dc09ebacb80cb20d553e93a577bb85b389ab056909a326212565a338bc.json @@ -0,0 +1,15 @@ +{ + "db_name": "PostgreSQL", + "query": "INSERT INTO eval_dataset (workspace_id, path, summary, scorers, extra_perms, created_at, created_by, edited_at, edited_by)\n SELECT $2, path, summary, scorers, extra_perms, created_at, created_by, edited_at, edited_by\n FROM eval_dataset WHERE workspace_id = $1", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "Text", + "Varchar" + ] + }, + "nullable": [] + }, + "hash": "479427dc09ebacb80cb20d553e93a577bb85b389ab056909a326212565a338bc" +} diff --git a/backend/.sqlx/query-4be42f447e10e420f2a909579398b01b5175e14eee47ca39fe63242a78da767e.json b/backend/.sqlx/query-4be42f447e10e420f2a909579398b01b5175e14eee47ca39fe63242a78da767e.json new file mode 100644 index 0000000000..2107f75f72 --- /dev/null +++ b/backend/.sqlx/query-4be42f447e10e420f2a909579398b01b5175e14eee47ca39fe63242a78da767e.json @@ -0,0 +1,16 @@ +{ + "db_name": "PostgreSQL", + "query": "UPDATE eval_case SET created_by = $1 WHERE created_by = $2 AND workspace_id = $3", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "Varchar", + "Text", + "Text" + ] + }, + "nullable": [] + }, + "hash": "4be42f447e10e420f2a909579398b01b5175e14eee47ca39fe63242a78da767e" +} diff --git a/backend/.sqlx/query-4bf7f1a0fd87e79bb789cee06a5da68e465ce7ce67e80770f22a36a9e8320657.json b/backend/.sqlx/query-4bf7f1a0fd87e79bb789cee06a5da68e465ce7ce67e80770f22a36a9e8320657.json new file mode 100644 index 0000000000..a4ddcb531a --- /dev/null +++ b/backend/.sqlx/query-4bf7f1a0fd87e79bb789cee06a5da68e465ce7ce67e80770f22a36a9e8320657.json @@ -0,0 +1,59 @@ +{ + "db_name": "PostgreSQL", + "query": "SELECT path, summary, scorers, created_at, created_by, edited_at, edited_by\n FROM eval_dataset WHERE workspace_id = $1 AND path = $2 FOR UPDATE", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "path", + "type_info": "Varchar" + }, + { + "ordinal": 1, + "name": "summary", + "type_info": "Varchar" + }, + { + "ordinal": 2, + "name": "scorers", + "type_info": "Jsonb" + }, + { + "ordinal": 3, + "name": "created_at", + "type_info": "Timestamptz" + }, + { + "ordinal": 4, + "name": "created_by", + "type_info": "Varchar" + }, + { + "ordinal": 5, + "name": "edited_at", + "type_info": "Timestamptz" + }, + { + "ordinal": 6, + "name": "edited_by", + "type_info": "Varchar" + } + ], + "parameters": { + "Left": [ + "Text", + "Text" + ] + }, + "nullable": [ + false, + true, + false, + false, + false, + false, + false + ] + }, + "hash": "4bf7f1a0fd87e79bb789cee06a5da68e465ce7ce67e80770f22a36a9e8320657" +} diff --git a/backend/.sqlx/query-e74e283951aa87627a46aa8286819cf7aa4fecbde17bc7a67ee6f49c427cee9e.json b/backend/.sqlx/query-4e4a9c7b6e95f81101a68a6cc6d29fe4424eaed36d5ac9e4cd0a2a9d8b878eeb.json similarity index 59% rename from backend/.sqlx/query-e74e283951aa87627a46aa8286819cf7aa4fecbde17bc7a67ee6f49c427cee9e.json rename to backend/.sqlx/query-4e4a9c7b6e95f81101a68a6cc6d29fe4424eaed36d5ac9e4cd0a2a9d8b878eeb.json index 77399cdf62..0796577849 100644 --- a/backend/.sqlx/query-e74e283951aa87627a46aa8286819cf7aa4fecbde17bc7a67ee6f49c427cee9e.json +++ b/backend/.sqlx/query-4e4a9c7b6e95f81101a68a6cc6d29fe4424eaed36d5ac9e4cd0a2a9d8b878eeb.json @@ -1,6 +1,6 @@ { "db_name": "PostgreSQL", - "query": "SELECT path, value FROM resource_version WHERE workspace_id = $1 AND id = $2", + "query": "SELECT path, scorers FROM eval_dataset WHERE workspace_id = $1 AND path = ANY($2)", "describe": { "columns": [ { @@ -10,20 +10,20 @@ }, { "ordinal": 1, - "name": "value", + "name": "scorers", "type_info": "Jsonb" } ], "parameters": { "Left": [ "Text", - "Int8" + "TextArray" ] }, "nullable": [ false, - true + false ] }, - "hash": "e74e283951aa87627a46aa8286819cf7aa4fecbde17bc7a67ee6f49c427cee9e" + "hash": "4e4a9c7b6e95f81101a68a6cc6d29fe4424eaed36d5ac9e4cd0a2a9d8b878eeb" } diff --git a/backend/.sqlx/query-5329ce41bbdc36698ea059fec5f05ded7dfe1c64517289ad67028001417d30a3.json b/backend/.sqlx/query-5329ce41bbdc36698ea059fec5f05ded7dfe1c64517289ad67028001417d30a3.json new file mode 100644 index 0000000000..54019fe5bc --- /dev/null +++ b/backend/.sqlx/query-5329ce41bbdc36698ea059fec5f05ded7dfe1c64517289ad67028001417d30a3.json @@ -0,0 +1,47 @@ +{ + "db_name": "PostgreSQL", + "query": "SELECT id, input, expected, created_at, created_by\n FROM eval_case\n WHERE workspace_id = $1 AND dataset_path = $2\n ORDER BY created_at, id", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "id", + "type_info": "Uuid" + }, + { + "ordinal": 1, + "name": "input", + "type_info": "Jsonb" + }, + { + "ordinal": 2, + "name": "expected", + "type_info": "Jsonb" + }, + { + "ordinal": 3, + "name": "created_at", + "type_info": "Timestamptz" + }, + { + "ordinal": 4, + "name": "created_by", + "type_info": "Varchar" + } + ], + "parameters": { + "Left": [ + "Text", + "Text" + ] + }, + "nullable": [ + false, + false, + true, + false, + false + ] + }, + "hash": "5329ce41bbdc36698ea059fec5f05ded7dfe1c64517289ad67028001417d30a3" +} diff --git a/backend/.sqlx/query-5a219a2532517869578c4504ff3153c43903f929ae5d62fbba12610f89c36d55.json b/backend/.sqlx/query-5a219a2532517869578c4504ff3153c43903f929ae5d62fbba12610f89c36d55.json index 36ddb8ab9f..713ccb9dd3 100644 --- a/backend/.sqlx/query-5a219a2532517869578c4504ff3153c43903f929ae5d62fbba12610f89c36d55.json +++ b/backend/.sqlx/query-5a219a2532517869578c4504ff3153c43903f929ae5d62fbba12610f89c36d55.json @@ -15,7 +15,7 @@ ] }, "nullable": [ - true + null ] }, "hash": "5a219a2532517869578c4504ff3153c43903f929ae5d62fbba12610f89c36d55" diff --git a/backend/.sqlx/query-5d3560d7a42f86436fec362a790cea189be47e344d11871a90faa9220d509646.json b/backend/.sqlx/query-5d3560d7a42f86436fec362a790cea189be47e344d11871a90faa9220d509646.json new file mode 100644 index 0000000000..548516f69c --- /dev/null +++ b/backend/.sqlx/query-5d3560d7a42f86436fec362a790cea189be47e344d11871a90faa9220d509646.json @@ -0,0 +1,16 @@ +{ + "db_name": "PostgreSQL", + "query": "UPDATE eval_experiment_case c SET job_id = j.id\n FROM v2_job j\n WHERE j.parent_job = $3 AND j.workspace_id = $2\n AND (j.args -> 'iter' -> 'value' ->> 'case_id')::uuid = c.case_id\n AND c.experiment_id = $1 AND c.job_id IS NULL", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "Uuid", + "Text", + "Uuid" + ] + }, + "nullable": [] + }, + "hash": "5d3560d7a42f86436fec362a790cea189be47e344d11871a90faa9220d509646" +} diff --git a/backend/.sqlx/query-5d5186bb17092425664d8c4f92ca6bfb8b08420cc2b2a52e90064d6a12f86113.json b/backend/.sqlx/query-5d5186bb17092425664d8c4f92ca6bfb8b08420cc2b2a52e90064d6a12f86113.json new file mode 100644 index 0000000000..8bf4956d79 --- /dev/null +++ b/backend/.sqlx/query-5d5186bb17092425664d8c4f92ca6bfb8b08420cc2b2a52e90064d6a12f86113.json @@ -0,0 +1,76 @@ +{ + "db_name": "PostgreSQL", + "query": "SELECT ordinal, case_id, input, expected, job_id, subject_version,\n subject_draft_hash, output, answered, status\n FROM eval_experiment_case\n WHERE experiment_id = $1 ORDER BY ordinal", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "ordinal", + "type_info": "Int4" + }, + { + "ordinal": 1, + "name": "case_id", + "type_info": "Uuid" + }, + { + "ordinal": 2, + "name": "input", + "type_info": "Jsonb" + }, + { + "ordinal": 3, + "name": "expected", + "type_info": "Jsonb" + }, + { + "ordinal": 4, + "name": "job_id", + "type_info": "Uuid" + }, + { + "ordinal": 5, + "name": "subject_version", + "type_info": "Int8" + }, + { + "ordinal": 6, + "name": "subject_draft_hash", + "type_info": "Varchar" + }, + { + "ordinal": 7, + "name": "output", + "type_info": "Text" + }, + { + "ordinal": 8, + "name": "answered", + "type_info": "Bool" + }, + { + "ordinal": 9, + "name": "status", + "type_info": "Varchar" + } + ], + "parameters": { + "Left": [ + "Uuid" + ] + }, + "nullable": [ + false, + false, + false, + true, + true, + true, + true, + true, + true, + true + ] + }, + "hash": "5d5186bb17092425664d8c4f92ca6bfb8b08420cc2b2a52e90064d6a12f86113" +} diff --git a/backend/.sqlx/query-5dcaea907b1ebb2854becc44c00eac5838995dce3d2a696769e00fb74c34288f.json b/backend/.sqlx/query-5dcaea907b1ebb2854becc44c00eac5838995dce3d2a696769e00fb74c34288f.json new file mode 100644 index 0000000000..5c3ee8da92 --- /dev/null +++ b/backend/.sqlx/query-5dcaea907b1ebb2854becc44c00eac5838995dce3d2a696769e00fb74c34288f.json @@ -0,0 +1,22 @@ +{ + "db_name": "PostgreSQL", + "query": "SELECT DISTINCT experiment_id FROM eval_score\n WHERE experiment_id = ANY($1) AND score IS NULL AND error IS NULL\n AND NOT not_applicable", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "experiment_id", + "type_info": "Uuid" + } + ], + "parameters": { + "Left": [ + "UuidArray" + ] + }, + "nullable": [ + false + ] + }, + "hash": "5dcaea907b1ebb2854becc44c00eac5838995dce3d2a696769e00fb74c34288f" +} diff --git a/backend/.sqlx/query-61773dd5d5952607eddfacb1717126a87ee5dc4bd6664eaf31c2c0096fea08db.json b/backend/.sqlx/query-61773dd5d5952607eddfacb1717126a87ee5dc4bd6664eaf31c2c0096fea08db.json new file mode 100644 index 0000000000..278ccbe48e --- /dev/null +++ b/backend/.sqlx/query-61773dd5d5952607eddfacb1717126a87ee5dc4bd6664eaf31c2c0096fea08db.json @@ -0,0 +1,26 @@ +{ + "db_name": "PostgreSQL", + "query": "UPDATE eval_case SET input = $4, expected = $5\n WHERE workspace_id = $1 AND dataset_path = $2 AND id = $3\n RETURNING id", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "id", + "type_info": "Uuid" + } + ], + "parameters": { + "Left": [ + "Text", + "Text", + "Uuid", + "Jsonb", + "Jsonb" + ] + }, + "nullable": [ + false + ] + }, + "hash": "61773dd5d5952607eddfacb1717126a87ee5dc4bd6664eaf31c2c0096fea08db" +} diff --git a/backend/.sqlx/query-6a17a1dfeb75808e5d0726f1f8bd573168161abe350b047402cda4ac4a4d13e4.json b/backend/.sqlx/query-6a17a1dfeb75808e5d0726f1f8bd573168161abe350b047402cda4ac4a4d13e4.json new file mode 100644 index 0000000000..6ced022760 --- /dev/null +++ b/backend/.sqlx/query-6a17a1dfeb75808e5d0726f1f8bd573168161abe350b047402cda4ac4a4d13e4.json @@ -0,0 +1,74 @@ +{ + "db_name": "PostgreSQL", + "query": "SELECT\n (CASE $3::text\n WHEN 'day' THEN day::text\n WHEN 'user' THEN email\n ELSE ''\n END) AS \"key!\",\n provider AS \"provider!\",\n model AS \"model!\",\n SUM(input_tokens)::bigint AS \"input_tokens!\",\n SUM(cache_read_tokens)::bigint AS \"cache_read_tokens!\",\n SUM(cache_write_tokens)::bigint AS \"cache_write_tokens!\",\n SUM(output_tokens)::bigint AS \"output_tokens!\",\n SUM(reported_cost_nano_usd)::bigint AS \"reported_cost_nano_usd\",\n SUM(requests)::bigint AS \"requests!\"\n FROM ai_token_usage\n WHERE workspace_id = $1 AND day > CURRENT_DATE - $2::int\n AND ($5::text IS NULL OR email = $5)\n GROUP BY 1, provider, model\n ORDER BY SUM(input_tokens + cache_read_tokens + cache_write_tokens + output_tokens) DESC\n LIMIT $4", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "key!", + "type_info": "Text" + }, + { + "ordinal": 1, + "name": "provider!", + "type_info": "Varchar" + }, + { + "ordinal": 2, + "name": "model!", + "type_info": "Varchar" + }, + { + "ordinal": 3, + "name": "input_tokens!", + "type_info": "Int8" + }, + { + "ordinal": 4, + "name": "cache_read_tokens!", + "type_info": "Int8" + }, + { + "ordinal": 5, + "name": "cache_write_tokens!", + "type_info": "Int8" + }, + { + "ordinal": 6, + "name": "output_tokens!", + "type_info": "Int8" + }, + { + "ordinal": 7, + "name": "reported_cost_nano_usd", + "type_info": "Int8" + }, + { + "ordinal": 8, + "name": "requests!", + "type_info": "Int8" + } + ], + "parameters": { + "Left": [ + "Text", + "Int4", + "Text", + "Int8", + "Text" + ] + }, + "nullable": [ + null, + false, + false, + null, + null, + null, + null, + null, + null + ] + }, + "hash": "6a17a1dfeb75808e5d0726f1f8bd573168161abe350b047402cda4ac4a4d13e4" +} diff --git a/backend/.sqlx/query-6bc9d682aabdf8e79beb693e0090d07f3514628a4970b5f25ddf3920ba5e9c53.json b/backend/.sqlx/query-6bc9d682aabdf8e79beb693e0090d07f3514628a4970b5f25ddf3920ba5e9c53.json new file mode 100644 index 0000000000..e2a97d9c11 --- /dev/null +++ b/backend/.sqlx/query-6bc9d682aabdf8e79beb693e0090d07f3514628a4970b5f25ddf3920ba5e9c53.json @@ -0,0 +1,23 @@ +{ + "db_name": "PostgreSQL", + "query": "SELECT scorers FROM eval_dataset WHERE workspace_id = $1 AND path = $2 FOR UPDATE", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "scorers", + "type_info": "Jsonb" + } + ], + "parameters": { + "Left": [ + "Text", + "Text" + ] + }, + "nullable": [ + false + ] + }, + "hash": "6bc9d682aabdf8e79beb693e0090d07f3514628a4970b5f25ddf3920ba5e9c53" +} diff --git a/backend/.sqlx/query-77db9fcef0d3f9c9eb9edba6c0a54469259621218616ef64a4923f3396b183a2.json b/backend/.sqlx/query-77db9fcef0d3f9c9eb9edba6c0a54469259621218616ef64a4923f3396b183a2.json new file mode 100644 index 0000000000..84bd3cc921 --- /dev/null +++ b/backend/.sqlx/query-77db9fcef0d3f9c9eb9edba6c0a54469259621218616ef64a4923f3396b183a2.json @@ -0,0 +1,14 @@ +{ + "db_name": "PostgreSQL", + "query": "DELETE FROM eval_experiment WHERE id = $1", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "Uuid" + ] + }, + "nullable": [] + }, + "hash": "77db9fcef0d3f9c9eb9edba6c0a54469259621218616ef64a4923f3396b183a2" +} diff --git a/backend/.sqlx/query-790e90a3aca284fd060aac049fcf27d2e39781fccc4dd8419f934085d5453fc7.json b/backend/.sqlx/query-790e90a3aca284fd060aac049fcf27d2e39781fccc4dd8419f934085d5453fc7.json new file mode 100644 index 0000000000..101f3e9703 --- /dev/null +++ b/backend/.sqlx/query-790e90a3aca284fd060aac049fcf27d2e39781fccc4dd8419f934085d5453fc7.json @@ -0,0 +1,20 @@ +{ + "db_name": "PostgreSQL", + "query": "INSERT INTO eval_experiment_case\n (experiment_id, ordinal, case_id, input, expected, subject_version,\n subject_draft_hash)\n SELECT $1, ordinal, case_id, input, expected, subject_version, subject_draft_hash\n FROM UNNEST($2::int[], $3::uuid[], $4::jsonb[], $5::jsonb[], $6::bigint[], $7::text[])\n AS t(ordinal, case_id, input, expected, subject_version, subject_draft_hash)", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "Uuid", + "Int4Array", + "UuidArray", + "JsonbArray", + "JsonbArray", + "Int8Array", + "TextArray" + ] + }, + "nullable": [] + }, + "hash": "790e90a3aca284fd060aac049fcf27d2e39781fccc4dd8419f934085d5453fc7" +} diff --git a/backend/.sqlx/query-7a14c6815a7acc912fd8836191263a373b5c427a0b2f48ca775fb8f5a0096e18.json b/backend/.sqlx/query-7a14c6815a7acc912fd8836191263a373b5c427a0b2f48ca775fb8f5a0096e18.json new file mode 100644 index 0000000000..ddcdb3a030 --- /dev/null +++ b/backend/.sqlx/query-7a14c6815a7acc912fd8836191263a373b5c427a0b2f48ca775fb8f5a0096e18.json @@ -0,0 +1,26 @@ +{ + "db_name": "PostgreSQL", + "query": "INSERT INTO eval_case\n (workspace_id, dataset_path, input, expected, created_by, created_at)\n VALUES ($1, $2, $3, $4, $5, clock_timestamp())\n RETURNING id", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "id", + "type_info": "Uuid" + } + ], + "parameters": { + "Left": [ + "Varchar", + "Varchar", + "Jsonb", + "Jsonb", + "Varchar" + ] + }, + "nullable": [ + false + ] + }, + "hash": "7a14c6815a7acc912fd8836191263a373b5c427a0b2f48ca775fb8f5a0096e18" +} diff --git a/backend/.sqlx/query-7b040feaa84e85fff1a5ad1ddfc3d57b0f69b05f96ffafae0325cf3ff04f0f40.json b/backend/.sqlx/query-7b040feaa84e85fff1a5ad1ddfc3d57b0f69b05f96ffafae0325cf3ff04f0f40.json new file mode 100644 index 0000000000..4665140ce9 --- /dev/null +++ b/backend/.sqlx/query-7b040feaa84e85fff1a5ad1ddfc3d57b0f69b05f96ffafae0325cf3ff04f0f40.json @@ -0,0 +1,60 @@ +{ + "db_name": "PostgreSQL", + "query": "SELECT s.experiment_id AS \"experiment_id!\", s.scorer_id AS \"scorer_id!\",\n avg(s.score) AS mean,\n count(s.score) AS \"scored!\",\n count(*) FILTER (WHERE s.error IS NOT NULL) AS \"failed!\",\n count(*) FILTER (WHERE t.pass_if IS NOT NULL AND s.score >= t.pass_if)\n AS \"passed!\",\n bool_or(t.pass_if IS NOT NULL) AS \"has_threshold!\"\n FROM eval_score s\n JOIN unnest($1::uuid[], $2::text[], $3::float8[])\n AS t(experiment_id, scorer_id, pass_if)\n ON t.experiment_id = s.experiment_id AND t.scorer_id = s.scorer_id\n GROUP BY s.experiment_id, s.scorer_id", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "experiment_id!", + "type_info": "Uuid" + }, + { + "ordinal": 1, + "name": "scorer_id!", + "type_info": "Varchar" + }, + { + "ordinal": 2, + "name": "mean", + "type_info": "Float8" + }, + { + "ordinal": 3, + "name": "scored!", + "type_info": "Int8" + }, + { + "ordinal": 4, + "name": "failed!", + "type_info": "Int8" + }, + { + "ordinal": 5, + "name": "passed!", + "type_info": "Int8" + }, + { + "ordinal": 6, + "name": "has_threshold!", + "type_info": "Bool" + } + ], + "parameters": { + "Left": [ + "UuidArray", + "TextArray", + "Float8Array" + ] + }, + "nullable": [ + false, + false, + null, + null, + null, + null, + null + ] + }, + "hash": "7b040feaa84e85fff1a5ad1ddfc3d57b0f69b05f96ffafae0325cf3ff04f0f40" +} diff --git a/backend/.sqlx/query-7f373cf063907999580d0541a1a4a216e3887fdc0a9bab0747a447c01a4b37a5.json b/backend/.sqlx/query-7f373cf063907999580d0541a1a4a216e3887fdc0a9bab0747a447c01a4b37a5.json new file mode 100644 index 0000000000..d73778feec --- /dev/null +++ b/backend/.sqlx/query-7f373cf063907999580d0541a1a4a216e3887fdc0a9bab0747a447c01a4b37a5.json @@ -0,0 +1,16 @@ +{ + "db_name": "PostgreSQL", + "query": "UPDATE eval_experiment_case\n SET subject_version = $3, subject_draft_hash = NULL\n WHERE experiment_id = $1 AND subject_draft_hash = $2", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "Uuid", + "Text", + "Int8" + ] + }, + "nullable": [] + }, + "hash": "7f373cf063907999580d0541a1a4a216e3887fdc0a9bab0747a447c01a4b37a5" +} diff --git a/backend/.sqlx/query-80a5dd06cc5b9f7cb0bcd3b8cdd92c62d40a4ab97fab0f2dd330bd7629c0a632.json b/backend/.sqlx/query-80a5dd06cc5b9f7cb0bcd3b8cdd92c62d40a4ab97fab0f2dd330bd7629c0a632.json new file mode 100644 index 0000000000..5b3af727e9 --- /dev/null +++ b/backend/.sqlx/query-80a5dd06cc5b9f7cb0bcd3b8cdd92c62d40a4ab97fab0f2dd330bd7629c0a632.json @@ -0,0 +1,64 @@ +{ + "db_name": "PostgreSQL", + "query": "SELECT ordinal, scorer_id, score, reason, checks, error, not_applicable, definition\n FROM eval_score WHERE experiment_id = $1", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "ordinal", + "type_info": "Int4" + }, + { + "ordinal": 1, + "name": "scorer_id", + "type_info": "Varchar" + }, + { + "ordinal": 2, + "name": "score", + "type_info": "Float8" + }, + { + "ordinal": 3, + "name": "reason", + "type_info": "Text" + }, + { + "ordinal": 4, + "name": "checks", + "type_info": "Jsonb" + }, + { + "ordinal": 5, + "name": "error", + "type_info": "Text" + }, + { + "ordinal": 6, + "name": "not_applicable", + "type_info": "Bool" + }, + { + "ordinal": 7, + "name": "definition", + "type_info": "Varchar" + } + ], + "parameters": { + "Left": [ + "Uuid" + ] + }, + "nullable": [ + false, + false, + true, + true, + true, + true, + false, + false + ] + }, + "hash": "80a5dd06cc5b9f7cb0bcd3b8cdd92c62d40a4ab97fab0f2dd330bd7629c0a632" +} diff --git a/backend/.sqlx/query-e27ed86394e6568afdf7a1dd72edd7943c87fdf1d05fba193cbe575cb7281db5.json b/backend/.sqlx/query-82b2e8383ae7e345e45fb8a1b2bb362439f5b12a3dd3290a8dab17ccf9c391f3.json similarity index 60% rename from backend/.sqlx/query-e27ed86394e6568afdf7a1dd72edd7943c87fdf1d05fba193cbe575cb7281db5.json rename to backend/.sqlx/query-82b2e8383ae7e345e45fb8a1b2bb362439f5b12a3dd3290a8dab17ccf9c391f3.json index 68825a19c7..a50a31419e 100644 --- a/backend/.sqlx/query-e27ed86394e6568afdf7a1dd72edd7943c87fdf1d05fba193cbe575cb7281db5.json +++ b/backend/.sqlx/query-82b2e8383ae7e345e45fb8a1b2bb362439f5b12a3dd3290a8dab17ccf9c391f3.json @@ -1,6 +1,6 @@ { "db_name": "PostgreSQL", - "query": "SELECT id, created_at, created_by FROM resource_version\n WHERE workspace_id = $1 AND path = $2 ORDER BY id DESC LIMIT $3", + "query": "SELECT id, version, created_at, created_by FROM resource_version\n WHERE workspace_id = $1 AND path = $2 ORDER BY id DESC LIMIT $3", "describe": { "columns": [ { @@ -10,11 +10,16 @@ }, { "ordinal": 1, + "name": "version", + "type_info": "Int8" + }, + { + "ordinal": 2, "name": "created_at", "type_info": "Timestamptz" }, { - "ordinal": 2, + "ordinal": 3, "name": "created_by", "type_info": "Varchar" } @@ -27,10 +32,11 @@ ] }, "nullable": [ + false, false, false, true ] }, - "hash": "e27ed86394e6568afdf7a1dd72edd7943c87fdf1d05fba193cbe575cb7281db5" + "hash": "82b2e8383ae7e345e45fb8a1b2bb362439f5b12a3dd3290a8dab17ccf9c391f3" } diff --git a/backend/.sqlx/query-858b5fe344d79913921d8c05e9e6219919f438e9a3ce0c3e996ea61d9b4f2ef8.json b/backend/.sqlx/query-858b5fe344d79913921d8c05e9e6219919f438e9a3ce0c3e996ea61d9b4f2ef8.json new file mode 100644 index 0000000000..0b8900d23a --- /dev/null +++ b/backend/.sqlx/query-858b5fe344d79913921d8c05e9e6219919f438e9a3ce0c3e996ea61d9b4f2ef8.json @@ -0,0 +1,16 @@ +{ + "db_name": "PostgreSQL", + "query": "UPDATE eval_dataset SET edited_by = $1 WHERE edited_by = $2 AND workspace_id = $3", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "Varchar", + "Text", + "Text" + ] + }, + "nullable": [] + }, + "hash": "858b5fe344d79913921d8c05e9e6219919f438e9a3ce0c3e996ea61d9b4f2ef8" +} diff --git a/backend/.sqlx/query-864184467477e73a45935bf9439ae60da7f9ecc12f8f98b926345a70f941845b.json b/backend/.sqlx/query-864184467477e73a45935bf9439ae60da7f9ecc12f8f98b926345a70f941845b.json new file mode 100644 index 0000000000..7da7df136e --- /dev/null +++ b/backend/.sqlx/query-864184467477e73a45935bf9439ae60da7f9ecc12f8f98b926345a70f941845b.json @@ -0,0 +1,24 @@ +{ + "db_name": "PostgreSQL", + "query": "SELECT value FROM resource_version\n WHERE version = $1 AND workspace_id = $2 AND path = $3", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "value", + "type_info": "Jsonb" + } + ], + "parameters": { + "Left": [ + "Int8", + "Text", + "Text" + ] + }, + "nullable": [ + true + ] + }, + "hash": "864184467477e73a45935bf9439ae60da7f9ecc12f8f98b926345a70f941845b" +} diff --git a/backend/.sqlx/query-897e8da49e4a5c3efdfb2c36fc7aff3d53432da9ef57f8a775850efcd50a965c.json b/backend/.sqlx/query-897e8da49e4a5c3efdfb2c36fc7aff3d53432da9ef57f8a775850efcd50a965c.json new file mode 100644 index 0000000000..8610e3372b --- /dev/null +++ b/backend/.sqlx/query-897e8da49e4a5c3efdfb2c36fc7aff3d53432da9ef57f8a775850efcd50a965c.json @@ -0,0 +1,29 @@ +{ + "db_name": "PostgreSQL", + "query": "SELECT r.value AS \"value: sqlx::types::Json\",\n (SELECT version FROM resource_version v\n WHERE v.workspace_id = r.workspace_id AND v.path = r.path\n ORDER BY v.version DESC LIMIT 1) AS version\n FROM resource r\n WHERE r.workspace_id = $1 AND r.path = $2 AND r.resource_type = 'ai_agent'", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "value: sqlx::types::Json", + "type_info": "Jsonb" + }, + { + "ordinal": 1, + "name": "version", + "type_info": "Int8" + } + ], + "parameters": { + "Left": [ + "Text", + "Text" + ] + }, + "nullable": [ + true, + null + ] + }, + "hash": "897e8da49e4a5c3efdfb2c36fc7aff3d53432da9ef57f8a775850efcd50a965c" +} diff --git a/backend/.sqlx/query-8c319ac3eb2a289a6709ea1ea1c164f1a56e328902dd46fd9b68d801c16f4ffa.json b/backend/.sqlx/query-8c319ac3eb2a289a6709ea1ea1c164f1a56e328902dd46fd9b68d801c16f4ffa.json new file mode 100644 index 0000000000..f60f76b002 --- /dev/null +++ b/backend/.sqlx/query-8c319ac3eb2a289a6709ea1ea1c164f1a56e328902dd46fd9b68d801c16f4ffa.json @@ -0,0 +1,16 @@ +{ + "db_name": "PostgreSQL", + "query": "UPDATE eval_dataset SET extra_perms = extra_perms - ('u/' || $2) || jsonb_build_object(('u/' || $1), extra_perms->('u/' || $2)) WHERE extra_perms ? ('u/' || $2) AND workspace_id = $3", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "Text", + "Text", + "Text" + ] + }, + "nullable": [] + }, + "hash": "8c319ac3eb2a289a6709ea1ea1c164f1a56e328902dd46fd9b68d801c16f4ffa" +} diff --git a/backend/.sqlx/query-9071eff54395ca39809a5d55d1e3fedb64cfafa9fe74cd9446eb6ee526c2d9a0.json b/backend/.sqlx/query-9071eff54395ca39809a5d55d1e3fedb64cfafa9fe74cd9446eb6ee526c2d9a0.json new file mode 100644 index 0000000000..dd4a781db0 --- /dev/null +++ b/backend/.sqlx/query-9071eff54395ca39809a5d55d1e3fedb64cfafa9fe74cd9446eb6ee526c2d9a0.json @@ -0,0 +1,58 @@ +{ + "db_name": "PostgreSQL", + "query": "SELECT path, summary, scorers, created_at, created_by,\n edited_at, edited_by\n FROM eval_dataset WHERE workspace_id = $1 ORDER BY path", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "path", + "type_info": "Varchar" + }, + { + "ordinal": 1, + "name": "summary", + "type_info": "Varchar" + }, + { + "ordinal": 2, + "name": "scorers", + "type_info": "Jsonb" + }, + { + "ordinal": 3, + "name": "created_at", + "type_info": "Timestamptz" + }, + { + "ordinal": 4, + "name": "created_by", + "type_info": "Varchar" + }, + { + "ordinal": 5, + "name": "edited_at", + "type_info": "Timestamptz" + }, + { + "ordinal": 6, + "name": "edited_by", + "type_info": "Varchar" + } + ], + "parameters": { + "Left": [ + "Text" + ] + }, + "nullable": [ + false, + true, + false, + false, + false, + false, + false + ] + }, + "hash": "9071eff54395ca39809a5d55d1e3fedb64cfafa9fe74cd9446eb6ee526c2d9a0" +} diff --git a/backend/.sqlx/query-97a701e16c0ce4b8c6a1394c2a71b8ae45e16c11c8003c5f64c9f53d0e7226aa.json b/backend/.sqlx/query-97a701e16c0ce4b8c6a1394c2a71b8ae45e16c11c8003c5f64c9f53d0e7226aa.json new file mode 100644 index 0000000000..d068fac4f4 --- /dev/null +++ b/backend/.sqlx/query-97a701e16c0ce4b8c6a1394c2a71b8ae45e16c11c8003c5f64c9f53d0e7226aa.json @@ -0,0 +1,23 @@ +{ + "db_name": "PostgreSQL", + "query": "SELECT COALESCE(s.flow_status, c.flow_status) -> 'modules' AS modules\n FROM v2_job j\n LEFT JOIN v2_job_status s ON s.id = j.id\n LEFT JOIN v2_job_completed c ON c.id = j.id\n WHERE j.id = $1 AND j.workspace_id = $2", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "modules", + "type_info": "Jsonb" + } + ], + "parameters": { + "Left": [ + "Uuid", + "Text" + ] + }, + "nullable": [ + null + ] + }, + "hash": "97a701e16c0ce4b8c6a1394c2a71b8ae45e16c11c8003c5f64c9f53d0e7226aa" +} diff --git a/backend/.sqlx/query-9b1ad1bbf0c2dca3ce1cc9433c35db87e3101bcbcb0fa359885ca3807c1c9ca2.json b/backend/.sqlx/query-9b1ad1bbf0c2dca3ce1cc9433c35db87e3101bcbcb0fa359885ca3807c1c9ca2.json new file mode 100644 index 0000000000..bc6e2eb971 --- /dev/null +++ b/backend/.sqlx/query-9b1ad1bbf0c2dca3ce1cc9433c35db87e3101bcbcb0fa359885ca3807c1c9ca2.json @@ -0,0 +1,23 @@ +{ + "db_name": "PostgreSQL", + "query": "SELECT j.id AS \"id!\" FROM v2_job j\n LEFT JOIN v2_job_completed c ON c.id = j.id AND c.workspace_id = $2\n WHERE j.id = ANY($1) AND j.workspace_id = $2 AND c.id IS NULL", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "id!", + "type_info": "Uuid" + } + ], + "parameters": { + "Left": [ + "UuidArray", + "Text" + ] + }, + "nullable": [ + false + ] + }, + "hash": "9b1ad1bbf0c2dca3ce1cc9433c35db87e3101bcbcb0fa359885ca3807c1c9ca2" +} diff --git a/backend/.sqlx/query-9d3ca755b323330033eb891ac7162a53440a5c713b58e6c4a467f6f0d5d7e37c.json b/backend/.sqlx/query-9d3ca755b323330033eb891ac7162a53440a5c713b58e6c4a467f6f0d5d7e37c.json new file mode 100644 index 0000000000..1c6b02aa9f --- /dev/null +++ b/backend/.sqlx/query-9d3ca755b323330033eb891ac7162a53440a5c713b58e6c4a467f6f0d5d7e37c.json @@ -0,0 +1,16 @@ +{ + "db_name": "PostgreSQL", + "query": "UPDATE eval_dataset SET scorers = COALESCE((\n SELECT jsonb_agg(\n CASE WHEN elem->>'path' LIKE ('u/' || $2 || '/%')\n THEN jsonb_set(elem, '{path}', to_jsonb(REGEXP_REPLACE(elem->>'path','u/' || $2 || '/(.*)','u/' || $1 || '/\\1')))\n ELSE elem END)\n FROM jsonb_array_elements(scorers) elem), '[]'::jsonb)\n WHERE workspace_id = $3\n AND EXISTS (SELECT 1 FROM jsonb_array_elements(scorers) e WHERE e->>'path' LIKE ('u/' || $2 || '/%'))", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "Text", + "Text", + "Text" + ] + }, + "nullable": [] + }, + "hash": "9d3ca755b323330033eb891ac7162a53440a5c713b58e6c4a467f6f0d5d7e37c" +} diff --git a/backend/.sqlx/query-a4c842e395714346d5178190793d8ae66f891421b72fc6adfa8a38060ca2f3fa.json b/backend/.sqlx/query-a4c842e395714346d5178190793d8ae66f891421b72fc6adfa8a38060ca2f3fa.json new file mode 100644 index 0000000000..23e4e95b2f --- /dev/null +++ b/backend/.sqlx/query-a4c842e395714346d5178190793d8ae66f891421b72fc6adfa8a38060ca2f3fa.json @@ -0,0 +1,15 @@ +{ + "db_name": "PostgreSQL", + "query": "INSERT INTO eval_case (workspace_id, dataset_path, input, expected, created_at, created_by)\n SELECT $2, dataset_path, input, expected, created_at, created_by\n FROM eval_case WHERE workspace_id = $1", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "Text", + "Varchar" + ] + }, + "nullable": [] + }, + "hash": "a4c842e395714346d5178190793d8ae66f891421b72fc6adfa8a38060ca2f3fa" +} diff --git a/backend/.sqlx/query-a5e8cf0e559742330d67d36d9ddb75ef7de6e0ad74e0cc6c4056b782155b86c7.json b/backend/.sqlx/query-a5e8cf0e559742330d67d36d9ddb75ef7de6e0ad74e0cc6c4056b782155b86c7.json new file mode 100644 index 0000000000..d4b1e174da --- /dev/null +++ b/backend/.sqlx/query-a5e8cf0e559742330d67d36d9ddb75ef7de6e0ad74e0cc6c4056b782155b86c7.json @@ -0,0 +1,53 @@ +{ + "db_name": "PostgreSQL", + "query": "SELECT s.ordinal, s.scorer_id, c.job_id AS \"job_id!\", d.status::text AS status,\n c.answered, (j.id IS NOT NULL) AS \"job_exists!\"\n FROM eval_score s\n JOIN eval_experiment_case c\n ON c.experiment_id = s.experiment_id AND c.ordinal = s.ordinal\n LEFT JOIN v2_job j ON j.id = c.job_id AND j.workspace_id = $2\n LEFT JOIN v2_job_completed d ON d.id = c.job_id AND d.workspace_id = $2\n WHERE s.experiment_id = $1 AND s.score IS NULL AND s.error IS NULL\n AND NOT s.not_applicable AND c.job_id IS NOT NULL", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "ordinal", + "type_info": "Int4" + }, + { + "ordinal": 1, + "name": "scorer_id", + "type_info": "Varchar" + }, + { + "ordinal": 2, + "name": "job_id!", + "type_info": "Uuid" + }, + { + "ordinal": 3, + "name": "status", + "type_info": "Text" + }, + { + "ordinal": 4, + "name": "answered", + "type_info": "Bool" + }, + { + "ordinal": 5, + "name": "job_exists!", + "type_info": "Bool" + } + ], + "parameters": { + "Left": [ + "Uuid", + "Text" + ] + }, + "nullable": [ + false, + false, + true, + null, + true, + null + ] + }, + "hash": "a5e8cf0e559742330d67d36d9ddb75ef7de6e0ad74e0cc6c4056b782155b86c7" +} diff --git a/backend/.sqlx/query-a7b589b8d5cded97905bc24412ad8f3f0339a2a088243f045a212f9357205407.json b/backend/.sqlx/query-a7b589b8d5cded97905bc24412ad8f3f0339a2a088243f045a212f9357205407.json new file mode 100644 index 0000000000..fb8f4593ab --- /dev/null +++ b/backend/.sqlx/query-a7b589b8d5cded97905bc24412ad8f3f0339a2a088243f045a212f9357205407.json @@ -0,0 +1,27 @@ +{ + "db_name": "PostgreSQL", + "query": "UPDATE eval_dataset\n SET path = COALESCE($6, path), summary = COALESCE($3, summary),\n scorers = COALESCE($4, scorers), edited_at = now(), edited_by = $5\n WHERE workspace_id = $1 AND path = $2\n RETURNING path", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "path", + "type_info": "Varchar" + } + ], + "parameters": { + "Left": [ + "Text", + "Text", + "Varchar", + "Jsonb", + "Varchar", + "Varchar" + ] + }, + "nullable": [ + false + ] + }, + "hash": "a7b589b8d5cded97905bc24412ad8f3f0339a2a088243f045a212f9357205407" +} diff --git a/backend/.sqlx/query-b2db4f32c615a99db7af23729682340328da39ac763bb26b6bcf82027fad48e5.json b/backend/.sqlx/query-b2db4f32c615a99db7af23729682340328da39ac763bb26b6bcf82027fad48e5.json new file mode 100644 index 0000000000..e0f8e62e11 --- /dev/null +++ b/backend/.sqlx/query-b2db4f32c615a99db7af23729682340328da39ac763bb26b6bcf82027fad48e5.json @@ -0,0 +1,29 @@ +{ + "db_name": "PostgreSQL", + "query": "SELECT id, run_job_id FROM eval_experiment WHERE workspace_id = $1 AND id = $2", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "id", + "type_info": "Uuid" + }, + { + "ordinal": 1, + "name": "run_job_id", + "type_info": "Uuid" + } + ], + "parameters": { + "Left": [ + "Text", + "Uuid" + ] + }, + "nullable": [ + false, + false + ] + }, + "hash": "b2db4f32c615a99db7af23729682340328da39ac763bb26b6bcf82027fad48e5" +} diff --git a/backend/.sqlx/query-b8e732a03969666444f73397ac153d23ec2af084b2f93da24c920532c1916384.json b/backend/.sqlx/query-b8e732a03969666444f73397ac153d23ec2af084b2f93da24c920532c1916384.json index b892061f56..b336210daf 100644 --- a/backend/.sqlx/query-b8e732a03969666444f73397ac153d23ec2af084b2f93da24c920532c1916384.json +++ b/backend/.sqlx/query-b8e732a03969666444f73397ac153d23ec2af084b2f93da24c920532c1916384.json @@ -98,12 +98,12 @@ null, null, null, - false, + true, null, null, null, - false, - false + true, + true ] }, "hash": "b8e732a03969666444f73397ac153d23ec2af084b2f93da24c920532c1916384" diff --git a/backend/.sqlx/query-bac36542b16b687a823067c013e3728f1cabe3774f581401514f522ea7522de5.json b/backend/.sqlx/query-bac36542b16b687a823067c013e3728f1cabe3774f581401514f522ea7522de5.json new file mode 100644 index 0000000000..b680e889c6 --- /dev/null +++ b/backend/.sqlx/query-bac36542b16b687a823067c013e3728f1cabe3774f581401514f522ea7522de5.json @@ -0,0 +1,29 @@ +{ + "db_name": "PostgreSQL", + "query": "SELECT status::text AS \"status!\", duration_ms FROM v2_job_completed\n WHERE id = $1 AND workspace_id = $2", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "status!", + "type_info": "Text" + }, + { + "ordinal": 1, + "name": "duration_ms", + "type_info": "Int8" + } + ], + "parameters": { + "Left": [ + "Uuid", + "Text" + ] + }, + "nullable": [ + null, + false + ] + }, + "hash": "bac36542b16b687a823067c013e3728f1cabe3774f581401514f522ea7522de5" +} diff --git a/backend/.sqlx/query-bbce2221f5724016543a5ac1db7fa74b8b35375dfb0f5d39b398369468ad0774.json b/backend/.sqlx/query-bbce2221f5724016543a5ac1db7fa74b8b35375dfb0f5d39b398369468ad0774.json new file mode 100644 index 0000000000..24ea423095 --- /dev/null +++ b/backend/.sqlx/query-bbce2221f5724016543a5ac1db7fa74b8b35375dfb0f5d39b398369468ad0774.json @@ -0,0 +1,54 @@ +{ + "db_name": "PostgreSQL", + "query": "SELECT e.subject, e.run_number, e.run_job_id, e.created_at,\n e.created_by,\n (SELECT count(*) FROM eval_experiment_case c WHERE c.experiment_id = e.id)\n AS \"case_count!\"\n FROM eval_experiment e\n WHERE e.workspace_id = $1 AND e.dataset_path = $2 AND e.id = $3", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "subject", + "type_info": "Jsonb" + }, + { + "ordinal": 1, + "name": "run_number", + "type_info": "Int4" + }, + { + "ordinal": 2, + "name": "run_job_id", + "type_info": "Uuid" + }, + { + "ordinal": 3, + "name": "created_at", + "type_info": "Timestamptz" + }, + { + "ordinal": 4, + "name": "created_by", + "type_info": "Varchar" + }, + { + "ordinal": 5, + "name": "case_count!", + "type_info": "Int8" + } + ], + "parameters": { + "Left": [ + "Text", + "Text", + "Uuid" + ] + }, + "nullable": [ + false, + false, + false, + false, + false, + null + ] + }, + "hash": "bbce2221f5724016543a5ac1db7fa74b8b35375dfb0f5d39b398369468ad0774" +} diff --git a/backend/.sqlx/query-bdb7185233941d1472a55c8ade3f61c82248f3f2c9a69c458bff3978ffd17379.json b/backend/.sqlx/query-bdb7185233941d1472a55c8ade3f61c82248f3f2c9a69c458bff3978ffd17379.json new file mode 100644 index 0000000000..0e31540e7c --- /dev/null +++ b/backend/.sqlx/query-bdb7185233941d1472a55c8ade3f61c82248f3f2c9a69c458bff3978ffd17379.json @@ -0,0 +1,66 @@ +{ + "db_name": "PostgreSQL", + "query": "SELECT e.id, e.dataset_path, e.subject, e.run_number, e.run_job_id, e.created_at,\n e.created_by,\n (SELECT count(*) FROM eval_experiment_case c WHERE c.experiment_id = e.id)\n AS \"case_count!\"\n FROM eval_experiment e\n JOIN eval_dataset d ON d.workspace_id = e.workspace_id AND d.path = e.dataset_path\n WHERE e.workspace_id = $1\n AND ($3::text IS NULL OR e.subject ->> 'path' = $3)\n ORDER BY e.created_at DESC\n LIMIT $2", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "id", + "type_info": "Uuid" + }, + { + "ordinal": 1, + "name": "dataset_path", + "type_info": "Varchar" + }, + { + "ordinal": 2, + "name": "subject", + "type_info": "Jsonb" + }, + { + "ordinal": 3, + "name": "run_number", + "type_info": "Int4" + }, + { + "ordinal": 4, + "name": "run_job_id", + "type_info": "Uuid" + }, + { + "ordinal": 5, + "name": "created_at", + "type_info": "Timestamptz" + }, + { + "ordinal": 6, + "name": "created_by", + "type_info": "Varchar" + }, + { + "ordinal": 7, + "name": "case_count!", + "type_info": "Int8" + } + ], + "parameters": { + "Left": [ + "Text", + "Int8", + "Text" + ] + }, + "nullable": [ + false, + false, + false, + false, + false, + false, + false, + null + ] + }, + "hash": "bdb7185233941d1472a55c8ade3f61c82248f3f2c9a69c458bff3978ffd17379" +} diff --git a/backend/.sqlx/query-be17a65f144cc21e849c7f8cfbf9d7271b953dda42e4ba09864da7bda731e752.json b/backend/.sqlx/query-be17a65f144cc21e849c7f8cfbf9d7271b953dda42e4ba09864da7bda731e752.json new file mode 100644 index 0000000000..f28916165f --- /dev/null +++ b/backend/.sqlx/query-be17a65f144cc21e849c7f8cfbf9d7271b953dda42e4ba09864da7bda731e752.json @@ -0,0 +1,54 @@ +{ + "db_name": "PostgreSQL", + "query": "SELECT j.id, j.args AS \"args: sqlx::types::Json>\",\n c.result AS \"result: sqlx::types::Json>\",\n c.status::text AS status, c.duration_ms,\n s.schema AS \"schema: sqlx::types::Json>\"\n FROM v2_job j\n LEFT JOIN v2_job_completed c ON c.id = j.id\n LEFT JOIN script s ON s.workspace_id = j.workspace_id AND s.hash = j.runnable_id\n WHERE j.id = ANY($1) AND j.workspace_id = $2 AND j.parent_job = $3", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "id", + "type_info": "Uuid" + }, + { + "ordinal": 1, + "name": "args: sqlx::types::Json>", + "type_info": "Jsonb" + }, + { + "ordinal": 2, + "name": "result: sqlx::types::Json>", + "type_info": "Jsonb" + }, + { + "ordinal": 3, + "name": "status", + "type_info": "Text" + }, + { + "ordinal": 4, + "name": "duration_ms", + "type_info": "Int8" + }, + { + "ordinal": 5, + "name": "schema: sqlx::types::Json>", + "type_info": "Json" + } + ], + "parameters": { + "Left": [ + "UuidArray", + "Text", + "Uuid" + ] + }, + "nullable": [ + false, + true, + true, + null, + false, + true + ] + }, + "hash": "be17a65f144cc21e849c7f8cfbf9d7271b953dda42e4ba09864da7bda731e752" +} diff --git a/backend/.sqlx/query-d10efb37765ac9a7f2e15f71dbbc02ec917b6ecb014f938e68ced3cc7fe9dc86.json b/backend/.sqlx/query-d10efb37765ac9a7f2e15f71dbbc02ec917b6ecb014f938e68ced3cc7fe9dc86.json new file mode 100644 index 0000000000..c124b8c3cb --- /dev/null +++ b/backend/.sqlx/query-d10efb37765ac9a7f2e15f71dbbc02ec917b6ecb014f938e68ced3cc7fe9dc86.json @@ -0,0 +1,16 @@ +{ + "db_name": "PostgreSQL", + "query": "UPDATE eval_experiment SET subject = jsonb_set(subject, '{path}', to_jsonb(REGEXP_REPLACE(subject->>'path','u/' || $2 || '/(.*)','u/' || $1 || '/\\1'))) WHERE subject->>'path' LIKE ('u/' || $2 || '/%') AND workspace_id = $3", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "Text", + "Text", + "Text" + ] + }, + "nullable": [] + }, + "hash": "d10efb37765ac9a7f2e15f71dbbc02ec917b6ecb014f938e68ced3cc7fe9dc86" +} diff --git a/backend/.sqlx/query-d5cdb1121f2a414c0a70f4f7e4ae630bad0715bbbb3597cb1b14e41abe9fec14.json b/backend/.sqlx/query-d5cdb1121f2a414c0a70f4f7e4ae630bad0715bbbb3597cb1b14e41abe9fec14.json new file mode 100644 index 0000000000..ad292f64cd --- /dev/null +++ b/backend/.sqlx/query-d5cdb1121f2a414c0a70f4f7e4ae630bad0715bbbb3597cb1b14e41abe9fec14.json @@ -0,0 +1,23 @@ +{ + "db_name": "PostgreSQL", + "query": "UPDATE eval_experiment_case SET status = $2, answered = false\n WHERE experiment_id = $1 AND job_id IS NULL AND status IS NULL\n RETURNING ordinal", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "ordinal", + "type_info": "Int4" + } + ], + "parameters": { + "Left": [ + "Uuid", + "Varchar" + ] + }, + "nullable": [ + false + ] + }, + "hash": "d5cdb1121f2a414c0a70f4f7e4ae630bad0715bbbb3597cb1b14e41abe9fec14" +} diff --git a/backend/.sqlx/query-db19932d940b2467147eb2c13b059f2ec33b0c83b5c4a332a4b6e54d2591767c.json b/backend/.sqlx/query-db19932d940b2467147eb2c13b059f2ec33b0c83b5c4a332a4b6e54d2591767c.json new file mode 100644 index 0000000000..1595cec076 --- /dev/null +++ b/backend/.sqlx/query-db19932d940b2467147eb2c13b059f2ec33b0c83b5c4a332a4b6e54d2591767c.json @@ -0,0 +1,18 @@ +{ + "db_name": "PostgreSQL", + "query": "INSERT INTO eval_case\n (workspace_id, dataset_path, input, expected, created_by, created_at)\n VALUES ($1, $2, $3, $4, $5, clock_timestamp())", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "Varchar", + "Varchar", + "Jsonb", + "Jsonb", + "Varchar" + ] + }, + "nullable": [] + }, + "hash": "db19932d940b2467147eb2c13b059f2ec33b0c83b5c4a332a4b6e54d2591767c" +} diff --git a/backend/.sqlx/query-db63d41718c50e264a949885d2d7c13719ae650e9c788ebf125627b785fb9ee0.json b/backend/.sqlx/query-db63d41718c50e264a949885d2d7c13719ae650e9c788ebf125627b785fb9ee0.json new file mode 100644 index 0000000000..4602a03f61 --- /dev/null +++ b/backend/.sqlx/query-db63d41718c50e264a949885d2d7c13719ae650e9c788ebf125627b785fb9ee0.json @@ -0,0 +1,23 @@ +{ + "db_name": "PostgreSQL", + "query": "SELECT path FROM eval_dataset WHERE workspace_id = $1 AND path = $2", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "path", + "type_info": "Varchar" + } + ], + "parameters": { + "Left": [ + "Text", + "Text" + ] + }, + "nullable": [ + false + ] + }, + "hash": "db63d41718c50e264a949885d2d7c13719ae650e9c788ebf125627b785fb9ee0" +} diff --git a/backend/.sqlx/query-e3c080b84f50622e0a74524111ae292c958e71293695ce18f9905af4dd940495.json b/backend/.sqlx/query-e3c080b84f50622e0a74524111ae292c958e71293695ce18f9905af4dd940495.json new file mode 100644 index 0000000000..ace8edc731 --- /dev/null +++ b/backend/.sqlx/query-e3c080b84f50622e0a74524111ae292c958e71293695ce18f9905af4dd940495.json @@ -0,0 +1,49 @@ +{ + "db_name": "PostgreSQL", + "query": "SELECT id, input, expected, created_at, created_by\n FROM eval_case\n WHERE workspace_id = $1 AND dataset_path = $2\n ORDER BY created_at, id\n LIMIT $3 OFFSET $4", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "id", + "type_info": "Uuid" + }, + { + "ordinal": 1, + "name": "input", + "type_info": "Jsonb" + }, + { + "ordinal": 2, + "name": "expected", + "type_info": "Jsonb" + }, + { + "ordinal": 3, + "name": "created_at", + "type_info": "Timestamptz" + }, + { + "ordinal": 4, + "name": "created_by", + "type_info": "Varchar" + } + ], + "parameters": { + "Left": [ + "Text", + "Text", + "Int8", + "Int8" + ] + }, + "nullable": [ + false, + false, + true, + false, + false + ] + }, + "hash": "e3c080b84f50622e0a74524111ae292c958e71293695ce18f9905af4dd940495" +} diff --git a/backend/.sqlx/query-e6d7e9779eaa6e584b613675ae8602c405fe620937a4563a17db80219a2930d2.json b/backend/.sqlx/query-e6d7e9779eaa6e584b613675ae8602c405fe620937a4563a17db80219a2930d2.json new file mode 100644 index 0000000000..d0e2c29c2d --- /dev/null +++ b/backend/.sqlx/query-e6d7e9779eaa6e584b613675ae8602c405fe620937a4563a17db80219a2930d2.json @@ -0,0 +1,23 @@ +{ + "db_name": "PostgreSQL", + "query": "SELECT args AS \"args: sqlx::types::Json>\" FROM v2_job\n WHERE id = $1 AND workspace_id = $2", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "args: sqlx::types::Json>", + "type_info": "Jsonb" + } + ], + "parameters": { + "Left": [ + "Uuid", + "Text" + ] + }, + "nullable": [ + true + ] + }, + "hash": "e6d7e9779eaa6e584b613675ae8602c405fe620937a4563a17db80219a2930d2" +} diff --git a/backend/.sqlx/query-e719e98cceef1383f882632c9398dd950ad0b1f30cde9c0392384c9a490290b9.json b/backend/.sqlx/query-e719e98cceef1383f882632c9398dd950ad0b1f30cde9c0392384c9a490290b9.json new file mode 100644 index 0000000000..b4f0820065 --- /dev/null +++ b/backend/.sqlx/query-e719e98cceef1383f882632c9398dd950ad0b1f30cde9c0392384c9a490290b9.json @@ -0,0 +1,17 @@ +{ + "db_name": "PostgreSQL", + "query": "INSERT INTO eval_score (experiment_id, ordinal, scorer_id, definition)\n SELECT $1, ordinal, scorer_id, definition\n FROM UNNEST($2::int[], $3::text[], $4::text[]) AS t(ordinal, scorer_id, definition)\n ON CONFLICT (experiment_id, ordinal, scorer_id)\n DO UPDATE SET definition = EXCLUDED.definition, score = NULL, reason = NULL,\n checks = NULL, error = NULL, not_applicable = false", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "Uuid", + "Int4Array", + "TextArray", + "TextArray" + ] + }, + "nullable": [] + }, + "hash": "e719e98cceef1383f882632c9398dd950ad0b1f30cde9c0392384c9a490290b9" +} diff --git a/backend/.sqlx/query-e72d71852b6c5998b8d46407d164fcee17400a0ffc172d2a257c3fc15617fbd9.json b/backend/.sqlx/query-e72d71852b6c5998b8d46407d164fcee17400a0ffc172d2a257c3fc15617fbd9.json new file mode 100644 index 0000000000..5ce8f9f4c6 --- /dev/null +++ b/backend/.sqlx/query-e72d71852b6c5998b8d46407d164fcee17400a0ffc172d2a257c3fc15617fbd9.json @@ -0,0 +1,41 @@ +{ + "db_name": "PostgreSQL", + "query": "SELECT c.ordinal, c.job_id AS \"job_id!\", d.status::text AS status,\n (j.id IS NOT NULL) AS \"job_exists!\"\n FROM eval_experiment_case c\n LEFT JOIN v2_job j ON j.id = c.job_id AND j.workspace_id = $2\n LEFT JOIN v2_job_completed d ON d.id = c.job_id AND d.workspace_id = $2\n WHERE c.experiment_id = $1 AND c.job_id IS NOT NULL AND c.status IS NULL", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "ordinal", + "type_info": "Int4" + }, + { + "ordinal": 1, + "name": "job_id!", + "type_info": "Uuid" + }, + { + "ordinal": 2, + "name": "status", + "type_info": "Text" + }, + { + "ordinal": 3, + "name": "job_exists!", + "type_info": "Bool" + } + ], + "parameters": { + "Left": [ + "Uuid", + "Text" + ] + }, + "nullable": [ + false, + true, + null, + null + ] + }, + "hash": "e72d71852b6c5998b8d46407d164fcee17400a0ffc172d2a257c3fc15617fbd9" +} diff --git a/backend/.sqlx/query-e942a74104771b192c33ea03dbcaa409e03b6bfb7423929c201f39db4228cc36.json b/backend/.sqlx/query-e942a74104771b192c33ea03dbcaa409e03b6bfb7423929c201f39db4228cc36.json new file mode 100644 index 0000000000..9daa8a1851 --- /dev/null +++ b/backend/.sqlx/query-e942a74104771b192c33ea03dbcaa409e03b6bfb7423929c201f39db4228cc36.json @@ -0,0 +1,23 @@ +{ + "db_name": "PostgreSQL", + "query": "DELETE FROM eval_dataset WHERE workspace_id = $1 AND path = $2 RETURNING path", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "path", + "type_info": "Varchar" + } + ], + "parameters": { + "Left": [ + "Text", + "Text" + ] + }, + "nullable": [ + false + ] + }, + "hash": "e942a74104771b192c33ea03dbcaa409e03b6bfb7423929c201f39db4228cc36" +} diff --git a/backend/.sqlx/query-eb0f25a10f4f1264e482674c06783204fddf0ccdca6d7c798a4ff6527548e96f.json b/backend/.sqlx/query-eb0f25a10f4f1264e482674c06783204fddf0ccdca6d7c798a4ff6527548e96f.json new file mode 100644 index 0000000000..9ed995c409 --- /dev/null +++ b/backend/.sqlx/query-eb0f25a10f4f1264e482674c06783204fddf0ccdca6d7c798a4ff6527548e96f.json @@ -0,0 +1,20 @@ +{ + "db_name": "PostgreSQL", + "query": "INSERT INTO eval_experiment\n (id, workspace_id, dataset_path, subject, run_number, created_by, run_job_id)\n VALUES ($1, $2, $3, $4, $5, $6, $7)", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "Uuid", + "Varchar", + "Varchar", + "Jsonb", + "Int4", + "Varchar", + "Uuid" + ] + }, + "nullable": [] + }, + "hash": "eb0f25a10f4f1264e482674c06783204fddf0ccdca6d7c798a4ff6527548e96f" +} diff --git a/backend/.sqlx/query-ee40e48afb5520b7ff84883204f064ce566cea6655e3cd11a9d7016cb08482cb.json b/backend/.sqlx/query-ee40e48afb5520b7ff84883204f064ce566cea6655e3cd11a9d7016cb08482cb.json new file mode 100644 index 0000000000..2280a368ea --- /dev/null +++ b/backend/.sqlx/query-ee40e48afb5520b7ff84883204f064ce566cea6655e3cd11a9d7016cb08482cb.json @@ -0,0 +1,21 @@ +{ + "db_name": "PostgreSQL", + "query": "UPDATE eval_score s\n SET score = t.score, reason = t.reason, checks = t.checks, error = t.error,\n not_applicable = t.not_applicable\n FROM UNNEST($2::int[], $3::text[], $4::double precision[], $5::text[], $6::jsonb[],\n $7::text[], $8::bool[])\n AS t(ordinal, scorer_id, score, reason, checks, error, not_applicable)\n WHERE s.experiment_id = $1 AND s.ordinal = t.ordinal AND s.scorer_id = t.scorer_id", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "Uuid", + "Int4Array", + "TextArray", + "Float8Array", + "TextArray", + "JsonbArray", + "TextArray", + "BoolArray" + ] + }, + "nullable": [] + }, + "hash": "ee40e48afb5520b7ff84883204f064ce566cea6655e3cd11a9d7016cb08482cb" +} diff --git a/backend/.sqlx/query-f0e943244b125d0c42a9b472701ec172af9dd334786275c391e80f1c38bbb45b.json b/backend/.sqlx/query-f0e943244b125d0c42a9b472701ec172af9dd334786275c391e80f1c38bbb45b.json new file mode 100644 index 0000000000..5e0fef8520 --- /dev/null +++ b/backend/.sqlx/query-f0e943244b125d0c42a9b472701ec172af9dd334786275c391e80f1c38bbb45b.json @@ -0,0 +1,18 @@ +{ + "db_name": "PostgreSQL", + "query": "UPDATE eval_experiment_case c\n SET output = COALESCE(c.output, t.output), answered = COALESCE(c.answered, t.answered),\n status = COALESCE(c.status, t.status)\n FROM UNNEST($2::int[], $3::text[], $4::bool[], $5::text[])\n AS t(ordinal, output, answered, status)\n WHERE c.experiment_id = $1 AND c.ordinal = t.ordinal", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "Uuid", + "Int4Array", + "TextArray", + "BoolArray", + "TextArray" + ] + }, + "nullable": [] + }, + "hash": "f0e943244b125d0c42a9b472701ec172af9dd334786275c391e80f1c38bbb45b" +} diff --git a/backend/.sqlx/query-f59afd524e3f216487ad0a780b1fda6341e215025dc6d8985891b91dad6d9dcb.json b/backend/.sqlx/query-f59afd524e3f216487ad0a780b1fda6341e215025dc6d8985891b91dad6d9dcb.json new file mode 100644 index 0000000000..2d8a83ba48 --- /dev/null +++ b/backend/.sqlx/query-f59afd524e3f216487ad0a780b1fda6341e215025dc6d8985891b91dad6d9dcb.json @@ -0,0 +1,26 @@ +{ + "db_name": "PostgreSQL", + "query": "INSERT INTO eval_dataset\n (workspace_id, path, summary, scorers, created_by, edited_by)\n VALUES ($1, $2, $3, $4, $5, $5)\n ON CONFLICT (workspace_id, path) DO NOTHING\n RETURNING path", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "path", + "type_info": "Varchar" + } + ], + "parameters": { + "Left": [ + "Varchar", + "Varchar", + "Varchar", + "Jsonb", + "Varchar" + ] + }, + "nullable": [ + false + ] + }, + "hash": "f59afd524e3f216487ad0a780b1fda6341e215025dc6d8985891b91dad6d9dcb" +} diff --git a/backend/.sqlx/query-f6c4c40b098ba06f4b3b057af1f5dce0711e90f7a4e1786d39a50d52400a3cb9.json b/backend/.sqlx/query-f6c4c40b098ba06f4b3b057af1f5dce0711e90f7a4e1786d39a50d52400a3cb9.json new file mode 100644 index 0000000000..035dc31c85 --- /dev/null +++ b/backend/.sqlx/query-f6c4c40b098ba06f4b3b057af1f5dce0711e90f7a4e1786d39a50d52400a3cb9.json @@ -0,0 +1,16 @@ +{ + "db_name": "PostgreSQL", + "query": "UPDATE eval_dataset SET created_by = $1 WHERE created_by = $2 AND workspace_id = $3", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "Varchar", + "Text", + "Text" + ] + }, + "nullable": [] + }, + "hash": "f6c4c40b098ba06f4b3b057af1f5dce0711e90f7a4e1786d39a50d52400a3cb9" +} diff --git a/backend/.sqlx/query-ef59abddc518f5213827e47a31aee49be917a61c46916c29d79c094438b1ff35.json b/backend/.sqlx/query-f907114909b1064a3d5eb603e5929a6ffebdcadb20e4c6a68768a9ad42fe9328.json similarity index 69% rename from backend/.sqlx/query-ef59abddc518f5213827e47a31aee49be917a61c46916c29d79c094438b1ff35.json rename to backend/.sqlx/query-f907114909b1064a3d5eb603e5929a6ffebdcadb20e4c6a68768a9ad42fe9328.json index 2a9c7cac4f..a62f4f95a1 100644 --- a/backend/.sqlx/query-ef59abddc518f5213827e47a31aee49be917a61c46916c29d79c094438b1ff35.json +++ b/backend/.sqlx/query-f907114909b1064a3d5eb603e5929a6ffebdcadb20e4c6a68768a9ad42fe9328.json @@ -1,6 +1,6 @@ { "db_name": "PostgreSQL", - "query": "SELECT id, path, created_at, created_by, value FROM resource_version\n WHERE workspace_id = $1 AND id = $2", + "query": "SELECT id, version, path, created_at, created_by, value FROM resource_version\n WHERE workspace_id = $1 AND id = $2", "describe": { "columns": [ { @@ -10,21 +10,26 @@ }, { "ordinal": 1, + "name": "version", + "type_info": "Int8" + }, + { + "ordinal": 2, "name": "path", "type_info": "Varchar" }, { - "ordinal": 2, + "ordinal": 3, "name": "created_at", "type_info": "Timestamptz" }, { - "ordinal": 3, + "ordinal": 4, "name": "created_by", "type_info": "Varchar" }, { - "ordinal": 4, + "ordinal": 5, "name": "value", "type_info": "Jsonb" } @@ -39,9 +44,10 @@ false, false, false, + false, true, true ] }, - "hash": "ef59abddc518f5213827e47a31aee49be917a61c46916c29d79c094438b1ff35" + "hash": "f907114909b1064a3d5eb603e5929a6ffebdcadb20e4c6a68768a9ad42fe9328" } diff --git a/backend/.sqlx/query-facbf7337d7ffa3f3e6287e2910ccd7ca7229f9d4e5c3af6c460ee7fe3c946e6.json b/backend/.sqlx/query-facbf7337d7ffa3f3e6287e2910ccd7ca7229f9d4e5c3af6c460ee7fe3c946e6.json new file mode 100644 index 0000000000..5f6f815e2a --- /dev/null +++ b/backend/.sqlx/query-facbf7337d7ffa3f3e6287e2910ccd7ca7229f9d4e5c3af6c460ee7fe3c946e6.json @@ -0,0 +1,23 @@ +{ + "db_name": "PostgreSQL", + "query": "SELECT path FROM eval_dataset WHERE workspace_id = $1 AND path = $2 FOR UPDATE", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "path", + "type_info": "Varchar" + } + ], + "parameters": { + "Left": [ + "Text", + "Text" + ] + }, + "nullable": [ + false + ] + }, + "hash": "facbf7337d7ffa3f3e6287e2910ccd7ca7229f9d4e5c3af6c460ee7fe3c946e6" +} diff --git a/backend/.sqlx/query-fcf570337b2ceeb0f9dcc311144d8ada02aa4353feb2f63e7a1f520a4000ed70.json b/backend/.sqlx/query-fcf570337b2ceeb0f9dcc311144d8ada02aa4353feb2f63e7a1f520a4000ed70.json new file mode 100644 index 0000000000..7dc41fa462 --- /dev/null +++ b/backend/.sqlx/query-fcf570337b2ceeb0f9dcc311144d8ada02aa4353feb2f63e7a1f520a4000ed70.json @@ -0,0 +1,36 @@ +{ + "db_name": "PostgreSQL", + "query": "SELECT branch.parent_job AS \"case_job!\", scorer.flow_step_id AS \"module!\",\n done.result AS \"result: sqlx::types::Json>\"\n FROM v2_job branch\n JOIN v2_job scorer ON scorer.parent_job = branch.id\n JOIN v2_job_completed done ON done.id = scorer.id\n WHERE branch.parent_job = ANY($1) AND branch.workspace_id = $2\n AND scorer.flow_step_id = ANY($3)", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "case_job!", + "type_info": "Uuid" + }, + { + "ordinal": 1, + "name": "module!", + "type_info": "Varchar" + }, + { + "ordinal": 2, + "name": "result: sqlx::types::Json>", + "type_info": "Jsonb" + } + ], + "parameters": { + "Left": [ + "UuidArray", + "Text", + "TextArray" + ] + }, + "nullable": [ + true, + true, + true + ] + }, + "hash": "fcf570337b2ceeb0f9dcc311144d8ada02aa4353feb2f63e7a1f520a4000ed70" +} diff --git a/backend/.sqlx/query-fd12bbe0605c80fced218a5a2e1288e6c04f4a1a76a2038679f8f538320cab1f.json b/backend/.sqlx/query-fd12bbe0605c80fced218a5a2e1288e6c04f4a1a76a2038679f8f538320cab1f.json new file mode 100644 index 0000000000..923a4b5f98 --- /dev/null +++ b/backend/.sqlx/query-fd12bbe0605c80fced218a5a2e1288e6c04f4a1a76a2038679f8f538320cab1f.json @@ -0,0 +1,24 @@ +{ + "db_name": "PostgreSQL", + "query": "SELECT path FROM eval_dataset\n WHERE scorers::text LIKE $1 AND NOT path LIKE $2 AND workspace_id = $3", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "path", + "type_info": "Varchar" + } + ], + "parameters": { + "Left": [ + "Text", + "Text", + "Text" + ] + }, + "nullable": [ + false + ] + }, + "hash": "fd12bbe0605c80fced218a5a2e1288e6c04f4a1a76a2038679f8f538320cab1f" +} diff --git a/backend/Cargo.lock b/backend/Cargo.lock index 43ff602ec1..2721fc7283 100644 --- a/backend/Cargo.lock +++ b/backend/Cargo.lock @@ -873,7 +873,7 @@ checksum = "82f6aeea286b8eb4dd3431a1be1b59d290ace00f5bfd8e2a159bc2a05e2c1667" dependencies = [ "proc-macro2", "quote", - "syn 3.0.3", + "syn 3.0.4", ] [[package]] @@ -1333,7 +1333,7 @@ dependencies = [ "aws-smithy-runtime-api", "aws-smithy-types", "h2 0.3.27", - "h2 0.4.18", + "h2 0.4.19", "http 0.2.12", "http 1.5.0", "http-body 0.4.6", @@ -1883,9 +1883,9 @@ dependencies = [ [[package]] name = "blocking" -version = "1.6.2" +version = "1.7.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e83f8d02be6967315521be875afa792a316e28d57b5a2d401897e2a7921b7f21" +checksum = "a70e4329df6cb94385eed412ec92375c3cdd8a6e502493d1229b6414e4036dfa" dependencies = [ "async-channel 2.5.0", "async-task", @@ -2128,7 +2128,7 @@ checksum = "fc0e56a716f1e132ff6bf4bdac1c944a3fcdc1cae65f70a4a2a1ac3b401d2d1f" dependencies = [ "proc-macro2", "quote", - "syn 3.0.3", + "syn 3.0.4", ] [[package]] @@ -2438,7 +2438,7 @@ dependencies = [ "heck", "proc-macro2", "quote", - "syn 3.0.3", + "syn 3.0.4", ] [[package]] @@ -2711,9 +2711,9 @@ checksum = "217698eaf96b4a3f0bc4f3662aaa55bdf913cd54d7204591faa790070c6d0853" [[package]] name = "crc32fast" -version = "1.5.0" +version = "1.5.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9481c1c90cbf2ac953f07c8d4a58aa3945c425b7185c9154d67a65e4230da511" +checksum = "8498c871161e1742aaa9d52551b2d6ebdd4c3d45a3be423e3728f33b955be550" dependencies = [ "cfg-if", ] @@ -3007,7 +3007,7 @@ dependencies = [ "proc-macro2", "quote", "strsim 0.11.1", - "syn 3.0.3", + "syn 3.0.4", ] [[package]] @@ -3062,7 +3062,7 @@ checksum = "2ac7135c3ef02b2f7833bbeb1be5ba7f966dcde8a87c6b87f65a778d71a02785" dependencies = [ "darling_core 0.24.1", "quote", - "syn 3.0.3", + "syn 3.0.4", ] [[package]] @@ -3951,7 +3951,7 @@ dependencies = [ "deno_tls", "dyn-clone", "error_reporter", - "h2 0.4.18", + "h2 0.4.19", "hickory-resolver", "http 1.5.0", "http-body-util", @@ -4552,7 +4552,7 @@ checksum = "c6232dd377dcc64799954cbd3a9bb882e9cdc1308ccd87b1c098f1fb2eaf82a8" dependencies = [ "proc-macro2", "quote", - "syn 3.0.3", + "syn 3.0.4", ] [[package]] @@ -4799,7 +4799,7 @@ checksum = "a65863d15a4ce2888bd2f0f543cc963d3879c3a022c8ee43f6141d479a3ac815" dependencies = [ "proc-macro2", "quote", - "syn 3.0.3", + "syn 3.0.4", ] [[package]] @@ -5262,7 +5262,7 @@ checksum = "9fb9654ba8355388abeb8dcb4fc62f511300867002afc858860463bdd9fe0c44" dependencies = [ "proc-macro2", "quote", - "syn 3.0.3", + "syn 3.0.4", ] [[package]] @@ -5749,9 +5749,9 @@ dependencies = [ [[package]] name = "h2" -version = "0.4.18" +version = "0.4.19" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "839c0e8a181239723652be9062bb56ca5bf5f64011f73b623f6f4fc59086a228" +checksum = "ef8e5e5a340588f4452631496976cf8636d4a7ecf600239fdc27615d2530bc16" dependencies = [ "atomic-waker", "bytes", @@ -6169,7 +6169,7 @@ dependencies = [ "bytes", "futures-channel", "futures-core", - "h2 0.4.18", + "h2 0.4.19", "http 1.5.0", "http-body 1.1.0", "httparse", @@ -7203,7 +7203,7 @@ dependencies = [ "bitflags 2.13.1", "libc", "plain", - "redox_syscall 0.9.2", + "redox_syscall 0.9.3", ] [[package]] @@ -7290,9 +7290,9 @@ dependencies = [ [[package]] name = "log" -version = "0.4.33" +version = "0.4.34" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0ceec5bc11778974d1bcb055b18002eba7f4b3518b6a0081b3af5f21666da9ad" +checksum = "f9f8bd3e56ce4dfc153cf470fffbfa98c7620958b312ca5c3a4b8d5181fd13c6" [[package]] name = "loom" @@ -10080,9 +10080,9 @@ dependencies = [ [[package]] name = "redox_syscall" -version = "0.9.2" +version = "0.9.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f1c93da5bb2c5d4e6c0ef7abeead62c89169a0a4882bfb83ac892f2423aea2fe" +checksum = "d678d17679829e73d371e96880897e98fee2ded7acc0a50bdf8af2affa4b2fe5" dependencies = [ "bitflags 2.13.1", ] @@ -10126,7 +10126,7 @@ checksum = "92ecd8964f8453721699a1ed72037b0db49ce2f5a5138486ee89bed6f67cdf3a" dependencies = [ "proc-macro2", "quote", - "syn 3.0.3", + "syn 3.0.4", ] [[package]] @@ -10200,7 +10200,7 @@ dependencies = [ "futures-channel", "futures-core", "futures-util", - "h2 0.4.18", + "h2 0.4.19", "http 1.5.0", "http-body 1.1.0", "http-body-util", @@ -10248,7 +10248,7 @@ dependencies = [ "encoding_rs", "futures-core", "futures-util", - "h2 0.4.18", + "h2 0.4.19", "http 1.5.0", "http-body 1.1.0", "http-body-util", @@ -10445,7 +10445,7 @@ dependencies = [ "proc-macro2", "quote", "serde_json", - "syn 3.0.3", + "syn 3.0.4", ] [[package]] @@ -11092,7 +11092,7 @@ dependencies = [ "proc-macro2", "quote", "serde_derive_internals 0.30.0", - "syn 3.0.3", + "syn 3.0.4", ] [[package]] @@ -11291,7 +11291,7 @@ checksum = "e7a5d71263a5a7d47b41f6b3f06ba276f10cc18b0931f1799f710578e2309348" dependencies = [ "proc-macro2", "quote", - "syn 3.0.3", + "syn 3.0.4", ] [[package]] @@ -11313,7 +11313,7 @@ checksum = "f852137cce035d6a4df67ccce505ff6b3e9fd3a10e3e52b24dc71e650bb1a9bd" dependencies = [ "proc-macro2", "quote", - "syn 3.0.3", + "syn 3.0.4", ] [[package]] @@ -11367,7 +11367,7 @@ checksum = "8d3b1629de253c70a0508c3899572da79ca359fdab27c7920ff00406df418906" dependencies = [ "proc-macro2", "quote", - "syn 3.0.3", + "syn 3.0.4", ] [[package]] @@ -11849,8 +11849,7 @@ dependencies = [ [[package]] name = "sqlx" version = "0.8.6" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1fefb893899429669dcdd979aff487bd78f4064e5e7907e4269081e0ef7d97dc" +source = "git+https://github.com/windmill-labs/sqlx?rev=6bdaee94fa62a01561125646da3f99eb341f2457#6bdaee94fa62a01561125646da3f99eb341f2457" dependencies = [ "sqlx-core", "sqlx-macros", @@ -11862,8 +11861,7 @@ dependencies = [ [[package]] name = "sqlx-core" version = "0.8.6" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ee6798b1838b6a0f69c007c133b8df5866302197e404e8b6ee8ed3e3a5e68dc6" +source = "git+https://github.com/windmill-labs/sqlx?rev=6bdaee94fa62a01561125646da3f99eb341f2457#6bdaee94fa62a01561125646da3f99eb341f2457" dependencies = [ "base64 0.22.1", "bigdecimal", @@ -11901,8 +11899,7 @@ dependencies = [ [[package]] name = "sqlx-macros" version = "0.8.6" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a2d452988ccaacfbf5e0bdbc348fb91d7c8af5bee192173ac3636b5fb6e6715d" +source = "git+https://github.com/windmill-labs/sqlx?rev=6bdaee94fa62a01561125646da3f99eb341f2457#6bdaee94fa62a01561125646da3f99eb341f2457" dependencies = [ "proc-macro2", "quote", @@ -11914,8 +11911,7 @@ dependencies = [ [[package]] name = "sqlx-macros-core" version = "0.8.6" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "19a9c1841124ac5a61741f96e1d9e2ec77424bf323962dd894bdb93f37d5219b" +source = "git+https://github.com/windmill-labs/sqlx?rev=6bdaee94fa62a01561125646da3f99eb341f2457#6bdaee94fa62a01561125646da3f99eb341f2457" dependencies = [ "dotenvy", "either", @@ -11939,8 +11935,7 @@ dependencies = [ [[package]] name = "sqlx-mysql" version = "0.8.6" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "aa003f0038df784eb8fecbbac13affe3da23b45194bd57dba231c8f48199c526" +source = "git+https://github.com/windmill-labs/sqlx?rev=6bdaee94fa62a01561125646da3f99eb341f2457#6bdaee94fa62a01561125646da3f99eb341f2457" dependencies = [ "atoi", "base64 0.22.1", @@ -11984,8 +11979,7 @@ dependencies = [ [[package]] name = "sqlx-postgres" version = "0.8.6" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "db58fcd5a53cf07c184b154801ff91347e4c30d17a3562a635ff028ad5deda46" +source = "git+https://github.com/windmill-labs/sqlx?rev=6bdaee94fa62a01561125646da3f99eb341f2457#6bdaee94fa62a01561125646da3f99eb341f2457" dependencies = [ "atoi", "base64 0.22.1", @@ -12025,8 +12019,7 @@ dependencies = [ [[package]] name = "sqlx-sqlite" version = "0.8.6" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c2d12fe70b2c1b4401038055f90f151b78208de1f9f89a7dbfd41587a10c3eea" +source = "git+https://github.com/windmill-labs/sqlx?rev=6bdaee94fa62a01561125646da3f99eb341f2457#6bdaee94fa62a01561125646da3f99eb341f2457" dependencies = [ "atoi", "chrono", @@ -12590,9 +12583,9 @@ dependencies = [ [[package]] name = "syn" -version = "3.0.3" +version = "3.0.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "53e9bae58849f64dfa4f5d5ae372c8341f7305f82a3868709269343628b659a3" +checksum = "e6275cddf4610d1775e6d1fe9469b2e77d0f39fd98fb7450901b821e0c53649f" dependencies = [ "proc-macro2", "quote", @@ -12985,7 +12978,7 @@ checksum = "bc04cd3e1236dd4a98afca4569f2deb3f120e5422a4023be2cb683f8486292af" dependencies = [ "proc-macro2", "quote", - "syn 3.0.3", + "syn 3.0.4", ] [[package]] @@ -13519,7 +13512,7 @@ dependencies = [ "base64 0.22.1", "bytes", "flate2", - "h2 0.4.18", + "h2 0.4.19", "http 1.5.0", "http-body 1.1.0", "http-body-util", @@ -13551,7 +13544,7 @@ dependencies = [ "axum 0.8.9", "base64 0.22.1", "bytes", - "h2 0.4.18", + "h2 0.4.19", "http 1.5.0", "http-body 1.1.0", "http-body-util", @@ -13951,7 +13944,7 @@ checksum = "f153acc4e99a5f2a5aefa09fb078be54e26271b2813f6041200b224c098d8328" dependencies = [ "proc-macro2", "quote", - "syn 3.0.3", + "syn 3.0.4", ] [[package]] @@ -14258,9 +14251,9 @@ checksum = "06abde3611657adf66d383f00b093d7faecc7fa57071cce2578660c9f1010821" [[package]] name = "uuid" -version = "1.24.1" +version = "1.25.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2cefc03fd367c0c6d4305de1b312cf00248c4114f4a0418ce6a6af769e3b0bd9" +checksum = "f053576934f05a761a402421fbbe3d425d9366f75f978806a037b3ca481abecc" dependencies = [ "getrandom 0.4.3", "js-sys", @@ -14670,7 +14663,7 @@ dependencies = [ [[package]] name = "windmill" -version = "1.795.0" +version = "1.796.0" dependencies = [ "anyhow", "async-nats", @@ -14755,7 +14748,7 @@ dependencies = [ [[package]] name = "windmill-ai" -version = "1.795.0" +version = "1.796.0" dependencies = [ "async-stream", "async-trait", @@ -14788,7 +14781,7 @@ dependencies = [ [[package]] name = "windmill-alerting" -version = "1.795.0" +version = "1.796.0" dependencies = [ "axum 0.8.9", "chrono", @@ -14801,7 +14794,7 @@ dependencies = [ [[package]] name = "windmill-api" -version = "1.795.0" +version = "1.796.0" dependencies = [ "anyhow", "argon2", @@ -14941,7 +14934,7 @@ dependencies = [ [[package]] name = "windmill-api-agent-workers" -version = "1.795.0" +version = "1.796.0" dependencies = [ "axum 0.8.9", "chrono", @@ -14964,7 +14957,7 @@ dependencies = [ [[package]] name = "windmill-api-assets" -version = "1.795.0" +version = "1.796.0" dependencies = [ "axum 0.8.9", "chrono", @@ -14981,7 +14974,7 @@ dependencies = [ [[package]] name = "windmill-api-auth" -version = "1.795.0" +version = "1.796.0" dependencies = [ "anyhow", "axum 0.8.9", @@ -15007,7 +15000,7 @@ dependencies = [ [[package]] name = "windmill-api-client" -version = "1.795.0" +version = "1.796.0" dependencies = [ "reqwest 0.12.28", "serde", @@ -15017,7 +15010,7 @@ dependencies = [ [[package]] name = "windmill-api-configs" -version = "1.795.0" +version = "1.796.0" dependencies = [ "axum 0.8.9", "chrono", @@ -15034,7 +15027,7 @@ dependencies = [ [[package]] name = "windmill-api-debug" -version = "1.795.0" +version = "1.796.0" dependencies = [ "axum 0.8.9", "base64 0.22.1", @@ -15056,7 +15049,7 @@ dependencies = [ [[package]] name = "windmill-api-embeddings" -version = "1.795.0" +version = "1.796.0" dependencies = [ "anyhow", "axum 0.8.9", @@ -15079,7 +15072,7 @@ dependencies = [ [[package]] name = "windmill-api-flow-conversations" -version = "1.795.0" +version = "1.796.0" dependencies = [ "axum 0.8.9", "chrono", @@ -15095,7 +15088,7 @@ dependencies = [ [[package]] name = "windmill-api-flows" -version = "1.795.0" +version = "1.796.0" dependencies = [ "axum 0.8.9", "chrono", @@ -15117,7 +15110,7 @@ dependencies = [ [[package]] name = "windmill-api-groups" -version = "1.795.0" +version = "1.796.0" dependencies = [ "axum 0.8.9", "chrono", @@ -15138,7 +15131,7 @@ dependencies = [ [[package]] name = "windmill-api-inputs" -version = "1.795.0" +version = "1.796.0" dependencies = [ "axum 0.8.9", "chrono", @@ -15152,7 +15145,7 @@ dependencies = [ [[package]] name = "windmill-api-integration-tests" -version = "1.795.0" +version = "1.796.0" dependencies = [ "anyhow", "async-nats", @@ -15187,7 +15180,7 @@ dependencies = [ [[package]] name = "windmill-api-jobs" -version = "1.795.0" +version = "1.796.0" dependencies = [ "anyhow", "axum 0.8.9", @@ -15212,7 +15205,7 @@ dependencies = [ [[package]] name = "windmill-api-npm-proxy" -version = "1.795.0" +version = "1.796.0" dependencies = [ "axum 0.8.9", "chrono", @@ -15240,7 +15233,7 @@ dependencies = [ [[package]] name = "windmill-api-openapi" -version = "1.795.0" +version = "1.796.0" dependencies = [ "anyhow", "axum 0.8.9", @@ -15262,7 +15255,7 @@ dependencies = [ [[package]] name = "windmill-api-schedule" -version = "1.795.0" +version = "1.796.0" dependencies = [ "axum 0.8.9", "chrono", @@ -15282,7 +15275,7 @@ dependencies = [ [[package]] name = "windmill-api-scripts" -version = "1.795.0" +version = "1.796.0" dependencies = [ "axum 0.8.9", "chrono", @@ -15320,7 +15313,7 @@ dependencies = [ [[package]] name = "windmill-api-settings" -version = "1.795.0" +version = "1.796.0" dependencies = [ "anyhow", "axum 0.8.9", @@ -15348,7 +15341,7 @@ dependencies = [ [[package]] name = "windmill-api-sse" -version = "1.795.0" +version = "1.796.0" dependencies = [ "lazy_static", "serde", @@ -15360,7 +15353,7 @@ dependencies = [ [[package]] name = "windmill-api-users" -version = "1.795.0" +version = "1.796.0" dependencies = [ "argon2", "axum 0.8.9", @@ -15384,7 +15377,7 @@ dependencies = [ [[package]] name = "windmill-api-workers" -version = "1.795.0" +version = "1.796.0" dependencies = [ "axum 0.8.9", "chrono", @@ -15398,7 +15391,7 @@ dependencies = [ [[package]] name = "windmill-api-workspaces" -version = "1.795.0" +version = "1.796.0" dependencies = [ "axum 0.8.9", "chrono", @@ -15433,7 +15426,7 @@ dependencies = [ [[package]] name = "windmill-audit" -version = "1.795.0" +version = "1.796.0" dependencies = [ "chrono", "lazy_static", @@ -15447,7 +15440,7 @@ dependencies = [ [[package]] name = "windmill-autoscaling" -version = "1.795.0" +version = "1.796.0" dependencies = [ "anyhow", "axum 0.8.9", @@ -15466,7 +15459,7 @@ dependencies = [ [[package]] name = "windmill-common" -version = "1.795.0" +version = "1.796.0" dependencies = [ "aes-gcm", "aho-corasick", @@ -15570,7 +15563,7 @@ dependencies = [ [[package]] name = "windmill-dep-map" -version = "1.795.0" +version = "1.796.0" dependencies = [ "chrono", "itertools 0.14.0", @@ -15589,7 +15582,7 @@ dependencies = [ [[package]] name = "windmill-git-sync" -version = "1.795.0" +version = "1.796.0" dependencies = [ "regex", "serde", @@ -15604,7 +15597,7 @@ dependencies = [ [[package]] name = "windmill-indexer" -version = "1.795.0" +version = "1.796.0" dependencies = [ "anyhow", "astral-tokio-tar", @@ -15628,7 +15621,7 @@ dependencies = [ [[package]] name = "windmill-jseval" -version = "1.795.0" +version = "1.796.0" dependencies = [ "anyhow", "futures", @@ -15645,7 +15638,7 @@ dependencies = [ [[package]] name = "windmill-macros" -version = "1.795.0" +version = "1.796.0" dependencies = [ "itertools 0.14.0", "lazy_static", @@ -15661,7 +15654,7 @@ dependencies = [ [[package]] name = "windmill-mcp" -version = "1.795.0" +version = "1.796.0" dependencies = [ "anyhow", "async-trait", @@ -15682,7 +15675,7 @@ dependencies = [ [[package]] name = "windmill-native-triggers" -version = "1.795.0" +version = "1.796.0" dependencies = [ "anyhow", "async-trait", @@ -15713,7 +15706,7 @@ dependencies = [ [[package]] name = "windmill-oauth" -version = "1.795.0" +version = "1.796.0" dependencies = [ "anyhow", "arc-swap", @@ -15738,7 +15731,7 @@ dependencies = [ [[package]] name = "windmill-object-store" -version = "1.795.0" +version = "1.796.0" dependencies = [ "anyhow", "async-stream", @@ -15772,7 +15765,7 @@ dependencies = [ [[package]] name = "windmill-operator" -version = "1.795.0" +version = "1.796.0" dependencies = [ "anyhow", "futures", @@ -15790,7 +15783,7 @@ dependencies = [ [[package]] name = "windmill-parser" -version = "1.795.0" +version = "1.796.0" dependencies = [ "convert_case 0.6.0", "serde", @@ -15799,7 +15792,7 @@ dependencies = [ [[package]] name = "windmill-parser-bash" -version = "1.795.0" +version = "1.796.0" dependencies = [ "anyhow", "lazy_static", @@ -15811,7 +15804,7 @@ dependencies = [ [[package]] name = "windmill-parser-csharp" -version = "1.795.0" +version = "1.796.0" dependencies = [ "anyhow", "serde_json", @@ -15823,7 +15816,7 @@ dependencies = [ [[package]] name = "windmill-parser-go" -version = "1.795.0" +version = "1.796.0" dependencies = [ "anyhow", "gosyn", @@ -15835,7 +15828,7 @@ dependencies = [ [[package]] name = "windmill-parser-graphql" -version = "1.795.0" +version = "1.796.0" dependencies = [ "anyhow", "lazy_static", @@ -15847,7 +15840,7 @@ dependencies = [ [[package]] name = "windmill-parser-java" -version = "1.795.0" +version = "1.796.0" dependencies = [ "anyhow", "serde_json", @@ -15859,7 +15852,7 @@ dependencies = [ [[package]] name = "windmill-parser-nu" -version = "1.795.0" +version = "1.796.0" dependencies = [ "anyhow", "nu-parser", @@ -15870,7 +15863,7 @@ dependencies = [ [[package]] name = "windmill-parser-php" -version = "1.795.0" +version = "1.796.0" dependencies = [ "anyhow", "itertools 0.14.0", @@ -15881,7 +15874,7 @@ dependencies = [ [[package]] name = "windmill-parser-py" -version = "1.795.0" +version = "1.796.0" dependencies = [ "anyhow", "itertools 0.14.0", @@ -15893,7 +15886,7 @@ dependencies = [ [[package]] name = "windmill-parser-py-asset" -version = "1.795.0" +version = "1.796.0" dependencies = [ "anyhow", "rustpython-ast", @@ -15904,7 +15897,7 @@ dependencies = [ [[package]] name = "windmill-parser-py-imports" -version = "1.795.0" +version = "1.796.0" dependencies = [ "anyhow", "async-recursion", @@ -15926,7 +15919,7 @@ dependencies = [ [[package]] name = "windmill-parser-r" -version = "1.795.0" +version = "1.796.0" dependencies = [ "anyhow", "serde_json", @@ -15938,7 +15931,7 @@ dependencies = [ [[package]] name = "windmill-parser-ruby" -version = "1.795.0" +version = "1.796.0" dependencies = [ "anyhow", "lazy_static", @@ -15952,7 +15945,7 @@ dependencies = [ [[package]] name = "windmill-parser-rust" -version = "1.795.0" +version = "1.796.0" dependencies = [ "anyhow", "convert_case 0.6.0", @@ -15969,7 +15962,7 @@ dependencies = [ [[package]] name = "windmill-parser-sql" -version = "1.795.0" +version = "1.796.0" dependencies = [ "anyhow", "lazy_static", @@ -15982,7 +15975,7 @@ dependencies = [ [[package]] name = "windmill-parser-sql-asset" -version = "1.795.0" +version = "1.796.0" dependencies = [ "anyhow", "serde", @@ -15994,7 +15987,7 @@ dependencies = [ [[package]] name = "windmill-parser-ts" -version = "1.795.0" +version = "1.796.0" dependencies = [ "anyhow", "lazy_static", @@ -16012,7 +16005,7 @@ dependencies = [ [[package]] name = "windmill-parser-ts-asset" -version = "1.795.0" +version = "1.796.0" dependencies = [ "anyhow", "serde-wasm-bindgen", @@ -16028,7 +16021,7 @@ dependencies = [ [[package]] name = "windmill-parser-wac" -version = "1.795.0" +version = "1.796.0" dependencies = [ "anyhow", "rustpython-ast", @@ -16044,7 +16037,7 @@ dependencies = [ [[package]] name = "windmill-parser-yaml" -version = "1.795.0" +version = "1.796.0" dependencies = [ "anyhow", "lazy_static", @@ -16058,7 +16051,7 @@ dependencies = [ [[package]] name = "windmill-queue" -version = "1.795.0" +version = "1.796.0" dependencies = [ "anyhow", "async-recursion", @@ -16097,7 +16090,7 @@ dependencies = [ [[package]] name = "windmill-runtime-nativets" -version = "1.795.0" +version = "1.796.0" dependencies = [ "anyhow", "const_format", @@ -16137,7 +16130,7 @@ dependencies = [ [[package]] name = "windmill-sql-datatype-parser-wasm" -version = "1.795.0" +version = "1.796.0" dependencies = [ "getrandom 0.3.4", "wasm-bindgen", @@ -16148,7 +16141,7 @@ dependencies = [ [[package]] name = "windmill-store" -version = "1.795.0" +version = "1.796.0" dependencies = [ "anyhow", "async-recursion", @@ -16183,7 +16176,7 @@ dependencies = [ [[package]] name = "windmill-test-utils" -version = "1.795.0" +version = "1.796.0" dependencies = [ "anyhow", "async-trait", @@ -16207,7 +16200,7 @@ dependencies = [ [[package]] name = "windmill-trigger" -version = "1.795.0" +version = "1.796.0" dependencies = [ "anyhow", "async-trait", @@ -16240,7 +16233,7 @@ dependencies = [ [[package]] name = "windmill-trigger-amqp" -version = "1.795.0" +version = "1.796.0" dependencies = [ "anyhow", "async-trait", @@ -16267,7 +16260,7 @@ dependencies = [ [[package]] name = "windmill-trigger-azure" -version = "1.795.0" +version = "1.796.0" dependencies = [ "anyhow", "async-trait", @@ -16300,7 +16293,7 @@ dependencies = [ [[package]] name = "windmill-trigger-email" -version = "1.795.0" +version = "1.796.0" dependencies = [ "anyhow", "async-trait", @@ -16320,7 +16313,7 @@ dependencies = [ [[package]] name = "windmill-trigger-gcp" -version = "1.795.0" +version = "1.796.0" dependencies = [ "anyhow", "async-trait", @@ -16354,7 +16347,7 @@ dependencies = [ [[package]] name = "windmill-trigger-http" -version = "1.795.0" +version = "1.796.0" dependencies = [ "anyhow", "async-trait", @@ -16390,7 +16383,7 @@ dependencies = [ [[package]] name = "windmill-trigger-kafka" -version = "1.795.0" +version = "1.796.0" dependencies = [ "anyhow", "async-trait", @@ -16413,7 +16406,7 @@ dependencies = [ [[package]] name = "windmill-trigger-mqtt" -version = "1.795.0" +version = "1.796.0" dependencies = [ "anyhow", "async-trait", @@ -16437,7 +16430,7 @@ dependencies = [ [[package]] name = "windmill-trigger-nats" -version = "1.795.0" +version = "1.796.0" dependencies = [ "anyhow", "async-nats", @@ -16461,7 +16454,7 @@ dependencies = [ [[package]] name = "windmill-trigger-postgres" -version = "1.795.0" +version = "1.796.0" dependencies = [ "anyhow", "async-trait", @@ -16496,7 +16489,7 @@ dependencies = [ [[package]] name = "windmill-trigger-sqs" -version = "1.795.0" +version = "1.796.0" dependencies = [ "anyhow", "async-trait", @@ -16524,7 +16517,7 @@ dependencies = [ [[package]] name = "windmill-trigger-websocket" -version = "1.795.0" +version = "1.796.0" dependencies = [ "anyhow", "async-trait", @@ -16549,7 +16542,7 @@ dependencies = [ [[package]] name = "windmill-types" -version = "1.795.0" +version = "1.796.0" dependencies = [ "anyhow", "bitflags 2.13.1", @@ -16568,7 +16561,7 @@ dependencies = [ [[package]] name = "windmill-worker" -version = "1.795.0" +version = "1.796.0" dependencies = [ "anyhow", "async-once-cell", @@ -16685,7 +16678,7 @@ dependencies = [ [[package]] name = "windmill-worker-volumes" -version = "1.795.0" +version = "1.796.0" dependencies = [ "bytes", "futures", @@ -17474,7 +17467,7 @@ checksum = "34df6fc39dbd26ddc9c10e6a2984476e13acce22e64e4487636ef494369225da" dependencies = [ "proc-macro2", "quote", - "syn 3.0.3", + "syn 3.0.4", ] [[package]] diff --git a/backend/Cargo.toml b/backend/Cargo.toml index 0bfa52d3e6..6dd9e452b8 100644 --- a/backend/Cargo.toml +++ b/backend/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "windmill" -version = "1.795.0" +version = "1.796.0" authors.workspace = true edition.workspace = true @@ -88,7 +88,7 @@ members = [ exclude = ["./windmill-duckdb-ffi-internal", "./parsers/windmill-parser-wasm"] [workspace.package] -version = "1.795.0" +version = "1.796.0" authors = ["Ruben Fiszel "] edition = "2021" @@ -100,6 +100,10 @@ path = "./src/main.rs" opt-level = 0 incremental = true split-debuginfo = "unpacked" +# Type and variable DWARF is the single largest thing in target/ and nothing in the dev loop +# reads it; backtraces only need the line tables, which this keeps. Raise to `true` when you +# actually need to inspect variables in gdb/lldb. +debug = "line-tables-only" [profile.dev.package."*"] debug = false @@ -214,6 +218,26 @@ all_sqlx_features = ["all_languages", "enterprise", "enterprise_saml", "embeddin "windmill-git-sync/all_sqlx_features"] [patch.crates-io] +# v0.8.6 plus one commit: `Pool::begin` is not cancel-safe on Postgres. sqlx raises the +# transaction depth its rollback-on-drop guard keys on only *after* the BEGIN round trip, so +# a cancelled caller (a disconnecting API client, a `timeout`, an aborted task) leaves the +# session in a transaction nothing will end, and the pool hands that connection out again — +# every later query on it fails with 25P02 until max_lifetime recycles it 30 minutes on. +# Reported upstream in 2022 (launchbadge/sqlx#2054), fixed for SQLite only, and still present +# in 0.9.0. Drop this the moment upstream carries the fix. +# The whole family has to move together: `sqlx-postgres` depends on `sqlx-core` by path +# inside the sqlx workspace, so patching it alone leaves two incompatible `sqlx-core`s and +# `Postgres` stops implementing the `Database` the macros expect. +# Changing any of this — a bump, a rebase of the fork, dropping these lines — still compiles +# clean, so run the guard that actually checks the behaviour is still there: +# cargo test -p windmill-common --test sqlx_begin_cancel_safe -- --ignored +sqlx = { git = "https://github.com/windmill-labs/sqlx", rev = "6bdaee94fa62a01561125646da3f99eb341f2457" } +sqlx-core = { git = "https://github.com/windmill-labs/sqlx", rev = "6bdaee94fa62a01561125646da3f99eb341f2457" } +sqlx-macros = { git = "https://github.com/windmill-labs/sqlx", rev = "6bdaee94fa62a01561125646da3f99eb341f2457" } +sqlx-macros-core = { git = "https://github.com/windmill-labs/sqlx", rev = "6bdaee94fa62a01561125646da3f99eb341f2457" } +sqlx-postgres = { git = "https://github.com/windmill-labs/sqlx", rev = "6bdaee94fa62a01561125646da3f99eb341f2457" } +sqlx-mysql = { git = "https://github.com/windmill-labs/sqlx", rev = "6bdaee94fa62a01561125646da3f99eb341f2457" } +sqlx-sqlite = { git = "https://github.com/windmill-labs/sqlx", rev = "6bdaee94fa62a01561125646da3f99eb341f2457" } object_store = { git = "https://github.com/apache/arrow-rs-object-store", rev = "36752c975d4f29e20b57c91f81a10872dcd48ae7" } # Use tiberius main branch for libgssapi 0.8.1 fix (https://github.com/prisma/tiberius/issues/343) tiberius = { git = "https://github.com/prisma/tiberius", rev = "59db57960a14b422fb3a1309aa4aa47880896ff8" } diff --git a/backend/migrations/20260812100659_ai_evals.down.sql b/backend/migrations/20260812100659_ai_evals.down.sql new file mode 100644 index 0000000000..2e9383c761 --- /dev/null +++ b/backend/migrations/20260812100659_ai_evals.down.sql @@ -0,0 +1,6 @@ +DROP TABLE IF EXISTS eval_score; +DROP TABLE IF EXISTS eval_experiment_case; +DROP TABLE IF EXISTS eval_experiment; +DROP TABLE IF EXISTS eval_case; +DROP TABLE IF EXISTS eval_dataset; +DROP FUNCTION IF EXISTS eval_dataset_writable(varchar, varchar); diff --git a/backend/migrations/20260812100659_ai_evals.up.sql b/backend/migrations/20260812100659_ai_evals.up.sql new file mode 100644 index 0000000000..7ef82d5f7e --- /dev/null +++ b/backend/migrations/20260812100659_ai_evals.up.sql @@ -0,0 +1,275 @@ +-- Eval datasets and the cases they hold. Path-addressed like every other Windmill object, so the +-- folder a dataset is named by is what grants access to it. +CREATE TABLE eval_dataset ( + workspace_id VARCHAR(50) NOT NULL, + path VARCHAR(255) NOT NULL, + summary VARCHAR(1000) NULL, + -- The scorers a dataset is scored by. One entry per column of the results table: + -- {id, name, kind, ...kind-specific config}. `id` is assigned once and never reused, so a + -- column stays the same column across experiments when it is renamed or its definition is + -- edited — which is what makes a delta between two experiments meaningful. + scorers JSONB NOT NULL DEFAULT '[]', + extra_perms JSONB NOT NULL DEFAULT '{}', + created_at TIMESTAMPTZ NOT NULL DEFAULT now(), + created_by VARCHAR(50) NOT NULL, + edited_at TIMESTAMPTZ NOT NULL DEFAULT now(), + edited_by VARCHAR(50) NOT NULL, + PRIMARY KEY (workspace_id, path), + FOREIGN KEY (workspace_id) REFERENCES workspace(id) ON DELETE CASCADE +); + +-- A case is the input half of one evaluation: what the agent is fed, and what it was expected to +-- answer. The generated output, the trajectory and every scorer's return value are the job's, not +-- this table's. +-- +-- ON UPDATE CASCADE so renaming a dataset carries its cases instead of stranding them. +CREATE TABLE eval_case ( + id UUID PRIMARY KEY DEFAULT gen_random_uuid(), + workspace_id VARCHAR(50) NOT NULL, + dataset_path VARCHAR(255) NOT NULL, + -- {user_message, user_attachments} + input JSONB NOT NULL DEFAULT '{}', + expected JSONB NULL, + created_at TIMESTAMPTZ NOT NULL DEFAULT now(), + created_by VARCHAR(50) NOT NULL, + FOREIGN KEY (workspace_id, dataset_path) REFERENCES eval_dataset (workspace_id, path) + ON DELETE CASCADE ON UPDATE CASCADE +); + +-- Serves the paginated case list, which is ordered oldest-first so a case keeps its position as +-- the dataset grows. +CREATE INDEX index_eval_case_dataset ON eval_case (workspace_id, dataset_path, created_at, id); + +GRANT ALL ON eval_dataset TO windmill_user; +GRANT ALL ON eval_dataset TO windmill_admin; +GRANT ALL ON eval_case TO windmill_user; +GRANT ALL ON eval_case TO windmill_admin; + +ALTER TABLE eval_dataset ENABLE ROW LEVEL SECURITY; +ALTER TABLE eval_case ENABLE ROW LEVEL SECURITY; + +CREATE POLICY admin_policy ON eval_dataset FOR ALL TO windmill_admin USING (true); +CREATE POLICY admin_policy ON eval_case FOR ALL TO windmill_admin USING (true); + +CREATE POLICY see_folder_extra_perms_user_select ON eval_dataset FOR SELECT TO windmill_user +USING (SPLIT_PART(eval_dataset.path, '/', 1) = 'f' AND SPLIT_PART(eval_dataset.path, '/', 2) = any((select regexp_split_to_array(current_setting('session.folders_read'), ','))::text[])); +CREATE POLICY see_folder_extra_perms_user_insert ON eval_dataset FOR INSERT TO windmill_user +WITH CHECK (SPLIT_PART(eval_dataset.path, '/', 1) = 'f' AND SPLIT_PART(eval_dataset.path, '/', 2) = any((select regexp_split_to_array(current_setting('session.folders_write'), ','))::text[])); +CREATE POLICY see_folder_extra_perms_user_update ON eval_dataset FOR UPDATE TO windmill_user +USING (SPLIT_PART(eval_dataset.path, '/', 1) = 'f' AND SPLIT_PART(eval_dataset.path, '/', 2) = any((select regexp_split_to_array(current_setting('session.folders_write'), ','))::text[])); +CREATE POLICY see_folder_extra_perms_user_delete ON eval_dataset FOR DELETE TO windmill_user +USING (SPLIT_PART(eval_dataset.path, '/', 1) = 'f' AND SPLIT_PART(eval_dataset.path, '/', 2) = any((select regexp_split_to_array(current_setting('session.folders_write'), ','))::text[])); + +CREATE POLICY see_own ON eval_dataset FOR ALL TO windmill_user +USING (SPLIT_PART(eval_dataset.path, '/', 1) = 'u' AND SPLIT_PART(eval_dataset.path, '/', 2) = (select current_setting('session.user'))); +CREATE POLICY see_member ON eval_dataset FOR ALL TO windmill_user +USING (SPLIT_PART(eval_dataset.path, '/', 1) = 'g' AND SPLIT_PART(eval_dataset.path, '/', 2) = any((select regexp_split_to_array(current_setting('session.groups'), ','))::text[])); + +CREATE POLICY see_extra_perms_user_select ON eval_dataset FOR SELECT TO windmill_user +USING (extra_perms ? (select concat('u/', current_setting('session.user')))); +CREATE POLICY see_extra_perms_user_insert ON eval_dataset FOR INSERT TO windmill_user +WITH CHECK ((extra_perms ->> (select concat('u/', current_setting('session.user'))))::boolean); +CREATE POLICY see_extra_perms_user_update ON eval_dataset FOR UPDATE TO windmill_user +USING ((extra_perms ->> (select concat('u/', current_setting('session.user'))))::boolean); +CREATE POLICY see_extra_perms_user_delete ON eval_dataset FOR DELETE TO windmill_user +USING ((extra_perms ->> (select concat('u/', current_setting('session.user'))))::boolean); + +CREATE POLICY see_extra_perms_groups_select ON eval_dataset FOR SELECT TO windmill_user +USING (extra_perms ?| (select regexp_split_to_array(current_setting('session.pgroups'), ','))::text[]); +CREATE POLICY see_extra_perms_groups_insert ON eval_dataset FOR INSERT TO windmill_user +WITH CHECK (exists( + SELECT key, value FROM jsonb_each_text(extra_perms) + WHERE SPLIT_PART(key, '/', 1) = 'g' AND key = ANY((select regexp_split_to_array(current_setting('session.pgroups'), ','))::text[]) + AND value::boolean)); +CREATE POLICY see_extra_perms_groups_update ON eval_dataset FOR UPDATE TO windmill_user +USING (exists( + SELECT key, value FROM jsonb_each_text(extra_perms) + WHERE SPLIT_PART(key, '/', 1) = 'g' AND key = ANY((select regexp_split_to_array(current_setting('session.pgroups'), ','))::text[]) + AND value::boolean)); +CREATE POLICY see_extra_perms_groups_delete ON eval_dataset FOR DELETE TO windmill_user +USING (exists( + SELECT key, value FROM jsonb_each_text(extra_perms) + WHERE SPLIT_PART(key, '/', 1) = 'g' AND key = ANY((select regexp_split_to_array(current_setting('session.pgroups'), ','))::text[]) + AND value::boolean)); + +-- Whether the session may *write* the dataset at (_workspace_id, _path): the same disjunction the +-- dataset's own write policies use, in one place so the cases that hang off a dataset are governed +-- by exactly the rule the dataset is. A read grant is not enough — writing a case is writing the +-- dataset's contents — so this checks write, not merely visibility. +CREATE OR REPLACE FUNCTION eval_dataset_writable(_workspace_id varchar, _path varchar) + RETURNS boolean LANGUAGE sql STABLE AS $$ + SELECT EXISTS ( + SELECT 1 FROM eval_dataset d + WHERE d.workspace_id = _workspace_id AND d.path = _path + AND ( + (SPLIT_PART(d.path, '/', 1) = 'f' AND SPLIT_PART(d.path, '/', 2) = any((select regexp_split_to_array(current_setting('session.folders_write'), ','))::text[])) + OR (SPLIT_PART(d.path, '/', 1) = 'u' AND SPLIT_PART(d.path, '/', 2) = (select current_setting('session.user'))) + OR (SPLIT_PART(d.path, '/', 1) = 'g' AND SPLIT_PART(d.path, '/', 2) = any((select regexp_split_to_array(current_setting('session.groups'), ','))::text[])) + OR ((d.extra_perms ->> (select concat('u/', current_setting('session.user'))))::boolean) + OR EXISTS ( + SELECT 1 FROM jsonb_each_text(d.extra_perms) ep + WHERE SPLIT_PART(ep.key, '/', 1) = 'g' + AND ep.key = ANY((select regexp_split_to_array(current_setting('session.pgroups'), ','))::text[]) + AND ep.value::boolean) + ) + ); +$$; + +-- Cases are the *contents* of a dataset, not independently addressable objects, so both their +-- visibility and who may change them are the parent's, stated once here instead of mirrored in the +-- API and left to drift. Read is the dataset's read (the subquery is itself subject to +-- eval_dataset's SELECT policies above); write is the dataset's write, which `eval_dataset_writable` +-- checks — so a read-only grant on a dataset can list its cases but not edit them. The whole edit +-- of a dataset and its cases therefore runs as one `user_db` transaction, governed by these +-- policies, rather than being split across the unrestricted pool after a hand-written check. +CREATE POLICY see_parent_dataset ON eval_case FOR SELECT TO windmill_user +USING ( + EXISTS ( + SELECT 1 FROM eval_dataset d + WHERE d.workspace_id = eval_case.workspace_id AND d.path = eval_case.dataset_path + ) +); +CREATE POLICY write_parent_dataset_insert ON eval_case FOR INSERT TO windmill_user +WITH CHECK (eval_dataset_writable(eval_case.workspace_id, eval_case.dataset_path)); +CREATE POLICY write_parent_dataset_update ON eval_case FOR UPDATE TO windmill_user +USING (eval_dataset_writable(eval_case.workspace_id, eval_case.dataset_path)) +WITH CHECK (eval_dataset_writable(eval_case.workspace_id, eval_case.dataset_path)); +CREATE POLICY write_parent_dataset_delete ON eval_case FOR DELETE TO windmill_user +USING (eval_dataset_writable(eval_case.workspace_id, eval_case.dataset_path)); +-- One run of a dataset: written once when the dataset is run, and only ever read afterwards, +-- which is what makes it worth comparing against. +CREATE TABLE eval_experiment ( + id UUID PRIMARY KEY, + workspace_id VARCHAR(50) NOT NULL, + dataset_path VARCHAR(255) NOT NULL, + -- {kind, path, version}: what was run, at the version it was at when the run was enqueued. + subject JSONB NOT NULL, + -- A run is named by the number it is: "Run 7" is stable, sorts, and survives history being + -- pruned, which a position computed at read time would not. Allocated per (dataset, subject + -- path) when the run is opened. + run_number INTEGER NOT NULL, + -- A run is one flow: a loop over the cases, each iteration answering and then scoring. This + -- is the job holding it, so the run can be watched, cancelled and rerun as the single thing + -- it is. Assigned before the flow is pushed, so a launch that dies partway leaves an + -- experiment naming a job that never started rather than a flow nothing accounts for. + run_job_id UUID NOT NULL, + created_at TIMESTAMPTZ NOT NULL DEFAULT now(), + created_by VARCHAR(50) NOT NULL, + FOREIGN KEY (workspace_id, dataset_path) REFERENCES eval_dataset (workspace_id, path) + ON DELETE CASCADE ON UPDATE CASCADE +); + +CREATE INDEX index_eval_experiment_dataset ON eval_experiment (workspace_id, dataset_path, created_at DESC); + +-- Serves the per-agent run list, which spans datasets: an agent's history is one list whichever +-- dataset each run was of. +CREATE INDEX index_eval_experiment_subject ON eval_experiment + (workspace_id, (subject ->> 'path'), created_at DESC); + +-- The exact case set an experiment ran, by value: a dataset keeps changing, and a result set that +-- cannot say which inputs produced it is not reproducible. `case_id` is therefore deliberately not +-- a foreign key — deleting a case must not rewrite the history of the runs that used it. +CREATE TABLE eval_experiment_case ( + experiment_id UUID NOT NULL REFERENCES eval_experiment (id) ON DELETE CASCADE, + ordinal INT NOT NULL, + case_id UUID NOT NULL, + input JSONB NOT NULL DEFAULT '{}', + expected JSONB NULL, + -- The iteration of the run's flow that answered this case. Minted by the flow engine, so the + -- case is recorded before it has one and the id is filled in once the iterations exist. + job_id UUID NULL, + -- What the run produced, copied out of the jobs once they have produced it. Jobs have their + -- own retention, and a recorded run has to still read as the run it was once they are gone. + -- `answered` is the agent step's own outcome, which is settled while the iteration around it + -- is still scoring; `status` is the iteration's, once it has one. + output TEXT NULL, + answered BOOLEAN NULL, + status VARCHAR(30) NULL, + -- The resource version the agent was at for this cell, and — for a draft, which has no + -- version to move — the hash of the configuration that actually ran: the only thing that can + -- say a row describes an agent that has since been edited. + subject_version BIGINT NULL, + subject_draft_hash VARCHAR(64) NULL, + started_at TIMESTAMPTZ NOT NULL DEFAULT now(), + PRIMARY KEY (experiment_id, ordinal), + -- A run holds each case once: the pair is what identifies a cell. + CONSTRAINT eval_experiment_case_unique_case UNIQUE (experiment_id, case_id) +); + +-- One scorer's verdict on one run. Separate from the run because scoring is separate from running: +-- a scorer's verdict is stored per run and scorer, independent of the agent execution that +-- produced the answers. +CREATE TABLE eval_score ( + experiment_id UUID NOT NULL, + ordinal INT NOT NULL, + scorer_id VARCHAR(64) NOT NULL, + -- NULL until the verdict has been read out of the run's flow, and when scoring failed. + score DOUBLE PRECISION NULL, + reason TEXT NULL, + -- [{name, passed, detail}], for scorers that report per-assertion results. + checks JSONB NULL, + error TEXT NULL, + -- The scorer read the run and said it had nothing to measure on this case. A verdict, not a + -- failure: the cell is left out of the column's mean and pass rate rather than counted as a + -- zero or reported as a scorer that produced nothing. + not_applicable BOOLEAN NOT NULL DEFAULT false, + -- Hash of the scorer configuration that produced this score, including the script hash or flow + -- version actually executed. Two scores of the same scorer whose definitions differ are still + -- compared, but the column says the scorer changed rather than letting it read as a change of + -- agent. + definition VARCHAR(64) NOT NULL, + created_at TIMESTAMPTZ NOT NULL DEFAULT now(), + PRIMARY KEY (experiment_id, ordinal, scorer_id), + FOREIGN KEY (experiment_id, ordinal) REFERENCES eval_experiment_case (experiment_id, ordinal) + ON DELETE CASCADE +); + +GRANT ALL ON eval_experiment TO windmill_user; +GRANT ALL ON eval_experiment TO windmill_admin; +GRANT ALL ON eval_experiment_case TO windmill_user; +GRANT ALL ON eval_experiment_case TO windmill_admin; +GRANT ALL ON eval_score TO windmill_user; +GRANT ALL ON eval_score TO windmill_admin; + +ALTER TABLE eval_experiment ENABLE ROW LEVEL SECURITY; +ALTER TABLE eval_experiment_case ENABLE ROW LEVEL SECURITY; +ALTER TABLE eval_score ENABLE ROW LEVEL SECURITY; + +CREATE POLICY admin_policy ON eval_experiment FOR ALL TO windmill_admin USING (true); +CREATE POLICY admin_policy ON eval_experiment_case FOR ALL TO windmill_admin USING (true); +CREATE POLICY admin_policy ON eval_score FOR ALL TO windmill_admin USING (true); + +-- Experiments are the *contents* of a dataset, not independently addressable objects, so their +-- visibility is the parent's: the subquery is itself subject to eval_dataset's policies, which +-- stay stated once instead of being mirrored here and left to drift. +-- +-- SELECT only, deliberately. A `FOR ALL ... USING` would be reused as the INSERT/UPDATE/DELETE +-- check expression, and since the subquery is a SELECT it applies the dataset's *read* policies — +-- which would let someone with read-only access to a dataset forge an experiment row naming a job +-- they cannot otherwise read. Writes are done on the unrestricted pool after the API has checked +-- write access to the parent, and a stray `user_db` write to these tables is meant to fail rather +-- than silently succeed. +CREATE POLICY see_parent_dataset ON eval_experiment FOR SELECT TO windmill_user +USING ( + EXISTS ( + SELECT 1 FROM eval_dataset d + WHERE d.workspace_id = eval_experiment.workspace_id AND d.path = eval_experiment.dataset_path + ) +); + +CREATE POLICY see_parent_experiment ON eval_experiment_case FOR SELECT TO windmill_user +USING ( + EXISTS ( + SELECT 1 FROM eval_experiment e + WHERE e.id = eval_experiment_case.experiment_id + ) +); + +-- Visibility is the experiment's, which is the dataset's. +CREATE POLICY see_parent_experiment ON eval_score FOR SELECT TO windmill_user +USING ( + EXISTS ( + SELECT 1 FROM eval_experiment e + WHERE e.id = eval_score.experiment_id + ) +); diff --git a/backend/migrations/20260813164338_ai_token_usage.down.sql b/backend/migrations/20260813164338_ai_token_usage.down.sql new file mode 100644 index 0000000000..0fa1b47462 --- /dev/null +++ b/backend/migrations/20260813164338_ai_token_usage.down.sql @@ -0,0 +1 @@ +DROP TABLE ai_token_usage; diff --git a/backend/migrations/20260813164338_ai_token_usage.up.sql b/backend/migrations/20260813164338_ai_token_usage.up.sql new file mode 100644 index 0000000000..b20e471db5 --- /dev/null +++ b/backend/migrations/20260813164338_ai_token_usage.up.sql @@ -0,0 +1,42 @@ +-- Per-workspace AI token spend, accumulated from the chat client. Rows hold token +-- counts rather than money: prices live in the frontend price table plus the +-- workspace's `ai_config.model_pricing` overrides and are applied at read time, so +-- correcting a price also corrects the history. `reported_cost_nano_usd` is the +-- exception — a few providers (OpenRouter) return what they actually charged, and +-- that figure wins over the estimate. +-- +-- Distinct from `feature_usage`, which is anonymous telemetry that leaves the +-- instance and is pruned after 60 days; spend is per-user and kept. +CREATE TABLE ai_token_usage ( + workspace_id VARCHAR(50) NOT NULL REFERENCES workspace(id) ON DELETE CASCADE, + day DATE NOT NULL DEFAULT CURRENT_DATE, + email VARCHAR(255) NOT NULL, + provider VARCHAR(50) NOT NULL, + model VARCHAR(255) NOT NULL, + -- Empty for chats that are not attached to an AI session. + session_id VARCHAR(50) NOT NULL DEFAULT '', + -- Uncached input only; the two cache columns hold the rest of the prompt, so + -- each column maps to exactly one price and they never double-count. + input_tokens BIGINT NOT NULL DEFAULT 0, + cache_read_tokens BIGINT NOT NULL DEFAULT 0, + cache_write_tokens BIGINT NOT NULL DEFAULT 0, + output_tokens BIGINT NOT NULL DEFAULT 0, + reported_cost_nano_usd BIGINT, + requests BIGINT NOT NULL DEFAULT 0, + updated_at TIMESTAMPTZ NOT NULL DEFAULT now(), + PRIMARY KEY (workspace_id, day, email, provider, model, session_id) +); + +-- The usage listing filters on workspace and a date range; the PK only reaches +-- `day` through `email`, so it cannot serve that on its own. +CREATE INDEX idx_ai_token_usage_ws_day ON ai_token_usage (workspace_id, day DESC); + +GRANT ALL ON ai_token_usage TO windmill_admin; +GRANT ALL ON ai_token_usage TO windmill_user; + +-- Both handlers go through the raw pool, so no policy is needed for them to work. +-- Enabling RLS with an admin-only policy is the backstop: a future query that +-- reaches this table through UserDB sees nothing rather than every user's spend. +ALTER TABLE ai_token_usage ENABLE ROW LEVEL SECURITY; + +CREATE POLICY admin_policy ON ai_token_usage FOR ALL TO windmill_admin USING (true); diff --git a/backend/migrations/20260819073729_resource_version_number.down.sql b/backend/migrations/20260819073729_resource_version_number.down.sql new file mode 100644 index 0000000000..13f95048ee --- /dev/null +++ b/backend/migrations/20260819073729_resource_version_number.down.sql @@ -0,0 +1,17 @@ +-- Back to numbering versions by the table-wide identity sequence, so the function must stop +-- writing a column that is about to go. +CREATE OR REPLACE FUNCTION record_resource_version() RETURNS trigger AS $$ +BEGIN + INSERT INTO resource_version (workspace_id, path, resource_type, value, created_by) + VALUES ( + NEW.workspace_id, NEW.path, NEW.resource_type, NEW.value, + COALESCE(NULLIF(current_setting('session.user', true), ''), NEW.created_by) + ); + + RETURN NEW; +END; +$$ LANGUAGE plpgsql SECURITY DEFINER SET search_path FROM CURRENT; + +DROP INDEX IF EXISTS index_resource_version_number; + +ALTER TABLE resource_version DROP COLUMN version; diff --git a/backend/migrations/20260819073729_resource_version_number.up.sql b/backend/migrations/20260819073729_resource_version_number.up.sql new file mode 100644 index 0000000000..e56dc7a09c --- /dev/null +++ b/backend/migrations/20260819073729_resource_version_number.up.sql @@ -0,0 +1,52 @@ +-- `id` is one identity sequence for the whole table and stays how a version is addressed; +-- `version` is the resource's own count, which is what a version is presented by. +ALTER TABLE resource_version ADD COLUMN version BIGINT; + +UPDATE resource_version rv SET version = ranked.rn +FROM ( + SELECT id, row_number() OVER (PARTITION BY workspace_id, path ORDER BY id) AS rn + FROM resource_version +) ranked +WHERE rv.id = ranked.id; + +ALTER TABLE resource_version ALTER COLUMN version SET NOT NULL; + +-- The number is only meaningful within a path, so the triple is the natural key: it serves the +-- lookup by number and makes a duplicate a hard error rather than two rows claiming v7. +CREATE UNIQUE INDEX index_resource_version_number ON resource_version (workspace_id, path, version); + +-- Numbering is assigned here rather than derived when read because both ways of deleting versions +-- take the oldest ones: the monitor's trim past MAX_RESOURCE_VERSIONS, and clearing a history down +-- to its current value. A number computed by counting the survivors would renumber under either, +-- so a run recorded against v3 would later name a different version. +CREATE OR REPLACE FUNCTION record_resource_version() RETURNS trigger AS $$ +BEGIN + -- `session.user` is set by UserDB::begin for authed requests; worker and system writes fall + -- back to the row's own author. NULLIF because a transaction-local set_config resets the + -- placeholder to the empty string rather than unsetting it, so a pooled connection that + -- previously served an authed request reports '' here, not NULL. + -- + -- MAX + 1 needs no lock of its own: this runs inside the transaction that wrote `resource`, and + -- a concurrent write to the same path blocks on that row's lock — or on the primary key, for an + -- insert — before its own trigger can run, so the maximum cannot be read stale. Deleting + -- versions never lowers it, since both deletions keep the newest row. + INSERT INTO resource_version (workspace_id, path, resource_type, value, created_by, version) + VALUES ( + NEW.workspace_id, NEW.path, NEW.resource_type, NEW.value, + COALESCE(NULLIF(current_setting('session.user', true), ''), NEW.created_by), + (SELECT COALESCE(MAX(version), 0) + 1 FROM resource_version + WHERE workspace_id = NEW.workspace_id AND path = NEW.path) + ); + + -- The per-path cap is enforced by trim_resource_versions in the monitor, not here: trimming + -- on every write would tax a path `setResource` can drive in a loop, to keep a bound that + -- does not need to hold instantaneously. + + RETURN NEW; +END; +-- SECURITY DEFINER so history is written on behalf of every writer without granting anyone direct +-- write access to the table, which users hold SELECT on only. `SET search_path FROM CURRENT` is the +-- injection hardening that goes with it, captured rather than hardcoded so installs running a +-- non-public PG_SCHEMA still resolve (see +-- 20260624103600_repair_folder_labels_search_path.up.sql). +$$ LANGUAGE plpgsql SECURITY DEFINER SET search_path FROM CURRENT; diff --git a/backend/oauth_connect.json b/backend/oauth_connect.json index cec49861cd..cb874eb2c7 100644 --- a/backend/oauth_connect.json +++ b/backend/oauth_connect.json @@ -16,9 +16,9 @@ "scopes": ["repository"] }, "slack": { - "auth_url": "https://slack.com/oauth/authorize", - "token_url": "https://slack.com/api/oauth.access", - "scopes": ["chat:write:user", "users:read", "users:read.email"] + "auth_url": "https://slack.com/oauth/v2/authorize", + "token_url": "https://slack.com/api/oauth.v2.access", + "scopes": ["chat:write", "chat:write.public", "channels:join", "files:write"] }, "supabase_wizard": { "auth_url": "https://api.supabase.com/v1/oauth/authorize", diff --git a/backend/parsers/windmill-parser-wasm/Cargo.lock b/backend/parsers/windmill-parser-wasm/Cargo.lock index c2e4b1c868..2f2fafedf4 100644 --- a/backend/parsers/windmill-parser-wasm/Cargo.lock +++ b/backend/parsers/windmill-parser-wasm/Cargo.lock @@ -6191,7 +6191,7 @@ checksum = "712e227841d057c1ee1cd2fb22fa7e5a5461ae8e48fa2ca79ec42cfc1931183f" [[package]] name = "windmill-common" -version = "1.795.0" +version = "1.796.0" dependencies = [ "aho-corasick", "anyhow", @@ -6274,7 +6274,7 @@ dependencies = [ [[package]] name = "windmill-macros" -version = "1.795.0" +version = "1.796.0" dependencies = [ "proc-macro2", "quote", @@ -6286,7 +6286,7 @@ dependencies = [ [[package]] name = "windmill-parser" -version = "1.795.0" +version = "1.796.0" dependencies = [ "convert_case", "serde", @@ -6295,7 +6295,7 @@ dependencies = [ [[package]] name = "windmill-parser-bash" -version = "1.795.0" +version = "1.796.0" dependencies = [ "anyhow", "lazy_static", @@ -6307,7 +6307,7 @@ dependencies = [ [[package]] name = "windmill-parser-csharp" -version = "1.795.0" +version = "1.796.0" dependencies = [ "anyhow", "serde_json", @@ -6319,7 +6319,7 @@ dependencies = [ [[package]] name = "windmill-parser-go" -version = "1.795.0" +version = "1.796.0" dependencies = [ "anyhow", "gosyn", @@ -6331,7 +6331,7 @@ dependencies = [ [[package]] name = "windmill-parser-graphql" -version = "1.795.0" +version = "1.796.0" dependencies = [ "anyhow", "lazy_static", @@ -6343,7 +6343,7 @@ dependencies = [ [[package]] name = "windmill-parser-java" -version = "1.795.0" +version = "1.796.0" dependencies = [ "anyhow", "serde_json", @@ -6355,7 +6355,7 @@ dependencies = [ [[package]] name = "windmill-parser-nu" -version = "1.795.0" +version = "1.796.0" dependencies = [ "anyhow", "nu-parser", @@ -6366,7 +6366,7 @@ dependencies = [ [[package]] name = "windmill-parser-php" -version = "1.795.0" +version = "1.796.0" dependencies = [ "anyhow", "itertools 0.14.0", @@ -6377,7 +6377,7 @@ dependencies = [ [[package]] name = "windmill-parser-py" -version = "1.795.0" +version = "1.796.0" dependencies = [ "anyhow", "itertools 0.14.0", @@ -6389,7 +6389,7 @@ dependencies = [ [[package]] name = "windmill-parser-py-asset" -version = "1.795.0" +version = "1.796.0" dependencies = [ "anyhow", "rustpython-ast", @@ -6400,7 +6400,7 @@ dependencies = [ [[package]] name = "windmill-parser-py-imports" -version = "1.795.0" +version = "1.796.0" dependencies = [ "anyhow", "async-recursion", @@ -6422,7 +6422,7 @@ dependencies = [ [[package]] name = "windmill-parser-r" -version = "1.795.0" +version = "1.796.0" dependencies = [ "anyhow", "serde_json", @@ -6434,7 +6434,7 @@ dependencies = [ [[package]] name = "windmill-parser-ruby" -version = "1.795.0" +version = "1.796.0" dependencies = [ "anyhow", "lazy_static", @@ -6448,7 +6448,7 @@ dependencies = [ [[package]] name = "windmill-parser-rust" -version = "1.795.0" +version = "1.796.0" dependencies = [ "anyhow", "convert_case", @@ -6465,7 +6465,7 @@ dependencies = [ [[package]] name = "windmill-parser-sql" -version = "1.795.0" +version = "1.796.0" dependencies = [ "anyhow", "lazy_static", @@ -6478,7 +6478,7 @@ dependencies = [ [[package]] name = "windmill-parser-sql-asset" -version = "1.795.0" +version = "1.796.0" dependencies = [ "anyhow", "serde", @@ -6490,7 +6490,7 @@ dependencies = [ [[package]] name = "windmill-parser-ts" -version = "1.795.0" +version = "1.796.0" dependencies = [ "anyhow", "lazy_static", @@ -6508,7 +6508,7 @@ dependencies = [ [[package]] name = "windmill-parser-ts-asset" -version = "1.795.0" +version = "1.796.0" dependencies = [ "anyhow", "serde-wasm-bindgen", @@ -6524,7 +6524,7 @@ dependencies = [ [[package]] name = "windmill-parser-wac" -version = "1.795.0" +version = "1.796.0" dependencies = [ "anyhow", "rustpython-ast", @@ -6540,7 +6540,7 @@ dependencies = [ [[package]] name = "windmill-parser-wasm" -version = "1.795.0" +version = "1.796.0" dependencies = [ "anyhow", "getrandom 0.2.17", @@ -6572,7 +6572,7 @@ dependencies = [ [[package]] name = "windmill-parser-yaml" -version = "1.795.0" +version = "1.796.0" dependencies = [ "anyhow", "lazy_static", @@ -6586,7 +6586,7 @@ dependencies = [ [[package]] name = "windmill-types" -version = "1.795.0" +version = "1.796.0" dependencies = [ "anyhow", "bitflags", diff --git a/backend/parsers/windmill-parser-wasm/Cargo.toml b/backend/parsers/windmill-parser-wasm/Cargo.toml index 84311d97d9..826e279c39 100644 --- a/backend/parsers/windmill-parser-wasm/Cargo.toml +++ b/backend/parsers/windmill-parser-wasm/Cargo.toml @@ -12,7 +12,7 @@ resolver = "2" members = ["."] [workspace.package] -version = "1.795.0" +version = "1.796.0" edition = "2021" authors = ["Ruben Fiszel "] diff --git a/backend/src/main.rs b/backend/src/main.rs index 1819146cbe..f8b150383a 100644 --- a/backend/src/main.rs +++ b/backend/src/main.rs @@ -42,12 +42,13 @@ use windmill_common::{ BASE_URL_SETTING, BUNFIG_INSTALL_SCOPES_SETTING, BUN_INSTALL_MIN_RELEASE_AGE_SETTING, CONCURRENCY_KEY_MAX_QUEUED_SETTING, CRITICAL_ALERTS_ON_DB_OVERSIZE_SETTING, CRITICAL_ALERTS_ON_TOKEN_EXPIRY_SETTING, CRITICAL_ALERT_MUTE_UI_SETTING, - CRITICAL_ERROR_CHANNELS_SETTING, CUSTOM_TAGS_SETTING, DEFAULT_TAGS_PER_WORKSPACE_SETTING, - DEFAULT_TAGS_WORKSPACES_SETTING, DISABLE_PASSWORD_LOGIN_SETTING, EMAIL_DOMAIN_SETTING, - ENV_SETTINGS, EXPOSE_DEBUG_METRICS_SETTING, EXPOSE_METRICS_SETTING, - EXTRA_PIP_INDEX_URL_SETTING, FORK_WORKSPACE_TAG_APPEND_FORK_SUFFIX_SETTING, - HTTP_ROUTE_WORKSPACED_ROUTE_SETTING, HUB_API_SECRET_SETTING, HUB_BASE_URL_SETTING, - INDEXER_SETTING, INSTANCE_EVENTS_WEBHOOK_SETTING, INSTANCE_PYTHON_VERSION_SETTING, + CRITICAL_ALERT_MUTE_ZOMBIE_JOB_RESTART_SETTING, CRITICAL_ERROR_CHANNELS_SETTING, + CUSTOM_TAGS_SETTING, DEFAULT_TAGS_PER_WORKSPACE_SETTING, DEFAULT_TAGS_WORKSPACES_SETTING, + DISABLE_PASSWORD_LOGIN_SETTING, EMAIL_DOMAIN_SETTING, ENV_SETTINGS, + EXPOSE_DEBUG_METRICS_SETTING, EXPOSE_METRICS_SETTING, EXTRA_PIP_INDEX_URL_SETTING, + FORK_WORKSPACE_TAG_APPEND_FORK_SUFFIX_SETTING, HTTP_ROUTE_WORKSPACED_ROUTE_SETTING, + HUB_API_SECRET_SETTING, HUB_BASE_URL_SETTING, INDEXER_SETTING, + INSTANCE_EVENTS_WEBHOOK_SETTING, INSTANCE_PYTHON_VERSION_SETTING, JOB_DEFAULT_TIMEOUT_SECS_SETTING, JOB_ISOLATION_SETTING, JWT_SECRET_SETTING, KEEP_JOB_DIR_SETTING, LICENSE_KEY_SETTING, MAVEN_REPOS_SETTING, MAVEN_SETTINGS_XML_SETTING, MONITOR_LOGS_ON_OBJECT_STORE_SETTING, NO_DEFAULT_MAVEN_SETTING, @@ -131,12 +132,13 @@ use crate::monitor::{ load_workspace_max_queued_jobs, monitor_db, reload_app_workspaced_route_setting, reload_audit_log_retention_days_setting, reload_base_url_setting, reload_bun_install_min_release_age_setting, reload_bunfig_install_scopes_setting, - reload_critical_alert_mute_ui_setting, reload_critical_alerts_on_token_expiry_setting, - reload_critical_error_channels_setting, reload_extra_pip_index_url_setting, - reload_http_route_workspaced_route_setting, reload_hub_api_secret_setting, - reload_hub_base_url_setting, reload_instance_events_webhook_setting, - reload_job_default_timeout_setting, reload_job_isolation_setting, reload_jwt_secret_setting, - reload_license_key, reload_npm_config_registry_setting, reload_nsjail_tmp_backing_setting, + reload_critical_alert_mute_ui_setting, reload_critical_alert_mute_zombie_job_restart_setting, + reload_critical_alerts_on_token_expiry_setting, reload_critical_error_channels_setting, + reload_extra_pip_index_url_setting, reload_http_route_workspaced_route_setting, + reload_hub_api_secret_setting, reload_hub_base_url_setting, + reload_instance_events_webhook_setting, reload_job_default_timeout_setting, + reload_job_isolation_setting, reload_jwt_secret_setting, reload_license_key, + reload_npm_config_registry_setting, reload_nsjail_tmp_backing_setting, reload_nsjail_tmpfs_size_setting, reload_otel_tracing_proxy_setting, reload_pip_index_url_setting, reload_retention_period_setting, reload_sandbox_image_cache_max_setting, reload_sandbox_image_default_registry_setting, @@ -1432,21 +1434,30 @@ Windmill Community Edition {GIT_VERSION} // Poll for new events from notify_event table match windmill_common::notify_events::poll_notify_events(&db, last_event_id).await { Ok(events) => { + let mut http_trigger_change_handled = false; for event in events { if !*windmill_common::QUIET_LOGS { tracing::info!("Processing notify event: channel={}, payload={}", event.channel, event.payload); } - process_notify_event( - &event.channel, - &event.payload, - &db, - &conn, - &tx, - server_mode, - worker_mode, - #[cfg(feature = "parquet")] - disable_s3_store, - ).await; + let is_http_trigger_change = event.channel == "notify_http_trigger_change"; + // Every changed http_trigger row emits its own event and each one forces + // a full router rebuild, but the batch's first successful rebuild already + // read every row the batch committed. A failed rebuild leaves the flag + // clear so the next event in the batch retries it. + if !(is_http_trigger_change && http_trigger_change_handled) { + let handled = process_notify_event( + &event.channel, + &event.payload, + &db, + &conn, + &tx, + server_mode, + worker_mode, + #[cfg(feature = "parquet")] + disable_s3_store, + ).await; + http_trigger_change_handled |= is_http_trigger_change && handled; + } last_event_id = last_event_id.max(event.id); } } @@ -1668,6 +1679,9 @@ Windmill Community Edition {GIT_VERSION} /// Process a single notify event from the polling-based event system. /// This replaces the old PgListener notification handling. +/// +/// Returns `false` when the event still needs handling. Only the HTTP router rebuild reports +/// that, because the poll loop coalesces those events and must not swallow the retry. #[allow(unused_variables)] async fn process_notify_event( channel: &str, @@ -1678,7 +1692,7 @@ async fn process_notify_event( server_mode: bool, worker_mode: bool, #[cfg(feature = "parquet")] disable_s3_store: bool, -) { +) -> bool { match channel { "notify_config_change" => { if payload == "server" && server_mode { @@ -1823,17 +1837,14 @@ async fn process_notify_event( #[cfg(feature = "http_trigger")] "notify_http_trigger_change" => { tracing::info!("HTTP trigger change detected: {}", payload); - match windmill_api::triggers::http::refresh_routers(db).await { - Ok((true, _)) => { + match windmill_api::triggers::http::refresh_routers(db, true).await { + Ok(_) => { tracing::info!("Refreshed HTTP routers (trigger change)"); } - Ok((false, _)) => { - tracing::warn!( - "Should have refreshed HTTP routers (trigger change) but did not" - ); - } Err(err) => { tracing::error!("Error refreshing HTTP routers (trigger change): {err:#}"); + windmill_api::triggers::http::invalidate_routers(); + return false; } }; } @@ -2057,7 +2068,7 @@ async fn process_notify_event( tracing::error!(error = %e, "Could not reload http route workspaced route setting"); } #[cfg(feature = "http_trigger")] - match windmill_api::triggers::http::refresh_routers(db).await { + match windmill_api::triggers::http::refresh_routers(db, false).await { Ok((true, _)) => { tracing::info!( "Refreshed HTTP routers (http workspaced route setting change)" @@ -2127,6 +2138,13 @@ async fn process_notify_event( tracing::error!(error = %e, "Could not reload critical alerts on token expiry setting"); } } + CRITICAL_ALERT_MUTE_ZOMBIE_JOB_RESTART_SETTING => { + if let Err(e) = + reload_critical_alert_mute_zombie_job_restart_setting(conn).await + { + tracing::error!(error = %e, "Could not reload zombie job restart alert mute setting"); + } + } INSTANCE_EVENTS_WEBHOOK_SETTING => { reload_instance_events_webhook_setting(db).await; } @@ -2170,6 +2188,7 @@ async fn process_notify_event( tracing::warn!("Unknown notification channel: {}", channel); } } + true } fn display_config(envs: &[&str]) { diff --git a/backend/src/monitor.rs b/backend/src/monitor.rs index 94dcbf0d46..430f3659c3 100644 --- a/backend/src/monitor.rs +++ b/backend/src/monitor.rs @@ -52,10 +52,10 @@ use windmill_common::{ flow_status::{FlowStatus, FlowStatusModule}, global_settings::{ get_or_create_jwt_secret, load_value_from_global_settings, - AUDIT_LOG_RETENTION_DAYS_SETTING, BASE_URL_SETTING, - BUNFIG_INSTALL_SCOPES_SETTING, BUN_INSTALL_MIN_RELEASE_AGE_SETTING, - CONCURRENCY_KEY_MAX_QUEUED_SETTING, CRITICAL_ALERTS_ON_DB_OVERSIZE_SETTING, - CRITICAL_ALERTS_ON_TOKEN_EXPIRY_SETTING, CRITICAL_ALERT_MUTE_UI_SETTING, + AUDIT_LOG_RETENTION_DAYS_SETTING, BASE_URL_SETTING, BUNFIG_INSTALL_SCOPES_SETTING, + BUN_INSTALL_MIN_RELEASE_AGE_SETTING, CONCURRENCY_KEY_MAX_QUEUED_SETTING, + CRITICAL_ALERTS_ON_DB_OVERSIZE_SETTING, CRITICAL_ALERTS_ON_TOKEN_EXPIRY_SETTING, + CRITICAL_ALERT_MUTE_UI_SETTING, CRITICAL_ALERT_MUTE_ZOMBIE_JOB_RESTART_SETTING, CRITICAL_ERROR_CHANNELS_SETTING, CUSTOM_TAGS_SETTING, DEFAULT_TAGS_PER_WORKSPACE_SETTING, DEFAULT_TAGS_WORKSPACES_SETTING, DISABLE_PASSWORD_LOGIN, DISABLE_PASSWORD_LOGIN_SETTING, EXPOSE_DEBUG_METRICS_SETTING, EXPOSE_METRICS_SETTING, EXTRA_PIP_INDEX_URL_SETTING, @@ -87,18 +87,18 @@ use windmill_common::{ worker::{ load_env_vars, load_init_bash_from_env, load_periodic_bash_script_from_env, load_periodic_bash_script_interval_from_env, load_whitelist_env_vars_from_env, - load_worker_config, store_pull_query, - store_suspended_pull_query, Connection, WorkerConfig, CLOUD_HOSTED, - CONCURRENCY_KEY_MAX_QUEUED, CONCURRENCY_KEY_MAX_QUEUED_DEFAULT, DEFAULT_TAGS_PER_WORKSPACE, - DEFAULT_TAGS_WORKSPACES, FORK_WORKSPACE_TAG_APPEND_FORK_SUFFIX, INDEXER_CONFIG, - PREVIEW_TAGS_OVERRIDE, SMTP_CONFIG, WINDMILL_DIR, WORKER_CONFIG, + load_worker_config, store_pull_query, store_suspended_pull_query, Connection, WorkerConfig, + CLOUD_HOSTED, CONCURRENCY_KEY_MAX_QUEUED, CONCURRENCY_KEY_MAX_QUEUED_DEFAULT, + DEFAULT_TAGS_PER_WORKSPACE, DEFAULT_TAGS_WORKSPACES, FORK_WORKSPACE_TAG_APPEND_FORK_SUFFIX, + INDEXER_CONFIG, PREVIEW_TAGS_OVERRIDE, SMTP_CONFIG, WINDMILL_DIR, WORKER_CONFIG, WORKER_GROUP, WORKSPACE_FAIRNESS_DURATION_SECS, WORKSPACE_FAIRNESS_ENABLED, WORKSPACE_FAIRNESS_MAX_PERCENT, WORKSPACE_FAIRNESS_MIN_TOTAL, WORKSPACE_MAX_QUEUED_JOBS, WORKSPACE_MAX_QUEUED_JOBS_DEFAULT, }, KillpillSender, AUDIT_LOG_RETENTION_DAYS, BASE_URL, CRITICAL_ALERTS_ON_DB_OVERSIZE, - CRITICAL_ALERTS_ON_TOKEN_EXPIRY, CRITICAL_ALERT_MUTE_UI_ENABLED, CRITICAL_ERROR_CHANNELS, DB, - DEFAULT_HUB_BASE_URL, HUB_BASE_URL, JOB_RETENTION_SECS, JOB_RETENTION_SECS_OVERRIDES, + CRITICAL_ALERTS_ON_TOKEN_EXPIRY, CRITICAL_ALERT_MUTE_UI_ENABLED, + CRITICAL_ALERT_MUTE_ZOMBIE_JOB_RESTART, CRITICAL_ERROR_CHANNELS, DB, DEFAULT_HUB_BASE_URL, + HUB_BASE_URL, JOB_RETENTION_SECS, JOB_RETENTION_SECS_OVERRIDES, JOB_RETENTION_SECS_OVERRIDES_LOADED, METRICS_DEBUG_ENABLED, METRICS_ENABLED, MONITOR_LOGS_ON_OBJECT_STORE, OTEL_LOGS_ENABLED, OTEL_METRICS_ENABLED, OTEL_TRACING_ENABLED, SERVICE_LOG_RETENTION_SECS, STORE_AUDIT_LOGS_S3, @@ -273,6 +273,11 @@ pub async fn initial_load( true, |v| async move { apply_critical_alerts_on_token_expiry_setting(v) }, ); + pass.setting( + CRITICAL_ALERT_MUTE_ZOMBIE_JOB_RESTART_SETTING, + true, + |v| async move { apply_critical_alert_mute_zombie_job_restart_setting(v) }, + ); if let Some(db) = conn.as_sql() { pass.setting(DEFAULT_TAGS_PER_WORKSPACE_SETTING, false, |v| async move { @@ -1070,6 +1075,31 @@ pub fn apply_critical_alerts_on_token_expiry_setting(value: Option error::Result<()> { + let v = load_value_from_global_settings_with_conn( + conn, + CRITICAL_ALERT_MUTE_ZOMBIE_JOB_RESTART_SETTING, + true, + ) + .await?; + apply_critical_alert_mute_zombie_job_restart_setting(v); + Ok(()) +} + +pub fn apply_critical_alert_mute_zombie_job_restart_setting(value: Option) { + match value { + Some(serde_json::Value::Bool(t)) => { + CRITICAL_ALERT_MUTE_ZOMBIE_JOB_RESTART.store(t, Ordering::Relaxed) + } + // Deleting the row must un-mute: keeping the last value would leave an instance + // silently muted until the next restart. + None => CRITICAL_ALERT_MUTE_ZOMBIE_JOB_RESTART.store(false, Ordering::Relaxed), + _ => (), + }; +} + pub async fn load_metrics_debug_enabled(conn: &Connection) -> error::Result<()> { let v = load_value_from_global_settings_with_conn(conn, EXPOSE_DEBUG_METRICS_SETTING, true).await?; @@ -5184,13 +5214,18 @@ async fn handle_zombie_jobs(db: &Pool, base_internal_url: &str, node_n .execute(db) .await; tracing::error!(critical_error_message); - report_critical_error( - critical_error_message, - db.clone(), - Some(&r.workspace_id), - None, - ) - .await; + // A restart that still has attempts left is self-healing, so an operator can mute it on + // an instance with flaky workers. Exhausting the attempts is a real failure and always + // alerts. + if !restart || !CRITICAL_ALERT_MUTE_ZOMBIE_JOB_RESTART.load(Ordering::Relaxed) { + report_critical_error( + critical_error_message, + db.clone(), + Some(&r.workspace_id), + None, + ) + .await; + } if !restart { zombie_jobs_uuid_restart_limit_reached.push(r.id); @@ -5661,7 +5696,10 @@ async fn handle_zombie_flows(db: &DB) -> error::Result<()> { flow.id, flow.workspace_id ); tracing::error!(error_message); - report_critical_error(error_message, db.clone(), Some(&flow.workspace_id), None).await; + if !CRITICAL_ALERT_MUTE_ZOMBIE_JOB_RESTART.load(Ordering::Relaxed) { + report_critical_error(error_message, db.clone(), Some(&flow.workspace_id), None) + .await; + } // if the flow hasn't started and is a zombie, we can simply restart it let mut tx = db.begin().await?; diff --git a/backend/summarized_schema.txt b/backend/summarized_schema.txt index 732f1ece52..a9865cebd3 100644 --- a/backend/summarized_schema.txt +++ b/backend/summarized_schema.txt @@ -38,6 +38,8 @@ account: workspace_id(char), id(int), expires_at(ts), refresh_token(char), clien FK: (workspace_id) -> workspace(id) agent_token_blacklist: token(char), expires_at(ts), blacklisted_at(ts), blacklisted_by(char) ai_agent_memory: workspace_id(char), conversation_id(uuid), step_id(char), messages(jsonb), created_at(ts), updated_at(ts) +ai_token_usage: workspace_id(char), day(date), email(char), provider(char), model(char), session_id(char), input_tokens(bigint), cache_read_tokens(bigint), cache_write_tokens(bigint), output_tokens(bigint), reported_cost_nano_usd(bigint), requests(bigint), updated_at(ts) + FK: (workspace_id) -> workspace(id) alerts: id(int), alert_type(char), message(text), created_at(ts), acknowledged(bool), workspace_id(text), acknowledged_workspace(bool), resource(text) app: id(bigint), workspace_id(char), path(char), summary(char), policy(jsonb), versions(bigint[]), extra_perms(jsonb), draft_only(bool), custom_path(text), labels(text[]) FK: (workspace_id) -> workspace(id) diff --git a/backend/tests/datatable_migration_add_down.rs b/backend/tests/datatable_migration_add_down.rs new file mode 100644 index 0000000000..3cd39e0fa3 --- /dev/null +++ b/backend/tests/datatable_migration_add_down.rs @@ -0,0 +1,144 @@ +//! A migration that has already run may still gain the down it was missing. +//! +//! Rewriting an applied migration is refused because its `_wm_migrations` record +//! would no longer match its SQL. Filling in an absent `code_down` is the one +//! edit that keeps that record true — and the only way to make an already-run +//! migration revertable — so it must stay allowed while every other edit stays +//! refused. + +use serde_json::json; +use sqlx::{Pool, Postgres}; + +use windmill_test_utils::*; + +const ROLE: &str = "wm_dtmig_down_role"; +const ROLE_PASSWORD: &str = "wm_dtmig_down_pwd"; +const VERSION: i64 = 20260101000000; +const CODE_UP: &str = "CREATE TABLE widgets (id int);"; +const CODE_DOWN: &str = "DROP TABLE widgets;"; + +fn authed(b: reqwest::RequestBuilder) -> reqwest::RequestBuilder { + b.header("Authorization", "Bearer DTMIG_ADMIN_TOKEN") +} + +/// Point the fixture's data table at this test's own database and put it in the +/// state that matters: one migration defined without a down, recorded as applied. +async fn setup_applied_migration_without_down(db: &Pool) -> anyhow::Result<()> { + let opts = (*db.connect_options()).clone(); + let dbname = opts.get_database().expect("test database name").to_string(); + + sqlx::query(&format!( + // Roles are cluster objects, so a leftover role or a parallel test + // session reaching here at the same time must not fail the setup. + "DO $$ BEGIN \ + CREATE ROLE {ROLE} LOGIN PASSWORD '{ROLE_PASSWORD}'; \ + EXCEPTION WHEN duplicate_object OR unique_violation THEN NULL; \ + END $$" + )) + .execute(db) + .await?; + sqlx::raw_sql(&format!( + "GRANT CONNECT ON DATABASE \"{dbname}\" TO {ROLE}; \ + GRANT USAGE ON SCHEMA public TO {ROLE}; \ + CREATE TABLE _wm_migrations ( \ + datatable TEXT NOT NULL, \ + version BIGINT NOT NULL, \ + installed_at TIMESTAMPTZ NOT NULL DEFAULT now(), \ + PRIMARY KEY (datatable, version)); \ + GRANT SELECT ON _wm_migrations TO {ROLE}; \ + INSERT INTO _wm_migrations (datatable, version) VALUES ('main', {VERSION});" + )) + .execute(db) + .await?; + + sqlx::query( + "INSERT INTO resource (workspace_id, path, value, resource_type, created_by) \ + VALUES ('dtmig-ws', 'u/dtmig-admin/pg', $1, 'postgresql', 'dtmig-admin')", + ) + .bind(json!({ + "host": opts.get_host(), + "port": opts.get_port(), + "dbname": dbname, + "user": ROLE, + "password": ROLE_PASSWORD, + "sslmode": "disable", + })) + .execute(db) + .await?; + + sqlx::query( + "INSERT INTO datatable_migrations (workspace_id, datatable, timestamp, name, code_up) \ + VALUES ('dtmig-ws', 'main', $1, 'create_widgets', $2)", + ) + .bind(VERSION) + .bind(CODE_UP) + .execute(db) + .await?; + + Ok(()) +} + +#[sqlx::test(fixtures("datatable_migrations_grants"))] +async fn test_add_down_to_an_applied_migration(db: Pool) -> anyhow::Result<()> { + initialize_tracing().await; + setup_applied_migration_without_down(&db).await?; + + let server = ApiServer::start(db.clone()).await?; + let port = server.addr.port(); + let url = format!( + "http://localhost:{port}/api/w/dtmig-ws/workspaces/upsert_datatable_migration/main" + ); + let upsert = |code_up: &str, code_down: &str| { + authed(reqwest::Client::new().post(&url)).json(&json!({ + "timestamp": VERSION, + "name": "create_widgets", + "code_up": code_up, + "code_down": code_down, + })) + }; + + let resp = upsert(CODE_UP, CODE_DOWN).send().await?; + let status = resp.status(); + let body = resp.text().await?; + assert_eq!( + status, 200, + "adding a missing down to an applied migration should be allowed: {body}" + ); + let stored = sqlx::query_scalar::<_, Option>( + "SELECT code_down FROM datatable_migrations \ + WHERE workspace_id = 'dtmig-ws' AND datatable = 'main' AND timestamp = $1", + ) + .bind(VERSION) + .fetch_one(&db) + .await?; + assert_eq!(stored.as_deref(), Some(CODE_DOWN)); + + // The up it ran is what the `_wm_migrations` record stands for: still frozen. + let resp = upsert("CREATE TABLE gadgets (id int);", CODE_DOWN) + .send() + .await?; + assert_eq!(resp.status(), 400); + assert!( + resp.text().await?.contains("has already been applied"), + "rewriting the up of an applied migration must stay refused" + ); + + // And so is a down that has already been recorded — only the absent-to-present + // step is exempt, in that direction alone. + let resp = upsert(CODE_UP, "DROP TABLE widgets CASCADE;") + .send() + .await?; + assert_eq!(resp.status(), 400); + + let resp = authed(reqwest::Client::new().post(&url)) + .json(&json!({ "timestamp": VERSION, "name": "create_widgets", "code_up": CODE_UP })) + .send() + .await?; + assert_eq!( + resp.status(), + 400, + "dropping the down of an applied migration must stay refused" + ); + + Ok(()) +} diff --git a/backend/tests/sign_s3_objects_authz.rs b/backend/tests/sign_s3_objects_authz.rs index a2baf9d449..511ef39ab5 100644 --- a/backend/tests/sign_s3_objects_authz.rs +++ b/backend/tests/sign_s3_objects_authz.rs @@ -17,7 +17,11 @@ //! so two test functions sharing the one fixture workspace serve each other's stale — by then //! deleted — filesystem root. //! -//! Advanced S3 permissions are an enterprise feature, so this test requires the +//! A second test pins the `expiry_secs` bounds: the signature's `exp` follows the caller's +//! request, defaults to 12h, and is clamped to [60s, 7d]. It only mints signatures and never +//! fetches through the proxy, so it never populates or reads that cache. +//! +//! Advanced S3 permissions are an enterprise feature, so these tests require the //! `enterprise` + `private` + `parquet` features. #![cfg(all(feature = "enterprise", feature = "private", feature = "parquet"))] @@ -175,3 +179,88 @@ async fn test_sign_s3_objects_enforces_read_authz(db: Pool) -> anyhow: Ok(()) } + +/// `exp` is signed into the HMAC message, so the only way a caller can influence +/// it is through `expiry_secs` — pin the default and both clamp bounds. +#[sqlx::test(fixtures("base"))] +async fn test_sign_s3_objects_expiry_secs(db: Pool) -> anyhow::Result<()> { + initialize_tracing().await; + + let server = ApiServer::start(db.clone()).await?; + let port = server.addr.port(); + let base = format!("http://localhost:{port}/api/w/test-workspace"); + + let storage_dir = tempfile::tempdir()?; + configure_lfs(&db, &storage_dir.path().to_string_lossy()).await?; + + async fn signed_exp(base: &str, body: serde_json::Value) -> anyhow::Result { + let resp = authed( + client().post(format!("{base}/apps/sign_s3_objects")), + "SECRET_TOKEN", + ) + .json(&body) + .send() + .await?; + let status = resp.status(); + let signed: serde_json::Value = resp.json().await?; + assert!(status.is_success(), "sign must succeed: {status} {signed}"); + let presigned = signed[0]["presigned"] + .as_str() + .expect("sign must return a presigned string"); + let exp = presigned + .split('&') + .find_map(|kv| kv.strip_prefix("exp=")) + .expect("presigned string must carry exp"); + Ok(exp.parse::()?) + } + + let key = json!([{ "s3": "allowed/file.txt" }]); + // The handler stamps `now` itself, so assert on a window rather than an exact value. + // Keep the window well under the 60s lower bound, or an unclamped 1s would pass. + let ttl_around = |exp: i64| exp - chrono::Utc::now().timestamp(); + let tolerance = 30; + + let default_ttl = ttl_around(signed_exp(&base, json!({ "s3_objects": key.clone() })).await?); + assert!( + (43200 - tolerance..=43200).contains(&default_ttl), + "omitting expiry_secs must keep the 12h default, got {default_ttl}s" + ); + + let honored = ttl_around( + signed_exp( + &base, + json!({ "s3_objects": key.clone(), "expiry_secs": 300 }), + ) + .await?, + ); + assert!( + (300 - tolerance..=300).contains(&honored), + "expiry_secs must be honored verbatim inside the bounds, got {honored}s" + ); + + let clamped_low = ttl_around( + signed_exp( + &base, + json!({ "s3_objects": key.clone(), "expiry_secs": 1 }), + ) + .await?, + ); + assert!( + (60 - tolerance..=60).contains(&clamped_low), + "expiry_secs below 60s must clamp up to 60s, got {clamped_low}s" + ); + + let clamped_high = ttl_around( + signed_exp( + &base, + json!({ "s3_objects": key.clone(), "expiry_secs": 99_999_999 }), + ) + .await?, + ); + assert!( + (604800 - tolerance..=604800).contains(&clamped_high), + "expiry_secs above 7d must clamp down to 7d, got {clamped_high}s" + ); + + Ok(()) +} diff --git a/backend/tests/workspace_export.rs b/backend/tests/workspace_export.rs index 8568b8ff24..5894ca8394 100644 --- a/backend/tests/workspace_export.rs +++ b/backend/tests/workspace_export.rs @@ -1,6 +1,6 @@ use sqlx::postgres::Postgres; use sqlx::Pool; -use windmill_test_utils::{initialize_tracing, ApiServer}; +use windmill_test_utils::{initialize_tracing, set_jwt_secret, ApiServer}; /// Integration test: exercises every explicit-column query in `tarball_workspace`. /// @@ -287,3 +287,77 @@ async fn test_tarball_export_gates_values_on_item_scopes(db: Pool) -> Ok(()) } + +/// `settings.json` carries the admin-managed integration config that `get_settings` +/// is admin-only for (the webhook URL, ai_config, git_sync, handler extra_args), so +/// `include_settings` takes the same admin check as `get_settings` rather than +/// riding on the route's `workspaces:read`. Git sync exports settings through the +/// same route, so the gate must still admit its system identity. +#[sqlx::test(fixtures("base"))] +async fn test_tarball_export_settings_are_admin_only(db: Pool) -> anyhow::Result<()> { + initialize_tracing().await; + set_jwt_secret().await; + let server = ApiServer::start(db.clone()).await?; + let base_url = format!("http://localhost:{}", server.addr.port()); + + sqlx::query( + r#"UPDATE workspace_settings + SET webhook = 'https://hook.example/?token=WEBHOOK_SECRET', + ai_config = '{"providers":{"openai":{"api_key":"AI_CONFIG_SECRET"}}}'::jsonb + WHERE workspace_id = 'test-workspace'"#, + ) + .execute(&db) + .await?; + + let export = async |token: &str| -> anyhow::Result<(u16, String)> { + let resp = reqwest::Client::new() + .get(format!( + "{base_url}/api/w/test-workspace/workspaces/tarball?include_settings=true&settings_version=v2" + )) + .bearer_auth(token) + .send() + .await?; + let status = resp.status().as_u16(); + // Lossy: a successful export is a tar, not UTF-8. Only the values matter here. + Ok(( + status, + String::from_utf8_lossy(&resp.bytes().await?).into_owned(), + )) + }; + + // SECRET_TOKEN_2 belongs to test-user-2, a non-admin member of test-workspace. + let (status, body) = export("SECRET_TOKEN_2").await?; + assert_eq!(status, 403, "non-admin exported settings: {body}"); + + let (status, body) = export("SECRET_TOKEN").await?; + assert_eq!(status, 200, "admin denied settings: {body}"); + assert!( + body.contains("WEBHOOK_SECRET") && body.contains("AI_CONFIG_SECRET"), + "admin got no settings" + ); + + // Git sync pushes the workspace to the repo by exporting it under + // `superadmin_sync@windmill.dev`, which belongs to no workspace: the job token + // it runs with is the export's only admin claim. + let sync_email = windmill_common::users::SUPERADMIN_SYNC_EMAIL; + let sync_token = windmill_common::auth::create_token_for_owner( + &db, + "test-workspace", + sync_email, + "git-sync", + 300, + sync_email, + &uuid::Uuid::new_v4(), + None, + None, + ) + .await?; + let (status, body) = export(&sync_token).await?; + assert_eq!(status, 200, "git-sync identity denied settings: {body}"); + assert!( + body.contains("WEBHOOK_SECRET"), + "git-sync identity got no settings" + ); + + Ok(()) +} diff --git a/backend/windmill-ai/src/ai_google.rs b/backend/windmill-ai/src/ai_google.rs index 4374cb8628..b90bda9e1c 100644 --- a/backend/windmill-ai/src/ai_google.rs +++ b/backend/windmill-ai/src/ai_google.rs @@ -277,7 +277,7 @@ pub struct GeminiSSECandidate { } /// Token usage from the `usageMetadata` field of a Gemini SSE event. -#[derive(Deserialize, Debug, Clone)] +#[derive(Deserialize, Debug, Clone, Default)] pub struct GeminiUsageMetadata { #[serde(rename = "promptTokenCount", default)] pub prompt_token_count: Option, @@ -285,6 +285,39 @@ pub struct GeminiUsageMetadata { pub candidates_token_count: Option, #[serde(rename = "totalTokenCount", default)] pub total_token_count: Option, + /// Subset of `promptTokenCount` served from context cache, billed at a reduced + /// rate. Reported separately so the client can price it separately. + #[serde(rename = "cachedContentTokenCount", default)] + pub cached_content_token_count: Option, + /// Thinking tokens, billed as output but counted apart from `candidatesTokenCount`. + #[serde(rename = "thoughtsTokenCount", default)] + pub thoughts_token_count: Option, + /// Input tokens spent on tool-use prompts, counted apart from `promptTokenCount` + /// rather than within it. + #[serde(rename = "toolUsePromptTokenCount", default)] + pub tool_use_prompt_token_count: Option, +} + +/// Input tokens as billed. Gemini reports tool-use prompts in their own field, and +/// they are disjoint from `promptTokenCount`: a live tool call returns 17 prompt + +/// 60 tool-use + 17 candidates + 52 thoughts against a `totalTokenCount` of 146, so +/// leaving them out under-reports the input of every tool-using turn. Cached tokens +/// are not added here, being already part of `promptTokenCount`. +fn gemini_prompt_tokens(usage: &GeminiUsageMetadata) -> i32 { + usage + .prompt_token_count + .unwrap_or(0) + .saturating_add(usage.tool_use_prompt_token_count.unwrap_or(0)) +} + +/// Output tokens as billed: Gemini counts thinking apart from `candidatesTokenCount` +/// but charges it at the output rate, so a reply that thought would otherwise be +/// reported as far cheaper than it was. +fn gemini_completion_tokens(usage: &GeminiUsageMetadata) -> i32 { + usage + .candidates_token_count + .unwrap_or(0) + .saturating_add(usage.thoughts_token_count.unwrap_or(0)) } /// Top-level structure of one Gemini SSE event. @@ -588,9 +621,12 @@ pub fn gemini_response_to_openai(parsed: &GeminiParsedEvent, model: &str) -> ser let usage = parsed.usage.as_ref().map(|u| { serde_json::json!({ - "prompt_tokens": u.prompt_token_count.unwrap_or(0), - "completion_tokens": u.candidates_token_count.unwrap_or(0), + "prompt_tokens": gemini_prompt_tokens(u), + "completion_tokens": gemini_completion_tokens(u), "total_tokens": u.total_token_count.unwrap_or(0), + "prompt_tokens_details": { + "cached_tokens": u.cached_content_token_count.unwrap_or(0) + }, }) }); @@ -680,8 +716,8 @@ pub fn gemini_event_to_openai_sse_chunks( // OpenAI's `stream_options.include_usage` terminal chunk (top-level `usage`, // empty `choices`) so the frontend's `'usage' in chunk` path records them. if let Some(usage) = &parsed.usage { - let prompt_tokens = usage.prompt_token_count.unwrap_or(0); - let completion_tokens = usage.candidates_token_count.unwrap_or(0); + let prompt_tokens = gemini_prompt_tokens(usage); + let completion_tokens = gemini_completion_tokens(usage); let total_tokens = usage .total_token_count .unwrap_or(prompt_tokens + completion_tokens); @@ -694,6 +730,9 @@ pub fn gemini_event_to_openai_sse_chunks( "prompt_tokens": prompt_tokens, "completion_tokens": completion_tokens, "total_tokens": total_tokens, + "prompt_tokens_details": { + "cached_tokens": usage.cached_content_token_count.unwrap_or(0) + }, } }); chunks.push(format!("data: {}\n\n", chunk)); @@ -943,6 +982,7 @@ mod tests { prompt_token_count: Some(12), candidates_token_count: Some(7), total_token_count: Some(19), + ..Default::default() }), ..Default::default() }; @@ -969,6 +1009,79 @@ mod tests { assert_eq!(usage_chunk["choices"], serde_json::json!([])); } + #[test] + fn gemini_usage_chunk_splits_cached_and_bills_thoughts() { + let parsed = GeminiParsedEvent { + text: Some("the answer".to_string()), + usage: Some(GeminiUsageMetadata { + prompt_token_count: Some(1000), + candidates_token_count: Some(20), + total_token_count: Some(1120), + cached_content_token_count: Some(900), + thoughts_token_count: Some(100), + ..Default::default() + }), + ..Default::default() + }; + + let mut tool_call_index = 0; + let chunks = gemini_event_to_openai_sse_chunks( + &parsed, + "chatcmpl-test", + "gemini-3-flash-preview", + &mut tool_call_index, + ); + let usage_chunk = chunks + .iter() + .map(|c| parse_sse_chunk(c)) + .find(|v| v.get("usage").map(|u| !u.is_null()).unwrap_or(false)) + .expect("a chunk should carry top-level usage"); + + // Gemini's prompt count already includes the cached tokens, so it passes + // through unchanged and the cached share is reported alongside it; thinking + // is billed as output but counted apart from the candidates. + assert_eq!(usage_chunk["usage"]["prompt_tokens"], 1000); + assert_eq!(usage_chunk["usage"]["prompt_tokens_details"]["cached_tokens"], 900); + assert_eq!(usage_chunk["usage"]["completion_tokens"], 120); + } + + #[test] + fn gemini_usage_chunk_counts_tool_use_prompt_tokens() { + let parsed = GeminiParsedEvent { + text: Some("Canberra".to_string()), + usage: Some(GeminiUsageMetadata { + prompt_token_count: Some(17), + candidates_token_count: Some(17), + total_token_count: Some(146), + tool_use_prompt_token_count: Some(60), + thoughts_token_count: Some(52), + ..Default::default() + }), + ..Default::default() + }; + + let mut tool_call_index = 0; + let chunks = gemini_event_to_openai_sse_chunks( + &parsed, + "chatcmpl-test", + "gemini-2.5-flash", + &mut tool_call_index, + ); + let usage_chunk = chunks + .iter() + .map(|c| parse_sse_chunk(c)) + .find(|v| v.get("usage").map(|u| !u.is_null()).unwrap_or(false)) + .expect("a chunk should carry top-level usage"); + + assert_eq!(usage_chunk["usage"]["prompt_tokens"], 77); + assert_eq!(usage_chunk["usage"]["completion_tokens"], 69); + assert_eq!( + usage_chunk["usage"]["prompt_tokens"].as_i64().unwrap() + + usage_chunk["usage"]["completion_tokens"].as_i64().unwrap(), + 146 + ); + } + #[test] fn gemini_streaming_usage_total_falls_back_to_prompt_plus_completion() { let parsed = GeminiParsedEvent { @@ -976,6 +1089,7 @@ mod tests { prompt_token_count: Some(5), candidates_token_count: Some(3), total_token_count: None, + ..Default::default() }), ..Default::default() }; diff --git a/backend/windmill-ai/src/ai_types.rs b/backend/windmill-ai/src/ai_types.rs index b80e237920..69c8ac4538 100644 --- a/backend/windmill-ai/src/ai_types.rs +++ b/backend/windmill-ai/src/ai_types.rs @@ -175,3 +175,57 @@ pub struct OpenAIMessage { #[serde(skip_serializing_if = "Option::is_none")] pub annotations: Option>, } + +// ============================================================================ +// Model pricing +// ============================================================================ + +/// Far above any real per-million-token rate, so a value beyond it is a unit +/// mistake rather than a price. The floor matters more: a negative rate would make +/// spend subtract, and NaN/infinity would poison every total derived from it. +pub const MAX_MODEL_RATE: f64 = 1000.0; + +/// Bound the `model_pricing` map of an AI config that is only available untyped — +/// the instance config is stored through the generic global-settings endpoint, +/// which never deserializes it into `AIConfig`, so the typed check on the +/// workspace path does not cover it. +pub fn validate_model_pricing_json(ai_config: &serde_json::Value) -> Result<(), String> { + // The container itself has to be checked too: a non-object `ai_config` persists + // here and then fails to deserialize as `AIConfig`, which drops the whole + // instance config back to its default for every workspace inheriting it. + if !ai_config.is_null() && !ai_config.is_object() { + return Err("ai_config must be an object".to_string()); + } + let pricing = match ai_config.get("model_pricing") { + None | Some(serde_json::Value::Null) => return Ok(()), + // A present-but-wrong shape must be rejected, not skipped: it would persist + // and then fail to deserialize as `AIConfig`, which silently drops the whole + // instance config back to its default for every workspace inheriting it. + Some(v) => v + .as_object() + .ok_or_else(|| "model_pricing must be an object".to_string())?, + }; + for (key, price) in pricing { + let Some(price) = price.as_object() else { + return Err(format!("Price override for {} is not an object", key)); + }; + for field in ["input", "output", "cache_read", "cache_write"] { + let Some(rate) = price.get(field) else { continue }; + let rate = rate + .as_f64() + .filter(|r| r.is_finite() && *r >= 0.0 && *r <= MAX_MODEL_RATE); + if rate.is_none() { + return Err(format!( + "Price override for {}: {} must be between 0 and {}", + key, field, MAX_MODEL_RATE + )); + } + } + for required in ["input", "output"] { + if !price.contains_key(required) { + return Err(format!("Price override for {} is missing {}", key, required)); + } + } + } + Ok(()) +} diff --git a/backend/windmill-ai/src/providers/bedrock.rs b/backend/windmill-ai/src/providers/bedrock.rs index 453607eea5..cf977f2484 100644 --- a/backend/windmill-ai/src/providers/bedrock.rs +++ b/backend/windmill-ai/src/providers/bedrock.rs @@ -660,6 +660,41 @@ fn bedrock_sse_chunks_for_event( chunks.push(Bytes::from(format!("data: {}\n\n", chunk))); } + // Usage arrives only on the trailing Metadata event, and only this converter + // reaches the chat: without a chunk for it a Bedrock chat reports no tokens at + // all. Bedrock counts cache reads and writes apart from `inputTokens`, while the + // OpenAI shape the client parses treats `prompt_tokens` as the whole input, so + // they are folded in here and split back out through `prompt_tokens_details`. + if let aws_sdk_bedrockruntime::types::ConverseStreamOutput::Metadata(metadata) = event { + if let Some(token_usage) = metadata.usage() { + let cache_read = token_usage.cache_read_input_tokens().unwrap_or(0); + let cache_write = token_usage.cache_write_input_tokens().unwrap_or(0); + let prompt_tokens = token_usage + .input_tokens() + .saturating_add(cache_read) + .saturating_add(cache_write); + + let chunk = serde_json::json!({ + "id": state.id, + "object": "chat.completion.chunk", + "created": state.created, + "model": state.model, + "choices": [], + "usage": { + "prompt_tokens": prompt_tokens, + "completion_tokens": token_usage.output_tokens(), + "total_tokens": token_usage.total_tokens(), + "prompt_tokens_details": { + "cached_tokens": cache_read, + "cache_write_tokens": cache_write + } + } + }); + + chunks.push(Bytes::from(format!("data: {}\n\n", chunk))); + } + } + chunks } @@ -1190,6 +1225,43 @@ mod tests { serde_json::from_str(payload).expect("chunk should contain JSON") } + #[test] + fn metadata_event_emits_usage_chunk_with_cache_split() { + let mut state = BedrockSseStreamState::new("id".to_string(), "model".to_string(), 0); + let event = ConverseStreamOutput::Metadata( + aws_sdk_bedrockruntime::types::ConverseStreamMetadataEvent::builder() + .usage( + aws_sdk_bedrockruntime::types::TokenUsage::builder() + .input_tokens(10) + .output_tokens(7) + .total_tokens(1017) + .cache_read_input_tokens(900) + .cache_write_input_tokens(100) + .build() + .expect("usage"), + ) + .build(), + ); + + let chunks = bedrock_sse_chunks_for_event(&event, &mut state); + let usage = chunks + .iter() + .map(sse_json) + .find(|v| v.get("usage").map(|u| !u.is_null()).unwrap_or(false)) + .expect("the metadata event should carry usage"); + + // Bedrock reports cache reads and writes apart from `inputTokens`; the OpenAI + // shape the client parses treats `prompt_tokens` as the whole input, and + // recovers the uncached share by subtracting the details back out. + assert_eq!(usage["usage"]["prompt_tokens"], 1010); + assert_eq!(usage["usage"]["completion_tokens"], 7); + assert_eq!(usage["usage"]["prompt_tokens_details"]["cached_tokens"], 900); + assert_eq!( + usage["usage"]["prompt_tokens_details"]["cache_write_tokens"], + 100 + ); + } + #[test] fn determine_auth_config_prioritizes_bearer_token() { let config = determine_auth_config( diff --git a/backend/windmill-api-auth/src/scopes.rs b/backend/windmill-api-auth/src/scopes.rs index da24453553..7109a04425 100644 --- a/backend/windmill-api-auth/src/scopes.rs +++ b/backend/windmill-api-auth/src/scopes.rs @@ -289,6 +289,7 @@ pub enum ScopeDomain { OAuth, AI, AiSkills, + AiEvals, // AI agent eval datasets Indexer, Teams, // Microsoft Teams integration @@ -349,6 +350,7 @@ impl ScopeDomain { Self::OAuth => "oauth", Self::AI => "ai", Self::AiSkills => "ai_skills", + Self::AiEvals => "ai_evals", Self::Capture => "capture", Self::Drafts => "drafts", Self::Favorites => "favorites", @@ -404,6 +406,7 @@ impl ScopeDomain { "oauth" => Some(Self::OAuth), "ai" => Some(Self::AI), "ai_skills" => Some(Self::AiSkills), + "ai_evals" => Some(Self::AiEvals), "indexer" | "srch" => Some(Self::Indexer), "teams" => Some(Self::Teams), "native_triggers" => Some(Self::NativeTriggers), diff --git a/backend/windmill-api-integration-tests/tests/jobs_authed.rs b/backend/windmill-api-integration-tests/tests/jobs_authed.rs index cd60275afc..c3da2f3fb9 100644 --- a/backend/windmill-api-integration-tests/tests/jobs_authed.rs +++ b/backend/windmill-api-integration-tests/tests/jobs_authed.rs @@ -286,13 +286,24 @@ async fn test_jobs_authed_reachability(db: Pool) -> anyhow::Result<()> "GET /jobs/result_by_id", ); + // Sent the way the generated client sends it. A handler whose `Path` tuple has drifted from + // the route is rejected by axum before it runs, which surfaces as a routing error rather + // than the handler's own answer, so reaching the handler is what this pins. let resp = authed(client().post(format!("{base}/restart/f/{fake}"))) + .json(&json!({ "step_id": "a" })) .send() .await?; - assert_route_reachable( - resp.status().as_u16(), - &resp.text().await?, - "POST /jobs/restart/f", + let status = resp.status().as_u16(); + let body = resp.text().await?; + assert_route_reachable(status, &body, "POST /jobs/restart/f"); + assert!( + !body.contains("path arguments"), + "POST /jobs/restart/f never reached its handler: {status} {body}", + ); + #[cfg(not(feature = "enterprise"))] + assert!( + body.contains("only available in enterprise version"), + "POST /jobs/restart/f must report the enterprise gate outside EE: {status} {body}", ); let resp = authed(client().post(format!("{base}/run/workflow_as_code/{fake}/main"))) diff --git a/backend/windmill-api-settings/src/lib.rs b/backend/windmill-api-settings/src/lib.rs index a9bc601591..cead2f4b27 100644 --- a/backend/windmill-api-settings/src/lib.rs +++ b/backend/windmill-api-settings/src/lib.rs @@ -891,6 +891,13 @@ async fn run_setting_pre_write_hook( value: &serde_json::Value, ) -> error::Result<()> { match key { + // The instance AI config is written as an untyped blob through this generic + // endpoint, so it never passes the typed check the workspace handler applies. + // Rates that reach a cost total unbounded would make it negative or infinite. + AI_CONFIG_SETTING => { + windmill_ai::ai_types::validate_model_pricing_json(value) + .map_err(error::Error::BadRequest)?; + } AUTOMATE_USERNAME_CREATION_SETTING => { if value.as_bool().unwrap_or(false) { generate_instance_username_for_all_users(db) diff --git a/backend/windmill-api-users/src/users.rs b/backend/windmill-api-users/src/users.rs index 0c83d5f6ec..8ea3bd0995 100644 --- a/backend/windmill-api-users/src/users.rs +++ b/backend/windmill-api-users/src/users.rs @@ -2399,6 +2399,7 @@ pub async fn delete_workspace_user_internal( "flow", "app", "resource", + "eval_dataset", "variable", "schedule", "group_", diff --git a/backend/windmill-api-workspaces/src/datatable_migrations.rs b/backend/windmill-api-workspaces/src/datatable_migrations.rs index 1214f06eab..e3a1b981fd 100644 --- a/backend/windmill-api-workspaces/src/datatable_migrations.rs +++ b/backend/windmill-api-workspaces/src/datatable_migrations.rs @@ -1288,7 +1288,9 @@ async fn upsert_datatable_migration( // its SQL, so a later `migrate up` would skip it and a rollback would run a // `down` that doesn't correspond to what was applied. Only an actual change // to an existing migration is guarded; unchanged re-pushes (e.g. - // `wmill sync push`) always proceed. + // `wmill sync push`) always proceed, and so does filling in a down migration + // that was missing — the up that ran is untouched, and that is the only way + // to make an already-applied migration revertable. let existing = sqlx::query!( "SELECT name, code_up, code_down FROM datatable_migrations \ WHERE workspace_id = $1 AND datatable = $2 AND timestamp = $3", @@ -1298,16 +1300,26 @@ async fn upsert_datatable_migration( ) .fetch_optional(&db) .await?; - // When modifying an existing definition, hold the run-serialization lock - // across the applied-check and the write below so an in-flight run can't - // record a version for the SQL we're about to overwrite. Held until the end - // of the handler (well past the write); a new/unchanged upsert needs no lock. - let _run_lock = match existing { - Some(existing) - if !(existing.name == payload.name - && existing.code_up == payload.code_up - && existing.code_down == payload.code_down) => - { + // When overwriting the SQL of an existing definition, hold the + // run-serialization lock across the applied-check and the write below so an + // in-flight run can't record a version for the SQL we're about to overwrite. + // Held until the end of the handler (well past the write). The exempt + // upserts need no lock: a new or unchanged one overwrites nothing, and one + // that only adds a down leaves the `code_up` a concurrent run is recording + // a version for untouched. + let only_adds_down = existing.as_ref().is_some_and(|existing| { + existing.name == payload.name + && existing.code_up == payload.code_up + && existing.code_down.is_none() + && payload.code_down.is_some() + }); + let unchanged = existing.as_ref().is_some_and(|existing| { + existing.name == payload.name + && existing.code_up == payload.code_up + && existing.code_down == payload.code_down + }); + let _run_lock = match existing.as_ref() { + Some(_) if !only_adds_down && !unchanged => { // Fail closed: if we can't lock/read the applied set (e.g. the // data-table database is temporarily unreachable), refuse the change // rather than risk overwriting a migration that has already run. @@ -1337,20 +1349,44 @@ async fn upsert_datatable_migration( _ => None, }; - sqlx::query!( + // An exempt upsert judged the row from an unlocked read and then writes + // without the lock, so that whole read is re-tested here, where `ON CONFLICT + // DO UPDATE` re-reads the row under a row lock. Otherwise a request working + // from a stale definition silently reverts whatever changed in between — a + // second addition's down, or a locked rewrite of the up whose new SQL a run + // may already have recorded a version for. Reading no row at all is part of + // the premise: the equality is NULL when `$8` is, so a version created in the + // meantime is refused rather than overwritten. + let recheck_observed = _run_lock.is_none(); + let written = sqlx::query!( "INSERT INTO datatable_migrations (workspace_id, datatable, timestamp, name, code_up, code_down) \ VALUES ($1, $2, $3, $4, $5, $6) \ ON CONFLICT (workspace_id, datatable, timestamp) DO UPDATE \ - SET name = EXCLUDED.name, code_up = EXCLUDED.code_up, code_down = EXCLUDED.code_down", + SET name = EXCLUDED.name, code_up = EXCLUDED.code_up, code_down = EXCLUDED.code_down \ + WHERE NOT $7 \ + OR (datatable_migrations.name = $8::text \ + AND datatable_migrations.code_up = $9::text \ + AND datatable_migrations.code_down IS NOT DISTINCT FROM $10::text)", &w_id, &datatable_name, payload.timestamp, &payload.name, &payload.code_up, payload.code_down.as_deref(), + recheck_observed, + existing.as_ref().map(|e| e.name.as_str()), + existing.as_ref().map(|e| e.code_up.as_str()), + existing.as_ref().and_then(|e| e.code_down.as_deref()), ) .execute(&db) .await?; + if written.rows_affected() == 0 { + return Err(Error::BadRequest(format!( + "Migration {} on data table '{}' changed while this change was being saved. \ + Reload it before editing.", + payload.timestamp, datatable_name + ))); + } // The definition is written; runs may resume (audit/deploy metadata below // don't need the lock). drop(_run_lock); diff --git a/backend/windmill-api-workspaces/src/workspaces.rs b/backend/windmill-api-workspaces/src/workspaces.rs index 81b98b6a7b..3238873eef 100644 --- a/backend/windmill-api-workspaces/src/workspaces.rs +++ b/backend/windmill-api-workspaces/src/workspaces.rs @@ -5574,6 +5574,8 @@ async fn clone_workspace_data( // Clone scripts with new hashes clone_scripts(tx, source_workspace_id, target_workspace_id).await?; + clone_eval_datasets(tx, source_workspace_id, target_workspace_id).await?; + // Clone the dbt graph sidecars. After `clone_scripts`, which keeps each // script's hash: these key on it, and a static descriptor never re-ingests, // so a fork without them shows dbt scripts with no models until someone @@ -6108,6 +6110,36 @@ async fn clone_resources( Ok(()) } +async fn clone_eval_datasets( + tx: &mut Transaction<'_, Postgres>, + source_workspace_id: &str, + target_workspace_id: &str, +) -> Result<()> { + // The authored evaluation data — datasets and their cases — travels with a fork like resources + // and scripts do; the runs (experiments) do not, since they name jobs the fork has no copy of. + sqlx::query!( + "INSERT INTO eval_dataset (workspace_id, path, summary, scorers, extra_perms, created_at, created_by, edited_at, edited_by) + SELECT $2, path, summary, scorers, extra_perms, created_at, created_by, edited_at, edited_by + FROM eval_dataset WHERE workspace_id = $1", + source_workspace_id, + target_workspace_id, + ) + .execute(&mut **tx) + .await?; + // A new id per cloned case: `eval_case`'s primary key is the id alone, unique across the whole + // table, so copying it would collide with the source's own rows. + sqlx::query!( + "INSERT INTO eval_case (workspace_id, dataset_path, input, expected, created_at, created_by) + SELECT $2, dataset_path, input, expected, created_at, created_by + FROM eval_case WHERE workspace_id = $1", + source_workspace_id, + target_workspace_id, + ) + .execute(&mut **tx) + .await?; + Ok(()) +} + async fn clone_variables( tx: &mut Transaction<'_, Postgres>, db: &DB, diff --git a/backend/windmill-api/openapi.yaml b/backend/windmill-api/openapi.yaml index abdfa9718a..9eee3c0513 100644 --- a/backend/windmill-api/openapi.yaml +++ b/backend/windmill-api/openapi.yaml @@ -1,7 +1,7 @@ openapi: "3.0.3" info: - version: 1.795.0 + version: 1.796.0 title: Windmill API contact: @@ -7933,7 +7933,7 @@ paths: schema: type: string - /w/{workspace}/resources/history/v/{version}: + /w/{workspace}/resources/history/v/{id}: get: summary: get a single resource version, with its value operationId: getResourceVersion @@ -7941,9 +7941,10 @@ paths: - resource parameters: - $ref: "#/components/parameters/WorkspaceId" - - name: version + - name: id in: path required: true + description: The version's id, not its number. schema: type: integer format: int64 @@ -7965,7 +7966,7 @@ paths: required: - missing_references - /w/{workspace}/resources/history/restore/v/{version}: + /w/{workspace}/resources/history/restore/v/{id}: post: summary: restore a resource to a previous version operationId: restoreResourceVersion @@ -7973,9 +7974,10 @@ paths: - resource parameters: - $ref: "#/components/parameters/WorkspaceId" - - name: version + - name: id in: path required: true + description: The version's id, not its number. schema: type: integer format: int64 @@ -11893,6 +11895,453 @@ paths: items: $ref: "#/components/schemas/FlowConversationMessage" + /w/{workspace}/ai_evals/datasets/list: + get: + summary: list eval datasets + operationId: listEvalDatasets + tags: + - ai_evals + parameters: + - $ref: "#/components/parameters/WorkspaceId" + responses: + "200": + description: eval datasets list + content: + application/json: + schema: + type: array + items: + $ref: "#/components/schemas/EvalDataset" + + /w/{workspace}/ai_evals/datasets/create: + post: + summary: create an eval dataset + operationId: createEvalDataset + tags: + - ai_evals + parameters: + - $ref: "#/components/parameters/WorkspaceId" + requestBody: + description: new eval dataset + required: true + content: + application/json: + schema: + type: object + required: [path] + properties: + path: + type: string + maxLength: 255 + summary: + type: string + maxLength: 1000 + scorers: + type: array + maxItems: 20 + items: + $ref: "#/components/schemas/Scorer" + cases: + type: array + maxItems: 1000 + description: The cases to create the dataset holding, so one can be assembled in a single act rather than created empty and filled in afterwards. + items: + $ref: "#/components/schemas/NewEvalCase" + responses: + "200": + description: eval dataset created + content: + text/plain: + schema: + type: string + + /w/{workspace}/ai_evals/datasets/get/{path}: + get: + summary: get an eval dataset + operationId: getEvalDataset + tags: + - ai_evals + parameters: + - $ref: "#/components/parameters/WorkspaceId" + - $ref: "#/components/parameters/Path" + responses: + "200": + description: eval dataset + content: + application/json: + schema: + $ref: "#/components/schemas/EvalDataset" + + /w/{workspace}/ai_evals/datasets/update/{path}: + post: + summary: update an eval dataset + operationId: updateEvalDataset + tags: + - ai_evals + parameters: + - $ref: "#/components/parameters/WorkspaceId" + - $ref: "#/components/parameters/Path" + requestBody: + description: updated eval dataset + required: true + content: + application/json: + schema: + type: object + properties: + path: + type: string + maxLength: 255 + description: > + Renames the dataset. Its cases and experiments follow through the foreign + keys, so a rename keeps the history it already has. + summary: + type: string + maxLength: 1000 + description: Left out to keep the stored summary; sent as "" to clear it. + scorers: + type: array + maxItems: 20 + description: > + Left out to keep the dataset's columns as they are; sent to replace them + wholesale. + items: + $ref: "#/components/schemas/Scorer" + cases: + type: array + maxItems: 1000 + description: > + The cases as they should stand afterwards: all of them, each carrying its id if + the dataset already has it. Sent with the rest of an edit so that a rename the + dataset refuses refuses the case edits with it. + items: + $ref: "#/components/schemas/SaveEvalCase" + responses: + "200": + description: eval dataset updated + content: + text/plain: + schema: + type: string + + /w/{workspace}/ai_evals/datasets/delete/{path}: + post: + summary: delete an eval dataset and all its cases + description: > + The cases, the runs and their recorded case sets go with it through the foreign keys; the + jobs those runs produced are left alone. + operationId: deleteEvalDataset + tags: + - ai_evals + parameters: + - $ref: "#/components/parameters/WorkspaceId" + - $ref: "#/components/parameters/Path" + responses: + "200": + description: eval dataset deleted + content: + text/plain: + schema: + type: string + + /w/{workspace}/ai_evals/cases/list/{path}: + get: + summary: list the cases of an eval dataset + operationId: listEvalCases + tags: + - ai_evals + parameters: + - $ref: "#/components/parameters/WorkspaceId" + - $ref: "#/components/parameters/Path" + - $ref: "#/components/parameters/Page" + - $ref: "#/components/parameters/PerPage" + responses: + "200": + description: eval cases + content: + application/json: + schema: + type: object + required: [cases] + properties: + cases: + type: array + items: + $ref: "#/components/schemas/EvalCase" + /w/{workspace}/ai_evals/subject_state: + get: + summary: what the agent under test is right now + description: > + The version it is deployed at. Small on purpose: the results endpoint reports the same + thing but harvests scores and reads every job to do it, so it is not something to ask for + on its own. + operationId: evalSubjectState + tags: + - ai_evals + parameters: + - $ref: "#/components/parameters/WorkspaceId" + - name: path + in: query + required: true + schema: + type: string + responses: + "200": + description: the subject as it is now + content: + application/json: + schema: + type: object + properties: + version: + type: integer + format: int64 + + /w/{workspace}/ai_evals/run_payload: + get: + summary: the run one iteration of an eval run answered, as its scorers read it + operationId: evalRunPayload + description: > + Called by the step a run's flow places between the agent and its scorers. Every tool call + is enriched with the arguments, result, status and duration of the job that ran it, and + with the schema of the script version it ran, none of which the flow itself can read. + tags: + - ai_evals + parameters: + - $ref: "#/components/parameters/WorkspaceId" + - name: job_id + in: query + required: true + description: The flow job that answered the case. + schema: + type: string + format: uuid + responses: + "200": + description: the run and its rendering + content: + application/json: + schema: + type: object + required: [run, rendered] + properties: + run: + type: object + description: The case, the answer, and every tool call the agent made. + rendered: + type: string + description: The same run as a judge agent is shown it. + + /w/{workspace}/ai_evals/scorer_defaults: + get: + summary: what a new judge agent and a new script scorer are created from + operationId: scorerDefaults + tags: + - ai_evals + parameters: + - $ref: "#/components/parameters/WorkspaceId" + responses: + "200": + description: scorer defaults + content: + application/json: + schema: + type: object + required: [judge_prompt, script_template] + properties: + judge_prompt: + type: string + description: The system prompt a judge agent is created with. + script_template: + type: string + + /w/{workspace}/ai_evals/scorers/recent: + get: + summary: list the scorers already in use in this workspace, most recent first + description: > + Filtered twice, both times by what the caller can read: the datasets they come from, and + the runnables themselves. A scorer they could not run does not appear. + operationId: recentScorers + tags: + - ai_evals + parameters: + - $ref: "#/components/parameters/WorkspaceId" + - name: kind + description: only scorers of this kind + in: query + required: false + schema: + type: string + enum: [script, agent] + responses: + "200": + description: recently used scorers + content: + application/json: + schema: + type: array + items: + allOf: + - $ref: "#/components/schemas/Scorer" + - type: object + required: [dataset] + properties: + dataset: + type: string + description: The dataset it is a column of. + + /w/{workspace}/ai_evals/experiments/run: + post: + summary: run every case of a dataset as one immutable experiment + operationId: runExperiment + tags: + - ai_evals + parameters: + - $ref: "#/components/parameters/WorkspaceId" + requestBody: + description: what to run + required: true + content: + application/json: + schema: + type: object + required: [dataset, subject] + properties: + dataset: + type: string + subject: + $ref: "#/components/schemas/EvalSubject" + responses: + "200": + description: id of the created experiment + content: + text/plain: + schema: + type: string + + /w/{workspace}/ai_evals/experiments/collect: + post: + summary: record what a run produced, so it outlives the jobs that produced it + description: > + Called by a run's own flow as its last step. The answers and scores a run produced live in + its jobs, which have their own retention; this copies them onto the run's rows. Reading a + run does the same, so this is what covers a run nobody opened. + operationId: collectExperiment + tags: + - ai_evals + parameters: + - $ref: "#/components/parameters/WorkspaceId" + - name: id + in: query + required: true + schema: + type: string + format: uuid + responses: + "200": + description: how many of the run's cases are recorded + content: + application/json: + schema: + type: integer + + /w/{workspace}/ai_evals/experiments/list_all: + get: + summary: list every experiment, across datasets + operationId: listAllExperiments + tags: + - ai_evals + parameters: + - $ref: "#/components/parameters/WorkspaceId" + - name: subject_path + description: > + Restrict to one agent's runs, which is what makes the list a history rather than a + log. Runs of what is deployed, of a past version, and of the edits waiting on top are + all that agent's, so this does not discriminate by kind. + in: query + required: false + schema: + type: string + responses: + "200": + description: > + The 100 newest experiments, each naming the dataset it is of. Restricted to datasets + the caller can read. + content: + application/json: + schema: + type: array + items: + $ref: "#/components/schemas/EvalExperiment" + + /w/{workspace}/ai_evals/experiments/results/{path}: + get: + summary: read an experiment's results, one row per case + operationId: experimentResults + tags: + - ai_evals + parameters: + - $ref: "#/components/parameters/WorkspaceId" + - $ref: "#/components/parameters/Path" + - name: id + description: the experiment to read + in: query + required: true + schema: + type: string + format: uuid + - name: baseline + description: > + The experiment every column is compared against. A delta is only computed between two + scores of the same scorer id, and a column the baseline was never scored with reports + it rather than showing a difference. + in: query + required: false + schema: + type: string + format: uuid + responses: + "200": + description: experiment results + content: + application/json: + schema: + type: object + required: [experiment, scorers, rows, means, regressed] + properties: + experiment: + $ref: "#/components/schemas/EvalExperiment" + baseline: + $ref: "#/components/schemas/EvalExperiment" + scorers: + type: array + description: The columns, which belong to the dataset rather than the experiment. + items: + $ref: "#/components/schemas/Scorer" + rows: + type: array + items: + $ref: "#/components/schemas/ExperimentRow" + means: + type: array + items: + $ref: "#/components/schemas/ScorerMean" + regressed: + type: integer + description: Cells scoring lower than the baseline, across every column. + subject_current_version: + type: integer + format: int64 + description: > + The version the subject is on now. A row that ran against an earlier one + describes an agent that no longer exists. + subject_deployed_hash: + type: string + description: > + What the agent hashes to as deployed. A run of unsaved edits carrying this + hash ran exactly what is deployed now — the edits were saved — so it is a run + of that version rather than of edits. + /w/{workspace}/path_autocomplete/list_paths: get: summary: list all paths in a workspace for client-side autocomplete @@ -12100,6 +12549,73 @@ paths: schema: type: string + /w/{workspace}/ai/usage: + post: + summary: record AI token usage for the calling user + operationId: recordAiUsage + tags: + - ai + parameters: + - $ref: "#/components/parameters/WorkspaceId" + requestBody: + required: true + content: + application/json: + schema: + type: object + required: + - events + properties: + events: + type: array + items: + $ref: "#/components/schemas/AITokenUsageEvent" + responses: + "204": + description: usage recorded + get: + summary: list aggregated AI token usage + operationId: listAiUsage + tags: + - ai + parameters: + - $ref: "#/components/parameters/WorkspaceId" + - name: days + in: query + schema: + type: integer + minimum: 1 + maximum: 365 + - name: group_by + in: query + schema: + type: string + enum: [day, user, model] + - name: scope + in: query + description: workspace-wide usage (admin only) or the calling user's own + schema: + type: string + enum: [workspace, self] + responses: + "200": + description: usage buckets + content: + application/json: + schema: + type: object + required: + - buckets + - truncated + properties: + buckets: + type: array + items: + $ref: "#/components/schemas/AITokenUsageBucket" + truncated: + type: boolean + description: more buckets matched than were returned, so summing them under-reports + /w/{workspace}/ai_skills/list: get: summary: list the workspace AI chat skills (name + description only) @@ -13066,6 +13582,10 @@ paths: type: array items: $ref: "#/components/schemas/S3Object" + expiry_secs: + type: integer + format: int64 + description: how long the signature stays valid, in seconds. Defaults to 43200 (12h) and is clamped server-side to [60, 604800] (1 minute to 7 days). required: - s3_objects responses: @@ -26225,6 +26745,348 @@ components: additionalProperties: true description: Array of JSON Web Keys for JWT verification + EvalSubject: + type: object + description: What an eval run is executed against. + required: [kind, path] + properties: + kind: + type: string + description: > + `agent` runs the ai_agent resource as it is deployed when the run opens, `agent_draft` + the caller's unsaved edits of it as the editor holds them (carried in `draft`), and + `agent_version` one past version named by `version`. The first and last are read + server-side; all three are inlined into the run, so every case of a run executes one + configuration: a deploy part-way through changes what the next run measures, never + this one. + enum: [agent, agent_draft, agent_version] + path: + type: string + description: Path of the ai_agent resource. + version: + type: integer + format: int64 + nullable: true + description: > + The agent's per-path version number when the run opened: how many times the resource + had been saved, not a resource_version row id. For `agent` and `agent_draft` it names + the configuration the run read and every case executed. For `agent_version` it is the + request's own, says which version to inline, and is required. + draft: + $ref: "#/components/schemas/AgentDraft" + draft_hash: + type: string + description: > + Hash of the configuration a draft run executed, stamped server-side. A draft moves + without the version moving, so this is what dates a run of one. It is also what + recognises a draft run whose configuration was later deployed: when it matches the + agent as deployed, the run's kind and version are rewritten to that version, once, and + the hash is kept as what the resolution rests on. + + AgentDraft: + type: object + description: > + The brain and tools of an agent, as the flow editor holds them. Carried by the request and + present exactly when the subject kind is `agent_draft` — the edits exist only in the editor + — where it is the whole definition of what ran: the run goes through the same unlinked + branch of the agent executor the editor's own test uses. + properties: + input_transforms: + type: object + additionalProperties: true + description: > + The agent's input transforms: provider, system prompt, output type and the rest. The + message and attachments come from the case and override anything named here. + tools: + type: array + items: + type: object + additionalProperties: true + + EvalDataset: + type: object + required: [path, created_at, created_by, edited_at, edited_by] + properties: + path: + type: string + maxLength: 255 + summary: + type: string + maxLength: 1000 + scorers: + type: array + description: The columns of the results table, in display order. + items: + $ref: "#/components/schemas/Scorer" + created_at: + type: string + format: date-time + created_by: + type: string + edited_at: + type: string + format: date-time + edited_by: + type: string + + EvalCaseInput: + type: object + description: The inputs a standalone run feeds the agent. + properties: + user_message: + type: string + user_attachments: + type: array + items: + type: object + + NewEvalCase: + type: object + properties: + input: + $ref: "#/components/schemas/EvalCaseInput" + expected: + description: Reference output a scorer compares a rerun against. + + SaveEvalCase: + allOf: + - type: object + properties: + id: + description: Absent for a case the dataset does not hold yet. + type: string + format: uuid + - $ref: "#/components/schemas/NewEvalCase" + + EvalCase: + allOf: + - type: object + required: [id, created_at, created_by] + properties: + id: + type: string + format: uuid + created_at: + type: string + format: date-time + created_by: + type: string + - $ref: "#/components/schemas/NewEvalCase" + + Scorer: + type: object + description: > + A scorer is a column of the results table, and it is always a runnable: an ai_agent + resource sent the run to grade, or a script handed the run as an argument. `id` is assigned + when the scorer is added to a dataset and never reused: it is what makes a column the same + column across experiments when the scorer is renamed, and a delta is only ever computed + between two scores carrying the same id. A scorer sent without an id is given one. + required: [kind, path] + properties: + id: + type: string + name: + type: string + description: Column header. Defaults to the last segment of the path. + pass_if: + type: number + description: > + A score at or above this counts as a pass, and the column reports a pass rate beside + its mean. Applied when results are read rather than when they are produced, so moving + the line re-reads every score already recorded instead of invalidating them. + kind: + type: string + enum: [script, agent] + path: + type: string + description: The script, or the ai_agent resource used as a judge. + + EvalExperiment: + type: object + description: >- + One run of a dataset: written once when the dataset is run, and only ever read afterwards. + The case set it executed is returned by the results endpoint, not here: a listing would + otherwise send the whole dataset back once per experiment. + required: [id, dataset, subject, run_number, run_job_id, case_count, created_at, created_by] + properties: + id: + type: string + format: uuid + dataset: + type: string + subject: + $ref: "#/components/schemas/EvalSubject" + run_number: + type: integer + description: > + This agent's nth run of this dataset, allocated once and never reused. What a run is + called. Numbered per agent rather than per subject kind: runs of what is deployed and + runs of its draft are the same agent's history. + run_job_id: + type: string + format: uuid + description: > + The flow executing the run: one job holding every case and its scores. + case_count: + type: integer + scores: + type: array + description: > + What the run scored, one entry per scorer that produced a number. Carried on the run + itself so a list of runs can say what each one scored without reading every cell of + every one of them. Empty on a run whose scores have not been read yet. + items: + $ref: "#/components/schemas/ExperimentScore" + running: + type: boolean + description: > + Whether the flow executing this run is still going. What makes a list of runs worth + watching rather than worth reloading. + created_at: + type: string + format: date-time + created_by: + type: string + + ExperimentScore: + type: object + description: >- + One scorer's headline for one run: the two numbers a column reports, over that run's cells. + required: [scorer_id, name, kind, scored, failed] + properties: + scorer_id: + type: string + name: + type: string + description: > + What the column is called in the dataset that ran it, resolved server-side because a + list of runs spanning datasets cannot hold every dataset's scorers to look it up. + kind: + type: string + enum: [agent, script] + mean: + type: number + pass_rate: + type: number + description: > + The share of scored cells at or above the column's threshold, for a column that has + one. Absent where the column has no threshold and the mean is the whole headline. + scored: + type: integer + failed: + type: integer + description: > + How many of the run's cells the column failed on. A column that failed on all of them + has no number to report and is still one of the columns that ran. + + CellScore: + type: object + description: One scorer's verdict on one run, and how it compares with the baseline. + required: [scorer_id, pending, definition_changed] + properties: + scorer_id: + type: string + score: + type: number + reason: + type: string + checks: {} + error: + type: string + not_applicable: + type: boolean + description: > + The scorer read this case and had nothing to measure on it. Left out of the column's + mean and pass rate rather than counted as a zero. + pending: + type: boolean + description: A scoring job is still running for this cell. + passed: + type: boolean + description: > + Which side of the scorer's `pass_if` threshold the score fell on. Absent when the + column has no threshold, or has no score yet. + baseline: + type: number + description: The same scorer's number on the baseline experiment. + definition_changed: + type: boolean + description: > + The baseline's score came from a different definition of this scorer, so the delta is + a change of scorer as much as a change of agent. + + ExperimentRow: + type: object + required: [case_id, input, status, scores] + properties: + case_id: + type: string + format: uuid + input: + $ref: "#/components/schemas/EvalCaseInput" + expected: {} + job_id: + type: string + format: uuid + description: > + The iteration that ran this case. Absent between a run being recorded and its flow + reaching this case, which reads as a case still to run. + status: + type: string + description: > + The case's status; `running` until its iteration completes, and `unavailable` for a + case whose job was retained away before anything read what it produced. + enum: [running, success, failure, canceled, skipped, unavailable] + output: + type: string + description: The agent's answer. The full trajectory stays reachable through job_id. + subject_version: + type: integer + format: int64 + description: > + The agent version this cell ran against. Cells of one experiment can differ, which the + table says rather than averaging two versions silently. + subject_draft_hash: + type: string + description: > + For a run of unsaved edits, the hash of the configuration this cell ran. Edits move + without a version changing, so this is what identifies what ran, and what recognises a + run whose edits were later saved as a run of that version. + scores: + type: array + description: One entry per scorer of the dataset, in column order. + items: + $ref: "#/components/schemas/CellScore" + + ScorerMean: + type: object + description: > + A column's summary. There is no single number for a dataset: averaging a judge with an + exact match would invent one. + required: [scorer_id, scored, missing_in_baseline, definition_changed] + properties: + scorer_id: + type: string + mean: + type: number + baseline_mean: + type: number + pass_rate: + type: number + description: > + The share of scored cells that passed, for a column with a threshold. Reported beside + the mean rather than instead of it: a pass rate says how many cases are good enough, + a mean says by how much, and neither answers the other's question. + baseline_pass_rate: + type: number + scored: + type: integer + missing_in_baseline: + type: integer + description: Cells the baseline has no score for, so a column the baseline never ran shows as unscored rather than as a spurious difference. + definition_changed: + type: boolean + FlowConversation: type: object required: @@ -26414,6 +27276,92 @@ components: type: integer minimum: 1 maximum: 2000000 + model_pricing: + type: object + additionalProperties: + $ref: "#/components/schemas/ModelPriceOverride" + + ModelPriceOverride: + type: object + description: negotiated rates in USD per million tokens, keyed `provider:model` + properties: + input: + type: number + minimum: 0 + maximum: 1000 + output: + type: number + minimum: 0 + maximum: 1000 + cache_read: + type: number + minimum: 0 + maximum: 1000 + cache_write: + type: number + minimum: 0 + maximum: 1000 + required: + - input + - output + + AITokenUsageEvent: + type: object + properties: + provider: + $ref: "#/components/schemas/AIProvider" + model: + type: string + session_id: + type: string + input_tokens: + type: integer + cache_read_tokens: + type: integer + cache_write_tokens: + type: integer + output_tokens: + type: integer + reported_cost_nano_usd: + type: integer + description: only set by providers that bill back an exact figure + requests: + type: integer + required: + - provider + - model + + AITokenUsageBucket: + type: object + properties: + key: + type: string + description: the grouped dimension's value; empty when grouping by model + provider: + type: string + model: + type: string + input_tokens: + type: integer + cache_read_tokens: + type: integer + cache_write_tokens: + type: integer + output_tokens: + type: integer + reported_cost_nano_usd: + type: integer + requests: + type: integer + required: + - key + - provider + - model + - input_tokens + - cache_read_tokens + - cache_write_tokens + - output_tokens + - requests InstanceAIProviderSummary: type: object @@ -28554,6 +29502,11 @@ components: id: type: integer format: int64 + description: How this version is addressed. Unique across every resource, so it says nothing about how many times this one has been saved. + version: + type: integer + format: int64 + description: Which version of this resource it is, counted from its first. What a version is called. created_at: type: string format: date-time @@ -28561,6 +29514,7 @@ components: type: string required: - id + - version - created_at ListableResource: diff --git a/backend/windmill-api/src/ai.rs b/backend/windmill-api/src/ai.rs index efc2ba7e60..1a5f9dba87 100644 --- a/backend/windmill-api/src/ai.rs +++ b/backend/windmill-api/src/ai.rs @@ -3,11 +3,16 @@ use crate::utils::check_scopes; #[cfg(feature = "bedrock")] use axum::routing::get; -#[cfg(feature = "bedrock")] use axum::Json; -use axum::{body::Bytes, extract::Path, response::IntoResponse, routing::post, Extension, Router}; +use axum::{ + body::Bytes, + extract::{DefaultBodyLimit, Path, Query}, + response::IntoResponse, + routing::post, + Extension, Router, +}; use futures::StreamExt; -use http::{HeaderMap, Method}; +use http::{HeaderMap, Method, StatusCode}; use quick_cache::sync::Cache; use reqwest::{Client, RequestBuilder}; use serde::{Deserialize, Serialize}; @@ -18,6 +23,7 @@ use windmill_ai::ai_cache::current_instance_ai_config_revision; use windmill_ai::ai_providers::{ empty_string_as_none, AIPlatform, AIProvider, ProviderConfig, ProviderModel, }; +use windmill_ai::ai_types::MAX_MODEL_RATE; use windmill_ai::credentials::ProviderCredentials; #[cfg(feature = "bedrock")] use windmill_ai::providers::bedrock::{ @@ -37,7 +43,7 @@ use windmill_ai::proxy::{ use windmill_audit::{audit_oss::audit_log, ActionKind}; use windmill_common::db::UserDB; use windmill_common::error::{to_anyhow, Error, Result}; -use windmill_common::utils::configure_client; +use windmill_common::utils::{configure_client, require_admin}; use windmill_common::variables::{get_variable_or_self, get_variable_or_self_as}; // AI timeout configuration constants @@ -417,9 +423,54 @@ pub struct AIConfig { pub custom_prompts: Option>, #[serde(skip_serializing_if = "Option::is_none")] pub max_tokens_per_model: Option>, + /// Per-model price overrides, keyed `provider:model` like `max_tokens_per_model`. + /// Only models whose rates differ from the built-in table are stored. + #[serde(skip_serializing_if = "Option::is_none")] + pub model_pricing: Option>, +} + +/// Negotiated rates in USD per million tokens. An unset cache rate is read as the +/// provider's own multiple of the input rate where the model has a published one, +/// and as the input rate itself where it does not — an unstated discount is never +/// filled in from another vendor's. +#[derive(Serialize, Deserialize, Debug, Clone)] +pub struct ModelPriceOverride { + pub input: f64, + pub output: f64, + #[serde(skip_serializing_if = "Option::is_none")] + pub cache_read: Option, + #[serde(skip_serializing_if = "Option::is_none")] + pub cache_write: Option, +} + +impl ModelPriceOverride { + pub fn validate(&self, key: &str) -> Result<()> { + for (field, rate) in [ + ("input", Some(self.input)), + ("output", Some(self.output)), + ("cache_read", self.cache_read), + ("cache_write", self.cache_write), + ] { + let Some(rate) = rate else { continue }; + if !rate.is_finite() || rate < 0.0 || rate > MAX_MODEL_RATE { + return Err(Error::BadRequest(format!( + "Price override for {}: {} must be between 0 and {}", + key, field, MAX_MODEL_RATE + ))); + } + } + Ok(()) + } } impl AIConfig { + pub fn validate_model_pricing(&self) -> Result<()> { + for (key, price) in self.model_pricing.iter().flatten() { + price.validate(key)?; + } + Ok(()) + } + pub fn has_providers(&self) -> bool { self.providers .as_ref() @@ -432,7 +483,18 @@ pub fn global_service() -> Router { } pub fn workspaced_service() -> Router { - let router = Router::new().route("/proxy/{*ai}", post(proxy).get(proxy)); + let router = Router::new() + .route("/proxy/{*ai}", post(proxy).get(proxy)) + .route( + "/usage", + post(record_ai_usage) + .get(list_ai_usage) + // The handler caps how many events it *stores*, but Json deserializes + // the whole array first — without a body limit an authenticated member + // could make the server allocate and parse an arbitrarily large one. + // Sized well above a full batch of the shape below. + .layer(DefaultBodyLimit::max(AI_USAGE_BODY_LIMIT)), + ); #[cfg(feature = "bedrock")] let router = router.route("/check_bedrock_credentials", get(check_bedrock_credentials)); @@ -440,6 +502,265 @@ pub fn workspaced_service() -> Router { router } +/// One provider request's worth of tokens, as counted by the chat client. +#[derive(Deserialize)] +struct AIUsageEvent { + provider: String, + model: String, + #[serde(default)] + session_id: String, + #[serde(default)] + input_tokens: i64, + #[serde(default)] + cache_read_tokens: i64, + #[serde(default)] + cache_write_tokens: i64, + #[serde(default)] + output_tokens: i64, + /// Only the providers that bill back an exact figure set this. + #[serde(default)] + reported_cost_nano_usd: Option, + #[serde(default)] + requests: Option, +} + +#[derive(Deserialize)] +struct RecordAIUsagePayload { + events: Vec, +} + +const MAX_AI_USAGE_EVENTS: usize = 50; +/// 64 KiB — a 50-event batch is a few kB even with the longest model ids. +const AI_USAGE_BODY_LIMIT: usize = 64 * 1024; +/// Well above any single conversation and far below an i64 overflow, so a client +/// bug caps out at one absurd row instead of poisoning the running total. +const MAX_TOKENS_PER_EVENT: i64 = 100_000_000; +/// $1000 in nano-USD. +const MAX_REPORTED_COST_PER_EVENT: i64 = 1_000_000_000_000; + +/// Model ids carry vendor prefixes and variant suffixes (`anthropic/claude-opus-5:thinking`), +/// so the shape check is looser than an identifier but still excludes whitespace and +/// anything that would not be a model id. +fn is_model_shaped(s: &str, max_len: usize) -> bool { + !s.is_empty() + && s.len() <= max_len + && s.chars() + .all(|c| c.is_ascii_alphanumeric() || matches!(c, '_' | '-' | ':' | '.' | '/' | '~')) +} + +/// Accumulate one workspace's AI token spend. Values are clamped and the caller's +/// email comes from the session, never the payload — the client is trusted to +/// report its own usage, not to attribute it to someone else. +async fn record_ai_usage( + authed: ApiAuthed, + Extension(db): Extension, + Path(w_id): Path, + Json(payload): Json, +) -> Result { + // Pre-sum duplicate keys: two rows hitting the same conflict target in a single + // INSERT error out ("cannot affect row a second time"). + let mut agg: HashMap<(String, String, String), AIUsageTotals> = HashMap::new(); + for e in payload.events.into_iter().take(MAX_AI_USAGE_EVENTS) { + if AIProvider::try_from(e.provider.as_str()).is_err() + || !is_model_shaped(&e.model, 255) + || !(e.session_id.is_empty() || is_model_shaped(&e.session_id, 50)) + { + continue; + } + let totals = agg + .entry((e.provider, e.model, e.session_id)) + .or_insert_with(AIUsageTotals::default); + totals.input += e.input_tokens.clamp(0, MAX_TOKENS_PER_EVENT); + totals.cache_read += e.cache_read_tokens.clamp(0, MAX_TOKENS_PER_EVENT); + totals.cache_write += e.cache_write_tokens.clamp(0, MAX_TOKENS_PER_EVENT); + totals.output += e.output_tokens.clamp(0, MAX_TOKENS_PER_EVENT); + totals.requests += e.requests.unwrap_or(1).clamp(0, MAX_AI_USAGE_EVENTS as i64); + if let Some(cost) = e.reported_cost_nano_usd { + totals.reported_cost = Some( + totals.reported_cost.unwrap_or(0) + cost.clamp(0, MAX_REPORTED_COST_PER_EVENT), + ); + } + } + if agg.is_empty() { + return Ok(StatusCode::NO_CONTENT); + } + + let mut providers = Vec::with_capacity(agg.len()); + let mut models = Vec::with_capacity(agg.len()); + let mut session_ids = Vec::with_capacity(agg.len()); + let mut inputs = Vec::with_capacity(agg.len()); + let mut cache_reads = Vec::with_capacity(agg.len()); + let mut cache_writes = Vec::with_capacity(agg.len()); + let mut outputs = Vec::with_capacity(agg.len()); + let mut reported_costs: Vec> = Vec::with_capacity(agg.len()); + let mut requests = Vec::with_capacity(agg.len()); + for ((provider, model, session_id), totals) in agg { + providers.push(provider); + models.push(model); + session_ids.push(session_id); + inputs.push(totals.input); + cache_reads.push(totals.cache_read); + cache_writes.push(totals.cache_write); + outputs.push(totals.output); + reported_costs.push(totals.reported_cost); + requests.push(totals.requests); + } + + sqlx::query!( + "INSERT INTO ai_token_usage (workspace_id, email, provider, model, session_id, \ + input_tokens, cache_read_tokens, cache_write_tokens, output_tokens, \ + reported_cost_nano_usd, requests) + SELECT $1, $2, * FROM UNNEST($3::text[], $4::text[], $5::text[], $6::bigint[], \ + $7::bigint[], $8::bigint[], $9::bigint[], $10::bigint[], $11::bigint[]) + ON CONFLICT (workspace_id, day, email, provider, model, session_id) + DO UPDATE SET + input_tokens = ai_token_usage.input_tokens + EXCLUDED.input_tokens, + cache_read_tokens = ai_token_usage.cache_read_tokens + EXCLUDED.cache_read_tokens, + cache_write_tokens = ai_token_usage.cache_write_tokens + EXCLUDED.cache_write_tokens, + output_tokens = ai_token_usage.output_tokens + EXCLUDED.output_tokens, + reported_cost_nano_usd = CASE + WHEN EXCLUDED.reported_cost_nano_usd IS NULL + THEN ai_token_usage.reported_cost_nano_usd + ELSE COALESCE(ai_token_usage.reported_cost_nano_usd, 0) + + EXCLUDED.reported_cost_nano_usd + END, + requests = ai_token_usage.requests + EXCLUDED.requests, + updated_at = now()", + &w_id, + &authed.email, + &providers, + &models, + &session_ids, + &inputs, + &cache_reads, + &cache_writes, + &outputs, + &reported_costs as &[Option], + &requests + ) + .execute(&db) + .await?; + + Ok(StatusCode::NO_CONTENT) +} + +#[derive(Default)] +struct AIUsageTotals { + input: i64, + cache_read: i64, + cache_write: i64, + output: i64, + reported_cost: Option, + requests: i64, +} + +#[derive(Deserialize)] +struct ListAIUsageQuery { + days: Option, + group_by: Option, + scope: Option, +} + +/// A bucket always carries its provider and model: the caller prices it from a +/// per-model rate table, which a bucket spanning several models could not be +/// resolved against. +#[derive(Serialize)] +struct AITokenUsageBucket { + key: String, + provider: String, + model: String, + input_tokens: i64, + cache_read_tokens: i64, + cache_write_tokens: i64, + output_tokens: i64, + reported_cost_nano_usd: Option, + requests: i64, +} + +/// Grouping by day over a long range, or by model across many models, can produce +/// more buckets than a table is worth rendering, so the listing is capped. +/// `truncated` says so explicitly — a caller that sums the rows into a total must be +/// able to tell that the total is partial rather than silently under-reporting spend. +#[derive(Serialize)] +struct AITokenUsageListing { + buckets: Vec, + truncated: bool, +} + +const AI_USAGE_MAX_BUCKETS: i64 = 1000; + +async fn list_ai_usage( + authed: ApiAuthed, + Extension(db): Extension, + Path(w_id): Path, + Query(query): Query, +) -> Result> { + // Reading the whole workspace's spend is an admin view; reading your own is + // not, so a member can see what they are costing without being shown their + // colleagues'. The filter is the session's email, never a parameter. + let own_email = match query.scope.as_deref().unwrap_or("workspace") { + "workspace" => { + require_admin(authed.is_admin, &authed.username)?; + None + } + "self" => Some(authed.email.clone()), + scope => return Err(Error::BadRequest(format!("Unsupported scope: {}", scope))), + }; + + let days = query.days.unwrap_or(30).clamp(1, 365); + let group_by = query.group_by.as_deref().unwrap_or("day"); + // No `session`: a session is identified by a client-generated id whose name + // lives only in the browser that made it, so a bucket keyed on one is a label + // nobody can resolve. `session_id` is still stored, at the grain the client + // batches on, should sessions ever gain a server-side name. + if !matches!(group_by, "day" | "user" | "model") { + return Err(Error::BadRequest(format!( + "Unsupported group_by: {}", + group_by + ))); + } + + // Fetch one past the cap to detect truncation. Ordering is by token volume, not + // by cost: rates are applied by the caller, so this query cannot know what a + // bucket cost. Volume is the closest proxy available here, and the caller is told + // the listing was capped rather than being left to sum a partial set silently. + let mut rows = sqlx::query_as!( + AITokenUsageBucket, + r#"SELECT + (CASE $3::text + WHEN 'day' THEN day::text + WHEN 'user' THEN email + ELSE '' + END) AS "key!", + provider AS "provider!", + model AS "model!", + SUM(input_tokens)::bigint AS "input_tokens!", + SUM(cache_read_tokens)::bigint AS "cache_read_tokens!", + SUM(cache_write_tokens)::bigint AS "cache_write_tokens!", + SUM(output_tokens)::bigint AS "output_tokens!", + SUM(reported_cost_nano_usd)::bigint AS "reported_cost_nano_usd", + SUM(requests)::bigint AS "requests!" + FROM ai_token_usage + WHERE workspace_id = $1 AND day > CURRENT_DATE - $2::int + AND ($5::text IS NULL OR email = $5) + GROUP BY 1, provider, model + ORDER BY SUM(input_tokens + cache_read_tokens + cache_write_tokens + output_tokens) DESC + LIMIT $4"#, + &w_id, + days, + group_by, + AI_USAGE_MAX_BUCKETS + 1, + own_email.as_deref() + ) + .fetch_all(&db) + .await?; + + let truncated = rows.len() as i64 > AI_USAGE_MAX_BUCKETS; + rows.truncate(AI_USAGE_MAX_BUCKETS as usize); + + Ok(Json(AITokenUsageListing { buckets: rows, truncated })) +} + /// Check if AWS Bedrock credentials are available from environment variables. #[cfg(feature = "bedrock")] async fn check_bedrock_credentials( diff --git a/backend/windmill-api/src/ai_evals/datasets.rs b/backend/windmill-api/src/ai_evals/datasets.rs new file mode 100644 index 0000000000..da901276b0 --- /dev/null +++ b/backend/windmill-api/src/ai_evals/datasets.rs @@ -0,0 +1,473 @@ +use super::*; + +#[derive(Serialize, Deserialize, Debug, Clone)] +pub struct EvalDataset { + pub path: String, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub summary: Option, + /// The columns of the results table, in display order. + #[serde(default)] + pub scorers: Vec, + pub created_at: DateTime, + pub created_by: String, + pub edited_at: DateTime, + pub edited_by: String, +} + +/// The agent-facing half of a case: exactly the inputs a standalone run feeds the agent. +#[derive(Serialize, Deserialize, Debug, Clone, Default)] +pub struct EvalCaseInput { + #[serde(default, skip_serializing_if = "Option::is_none")] + pub user_message: Option, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub user_attachments: Option>, +} + +#[derive(Serialize, Deserialize, Debug, Clone)] +pub struct EvalCase { + pub id: Uuid, + pub input: EvalCaseInput, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub expected: Option>, + pub created_at: DateTime, + pub created_by: String, +} + +/// The case fields a caller may set. `id`/`created_at`/`created_by` are assigned server-side so +/// a client cannot forge provenance or collide with an existing case. +#[derive(Deserialize, Debug)] +pub struct NewEvalCase { + #[serde(default)] + pub input: EvalCaseInput, + #[serde(default)] + pub expected: Option>, +} + +#[derive(Deserialize)] +pub struct CreateDataset { + pub path: String, + #[serde(default)] + pub summary: Option, + #[serde(default)] + pub scorers: Vec, + /// The cases to create it holding. A case cannot be written before there is a dataset for it + /// to be a row of, so they are sent with it rather than added afterwards. + #[serde(default)] + pub cases: Vec, +} + +#[derive(Deserialize)] +pub struct EditDataset { + /// Renames the dataset. Its cases and experiments follow through the foreign keys. + #[serde(default)] + pub path: Option, + /// Left out to keep the stored summary; sent as `""` to clear it. + #[serde(default)] + pub summary: Option, + /// Left out to keep the dataset's columns as they are; sent to replace them wholesale. + #[serde(default)] + pub scorers: Option>, + /// The cases as they should stand afterwards: all of them, each carrying its `id` if the + /// dataset already has it. Sent with the rest of an edit so a rename the dataset refuses + /// refuses the case edits with it, rather than leaving them written under the old name. + #[serde(default)] + pub cases: Option>, +} + +#[derive(Deserialize)] +pub struct SaveCase { + #[serde(default)] + pub id: Option, + #[serde(default)] + pub input: EvalCaseInput, + #[serde(default)] + pub expected: Option>, +} + +#[derive(Serialize)] +pub struct ListCasesResponse { + pub cases: Vec, +} + +pub async fn list_datasets( + authed: ApiAuthed, + Extension(user_db): Extension, + Path(w_id): Path, +) -> JsonResult> { + let mut tx = user_db.begin(&authed).await?; + let rows = sqlx::query!( + "SELECT path, summary, scorers, created_at, created_by, + edited_at, edited_by + FROM eval_dataset WHERE workspace_id = $1 ORDER BY path", + w_id + ) + .fetch_all(&mut *tx) + .await?; + tx.commit().await?; + Ok(Json( + rows.into_iter() + .map(|row| { + dataset_from_row( + row.path, + row.summary, + row.scorers, + row.created_at, + row.created_by, + row.edited_at, + row.edited_by, + ) + }) + .collect::>>()?, + )) +} + +pub async fn create_dataset( + authed: ApiAuthed, + Extension(user_db): Extension, + Path(w_id): Path, + Json(payload): Json, +) -> Result { + check_proper_path(&payload.path)?; + check_summary(payload.summary.as_deref())?; + if authed.is_operator { + return Err(Error::NotAuthorized( + "Operators cannot create eval datasets".to_string(), + )); + } + check_case_set( + payload + .cases + .iter() + .map(|case| (&case.input, case.expected.as_ref())), + )?; + let mut scorers = payload.scorers; + // A dataset being created has no columns yet, so every id is minted. + assign_scorer_ids(&mut scorers, &std::collections::HashSet::new())?; + let scorers = serde_json::to_value(&scorers)?; + // One `user_db` transaction: the row's insert policy gates the dataset, the cases' insert + // policy gates each case, and the two land together or not at all. + let mut tx = user_db.begin(&authed).await?; + // A path already taken returns no row; a path the caller may not create raises the insert + // policy, which `map_rls_denied` turns into an access error. + let created = sqlx::query_scalar!( + "INSERT INTO eval_dataset + (workspace_id, path, summary, scorers, created_by, edited_by) + VALUES ($1, $2, $3, $4, $5, $5) + ON CONFLICT (workspace_id, path) DO NOTHING + RETURNING path", + w_id, + payload.path, + payload.summary, + scorers, + authed.username, + ) + .fetch_optional(&mut *tx) + .await + .map_err(|e| map_rls_denied(&payload.path, "create", e))?; + if created.is_none() { + return Err(Error::BadRequest(format!( + "Eval dataset {} already exists", + payload.path + ))); + } + for case in &payload.cases { + sqlx::query!( + // clock_timestamp() (not the now() default, which is transaction-stable) so cases + // saved together get strictly increasing created_at and reload in insertion order; + // ORDER BY created_at, id would otherwise tie-break a same-transaction batch on the + // random uuid id. + "INSERT INTO eval_case + (workspace_id, dataset_path, input, expected, created_by, created_at) + VALUES ($1, $2, $3, $4, $5, clock_timestamp())", + w_id, + payload.path, + serde_json::to_value(&case.input)?, + opt_from_raw(case.expected.as_ref())?, + authed.username, + ) + .execute(&mut *tx) + .await?; + } + tx.commit().await?; + + Ok(format!("Created eval dataset {}", payload.path)) +} + +pub async fn get_dataset( + authed: ApiAuthed, + Extension(user_db): Extension, + Path((w_id, path)): Path<(String, String)>, +) -> JsonResult { + Ok(Json(read_dataset(&authed, &user_db, &w_id, &path).await?)) +} + +/// An edit is one transaction: the rename, the summary, the columns and the cases land together +/// or not at all. +pub async fn update_dataset( + authed: ApiAuthed, + Extension(user_db): Extension, + Path((w_id, path)): Path<(String, String)>, + Json(payload): Json, +) -> Result { + if authed.is_operator { + return Err(Error::NotAuthorized( + "Operators cannot modify eval datasets".to_string(), + )); + } + check_summary(payload.summary.as_deref())?; + let new_path = match payload.path.filter(|p| *p != path) { + Some(new_path) => { + check_proper_path(&new_path)?; + // A rename is owner-only, as for every other renamable object. RLS write access is not + // enough: the UPDATE policies carry no explicit WITH CHECK, so Postgres reuses their + // USING, and the row's own extra_perms travels with the rename and would satisfy it + // for any destination. + windmill_api_auth::require_owner_of_path(&authed, &path)?; + Some(new_path) + } + None => None, + }; + if let Some(cases) = &payload.cases { + check_cases(cases)?; + } + // One `user_db` transaction, governed by the row-level policies throughout. The row is read + // `FOR UPDATE` — its UPDATE policy decides who may — which also pins its cases, so a + // concurrent edit cannot restore a removed scorer's id or interleave with the case write. + let mut tx = user_db.clone().begin(&authed).await?; + let current = sqlx::query_scalar!( + "SELECT scorers FROM eval_dataset WHERE workspace_id = $1 AND path = $2 FOR UPDATE", + w_id, + path + ) + .fetch_optional(&mut *tx) + .await?; + let Some(current) = current else { + drop(tx); + return Err(write_refused(&authed, &user_db, &w_id, &path).await); + }; + let existing: std::collections::HashSet = + parse_scorers(current)?.into_iter().map(|s| s.id).collect(); + let scorers = match payload.scorers { + Some(mut scorers) => { + assign_scorer_ids(&mut scorers, &existing)?; + Some(serde_json::to_value(&scorers)?) + } + None => None, + }; + + let updated = sqlx::query_scalar!( + "UPDATE eval_dataset + SET path = COALESCE($6, path), summary = COALESCE($3, summary), + scorers = COALESCE($4, scorers), edited_at = now(), edited_by = $5 + WHERE workspace_id = $1 AND path = $2 + RETURNING path", + w_id, + path, + payload.summary, + scorers, + authed.username, + new_path.as_deref(), + ) + .fetch_optional(&mut *tx) + .await + .map_err(|e| { + if e.as_database_error().and_then(|e| e.code()).as_deref() == Some("23505") { + Error::BadRequest(format!( + "Eval dataset {} already exists", + new_path.as_deref().unwrap_or(&path) + )) + } else { + map_rls_denied(new_path.as_deref().unwrap_or(&path), "rename", e) + } + })?; + // No row updated: the caller cannot write this dataset (its UPDATE policy denied the row) or it + // is gone. A refused rename destination raises 42501 instead, handled just above. + let Some(updated) = updated else { + drop(tx); + return Err(write_refused(&authed, &user_db, &w_id, &path).await); + }; + // Under the name the dataset now has: the cases followed the rename through the foreign key. + if let Some(cases) = &payload.cases { + write_cases(&mut tx, &w_id, &updated, cases, &authed.username).await?; + } + tx.commit().await?; + Ok(format!("Updated eval dataset {}", updated)) +} + +/// The cases, the experiments and their recorded case sets go with the dataset, through the +/// foreign keys. The jobs those experiments produced are not touched: they are jobs, with their +/// own retention. +pub async fn delete_dataset( + authed: ApiAuthed, + Extension(user_db): Extension, + Path((w_id, path)): Path<(String, String)>, +) -> Result { + check_proper_path(&path)?; + if authed.is_operator { + return Err(Error::NotAuthorized( + "Operators cannot delete eval datasets".to_string(), + )); + } + let mut tx = user_db.clone().begin(&authed).await?; + let deleted = sqlx::query_scalar!( + "DELETE FROM eval_dataset WHERE workspace_id = $1 AND path = $2 RETURNING path", + w_id, + path + ) + .fetch_optional(&mut *tx) + .await?; + tx.commit().await?; + if deleted.is_none() { + return Err(write_refused(&authed, &user_db, &w_id, &path).await); + } + Ok(format!("Deleted eval dataset {}", path)) +} + +// ----------------------------------------------------------------------------------------------- +// Cases +// ----------------------------------------------------------------------------------------------- + +async fn read_cases( + tx: &mut sqlx::Transaction<'_, sqlx::Postgres>, + w_id: &str, + dataset: &str, + per_page: usize, + offset: usize, +) -> Result> { + let rows = sqlx::query!( + "SELECT id, input, expected, created_at, created_by + FROM eval_case + WHERE workspace_id = $1 AND dataset_path = $2 + ORDER BY created_at, id + LIMIT $3 OFFSET $4", + w_id, + dataset, + per_page as i64, + offset as i64 + ) + .fetch_all(&mut **tx) + .await?; + rows.into_iter() + .map(|row| { + Ok(EvalCase { + id: row.id, + input: serde_json::from_value(row.input)?, + expected: opt_to_raw(row.expected)?, + created_at: row.created_at, + created_by: row.created_by, + }) + }) + .collect() +} + +pub async fn list_cases( + authed: ApiAuthed, + Extension(user_db): Extension, + Path((w_id, path)): Path<(String, String)>, + Query(pagination): Query, +) -> JsonResult { + check_proper_path(&path)?; + let (per_page, offset) = paginate(pagination); + let mut tx = user_db.begin(&authed).await?; + // The dataset first, so an unknown or unreadable one is a 404 rather than an empty dataset: + // the case rows are invisible in both cases. + let dataset = sqlx::query_scalar!( + "SELECT path FROM eval_dataset WHERE workspace_id = $1 AND path = $2", + w_id, + path + ) + .fetch_optional(&mut *tx) + .await?; + if dataset.is_none() { + return Err(Error::NotFound(format!("Eval dataset {} not found", path))); + } + let cases = read_cases(&mut tx, &w_id, &path, per_page, offset).await?; + tx.commit().await?; + Ok(Json(ListCasesResponse { cases })) +} + +/// What a whole list of cases can be refused for, before any of it is written. +fn check_cases(cases: &[SaveCase]) -> Result<()> { + check_case_set( + cases + .iter() + .map(|case| (&case.input, case.expected.as_ref())), + )?; + // One row per id: the same id twice would write one row twice and return a list longer than + // the dataset it describes, and the save would read as having kept a case it dropped. + let mut ids: Vec = cases.iter().filter_map(|c| c.id).collect(); + ids.sort(); + let submitted = ids.len(); + ids.dedup(); + if ids.len() != submitted { + return Err(Error::BadRequest( + "A case id appears more than once in the dataset".to_string(), + )); + } + Ok(()) +} + +/// Replace a dataset's cases with `cases`, in the caller's transaction: rows not in the list go, +/// rows carrying an id are updated, the rest are added. Returns one id per case, in order. +async fn write_cases( + tx: &mut sqlx::Transaction<'_, sqlx::Postgres>, + w_id: &str, + path: &str, + cases: &[SaveCase], + username: &str, +) -> Result> { + let kept: Vec = cases.iter().filter_map(|c| c.id).collect(); + sqlx::query!( + "DELETE FROM eval_case + WHERE workspace_id = $1 AND dataset_path = $2 AND NOT (id = ANY($3))", + w_id, + path, + &kept + ) + .execute(&mut **tx) + .await?; + + let mut ids = Vec::with_capacity(cases.len()); + for case in cases { + let input = serde_json::to_value(&case.input)?; + let expected = opt_from_raw(case.expected.as_ref())?; + let id = match case.id { + Some(id) => sqlx::query_scalar!( + "UPDATE eval_case SET input = $4, expected = $5 + WHERE workspace_id = $1 AND dataset_path = $2 AND id = $3 + RETURNING id", + w_id, + path, + id, + input, + expected, + ) + .fetch_optional(&mut **tx) + .await? + .ok_or_else(|| Error::NotFound(format!("Eval case {} not found in {}", id, path)))?, + None => sqlx::query_scalar!( + // clock_timestamp() keeps a same-transaction batch in insertion order on reload. + "INSERT INTO eval_case + (workspace_id, dataset_path, input, expected, created_by, created_at) + VALUES ($1, $2, $3, $4, $5, clock_timestamp()) + RETURNING id", + w_id, + path, + input, + expected, + username, + ) + .fetch_one(&mut **tx) + .await + .map_err(|e| { + if is_missing_dataset(&e) { + Error::NotFound(format!("Eval dataset {} not found", path)) + } else { + e.into() + } + })?, + }; + ids.push(id); + } + Ok(ids) +} diff --git a/backend/windmill-api/src/ai_evals/mod.rs b/backend/windmill-api/src/ai_evals/mod.rs new file mode 100644 index 0000000000..b4d6009857 --- /dev/null +++ b/backend/windmill-api/src/ai_evals/mod.rs @@ -0,0 +1,371 @@ +//! Eval datasets for reusable AI agents. +//! +//! Five tables: `eval_dataset` and the `eval_case` rows it holds are the curated inputs; +//! `eval_experiment`, `eval_experiment_case` and `eval_score` are one run of them, written once +//! and only ever read afterwards. +//! +//! Datasets and cases go through `user_db`, so row-level security is the only access authority: +//! `eval_case`'s policies derive from its dataset's (`eval_dataset_writable`, in the migration). +//! The experiment tables carry read policies only and are written on the unrestricted pool after +//! the API has checked access — see `run_experiment` and `collect_experiment`. + +use axum::{ + extract::{Path, Query}, + routing::{get, post}, + Extension, Json, Router, +}; +use chrono::{DateTime, Utc}; +use serde::{Deserialize, Serialize}; +use serde_json::value::RawValue; +use uuid::Uuid; +use windmill_common::{ + db::UserDB, + error::{Error, JsonResult, Result}, + utils::{check_proper_path, paginate, Pagination}, +}; + +use crate::db::{ApiAuthed, DB}; +use windmill_api_auth::check_scopes; + +pub(crate) mod datasets; +pub(crate) mod payload; +pub(crate) mod results; +pub(crate) mod run; +pub(crate) mod scorers; +pub(crate) mod scoring; +pub(crate) mod subject; +pub(crate) mod template; + +pub(crate) use datasets::*; +pub(crate) use payload::*; +pub(crate) use results::*; +pub(crate) use run::*; +pub(crate) use scorers::*; +pub(crate) use scoring::*; +pub(crate) use subject::*; +pub(crate) use template::*; + +pub fn workspaced_service() -> Router { + Router::new() + .route("/datasets/list", get(list_datasets)) + .route("/datasets/create", post(create_dataset)) + .route("/datasets/get/{*path}", get(get_dataset)) + .route("/datasets/update/{*path}", post(update_dataset)) + .route("/datasets/delete/{*path}", post(delete_dataset)) + .route("/cases/list/{*path}", get(list_cases)) + .route("/scorer_defaults", get(scorer_defaults)) + .route("/run_payload", get(run_payload)) + .route("/experiments/run", post(run_experiment)) + .route("/experiments/collect", post(collect_experiment)) + .route("/scorers/recent", get(recent_scorers)) + .route("/subject_state", get(subject_state)) + .route("/experiments/list_all", get(list_all_experiments)) + .route("/experiments/results/{*path}", get(experiment_results)) +} + +/// Checked here rather than left to the column, whose own refusal comes back as an internal +/// database error naming no field. +const MAX_DATASET_SUMMARY_CHARS: usize = 1000; + +fn check_summary(summary: Option<&str>) -> Result<()> { + match summary { + Some(summary) if summary.chars().count() > MAX_DATASET_SUMMARY_CHARS => { + Err(Error::BadRequest(format!( + "This dataset's summary is {} characters, over the {} the column holds.", + summary.chars().count(), + MAX_DATASET_SUMMARY_CHARS + ))) + } + _ => Ok(()), + } +} + +/// A case is text — attachments are S3 references rather than inline bytes. +const MAX_CASE_BYTES: usize = 256 * 1024; +/// The whole case set together, so cases at the per-case cap cannot add up to a dataset a listing +/// or a run must hold hundreds of megabytes of at once. +const MAX_DATASET_BYTES: usize = 16 * 1024 * 1024; +/// Also what a listing returns in one page, so a dataset is always read whole: the editor holds +/// every case at once and writes them together, and half a set on screen is a Save that drops the +/// rest. +const MAX_CASES_PER_DATASET: i64 = 1_000; + +const MAX_EXPERIMENTS_LISTED: i64 = 100; + +const MAX_RECENT_SCORERS: usize = 12; + +/// A run's work is cases × scorers, so this bounds how far one request fans out. +const MAX_SCORERS_PER_DATASET: usize = 20; + +/// The dataset a write was aimed at is gone. Raised from the foreign key rather than from a +/// preceding existence check, so a dataset deleted mid-request cannot slip between the two. +fn is_missing_dataset(e: &sqlx::Error) -> bool { + e.as_database_error().and_then(|d| d.code()).as_deref() == Some("23503") +} + +/// A `user_db` write the row-level policies refused surfaces as SQLSTATE 42501, whose message +/// names the table and the policy. Turn it into one about access. +fn map_rls_denied(path: &str, action: &str, e: sqlx::Error) -> Error { + if e.as_database_error().and_then(|d| d.code()).as_deref() == Some("42501") { + return Error::NotAuthorized(format!("Not allowed to {} eval dataset {}", action, path)); + } + e.into() +} + +/// A write that matched no row is either a dataset that does not exist or one the caller can read +/// but not write. Row-level security cannot distinguish them — both are simply invisible to the +/// statement — so ask again with a plain read. +async fn write_refused(authed: &ApiAuthed, user_db: &UserDB, w_id: &str, path: &str) -> Error { + let visible = async { + let mut tx = user_db.clone().begin(authed).await?; + let found = sqlx::query_scalar!( + "SELECT path FROM eval_dataset WHERE workspace_id = $1 AND path = $2", + w_id, + path + ) + .fetch_optional(&mut *tx) + .await?; + tx.commit().await?; + Ok::<_, Error>(found.is_some()) + } + .await; + match visible { + Ok(true) => Error::NotAuthorized(format!( + "User {} does not have write access to eval dataset {}", + authed.username, path + )), + Ok(false) => Error::NotFound(format!("Eval dataset {} not found", path)), + Err(e) => e, + } +} + +/// One `eval_dataset` row, from the columns every read of the table selects. +fn dataset_from_row( + path: String, + summary: Option, + scorers: serde_json::Value, + created_at: DateTime, + created_by: String, + edited_at: DateTime, + edited_by: String, +) -> Result { + Ok(EvalDataset { + path, + summary, + scorers: parse_scorers(scorers)?, + created_at, + created_by, + edited_at, + edited_by, + }) +} + +/// A dataset's columns. Only this module writes them, through serde, so a value that does not +/// parse is corruption rather than input: defaulting to no columns would let the next save mint +/// fresh scorer ids and orphan every score already recorded. +pub(crate) fn parse_scorers(scorers: serde_json::Value) -> Result> { + serde_json::from_value(scorers) + .map_err(|e| Error::internal_err(format!("eval dataset scorers are not readable: {e}"))) +} + +/// Read the dataset the request names, through `user_db` so that a caller who cannot see it gets +/// the same answer as one asking for a dataset that does not exist. +async fn read_dataset( + authed: &ApiAuthed, + user_db: &UserDB, + w_id: &str, + path: &str, +) -> Result { + check_proper_path(path)?; + let mut tx = user_db.clone().begin(authed).await?; + let row = sqlx::query!( + "SELECT path, summary, scorers, created_at, created_by, + edited_at, edited_by + FROM eval_dataset WHERE workspace_id = $1 AND path = $2", + w_id, + path + ) + .fetch_optional(&mut *tx) + .await?; + tx.commit().await?; + let row = row.ok_or_else(|| Error::NotFound(format!("Eval dataset {} not found", path)))?; + dataset_from_row( + row.path, + row.summary, + row.scorers, + row.created_at, + row.created_by, + row.edited_at, + row.edited_by, + ) +} + +/// The dataset and its cases as one snapshot, so a launch cannot record the cases from before an +/// edit beside the scorers from after it. One transaction is not enough: `user_db` runs at READ +/// COMMITTED, where each statement takes a fresh snapshot, so the row is taken `FOR UPDATE` — +/// which an edit's own `FOR UPDATE` and a case write's foreign-key lock both conflict with. +pub(crate) async fn read_dataset_and_cases( + authed: &ApiAuthed, + user_db: &UserDB, + w_id: &str, + path: &str, +) -> Result<(EvalDataset, Vec)> { + check_proper_path(path)?; + let mut tx = user_db.clone().begin(authed).await?; + let row = sqlx::query!( + "SELECT path, summary, scorers, created_at, created_by, edited_at, edited_by + FROM eval_dataset WHERE workspace_id = $1 AND path = $2 FOR UPDATE", + w_id, + path + ) + .fetch_optional(&mut *tx) + .await?; + let Some(row) = row else { + tx.commit().await?; + return Err(Error::NotFound(format!("Eval dataset {} not found", path))); + }; + let case_rows = sqlx::query!( + "SELECT id, input, expected, created_at, created_by + FROM eval_case + WHERE workspace_id = $1 AND dataset_path = $2 + ORDER BY created_at, id", + w_id, + path + ) + .fetch_all(&mut *tx) + .await?; + tx.commit().await?; + let dataset = dataset_from_row( + row.path, + row.summary, + row.scorers, + row.created_at, + row.created_by, + row.edited_at, + row.edited_by, + )?; + let cases = case_rows + .into_iter() + .map(|row| { + Ok(EvalCase { + id: row.id, + input: serde_json::from_value(row.input)?, + expected: opt_to_raw(row.expected)?, + created_at: row.created_at, + created_by: row.created_by, + }) + }) + .collect::>>()?; + Ok((dataset, cases)) +} + +/// Whether this caller may write a dataset's contents: its cases, and the experiments that run +/// them. +/// +/// `SELECT … FOR UPDATE` applies `eval_dataset`'s UPDATE policies on top of its SELECT policies, +/// so the row itself answers who may write it, and a grant in `extra_perms` is honoured without +/// being mirrored here. +async fn require_dataset_writable( + authed: &ApiAuthed, + user_db: &UserDB, + w_id: &str, + path: &str, +) -> Result<()> { + check_proper_path(path)?; + if authed.is_operator { + return Err(Error::NotAuthorized( + "Operators cannot modify eval datasets".to_string(), + )); + } + let mut tx = user_db.clone().begin(authed).await?; + let writable = sqlx::query_scalar!( + "SELECT path FROM eval_dataset WHERE workspace_id = $1 AND path = $2 FOR UPDATE", + w_id, + path + ) + .fetch_optional(&mut *tx) + .await?; + tx.commit().await?; + if writable.is_some() { + Ok(()) + } else { + Err(write_refused(authed, user_db, w_id, path).await) + } +} + +/// jsonb columns are read as `serde_json::Value` and handed on as `RawValue`: a case's `expected` +/// is arbitrary user JSON that this module never looks inside. +fn opt_to_raw(value: Option) -> Result>> { + value + .map(|v| Ok(serde_json::value::to_raw_value(&v)?)) + .transpose() +} + +fn opt_from_raw(value: Option<&Box>) -> Result> { + value + .map(|v| Ok(serde_json::from_str(v.get())?)) + .transpose() +} + +fn check_case(input: &EvalCaseInput, expected: Option<&Box>) -> Result<()> { + // The shape the agent step reads its attachments in, checked when the case is written rather + // than when a run deserialises the step's arguments, which is after the case was queued. + if let Some(attachments) = &input.user_attachments { + if serde_json::from_str::>(attachments.get()).is_err() { + return Err(Error::BadRequest( + "A case's user_attachments is a list of S3 objects, each with an `s3` key naming \ + the file" + .to_string(), + )); + } + } + check_case_size(input, expected) +} + +/// The bytes one case weighs against its own and the dataset's caps. +fn case_bytes(input: &EvalCaseInput, expected: Option<&Box>) -> Result { + let mut bytes = serde_json::to_vec(input)?.len(); + if let Some(expected) = expected { + bytes += expected.get().len(); + } + Ok(bytes) +} + +/// What a whole case set can be refused for, before any of it is written. +fn check_case_set<'a>( + cases: impl ExactSizeIterator>)>, +) -> Result<()> { + if cases.len() as i64 > MAX_CASES_PER_DATASET { + return Err(Error::BadRequest(format!( + "An eval dataset holds at most {} cases. Split them into several datasets.", + MAX_CASES_PER_DATASET + ))); + } + let mut total = 0usize; + for (input, expected) in cases { + check_case(input, expected)?; + total += case_bytes(input, expected)?; + } + if total > MAX_DATASET_BYTES { + return Err(Error::BadRequest(format!( + "This dataset is {} KiB of cases, over the {} KiB limit. Attachments belong in \ + workspace storage and are referenced by a case, not stored inside it.", + total / 1024, + MAX_DATASET_BYTES / 1024 + ))); + } + Ok(()) +} + +fn check_case_size(input: &EvalCaseInput, expected: Option<&Box>) -> Result<()> { + let bytes = case_bytes(input, expected)?; + if bytes > MAX_CASE_BYTES { + return Err(Error::BadRequest(format!( + "This eval case is {} KiB, over the {} KiB limit. Attachments belong in workspace \ + storage and are referenced by a case, not stored inside it.", + bytes / 1024, + MAX_CASE_BYTES / 1024 + ))); + } + Ok(()) +} diff --git a/backend/windmill-api/src/ai_evals/payload.rs b/backend/windmill-api/src/ai_evals/payload.rs new file mode 100644 index 0000000000..e89948e9b5 --- /dev/null +++ b/backend/windmill-api/src/ai_evals/payload.rs @@ -0,0 +1,410 @@ +use super::*; + +/// What every scorer is handed: the answer, and the calls the agent made to reach it. +/// +/// Built from the job the run already stored, which is what lets a scorer added later score an +/// experiment that has already run. +#[derive(Serialize, Debug, Clone)] +pub struct EvalRunPayload { + pub input: EvalCaseInput, + #[serde(skip_serializing_if = "Option::is_none")] + pub output: Option>, + #[serde(skip_serializing_if = "Option::is_none")] + pub expected: Option>, + pub tool_calls: Vec, + /// The tools that were actually called, with the schema they were called against. A tool + /// whose schema could not be resolved carries `null`, and a scorer validating arguments must + /// treat that as unchecked rather than as a failure. + pub tools: Vec, + pub metrics: EvalMetrics, + pub status: String, + pub job_id: Uuid, +} + +#[derive(Serialize, Debug, Clone)] +pub struct EvalToolCall { + pub name: String, + #[serde(skip_serializing_if = "Option::is_none")] + pub args: Option>, + #[serde(skip_serializing_if = "Option::is_none")] + pub result: Option>, + #[serde(skip_serializing_if = "Option::is_none")] + pub error: Option, + #[serde(skip_serializing_if = "Option::is_none")] + pub duration_ms: Option, + /// Set when the result was too large to carry and was cut down. + #[serde(skip_serializing_if = "std::ops::Not::not")] + pub truncated: bool, +} + +#[derive(Serialize, Debug, Clone)] +pub struct EvalToolDef { + pub name: String, + #[serde(skip_serializing_if = "Option::is_none")] + pub schema: Option>, +} + +#[derive(Serialize, Debug, Clone)] +pub struct EvalMetrics { + pub steps: usize, + #[serde(skip_serializing_if = "Option::is_none")] + pub duration_ms: Option, + /// The provider's token counts, when it reported any. + #[serde(skip_serializing_if = "Option::is_none")] + pub usage: Option>, +} + +/// A tool result large enough to swamp a judge's context is cut here. The scorer is told, so a +/// check reading a truncated result can say so instead of failing on the missing tail. +const MAX_TOOL_RESULT_BYTES: usize = 4 * 1024; + +fn truncate_value(value: Box) -> (Box, bool) { + if value.get().len() <= MAX_TOOL_RESULT_BYTES { + return (value, false); + } + let text = value.get(); + let mut end = MAX_TOOL_RESULT_BYTES; + while !text.is_char_boundary(end) { + end -= 1; + } + match serde_json::value::to_raw_value(&format!("{}… [truncated]", &text[..end])) { + Ok(v) => (v, true), + Err(_) => (value, false), + } +} + +/// Assemble the payload from a completed case job: the agent step's own result carries the answer +/// and the message list, and every message that made a tool call names the job that ran it. +async fn build_run_payload( + db: &DB, + w_id: &str, + job_id: Uuid, + agent_job: Uuid, + input: EvalCaseInput, + expected: Option>, + status: String, + duration_ms: Option, +) -> Result { + // A read that failed is not a run with no answer: handing the scorers an empty payload would + // have them grade the absence of evidence and record that verdict permanently. + let agent_result = agent_result(db, w_id, job_id).await?.map(|(r, _)| r); + + let parsed: Option = agent_result + .as_ref() + .and_then(|r| serde_json::from_str(r.get()).ok()); + let output = parsed + .as_ref() + .and_then(|p| p.get("output")) + .map(|o| serde_json::value::to_raw_value(o)) + .transpose()?; + let usage = parsed + .as_ref() + .and_then(|p| p.get("usage")) + .map(|u| serde_json::value::to_raw_value(u)) + .transpose()?; + + // Walk the messages in order: a tool call is an `agent_action` on the message that made it. + let mut calls: Vec<(String, Option, Option>)> = vec![]; + if let Some(messages) = parsed + .as_ref() + .and_then(|p| p.get("messages")) + .and_then(|m| m.as_array()) + { + for message in messages { + let Some(action) = message.get("agent_action") else { + continue; + }; + match action.get("type").and_then(|t| t.as_str()) { + Some("tool_call") => calls.push(( + action + .get("function_name") + .and_then(|f| f.as_str()) + .unwrap_or("tool") + .to_string(), + action + .get("job_id") + .and_then(|j| j.as_str()) + .and_then(|j| Uuid::parse_str(j).ok()), + None, + )), + // An MCP call runs inside the agent rather than as a job, so its arguments are on + // the action itself. Its result lives in a later `role: "tool"` message rather + // than a child-job row, and is not surfaced to scorers yet. + Some("mcp_tool_call") => calls.push(( + action + .get("function_name") + .and_then(|f| f.as_str()) + .unwrap_or("tool") + .to_string(), + None, + action + .get("arguments") + .map(|a| serde_json::value::to_raw_value(a)) + .transpose()?, + )), + _ => {} + } + } + } + + let call_job_ids: Vec = calls.iter().filter_map(|(_, id, _)| *id).collect(); + let mut jobs = std::collections::HashMap::new(); + if !call_job_ids.is_empty() { + // Constrained to the agent step's own children rather than to the workspace: these ids + // come out of a job result, so a caller who can run a flow can put any id there. A tool + // call is pushed as a child of the agent that made it, which is what makes that the + // boundary. + let rows = sqlx::query!( + "SELECT j.id, j.args AS \"args: sqlx::types::Json>\", + c.result AS \"result: sqlx::types::Json>\", + c.status::text AS status, c.duration_ms, + s.schema AS \"schema: sqlx::types::Json>\" + FROM v2_job j + LEFT JOIN v2_job_completed c ON c.id = j.id + LEFT JOIN script s ON s.workspace_id = j.workspace_id AND s.hash = j.runnable_id + WHERE j.id = ANY($1) AND j.workspace_id = $2 AND j.parent_job = $3", + &call_job_ids, + w_id, + agent_job + ) + .fetch_all(db) + .await?; + for row in rows { + jobs.insert(row.id, row); + } + } + + let mut tool_calls = Vec::with_capacity(calls.len()); + let mut tools: Vec = vec![]; + for (name, call_job_id, inline_args) in calls { + let row = call_job_id.and_then(|id| jobs.get(&id)); + let (result, truncated) = match row.and_then(|r| r.result.as_ref()) { + Some(result) => { + let (value, truncated) = truncate_value(result.0.clone()); + (Some(value), truncated) + } + None => (None, false), + }; + let failed = row + .and_then(|r| r.status.as_deref()) + .map(|s| s != "success") + .unwrap_or(false); + if !tools.iter().any(|t| t.name == name) { + tools.push(EvalToolDef { + name: name.clone(), + schema: row.and_then(|r| r.schema.as_ref()).map(|s| s.0.clone()), + }); + } + // The already truncated result restated. `render_tool_calls` shows `error` and not + // `result` for a failed call, so the judge's context carries the payload once and bounded; + // `result` stays on the raw call for a script scorer. + let error = failed + .then(|| result.as_ref().map(|r| r.get().to_string())) + .flatten(); + tool_calls.push(EvalToolCall { + name, + args: inline_args.or_else(|| row.and_then(|r| r.args.as_ref()).map(|a| a.0.clone())), + result, + error, + duration_ms: row.map(|r| r.duration_ms), + truncated, + }); + } + + Ok(EvalRunPayload { + metrics: EvalMetrics { steps: tool_calls.len(), duration_ms, usage }, + input, + output, + expected, + tool_calls, + tools, + status, + job_id, + }) +} + +#[derive(Deserialize)] +pub struct RunPayloadQuery { + /// The flow job that answered the case: an iteration of a run. + pub job_id: Uuid, +} + +/// What the scorers of one iteration are handed. +#[derive(Serialize)] +pub struct RunPayloadResponse { + pub run: EvalRunPayload, + /// The same run as a judge reads it. Rendered once per case rather than once per judge. + pub rendered: String, +} + +/// Assemble the payload for one answered case, for the step that feeds the scorers. +/// +/// The case is read from the job's arguments rather than from the experiment, so this works for an +/// iteration whose row has not been filled in yet. +pub async fn run_payload( + authed: ApiAuthed, + Extension(db): Extension, + Extension(user_db): Extension, + Path(w_id): Path, + Query(query): Query, +) -> JsonResult { + // `UserDB` enforces row permissions but not a token's scopes, so without this an + // `ai_evals:read` token would read job arguments, results and tool calls that `jobs:read` + // is what actually gates. Job tokens are unscoped, so the run flow's payload step passes. + check_scopes(&authed, || "jobs:read".to_string())?; + // Through `user_db`: the caller is a job token, and it reads what its runner can read. + let mut tx = user_db.begin(&authed).await?; + let args = sqlx::query_scalar!( + "SELECT args AS \"args: sqlx::types::Json>\" FROM v2_job + WHERE id = $1 AND workspace_id = $2", + query.job_id, + w_id + ) + .fetch_optional(&mut *tx) + .await? + .flatten() + .ok_or_else(|| Error::NotFound(format!("Job {} not found", query.job_id)))?; + tx.commit().await?; + + let args: serde_json::Value = serde_json::from_str(args.0.get())?; + // An iteration carries its case; a run recorded one job per case carries the same input under + // the stamp that job was pushed with. + let case = args.get("iter").and_then(|i| i.get("value")); + let input = case + .and_then(|c| c.get("input")) + .or_else(|| args.get("_eval_input")) + .cloned() + .unwrap_or_else(|| serde_json::json!({})); + let expected = case + .and_then(|c| c.get("expected")) + .or_else(|| args.get("expected")) + .cloned(); + + // The agent step's own status and duration, never the iteration's: the iteration goes on to + // assemble this payload and run the scorers, so a scorer reading the iteration's duration + // would be measuring itself. + let agent_job = agent_step_job(&db, &w_id, query.job_id) + .await? + .unwrap_or(query.job_id); + let completed = sqlx::query!( + "SELECT status::text AS \"status!\", duration_ms FROM v2_job_completed + WHERE id = $1 AND workspace_id = $2", + agent_job, + w_id + ) + .fetch_optional(&db) + .await?; + + let run = build_run_payload( + &db, + &w_id, + query.job_id, + agent_job, + serde_json::from_value(input)?, + expected + .map(|e| serde_json::value::to_raw_value(&e)) + .transpose()?, + completed + .as_ref() + .map(|c| c.status.clone()) + // The iteration asking is itself still running: its agent step is what finished. + .unwrap_or_else(|| "success".to_string()), + completed.as_ref().map(|c| c.duration_ms), + ) + .await?; + let rendered = render_run(&run); + Ok(Json(RunPayloadResponse { run, rendered })) +} + +/// The job of the agent step inside a run's flow, from the flow status of either a running or a +/// finished one. +async fn agent_step_job(db: &DB, w_id: &str, flow_job: Uuid) -> Result> { + let modules = sqlx::query_scalar!( + "SELECT COALESCE(s.flow_status, c.flow_status) -> 'modules' AS modules + FROM v2_job j + LEFT JOIN v2_job_status s ON s.id = j.id + LEFT JOIN v2_job_completed c ON c.id = j.id + WHERE j.id = $1 AND j.workspace_id = $2", + flow_job, + w_id + ) + .fetch_optional(db) + .await? + .flatten(); + Ok(modules + .as_ref() + .and_then(|m| m.as_array()) + .and_then(|modules| { + modules + .iter() + .find(|m| m.get("id").and_then(|i| i.as_str()) == Some(AGENT_NODE_ID)) + }) + .and_then(|m| m.get("job")) + .and_then(|j| j.as_str()) + .and_then(|j| Uuid::parse_str(j).ok())) +} + +/// The system prompt a judge agent is created with. It is the agent's own, so editing a judge is +/// editing that resource — there is no second copy of the grading contract on the dataset. +pub const JUDGE_SYSTEM_PROMPT: &str = r#"You are grading one run of an AI agent. + +Score how well the agent handled the request, from 0 to 1. Judge the whole trajectory, not only the +final answer. Penalise asking for information already in the request, calling a tool twice with the +same arguments, and tool errors left unrecovered. + +Reply with JSON only, of the form {"score": , "reason": }."#; + +fn render_json(value: Option<&RawValue>) -> String { + value + .map(|v| v.get().to_string()) + .unwrap_or_else(|| "(none)".to_string()) +} + +/// Tool calls as the judge reads them: numbered, in order, with arguments, result and duration. +fn render_tool_calls(calls: &[EvalToolCall]) -> String { + if calls.is_empty() { + return "(none)".to_string(); + } + calls + .iter() + .enumerate() + .map(|(index, call)| { + let args = call.args.as_ref().map(|a| a.get()).unwrap_or("{}"); + let outcome = match (&call.error, &call.result) { + (Some(error), _) => format!("error: {}", error), + (None, Some(result)) => result.get().to_string(), + (None, None) => "(no result)".to_string(), + }; + let timing = call + .duration_ms + .map(|ms| format!(" ({}ms)", ms)) + .unwrap_or_default(); + format!( + "{}. {}({}) -> {}{}", + index + 1, + call.name, + args, + outcome, + timing + ) + }) + .collect::>() + .join("\n") +} + +/// One run, as a judge is shown it. +fn render_run(run: &EvalRunPayload) -> String { + format!( + "Request: {}\nTool calls, in order:\n{}\nAnswer: {}\nExpected: {}", + run.input.user_message.as_deref().unwrap_or("(none)"), + render_tool_calls(&run.tool_calls), + render_json(run.output.as_deref()), + render_json(run.expected.as_deref()), + ) +} + +/// Module id of a scorer inside a scoring job. `assign_scorer_ids` keeps ids to +/// `[A-Za-z0-9_]`, so this is a valid identifier. +pub(crate) fn scorer_module_id(scorer_id: &str) -> String { + format!("s_{}", scorer_id) +} diff --git a/backend/windmill-api/src/ai_evals/results.rs b/backend/windmill-api/src/ai_evals/results.rs new file mode 100644 index 0000000000..500af88d72 --- /dev/null +++ b/backend/windmill-api/src/ai_evals/results.rs @@ -0,0 +1,856 @@ +use super::*; + +/// One run of a dataset: written once when the dataset is run, and only ever read afterwards. +#[derive(Serialize, Deserialize, Debug, Clone)] +pub struct EvalExperiment { + pub id: Uuid, + pub dataset: String, + pub subject: EvalSubject, + /// This subject's nth run of this dataset, allocated once and never reused: "Run 7" survives + /// history being pruned, which a position computed when the list is read would not. + pub run_number: i32, + /// The flow executing the run: one job holding every case and its scores. + pub run_job_id: Uuid, + pub case_count: i64, + /// What the run scored, one entry per scorer that produced a number. Carried on the run so a + /// list can say what each one scored without reading every cell of every one of them. + #[serde(default, skip_serializing_if = "Vec::is_empty")] + pub scores: Vec, + /// Whether the flow executing this run is still going. What makes a list of runs worth + /// watching rather than worth reloading. + #[serde(default, skip_serializing_if = "std::ops::Not::not")] + pub running: bool, + pub created_at: DateTime, + pub created_by: String, +} + +/// One scorer's headline for one run: the two numbers a column reports, over that run's cells. +#[derive(Serialize, Deserialize, Debug, Clone)] +pub struct ExperimentScore { + pub scorer_id: String, + /// What the column is called in the dataset that ran it, resolved here because a list of runs + /// spanning datasets cannot hold every dataset's scorers to look it up. + pub name: String, + /// `agent` or `script`, for the badge to say which kind of thing produced the number. + pub kind: String, + #[serde(skip_serializing_if = "Option::is_none")] + pub mean: Option, + /// The share of scored cells at or above the column's threshold, for a column that has one. + /// Absent where the column has no threshold and the mean is the whole headline. + #[serde(skip_serializing_if = "Option::is_none")] + pub pass_rate: Option, + pub scored: i64, + /// How many of this run's cells the column failed on. A column that failed on all of them + /// still ran, which is the difference between a headline of nothing and no headline at all. + pub failed: i64, +} + +#[derive(Deserialize)] +pub struct ListExperimentsQuery { + /// Restrict to one agent's runs. Both what was deployed and what was drafted are that agent's + /// history, so this does not discriminate by kind. + #[serde(default)] + pub subject_path: Option, +} + +/// Every run of this agent, across every dataset it has been measured on. +/// +/// Filtered by `user_db`: a run is visible exactly when the dataset it belongs to is. +pub async fn list_all_experiments( + authed: ApiAuthed, + Extension(db): Extension, + Extension(user_db): Extension, + Path(w_id): Path, + Query(query): Query, +) -> JsonResult> { + let mut tx = user_db.clone().begin(&authed).await?; + let rows = sqlx::query!( + "SELECT e.id, e.dataset_path, e.subject, e.run_number, e.run_job_id, e.created_at, + e.created_by, + (SELECT count(*) FROM eval_experiment_case c WHERE c.experiment_id = e.id) + AS \"case_count!\" + FROM eval_experiment e + JOIN eval_dataset d ON d.workspace_id = e.workspace_id AND d.path = e.dataset_path + WHERE e.workspace_id = $1 + AND ($3::text IS NULL OR e.subject ->> 'path' = $3) + ORDER BY e.created_at DESC + LIMIT $2", + w_id, + MAX_EXPERIMENTS_LISTED, + query.subject_path, + ) + .fetch_all(&mut *tx) + .await?; + tx.commit().await?; + let mut experiments = rows + .into_iter() + .map(|row| { + experiment_from_row( + row.id, + row.dataset_path, + row.subject, + row.run_number, + row.run_job_id, + row.case_count, + row.created_at, + row.created_by, + ) + }) + .collect::>>()?; + + resolve_listed_drafts(&authed, &db, &user_db, &w_id, &mut experiments).await?; + let scorers_by_dataset = scorers_of_listed(&authed, &user_db, &w_id, &experiments).await?; + mark_running(&db, &w_id, &mut experiments).await?; + sync_listed_runs(&db, &w_id, &experiments).await?; + let mut scores = experiment_scores(&db, &experiments, &scorers_by_dataset).await?; + for experiment in experiments.iter_mut() { + experiment.scores = scores.remove(&experiment.id).unwrap_or_default(); + } + Ok(Json(experiments)) +} + +/// Which listed runs are still going, read from the flows executing them. A run whose flow is no +/// longer there at all is over: jobs have their own retention, and reading a missing one as +/// unfinished would leave every run older than it spinning. +async fn mark_running(db: &DB, w_id: &str, experiments: &mut [EvalExperiment]) -> Result<()> { + let job_ids: Vec = experiments.iter().map(|e| e.run_job_id).collect(); + if job_ids.is_empty() { + return Ok(()); + } + let unfinished: std::collections::HashSet = sqlx::query_scalar!( + "SELECT j.id AS \"id!\" FROM v2_job j + LEFT JOIN v2_job_completed c ON c.id = j.id AND c.workspace_id = $2 + WHERE j.id = ANY($1) AND j.workspace_id = $2 AND c.id IS NULL", + &job_ids, + w_id + ) + .fetch_all(db) + .await? + .into_iter() + .collect(); + for experiment in experiments.iter_mut() { + experiment.running = unfinished.contains(&experiment.run_job_id); + } + Ok(()) +} + +/// A run of a draft whose edits have since been deployed is a run of that version. Resolved once +/// per subject rather than once per run, because a listing is usually one agent's history. +async fn resolve_listed_drafts( + authed: &ApiAuthed, + db: &DB, + user_db: &UserDB, + w_id: &str, + experiments: &mut [EvalExperiment], +) -> Result<()> { + let drafted: std::collections::HashSet = experiments + .iter() + .filter(|e| e.subject.kind == EvalSubjectKind::AgentDraft) + .map(|e| e.subject.path.clone()) + .collect(); + if drafted.is_empty() { + return Ok(()); + } + // Read each subject as the caller (see experiment_results): an agent the caller cannot read + // yields no hash or version, so its config fingerprint never leaks through the list either. + let mut deployed = std::collections::HashMap::new(); + for path in drafted { + let (hash, version) = match readable_agent_state(authed, user_db, w_id, &path).await? { + Some((config, version)) => (Some(draft_hash(&config)), Some(version)), + None => (None, None), + }; + deployed.insert(path.clone(), (hash, version)); + } + for experiment in experiments.iter_mut() { + let Some((hash, version)) = deployed.get(&experiment.subject.path) else { + continue; + }; + // Each run's own dataset: the list may span them, and the update is keyed on both. + let dataset = experiment.dataset.clone(); + resolve_deployed_draft(db, w_id, &dataset, experiment, hash.as_deref(), *version).await?; + } + Ok(()) +} + +/// How many listed runs one list call reads out of their flows. A run's scores live in its flow +/// until something reads them into `eval_score`, so an unopened run has nothing to report; the cap +/// keeps a long history from turning one list call into a hundred flow reads. +const MAX_RUNS_SYNCED_PER_LIST: usize = 10; + +/// Read the flows of listed runs that still have scores to collect. Runs already collected are +/// skipped, so the steady-state cost of listing is one query rather than one read per run. +async fn sync_listed_runs(db: &DB, w_id: &str, experiments: &[EvalExperiment]) -> Result<()> { + if experiments.is_empty() { + return Ok(()); + } + let ids: Vec = experiments.iter().map(|e| e.id).collect(); + let unread = sqlx::query_scalar!( + "SELECT DISTINCT experiment_id FROM eval_score + WHERE experiment_id = ANY($1) AND score IS NULL AND error IS NULL + AND NOT not_applicable", + &ids + ) + .fetch_all(db) + .await? + .into_iter() + .collect::>(); + for experiment in experiments + .iter() + .filter(|e| unread.contains(&e.id)) + .take(MAX_RUNS_SYNCED_PER_LIST) + { + // Best-effort, for the same reason reading one run is: this is the home screen, and one + // run with an unreadable cell must not cost the list of every other run. + if let Err(e) = sync_run(db, w_id, experiment.id, experiment.run_job_id, false).await { + tracing::warn!("could not collect eval run {}: {e:#}", experiment.id); + } + } + Ok(()) +} + +/// Every listed run's per-scorer headline, in one grouped query. +/// +/// Thresholds come from each run's own dataset as its scorers are *now*, joined per (run, scorer) +/// rather than per scorer: a list spanning datasets is a list of runs whose columns are not the +/// same columns. +async fn experiment_scores( + db: &DB, + experiments: &[EvalExperiment], + scorers_by_dataset: &std::collections::HashMap>, +) -> Result>> { + let mut by_experiment: std::collections::HashMap> = + Default::default(); + // One entry per (run, column) it could have scored, which is what carries the threshold and + // the column's order into the query. + let mut ids: Vec = vec![]; + let mut scorer_ids: Vec = vec![]; + let mut thresholds: Vec> = vec![]; + for experiment in experiments { + for scorer in scorers_by_dataset + .get(&experiment.dataset) + .map(|s| s.as_slice()) + .unwrap_or(&[]) + { + ids.push(experiment.id); + scorer_ids.push(scorer.id.clone()); + thresholds.push(scorer.pass_if); + } + } + if ids.is_empty() { + return Ok(by_experiment); + } + let rows = sqlx::query!( + "SELECT s.experiment_id AS \"experiment_id!\", s.scorer_id AS \"scorer_id!\", + avg(s.score) AS mean, + count(s.score) AS \"scored!\", + count(*) FILTER (WHERE s.error IS NOT NULL) AS \"failed!\", + count(*) FILTER (WHERE t.pass_if IS NOT NULL AND s.score >= t.pass_if) + AS \"passed!\", + bool_or(t.pass_if IS NOT NULL) AS \"has_threshold!\" + FROM eval_score s + JOIN unnest($1::uuid[], $2::text[], $3::float8[]) + AS t(experiment_id, scorer_id, pass_if) + ON t.experiment_id = s.experiment_id AND t.scorer_id = s.scorer_id + GROUP BY s.experiment_id, s.scorer_id", + &ids, + &scorer_ids, + &thresholds as &[Option], + ) + .fetch_all(db) + .await?; + let mut headline: std::collections::HashMap< + (Uuid, String), + (Option, i64, i64, i64, bool), + > = Default::default(); + for row in rows { + headline.insert( + (row.experiment_id, row.scorer_id), + ( + row.mean, + row.scored, + row.failed, + row.passed, + row.has_threshold, + ), + ); + } + // Emitted in the dataset's column order rather than the query's, so the badges on a row read + // left to right the way that dataset's table does. + for experiment in experiments { + for scorer in scorers_by_dataset + .get(&experiment.dataset) + .map(|s| s.as_slice()) + .unwrap_or(&[]) + { + // A column with no cells at all on this run is one added after it. A column that has + // cells is reported even where none produced a number, which is what a column that + // failed throughout looks like. + let Some((mean, scored, failed, passed, has_threshold)) = + headline.get(&(experiment.id, scorer.id.clone())) + else { + continue; + }; + by_experiment + .entry(experiment.id) + .or_default() + .push(ExperimentScore { + scorer_id: scorer.id.clone(), + name: scorer_name(scorer), + kind: scorer.def.kind_str().to_string(), + mean: *mean, + pass_rate: (*has_threshold && *scored > 0) + .then(|| *passed as f64 / *scored as f64), + scored: *scored, + failed: *failed, + }); + } + } + Ok(by_experiment) +} + +/// The scorers of every dataset named by a listed run, read through `user_db` so a run of a +/// dataset the caller cannot read contributes nothing. +async fn scorers_of_listed( + authed: &ApiAuthed, + user_db: &UserDB, + w_id: &str, + experiments: &[EvalExperiment], +) -> Result>> { + let paths: Vec = experiments + .iter() + .map(|e| e.dataset.clone()) + .collect::>() + .into_iter() + .collect(); + if paths.is_empty() { + return Ok(Default::default()); + } + let mut tx = user_db.clone().begin(authed).await?; + let rows = sqlx::query!( + "SELECT path, scorers FROM eval_dataset WHERE workspace_id = $1 AND path = ANY($2)", + w_id, + &paths + ) + .fetch_all(&mut *tx) + .await?; + tx.commit().await?; + rows.into_iter() + .map(|row| Ok((row.path, parse_scorers(row.scorers)?))) + .collect() +} + +#[derive(Deserialize)] +pub struct ExperimentRef { + pub id: Uuid, + /// The experiment every column is compared against. A delta is only ever computed between two + /// scores of the same scorer id. + #[serde(default)] + pub baseline: Option, +} + +/// One scorer's verdict on one run, and how it compares with the baseline. +#[derive(Serialize)] +pub struct CellScore { + pub scorer_id: String, + #[serde(skip_serializing_if = "Option::is_none")] + pub score: Option, + #[serde(skip_serializing_if = "Option::is_none")] + pub reason: Option, + #[serde(skip_serializing_if = "Option::is_none")] + pub checks: Option>, + #[serde(skip_serializing_if = "Option::is_none")] + pub error: Option, + /// The scorer read this case and had nothing to measure on it. Left out of the column's mean + /// and pass rate rather than counted as a zero. + #[serde(skip_serializing_if = "std::ops::Not::not")] + pub not_applicable: bool, + /// A scoring job is still running for this cell. + pub pending: bool, + /// Which side of the scorer's threshold the score fell on, when it has one. + #[serde(skip_serializing_if = "Option::is_none")] + pub passed: Option, + #[serde(skip_serializing_if = "Option::is_none")] + pub baseline: Option, + /// The baseline's score for this scorer was produced by a different definition of it, so the + /// delta is a change of scorer as much as a change of agent. + pub definition_changed: bool, +} + +/// One row per case: what it was asked, what the agent answered, and each scorer's cell. +#[derive(Serialize)] +pub struct ExperimentRow { + pub case_id: Uuid, + pub input: EvalCaseInput, + #[serde(skip_serializing_if = "Option::is_none")] + pub expected: Option>, + /// The iteration that ran this case. Absent between a run being recorded and its flow + /// reaching this case, which reads as a case still to run. + #[serde(skip_serializing_if = "Option::is_none")] + pub job_id: Option, + /// What happened to the answer: the iteration's own `success`/`failure`/`canceled`/`skipped` + /// once it has finished, and until then the agent step's, since the answer is written before + /// the scorers that keep the iteration running have read it. `unavailable` for a case whose + /// job was retained away before anything read what it produced. + pub status: String, + /// The agent's answer, which is what a table cell shows. The whole trajectory stays + /// reachable through `job_id`, so the row carries the text rather than the result object. + #[serde(skip_serializing_if = "Option::is_none")] + pub output: Option, + /// The agent version this cell ran against. Cells of one experiment can differ, which is what + /// the table says instead of averaging two versions silently. + #[serde(skip_serializing_if = "Option::is_none")] + pub subject_version: Option, + /// For a run of unsaved edits, the hash of the configuration this cell ran: edits move without + /// a version changing, and `resolve_deployed_draft` matches this against what is deployed. + #[serde(skip_serializing_if = "Option::is_none")] + pub subject_draft_hash: Option, + /// One entry per scorer of the dataset, in column order. + pub scores: Vec, +} + +/// A column's summary. There is no single number for a dataset: averaging a judge with an exact +/// match would invent one. +#[derive(Serialize)] +pub struct ScorerMean { + pub scorer_id: String, + #[serde(skip_serializing_if = "Option::is_none")] + pub mean: Option, + #[serde(skip_serializing_if = "Option::is_none")] + pub baseline_mean: Option, + /// The share of scored cells that passed, for a column with a threshold. Reported beside the + /// mean rather than instead of it: neither number answers the other's question. + #[serde(skip_serializing_if = "Option::is_none")] + pub pass_rate: Option, + #[serde(skip_serializing_if = "Option::is_none")] + pub baseline_pass_rate: Option, + pub scored: usize, + /// Cells the baseline has no score for, reported so a column the baseline never ran shows as + /// unscored rather than as a spurious difference. + pub missing_in_baseline: usize, + pub definition_changed: bool, +} + +#[derive(Serialize)] +pub struct ExperimentResults { + pub experiment: EvalExperiment, + #[serde(skip_serializing_if = "Option::is_none")] + pub baseline: Option, + /// The columns, which belong to the dataset rather than to the experiment. + pub scorers: Vec, + pub rows: Vec, + pub means: Vec, + /// Cells scoring lower than the baseline, across every column. + pub regressed: usize, + /// The version the subject is on now. A row that ran against an earlier one describes an + /// agent that no longer exists. + #[serde(skip_serializing_if = "Option::is_none")] + pub subject_current_version: Option, + /// What the agent hashes to as deployed. A run of unsaved edits carrying this hash ran exactly + /// what is deployed now — the edits were saved — so it is a run of that version. + #[serde(skip_serializing_if = "Option::is_none")] + pub subject_deployed_hash: Option, +} + +/// The agent's own result is `{output, messages}`; the answer is its `output`. +pub(crate) fn agent_answer(result: &RawValue) -> Option { + let parsed: serde_json::Value = serde_json::from_str(result.get()).ok()?; + match parsed.get("output") { + Some(serde_json::Value::String(s)) => Some(s.clone()), + Some(other) => Some(other.to_string()), + None => None, + } +} + +struct ScoreRow { + score: Option, + reason: Option, + checks: Option, + error: Option, + not_applicable: bool, + definition: String, +} + +/// Every score of one experiment, keyed by the cell and the scorer that produced it. +async fn load_scores( + db: &DB, + experiment_id: Uuid, +) -> Result> { + Ok(sqlx::query!( + "SELECT ordinal, scorer_id, score, reason, checks, error, not_applicable, definition + FROM eval_score WHERE experiment_id = $1", + experiment_id + ) + .fetch_all(db) + .await? + .into_iter() + .map(|r| { + ( + (r.ordinal, r.scorer_id), + ScoreRow { + score: r.score, + reason: r.reason, + checks: r.checks, + error: r.error, + not_applicable: r.not_applicable, + definition: r.definition, + }, + ) + }) + .collect()) +} + +async fn read_experiment(db: &DB, w_id: &str, dataset: &str, id: Uuid) -> Result { + let row = sqlx::query!( + "SELECT e.subject, e.run_number, e.run_job_id, e.created_at, + e.created_by, + (SELECT count(*) FROM eval_experiment_case c WHERE c.experiment_id = e.id) + AS \"case_count!\" + FROM eval_experiment e + WHERE e.workspace_id = $1 AND e.dataset_path = $2 AND e.id = $3", + w_id, + dataset, + id + ) + .fetch_optional(db) + .await? + .ok_or_else(|| { + Error::NotFound(format!( + "Experiment {} not found in eval dataset {}", + id, dataset + )) + })?; + experiment_from_row( + id, + dataset.to_string(), + row.subject, + row.run_number, + row.run_job_id, + row.case_count, + row.created_at, + row.created_by, + ) +} + +/// Recognise a draft run that has since been deployed, and record it as the version it became. +/// +/// Written once rather than derived per read: derived against what is deployed *now*, the next +/// deployment would send a run that already read `v21` back to `v18 + edits`. +async fn resolve_deployed_draft( + db: &DB, + w_id: &str, + dataset: &str, + experiment: &mut EvalExperiment, + deployed_hash: Option<&str>, + deployed_version: Option, +) -> Result<()> { + if experiment.subject.kind != EvalSubjectKind::AgentDraft { + return Ok(()); + } + let (Some(hash), Some(deployed_hash), Some(version)) = ( + experiment.subject.draft_hash.as_deref(), + deployed_hash, + deployed_version, + ) else { + return Ok(()); + }; + if hash != deployed_hash { + return Ok(()); + } + // The hash stays: it is what identifies the configuration, and what this resolution rests on. + experiment.subject.kind = EvalSubjectKind::Agent; + experiment.subject.version = Some(version); + // Both writes in one transaction: a failure between them would leave the experiment promoted + // to a version while its cells stayed a draft's, a split no later read repairs since the + // experiment is no longer a draft. + let mut tx = db.begin().await?; + sqlx::query!( + "UPDATE eval_experiment + SET subject = jsonb_set( + jsonb_set(subject, '{kind}', '\"agent\"'), + '{version}', to_jsonb($4::bigint)) + WHERE workspace_id = $1 AND dataset_path = $2 AND id = $3 + AND subject ->> 'kind' = 'agent_draft'", + w_id, + dataset, + experiment.id, + version, + ) + .execute(&mut *tx) + .await?; + // The cells that ran that configuration are dated by the version too; leaving their hash would + // make the run go on reading as a draft's after the next deployment. + sqlx::query!( + "UPDATE eval_experiment_case + SET subject_version = $3, subject_draft_hash = NULL + WHERE experiment_id = $1 AND subject_draft_hash = $2", + experiment.id, + hash, + version, + ) + .execute(&mut *tx) + .await?; + tx.commit().await?; + Ok(()) +} + +/// Record what a run produced, from inside the run: the last step of a run's own flow calls this. +/// +/// Gated on reading the run rather than on writing its dataset, unlike everything else here: it is +/// the same harvest `experiment_results` performs behind the same check, over the run's own cells, +/// and it reports a count rather than any of what it read. +pub async fn collect_experiment( + authed: ApiAuthed, + Extension(db): Extension, + Extension(user_db): Extension, + Path(w_id): Path, + Query(query): Query, +) -> JsonResult { + // Through `user_db`, so the run is one the caller can see. The row carries the job to read it + // out of, so nothing that is read afterwards is caller-supplied. + let mut tx = user_db.begin(&authed).await?; + let experiment = sqlx::query!( + "SELECT id, run_job_id FROM eval_experiment WHERE workspace_id = $1 AND id = $2", + w_id, + query.id + ) + .fetch_optional(&mut *tx) + .await?; + tx.commit().await?; + let experiment = + experiment.ok_or_else(|| Error::NotFound(format!("Eval run {} not found", query.id)))?; + sync_run(&db, &w_id, experiment.id, experiment.run_job_id, true).await?; + let recorded = sqlx::query_scalar!( + "SELECT count(*) AS \"count!\" FROM eval_experiment_case + WHERE experiment_id = $1 AND status IS NOT NULL", + experiment.id + ) + .fetch_one(&db) + .await?; + Ok(Json(recorded as usize)) +} + +#[derive(Deserialize)] +pub struct ExperimentId { + pub id: Uuid, +} + +/// Collect a run for a reader, without letting the collection decide whether the read succeeds. +/// `collect_experiment` propagates instead: it is the run reporting on itself, and a failure there +/// is worth surfacing to the step that called it. +async fn collect_quietly(db: &DB, w_id: &str, experiment_id: Uuid, run_job_id: Uuid) { + if let Err(e) = sync_run(db, w_id, experiment_id, run_job_id, true).await { + tracing::warn!("could not collect eval run {}: {e:#}", experiment_id); + } +} + +/// The rows a results table is built from. The job ids come out of `eval_experiment_case`, which +/// only this module writes, so they can be read on the unrestricted pool once the dataset read +/// below has established the caller's access. +pub async fn experiment_results( + authed: ApiAuthed, + Extension(db): Extension, + Extension(user_db): Extension, + Path((w_id, dataset)): Path<(String, String)>, + Query(query): Query, +) -> JsonResult { + // The rows carry what the run's jobs produced, which `jobs:read` gates. `UserDB` settles who + // may see the dataset; a token's scopes are a separate question. + check_scopes(&authed, || "jobs:read".to_string())?; + let dataset_row = read_dataset(&authed, &user_db, &w_id, &dataset).await?; + let scorers = dataset_row.scorers; + + let mut experiment = read_experiment(&db, &w_id, &dataset, query.id).await?; + // Best-effort: collecting is what the run's own step is for, and a cell that could not be read + // — a job retained away between the iteration and its children — must not take the whole table + // down with it. The rows already recorded are still the run. + collect_quietly(&db, &w_id, query.id, experiment.run_job_id).await; + let scores = load_scores(&db, query.id).await?; + + let baseline = match query.baseline.filter(|id| *id != query.id) { + Some(id) => { + let baseline = read_experiment(&db, &w_id, &dataset, id).await?; + collect_quietly(&db, &w_id, id, baseline.run_job_id).await; + Some((baseline, load_scores(&db, id).await?)) + } + None => None, + }; + // The baseline is compared case by case, so its cells are keyed by the case they ran. + let baseline_ordinals = match &baseline { + Some((baseline, _)) => sqlx::query!( + "SELECT case_id, ordinal FROM eval_experiment_case WHERE experiment_id = $1", + baseline.id + ) + .fetch_all(&db) + .await? + .into_iter() + .map(|r| (r.case_id, r.ordinal)) + .collect::>(), + None => Default::default(), + }; + + let case_rows = sqlx::query!( + "SELECT ordinal, case_id, input, expected, job_id, subject_version, + subject_draft_hash, output, answered, status + FROM eval_experiment_case + WHERE experiment_id = $1 ORDER BY ordinal", + query.id + ) + .fetch_all(&db) + .await?; + + let mut sums = vec![(0.0f64, 0usize); scorers.len()]; + let mut baseline_sums = vec![(0.0f64, 0usize); scorers.len()]; + let mut passes = vec![0usize; scorers.len()]; + let mut baseline_passes = vec![0usize; scorers.len()]; + let mut missing_in_baseline = vec![0usize; scorers.len()]; + let mut definition_changed = vec![false; scorers.len()]; + let mut regressed = 0usize; + let mut rows = Vec::with_capacity(case_rows.len()); + + for case in case_rows { + let mut cells = Vec::with_capacity(scorers.len()); + for (index, scorer) in scorers.iter().enumerate() { + let current = scores.get(&(case.ordinal, scorer.id.clone())); + let baseline_score = baseline.as_ref().and_then(|(_, baseline_scores)| { + baseline_ordinals + .get(&case.case_id) + .and_then(|ordinal| baseline_scores.get(&(*ordinal, scorer.id.clone()))) + }); + if let Some(score) = current.and_then(|c| c.score) { + sums[index].0 += score; + sums[index].1 += 1; + if scorer.passed(Some(score)) == Some(true) { + passes[index] += 1; + } + } + if let Some(score) = baseline_score.and_then(|b| b.score) { + baseline_sums[index].0 += score; + baseline_sums[index].1 += 1; + if scorer.passed(Some(score)) == Some(true) { + baseline_passes[index] += 1; + } + } else if baseline.is_some() { + missing_in_baseline[index] += 1; + } + let changed = match (current, baseline_score) { + (Some(current), Some(baseline)) => current.definition != baseline.definition, + _ => false, + }; + if changed { + definition_changed[index] = true; + } + if let (Some(score), Some(previous)) = ( + current.and_then(|c| c.score), + baseline_score.and_then(|b| b.score), + ) { + if score < previous { + regressed += 1; + } + } + cells.push(CellScore { + scorer_id: scorer.id.clone(), + score: current.and_then(|c| c.score), + reason: current.and_then(|c| c.reason.clone()), + checks: current + .and_then(|c| c.checks.clone()) + .map(|c| serde_json::value::to_raw_value(&c)) + .transpose()?, + error: current.and_then(|c| c.error.clone()), + not_applicable: current.map(|c| c.not_applicable).unwrap_or(false), + // A row exists because the run was launched with this scorer, so an empty one is a + // score still to come, unless the scorer has already said this case is not one it + // measures. + pending: current + .map(|c| c.score.is_none() && c.error.is_none() && !c.not_applicable) + .unwrap_or(false), + passed: scorer.passed(current.and_then(|c| c.score)), + baseline: baseline_score.and_then(|b| b.score), + definition_changed: changed, + }); + } + rows.push(ExperimentRow { + case_id: case.case_id, + input: serde_json::from_value(case.input)?, + expected: opt_to_raw(case.expected)?, + // The iteration's verdict once it has one. While it is still running, the agent step's: + // the answer is written before the scorers read it, and a spinner beside an answer + // already there reads as an answer still being written. + status: case + .status + .or_else(|| { + case.answered + .map(|ok| if ok { "success" } else { "failure" }.to_string()) + }) + .unwrap_or_else(|| "running".to_string()), + output: case.output, + subject_version: case.subject_version, + subject_draft_hash: case.subject_draft_hash, + job_id: case.job_id, + scores: cells, + }); + } + + let means = scorers + .iter() + .enumerate() + .map(|(index, scorer)| ScorerMean { + scorer_id: scorer.id.clone(), + mean: (sums[index].1 > 0).then(|| sums[index].0 / sums[index].1 as f64), + baseline_mean: (baseline_sums[index].1 > 0) + .then(|| baseline_sums[index].0 / baseline_sums[index].1 as f64), + pass_rate: (scorer.pass_if.is_some() && sums[index].1 > 0) + .then(|| passes[index] as f64 / sums[index].1 as f64), + baseline_pass_rate: (scorer.pass_if.is_some() && baseline_sums[index].1 > 0) + .then(|| baseline_passes[index] as f64 / baseline_sums[index].1 as f64), + scored: sums[index].1, + missing_in_baseline: missing_in_baseline[index], + definition_changed: definition_changed[index], + }) + .collect(); + + // Read as the caller, so a viewer who can see the dataset but not the agent gets neither: the + // agent's version and configuration fingerprint must not leak past its own read permission. + let (subject_deployed_hash, subject_current_version) = + match readable_agent_state(&authed, &user_db, &w_id, &experiment.subject.path).await? { + Some((config, version)) => (Some(draft_hash(&config)), Some(version)), + None => (None, None), + }; + + // A run of unsaved edits whose configuration has since been deployed is a run of that version. + let mut baseline = baseline.map(|(baseline, _)| baseline); + resolve_deployed_draft( + &db, + &w_id, + &dataset, + &mut experiment, + subject_deployed_hash.as_deref(), + subject_current_version, + ) + .await?; + if let Some(baseline) = baseline.as_mut() { + // The compare-to list holds this agent's runs, but the id is the caller's: a run of another + // agent must not be stamped with this one's version. + if baseline.subject.path == experiment.subject.path { + resolve_deployed_draft( + &db, + &w_id, + &dataset, + baseline, + subject_deployed_hash.as_deref(), + subject_current_version, + ) + .await?; + } + } + + Ok(Json(ExperimentResults { + experiment, + baseline, + scorers, + rows, + means, + regressed, + subject_current_version, + subject_deployed_hash, + })) +} diff --git a/backend/windmill-api/src/ai_evals/run.rs b/backend/windmill-api/src/ai_evals/run.rs new file mode 100644 index 0000000000..e102abaf4c --- /dev/null +++ b/backend/windmill-api/src/ai_evals/run.rs @@ -0,0 +1,950 @@ +use super::*; + +/// Node id of the agent step. The answer is read back by this id, so it is part of the stored +/// shape rather than an implementation detail. +pub const AGENT_NODE_ID: &str = "a"; + +/// Node id of the step that assembles what the scorers are handed. +const PAYLOAD_NODE_ID: &str = "p"; +/// Node id of the loop over the dataset's cases. +const CASES_NODE_ID: &str = "cases"; +/// The branch holding every scorer of a case, so they measure it at the same time. +const SCORERS_NODE_ID: &str = "scores"; + +/// In-flight iterations. A dataset is a burst of calls to one provider, so answering every case at +/// once is a run that spends its time being rate-limited. +const RUN_PARALLELISM: u16 = 8; + +/// What each iteration is handed: the case, small enough to sit in every iteration's arguments. +#[derive(Serialize)] +struct CaseIteration { + case_id: Uuid, + ordinal: i32, + input: EvalCaseInput, + #[serde(skip_serializing_if = "Option::is_none")] + expected: Option>, +} + +/// Assembles the payload the scorers read. +/// +/// A step rather than an input transform: every tool call is enriched with the arguments, result, +/// status and duration of the job that ran it, none of which the flow can see. +const PAYLOAD_SCRIPT: &str = r#"//native +// Generated by Windmill: reads the run this iteration answered. +export async function main() { + const id = process.env.WM_FLOW_JOB_ID + const base = process.env.BASE_URL || process.env.BASE_INTERNAL_URL + const res = await fetch( + `${base}/api/w/${process.env.WM_WORKSPACE}/ai_evals/run_payload?job_id=${id}`, + { headers: { Authorization: `Bearer ${process.env.WM_TOKEN}` } } + ) + if (!res.ok) { + throw new Error(`could not read the run of job ${id}: ${res.status} ${await res.text()}`) + } + return await res.json() +} +"#; + +fn payload_module() -> serde_json::Value { + serde_json::json!({ + "id": PAYLOAD_NODE_ID, + "summary": "Assemble the run the scorers read", + "value": { + "type": "rawscript", + // `bunnative` (tag `nativets`), matching the `//native` the script carries. That tag + // belongs to the `native` worker group rather than the default one, so a queued + // iteration never starts when nothing serves it. + "language": "bunnative", + "content": PAYLOAD_SCRIPT, + "lock": EMPTY_BUN_LOCK, + "input_transforms": {} + } + }) +} + +/// Node id of the step that records what the run produced. +const COLLECT_NODE_ID: &str = "collect"; + +/// Copies the run's answers and scores into its own rows, from inside the run. +/// +/// The tables know nothing about the flow, so without this a run started and left is only ever +/// recorded by someone looking at it — after its jobs have been retained away, there is nothing +/// left to record. +const COLLECT_SCRIPT: &str = r#"//native +// Generated by Windmill: records what this run produced, so it outlives the jobs that produced it. +export async function main(experiment_id: string) { + const base = process.env.BASE_URL || process.env.BASE_INTERNAL_URL + const res = await fetch( + `${base}/api/w/${process.env.WM_WORKSPACE}/ai_evals/experiments/collect?id=${experiment_id}`, + { method: 'POST', headers: { Authorization: `Bearer ${process.env.WM_TOKEN}` } } + ) + if (!res.ok) { + throw new Error(`could not record run ${experiment_id}: ${res.status} ${await res.text()}`) + } + return await res.json() +} +"#; + +fn collect_module(experiment_id: Uuid) -> serde_json::Value { + serde_json::json!({ + "id": COLLECT_NODE_ID, + "summary": "Record what the run produced", + // Bookkeeping, so it does not decide whether the run succeeded. What it would have written + // is written again by the first read of the run. + "continue_on_error": true, + "value": { + "type": "rawscript", + "language": "bunnative", + "content": COLLECT_SCRIPT, + "lock": EMPTY_BUN_LOCK, + "input_transforms": { + "experiment_id": { + "type": "static", + "value": experiment_id.to_string(), + }, + } + } + }) +} + +/// The script imports nothing, so its lockfile is the empty one, spelled the way the bun executor +/// splits it. Without a lock a worker running this as bun would resolve dependencies every time. +const EMPTY_BUN_LOCK: &str = "{\n \"dependencies\": {}\n}\n//bun.lock\n"; + +/// What a judge is asked about the case: the run as it reads it, and the case's own attachments, +/// handed to it as they were handed to the agent. +fn judge_case_transforms() -> serde_json::Map { + let mut transforms = serde_json::Map::new(); + transforms.insert( + "user_message".to_string(), + serde_json::json!({ + "type": "javascript", + "expr": format!("results.{}.rendered", PAYLOAD_NODE_ID), + }), + ); + transforms.insert( + "user_attachments".to_string(), + serde_json::json!({ + "type": "javascript", + "expr": format!("results.{}.run.input.user_attachments", PAYLOAD_NODE_ID), + }), + ); + transforms +} + +/// The scorer steps of one iteration, reading the payload the step before them assembled. +/// +/// Each scorer is baked in as `resolve_scorer` resolved it at launch, never linked by path: a +/// linked step resolves the resource when the case reaches it, so a scorer edited mid-run would +/// grade the later cases while every score still names the definition recorded at launch. +fn scorer_modules(scorers: &[(&Scorer, ResolvedScorer)]) -> Vec { + scorers + .iter() + .map(|(scorer, resolved)| { + let value = match resolved { + // A judge is an agent handed the run as its message; its own system prompt is the + // grading contract, which is why editing a judge means editing that agent. + ResolvedScorer::Judge { config } => { + let mut transforms = match &config.input_transforms { + serde_json::Value::Object(map) => map.clone(), + _ => serde_json::Map::new(), + }; + transforms.extend(judge_case_transforms()); + serde_json::json!({ + "type": "aiagent", + "tools": config.tools, + "input_transforms": serde_json::Value::Object(transforms), + }) + } + // `run` is the whole payload; `input`, `output` and `expected` are the same values + // spelled out, so a three-line scorer does not have to reach into it. + ResolvedScorer::Script { hash } => serde_json::json!({ + "type": "script", + "path": scorer.def.path(), + // Serialized as `ScriptHash` (a hex string), which is the only shape a flow + // module's `hash` field deserializes from — a bare number fails in the worker. + "hash": windmill_common::scripts::ScriptHash(*hash), + "input_transforms": { + "run": { + "type": "javascript", + "expr": format!("results.{}.run", PAYLOAD_NODE_ID), + }, + "input": { + "type": "javascript", + "expr": format!("results.{}.run.input", PAYLOAD_NODE_ID), + }, + "output": { + "type": "javascript", + "expr": format!("results.{}.run.output", PAYLOAD_NODE_ID), + }, + "expected": { + "type": "javascript", + "expr": format!("results.{}.run.expected", PAYLOAD_NODE_ID), + }, + } + }), + }; + serde_json::json!({ "id": scorer_module_id(&scorer.id), "value": value }) + }) + .collect() +} + +/// The flow a whole run is: one loop over the dataset's cases, each iteration answering the case +/// and then scoring the answer. +/// +/// One job rather than one per case: a run outlives the tab that started it, and only a worker can +/// notice that the last case finished. The cases live in the flow's value, stored once, rather +/// than in its arguments, which every iteration inherits a copy of. +fn build_run_flow( + config: &AgentDraft, + cases: &[CaseIteration], + scorers: &[(&Scorer, ResolvedScorer)], + experiment_id: Uuid, +) -> Result { + let mut modules: Vec = vec![agent_module(config)?]; + if !scorers.is_empty() { + modules.push(payload_module()); + // One branch each, run together: scorers read the answer and never each other. Each branch + // keeps its own failure, so a judge that errors costs its own column and no other. + modules.push(serde_json::json!({ + "id": SCORERS_NODE_ID, + "value": { + "type": "branchall", + "parallel": true, + "branches": scorers + .iter() + .zip(scorer_modules(scorers)) + .map(|((scorer, _), module)| serde_json::json!({ + // Named for the column it produces: the graph of a run is read to see which + // scorer did what, and a module id is not what a scorer is called. + "summary": scorer_name(scorer), + "skip_failure": true, + "modules": [module], + })) + .collect::>(), + } + })); + } + + Ok(serde_json::from_value(serde_json::json!({ + "modules": [ + { + "id": CASES_NODE_ID, + "value": { + "type": "forloopflow", + "iterator": { "type": "static", "value": cases }, + "parallel": true, + "parallelism": RUN_PARALLELISM, + // One case failing is one cell of the run, not the end of it. + "skip_failures": true, + "modules": modules, + } + }, + // After the loop, so every case has both answered and been scored by the time it runs. + collect_module(experiment_id), + ] + }))?) +} + +/// The agent step, reading its case from the iteration rather than from the flow's arguments. +fn agent_module(config: &AgentDraft) -> Result { + let flow = build_case_flow(config)?; + let mut value = serde_json::to_value(&flow.modules[0].value)?; + if let Some(map) = value.as_object_mut() { + let transforms = map + .entry("input_transforms") + .or_insert_with(|| serde_json::json!({})); + if let Some(transforms) = transforms.as_object_mut() { + for key in ["user_message", "user_attachments"] { + transforms.insert( + key.to_string(), + serde_json::json!({ + "type": "javascript", + "expr": format!("flow_input.iter.value.input.{}", key), + }), + ); + } + } + } + Ok(serde_json::json!({ "id": AGENT_NODE_ID, "value": value })) +} + +/// The agent step as a one-module flow, so the module shape is validated by deserializing +/// through `FlowValue` rather than trusted as raw JSON. +fn build_case_flow(config: &AgentDraft) -> Result { + // The configuration runs exactly as authored: its own brain transforms are the module's, and + // the case supplies the message and the attachments over the top. + let mut input_transforms = match &config.input_transforms { + serde_json::Value::Object(map) => map.clone(), + _ => serde_json::Map::new(), + }; + for key in ["user_message", "user_attachments"] { + input_transforms.insert( + key.to_string(), + serde_json::json!({ "type": "javascript", "expr": format!("flow_input.{}", key) }), + ); + } + + // Always inlined, never a link to the resource: a linked step would resolve the agent when + // each case runs, which is the one thing a run of a named version must not do. + let mut agent_value = serde_json::Map::new(); + agent_value.insert("type".to_string(), serde_json::json!("aiagent")); + agent_value.insert("tools".to_string(), serde_json::json!(config.tools)); + agent_value.insert( + "input_transforms".to_string(), + serde_json::Value::Object(input_transforms), + ); + Ok(serde_json::from_value(serde_json::json!({ + "modules": [{ "id": AGENT_NODE_ID, "value": serde_json::Value::Object(agent_value) }] + }))?) +} + +/// How many times the agent has been saved, not the identity of the row holding that value: runs +/// are named by it and compared by it, so it has to be the resource's own count rather than a +/// sequence the whole instance shares. +pub(crate) async fn current_resource_version( + db: &DB, + w_id: &str, + path: &str, +) -> Result> { + let version = sqlx::query_scalar!( + "SELECT version FROM resource_version WHERE workspace_id = $1 AND path = $2 + ORDER BY version DESC LIMIT 1", + w_id, + path + ) + .fetch_optional(db) + .await?; + Ok(version) +} + +/// Read the agent through `user_db` so a caller who cannot read the resource cannot run it. +pub(crate) async fn require_agent( + authed: &ApiAuthed, + user_db: &UserDB, + w_id: &str, + agent_path: &str, +) -> Result<()> { + let mut tx = user_db.clone().begin(authed).await?; + let resource_type = sqlx::query_scalar!( + "SELECT resource_type FROM resource WHERE workspace_id = $1 AND path = $2", + w_id, + agent_path + ) + .fetch_optional(&mut *tx) + .await?; + tx.commit().await?; + match resource_type.as_deref() { + Some("ai_agent") => Ok(()), + Some(other) => Err(Error::BadRequest(format!( + "Resource {} is a {}, not an ai_agent", + agent_path, other + ))), + None => Err(Error::NotFound(format!("Agent {} not found", agent_path))), + } +} + +/// An `ai_agent` value as the configuration to run it with: its brain becomes the module's input +/// transforms, its tools the module's tools. The same conversion for a draft and for what is +/// deployed, so the two hash comparably — which is what lets a draft run be recognised as the +/// version it became. +fn config_to_draft(value: serde_json::Value) -> Result { + let mut config = match value { + serde_json::Value::Object(map) => map, + _ => return Err(Error::BadRequest("The agent is not an object".to_string())), + }; + let tools = match config.remove("tools") { + Some(serde_json::Value::Array(tools)) => tools, + _ => vec![], + }; + // Every brain key becomes a static transform: `$res:`/`$var:` in them are resolved by the + // same argument machinery a linked step's resource goes through. + let input_transforms = config + .into_iter() + .map(|(key, value)| (key, serde_json::json!({ "type": "static", "value": value }))) + .collect::>(); + Ok(AgentDraft { input_transforms: serde_json::Value::Object(input_transforms), tools }) +} + +/// An agent's deployed value and the version that names it, in the shape a step runs. `None` when +/// the caller cannot see the resource, or it is not a usable agent. +/// +/// Both from one read: a deploy landing between two reads would pair one version's configuration +/// with another's number, and what a run records of its subject is permanent. +pub(crate) async fn readable_agent_state( + authed: &ApiAuthed, + user_db: &UserDB, + w_id: &str, + path: &str, +) -> Result> { + let mut tx = user_db.clone().begin(authed).await?; + let row = sqlx::query!( + "SELECT r.value AS \"value: sqlx::types::Json\", + (SELECT version FROM resource_version v + WHERE v.workspace_id = r.workspace_id AND v.path = r.path + ORDER BY v.version DESC LIMIT 1) AS version + FROM resource r + WHERE r.workspace_id = $1 AND r.path = $2 AND r.resource_type = 'ai_agent'", + w_id, + path + ) + .fetch_optional(&mut *tx) + .await?; + tx.commit().await?; + let Some(row) = row else { + return Ok(None); + }; + let (Some(value), Some(version)) = (row.value, row.version) else { + return Ok(None); + }; + // A resource's value isn't validated against its type on write, so an ai_agent whose value + // isn't a valid config is no usable state rather than an error: one bad row would otherwise + // 400 the whole results page or run list. + match config_to_draft(value.0) { + Ok(config) => Ok(Some((config, version))), + Err(_) => Ok(None), + } +} + +/// Fill in what the client cannot: the version a saved agent is at, or the configuration a past +/// version held. +/// +/// Returns the configuration the run executes, read once here. Every case then executes that one +/// configuration: resolved per case instead, an agent deployed mid-run would be executed by the +/// cases after it while every row still names the version the run started against. +async fn resolve_subject( + authed: &ApiAuthed, + db: &DB, + user_db: &UserDB, + w_id: &str, + subject: &mut EvalSubject, + draft: Option, +) -> Result { + Ok(match subject.kind { + EvalSubjectKind::Agent => { + let Some((config, version)) = + readable_agent_state(authed, user_db, w_id, &subject.path).await? + else { + return Err(Error::BadRequest(format!( + "Agent {} is not a readable ai_agent resource", + subject.path + ))); + }; + subject.version = Some(version); + config + } + EvalSubjectKind::AgentDraft => { + // The edits live nowhere the server can read them, so the request carries them. The + // agent is still read, so a run can only be filed under one the caller can see. + require_agent(authed, user_db, w_id, &subject.path).await?; + // The version the edits are an edit of, as of now: edits record no version of their + // own, so "v15 plus unsaved edits" means the edits and whatever was deployed when the + // run started. + subject.version = current_resource_version(db, w_id, &subject.path).await?; + draft.ok_or_else(|| Error::BadRequest(DRAFT_REQUIRED.to_string()))? + } + EvalSubjectKind::AgentVersion => { + let Some(version) = subject.version else { + return Err(Error::BadRequest( + "A run of a past version must say which version".to_string(), + )); + }; + let config = + agent_version_config(authed, user_db, db, w_id, &subject.path, version).await?; + subject.draft = Some(config.clone()); + config + } + }) +} + +/// One version of an agent out of its history, in the shape a step runs. +/// +/// Read through `user_db` for the agent itself first: a version is the resource as it was, so +/// seeing one is seeing the resource. +async fn agent_version_config( + authed: &ApiAuthed, + user_db: &UserDB, + db: &DB, + w_id: &str, + agent_path: &str, + version: i64, +) -> Result { + require_agent(authed, user_db, w_id, agent_path).await?; + let value = sqlx::query_scalar!( + "SELECT value FROM resource_version + WHERE version = $1 AND workspace_id = $2 AND path = $3", + version, + w_id, + agent_path + ) + .fetch_optional(db) + .await? + .flatten() + .ok_or_else(|| Error::NotFound(format!("Agent {} has no version {}", agent_path, version)))?; + config_to_draft(value).map_err(|_| { + Error::BadRequest(format!( + "Version {} of {} is not an object", + version, agent_path + )) + }) +} + +const DRAFT_REQUIRED: &str = "A run of unsaved edits must carry the configuration being edited"; + +/// The configuration the request may carry, taken out of the subject it belongs to. +/// +/// A saved agent and a past version are read from the workspace by the path they name, so a +/// request carrying a configuration for them would run something other than the agent it claims to +/// be a run of. Unsaved edits are the one kind the request has to carry: they exist only in the +/// editor. +fn validate_subject(subject: &EvalSubject) -> Result> { + if subject.path.trim().is_empty() { + return Err(Error::BadRequest( + "The subject needs a path: it is the agent a run is filed under".to_string(), + )); + } + match (&subject.draft, &subject.kind) { + (Some(draft), EvalSubjectKind::AgentDraft) => Ok(Some(draft.clone())), + (Some(_), _) => Err(Error::BadRequest( + "A saved agent's configuration is read from the workspace; remove it from the request" + .to_string(), + )), + (None, EvalSubjectKind::AgentDraft) => Err(Error::BadRequest(DRAFT_REQUIRED.to_string())), + (None, _) => Ok(None), + } +} + +// ----------------------------------------------------------------------------------------------- +// Experiments +// ----------------------------------------------------------------------------------------------- + +#[derive(Deserialize)] +pub struct RunExperiment { + pub dataset: String, + pub subject: EvalSubject, +} + +/// Open a run of this dataset. +/// +/// Runs are numbered per (dataset, agent) pair, and the deployed agent and its draft share that +/// numbering: they are the same agent, so "Run 7" of a dataset means one thing whether it ran the +/// deployed value or the edits waiting on top of it. +async fn new_run( + tx: &mut sqlx::Transaction<'_, sqlx::Postgres>, + w_id: &str, + dataset: &str, + subject: &EvalSubject, + username: &str, + run_job_id: Uuid, + id: Uuid, +) -> Result { + // Two runs starting together would otherwise read the same run number. Held for the rest of + // this transaction, which pushes no jobs. + sqlx::query!( + "SELECT pg_advisory_xact_lock(hashtext('ai_eval_open:' || $1 || '/' || $2 || '/' || $3))", + w_id, + dataset, + subject.path, + ) + .execute(&mut **tx) + .await?; + let run_number = sqlx::query_scalar!( + "SELECT coalesce(max(run_number), 0) + 1 FROM eval_experiment + WHERE workspace_id = $1 AND dataset_path = $2 AND subject ->> 'path' = $3", + w_id, + dataset, + subject.path, + ) + .fetch_one(&mut **tx) + .await? + .unwrap_or(1); + sqlx::query!( + "INSERT INTO eval_experiment + (id, workspace_id, dataset_path, subject, run_number, created_by, run_job_id) + VALUES ($1, $2, $3, $4, $5, $6, $7)", + id, + w_id, + dataset, + serde_json::to_value(subject.stamp())?, + run_number, + username, + run_job_id, + ) + .execute(&mut **tx) + .await + .map_err(|e| { + if is_missing_dataset(&e) { + Error::NotFound(format!("Eval dataset {} not found", dataset)) + } else { + e.into() + } + })?; + Ok(id) +} + +pub async fn run_experiment( + authed: ApiAuthed, + Extension(db): Extension, + Extension(user_db): Extension, + Path(w_id): Path, + Json(payload): Json, +) -> Result { + check_scopes(&authed, || "jobs:run".to_string())?; + // A write, not a read: it persists an experiment into the dataset. + require_dataset_writable(&authed, &user_db, &w_id, &payload.dataset).await?; + + let mut subject = payload.subject; + let draft = validate_subject(&subject)?; + let config = resolve_subject(&authed, &db, &user_db, &w_id, &mut subject, draft).await?; + + // One snapshot of the dataset: the scorers and the cases a run records must be the same + // revision, or a Save landing between two reads files a run under cases and columns that never + // stood together. + let (dataset, cases) = + read_dataset_and_cases(&authed, &user_db, &w_id, &payload.dataset).await?; + if cases.is_empty() { + return Err(Error::BadRequest(format!( + "Eval dataset {} has no case to run", + payload.dataset + ))); + } + + let case_count = cases.len(); + // Resolved through the caller's own db, so a run executes only runnables the caller may read, + // and what resolving pinned is baked into the flow. Recorded per cell at launch rather than + // when the score comes back, so a scorer edited mid-run reads as the change of scorer it is. + let mut definitions = Vec::with_capacity(dataset.scorers.len()); + let mut scorers: Vec<(&Scorer, ResolvedScorer)> = Vec::with_capacity(dataset.scorers.len()); + for scorer in &dataset.scorers { + let (definition, resolved) = resolve_scorer(&user_db, &authed, &w_id, scorer).await?; + definitions.push(definition); + scorers.push((scorer, resolved)); + } + + let iterations = cases + .iter() + .enumerate() + .map(|(index, case)| CaseIteration { + case_id: case.id, + ordinal: index as i32, + input: case.input.clone(), + expected: case.expected.clone(), + }) + .collect::>(); + // Both ids are chosen here: the run's own collect step is handed the experiment id, and the + // experiment names its job before that job exists. + let experiment_id = Uuid::new_v4(); + let run_job_id = Uuid::new_v4(); + let flow_value = build_run_flow(&config, &iterations, &scorers, experiment_id)?; + + // Recorded before the job is queued, so a launch that dies partway leaves an experiment naming + // a job that never started rather than a flow no experiment accounts for and nothing collects. + let mut tx = db.begin().await?; + let experiment_id = new_run( + &mut tx, + &w_id, + &payload.dataset, + &subject, + &authed.username, + run_job_id, + experiment_id, + ) + .await?; + + let ordinals = (0..case_count as i32).collect::>(); + let case_ids = cases.iter().map(|c| c.id).collect::>(); + let inputs = cases + .iter() + .map(|c| serde_json::to_value(&c.input)) + .collect::, _>>()?; + let expecteds = cases + .iter() + .map(|c| opt_from_raw(c.expected.as_ref())) + .collect::>>()?; + let versions = vec![subject.version; case_count]; + let hashes = vec![subject.draft.as_ref().map(draft_hash); case_count]; + // No job id: the iteration that answers a case is minted by the flow engine, and the case is + // matched back to it once it exists. + sqlx::query!( + "INSERT INTO eval_experiment_case + (experiment_id, ordinal, case_id, input, expected, subject_version, + subject_draft_hash) + SELECT $1, ordinal, case_id, input, expected, subject_version, subject_draft_hash + FROM UNNEST($2::int[], $3::uuid[], $4::jsonb[], $5::jsonb[], $6::bigint[], $7::text[]) + AS t(ordinal, case_id, input, expected, subject_version, subject_draft_hash)", + experiment_id, + &ordinals, + &case_ids, + &inputs, + &expecteds as &[Option], + &versions as &[Option], + &hashes as &[Option], + ) + .execute(&mut *tx) + .await?; + insert_pending_scores(&mut tx, experiment_id, &ordinals, &scorers, &definitions).await?; + // The foreign key makes a delete racing this assembly fail the commit, so nothing is queued. + // A delete landing between this commit and the push below still cascades the experiment away + // while the flow queues; that launch/delete race is a known beta limitation. + tx.commit().await?; + + if let Err(e) = push_run_flow( + &authed, + &db, + &user_db, + &w_id, + &payload.dataset, + &subject, + experiment_id, + run_job_id, + flow_value, + ) + .await + { + // Nothing ran, so there is nothing to keep: one failed push is the whole run. + sqlx::query!("DELETE FROM eval_experiment WHERE id = $1", experiment_id) + .execute(&db) + .await?; + return Err(e); + } + Ok(experiment_id.to_string()) +} + +/// The cells a run will fill in, written at launch. A pending row is what the table reads as a +/// score still being produced, and it is where the definition that produced it is recorded. +async fn insert_pending_scores( + tx: &mut sqlx::Transaction<'_, sqlx::Postgres>, + experiment_id: Uuid, + ordinals: &[i32], + scorers: &[(&Scorer, ResolvedScorer)], + definitions: &[String], +) -> Result<()> { + if scorers.is_empty() || ordinals.is_empty() { + return Ok(()); + } + let mut rows_ordinal = vec![]; + let mut rows_scorer = vec![]; + let mut rows_definition = vec![]; + for ordinal in ordinals { + for ((scorer, _), definition) in scorers.iter().zip(definitions.iter()) { + rows_ordinal.push(*ordinal); + rows_scorer.push(scorer.id.clone()); + rows_definition.push(definition.clone()); + } + } + sqlx::query!( + "INSERT INTO eval_score (experiment_id, ordinal, scorer_id, definition) + SELECT $1, ordinal, scorer_id, definition + FROM UNNEST($2::int[], $3::text[], $4::text[]) AS t(ordinal, scorer_id, definition) + ON CONFLICT (experiment_id, ordinal, scorer_id) + DO UPDATE SET definition = EXCLUDED.definition, score = NULL, reason = NULL, + checks = NULL, error = NULL, not_applicable = false", + experiment_id, + &rows_ordinal, + &rows_scorer, + &rows_definition, + ) + .execute(&mut **tx) + .await?; + Ok(()) +} + +/// Queue the flow a run is. Its id is chosen by the caller, so the experiment can name it before +/// it exists. +async fn push_run_flow( + authed: &ApiAuthed, + db: &DB, + user_db: &UserDB, + w_id: &str, + dataset: &str, + subject: &EvalSubject, + experiment_id: Uuid, + run_job_id: Uuid, + flow_value: windmill_common::flows::FlowValue, +) -> Result { + use windmill_common::{jobs::JobPayload, users::username_to_permissioned_as}; + use windmill_queue::{push, PushArgs, PushIsolationLevel}; + + let mut args = std::collections::HashMap::new(); + // So the job says what it was evaluating when opened cold from the runs page. Every iteration + // inherits these, so they are the stamp and nothing bulkier. + args.insert( + "_eval".to_string(), + serde_json::value::to_raw_value(&serde_json::json!({ + "subject": subject.stamp(), + "dataset": dataset, + "experiment_id": experiment_id, + }))?, + ); + + let path = subject.path.clone(); + let tx = PushIsolationLevel::Isolated(user_db.clone(), authed.clone().into()); + let (uuid, tx) = push( + db, + tx, + w_id, + JobPayload::RawFlow { value: flow_value, path: Some(path), restarted_from: None }, + PushArgs::from(&args), + authed.display_username(), + &authed.email, + username_to_permissioned_as(&authed.username), + authed.token_prefix.as_deref(), + authed.username_override.as_deref(), + None, + None, + None, + None, + None, + Some(run_job_id), + false, + false, + None, + true, + None, + None, + None, + None, + Some(&authed.clone().into()), + false, + None, + authed.trigger_or_fallback(None), + None, + ) + .await?; + tx.commit().await?; + Ok(uuid) +} + +pub(crate) fn experiment_from_row( + id: Uuid, + dataset: String, + subject: serde_json::Value, + run_number: i32, + run_job_id: Uuid, + case_count: i64, + created_at: DateTime, + created_by: String, +) -> Result { + Ok(EvalExperiment { + id, + dataset, + subject: serde_json::from_value(subject)?, + run_number, + run_job_id, + case_count, + // Filled in by the list, which reads every listed run's scores in one query. + scores: vec![], + running: false, + created_at, + created_by, + }) +} + +#[cfg(test)] +mod tests { + use super::*; + + fn agent_config() -> AgentDraft { + AgentDraft { input_transforms: serde_json::json!({}), tools: vec![] } + } + + fn scorer(kind: ScorerDef) -> Scorer { + Scorer { id: "s1".to_string(), name: None, pass_if: None, def: kind } + } + + fn subject(kind: EvalSubjectKind, draft: Option) -> EvalSubject { + EvalSubject { kind, path: "u/me/agent".to_string(), version: None, draft, draft_hash: None } + } + + /// The whole argument for accepting a configuration from the request is that it is accepted + /// for exactly one kind: the edits in progress, which exist nowhere the server can read. A + /// saved agent or a past version carrying one would run something other than what it names. + #[test] + fn a_configuration_is_required_for_edits_and_refused_for_anything_saved() { + assert!( + validate_subject(&subject(EvalSubjectKind::AgentDraft, Some(agent_config()))).is_ok() + ); + assert!(validate_subject(&subject(EvalSubjectKind::Agent, None)).is_ok()); + assert!(validate_subject(&subject(EvalSubjectKind::AgentDraft, None)).is_err()); + assert!(validate_subject(&subject(EvalSubjectKind::Agent, Some(agent_config()))).is_err()); + assert!(validate_subject(&subject( + EvalSubjectKind::AgentVersion, + Some(agent_config()) + )) + .is_err()); + } + + /// Where the collect step sits is load-bearing twice over: inside the loop it would run once + /// per case, and `backfill_case_jobs` matches a case to any child of the run carrying an + /// `iter` argument, which the collect job must therefore never be. + #[test] + fn the_collect_step_runs_once_after_the_loop() { + let experiment = Uuid::new_v4(); + let flow = build_run_flow(&agent_config(), &[], &[], experiment).unwrap(); + let value = serde_json::to_value(&flow).unwrap(); + let modules = value["modules"].as_array().unwrap(); + assert_eq!( + modules + .iter() + .map(|m| m["id"].as_str().unwrap()) + .collect::>(), + vec![CASES_NODE_ID, COLLECT_NODE_ID] + ); + let collect = &modules[1]; + // The run it records is baked in rather than read from the iteration around it, which is + // what makes it a step of the run and not of a case. + assert_eq!( + collect["value"]["input_transforms"]["experiment_id"]["value"] + .as_str() + .unwrap(), + experiment.to_string() + ); + assert!(collect["value"]["input_transforms"]["iter"].is_null()); + // A failed record must not fail a run whose cases all answered. + assert_eq!(collect["continue_on_error"].as_bool(), Some(true)); + } + + /// A code scorer pins the deployed hash it resolved to, and that hash must reach the flow as a + /// `ScriptHash` (a hex string), not the bare number it is in the database. + #[test] + fn a_script_scorer_pins_its_resolved_hash_as_a_hex_string() { + let s = scorer(ScorerDef::Script { path: "f/e/scorer".to_string() }); + let scorers = vec![(&s, ResolvedScorer::Script { hash: 8816320759749465854i64 })]; + let modules = scorer_modules(&scorers); + // A flow module's `hash` deserializes only from a `ScriptHash` (a hex string); emitted as a + // bare number it fails in the worker and every code-scorer column breaks at runtime. + assert!( + modules[0]["value"]["hash"].is_string(), + "the pinned scorer hash must serialize as a hex string, not a number" + ); + } + + /// A judge is pinned by inlining the configuration resolved at launch rather than linked by + /// path, which is the difference between a run that grades against one definition and one that + /// resolves the judge per case. + #[test] + fn a_judge_is_inlined_rather_than_linked() { + let judge = scorer(ScorerDef::Agent { path: "f/e/judge".to_string() }); + let scorers = vec![( + &judge, + ResolvedScorer::Judge { + config: AgentDraft { + input_transforms: serde_json::json!({ + "system_prompt": { "type": "static", "value": "grade it" } + }), + tools: vec![], + }, + }, + )]; + let pinned = scorer_modules(&scorers); + let value = &pinned[0]["value"]; + assert!(value["agent"].is_null()); + assert_eq!( + value["input_transforms"]["system_prompt"]["value"].as_str(), + Some("grade it") + ); + // The case reaches the judge alongside the judge's own transforms. + assert!(value["input_transforms"]["user_message"]["expr"].is_string()); + assert!(value["input_transforms"]["user_attachments"]["expr"].is_string()); + } +} diff --git a/backend/windmill-api/src/ai_evals/scorers.rs b/backend/windmill-api/src/ai_evals/scorers.rs new file mode 100644 index 0000000000..6d6af3096e --- /dev/null +++ b/backend/windmill-api/src/ai_evals/scorers.rs @@ -0,0 +1,285 @@ +use super::*; + +/// A scorer is a column of the results table. +/// +/// `id` is assigned when the scorer is added to a dataset and never reused: it is what makes a +/// column the same column across experiments when the scorer is renamed or its definition is +/// edited, and a delta is only ever computed between two scores carrying the same id. +#[derive(Serialize, Deserialize, Debug, Clone)] +pub struct Scorer { + /// Assigned on write when a new scorer arrives without one, so a client cannot collide two + /// columns onto one id. + #[serde(default)] + pub id: String, + /// The column header. Defaults to the kind, or the last segment of the path. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub name: Option, + /// A score at or above this counts as a pass. Deliberately outside `definition`: where the + /// line sits interprets a score rather than produces it, so moving it re-reads every score + /// already recorded instead of invalidating them. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub pass_if: Option, + #[serde(flatten)] + pub def: ScorerDef, +} + +/// A judge is an `ai_agent` resource sent the run to grade; a script receives the run as an +/// argument. Both are runnables, so every column has a path, a version and code you can open. +#[derive(Serialize, Deserialize, Debug, Clone, PartialEq)] +#[serde(tag = "kind", rename_all = "snake_case")] +pub enum ScorerDef { + Script { path: String }, + Agent { path: String }, +} + +impl ScorerDef { + pub fn path(&self) -> &str { + match self { + ScorerDef::Script { path } | ScorerDef::Agent { path } => path, + } + } + + /// The wire name of the kind, as the client sends it. + pub(crate) fn kind_str(&self) -> &'static str { + match self { + ScorerDef::Script { .. } => "script", + ScorerDef::Agent { .. } => "agent", + } + } + + fn kind_label(&self) -> &'static str { + match self { + ScorerDef::Script { .. } => "Script", + ScorerDef::Agent { .. } => "Judge agent", + } + } +} + +impl Scorer { + /// Whether a score counts as a pass. `None` when the column has no threshold, which keeps a + /// column of plain numbers from being rendered as if it had one. + pub fn passed(&self, score: Option) -> Option { + match (self.pass_if, score) { + (Some(threshold), Some(score)) => Some(score >= threshold), + _ => None, + } + } + + /// What produced a score, recorded with it so a comparison can say the scorer changed instead + /// of letting that read as a difference between two agents. `resolved` is the script hash or + /// resource version that actually ran, which the path alone does not pin. + pub fn definition(&self, resolved: Option<&str>) -> String { + use sha2::{Digest, Sha256}; + let mut hasher = Sha256::new(); + hasher.update(self.def.kind_label().as_bytes()); + hasher.update(b":"); + hasher.update(self.def.path().as_bytes()); + if let Some(resolved) = resolved { + hasher.update(b"@"); + hasher.update(resolved.as_bytes()); + } + hex::encode(hasher.finalize())[..32].to_string() + } +} + +const MAX_SCORER_NAME_CHARS: usize = 120; + +/// Ids are assigned here rather than trusted from the client: an id is kept only when it names a +/// column the dataset already has, so a removed column cannot come back and inherit the scores +/// recorded against it. Anything else is minted as a valid flow module identifier, which the +/// scoring flows it is baked into require (see `scorer_module_id`). +pub(crate) fn assign_scorer_ids( + scorers: &mut Vec, + existing: &std::collections::HashSet, +) -> Result<()> { + if scorers.len() > MAX_SCORERS_PER_DATASET { + return Err(Error::BadRequest(format!( + "An eval dataset holds at most {} scorers", + MAX_SCORERS_PER_DATASET + ))); + } + let mut seen = std::collections::HashSet::new(); + for scorer in scorers.iter_mut() { + if !existing.contains(&scorer.id) || !seen.insert(scorer.id.clone()) { + scorer.id = Uuid::new_v4().simple().to_string(); + seen.insert(scorer.id.clone()); + } + if let Some(name) = &scorer.name { + if name.chars().count() > MAX_SCORER_NAME_CHARS { + return Err(Error::BadRequest(format!( + "Scorer name {} is too long, {} characters at most", + name, MAX_SCORER_NAME_CHARS + ))); + } + } + // A score is 0 to 1, so a threshold outside that range would pass everything or nothing + // regardless of what the scorer measured. + if let Some(pass_if) = scorer.pass_if { + if !(0.0..=1.0).contains(&pass_if) { + return Err(Error::BadRequest(format!( + "Scorer pass threshold {} must be between 0 and 1", + pass_if + ))); + } + } + check_proper_path(scorer.def.path())?; + } + Ok(()) +} + +/// What a column is called: the dataset's own name for it, or the last segment of what it points +/// at. The same fallback the column header uses. +pub(crate) fn scorer_name(scorer: &Scorer) -> String { + scorer + .name + .clone() + .filter(|n| !n.trim().is_empty()) + .unwrap_or_else(|| { + let path = scorer.def.path(); + path.rsplit('/').next().unwrap_or(path).to_string() + }) +} + +#[derive(Serialize)] +pub struct RecentScorer { + #[serde(flatten)] + pub scorer: Scorer, + /// The dataset it is a column of, which is where the user last saw it. + pub dataset: String, +} + +#[derive(Deserialize)] +pub struct RecentScorersQuery { + /// Only scorers of this kind, which is the one the add form was opened for. + #[serde(default)] + pub kind: Option, +} + +/// The scorers already in use in this workspace, most recently edited dataset first. +/// +/// Filtered twice through `user_db`: a scorer appears only if its dataset does, and the runnable +/// is checked the same way, so the list is scorers the caller could actually run. +pub async fn recent_scorers( + authed: ApiAuthed, + Extension(user_db): Extension, + Path(w_id): Path, + Query(query): Query, +) -> JsonResult> { + let mut tx = user_db.begin(&authed).await?; + let datasets = sqlx::query!( + "SELECT path, scorers FROM eval_dataset + WHERE workspace_id = $1 ORDER BY edited_at DESC LIMIT 100", + w_id + ) + .fetch_all(&mut *tx) + .await?; + + let mut seen = std::collections::HashSet::new(); + let mut recent: Vec = vec![]; + for row in datasets { + for scorer in parse_scorers(row.scorers)? { + if query + .kind + .as_deref() + .is_some_and(|kind| kind != scorer.def.kind_str()) + { + continue; + } + let key = (scorer.def.kind_str(), scorer.def.path().to_string()); + if seen.insert(key) { + recent.push(RecentScorer { scorer, dataset: row.path.clone() }); + } + } + } + // Readability is resolved over every candidate, then the list is cut: an unreadable scorer must + // not take a slot a readable one further down would have filled. + let script_paths = recent + .iter() + .filter(|r| matches!(r.scorer.def, ScorerDef::Script { .. })) + .map(|r| r.scorer.def.path().to_string()) + .collect::>(); + let agent_paths = recent + .iter() + .filter(|r| matches!(r.scorer.def, ScorerDef::Agent { .. })) + .map(|r| r.scorer.def.path().to_string()) + .collect::>(); + // Same deployed-version predicate as get_latest_script_hash: a script with no successfully + // locked version can't be resolved at launch, so it must not offer itself as a scorer here. + let readable_scripts = sqlx::query_scalar!( + "SELECT DISTINCT path FROM script + WHERE workspace_id = $1 AND path = ANY($2) + AND deleted = false AND lock IS NOT NULL AND lock_error_logs IS NULL", + w_id, + &script_paths + ) + .fetch_all(&mut *tx) + .await? + .into_iter() + .collect::>(); + let readable_agents = sqlx::query_scalar!( + "SELECT path FROM resource WHERE workspace_id = $1 AND path = ANY($2) AND resource_type = 'ai_agent'", + w_id, + &agent_paths + ) + .fetch_all(&mut *tx) + .await? + .into_iter() + .collect::>(); + tx.commit().await?; + + recent.retain(|r| match &r.scorer.def { + ScorerDef::Script { path } => readable_scripts.contains(path), + ScorerDef::Agent { path } => readable_agents.contains(path), + }); + recent.truncate(MAX_RECENT_SCORERS); + Ok(Json(recent)) +} + +#[cfg(test)] +mod tests { + use super::*; + + /// The definition hash is what tells a comparison that the scorer changed; the path alone + /// would miss an edit to the script itself. + #[test] + fn definition_moves_with_the_runnable_and_not_with_its_name() { + let script = |path: &str, name: Option<&str>| Scorer { + id: "s1".to_string(), + name: name.map(|n| n.to_string()), + pass_if: None, + def: ScorerDef::Script { path: path.to_string() }, + }; + // Renaming a column is not a change of scorer: same runnable, same version. + assert_eq!( + script("f/e/s", None).definition(Some("1234")), + script("f/e/s", Some("Tool discipline")).definition(Some("1234")) + ); + // Same script, newly deployed: the column says the scorer changed. + assert_ne!( + script("f/e/s", None).definition(Some("1234")), + script("f/e/s", None).definition(Some("5678")) + ); + // A judge agent and a script sharing a path are not the same column. + let agent = Scorer { + id: "s1".to_string(), + name: None, + pass_if: None, + def: ScorerDef::Agent { path: "f/e/s".to_string() }, + }; + assert_ne!( + agent.definition(Some("1")), + script("f/e/s", None).definition(Some("1")) + ); + // If the pass line entered the hash, setting a threshold would mark every score already + // recorded as coming from a different scorer. + let mut thresholded = script("f/e/s", None); + thresholded.pass_if = Some(0.7); + assert_eq!( + thresholded.definition(Some("1234")), + script("f/e/s", None).definition(Some("1234")) + ); + assert_eq!(thresholded.passed(Some(0.7)), Some(true)); + assert_eq!(thresholded.passed(Some(0.69)), Some(false)); + assert_eq!(script("f/e/s", None).passed(Some(0.1)), None); + } +} diff --git a/backend/windmill-api/src/ai_evals/scoring.rs b/backend/windmill-api/src/ai_evals/scoring.rs new file mode 100644 index 0000000000..8fff9d040f --- /dev/null +++ b/backend/windmill-api/src/ai_evals/scoring.rs @@ -0,0 +1,712 @@ +use super::*; + +/// What a scorer resolves to, alongside the definition to record: a script by its pinned hash, or +/// a judge by the configuration to inline. +pub(crate) enum ResolvedScorer { + Script { hash: i64 }, + Judge { config: AgentDraft }, +} + +/// The runnable a scorer names, resolved through the caller's *own* database so a run can only +/// execute code the caller may read: a scorer is added with a bare path and nothing checks read +/// access there. +/// +/// Returns the definition to record and what to run: a script by its deployed hash to pin, or a +/// judge by the configuration to inline, so a redeploy midway through a run cannot swap the code +/// out from under a score labelled with the old version. +pub(crate) async fn resolve_scorer( + user_db: &UserDB, + authed: &ApiAuthed, + w_id: &str, + scorer: &Scorer, +) -> Result<(String, ResolvedScorer)> { + match &scorer.def { + ScorerDef::Script { path } => { + // The latest *deployed* hash (no draft, no failed deploy), through the canonical helper + // so the version a scorer pins is the one everything else runs. + let mut tx = user_db.clone().begin(authed).await?; + let hash = windmill_common::get_latest_script_hash(&mut *tx, path, w_id).await?; + tx.commit().await?; + let Some(hash) = hash else { + return Err(Error::BadRequest(format!( + "Scorer script {} is not deployed or not readable", + path + ))); + }; + Ok(( + scorer.definition(Some(&hash.to_string())), + ResolvedScorer::Script { hash }, + )) + } + ScorerDef::Agent { path } => { + let Some((config, version)) = readable_agent_state(authed, user_db, w_id, path).await? + else { + return Err(Error::BadRequest(format!( + "Judge scorer {} is not a readable ai_agent resource", + path + ))); + }; + Ok(( + scorer.definition(Some(&version.to_string())), + ResolvedScorer::Judge { config }, + )) + } + } +} + +/// Bring a run's record up to date with the flow that executed it: which iteration answered which +/// case, what the agent answered, and what its scorers returned. +/// +/// `answers` is what separates the two callers: a listing reports each run's score aggregates and +/// never shows an answer, so harvesting them there reads a column of every case of every listed +/// run to display none of it. +pub(crate) async fn sync_run( + db: &DB, + w_id: &str, + experiment_id: Uuid, + run_job_id: Uuid, + answers: bool, +) -> Result<()> { + backfill_case_jobs(db, w_id, experiment_id, run_job_id).await?; + settle_unspawned_cases(db, w_id, experiment_id, run_job_id).await?; + if answers { + record_case_answers(db, w_id, experiment_id).await?; + } + harvest_flow_scores(db, w_id, experiment_id).await?; + Ok(()) +} + +/// Give a terminal status to cases the run never spawned an iteration for: with no `job_id` there +/// is nothing to read an answer or a score out of, so they would report "running" indefinitely. +async fn settle_unspawned_cases( + db: &DB, + w_id: &str, + experiment_id: Uuid, + run_job_id: Uuid, +) -> Result<()> { + // Only a run that has reached `v2_job_completed` is settled from here. A job absent from the + // tables is as likely mid-launch — the experiment is committed before its job is pushed — as + // aged out, and settling then would cancel the cases of a run about to start. A cancelled run + // lands in `v2_job_completed`, so a cancel before an iteration spawned is still covered. + let Some(terminal_status) = sqlx::query_scalar!( + "SELECT status::text AS \"status!\" FROM v2_job_completed WHERE id = $1 AND workspace_id = $2", + run_job_id, + w_id + ) + .fetch_optional(db) + .await? + else { + return Ok(()); + }; + let settled = sqlx::query_scalar!( + "UPDATE eval_experiment_case SET status = $2, answered = false + WHERE experiment_id = $1 AND job_id IS NULL AND status IS NULL + RETURNING ordinal", + experiment_id, + terminal_status + ) + .fetch_all(db) + .await?; + // The score cells of a case that never ran have no job to read a verdict out of either. + if !settled.is_empty() { + sqlx::query!( + "UPDATE eval_score SET error = 'The case did not run' + WHERE experiment_id = $1 AND ordinal = ANY($2) + AND score IS NULL AND error IS NULL AND NOT not_applicable", + experiment_id, + &settled + ) + .execute(db) + .await?; + } + Ok(()) +} + +/// In-flight reads of what a case's agent step produced. Each is several queries and a run holds +/// up to `MAX_CASES_PER_DATASET` cases, so they go a few at a time. +const HARVEST_CONCURRENCY: usize = 8; + +/// Cases whose scorer results are read in one query: every scorer of every case in the batch, so +/// the batch bounds how much of a run's worth of judge conversations is held at once. +const HARVEST_BATCH_CASES: usize = 100; + +/// Copy what each iteration produced into its row: the agent's answer, whether producing it +/// succeeded, and how the iteration ended. +/// +/// Written once, when it becomes readable, rather than read back out of the jobs whenever the +/// table is displayed — jobs have their own retention, and a run whose rows are kept has to still +/// read as the run it was after they have aged out. +async fn record_case_answers(db: &DB, w_id: &str, experiment_id: Uuid) -> Result<()> { + let unrecorded = sqlx::query!( + "SELECT c.ordinal, c.job_id AS \"job_id!\", d.status::text AS status, + (j.id IS NOT NULL) AS \"job_exists!\" + FROM eval_experiment_case c + LEFT JOIN v2_job j ON j.id = c.job_id AND j.workspace_id = $2 + LEFT JOIN v2_job_completed d ON d.id = c.job_id AND d.workspace_id = $2 + WHERE c.experiment_id = $1 AND c.job_id IS NOT NULL AND c.status IS NULL", + experiment_id, + w_id + ) + .fetch_all(db) + .await?; + + use futures::StreamExt; + let answers = futures::stream::iter(unrecorded.into_iter().map(|row| async move { + // The job was retained away before anything read it: nothing to read, and nothing more + // will ever be there to read. + if !row.job_exists { + return Ok((row.ordinal, None, None, Some("unavailable".to_string()))); + } + // The agent step's own result, never the iteration's: the iteration goes on to score the + // answer, so the answer is settled long before the iteration is. + let agent = agent_result(db, w_id, row.job_id).await?; + // An iteration that ended without an answer — skipped, cancelled, or an agent that failed + // outright — produced none, and saying so is what stops this re-reading it. + let answered = agent + .as_ref() + .map(|(_, success)| *success) + .or_else(|| row.status.is_some().then_some(false)); + let output = agent.as_ref().and_then(|(result, _)| agent_answer(result)); + Ok::<_, Error>((row.ordinal, output, answered, row.status)) + })) + .buffered(HARVEST_CONCURRENCY) + .collect::>() + .await + .into_iter() + .collect::>>()?; + + // One statement for the whole run: the run's own collect step reaches every case at once, and + // a thousand of them one at a time is a thousand round trips. + let mut ordinals = vec![]; + let mut outputs = vec![]; + let mut answered = vec![]; + let mut statuses = vec![]; + for (ordinal, output, was_answered, status) in answers { + // Nothing to record yet, and the iteration may still produce it. + if was_answered.is_none() && status.is_none() { + continue; + } + ordinals.push(ordinal); + outputs.push(output); + answered.push(was_answered); + statuses.push(status); + } + if ordinals.is_empty() { + return Ok(()); + } + sqlx::query!( + "UPDATE eval_experiment_case c + SET output = COALESCE(c.output, t.output), answered = COALESCE(c.answered, t.answered), + status = COALESCE(c.status, t.status) + FROM UNNEST($2::int[], $3::text[], $4::bool[], $5::text[]) + AS t(ordinal, output, answered, status) + WHERE c.experiment_id = $1 AND c.ordinal = t.ordinal", + experiment_id, + &ordinals, + &outputs as &[Option], + &answered as &[Option], + &statuses as &[Option], + ) + .execute(db) + .await?; + Ok(()) +} + +/// The agent step's result, with "there is none" kept apart from "it could not be read": a lookup +/// that failed for any other reason must not be recorded as a case that produced no answer, +/// because nothing reads that row again. +pub(crate) async fn agent_result( + db: &DB, + w_id: &str, + job_id: Uuid, +) -> Result, bool)>> { + match windmill_queue::get_result_and_success_by_id_from_flow( + db, + w_id, + &job_id, + AGENT_NODE_ID, + None, + ) + .await + { + Ok(found) => Ok(Some(found)), + Err(Error::NotFound(_)) => Ok(None), + Err(e) => Err(e), + } +} + +/// Match each case to the iteration that ran it. The flow engine mints those job ids, so the case +/// they belong to is read back from the iteration's own arguments, which survives iterations +/// finishing in any order. +async fn backfill_case_jobs( + db: &DB, + w_id: &str, + experiment_id: Uuid, + run_job_id: Uuid, +) -> Result<()> { + sqlx::query!( + "UPDATE eval_experiment_case c SET job_id = j.id + FROM v2_job j + WHERE j.parent_job = $3 AND j.workspace_id = $2 + AND (j.args -> 'iter' -> 'value' ->> 'case_id')::uuid = c.case_id + AND c.experiment_id = $1 AND c.job_id IS NULL", + experiment_id, + w_id, + run_job_id + ) + .execute(db) + .await?; + Ok(()) +} + +/// Read the scores a run's own flow produced into `eval_score`, so a score outlives the flow +/// that produced it and the retention on its jobs. +async fn harvest_flow_scores(db: &DB, w_id: &str, experiment_id: Uuid) -> Result<()> { + let pending = sqlx::query!( + // Left-joined, so an iteration still running is read too: a scorer runs after the agent + // within that iteration, so its verdict is there to be read as soon as its own step is + // done, and waiting for the iteration to end would hold every column of a case back until + // the last of them finished. + "SELECT s.ordinal, s.scorer_id, c.job_id AS \"job_id!\", d.status::text AS status, + c.answered, (j.id IS NOT NULL) AS \"job_exists!\" + FROM eval_score s + JOIN eval_experiment_case c + ON c.experiment_id = s.experiment_id AND c.ordinal = s.ordinal + LEFT JOIN v2_job j ON j.id = c.job_id AND j.workspace_id = $2 + LEFT JOIN v2_job_completed d ON d.id = c.job_id AND d.workspace_id = $2 + WHERE s.experiment_id = $1 AND s.score IS NULL AND s.error IS NULL + AND NOT s.not_applicable AND c.job_id IS NOT NULL", + experiment_id, + w_id + ) + .fetch_all(db) + .await?; + if pending.is_empty() { + return Ok(()); + } + + // The job tree is walked in SQL rather than once per cell: a live run is read every couple of + // seconds and a full one is up to MAX_CASES_PER_DATASET × MAX_SCORERS_PER_DATASET cells. The + // shape is `build_run_flow`'s: a scorer is the one module of its own branch of the scoring + // step, so its job's parent is that branch and the branch's parent is the case. + let mut case_jobs: Vec = pending.iter().map(|row| row.job_id).collect(); + case_jobs.sort(); + case_jobs.dedup(); + let mut modules: Vec = pending + .iter() + .map(|row| scorer_module_id(&row.scorer_id)) + .collect(); + modules.sort(); + modules.dedup(); + let mut verdicts: Vec<(i32, String, Option<(Verdict, Option)>)> = + Vec::with_capacity(pending.len()); + for batch in case_jobs.chunks(HARVEST_BATCH_CASES) { + let results: std::collections::HashMap<(Uuid, String), Box> = sqlx::query!( + "SELECT branch.parent_job AS \"case_job!\", scorer.flow_step_id AS \"module!\", + done.result AS \"result: sqlx::types::Json>\" + FROM v2_job branch + JOIN v2_job scorer ON scorer.parent_job = branch.id + JOIN v2_job_completed done ON done.id = scorer.id + WHERE branch.parent_job = ANY($1) AND branch.workspace_id = $2 + AND scorer.flow_step_id = ANY($3)", + batch, + w_id, + &modules + ) + .fetch_all(db) + .await? + .into_iter() + .map(|row| { + let result = row + .result + .map(|json| json.0) + .unwrap_or_else(|| RawValue::from_string("null".to_string()).expect("a literal")); + ((row.case_job, row.module), result) + }) + .collect(); + let in_batch: std::collections::HashSet = batch.iter().copied().collect(); + for row in pending.iter().filter(|row| in_batch.contains(&row.job_id)) { + // Nothing left to read the verdict out of. Settled here, since a cell left pending is + // one every later listing would go back to this same absent job for. + if !row.job_exists { + verdicts.push(( + row.ordinal, + row.scorer_id.clone(), + Some(( + Verdict::default(), + Some("The run that produced this score is no longer available".to_string()), + )), + )); + continue; + } + // What to say when the job is over and this scorer left nothing. Only + // `record_case_answers` tells the two states apart and a listing syncs without it, so + // `None` withholds the sentence — not the harvest: a scorer that returned a number is + // read and recorded either way. + let missing = row.answered.map(|answered| { + if answered { + "This scorer did not run for the case" + } else { + "The case produced no answer to score" + } + }); + let result = results + .get(&(row.job_id, scorer_module_id(&row.scorer_id))) + .map(|r| r.as_ref()); + let verdict = settle_verdict(result, row.status.as_deref(), missing); + verdicts.push((row.ordinal, row.scorer_id.clone(), verdict)); + } + } + + // One statement for every cell read, for the same reason the answers are written that way. + let mut ordinals = vec![]; + let mut scorer_ids = vec![]; + let mut scores = vec![]; + let mut reasons = vec![]; + let mut checks = vec![]; + let mut errors = vec![]; + let mut not_applicable = vec![]; + for (ordinal, scorer_id, read) in verdicts { + // Still to come: a scorer whose own step has not run yet. + let Some((verdict, error)) = read else { + continue; + }; + ordinals.push(ordinal); + scorer_ids.push(scorer_id); + scores.push(verdict.score); + reasons.push(verdict.reason); + checks.push(verdict.checks); + errors.push(error); + not_applicable.push(verdict.not_applicable); + } + if ordinals.is_empty() { + return Ok(()); + } + sqlx::query!( + "UPDATE eval_score s + SET score = t.score, reason = t.reason, checks = t.checks, error = t.error, + not_applicable = t.not_applicable + FROM UNNEST($2::int[], $3::text[], $4::double precision[], $5::text[], $6::jsonb[], + $7::text[], $8::bool[]) + AS t(ordinal, scorer_id, score, reason, checks, error, not_applicable) + WHERE s.experiment_id = $1 AND s.ordinal = t.ordinal AND s.scorer_id = t.scorer_id", + experiment_id, + &ordinals, + &scorer_ids, + &scores as &[Option], + &reasons as &[Option], + &checks as &[Option], + &errors as &[Option], + ¬_applicable, + ) + .execute(db) + .await?; + Ok(()) +} + +/// One scorer's verdict, from the result of the step that produced it, inside a job that may +/// still be running: a scorer's own step can be done while the iteration around it is not. `None` +/// while the result is not readable yet, which is a state to wait through rather than to record +/// as a failure; `Some` with an error is a scorer that produced nothing, worded by where it ran. +fn settle_verdict( + result: Option<&RawValue>, + job_status: Option<&str>, + // What to record when the job is over and this scorer produced nothing. A different statement + // depending on where the scorer ran: its own job failed, or the case it was to score never + // produced an answer. `None` when the caller cannot yet tell those apart, which leaves the + // cell pending for a read that can, rather than settling it on the wrong one of the two. + missing_error: Option<&str>, +) -> Option<(Verdict, Option)> { + Some(match result { + Some(value) => { + let verdict = extract_verdict(value); + match verdict { + // A score is a fraction: the mean and the pass rate read it as one, so a number + // outside that range is recorded as an error rather than a value that would + // quietly skew the column. + Verdict { score: Some(score), .. } if !(0.0..=1.0).contains(&score) => ( + Verdict::default(), + Some(format!( + "The scorer returned {}, outside the 0 to 1 range a score must be in", + score + )), + ), + // A number in range, or the scorer saying this case is not one it measures. Both + // are answers, so neither is an error. + Verdict { score: Some(_), .. } | Verdict { not_applicable: true, .. } => { + (verdict, None) + } + // The job around this scorer is still going, so a module with no number in it is + // one that has not run yet. Recording a failure here would make it permanent. + _ if job_status.is_none() => return None, + _ if job_status == Some("success") => ( + verdict, + Some("The scorer returned no number to plot".to_string()), + ), + _ => match missing_error { + Some(missing) => (verdict, Some(missing.to_string())), + None => return None, + }, + } + } + // The iteration is over, so a scorer step with no readable result produced nothing and + // never will; left pending it would be re-read on every listing. + None if job_status == Some("success") => ( + Verdict::default(), + Some("The scorer step produced no result".to_string()), + ), + // The job holding this scorer has not finished, so a module with nothing in it yet is a + // step that has not run rather than one that produced nothing. + None if job_status.is_none() => return None, + None => match missing_error { + Some(missing) => (Verdict::default(), Some(missing.to_string())), + None => return None, + }, + }) +} + +/// The score and reason read straight out of text that failed to parse as JSON. Deliberately not a +/// second JSON parser: it looks for the two keys and takes what follows, which is what survives a +/// model writing an unescaped quote in the middle of a sentence. +fn salvage_verdict(text: &str) -> (Option, Option) { + fn after_key<'a>(text: &'a str, key: &str) -> Option<&'a str> { + let start = text.find(key)? + key.len(); + Some(text[start..].trim_start().strip_prefix(':')?.trim_start()) + } + + let score = after_key(text, "\"score\"").and_then(|rest| { + if rest.starts_with("true") { + return Some(1.0); + } + if rest.starts_with("false") { + return Some(0.0); + } + let end = rest + .find(|c: char| !matches!(c, '0'..='9' | '.' | '-' | '+' | 'e' | 'E')) + .unwrap_or(rest.len()); + rest[..end].parse::().ok() + }); + + // To the last quote of the object, so an unescaped one inside the sentence stays part of it. + let reason = after_key(text, "\"reason\"") + .and_then(|rest| rest.strip_prefix('"')) + .and_then(|rest| { + let body = match rest.rfind('}') { + Some(brace) => &rest[..brace], + None => rest, + }; + let end = body.rfind('"')?; + Some(body[..end].to_string()) + }) + .filter(|reason| !reason.is_empty()); + + (score, reason) +} + +/// A fenced code block as the model wrote it, reduced to what is inside the fence. The opening +/// fence carries a language tag often enough that the first line goes with it. +fn unfence(text: &str) -> &str { + let trimmed = text.trim(); + let Some(rest) = trimmed.strip_prefix("```") else { + return trimmed; + }; + let inner = match rest.split_once('\n') { + Some((_language, body)) => body, + None => rest, + }; + inner.trim_end().trim_end_matches("```").trim() +} + +/// What a scorer said about one run. `not_applicable` is the scorer declining to measure this +/// case: an explicit `{"score": null}`. A bare `null` stays an error, since a scorer that forgot +/// to return is indistinguishable from one that returned nothing on purpose. +#[derive(Default)] +struct Verdict { + score: Option, + reason: Option, + checks: Option, + not_applicable: bool, +} + +impl Verdict { + fn scored(score: f64) -> Self { + Verdict { score: Some(score), ..Default::default() } + } +} + +/// A scorer may return a bare number, a boolean, or `{score, reason, checks}`; an agent wraps its +/// answer in `output`, sometimes as a string holding any of those. Anything with no number in it +/// is left empty rather than guessed at. +fn extract_verdict(value: &RawValue) -> Verdict { + let Ok(parsed) = serde_json::from_str::(value.get()) else { + return Verdict::default(); + }; + fn as_number(value: &serde_json::Value) -> Option { + match value { + serde_json::Value::Number(n) => n.as_f64(), + serde_json::Value::Bool(b) => Some(if *b { 1.0 } else { 0.0 }), + _ => None, + } + } + if let Some(number) = as_number(&parsed) { + return Verdict::scored(number); + } + let serde_json::Value::Object(map) = &parsed else { + // A judge often answers with JSON inside a string, and often fences it as markdown even + // when told to reply with JSON only. + if let serde_json::Value::String(text) = &parsed { + let text = unfence(text); + if let Ok(inner) = serde_json::from_str::(text) { + if let Ok(raw) = serde_json::value::to_raw_value(&inner) { + return extract_verdict(&raw); + } + } + // Nearly JSON: a judge that quotes the agent inside its own reason writes those quotes + // unescaped, which is invalid and also the most ordinary thing for it to say. The + // number is what the column plots, so it is read out of the text rather than lost with + // the object around it. + let (score, reason) = salvage_verdict(text); + return Verdict { score, reason, checks: None, not_applicable: false }; + } + return Verdict::default(); + }; + let reason = || { + map.get("reason") + .or_else(|| map.get("comment")) + .and_then(|r| r.as_str()) + .map(|r| r.to_string()) + }; + if let Some(score) = map.get("score").and_then(as_number) { + return Verdict { + score: Some(score), + reason: reason(), + checks: map.get("checks").cloned(), + not_applicable: false, + }; + } + // Written out rather than merely absent, which is what separates it from a scorer that + // returned an object with no verdict in it at all. + if map.get("score").is_some_and(|s| s.is_null()) { + return Verdict { + score: None, + reason: reason(), + checks: map.get("checks").cloned(), + not_applicable: true, + }; + } + match map.get("output") { + Some(output) => match serde_json::value::to_raw_value(output) { + Ok(raw) => extract_verdict(&raw), + Err(_) => Verdict::default(), + }, + None => Verdict::default(), + } +} + +#[cfg(test)] +mod tests { + use super::*; + + fn raw(json: &str) -> Box { + serde_json::from_str(json).unwrap() + } + + /// A scorer's answer arrives in whatever shape its runnable returns: a script's bare value or + /// object, or a judge's answer wrapped in `output` and often stringified. A shape that goes + /// unrecognised is a silently empty cell rather than an error. + #[test] + fn extract_verdict_reads_every_documented_scorer_shape() { + let score = |json: &str| extract_verdict(&raw(json)).score; + assert_eq!(score("0.75"), Some(0.75)); + assert_eq!(score("true"), Some(1.0)); + assert_eq!(score(r#"{"score": 0.5}"#), Some(0.5)); + assert_eq!(score(r#"{"score": false}"#), Some(0.0)); + + // judges and agent scorers: the answer is under `output`, sometimes as a string + assert_eq!(score(r#"{"output": 0.25}"#), Some(0.25)); + assert_eq!(score(r#"{"output": "0.9"}"#), Some(0.9)); + assert_eq!(score(r#"{"output": {"score": 0.8}}"#), Some(0.8)); + assert_eq!(score(r#"{"output": "{\"score\": 0.4}"}"#), Some(0.4)); + + // a judge told to reply with JSON only, replying with JSON only, in a code fence + assert_eq!( + score("{\"output\": \"```json\\n{\\\"score\\\": 0.15}\\n```\"}"), + Some(0.15) + ); + assert_eq!(score("{\"output\": \"```\\n0.6\\n```\"}"), Some(0.6)); + + // A judge quoting the agent inside its own reason, which is invalid JSON. + let quoted = extract_verdict(&raw( + r#"{"output": "{\"score\": 0.8, \"reason\": \"invented context (\"stop asking me\", never said) here\"}"}"#, + )); + assert_eq!(quoted.score, Some(0.8)); + assert_eq!( + quoted.reason.as_deref(), + Some(r#"invented context ("stop asking me", never said) here"#) + ); + + // nothing numeric to plot: left empty rather than guessed at + assert_eq!(score(r#"{"output": "not a score"}"#), None); + assert_eq!(score(r#"{"verdict": "good"}"#), None); + + let full = extract_verdict(&raw( + r#"{"score": 0.5, "reason": "half", "checks": [{"name": "a"}]}"#, + )); + assert_eq!( + (full.score, full.reason), + (Some(0.5), Some("half".to_string())) + ); + assert!(full.checks.is_some()); + assert!(!full.not_applicable); + + // `comment` as the rationale, which is what a scorer written for LangSmith or Langfuse + // returns. Read rather than dropped, since the number arrives either way. + assert_eq!( + extract_verdict(&raw(r#"{"score": 1, "comment": "fine"}"#)) + .reason + .as_deref(), + Some("fine") + ); + } + + /// A score is a fraction: anything outside 0..=1 (a scorer that returned a count, say) is + /// recorded as an error naming the value rather than plotted as a bogus point. + #[test] + fn an_out_of_range_score_is_recorded_as_an_error_not_a_value() { + // In range: recorded as the score it is. + let (v, e) = settle_verdict(Some(&raw("0.5")), Some("success"), None).unwrap(); + assert_eq!(v.score, Some(0.5)); + assert!(e.is_none()); + // Out of range (a scorer returning a count, say): no score, an error naming the value. + let (v, e) = settle_verdict(Some(&raw("100")), Some("success"), None).unwrap(); + assert_eq!(v.score, None); + assert!(e.unwrap().contains("100")); + let (v, _) = settle_verdict(Some(&raw("-5")), Some("success"), None).unwrap(); + assert_eq!(v.score, None); + // No result at all once the iteration is over: an error, not a cell pending forever. + let (v, e) = settle_verdict(None, Some("success"), None).unwrap(); + assert_eq!(v.score, None); + assert!(e.is_some()); + // Still running: nothing to settle yet. + assert!(settle_verdict(None, None, None).is_none()); + } + + /// A scorer saying it has nothing to measure on a case is a verdict rather than a failure: the + /// cell is left out of the mean instead of counted as a zero. Spelled out, so a scorer that + /// returns nothing at all is still an error rather than silently excused. + #[test] + fn an_explicit_null_score_is_not_applicable_rather_than_missing() { + let na = extract_verdict(&raw(r#"{"score": null, "reason": "no sources to cite"}"#)); + assert!(na.not_applicable); + assert_eq!(na.score, None); + assert_eq!(na.reason.as_deref(), Some("no sources to cite")); + + // Through a judge's wrapper, as any other verdict is. + assert!(extract_verdict(&raw(r#"{"output": {"score": null}}"#)).not_applicable); + assert!(extract_verdict(&raw(r#"{"output": "{\"score\": null}"}"#)).not_applicable); + + // Not the same as a scorer that returned nothing, or an object with no verdict in it. + assert!(!extract_verdict(&raw("null")).not_applicable); + assert!(!extract_verdict(&raw(r#"{"verdict": "good"}"#)).not_applicable); + } +} diff --git a/backend/windmill-api/src/ai_evals/subject.rs b/backend/windmill-api/src/ai_evals/subject.rs new file mode 100644 index 0000000000..52b8ff0519 --- /dev/null +++ b/backend/windmill-api/src/ai_evals/subject.rs @@ -0,0 +1,135 @@ +use super::*; + +/// What a run is executed against. Kept as `(kind, path, version)` rather than a bare agent +/// path so flow-scoped evaluation is a later superset instead of a rewrite. +#[derive(Serialize, Deserialize, Debug, Clone)] +pub struct EvalSubject { + #[serde(default = "default_subject_kind")] + pub kind: EvalSubjectKind, + /// The agent resource under test. + pub path: String, + /// Which version of the agent, counted per path: how many times it had been saved. The + /// request's to choose for a pinned run, and otherwise the version the run was enqueued + /// against. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub version: Option, + /// The agent's unsaved edits, as the editor holds them. Present exactly when `kind` is + /// `agent_draft`, since the edits exist nowhere else. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub draft: Option, + /// Hash of that configuration. A draft moves without the version moving, so this is the only + /// thing that can say a run describes an agent that has since been edited. Stamped + /// server-side. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub draft_hash: Option, +} + +/// Key order is not meaningful and `serde_json` preserves insertion order here, so it is sorted +/// away before hashing: the same configuration must hash the same however it was assembled. +fn canonical_json(value: &serde_json::Value) -> String { + match value { + serde_json::Value::Object(map) => { + let sorted = map + .iter() + .collect::>() + .into_iter() + .map(|(k, v)| { + format!( + "{}:{}", + serde_json::to_string(k).unwrap_or_default(), + canonical_json(v) + ) + }) + .collect::>() + .join(","); + format!("{{{}}}", sorted) + } + serde_json::Value::Array(items) => format!( + "[{}]", + items + .iter() + .map(canonical_json) + .collect::>() + .join(",") + ), + other => other.to_string(), + } +} + +pub(crate) fn draft_hash(draft: &AgentDraft) -> String { + use sha2::{Digest, Sha256}; + let mut hasher = Sha256::new(); + hasher.update(canonical_json(&draft.input_transforms).as_bytes()); + hasher.update(b"|"); + hasher.update(canonical_json(&serde_json::Value::Array(draft.tools.clone())).as_bytes()); + hex::encode(hasher.finalize())[..32].to_string() +} + +fn default_subject_kind() -> EvalSubjectKind { + EvalSubjectKind::Agent +} + +#[derive(Serialize, Deserialize, Debug, Clone, PartialEq)] +#[serde(rename_all = "snake_case")] +pub enum EvalSubjectKind { + Agent, + /// A saved agent's unsaved edits, carried by the request and inlined: a linked step resolves + /// the resource live and so would run what the edits replace. + AgentDraft, + /// One past version of a saved agent, inlined for the same reason. `version` says which, and + /// it is the request's to choose rather than the server's. + AgentVersion, +} + +/// The brain and tools of an agent, as the flow editor holds them. +#[derive(Serialize, Deserialize, Debug, Clone)] +pub struct AgentDraft { + /// The agent's input transforms: provider, system prompt, output type and the rest. The + /// message and attachments are supplied by the case and override anything named here. + #[serde(default)] + pub input_transforms: serde_json::Value, + #[serde(default)] + pub tools: Vec, +} + +impl EvalSubject { + /// What is recorded of a subject: enough to say what ran, without the configuration itself. + pub(crate) fn stamp(&self) -> EvalSubject { + EvalSubject { + kind: self.kind.clone(), + path: self.path.clone(), + version: self.version, + draft: None, + // Only ever derived from the draft this request carries: a hash the client supplies on + // its own could relabel a run as the deployed version. + draft_hash: self.draft.as_ref().map(draft_hash), + } + } +} + +#[derive(Deserialize)] +pub struct SubjectStateQuery { + pub path: String, +} + +#[derive(Serialize)] +pub struct SubjectState { + /// The version the agent is on now. + #[serde(skip_serializing_if = "Option::is_none")] + pub version: Option, +} + +/// The version the agent is deployed at. Small on purpose: the results endpoint reports the same +/// thing, but it harvests scores and reads every job to do it. +pub async fn subject_state( + authed: ApiAuthed, + Extension(user_db): Extension, + Path(w_id): Path, + Query(query): Query, +) -> JsonResult { + let Some((_, version)) = readable_agent_state(&authed, &user_db, &w_id, &query.path).await? + else { + return Err(Error::NotFound(format!("Agent {} not found", query.path))); + }; + Ok(Json(SubjectState { version: Some(version) })) +} diff --git a/backend/windmill-api/src/ai_evals/template.rs b/backend/windmill-api/src/ai_evals/template.rs new file mode 100644 index 0000000000..22d37d1e85 --- /dev/null +++ b/backend/windmill-api/src/ai_evals/template.rs @@ -0,0 +1,132 @@ +use super::*; + +/// What a script scorer starts from. +pub const SCORER_SCRIPT_TEMPLATE: &str = r#"// A scorer receives one run and returns a number between 0 and 1, a boolean, or +// { score, reason, checks } — checks show up in the case detail. +// Return { score: null } for a case this scorer has nothing to measure on: the cell +// is left out of the column's mean and pass rate rather than counted as a zero. +// +// The run is also handed to you spelled out, so a short scorer can skip the type below +// entirely: export async function main(output: unknown, expected: unknown) { ... } +type ToolCall = { + name: string + args?: Record + result?: unknown + error?: string + duration_ms?: number + truncated?: boolean +} + +type EvalRun = { + input: { user_message?: string; user_attachments?: unknown[] } + output?: unknown + expected?: unknown + tool_calls: ToolCall[] + tools: { name: string; schema?: Record }[] + metrics: { steps: number; duration_ms?: number; usage?: Record } + status: string + job_id: string +} + +export async function main(run: EvalRun) { + // How the agent got to its answer. Reported rather than scored: checks render in the case + // detail either way, so they explain the number without being averaged into it. + const checks = [ + check('arguments match the schema', args_schema_valid(run)), + check('no repeated calls', no_repeated_calls(run)), + check('no failed tool calls', no_step_errors(run)), + check('under 6 steps', run.metrics.steps <= 6, `${run.metrics.steps} steps`), + check('under 30 seconds', under_ms(run, 30_000), `${run.metrics.duration_ms ?? '?'} ms`) + ] + + // Nothing to compare the answer against, so this column has no verdict on this case rather + // than a failing one. The cell reads n/a and the column's mean is of the cases it measured. + if (run.expected == undefined) { + return { score: null, reason: 'this case has no expected answer', checks } + } + + // One question per column, and this column's question is whether the answer is right. + // Deliberately not the share of checks above that passed: a right answer that was slow and a + // wrong answer that was fast would score the same, and the column could not say which it was. + const correct = contains(run.output, text(run.expected)) + return { + score: correct ? 1 : 0, + reason: correct ? undefined : `expected ${text(run.expected)}`, + checks + } +} + +// Helpers. Edit or delete freely. + +function check(name: string, passed: boolean, detail?: string) { + return { name, passed, detail } +} + +function text(value: unknown): string { + return typeof value === 'string' ? value : JSON.stringify(value ?? '') +} + +function contains(output: unknown, needle: string): boolean { + return needle.trim().length > 0 && text(output).toLowerCase().includes(needle.trim().toLowerCase()) +} + +// Every call validated against the schema of the tool it called. A tool whose schema could not be +// resolved is not checked rather than failed. +function args_schema_valid(run: EvalRun): boolean { + return run.tool_calls.every((call) => { + const schema = run.tools.find((tool) => tool.name === call.name)?.schema as + | { properties?: Record; required?: string[] } + | undefined + if (!schema?.properties) return true + const args = call.args ?? {} + for (const key of schema.required ?? []) { + if (args[key] === undefined || args[key] === null) return false + } + for (const [key, value] of Object.entries(args)) { + const expected = schema.properties[key]?.type + if (!expected) continue + const actual = Array.isArray(value) ? 'array' : value === null ? 'null' : typeof value + if (expected === 'integer' ? !Number.isInteger(value) : expected !== actual) return false + } + return true + }) +} + +// The same tool called twice with the same arguments. +function no_repeated_calls(run: EvalRun): boolean { + const seen = new Set() + for (const call of run.tool_calls) { + const key = `${call.name}:${JSON.stringify(call.args ?? {})}` + if (seen.has(key)) return false + seen.add(key) + } + return true +} + +function no_step_errors(run: EvalRun): boolean { + return run.status === 'success' && run.tool_calls.every((call) => !call.error) +} + +// A run with no recorded duration is not under the limit: a check that could not be evaluated +// should not report as one that passed. +function under_ms(run: EvalRun, max: number): boolean { + const ms = run.metrics.duration_ms + return ms != undefined && ms <= max +} +"#; + +#[derive(Serialize)] +pub struct ScorerDefaults { + /// The system prompt a judge agent is created with. It lives on that agent afterwards. + pub judge_prompt: String, + /// The starting point for a script scorer, held here so the shape a scorer is handed and the + /// template that reads it cannot drift apart. + pub script_template: String, +} + +pub async fn scorer_defaults() -> JsonResult { + Ok(Json(ScorerDefaults { + judge_prompt: JUDGE_SYSTEM_PROMPT.to_string(), + script_template: SCORER_SCRIPT_TEMPLATE.to_string(), + })) +} diff --git a/backend/windmill-api/src/apps.rs b/backend/windmill-api/src/apps.rs index 0e64d1cba2..7bc995844d 100644 --- a/backend/windmill-api/src/apps.rs +++ b/backend/windmill-api/src/apps.rs @@ -4112,10 +4112,18 @@ struct S3DeleteTokenClaims { pub exp: usize, } +#[cfg(feature = "parquet")] +const SIGN_S3_DEFAULT_EXPIRY_SECS: i64 = 12 * 60 * 60; +#[cfg(feature = "parquet")] +const SIGN_S3_MIN_EXPIRY_SECS: i64 = 60; +#[cfg(feature = "parquet")] +const SIGN_S3_MAX_EXPIRY_SECS: i64 = 7 * 24 * 60 * 60; + #[cfg(feature = "parquet")] #[derive(Deserialize)] struct S3TokenRequestBody { s3_objects: Vec, + expiry_secs: Option, } #[cfg(feature = "parquet")] async fn sign_s3_objects( @@ -4126,6 +4134,12 @@ async fn sign_s3_objects( ) -> Result>> { let workspace_key = get_workspace_key(&w_id, &db).await?; + let expiry_secs = body + .expiry_secs + .unwrap_or(SIGN_S3_DEFAULT_EXPIRY_SECS) + .clamp(SIGN_S3_MIN_EXPIRY_SECS, SIGN_S3_MAX_EXPIRY_SECS); + let exp = (chrono::Utc::now() + chrono::Duration::seconds(expiry_secs)).timestamp(); + let futures = body.s3_objects.into_iter().map(|s3_object| async { // The signature this mints is a transferable bearer capability: `validate_s3_signature` // only checks the HMAC and expiry, so anyone who obtains the string can read this key. @@ -4156,7 +4170,6 @@ async fn sign_s3_objects( ) .await?; - let exp = (chrono::Utc::now() + chrono::Duration::hours(12)).timestamp(); let message = format!( "file_key={}&exp={}{}", s3_object.s3.clone(), diff --git a/backend/windmill-api/src/jobs.rs b/backend/windmill-api/src/jobs.rs index 513341b6c1..a30cffbfbf 100644 --- a/backend/windmill-api/src/jobs.rs +++ b/backend/windmill-api/src/jobs.rs @@ -6559,17 +6559,18 @@ pub async fn run_flow_by_version_inner( Ok((uuid, early_return, has_failure_module)) } +/// Path parameters of `POST /w/{workspace}/jobs/restart/f/{job_id}`, shared by the CE and EE +/// handlers. Axum only checks the tuple against the route at request time and rejects a +/// mismatch with an opaque 500 before the handler runs, so both must be declared from here: +/// an arity that drifts from the route hides the handler behind what reads as a broken route. +type RestartFlowPath = Path<(String, Uuid)>; + #[cfg(not(feature = "enterprise"))] pub async fn restart_flow( _authed: ApiAuthed, Extension(_db): Extension, Extension(_user_db): Extension, - Path((_w_id, _job_id, _step_id, _branch_or_iteration_n)): Path<( - String, - Uuid, - String, - Option, - )>, + Path((_w_id, _job_id)): RestartFlowPath, Query(_run_query): Query, ) -> error::Result<(StatusCode, String)> { return Err(Error::BadRequest( @@ -6798,7 +6799,7 @@ pub async fn restart_flow( authed: ApiAuthed, Extension(db): Extension, Extension(user_db): Extension, - Path((w_id, job_id)): Path<(String, Uuid)>, + Path((w_id, job_id)): RestartFlowPath, Query(run_query): Query, Json(RestartFlowRequestBody { step_id, diff --git a/backend/windmill-api/src/lib.rs b/backend/windmill-api/src/lib.rs index 9963219993..de49949043 100644 --- a/backend/windmill-api/src/lib.rs +++ b/backend/windmill-api/src/lib.rs @@ -113,6 +113,7 @@ pub mod storage_list_ee; mod storage_list_oss; mod workspace_dependencies; +mod ai_evals; mod approvals; #[cfg(all(feature = "enterprise", feature = "private"))] pub mod apps_ee; @@ -674,6 +675,7 @@ pub async fn run_server( .route("/labels/list", get(list_workspace_labels)) .nest("/job_metrics", job_metrics::workspaced_service()) .nest("/job_helpers", job_helpers_service) + .nest("/ai_evals", ai_evals::workspaced_service()) .nest("/jobs", jobs::workspaced_service()) .nest("/debug", windmill_api_debug::workspaced_service()) .nest("/native_triggers", { diff --git a/backend/windmill-api/src/offboarding.rs b/backend/windmill-api/src/offboarding.rs index 0d9529c0ce..a69441ea75 100644 --- a/backend/windmill-api/src/offboarding.rs +++ b/backend/windmill-api/src/offboarding.rs @@ -52,6 +52,8 @@ struct OffboardAffectedPaths { variables: Vec, #[serde(skip_serializing_if = "Vec::is_empty")] schedules: Vec, + #[serde(skip_serializing_if = "Vec::is_empty")] + eval_datasets: Vec, #[serde(skip_serializing_if = "HashMap::is_empty")] triggers: HashMap>, } @@ -91,6 +93,7 @@ struct OffboardSummary { flows_reassigned: i64, apps_reassigned: i64, resources_reassigned: i64, + eval_datasets_reassigned: i64, variables_reassigned: i64, schedules_reassigned: i64, triggers_reassigned: i64, @@ -168,6 +171,14 @@ async fn get_offboard_preview( .fetch_all(db) .await?; + let eval_datasets = sqlx::query_scalar!( + "SELECT path FROM eval_dataset WHERE path LIKE $1 AND workspace_id = $2", + &user_prefix, + w_id + ) + .fetch_all(db) + .await?; + let variables = sqlx::query_scalar!( "SELECT path FROM variable WHERE path LIKE $1 AND workspace_id = $2", &user_prefix, @@ -285,6 +296,16 @@ async fn get_offboard_preview( &ref_pattern, &user_prefix, w_id ).fetch_all(db).await?; + let ref_eval_datasets = sqlx::query_scalar!( + "SELECT path FROM eval_dataset + WHERE scorers::text LIKE $1 AND NOT path LIKE $2 AND workspace_id = $3", + &ref_pattern, + &user_prefix, + w_id + ) + .fetch_all(db) + .await?; + let ref_resources = sqlx::query_scalar!( "SELECT DISTINCT path FROM resource WHERE value::text LIKE $1 AND NOT path LIKE $2 AND workspace_id = $3", &ref_pattern, &user_prefix, w_id @@ -317,6 +338,7 @@ async fn get_offboard_preview( resources, variables, schedules, + eval_datasets, triggers, }, executing_on_behalf: OffboardAffectedPaths { @@ -332,6 +354,7 @@ async fn get_offboard_preview( flows: ref_flows, apps: ref_apps, resources: ref_resources, + eval_datasets: ref_eval_datasets, ..Default::default() }, tokens, @@ -548,6 +571,7 @@ pub(crate) async fn offboard_global_user( flows_reassigned: 0, apps_reassigned: 0, resources_reassigned: 0, + eval_datasets_reassigned: 0, variables_reassigned: 0, schedules_reassigned: 0, triggers_reassigned: 0, @@ -574,6 +598,7 @@ pub(crate) async fn offboard_global_user( total_summary.flows_reassigned += ws_summary.flows_reassigned; total_summary.apps_reassigned += ws_summary.apps_reassigned; total_summary.resources_reassigned += ws_summary.resources_reassigned; + total_summary.eval_datasets_reassigned += ws_summary.eval_datasets_reassigned; total_summary.variables_reassigned += ws_summary.variables_reassigned; total_summary.schedules_reassigned += ws_summary.schedules_reassigned; total_summary.triggers_reassigned += ws_summary.triggers_reassigned; @@ -746,6 +771,7 @@ async fn check_path_conflicts( "flow", "app", "resource", + "eval_dataset", "variable", "schedule", "http_trigger", @@ -959,6 +985,47 @@ async fn offboard_user_from_workspace<'c>( .await? .unwrap_or(0); + // ---- eval datasets ---- + // The foreign keys cascade the rename onto cases and experiments; the paths held inside JSONB + // (an experiment's subject, a dataset's scorers) are rewritten separately since the cascade + // cannot reach them and those runnables move with the user. + let eval_datasets_reassigned = sqlx::query_scalar!( + r#"WITH updated AS ( + UPDATE eval_dataset SET path = REGEXP_REPLACE(path, 'u/' || $2 || '/(.*)', $1 || '/\1') + WHERE path LIKE ('u/' || $2 || '/%') AND workspace_id = $3 + RETURNING 1 + ) SELECT COUNT(*) FROM updated"#, + &new_prefix, + username, + w_id + ) + .fetch_one(&mut **tx) + .await? + .unwrap_or(0); + sqlx::query!( + r#"UPDATE eval_experiment SET subject = jsonb_set(subject, '{path}', to_jsonb(REGEXP_REPLACE(subject->>'path', 'u/' || $2 || '/(.*)', $1 || '/\1'))) WHERE subject->>'path' LIKE ('u/' || $2 || '/%') AND workspace_id = $3"#, + &new_prefix, + username, + w_id + ) + .execute(&mut **tx) + .await?; + sqlx::query!( + r#"UPDATE eval_dataset SET scorers = COALESCE(( + SELECT jsonb_agg( + CASE WHEN elem->>'path' LIKE ('u/' || $2 || '/%') + THEN jsonb_set(elem, '{path}', to_jsonb(REGEXP_REPLACE(elem->>'path', 'u/' || $2 || '/(.*)', $1 || '/\1'))) + ELSE elem END) + FROM jsonb_array_elements(scorers) elem), '[]'::jsonb) + WHERE workspace_id = $3 + AND EXISTS (SELECT 1 FROM jsonb_array_elements(scorers) e WHERE e->>'path' LIKE ('u/' || $2 || '/%'))"#, + &new_prefix, + username, + w_id + ) + .execute(&mut **tx) + .await?; + // ---- variables (with Vault secret handling) ---- let old_var_prefix = format!("u/{}/", username); let new_var_prefix = format!("{}/", reassign_to); @@ -1152,6 +1219,7 @@ async fn offboard_user_from_workspace<'c>( flows_reassigned, apps_reassigned, resources_reassigned, + eval_datasets_reassigned, variables_reassigned, schedules_reassigned, triggers_reassigned, diff --git a/backend/windmill-api/src/token.rs b/backend/windmill-api/src/token.rs index fc808767ec..2268a63894 100644 --- a/backend/windmill-api/src/token.rs +++ b/backend/windmill-api/src/token.rs @@ -100,6 +100,12 @@ fn build_standard_scope_domains() -> Vec { ("oauth", "OAuth", "OAuth management", false), ("ai", "AI", "AI feature management", false), ("ai_skills", "AI Skills", "AI skill management", false), + ( + "ai_evals", + "AI Evals", + "AI agent eval datasets and standalone runs", + false, + ), ( "agent_workers", "Agent Workers", diff --git a/backend/windmill-api/src/triggers/http/handler.rs b/backend/windmill-api/src/triggers/http/handler.rs index 37442c8111..e58aab44a2 100644 --- a/backend/windmill-api/src/triggers/http/handler.rs +++ b/backend/windmill-api/src/triggers/http/handler.rs @@ -123,7 +123,9 @@ async fn get_http_route_trigger( let routers_cache = if routers_cache.routers.is_empty() { tracing::warn!("HTTP routers are not loaded, loading from db"); - let (_, routers_cache) = refresh_routers(db).await?; + // refresh_routers takes the write lock, so holding this read guard across it deadlocks. + drop(routers_cache); + let (_, routers_cache) = refresh_routers(db, false).await?; routers_cache } else { routers_cache diff --git a/backend/windmill-api/src/users.rs b/backend/windmill-api/src/users.rs index 37946f68a4..8804004f2b 100644 --- a/backend/windmill-api/src/users.rs +++ b/backend/windmill-api/src/users.rs @@ -373,6 +373,87 @@ async fn update_username_in_workpsace<'c>( .execute(&mut **tx) .await?; + // Eval datasets are path-addressed like every other object, so a username change moves them + // too. The foreign keys cascade the rename onto their cases and experiments; the experiment + // subject (the agent a run was of) is a `u//` path of its own inside JSONB, so it is + // rewritten separately or a user's own runs would detach from their renamed agent. + sqlx::query!( + r#"UPDATE eval_dataset SET path = REGEXP_REPLACE(path,'u/' || $2 || '/(.*)','u/' || $1 || '/\1') WHERE path LIKE ('u/' || $2 || '/%') AND workspace_id = $3"#, + new_username, + old_username, + w_id + ).execute(&mut **tx) + .await?; + + sqlx::query!( + "UPDATE eval_dataset SET extra_perms = extra_perms - ('u/' || $2) || jsonb_build_object(('u/' || $1), extra_perms->('u/' || $2)) WHERE extra_perms ? ('u/' || $2) AND workspace_id = $3", + new_username, + old_username, + w_id + ) + .execute(&mut **tx) + .await?; + + sqlx::query!( + r#"UPDATE eval_experiment SET subject = jsonb_set(subject, '{path}', to_jsonb(REGEXP_REPLACE(subject->>'path','u/' || $2 || '/(.*)','u/' || $1 || '/\1'))) WHERE subject->>'path' LIKE ('u/' || $2 || '/%') AND workspace_id = $3"#, + new_username, + old_username, + w_id + ).execute(&mut **tx) + .await?; + + sqlx::query!( + "UPDATE eval_dataset SET created_by = $1 WHERE created_by = $2 AND workspace_id = $3", + new_username, + old_username, + w_id + ) + .execute(&mut **tx) + .await?; + sqlx::query!( + "UPDATE eval_dataset SET edited_by = $1 WHERE edited_by = $2 AND workspace_id = $3", + new_username, + old_username, + w_id + ) + .execute(&mut **tx) + .await?; + sqlx::query!( + "UPDATE eval_case SET created_by = $1 WHERE created_by = $2 AND workspace_id = $3", + new_username, + old_username, + w_id + ) + .execute(&mut **tx) + .await?; + sqlx::query!( + "UPDATE eval_experiment SET created_by = $1 WHERE created_by = $2 AND workspace_id = $3", + new_username, + old_username, + w_id + ) + .execute(&mut **tx) + .await?; + + // A dataset's scorers name scripts and agents by path in a JSONB array, which the rewrites + // above do not reach; those runnables are renamed elsewhere in this transaction, so each + // scorer path under the old username is rewritten too or the dataset points at a runnable that + // no longer exists. + sqlx::query!( + r#"UPDATE eval_dataset SET scorers = COALESCE(( + SELECT jsonb_agg( + CASE WHEN elem->>'path' LIKE ('u/' || $2 || '/%') + THEN jsonb_set(elem, '{path}', to_jsonb(REGEXP_REPLACE(elem->>'path','u/' || $2 || '/(.*)','u/' || $1 || '/\1'))) + ELSE elem END) + FROM jsonb_array_elements(scorers) elem), '[]'::jsonb) + WHERE workspace_id = $3 + AND EXISTS (SELECT 1 FROM jsonb_array_elements(scorers) e WHERE e->>'path' LIKE ('u/' || $2 || '/%'))"#, + new_username, + old_username, + w_id + ).execute(&mut **tx) + .await?; + // ---- variables ---- // Handle Vault secret renames before updating paths in DB diff --git a/backend/windmill-api/src/workspaces.rs b/backend/windmill-api/src/workspaces.rs index 23ce68fb80..01eb32b7f6 100644 --- a/backend/windmill-api/src/workspaces.rs +++ b/backend/windmill-api/src/workspaces.rs @@ -108,6 +108,8 @@ async fn edit_copilot_config( } } + ai_config.validate_model_pricing()?; + let mut tx = db.begin().await?; sqlx::query!( diff --git a/backend/windmill-api/src/workspaces_export.rs b/backend/windmill-api/src/workspaces_export.rs index efe0ce157c..58deadfca1 100644 --- a/backend/windmill-api/src/workspaces_export.rs +++ b/backend/windmill-api/src/workspaces_export.rs @@ -644,6 +644,16 @@ pub(crate) async fn tarball_workspace( windmill_api_auth::forbid_scoped_token_workspace_key(&authed)?; } + // settings.json carries the admin-managed integration config that `get_settings` + // is admin-only for (ai_config, the webhook URL, git_sync, handler extra_args), + // so it takes the same check. Not a per-field redaction: fields silently dropped + // from settings.json come back as null on the next `wmill sync push`. + if include_settings.unwrap_or(false) && !authed.is_admin { + return Err(Error::PermissionDenied( + "include_settings requires workspace admin".to_string(), + )); + } + // The route is gated by workspaces:read, but the tarball also carries the item // values that the per-item routes gate on their own domain (get_resource_value, // get_variable). A whole-workspace export cannot be confined to a path, so it @@ -1627,13 +1637,7 @@ pub(crate) async fn tarball_workspace( slack_name: row.slack_name.clone(), slack_command_script: row.slack_command_script.clone(), slack_oauth_client_id: row.slack_oauth_client_id.clone(), - // Mirror the non-admin redaction in `get_settings`: the OAuth - // client secret is admin-only and must not leak via tarball. - slack_oauth_client_secret: if authed.is_admin { - row.slack_oauth_client_secret.clone() - } else { - None - }, + slack_oauth_client_secret: row.slack_oauth_client_secret.clone(), }; serde_json::to_value(settings) .map(|v| serde_json::to_string_pretty(&v).ok()) diff --git a/backend/windmill-common/src/global_settings.rs b/backend/windmill-common/src/global_settings.rs index 37b4b60da4..b539c1c6b3 100644 --- a/backend/windmill-common/src/global_settings.rs +++ b/backend/windmill-common/src/global_settings.rs @@ -100,6 +100,8 @@ pub const CRITICAL_ERROR_CHANNELS_SETTING: &str = "critical_error_channels"; pub const CRITICAL_ALERT_MUTE_UI_SETTING: &str = "critical_alert_mute_ui"; pub const CRITICAL_ALERTS_ON_DB_OVERSIZE_SETTING: &str = "critical_alerts_on_db_oversize"; pub const CRITICAL_ALERTS_ON_TOKEN_EXPIRY_SETTING: &str = "critical_alerts_on_token_expiry"; +pub const CRITICAL_ALERT_MUTE_ZOMBIE_JOB_RESTART_SETTING: &str = + "critical_alert_mute_zombie_job_restart"; pub const DEV_INSTANCE_SETTING: &str = "dev_instance"; pub const JWT_SECRET_SETTING: &str = "jwt_secret"; pub const EMAIL_DOMAIN_SETTING: &str = "email_domain"; diff --git a/backend/windmill-common/src/lib.rs b/backend/windmill-common/src/lib.rs index e7cb368d25..89db0ee769 100644 --- a/backend/windmill-common/src/lib.rs +++ b/backend/windmill-common/src/lib.rs @@ -354,6 +354,7 @@ lazy_static::lazy_static! { pub static ref CRITICAL_ALERT_MUTE_UI_ENABLED: AtomicBool = AtomicBool::new(false); pub static ref CRITICAL_ALERTS_ON_TOKEN_EXPIRY: AtomicBool = AtomicBool::new(false); + pub static ref CRITICAL_ALERT_MUTE_ZOMBIE_JOB_RESTART: AtomicBool = AtomicBool::new(false); pub static ref BASE_URL: arc_swap::ArcSwap = arc_swap::ArcSwap::from_pointee("".to_string()); pub static ref IS_READY: std::sync::atomic::AtomicBool = std::sync::atomic::AtomicBool::new(false); diff --git a/backend/windmill-common/tests/sqlx_begin_cancel_safe.rs b/backend/windmill-common/tests/sqlx_begin_cancel_safe.rs new file mode 100644 index 0000000000..1f387984b0 --- /dev/null +++ b/backend/windmill-common/tests/sqlx_begin_cancel_safe.rs @@ -0,0 +1,72 @@ +//! Guards the `sqlx` entries in `[patch.crates-io]` — `backend/Cargo.toml` carries the why. +//! Dropping the patch still compiles, so a test is what notices. +//! +//! Ignored by default: it only has something to say when the sqlx dependency moves, and it +//! spends a couple of seconds waiting on a deliberately slow round trip. Run it whenever you +//! touch sqlx — a version bump, a change to the patch entries, a fork rebase: +//! +//! ```text +//! cargo test -p windmill-common --test sqlx_begin_cancel_safe -- --ignored +//! ``` + +use sqlx::{Connection, PgConnection, Pool, Postgres}; +use std::time::{Duration, Instant}; + +#[sqlx::test] +#[ignore = "run with --ignored after any sqlx bump or change to [patch.crates-io]"] +async fn begin_cancelled_mid_round_trip_leaves_no_open_transaction(db: Pool) { + // One connection, so the session inspected below is the one the cancelled begin used. + let pool = sqlx::postgres::PgPoolOptions::new() + .max_connections(1) + .min_connections(0) + .connect_with((*db.connect_options()).clone()) + .await + .expect("failed to build pool"); + let pid: i32 = sqlx::query_scalar("SELECT pg_backend_pid()") + .fetch_one(&pool) + .await + .unwrap(); + + // A plain `BEGIN` answers in well under a millisecond, which is too narrow to cancel + // reliably; appending a sleep widens the round trip and runs through the same + // `PgTransactionManager::begin` the patch fixes. + let cancelled = tokio::time::timeout( + Duration::from_millis(300), + pool.begin_with("BEGIN; SELECT pg_sleep(2);"), + ) + .await; + assert!(cancelled.is_err(), "the begin must not have completed"); + + let mut admin = PgConnection::connect_with(&(*db.connect_options()).clone()) + .await + .expect("failed to open an observing connection"); + + // sqlx only flushes the queued ROLLBACK once the abandoned statement has answered, so + // wait for the session to stop running rather than sleeping a fixed time a loaded runner + // could overshoot. + let deadline = Instant::now() + Duration::from_secs(30); + let state = loop { + let state: String = sqlx::query_scalar("SELECT state FROM pg_stat_activity WHERE pid = $1") + .bind(pid) + .fetch_optional(&mut admin) + .await + .unwrap() + .flatten() + .unwrap_or_default(); + if state != "active" || Instant::now() >= deadline { + break state; + } + tokio::time::sleep(Duration::from_millis(100)).await; + }; + + assert!( + !state.starts_with("idle in transaction"), + "connection returned to the pool still inside a transaction (state {state:?}) — is \ + the sqlx patch in backend/Cargo.toml still applied?" + ); + + sqlx::query_scalar::<_, i32>("SELECT 1") + .fetch_one(&pool) + .await + .expect("pool must still serve queries"); +} diff --git a/backend/windmill-store/src/resources.rs b/backend/windmill-store/src/resources.rs index 1a051db8fe..b107d34be4 100644 --- a/backend/windmill-store/src/resources.rs +++ b/backend/windmill-store/src/resources.rs @@ -81,11 +81,8 @@ pub fn workspaced_service() -> Router { "/history/p/{*path}", get(get_resource_history).delete(clear_resource_history), ) - .route("/history/v/{version}", get(get_resource_version)) - .route( - "/history/restore/v/{version}", - post(restore_resource_version), - ) + .route("/history/v/{id}", get(get_resource_version)) + .route("/history/restore/v/{id}", post(restore_resource_version)) .route("/delete/{*path}", delete(delete_resource)) .route("/delete_bulk", delete(delete_resources_bulk)) .route("/create", post(create_resource)) @@ -2200,7 +2197,9 @@ async fn set_resource_value( #[derive(Serialize)] struct ResourceVersion { + /// Addresses a version; `version` is the per-resource number it is presented by. id: i64, + version: i64, created_at: chrono::DateTime, created_by: Option, } @@ -2208,6 +2207,7 @@ struct ResourceVersion { #[derive(Serialize)] struct ResourceVersionWithValue { id: i64, + version: i64, created_at: chrono::DateTime, created_by: Option, value: Option, @@ -2243,7 +2243,7 @@ async fn get_resource_history( let versions = sqlx::query_as!( ResourceVersion, - "SELECT id, created_at, created_by FROM resource_version + "SELECT id, version, created_at, created_by FROM resource_version WHERE workspace_id = $1 AND path = $2 ORDER BY id DESC LIMIT $3", w_id, path, @@ -2357,19 +2357,19 @@ async fn missing_references( async fn get_resource_version( authed: ApiAuthed, Extension(user_db): Extension, - Path((w_id, version)): Path<(String, i64)>, + Path((w_id, id)): Path<(String, i64)>, ) -> JsonResult { let mut tx = user_db.begin(&authed).await?; let row = sqlx::query!( - "SELECT id, path, created_at, created_by, value FROM resource_version + "SELECT id, version, path, created_at, created_by, value FROM resource_version WHERE workspace_id = $1 AND id = $2", w_id, - version + id ) .fetch_optional(&mut *tx) .await?; - let row = not_found_if_none(row, "ResourceVersion", version.to_string())?; + let row = not_found_if_none(row, "ResourceVersion", id.to_string())?; check_scopes(&authed, || format!("resources:read:{}", row.path))?; let missing = missing_references(&mut tx, &w_id, row.value.as_ref()).await?; @@ -2377,6 +2377,7 @@ async fn get_resource_version( Ok(Json(ResourceVersionWithValue { id: row.id, + version: row.version, created_at: row.created_at, created_by: row.created_by, value: row.value, @@ -2455,17 +2456,17 @@ async fn restore_resource_version( Extension(db): Extension, Extension(user_db): Extension, Extension(webhook): Extension, - Path((w_id, version)): Path<(String, i64)>, + Path((w_id, id)): Path<(String, i64)>, ) -> Result { let mut tx = user_db.clone().begin(&authed).await?; let row = sqlx::query!( - "SELECT path, value FROM resource_version WHERE workspace_id = $1 AND id = $2", + "SELECT path, value, version FROM resource_version WHERE workspace_id = $1 AND id = $2", w_id, - version + id ) .fetch_optional(&mut *tx) .await?; - let row = not_found_if_none(row, "ResourceVersion", version.to_string())?; + let row = not_found_if_none(row, "ResourceVersion", id.to_string())?; tx.commit().await?; check_scopes(&authed, || format!("resources:write:{}", row.path))?; @@ -2486,7 +2487,7 @@ async fn restore_resource_version( Ok(format!( "resource {} restored to version {}", - row.path, version + row.path, row.version )) } diff --git a/backend/windmill-trigger-http/src/lib.rs b/backend/windmill-trigger-http/src/lib.rs index f89247dd16..b78276c28d 100644 --- a/backend/windmill-trigger-http/src/lib.rs +++ b/backend/windmill-trigger-http/src/lib.rs @@ -1,4 +1,5 @@ use std::collections::HashMap; +use std::sync::atomic::{AtomicU64, Ordering}; use quick_cache::sync::Cache; use serde::{Deserialize, Serialize}; @@ -27,9 +28,12 @@ lazy_static::lazy_static! { pub static ref HTTP_ROUTERS_CACHE: RwLock = RwLock::new(RoutersCache { routers: HashMap::new(), version: 0, + invalidations: 0, }); } +static HTTP_ROUTERS_INVALIDATIONS: AtomicU64 = AtomicU64::new(0); + #[derive(Debug, Deserialize, Clone)] pub struct TriggerRoute { pub path: String, @@ -56,6 +60,10 @@ pub struct TriggerRoute { pub struct RoutersCache { pub routers: HashMap>, pub version: i64, + /// `HTTP_ROUTERS_INVALIDATIONS` as of the moment these rows were read. A rebuild that + /// started before an invalidation publishes a count behind the current one, which is what + /// stops it from passing its own stale rows off as covering that invalidation. + invalidations: u64, } #[derive(Serialize, Deserialize, sqlx::Type, Debug, Clone, Copy, Hash, Eq, PartialEq)] @@ -223,12 +231,24 @@ pub fn validate_authentication_method( } } -pub async fn refresh_routers(db: &DB) -> Result<(bool, RwLockReadGuard<'_, RoutersCache>)> { +/// `force` rebuilds unconditionally. `nextval` on `http_trigger_version_seq` runs inside the +/// writing transaction and sequences are non-transactional, so another session can cache the +/// bumped version against still-uncommitted rows, after which every version-gated refresh is a +/// no-op. Force when reacting to a bump that could have been observed before its own rows were. +pub async fn refresh_routers( + db: &DB, + force: bool, +) -> Result<(bool, RwLockReadGuard<'_, RoutersCache>)> { + let invalidations = HTTP_ROUTERS_INVALIDATIONS.load(Ordering::Relaxed); let version = sqlx::query_scalar!("SELECT last_value FROM http_trigger_version_seq",) .fetch_one(db) .await?; let routers_cache = HTTP_ROUTERS_CACHE.read().await; - if routers_cache.version == 0 || version > routers_cache.version { + if force + || routers_cache.version == 0 + || version > routers_cache.version + || invalidations != routers_cache.invalidations + { drop(routers_cache); let mut routers = HashMap::new(); @@ -274,7 +294,8 @@ pub async fn refresh_routers(db: &DB) -> Result<(bool, RwLockReadGuard<'_, Route .await?; let mut router = matchit::Router::new(); - let http_route_workspaced = HTTP_ROUTE_WORKSPACED_ROUTE.load(std::sync::atomic::Ordering::Relaxed); + let http_route_workspaced = + HTTP_ROUTE_WORKSPACED_ROUTE.load(std::sync::atomic::Ordering::Relaxed); for trigger in triggers { let full_path = @@ -306,7 +327,7 @@ pub async fn refresh_routers(db: &DB) -> Result<(bool, RwLockReadGuard<'_, Route } let mut routers_cache = HTTP_ROUTERS_CACHE.write().await; - *routers_cache = RoutersCache { routers, version }; + *routers_cache = RoutersCache { routers, version, invalidations }; Ok((true, routers_cache.downgrade())) } else { @@ -315,11 +336,19 @@ pub async fn refresh_routers(db: &DB) -> Result<(bool, RwLockReadGuard<'_, Route } } +/// Record that the cache no longer covers everything committed, so the next refresh rebuilds +/// whatever the version says. The routes already loaded keep being served in the meantime. Use +/// after a forced refresh fails: its change is inside the cached version, so nothing else would +/// retry it. +pub fn invalidate_routers() { + HTTP_ROUTERS_INVALIDATIONS.fetch_add(1, Ordering::Relaxed); +} + pub async fn refresh_routers_loop( db: &DB, mut killpill_rx: tokio::sync::broadcast::Receiver<()>, ) -> () { - match refresh_routers(db).await { + match refresh_routers(db, false).await { Ok(_) => { tracing::info!("Loaded HTTP routers"); } @@ -335,7 +364,7 @@ pub async fn refresh_routers_loop( break; } _ = tokio::time::sleep(std::time::Duration::from_secs(60)) => { - match refresh_routers(&db).await { + match refresh_routers(&db, false).await { Ok((true, _)) => { tracing::info!("Refreshed HTTP routers"); } diff --git a/backend/windmill-trigger-http/tests/refresh_routers.rs b/backend/windmill-trigger-http/tests/refresh_routers.rs new file mode 100644 index 0000000000..f47da2ee0e --- /dev/null +++ b/backend/windmill-trigger-http/tests/refresh_routers.rs @@ -0,0 +1,62 @@ +use sqlx::{Pool, Postgres}; +use windmill_trigger_http::{invalidate_routers, refresh_routers, HttpMethod, RoutersCache}; + +async fn insert_trigger(db: &Pool, path: &str, route_path: &str) { + sqlx::query( + "INSERT INTO http_trigger ( + path, route_path, route_path_key, script_path, is_flow, workspace_id, edited_by, + permissioned_as, http_method, authentication_method, request_type, is_static_website, + workspaced_route, wrap_body, raw_string, mode + ) VALUES ($1, $2, $2, 'f/test/handler', false, 'test-workspace', 'test-user', + 'u/test-user', 'get', 'none', 'async', false, false, false, false, 'enabled')", + ) + .bind(path) + .bind(route_path) + .execute(db) + .await + .expect("insert http_trigger"); +} + +fn routes(cache: &RoutersCache, path: &str) -> bool { + cache.routers[&HttpMethod::Get].at(path).is_ok() +} + +// A trigger row can commit without advancing http_trigger_version_seq past what the cache +// already holds, because `nextval` runs ahead of the commit it belongs to. The version gate +// cannot see such a row; only forcing, or an invalidation, recovers the route. +#[sqlx::test(migrations = "../migrations")] +async fn rebuilds_a_change_the_cached_version_does_not_cover(db: Pool) { + insert_trigger(&db, "f/test/first", "first").await; + let (rebuilt, cache) = refresh_routers(&db, false).await.unwrap(); + assert!(rebuilt); + assert!(routes(&cache, "/first")); + drop(cache); + + insert_trigger(&db, "f/test/second", "second").await; + + let (rebuilt, cache) = refresh_routers(&db, false).await.unwrap(); + assert!(!rebuilt, "an unchanged version must not rebuild"); + assert!(!routes(&cache, "/second")); + drop(cache); + + let (rebuilt, cache) = refresh_routers(&db, true).await.unwrap(); + assert!(rebuilt, "force must rebuild whatever the version says"); + assert!(routes(&cache, "/second")); + drop(cache); + + // A forced refresh that failed leaves its change inside the cached version, so the periodic + // version-gated refresh has to rebuild on the invalidation alone. + insert_trigger(&db, "f/test/third", "third").await; + invalidate_routers(); + + let (rebuilt, cache) = refresh_routers(&db, false).await.unwrap(); + assert!( + rebuilt, + "an invalidation must rebuild through the version gate" + ); + assert!(routes(&cache, "/third")); + drop(cache); + + let (rebuilt, _) = refresh_routers(&db, false).await.unwrap(); + assert!(!rebuilt, "a served invalidation must not rebuild forever"); +} diff --git a/backend/windmill-worker/src/ai_executor.rs b/backend/windmill-worker/src/ai_executor.rs index b11ecebc6c..d948fee088 100644 --- a/backend/windmill-worker/src/ai_executor.rs +++ b/backend/windmill-worker/src/ai_executor.rs @@ -451,9 +451,9 @@ pub async fn handle_ai_agent_job( (args, tools) } else { let args = serde_json::from_str::(&serde_json::to_string(&local_args)?)?; - // "Edit" on a linked step clears `agent` but keeps the host's `tool_inputs` until Save or - // Cancel folds them back, so overlay them here too: a flow persisted mid-edit must still - // bind its tools to this flow's context rather than the agent author's. + // "Edit" on a linked step clears `agent` but keeps the host's `tool_inputs`, so overlay them + // here too: a flow persisted mid-edit must still bind its tools to this flow's context + // rather than the agent author's. let mut tools = module_tools; overlay_tool_inputs(&mut tools, &tool_inputs); (args, tools) diff --git a/benchmarks/lib.ts b/benchmarks/lib.ts index 7b78ff0fa8..55734e52e8 100644 --- a/benchmarks/lib.ts +++ b/benchmarks/lib.ts @@ -2,7 +2,7 @@ import { sleep } from "https://deno.land/x/sleep@v1.2.1/mod.ts"; import * as windmill from "https://deno.land/x/windmill@v1.174.0/mod.ts"; import * as api from "https://deno.land/x/windmill@v1.174.0/windmill-api/index.ts"; -export const VERSION = "v1.795.0"; +export const VERSION = "v1.796.0"; export async function login(email: string, password: string): Promise { return await windmill.UserService.login({ diff --git a/cli/src/core/constants.ts b/cli/src/core/constants.ts index 64b77a690d..2f80eb41fc 100644 --- a/cli/src/core/constants.ts +++ b/cli/src/core/constants.ts @@ -10,4 +10,4 @@ export const WM_FORK_PREFIX = "wm-fork"; // (e.g. utils.ts) can read it without importing main.ts and creating a circular // dependency (main → workspace → utils → main) that triggers a TDZ. // Re-exported from main.ts for backwards compatibility. -export const VERSION = "1.795.0"; +export const VERSION = "1.796.0"; diff --git a/cli/src/guidance/skills.gen.ts b/cli/src/guidance/skills.gen.ts index 9a6c75e328..a263401764 100644 --- a/cli/src/guidance/skills.gen.ts +++ b/cli/src/guidance/skills.gen.ts @@ -848,31 +848,35 @@ async deleteS3File(s3object: S3Object, workspace: string | undefined = undefined /** * Sign S3 objects to be used by anonymous users in public apps * @param s3objects s3 objects to sign + * @param expirySecs how long the signature stays valid, in seconds (default 43200 = 12h, clamped to [60, 604800]) * @returns signed s3 objects */ -async signS3Objects(s3objects: S3Object[]): Promise +async signS3Objects(s3objects: S3Object[], { expirySecs }: { expirySecs?: number } = {}): Promise /** * Sign S3 object to be used by anonymous users in public apps * @param s3object s3 object to sign + * @param expirySecs how long the signature stays valid, in seconds (default 43200 = 12h, clamped to [60, 604800]) * @returns signed s3 object */ -async signS3Object(s3object: S3Object): Promise +async signS3Object(s3object: S3Object, { expirySecs }: { expirySecs?: number } = {}): Promise /** * Generate a presigned public URL for an array of S3 objects. * If an S3 object is not signed yet, it will be signed first. * @param s3Objects s3 objects to sign + * @param expirySecs how long the signature stays valid, in seconds (default 43200 = 12h, clamped to [60, 604800]) * @returns list of signed public URLs */ -async getPresignedS3PublicUrls(s3Objects: S3Object[], { baseUrl }: { baseUrl?: string } = {}): Promise +async getPresignedS3PublicUrls(s3Objects: S3Object[], { baseUrl, expirySecs }: { baseUrl?: string; expirySecs?: number } = {}): Promise /** * Generate a presigned public URL for an S3 object. If the S3 object is not signed yet, it will be signed first. * @param s3Object s3 object to sign + * @param expirySecs how long the signature stays valid, in seconds (default 43200 = 12h, clamped to [60, 604800]) * @returns signed public URL */ -async getPresignedS3PublicUrl(s3Objects: S3Object, { baseUrl }: { baseUrl?: string } = {}): Promise +async getPresignedS3PublicUrl(s3Objects: S3Object, { baseUrl, expirySecs }: { baseUrl?: string; expirySecs?: number } = {}): Promise /** * Get URLs needed for resuming a flow after this step @@ -1635,31 +1639,35 @@ async deleteS3File(s3object: S3Object, workspace: string | undefined = undefined /** * Sign S3 objects to be used by anonymous users in public apps * @param s3objects s3 objects to sign + * @param expirySecs how long the signature stays valid, in seconds (default 43200 = 12h, clamped to [60, 604800]) * @returns signed s3 objects */ -async signS3Objects(s3objects: S3Object[]): Promise +async signS3Objects(s3objects: S3Object[], { expirySecs }: { expirySecs?: number } = {}): Promise /** * Sign S3 object to be used by anonymous users in public apps * @param s3object s3 object to sign + * @param expirySecs how long the signature stays valid, in seconds (default 43200 = 12h, clamped to [60, 604800]) * @returns signed s3 object */ -async signS3Object(s3object: S3Object): Promise +async signS3Object(s3object: S3Object, { expirySecs }: { expirySecs?: number } = {}): Promise /** * Generate a presigned public URL for an array of S3 objects. * If an S3 object is not signed yet, it will be signed first. * @param s3Objects s3 objects to sign + * @param expirySecs how long the signature stays valid, in seconds (default 43200 = 12h, clamped to [60, 604800]) * @returns list of signed public URLs */ -async getPresignedS3PublicUrls(s3Objects: S3Object[], { baseUrl }: { baseUrl?: string } = {}): Promise +async getPresignedS3PublicUrls(s3Objects: S3Object[], { baseUrl, expirySecs }: { baseUrl?: string; expirySecs?: number } = {}): Promise /** * Generate a presigned public URL for an S3 object. If the S3 object is not signed yet, it will be signed first. * @param s3Object s3 object to sign + * @param expirySecs how long the signature stays valid, in seconds (default 43200 = 12h, clamped to [60, 604800]) * @returns signed public URL */ -async getPresignedS3PublicUrl(s3Objects: S3Object, { baseUrl }: { baseUrl?: string } = {}): Promise +async getPresignedS3PublicUrl(s3Objects: S3Object, { baseUrl, expirySecs }: { baseUrl?: string; expirySecs?: number } = {}): Promise /** * Get URLs needed for resuming a flow after this step @@ -2516,31 +2524,35 @@ async deleteS3File(s3object: S3Object, workspace: string | undefined = undefined /** * Sign S3 objects to be used by anonymous users in public apps * @param s3objects s3 objects to sign + * @param expirySecs how long the signature stays valid, in seconds (default 43200 = 12h, clamped to [60, 604800]) * @returns signed s3 objects */ -async signS3Objects(s3objects: S3Object[]): Promise +async signS3Objects(s3objects: S3Object[], { expirySecs }: { expirySecs?: number } = {}): Promise /** * Sign S3 object to be used by anonymous users in public apps * @param s3object s3 object to sign + * @param expirySecs how long the signature stays valid, in seconds (default 43200 = 12h, clamped to [60, 604800]) * @returns signed s3 object */ -async signS3Object(s3object: S3Object): Promise +async signS3Object(s3object: S3Object, { expirySecs }: { expirySecs?: number } = {}): Promise /** * Generate a presigned public URL for an array of S3 objects. * If an S3 object is not signed yet, it will be signed first. * @param s3Objects s3 objects to sign + * @param expirySecs how long the signature stays valid, in seconds (default 43200 = 12h, clamped to [60, 604800]) * @returns list of signed public URLs */ -async getPresignedS3PublicUrls(s3Objects: S3Object[], { baseUrl }: { baseUrl?: string } = {}): Promise +async getPresignedS3PublicUrls(s3Objects: S3Object[], { baseUrl, expirySecs }: { baseUrl?: string; expirySecs?: number } = {}): Promise /** * Generate a presigned public URL for an S3 object. If the S3 object is not signed yet, it will be signed first. * @param s3Object s3 object to sign + * @param expirySecs how long the signature stays valid, in seconds (default 43200 = 12h, clamped to [60, 604800]) * @returns signed public URL */ -async getPresignedS3PublicUrl(s3Objects: S3Object, { baseUrl }: { baseUrl?: string } = {}): Promise +async getPresignedS3PublicUrl(s3Objects: S3Object, { baseUrl, expirySecs }: { baseUrl?: string; expirySecs?: number } = {}): Promise /** * Get URLs needed for resuming a flow after this step @@ -4237,19 +4249,23 @@ def delete_s3_object(s3object: S3Object | str, s3_resource_path: str | None = No # # Args: # s3_objects: List of S3 objects to sign +# expiry_secs: How long the signature stays valid, in seconds +# (defaults to 43200 = 12h, clamped to [60, 604800]) # # Returns: # List of signed S3 objects -def sign_s3_objects(s3_objects: list[S3Object | str]) -> list[S3Object] +def sign_s3_objects(s3_objects: list[S3Object | str], expiry_secs: int | None = None) -> list[S3Object] # Sign a single S3 object for use by anonymous users in public apps. # # Args: # s3_object: S3 object to sign +# expiry_secs: How long the signature stays valid, in seconds +# (defaults to 43200 = 12h, clamped to [60, 604800]) # # Returns: # Signed S3 object -def sign_s3_object(s3_object: S3Object | str) -> S3Object +def sign_s3_object(s3_object: S3Object | str, expiry_secs: int | None = None) -> S3Object # Generate presigned public URLs for an array of S3 objects. # If an S3 object is not signed yet, it will be signed first. @@ -4257,6 +4273,8 @@ def sign_s3_object(s3_object: S3Object | str) -> S3Object # Args: # s3_objects: List of S3 objects to sign # base_url: Optional base URL for the presigned URLs (defaults to WM_BASE_URL) +# expiry_secs: How long the signatures stay valid, in seconds +# (defaults to 43200 = 12h, clamped to [60, 604800]) # # Returns: # List of signed public URLs @@ -4264,7 +4282,7 @@ def sign_s3_object(s3_object: S3Object | str) -> S3Object # Example: # >>> s3_objs = [S3Object(s3="/path/to/file1.txt"), S3Object(s3="/path/to/file2.txt")] # >>> urls = client.get_presigned_s3_public_urls(s3_objs) -def get_presigned_s3_public_urls(s3_objects: list[S3Object | str], base_url: str | None = None) -> list[str] +def get_presigned_s3_public_urls(s3_objects: list[S3Object | str], base_url: str | None = None, expiry_secs: int | None = None) -> list[str] # Generate a presigned public URL for an S3 object. # If the S3 object is not signed yet, it will be signed first. @@ -4272,6 +4290,8 @@ def get_presigned_s3_public_urls(s3_objects: list[S3Object | str], base_url: str # Args: # s3_object: S3 object to sign # base_url: Optional base URL for the presigned URL (defaults to WM_BASE_URL) +# expiry_secs: How long the signature stays valid, in seconds +# (defaults to 43200 = 12h, clamped to [60, 604800]) # # Returns: # Signed public URL @@ -4279,7 +4299,7 @@ def get_presigned_s3_public_urls(s3_objects: list[S3Object | str], base_url: str # Example: # >>> s3_obj = S3Object(s3="/path/to/file.txt") # >>> url = client.get_presigned_s3_public_url(s3_obj) -def get_presigned_s3_public_url(s3_object: S3Object | str, base_url: str | None = None) -> str +def get_presigned_s3_public_url(s3_object: S3Object | str, base_url: str | None = None, expiry_secs: int | None = None) -> str # Get the current user information. # diff --git a/debugger/dap_websocket_server_bun.ts b/debugger/dap_websocket_server_bun.ts index 3b5072b1e6..d1e0ea1f14 100644 --- a/debugger/dap_websocket_server_bun.ts +++ b/debugger/dap_websocket_server_bun.ts @@ -1651,8 +1651,17 @@ export class DebugSession { try { await this.startBunProcess(cwd) } catch (error) { - this.sendEvent('output', { category: 'stderr', output: `Failed to start Bun: ${error}\n` }) - this.sendEvent('terminated', { error: String(error) }) + // A launch failure is reported here, a finished script from onclose; whichever gets + // there first owns the terminated event, so a client that already has a result is + // never told afterwards that the launch failed. + if (!this.terminatedSent) { + this.terminatedSent = true + this.sendEvent('output', { category: 'stderr', output: `Failed to start Bun: ${error}\n` }) + this.sendEvent('terminated', { error: String(error) }) + } + // --inspect-wait blocks until a debugger attaches, so a bun we failed to attach to + // waits forever unless it is reaped here. + await this.cleanup() } } @@ -1956,9 +1965,12 @@ export class DebugSession { const text = decoder.decode(value) buffer += text - // Look for the WebSocket URL in Bun's inspector output - // Format: "ws://127.0.0.1:9229/xxxxx" - const wsMatch = buffer.match(/ws:\/\/[\d.]+:\d+\/[a-z0-9]+/i) + // Look for the WebSocket URL in Bun's inspector banner, e.g. + // " ws://127.0.0.1:9229/848c719d-a52e-4610-8e94-99cd60f34af9". + // The token's alphabet is Bun's to change (it became a hyphenated UUID in 1.4), so + // take the whole path, and only once whitespace proves it complete: a stderr chunk + // can end mid-URL, and connecting to a truncated path gets a 404 from the inspector. + const wsMatch = buffer.match(/ws:\/\/[\d.]+:\d+\/\S+(?=\s)/) if (wsMatch && this.inspectorWsUrlPromise) { const wsUrl = wsMatch[0] logger.info(`Found inspector WebSocket URL in stderr: ${wsUrl}`) @@ -1988,12 +2000,22 @@ export class DebugSession { return new Promise((resolve, reject) => { this.inspectorWs = new WebSocket(wsUrl) + // A close before the script is running is a failed connection, not a finished script, + // and the two are reported to the client in opposite ways. The socket opening is not + // the line: the setup commands below run over an open socket and none of them reject + // when it drops (sendInspectorCommand only has its own timer), so a drop mid-setup + // would otherwise be indistinguishable from a clean exit. + let opened = false + let executionStarted = false + let handshakeError: string | null = null + const timeout = setTimeout(() => { reject(new Error('Inspector connection timeout')) }, 5000) this.inspectorWs.onopen = async () => { clearTimeout(timeout) + opened = true logger.info('Connected to inspector') try { @@ -2030,6 +2052,11 @@ export class DebugSession { logger.info('Starting script execution with Inspector.initialized...') await this.sendInspectorCommand('Inspector.initialized', {}) + // Only past its reply is a later close a finished script rather than a lost + // connection. The reply precedes any close on this socket, so the continuation + // runs first and a real run is never misread as a failure. + executionStarted = true + resolve() } catch (error) { reject(error) @@ -2042,12 +2069,25 @@ export class DebugSession { this.inspectorWs.onerror = (error) => { logger.error('Inspector WebSocket error:', error) + if (!opened) { + handshakeError = (error as ErrorEvent)?.message || String(error) + } } this.inspectorWs.onclose = () => { logger.info('Inspector WebSocket closed') this.inspectorWs = null + if (!executionStarted) { + clearTimeout(timeout) + reject( + new Error( + `Inspector connection failed: ${handshakeError ?? (opened ? 'closed before setup completed' : 'closed before the handshake completed')}` + ) + ) + return + } + // When inspector closes, the script has ended - send terminated event if (!this.terminatedSent) { this.terminatedSent = true diff --git a/docker/test_windmill_extra.ts b/docker/test_windmill_extra.ts index 46ffce54c9..5fd1a74108 100644 --- a/docker/test_windmill_extra.ts +++ b/docker/test_windmill_extra.ts @@ -206,7 +206,11 @@ class DAPTestClient { private events: DAPMessage[] = [] private output: string[] = [] private result: unknown = undefined - private eventHandlers = new Map void)[]>() + private eventWaiters = new Map void)[]>() + // An event that arrives before its waiter is registered is queued rather than dropped: the + // server sends 'initialized' right behind the 'initialize' response, and 'terminated' can + // land before the launch call the test awaits has even returned. + private bufferedEvents = new Map() async connect(endpoint: string): Promise { const url = `ws://${HOST}:${DEBUGGER_PORT}${endpoint}` @@ -273,9 +277,13 @@ class DAPTestClient { this.result = msg.body.result } - const handlers = this.eventHandlers.get(msg.event!) || [] - for (const handler of handlers) { - handler(msg) + const waiters = this.eventWaiters.get(msg.event!) + if (waiters && waiters.length > 0) { + waiters.shift()!(msg) + } else { + const buffered = this.bufferedEvents.get(msg.event!) || [] + buffered.push(msg) + this.bufferedEvents.set(msg.event!, buffered) } } } catch { @@ -313,23 +321,27 @@ class DAPTestClient { } waitForEvent(eventName: string, timeout = 10000): Promise { + const buffered = this.bufferedEvents.get(eventName) + if (buffered && buffered.length > 0) { + return Promise.resolve(buffered.shift()!) + } + return new Promise((resolve, reject) => { + const waiters = this.eventWaiters.get(eventName) || [] + this.eventWaiters.set(eventName, waiters) + const timer = setTimeout(() => { + const idx = waiters.indexOf(handler) + if (idx >= 0) waiters.splice(idx, 1) reject(new Error(`Timeout waiting for event: ${eventName}`)) }, timeout) const handler = (event: DAPMessage) => { clearTimeout(timer) - const handlers = this.eventHandlers.get(eventName) || [] - const idx = handlers.indexOf(handler) - if (idx >= 0) handlers.splice(idx, 1) resolve(event) } - if (!this.eventHandlers.has(eventName)) { - this.eventHandlers.set(eventName, []) - } - this.eventHandlers.get(eventName)!.push(handler) + waiters.push(handler) }) } @@ -379,6 +391,7 @@ class DAPTestClient { this.output = [] this.result = undefined this.events = [] + this.bufferedEvents.clear() } } diff --git a/docs/ai-agent-evals.md b/docs/ai-agent-evals.md new file mode 100644 index 0000000000..53d6ccf272 --- /dev/null +++ b/docs/ai-agent-evals.md @@ -0,0 +1,482 @@ +# AI agent evals + +A reusable AI agent (`docs/reusable-ai-agents.md`) can be run on its own, against a curated set +of **cases** — the inputs it is expected to keep handling. + +Three words, and no fourth: + +- a **case** is one input the agent should handle, held in a **dataset**; +- a **run** (stored as an **experiment**) is one execution of a whole dataset: a single flow job + that answers every case, which is what the UI labels "Run N"; +- each case is answered as one **iteration** of that run. + +The surface is a dialog, two screens deep: **this agent's runs** across every dataset it has been +measured on (one row per run, a badge per scorer), and, on opening a run, **one table** — a row per +case, a column per scorer, the cell being that scorer's verdict — with a case's detail beside it. +Editing a dataset is a drawer over both. It opens from where an agent already is: the agent card at +the top of an AI agent step's inputs in the flow editor, and the `ai_agent` row on `/resources`. + +**Evals belong to a saved agent.** A dataset and its runs hang off an `ai_agent` resource, so they +outlive the step being renamed, copied or deleted, and two runs are comparable because they name +the same thing. A step whose agent is written inline has nothing to hang them on: what stands in +its place is **Save as reusable agent**, which is the only setup evals ask for. + +## What runs + +**A run is one flow**: a loop over the dataset's cases, each iteration answering its case and then +scoring the answer. Pushed as a `RawFlow`, so the agent step is the same vehicle +`ModuleTest.svelte` uses to test an agent step and a case exercises the production branch of +`ai_executor.rs` rather than a parallel one. + +One flow rather than one job per case, because **a run outlives the tab that started it**: a +dataset of two hundred cases against a slow provider takes long enough that nobody watches it, and +only a worker can notice that the last case finished. Scoring is therefore a step, not something +the client does afterwards, and the run is one thing to watch, cancel, or point a schedule at. Each +iteration is read back by node id (`get_result_and_success_by_id_from_flow`), so an answer is +fetched without walking the loop's status. + +The loop is `parallel` with a bounded `parallelism` and `skip_failures`: a dataset is a burst of +calls to one provider, and one case failing is one cell of the run rather than the end of it. + +The cases are the loop's **static iterator**, so they live in the flow's value, which is stored +once. Passing them as an argument would put a copy of the whole dataset in every iteration's +arguments. + +Whichever state of the agent is chosen — what is deployed, the edits in progress, or a past version +— its configuration is fixed once, when the run is opened, and inlined into the step every case +runs. A linked step would resolve the resource when each case reaches it, so a deploy part-way +through a run would be executed by the cases after it while every row still named the version the +run started against. One run measures one configuration; the cost is that a run does not exercise +the linked branch the production step takes. The edits and a past version could not be run any +other way: a reference resolves to what is deployed, which is exactly what neither of them is. + +A saved agent's and a past version's configuration are never taken from the request: both are read +from the workspace by the path they name, and a subject carrying one is refused. The edits in +progress are the one kind the request has to carry — they exist only in the editor — and the run +records what it was handed, inlined into its flow and hashed, so it is reproducible and +attributable to "this version plus these edits"; what the server cannot assert about them is that +they derive from that version. + +Each iteration is an agent step, then a **payload step**, then one step per scorer. The payload +step exists because the flow cannot see what it needs to: the agent's own result carries the +answer and every message, but each tool call's arguments, result, status, duration and schema +belong to the job that ran it. The step reads them back through `GET /ai_evals/run_payload`, whose +one argument is the iteration's own job id, and hands the scorers exactly what a scorer receives +anywhere else. A scorer therefore measures the agent's latency and not its own: the payload +reports the *agent step's* duration, never the iteration's. + +The edits are the transforms as authored, expressions included. One that reads `results..x` +or a `flow_input` the case does not supply resolves to nothing here, the same way it would in any +run of that step outside its flow. + +A linked agent is not fully self-contained: a host flow can override its tools' inputs through +the step's `tool_inputs`. A run does not reproduce that wiring — the agent runs with its own +authored defaults — so an agent whose behaviour depends on one flow's overrides is measured here +without them. + +A case carries **no conversation**: one question and the answer it should produce, so a run starts +from the agent's own memory configuration and nothing is replayed into it. + +## Where results live + +Results are jobs. A run's logs, trajectory, tool-call child jobs, permissions and retention are +already `v2_job` / `v2_job_completed` and the flow status's `agent_actions`, and none of that is +stored a second time. + +What the table itself is made of is the exception: each cell's answer, its outcome and every +scorer's verdict are copied into the run's own rows the first time they can be read. Jobs have +their own retention, and a recorded run is meant to still read as the run it was long after the +jobs that produced it are gone. + +The pane shows the **answer** and nothing else of a job; the trajectory is the run page's, and +`job_id` is the way there. For a recorded row the answer is read off the row rather than out of +the job, because `job_id` is the whole iteration — the agent and then the scorers that measured it +— and its result is the last scorer's verdict, not the answer. + +What makes a job findable again is stamped on it at push: + +- `runnable_path` is the agent's own path, so the existing `script_path_start` job filter + answers "every run of this agent" with no new state. +- `_eval` in the flow's args records `{subject: {kind, path, version}, dataset, experiment_id}`, + and every iteration inherits it, so a job opened cold from the runs page explains itself. Which + case an iteration ran is in its own `iter.value`, which is also how a cell finds its job again. + Extra flow inputs are inert — the agent step reads only `user_message`/`user_attachments`. + +## Versioning + +`subject.version` is the agent's version number: how many times it has been saved. It is counted +per resource rather than read off `resource_version.id`, which is one identity sequence for the +whole table — an agent saved nine times reads v4 … v24 under it, and the gaps count writes in +workspaces the reader cannot see. The id stays how a version is addressed, by the history routes +and by restore; the number is what a version is called, and what runs are named and compared by. + +The number is stored on the row rather than counted when read, because both ways of deleting +versions take the oldest: the monitor's trim past `MAX_RESOURCE_VERSIONS`, and clearing a history +down to its current value. Counting the survivors would renumber under either, so a run recorded +against v3 would later name a different version. + +For an `agent` run the version names the configuration the run read when it opened, which is the +one every case executes. Pinning an *older* version is a subject kind of its own — an +`agent_version` run says which version to read, where an `agent` run reads whatever is deployed at +the moment it starts. + +A version captures the resource, not its transitive closure. Two byte-identical versions can +behave differently because a `$var:`/`$res:` they reference changed underneath them, so a +recorded version is necessary for attribution but not sufficient. + +## Experiments + +Running a dataset produces an experiment: every case executed against one subject, with a row per +case. The experiment records the **exact case set it ran**, by value — a dataset keeps changing, +and a result set that cannot say which inputs produced it is not reproducible. + +### What a run is called + +An experiment is `Run N`: `run_number` is allocated per `(dataset, agent path)` when the run is +opened, once, and never reused. Stored rather than counted at read time, so a run keeps the name it +was given as history is pruned around it. There is no user-given label. + +Numbering is per agent and not per subject kind, so runs of what is deployed and runs of the edits +on top of it share one sequence: "Run 7" means one thing, and which of the two ran it is what the +run says beside its number. + +### A run is permanent + +Every run is written once and then only ever read: there is no writable experiment, no partial +rerun, and no cell that can be edited after the fact. A run in which some cells came from one +version and some from another would not be worth comparing, and running the dataset is the only +way a run appears. + +- **One experiment holds one subject, and one agent keeps one history.** The experiment list is + filtered to the agent the pane was opened on, across both kinds, so a dataset shared by two + agents never shows one agent's runs when the other is opened. +- **A version is per cell** (`eval_experiment_case.subject_version`), not per experiment. The + subject is resolved once when the run is opened and every cell is stamped from it, so the column + is uniform today; it is per cell so that a run which one day executes cell by cell can say so + rather than averaging two versions silently. +- **Edits are dated by their hash**, not by a version, because editing moves nothing a version + could record. Each cell carries `subject_draft_hash`, the hash of the configuration it ran + (canonicalised: key order is not meaningful and `serde_json` preserves insertion order). +- **An agent's unsaved edits are their own subject.** Their runs are keyed under `agent_draft`, so + a number produced by edits is never quietly read as the deployed agent's. The run dialog offers + them only when it was opened from the editing card, preselected there; from anywhere else the + agent is what is deployed. + +The table asks what version the agent is on when it opens and whenever the tab regains focus — a +small `subject_state` read — rather than polling for it; the results endpoint reports the same +version but collects the run as it goes, so it is polled only while a run is in flight, one pass +at a time. An agent saved in another tab while this one stays focused is noticed on the next focus +or the next run. + +A run of an older version is history and says so (`Run 14 · v23` beside an agent on v24); nothing +flags it, since that would flag every past run the moment anything is deployed. A run whose edits +were later deployed is a run of that version, and the results endpoint recognises and restamps it +(see "What a run says it ran"). The hash each run carries is recorded, not shown. + +In the flow editor, editing a linked agent forks the configuration into the step and clears the +link; the step is the only copy of the edits until Save changes, Cancel or Discard +(`docs/reusable-ai-agents.md`). Evals open from the agent card in both of its states: from the +editing card they run the edits as the step holds them when Run is pressed; from the linked card +they run the deployed agent, the same reading a linked step makes at run time. The agent's own +resource draft — the one the resource editor writes — is never read by evals. The card also names +the version a run is recorded against (`v24`, and `v24` beside an unsaved-changes badge for edits +on top of it), read from the resource's newest history entry since the resource itself does not +carry its version. + +## Scoring + +Scoring is not a second act with a button of its own: **Run** produces an answer and then scores +it. Each iteration of the run's flow scores its own answer as a step, and the numbers are harvested +into rows when results are read. + +A run's cells are therefore measured by the scorers as they stood when it ran, and never again. +A scorer edited or added afterwards has no cell in the runs that predate it: rescoring a run in +place would make a permanent run editable. The one thing read through the present is the pass line +— `pass_if` is applied when a score is read, so moving it re-reads every run with no model call. + +The columns themselves are the dataset's current scorers, so the table stays comparable across the +runs it lists rather than growing a column per run. Removing a scorer therefore takes its column +off the runs already recorded as well: the rows it produced are not deleted, but nothing renders +them, and adding the scorer back mints a new column that fills from the next run on. The removal +asks first, and says that. + +Two things are deliberately absent. Rescoring stored answers under edited scorers would need a run +of its own that reuses a parent run's answers and is attributed to the version that produced them, +so it never reads as the agent having answered again. A result cache keyed on (agent configuration, +case, scorer definition) would assert the agent is deterministic, which it is not, so it has to be +an explicit choice with its own answer to what a run means when half of it was computed last week. + +### A score is a number, and optionally a line through it + +Every scorer returns a number **between 0 and 1** — both templates say so, and the mean and the +pass rate read it as a fraction; a scorer returning anything outside that range has its result +recorded as an error rather than counted, and a `pass_if` threshold is held to the same range. Pass +or fail is not a second kind of score: a column carries an optional `pass_if`, and a case scoring at +or above it counts as a pass. A boolean scorer is one that returns 0 or 1 with the line at 0.5. + +A column with a threshold reports a **pass rate** beside its mean and marks each cell; a column +without one is a plain number and is not dressed up as a verdict. + +The line is deliberately outside the score's **definition** hash: where it sits is an +interpretation of a score rather than part of producing it, so moving it re-reads every run already +recorded with nothing re-run. It is set when the column is added and changed later under **Scorer +settings** in the dataset drawer, which is also where the column is named. + +The name is this dataset's own name for the scorer, seeded from the summary given when it was +added. It is a copy, not a link: the script or judge agent keeps whatever it is called, so renaming +a column here does not rename anything a second dataset shows. Reading it live from the runnable +would cost a fetch per column and leave a column blank for anyone who cannot read what it points +at. + +### What a scorer receives + +An agent is judged on its behaviour, so the final answer is the smaller half of the evidence. Every +scorer — a judge prompt or a script — is handed the same `EvalRun`, built from the job the run +already stored: + +| field | from | +|---|---| +| `input`, `expected` | the case as the experiment recorded it | +| `output` | the agent step's own result | +| `tool_calls` | every message carrying an `agent_action`, in order, with the arguments, result, error and duration of the job that call ran | +| `tools` | the tools that were called, with the schema of the script version that ran | +| `metrics` | `steps`, `duration_ms`, and the provider's `usage` when it reported any | + +Tool results are truncated at 4 KiB with `truncated: true`, so a large one cannot swamp a judge's +context, and a check that reads a truncated result can say so rather than failing on the missing +tail. A tool whose schema could not be resolved carries `null`, and a scorer validating arguments +must treat that as unchecked rather than as a failure. There is no cost field: Windmill keeps no +provider price table — the script template takes a rate as an argument instead. + +### The kinds + +Two, and both are runnables: + +| kind | is | receives | +|---|---|---| +| `agent` | an `ai_agent` resource used as a judge | the run, rendered as a message | +| `script` | a workspace script | `run`, with `input`, `output` and `expected` also spelled out | + +Keeping every scorer a runnable is what makes columns comparable: each has a path, a version, and +code you can open. There is no third kind stored as configuration on the dataset — a judge's model +and grading prompt live on the agent resource, so editing a judge is editing that agent, and the +column is not something you edit at all. Editing a column is editing the runnable it points at, so +the dataset drawer opens it in place: a script in the script editor, a judge in the resource +editor. + +Adding a scorer chooses the kind before the form opens: a judge is created next to the dataset +from the model you pick and a grading prompt that starts at the default; a script is created from +the template and opened in the editor. Both are named by a summary of what they score, which +becomes the column header and, prefixed with the dataset, the path. + +A `reason` is worth returning: it is what the cell shows on hover, together with the per-assertion +`checks`, so a number that looks wrong can be read rather than re-derived from the trajectory. + +A scorer may return a bare number, a boolean, or `{score, reason, checks}`; a judge's answer arrives +under `output`, sometimes as a string holding one of those, and often as a markdown code fence +around it, which is still read. `comment` is read as `reason`, so a scorer written for another +platform keeps its rationale. Anything with no number in it is left empty rather than guessed at, +and means skip the empty ones — a missing score counted as zero would read as a regression. + +`{score: null}` is the one exception, and it means the scorer read the case and had nothing to +measure on it: a column asking whether sources were cited has no verdict on a case with nothing to +cite. The cell shows `n/a` and is left out of the column's mean and pass rate, which is not the +same as the scorer failing — that is an error, and the column reports it as one. Written out +rather than merely absent, since a scorer that returns nothing at all is a scorer that is broken. + +### What a run says it ran + +A run is `v15` when it ran the deployed agent and `v15 + edits` when it ran that version with +undeployed changes on top. + +An `agent_draft` run records the version it is an edit of, because "the draft" is not attributable +without saying which deployed state it is a draft of. It also stops being a draft by itself: the +agent is hashed as deployed, in the same shape a draft is hashed in, so a run whose configuration +was later saved is recognised as the version it became. Edit, run, deploy, and the run you made +reads as `v16` rather than staying an edit of `v15` forever. + +That recognition is **written, not derived**. When the hashes match, the run's subject is rewritten +to `agent` at that version, once, keeping the hash it is founded on. Deriving it on every read +would make the answer expire: it would only ever mean "this ran what is deployed right now", so the +next deployment would send a run that already read `v16` back to `v15 + edits`. The write goes to +the unrestricted pool alongside the scores harvested in the same read, and nothing in it comes from +the caller — the hash is the proof, and a run of a configuration that was never deployed simply +stays an edit. + +It follows that the resolution needs someone to look: a run is stamped by the first results read +after its configuration is deployed. A run whose configuration was deployed and then replaced +without anyone opening the table keeps saying `+ edits`, which is the honest answer when the only +evidence is a hash that matches nothing deployed. + +### Reusing a scorer + +The add form lists the scorers this workspace already uses, most recently edited dataset first, +read out of the datasets' own `scorers` rather than stored anywhere new. It is filtered twice, +both times by what the caller can read: the datasets are read through `user_db`, so a scorer only +appears if the dataset carrying it does; then the runnables themselves are checked the same way, so +a script or agent the caller cannot open is never suggested. + +### A scorer is a column + +A scorer is stored on the dataset as `{id, name?, pass_if?, kind, path}`, with the `id` assigned +once and never reused: on a write, an incoming id is kept only when it names a column the dataset +already holds, and anything else is minted, so a column that was removed cannot come back under +its old id and inherit the scores recorded against it. That id is what makes a column the same +column across experiments when the scorer is renamed or its definition edited, and a delta is only +ever computed between two scores carrying the same id. Two scorers pointing at the same script are +two columns. + +A score is keyed `(experiment_id, ordinal, scorer_id)`, not baked into the experiment, so a frozen +experiment can gain a score without becoming mutable in any way that matters: what is frozen is +which runs are in it. + +Each score also records the **definition** that produced it — the kind, the path, and the script +hash or resource version that actually ran, so a path alone cannot hide an edit. When two scores +of one column carry different definitions the delta is still shown, marked: hiding the number +would force model calls just to see anything, and showing it unmarked would let a change of judge +read as a change of agent. + +### The surface + +Opening evals selects the dataset this agent was last worked in, remembered per agent in +`localStorage` and only restored while it still exists and is still readable; no run is opened for +you. The picker lists this agent's own datasets first and everyone else's below, sorted rather +than filtered, since running one dataset against a second agent is a comparison the picker exists +for. + +A dataset is named the way a script is: a **summary** of what the cases are for, from which the +path follows, prefixed with the agent so it sorts with the agent's own. With no summary the +fallback is `_dataset1`, taking the next free number. One path segment rather than a folder +under the agent, because a Windmill path is `//` and the picker that edits it +cannot express a deeper one. + +The dataset is edited in a drawer over the table: the summary and the path, the **scorers**, then +the cases in a grid. **Every way of managing a scorer is in that drawer** — adding, renaming, +moving its pass line, opening the runnable behind it, removing it; the column header over a run's +table reports and does not edit, since a run is permanent. Creating a dataset is the same drawer +with no cases yet, reached from the dataset named on a row of the runs list and from the run +dialog. Renaming moves the dataset, and its cases and its runs follow through the foreign keys. +The drawer edits a working copy and writes it in one request when **Save** is pressed — the +rename, the summary and the cases together — so a rename the server refuses leaves the cases as +they were, and a half-finished edit is never what the next run executes. A row's panel in the +results table is read-only and shows the case *as the run executed it*, not as the dataset holds +it now; deleting a case is in the drawer, and asks first. + +A case is its message, what it expects, and nothing else; the message is what identifies it. +`expected` is what a scorer compares an answer against: plain text, or JSON when the answer has +structure. + +The runs list is one row per run of this agent, newest first, whichever dataset it was of: the +run's number and what executed it (`v24`, `v24 + edits`, or a pinned `v18`), how many cases, one +badge per scorer, the dataset, and when. Each badge is the headline that column reports — a pass +rate where the column has a line, the mean where it does not — read through the thresholds as they +are **now**. A column that never scored a run reads `—`; a run still going spins. The badges are +named and resolved server-side: a list spanning datasets cannot hold every dataset's scorers to +look a column's name up, so the name and the kind ride along with the number, and the thresholds +are joined in per (run, column) — one grouped query over `eval_score` rather than a read of each +run's cells. A run whose scores are still in its flow is read out of it by the list itself, capped +per call and skipped for runs already collected, so the steady state is one query. + +**Run** asks two questions: which state of the agent (`v24 (latest deployed)` as it is saved when +you press Run; a past version as it was then; `v24 + edits (current)` running the step's edits as +they are when you press Run, offered and preselected only from the editing card), and which +dataset, with an edit button on the row and a way to start a new one without leaving. A pinned +version reproduces the configuration, not the world around it: `$var:` and `$res:` references +inside it still resolve at run time. The run that was just started opens straight away. + +The results table's rows are the dataset's cases, in dataset order, each carrying its result in +the selected experiment when it has one, so a dataset that has never been run is not an empty +table. A case the experiment ran but the dataset no longer holds keeps its row at the end: the run +happened, and deleting the case does not unmake it. Each column's mean sits under its header, with +its delta beside it when a baseline is selected. + +Picking a baseline adds a per-scorer delta to every cell and to each column's mean, and counts the +cells that regressed. Every delta names its scorer; there is no single number for a dataset, since +averaging a judge with an exact match would invent one. Rows are joined by case id, so a case added +after the baseline ran has no delta rather than counting as a change, and a column the baseline +was never scored with reports that rather than a difference that does not exist. + +## Storage + +Datasets, cases and experiments are rows: + +| table | holds | +|---|---| +| `eval_dataset` | one dataset, addressed by a workspace path, and the scorers that are its columns | +| `eval_case` | one case: its inputs and the answer it was expected to produce | +| `eval_experiment` | one run over a dataset, against one subject; written once, then only read | +| `eval_experiment_case` | the case set it executed, the job each case became, and the version or draft hash each ran against | +| `eval_score` | one scorer's verdict on one run, with the definition that produced it | + +An experiment records its cases by value instead of pointing at `eval_case`, because a dataset +keeps changing and a result set that cannot say which inputs produced it is not reproducible. For +the same reason `case_id` is a plain column rather than a foreign key: deleting a case must not +rewrite the history of the runs that used it. + +Deleting a dataset takes its cases, its experiments, their recorded case sets and every score with +it through the foreign keys. The jobs those experiments produced are left alone — they are jobs, +with their own retention. + +A case is text: a message and an expected answer. Attachments are S3 references rather than inline +bytes, so nothing in a case is meant to be large, and three caps keep it that way — 256 KiB per +case, 16 MiB and 1 000 cases per dataset — all refused at the API rather than truncated. A run +scores every case by every scorer, so a dataset also holds at most 20 scorers, refused the same +way. + +### Permissions + +A dataset is permissioned like any other path-addressed object: row-level security on +`eval_dataset` decides who may see it (readers of its folder, `u/`, a group, or an +`extra_perms` grant) and who may change it. Operators cannot write at all. Recording an experiment +counts as a write, since it persists into the dataset. + +Cases are the contents of a dataset rather than objects in their own right. `eval_case` carries a +read policy derived from its dataset (`see_parent_dataset`) and write policies that check the +dataset is *writable* — `eval_dataset_writable`, one function holding the same disjunction the +dataset's own write policies use, so a read-only grant can list a dataset's cases but not edit +them. A dataset and its cases therefore move in one `user_db` transaction, governed by the same +policies, and a rename is checked against the destination path the same way. The experiment tables +are the exception: their rows are written both by a launch (which holds dataset write) and by the +harvest (which holds only *read* of the run it copies onto its rows), so they carry read policies +only and are written on the unrestricted pool after the API has checked the right access. + +### Why an experiment is recorded before it is launched + +Launching picks the run job's id up front, writes the experiment, its case set and a pending score +per cell in one transaction, and only then queues the flow. Queueing first and recording afterwards +leaves a window in which a flow is running that no experiment accounts for, that nothing will +collect and that a retry would silently duplicate. In this order, a launch that dies before the +push leaves an experiment naming a job that never started — a run that did not run — and a push +that fails deletes it, because one failed push is the whole run. + +The dataset's foreign key guards a delete that races the assembly: the transaction fails, and at +that point nothing has been queued. It does not cover a delete that lands after this transaction +commits and before the flow is queued, which cascades the experiment away while the run still +starts. + +### How a cell finds its job, and its score + +The flow engine mints the iteration job ids, so a case is recorded before it has one. Three things +fill the gap, each copied out of the flow the first time it can be read: + +- **Which iteration ran which case.** The case is what the loop iterates over, so it is in the + iteration's own arguments by construction: `args -> 'iter' -> 'value' ->> 'case_id'` matches the + cell, whatever order the iterations finish in. +- **What the agent answered.** The agent step's result and outcome, copied onto the cell as soon + as that step is done — which is well before the iteration around it, since the scorers are still + reading it. +- **What the scorers returned.** Each scorer step's result is read out of the iteration's flow + status into the pending row that was written for it at launch. + +All three are written once, when they first become readable, and every later read is of the rows. +A job that was retained away before anything read it leaves the cell saying so, rather than +looking like a case still being answered. + +The flow itself cannot write them: it runs on workers that know nothing about these tables. So two +things call the collector. A run's flow ends with a step that calls `POST +/ai_evals/experiments/collect` on itself, which is what records a run nobody watched finish. +Reading a run collects it too, which covers the run whose flow never reached that step: one +cancelled part-way, or started while nothing served the `nativets` tag. + +That step is bookkeeping, so it is `continue_on_error`: a run whose every case answered and scored +does not become a failed job because the call did not land. diff --git a/docs/reusable-ai-agents.md b/docs/reusable-ai-agents.md index 8fa9facd74..8383bc2836 100644 --- a/docs/reusable-ai-agents.md +++ b/docs/reusable-ai-agents.md @@ -28,6 +28,11 @@ In the flow editor, the AI agent step's **Step Input** tab shows a single read-o (*linked to *, with the inherited brain + tools and an explanatory tooltip) plus *Edit* (fork into the editable step, Save changes upserts back and re-links) and *Unlink* (fork the resolved config — including any `tool_inputs` — back into the step as a one-off). +While editing, the step is the only copy of the edits: Cancel drops them and re-links (asking +first when there is something to drop), and the unsaved-changes badge opens a diff against the +deployed agent whose Discard changes is Cancel without the question. What a fork is an edit of, +and the deployed baseline the edits are judged against, live in `agentEditStore` (in memory), so +a reload brings the step back as a standalone agent with no path to save back to. A linked agent's tools appear as display-only graph tool nodes (clicking one selects the agent step); below the step's inputs, each tool gets a section with the standard schema-aware input editors (prop picker included) and a read-only view of its code — edits persist into @@ -62,6 +67,9 @@ as the reference, so two versions can be byte-identical while the agent behaves because the referenced variable changed underneath them. Anything comparing agent runs across versions has to account for that. +An eval run records the version its agent was at when the run was enqueued, which is what makes a +result attributable to a prompt state — see `docs/ai-agent-evals.md`. + A superseded value is retained for up to 100 versions. Values written through the UI keep their secrets in linked variables, but one pushed by `wmill` or written by `setResource` can hold an inline credential, and overwriting it no longer removes it from the database — anyone who can diff --git a/docs/validation.md b/docs/validation.md index 050eacd75b..74e9cf5677 100644 --- a/docs/validation.md +++ b/docs/validation.md @@ -16,6 +16,7 @@ After making changes, run the appropriate checks and fix all errors before consi | Multiple gated modules | `cargo check --features enterprise,parquet` | Combine only the flags you need | | API route changes | `cargo check` | Then update `openapi.yaml` and run `npm run generate-backend-client` | | Database migrations | `cargo check` | Test migration applies cleanly with `sqlx migrate run` | +| The `sqlx` dependency (version bump, `[patch.crates-io]` entries, fork rebase) | `cargo test -p windmill-common --test sqlx_begin_cancel_safe -- --ignored` | Windmill runs a patched `sqlx`: upstream's `Pool::begin` is not cancel-safe on Postgres, and a cancelled one poisons the pooled connection for 30 minutes. Losing the patch still compiles, so this ignored test is the only thing that notices. `backend/Cargo.toml` has the detail | **Never** use `--features all_sqlx_features` — it compiles everything and is very slow. Check `backend/Cargo.toml` `[features]` to find the right flags. diff --git a/frontend/package-lock.json b/frontend/package-lock.json index 17e2718f7f..6d72964274 100644 --- a/frontend/package-lock.json +++ b/frontend/package-lock.json @@ -1,12 +1,12 @@ { "name": "@windmill-labs/components", - "version": "1.795.0", + "version": "1.796.0", "lockfileVersion": 3, "requires": true, "packages": { "": { "name": "@windmill-labs/components", - "version": "1.795.0", + "version": "1.796.0", "hasInstallScript": true, "license": "AGPL-3.0", "dependencies": { diff --git a/frontend/package.json b/frontend/package.json index 226c6d76ad..1d984c8587 100644 --- a/frontend/package.json +++ b/frontend/package.json @@ -1,6 +1,6 @@ { "name": "@windmill-labs/components", - "version": "1.795.0", + "version": "1.796.0", "scripts": { "dev": "vite dev", "dev:ui-builder": "mv static/ui_builder static/ui_builder.dev-disabled 2>/dev/null || true ; trap 'mv static/ui_builder.dev-disabled static/ui_builder 2>/dev/null || true' EXIT ; vite dev", diff --git a/frontend/src/lib/aiStore.ts b/frontend/src/lib/aiStore.ts index 1e8fb6ac61..0955f68ee5 100644 --- a/frontend/src/lib/aiStore.ts +++ b/frontend/src/lib/aiStore.ts @@ -3,7 +3,12 @@ // import aiStore back, and such a cycle crashes the app once the bundler splits it across // chunks (docs/frontend-import-cycles.md; the build fails on the chunk cycle, not on this). import { writable, get } from 'svelte/store' -import { type AIProviderModel, type AIProvider, type AIConfig } from './gen' +import { + type AIProviderModel, + type AIProvider, + type AIConfig, + type ModelPriceOverride +} from './gen' import { aiUserDisabled, COPILOT_SESSION_MODEL_SETTING_NAME, @@ -41,6 +46,8 @@ export const copilotInfo = writable<{ aiModels: AIProviderModel[] customPrompts?: Record maxTokensPerModel?: Record + /** Negotiated rates per `provider:model`, overriding the built-in price table. */ + modelPricing?: Record webSearchEnabledProviders?: Partial> }>({ enabled: false, @@ -50,6 +57,7 @@ export const copilotInfo = writable<{ aiModels: [], customPrompts: {}, maxTokensPerModel: {}, + modelPricing: {}, webSearchEnabledProviders: {} }) @@ -124,6 +132,7 @@ export function setCopilotInfo(aiConfig: AIConfig) { aiModels: aiModels, customPrompts: aiConfig.custom_prompts ?? {}, maxTokensPerModel: aiConfig.max_tokens_per_model ?? {}, + modelPricing: aiConfig.model_pricing ?? {}, webSearchEnabledProviders }) } else { @@ -137,6 +146,7 @@ export function setCopilotInfo(aiConfig: AIConfig) { aiModels: [], customPrompts: {}, maxTokensPerModel: {}, + modelPricing: {}, webSearchEnabledProviders: {} }) } diff --git a/frontend/src/lib/components/ArgInput.svelte b/frontend/src/lib/components/ArgInput.svelte index 5ee7a5f429..8550a58b57 100644 --- a/frontend/src/lib/components/ArgInput.svelte +++ b/frontend/src/lib/components/ArgInput.svelte @@ -841,6 +841,7 @@ {disablePortal} {disabled} {prettifyHeader} + {workspace} {schema} bind:args={value} /> @@ -983,6 +984,7 @@ {disablePortal} {disabled} {prettifyHeader} + {workspace} schema={getSchemaFromProperties(itemsType?.properties)} bind:args={value[i]} /> @@ -1150,6 +1152,7 @@ {disablePortal} {disabled} {prettifyHeader} + {workspace} bind:schema={ () => ({ properties: obj.properties ?? {}, @@ -1186,6 +1189,7 @@ {disabled} {prettifyHeader} {chatInputEnabled} + {workspace} hiddenArgs={['label', 'kind']} schema={{ properties: obj.properties, @@ -1270,6 +1274,7 @@ {disablePortal} {disabled} {prettifyHeader} + {workspace} schema={{ properties, $schema: '', @@ -1301,6 +1306,7 @@ {disablePortal} {disabled} {prettifyHeader} + {workspace} schema={{ properties, order, @@ -1471,7 +1477,7 @@ /> {/if} {:else} - + {/if} {:else} {#key extra?.['minRows']} diff --git a/frontend/src/lib/components/ArrayTypeNarrowing.svelte b/frontend/src/lib/components/ArrayTypeNarrowing.svelte index 9c68f4523a..2dc5956ffd 100644 --- a/frontend/src/lib/components/ArrayTypeNarrowing.svelte +++ b/frontend/src/lib/components/ArrayTypeNarrowing.svelte @@ -25,13 +25,15 @@ } | undefined nonEmpty?: boolean | undefined + workspace?: string | undefined } let { canEditResourceType = false, originalType = undefined, itemsType = $bindable(), - nonEmpty = $bindable() + nonEmpty = $bindable(), + workspace }: Props = $props() let selected: @@ -199,6 +201,7 @@ /> {#if itemsType?.properties != undefined} { return { diff --git a/frontend/src/lib/components/DatatableSchemaDiff.svelte b/frontend/src/lib/components/DatatableSchemaDiff.svelte index 579b7a1923..1fb6918580 100644 --- a/frontend/src/lib/components/DatatableSchemaDiff.svelte +++ b/frontend/src/lib/components/DatatableSchemaDiff.svelte @@ -48,6 +48,9 @@ // migrations, its changes flow through the normal item diff instead, so it is // excluded here and the whole section hides once none remain. let applicableCount = $state(0) + let applicableLabel = $derived( + applicableCount === 1 ? 'this data table has' : 'these data tables have' + ) let expandedDatatables: Set = $state(new Set()) // Drawer state @@ -293,6 +296,12 @@ {#if applicableCount > 0}

Datatable schema changes

+ + This section diffs data table schemas directly because {applicableLabel} not opted in to migrations. + With migrations enabled, schema changes are tracked as migrations and deployed like any other item. + Opt in from the parent workspace's data table settings — a fork only inherits the flag when it + is created, so this fork keeps using the legacy diff. + {#if loading}
Loading datatable diffs... diff --git a/frontend/src/lib/components/EditableSchemaForm.svelte b/frontend/src/lib/components/EditableSchemaForm.svelte index 5ceef0fbf0..67dfeed0b8 100644 --- a/frontend/src/lib/components/EditableSchemaForm.svelte +++ b/frontend/src/lib/components/EditableSchemaForm.svelte @@ -15,6 +15,7 @@ import PropertyEditor from './schema/PropertyEditor.svelte' import SimpleEditor from './SimpleEditor.svelte' import { createEventDispatcher, untrack } from 'svelte' + import { watch } from 'runed' import ToggleButton from './common/toggleButton-v2/ToggleButton.svelte' import ToggleButtonGroup from './common/toggleButton-v2/ToggleButtonGroup.svelte' import Label from './Label.svelte' @@ -82,6 +83,7 @@ extraTab?: import('svelte').Snippet schemaFormClassName?: string onChange?: (args: Record) => void + workspace?: string | undefined } let { @@ -119,9 +121,12 @@ runButton, extraTab, schemaFormClassName = undefined, - onChange = undefined + onChange = undefined, + workspace = undefined }: Props = $props() + let ws = $derived(workspace ?? $workspaceStore) + $effect.pre(() => { if (args == undefined) { args = {} @@ -169,6 +174,11 @@ let itemPicker: ItemPicker | undefined = $state(undefined) let variableEditor: VariableEditor | undefined = $state(undefined) + watch( + () => ws, + () => itemPicker?.reloadItems() + ) + let keys: string[] = $state( (Array.isArray(schema?.order) ? [...schema.order] @@ -437,6 +447,7 @@ {hiddenArgs} {disableDnd} {onlyMaskPassword} + {workspace} bind:args on:click={(e) => { opened = e.detail @@ -682,6 +693,7 @@ {isFlowInput} {isAppInput} {showSensitiveToggle} + {workspace} > {#snippet typeeditor()} {#if isFlowInput || isAppInput} @@ -809,6 +821,7 @@ {#if isFlowInput || isAppInput} { dndType = generateRandomString() }} @@ -909,7 +922,7 @@ documentationLink="https://www.windmill.dev/docs/core_concepts/variables_and_secrets" extraField="path" loadItems={async () => - (await VariableService.listVariable({ workspace: $workspaceStore ?? '' })).map((x) => ({ + (await VariableService.listVariable({ workspace: ws ?? '' })).map((x) => ({ name: x.path, ...x }))} @@ -928,7 +941,7 @@ {/snippet} - + {/if}