diff --git a/backend/.sqlx/query-01bce88dd622f314d1a09c24cd12df5e7d3ff6a15a93e1c0c95e262f7b3d0ef1.json b/backend/.sqlx/query-01bce88dd622f314d1a09c24cd12df5e7d3ff6a15a93e1c0c95e262f7b3d0ef1.json new file mode 100644 index 0000000000..64bf73746f --- /dev/null +++ b/backend/.sqlx/query-01bce88dd622f314d1a09c24cd12df5e7d3ff6a15a93e1c0c95e262f7b3d0ef1.json @@ -0,0 +1,17 @@ +{ + "db_name": "PostgreSQL", + "query": "UPDATE eval_experiment\n SET subject = jsonb_set(\n jsonb_set(subject, '{kind}', '\"agent\"'),\n '{version}', to_jsonb($4::bigint))\n WHERE workspace_id = $1 AND dataset_path = $2 AND id = $3\n AND subject ->> 'kind' = 'agent_draft'", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "Text", + "Text", + "Uuid", + "Int8" + ] + }, + "nullable": [] + }, + "hash": "01bce88dd622f314d1a09c24cd12df5e7d3ff6a15a93e1c0c95e262f7b3d0ef1" +} diff --git a/backend/.sqlx/query-0276e6030abb2eb00a68c568a9cc60f3e7c2af0331388c4b358035de865a121a.json b/backend/.sqlx/query-0276e6030abb2eb00a68c568a9cc60f3e7c2af0331388c4b358035de865a121a.json new file mode 100644 index 0000000000..a7452b0602 --- /dev/null +++ b/backend/.sqlx/query-0276e6030abb2eb00a68c568a9cc60f3e7c2af0331388c4b358035de865a121a.json @@ -0,0 +1,35 @@ +{ + "db_name": "PostgreSQL", + "query": "SELECT path, value, version FROM resource_version WHERE workspace_id = $1 AND id = $2", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "path", + "type_info": "Varchar" + }, + { + "ordinal": 1, + "name": "value", + "type_info": "Jsonb" + }, + { + "ordinal": 2, + "name": "version", + "type_info": "Int8" + } + ], + "parameters": { + "Left": [ + "Text", + "Int8" + ] + }, + "nullable": [ + false, + true, + false + ] + }, + "hash": "0276e6030abb2eb00a68c568a9cc60f3e7c2af0331388c4b358035de865a121a" +} diff --git a/backend/.sqlx/query-0335de6713de6678b9bf266121af23abc46d5db95da095bb15726c5a2db7ad2f.json b/backend/.sqlx/query-0335de6713de6678b9bf266121af23abc46d5db95da095bb15726c5a2db7ad2f.json new file mode 100644 index 0000000000..f56ef7ead0 --- /dev/null +++ b/backend/.sqlx/query-0335de6713de6678b9bf266121af23abc46d5db95da095bb15726c5a2db7ad2f.json @@ -0,0 +1,24 @@ +{ + "db_name": "PostgreSQL", + "query": "SELECT coalesce(max(run_number), 0) + 1 FROM eval_experiment\n WHERE workspace_id = $1 AND dataset_path = $2 AND subject ->> 'path' = $3", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "?column?", + "type_info": "Int4" + } + ], + "parameters": { + "Left": [ + "Text", + "Text", + "Text" + ] + }, + "nullable": [ + null + ] + }, + "hash": "0335de6713de6678b9bf266121af23abc46d5db95da095bb15726c5a2db7ad2f" +} diff --git a/backend/.sqlx/query-0aae275d9196e742b5783df4e67c72459d45e275bfeafa2952349cae259ac9f0.json b/backend/.sqlx/query-0aae275d9196e742b5783df4e67c72459d45e275bfeafa2952349cae259ac9f0.json new file mode 100644 index 0000000000..25e389fd30 --- /dev/null +++ b/backend/.sqlx/query-0aae275d9196e742b5783df4e67c72459d45e275bfeafa2952349cae259ac9f0.json @@ -0,0 +1,16 @@ +{ + "db_name": "PostgreSQL", + "query": "UPDATE eval_dataset SET scorers = COALESCE((\n SELECT jsonb_agg(\n CASE WHEN elem->>'path' LIKE ('u/' || $2 || '/%')\n THEN jsonb_set(elem, '{path}', to_jsonb(REGEXP_REPLACE(elem->>'path', 'u/' || $2 || '/(.*)', $1 || '/\\1')))\n ELSE elem END)\n FROM jsonb_array_elements(scorers) elem), '[]'::jsonb)\n WHERE workspace_id = $3\n AND EXISTS (SELECT 1 FROM jsonb_array_elements(scorers) e WHERE e->>'path' LIKE ('u/' || $2 || '/%'))", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "Text", + "Text", + "Text" + ] + }, + "nullable": [] + }, + "hash": "0aae275d9196e742b5783df4e67c72459d45e275bfeafa2952349cae259ac9f0" +} diff --git a/backend/.sqlx/query-0d6700ccffb8179e365bbc1f03398e474f23b013f642ba29ad6f68e6f047c1e5.json b/backend/.sqlx/query-0d6700ccffb8179e365bbc1f03398e474f23b013f642ba29ad6f68e6f047c1e5.json new file mode 100644 index 0000000000..2ec3036fb0 --- /dev/null +++ b/backend/.sqlx/query-0d6700ccffb8179e365bbc1f03398e474f23b013f642ba29ad6f68e6f047c1e5.json @@ -0,0 +1,24 @@ +{ + "db_name": "PostgreSQL", + "query": "SELECT pg_advisory_xact_lock(hashtext('ai_eval_open:' || $1 || '/' || $2 || '/' || $3))", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "pg_advisory_xact_lock", + "type_info": "Void" + } + ], + "parameters": { + "Left": [ + "Text", + "Text", + "Text" + ] + }, + "nullable": [ + null + ] + }, + "hash": "0d6700ccffb8179e365bbc1f03398e474f23b013f642ba29ad6f68e6f047c1e5" +} diff --git a/backend/.sqlx/query-1815730982dcaf7239ddcb22f88ae5c79794213cf6278167f8afdbca30b1b15c.json b/backend/.sqlx/query-1815730982dcaf7239ddcb22f88ae5c79794213cf6278167f8afdbca30b1b15c.json new file mode 100644 index 0000000000..a8134921e5 --- /dev/null +++ b/backend/.sqlx/query-1815730982dcaf7239ddcb22f88ae5c79794213cf6278167f8afdbca30b1b15c.json @@ -0,0 +1,16 @@ +{ + "db_name": "PostgreSQL", + "query": "UPDATE eval_experiment SET subject = jsonb_set(subject, '{path}', to_jsonb(REGEXP_REPLACE(subject->>'path', 'u/' || $2 || '/(.*)', $1 || '/\\1'))) WHERE subject->>'path' LIKE ('u/' || $2 || '/%') AND workspace_id = $3", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "Text", + "Text", + "Text" + ] + }, + "nullable": [] + }, + "hash": "1815730982dcaf7239ddcb22f88ae5c79794213cf6278167f8afdbca30b1b15c" +} diff --git a/backend/.sqlx/query-186c663249ffada82abf61ce214f52e2730501774a0ca4dc855380e6c6487917.json b/backend/.sqlx/query-186c663249ffada82abf61ce214f52e2730501774a0ca4dc855380e6c6487917.json new file mode 100644 index 0000000000..5cead4df77 --- /dev/null +++ b/backend/.sqlx/query-186c663249ffada82abf61ce214f52e2730501774a0ca4dc855380e6c6487917.json @@ -0,0 +1,22 @@ +{ + "db_name": "PostgreSQL", + "query": "SELECT count(*) AS \"count!\" FROM eval_experiment_case\n WHERE experiment_id = $1 AND status IS NOT NULL", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "count!", + "type_info": "Int8" + } + ], + "parameters": { + "Left": [ + "Uuid" + ] + }, + "nullable": [ + null + ] + }, + "hash": "186c663249ffada82abf61ce214f52e2730501774a0ca4dc855380e6c6487917" +} diff --git a/backend/.sqlx/query-196939257a334f7d37aa6d66153b251446d7701a893cb7852b81cf842c0fa228.json b/backend/.sqlx/query-196939257a334f7d37aa6d66153b251446d7701a893cb7852b81cf842c0fa228.json new file mode 100644 index 0000000000..d9643875cf --- /dev/null +++ b/backend/.sqlx/query-196939257a334f7d37aa6d66153b251446d7701a893cb7852b81cf842c0fa228.json @@ -0,0 +1,59 @@ +{ + "db_name": "PostgreSQL", + "query": "SELECT path, summary, scorers, created_at, created_by,\n edited_at, edited_by\n FROM eval_dataset WHERE workspace_id = $1 AND path = $2", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "path", + "type_info": "Varchar" + }, + { + "ordinal": 1, + "name": "summary", + "type_info": "Varchar" + }, + { + "ordinal": 2, + "name": "scorers", + "type_info": "Jsonb" + }, + { + "ordinal": 3, + "name": "created_at", + "type_info": "Timestamptz" + }, + { + "ordinal": 4, + "name": "created_by", + "type_info": "Varchar" + }, + { + "ordinal": 5, + "name": "edited_at", + "type_info": "Timestamptz" + }, + { + "ordinal": 6, + "name": "edited_by", + "type_info": "Varchar" + } + ], + "parameters": { + "Left": [ + "Text", + "Text" + ] + }, + "nullable": [ + false, + true, + false, + false, + false, + false, + false + ] + }, + "hash": "196939257a334f7d37aa6d66153b251446d7701a893cb7852b81cf842c0fa228" +} diff --git a/backend/.sqlx/query-1b6e229545f6b877e72d21728257d1bddaba15ef0fbe72bb4f43b45140f184ce.json b/backend/.sqlx/query-1b6e229545f6b877e72d21728257d1bddaba15ef0fbe72bb4f43b45140f184ce.json new file mode 100644 index 0000000000..54e381be1a --- /dev/null +++ b/backend/.sqlx/query-1b6e229545f6b877e72d21728257d1bddaba15ef0fbe72bb4f43b45140f184ce.json @@ -0,0 +1,28 @@ +{ + "db_name": "PostgreSQL", + "query": "SELECT case_id, ordinal FROM eval_experiment_case WHERE experiment_id = $1", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "case_id", + "type_info": "Uuid" + }, + { + "ordinal": 1, + "name": "ordinal", + "type_info": "Int4" + } + ], + "parameters": { + "Left": [ + "Uuid" + ] + }, + "nullable": [ + false, + false + ] + }, + "hash": "1b6e229545f6b877e72d21728257d1bddaba15ef0fbe72bb4f43b45140f184ce" +} diff --git a/backend/.sqlx/query-1db80f3ba2c6c769a98424ebf9aaf168a4fa2c64e446a824038cf267236fe979.json b/backend/.sqlx/query-1db80f3ba2c6c769a98424ebf9aaf168a4fa2c64e446a824038cf267236fe979.json new file mode 100644 index 0000000000..01bae0f126 --- /dev/null +++ b/backend/.sqlx/query-1db80f3ba2c6c769a98424ebf9aaf168a4fa2c64e446a824038cf267236fe979.json @@ -0,0 +1,23 @@ +{ + "db_name": "PostgreSQL", + "query": "SELECT DISTINCT path FROM script\n WHERE workspace_id = $1 AND path = ANY($2)\n AND deleted = false AND lock IS NOT NULL AND lock_error_logs IS NULL", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "path", + "type_info": "Varchar" + } + ], + "parameters": { + "Left": [ + "Text", + "TextArray" + ] + }, + "nullable": [ + false + ] + }, + "hash": "1db80f3ba2c6c769a98424ebf9aaf168a4fa2c64e446a824038cf267236fe979" +} diff --git a/backend/.sqlx/query-242845c86084e010ab33c2197d44af9aeb181672a2f2330dbe65bfe586376450.json b/backend/.sqlx/query-242845c86084e010ab33c2197d44af9aeb181672a2f2330dbe65bfe586376450.json new file mode 100644 index 0000000000..38dcbadfb0 --- /dev/null +++ b/backend/.sqlx/query-242845c86084e010ab33c2197d44af9aeb181672a2f2330dbe65bfe586376450.json @@ -0,0 +1,15 @@ +{ + "db_name": "PostgreSQL", + "query": "UPDATE eval_score SET error = 'The case did not run'\n WHERE experiment_id = $1 AND ordinal = ANY($2)\n AND score IS NULL AND error IS NULL AND NOT not_applicable", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "Uuid", + "Int4Array" + ] + }, + "nullable": [] + }, + "hash": "242845c86084e010ab33c2197d44af9aeb181672a2f2330dbe65bfe586376450" +} diff --git a/backend/.sqlx/query-24fcc2b69f30953915f0cbf246e1c19b2310075a644d81c21784c991e52b4001.json b/backend/.sqlx/query-24fcc2b69f30953915f0cbf246e1c19b2310075a644d81c21784c991e52b4001.json new file mode 100644 index 0000000000..a2cd96857a --- /dev/null +++ b/backend/.sqlx/query-24fcc2b69f30953915f0cbf246e1c19b2310075a644d81c21784c991e52b4001.json @@ -0,0 +1,24 @@ +{ + "db_name": "PostgreSQL", + "query": "INSERT INTO ai_token_usage (workspace_id, email, provider, model, session_id, input_tokens, cache_read_tokens, cache_write_tokens, output_tokens, reported_cost_nano_usd, requests)\n SELECT $1, $2, * FROM UNNEST($3::text[], $4::text[], $5::text[], $6::bigint[], $7::bigint[], $8::bigint[], $9::bigint[], $10::bigint[], $11::bigint[])\n ON CONFLICT (workspace_id, day, email, provider, model, session_id)\n DO UPDATE SET\n input_tokens = ai_token_usage.input_tokens + EXCLUDED.input_tokens,\n cache_read_tokens = ai_token_usage.cache_read_tokens + EXCLUDED.cache_read_tokens,\n cache_write_tokens = ai_token_usage.cache_write_tokens + EXCLUDED.cache_write_tokens,\n output_tokens = ai_token_usage.output_tokens + EXCLUDED.output_tokens,\n reported_cost_nano_usd = CASE\n WHEN EXCLUDED.reported_cost_nano_usd IS NULL\n THEN ai_token_usage.reported_cost_nano_usd\n ELSE COALESCE(ai_token_usage.reported_cost_nano_usd, 0)\n + EXCLUDED.reported_cost_nano_usd\n END,\n requests = ai_token_usage.requests + EXCLUDED.requests,\n updated_at = now()", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "Varchar", + "Varchar", + "TextArray", + "TextArray", + "TextArray", + "Int8Array", + "Int8Array", + "Int8Array", + "Int8Array", + "Int8Array", + "Int8Array" + ] + }, + "nullable": [] + }, + "hash": "24fcc2b69f30953915f0cbf246e1c19b2310075a644d81c21784c991e52b4001" +} diff --git a/backend/.sqlx/query-2b41dc4d872af0e230c31bef1a496604c1562cdfb0bdb54bd3b0c6579d73e46d.json b/backend/.sqlx/query-2b41dc4d872af0e230c31bef1a496604c1562cdfb0bdb54bd3b0c6579d73e46d.json new file mode 100644 index 0000000000..4506a038fc --- /dev/null +++ b/backend/.sqlx/query-2b41dc4d872af0e230c31bef1a496604c1562cdfb0bdb54bd3b0c6579d73e46d.json @@ -0,0 +1,28 @@ +{ + "db_name": "PostgreSQL", + "query": "SELECT path, scorers FROM eval_dataset\n WHERE workspace_id = $1 ORDER BY edited_at DESC LIMIT 100", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "path", + "type_info": "Varchar" + }, + { + "ordinal": 1, + "name": "scorers", + "type_info": "Jsonb" + } + ], + "parameters": { + "Left": [ + "Text" + ] + }, + "nullable": [ + false, + false + ] + }, + "hash": "2b41dc4d872af0e230c31bef1a496604c1562cdfb0bdb54bd3b0c6579d73e46d" +} diff --git a/backend/.sqlx/query-307d5b797e51122dbf087e5dc9f33235a349731828d500c74a28b577469e2624.json b/backend/.sqlx/query-307d5b797e51122dbf087e5dc9f33235a349731828d500c74a28b577469e2624.json new file mode 100644 index 0000000000..2cfec01e9b --- /dev/null +++ b/backend/.sqlx/query-307d5b797e51122dbf087e5dc9f33235a349731828d500c74a28b577469e2624.json @@ -0,0 +1,23 @@ +{ + "db_name": "PostgreSQL", + "query": "SELECT path FROM resource WHERE workspace_id = $1 AND path = ANY($2) AND resource_type = 'ai_agent'", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "path", + "type_info": "Varchar" + } + ], + "parameters": { + "Left": [ + "Text", + "TextArray" + ] + }, + "nullable": [ + false + ] + }, + "hash": "307d5b797e51122dbf087e5dc9f33235a349731828d500c74a28b577469e2624" +} diff --git a/backend/.sqlx/query-30b8590939bf3d6770cabca9f4ab3527e750d85d3b8cc0eadef55461de4a2687.json b/backend/.sqlx/query-30b8590939bf3d6770cabca9f4ab3527e750d85d3b8cc0eadef55461de4a2687.json new file mode 100644 index 0000000000..d3a443a0bf --- /dev/null +++ b/backend/.sqlx/query-30b8590939bf3d6770cabca9f4ab3527e750d85d3b8cc0eadef55461de4a2687.json @@ -0,0 +1,23 @@ +{ + "db_name": "PostgreSQL", + "query": "SELECT path FROM eval_dataset WHERE path LIKE $1 AND workspace_id = $2", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "path", + "type_info": "Varchar" + } + ], + "parameters": { + "Left": [ + "Text", + "Text" + ] + }, + "nullable": [ + false + ] + }, + "hash": "30b8590939bf3d6770cabca9f4ab3527e750d85d3b8cc0eadef55461de4a2687" +} diff --git a/backend/.sqlx/query-316e7c86082b6ee2864b88674cfac5b5b34c5de851f1c6f832b868101edcb052.json b/backend/.sqlx/query-316e7c86082b6ee2864b88674cfac5b5b34c5de851f1c6f832b868101edcb052.json new file mode 100644 index 0000000000..a3443236ea --- /dev/null +++ b/backend/.sqlx/query-316e7c86082b6ee2864b88674cfac5b5b34c5de851f1c6f832b868101edcb052.json @@ -0,0 +1,23 @@ +{ + "db_name": "PostgreSQL", + "query": "SELECT version FROM resource_version WHERE workspace_id = $1 AND path = $2\n ORDER BY version DESC LIMIT 1", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "version", + "type_info": "Int8" + } + ], + "parameters": { + "Left": [ + "Text", + "Text" + ] + }, + "nullable": [ + false + ] + }, + "hash": "316e7c86082b6ee2864b88674cfac5b5b34c5de851f1c6f832b868101edcb052" +} diff --git a/backend/.sqlx/query-34fbb2b141ad691e0cdc55bf2ebd6974e02a7c81d91217d3f74e03113edc9b0a.json b/backend/.sqlx/query-34fbb2b141ad691e0cdc55bf2ebd6974e02a7c81d91217d3f74e03113edc9b0a.json new file mode 100644 index 0000000000..c79145adaf --- /dev/null +++ b/backend/.sqlx/query-34fbb2b141ad691e0cdc55bf2ebd6974e02a7c81d91217d3f74e03113edc9b0a.json @@ -0,0 +1,16 @@ +{ + "db_name": "PostgreSQL", + "query": "DELETE FROM eval_case\n WHERE workspace_id = $1 AND dataset_path = $2 AND NOT (id = ANY($3))", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "Text", + "Text", + "UuidArray" + ] + }, + "nullable": [] + }, + "hash": "34fbb2b141ad691e0cdc55bf2ebd6974e02a7c81d91217d3f74e03113edc9b0a" +} diff --git a/backend/.sqlx/query-3ca2f72d2917f48644cb79daba9858f472be9f1bb8e504cc3dc2eb998e3f7b91.json b/backend/.sqlx/query-3ca2f72d2917f48644cb79daba9858f472be9f1bb8e504cc3dc2eb998e3f7b91.json new file mode 100644 index 0000000000..fd5e4325a0 --- /dev/null +++ b/backend/.sqlx/query-3ca2f72d2917f48644cb79daba9858f472be9f1bb8e504cc3dc2eb998e3f7b91.json @@ -0,0 +1,16 @@ +{ + "db_name": "PostgreSQL", + "query": "UPDATE eval_experiment SET created_by = $1 WHERE created_by = $2 AND workspace_id = $3", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "Varchar", + "Text", + "Text" + ] + }, + "nullable": [] + }, + "hash": "3ca2f72d2917f48644cb79daba9858f472be9f1bb8e504cc3dc2eb998e3f7b91" +} diff --git a/backend/.sqlx/query-44a39475ba202bd5852b666335308360569aae717eff3fe9cc9253a261ae2fa7.json b/backend/.sqlx/query-44a39475ba202bd5852b666335308360569aae717eff3fe9cc9253a261ae2fa7.json new file mode 100644 index 0000000000..cf0806757d --- /dev/null +++ b/backend/.sqlx/query-44a39475ba202bd5852b666335308360569aae717eff3fe9cc9253a261ae2fa7.json @@ -0,0 +1,23 @@ +{ + "db_name": "PostgreSQL", + "query": "SELECT status::text AS \"status!\" FROM v2_job_completed WHERE id = $1 AND workspace_id = $2", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "status!", + "type_info": "Text" + } + ], + "parameters": { + "Left": [ + "Uuid", + "Text" + ] + }, + "nullable": [ + null + ] + }, + "hash": "44a39475ba202bd5852b666335308360569aae717eff3fe9cc9253a261ae2fa7" +} diff --git a/backend/.sqlx/query-4550ae568abf23045259f95195bf1a35f5dc8f6828c2f3df51520683af48ebbd.json b/backend/.sqlx/query-4550ae568abf23045259f95195bf1a35f5dc8f6828c2f3df51520683af48ebbd.json new file mode 100644 index 0000000000..fd2ab8fbee --- /dev/null +++ b/backend/.sqlx/query-4550ae568abf23045259f95195bf1a35f5dc8f6828c2f3df51520683af48ebbd.json @@ -0,0 +1,16 @@ +{ + "db_name": "PostgreSQL", + "query": "UPDATE eval_dataset SET path = REGEXP_REPLACE(path,'u/' || $2 || '/(.*)','u/' || $1 || '/\\1') WHERE path LIKE ('u/' || $2 || '/%') AND workspace_id = $3", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "Text", + "Text", + "Text" + ] + }, + "nullable": [] + }, + "hash": "4550ae568abf23045259f95195bf1a35f5dc8f6828c2f3df51520683af48ebbd" +} diff --git a/backend/.sqlx/query-467ed4d282af003dd3b0d9542caec73feba3141f7345ffd1a6554fe44a6a9171.json b/backend/.sqlx/query-467ed4d282af003dd3b0d9542caec73feba3141f7345ffd1a6554fe44a6a9171.json new file mode 100644 index 0000000000..5399d071a9 --- /dev/null +++ b/backend/.sqlx/query-467ed4d282af003dd3b0d9542caec73feba3141f7345ffd1a6554fe44a6a9171.json @@ -0,0 +1,24 @@ +{ + "db_name": "PostgreSQL", + "query": "WITH updated AS (\n UPDATE eval_dataset SET path = REGEXP_REPLACE(path, 'u/' || $2 || '/(.*)', $1 || '/\\1')\n WHERE path LIKE ('u/' || $2 || '/%') AND workspace_id = $3\n RETURNING 1\n ) SELECT COUNT(*) FROM updated", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "count", + "type_info": "Int8" + } + ], + "parameters": { + "Left": [ + "Text", + "Text", + "Text" + ] + }, + "nullable": [ + null + ] + }, + "hash": "467ed4d282af003dd3b0d9542caec73feba3141f7345ffd1a6554fe44a6a9171" +} diff --git a/backend/.sqlx/query-479427dc09ebacb80cb20d553e93a577bb85b389ab056909a326212565a338bc.json b/backend/.sqlx/query-479427dc09ebacb80cb20d553e93a577bb85b389ab056909a326212565a338bc.json new file mode 100644 index 0000000000..8dd3b1ec7d --- /dev/null +++ b/backend/.sqlx/query-479427dc09ebacb80cb20d553e93a577bb85b389ab056909a326212565a338bc.json @@ -0,0 +1,15 @@ +{ + "db_name": "PostgreSQL", + "query": "INSERT INTO eval_dataset (workspace_id, path, summary, scorers, extra_perms, created_at, created_by, edited_at, edited_by)\n SELECT $2, path, summary, scorers, extra_perms, created_at, created_by, edited_at, edited_by\n FROM eval_dataset WHERE workspace_id = $1", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "Text", + "Varchar" + ] + }, + "nullable": [] + }, + "hash": "479427dc09ebacb80cb20d553e93a577bb85b389ab056909a326212565a338bc" +} diff --git a/backend/.sqlx/query-4be42f447e10e420f2a909579398b01b5175e14eee47ca39fe63242a78da767e.json b/backend/.sqlx/query-4be42f447e10e420f2a909579398b01b5175e14eee47ca39fe63242a78da767e.json new file mode 100644 index 0000000000..2107f75f72 --- /dev/null +++ b/backend/.sqlx/query-4be42f447e10e420f2a909579398b01b5175e14eee47ca39fe63242a78da767e.json @@ -0,0 +1,16 @@ +{ + "db_name": "PostgreSQL", + "query": "UPDATE eval_case SET created_by = $1 WHERE created_by = $2 AND workspace_id = $3", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "Varchar", + "Text", + "Text" + ] + }, + "nullable": [] + }, + "hash": "4be42f447e10e420f2a909579398b01b5175e14eee47ca39fe63242a78da767e" +} diff --git a/backend/.sqlx/query-4bf7f1a0fd87e79bb789cee06a5da68e465ce7ce67e80770f22a36a9e8320657.json b/backend/.sqlx/query-4bf7f1a0fd87e79bb789cee06a5da68e465ce7ce67e80770f22a36a9e8320657.json new file mode 100644 index 0000000000..a4ddcb531a --- /dev/null +++ b/backend/.sqlx/query-4bf7f1a0fd87e79bb789cee06a5da68e465ce7ce67e80770f22a36a9e8320657.json @@ -0,0 +1,59 @@ +{ + "db_name": "PostgreSQL", + "query": "SELECT path, summary, scorers, created_at, created_by, edited_at, edited_by\n FROM eval_dataset WHERE workspace_id = $1 AND path = $2 FOR UPDATE", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "path", + "type_info": "Varchar" + }, + { + "ordinal": 1, + "name": "summary", + "type_info": "Varchar" + }, + { + "ordinal": 2, + "name": "scorers", + "type_info": "Jsonb" + }, + { + "ordinal": 3, + "name": "created_at", + "type_info": "Timestamptz" + }, + { + "ordinal": 4, + "name": "created_by", + "type_info": "Varchar" + }, + { + "ordinal": 5, + "name": "edited_at", + "type_info": "Timestamptz" + }, + { + "ordinal": 6, + "name": "edited_by", + "type_info": "Varchar" + } + ], + "parameters": { + "Left": [ + "Text", + "Text" + ] + }, + "nullable": [ + false, + true, + false, + false, + false, + false, + false + ] + }, + "hash": "4bf7f1a0fd87e79bb789cee06a5da68e465ce7ce67e80770f22a36a9e8320657" +} diff --git a/backend/.sqlx/query-e74e283951aa87627a46aa8286819cf7aa4fecbde17bc7a67ee6f49c427cee9e.json b/backend/.sqlx/query-4e4a9c7b6e95f81101a68a6cc6d29fe4424eaed36d5ac9e4cd0a2a9d8b878eeb.json similarity index 59% rename from backend/.sqlx/query-e74e283951aa87627a46aa8286819cf7aa4fecbde17bc7a67ee6f49c427cee9e.json rename to backend/.sqlx/query-4e4a9c7b6e95f81101a68a6cc6d29fe4424eaed36d5ac9e4cd0a2a9d8b878eeb.json index 77399cdf62..0796577849 100644 --- a/backend/.sqlx/query-e74e283951aa87627a46aa8286819cf7aa4fecbde17bc7a67ee6f49c427cee9e.json +++ b/backend/.sqlx/query-4e4a9c7b6e95f81101a68a6cc6d29fe4424eaed36d5ac9e4cd0a2a9d8b878eeb.json @@ -1,6 +1,6 @@ { "db_name": "PostgreSQL", - "query": "SELECT path, value FROM resource_version WHERE workspace_id = $1 AND id = $2", + "query": "SELECT path, scorers FROM eval_dataset WHERE workspace_id = $1 AND path = ANY($2)", "describe": { "columns": [ { @@ -10,20 +10,20 @@ }, { "ordinal": 1, - "name": "value", + "name": "scorers", "type_info": "Jsonb" } ], "parameters": { "Left": [ "Text", - "Int8" + "TextArray" ] }, "nullable": [ false, - true + false ] }, - "hash": "e74e283951aa87627a46aa8286819cf7aa4fecbde17bc7a67ee6f49c427cee9e" + "hash": "4e4a9c7b6e95f81101a68a6cc6d29fe4424eaed36d5ac9e4cd0a2a9d8b878eeb" } diff --git a/backend/.sqlx/query-5329ce41bbdc36698ea059fec5f05ded7dfe1c64517289ad67028001417d30a3.json b/backend/.sqlx/query-5329ce41bbdc36698ea059fec5f05ded7dfe1c64517289ad67028001417d30a3.json new file mode 100644 index 0000000000..54019fe5bc --- /dev/null +++ b/backend/.sqlx/query-5329ce41bbdc36698ea059fec5f05ded7dfe1c64517289ad67028001417d30a3.json @@ -0,0 +1,47 @@ +{ + "db_name": "PostgreSQL", + "query": "SELECT id, input, expected, created_at, created_by\n FROM eval_case\n WHERE workspace_id = $1 AND dataset_path = $2\n ORDER BY created_at, id", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "id", + "type_info": "Uuid" + }, + { + "ordinal": 1, + "name": "input", + "type_info": "Jsonb" + }, + { + "ordinal": 2, + "name": "expected", + "type_info": "Jsonb" + }, + { + "ordinal": 3, + "name": "created_at", + "type_info": "Timestamptz" + }, + { + "ordinal": 4, + "name": "created_by", + "type_info": "Varchar" + } + ], + "parameters": { + "Left": [ + "Text", + "Text" + ] + }, + "nullable": [ + false, + false, + true, + false, + false + ] + }, + "hash": "5329ce41bbdc36698ea059fec5f05ded7dfe1c64517289ad67028001417d30a3" +} diff --git a/backend/.sqlx/query-5a219a2532517869578c4504ff3153c43903f929ae5d62fbba12610f89c36d55.json b/backend/.sqlx/query-5a219a2532517869578c4504ff3153c43903f929ae5d62fbba12610f89c36d55.json index 36ddb8ab9f..713ccb9dd3 100644 --- a/backend/.sqlx/query-5a219a2532517869578c4504ff3153c43903f929ae5d62fbba12610f89c36d55.json +++ b/backend/.sqlx/query-5a219a2532517869578c4504ff3153c43903f929ae5d62fbba12610f89c36d55.json @@ -15,7 +15,7 @@ ] }, "nullable": [ - true + null ] }, "hash": "5a219a2532517869578c4504ff3153c43903f929ae5d62fbba12610f89c36d55" diff --git a/backend/.sqlx/query-5d3560d7a42f86436fec362a790cea189be47e344d11871a90faa9220d509646.json b/backend/.sqlx/query-5d3560d7a42f86436fec362a790cea189be47e344d11871a90faa9220d509646.json new file mode 100644 index 0000000000..548516f69c --- /dev/null +++ b/backend/.sqlx/query-5d3560d7a42f86436fec362a790cea189be47e344d11871a90faa9220d509646.json @@ -0,0 +1,16 @@ +{ + "db_name": "PostgreSQL", + "query": "UPDATE eval_experiment_case c SET job_id = j.id\n FROM v2_job j\n WHERE j.parent_job = $3 AND j.workspace_id = $2\n AND (j.args -> 'iter' -> 'value' ->> 'case_id')::uuid = c.case_id\n AND c.experiment_id = $1 AND c.job_id IS NULL", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "Uuid", + "Text", + "Uuid" + ] + }, + "nullable": [] + }, + "hash": "5d3560d7a42f86436fec362a790cea189be47e344d11871a90faa9220d509646" +} diff --git a/backend/.sqlx/query-5d5186bb17092425664d8c4f92ca6bfb8b08420cc2b2a52e90064d6a12f86113.json b/backend/.sqlx/query-5d5186bb17092425664d8c4f92ca6bfb8b08420cc2b2a52e90064d6a12f86113.json new file mode 100644 index 0000000000..8bf4956d79 --- /dev/null +++ b/backend/.sqlx/query-5d5186bb17092425664d8c4f92ca6bfb8b08420cc2b2a52e90064d6a12f86113.json @@ -0,0 +1,76 @@ +{ + "db_name": "PostgreSQL", + "query": "SELECT ordinal, case_id, input, expected, job_id, subject_version,\n subject_draft_hash, output, answered, status\n FROM eval_experiment_case\n WHERE experiment_id = $1 ORDER BY ordinal", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "ordinal", + "type_info": "Int4" + }, + { + "ordinal": 1, + "name": "case_id", + "type_info": "Uuid" + }, + { + "ordinal": 2, + "name": "input", + "type_info": "Jsonb" + }, + { + "ordinal": 3, + "name": "expected", + "type_info": "Jsonb" + }, + { + "ordinal": 4, + "name": "job_id", + "type_info": "Uuid" + }, + { + "ordinal": 5, + "name": "subject_version", + "type_info": "Int8" + }, + { + "ordinal": 6, + "name": "subject_draft_hash", + "type_info": "Varchar" + }, + { + "ordinal": 7, + "name": "output", + "type_info": "Text" + }, + { + "ordinal": 8, + "name": "answered", + "type_info": "Bool" + }, + { + "ordinal": 9, + "name": "status", + "type_info": "Varchar" + } + ], + "parameters": { + "Left": [ + "Uuid" + ] + }, + "nullable": [ + false, + false, + false, + true, + true, + true, + true, + true, + true, + true + ] + }, + "hash": "5d5186bb17092425664d8c4f92ca6bfb8b08420cc2b2a52e90064d6a12f86113" +} diff --git a/backend/.sqlx/query-5dcaea907b1ebb2854becc44c00eac5838995dce3d2a696769e00fb74c34288f.json b/backend/.sqlx/query-5dcaea907b1ebb2854becc44c00eac5838995dce3d2a696769e00fb74c34288f.json new file mode 100644 index 0000000000..5c3ee8da92 --- /dev/null +++ b/backend/.sqlx/query-5dcaea907b1ebb2854becc44c00eac5838995dce3d2a696769e00fb74c34288f.json @@ -0,0 +1,22 @@ +{ + "db_name": "PostgreSQL", + "query": "SELECT DISTINCT experiment_id FROM eval_score\n WHERE experiment_id = ANY($1) AND score IS NULL AND error IS NULL\n AND NOT not_applicable", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "experiment_id", + "type_info": "Uuid" + } + ], + "parameters": { + "Left": [ + "UuidArray" + ] + }, + "nullable": [ + false + ] + }, + "hash": "5dcaea907b1ebb2854becc44c00eac5838995dce3d2a696769e00fb74c34288f" +} diff --git a/backend/.sqlx/query-61773dd5d5952607eddfacb1717126a87ee5dc4bd6664eaf31c2c0096fea08db.json b/backend/.sqlx/query-61773dd5d5952607eddfacb1717126a87ee5dc4bd6664eaf31c2c0096fea08db.json new file mode 100644 index 0000000000..278ccbe48e --- /dev/null +++ b/backend/.sqlx/query-61773dd5d5952607eddfacb1717126a87ee5dc4bd6664eaf31c2c0096fea08db.json @@ -0,0 +1,26 @@ +{ + "db_name": "PostgreSQL", + "query": "UPDATE eval_case SET input = $4, expected = $5\n WHERE workspace_id = $1 AND dataset_path = $2 AND id = $3\n RETURNING id", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "id", + "type_info": "Uuid" + } + ], + "parameters": { + "Left": [ + "Text", + "Text", + "Uuid", + "Jsonb", + "Jsonb" + ] + }, + "nullable": [ + false + ] + }, + "hash": "61773dd5d5952607eddfacb1717126a87ee5dc4bd6664eaf31c2c0096fea08db" +} diff --git a/backend/.sqlx/query-6a17a1dfeb75808e5d0726f1f8bd573168161abe350b047402cda4ac4a4d13e4.json b/backend/.sqlx/query-6a17a1dfeb75808e5d0726f1f8bd573168161abe350b047402cda4ac4a4d13e4.json new file mode 100644 index 0000000000..6ced022760 --- /dev/null +++ b/backend/.sqlx/query-6a17a1dfeb75808e5d0726f1f8bd573168161abe350b047402cda4ac4a4d13e4.json @@ -0,0 +1,74 @@ +{ + "db_name": "PostgreSQL", + "query": "SELECT\n (CASE $3::text\n WHEN 'day' THEN day::text\n WHEN 'user' THEN email\n ELSE ''\n END) AS \"key!\",\n provider AS \"provider!\",\n model AS \"model!\",\n SUM(input_tokens)::bigint AS \"input_tokens!\",\n SUM(cache_read_tokens)::bigint AS \"cache_read_tokens!\",\n SUM(cache_write_tokens)::bigint AS \"cache_write_tokens!\",\n SUM(output_tokens)::bigint AS \"output_tokens!\",\n SUM(reported_cost_nano_usd)::bigint AS \"reported_cost_nano_usd\",\n SUM(requests)::bigint AS \"requests!\"\n FROM ai_token_usage\n WHERE workspace_id = $1 AND day > CURRENT_DATE - $2::int\n AND ($5::text IS NULL OR email = $5)\n GROUP BY 1, provider, model\n ORDER BY SUM(input_tokens + cache_read_tokens + cache_write_tokens + output_tokens) DESC\n LIMIT $4", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "key!", + "type_info": "Text" + }, + { + "ordinal": 1, + "name": "provider!", + "type_info": "Varchar" + }, + { + "ordinal": 2, + "name": "model!", + "type_info": "Varchar" + }, + { + "ordinal": 3, + "name": "input_tokens!", + "type_info": "Int8" + }, + { + "ordinal": 4, + "name": "cache_read_tokens!", + "type_info": "Int8" + }, + { + "ordinal": 5, + "name": "cache_write_tokens!", + "type_info": "Int8" + }, + { + "ordinal": 6, + "name": "output_tokens!", + "type_info": "Int8" + }, + { + "ordinal": 7, + "name": "reported_cost_nano_usd", + "type_info": "Int8" + }, + { + "ordinal": 8, + "name": "requests!", + "type_info": "Int8" + } + ], + "parameters": { + "Left": [ + "Text", + "Int4", + "Text", + "Int8", + "Text" + ] + }, + "nullable": [ + null, + false, + false, + null, + null, + null, + null, + null, + null + ] + }, + "hash": "6a17a1dfeb75808e5d0726f1f8bd573168161abe350b047402cda4ac4a4d13e4" +} diff --git a/backend/.sqlx/query-6bc9d682aabdf8e79beb693e0090d07f3514628a4970b5f25ddf3920ba5e9c53.json b/backend/.sqlx/query-6bc9d682aabdf8e79beb693e0090d07f3514628a4970b5f25ddf3920ba5e9c53.json new file mode 100644 index 0000000000..e2a97d9c11 --- /dev/null +++ b/backend/.sqlx/query-6bc9d682aabdf8e79beb693e0090d07f3514628a4970b5f25ddf3920ba5e9c53.json @@ -0,0 +1,23 @@ +{ + "db_name": "PostgreSQL", + "query": "SELECT scorers FROM eval_dataset WHERE workspace_id = $1 AND path = $2 FOR UPDATE", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "scorers", + "type_info": "Jsonb" + } + ], + "parameters": { + "Left": [ + "Text", + "Text" + ] + }, + "nullable": [ + false + ] + }, + "hash": "6bc9d682aabdf8e79beb693e0090d07f3514628a4970b5f25ddf3920ba5e9c53" +} diff --git a/backend/.sqlx/query-77db9fcef0d3f9c9eb9edba6c0a54469259621218616ef64a4923f3396b183a2.json b/backend/.sqlx/query-77db9fcef0d3f9c9eb9edba6c0a54469259621218616ef64a4923f3396b183a2.json new file mode 100644 index 0000000000..84bd3cc921 --- /dev/null +++ b/backend/.sqlx/query-77db9fcef0d3f9c9eb9edba6c0a54469259621218616ef64a4923f3396b183a2.json @@ -0,0 +1,14 @@ +{ + "db_name": "PostgreSQL", + "query": "DELETE FROM eval_experiment WHERE id = $1", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "Uuid" + ] + }, + "nullable": [] + }, + "hash": "77db9fcef0d3f9c9eb9edba6c0a54469259621218616ef64a4923f3396b183a2" +} diff --git a/backend/.sqlx/query-790e90a3aca284fd060aac049fcf27d2e39781fccc4dd8419f934085d5453fc7.json b/backend/.sqlx/query-790e90a3aca284fd060aac049fcf27d2e39781fccc4dd8419f934085d5453fc7.json new file mode 100644 index 0000000000..101f3e9703 --- /dev/null +++ b/backend/.sqlx/query-790e90a3aca284fd060aac049fcf27d2e39781fccc4dd8419f934085d5453fc7.json @@ -0,0 +1,20 @@ +{ + "db_name": "PostgreSQL", + "query": "INSERT INTO eval_experiment_case\n (experiment_id, ordinal, case_id, input, expected, subject_version,\n subject_draft_hash)\n SELECT $1, ordinal, case_id, input, expected, subject_version, subject_draft_hash\n FROM UNNEST($2::int[], $3::uuid[], $4::jsonb[], $5::jsonb[], $6::bigint[], $7::text[])\n AS t(ordinal, case_id, input, expected, subject_version, subject_draft_hash)", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "Uuid", + "Int4Array", + "UuidArray", + "JsonbArray", + "JsonbArray", + "Int8Array", + "TextArray" + ] + }, + "nullable": [] + }, + "hash": "790e90a3aca284fd060aac049fcf27d2e39781fccc4dd8419f934085d5453fc7" +} diff --git a/backend/.sqlx/query-7a14c6815a7acc912fd8836191263a373b5c427a0b2f48ca775fb8f5a0096e18.json b/backend/.sqlx/query-7a14c6815a7acc912fd8836191263a373b5c427a0b2f48ca775fb8f5a0096e18.json new file mode 100644 index 0000000000..ddcdb3a030 --- /dev/null +++ b/backend/.sqlx/query-7a14c6815a7acc912fd8836191263a373b5c427a0b2f48ca775fb8f5a0096e18.json @@ -0,0 +1,26 @@ +{ + "db_name": "PostgreSQL", + "query": "INSERT INTO eval_case\n (workspace_id, dataset_path, input, expected, created_by, created_at)\n VALUES ($1, $2, $3, $4, $5, clock_timestamp())\n RETURNING id", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "id", + "type_info": "Uuid" + } + ], + "parameters": { + "Left": [ + "Varchar", + "Varchar", + "Jsonb", + "Jsonb", + "Varchar" + ] + }, + "nullable": [ + false + ] + }, + "hash": "7a14c6815a7acc912fd8836191263a373b5c427a0b2f48ca775fb8f5a0096e18" +} diff --git a/backend/.sqlx/query-7b040feaa84e85fff1a5ad1ddfc3d57b0f69b05f96ffafae0325cf3ff04f0f40.json b/backend/.sqlx/query-7b040feaa84e85fff1a5ad1ddfc3d57b0f69b05f96ffafae0325cf3ff04f0f40.json new file mode 100644 index 0000000000..4665140ce9 --- /dev/null +++ b/backend/.sqlx/query-7b040feaa84e85fff1a5ad1ddfc3d57b0f69b05f96ffafae0325cf3ff04f0f40.json @@ -0,0 +1,60 @@ +{ + "db_name": "PostgreSQL", + "query": "SELECT s.experiment_id AS \"experiment_id!\", s.scorer_id AS \"scorer_id!\",\n avg(s.score) AS mean,\n count(s.score) AS \"scored!\",\n count(*) FILTER (WHERE s.error IS NOT NULL) AS \"failed!\",\n count(*) FILTER (WHERE t.pass_if IS NOT NULL AND s.score >= t.pass_if)\n AS \"passed!\",\n bool_or(t.pass_if IS NOT NULL) AS \"has_threshold!\"\n FROM eval_score s\n JOIN unnest($1::uuid[], $2::text[], $3::float8[])\n AS t(experiment_id, scorer_id, pass_if)\n ON t.experiment_id = s.experiment_id AND t.scorer_id = s.scorer_id\n GROUP BY s.experiment_id, s.scorer_id", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "experiment_id!", + "type_info": "Uuid" + }, + { + "ordinal": 1, + "name": "scorer_id!", + "type_info": "Varchar" + }, + { + "ordinal": 2, + "name": "mean", + "type_info": "Float8" + }, + { + "ordinal": 3, + "name": "scored!", + "type_info": "Int8" + }, + { + "ordinal": 4, + "name": "failed!", + "type_info": "Int8" + }, + { + "ordinal": 5, + "name": "passed!", + "type_info": "Int8" + }, + { + "ordinal": 6, + "name": "has_threshold!", + "type_info": "Bool" + } + ], + "parameters": { + "Left": [ + "UuidArray", + "TextArray", + "Float8Array" + ] + }, + "nullable": [ + false, + false, + null, + null, + null, + null, + null + ] + }, + "hash": "7b040feaa84e85fff1a5ad1ddfc3d57b0f69b05f96ffafae0325cf3ff04f0f40" +} diff --git a/backend/.sqlx/query-7f373cf063907999580d0541a1a4a216e3887fdc0a9bab0747a447c01a4b37a5.json b/backend/.sqlx/query-7f373cf063907999580d0541a1a4a216e3887fdc0a9bab0747a447c01a4b37a5.json new file mode 100644 index 0000000000..d73778feec --- /dev/null +++ b/backend/.sqlx/query-7f373cf063907999580d0541a1a4a216e3887fdc0a9bab0747a447c01a4b37a5.json @@ -0,0 +1,16 @@ +{ + "db_name": "PostgreSQL", + "query": "UPDATE eval_experiment_case\n SET subject_version = $3, subject_draft_hash = NULL\n WHERE experiment_id = $1 AND subject_draft_hash = $2", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "Uuid", + "Text", + "Int8" + ] + }, + "nullable": [] + }, + "hash": "7f373cf063907999580d0541a1a4a216e3887fdc0a9bab0747a447c01a4b37a5" +} diff --git a/backend/.sqlx/query-80a5dd06cc5b9f7cb0bcd3b8cdd92c62d40a4ab97fab0f2dd330bd7629c0a632.json b/backend/.sqlx/query-80a5dd06cc5b9f7cb0bcd3b8cdd92c62d40a4ab97fab0f2dd330bd7629c0a632.json new file mode 100644 index 0000000000..5b3af727e9 --- /dev/null +++ b/backend/.sqlx/query-80a5dd06cc5b9f7cb0bcd3b8cdd92c62d40a4ab97fab0f2dd330bd7629c0a632.json @@ -0,0 +1,64 @@ +{ + "db_name": "PostgreSQL", + "query": "SELECT ordinal, scorer_id, score, reason, checks, error, not_applicable, definition\n FROM eval_score WHERE experiment_id = $1", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "ordinal", + "type_info": "Int4" + }, + { + "ordinal": 1, + "name": "scorer_id", + "type_info": "Varchar" + }, + { + "ordinal": 2, + "name": "score", + "type_info": "Float8" + }, + { + "ordinal": 3, + "name": "reason", + "type_info": "Text" + }, + { + "ordinal": 4, + "name": "checks", + "type_info": "Jsonb" + }, + { + "ordinal": 5, + "name": "error", + "type_info": "Text" + }, + { + "ordinal": 6, + "name": "not_applicable", + "type_info": "Bool" + }, + { + "ordinal": 7, + "name": "definition", + "type_info": "Varchar" + } + ], + "parameters": { + "Left": [ + "Uuid" + ] + }, + "nullable": [ + false, + false, + true, + true, + true, + true, + false, + false + ] + }, + "hash": "80a5dd06cc5b9f7cb0bcd3b8cdd92c62d40a4ab97fab0f2dd330bd7629c0a632" +} diff --git a/backend/.sqlx/query-e27ed86394e6568afdf7a1dd72edd7943c87fdf1d05fba193cbe575cb7281db5.json b/backend/.sqlx/query-82b2e8383ae7e345e45fb8a1b2bb362439f5b12a3dd3290a8dab17ccf9c391f3.json similarity index 60% rename from backend/.sqlx/query-e27ed86394e6568afdf7a1dd72edd7943c87fdf1d05fba193cbe575cb7281db5.json rename to backend/.sqlx/query-82b2e8383ae7e345e45fb8a1b2bb362439f5b12a3dd3290a8dab17ccf9c391f3.json index 68825a19c7..a50a31419e 100644 --- a/backend/.sqlx/query-e27ed86394e6568afdf7a1dd72edd7943c87fdf1d05fba193cbe575cb7281db5.json +++ b/backend/.sqlx/query-82b2e8383ae7e345e45fb8a1b2bb362439f5b12a3dd3290a8dab17ccf9c391f3.json @@ -1,6 +1,6 @@ { "db_name": "PostgreSQL", - "query": "SELECT id, created_at, created_by FROM resource_version\n WHERE workspace_id = $1 AND path = $2 ORDER BY id DESC LIMIT $3", + "query": "SELECT id, version, created_at, created_by FROM resource_version\n WHERE workspace_id = $1 AND path = $2 ORDER BY id DESC LIMIT $3", "describe": { "columns": [ { @@ -10,11 +10,16 @@ }, { "ordinal": 1, + "name": "version", + "type_info": "Int8" + }, + { + "ordinal": 2, "name": "created_at", "type_info": "Timestamptz" }, { - "ordinal": 2, + "ordinal": 3, "name": "created_by", "type_info": "Varchar" } @@ -27,10 +32,11 @@ ] }, "nullable": [ + false, false, false, true ] }, - "hash": "e27ed86394e6568afdf7a1dd72edd7943c87fdf1d05fba193cbe575cb7281db5" + "hash": "82b2e8383ae7e345e45fb8a1b2bb362439f5b12a3dd3290a8dab17ccf9c391f3" } diff --git a/backend/.sqlx/query-858b5fe344d79913921d8c05e9e6219919f438e9a3ce0c3e996ea61d9b4f2ef8.json b/backend/.sqlx/query-858b5fe344d79913921d8c05e9e6219919f438e9a3ce0c3e996ea61d9b4f2ef8.json new file mode 100644 index 0000000000..0b8900d23a --- /dev/null +++ b/backend/.sqlx/query-858b5fe344d79913921d8c05e9e6219919f438e9a3ce0c3e996ea61d9b4f2ef8.json @@ -0,0 +1,16 @@ +{ + "db_name": "PostgreSQL", + "query": "UPDATE eval_dataset SET edited_by = $1 WHERE edited_by = $2 AND workspace_id = $3", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "Varchar", + "Text", + "Text" + ] + }, + "nullable": [] + }, + "hash": "858b5fe344d79913921d8c05e9e6219919f438e9a3ce0c3e996ea61d9b4f2ef8" +} diff --git a/backend/.sqlx/query-864184467477e73a45935bf9439ae60da7f9ecc12f8f98b926345a70f941845b.json b/backend/.sqlx/query-864184467477e73a45935bf9439ae60da7f9ecc12f8f98b926345a70f941845b.json new file mode 100644 index 0000000000..7da7df136e --- /dev/null +++ b/backend/.sqlx/query-864184467477e73a45935bf9439ae60da7f9ecc12f8f98b926345a70f941845b.json @@ -0,0 +1,24 @@ +{ + "db_name": "PostgreSQL", + "query": "SELECT value FROM resource_version\n WHERE version = $1 AND workspace_id = $2 AND path = $3", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "value", + "type_info": "Jsonb" + } + ], + "parameters": { + "Left": [ + "Int8", + "Text", + "Text" + ] + }, + "nullable": [ + true + ] + }, + "hash": "864184467477e73a45935bf9439ae60da7f9ecc12f8f98b926345a70f941845b" +} diff --git a/backend/.sqlx/query-897e8da49e4a5c3efdfb2c36fc7aff3d53432da9ef57f8a775850efcd50a965c.json b/backend/.sqlx/query-897e8da49e4a5c3efdfb2c36fc7aff3d53432da9ef57f8a775850efcd50a965c.json new file mode 100644 index 0000000000..8610e3372b --- /dev/null +++ b/backend/.sqlx/query-897e8da49e4a5c3efdfb2c36fc7aff3d53432da9ef57f8a775850efcd50a965c.json @@ -0,0 +1,29 @@ +{ + "db_name": "PostgreSQL", + "query": "SELECT r.value AS \"value: sqlx::types::Json\",\n (SELECT version FROM resource_version v\n WHERE v.workspace_id = r.workspace_id AND v.path = r.path\n ORDER BY v.version DESC LIMIT 1) AS version\n FROM resource r\n WHERE r.workspace_id = $1 AND r.path = $2 AND r.resource_type = 'ai_agent'", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "value: sqlx::types::Json", + "type_info": "Jsonb" + }, + { + "ordinal": 1, + "name": "version", + "type_info": "Int8" + } + ], + "parameters": { + "Left": [ + "Text", + "Text" + ] + }, + "nullable": [ + true, + null + ] + }, + "hash": "897e8da49e4a5c3efdfb2c36fc7aff3d53432da9ef57f8a775850efcd50a965c" +} diff --git a/backend/.sqlx/query-8c319ac3eb2a289a6709ea1ea1c164f1a56e328902dd46fd9b68d801c16f4ffa.json b/backend/.sqlx/query-8c319ac3eb2a289a6709ea1ea1c164f1a56e328902dd46fd9b68d801c16f4ffa.json new file mode 100644 index 0000000000..f60f76b002 --- /dev/null +++ b/backend/.sqlx/query-8c319ac3eb2a289a6709ea1ea1c164f1a56e328902dd46fd9b68d801c16f4ffa.json @@ -0,0 +1,16 @@ +{ + "db_name": "PostgreSQL", + "query": "UPDATE eval_dataset SET extra_perms = extra_perms - ('u/' || $2) || jsonb_build_object(('u/' || $1), extra_perms->('u/' || $2)) WHERE extra_perms ? ('u/' || $2) AND workspace_id = $3", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "Text", + "Text", + "Text" + ] + }, + "nullable": [] + }, + "hash": "8c319ac3eb2a289a6709ea1ea1c164f1a56e328902dd46fd9b68d801c16f4ffa" +} diff --git a/backend/.sqlx/query-9071eff54395ca39809a5d55d1e3fedb64cfafa9fe74cd9446eb6ee526c2d9a0.json b/backend/.sqlx/query-9071eff54395ca39809a5d55d1e3fedb64cfafa9fe74cd9446eb6ee526c2d9a0.json new file mode 100644 index 0000000000..dd4a781db0 --- /dev/null +++ b/backend/.sqlx/query-9071eff54395ca39809a5d55d1e3fedb64cfafa9fe74cd9446eb6ee526c2d9a0.json @@ -0,0 +1,58 @@ +{ + "db_name": "PostgreSQL", + "query": "SELECT path, summary, scorers, created_at, created_by,\n edited_at, edited_by\n FROM eval_dataset WHERE workspace_id = $1 ORDER BY path", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "path", + "type_info": "Varchar" + }, + { + "ordinal": 1, + "name": "summary", + "type_info": "Varchar" + }, + { + "ordinal": 2, + "name": "scorers", + "type_info": "Jsonb" + }, + { + "ordinal": 3, + "name": "created_at", + "type_info": "Timestamptz" + }, + { + "ordinal": 4, + "name": "created_by", + "type_info": "Varchar" + }, + { + "ordinal": 5, + "name": "edited_at", + "type_info": "Timestamptz" + }, + { + "ordinal": 6, + "name": "edited_by", + "type_info": "Varchar" + } + ], + "parameters": { + "Left": [ + "Text" + ] + }, + "nullable": [ + false, + true, + false, + false, + false, + false, + false + ] + }, + "hash": "9071eff54395ca39809a5d55d1e3fedb64cfafa9fe74cd9446eb6ee526c2d9a0" +} diff --git a/backend/.sqlx/query-97a701e16c0ce4b8c6a1394c2a71b8ae45e16c11c8003c5f64c9f53d0e7226aa.json b/backend/.sqlx/query-97a701e16c0ce4b8c6a1394c2a71b8ae45e16c11c8003c5f64c9f53d0e7226aa.json new file mode 100644 index 0000000000..d068fac4f4 --- /dev/null +++ b/backend/.sqlx/query-97a701e16c0ce4b8c6a1394c2a71b8ae45e16c11c8003c5f64c9f53d0e7226aa.json @@ -0,0 +1,23 @@ +{ + "db_name": "PostgreSQL", + "query": "SELECT COALESCE(s.flow_status, c.flow_status) -> 'modules' AS modules\n FROM v2_job j\n LEFT JOIN v2_job_status s ON s.id = j.id\n LEFT JOIN v2_job_completed c ON c.id = j.id\n WHERE j.id = $1 AND j.workspace_id = $2", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "modules", + "type_info": "Jsonb" + } + ], + "parameters": { + "Left": [ + "Uuid", + "Text" + ] + }, + "nullable": [ + null + ] + }, + "hash": "97a701e16c0ce4b8c6a1394c2a71b8ae45e16c11c8003c5f64c9f53d0e7226aa" +} diff --git a/backend/.sqlx/query-9b1ad1bbf0c2dca3ce1cc9433c35db87e3101bcbcb0fa359885ca3807c1c9ca2.json b/backend/.sqlx/query-9b1ad1bbf0c2dca3ce1cc9433c35db87e3101bcbcb0fa359885ca3807c1c9ca2.json new file mode 100644 index 0000000000..bc6e2eb971 --- /dev/null +++ b/backend/.sqlx/query-9b1ad1bbf0c2dca3ce1cc9433c35db87e3101bcbcb0fa359885ca3807c1c9ca2.json @@ -0,0 +1,23 @@ +{ + "db_name": "PostgreSQL", + "query": "SELECT j.id AS \"id!\" FROM v2_job j\n LEFT JOIN v2_job_completed c ON c.id = j.id AND c.workspace_id = $2\n WHERE j.id = ANY($1) AND j.workspace_id = $2 AND c.id IS NULL", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "id!", + "type_info": "Uuid" + } + ], + "parameters": { + "Left": [ + "UuidArray", + "Text" + ] + }, + "nullable": [ + false + ] + }, + "hash": "9b1ad1bbf0c2dca3ce1cc9433c35db87e3101bcbcb0fa359885ca3807c1c9ca2" +} diff --git a/backend/.sqlx/query-9d3ca755b323330033eb891ac7162a53440a5c713b58e6c4a467f6f0d5d7e37c.json b/backend/.sqlx/query-9d3ca755b323330033eb891ac7162a53440a5c713b58e6c4a467f6f0d5d7e37c.json new file mode 100644 index 0000000000..1c6b02aa9f --- /dev/null +++ b/backend/.sqlx/query-9d3ca755b323330033eb891ac7162a53440a5c713b58e6c4a467f6f0d5d7e37c.json @@ -0,0 +1,16 @@ +{ + "db_name": "PostgreSQL", + "query": "UPDATE eval_dataset SET scorers = COALESCE((\n SELECT jsonb_agg(\n CASE WHEN elem->>'path' LIKE ('u/' || $2 || '/%')\n THEN jsonb_set(elem, '{path}', to_jsonb(REGEXP_REPLACE(elem->>'path','u/' || $2 || '/(.*)','u/' || $1 || '/\\1')))\n ELSE elem END)\n FROM jsonb_array_elements(scorers) elem), '[]'::jsonb)\n WHERE workspace_id = $3\n AND EXISTS (SELECT 1 FROM jsonb_array_elements(scorers) e WHERE e->>'path' LIKE ('u/' || $2 || '/%'))", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "Text", + "Text", + "Text" + ] + }, + "nullable": [] + }, + "hash": "9d3ca755b323330033eb891ac7162a53440a5c713b58e6c4a467f6f0d5d7e37c" +} diff --git a/backend/.sqlx/query-a4c842e395714346d5178190793d8ae66f891421b72fc6adfa8a38060ca2f3fa.json b/backend/.sqlx/query-a4c842e395714346d5178190793d8ae66f891421b72fc6adfa8a38060ca2f3fa.json new file mode 100644 index 0000000000..23e4e95b2f --- /dev/null +++ b/backend/.sqlx/query-a4c842e395714346d5178190793d8ae66f891421b72fc6adfa8a38060ca2f3fa.json @@ -0,0 +1,15 @@ +{ + "db_name": "PostgreSQL", + "query": "INSERT INTO eval_case (workspace_id, dataset_path, input, expected, created_at, created_by)\n SELECT $2, dataset_path, input, expected, created_at, created_by\n FROM eval_case WHERE workspace_id = $1", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "Text", + "Varchar" + ] + }, + "nullable": [] + }, + "hash": "a4c842e395714346d5178190793d8ae66f891421b72fc6adfa8a38060ca2f3fa" +} diff --git a/backend/.sqlx/query-a5e8cf0e559742330d67d36d9ddb75ef7de6e0ad74e0cc6c4056b782155b86c7.json b/backend/.sqlx/query-a5e8cf0e559742330d67d36d9ddb75ef7de6e0ad74e0cc6c4056b782155b86c7.json new file mode 100644 index 0000000000..d4b1e174da --- /dev/null +++ b/backend/.sqlx/query-a5e8cf0e559742330d67d36d9ddb75ef7de6e0ad74e0cc6c4056b782155b86c7.json @@ -0,0 +1,53 @@ +{ + "db_name": "PostgreSQL", + "query": "SELECT s.ordinal, s.scorer_id, c.job_id AS \"job_id!\", d.status::text AS status,\n c.answered, (j.id IS NOT NULL) AS \"job_exists!\"\n FROM eval_score s\n JOIN eval_experiment_case c\n ON c.experiment_id = s.experiment_id AND c.ordinal = s.ordinal\n LEFT JOIN v2_job j ON j.id = c.job_id AND j.workspace_id = $2\n LEFT JOIN v2_job_completed d ON d.id = c.job_id AND d.workspace_id = $2\n WHERE s.experiment_id = $1 AND s.score IS NULL AND s.error IS NULL\n AND NOT s.not_applicable AND c.job_id IS NOT NULL", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "ordinal", + "type_info": "Int4" + }, + { + "ordinal": 1, + "name": "scorer_id", + "type_info": "Varchar" + }, + { + "ordinal": 2, + "name": "job_id!", + "type_info": "Uuid" + }, + { + "ordinal": 3, + "name": "status", + "type_info": "Text" + }, + { + "ordinal": 4, + "name": "answered", + "type_info": "Bool" + }, + { + "ordinal": 5, + "name": "job_exists!", + "type_info": "Bool" + } + ], + "parameters": { + "Left": [ + "Uuid", + "Text" + ] + }, + "nullable": [ + false, + false, + true, + null, + true, + null + ] + }, + "hash": "a5e8cf0e559742330d67d36d9ddb75ef7de6e0ad74e0cc6c4056b782155b86c7" +} diff --git a/backend/.sqlx/query-a7b589b8d5cded97905bc24412ad8f3f0339a2a088243f045a212f9357205407.json b/backend/.sqlx/query-a7b589b8d5cded97905bc24412ad8f3f0339a2a088243f045a212f9357205407.json new file mode 100644 index 0000000000..fb8f4593ab --- /dev/null +++ b/backend/.sqlx/query-a7b589b8d5cded97905bc24412ad8f3f0339a2a088243f045a212f9357205407.json @@ -0,0 +1,27 @@ +{ + "db_name": "PostgreSQL", + "query": "UPDATE eval_dataset\n SET path = COALESCE($6, path), summary = COALESCE($3, summary),\n scorers = COALESCE($4, scorers), edited_at = now(), edited_by = $5\n WHERE workspace_id = $1 AND path = $2\n RETURNING path", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "path", + "type_info": "Varchar" + } + ], + "parameters": { + "Left": [ + "Text", + "Text", + "Varchar", + "Jsonb", + "Varchar", + "Varchar" + ] + }, + "nullable": [ + false + ] + }, + "hash": "a7b589b8d5cded97905bc24412ad8f3f0339a2a088243f045a212f9357205407" +} diff --git a/backend/.sqlx/query-b2db4f32c615a99db7af23729682340328da39ac763bb26b6bcf82027fad48e5.json b/backend/.sqlx/query-b2db4f32c615a99db7af23729682340328da39ac763bb26b6bcf82027fad48e5.json new file mode 100644 index 0000000000..e0f8e62e11 --- /dev/null +++ b/backend/.sqlx/query-b2db4f32c615a99db7af23729682340328da39ac763bb26b6bcf82027fad48e5.json @@ -0,0 +1,29 @@ +{ + "db_name": "PostgreSQL", + "query": "SELECT id, run_job_id FROM eval_experiment WHERE workspace_id = $1 AND id = $2", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "id", + "type_info": "Uuid" + }, + { + "ordinal": 1, + "name": "run_job_id", + "type_info": "Uuid" + } + ], + "parameters": { + "Left": [ + "Text", + "Uuid" + ] + }, + "nullable": [ + false, + false + ] + }, + "hash": "b2db4f32c615a99db7af23729682340328da39ac763bb26b6bcf82027fad48e5" +} diff --git a/backend/.sqlx/query-b8e732a03969666444f73397ac153d23ec2af084b2f93da24c920532c1916384.json b/backend/.sqlx/query-b8e732a03969666444f73397ac153d23ec2af084b2f93da24c920532c1916384.json index b892061f56..b336210daf 100644 --- a/backend/.sqlx/query-b8e732a03969666444f73397ac153d23ec2af084b2f93da24c920532c1916384.json +++ b/backend/.sqlx/query-b8e732a03969666444f73397ac153d23ec2af084b2f93da24c920532c1916384.json @@ -98,12 +98,12 @@ null, null, null, - false, + true, null, null, null, - false, - false + true, + true ] }, "hash": "b8e732a03969666444f73397ac153d23ec2af084b2f93da24c920532c1916384" diff --git a/backend/.sqlx/query-bac36542b16b687a823067c013e3728f1cabe3774f581401514f522ea7522de5.json b/backend/.sqlx/query-bac36542b16b687a823067c013e3728f1cabe3774f581401514f522ea7522de5.json new file mode 100644 index 0000000000..b680e889c6 --- /dev/null +++ b/backend/.sqlx/query-bac36542b16b687a823067c013e3728f1cabe3774f581401514f522ea7522de5.json @@ -0,0 +1,29 @@ +{ + "db_name": "PostgreSQL", + "query": "SELECT status::text AS \"status!\", duration_ms FROM v2_job_completed\n WHERE id = $1 AND workspace_id = $2", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "status!", + "type_info": "Text" + }, + { + "ordinal": 1, + "name": "duration_ms", + "type_info": "Int8" + } + ], + "parameters": { + "Left": [ + "Uuid", + "Text" + ] + }, + "nullable": [ + null, + false + ] + }, + "hash": "bac36542b16b687a823067c013e3728f1cabe3774f581401514f522ea7522de5" +} diff --git a/backend/.sqlx/query-bbce2221f5724016543a5ac1db7fa74b8b35375dfb0f5d39b398369468ad0774.json b/backend/.sqlx/query-bbce2221f5724016543a5ac1db7fa74b8b35375dfb0f5d39b398369468ad0774.json new file mode 100644 index 0000000000..24ea423095 --- /dev/null +++ b/backend/.sqlx/query-bbce2221f5724016543a5ac1db7fa74b8b35375dfb0f5d39b398369468ad0774.json @@ -0,0 +1,54 @@ +{ + "db_name": "PostgreSQL", + "query": "SELECT e.subject, e.run_number, e.run_job_id, e.created_at,\n e.created_by,\n (SELECT count(*) FROM eval_experiment_case c WHERE c.experiment_id = e.id)\n AS \"case_count!\"\n FROM eval_experiment e\n WHERE e.workspace_id = $1 AND e.dataset_path = $2 AND e.id = $3", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "subject", + "type_info": "Jsonb" + }, + { + "ordinal": 1, + "name": "run_number", + "type_info": "Int4" + }, + { + "ordinal": 2, + "name": "run_job_id", + "type_info": "Uuid" + }, + { + "ordinal": 3, + "name": "created_at", + "type_info": "Timestamptz" + }, + { + "ordinal": 4, + "name": "created_by", + "type_info": "Varchar" + }, + { + "ordinal": 5, + "name": "case_count!", + "type_info": "Int8" + } + ], + "parameters": { + "Left": [ + "Text", + "Text", + "Uuid" + ] + }, + "nullable": [ + false, + false, + false, + false, + false, + null + ] + }, + "hash": "bbce2221f5724016543a5ac1db7fa74b8b35375dfb0f5d39b398369468ad0774" +} diff --git a/backend/.sqlx/query-bdb7185233941d1472a55c8ade3f61c82248f3f2c9a69c458bff3978ffd17379.json b/backend/.sqlx/query-bdb7185233941d1472a55c8ade3f61c82248f3f2c9a69c458bff3978ffd17379.json new file mode 100644 index 0000000000..0e31540e7c --- /dev/null +++ b/backend/.sqlx/query-bdb7185233941d1472a55c8ade3f61c82248f3f2c9a69c458bff3978ffd17379.json @@ -0,0 +1,66 @@ +{ + "db_name": "PostgreSQL", + "query": "SELECT e.id, e.dataset_path, e.subject, e.run_number, e.run_job_id, e.created_at,\n e.created_by,\n (SELECT count(*) FROM eval_experiment_case c WHERE c.experiment_id = e.id)\n AS \"case_count!\"\n FROM eval_experiment e\n JOIN eval_dataset d ON d.workspace_id = e.workspace_id AND d.path = e.dataset_path\n WHERE e.workspace_id = $1\n AND ($3::text IS NULL OR e.subject ->> 'path' = $3)\n ORDER BY e.created_at DESC\n LIMIT $2", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "id", + "type_info": "Uuid" + }, + { + "ordinal": 1, + "name": "dataset_path", + "type_info": "Varchar" + }, + { + "ordinal": 2, + "name": "subject", + "type_info": "Jsonb" + }, + { + "ordinal": 3, + "name": "run_number", + "type_info": "Int4" + }, + { + "ordinal": 4, + "name": "run_job_id", + "type_info": "Uuid" + }, + { + "ordinal": 5, + "name": "created_at", + "type_info": "Timestamptz" + }, + { + "ordinal": 6, + "name": "created_by", + "type_info": "Varchar" + }, + { + "ordinal": 7, + "name": "case_count!", + "type_info": "Int8" + } + ], + "parameters": { + "Left": [ + "Text", + "Int8", + "Text" + ] + }, + "nullable": [ + false, + false, + false, + false, + false, + false, + false, + null + ] + }, + "hash": "bdb7185233941d1472a55c8ade3f61c82248f3f2c9a69c458bff3978ffd17379" +} diff --git a/backend/.sqlx/query-be17a65f144cc21e849c7f8cfbf9d7271b953dda42e4ba09864da7bda731e752.json b/backend/.sqlx/query-be17a65f144cc21e849c7f8cfbf9d7271b953dda42e4ba09864da7bda731e752.json new file mode 100644 index 0000000000..f28916165f --- /dev/null +++ b/backend/.sqlx/query-be17a65f144cc21e849c7f8cfbf9d7271b953dda42e4ba09864da7bda731e752.json @@ -0,0 +1,54 @@ +{ + "db_name": "PostgreSQL", + "query": "SELECT j.id, j.args AS \"args: sqlx::types::Json>\",\n c.result AS \"result: sqlx::types::Json>\",\n c.status::text AS status, c.duration_ms,\n s.schema AS \"schema: sqlx::types::Json>\"\n FROM v2_job j\n LEFT JOIN v2_job_completed c ON c.id = j.id\n LEFT JOIN script s ON s.workspace_id = j.workspace_id AND s.hash = j.runnable_id\n WHERE j.id = ANY($1) AND j.workspace_id = $2 AND j.parent_job = $3", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "id", + "type_info": "Uuid" + }, + { + "ordinal": 1, + "name": "args: sqlx::types::Json>", + "type_info": "Jsonb" + }, + { + "ordinal": 2, + "name": "result: sqlx::types::Json>", + "type_info": "Jsonb" + }, + { + "ordinal": 3, + "name": "status", + "type_info": "Text" + }, + { + "ordinal": 4, + "name": "duration_ms", + "type_info": "Int8" + }, + { + "ordinal": 5, + "name": "schema: sqlx::types::Json>", + "type_info": "Json" + } + ], + "parameters": { + "Left": [ + "UuidArray", + "Text", + "Uuid" + ] + }, + "nullable": [ + false, + true, + true, + null, + false, + true + ] + }, + "hash": "be17a65f144cc21e849c7f8cfbf9d7271b953dda42e4ba09864da7bda731e752" +} diff --git a/backend/.sqlx/query-d10efb37765ac9a7f2e15f71dbbc02ec917b6ecb014f938e68ced3cc7fe9dc86.json b/backend/.sqlx/query-d10efb37765ac9a7f2e15f71dbbc02ec917b6ecb014f938e68ced3cc7fe9dc86.json new file mode 100644 index 0000000000..c124b8c3cb --- /dev/null +++ b/backend/.sqlx/query-d10efb37765ac9a7f2e15f71dbbc02ec917b6ecb014f938e68ced3cc7fe9dc86.json @@ -0,0 +1,16 @@ +{ + "db_name": "PostgreSQL", + "query": "UPDATE eval_experiment SET subject = jsonb_set(subject, '{path}', to_jsonb(REGEXP_REPLACE(subject->>'path','u/' || $2 || '/(.*)','u/' || $1 || '/\\1'))) WHERE subject->>'path' LIKE ('u/' || $2 || '/%') AND workspace_id = $3", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "Text", + "Text", + "Text" + ] + }, + "nullable": [] + }, + "hash": "d10efb37765ac9a7f2e15f71dbbc02ec917b6ecb014f938e68ced3cc7fe9dc86" +} diff --git a/backend/.sqlx/query-d5cdb1121f2a414c0a70f4f7e4ae630bad0715bbbb3597cb1b14e41abe9fec14.json b/backend/.sqlx/query-d5cdb1121f2a414c0a70f4f7e4ae630bad0715bbbb3597cb1b14e41abe9fec14.json new file mode 100644 index 0000000000..ad292f64cd --- /dev/null +++ b/backend/.sqlx/query-d5cdb1121f2a414c0a70f4f7e4ae630bad0715bbbb3597cb1b14e41abe9fec14.json @@ -0,0 +1,23 @@ +{ + "db_name": "PostgreSQL", + "query": "UPDATE eval_experiment_case SET status = $2, answered = false\n WHERE experiment_id = $1 AND job_id IS NULL AND status IS NULL\n RETURNING ordinal", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "ordinal", + "type_info": "Int4" + } + ], + "parameters": { + "Left": [ + "Uuid", + "Varchar" + ] + }, + "nullable": [ + false + ] + }, + "hash": "d5cdb1121f2a414c0a70f4f7e4ae630bad0715bbbb3597cb1b14e41abe9fec14" +} diff --git a/backend/.sqlx/query-db19932d940b2467147eb2c13b059f2ec33b0c83b5c4a332a4b6e54d2591767c.json b/backend/.sqlx/query-db19932d940b2467147eb2c13b059f2ec33b0c83b5c4a332a4b6e54d2591767c.json new file mode 100644 index 0000000000..1595cec076 --- /dev/null +++ b/backend/.sqlx/query-db19932d940b2467147eb2c13b059f2ec33b0c83b5c4a332a4b6e54d2591767c.json @@ -0,0 +1,18 @@ +{ + "db_name": "PostgreSQL", + "query": "INSERT INTO eval_case\n (workspace_id, dataset_path, input, expected, created_by, created_at)\n VALUES ($1, $2, $3, $4, $5, clock_timestamp())", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "Varchar", + "Varchar", + "Jsonb", + "Jsonb", + "Varchar" + ] + }, + "nullable": [] + }, + "hash": "db19932d940b2467147eb2c13b059f2ec33b0c83b5c4a332a4b6e54d2591767c" +} diff --git a/backend/.sqlx/query-db63d41718c50e264a949885d2d7c13719ae650e9c788ebf125627b785fb9ee0.json b/backend/.sqlx/query-db63d41718c50e264a949885d2d7c13719ae650e9c788ebf125627b785fb9ee0.json new file mode 100644 index 0000000000..4602a03f61 --- /dev/null +++ b/backend/.sqlx/query-db63d41718c50e264a949885d2d7c13719ae650e9c788ebf125627b785fb9ee0.json @@ -0,0 +1,23 @@ +{ + "db_name": "PostgreSQL", + "query": "SELECT path FROM eval_dataset WHERE workspace_id = $1 AND path = $2", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "path", + "type_info": "Varchar" + } + ], + "parameters": { + "Left": [ + "Text", + "Text" + ] + }, + "nullable": [ + false + ] + }, + "hash": "db63d41718c50e264a949885d2d7c13719ae650e9c788ebf125627b785fb9ee0" +} diff --git a/backend/.sqlx/query-e3c080b84f50622e0a74524111ae292c958e71293695ce18f9905af4dd940495.json b/backend/.sqlx/query-e3c080b84f50622e0a74524111ae292c958e71293695ce18f9905af4dd940495.json new file mode 100644 index 0000000000..ace8edc731 --- /dev/null +++ b/backend/.sqlx/query-e3c080b84f50622e0a74524111ae292c958e71293695ce18f9905af4dd940495.json @@ -0,0 +1,49 @@ +{ + "db_name": "PostgreSQL", + "query": "SELECT id, input, expected, created_at, created_by\n FROM eval_case\n WHERE workspace_id = $1 AND dataset_path = $2\n ORDER BY created_at, id\n LIMIT $3 OFFSET $4", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "id", + "type_info": "Uuid" + }, + { + "ordinal": 1, + "name": "input", + "type_info": "Jsonb" + }, + { + "ordinal": 2, + "name": "expected", + "type_info": "Jsonb" + }, + { + "ordinal": 3, + "name": "created_at", + "type_info": "Timestamptz" + }, + { + "ordinal": 4, + "name": "created_by", + "type_info": "Varchar" + } + ], + "parameters": { + "Left": [ + "Text", + "Text", + "Int8", + "Int8" + ] + }, + "nullable": [ + false, + false, + true, + false, + false + ] + }, + "hash": "e3c080b84f50622e0a74524111ae292c958e71293695ce18f9905af4dd940495" +} diff --git a/backend/.sqlx/query-e6d7e9779eaa6e584b613675ae8602c405fe620937a4563a17db80219a2930d2.json b/backend/.sqlx/query-e6d7e9779eaa6e584b613675ae8602c405fe620937a4563a17db80219a2930d2.json new file mode 100644 index 0000000000..d0e2c29c2d --- /dev/null +++ b/backend/.sqlx/query-e6d7e9779eaa6e584b613675ae8602c405fe620937a4563a17db80219a2930d2.json @@ -0,0 +1,23 @@ +{ + "db_name": "PostgreSQL", + "query": "SELECT args AS \"args: sqlx::types::Json>\" FROM v2_job\n WHERE id = $1 AND workspace_id = $2", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "args: sqlx::types::Json>", + "type_info": "Jsonb" + } + ], + "parameters": { + "Left": [ + "Uuid", + "Text" + ] + }, + "nullable": [ + true + ] + }, + "hash": "e6d7e9779eaa6e584b613675ae8602c405fe620937a4563a17db80219a2930d2" +} diff --git a/backend/.sqlx/query-e719e98cceef1383f882632c9398dd950ad0b1f30cde9c0392384c9a490290b9.json b/backend/.sqlx/query-e719e98cceef1383f882632c9398dd950ad0b1f30cde9c0392384c9a490290b9.json new file mode 100644 index 0000000000..b4f0820065 --- /dev/null +++ b/backend/.sqlx/query-e719e98cceef1383f882632c9398dd950ad0b1f30cde9c0392384c9a490290b9.json @@ -0,0 +1,17 @@ +{ + "db_name": "PostgreSQL", + "query": "INSERT INTO eval_score (experiment_id, ordinal, scorer_id, definition)\n SELECT $1, ordinal, scorer_id, definition\n FROM UNNEST($2::int[], $3::text[], $4::text[]) AS t(ordinal, scorer_id, definition)\n ON CONFLICT (experiment_id, ordinal, scorer_id)\n DO UPDATE SET definition = EXCLUDED.definition, score = NULL, reason = NULL,\n checks = NULL, error = NULL, not_applicable = false", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "Uuid", + "Int4Array", + "TextArray", + "TextArray" + ] + }, + "nullable": [] + }, + "hash": "e719e98cceef1383f882632c9398dd950ad0b1f30cde9c0392384c9a490290b9" +} diff --git a/backend/.sqlx/query-e72d71852b6c5998b8d46407d164fcee17400a0ffc172d2a257c3fc15617fbd9.json b/backend/.sqlx/query-e72d71852b6c5998b8d46407d164fcee17400a0ffc172d2a257c3fc15617fbd9.json new file mode 100644 index 0000000000..5ce8f9f4c6 --- /dev/null +++ b/backend/.sqlx/query-e72d71852b6c5998b8d46407d164fcee17400a0ffc172d2a257c3fc15617fbd9.json @@ -0,0 +1,41 @@ +{ + "db_name": "PostgreSQL", + "query": "SELECT c.ordinal, c.job_id AS \"job_id!\", d.status::text AS status,\n (j.id IS NOT NULL) AS \"job_exists!\"\n FROM eval_experiment_case c\n LEFT JOIN v2_job j ON j.id = c.job_id AND j.workspace_id = $2\n LEFT JOIN v2_job_completed d ON d.id = c.job_id AND d.workspace_id = $2\n WHERE c.experiment_id = $1 AND c.job_id IS NOT NULL AND c.status IS NULL", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "ordinal", + "type_info": "Int4" + }, + { + "ordinal": 1, + "name": "job_id!", + "type_info": "Uuid" + }, + { + "ordinal": 2, + "name": "status", + "type_info": "Text" + }, + { + "ordinal": 3, + "name": "job_exists!", + "type_info": "Bool" + } + ], + "parameters": { + "Left": [ + "Uuid", + "Text" + ] + }, + "nullable": [ + false, + true, + null, + null + ] + }, + "hash": "e72d71852b6c5998b8d46407d164fcee17400a0ffc172d2a257c3fc15617fbd9" +} diff --git a/backend/.sqlx/query-e942a74104771b192c33ea03dbcaa409e03b6bfb7423929c201f39db4228cc36.json b/backend/.sqlx/query-e942a74104771b192c33ea03dbcaa409e03b6bfb7423929c201f39db4228cc36.json new file mode 100644 index 0000000000..9daa8a1851 --- /dev/null +++ b/backend/.sqlx/query-e942a74104771b192c33ea03dbcaa409e03b6bfb7423929c201f39db4228cc36.json @@ -0,0 +1,23 @@ +{ + "db_name": "PostgreSQL", + "query": "DELETE FROM eval_dataset WHERE workspace_id = $1 AND path = $2 RETURNING path", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "path", + "type_info": "Varchar" + } + ], + "parameters": { + "Left": [ + "Text", + "Text" + ] + }, + "nullable": [ + false + ] + }, + "hash": "e942a74104771b192c33ea03dbcaa409e03b6bfb7423929c201f39db4228cc36" +} diff --git a/backend/.sqlx/query-eb0f25a10f4f1264e482674c06783204fddf0ccdca6d7c798a4ff6527548e96f.json b/backend/.sqlx/query-eb0f25a10f4f1264e482674c06783204fddf0ccdca6d7c798a4ff6527548e96f.json new file mode 100644 index 0000000000..9ed995c409 --- /dev/null +++ b/backend/.sqlx/query-eb0f25a10f4f1264e482674c06783204fddf0ccdca6d7c798a4ff6527548e96f.json @@ -0,0 +1,20 @@ +{ + "db_name": "PostgreSQL", + "query": "INSERT INTO eval_experiment\n (id, workspace_id, dataset_path, subject, run_number, created_by, run_job_id)\n VALUES ($1, $2, $3, $4, $5, $6, $7)", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "Uuid", + "Varchar", + "Varchar", + "Jsonb", + "Int4", + "Varchar", + "Uuid" + ] + }, + "nullable": [] + }, + "hash": "eb0f25a10f4f1264e482674c06783204fddf0ccdca6d7c798a4ff6527548e96f" +} diff --git a/backend/.sqlx/query-ee40e48afb5520b7ff84883204f064ce566cea6655e3cd11a9d7016cb08482cb.json b/backend/.sqlx/query-ee40e48afb5520b7ff84883204f064ce566cea6655e3cd11a9d7016cb08482cb.json new file mode 100644 index 0000000000..2280a368ea --- /dev/null +++ b/backend/.sqlx/query-ee40e48afb5520b7ff84883204f064ce566cea6655e3cd11a9d7016cb08482cb.json @@ -0,0 +1,21 @@ +{ + "db_name": "PostgreSQL", + "query": "UPDATE eval_score s\n SET score = t.score, reason = t.reason, checks = t.checks, error = t.error,\n not_applicable = t.not_applicable\n FROM UNNEST($2::int[], $3::text[], $4::double precision[], $5::text[], $6::jsonb[],\n $7::text[], $8::bool[])\n AS t(ordinal, scorer_id, score, reason, checks, error, not_applicable)\n WHERE s.experiment_id = $1 AND s.ordinal = t.ordinal AND s.scorer_id = t.scorer_id", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "Uuid", + "Int4Array", + "TextArray", + "Float8Array", + "TextArray", + "JsonbArray", + "TextArray", + "BoolArray" + ] + }, + "nullable": [] + }, + "hash": "ee40e48afb5520b7ff84883204f064ce566cea6655e3cd11a9d7016cb08482cb" +} diff --git a/backend/.sqlx/query-f0e943244b125d0c42a9b472701ec172af9dd334786275c391e80f1c38bbb45b.json b/backend/.sqlx/query-f0e943244b125d0c42a9b472701ec172af9dd334786275c391e80f1c38bbb45b.json new file mode 100644 index 0000000000..5e0fef8520 --- /dev/null +++ b/backend/.sqlx/query-f0e943244b125d0c42a9b472701ec172af9dd334786275c391e80f1c38bbb45b.json @@ -0,0 +1,18 @@ +{ + "db_name": "PostgreSQL", + "query": "UPDATE eval_experiment_case c\n SET output = COALESCE(c.output, t.output), answered = COALESCE(c.answered, t.answered),\n status = COALESCE(c.status, t.status)\n FROM UNNEST($2::int[], $3::text[], $4::bool[], $5::text[])\n AS t(ordinal, output, answered, status)\n WHERE c.experiment_id = $1 AND c.ordinal = t.ordinal", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "Uuid", + "Int4Array", + "TextArray", + "BoolArray", + "TextArray" + ] + }, + "nullable": [] + }, + "hash": "f0e943244b125d0c42a9b472701ec172af9dd334786275c391e80f1c38bbb45b" +} diff --git a/backend/.sqlx/query-f59afd524e3f216487ad0a780b1fda6341e215025dc6d8985891b91dad6d9dcb.json b/backend/.sqlx/query-f59afd524e3f216487ad0a780b1fda6341e215025dc6d8985891b91dad6d9dcb.json new file mode 100644 index 0000000000..2d8a83ba48 --- /dev/null +++ b/backend/.sqlx/query-f59afd524e3f216487ad0a780b1fda6341e215025dc6d8985891b91dad6d9dcb.json @@ -0,0 +1,26 @@ +{ + "db_name": "PostgreSQL", + "query": "INSERT INTO eval_dataset\n (workspace_id, path, summary, scorers, created_by, edited_by)\n VALUES ($1, $2, $3, $4, $5, $5)\n ON CONFLICT (workspace_id, path) DO NOTHING\n RETURNING path", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "path", + "type_info": "Varchar" + } + ], + "parameters": { + "Left": [ + "Varchar", + "Varchar", + "Varchar", + "Jsonb", + "Varchar" + ] + }, + "nullable": [ + false + ] + }, + "hash": "f59afd524e3f216487ad0a780b1fda6341e215025dc6d8985891b91dad6d9dcb" +} diff --git a/backend/.sqlx/query-f6c4c40b098ba06f4b3b057af1f5dce0711e90f7a4e1786d39a50d52400a3cb9.json b/backend/.sqlx/query-f6c4c40b098ba06f4b3b057af1f5dce0711e90f7a4e1786d39a50d52400a3cb9.json new file mode 100644 index 0000000000..035dc31c85 --- /dev/null +++ b/backend/.sqlx/query-f6c4c40b098ba06f4b3b057af1f5dce0711e90f7a4e1786d39a50d52400a3cb9.json @@ -0,0 +1,16 @@ +{ + "db_name": "PostgreSQL", + "query": "UPDATE eval_dataset SET created_by = $1 WHERE created_by = $2 AND workspace_id = $3", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "Varchar", + "Text", + "Text" + ] + }, + "nullable": [] + }, + "hash": "f6c4c40b098ba06f4b3b057af1f5dce0711e90f7a4e1786d39a50d52400a3cb9" +} diff --git a/backend/.sqlx/query-ef59abddc518f5213827e47a31aee49be917a61c46916c29d79c094438b1ff35.json b/backend/.sqlx/query-f907114909b1064a3d5eb603e5929a6ffebdcadb20e4c6a68768a9ad42fe9328.json similarity index 69% rename from backend/.sqlx/query-ef59abddc518f5213827e47a31aee49be917a61c46916c29d79c094438b1ff35.json rename to backend/.sqlx/query-f907114909b1064a3d5eb603e5929a6ffebdcadb20e4c6a68768a9ad42fe9328.json index 2a9c7cac4f..a62f4f95a1 100644 --- a/backend/.sqlx/query-ef59abddc518f5213827e47a31aee49be917a61c46916c29d79c094438b1ff35.json +++ b/backend/.sqlx/query-f907114909b1064a3d5eb603e5929a6ffebdcadb20e4c6a68768a9ad42fe9328.json @@ -1,6 +1,6 @@ { "db_name": "PostgreSQL", - "query": "SELECT id, path, created_at, created_by, value FROM resource_version\n WHERE workspace_id = $1 AND id = $2", + "query": "SELECT id, version, path, created_at, created_by, value FROM resource_version\n WHERE workspace_id = $1 AND id = $2", "describe": { "columns": [ { @@ -10,21 +10,26 @@ }, { "ordinal": 1, + "name": "version", + "type_info": "Int8" + }, + { + "ordinal": 2, "name": "path", "type_info": "Varchar" }, { - "ordinal": 2, + "ordinal": 3, "name": "created_at", "type_info": "Timestamptz" }, { - "ordinal": 3, + "ordinal": 4, "name": "created_by", "type_info": "Varchar" }, { - "ordinal": 4, + "ordinal": 5, "name": "value", "type_info": "Jsonb" } @@ -39,9 +44,10 @@ false, false, false, + false, true, true ] }, - "hash": "ef59abddc518f5213827e47a31aee49be917a61c46916c29d79c094438b1ff35" + "hash": "f907114909b1064a3d5eb603e5929a6ffebdcadb20e4c6a68768a9ad42fe9328" } diff --git a/backend/.sqlx/query-facbf7337d7ffa3f3e6287e2910ccd7ca7229f9d4e5c3af6c460ee7fe3c946e6.json b/backend/.sqlx/query-facbf7337d7ffa3f3e6287e2910ccd7ca7229f9d4e5c3af6c460ee7fe3c946e6.json new file mode 100644 index 0000000000..5f6f815e2a --- /dev/null +++ b/backend/.sqlx/query-facbf7337d7ffa3f3e6287e2910ccd7ca7229f9d4e5c3af6c460ee7fe3c946e6.json @@ -0,0 +1,23 @@ +{ + "db_name": "PostgreSQL", + "query": "SELECT path FROM eval_dataset WHERE workspace_id = $1 AND path = $2 FOR UPDATE", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "path", + "type_info": "Varchar" + } + ], + "parameters": { + "Left": [ + "Text", + "Text" + ] + }, + "nullable": [ + false + ] + }, + "hash": "facbf7337d7ffa3f3e6287e2910ccd7ca7229f9d4e5c3af6c460ee7fe3c946e6" +} diff --git a/backend/.sqlx/query-fcf570337b2ceeb0f9dcc311144d8ada02aa4353feb2f63e7a1f520a4000ed70.json b/backend/.sqlx/query-fcf570337b2ceeb0f9dcc311144d8ada02aa4353feb2f63e7a1f520a4000ed70.json new file mode 100644 index 0000000000..7dc41fa462 --- /dev/null +++ b/backend/.sqlx/query-fcf570337b2ceeb0f9dcc311144d8ada02aa4353feb2f63e7a1f520a4000ed70.json @@ -0,0 +1,36 @@ +{ + "db_name": "PostgreSQL", + "query": "SELECT branch.parent_job AS \"case_job!\", scorer.flow_step_id AS \"module!\",\n done.result AS \"result: sqlx::types::Json>\"\n FROM v2_job branch\n JOIN v2_job scorer ON scorer.parent_job = branch.id\n JOIN v2_job_completed done ON done.id = scorer.id\n WHERE branch.parent_job = ANY($1) AND branch.workspace_id = $2\n AND scorer.flow_step_id = ANY($3)", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "case_job!", + "type_info": "Uuid" + }, + { + "ordinal": 1, + "name": "module!", + "type_info": "Varchar" + }, + { + "ordinal": 2, + "name": "result: sqlx::types::Json>", + "type_info": "Jsonb" + } + ], + "parameters": { + "Left": [ + "UuidArray", + "Text", + "TextArray" + ] + }, + "nullable": [ + true, + true, + true + ] + }, + "hash": "fcf570337b2ceeb0f9dcc311144d8ada02aa4353feb2f63e7a1f520a4000ed70" +} diff --git a/backend/.sqlx/query-fd12bbe0605c80fced218a5a2e1288e6c04f4a1a76a2038679f8f538320cab1f.json b/backend/.sqlx/query-fd12bbe0605c80fced218a5a2e1288e6c04f4a1a76a2038679f8f538320cab1f.json new file mode 100644 index 0000000000..923a4b5f98 --- /dev/null +++ b/backend/.sqlx/query-fd12bbe0605c80fced218a5a2e1288e6c04f4a1a76a2038679f8f538320cab1f.json @@ -0,0 +1,24 @@ +{ + "db_name": "PostgreSQL", + "query": "SELECT path FROM eval_dataset\n WHERE scorers::text LIKE $1 AND NOT path LIKE $2 AND workspace_id = $3", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "path", + "type_info": "Varchar" + } + ], + "parameters": { + "Left": [ + "Text", + "Text", + "Text" + ] + }, + "nullable": [ + false + ] + }, + "hash": "fd12bbe0605c80fced218a5a2e1288e6c04f4a1a76a2038679f8f538320cab1f" +} diff --git a/backend/migrations/20260812100659_ai_evals.down.sql b/backend/migrations/20260812100659_ai_evals.down.sql new file mode 100644 index 0000000000..2e9383c761 --- /dev/null +++ b/backend/migrations/20260812100659_ai_evals.down.sql @@ -0,0 +1,6 @@ +DROP TABLE IF EXISTS eval_score; +DROP TABLE IF EXISTS eval_experiment_case; +DROP TABLE IF EXISTS eval_experiment; +DROP TABLE IF EXISTS eval_case; +DROP TABLE IF EXISTS eval_dataset; +DROP FUNCTION IF EXISTS eval_dataset_writable(varchar, varchar); diff --git a/backend/migrations/20260812100659_ai_evals.up.sql b/backend/migrations/20260812100659_ai_evals.up.sql new file mode 100644 index 0000000000..7ef82d5f7e --- /dev/null +++ b/backend/migrations/20260812100659_ai_evals.up.sql @@ -0,0 +1,275 @@ +-- Eval datasets and the cases they hold. Path-addressed like every other Windmill object, so the +-- folder a dataset is named by is what grants access to it. +CREATE TABLE eval_dataset ( + workspace_id VARCHAR(50) NOT NULL, + path VARCHAR(255) NOT NULL, + summary VARCHAR(1000) NULL, + -- The scorers a dataset is scored by. One entry per column of the results table: + -- {id, name, kind, ...kind-specific config}. `id` is assigned once and never reused, so a + -- column stays the same column across experiments when it is renamed or its definition is + -- edited — which is what makes a delta between two experiments meaningful. + scorers JSONB NOT NULL DEFAULT '[]', + extra_perms JSONB NOT NULL DEFAULT '{}', + created_at TIMESTAMPTZ NOT NULL DEFAULT now(), + created_by VARCHAR(50) NOT NULL, + edited_at TIMESTAMPTZ NOT NULL DEFAULT now(), + edited_by VARCHAR(50) NOT NULL, + PRIMARY KEY (workspace_id, path), + FOREIGN KEY (workspace_id) REFERENCES workspace(id) ON DELETE CASCADE +); + +-- A case is the input half of one evaluation: what the agent is fed, and what it was expected to +-- answer. The generated output, the trajectory and every scorer's return value are the job's, not +-- this table's. +-- +-- ON UPDATE CASCADE so renaming a dataset carries its cases instead of stranding them. +CREATE TABLE eval_case ( + id UUID PRIMARY KEY DEFAULT gen_random_uuid(), + workspace_id VARCHAR(50) NOT NULL, + dataset_path VARCHAR(255) NOT NULL, + -- {user_message, user_attachments} + input JSONB NOT NULL DEFAULT '{}', + expected JSONB NULL, + created_at TIMESTAMPTZ NOT NULL DEFAULT now(), + created_by VARCHAR(50) NOT NULL, + FOREIGN KEY (workspace_id, dataset_path) REFERENCES eval_dataset (workspace_id, path) + ON DELETE CASCADE ON UPDATE CASCADE +); + +-- Serves the paginated case list, which is ordered oldest-first so a case keeps its position as +-- the dataset grows. +CREATE INDEX index_eval_case_dataset ON eval_case (workspace_id, dataset_path, created_at, id); + +GRANT ALL ON eval_dataset TO windmill_user; +GRANT ALL ON eval_dataset TO windmill_admin; +GRANT ALL ON eval_case TO windmill_user; +GRANT ALL ON eval_case TO windmill_admin; + +ALTER TABLE eval_dataset ENABLE ROW LEVEL SECURITY; +ALTER TABLE eval_case ENABLE ROW LEVEL SECURITY; + +CREATE POLICY admin_policy ON eval_dataset FOR ALL TO windmill_admin USING (true); +CREATE POLICY admin_policy ON eval_case FOR ALL TO windmill_admin USING (true); + +CREATE POLICY see_folder_extra_perms_user_select ON eval_dataset FOR SELECT TO windmill_user +USING (SPLIT_PART(eval_dataset.path, '/', 1) = 'f' AND SPLIT_PART(eval_dataset.path, '/', 2) = any((select regexp_split_to_array(current_setting('session.folders_read'), ','))::text[])); +CREATE POLICY see_folder_extra_perms_user_insert ON eval_dataset FOR INSERT TO windmill_user +WITH CHECK (SPLIT_PART(eval_dataset.path, '/', 1) = 'f' AND SPLIT_PART(eval_dataset.path, '/', 2) = any((select regexp_split_to_array(current_setting('session.folders_write'), ','))::text[])); +CREATE POLICY see_folder_extra_perms_user_update ON eval_dataset FOR UPDATE TO windmill_user +USING (SPLIT_PART(eval_dataset.path, '/', 1) = 'f' AND SPLIT_PART(eval_dataset.path, '/', 2) = any((select regexp_split_to_array(current_setting('session.folders_write'), ','))::text[])); +CREATE POLICY see_folder_extra_perms_user_delete ON eval_dataset FOR DELETE TO windmill_user +USING (SPLIT_PART(eval_dataset.path, '/', 1) = 'f' AND SPLIT_PART(eval_dataset.path, '/', 2) = any((select regexp_split_to_array(current_setting('session.folders_write'), ','))::text[])); + +CREATE POLICY see_own ON eval_dataset FOR ALL TO windmill_user +USING (SPLIT_PART(eval_dataset.path, '/', 1) = 'u' AND SPLIT_PART(eval_dataset.path, '/', 2) = (select current_setting('session.user'))); +CREATE POLICY see_member ON eval_dataset FOR ALL TO windmill_user +USING (SPLIT_PART(eval_dataset.path, '/', 1) = 'g' AND SPLIT_PART(eval_dataset.path, '/', 2) = any((select regexp_split_to_array(current_setting('session.groups'), ','))::text[])); + +CREATE POLICY see_extra_perms_user_select ON eval_dataset FOR SELECT TO windmill_user +USING (extra_perms ? (select concat('u/', current_setting('session.user')))); +CREATE POLICY see_extra_perms_user_insert ON eval_dataset FOR INSERT TO windmill_user +WITH CHECK ((extra_perms ->> (select concat('u/', current_setting('session.user'))))::boolean); +CREATE POLICY see_extra_perms_user_update ON eval_dataset FOR UPDATE TO windmill_user +USING ((extra_perms ->> (select concat('u/', current_setting('session.user'))))::boolean); +CREATE POLICY see_extra_perms_user_delete ON eval_dataset FOR DELETE TO windmill_user +USING ((extra_perms ->> (select concat('u/', current_setting('session.user'))))::boolean); + +CREATE POLICY see_extra_perms_groups_select ON eval_dataset FOR SELECT TO windmill_user +USING (extra_perms ?| (select regexp_split_to_array(current_setting('session.pgroups'), ','))::text[]); +CREATE POLICY see_extra_perms_groups_insert ON eval_dataset FOR INSERT TO windmill_user +WITH CHECK (exists( + SELECT key, value FROM jsonb_each_text(extra_perms) + WHERE SPLIT_PART(key, '/', 1) = 'g' AND key = ANY((select regexp_split_to_array(current_setting('session.pgroups'), ','))::text[]) + AND value::boolean)); +CREATE POLICY see_extra_perms_groups_update ON eval_dataset FOR UPDATE TO windmill_user +USING (exists( + SELECT key, value FROM jsonb_each_text(extra_perms) + WHERE SPLIT_PART(key, '/', 1) = 'g' AND key = ANY((select regexp_split_to_array(current_setting('session.pgroups'), ','))::text[]) + AND value::boolean)); +CREATE POLICY see_extra_perms_groups_delete ON eval_dataset FOR DELETE TO windmill_user +USING (exists( + SELECT key, value FROM jsonb_each_text(extra_perms) + WHERE SPLIT_PART(key, '/', 1) = 'g' AND key = ANY((select regexp_split_to_array(current_setting('session.pgroups'), ','))::text[]) + AND value::boolean)); + +-- Whether the session may *write* the dataset at (_workspace_id, _path): the same disjunction the +-- dataset's own write policies use, in one place so the cases that hang off a dataset are governed +-- by exactly the rule the dataset is. A read grant is not enough — writing a case is writing the +-- dataset's contents — so this checks write, not merely visibility. +CREATE OR REPLACE FUNCTION eval_dataset_writable(_workspace_id varchar, _path varchar) + RETURNS boolean LANGUAGE sql STABLE AS $$ + SELECT EXISTS ( + SELECT 1 FROM eval_dataset d + WHERE d.workspace_id = _workspace_id AND d.path = _path + AND ( + (SPLIT_PART(d.path, '/', 1) = 'f' AND SPLIT_PART(d.path, '/', 2) = any((select regexp_split_to_array(current_setting('session.folders_write'), ','))::text[])) + OR (SPLIT_PART(d.path, '/', 1) = 'u' AND SPLIT_PART(d.path, '/', 2) = (select current_setting('session.user'))) + OR (SPLIT_PART(d.path, '/', 1) = 'g' AND SPLIT_PART(d.path, '/', 2) = any((select regexp_split_to_array(current_setting('session.groups'), ','))::text[])) + OR ((d.extra_perms ->> (select concat('u/', current_setting('session.user'))))::boolean) + OR EXISTS ( + SELECT 1 FROM jsonb_each_text(d.extra_perms) ep + WHERE SPLIT_PART(ep.key, '/', 1) = 'g' + AND ep.key = ANY((select regexp_split_to_array(current_setting('session.pgroups'), ','))::text[]) + AND ep.value::boolean) + ) + ); +$$; + +-- Cases are the *contents* of a dataset, not independently addressable objects, so both their +-- visibility and who may change them are the parent's, stated once here instead of mirrored in the +-- API and left to drift. Read is the dataset's read (the subquery is itself subject to +-- eval_dataset's SELECT policies above); write is the dataset's write, which `eval_dataset_writable` +-- checks — so a read-only grant on a dataset can list its cases but not edit them. The whole edit +-- of a dataset and its cases therefore runs as one `user_db` transaction, governed by these +-- policies, rather than being split across the unrestricted pool after a hand-written check. +CREATE POLICY see_parent_dataset ON eval_case FOR SELECT TO windmill_user +USING ( + EXISTS ( + SELECT 1 FROM eval_dataset d + WHERE d.workspace_id = eval_case.workspace_id AND d.path = eval_case.dataset_path + ) +); +CREATE POLICY write_parent_dataset_insert ON eval_case FOR INSERT TO windmill_user +WITH CHECK (eval_dataset_writable(eval_case.workspace_id, eval_case.dataset_path)); +CREATE POLICY write_parent_dataset_update ON eval_case FOR UPDATE TO windmill_user +USING (eval_dataset_writable(eval_case.workspace_id, eval_case.dataset_path)) +WITH CHECK (eval_dataset_writable(eval_case.workspace_id, eval_case.dataset_path)); +CREATE POLICY write_parent_dataset_delete ON eval_case FOR DELETE TO windmill_user +USING (eval_dataset_writable(eval_case.workspace_id, eval_case.dataset_path)); +-- One run of a dataset: written once when the dataset is run, and only ever read afterwards, +-- which is what makes it worth comparing against. +CREATE TABLE eval_experiment ( + id UUID PRIMARY KEY, + workspace_id VARCHAR(50) NOT NULL, + dataset_path VARCHAR(255) NOT NULL, + -- {kind, path, version}: what was run, at the version it was at when the run was enqueued. + subject JSONB NOT NULL, + -- A run is named by the number it is: "Run 7" is stable, sorts, and survives history being + -- pruned, which a position computed at read time would not. Allocated per (dataset, subject + -- path) when the run is opened. + run_number INTEGER NOT NULL, + -- A run is one flow: a loop over the cases, each iteration answering and then scoring. This + -- is the job holding it, so the run can be watched, cancelled and rerun as the single thing + -- it is. Assigned before the flow is pushed, so a launch that dies partway leaves an + -- experiment naming a job that never started rather than a flow nothing accounts for. + run_job_id UUID NOT NULL, + created_at TIMESTAMPTZ NOT NULL DEFAULT now(), + created_by VARCHAR(50) NOT NULL, + FOREIGN KEY (workspace_id, dataset_path) REFERENCES eval_dataset (workspace_id, path) + ON DELETE CASCADE ON UPDATE CASCADE +); + +CREATE INDEX index_eval_experiment_dataset ON eval_experiment (workspace_id, dataset_path, created_at DESC); + +-- Serves the per-agent run list, which spans datasets: an agent's history is one list whichever +-- dataset each run was of. +CREATE INDEX index_eval_experiment_subject ON eval_experiment + (workspace_id, (subject ->> 'path'), created_at DESC); + +-- The exact case set an experiment ran, by value: a dataset keeps changing, and a result set that +-- cannot say which inputs produced it is not reproducible. `case_id` is therefore deliberately not +-- a foreign key — deleting a case must not rewrite the history of the runs that used it. +CREATE TABLE eval_experiment_case ( + experiment_id UUID NOT NULL REFERENCES eval_experiment (id) ON DELETE CASCADE, + ordinal INT NOT NULL, + case_id UUID NOT NULL, + input JSONB NOT NULL DEFAULT '{}', + expected JSONB NULL, + -- The iteration of the run's flow that answered this case. Minted by the flow engine, so the + -- case is recorded before it has one and the id is filled in once the iterations exist. + job_id UUID NULL, + -- What the run produced, copied out of the jobs once they have produced it. Jobs have their + -- own retention, and a recorded run has to still read as the run it was once they are gone. + -- `answered` is the agent step's own outcome, which is settled while the iteration around it + -- is still scoring; `status` is the iteration's, once it has one. + output TEXT NULL, + answered BOOLEAN NULL, + status VARCHAR(30) NULL, + -- The resource version the agent was at for this cell, and — for a draft, which has no + -- version to move — the hash of the configuration that actually ran: the only thing that can + -- say a row describes an agent that has since been edited. + subject_version BIGINT NULL, + subject_draft_hash VARCHAR(64) NULL, + started_at TIMESTAMPTZ NOT NULL DEFAULT now(), + PRIMARY KEY (experiment_id, ordinal), + -- A run holds each case once: the pair is what identifies a cell. + CONSTRAINT eval_experiment_case_unique_case UNIQUE (experiment_id, case_id) +); + +-- One scorer's verdict on one run. Separate from the run because scoring is separate from running: +-- a scorer's verdict is stored per run and scorer, independent of the agent execution that +-- produced the answers. +CREATE TABLE eval_score ( + experiment_id UUID NOT NULL, + ordinal INT NOT NULL, + scorer_id VARCHAR(64) NOT NULL, + -- NULL until the verdict has been read out of the run's flow, and when scoring failed. + score DOUBLE PRECISION NULL, + reason TEXT NULL, + -- [{name, passed, detail}], for scorers that report per-assertion results. + checks JSONB NULL, + error TEXT NULL, + -- The scorer read the run and said it had nothing to measure on this case. A verdict, not a + -- failure: the cell is left out of the column's mean and pass rate rather than counted as a + -- zero or reported as a scorer that produced nothing. + not_applicable BOOLEAN NOT NULL DEFAULT false, + -- Hash of the scorer configuration that produced this score, including the script hash or flow + -- version actually executed. Two scores of the same scorer whose definitions differ are still + -- compared, but the column says the scorer changed rather than letting it read as a change of + -- agent. + definition VARCHAR(64) NOT NULL, + created_at TIMESTAMPTZ NOT NULL DEFAULT now(), + PRIMARY KEY (experiment_id, ordinal, scorer_id), + FOREIGN KEY (experiment_id, ordinal) REFERENCES eval_experiment_case (experiment_id, ordinal) + ON DELETE CASCADE +); + +GRANT ALL ON eval_experiment TO windmill_user; +GRANT ALL ON eval_experiment TO windmill_admin; +GRANT ALL ON eval_experiment_case TO windmill_user; +GRANT ALL ON eval_experiment_case TO windmill_admin; +GRANT ALL ON eval_score TO windmill_user; +GRANT ALL ON eval_score TO windmill_admin; + +ALTER TABLE eval_experiment ENABLE ROW LEVEL SECURITY; +ALTER TABLE eval_experiment_case ENABLE ROW LEVEL SECURITY; +ALTER TABLE eval_score ENABLE ROW LEVEL SECURITY; + +CREATE POLICY admin_policy ON eval_experiment FOR ALL TO windmill_admin USING (true); +CREATE POLICY admin_policy ON eval_experiment_case FOR ALL TO windmill_admin USING (true); +CREATE POLICY admin_policy ON eval_score FOR ALL TO windmill_admin USING (true); + +-- Experiments are the *contents* of a dataset, not independently addressable objects, so their +-- visibility is the parent's: the subquery is itself subject to eval_dataset's policies, which +-- stay stated once instead of being mirrored here and left to drift. +-- +-- SELECT only, deliberately. A `FOR ALL ... USING` would be reused as the INSERT/UPDATE/DELETE +-- check expression, and since the subquery is a SELECT it applies the dataset's *read* policies — +-- which would let someone with read-only access to a dataset forge an experiment row naming a job +-- they cannot otherwise read. Writes are done on the unrestricted pool after the API has checked +-- write access to the parent, and a stray `user_db` write to these tables is meant to fail rather +-- than silently succeed. +CREATE POLICY see_parent_dataset ON eval_experiment FOR SELECT TO windmill_user +USING ( + EXISTS ( + SELECT 1 FROM eval_dataset d + WHERE d.workspace_id = eval_experiment.workspace_id AND d.path = eval_experiment.dataset_path + ) +); + +CREATE POLICY see_parent_experiment ON eval_experiment_case FOR SELECT TO windmill_user +USING ( + EXISTS ( + SELECT 1 FROM eval_experiment e + WHERE e.id = eval_experiment_case.experiment_id + ) +); + +-- Visibility is the experiment's, which is the dataset's. +CREATE POLICY see_parent_experiment ON eval_score FOR SELECT TO windmill_user +USING ( + EXISTS ( + SELECT 1 FROM eval_experiment e + WHERE e.id = eval_score.experiment_id + ) +); diff --git a/backend/migrations/20260813164338_ai_token_usage.down.sql b/backend/migrations/20260813164338_ai_token_usage.down.sql new file mode 100644 index 0000000000..0fa1b47462 --- /dev/null +++ b/backend/migrations/20260813164338_ai_token_usage.down.sql @@ -0,0 +1 @@ +DROP TABLE ai_token_usage; diff --git a/backend/migrations/20260813164338_ai_token_usage.up.sql b/backend/migrations/20260813164338_ai_token_usage.up.sql new file mode 100644 index 0000000000..b20e471db5 --- /dev/null +++ b/backend/migrations/20260813164338_ai_token_usage.up.sql @@ -0,0 +1,42 @@ +-- Per-workspace AI token spend, accumulated from the chat client. Rows hold token +-- counts rather than money: prices live in the frontend price table plus the +-- workspace's `ai_config.model_pricing` overrides and are applied at read time, so +-- correcting a price also corrects the history. `reported_cost_nano_usd` is the +-- exception — a few providers (OpenRouter) return what they actually charged, and +-- that figure wins over the estimate. +-- +-- Distinct from `feature_usage`, which is anonymous telemetry that leaves the +-- instance and is pruned after 60 days; spend is per-user and kept. +CREATE TABLE ai_token_usage ( + workspace_id VARCHAR(50) NOT NULL REFERENCES workspace(id) ON DELETE CASCADE, + day DATE NOT NULL DEFAULT CURRENT_DATE, + email VARCHAR(255) NOT NULL, + provider VARCHAR(50) NOT NULL, + model VARCHAR(255) NOT NULL, + -- Empty for chats that are not attached to an AI session. + session_id VARCHAR(50) NOT NULL DEFAULT '', + -- Uncached input only; the two cache columns hold the rest of the prompt, so + -- each column maps to exactly one price and they never double-count. + input_tokens BIGINT NOT NULL DEFAULT 0, + cache_read_tokens BIGINT NOT NULL DEFAULT 0, + cache_write_tokens BIGINT NOT NULL DEFAULT 0, + output_tokens BIGINT NOT NULL DEFAULT 0, + reported_cost_nano_usd BIGINT, + requests BIGINT NOT NULL DEFAULT 0, + updated_at TIMESTAMPTZ NOT NULL DEFAULT now(), + PRIMARY KEY (workspace_id, day, email, provider, model, session_id) +); + +-- The usage listing filters on workspace and a date range; the PK only reaches +-- `day` through `email`, so it cannot serve that on its own. +CREATE INDEX idx_ai_token_usage_ws_day ON ai_token_usage (workspace_id, day DESC); + +GRANT ALL ON ai_token_usage TO windmill_admin; +GRANT ALL ON ai_token_usage TO windmill_user; + +-- Both handlers go through the raw pool, so no policy is needed for them to work. +-- Enabling RLS with an admin-only policy is the backstop: a future query that +-- reaches this table through UserDB sees nothing rather than every user's spend. +ALTER TABLE ai_token_usage ENABLE ROW LEVEL SECURITY; + +CREATE POLICY admin_policy ON ai_token_usage FOR ALL TO windmill_admin USING (true); diff --git a/backend/migrations/20260819073729_resource_version_number.down.sql b/backend/migrations/20260819073729_resource_version_number.down.sql new file mode 100644 index 0000000000..13f95048ee --- /dev/null +++ b/backend/migrations/20260819073729_resource_version_number.down.sql @@ -0,0 +1,17 @@ +-- Back to numbering versions by the table-wide identity sequence, so the function must stop +-- writing a column that is about to go. +CREATE OR REPLACE FUNCTION record_resource_version() RETURNS trigger AS $$ +BEGIN + INSERT INTO resource_version (workspace_id, path, resource_type, value, created_by) + VALUES ( + NEW.workspace_id, NEW.path, NEW.resource_type, NEW.value, + COALESCE(NULLIF(current_setting('session.user', true), ''), NEW.created_by) + ); + + RETURN NEW; +END; +$$ LANGUAGE plpgsql SECURITY DEFINER SET search_path FROM CURRENT; + +DROP INDEX IF EXISTS index_resource_version_number; + +ALTER TABLE resource_version DROP COLUMN version; diff --git a/backend/migrations/20260819073729_resource_version_number.up.sql b/backend/migrations/20260819073729_resource_version_number.up.sql new file mode 100644 index 0000000000..e56dc7a09c --- /dev/null +++ b/backend/migrations/20260819073729_resource_version_number.up.sql @@ -0,0 +1,52 @@ +-- `id` is one identity sequence for the whole table and stays how a version is addressed; +-- `version` is the resource's own count, which is what a version is presented by. +ALTER TABLE resource_version ADD COLUMN version BIGINT; + +UPDATE resource_version rv SET version = ranked.rn +FROM ( + SELECT id, row_number() OVER (PARTITION BY workspace_id, path ORDER BY id) AS rn + FROM resource_version +) ranked +WHERE rv.id = ranked.id; + +ALTER TABLE resource_version ALTER COLUMN version SET NOT NULL; + +-- The number is only meaningful within a path, so the triple is the natural key: it serves the +-- lookup by number and makes a duplicate a hard error rather than two rows claiming v7. +CREATE UNIQUE INDEX index_resource_version_number ON resource_version (workspace_id, path, version); + +-- Numbering is assigned here rather than derived when read because both ways of deleting versions +-- take the oldest ones: the monitor's trim past MAX_RESOURCE_VERSIONS, and clearing a history down +-- to its current value. A number computed by counting the survivors would renumber under either, +-- so a run recorded against v3 would later name a different version. +CREATE OR REPLACE FUNCTION record_resource_version() RETURNS trigger AS $$ +BEGIN + -- `session.user` is set by UserDB::begin for authed requests; worker and system writes fall + -- back to the row's own author. NULLIF because a transaction-local set_config resets the + -- placeholder to the empty string rather than unsetting it, so a pooled connection that + -- previously served an authed request reports '' here, not NULL. + -- + -- MAX + 1 needs no lock of its own: this runs inside the transaction that wrote `resource`, and + -- a concurrent write to the same path blocks on that row's lock — or on the primary key, for an + -- insert — before its own trigger can run, so the maximum cannot be read stale. Deleting + -- versions never lowers it, since both deletions keep the newest row. + INSERT INTO resource_version (workspace_id, path, resource_type, value, created_by, version) + VALUES ( + NEW.workspace_id, NEW.path, NEW.resource_type, NEW.value, + COALESCE(NULLIF(current_setting('session.user', true), ''), NEW.created_by), + (SELECT COALESCE(MAX(version), 0) + 1 FROM resource_version + WHERE workspace_id = NEW.workspace_id AND path = NEW.path) + ); + + -- The per-path cap is enforced by trim_resource_versions in the monitor, not here: trimming + -- on every write would tax a path `setResource` can drive in a loop, to keep a bound that + -- does not need to hold instantaneously. + + RETURN NEW; +END; +-- SECURITY DEFINER so history is written on behalf of every writer without granting anyone direct +-- write access to the table, which users hold SELECT on only. `SET search_path FROM CURRENT` is the +-- injection hardening that goes with it, captured rather than hardcoded so installs running a +-- non-public PG_SCHEMA still resolve (see +-- 20260624103600_repair_folder_labels_search_path.up.sql). +$$ LANGUAGE plpgsql SECURITY DEFINER SET search_path FROM CURRENT; diff --git a/backend/summarized_schema.txt b/backend/summarized_schema.txt index 52790b6ec6..f1a72e8ee2 100644 --- a/backend/summarized_schema.txt +++ b/backend/summarized_schema.txt @@ -40,6 +40,8 @@ agent_token_blacklist: token(char), expires_at(ts), blacklisted_at(ts), blacklis ai_agent_memory: workspace_id(char), conversation_id(uuid), step_id(char), messages(jsonb), created_at(ts), updated_at(ts) ai_free_token_daily_usage: day(date), cost_nanos(bigint), updated_at(ts) ai_free_token_usage: email(char), cost_nanos(bigint), updated_at(ts) +ai_token_usage: workspace_id(char), day(date), email(char), provider(char), model(char), session_id(char), input_tokens(bigint), cache_read_tokens(bigint), cache_write_tokens(bigint), output_tokens(bigint), reported_cost_nano_usd(bigint), requests(bigint), updated_at(ts) + FK: (workspace_id) -> workspace(id) alerts: id(int), alert_type(char), message(text), created_at(ts), acknowledged(bool), workspace_id(text), acknowledged_workspace(bool), resource(text) app: id(bigint), workspace_id(char), path(char), summary(char), policy(jsonb), versions(bigint[]), extra_perms(jsonb), draft_only(bool), custom_path(text), labels(text[]) FK: (workspace_id) -> workspace(id) diff --git a/backend/windmill-ai/src/ai_google.rs b/backend/windmill-ai/src/ai_google.rs index 4374cb8628..b90bda9e1c 100644 --- a/backend/windmill-ai/src/ai_google.rs +++ b/backend/windmill-ai/src/ai_google.rs @@ -277,7 +277,7 @@ pub struct GeminiSSECandidate { } /// Token usage from the `usageMetadata` field of a Gemini SSE event. -#[derive(Deserialize, Debug, Clone)] +#[derive(Deserialize, Debug, Clone, Default)] pub struct GeminiUsageMetadata { #[serde(rename = "promptTokenCount", default)] pub prompt_token_count: Option, @@ -285,6 +285,39 @@ pub struct GeminiUsageMetadata { pub candidates_token_count: Option, #[serde(rename = "totalTokenCount", default)] pub total_token_count: Option, + /// Subset of `promptTokenCount` served from context cache, billed at a reduced + /// rate. Reported separately so the client can price it separately. + #[serde(rename = "cachedContentTokenCount", default)] + pub cached_content_token_count: Option, + /// Thinking tokens, billed as output but counted apart from `candidatesTokenCount`. + #[serde(rename = "thoughtsTokenCount", default)] + pub thoughts_token_count: Option, + /// Input tokens spent on tool-use prompts, counted apart from `promptTokenCount` + /// rather than within it. + #[serde(rename = "toolUsePromptTokenCount", default)] + pub tool_use_prompt_token_count: Option, +} + +/// Input tokens as billed. Gemini reports tool-use prompts in their own field, and +/// they are disjoint from `promptTokenCount`: a live tool call returns 17 prompt + +/// 60 tool-use + 17 candidates + 52 thoughts against a `totalTokenCount` of 146, so +/// leaving them out under-reports the input of every tool-using turn. Cached tokens +/// are not added here, being already part of `promptTokenCount`. +fn gemini_prompt_tokens(usage: &GeminiUsageMetadata) -> i32 { + usage + .prompt_token_count + .unwrap_or(0) + .saturating_add(usage.tool_use_prompt_token_count.unwrap_or(0)) +} + +/// Output tokens as billed: Gemini counts thinking apart from `candidatesTokenCount` +/// but charges it at the output rate, so a reply that thought would otherwise be +/// reported as far cheaper than it was. +fn gemini_completion_tokens(usage: &GeminiUsageMetadata) -> i32 { + usage + .candidates_token_count + .unwrap_or(0) + .saturating_add(usage.thoughts_token_count.unwrap_or(0)) } /// Top-level structure of one Gemini SSE event. @@ -588,9 +621,12 @@ pub fn gemini_response_to_openai(parsed: &GeminiParsedEvent, model: &str) -> ser let usage = parsed.usage.as_ref().map(|u| { serde_json::json!({ - "prompt_tokens": u.prompt_token_count.unwrap_or(0), - "completion_tokens": u.candidates_token_count.unwrap_or(0), + "prompt_tokens": gemini_prompt_tokens(u), + "completion_tokens": gemini_completion_tokens(u), "total_tokens": u.total_token_count.unwrap_or(0), + "prompt_tokens_details": { + "cached_tokens": u.cached_content_token_count.unwrap_or(0) + }, }) }); @@ -680,8 +716,8 @@ pub fn gemini_event_to_openai_sse_chunks( // OpenAI's `stream_options.include_usage` terminal chunk (top-level `usage`, // empty `choices`) so the frontend's `'usage' in chunk` path records them. if let Some(usage) = &parsed.usage { - let prompt_tokens = usage.prompt_token_count.unwrap_or(0); - let completion_tokens = usage.candidates_token_count.unwrap_or(0); + let prompt_tokens = gemini_prompt_tokens(usage); + let completion_tokens = gemini_completion_tokens(usage); let total_tokens = usage .total_token_count .unwrap_or(prompt_tokens + completion_tokens); @@ -694,6 +730,9 @@ pub fn gemini_event_to_openai_sse_chunks( "prompt_tokens": prompt_tokens, "completion_tokens": completion_tokens, "total_tokens": total_tokens, + "prompt_tokens_details": { + "cached_tokens": usage.cached_content_token_count.unwrap_or(0) + }, } }); chunks.push(format!("data: {}\n\n", chunk)); @@ -943,6 +982,7 @@ mod tests { prompt_token_count: Some(12), candidates_token_count: Some(7), total_token_count: Some(19), + ..Default::default() }), ..Default::default() }; @@ -969,6 +1009,79 @@ mod tests { assert_eq!(usage_chunk["choices"], serde_json::json!([])); } + #[test] + fn gemini_usage_chunk_splits_cached_and_bills_thoughts() { + let parsed = GeminiParsedEvent { + text: Some("the answer".to_string()), + usage: Some(GeminiUsageMetadata { + prompt_token_count: Some(1000), + candidates_token_count: Some(20), + total_token_count: Some(1120), + cached_content_token_count: Some(900), + thoughts_token_count: Some(100), + ..Default::default() + }), + ..Default::default() + }; + + let mut tool_call_index = 0; + let chunks = gemini_event_to_openai_sse_chunks( + &parsed, + "chatcmpl-test", + "gemini-3-flash-preview", + &mut tool_call_index, + ); + let usage_chunk = chunks + .iter() + .map(|c| parse_sse_chunk(c)) + .find(|v| v.get("usage").map(|u| !u.is_null()).unwrap_or(false)) + .expect("a chunk should carry top-level usage"); + + // Gemini's prompt count already includes the cached tokens, so it passes + // through unchanged and the cached share is reported alongside it; thinking + // is billed as output but counted apart from the candidates. + assert_eq!(usage_chunk["usage"]["prompt_tokens"], 1000); + assert_eq!(usage_chunk["usage"]["prompt_tokens_details"]["cached_tokens"], 900); + assert_eq!(usage_chunk["usage"]["completion_tokens"], 120); + } + + #[test] + fn gemini_usage_chunk_counts_tool_use_prompt_tokens() { + let parsed = GeminiParsedEvent { + text: Some("Canberra".to_string()), + usage: Some(GeminiUsageMetadata { + prompt_token_count: Some(17), + candidates_token_count: Some(17), + total_token_count: Some(146), + tool_use_prompt_token_count: Some(60), + thoughts_token_count: Some(52), + ..Default::default() + }), + ..Default::default() + }; + + let mut tool_call_index = 0; + let chunks = gemini_event_to_openai_sse_chunks( + &parsed, + "chatcmpl-test", + "gemini-2.5-flash", + &mut tool_call_index, + ); + let usage_chunk = chunks + .iter() + .map(|c| parse_sse_chunk(c)) + .find(|v| v.get("usage").map(|u| !u.is_null()).unwrap_or(false)) + .expect("a chunk should carry top-level usage"); + + assert_eq!(usage_chunk["usage"]["prompt_tokens"], 77); + assert_eq!(usage_chunk["usage"]["completion_tokens"], 69); + assert_eq!( + usage_chunk["usage"]["prompt_tokens"].as_i64().unwrap() + + usage_chunk["usage"]["completion_tokens"].as_i64().unwrap(), + 146 + ); + } + #[test] fn gemini_streaming_usage_total_falls_back_to_prompt_plus_completion() { let parsed = GeminiParsedEvent { @@ -976,6 +1089,7 @@ mod tests { prompt_token_count: Some(5), candidates_token_count: Some(3), total_token_count: None, + ..Default::default() }), ..Default::default() }; diff --git a/backend/windmill-ai/src/ai_types.rs b/backend/windmill-ai/src/ai_types.rs index b80e237920..69c8ac4538 100644 --- a/backend/windmill-ai/src/ai_types.rs +++ b/backend/windmill-ai/src/ai_types.rs @@ -175,3 +175,57 @@ pub struct OpenAIMessage { #[serde(skip_serializing_if = "Option::is_none")] pub annotations: Option>, } + +// ============================================================================ +// Model pricing +// ============================================================================ + +/// Far above any real per-million-token rate, so a value beyond it is a unit +/// mistake rather than a price. The floor matters more: a negative rate would make +/// spend subtract, and NaN/infinity would poison every total derived from it. +pub const MAX_MODEL_RATE: f64 = 1000.0; + +/// Bound the `model_pricing` map of an AI config that is only available untyped — +/// the instance config is stored through the generic global-settings endpoint, +/// which never deserializes it into `AIConfig`, so the typed check on the +/// workspace path does not cover it. +pub fn validate_model_pricing_json(ai_config: &serde_json::Value) -> Result<(), String> { + // The container itself has to be checked too: a non-object `ai_config` persists + // here and then fails to deserialize as `AIConfig`, which drops the whole + // instance config back to its default for every workspace inheriting it. + if !ai_config.is_null() && !ai_config.is_object() { + return Err("ai_config must be an object".to_string()); + } + let pricing = match ai_config.get("model_pricing") { + None | Some(serde_json::Value::Null) => return Ok(()), + // A present-but-wrong shape must be rejected, not skipped: it would persist + // and then fail to deserialize as `AIConfig`, which silently drops the whole + // instance config back to its default for every workspace inheriting it. + Some(v) => v + .as_object() + .ok_or_else(|| "model_pricing must be an object".to_string())?, + }; + for (key, price) in pricing { + let Some(price) = price.as_object() else { + return Err(format!("Price override for {} is not an object", key)); + }; + for field in ["input", "output", "cache_read", "cache_write"] { + let Some(rate) = price.get(field) else { continue }; + let rate = rate + .as_f64() + .filter(|r| r.is_finite() && *r >= 0.0 && *r <= MAX_MODEL_RATE); + if rate.is_none() { + return Err(format!( + "Price override for {}: {} must be between 0 and {}", + key, field, MAX_MODEL_RATE + )); + } + } + for required in ["input", "output"] { + if !price.contains_key(required) { + return Err(format!("Price override for {} is missing {}", key, required)); + } + } + } + Ok(()) +} diff --git a/backend/windmill-ai/src/providers/bedrock.rs b/backend/windmill-ai/src/providers/bedrock.rs index 453607eea5..cf977f2484 100644 --- a/backend/windmill-ai/src/providers/bedrock.rs +++ b/backend/windmill-ai/src/providers/bedrock.rs @@ -660,6 +660,41 @@ fn bedrock_sse_chunks_for_event( chunks.push(Bytes::from(format!("data: {}\n\n", chunk))); } + // Usage arrives only on the trailing Metadata event, and only this converter + // reaches the chat: without a chunk for it a Bedrock chat reports no tokens at + // all. Bedrock counts cache reads and writes apart from `inputTokens`, while the + // OpenAI shape the client parses treats `prompt_tokens` as the whole input, so + // they are folded in here and split back out through `prompt_tokens_details`. + if let aws_sdk_bedrockruntime::types::ConverseStreamOutput::Metadata(metadata) = event { + if let Some(token_usage) = metadata.usage() { + let cache_read = token_usage.cache_read_input_tokens().unwrap_or(0); + let cache_write = token_usage.cache_write_input_tokens().unwrap_or(0); + let prompt_tokens = token_usage + .input_tokens() + .saturating_add(cache_read) + .saturating_add(cache_write); + + let chunk = serde_json::json!({ + "id": state.id, + "object": "chat.completion.chunk", + "created": state.created, + "model": state.model, + "choices": [], + "usage": { + "prompt_tokens": prompt_tokens, + "completion_tokens": token_usage.output_tokens(), + "total_tokens": token_usage.total_tokens(), + "prompt_tokens_details": { + "cached_tokens": cache_read, + "cache_write_tokens": cache_write + } + } + }); + + chunks.push(Bytes::from(format!("data: {}\n\n", chunk))); + } + } + chunks } @@ -1190,6 +1225,43 @@ mod tests { serde_json::from_str(payload).expect("chunk should contain JSON") } + #[test] + fn metadata_event_emits_usage_chunk_with_cache_split() { + let mut state = BedrockSseStreamState::new("id".to_string(), "model".to_string(), 0); + let event = ConverseStreamOutput::Metadata( + aws_sdk_bedrockruntime::types::ConverseStreamMetadataEvent::builder() + .usage( + aws_sdk_bedrockruntime::types::TokenUsage::builder() + .input_tokens(10) + .output_tokens(7) + .total_tokens(1017) + .cache_read_input_tokens(900) + .cache_write_input_tokens(100) + .build() + .expect("usage"), + ) + .build(), + ); + + let chunks = bedrock_sse_chunks_for_event(&event, &mut state); + let usage = chunks + .iter() + .map(sse_json) + .find(|v| v.get("usage").map(|u| !u.is_null()).unwrap_or(false)) + .expect("the metadata event should carry usage"); + + // Bedrock reports cache reads and writes apart from `inputTokens`; the OpenAI + // shape the client parses treats `prompt_tokens` as the whole input, and + // recovers the uncached share by subtracting the details back out. + assert_eq!(usage["usage"]["prompt_tokens"], 1010); + assert_eq!(usage["usage"]["completion_tokens"], 7); + assert_eq!(usage["usage"]["prompt_tokens_details"]["cached_tokens"], 900); + assert_eq!( + usage["usage"]["prompt_tokens_details"]["cache_write_tokens"], + 100 + ); + } + #[test] fn determine_auth_config_prioritizes_bearer_token() { let config = determine_auth_config( diff --git a/backend/windmill-api-auth/src/scopes.rs b/backend/windmill-api-auth/src/scopes.rs index da24453553..7109a04425 100644 --- a/backend/windmill-api-auth/src/scopes.rs +++ b/backend/windmill-api-auth/src/scopes.rs @@ -289,6 +289,7 @@ pub enum ScopeDomain { OAuth, AI, AiSkills, + AiEvals, // AI agent eval datasets Indexer, Teams, // Microsoft Teams integration @@ -349,6 +350,7 @@ impl ScopeDomain { Self::OAuth => "oauth", Self::AI => "ai", Self::AiSkills => "ai_skills", + Self::AiEvals => "ai_evals", Self::Capture => "capture", Self::Drafts => "drafts", Self::Favorites => "favorites", @@ -404,6 +406,7 @@ impl ScopeDomain { "oauth" => Some(Self::OAuth), "ai" => Some(Self::AI), "ai_skills" => Some(Self::AiSkills), + "ai_evals" => Some(Self::AiEvals), "indexer" | "srch" => Some(Self::Indexer), "teams" => Some(Self::Teams), "native_triggers" => Some(Self::NativeTriggers), diff --git a/backend/windmill-api-settings/src/lib.rs b/backend/windmill-api-settings/src/lib.rs index a9bc601591..cead2f4b27 100644 --- a/backend/windmill-api-settings/src/lib.rs +++ b/backend/windmill-api-settings/src/lib.rs @@ -891,6 +891,13 @@ async fn run_setting_pre_write_hook( value: &serde_json::Value, ) -> error::Result<()> { match key { + // The instance AI config is written as an untyped blob through this generic + // endpoint, so it never passes the typed check the workspace handler applies. + // Rates that reach a cost total unbounded would make it negative or infinite. + AI_CONFIG_SETTING => { + windmill_ai::ai_types::validate_model_pricing_json(value) + .map_err(error::Error::BadRequest)?; + } AUTOMATE_USERNAME_CREATION_SETTING => { if value.as_bool().unwrap_or(false) { generate_instance_username_for_all_users(db) diff --git a/backend/windmill-api-users/src/users.rs b/backend/windmill-api-users/src/users.rs index 0c83d5f6ec..8ea3bd0995 100644 --- a/backend/windmill-api-users/src/users.rs +++ b/backend/windmill-api-users/src/users.rs @@ -2399,6 +2399,7 @@ pub async fn delete_workspace_user_internal( "flow", "app", "resource", + "eval_dataset", "variable", "schedule", "group_", diff --git a/backend/windmill-api-workspaces/src/workspaces.rs b/backend/windmill-api-workspaces/src/workspaces.rs index 44018736eb..e04e1c1ab9 100644 --- a/backend/windmill-api-workspaces/src/workspaces.rs +++ b/backend/windmill-api-workspaces/src/workspaces.rs @@ -5471,6 +5471,8 @@ async fn clone_workspace_data( // Clone scripts with new hashes clone_scripts(tx, source_workspace_id, target_workspace_id).await?; + clone_eval_datasets(tx, source_workspace_id, target_workspace_id).await?; + // Clone the dbt graph sidecars. After `clone_scripts`, which keeps each // script's hash: these key on it, and a static descriptor never re-ingests, // so a fork without them shows dbt scripts with no models until someone @@ -6005,6 +6007,36 @@ async fn clone_resources( Ok(()) } +async fn clone_eval_datasets( + tx: &mut Transaction<'_, Postgres>, + source_workspace_id: &str, + target_workspace_id: &str, +) -> Result<()> { + // The authored evaluation data — datasets and their cases — travels with a fork like resources + // and scripts do; the runs (experiments) do not, since they name jobs the fork has no copy of. + sqlx::query!( + "INSERT INTO eval_dataset (workspace_id, path, summary, scorers, extra_perms, created_at, created_by, edited_at, edited_by) + SELECT $2, path, summary, scorers, extra_perms, created_at, created_by, edited_at, edited_by + FROM eval_dataset WHERE workspace_id = $1", + source_workspace_id, + target_workspace_id, + ) + .execute(&mut **tx) + .await?; + // A new id per cloned case: `eval_case`'s primary key is the id alone, unique across the whole + // table, so copying it would collide with the source's own rows. + sqlx::query!( + "INSERT INTO eval_case (workspace_id, dataset_path, input, expected, created_at, created_by) + SELECT $2, dataset_path, input, expected, created_at, created_by + FROM eval_case WHERE workspace_id = $1", + source_workspace_id, + target_workspace_id, + ) + .execute(&mut **tx) + .await?; + Ok(()) +} + async fn clone_variables( tx: &mut Transaction<'_, Postgres>, db: &DB, diff --git a/backend/windmill-api/openapi.yaml b/backend/windmill-api/openapi.yaml index 2d6b98ea75..6031665fad 100644 --- a/backend/windmill-api/openapi.yaml +++ b/backend/windmill-api/openapi.yaml @@ -7819,7 +7819,7 @@ paths: schema: type: string - /w/{workspace}/resources/history/v/{version}: + /w/{workspace}/resources/history/v/{id}: get: summary: get a single resource version, with its value operationId: getResourceVersion @@ -7827,9 +7827,10 @@ paths: - resource parameters: - $ref: "#/components/parameters/WorkspaceId" - - name: version + - name: id in: path required: true + description: The version's id, not its number. schema: type: integer format: int64 @@ -7851,7 +7852,7 @@ paths: required: - missing_references - /w/{workspace}/resources/history/restore/v/{version}: + /w/{workspace}/resources/history/restore/v/{id}: post: summary: restore a resource to a previous version operationId: restoreResourceVersion @@ -7859,9 +7860,10 @@ paths: - resource parameters: - $ref: "#/components/parameters/WorkspaceId" - - name: version + - name: id in: path required: true + description: The version's id, not its number. schema: type: integer format: int64 @@ -11779,6 +11781,453 @@ paths: items: $ref: "#/components/schemas/FlowConversationMessage" + /w/{workspace}/ai_evals/datasets/list: + get: + summary: list eval datasets + operationId: listEvalDatasets + tags: + - ai_evals + parameters: + - $ref: "#/components/parameters/WorkspaceId" + responses: + "200": + description: eval datasets list + content: + application/json: + schema: + type: array + items: + $ref: "#/components/schemas/EvalDataset" + + /w/{workspace}/ai_evals/datasets/create: + post: + summary: create an eval dataset + operationId: createEvalDataset + tags: + - ai_evals + parameters: + - $ref: "#/components/parameters/WorkspaceId" + requestBody: + description: new eval dataset + required: true + content: + application/json: + schema: + type: object + required: [path] + properties: + path: + type: string + maxLength: 255 + summary: + type: string + maxLength: 1000 + scorers: + type: array + maxItems: 20 + items: + $ref: "#/components/schemas/Scorer" + cases: + type: array + maxItems: 1000 + description: The cases to create the dataset holding, so one can be assembled in a single act rather than created empty and filled in afterwards. + items: + $ref: "#/components/schemas/NewEvalCase" + responses: + "200": + description: eval dataset created + content: + text/plain: + schema: + type: string + + /w/{workspace}/ai_evals/datasets/get/{path}: + get: + summary: get an eval dataset + operationId: getEvalDataset + tags: + - ai_evals + parameters: + - $ref: "#/components/parameters/WorkspaceId" + - $ref: "#/components/parameters/Path" + responses: + "200": + description: eval dataset + content: + application/json: + schema: + $ref: "#/components/schemas/EvalDataset" + + /w/{workspace}/ai_evals/datasets/update/{path}: + post: + summary: update an eval dataset + operationId: updateEvalDataset + tags: + - ai_evals + parameters: + - $ref: "#/components/parameters/WorkspaceId" + - $ref: "#/components/parameters/Path" + requestBody: + description: updated eval dataset + required: true + content: + application/json: + schema: + type: object + properties: + path: + type: string + maxLength: 255 + description: > + Renames the dataset. Its cases and experiments follow through the foreign + keys, so a rename keeps the history it already has. + summary: + type: string + maxLength: 1000 + description: Left out to keep the stored summary; sent as "" to clear it. + scorers: + type: array + maxItems: 20 + description: > + Left out to keep the dataset's columns as they are; sent to replace them + wholesale. + items: + $ref: "#/components/schemas/Scorer" + cases: + type: array + maxItems: 1000 + description: > + The cases as they should stand afterwards: all of them, each carrying its id if + the dataset already has it. Sent with the rest of an edit so that a rename the + dataset refuses refuses the case edits with it. + items: + $ref: "#/components/schemas/SaveEvalCase" + responses: + "200": + description: eval dataset updated + content: + text/plain: + schema: + type: string + + /w/{workspace}/ai_evals/datasets/delete/{path}: + post: + summary: delete an eval dataset and all its cases + description: > + The cases, the runs and their recorded case sets go with it through the foreign keys; the + jobs those runs produced are left alone. + operationId: deleteEvalDataset + tags: + - ai_evals + parameters: + - $ref: "#/components/parameters/WorkspaceId" + - $ref: "#/components/parameters/Path" + responses: + "200": + description: eval dataset deleted + content: + text/plain: + schema: + type: string + + /w/{workspace}/ai_evals/cases/list/{path}: + get: + summary: list the cases of an eval dataset + operationId: listEvalCases + tags: + - ai_evals + parameters: + - $ref: "#/components/parameters/WorkspaceId" + - $ref: "#/components/parameters/Path" + - $ref: "#/components/parameters/Page" + - $ref: "#/components/parameters/PerPage" + responses: + "200": + description: eval cases + content: + application/json: + schema: + type: object + required: [cases] + properties: + cases: + type: array + items: + $ref: "#/components/schemas/EvalCase" + /w/{workspace}/ai_evals/subject_state: + get: + summary: what the agent under test is right now + description: > + The version it is deployed at. Small on purpose: the results endpoint reports the same + thing but harvests scores and reads every job to do it, so it is not something to ask for + on its own. + operationId: evalSubjectState + tags: + - ai_evals + parameters: + - $ref: "#/components/parameters/WorkspaceId" + - name: path + in: query + required: true + schema: + type: string + responses: + "200": + description: the subject as it is now + content: + application/json: + schema: + type: object + properties: + version: + type: integer + format: int64 + + /w/{workspace}/ai_evals/run_payload: + get: + summary: the run one iteration of an eval run answered, as its scorers read it + operationId: evalRunPayload + description: > + Called by the step a run's flow places between the agent and its scorers. Every tool call + is enriched with the arguments, result, status and duration of the job that ran it, and + with the schema of the script version it ran, none of which the flow itself can read. + tags: + - ai_evals + parameters: + - $ref: "#/components/parameters/WorkspaceId" + - name: job_id + in: query + required: true + description: The flow job that answered the case. + schema: + type: string + format: uuid + responses: + "200": + description: the run and its rendering + content: + application/json: + schema: + type: object + required: [run, rendered] + properties: + run: + type: object + description: The case, the answer, and every tool call the agent made. + rendered: + type: string + description: The same run as a judge agent is shown it. + + /w/{workspace}/ai_evals/scorer_defaults: + get: + summary: what a new judge agent and a new script scorer are created from + operationId: scorerDefaults + tags: + - ai_evals + parameters: + - $ref: "#/components/parameters/WorkspaceId" + responses: + "200": + description: scorer defaults + content: + application/json: + schema: + type: object + required: [judge_prompt, script_template] + properties: + judge_prompt: + type: string + description: The system prompt a judge agent is created with. + script_template: + type: string + + /w/{workspace}/ai_evals/scorers/recent: + get: + summary: list the scorers already in use in this workspace, most recent first + description: > + Filtered twice, both times by what the caller can read: the datasets they come from, and + the runnables themselves. A scorer they could not run does not appear. + operationId: recentScorers + tags: + - ai_evals + parameters: + - $ref: "#/components/parameters/WorkspaceId" + - name: kind + description: only scorers of this kind + in: query + required: false + schema: + type: string + enum: [script, agent] + responses: + "200": + description: recently used scorers + content: + application/json: + schema: + type: array + items: + allOf: + - $ref: "#/components/schemas/Scorer" + - type: object + required: [dataset] + properties: + dataset: + type: string + description: The dataset it is a column of. + + /w/{workspace}/ai_evals/experiments/run: + post: + summary: run every case of a dataset as one immutable experiment + operationId: runExperiment + tags: + - ai_evals + parameters: + - $ref: "#/components/parameters/WorkspaceId" + requestBody: + description: what to run + required: true + content: + application/json: + schema: + type: object + required: [dataset, subject] + properties: + dataset: + type: string + subject: + $ref: "#/components/schemas/EvalSubject" + responses: + "200": + description: id of the created experiment + content: + text/plain: + schema: + type: string + + /w/{workspace}/ai_evals/experiments/collect: + post: + summary: record what a run produced, so it outlives the jobs that produced it + description: > + Called by a run's own flow as its last step. The answers and scores a run produced live in + its jobs, which have their own retention; this copies them onto the run's rows. Reading a + run does the same, so this is what covers a run nobody opened. + operationId: collectExperiment + tags: + - ai_evals + parameters: + - $ref: "#/components/parameters/WorkspaceId" + - name: id + in: query + required: true + schema: + type: string + format: uuid + responses: + "200": + description: how many of the run's cases are recorded + content: + application/json: + schema: + type: integer + + /w/{workspace}/ai_evals/experiments/list_all: + get: + summary: list every experiment, across datasets + operationId: listAllExperiments + tags: + - ai_evals + parameters: + - $ref: "#/components/parameters/WorkspaceId" + - name: subject_path + description: > + Restrict to one agent's runs, which is what makes the list a history rather than a + log. Runs of what is deployed, of a past version, and of the edits waiting on top are + all that agent's, so this does not discriminate by kind. + in: query + required: false + schema: + type: string + responses: + "200": + description: > + The 100 newest experiments, each naming the dataset it is of. Restricted to datasets + the caller can read. + content: + application/json: + schema: + type: array + items: + $ref: "#/components/schemas/EvalExperiment" + + /w/{workspace}/ai_evals/experiments/results/{path}: + get: + summary: read an experiment's results, one row per case + operationId: experimentResults + tags: + - ai_evals + parameters: + - $ref: "#/components/parameters/WorkspaceId" + - $ref: "#/components/parameters/Path" + - name: id + description: the experiment to read + in: query + required: true + schema: + type: string + format: uuid + - name: baseline + description: > + The experiment every column is compared against. A delta is only computed between two + scores of the same scorer id, and a column the baseline was never scored with reports + it rather than showing a difference. + in: query + required: false + schema: + type: string + format: uuid + responses: + "200": + description: experiment results + content: + application/json: + schema: + type: object + required: [experiment, scorers, rows, means, regressed] + properties: + experiment: + $ref: "#/components/schemas/EvalExperiment" + baseline: + $ref: "#/components/schemas/EvalExperiment" + scorers: + type: array + description: The columns, which belong to the dataset rather than the experiment. + items: + $ref: "#/components/schemas/Scorer" + rows: + type: array + items: + $ref: "#/components/schemas/ExperimentRow" + means: + type: array + items: + $ref: "#/components/schemas/ScorerMean" + regressed: + type: integer + description: Cells scoring lower than the baseline, across every column. + subject_current_version: + type: integer + format: int64 + description: > + The version the subject is on now. A row that ran against an earlier one + describes an agent that no longer exists. + subject_deployed_hash: + type: string + description: > + What the agent hashes to as deployed. A run of unsaved edits carrying this + hash ran exactly what is deployed now — the edits were saved — so it is a run + of that version rather than of edits. + /w/{workspace}/path_autocomplete/list_paths: get: summary: list all paths in a workspace for client-side autocomplete @@ -11986,6 +12435,73 @@ paths: schema: type: string + /w/{workspace}/ai/usage: + post: + summary: record AI token usage for the calling user + operationId: recordAiUsage + tags: + - ai + parameters: + - $ref: "#/components/parameters/WorkspaceId" + requestBody: + required: true + content: + application/json: + schema: + type: object + required: + - events + properties: + events: + type: array + items: + $ref: "#/components/schemas/AITokenUsageEvent" + responses: + "204": + description: usage recorded + get: + summary: list aggregated AI token usage + operationId: listAiUsage + tags: + - ai + parameters: + - $ref: "#/components/parameters/WorkspaceId" + - name: days + in: query + schema: + type: integer + minimum: 1 + maximum: 365 + - name: group_by + in: query + schema: + type: string + enum: [day, user, model] + - name: scope + in: query + description: workspace-wide usage (admin only) or the calling user's own + schema: + type: string + enum: [workspace, self] + responses: + "200": + description: usage buckets + content: + application/json: + schema: + type: object + required: + - buckets + - truncated + properties: + buckets: + type: array + items: + $ref: "#/components/schemas/AITokenUsageBucket" + truncated: + type: boolean + description: more buckets matched than were returned, so summing them under-reports + /w/{workspace}/ai_skills/list: get: summary: list the workspace AI chat skills (name + description only) @@ -26111,6 +26627,348 @@ components: additionalProperties: true description: Array of JSON Web Keys for JWT verification + EvalSubject: + type: object + description: What an eval run is executed against. + required: [kind, path] + properties: + kind: + type: string + description: > + `agent` runs the ai_agent resource as it is deployed when the run opens, `agent_draft` + the caller's unsaved edits of it as the editor holds them (carried in `draft`), and + `agent_version` one past version named by `version`. The first and last are read + server-side; all three are inlined into the run, so every case of a run executes one + configuration: a deploy part-way through changes what the next run measures, never + this one. + enum: [agent, agent_draft, agent_version] + path: + type: string + description: Path of the ai_agent resource. + version: + type: integer + format: int64 + nullable: true + description: > + The agent's per-path version number when the run opened: how many times the resource + had been saved, not a resource_version row id. For `agent` and `agent_draft` it names + the configuration the run read and every case executed. For `agent_version` it is the + request's own, says which version to inline, and is required. + draft: + $ref: "#/components/schemas/AgentDraft" + draft_hash: + type: string + description: > + Hash of the configuration a draft run executed, stamped server-side. A draft moves + without the version moving, so this is what dates a run of one. It is also what + recognises a draft run whose configuration was later deployed: when it matches the + agent as deployed, the run's kind and version are rewritten to that version, once, and + the hash is kept as what the resolution rests on. + + AgentDraft: + type: object + description: > + The brain and tools of an agent, as the flow editor holds them. Carried by the request and + present exactly when the subject kind is `agent_draft` — the edits exist only in the editor + — where it is the whole definition of what ran: the run goes through the same unlinked + branch of the agent executor the editor's own test uses. + properties: + input_transforms: + type: object + additionalProperties: true + description: > + The agent's input transforms: provider, system prompt, output type and the rest. The + message and attachments come from the case and override anything named here. + tools: + type: array + items: + type: object + additionalProperties: true + + EvalDataset: + type: object + required: [path, created_at, created_by, edited_at, edited_by] + properties: + path: + type: string + maxLength: 255 + summary: + type: string + maxLength: 1000 + scorers: + type: array + description: The columns of the results table, in display order. + items: + $ref: "#/components/schemas/Scorer" + created_at: + type: string + format: date-time + created_by: + type: string + edited_at: + type: string + format: date-time + edited_by: + type: string + + EvalCaseInput: + type: object + description: The inputs a standalone run feeds the agent. + properties: + user_message: + type: string + user_attachments: + type: array + items: + type: object + + NewEvalCase: + type: object + properties: + input: + $ref: "#/components/schemas/EvalCaseInput" + expected: + description: Reference output a scorer compares a rerun against. + + SaveEvalCase: + allOf: + - type: object + properties: + id: + description: Absent for a case the dataset does not hold yet. + type: string + format: uuid + - $ref: "#/components/schemas/NewEvalCase" + + EvalCase: + allOf: + - type: object + required: [id, created_at, created_by] + properties: + id: + type: string + format: uuid + created_at: + type: string + format: date-time + created_by: + type: string + - $ref: "#/components/schemas/NewEvalCase" + + Scorer: + type: object + description: > + A scorer is a column of the results table, and it is always a runnable: an ai_agent + resource sent the run to grade, or a script handed the run as an argument. `id` is assigned + when the scorer is added to a dataset and never reused: it is what makes a column the same + column across experiments when the scorer is renamed, and a delta is only ever computed + between two scores carrying the same id. A scorer sent without an id is given one. + required: [kind, path] + properties: + id: + type: string + name: + type: string + description: Column header. Defaults to the last segment of the path. + pass_if: + type: number + description: > + A score at or above this counts as a pass, and the column reports a pass rate beside + its mean. Applied when results are read rather than when they are produced, so moving + the line re-reads every score already recorded instead of invalidating them. + kind: + type: string + enum: [script, agent] + path: + type: string + description: The script, or the ai_agent resource used as a judge. + + EvalExperiment: + type: object + description: >- + One run of a dataset: written once when the dataset is run, and only ever read afterwards. + The case set it executed is returned by the results endpoint, not here: a listing would + otherwise send the whole dataset back once per experiment. + required: [id, dataset, subject, run_number, run_job_id, case_count, created_at, created_by] + properties: + id: + type: string + format: uuid + dataset: + type: string + subject: + $ref: "#/components/schemas/EvalSubject" + run_number: + type: integer + description: > + This agent's nth run of this dataset, allocated once and never reused. What a run is + called. Numbered per agent rather than per subject kind: runs of what is deployed and + runs of its draft are the same agent's history. + run_job_id: + type: string + format: uuid + description: > + The flow executing the run: one job holding every case and its scores. + case_count: + type: integer + scores: + type: array + description: > + What the run scored, one entry per scorer that produced a number. Carried on the run + itself so a list of runs can say what each one scored without reading every cell of + every one of them. Empty on a run whose scores have not been read yet. + items: + $ref: "#/components/schemas/ExperimentScore" + running: + type: boolean + description: > + Whether the flow executing this run is still going. What makes a list of runs worth + watching rather than worth reloading. + created_at: + type: string + format: date-time + created_by: + type: string + + ExperimentScore: + type: object + description: >- + One scorer's headline for one run: the two numbers a column reports, over that run's cells. + required: [scorer_id, name, kind, scored, failed] + properties: + scorer_id: + type: string + name: + type: string + description: > + What the column is called in the dataset that ran it, resolved server-side because a + list of runs spanning datasets cannot hold every dataset's scorers to look it up. + kind: + type: string + enum: [agent, script] + mean: + type: number + pass_rate: + type: number + description: > + The share of scored cells at or above the column's threshold, for a column that has + one. Absent where the column has no threshold and the mean is the whole headline. + scored: + type: integer + failed: + type: integer + description: > + How many of the run's cells the column failed on. A column that failed on all of them + has no number to report and is still one of the columns that ran. + + CellScore: + type: object + description: One scorer's verdict on one run, and how it compares with the baseline. + required: [scorer_id, pending, definition_changed] + properties: + scorer_id: + type: string + score: + type: number + reason: + type: string + checks: {} + error: + type: string + not_applicable: + type: boolean + description: > + The scorer read this case and had nothing to measure on it. Left out of the column's + mean and pass rate rather than counted as a zero. + pending: + type: boolean + description: A scoring job is still running for this cell. + passed: + type: boolean + description: > + Which side of the scorer's `pass_if` threshold the score fell on. Absent when the + column has no threshold, or has no score yet. + baseline: + type: number + description: The same scorer's number on the baseline experiment. + definition_changed: + type: boolean + description: > + The baseline's score came from a different definition of this scorer, so the delta is + a change of scorer as much as a change of agent. + + ExperimentRow: + type: object + required: [case_id, input, status, scores] + properties: + case_id: + type: string + format: uuid + input: + $ref: "#/components/schemas/EvalCaseInput" + expected: {} + job_id: + type: string + format: uuid + description: > + The iteration that ran this case. Absent between a run being recorded and its flow + reaching this case, which reads as a case still to run. + status: + type: string + description: > + The case's status; `running` until its iteration completes, and `unavailable` for a + case whose job was retained away before anything read what it produced. + enum: [running, success, failure, canceled, skipped, unavailable] + output: + type: string + description: The agent's answer. The full trajectory stays reachable through job_id. + subject_version: + type: integer + format: int64 + description: > + The agent version this cell ran against. Cells of one experiment can differ, which the + table says rather than averaging two versions silently. + subject_draft_hash: + type: string + description: > + For a run of unsaved edits, the hash of the configuration this cell ran. Edits move + without a version changing, so this is what identifies what ran, and what recognises a + run whose edits were later saved as a run of that version. + scores: + type: array + description: One entry per scorer of the dataset, in column order. + items: + $ref: "#/components/schemas/CellScore" + + ScorerMean: + type: object + description: > + A column's summary. There is no single number for a dataset: averaging a judge with an + exact match would invent one. + required: [scorer_id, scored, missing_in_baseline, definition_changed] + properties: + scorer_id: + type: string + mean: + type: number + baseline_mean: + type: number + pass_rate: + type: number + description: > + The share of scored cells that passed, for a column with a threshold. Reported beside + the mean rather than instead of it: a pass rate says how many cases are good enough, + a mean says by how much, and neither answers the other's question. + baseline_pass_rate: + type: number + scored: + type: integer + missing_in_baseline: + type: integer + description: Cells the baseline has no score for, so a column the baseline never ran shows as unscored rather than as a spurious difference. + definition_changed: + type: boolean + FlowConversation: type: object required: @@ -26302,6 +27160,10 @@ components: maximum: 2000000 free_tier: $ref: "#/components/schemas/FreeTierInfo" + model_pricing: + type: object + additionalProperties: + $ref: "#/components/schemas/ModelPriceOverride" FreeTierInfo: type: object @@ -26319,6 +27181,88 @@ components: - exhausted - used_ratio + ModelPriceOverride: + type: object + description: negotiated rates in USD per million tokens, keyed `provider:model` + properties: + input: + type: number + minimum: 0 + maximum: 1000 + output: + type: number + minimum: 0 + maximum: 1000 + cache_read: + type: number + minimum: 0 + maximum: 1000 + cache_write: + type: number + minimum: 0 + maximum: 1000 + required: + - input + - output + + AITokenUsageEvent: + type: object + properties: + provider: + $ref: "#/components/schemas/AIProvider" + model: + type: string + session_id: + type: string + input_tokens: + type: integer + cache_read_tokens: + type: integer + cache_write_tokens: + type: integer + output_tokens: + type: integer + reported_cost_nano_usd: + type: integer + description: only set by providers that bill back an exact figure + requests: + type: integer + required: + - provider + - model + + AITokenUsageBucket: + type: object + properties: + key: + type: string + description: the grouped dimension's value; empty when grouping by model + provider: + type: string + model: + type: string + input_tokens: + type: integer + cache_read_tokens: + type: integer + cache_write_tokens: + type: integer + output_tokens: + type: integer + reported_cost_nano_usd: + type: integer + requests: + type: integer + required: + - key + - provider + - model + - input_tokens + - cache_read_tokens + - cache_write_tokens + - output_tokens + - requests + InstanceAIProviderSummary: type: object properties: @@ -28458,6 +29402,11 @@ components: id: type: integer format: int64 + description: How this version is addressed. Unique across every resource, so it says nothing about how many times this one has been saved. + version: + type: integer + format: int64 + description: Which version of this resource it is, counted from its first. What a version is called. created_at: type: string format: date-time @@ -28465,6 +29414,7 @@ components: type: string required: - id + - version - created_at ListableResource: diff --git a/backend/windmill-api/src/ai.rs b/backend/windmill-api/src/ai.rs index f722b9b565..64e24367b0 100644 --- a/backend/windmill-api/src/ai.rs +++ b/backend/windmill-api/src/ai.rs @@ -3,11 +3,16 @@ use crate::utils::check_scopes; #[cfg(feature = "bedrock")] use axum::routing::get; -#[cfg(feature = "bedrock")] use axum::Json; -use axum::{body::Bytes, extract::Path, response::IntoResponse, routing::post, Extension, Router}; +use axum::{ + body::Bytes, + extract::{DefaultBodyLimit, Path, Query}, + response::IntoResponse, + routing::post, + Extension, Router, +}; use futures::StreamExt; -use http::{HeaderMap, Method}; +use http::{HeaderMap, Method, StatusCode}; use quick_cache::sync::Cache; use reqwest::{Client, RequestBuilder}; use serde::{Deserialize, Serialize}; @@ -18,6 +23,7 @@ use windmill_ai::ai_cache::current_instance_ai_config_revision; use windmill_ai::ai_providers::{ empty_string_as_none, AIPlatform, AIProvider, ProviderConfig, ProviderModel, }; +use windmill_ai::ai_types::MAX_MODEL_RATE; use windmill_ai::credentials::ProviderCredentials; #[cfg(feature = "bedrock")] use windmill_ai::providers::bedrock::{ @@ -37,7 +43,7 @@ use windmill_ai::proxy::{ use windmill_audit::{audit_oss::audit_log, ActionKind}; use windmill_common::db::UserDB; use windmill_common::error::{to_anyhow, Error, Result}; -use windmill_common::utils::configure_client; +use windmill_common::utils::{configure_client, require_admin}; use windmill_common::variables::{get_variable_or_self, get_variable_or_self_as}; // AI timeout configuration constants @@ -435,9 +441,54 @@ pub struct AIConfig { /// free-tier marker. Only the server sets it, per-request. #[serde(skip_serializing_if = "Option::is_none", skip_deserializing)] pub free_tier: Option, + /// Per-model price overrides, keyed `provider:model` like `max_tokens_per_model`. + /// Only models whose rates differ from the built-in table are stored. + #[serde(skip_serializing_if = "Option::is_none")] + pub model_pricing: Option>, +} + +/// Negotiated rates in USD per million tokens. An unset cache rate is read as the +/// provider's own multiple of the input rate where the model has a published one, +/// and as the input rate itself where it does not — an unstated discount is never +/// filled in from another vendor's. +#[derive(Serialize, Deserialize, Debug, Clone)] +pub struct ModelPriceOverride { + pub input: f64, + pub output: f64, + #[serde(skip_serializing_if = "Option::is_none")] + pub cache_read: Option, + #[serde(skip_serializing_if = "Option::is_none")] + pub cache_write: Option, +} + +impl ModelPriceOverride { + pub fn validate(&self, key: &str) -> Result<()> { + for (field, rate) in [ + ("input", Some(self.input)), + ("output", Some(self.output)), + ("cache_read", self.cache_read), + ("cache_write", self.cache_write), + ] { + let Some(rate) = rate else { continue }; + if !rate.is_finite() || rate < 0.0 || rate > MAX_MODEL_RATE { + return Err(Error::BadRequest(format!( + "Price override for {}: {} must be between 0 and {}", + key, field, MAX_MODEL_RATE + ))); + } + } + Ok(()) + } } impl AIConfig { + pub fn validate_model_pricing(&self) -> Result<()> { + for (key, price) in self.model_pricing.iter().flatten() { + price.validate(key)?; + } + Ok(()) + } + pub fn has_providers(&self) -> bool { self.providers .as_ref() @@ -450,7 +501,18 @@ pub fn global_service() -> Router { } pub fn workspaced_service() -> Router { - let router = Router::new().route("/proxy/{*ai}", post(proxy).get(proxy)); + let router = Router::new() + .route("/proxy/{*ai}", post(proxy).get(proxy)) + .route( + "/usage", + post(record_ai_usage) + .get(list_ai_usage) + // The handler caps how many events it *stores*, but Json deserializes + // the whole array first — without a body limit an authenticated member + // could make the server allocate and parse an arbitrarily large one. + // Sized well above a full batch of the shape below. + .layer(DefaultBodyLimit::max(AI_USAGE_BODY_LIMIT)), + ); #[cfg(feature = "bedrock")] let router = router.route("/check_bedrock_credentials", get(check_bedrock_credentials)); @@ -458,6 +520,265 @@ pub fn workspaced_service() -> Router { router } +/// One provider request's worth of tokens, as counted by the chat client. +#[derive(Deserialize)] +struct AIUsageEvent { + provider: String, + model: String, + #[serde(default)] + session_id: String, + #[serde(default)] + input_tokens: i64, + #[serde(default)] + cache_read_tokens: i64, + #[serde(default)] + cache_write_tokens: i64, + #[serde(default)] + output_tokens: i64, + /// Only the providers that bill back an exact figure set this. + #[serde(default)] + reported_cost_nano_usd: Option, + #[serde(default)] + requests: Option, +} + +#[derive(Deserialize)] +struct RecordAIUsagePayload { + events: Vec, +} + +const MAX_AI_USAGE_EVENTS: usize = 50; +/// 64 KiB — a 50-event batch is a few kB even with the longest model ids. +const AI_USAGE_BODY_LIMIT: usize = 64 * 1024; +/// Well above any single conversation and far below an i64 overflow, so a client +/// bug caps out at one absurd row instead of poisoning the running total. +const MAX_TOKENS_PER_EVENT: i64 = 100_000_000; +/// $1000 in nano-USD. +const MAX_REPORTED_COST_PER_EVENT: i64 = 1_000_000_000_000; + +/// Model ids carry vendor prefixes and variant suffixes (`anthropic/claude-opus-5:thinking`), +/// so the shape check is looser than an identifier but still excludes whitespace and +/// anything that would not be a model id. +fn is_model_shaped(s: &str, max_len: usize) -> bool { + !s.is_empty() + && s.len() <= max_len + && s.chars() + .all(|c| c.is_ascii_alphanumeric() || matches!(c, '_' | '-' | ':' | '.' | '/' | '~')) +} + +/// Accumulate one workspace's AI token spend. Values are clamped and the caller's +/// email comes from the session, never the payload — the client is trusted to +/// report its own usage, not to attribute it to someone else. +async fn record_ai_usage( + authed: ApiAuthed, + Extension(db): Extension, + Path(w_id): Path, + Json(payload): Json, +) -> Result { + // Pre-sum duplicate keys: two rows hitting the same conflict target in a single + // INSERT error out ("cannot affect row a second time"). + let mut agg: HashMap<(String, String, String), AIUsageTotals> = HashMap::new(); + for e in payload.events.into_iter().take(MAX_AI_USAGE_EVENTS) { + if AIProvider::try_from(e.provider.as_str()).is_err() + || !is_model_shaped(&e.model, 255) + || !(e.session_id.is_empty() || is_model_shaped(&e.session_id, 50)) + { + continue; + } + let totals = agg + .entry((e.provider, e.model, e.session_id)) + .or_insert_with(AIUsageTotals::default); + totals.input += e.input_tokens.clamp(0, MAX_TOKENS_PER_EVENT); + totals.cache_read += e.cache_read_tokens.clamp(0, MAX_TOKENS_PER_EVENT); + totals.cache_write += e.cache_write_tokens.clamp(0, MAX_TOKENS_PER_EVENT); + totals.output += e.output_tokens.clamp(0, MAX_TOKENS_PER_EVENT); + totals.requests += e.requests.unwrap_or(1).clamp(0, MAX_AI_USAGE_EVENTS as i64); + if let Some(cost) = e.reported_cost_nano_usd { + totals.reported_cost = Some( + totals.reported_cost.unwrap_or(0) + cost.clamp(0, MAX_REPORTED_COST_PER_EVENT), + ); + } + } + if agg.is_empty() { + return Ok(StatusCode::NO_CONTENT); + } + + let mut providers = Vec::with_capacity(agg.len()); + let mut models = Vec::with_capacity(agg.len()); + let mut session_ids = Vec::with_capacity(agg.len()); + let mut inputs = Vec::with_capacity(agg.len()); + let mut cache_reads = Vec::with_capacity(agg.len()); + let mut cache_writes = Vec::with_capacity(agg.len()); + let mut outputs = Vec::with_capacity(agg.len()); + let mut reported_costs: Vec> = Vec::with_capacity(agg.len()); + let mut requests = Vec::with_capacity(agg.len()); + for ((provider, model, session_id), totals) in agg { + providers.push(provider); + models.push(model); + session_ids.push(session_id); + inputs.push(totals.input); + cache_reads.push(totals.cache_read); + cache_writes.push(totals.cache_write); + outputs.push(totals.output); + reported_costs.push(totals.reported_cost); + requests.push(totals.requests); + } + + sqlx::query!( + "INSERT INTO ai_token_usage (workspace_id, email, provider, model, session_id, \ + input_tokens, cache_read_tokens, cache_write_tokens, output_tokens, \ + reported_cost_nano_usd, requests) + SELECT $1, $2, * FROM UNNEST($3::text[], $4::text[], $5::text[], $6::bigint[], \ + $7::bigint[], $8::bigint[], $9::bigint[], $10::bigint[], $11::bigint[]) + ON CONFLICT (workspace_id, day, email, provider, model, session_id) + DO UPDATE SET + input_tokens = ai_token_usage.input_tokens + EXCLUDED.input_tokens, + cache_read_tokens = ai_token_usage.cache_read_tokens + EXCLUDED.cache_read_tokens, + cache_write_tokens = ai_token_usage.cache_write_tokens + EXCLUDED.cache_write_tokens, + output_tokens = ai_token_usage.output_tokens + EXCLUDED.output_tokens, + reported_cost_nano_usd = CASE + WHEN EXCLUDED.reported_cost_nano_usd IS NULL + THEN ai_token_usage.reported_cost_nano_usd + ELSE COALESCE(ai_token_usage.reported_cost_nano_usd, 0) + + EXCLUDED.reported_cost_nano_usd + END, + requests = ai_token_usage.requests + EXCLUDED.requests, + updated_at = now()", + &w_id, + &authed.email, + &providers, + &models, + &session_ids, + &inputs, + &cache_reads, + &cache_writes, + &outputs, + &reported_costs as &[Option], + &requests + ) + .execute(&db) + .await?; + + Ok(StatusCode::NO_CONTENT) +} + +#[derive(Default)] +struct AIUsageTotals { + input: i64, + cache_read: i64, + cache_write: i64, + output: i64, + reported_cost: Option, + requests: i64, +} + +#[derive(Deserialize)] +struct ListAIUsageQuery { + days: Option, + group_by: Option, + scope: Option, +} + +/// A bucket always carries its provider and model: the caller prices it from a +/// per-model rate table, which a bucket spanning several models could not be +/// resolved against. +#[derive(Serialize)] +struct AITokenUsageBucket { + key: String, + provider: String, + model: String, + input_tokens: i64, + cache_read_tokens: i64, + cache_write_tokens: i64, + output_tokens: i64, + reported_cost_nano_usd: Option, + requests: i64, +} + +/// Grouping by day over a long range, or by model across many models, can produce +/// more buckets than a table is worth rendering, so the listing is capped. +/// `truncated` says so explicitly — a caller that sums the rows into a total must be +/// able to tell that the total is partial rather than silently under-reporting spend. +#[derive(Serialize)] +struct AITokenUsageListing { + buckets: Vec, + truncated: bool, +} + +const AI_USAGE_MAX_BUCKETS: i64 = 1000; + +async fn list_ai_usage( + authed: ApiAuthed, + Extension(db): Extension, + Path(w_id): Path, + Query(query): Query, +) -> Result> { + // Reading the whole workspace's spend is an admin view; reading your own is + // not, so a member can see what they are costing without being shown their + // colleagues'. The filter is the session's email, never a parameter. + let own_email = match query.scope.as_deref().unwrap_or("workspace") { + "workspace" => { + require_admin(authed.is_admin, &authed.username)?; + None + } + "self" => Some(authed.email.clone()), + scope => return Err(Error::BadRequest(format!("Unsupported scope: {}", scope))), + }; + + let days = query.days.unwrap_or(30).clamp(1, 365); + let group_by = query.group_by.as_deref().unwrap_or("day"); + // No `session`: a session is identified by a client-generated id whose name + // lives only in the browser that made it, so a bucket keyed on one is a label + // nobody can resolve. `session_id` is still stored, at the grain the client + // batches on, should sessions ever gain a server-side name. + if !matches!(group_by, "day" | "user" | "model") { + return Err(Error::BadRequest(format!( + "Unsupported group_by: {}", + group_by + ))); + } + + // Fetch one past the cap to detect truncation. Ordering is by token volume, not + // by cost: rates are applied by the caller, so this query cannot know what a + // bucket cost. Volume is the closest proxy available here, and the caller is told + // the listing was capped rather than being left to sum a partial set silently. + let mut rows = sqlx::query_as!( + AITokenUsageBucket, + r#"SELECT + (CASE $3::text + WHEN 'day' THEN day::text + WHEN 'user' THEN email + ELSE '' + END) AS "key!", + provider AS "provider!", + model AS "model!", + SUM(input_tokens)::bigint AS "input_tokens!", + SUM(cache_read_tokens)::bigint AS "cache_read_tokens!", + SUM(cache_write_tokens)::bigint AS "cache_write_tokens!", + SUM(output_tokens)::bigint AS "output_tokens!", + SUM(reported_cost_nano_usd)::bigint AS "reported_cost_nano_usd", + SUM(requests)::bigint AS "requests!" + FROM ai_token_usage + WHERE workspace_id = $1 AND day > CURRENT_DATE - $2::int + AND ($5::text IS NULL OR email = $5) + GROUP BY 1, provider, model + ORDER BY SUM(input_tokens + cache_read_tokens + cache_write_tokens + output_tokens) DESC + LIMIT $4"#, + &w_id, + days, + group_by, + AI_USAGE_MAX_BUCKETS + 1, + own_email.as_deref() + ) + .fetch_all(&db) + .await?; + + let truncated = rows.len() as i64 > AI_USAGE_MAX_BUCKETS; + rows.truncate(AI_USAGE_MAX_BUCKETS as usize); + + Ok(Json(AITokenUsageListing { buckets: rows, truncated })) +} + /// Check if AWS Bedrock credentials are available from environment variables. #[cfg(feature = "bedrock")] async fn check_bedrock_credentials( diff --git a/backend/windmill-api/src/ai_evals/datasets.rs b/backend/windmill-api/src/ai_evals/datasets.rs new file mode 100644 index 0000000000..da901276b0 --- /dev/null +++ b/backend/windmill-api/src/ai_evals/datasets.rs @@ -0,0 +1,473 @@ +use super::*; + +#[derive(Serialize, Deserialize, Debug, Clone)] +pub struct EvalDataset { + pub path: String, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub summary: Option, + /// The columns of the results table, in display order. + #[serde(default)] + pub scorers: Vec, + pub created_at: DateTime, + pub created_by: String, + pub edited_at: DateTime, + pub edited_by: String, +} + +/// The agent-facing half of a case: exactly the inputs a standalone run feeds the agent. +#[derive(Serialize, Deserialize, Debug, Clone, Default)] +pub struct EvalCaseInput { + #[serde(default, skip_serializing_if = "Option::is_none")] + pub user_message: Option, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub user_attachments: Option>, +} + +#[derive(Serialize, Deserialize, Debug, Clone)] +pub struct EvalCase { + pub id: Uuid, + pub input: EvalCaseInput, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub expected: Option>, + pub created_at: DateTime, + pub created_by: String, +} + +/// The case fields a caller may set. `id`/`created_at`/`created_by` are assigned server-side so +/// a client cannot forge provenance or collide with an existing case. +#[derive(Deserialize, Debug)] +pub struct NewEvalCase { + #[serde(default)] + pub input: EvalCaseInput, + #[serde(default)] + pub expected: Option>, +} + +#[derive(Deserialize)] +pub struct CreateDataset { + pub path: String, + #[serde(default)] + pub summary: Option, + #[serde(default)] + pub scorers: Vec, + /// The cases to create it holding. A case cannot be written before there is a dataset for it + /// to be a row of, so they are sent with it rather than added afterwards. + #[serde(default)] + pub cases: Vec, +} + +#[derive(Deserialize)] +pub struct EditDataset { + /// Renames the dataset. Its cases and experiments follow through the foreign keys. + #[serde(default)] + pub path: Option, + /// Left out to keep the stored summary; sent as `""` to clear it. + #[serde(default)] + pub summary: Option, + /// Left out to keep the dataset's columns as they are; sent to replace them wholesale. + #[serde(default)] + pub scorers: Option>, + /// The cases as they should stand afterwards: all of them, each carrying its `id` if the + /// dataset already has it. Sent with the rest of an edit so a rename the dataset refuses + /// refuses the case edits with it, rather than leaving them written under the old name. + #[serde(default)] + pub cases: Option>, +} + +#[derive(Deserialize)] +pub struct SaveCase { + #[serde(default)] + pub id: Option, + #[serde(default)] + pub input: EvalCaseInput, + #[serde(default)] + pub expected: Option>, +} + +#[derive(Serialize)] +pub struct ListCasesResponse { + pub cases: Vec, +} + +pub async fn list_datasets( + authed: ApiAuthed, + Extension(user_db): Extension, + Path(w_id): Path, +) -> JsonResult> { + let mut tx = user_db.begin(&authed).await?; + let rows = sqlx::query!( + "SELECT path, summary, scorers, created_at, created_by, + edited_at, edited_by + FROM eval_dataset WHERE workspace_id = $1 ORDER BY path", + w_id + ) + .fetch_all(&mut *tx) + .await?; + tx.commit().await?; + Ok(Json( + rows.into_iter() + .map(|row| { + dataset_from_row( + row.path, + row.summary, + row.scorers, + row.created_at, + row.created_by, + row.edited_at, + row.edited_by, + ) + }) + .collect::>>()?, + )) +} + +pub async fn create_dataset( + authed: ApiAuthed, + Extension(user_db): Extension, + Path(w_id): Path, + Json(payload): Json, +) -> Result { + check_proper_path(&payload.path)?; + check_summary(payload.summary.as_deref())?; + if authed.is_operator { + return Err(Error::NotAuthorized( + "Operators cannot create eval datasets".to_string(), + )); + } + check_case_set( + payload + .cases + .iter() + .map(|case| (&case.input, case.expected.as_ref())), + )?; + let mut scorers = payload.scorers; + // A dataset being created has no columns yet, so every id is minted. + assign_scorer_ids(&mut scorers, &std::collections::HashSet::new())?; + let scorers = serde_json::to_value(&scorers)?; + // One `user_db` transaction: the row's insert policy gates the dataset, the cases' insert + // policy gates each case, and the two land together or not at all. + let mut tx = user_db.begin(&authed).await?; + // A path already taken returns no row; a path the caller may not create raises the insert + // policy, which `map_rls_denied` turns into an access error. + let created = sqlx::query_scalar!( + "INSERT INTO eval_dataset + (workspace_id, path, summary, scorers, created_by, edited_by) + VALUES ($1, $2, $3, $4, $5, $5) + ON CONFLICT (workspace_id, path) DO NOTHING + RETURNING path", + w_id, + payload.path, + payload.summary, + scorers, + authed.username, + ) + .fetch_optional(&mut *tx) + .await + .map_err(|e| map_rls_denied(&payload.path, "create", e))?; + if created.is_none() { + return Err(Error::BadRequest(format!( + "Eval dataset {} already exists", + payload.path + ))); + } + for case in &payload.cases { + sqlx::query!( + // clock_timestamp() (not the now() default, which is transaction-stable) so cases + // saved together get strictly increasing created_at and reload in insertion order; + // ORDER BY created_at, id would otherwise tie-break a same-transaction batch on the + // random uuid id. + "INSERT INTO eval_case + (workspace_id, dataset_path, input, expected, created_by, created_at) + VALUES ($1, $2, $3, $4, $5, clock_timestamp())", + w_id, + payload.path, + serde_json::to_value(&case.input)?, + opt_from_raw(case.expected.as_ref())?, + authed.username, + ) + .execute(&mut *tx) + .await?; + } + tx.commit().await?; + + Ok(format!("Created eval dataset {}", payload.path)) +} + +pub async fn get_dataset( + authed: ApiAuthed, + Extension(user_db): Extension, + Path((w_id, path)): Path<(String, String)>, +) -> JsonResult { + Ok(Json(read_dataset(&authed, &user_db, &w_id, &path).await?)) +} + +/// An edit is one transaction: the rename, the summary, the columns and the cases land together +/// or not at all. +pub async fn update_dataset( + authed: ApiAuthed, + Extension(user_db): Extension, + Path((w_id, path)): Path<(String, String)>, + Json(payload): Json, +) -> Result { + if authed.is_operator { + return Err(Error::NotAuthorized( + "Operators cannot modify eval datasets".to_string(), + )); + } + check_summary(payload.summary.as_deref())?; + let new_path = match payload.path.filter(|p| *p != path) { + Some(new_path) => { + check_proper_path(&new_path)?; + // A rename is owner-only, as for every other renamable object. RLS write access is not + // enough: the UPDATE policies carry no explicit WITH CHECK, so Postgres reuses their + // USING, and the row's own extra_perms travels with the rename and would satisfy it + // for any destination. + windmill_api_auth::require_owner_of_path(&authed, &path)?; + Some(new_path) + } + None => None, + }; + if let Some(cases) = &payload.cases { + check_cases(cases)?; + } + // One `user_db` transaction, governed by the row-level policies throughout. The row is read + // `FOR UPDATE` — its UPDATE policy decides who may — which also pins its cases, so a + // concurrent edit cannot restore a removed scorer's id or interleave with the case write. + let mut tx = user_db.clone().begin(&authed).await?; + let current = sqlx::query_scalar!( + "SELECT scorers FROM eval_dataset WHERE workspace_id = $1 AND path = $2 FOR UPDATE", + w_id, + path + ) + .fetch_optional(&mut *tx) + .await?; + let Some(current) = current else { + drop(tx); + return Err(write_refused(&authed, &user_db, &w_id, &path).await); + }; + let existing: std::collections::HashSet = + parse_scorers(current)?.into_iter().map(|s| s.id).collect(); + let scorers = match payload.scorers { + Some(mut scorers) => { + assign_scorer_ids(&mut scorers, &existing)?; + Some(serde_json::to_value(&scorers)?) + } + None => None, + }; + + let updated = sqlx::query_scalar!( + "UPDATE eval_dataset + SET path = COALESCE($6, path), summary = COALESCE($3, summary), + scorers = COALESCE($4, scorers), edited_at = now(), edited_by = $5 + WHERE workspace_id = $1 AND path = $2 + RETURNING path", + w_id, + path, + payload.summary, + scorers, + authed.username, + new_path.as_deref(), + ) + .fetch_optional(&mut *tx) + .await + .map_err(|e| { + if e.as_database_error().and_then(|e| e.code()).as_deref() == Some("23505") { + Error::BadRequest(format!( + "Eval dataset {} already exists", + new_path.as_deref().unwrap_or(&path) + )) + } else { + map_rls_denied(new_path.as_deref().unwrap_or(&path), "rename", e) + } + })?; + // No row updated: the caller cannot write this dataset (its UPDATE policy denied the row) or it + // is gone. A refused rename destination raises 42501 instead, handled just above. + let Some(updated) = updated else { + drop(tx); + return Err(write_refused(&authed, &user_db, &w_id, &path).await); + }; + // Under the name the dataset now has: the cases followed the rename through the foreign key. + if let Some(cases) = &payload.cases { + write_cases(&mut tx, &w_id, &updated, cases, &authed.username).await?; + } + tx.commit().await?; + Ok(format!("Updated eval dataset {}", updated)) +} + +/// The cases, the experiments and their recorded case sets go with the dataset, through the +/// foreign keys. The jobs those experiments produced are not touched: they are jobs, with their +/// own retention. +pub async fn delete_dataset( + authed: ApiAuthed, + Extension(user_db): Extension, + Path((w_id, path)): Path<(String, String)>, +) -> Result { + check_proper_path(&path)?; + if authed.is_operator { + return Err(Error::NotAuthorized( + "Operators cannot delete eval datasets".to_string(), + )); + } + let mut tx = user_db.clone().begin(&authed).await?; + let deleted = sqlx::query_scalar!( + "DELETE FROM eval_dataset WHERE workspace_id = $1 AND path = $2 RETURNING path", + w_id, + path + ) + .fetch_optional(&mut *tx) + .await?; + tx.commit().await?; + if deleted.is_none() { + return Err(write_refused(&authed, &user_db, &w_id, &path).await); + } + Ok(format!("Deleted eval dataset {}", path)) +} + +// ----------------------------------------------------------------------------------------------- +// Cases +// ----------------------------------------------------------------------------------------------- + +async fn read_cases( + tx: &mut sqlx::Transaction<'_, sqlx::Postgres>, + w_id: &str, + dataset: &str, + per_page: usize, + offset: usize, +) -> Result> { + let rows = sqlx::query!( + "SELECT id, input, expected, created_at, created_by + FROM eval_case + WHERE workspace_id = $1 AND dataset_path = $2 + ORDER BY created_at, id + LIMIT $3 OFFSET $4", + w_id, + dataset, + per_page as i64, + offset as i64 + ) + .fetch_all(&mut **tx) + .await?; + rows.into_iter() + .map(|row| { + Ok(EvalCase { + id: row.id, + input: serde_json::from_value(row.input)?, + expected: opt_to_raw(row.expected)?, + created_at: row.created_at, + created_by: row.created_by, + }) + }) + .collect() +} + +pub async fn list_cases( + authed: ApiAuthed, + Extension(user_db): Extension, + Path((w_id, path)): Path<(String, String)>, + Query(pagination): Query, +) -> JsonResult { + check_proper_path(&path)?; + let (per_page, offset) = paginate(pagination); + let mut tx = user_db.begin(&authed).await?; + // The dataset first, so an unknown or unreadable one is a 404 rather than an empty dataset: + // the case rows are invisible in both cases. + let dataset = sqlx::query_scalar!( + "SELECT path FROM eval_dataset WHERE workspace_id = $1 AND path = $2", + w_id, + path + ) + .fetch_optional(&mut *tx) + .await?; + if dataset.is_none() { + return Err(Error::NotFound(format!("Eval dataset {} not found", path))); + } + let cases = read_cases(&mut tx, &w_id, &path, per_page, offset).await?; + tx.commit().await?; + Ok(Json(ListCasesResponse { cases })) +} + +/// What a whole list of cases can be refused for, before any of it is written. +fn check_cases(cases: &[SaveCase]) -> Result<()> { + check_case_set( + cases + .iter() + .map(|case| (&case.input, case.expected.as_ref())), + )?; + // One row per id: the same id twice would write one row twice and return a list longer than + // the dataset it describes, and the save would read as having kept a case it dropped. + let mut ids: Vec = cases.iter().filter_map(|c| c.id).collect(); + ids.sort(); + let submitted = ids.len(); + ids.dedup(); + if ids.len() != submitted { + return Err(Error::BadRequest( + "A case id appears more than once in the dataset".to_string(), + )); + } + Ok(()) +} + +/// Replace a dataset's cases with `cases`, in the caller's transaction: rows not in the list go, +/// rows carrying an id are updated, the rest are added. Returns one id per case, in order. +async fn write_cases( + tx: &mut sqlx::Transaction<'_, sqlx::Postgres>, + w_id: &str, + path: &str, + cases: &[SaveCase], + username: &str, +) -> Result> { + let kept: Vec = cases.iter().filter_map(|c| c.id).collect(); + sqlx::query!( + "DELETE FROM eval_case + WHERE workspace_id = $1 AND dataset_path = $2 AND NOT (id = ANY($3))", + w_id, + path, + &kept + ) + .execute(&mut **tx) + .await?; + + let mut ids = Vec::with_capacity(cases.len()); + for case in cases { + let input = serde_json::to_value(&case.input)?; + let expected = opt_from_raw(case.expected.as_ref())?; + let id = match case.id { + Some(id) => sqlx::query_scalar!( + "UPDATE eval_case SET input = $4, expected = $5 + WHERE workspace_id = $1 AND dataset_path = $2 AND id = $3 + RETURNING id", + w_id, + path, + id, + input, + expected, + ) + .fetch_optional(&mut **tx) + .await? + .ok_or_else(|| Error::NotFound(format!("Eval case {} not found in {}", id, path)))?, + None => sqlx::query_scalar!( + // clock_timestamp() keeps a same-transaction batch in insertion order on reload. + "INSERT INTO eval_case + (workspace_id, dataset_path, input, expected, created_by, created_at) + VALUES ($1, $2, $3, $4, $5, clock_timestamp()) + RETURNING id", + w_id, + path, + input, + expected, + username, + ) + .fetch_one(&mut **tx) + .await + .map_err(|e| { + if is_missing_dataset(&e) { + Error::NotFound(format!("Eval dataset {} not found", path)) + } else { + e.into() + } + })?, + }; + ids.push(id); + } + Ok(ids) +} diff --git a/backend/windmill-api/src/ai_evals/mod.rs b/backend/windmill-api/src/ai_evals/mod.rs new file mode 100644 index 0000000000..b4d6009857 --- /dev/null +++ b/backend/windmill-api/src/ai_evals/mod.rs @@ -0,0 +1,371 @@ +//! Eval datasets for reusable AI agents. +//! +//! Five tables: `eval_dataset` and the `eval_case` rows it holds are the curated inputs; +//! `eval_experiment`, `eval_experiment_case` and `eval_score` are one run of them, written once +//! and only ever read afterwards. +//! +//! Datasets and cases go through `user_db`, so row-level security is the only access authority: +//! `eval_case`'s policies derive from its dataset's (`eval_dataset_writable`, in the migration). +//! The experiment tables carry read policies only and are written on the unrestricted pool after +//! the API has checked access — see `run_experiment` and `collect_experiment`. + +use axum::{ + extract::{Path, Query}, + routing::{get, post}, + Extension, Json, Router, +}; +use chrono::{DateTime, Utc}; +use serde::{Deserialize, Serialize}; +use serde_json::value::RawValue; +use uuid::Uuid; +use windmill_common::{ + db::UserDB, + error::{Error, JsonResult, Result}, + utils::{check_proper_path, paginate, Pagination}, +}; + +use crate::db::{ApiAuthed, DB}; +use windmill_api_auth::check_scopes; + +pub(crate) mod datasets; +pub(crate) mod payload; +pub(crate) mod results; +pub(crate) mod run; +pub(crate) mod scorers; +pub(crate) mod scoring; +pub(crate) mod subject; +pub(crate) mod template; + +pub(crate) use datasets::*; +pub(crate) use payload::*; +pub(crate) use results::*; +pub(crate) use run::*; +pub(crate) use scorers::*; +pub(crate) use scoring::*; +pub(crate) use subject::*; +pub(crate) use template::*; + +pub fn workspaced_service() -> Router { + Router::new() + .route("/datasets/list", get(list_datasets)) + .route("/datasets/create", post(create_dataset)) + .route("/datasets/get/{*path}", get(get_dataset)) + .route("/datasets/update/{*path}", post(update_dataset)) + .route("/datasets/delete/{*path}", post(delete_dataset)) + .route("/cases/list/{*path}", get(list_cases)) + .route("/scorer_defaults", get(scorer_defaults)) + .route("/run_payload", get(run_payload)) + .route("/experiments/run", post(run_experiment)) + .route("/experiments/collect", post(collect_experiment)) + .route("/scorers/recent", get(recent_scorers)) + .route("/subject_state", get(subject_state)) + .route("/experiments/list_all", get(list_all_experiments)) + .route("/experiments/results/{*path}", get(experiment_results)) +} + +/// Checked here rather than left to the column, whose own refusal comes back as an internal +/// database error naming no field. +const MAX_DATASET_SUMMARY_CHARS: usize = 1000; + +fn check_summary(summary: Option<&str>) -> Result<()> { + match summary { + Some(summary) if summary.chars().count() > MAX_DATASET_SUMMARY_CHARS => { + Err(Error::BadRequest(format!( + "This dataset's summary is {} characters, over the {} the column holds.", + summary.chars().count(), + MAX_DATASET_SUMMARY_CHARS + ))) + } + _ => Ok(()), + } +} + +/// A case is text — attachments are S3 references rather than inline bytes. +const MAX_CASE_BYTES: usize = 256 * 1024; +/// The whole case set together, so cases at the per-case cap cannot add up to a dataset a listing +/// or a run must hold hundreds of megabytes of at once. +const MAX_DATASET_BYTES: usize = 16 * 1024 * 1024; +/// Also what a listing returns in one page, so a dataset is always read whole: the editor holds +/// every case at once and writes them together, and half a set on screen is a Save that drops the +/// rest. +const MAX_CASES_PER_DATASET: i64 = 1_000; + +const MAX_EXPERIMENTS_LISTED: i64 = 100; + +const MAX_RECENT_SCORERS: usize = 12; + +/// A run's work is cases × scorers, so this bounds how far one request fans out. +const MAX_SCORERS_PER_DATASET: usize = 20; + +/// The dataset a write was aimed at is gone. Raised from the foreign key rather than from a +/// preceding existence check, so a dataset deleted mid-request cannot slip between the two. +fn is_missing_dataset(e: &sqlx::Error) -> bool { + e.as_database_error().and_then(|d| d.code()).as_deref() == Some("23503") +} + +/// A `user_db` write the row-level policies refused surfaces as SQLSTATE 42501, whose message +/// names the table and the policy. Turn it into one about access. +fn map_rls_denied(path: &str, action: &str, e: sqlx::Error) -> Error { + if e.as_database_error().and_then(|d| d.code()).as_deref() == Some("42501") { + return Error::NotAuthorized(format!("Not allowed to {} eval dataset {}", action, path)); + } + e.into() +} + +/// A write that matched no row is either a dataset that does not exist or one the caller can read +/// but not write. Row-level security cannot distinguish them — both are simply invisible to the +/// statement — so ask again with a plain read. +async fn write_refused(authed: &ApiAuthed, user_db: &UserDB, w_id: &str, path: &str) -> Error { + let visible = async { + let mut tx = user_db.clone().begin(authed).await?; + let found = sqlx::query_scalar!( + "SELECT path FROM eval_dataset WHERE workspace_id = $1 AND path = $2", + w_id, + path + ) + .fetch_optional(&mut *tx) + .await?; + tx.commit().await?; + Ok::<_, Error>(found.is_some()) + } + .await; + match visible { + Ok(true) => Error::NotAuthorized(format!( + "User {} does not have write access to eval dataset {}", + authed.username, path + )), + Ok(false) => Error::NotFound(format!("Eval dataset {} not found", path)), + Err(e) => e, + } +} + +/// One `eval_dataset` row, from the columns every read of the table selects. +fn dataset_from_row( + path: String, + summary: Option, + scorers: serde_json::Value, + created_at: DateTime, + created_by: String, + edited_at: DateTime, + edited_by: String, +) -> Result { + Ok(EvalDataset { + path, + summary, + scorers: parse_scorers(scorers)?, + created_at, + created_by, + edited_at, + edited_by, + }) +} + +/// A dataset's columns. Only this module writes them, through serde, so a value that does not +/// parse is corruption rather than input: defaulting to no columns would let the next save mint +/// fresh scorer ids and orphan every score already recorded. +pub(crate) fn parse_scorers(scorers: serde_json::Value) -> Result> { + serde_json::from_value(scorers) + .map_err(|e| Error::internal_err(format!("eval dataset scorers are not readable: {e}"))) +} + +/// Read the dataset the request names, through `user_db` so that a caller who cannot see it gets +/// the same answer as one asking for a dataset that does not exist. +async fn read_dataset( + authed: &ApiAuthed, + user_db: &UserDB, + w_id: &str, + path: &str, +) -> Result { + check_proper_path(path)?; + let mut tx = user_db.clone().begin(authed).await?; + let row = sqlx::query!( + "SELECT path, summary, scorers, created_at, created_by, + edited_at, edited_by + FROM eval_dataset WHERE workspace_id = $1 AND path = $2", + w_id, + path + ) + .fetch_optional(&mut *tx) + .await?; + tx.commit().await?; + let row = row.ok_or_else(|| Error::NotFound(format!("Eval dataset {} not found", path)))?; + dataset_from_row( + row.path, + row.summary, + row.scorers, + row.created_at, + row.created_by, + row.edited_at, + row.edited_by, + ) +} + +/// The dataset and its cases as one snapshot, so a launch cannot record the cases from before an +/// edit beside the scorers from after it. One transaction is not enough: `user_db` runs at READ +/// COMMITTED, where each statement takes a fresh snapshot, so the row is taken `FOR UPDATE` — +/// which an edit's own `FOR UPDATE` and a case write's foreign-key lock both conflict with. +pub(crate) async fn read_dataset_and_cases( + authed: &ApiAuthed, + user_db: &UserDB, + w_id: &str, + path: &str, +) -> Result<(EvalDataset, Vec)> { + check_proper_path(path)?; + let mut tx = user_db.clone().begin(authed).await?; + let row = sqlx::query!( + "SELECT path, summary, scorers, created_at, created_by, edited_at, edited_by + FROM eval_dataset WHERE workspace_id = $1 AND path = $2 FOR UPDATE", + w_id, + path + ) + .fetch_optional(&mut *tx) + .await?; + let Some(row) = row else { + tx.commit().await?; + return Err(Error::NotFound(format!("Eval dataset {} not found", path))); + }; + let case_rows = sqlx::query!( + "SELECT id, input, expected, created_at, created_by + FROM eval_case + WHERE workspace_id = $1 AND dataset_path = $2 + ORDER BY created_at, id", + w_id, + path + ) + .fetch_all(&mut *tx) + .await?; + tx.commit().await?; + let dataset = dataset_from_row( + row.path, + row.summary, + row.scorers, + row.created_at, + row.created_by, + row.edited_at, + row.edited_by, + )?; + let cases = case_rows + .into_iter() + .map(|row| { + Ok(EvalCase { + id: row.id, + input: serde_json::from_value(row.input)?, + expected: opt_to_raw(row.expected)?, + created_at: row.created_at, + created_by: row.created_by, + }) + }) + .collect::>>()?; + Ok((dataset, cases)) +} + +/// Whether this caller may write a dataset's contents: its cases, and the experiments that run +/// them. +/// +/// `SELECT … FOR UPDATE` applies `eval_dataset`'s UPDATE policies on top of its SELECT policies, +/// so the row itself answers who may write it, and a grant in `extra_perms` is honoured without +/// being mirrored here. +async fn require_dataset_writable( + authed: &ApiAuthed, + user_db: &UserDB, + w_id: &str, + path: &str, +) -> Result<()> { + check_proper_path(path)?; + if authed.is_operator { + return Err(Error::NotAuthorized( + "Operators cannot modify eval datasets".to_string(), + )); + } + let mut tx = user_db.clone().begin(authed).await?; + let writable = sqlx::query_scalar!( + "SELECT path FROM eval_dataset WHERE workspace_id = $1 AND path = $2 FOR UPDATE", + w_id, + path + ) + .fetch_optional(&mut *tx) + .await?; + tx.commit().await?; + if writable.is_some() { + Ok(()) + } else { + Err(write_refused(authed, user_db, w_id, path).await) + } +} + +/// jsonb columns are read as `serde_json::Value` and handed on as `RawValue`: a case's `expected` +/// is arbitrary user JSON that this module never looks inside. +fn opt_to_raw(value: Option) -> Result>> { + value + .map(|v| Ok(serde_json::value::to_raw_value(&v)?)) + .transpose() +} + +fn opt_from_raw(value: Option<&Box>) -> Result> { + value + .map(|v| Ok(serde_json::from_str(v.get())?)) + .transpose() +} + +fn check_case(input: &EvalCaseInput, expected: Option<&Box>) -> Result<()> { + // The shape the agent step reads its attachments in, checked when the case is written rather + // than when a run deserialises the step's arguments, which is after the case was queued. + if let Some(attachments) = &input.user_attachments { + if serde_json::from_str::>(attachments.get()).is_err() { + return Err(Error::BadRequest( + "A case's user_attachments is a list of S3 objects, each with an `s3` key naming \ + the file" + .to_string(), + )); + } + } + check_case_size(input, expected) +} + +/// The bytes one case weighs against its own and the dataset's caps. +fn case_bytes(input: &EvalCaseInput, expected: Option<&Box>) -> Result { + let mut bytes = serde_json::to_vec(input)?.len(); + if let Some(expected) = expected { + bytes += expected.get().len(); + } + Ok(bytes) +} + +/// What a whole case set can be refused for, before any of it is written. +fn check_case_set<'a>( + cases: impl ExactSizeIterator>)>, +) -> Result<()> { + if cases.len() as i64 > MAX_CASES_PER_DATASET { + return Err(Error::BadRequest(format!( + "An eval dataset holds at most {} cases. Split them into several datasets.", + MAX_CASES_PER_DATASET + ))); + } + let mut total = 0usize; + for (input, expected) in cases { + check_case(input, expected)?; + total += case_bytes(input, expected)?; + } + if total > MAX_DATASET_BYTES { + return Err(Error::BadRequest(format!( + "This dataset is {} KiB of cases, over the {} KiB limit. Attachments belong in \ + workspace storage and are referenced by a case, not stored inside it.", + total / 1024, + MAX_DATASET_BYTES / 1024 + ))); + } + Ok(()) +} + +fn check_case_size(input: &EvalCaseInput, expected: Option<&Box>) -> Result<()> { + let bytes = case_bytes(input, expected)?; + if bytes > MAX_CASE_BYTES { + return Err(Error::BadRequest(format!( + "This eval case is {} KiB, over the {} KiB limit. Attachments belong in workspace \ + storage and are referenced by a case, not stored inside it.", + bytes / 1024, + MAX_CASE_BYTES / 1024 + ))); + } + Ok(()) +} diff --git a/backend/windmill-api/src/ai_evals/payload.rs b/backend/windmill-api/src/ai_evals/payload.rs new file mode 100644 index 0000000000..e89948e9b5 --- /dev/null +++ b/backend/windmill-api/src/ai_evals/payload.rs @@ -0,0 +1,410 @@ +use super::*; + +/// What every scorer is handed: the answer, and the calls the agent made to reach it. +/// +/// Built from the job the run already stored, which is what lets a scorer added later score an +/// experiment that has already run. +#[derive(Serialize, Debug, Clone)] +pub struct EvalRunPayload { + pub input: EvalCaseInput, + #[serde(skip_serializing_if = "Option::is_none")] + pub output: Option>, + #[serde(skip_serializing_if = "Option::is_none")] + pub expected: Option>, + pub tool_calls: Vec, + /// The tools that were actually called, with the schema they were called against. A tool + /// whose schema could not be resolved carries `null`, and a scorer validating arguments must + /// treat that as unchecked rather than as a failure. + pub tools: Vec, + pub metrics: EvalMetrics, + pub status: String, + pub job_id: Uuid, +} + +#[derive(Serialize, Debug, Clone)] +pub struct EvalToolCall { + pub name: String, + #[serde(skip_serializing_if = "Option::is_none")] + pub args: Option>, + #[serde(skip_serializing_if = "Option::is_none")] + pub result: Option>, + #[serde(skip_serializing_if = "Option::is_none")] + pub error: Option, + #[serde(skip_serializing_if = "Option::is_none")] + pub duration_ms: Option, + /// Set when the result was too large to carry and was cut down. + #[serde(skip_serializing_if = "std::ops::Not::not")] + pub truncated: bool, +} + +#[derive(Serialize, Debug, Clone)] +pub struct EvalToolDef { + pub name: String, + #[serde(skip_serializing_if = "Option::is_none")] + pub schema: Option>, +} + +#[derive(Serialize, Debug, Clone)] +pub struct EvalMetrics { + pub steps: usize, + #[serde(skip_serializing_if = "Option::is_none")] + pub duration_ms: Option, + /// The provider's token counts, when it reported any. + #[serde(skip_serializing_if = "Option::is_none")] + pub usage: Option>, +} + +/// A tool result large enough to swamp a judge's context is cut here. The scorer is told, so a +/// check reading a truncated result can say so instead of failing on the missing tail. +const MAX_TOOL_RESULT_BYTES: usize = 4 * 1024; + +fn truncate_value(value: Box) -> (Box, bool) { + if value.get().len() <= MAX_TOOL_RESULT_BYTES { + return (value, false); + } + let text = value.get(); + let mut end = MAX_TOOL_RESULT_BYTES; + while !text.is_char_boundary(end) { + end -= 1; + } + match serde_json::value::to_raw_value(&format!("{}… [truncated]", &text[..end])) { + Ok(v) => (v, true), + Err(_) => (value, false), + } +} + +/// Assemble the payload from a completed case job: the agent step's own result carries the answer +/// and the message list, and every message that made a tool call names the job that ran it. +async fn build_run_payload( + db: &DB, + w_id: &str, + job_id: Uuid, + agent_job: Uuid, + input: EvalCaseInput, + expected: Option>, + status: String, + duration_ms: Option, +) -> Result { + // A read that failed is not a run with no answer: handing the scorers an empty payload would + // have them grade the absence of evidence and record that verdict permanently. + let agent_result = agent_result(db, w_id, job_id).await?.map(|(r, _)| r); + + let parsed: Option = agent_result + .as_ref() + .and_then(|r| serde_json::from_str(r.get()).ok()); + let output = parsed + .as_ref() + .and_then(|p| p.get("output")) + .map(|o| serde_json::value::to_raw_value(o)) + .transpose()?; + let usage = parsed + .as_ref() + .and_then(|p| p.get("usage")) + .map(|u| serde_json::value::to_raw_value(u)) + .transpose()?; + + // Walk the messages in order: a tool call is an `agent_action` on the message that made it. + let mut calls: Vec<(String, Option, Option>)> = vec![]; + if let Some(messages) = parsed + .as_ref() + .and_then(|p| p.get("messages")) + .and_then(|m| m.as_array()) + { + for message in messages { + let Some(action) = message.get("agent_action") else { + continue; + }; + match action.get("type").and_then(|t| t.as_str()) { + Some("tool_call") => calls.push(( + action + .get("function_name") + .and_then(|f| f.as_str()) + .unwrap_or("tool") + .to_string(), + action + .get("job_id") + .and_then(|j| j.as_str()) + .and_then(|j| Uuid::parse_str(j).ok()), + None, + )), + // An MCP call runs inside the agent rather than as a job, so its arguments are on + // the action itself. Its result lives in a later `role: "tool"` message rather + // than a child-job row, and is not surfaced to scorers yet. + Some("mcp_tool_call") => calls.push(( + action + .get("function_name") + .and_then(|f| f.as_str()) + .unwrap_or("tool") + .to_string(), + None, + action + .get("arguments") + .map(|a| serde_json::value::to_raw_value(a)) + .transpose()?, + )), + _ => {} + } + } + } + + let call_job_ids: Vec = calls.iter().filter_map(|(_, id, _)| *id).collect(); + let mut jobs = std::collections::HashMap::new(); + if !call_job_ids.is_empty() { + // Constrained to the agent step's own children rather than to the workspace: these ids + // come out of a job result, so a caller who can run a flow can put any id there. A tool + // call is pushed as a child of the agent that made it, which is what makes that the + // boundary. + let rows = sqlx::query!( + "SELECT j.id, j.args AS \"args: sqlx::types::Json>\", + c.result AS \"result: sqlx::types::Json>\", + c.status::text AS status, c.duration_ms, + s.schema AS \"schema: sqlx::types::Json>\" + FROM v2_job j + LEFT JOIN v2_job_completed c ON c.id = j.id + LEFT JOIN script s ON s.workspace_id = j.workspace_id AND s.hash = j.runnable_id + WHERE j.id = ANY($1) AND j.workspace_id = $2 AND j.parent_job = $3", + &call_job_ids, + w_id, + agent_job + ) + .fetch_all(db) + .await?; + for row in rows { + jobs.insert(row.id, row); + } + } + + let mut tool_calls = Vec::with_capacity(calls.len()); + let mut tools: Vec = vec![]; + for (name, call_job_id, inline_args) in calls { + let row = call_job_id.and_then(|id| jobs.get(&id)); + let (result, truncated) = match row.and_then(|r| r.result.as_ref()) { + Some(result) => { + let (value, truncated) = truncate_value(result.0.clone()); + (Some(value), truncated) + } + None => (None, false), + }; + let failed = row + .and_then(|r| r.status.as_deref()) + .map(|s| s != "success") + .unwrap_or(false); + if !tools.iter().any(|t| t.name == name) { + tools.push(EvalToolDef { + name: name.clone(), + schema: row.and_then(|r| r.schema.as_ref()).map(|s| s.0.clone()), + }); + } + // The already truncated result restated. `render_tool_calls` shows `error` and not + // `result` for a failed call, so the judge's context carries the payload once and bounded; + // `result` stays on the raw call for a script scorer. + let error = failed + .then(|| result.as_ref().map(|r| r.get().to_string())) + .flatten(); + tool_calls.push(EvalToolCall { + name, + args: inline_args.or_else(|| row.and_then(|r| r.args.as_ref()).map(|a| a.0.clone())), + result, + error, + duration_ms: row.map(|r| r.duration_ms), + truncated, + }); + } + + Ok(EvalRunPayload { + metrics: EvalMetrics { steps: tool_calls.len(), duration_ms, usage }, + input, + output, + expected, + tool_calls, + tools, + status, + job_id, + }) +} + +#[derive(Deserialize)] +pub struct RunPayloadQuery { + /// The flow job that answered the case: an iteration of a run. + pub job_id: Uuid, +} + +/// What the scorers of one iteration are handed. +#[derive(Serialize)] +pub struct RunPayloadResponse { + pub run: EvalRunPayload, + /// The same run as a judge reads it. Rendered once per case rather than once per judge. + pub rendered: String, +} + +/// Assemble the payload for one answered case, for the step that feeds the scorers. +/// +/// The case is read from the job's arguments rather than from the experiment, so this works for an +/// iteration whose row has not been filled in yet. +pub async fn run_payload( + authed: ApiAuthed, + Extension(db): Extension, + Extension(user_db): Extension, + Path(w_id): Path, + Query(query): Query, +) -> JsonResult { + // `UserDB` enforces row permissions but not a token's scopes, so without this an + // `ai_evals:read` token would read job arguments, results and tool calls that `jobs:read` + // is what actually gates. Job tokens are unscoped, so the run flow's payload step passes. + check_scopes(&authed, || "jobs:read".to_string())?; + // Through `user_db`: the caller is a job token, and it reads what its runner can read. + let mut tx = user_db.begin(&authed).await?; + let args = sqlx::query_scalar!( + "SELECT args AS \"args: sqlx::types::Json>\" FROM v2_job + WHERE id = $1 AND workspace_id = $2", + query.job_id, + w_id + ) + .fetch_optional(&mut *tx) + .await? + .flatten() + .ok_or_else(|| Error::NotFound(format!("Job {} not found", query.job_id)))?; + tx.commit().await?; + + let args: serde_json::Value = serde_json::from_str(args.0.get())?; + // An iteration carries its case; a run recorded one job per case carries the same input under + // the stamp that job was pushed with. + let case = args.get("iter").and_then(|i| i.get("value")); + let input = case + .and_then(|c| c.get("input")) + .or_else(|| args.get("_eval_input")) + .cloned() + .unwrap_or_else(|| serde_json::json!({})); + let expected = case + .and_then(|c| c.get("expected")) + .or_else(|| args.get("expected")) + .cloned(); + + // The agent step's own status and duration, never the iteration's: the iteration goes on to + // assemble this payload and run the scorers, so a scorer reading the iteration's duration + // would be measuring itself. + let agent_job = agent_step_job(&db, &w_id, query.job_id) + .await? + .unwrap_or(query.job_id); + let completed = sqlx::query!( + "SELECT status::text AS \"status!\", duration_ms FROM v2_job_completed + WHERE id = $1 AND workspace_id = $2", + agent_job, + w_id + ) + .fetch_optional(&db) + .await?; + + let run = build_run_payload( + &db, + &w_id, + query.job_id, + agent_job, + serde_json::from_value(input)?, + expected + .map(|e| serde_json::value::to_raw_value(&e)) + .transpose()?, + completed + .as_ref() + .map(|c| c.status.clone()) + // The iteration asking is itself still running: its agent step is what finished. + .unwrap_or_else(|| "success".to_string()), + completed.as_ref().map(|c| c.duration_ms), + ) + .await?; + let rendered = render_run(&run); + Ok(Json(RunPayloadResponse { run, rendered })) +} + +/// The job of the agent step inside a run's flow, from the flow status of either a running or a +/// finished one. +async fn agent_step_job(db: &DB, w_id: &str, flow_job: Uuid) -> Result> { + let modules = sqlx::query_scalar!( + "SELECT COALESCE(s.flow_status, c.flow_status) -> 'modules' AS modules + FROM v2_job j + LEFT JOIN v2_job_status s ON s.id = j.id + LEFT JOIN v2_job_completed c ON c.id = j.id + WHERE j.id = $1 AND j.workspace_id = $2", + flow_job, + w_id + ) + .fetch_optional(db) + .await? + .flatten(); + Ok(modules + .as_ref() + .and_then(|m| m.as_array()) + .and_then(|modules| { + modules + .iter() + .find(|m| m.get("id").and_then(|i| i.as_str()) == Some(AGENT_NODE_ID)) + }) + .and_then(|m| m.get("job")) + .and_then(|j| j.as_str()) + .and_then(|j| Uuid::parse_str(j).ok())) +} + +/// The system prompt a judge agent is created with. It is the agent's own, so editing a judge is +/// editing that resource — there is no second copy of the grading contract on the dataset. +pub const JUDGE_SYSTEM_PROMPT: &str = r#"You are grading one run of an AI agent. + +Score how well the agent handled the request, from 0 to 1. Judge the whole trajectory, not only the +final answer. Penalise asking for information already in the request, calling a tool twice with the +same arguments, and tool errors left unrecovered. + +Reply with JSON only, of the form {"score": , "reason": }."#; + +fn render_json(value: Option<&RawValue>) -> String { + value + .map(|v| v.get().to_string()) + .unwrap_or_else(|| "(none)".to_string()) +} + +/// Tool calls as the judge reads them: numbered, in order, with arguments, result and duration. +fn render_tool_calls(calls: &[EvalToolCall]) -> String { + if calls.is_empty() { + return "(none)".to_string(); + } + calls + .iter() + .enumerate() + .map(|(index, call)| { + let args = call.args.as_ref().map(|a| a.get()).unwrap_or("{}"); + let outcome = match (&call.error, &call.result) { + (Some(error), _) => format!("error: {}", error), + (None, Some(result)) => result.get().to_string(), + (None, None) => "(no result)".to_string(), + }; + let timing = call + .duration_ms + .map(|ms| format!(" ({}ms)", ms)) + .unwrap_or_default(); + format!( + "{}. {}({}) -> {}{}", + index + 1, + call.name, + args, + outcome, + timing + ) + }) + .collect::>() + .join("\n") +} + +/// One run, as a judge is shown it. +fn render_run(run: &EvalRunPayload) -> String { + format!( + "Request: {}\nTool calls, in order:\n{}\nAnswer: {}\nExpected: {}", + run.input.user_message.as_deref().unwrap_or("(none)"), + render_tool_calls(&run.tool_calls), + render_json(run.output.as_deref()), + render_json(run.expected.as_deref()), + ) +} + +/// Module id of a scorer inside a scoring job. `assign_scorer_ids` keeps ids to +/// `[A-Za-z0-9_]`, so this is a valid identifier. +pub(crate) fn scorer_module_id(scorer_id: &str) -> String { + format!("s_{}", scorer_id) +} diff --git a/backend/windmill-api/src/ai_evals/results.rs b/backend/windmill-api/src/ai_evals/results.rs new file mode 100644 index 0000000000..500af88d72 --- /dev/null +++ b/backend/windmill-api/src/ai_evals/results.rs @@ -0,0 +1,856 @@ +use super::*; + +/// One run of a dataset: written once when the dataset is run, and only ever read afterwards. +#[derive(Serialize, Deserialize, Debug, Clone)] +pub struct EvalExperiment { + pub id: Uuid, + pub dataset: String, + pub subject: EvalSubject, + /// This subject's nth run of this dataset, allocated once and never reused: "Run 7" survives + /// history being pruned, which a position computed when the list is read would not. + pub run_number: i32, + /// The flow executing the run: one job holding every case and its scores. + pub run_job_id: Uuid, + pub case_count: i64, + /// What the run scored, one entry per scorer that produced a number. Carried on the run so a + /// list can say what each one scored without reading every cell of every one of them. + #[serde(default, skip_serializing_if = "Vec::is_empty")] + pub scores: Vec, + /// Whether the flow executing this run is still going. What makes a list of runs worth + /// watching rather than worth reloading. + #[serde(default, skip_serializing_if = "std::ops::Not::not")] + pub running: bool, + pub created_at: DateTime, + pub created_by: String, +} + +/// One scorer's headline for one run: the two numbers a column reports, over that run's cells. +#[derive(Serialize, Deserialize, Debug, Clone)] +pub struct ExperimentScore { + pub scorer_id: String, + /// What the column is called in the dataset that ran it, resolved here because a list of runs + /// spanning datasets cannot hold every dataset's scorers to look it up. + pub name: String, + /// `agent` or `script`, for the badge to say which kind of thing produced the number. + pub kind: String, + #[serde(skip_serializing_if = "Option::is_none")] + pub mean: Option, + /// The share of scored cells at or above the column's threshold, for a column that has one. + /// Absent where the column has no threshold and the mean is the whole headline. + #[serde(skip_serializing_if = "Option::is_none")] + pub pass_rate: Option, + pub scored: i64, + /// How many of this run's cells the column failed on. A column that failed on all of them + /// still ran, which is the difference between a headline of nothing and no headline at all. + pub failed: i64, +} + +#[derive(Deserialize)] +pub struct ListExperimentsQuery { + /// Restrict to one agent's runs. Both what was deployed and what was drafted are that agent's + /// history, so this does not discriminate by kind. + #[serde(default)] + pub subject_path: Option, +} + +/// Every run of this agent, across every dataset it has been measured on. +/// +/// Filtered by `user_db`: a run is visible exactly when the dataset it belongs to is. +pub async fn list_all_experiments( + authed: ApiAuthed, + Extension(db): Extension, + Extension(user_db): Extension, + Path(w_id): Path, + Query(query): Query, +) -> JsonResult> { + let mut tx = user_db.clone().begin(&authed).await?; + let rows = sqlx::query!( + "SELECT e.id, e.dataset_path, e.subject, e.run_number, e.run_job_id, e.created_at, + e.created_by, + (SELECT count(*) FROM eval_experiment_case c WHERE c.experiment_id = e.id) + AS \"case_count!\" + FROM eval_experiment e + JOIN eval_dataset d ON d.workspace_id = e.workspace_id AND d.path = e.dataset_path + WHERE e.workspace_id = $1 + AND ($3::text IS NULL OR e.subject ->> 'path' = $3) + ORDER BY e.created_at DESC + LIMIT $2", + w_id, + MAX_EXPERIMENTS_LISTED, + query.subject_path, + ) + .fetch_all(&mut *tx) + .await?; + tx.commit().await?; + let mut experiments = rows + .into_iter() + .map(|row| { + experiment_from_row( + row.id, + row.dataset_path, + row.subject, + row.run_number, + row.run_job_id, + row.case_count, + row.created_at, + row.created_by, + ) + }) + .collect::>>()?; + + resolve_listed_drafts(&authed, &db, &user_db, &w_id, &mut experiments).await?; + let scorers_by_dataset = scorers_of_listed(&authed, &user_db, &w_id, &experiments).await?; + mark_running(&db, &w_id, &mut experiments).await?; + sync_listed_runs(&db, &w_id, &experiments).await?; + let mut scores = experiment_scores(&db, &experiments, &scorers_by_dataset).await?; + for experiment in experiments.iter_mut() { + experiment.scores = scores.remove(&experiment.id).unwrap_or_default(); + } + Ok(Json(experiments)) +} + +/// Which listed runs are still going, read from the flows executing them. A run whose flow is no +/// longer there at all is over: jobs have their own retention, and reading a missing one as +/// unfinished would leave every run older than it spinning. +async fn mark_running(db: &DB, w_id: &str, experiments: &mut [EvalExperiment]) -> Result<()> { + let job_ids: Vec = experiments.iter().map(|e| e.run_job_id).collect(); + if job_ids.is_empty() { + return Ok(()); + } + let unfinished: std::collections::HashSet = sqlx::query_scalar!( + "SELECT j.id AS \"id!\" FROM v2_job j + LEFT JOIN v2_job_completed c ON c.id = j.id AND c.workspace_id = $2 + WHERE j.id = ANY($1) AND j.workspace_id = $2 AND c.id IS NULL", + &job_ids, + w_id + ) + .fetch_all(db) + .await? + .into_iter() + .collect(); + for experiment in experiments.iter_mut() { + experiment.running = unfinished.contains(&experiment.run_job_id); + } + Ok(()) +} + +/// A run of a draft whose edits have since been deployed is a run of that version. Resolved once +/// per subject rather than once per run, because a listing is usually one agent's history. +async fn resolve_listed_drafts( + authed: &ApiAuthed, + db: &DB, + user_db: &UserDB, + w_id: &str, + experiments: &mut [EvalExperiment], +) -> Result<()> { + let drafted: std::collections::HashSet = experiments + .iter() + .filter(|e| e.subject.kind == EvalSubjectKind::AgentDraft) + .map(|e| e.subject.path.clone()) + .collect(); + if drafted.is_empty() { + return Ok(()); + } + // Read each subject as the caller (see experiment_results): an agent the caller cannot read + // yields no hash or version, so its config fingerprint never leaks through the list either. + let mut deployed = std::collections::HashMap::new(); + for path in drafted { + let (hash, version) = match readable_agent_state(authed, user_db, w_id, &path).await? { + Some((config, version)) => (Some(draft_hash(&config)), Some(version)), + None => (None, None), + }; + deployed.insert(path.clone(), (hash, version)); + } + for experiment in experiments.iter_mut() { + let Some((hash, version)) = deployed.get(&experiment.subject.path) else { + continue; + }; + // Each run's own dataset: the list may span them, and the update is keyed on both. + let dataset = experiment.dataset.clone(); + resolve_deployed_draft(db, w_id, &dataset, experiment, hash.as_deref(), *version).await?; + } + Ok(()) +} + +/// How many listed runs one list call reads out of their flows. A run's scores live in its flow +/// until something reads them into `eval_score`, so an unopened run has nothing to report; the cap +/// keeps a long history from turning one list call into a hundred flow reads. +const MAX_RUNS_SYNCED_PER_LIST: usize = 10; + +/// Read the flows of listed runs that still have scores to collect. Runs already collected are +/// skipped, so the steady-state cost of listing is one query rather than one read per run. +async fn sync_listed_runs(db: &DB, w_id: &str, experiments: &[EvalExperiment]) -> Result<()> { + if experiments.is_empty() { + return Ok(()); + } + let ids: Vec = experiments.iter().map(|e| e.id).collect(); + let unread = sqlx::query_scalar!( + "SELECT DISTINCT experiment_id FROM eval_score + WHERE experiment_id = ANY($1) AND score IS NULL AND error IS NULL + AND NOT not_applicable", + &ids + ) + .fetch_all(db) + .await? + .into_iter() + .collect::>(); + for experiment in experiments + .iter() + .filter(|e| unread.contains(&e.id)) + .take(MAX_RUNS_SYNCED_PER_LIST) + { + // Best-effort, for the same reason reading one run is: this is the home screen, and one + // run with an unreadable cell must not cost the list of every other run. + if let Err(e) = sync_run(db, w_id, experiment.id, experiment.run_job_id, false).await { + tracing::warn!("could not collect eval run {}: {e:#}", experiment.id); + } + } + Ok(()) +} + +/// Every listed run's per-scorer headline, in one grouped query. +/// +/// Thresholds come from each run's own dataset as its scorers are *now*, joined per (run, scorer) +/// rather than per scorer: a list spanning datasets is a list of runs whose columns are not the +/// same columns. +async fn experiment_scores( + db: &DB, + experiments: &[EvalExperiment], + scorers_by_dataset: &std::collections::HashMap>, +) -> Result>> { + let mut by_experiment: std::collections::HashMap> = + Default::default(); + // One entry per (run, column) it could have scored, which is what carries the threshold and + // the column's order into the query. + let mut ids: Vec = vec![]; + let mut scorer_ids: Vec = vec![]; + let mut thresholds: Vec> = vec![]; + for experiment in experiments { + for scorer in scorers_by_dataset + .get(&experiment.dataset) + .map(|s| s.as_slice()) + .unwrap_or(&[]) + { + ids.push(experiment.id); + scorer_ids.push(scorer.id.clone()); + thresholds.push(scorer.pass_if); + } + } + if ids.is_empty() { + return Ok(by_experiment); + } + let rows = sqlx::query!( + "SELECT s.experiment_id AS \"experiment_id!\", s.scorer_id AS \"scorer_id!\", + avg(s.score) AS mean, + count(s.score) AS \"scored!\", + count(*) FILTER (WHERE s.error IS NOT NULL) AS \"failed!\", + count(*) FILTER (WHERE t.pass_if IS NOT NULL AND s.score >= t.pass_if) + AS \"passed!\", + bool_or(t.pass_if IS NOT NULL) AS \"has_threshold!\" + FROM eval_score s + JOIN unnest($1::uuid[], $2::text[], $3::float8[]) + AS t(experiment_id, scorer_id, pass_if) + ON t.experiment_id = s.experiment_id AND t.scorer_id = s.scorer_id + GROUP BY s.experiment_id, s.scorer_id", + &ids, + &scorer_ids, + &thresholds as &[Option], + ) + .fetch_all(db) + .await?; + let mut headline: std::collections::HashMap< + (Uuid, String), + (Option, i64, i64, i64, bool), + > = Default::default(); + for row in rows { + headline.insert( + (row.experiment_id, row.scorer_id), + ( + row.mean, + row.scored, + row.failed, + row.passed, + row.has_threshold, + ), + ); + } + // Emitted in the dataset's column order rather than the query's, so the badges on a row read + // left to right the way that dataset's table does. + for experiment in experiments { + for scorer in scorers_by_dataset + .get(&experiment.dataset) + .map(|s| s.as_slice()) + .unwrap_or(&[]) + { + // A column with no cells at all on this run is one added after it. A column that has + // cells is reported even where none produced a number, which is what a column that + // failed throughout looks like. + let Some((mean, scored, failed, passed, has_threshold)) = + headline.get(&(experiment.id, scorer.id.clone())) + else { + continue; + }; + by_experiment + .entry(experiment.id) + .or_default() + .push(ExperimentScore { + scorer_id: scorer.id.clone(), + name: scorer_name(scorer), + kind: scorer.def.kind_str().to_string(), + mean: *mean, + pass_rate: (*has_threshold && *scored > 0) + .then(|| *passed as f64 / *scored as f64), + scored: *scored, + failed: *failed, + }); + } + } + Ok(by_experiment) +} + +/// The scorers of every dataset named by a listed run, read through `user_db` so a run of a +/// dataset the caller cannot read contributes nothing. +async fn scorers_of_listed( + authed: &ApiAuthed, + user_db: &UserDB, + w_id: &str, + experiments: &[EvalExperiment], +) -> Result>> { + let paths: Vec = experiments + .iter() + .map(|e| e.dataset.clone()) + .collect::>() + .into_iter() + .collect(); + if paths.is_empty() { + return Ok(Default::default()); + } + let mut tx = user_db.clone().begin(authed).await?; + let rows = sqlx::query!( + "SELECT path, scorers FROM eval_dataset WHERE workspace_id = $1 AND path = ANY($2)", + w_id, + &paths + ) + .fetch_all(&mut *tx) + .await?; + tx.commit().await?; + rows.into_iter() + .map(|row| Ok((row.path, parse_scorers(row.scorers)?))) + .collect() +} + +#[derive(Deserialize)] +pub struct ExperimentRef { + pub id: Uuid, + /// The experiment every column is compared against. A delta is only ever computed between two + /// scores of the same scorer id. + #[serde(default)] + pub baseline: Option, +} + +/// One scorer's verdict on one run, and how it compares with the baseline. +#[derive(Serialize)] +pub struct CellScore { + pub scorer_id: String, + #[serde(skip_serializing_if = "Option::is_none")] + pub score: Option, + #[serde(skip_serializing_if = "Option::is_none")] + pub reason: Option, + #[serde(skip_serializing_if = "Option::is_none")] + pub checks: Option>, + #[serde(skip_serializing_if = "Option::is_none")] + pub error: Option, + /// The scorer read this case and had nothing to measure on it. Left out of the column's mean + /// and pass rate rather than counted as a zero. + #[serde(skip_serializing_if = "std::ops::Not::not")] + pub not_applicable: bool, + /// A scoring job is still running for this cell. + pub pending: bool, + /// Which side of the scorer's threshold the score fell on, when it has one. + #[serde(skip_serializing_if = "Option::is_none")] + pub passed: Option, + #[serde(skip_serializing_if = "Option::is_none")] + pub baseline: Option, + /// The baseline's score for this scorer was produced by a different definition of it, so the + /// delta is a change of scorer as much as a change of agent. + pub definition_changed: bool, +} + +/// One row per case: what it was asked, what the agent answered, and each scorer's cell. +#[derive(Serialize)] +pub struct ExperimentRow { + pub case_id: Uuid, + pub input: EvalCaseInput, + #[serde(skip_serializing_if = "Option::is_none")] + pub expected: Option>, + /// The iteration that ran this case. Absent between a run being recorded and its flow + /// reaching this case, which reads as a case still to run. + #[serde(skip_serializing_if = "Option::is_none")] + pub job_id: Option, + /// What happened to the answer: the iteration's own `success`/`failure`/`canceled`/`skipped` + /// once it has finished, and until then the agent step's, since the answer is written before + /// the scorers that keep the iteration running have read it. `unavailable` for a case whose + /// job was retained away before anything read what it produced. + pub status: String, + /// The agent's answer, which is what a table cell shows. The whole trajectory stays + /// reachable through `job_id`, so the row carries the text rather than the result object. + #[serde(skip_serializing_if = "Option::is_none")] + pub output: Option, + /// The agent version this cell ran against. Cells of one experiment can differ, which is what + /// the table says instead of averaging two versions silently. + #[serde(skip_serializing_if = "Option::is_none")] + pub subject_version: Option, + /// For a run of unsaved edits, the hash of the configuration this cell ran: edits move without + /// a version changing, and `resolve_deployed_draft` matches this against what is deployed. + #[serde(skip_serializing_if = "Option::is_none")] + pub subject_draft_hash: Option, + /// One entry per scorer of the dataset, in column order. + pub scores: Vec, +} + +/// A column's summary. There is no single number for a dataset: averaging a judge with an exact +/// match would invent one. +#[derive(Serialize)] +pub struct ScorerMean { + pub scorer_id: String, + #[serde(skip_serializing_if = "Option::is_none")] + pub mean: Option, + #[serde(skip_serializing_if = "Option::is_none")] + pub baseline_mean: Option, + /// The share of scored cells that passed, for a column with a threshold. Reported beside the + /// mean rather than instead of it: neither number answers the other's question. + #[serde(skip_serializing_if = "Option::is_none")] + pub pass_rate: Option, + #[serde(skip_serializing_if = "Option::is_none")] + pub baseline_pass_rate: Option, + pub scored: usize, + /// Cells the baseline has no score for, reported so a column the baseline never ran shows as + /// unscored rather than as a spurious difference. + pub missing_in_baseline: usize, + pub definition_changed: bool, +} + +#[derive(Serialize)] +pub struct ExperimentResults { + pub experiment: EvalExperiment, + #[serde(skip_serializing_if = "Option::is_none")] + pub baseline: Option, + /// The columns, which belong to the dataset rather than to the experiment. + pub scorers: Vec, + pub rows: Vec, + pub means: Vec, + /// Cells scoring lower than the baseline, across every column. + pub regressed: usize, + /// The version the subject is on now. A row that ran against an earlier one describes an + /// agent that no longer exists. + #[serde(skip_serializing_if = "Option::is_none")] + pub subject_current_version: Option, + /// What the agent hashes to as deployed. A run of unsaved edits carrying this hash ran exactly + /// what is deployed now — the edits were saved — so it is a run of that version. + #[serde(skip_serializing_if = "Option::is_none")] + pub subject_deployed_hash: Option, +} + +/// The agent's own result is `{output, messages}`; the answer is its `output`. +pub(crate) fn agent_answer(result: &RawValue) -> Option { + let parsed: serde_json::Value = serde_json::from_str(result.get()).ok()?; + match parsed.get("output") { + Some(serde_json::Value::String(s)) => Some(s.clone()), + Some(other) => Some(other.to_string()), + None => None, + } +} + +struct ScoreRow { + score: Option, + reason: Option, + checks: Option, + error: Option, + not_applicable: bool, + definition: String, +} + +/// Every score of one experiment, keyed by the cell and the scorer that produced it. +async fn load_scores( + db: &DB, + experiment_id: Uuid, +) -> Result> { + Ok(sqlx::query!( + "SELECT ordinal, scorer_id, score, reason, checks, error, not_applicable, definition + FROM eval_score WHERE experiment_id = $1", + experiment_id + ) + .fetch_all(db) + .await? + .into_iter() + .map(|r| { + ( + (r.ordinal, r.scorer_id), + ScoreRow { + score: r.score, + reason: r.reason, + checks: r.checks, + error: r.error, + not_applicable: r.not_applicable, + definition: r.definition, + }, + ) + }) + .collect()) +} + +async fn read_experiment(db: &DB, w_id: &str, dataset: &str, id: Uuid) -> Result { + let row = sqlx::query!( + "SELECT e.subject, e.run_number, e.run_job_id, e.created_at, + e.created_by, + (SELECT count(*) FROM eval_experiment_case c WHERE c.experiment_id = e.id) + AS \"case_count!\" + FROM eval_experiment e + WHERE e.workspace_id = $1 AND e.dataset_path = $2 AND e.id = $3", + w_id, + dataset, + id + ) + .fetch_optional(db) + .await? + .ok_or_else(|| { + Error::NotFound(format!( + "Experiment {} not found in eval dataset {}", + id, dataset + )) + })?; + experiment_from_row( + id, + dataset.to_string(), + row.subject, + row.run_number, + row.run_job_id, + row.case_count, + row.created_at, + row.created_by, + ) +} + +/// Recognise a draft run that has since been deployed, and record it as the version it became. +/// +/// Written once rather than derived per read: derived against what is deployed *now*, the next +/// deployment would send a run that already read `v21` back to `v18 + edits`. +async fn resolve_deployed_draft( + db: &DB, + w_id: &str, + dataset: &str, + experiment: &mut EvalExperiment, + deployed_hash: Option<&str>, + deployed_version: Option, +) -> Result<()> { + if experiment.subject.kind != EvalSubjectKind::AgentDraft { + return Ok(()); + } + let (Some(hash), Some(deployed_hash), Some(version)) = ( + experiment.subject.draft_hash.as_deref(), + deployed_hash, + deployed_version, + ) else { + return Ok(()); + }; + if hash != deployed_hash { + return Ok(()); + } + // The hash stays: it is what identifies the configuration, and what this resolution rests on. + experiment.subject.kind = EvalSubjectKind::Agent; + experiment.subject.version = Some(version); + // Both writes in one transaction: a failure between them would leave the experiment promoted + // to a version while its cells stayed a draft's, a split no later read repairs since the + // experiment is no longer a draft. + let mut tx = db.begin().await?; + sqlx::query!( + "UPDATE eval_experiment + SET subject = jsonb_set( + jsonb_set(subject, '{kind}', '\"agent\"'), + '{version}', to_jsonb($4::bigint)) + WHERE workspace_id = $1 AND dataset_path = $2 AND id = $3 + AND subject ->> 'kind' = 'agent_draft'", + w_id, + dataset, + experiment.id, + version, + ) + .execute(&mut *tx) + .await?; + // The cells that ran that configuration are dated by the version too; leaving their hash would + // make the run go on reading as a draft's after the next deployment. + sqlx::query!( + "UPDATE eval_experiment_case + SET subject_version = $3, subject_draft_hash = NULL + WHERE experiment_id = $1 AND subject_draft_hash = $2", + experiment.id, + hash, + version, + ) + .execute(&mut *tx) + .await?; + tx.commit().await?; + Ok(()) +} + +/// Record what a run produced, from inside the run: the last step of a run's own flow calls this. +/// +/// Gated on reading the run rather than on writing its dataset, unlike everything else here: it is +/// the same harvest `experiment_results` performs behind the same check, over the run's own cells, +/// and it reports a count rather than any of what it read. +pub async fn collect_experiment( + authed: ApiAuthed, + Extension(db): Extension, + Extension(user_db): Extension, + Path(w_id): Path, + Query(query): Query, +) -> JsonResult { + // Through `user_db`, so the run is one the caller can see. The row carries the job to read it + // out of, so nothing that is read afterwards is caller-supplied. + let mut tx = user_db.begin(&authed).await?; + let experiment = sqlx::query!( + "SELECT id, run_job_id FROM eval_experiment WHERE workspace_id = $1 AND id = $2", + w_id, + query.id + ) + .fetch_optional(&mut *tx) + .await?; + tx.commit().await?; + let experiment = + experiment.ok_or_else(|| Error::NotFound(format!("Eval run {} not found", query.id)))?; + sync_run(&db, &w_id, experiment.id, experiment.run_job_id, true).await?; + let recorded = sqlx::query_scalar!( + "SELECT count(*) AS \"count!\" FROM eval_experiment_case + WHERE experiment_id = $1 AND status IS NOT NULL", + experiment.id + ) + .fetch_one(&db) + .await?; + Ok(Json(recorded as usize)) +} + +#[derive(Deserialize)] +pub struct ExperimentId { + pub id: Uuid, +} + +/// Collect a run for a reader, without letting the collection decide whether the read succeeds. +/// `collect_experiment` propagates instead: it is the run reporting on itself, and a failure there +/// is worth surfacing to the step that called it. +async fn collect_quietly(db: &DB, w_id: &str, experiment_id: Uuid, run_job_id: Uuid) { + if let Err(e) = sync_run(db, w_id, experiment_id, run_job_id, true).await { + tracing::warn!("could not collect eval run {}: {e:#}", experiment_id); + } +} + +/// The rows a results table is built from. The job ids come out of `eval_experiment_case`, which +/// only this module writes, so they can be read on the unrestricted pool once the dataset read +/// below has established the caller's access. +pub async fn experiment_results( + authed: ApiAuthed, + Extension(db): Extension, + Extension(user_db): Extension, + Path((w_id, dataset)): Path<(String, String)>, + Query(query): Query, +) -> JsonResult { + // The rows carry what the run's jobs produced, which `jobs:read` gates. `UserDB` settles who + // may see the dataset; a token's scopes are a separate question. + check_scopes(&authed, || "jobs:read".to_string())?; + let dataset_row = read_dataset(&authed, &user_db, &w_id, &dataset).await?; + let scorers = dataset_row.scorers; + + let mut experiment = read_experiment(&db, &w_id, &dataset, query.id).await?; + // Best-effort: collecting is what the run's own step is for, and a cell that could not be read + // — a job retained away between the iteration and its children — must not take the whole table + // down with it. The rows already recorded are still the run. + collect_quietly(&db, &w_id, query.id, experiment.run_job_id).await; + let scores = load_scores(&db, query.id).await?; + + let baseline = match query.baseline.filter(|id| *id != query.id) { + Some(id) => { + let baseline = read_experiment(&db, &w_id, &dataset, id).await?; + collect_quietly(&db, &w_id, id, baseline.run_job_id).await; + Some((baseline, load_scores(&db, id).await?)) + } + None => None, + }; + // The baseline is compared case by case, so its cells are keyed by the case they ran. + let baseline_ordinals = match &baseline { + Some((baseline, _)) => sqlx::query!( + "SELECT case_id, ordinal FROM eval_experiment_case WHERE experiment_id = $1", + baseline.id + ) + .fetch_all(&db) + .await? + .into_iter() + .map(|r| (r.case_id, r.ordinal)) + .collect::>(), + None => Default::default(), + }; + + let case_rows = sqlx::query!( + "SELECT ordinal, case_id, input, expected, job_id, subject_version, + subject_draft_hash, output, answered, status + FROM eval_experiment_case + WHERE experiment_id = $1 ORDER BY ordinal", + query.id + ) + .fetch_all(&db) + .await?; + + let mut sums = vec![(0.0f64, 0usize); scorers.len()]; + let mut baseline_sums = vec![(0.0f64, 0usize); scorers.len()]; + let mut passes = vec![0usize; scorers.len()]; + let mut baseline_passes = vec![0usize; scorers.len()]; + let mut missing_in_baseline = vec![0usize; scorers.len()]; + let mut definition_changed = vec![false; scorers.len()]; + let mut regressed = 0usize; + let mut rows = Vec::with_capacity(case_rows.len()); + + for case in case_rows { + let mut cells = Vec::with_capacity(scorers.len()); + for (index, scorer) in scorers.iter().enumerate() { + let current = scores.get(&(case.ordinal, scorer.id.clone())); + let baseline_score = baseline.as_ref().and_then(|(_, baseline_scores)| { + baseline_ordinals + .get(&case.case_id) + .and_then(|ordinal| baseline_scores.get(&(*ordinal, scorer.id.clone()))) + }); + if let Some(score) = current.and_then(|c| c.score) { + sums[index].0 += score; + sums[index].1 += 1; + if scorer.passed(Some(score)) == Some(true) { + passes[index] += 1; + } + } + if let Some(score) = baseline_score.and_then(|b| b.score) { + baseline_sums[index].0 += score; + baseline_sums[index].1 += 1; + if scorer.passed(Some(score)) == Some(true) { + baseline_passes[index] += 1; + } + } else if baseline.is_some() { + missing_in_baseline[index] += 1; + } + let changed = match (current, baseline_score) { + (Some(current), Some(baseline)) => current.definition != baseline.definition, + _ => false, + }; + if changed { + definition_changed[index] = true; + } + if let (Some(score), Some(previous)) = ( + current.and_then(|c| c.score), + baseline_score.and_then(|b| b.score), + ) { + if score < previous { + regressed += 1; + } + } + cells.push(CellScore { + scorer_id: scorer.id.clone(), + score: current.and_then(|c| c.score), + reason: current.and_then(|c| c.reason.clone()), + checks: current + .and_then(|c| c.checks.clone()) + .map(|c| serde_json::value::to_raw_value(&c)) + .transpose()?, + error: current.and_then(|c| c.error.clone()), + not_applicable: current.map(|c| c.not_applicable).unwrap_or(false), + // A row exists because the run was launched with this scorer, so an empty one is a + // score still to come, unless the scorer has already said this case is not one it + // measures. + pending: current + .map(|c| c.score.is_none() && c.error.is_none() && !c.not_applicable) + .unwrap_or(false), + passed: scorer.passed(current.and_then(|c| c.score)), + baseline: baseline_score.and_then(|b| b.score), + definition_changed: changed, + }); + } + rows.push(ExperimentRow { + case_id: case.case_id, + input: serde_json::from_value(case.input)?, + expected: opt_to_raw(case.expected)?, + // The iteration's verdict once it has one. While it is still running, the agent step's: + // the answer is written before the scorers read it, and a spinner beside an answer + // already there reads as an answer still being written. + status: case + .status + .or_else(|| { + case.answered + .map(|ok| if ok { "success" } else { "failure" }.to_string()) + }) + .unwrap_or_else(|| "running".to_string()), + output: case.output, + subject_version: case.subject_version, + subject_draft_hash: case.subject_draft_hash, + job_id: case.job_id, + scores: cells, + }); + } + + let means = scorers + .iter() + .enumerate() + .map(|(index, scorer)| ScorerMean { + scorer_id: scorer.id.clone(), + mean: (sums[index].1 > 0).then(|| sums[index].0 / sums[index].1 as f64), + baseline_mean: (baseline_sums[index].1 > 0) + .then(|| baseline_sums[index].0 / baseline_sums[index].1 as f64), + pass_rate: (scorer.pass_if.is_some() && sums[index].1 > 0) + .then(|| passes[index] as f64 / sums[index].1 as f64), + baseline_pass_rate: (scorer.pass_if.is_some() && baseline_sums[index].1 > 0) + .then(|| baseline_passes[index] as f64 / baseline_sums[index].1 as f64), + scored: sums[index].1, + missing_in_baseline: missing_in_baseline[index], + definition_changed: definition_changed[index], + }) + .collect(); + + // Read as the caller, so a viewer who can see the dataset but not the agent gets neither: the + // agent's version and configuration fingerprint must not leak past its own read permission. + let (subject_deployed_hash, subject_current_version) = + match readable_agent_state(&authed, &user_db, &w_id, &experiment.subject.path).await? { + Some((config, version)) => (Some(draft_hash(&config)), Some(version)), + None => (None, None), + }; + + // A run of unsaved edits whose configuration has since been deployed is a run of that version. + let mut baseline = baseline.map(|(baseline, _)| baseline); + resolve_deployed_draft( + &db, + &w_id, + &dataset, + &mut experiment, + subject_deployed_hash.as_deref(), + subject_current_version, + ) + .await?; + if let Some(baseline) = baseline.as_mut() { + // The compare-to list holds this agent's runs, but the id is the caller's: a run of another + // agent must not be stamped with this one's version. + if baseline.subject.path == experiment.subject.path { + resolve_deployed_draft( + &db, + &w_id, + &dataset, + baseline, + subject_deployed_hash.as_deref(), + subject_current_version, + ) + .await?; + } + } + + Ok(Json(ExperimentResults { + experiment, + baseline, + scorers, + rows, + means, + regressed, + subject_current_version, + subject_deployed_hash, + })) +} diff --git a/backend/windmill-api/src/ai_evals/run.rs b/backend/windmill-api/src/ai_evals/run.rs new file mode 100644 index 0000000000..e102abaf4c --- /dev/null +++ b/backend/windmill-api/src/ai_evals/run.rs @@ -0,0 +1,950 @@ +use super::*; + +/// Node id of the agent step. The answer is read back by this id, so it is part of the stored +/// shape rather than an implementation detail. +pub const AGENT_NODE_ID: &str = "a"; + +/// Node id of the step that assembles what the scorers are handed. +const PAYLOAD_NODE_ID: &str = "p"; +/// Node id of the loop over the dataset's cases. +const CASES_NODE_ID: &str = "cases"; +/// The branch holding every scorer of a case, so they measure it at the same time. +const SCORERS_NODE_ID: &str = "scores"; + +/// In-flight iterations. A dataset is a burst of calls to one provider, so answering every case at +/// once is a run that spends its time being rate-limited. +const RUN_PARALLELISM: u16 = 8; + +/// What each iteration is handed: the case, small enough to sit in every iteration's arguments. +#[derive(Serialize)] +struct CaseIteration { + case_id: Uuid, + ordinal: i32, + input: EvalCaseInput, + #[serde(skip_serializing_if = "Option::is_none")] + expected: Option>, +} + +/// Assembles the payload the scorers read. +/// +/// A step rather than an input transform: every tool call is enriched with the arguments, result, +/// status and duration of the job that ran it, none of which the flow can see. +const PAYLOAD_SCRIPT: &str = r#"//native +// Generated by Windmill: reads the run this iteration answered. +export async function main() { + const id = process.env.WM_FLOW_JOB_ID + const base = process.env.BASE_URL || process.env.BASE_INTERNAL_URL + const res = await fetch( + `${base}/api/w/${process.env.WM_WORKSPACE}/ai_evals/run_payload?job_id=${id}`, + { headers: { Authorization: `Bearer ${process.env.WM_TOKEN}` } } + ) + if (!res.ok) { + throw new Error(`could not read the run of job ${id}: ${res.status} ${await res.text()}`) + } + return await res.json() +} +"#; + +fn payload_module() -> serde_json::Value { + serde_json::json!({ + "id": PAYLOAD_NODE_ID, + "summary": "Assemble the run the scorers read", + "value": { + "type": "rawscript", + // `bunnative` (tag `nativets`), matching the `//native` the script carries. That tag + // belongs to the `native` worker group rather than the default one, so a queued + // iteration never starts when nothing serves it. + "language": "bunnative", + "content": PAYLOAD_SCRIPT, + "lock": EMPTY_BUN_LOCK, + "input_transforms": {} + } + }) +} + +/// Node id of the step that records what the run produced. +const COLLECT_NODE_ID: &str = "collect"; + +/// Copies the run's answers and scores into its own rows, from inside the run. +/// +/// The tables know nothing about the flow, so without this a run started and left is only ever +/// recorded by someone looking at it — after its jobs have been retained away, there is nothing +/// left to record. +const COLLECT_SCRIPT: &str = r#"//native +// Generated by Windmill: records what this run produced, so it outlives the jobs that produced it. +export async function main(experiment_id: string) { + const base = process.env.BASE_URL || process.env.BASE_INTERNAL_URL + const res = await fetch( + `${base}/api/w/${process.env.WM_WORKSPACE}/ai_evals/experiments/collect?id=${experiment_id}`, + { method: 'POST', headers: { Authorization: `Bearer ${process.env.WM_TOKEN}` } } + ) + if (!res.ok) { + throw new Error(`could not record run ${experiment_id}: ${res.status} ${await res.text()}`) + } + return await res.json() +} +"#; + +fn collect_module(experiment_id: Uuid) -> serde_json::Value { + serde_json::json!({ + "id": COLLECT_NODE_ID, + "summary": "Record what the run produced", + // Bookkeeping, so it does not decide whether the run succeeded. What it would have written + // is written again by the first read of the run. + "continue_on_error": true, + "value": { + "type": "rawscript", + "language": "bunnative", + "content": COLLECT_SCRIPT, + "lock": EMPTY_BUN_LOCK, + "input_transforms": { + "experiment_id": { + "type": "static", + "value": experiment_id.to_string(), + }, + } + } + }) +} + +/// The script imports nothing, so its lockfile is the empty one, spelled the way the bun executor +/// splits it. Without a lock a worker running this as bun would resolve dependencies every time. +const EMPTY_BUN_LOCK: &str = "{\n \"dependencies\": {}\n}\n//bun.lock\n"; + +/// What a judge is asked about the case: the run as it reads it, and the case's own attachments, +/// handed to it as they were handed to the agent. +fn judge_case_transforms() -> serde_json::Map { + let mut transforms = serde_json::Map::new(); + transforms.insert( + "user_message".to_string(), + serde_json::json!({ + "type": "javascript", + "expr": format!("results.{}.rendered", PAYLOAD_NODE_ID), + }), + ); + transforms.insert( + "user_attachments".to_string(), + serde_json::json!({ + "type": "javascript", + "expr": format!("results.{}.run.input.user_attachments", PAYLOAD_NODE_ID), + }), + ); + transforms +} + +/// The scorer steps of one iteration, reading the payload the step before them assembled. +/// +/// Each scorer is baked in as `resolve_scorer` resolved it at launch, never linked by path: a +/// linked step resolves the resource when the case reaches it, so a scorer edited mid-run would +/// grade the later cases while every score still names the definition recorded at launch. +fn scorer_modules(scorers: &[(&Scorer, ResolvedScorer)]) -> Vec { + scorers + .iter() + .map(|(scorer, resolved)| { + let value = match resolved { + // A judge is an agent handed the run as its message; its own system prompt is the + // grading contract, which is why editing a judge means editing that agent. + ResolvedScorer::Judge { config } => { + let mut transforms = match &config.input_transforms { + serde_json::Value::Object(map) => map.clone(), + _ => serde_json::Map::new(), + }; + transforms.extend(judge_case_transforms()); + serde_json::json!({ + "type": "aiagent", + "tools": config.tools, + "input_transforms": serde_json::Value::Object(transforms), + }) + } + // `run` is the whole payload; `input`, `output` and `expected` are the same values + // spelled out, so a three-line scorer does not have to reach into it. + ResolvedScorer::Script { hash } => serde_json::json!({ + "type": "script", + "path": scorer.def.path(), + // Serialized as `ScriptHash` (a hex string), which is the only shape a flow + // module's `hash` field deserializes from — a bare number fails in the worker. + "hash": windmill_common::scripts::ScriptHash(*hash), + "input_transforms": { + "run": { + "type": "javascript", + "expr": format!("results.{}.run", PAYLOAD_NODE_ID), + }, + "input": { + "type": "javascript", + "expr": format!("results.{}.run.input", PAYLOAD_NODE_ID), + }, + "output": { + "type": "javascript", + "expr": format!("results.{}.run.output", PAYLOAD_NODE_ID), + }, + "expected": { + "type": "javascript", + "expr": format!("results.{}.run.expected", PAYLOAD_NODE_ID), + }, + } + }), + }; + serde_json::json!({ "id": scorer_module_id(&scorer.id), "value": value }) + }) + .collect() +} + +/// The flow a whole run is: one loop over the dataset's cases, each iteration answering the case +/// and then scoring the answer. +/// +/// One job rather than one per case: a run outlives the tab that started it, and only a worker can +/// notice that the last case finished. The cases live in the flow's value, stored once, rather +/// than in its arguments, which every iteration inherits a copy of. +fn build_run_flow( + config: &AgentDraft, + cases: &[CaseIteration], + scorers: &[(&Scorer, ResolvedScorer)], + experiment_id: Uuid, +) -> Result { + let mut modules: Vec = vec![agent_module(config)?]; + if !scorers.is_empty() { + modules.push(payload_module()); + // One branch each, run together: scorers read the answer and never each other. Each branch + // keeps its own failure, so a judge that errors costs its own column and no other. + modules.push(serde_json::json!({ + "id": SCORERS_NODE_ID, + "value": { + "type": "branchall", + "parallel": true, + "branches": scorers + .iter() + .zip(scorer_modules(scorers)) + .map(|((scorer, _), module)| serde_json::json!({ + // Named for the column it produces: the graph of a run is read to see which + // scorer did what, and a module id is not what a scorer is called. + "summary": scorer_name(scorer), + "skip_failure": true, + "modules": [module], + })) + .collect::>(), + } + })); + } + + Ok(serde_json::from_value(serde_json::json!({ + "modules": [ + { + "id": CASES_NODE_ID, + "value": { + "type": "forloopflow", + "iterator": { "type": "static", "value": cases }, + "parallel": true, + "parallelism": RUN_PARALLELISM, + // One case failing is one cell of the run, not the end of it. + "skip_failures": true, + "modules": modules, + } + }, + // After the loop, so every case has both answered and been scored by the time it runs. + collect_module(experiment_id), + ] + }))?) +} + +/// The agent step, reading its case from the iteration rather than from the flow's arguments. +fn agent_module(config: &AgentDraft) -> Result { + let flow = build_case_flow(config)?; + let mut value = serde_json::to_value(&flow.modules[0].value)?; + if let Some(map) = value.as_object_mut() { + let transforms = map + .entry("input_transforms") + .or_insert_with(|| serde_json::json!({})); + if let Some(transforms) = transforms.as_object_mut() { + for key in ["user_message", "user_attachments"] { + transforms.insert( + key.to_string(), + serde_json::json!({ + "type": "javascript", + "expr": format!("flow_input.iter.value.input.{}", key), + }), + ); + } + } + } + Ok(serde_json::json!({ "id": AGENT_NODE_ID, "value": value })) +} + +/// The agent step as a one-module flow, so the module shape is validated by deserializing +/// through `FlowValue` rather than trusted as raw JSON. +fn build_case_flow(config: &AgentDraft) -> Result { + // The configuration runs exactly as authored: its own brain transforms are the module's, and + // the case supplies the message and the attachments over the top. + let mut input_transforms = match &config.input_transforms { + serde_json::Value::Object(map) => map.clone(), + _ => serde_json::Map::new(), + }; + for key in ["user_message", "user_attachments"] { + input_transforms.insert( + key.to_string(), + serde_json::json!({ "type": "javascript", "expr": format!("flow_input.{}", key) }), + ); + } + + // Always inlined, never a link to the resource: a linked step would resolve the agent when + // each case runs, which is the one thing a run of a named version must not do. + let mut agent_value = serde_json::Map::new(); + agent_value.insert("type".to_string(), serde_json::json!("aiagent")); + agent_value.insert("tools".to_string(), serde_json::json!(config.tools)); + agent_value.insert( + "input_transforms".to_string(), + serde_json::Value::Object(input_transforms), + ); + Ok(serde_json::from_value(serde_json::json!({ + "modules": [{ "id": AGENT_NODE_ID, "value": serde_json::Value::Object(agent_value) }] + }))?) +} + +/// How many times the agent has been saved, not the identity of the row holding that value: runs +/// are named by it and compared by it, so it has to be the resource's own count rather than a +/// sequence the whole instance shares. +pub(crate) async fn current_resource_version( + db: &DB, + w_id: &str, + path: &str, +) -> Result> { + let version = sqlx::query_scalar!( + "SELECT version FROM resource_version WHERE workspace_id = $1 AND path = $2 + ORDER BY version DESC LIMIT 1", + w_id, + path + ) + .fetch_optional(db) + .await?; + Ok(version) +} + +/// Read the agent through `user_db` so a caller who cannot read the resource cannot run it. +pub(crate) async fn require_agent( + authed: &ApiAuthed, + user_db: &UserDB, + w_id: &str, + agent_path: &str, +) -> Result<()> { + let mut tx = user_db.clone().begin(authed).await?; + let resource_type = sqlx::query_scalar!( + "SELECT resource_type FROM resource WHERE workspace_id = $1 AND path = $2", + w_id, + agent_path + ) + .fetch_optional(&mut *tx) + .await?; + tx.commit().await?; + match resource_type.as_deref() { + Some("ai_agent") => Ok(()), + Some(other) => Err(Error::BadRequest(format!( + "Resource {} is a {}, not an ai_agent", + agent_path, other + ))), + None => Err(Error::NotFound(format!("Agent {} not found", agent_path))), + } +} + +/// An `ai_agent` value as the configuration to run it with: its brain becomes the module's input +/// transforms, its tools the module's tools. The same conversion for a draft and for what is +/// deployed, so the two hash comparably — which is what lets a draft run be recognised as the +/// version it became. +fn config_to_draft(value: serde_json::Value) -> Result { + let mut config = match value { + serde_json::Value::Object(map) => map, + _ => return Err(Error::BadRequest("The agent is not an object".to_string())), + }; + let tools = match config.remove("tools") { + Some(serde_json::Value::Array(tools)) => tools, + _ => vec![], + }; + // Every brain key becomes a static transform: `$res:`/`$var:` in them are resolved by the + // same argument machinery a linked step's resource goes through. + let input_transforms = config + .into_iter() + .map(|(key, value)| (key, serde_json::json!({ "type": "static", "value": value }))) + .collect::>(); + Ok(AgentDraft { input_transforms: serde_json::Value::Object(input_transforms), tools }) +} + +/// An agent's deployed value and the version that names it, in the shape a step runs. `None` when +/// the caller cannot see the resource, or it is not a usable agent. +/// +/// Both from one read: a deploy landing between two reads would pair one version's configuration +/// with another's number, and what a run records of its subject is permanent. +pub(crate) async fn readable_agent_state( + authed: &ApiAuthed, + user_db: &UserDB, + w_id: &str, + path: &str, +) -> Result> { + let mut tx = user_db.clone().begin(authed).await?; + let row = sqlx::query!( + "SELECT r.value AS \"value: sqlx::types::Json\", + (SELECT version FROM resource_version v + WHERE v.workspace_id = r.workspace_id AND v.path = r.path + ORDER BY v.version DESC LIMIT 1) AS version + FROM resource r + WHERE r.workspace_id = $1 AND r.path = $2 AND r.resource_type = 'ai_agent'", + w_id, + path + ) + .fetch_optional(&mut *tx) + .await?; + tx.commit().await?; + let Some(row) = row else { + return Ok(None); + }; + let (Some(value), Some(version)) = (row.value, row.version) else { + return Ok(None); + }; + // A resource's value isn't validated against its type on write, so an ai_agent whose value + // isn't a valid config is no usable state rather than an error: one bad row would otherwise + // 400 the whole results page or run list. + match config_to_draft(value.0) { + Ok(config) => Ok(Some((config, version))), + Err(_) => Ok(None), + } +} + +/// Fill in what the client cannot: the version a saved agent is at, or the configuration a past +/// version held. +/// +/// Returns the configuration the run executes, read once here. Every case then executes that one +/// configuration: resolved per case instead, an agent deployed mid-run would be executed by the +/// cases after it while every row still names the version the run started against. +async fn resolve_subject( + authed: &ApiAuthed, + db: &DB, + user_db: &UserDB, + w_id: &str, + subject: &mut EvalSubject, + draft: Option, +) -> Result { + Ok(match subject.kind { + EvalSubjectKind::Agent => { + let Some((config, version)) = + readable_agent_state(authed, user_db, w_id, &subject.path).await? + else { + return Err(Error::BadRequest(format!( + "Agent {} is not a readable ai_agent resource", + subject.path + ))); + }; + subject.version = Some(version); + config + } + EvalSubjectKind::AgentDraft => { + // The edits live nowhere the server can read them, so the request carries them. The + // agent is still read, so a run can only be filed under one the caller can see. + require_agent(authed, user_db, w_id, &subject.path).await?; + // The version the edits are an edit of, as of now: edits record no version of their + // own, so "v15 plus unsaved edits" means the edits and whatever was deployed when the + // run started. + subject.version = current_resource_version(db, w_id, &subject.path).await?; + draft.ok_or_else(|| Error::BadRequest(DRAFT_REQUIRED.to_string()))? + } + EvalSubjectKind::AgentVersion => { + let Some(version) = subject.version else { + return Err(Error::BadRequest( + "A run of a past version must say which version".to_string(), + )); + }; + let config = + agent_version_config(authed, user_db, db, w_id, &subject.path, version).await?; + subject.draft = Some(config.clone()); + config + } + }) +} + +/// One version of an agent out of its history, in the shape a step runs. +/// +/// Read through `user_db` for the agent itself first: a version is the resource as it was, so +/// seeing one is seeing the resource. +async fn agent_version_config( + authed: &ApiAuthed, + user_db: &UserDB, + db: &DB, + w_id: &str, + agent_path: &str, + version: i64, +) -> Result { + require_agent(authed, user_db, w_id, agent_path).await?; + let value = sqlx::query_scalar!( + "SELECT value FROM resource_version + WHERE version = $1 AND workspace_id = $2 AND path = $3", + version, + w_id, + agent_path + ) + .fetch_optional(db) + .await? + .flatten() + .ok_or_else(|| Error::NotFound(format!("Agent {} has no version {}", agent_path, version)))?; + config_to_draft(value).map_err(|_| { + Error::BadRequest(format!( + "Version {} of {} is not an object", + version, agent_path + )) + }) +} + +const DRAFT_REQUIRED: &str = "A run of unsaved edits must carry the configuration being edited"; + +/// The configuration the request may carry, taken out of the subject it belongs to. +/// +/// A saved agent and a past version are read from the workspace by the path they name, so a +/// request carrying a configuration for them would run something other than the agent it claims to +/// be a run of. Unsaved edits are the one kind the request has to carry: they exist only in the +/// editor. +fn validate_subject(subject: &EvalSubject) -> Result> { + if subject.path.trim().is_empty() { + return Err(Error::BadRequest( + "The subject needs a path: it is the agent a run is filed under".to_string(), + )); + } + match (&subject.draft, &subject.kind) { + (Some(draft), EvalSubjectKind::AgentDraft) => Ok(Some(draft.clone())), + (Some(_), _) => Err(Error::BadRequest( + "A saved agent's configuration is read from the workspace; remove it from the request" + .to_string(), + )), + (None, EvalSubjectKind::AgentDraft) => Err(Error::BadRequest(DRAFT_REQUIRED.to_string())), + (None, _) => Ok(None), + } +} + +// ----------------------------------------------------------------------------------------------- +// Experiments +// ----------------------------------------------------------------------------------------------- + +#[derive(Deserialize)] +pub struct RunExperiment { + pub dataset: String, + pub subject: EvalSubject, +} + +/// Open a run of this dataset. +/// +/// Runs are numbered per (dataset, agent) pair, and the deployed agent and its draft share that +/// numbering: they are the same agent, so "Run 7" of a dataset means one thing whether it ran the +/// deployed value or the edits waiting on top of it. +async fn new_run( + tx: &mut sqlx::Transaction<'_, sqlx::Postgres>, + w_id: &str, + dataset: &str, + subject: &EvalSubject, + username: &str, + run_job_id: Uuid, + id: Uuid, +) -> Result { + // Two runs starting together would otherwise read the same run number. Held for the rest of + // this transaction, which pushes no jobs. + sqlx::query!( + "SELECT pg_advisory_xact_lock(hashtext('ai_eval_open:' || $1 || '/' || $2 || '/' || $3))", + w_id, + dataset, + subject.path, + ) + .execute(&mut **tx) + .await?; + let run_number = sqlx::query_scalar!( + "SELECT coalesce(max(run_number), 0) + 1 FROM eval_experiment + WHERE workspace_id = $1 AND dataset_path = $2 AND subject ->> 'path' = $3", + w_id, + dataset, + subject.path, + ) + .fetch_one(&mut **tx) + .await? + .unwrap_or(1); + sqlx::query!( + "INSERT INTO eval_experiment + (id, workspace_id, dataset_path, subject, run_number, created_by, run_job_id) + VALUES ($1, $2, $3, $4, $5, $6, $7)", + id, + w_id, + dataset, + serde_json::to_value(subject.stamp())?, + run_number, + username, + run_job_id, + ) + .execute(&mut **tx) + .await + .map_err(|e| { + if is_missing_dataset(&e) { + Error::NotFound(format!("Eval dataset {} not found", dataset)) + } else { + e.into() + } + })?; + Ok(id) +} + +pub async fn run_experiment( + authed: ApiAuthed, + Extension(db): Extension, + Extension(user_db): Extension, + Path(w_id): Path, + Json(payload): Json, +) -> Result { + check_scopes(&authed, || "jobs:run".to_string())?; + // A write, not a read: it persists an experiment into the dataset. + require_dataset_writable(&authed, &user_db, &w_id, &payload.dataset).await?; + + let mut subject = payload.subject; + let draft = validate_subject(&subject)?; + let config = resolve_subject(&authed, &db, &user_db, &w_id, &mut subject, draft).await?; + + // One snapshot of the dataset: the scorers and the cases a run records must be the same + // revision, or a Save landing between two reads files a run under cases and columns that never + // stood together. + let (dataset, cases) = + read_dataset_and_cases(&authed, &user_db, &w_id, &payload.dataset).await?; + if cases.is_empty() { + return Err(Error::BadRequest(format!( + "Eval dataset {} has no case to run", + payload.dataset + ))); + } + + let case_count = cases.len(); + // Resolved through the caller's own db, so a run executes only runnables the caller may read, + // and what resolving pinned is baked into the flow. Recorded per cell at launch rather than + // when the score comes back, so a scorer edited mid-run reads as the change of scorer it is. + let mut definitions = Vec::with_capacity(dataset.scorers.len()); + let mut scorers: Vec<(&Scorer, ResolvedScorer)> = Vec::with_capacity(dataset.scorers.len()); + for scorer in &dataset.scorers { + let (definition, resolved) = resolve_scorer(&user_db, &authed, &w_id, scorer).await?; + definitions.push(definition); + scorers.push((scorer, resolved)); + } + + let iterations = cases + .iter() + .enumerate() + .map(|(index, case)| CaseIteration { + case_id: case.id, + ordinal: index as i32, + input: case.input.clone(), + expected: case.expected.clone(), + }) + .collect::>(); + // Both ids are chosen here: the run's own collect step is handed the experiment id, and the + // experiment names its job before that job exists. + let experiment_id = Uuid::new_v4(); + let run_job_id = Uuid::new_v4(); + let flow_value = build_run_flow(&config, &iterations, &scorers, experiment_id)?; + + // Recorded before the job is queued, so a launch that dies partway leaves an experiment naming + // a job that never started rather than a flow no experiment accounts for and nothing collects. + let mut tx = db.begin().await?; + let experiment_id = new_run( + &mut tx, + &w_id, + &payload.dataset, + &subject, + &authed.username, + run_job_id, + experiment_id, + ) + .await?; + + let ordinals = (0..case_count as i32).collect::>(); + let case_ids = cases.iter().map(|c| c.id).collect::>(); + let inputs = cases + .iter() + .map(|c| serde_json::to_value(&c.input)) + .collect::, _>>()?; + let expecteds = cases + .iter() + .map(|c| opt_from_raw(c.expected.as_ref())) + .collect::>>()?; + let versions = vec![subject.version; case_count]; + let hashes = vec![subject.draft.as_ref().map(draft_hash); case_count]; + // No job id: the iteration that answers a case is minted by the flow engine, and the case is + // matched back to it once it exists. + sqlx::query!( + "INSERT INTO eval_experiment_case + (experiment_id, ordinal, case_id, input, expected, subject_version, + subject_draft_hash) + SELECT $1, ordinal, case_id, input, expected, subject_version, subject_draft_hash + FROM UNNEST($2::int[], $3::uuid[], $4::jsonb[], $5::jsonb[], $6::bigint[], $7::text[]) + AS t(ordinal, case_id, input, expected, subject_version, subject_draft_hash)", + experiment_id, + &ordinals, + &case_ids, + &inputs, + &expecteds as &[Option], + &versions as &[Option], + &hashes as &[Option], + ) + .execute(&mut *tx) + .await?; + insert_pending_scores(&mut tx, experiment_id, &ordinals, &scorers, &definitions).await?; + // The foreign key makes a delete racing this assembly fail the commit, so nothing is queued. + // A delete landing between this commit and the push below still cascades the experiment away + // while the flow queues; that launch/delete race is a known beta limitation. + tx.commit().await?; + + if let Err(e) = push_run_flow( + &authed, + &db, + &user_db, + &w_id, + &payload.dataset, + &subject, + experiment_id, + run_job_id, + flow_value, + ) + .await + { + // Nothing ran, so there is nothing to keep: one failed push is the whole run. + sqlx::query!("DELETE FROM eval_experiment WHERE id = $1", experiment_id) + .execute(&db) + .await?; + return Err(e); + } + Ok(experiment_id.to_string()) +} + +/// The cells a run will fill in, written at launch. A pending row is what the table reads as a +/// score still being produced, and it is where the definition that produced it is recorded. +async fn insert_pending_scores( + tx: &mut sqlx::Transaction<'_, sqlx::Postgres>, + experiment_id: Uuid, + ordinals: &[i32], + scorers: &[(&Scorer, ResolvedScorer)], + definitions: &[String], +) -> Result<()> { + if scorers.is_empty() || ordinals.is_empty() { + return Ok(()); + } + let mut rows_ordinal = vec![]; + let mut rows_scorer = vec![]; + let mut rows_definition = vec![]; + for ordinal in ordinals { + for ((scorer, _), definition) in scorers.iter().zip(definitions.iter()) { + rows_ordinal.push(*ordinal); + rows_scorer.push(scorer.id.clone()); + rows_definition.push(definition.clone()); + } + } + sqlx::query!( + "INSERT INTO eval_score (experiment_id, ordinal, scorer_id, definition) + SELECT $1, ordinal, scorer_id, definition + FROM UNNEST($2::int[], $3::text[], $4::text[]) AS t(ordinal, scorer_id, definition) + ON CONFLICT (experiment_id, ordinal, scorer_id) + DO UPDATE SET definition = EXCLUDED.definition, score = NULL, reason = NULL, + checks = NULL, error = NULL, not_applicable = false", + experiment_id, + &rows_ordinal, + &rows_scorer, + &rows_definition, + ) + .execute(&mut **tx) + .await?; + Ok(()) +} + +/// Queue the flow a run is. Its id is chosen by the caller, so the experiment can name it before +/// it exists. +async fn push_run_flow( + authed: &ApiAuthed, + db: &DB, + user_db: &UserDB, + w_id: &str, + dataset: &str, + subject: &EvalSubject, + experiment_id: Uuid, + run_job_id: Uuid, + flow_value: windmill_common::flows::FlowValue, +) -> Result { + use windmill_common::{jobs::JobPayload, users::username_to_permissioned_as}; + use windmill_queue::{push, PushArgs, PushIsolationLevel}; + + let mut args = std::collections::HashMap::new(); + // So the job says what it was evaluating when opened cold from the runs page. Every iteration + // inherits these, so they are the stamp and nothing bulkier. + args.insert( + "_eval".to_string(), + serde_json::value::to_raw_value(&serde_json::json!({ + "subject": subject.stamp(), + "dataset": dataset, + "experiment_id": experiment_id, + }))?, + ); + + let path = subject.path.clone(); + let tx = PushIsolationLevel::Isolated(user_db.clone(), authed.clone().into()); + let (uuid, tx) = push( + db, + tx, + w_id, + JobPayload::RawFlow { value: flow_value, path: Some(path), restarted_from: None }, + PushArgs::from(&args), + authed.display_username(), + &authed.email, + username_to_permissioned_as(&authed.username), + authed.token_prefix.as_deref(), + authed.username_override.as_deref(), + None, + None, + None, + None, + None, + Some(run_job_id), + false, + false, + None, + true, + None, + None, + None, + None, + Some(&authed.clone().into()), + false, + None, + authed.trigger_or_fallback(None), + None, + ) + .await?; + tx.commit().await?; + Ok(uuid) +} + +pub(crate) fn experiment_from_row( + id: Uuid, + dataset: String, + subject: serde_json::Value, + run_number: i32, + run_job_id: Uuid, + case_count: i64, + created_at: DateTime, + created_by: String, +) -> Result { + Ok(EvalExperiment { + id, + dataset, + subject: serde_json::from_value(subject)?, + run_number, + run_job_id, + case_count, + // Filled in by the list, which reads every listed run's scores in one query. + scores: vec![], + running: false, + created_at, + created_by, + }) +} + +#[cfg(test)] +mod tests { + use super::*; + + fn agent_config() -> AgentDraft { + AgentDraft { input_transforms: serde_json::json!({}), tools: vec![] } + } + + fn scorer(kind: ScorerDef) -> Scorer { + Scorer { id: "s1".to_string(), name: None, pass_if: None, def: kind } + } + + fn subject(kind: EvalSubjectKind, draft: Option) -> EvalSubject { + EvalSubject { kind, path: "u/me/agent".to_string(), version: None, draft, draft_hash: None } + } + + /// The whole argument for accepting a configuration from the request is that it is accepted + /// for exactly one kind: the edits in progress, which exist nowhere the server can read. A + /// saved agent or a past version carrying one would run something other than what it names. + #[test] + fn a_configuration_is_required_for_edits_and_refused_for_anything_saved() { + assert!( + validate_subject(&subject(EvalSubjectKind::AgentDraft, Some(agent_config()))).is_ok() + ); + assert!(validate_subject(&subject(EvalSubjectKind::Agent, None)).is_ok()); + assert!(validate_subject(&subject(EvalSubjectKind::AgentDraft, None)).is_err()); + assert!(validate_subject(&subject(EvalSubjectKind::Agent, Some(agent_config()))).is_err()); + assert!(validate_subject(&subject( + EvalSubjectKind::AgentVersion, + Some(agent_config()) + )) + .is_err()); + } + + /// Where the collect step sits is load-bearing twice over: inside the loop it would run once + /// per case, and `backfill_case_jobs` matches a case to any child of the run carrying an + /// `iter` argument, which the collect job must therefore never be. + #[test] + fn the_collect_step_runs_once_after_the_loop() { + let experiment = Uuid::new_v4(); + let flow = build_run_flow(&agent_config(), &[], &[], experiment).unwrap(); + let value = serde_json::to_value(&flow).unwrap(); + let modules = value["modules"].as_array().unwrap(); + assert_eq!( + modules + .iter() + .map(|m| m["id"].as_str().unwrap()) + .collect::>(), + vec![CASES_NODE_ID, COLLECT_NODE_ID] + ); + let collect = &modules[1]; + // The run it records is baked in rather than read from the iteration around it, which is + // what makes it a step of the run and not of a case. + assert_eq!( + collect["value"]["input_transforms"]["experiment_id"]["value"] + .as_str() + .unwrap(), + experiment.to_string() + ); + assert!(collect["value"]["input_transforms"]["iter"].is_null()); + // A failed record must not fail a run whose cases all answered. + assert_eq!(collect["continue_on_error"].as_bool(), Some(true)); + } + + /// A code scorer pins the deployed hash it resolved to, and that hash must reach the flow as a + /// `ScriptHash` (a hex string), not the bare number it is in the database. + #[test] + fn a_script_scorer_pins_its_resolved_hash_as_a_hex_string() { + let s = scorer(ScorerDef::Script { path: "f/e/scorer".to_string() }); + let scorers = vec![(&s, ResolvedScorer::Script { hash: 8816320759749465854i64 })]; + let modules = scorer_modules(&scorers); + // A flow module's `hash` deserializes only from a `ScriptHash` (a hex string); emitted as a + // bare number it fails in the worker and every code-scorer column breaks at runtime. + assert!( + modules[0]["value"]["hash"].is_string(), + "the pinned scorer hash must serialize as a hex string, not a number" + ); + } + + /// A judge is pinned by inlining the configuration resolved at launch rather than linked by + /// path, which is the difference between a run that grades against one definition and one that + /// resolves the judge per case. + #[test] + fn a_judge_is_inlined_rather_than_linked() { + let judge = scorer(ScorerDef::Agent { path: "f/e/judge".to_string() }); + let scorers = vec![( + &judge, + ResolvedScorer::Judge { + config: AgentDraft { + input_transforms: serde_json::json!({ + "system_prompt": { "type": "static", "value": "grade it" } + }), + tools: vec![], + }, + }, + )]; + let pinned = scorer_modules(&scorers); + let value = &pinned[0]["value"]; + assert!(value["agent"].is_null()); + assert_eq!( + value["input_transforms"]["system_prompt"]["value"].as_str(), + Some("grade it") + ); + // The case reaches the judge alongside the judge's own transforms. + assert!(value["input_transforms"]["user_message"]["expr"].is_string()); + assert!(value["input_transforms"]["user_attachments"]["expr"].is_string()); + } +} diff --git a/backend/windmill-api/src/ai_evals/scorers.rs b/backend/windmill-api/src/ai_evals/scorers.rs new file mode 100644 index 0000000000..6d6af3096e --- /dev/null +++ b/backend/windmill-api/src/ai_evals/scorers.rs @@ -0,0 +1,285 @@ +use super::*; + +/// A scorer is a column of the results table. +/// +/// `id` is assigned when the scorer is added to a dataset and never reused: it is what makes a +/// column the same column across experiments when the scorer is renamed or its definition is +/// edited, and a delta is only ever computed between two scores carrying the same id. +#[derive(Serialize, Deserialize, Debug, Clone)] +pub struct Scorer { + /// Assigned on write when a new scorer arrives without one, so a client cannot collide two + /// columns onto one id. + #[serde(default)] + pub id: String, + /// The column header. Defaults to the kind, or the last segment of the path. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub name: Option, + /// A score at or above this counts as a pass. Deliberately outside `definition`: where the + /// line sits interprets a score rather than produces it, so moving it re-reads every score + /// already recorded instead of invalidating them. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub pass_if: Option, + #[serde(flatten)] + pub def: ScorerDef, +} + +/// A judge is an `ai_agent` resource sent the run to grade; a script receives the run as an +/// argument. Both are runnables, so every column has a path, a version and code you can open. +#[derive(Serialize, Deserialize, Debug, Clone, PartialEq)] +#[serde(tag = "kind", rename_all = "snake_case")] +pub enum ScorerDef { + Script { path: String }, + Agent { path: String }, +} + +impl ScorerDef { + pub fn path(&self) -> &str { + match self { + ScorerDef::Script { path } | ScorerDef::Agent { path } => path, + } + } + + /// The wire name of the kind, as the client sends it. + pub(crate) fn kind_str(&self) -> &'static str { + match self { + ScorerDef::Script { .. } => "script", + ScorerDef::Agent { .. } => "agent", + } + } + + fn kind_label(&self) -> &'static str { + match self { + ScorerDef::Script { .. } => "Script", + ScorerDef::Agent { .. } => "Judge agent", + } + } +} + +impl Scorer { + /// Whether a score counts as a pass. `None` when the column has no threshold, which keeps a + /// column of plain numbers from being rendered as if it had one. + pub fn passed(&self, score: Option) -> Option { + match (self.pass_if, score) { + (Some(threshold), Some(score)) => Some(score >= threshold), + _ => None, + } + } + + /// What produced a score, recorded with it so a comparison can say the scorer changed instead + /// of letting that read as a difference between two agents. `resolved` is the script hash or + /// resource version that actually ran, which the path alone does not pin. + pub fn definition(&self, resolved: Option<&str>) -> String { + use sha2::{Digest, Sha256}; + let mut hasher = Sha256::new(); + hasher.update(self.def.kind_label().as_bytes()); + hasher.update(b":"); + hasher.update(self.def.path().as_bytes()); + if let Some(resolved) = resolved { + hasher.update(b"@"); + hasher.update(resolved.as_bytes()); + } + hex::encode(hasher.finalize())[..32].to_string() + } +} + +const MAX_SCORER_NAME_CHARS: usize = 120; + +/// Ids are assigned here rather than trusted from the client: an id is kept only when it names a +/// column the dataset already has, so a removed column cannot come back and inherit the scores +/// recorded against it. Anything else is minted as a valid flow module identifier, which the +/// scoring flows it is baked into require (see `scorer_module_id`). +pub(crate) fn assign_scorer_ids( + scorers: &mut Vec, + existing: &std::collections::HashSet, +) -> Result<()> { + if scorers.len() > MAX_SCORERS_PER_DATASET { + return Err(Error::BadRequest(format!( + "An eval dataset holds at most {} scorers", + MAX_SCORERS_PER_DATASET + ))); + } + let mut seen = std::collections::HashSet::new(); + for scorer in scorers.iter_mut() { + if !existing.contains(&scorer.id) || !seen.insert(scorer.id.clone()) { + scorer.id = Uuid::new_v4().simple().to_string(); + seen.insert(scorer.id.clone()); + } + if let Some(name) = &scorer.name { + if name.chars().count() > MAX_SCORER_NAME_CHARS { + return Err(Error::BadRequest(format!( + "Scorer name {} is too long, {} characters at most", + name, MAX_SCORER_NAME_CHARS + ))); + } + } + // A score is 0 to 1, so a threshold outside that range would pass everything or nothing + // regardless of what the scorer measured. + if let Some(pass_if) = scorer.pass_if { + if !(0.0..=1.0).contains(&pass_if) { + return Err(Error::BadRequest(format!( + "Scorer pass threshold {} must be between 0 and 1", + pass_if + ))); + } + } + check_proper_path(scorer.def.path())?; + } + Ok(()) +} + +/// What a column is called: the dataset's own name for it, or the last segment of what it points +/// at. The same fallback the column header uses. +pub(crate) fn scorer_name(scorer: &Scorer) -> String { + scorer + .name + .clone() + .filter(|n| !n.trim().is_empty()) + .unwrap_or_else(|| { + let path = scorer.def.path(); + path.rsplit('/').next().unwrap_or(path).to_string() + }) +} + +#[derive(Serialize)] +pub struct RecentScorer { + #[serde(flatten)] + pub scorer: Scorer, + /// The dataset it is a column of, which is where the user last saw it. + pub dataset: String, +} + +#[derive(Deserialize)] +pub struct RecentScorersQuery { + /// Only scorers of this kind, which is the one the add form was opened for. + #[serde(default)] + pub kind: Option, +} + +/// The scorers already in use in this workspace, most recently edited dataset first. +/// +/// Filtered twice through `user_db`: a scorer appears only if its dataset does, and the runnable +/// is checked the same way, so the list is scorers the caller could actually run. +pub async fn recent_scorers( + authed: ApiAuthed, + Extension(user_db): Extension, + Path(w_id): Path, + Query(query): Query, +) -> JsonResult> { + let mut tx = user_db.begin(&authed).await?; + let datasets = sqlx::query!( + "SELECT path, scorers FROM eval_dataset + WHERE workspace_id = $1 ORDER BY edited_at DESC LIMIT 100", + w_id + ) + .fetch_all(&mut *tx) + .await?; + + let mut seen = std::collections::HashSet::new(); + let mut recent: Vec = vec![]; + for row in datasets { + for scorer in parse_scorers(row.scorers)? { + if query + .kind + .as_deref() + .is_some_and(|kind| kind != scorer.def.kind_str()) + { + continue; + } + let key = (scorer.def.kind_str(), scorer.def.path().to_string()); + if seen.insert(key) { + recent.push(RecentScorer { scorer, dataset: row.path.clone() }); + } + } + } + // Readability is resolved over every candidate, then the list is cut: an unreadable scorer must + // not take a slot a readable one further down would have filled. + let script_paths = recent + .iter() + .filter(|r| matches!(r.scorer.def, ScorerDef::Script { .. })) + .map(|r| r.scorer.def.path().to_string()) + .collect::>(); + let agent_paths = recent + .iter() + .filter(|r| matches!(r.scorer.def, ScorerDef::Agent { .. })) + .map(|r| r.scorer.def.path().to_string()) + .collect::>(); + // Same deployed-version predicate as get_latest_script_hash: a script with no successfully + // locked version can't be resolved at launch, so it must not offer itself as a scorer here. + let readable_scripts = sqlx::query_scalar!( + "SELECT DISTINCT path FROM script + WHERE workspace_id = $1 AND path = ANY($2) + AND deleted = false AND lock IS NOT NULL AND lock_error_logs IS NULL", + w_id, + &script_paths + ) + .fetch_all(&mut *tx) + .await? + .into_iter() + .collect::>(); + let readable_agents = sqlx::query_scalar!( + "SELECT path FROM resource WHERE workspace_id = $1 AND path = ANY($2) AND resource_type = 'ai_agent'", + w_id, + &agent_paths + ) + .fetch_all(&mut *tx) + .await? + .into_iter() + .collect::>(); + tx.commit().await?; + + recent.retain(|r| match &r.scorer.def { + ScorerDef::Script { path } => readable_scripts.contains(path), + ScorerDef::Agent { path } => readable_agents.contains(path), + }); + recent.truncate(MAX_RECENT_SCORERS); + Ok(Json(recent)) +} + +#[cfg(test)] +mod tests { + use super::*; + + /// The definition hash is what tells a comparison that the scorer changed; the path alone + /// would miss an edit to the script itself. + #[test] + fn definition_moves_with_the_runnable_and_not_with_its_name() { + let script = |path: &str, name: Option<&str>| Scorer { + id: "s1".to_string(), + name: name.map(|n| n.to_string()), + pass_if: None, + def: ScorerDef::Script { path: path.to_string() }, + }; + // Renaming a column is not a change of scorer: same runnable, same version. + assert_eq!( + script("f/e/s", None).definition(Some("1234")), + script("f/e/s", Some("Tool discipline")).definition(Some("1234")) + ); + // Same script, newly deployed: the column says the scorer changed. + assert_ne!( + script("f/e/s", None).definition(Some("1234")), + script("f/e/s", None).definition(Some("5678")) + ); + // A judge agent and a script sharing a path are not the same column. + let agent = Scorer { + id: "s1".to_string(), + name: None, + pass_if: None, + def: ScorerDef::Agent { path: "f/e/s".to_string() }, + }; + assert_ne!( + agent.definition(Some("1")), + script("f/e/s", None).definition(Some("1")) + ); + // If the pass line entered the hash, setting a threshold would mark every score already + // recorded as coming from a different scorer. + let mut thresholded = script("f/e/s", None); + thresholded.pass_if = Some(0.7); + assert_eq!( + thresholded.definition(Some("1234")), + script("f/e/s", None).definition(Some("1234")) + ); + assert_eq!(thresholded.passed(Some(0.7)), Some(true)); + assert_eq!(thresholded.passed(Some(0.69)), Some(false)); + assert_eq!(script("f/e/s", None).passed(Some(0.1)), None); + } +} diff --git a/backend/windmill-api/src/ai_evals/scoring.rs b/backend/windmill-api/src/ai_evals/scoring.rs new file mode 100644 index 0000000000..8fff9d040f --- /dev/null +++ b/backend/windmill-api/src/ai_evals/scoring.rs @@ -0,0 +1,712 @@ +use super::*; + +/// What a scorer resolves to, alongside the definition to record: a script by its pinned hash, or +/// a judge by the configuration to inline. +pub(crate) enum ResolvedScorer { + Script { hash: i64 }, + Judge { config: AgentDraft }, +} + +/// The runnable a scorer names, resolved through the caller's *own* database so a run can only +/// execute code the caller may read: a scorer is added with a bare path and nothing checks read +/// access there. +/// +/// Returns the definition to record and what to run: a script by its deployed hash to pin, or a +/// judge by the configuration to inline, so a redeploy midway through a run cannot swap the code +/// out from under a score labelled with the old version. +pub(crate) async fn resolve_scorer( + user_db: &UserDB, + authed: &ApiAuthed, + w_id: &str, + scorer: &Scorer, +) -> Result<(String, ResolvedScorer)> { + match &scorer.def { + ScorerDef::Script { path } => { + // The latest *deployed* hash (no draft, no failed deploy), through the canonical helper + // so the version a scorer pins is the one everything else runs. + let mut tx = user_db.clone().begin(authed).await?; + let hash = windmill_common::get_latest_script_hash(&mut *tx, path, w_id).await?; + tx.commit().await?; + let Some(hash) = hash else { + return Err(Error::BadRequest(format!( + "Scorer script {} is not deployed or not readable", + path + ))); + }; + Ok(( + scorer.definition(Some(&hash.to_string())), + ResolvedScorer::Script { hash }, + )) + } + ScorerDef::Agent { path } => { + let Some((config, version)) = readable_agent_state(authed, user_db, w_id, path).await? + else { + return Err(Error::BadRequest(format!( + "Judge scorer {} is not a readable ai_agent resource", + path + ))); + }; + Ok(( + scorer.definition(Some(&version.to_string())), + ResolvedScorer::Judge { config }, + )) + } + } +} + +/// Bring a run's record up to date with the flow that executed it: which iteration answered which +/// case, what the agent answered, and what its scorers returned. +/// +/// `answers` is what separates the two callers: a listing reports each run's score aggregates and +/// never shows an answer, so harvesting them there reads a column of every case of every listed +/// run to display none of it. +pub(crate) async fn sync_run( + db: &DB, + w_id: &str, + experiment_id: Uuid, + run_job_id: Uuid, + answers: bool, +) -> Result<()> { + backfill_case_jobs(db, w_id, experiment_id, run_job_id).await?; + settle_unspawned_cases(db, w_id, experiment_id, run_job_id).await?; + if answers { + record_case_answers(db, w_id, experiment_id).await?; + } + harvest_flow_scores(db, w_id, experiment_id).await?; + Ok(()) +} + +/// Give a terminal status to cases the run never spawned an iteration for: with no `job_id` there +/// is nothing to read an answer or a score out of, so they would report "running" indefinitely. +async fn settle_unspawned_cases( + db: &DB, + w_id: &str, + experiment_id: Uuid, + run_job_id: Uuid, +) -> Result<()> { + // Only a run that has reached `v2_job_completed` is settled from here. A job absent from the + // tables is as likely mid-launch — the experiment is committed before its job is pushed — as + // aged out, and settling then would cancel the cases of a run about to start. A cancelled run + // lands in `v2_job_completed`, so a cancel before an iteration spawned is still covered. + let Some(terminal_status) = sqlx::query_scalar!( + "SELECT status::text AS \"status!\" FROM v2_job_completed WHERE id = $1 AND workspace_id = $2", + run_job_id, + w_id + ) + .fetch_optional(db) + .await? + else { + return Ok(()); + }; + let settled = sqlx::query_scalar!( + "UPDATE eval_experiment_case SET status = $2, answered = false + WHERE experiment_id = $1 AND job_id IS NULL AND status IS NULL + RETURNING ordinal", + experiment_id, + terminal_status + ) + .fetch_all(db) + .await?; + // The score cells of a case that never ran have no job to read a verdict out of either. + if !settled.is_empty() { + sqlx::query!( + "UPDATE eval_score SET error = 'The case did not run' + WHERE experiment_id = $1 AND ordinal = ANY($2) + AND score IS NULL AND error IS NULL AND NOT not_applicable", + experiment_id, + &settled + ) + .execute(db) + .await?; + } + Ok(()) +} + +/// In-flight reads of what a case's agent step produced. Each is several queries and a run holds +/// up to `MAX_CASES_PER_DATASET` cases, so they go a few at a time. +const HARVEST_CONCURRENCY: usize = 8; + +/// Cases whose scorer results are read in one query: every scorer of every case in the batch, so +/// the batch bounds how much of a run's worth of judge conversations is held at once. +const HARVEST_BATCH_CASES: usize = 100; + +/// Copy what each iteration produced into its row: the agent's answer, whether producing it +/// succeeded, and how the iteration ended. +/// +/// Written once, when it becomes readable, rather than read back out of the jobs whenever the +/// table is displayed — jobs have their own retention, and a run whose rows are kept has to still +/// read as the run it was after they have aged out. +async fn record_case_answers(db: &DB, w_id: &str, experiment_id: Uuid) -> Result<()> { + let unrecorded = sqlx::query!( + "SELECT c.ordinal, c.job_id AS \"job_id!\", d.status::text AS status, + (j.id IS NOT NULL) AS \"job_exists!\" + FROM eval_experiment_case c + LEFT JOIN v2_job j ON j.id = c.job_id AND j.workspace_id = $2 + LEFT JOIN v2_job_completed d ON d.id = c.job_id AND d.workspace_id = $2 + WHERE c.experiment_id = $1 AND c.job_id IS NOT NULL AND c.status IS NULL", + experiment_id, + w_id + ) + .fetch_all(db) + .await?; + + use futures::StreamExt; + let answers = futures::stream::iter(unrecorded.into_iter().map(|row| async move { + // The job was retained away before anything read it: nothing to read, and nothing more + // will ever be there to read. + if !row.job_exists { + return Ok((row.ordinal, None, None, Some("unavailable".to_string()))); + } + // The agent step's own result, never the iteration's: the iteration goes on to score the + // answer, so the answer is settled long before the iteration is. + let agent = agent_result(db, w_id, row.job_id).await?; + // An iteration that ended without an answer — skipped, cancelled, or an agent that failed + // outright — produced none, and saying so is what stops this re-reading it. + let answered = agent + .as_ref() + .map(|(_, success)| *success) + .or_else(|| row.status.is_some().then_some(false)); + let output = agent.as_ref().and_then(|(result, _)| agent_answer(result)); + Ok::<_, Error>((row.ordinal, output, answered, row.status)) + })) + .buffered(HARVEST_CONCURRENCY) + .collect::>() + .await + .into_iter() + .collect::>>()?; + + // One statement for the whole run: the run's own collect step reaches every case at once, and + // a thousand of them one at a time is a thousand round trips. + let mut ordinals = vec![]; + let mut outputs = vec![]; + let mut answered = vec![]; + let mut statuses = vec![]; + for (ordinal, output, was_answered, status) in answers { + // Nothing to record yet, and the iteration may still produce it. + if was_answered.is_none() && status.is_none() { + continue; + } + ordinals.push(ordinal); + outputs.push(output); + answered.push(was_answered); + statuses.push(status); + } + if ordinals.is_empty() { + return Ok(()); + } + sqlx::query!( + "UPDATE eval_experiment_case c + SET output = COALESCE(c.output, t.output), answered = COALESCE(c.answered, t.answered), + status = COALESCE(c.status, t.status) + FROM UNNEST($2::int[], $3::text[], $4::bool[], $5::text[]) + AS t(ordinal, output, answered, status) + WHERE c.experiment_id = $1 AND c.ordinal = t.ordinal", + experiment_id, + &ordinals, + &outputs as &[Option], + &answered as &[Option], + &statuses as &[Option], + ) + .execute(db) + .await?; + Ok(()) +} + +/// The agent step's result, with "there is none" kept apart from "it could not be read": a lookup +/// that failed for any other reason must not be recorded as a case that produced no answer, +/// because nothing reads that row again. +pub(crate) async fn agent_result( + db: &DB, + w_id: &str, + job_id: Uuid, +) -> Result, bool)>> { + match windmill_queue::get_result_and_success_by_id_from_flow( + db, + w_id, + &job_id, + AGENT_NODE_ID, + None, + ) + .await + { + Ok(found) => Ok(Some(found)), + Err(Error::NotFound(_)) => Ok(None), + Err(e) => Err(e), + } +} + +/// Match each case to the iteration that ran it. The flow engine mints those job ids, so the case +/// they belong to is read back from the iteration's own arguments, which survives iterations +/// finishing in any order. +async fn backfill_case_jobs( + db: &DB, + w_id: &str, + experiment_id: Uuid, + run_job_id: Uuid, +) -> Result<()> { + sqlx::query!( + "UPDATE eval_experiment_case c SET job_id = j.id + FROM v2_job j + WHERE j.parent_job = $3 AND j.workspace_id = $2 + AND (j.args -> 'iter' -> 'value' ->> 'case_id')::uuid = c.case_id + AND c.experiment_id = $1 AND c.job_id IS NULL", + experiment_id, + w_id, + run_job_id + ) + .execute(db) + .await?; + Ok(()) +} + +/// Read the scores a run's own flow produced into `eval_score`, so a score outlives the flow +/// that produced it and the retention on its jobs. +async fn harvest_flow_scores(db: &DB, w_id: &str, experiment_id: Uuid) -> Result<()> { + let pending = sqlx::query!( + // Left-joined, so an iteration still running is read too: a scorer runs after the agent + // within that iteration, so its verdict is there to be read as soon as its own step is + // done, and waiting for the iteration to end would hold every column of a case back until + // the last of them finished. + "SELECT s.ordinal, s.scorer_id, c.job_id AS \"job_id!\", d.status::text AS status, + c.answered, (j.id IS NOT NULL) AS \"job_exists!\" + FROM eval_score s + JOIN eval_experiment_case c + ON c.experiment_id = s.experiment_id AND c.ordinal = s.ordinal + LEFT JOIN v2_job j ON j.id = c.job_id AND j.workspace_id = $2 + LEFT JOIN v2_job_completed d ON d.id = c.job_id AND d.workspace_id = $2 + WHERE s.experiment_id = $1 AND s.score IS NULL AND s.error IS NULL + AND NOT s.not_applicable AND c.job_id IS NOT NULL", + experiment_id, + w_id + ) + .fetch_all(db) + .await?; + if pending.is_empty() { + return Ok(()); + } + + // The job tree is walked in SQL rather than once per cell: a live run is read every couple of + // seconds and a full one is up to MAX_CASES_PER_DATASET × MAX_SCORERS_PER_DATASET cells. The + // shape is `build_run_flow`'s: a scorer is the one module of its own branch of the scoring + // step, so its job's parent is that branch and the branch's parent is the case. + let mut case_jobs: Vec = pending.iter().map(|row| row.job_id).collect(); + case_jobs.sort(); + case_jobs.dedup(); + let mut modules: Vec = pending + .iter() + .map(|row| scorer_module_id(&row.scorer_id)) + .collect(); + modules.sort(); + modules.dedup(); + let mut verdicts: Vec<(i32, String, Option<(Verdict, Option)>)> = + Vec::with_capacity(pending.len()); + for batch in case_jobs.chunks(HARVEST_BATCH_CASES) { + let results: std::collections::HashMap<(Uuid, String), Box> = sqlx::query!( + "SELECT branch.parent_job AS \"case_job!\", scorer.flow_step_id AS \"module!\", + done.result AS \"result: sqlx::types::Json>\" + FROM v2_job branch + JOIN v2_job scorer ON scorer.parent_job = branch.id + JOIN v2_job_completed done ON done.id = scorer.id + WHERE branch.parent_job = ANY($1) AND branch.workspace_id = $2 + AND scorer.flow_step_id = ANY($3)", + batch, + w_id, + &modules + ) + .fetch_all(db) + .await? + .into_iter() + .map(|row| { + let result = row + .result + .map(|json| json.0) + .unwrap_or_else(|| RawValue::from_string("null".to_string()).expect("a literal")); + ((row.case_job, row.module), result) + }) + .collect(); + let in_batch: std::collections::HashSet = batch.iter().copied().collect(); + for row in pending.iter().filter(|row| in_batch.contains(&row.job_id)) { + // Nothing left to read the verdict out of. Settled here, since a cell left pending is + // one every later listing would go back to this same absent job for. + if !row.job_exists { + verdicts.push(( + row.ordinal, + row.scorer_id.clone(), + Some(( + Verdict::default(), + Some("The run that produced this score is no longer available".to_string()), + )), + )); + continue; + } + // What to say when the job is over and this scorer left nothing. Only + // `record_case_answers` tells the two states apart and a listing syncs without it, so + // `None` withholds the sentence — not the harvest: a scorer that returned a number is + // read and recorded either way. + let missing = row.answered.map(|answered| { + if answered { + "This scorer did not run for the case" + } else { + "The case produced no answer to score" + } + }); + let result = results + .get(&(row.job_id, scorer_module_id(&row.scorer_id))) + .map(|r| r.as_ref()); + let verdict = settle_verdict(result, row.status.as_deref(), missing); + verdicts.push((row.ordinal, row.scorer_id.clone(), verdict)); + } + } + + // One statement for every cell read, for the same reason the answers are written that way. + let mut ordinals = vec![]; + let mut scorer_ids = vec![]; + let mut scores = vec![]; + let mut reasons = vec![]; + let mut checks = vec![]; + let mut errors = vec![]; + let mut not_applicable = vec![]; + for (ordinal, scorer_id, read) in verdicts { + // Still to come: a scorer whose own step has not run yet. + let Some((verdict, error)) = read else { + continue; + }; + ordinals.push(ordinal); + scorer_ids.push(scorer_id); + scores.push(verdict.score); + reasons.push(verdict.reason); + checks.push(verdict.checks); + errors.push(error); + not_applicable.push(verdict.not_applicable); + } + if ordinals.is_empty() { + return Ok(()); + } + sqlx::query!( + "UPDATE eval_score s + SET score = t.score, reason = t.reason, checks = t.checks, error = t.error, + not_applicable = t.not_applicable + FROM UNNEST($2::int[], $3::text[], $4::double precision[], $5::text[], $6::jsonb[], + $7::text[], $8::bool[]) + AS t(ordinal, scorer_id, score, reason, checks, error, not_applicable) + WHERE s.experiment_id = $1 AND s.ordinal = t.ordinal AND s.scorer_id = t.scorer_id", + experiment_id, + &ordinals, + &scorer_ids, + &scores as &[Option], + &reasons as &[Option], + &checks as &[Option], + &errors as &[Option], + ¬_applicable, + ) + .execute(db) + .await?; + Ok(()) +} + +/// One scorer's verdict, from the result of the step that produced it, inside a job that may +/// still be running: a scorer's own step can be done while the iteration around it is not. `None` +/// while the result is not readable yet, which is a state to wait through rather than to record +/// as a failure; `Some` with an error is a scorer that produced nothing, worded by where it ran. +fn settle_verdict( + result: Option<&RawValue>, + job_status: Option<&str>, + // What to record when the job is over and this scorer produced nothing. A different statement + // depending on where the scorer ran: its own job failed, or the case it was to score never + // produced an answer. `None` when the caller cannot yet tell those apart, which leaves the + // cell pending for a read that can, rather than settling it on the wrong one of the two. + missing_error: Option<&str>, +) -> Option<(Verdict, Option)> { + Some(match result { + Some(value) => { + let verdict = extract_verdict(value); + match verdict { + // A score is a fraction: the mean and the pass rate read it as one, so a number + // outside that range is recorded as an error rather than a value that would + // quietly skew the column. + Verdict { score: Some(score), .. } if !(0.0..=1.0).contains(&score) => ( + Verdict::default(), + Some(format!( + "The scorer returned {}, outside the 0 to 1 range a score must be in", + score + )), + ), + // A number in range, or the scorer saying this case is not one it measures. Both + // are answers, so neither is an error. + Verdict { score: Some(_), .. } | Verdict { not_applicable: true, .. } => { + (verdict, None) + } + // The job around this scorer is still going, so a module with no number in it is + // one that has not run yet. Recording a failure here would make it permanent. + _ if job_status.is_none() => return None, + _ if job_status == Some("success") => ( + verdict, + Some("The scorer returned no number to plot".to_string()), + ), + _ => match missing_error { + Some(missing) => (verdict, Some(missing.to_string())), + None => return None, + }, + } + } + // The iteration is over, so a scorer step with no readable result produced nothing and + // never will; left pending it would be re-read on every listing. + None if job_status == Some("success") => ( + Verdict::default(), + Some("The scorer step produced no result".to_string()), + ), + // The job holding this scorer has not finished, so a module with nothing in it yet is a + // step that has not run rather than one that produced nothing. + None if job_status.is_none() => return None, + None => match missing_error { + Some(missing) => (Verdict::default(), Some(missing.to_string())), + None => return None, + }, + }) +} + +/// The score and reason read straight out of text that failed to parse as JSON. Deliberately not a +/// second JSON parser: it looks for the two keys and takes what follows, which is what survives a +/// model writing an unescaped quote in the middle of a sentence. +fn salvage_verdict(text: &str) -> (Option, Option) { + fn after_key<'a>(text: &'a str, key: &str) -> Option<&'a str> { + let start = text.find(key)? + key.len(); + Some(text[start..].trim_start().strip_prefix(':')?.trim_start()) + } + + let score = after_key(text, "\"score\"").and_then(|rest| { + if rest.starts_with("true") { + return Some(1.0); + } + if rest.starts_with("false") { + return Some(0.0); + } + let end = rest + .find(|c: char| !matches!(c, '0'..='9' | '.' | '-' | '+' | 'e' | 'E')) + .unwrap_or(rest.len()); + rest[..end].parse::().ok() + }); + + // To the last quote of the object, so an unescaped one inside the sentence stays part of it. + let reason = after_key(text, "\"reason\"") + .and_then(|rest| rest.strip_prefix('"')) + .and_then(|rest| { + let body = match rest.rfind('}') { + Some(brace) => &rest[..brace], + None => rest, + }; + let end = body.rfind('"')?; + Some(body[..end].to_string()) + }) + .filter(|reason| !reason.is_empty()); + + (score, reason) +} + +/// A fenced code block as the model wrote it, reduced to what is inside the fence. The opening +/// fence carries a language tag often enough that the first line goes with it. +fn unfence(text: &str) -> &str { + let trimmed = text.trim(); + let Some(rest) = trimmed.strip_prefix("```") else { + return trimmed; + }; + let inner = match rest.split_once('\n') { + Some((_language, body)) => body, + None => rest, + }; + inner.trim_end().trim_end_matches("```").trim() +} + +/// What a scorer said about one run. `not_applicable` is the scorer declining to measure this +/// case: an explicit `{"score": null}`. A bare `null` stays an error, since a scorer that forgot +/// to return is indistinguishable from one that returned nothing on purpose. +#[derive(Default)] +struct Verdict { + score: Option, + reason: Option, + checks: Option, + not_applicable: bool, +} + +impl Verdict { + fn scored(score: f64) -> Self { + Verdict { score: Some(score), ..Default::default() } + } +} + +/// A scorer may return a bare number, a boolean, or `{score, reason, checks}`; an agent wraps its +/// answer in `output`, sometimes as a string holding any of those. Anything with no number in it +/// is left empty rather than guessed at. +fn extract_verdict(value: &RawValue) -> Verdict { + let Ok(parsed) = serde_json::from_str::(value.get()) else { + return Verdict::default(); + }; + fn as_number(value: &serde_json::Value) -> Option { + match value { + serde_json::Value::Number(n) => n.as_f64(), + serde_json::Value::Bool(b) => Some(if *b { 1.0 } else { 0.0 }), + _ => None, + } + } + if let Some(number) = as_number(&parsed) { + return Verdict::scored(number); + } + let serde_json::Value::Object(map) = &parsed else { + // A judge often answers with JSON inside a string, and often fences it as markdown even + // when told to reply with JSON only. + if let serde_json::Value::String(text) = &parsed { + let text = unfence(text); + if let Ok(inner) = serde_json::from_str::(text) { + if let Ok(raw) = serde_json::value::to_raw_value(&inner) { + return extract_verdict(&raw); + } + } + // Nearly JSON: a judge that quotes the agent inside its own reason writes those quotes + // unescaped, which is invalid and also the most ordinary thing for it to say. The + // number is what the column plots, so it is read out of the text rather than lost with + // the object around it. + let (score, reason) = salvage_verdict(text); + return Verdict { score, reason, checks: None, not_applicable: false }; + } + return Verdict::default(); + }; + let reason = || { + map.get("reason") + .or_else(|| map.get("comment")) + .and_then(|r| r.as_str()) + .map(|r| r.to_string()) + }; + if let Some(score) = map.get("score").and_then(as_number) { + return Verdict { + score: Some(score), + reason: reason(), + checks: map.get("checks").cloned(), + not_applicable: false, + }; + } + // Written out rather than merely absent, which is what separates it from a scorer that + // returned an object with no verdict in it at all. + if map.get("score").is_some_and(|s| s.is_null()) { + return Verdict { + score: None, + reason: reason(), + checks: map.get("checks").cloned(), + not_applicable: true, + }; + } + match map.get("output") { + Some(output) => match serde_json::value::to_raw_value(output) { + Ok(raw) => extract_verdict(&raw), + Err(_) => Verdict::default(), + }, + None => Verdict::default(), + } +} + +#[cfg(test)] +mod tests { + use super::*; + + fn raw(json: &str) -> Box { + serde_json::from_str(json).unwrap() + } + + /// A scorer's answer arrives in whatever shape its runnable returns: a script's bare value or + /// object, or a judge's answer wrapped in `output` and often stringified. A shape that goes + /// unrecognised is a silently empty cell rather than an error. + #[test] + fn extract_verdict_reads_every_documented_scorer_shape() { + let score = |json: &str| extract_verdict(&raw(json)).score; + assert_eq!(score("0.75"), Some(0.75)); + assert_eq!(score("true"), Some(1.0)); + assert_eq!(score(r#"{"score": 0.5}"#), Some(0.5)); + assert_eq!(score(r#"{"score": false}"#), Some(0.0)); + + // judges and agent scorers: the answer is under `output`, sometimes as a string + assert_eq!(score(r#"{"output": 0.25}"#), Some(0.25)); + assert_eq!(score(r#"{"output": "0.9"}"#), Some(0.9)); + assert_eq!(score(r#"{"output": {"score": 0.8}}"#), Some(0.8)); + assert_eq!(score(r#"{"output": "{\"score\": 0.4}"}"#), Some(0.4)); + + // a judge told to reply with JSON only, replying with JSON only, in a code fence + assert_eq!( + score("{\"output\": \"```json\\n{\\\"score\\\": 0.15}\\n```\"}"), + Some(0.15) + ); + assert_eq!(score("{\"output\": \"```\\n0.6\\n```\"}"), Some(0.6)); + + // A judge quoting the agent inside its own reason, which is invalid JSON. + let quoted = extract_verdict(&raw( + r#"{"output": "{\"score\": 0.8, \"reason\": \"invented context (\"stop asking me\", never said) here\"}"}"#, + )); + assert_eq!(quoted.score, Some(0.8)); + assert_eq!( + quoted.reason.as_deref(), + Some(r#"invented context ("stop asking me", never said) here"#) + ); + + // nothing numeric to plot: left empty rather than guessed at + assert_eq!(score(r#"{"output": "not a score"}"#), None); + assert_eq!(score(r#"{"verdict": "good"}"#), None); + + let full = extract_verdict(&raw( + r#"{"score": 0.5, "reason": "half", "checks": [{"name": "a"}]}"#, + )); + assert_eq!( + (full.score, full.reason), + (Some(0.5), Some("half".to_string())) + ); + assert!(full.checks.is_some()); + assert!(!full.not_applicable); + + // `comment` as the rationale, which is what a scorer written for LangSmith or Langfuse + // returns. Read rather than dropped, since the number arrives either way. + assert_eq!( + extract_verdict(&raw(r#"{"score": 1, "comment": "fine"}"#)) + .reason + .as_deref(), + Some("fine") + ); + } + + /// A score is a fraction: anything outside 0..=1 (a scorer that returned a count, say) is + /// recorded as an error naming the value rather than plotted as a bogus point. + #[test] + fn an_out_of_range_score_is_recorded_as_an_error_not_a_value() { + // In range: recorded as the score it is. + let (v, e) = settle_verdict(Some(&raw("0.5")), Some("success"), None).unwrap(); + assert_eq!(v.score, Some(0.5)); + assert!(e.is_none()); + // Out of range (a scorer returning a count, say): no score, an error naming the value. + let (v, e) = settle_verdict(Some(&raw("100")), Some("success"), None).unwrap(); + assert_eq!(v.score, None); + assert!(e.unwrap().contains("100")); + let (v, _) = settle_verdict(Some(&raw("-5")), Some("success"), None).unwrap(); + assert_eq!(v.score, None); + // No result at all once the iteration is over: an error, not a cell pending forever. + let (v, e) = settle_verdict(None, Some("success"), None).unwrap(); + assert_eq!(v.score, None); + assert!(e.is_some()); + // Still running: nothing to settle yet. + assert!(settle_verdict(None, None, None).is_none()); + } + + /// A scorer saying it has nothing to measure on a case is a verdict rather than a failure: the + /// cell is left out of the mean instead of counted as a zero. Spelled out, so a scorer that + /// returns nothing at all is still an error rather than silently excused. + #[test] + fn an_explicit_null_score_is_not_applicable_rather_than_missing() { + let na = extract_verdict(&raw(r#"{"score": null, "reason": "no sources to cite"}"#)); + assert!(na.not_applicable); + assert_eq!(na.score, None); + assert_eq!(na.reason.as_deref(), Some("no sources to cite")); + + // Through a judge's wrapper, as any other verdict is. + assert!(extract_verdict(&raw(r#"{"output": {"score": null}}"#)).not_applicable); + assert!(extract_verdict(&raw(r#"{"output": "{\"score\": null}"}"#)).not_applicable); + + // Not the same as a scorer that returned nothing, or an object with no verdict in it. + assert!(!extract_verdict(&raw("null")).not_applicable); + assert!(!extract_verdict(&raw(r#"{"verdict": "good"}"#)).not_applicable); + } +} diff --git a/backend/windmill-api/src/ai_evals/subject.rs b/backend/windmill-api/src/ai_evals/subject.rs new file mode 100644 index 0000000000..52b8ff0519 --- /dev/null +++ b/backend/windmill-api/src/ai_evals/subject.rs @@ -0,0 +1,135 @@ +use super::*; + +/// What a run is executed against. Kept as `(kind, path, version)` rather than a bare agent +/// path so flow-scoped evaluation is a later superset instead of a rewrite. +#[derive(Serialize, Deserialize, Debug, Clone)] +pub struct EvalSubject { + #[serde(default = "default_subject_kind")] + pub kind: EvalSubjectKind, + /// The agent resource under test. + pub path: String, + /// Which version of the agent, counted per path: how many times it had been saved. The + /// request's to choose for a pinned run, and otherwise the version the run was enqueued + /// against. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub version: Option, + /// The agent's unsaved edits, as the editor holds them. Present exactly when `kind` is + /// `agent_draft`, since the edits exist nowhere else. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub draft: Option, + /// Hash of that configuration. A draft moves without the version moving, so this is the only + /// thing that can say a run describes an agent that has since been edited. Stamped + /// server-side. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub draft_hash: Option, +} + +/// Key order is not meaningful and `serde_json` preserves insertion order here, so it is sorted +/// away before hashing: the same configuration must hash the same however it was assembled. +fn canonical_json(value: &serde_json::Value) -> String { + match value { + serde_json::Value::Object(map) => { + let sorted = map + .iter() + .collect::>() + .into_iter() + .map(|(k, v)| { + format!( + "{}:{}", + serde_json::to_string(k).unwrap_or_default(), + canonical_json(v) + ) + }) + .collect::>() + .join(","); + format!("{{{}}}", sorted) + } + serde_json::Value::Array(items) => format!( + "[{}]", + items + .iter() + .map(canonical_json) + .collect::>() + .join(",") + ), + other => other.to_string(), + } +} + +pub(crate) fn draft_hash(draft: &AgentDraft) -> String { + use sha2::{Digest, Sha256}; + let mut hasher = Sha256::new(); + hasher.update(canonical_json(&draft.input_transforms).as_bytes()); + hasher.update(b"|"); + hasher.update(canonical_json(&serde_json::Value::Array(draft.tools.clone())).as_bytes()); + hex::encode(hasher.finalize())[..32].to_string() +} + +fn default_subject_kind() -> EvalSubjectKind { + EvalSubjectKind::Agent +} + +#[derive(Serialize, Deserialize, Debug, Clone, PartialEq)] +#[serde(rename_all = "snake_case")] +pub enum EvalSubjectKind { + Agent, + /// A saved agent's unsaved edits, carried by the request and inlined: a linked step resolves + /// the resource live and so would run what the edits replace. + AgentDraft, + /// One past version of a saved agent, inlined for the same reason. `version` says which, and + /// it is the request's to choose rather than the server's. + AgentVersion, +} + +/// The brain and tools of an agent, as the flow editor holds them. +#[derive(Serialize, Deserialize, Debug, Clone)] +pub struct AgentDraft { + /// The agent's input transforms: provider, system prompt, output type and the rest. The + /// message and attachments are supplied by the case and override anything named here. + #[serde(default)] + pub input_transforms: serde_json::Value, + #[serde(default)] + pub tools: Vec, +} + +impl EvalSubject { + /// What is recorded of a subject: enough to say what ran, without the configuration itself. + pub(crate) fn stamp(&self) -> EvalSubject { + EvalSubject { + kind: self.kind.clone(), + path: self.path.clone(), + version: self.version, + draft: None, + // Only ever derived from the draft this request carries: a hash the client supplies on + // its own could relabel a run as the deployed version. + draft_hash: self.draft.as_ref().map(draft_hash), + } + } +} + +#[derive(Deserialize)] +pub struct SubjectStateQuery { + pub path: String, +} + +#[derive(Serialize)] +pub struct SubjectState { + /// The version the agent is on now. + #[serde(skip_serializing_if = "Option::is_none")] + pub version: Option, +} + +/// The version the agent is deployed at. Small on purpose: the results endpoint reports the same +/// thing, but it harvests scores and reads every job to do it. +pub async fn subject_state( + authed: ApiAuthed, + Extension(user_db): Extension, + Path(w_id): Path, + Query(query): Query, +) -> JsonResult { + let Some((_, version)) = readable_agent_state(&authed, &user_db, &w_id, &query.path).await? + else { + return Err(Error::NotFound(format!("Agent {} not found", query.path))); + }; + Ok(Json(SubjectState { version: Some(version) })) +} diff --git a/backend/windmill-api/src/ai_evals/template.rs b/backend/windmill-api/src/ai_evals/template.rs new file mode 100644 index 0000000000..22d37d1e85 --- /dev/null +++ b/backend/windmill-api/src/ai_evals/template.rs @@ -0,0 +1,132 @@ +use super::*; + +/// What a script scorer starts from. +pub const SCORER_SCRIPT_TEMPLATE: &str = r#"// A scorer receives one run and returns a number between 0 and 1, a boolean, or +// { score, reason, checks } — checks show up in the case detail. +// Return { score: null } for a case this scorer has nothing to measure on: the cell +// is left out of the column's mean and pass rate rather than counted as a zero. +// +// The run is also handed to you spelled out, so a short scorer can skip the type below +// entirely: export async function main(output: unknown, expected: unknown) { ... } +type ToolCall = { + name: string + args?: Record + result?: unknown + error?: string + duration_ms?: number + truncated?: boolean +} + +type EvalRun = { + input: { user_message?: string; user_attachments?: unknown[] } + output?: unknown + expected?: unknown + tool_calls: ToolCall[] + tools: { name: string; schema?: Record }[] + metrics: { steps: number; duration_ms?: number; usage?: Record } + status: string + job_id: string +} + +export async function main(run: EvalRun) { + // How the agent got to its answer. Reported rather than scored: checks render in the case + // detail either way, so they explain the number without being averaged into it. + const checks = [ + check('arguments match the schema', args_schema_valid(run)), + check('no repeated calls', no_repeated_calls(run)), + check('no failed tool calls', no_step_errors(run)), + check('under 6 steps', run.metrics.steps <= 6, `${run.metrics.steps} steps`), + check('under 30 seconds', under_ms(run, 30_000), `${run.metrics.duration_ms ?? '?'} ms`) + ] + + // Nothing to compare the answer against, so this column has no verdict on this case rather + // than a failing one. The cell reads n/a and the column's mean is of the cases it measured. + if (run.expected == undefined) { + return { score: null, reason: 'this case has no expected answer', checks } + } + + // One question per column, and this column's question is whether the answer is right. + // Deliberately not the share of checks above that passed: a right answer that was slow and a + // wrong answer that was fast would score the same, and the column could not say which it was. + const correct = contains(run.output, text(run.expected)) + return { + score: correct ? 1 : 0, + reason: correct ? undefined : `expected ${text(run.expected)}`, + checks + } +} + +// Helpers. Edit or delete freely. + +function check(name: string, passed: boolean, detail?: string) { + return { name, passed, detail } +} + +function text(value: unknown): string { + return typeof value === 'string' ? value : JSON.stringify(value ?? '') +} + +function contains(output: unknown, needle: string): boolean { + return needle.trim().length > 0 && text(output).toLowerCase().includes(needle.trim().toLowerCase()) +} + +// Every call validated against the schema of the tool it called. A tool whose schema could not be +// resolved is not checked rather than failed. +function args_schema_valid(run: EvalRun): boolean { + return run.tool_calls.every((call) => { + const schema = run.tools.find((tool) => tool.name === call.name)?.schema as + | { properties?: Record; required?: string[] } + | undefined + if (!schema?.properties) return true + const args = call.args ?? {} + for (const key of schema.required ?? []) { + if (args[key] === undefined || args[key] === null) return false + } + for (const [key, value] of Object.entries(args)) { + const expected = schema.properties[key]?.type + if (!expected) continue + const actual = Array.isArray(value) ? 'array' : value === null ? 'null' : typeof value + if (expected === 'integer' ? !Number.isInteger(value) : expected !== actual) return false + } + return true + }) +} + +// The same tool called twice with the same arguments. +function no_repeated_calls(run: EvalRun): boolean { + const seen = new Set() + for (const call of run.tool_calls) { + const key = `${call.name}:${JSON.stringify(call.args ?? {})}` + if (seen.has(key)) return false + seen.add(key) + } + return true +} + +function no_step_errors(run: EvalRun): boolean { + return run.status === 'success' && run.tool_calls.every((call) => !call.error) +} + +// A run with no recorded duration is not under the limit: a check that could not be evaluated +// should not report as one that passed. +function under_ms(run: EvalRun, max: number): boolean { + const ms = run.metrics.duration_ms + return ms != undefined && ms <= max +} +"#; + +#[derive(Serialize)] +pub struct ScorerDefaults { + /// The system prompt a judge agent is created with. It lives on that agent afterwards. + pub judge_prompt: String, + /// The starting point for a script scorer, held here so the shape a scorer is handed and the + /// template that reads it cannot drift apart. + pub script_template: String, +} + +pub async fn scorer_defaults() -> JsonResult { + Ok(Json(ScorerDefaults { + judge_prompt: JUDGE_SYSTEM_PROMPT.to_string(), + script_template: SCORER_SCRIPT_TEMPLATE.to_string(), + })) +} diff --git a/backend/windmill-api/src/lib.rs b/backend/windmill-api/src/lib.rs index 307ed3ac5d..fc98020d1b 100644 --- a/backend/windmill-api/src/lib.rs +++ b/backend/windmill-api/src/lib.rs @@ -116,6 +116,7 @@ pub mod storage_list_ee; mod storage_list_oss; mod workspace_dependencies; +mod ai_evals; mod approvals; #[cfg(all(feature = "enterprise", feature = "private"))] pub mod apps_ee; @@ -677,6 +678,7 @@ pub async fn run_server( .route("/labels/list", get(list_workspace_labels)) .nest("/job_metrics", job_metrics::workspaced_service()) .nest("/job_helpers", job_helpers_service) + .nest("/ai_evals", ai_evals::workspaced_service()) .nest("/jobs", jobs::workspaced_service()) .nest("/debug", windmill_api_debug::workspaced_service()) .nest("/native_triggers", { diff --git a/backend/windmill-api/src/offboarding.rs b/backend/windmill-api/src/offboarding.rs index 0d9529c0ce..a69441ea75 100644 --- a/backend/windmill-api/src/offboarding.rs +++ b/backend/windmill-api/src/offboarding.rs @@ -52,6 +52,8 @@ struct OffboardAffectedPaths { variables: Vec, #[serde(skip_serializing_if = "Vec::is_empty")] schedules: Vec, + #[serde(skip_serializing_if = "Vec::is_empty")] + eval_datasets: Vec, #[serde(skip_serializing_if = "HashMap::is_empty")] triggers: HashMap>, } @@ -91,6 +93,7 @@ struct OffboardSummary { flows_reassigned: i64, apps_reassigned: i64, resources_reassigned: i64, + eval_datasets_reassigned: i64, variables_reassigned: i64, schedules_reassigned: i64, triggers_reassigned: i64, @@ -168,6 +171,14 @@ async fn get_offboard_preview( .fetch_all(db) .await?; + let eval_datasets = sqlx::query_scalar!( + "SELECT path FROM eval_dataset WHERE path LIKE $1 AND workspace_id = $2", + &user_prefix, + w_id + ) + .fetch_all(db) + .await?; + let variables = sqlx::query_scalar!( "SELECT path FROM variable WHERE path LIKE $1 AND workspace_id = $2", &user_prefix, @@ -285,6 +296,16 @@ async fn get_offboard_preview( &ref_pattern, &user_prefix, w_id ).fetch_all(db).await?; + let ref_eval_datasets = sqlx::query_scalar!( + "SELECT path FROM eval_dataset + WHERE scorers::text LIKE $1 AND NOT path LIKE $2 AND workspace_id = $3", + &ref_pattern, + &user_prefix, + w_id + ) + .fetch_all(db) + .await?; + let ref_resources = sqlx::query_scalar!( "SELECT DISTINCT path FROM resource WHERE value::text LIKE $1 AND NOT path LIKE $2 AND workspace_id = $3", &ref_pattern, &user_prefix, w_id @@ -317,6 +338,7 @@ async fn get_offboard_preview( resources, variables, schedules, + eval_datasets, triggers, }, executing_on_behalf: OffboardAffectedPaths { @@ -332,6 +354,7 @@ async fn get_offboard_preview( flows: ref_flows, apps: ref_apps, resources: ref_resources, + eval_datasets: ref_eval_datasets, ..Default::default() }, tokens, @@ -548,6 +571,7 @@ pub(crate) async fn offboard_global_user( flows_reassigned: 0, apps_reassigned: 0, resources_reassigned: 0, + eval_datasets_reassigned: 0, variables_reassigned: 0, schedules_reassigned: 0, triggers_reassigned: 0, @@ -574,6 +598,7 @@ pub(crate) async fn offboard_global_user( total_summary.flows_reassigned += ws_summary.flows_reassigned; total_summary.apps_reassigned += ws_summary.apps_reassigned; total_summary.resources_reassigned += ws_summary.resources_reassigned; + total_summary.eval_datasets_reassigned += ws_summary.eval_datasets_reassigned; total_summary.variables_reassigned += ws_summary.variables_reassigned; total_summary.schedules_reassigned += ws_summary.schedules_reassigned; total_summary.triggers_reassigned += ws_summary.triggers_reassigned; @@ -746,6 +771,7 @@ async fn check_path_conflicts( "flow", "app", "resource", + "eval_dataset", "variable", "schedule", "http_trigger", @@ -959,6 +985,47 @@ async fn offboard_user_from_workspace<'c>( .await? .unwrap_or(0); + // ---- eval datasets ---- + // The foreign keys cascade the rename onto cases and experiments; the paths held inside JSONB + // (an experiment's subject, a dataset's scorers) are rewritten separately since the cascade + // cannot reach them and those runnables move with the user. + let eval_datasets_reassigned = sqlx::query_scalar!( + r#"WITH updated AS ( + UPDATE eval_dataset SET path = REGEXP_REPLACE(path, 'u/' || $2 || '/(.*)', $1 || '/\1') + WHERE path LIKE ('u/' || $2 || '/%') AND workspace_id = $3 + RETURNING 1 + ) SELECT COUNT(*) FROM updated"#, + &new_prefix, + username, + w_id + ) + .fetch_one(&mut **tx) + .await? + .unwrap_or(0); + sqlx::query!( + r#"UPDATE eval_experiment SET subject = jsonb_set(subject, '{path}', to_jsonb(REGEXP_REPLACE(subject->>'path', 'u/' || $2 || '/(.*)', $1 || '/\1'))) WHERE subject->>'path' LIKE ('u/' || $2 || '/%') AND workspace_id = $3"#, + &new_prefix, + username, + w_id + ) + .execute(&mut **tx) + .await?; + sqlx::query!( + r#"UPDATE eval_dataset SET scorers = COALESCE(( + SELECT jsonb_agg( + CASE WHEN elem->>'path' LIKE ('u/' || $2 || '/%') + THEN jsonb_set(elem, '{path}', to_jsonb(REGEXP_REPLACE(elem->>'path', 'u/' || $2 || '/(.*)', $1 || '/\1'))) + ELSE elem END) + FROM jsonb_array_elements(scorers) elem), '[]'::jsonb) + WHERE workspace_id = $3 + AND EXISTS (SELECT 1 FROM jsonb_array_elements(scorers) e WHERE e->>'path' LIKE ('u/' || $2 || '/%'))"#, + &new_prefix, + username, + w_id + ) + .execute(&mut **tx) + .await?; + // ---- variables (with Vault secret handling) ---- let old_var_prefix = format!("u/{}/", username); let new_var_prefix = format!("{}/", reassign_to); @@ -1152,6 +1219,7 @@ async fn offboard_user_from_workspace<'c>( flows_reassigned, apps_reassigned, resources_reassigned, + eval_datasets_reassigned, variables_reassigned, schedules_reassigned, triggers_reassigned, diff --git a/backend/windmill-api/src/token.rs b/backend/windmill-api/src/token.rs index fc808767ec..2268a63894 100644 --- a/backend/windmill-api/src/token.rs +++ b/backend/windmill-api/src/token.rs @@ -100,6 +100,12 @@ fn build_standard_scope_domains() -> Vec { ("oauth", "OAuth", "OAuth management", false), ("ai", "AI", "AI feature management", false), ("ai_skills", "AI Skills", "AI skill management", false), + ( + "ai_evals", + "AI Evals", + "AI agent eval datasets and standalone runs", + false, + ), ( "agent_workers", "Agent Workers", diff --git a/backend/windmill-api/src/users.rs b/backend/windmill-api/src/users.rs index 37946f68a4..8804004f2b 100644 --- a/backend/windmill-api/src/users.rs +++ b/backend/windmill-api/src/users.rs @@ -373,6 +373,87 @@ async fn update_username_in_workpsace<'c>( .execute(&mut **tx) .await?; + // Eval datasets are path-addressed like every other object, so a username change moves them + // too. The foreign keys cascade the rename onto their cases and experiments; the experiment + // subject (the agent a run was of) is a `u//` path of its own inside JSONB, so it is + // rewritten separately or a user's own runs would detach from their renamed agent. + sqlx::query!( + r#"UPDATE eval_dataset SET path = REGEXP_REPLACE(path,'u/' || $2 || '/(.*)','u/' || $1 || '/\1') WHERE path LIKE ('u/' || $2 || '/%') AND workspace_id = $3"#, + new_username, + old_username, + w_id + ).execute(&mut **tx) + .await?; + + sqlx::query!( + "UPDATE eval_dataset SET extra_perms = extra_perms - ('u/' || $2) || jsonb_build_object(('u/' || $1), extra_perms->('u/' || $2)) WHERE extra_perms ? ('u/' || $2) AND workspace_id = $3", + new_username, + old_username, + w_id + ) + .execute(&mut **tx) + .await?; + + sqlx::query!( + r#"UPDATE eval_experiment SET subject = jsonb_set(subject, '{path}', to_jsonb(REGEXP_REPLACE(subject->>'path','u/' || $2 || '/(.*)','u/' || $1 || '/\1'))) WHERE subject->>'path' LIKE ('u/' || $2 || '/%') AND workspace_id = $3"#, + new_username, + old_username, + w_id + ).execute(&mut **tx) + .await?; + + sqlx::query!( + "UPDATE eval_dataset SET created_by = $1 WHERE created_by = $2 AND workspace_id = $3", + new_username, + old_username, + w_id + ) + .execute(&mut **tx) + .await?; + sqlx::query!( + "UPDATE eval_dataset SET edited_by = $1 WHERE edited_by = $2 AND workspace_id = $3", + new_username, + old_username, + w_id + ) + .execute(&mut **tx) + .await?; + sqlx::query!( + "UPDATE eval_case SET created_by = $1 WHERE created_by = $2 AND workspace_id = $3", + new_username, + old_username, + w_id + ) + .execute(&mut **tx) + .await?; + sqlx::query!( + "UPDATE eval_experiment SET created_by = $1 WHERE created_by = $2 AND workspace_id = $3", + new_username, + old_username, + w_id + ) + .execute(&mut **tx) + .await?; + + // A dataset's scorers name scripts and agents by path in a JSONB array, which the rewrites + // above do not reach; those runnables are renamed elsewhere in this transaction, so each + // scorer path under the old username is rewritten too or the dataset points at a runnable that + // no longer exists. + sqlx::query!( + r#"UPDATE eval_dataset SET scorers = COALESCE(( + SELECT jsonb_agg( + CASE WHEN elem->>'path' LIKE ('u/' || $2 || '/%') + THEN jsonb_set(elem, '{path}', to_jsonb(REGEXP_REPLACE(elem->>'path','u/' || $2 || '/(.*)','u/' || $1 || '/\1'))) + ELSE elem END) + FROM jsonb_array_elements(scorers) elem), '[]'::jsonb) + WHERE workspace_id = $3 + AND EXISTS (SELECT 1 FROM jsonb_array_elements(scorers) e WHERE e->>'path' LIKE ('u/' || $2 || '/%'))"#, + new_username, + old_username, + w_id + ).execute(&mut **tx) + .await?; + // ---- variables ---- // Handle Vault secret renames before updating paths in DB diff --git a/backend/windmill-api/src/workspaces.rs b/backend/windmill-api/src/workspaces.rs index bd5ed62a04..ec240d2f1a 100644 --- a/backend/windmill-api/src/workspaces.rs +++ b/backend/windmill-api/src/workspaces.rs @@ -107,6 +107,8 @@ async fn edit_copilot_config( } } + ai_config.validate_model_pricing()?; + let mut tx = db.begin().await?; sqlx::query!( diff --git a/backend/windmill-store/src/resources.rs b/backend/windmill-store/src/resources.rs index 1a051db8fe..b107d34be4 100644 --- a/backend/windmill-store/src/resources.rs +++ b/backend/windmill-store/src/resources.rs @@ -81,11 +81,8 @@ pub fn workspaced_service() -> Router { "/history/p/{*path}", get(get_resource_history).delete(clear_resource_history), ) - .route("/history/v/{version}", get(get_resource_version)) - .route( - "/history/restore/v/{version}", - post(restore_resource_version), - ) + .route("/history/v/{id}", get(get_resource_version)) + .route("/history/restore/v/{id}", post(restore_resource_version)) .route("/delete/{*path}", delete(delete_resource)) .route("/delete_bulk", delete(delete_resources_bulk)) .route("/create", post(create_resource)) @@ -2200,7 +2197,9 @@ async fn set_resource_value( #[derive(Serialize)] struct ResourceVersion { + /// Addresses a version; `version` is the per-resource number it is presented by. id: i64, + version: i64, created_at: chrono::DateTime, created_by: Option, } @@ -2208,6 +2207,7 @@ struct ResourceVersion { #[derive(Serialize)] struct ResourceVersionWithValue { id: i64, + version: i64, created_at: chrono::DateTime, created_by: Option, value: Option, @@ -2243,7 +2243,7 @@ async fn get_resource_history( let versions = sqlx::query_as!( ResourceVersion, - "SELECT id, created_at, created_by FROM resource_version + "SELECT id, version, created_at, created_by FROM resource_version WHERE workspace_id = $1 AND path = $2 ORDER BY id DESC LIMIT $3", w_id, path, @@ -2357,19 +2357,19 @@ async fn missing_references( async fn get_resource_version( authed: ApiAuthed, Extension(user_db): Extension, - Path((w_id, version)): Path<(String, i64)>, + Path((w_id, id)): Path<(String, i64)>, ) -> JsonResult { let mut tx = user_db.begin(&authed).await?; let row = sqlx::query!( - "SELECT id, path, created_at, created_by, value FROM resource_version + "SELECT id, version, path, created_at, created_by, value FROM resource_version WHERE workspace_id = $1 AND id = $2", w_id, - version + id ) .fetch_optional(&mut *tx) .await?; - let row = not_found_if_none(row, "ResourceVersion", version.to_string())?; + let row = not_found_if_none(row, "ResourceVersion", id.to_string())?; check_scopes(&authed, || format!("resources:read:{}", row.path))?; let missing = missing_references(&mut tx, &w_id, row.value.as_ref()).await?; @@ -2377,6 +2377,7 @@ async fn get_resource_version( Ok(Json(ResourceVersionWithValue { id: row.id, + version: row.version, created_at: row.created_at, created_by: row.created_by, value: row.value, @@ -2455,17 +2456,17 @@ async fn restore_resource_version( Extension(db): Extension, Extension(user_db): Extension, Extension(webhook): Extension, - Path((w_id, version)): Path<(String, i64)>, + Path((w_id, id)): Path<(String, i64)>, ) -> Result { let mut tx = user_db.clone().begin(&authed).await?; let row = sqlx::query!( - "SELECT path, value FROM resource_version WHERE workspace_id = $1 AND id = $2", + "SELECT path, value, version FROM resource_version WHERE workspace_id = $1 AND id = $2", w_id, - version + id ) .fetch_optional(&mut *tx) .await?; - let row = not_found_if_none(row, "ResourceVersion", version.to_string())?; + let row = not_found_if_none(row, "ResourceVersion", id.to_string())?; tx.commit().await?; check_scopes(&authed, || format!("resources:write:{}", row.path))?; @@ -2486,7 +2487,7 @@ async fn restore_resource_version( Ok(format!( "resource {} restored to version {}", - row.path, version + row.path, row.version )) } diff --git a/backend/windmill-worker/src/ai_executor.rs b/backend/windmill-worker/src/ai_executor.rs index b11ecebc6c..d948fee088 100644 --- a/backend/windmill-worker/src/ai_executor.rs +++ b/backend/windmill-worker/src/ai_executor.rs @@ -451,9 +451,9 @@ pub async fn handle_ai_agent_job( (args, tools) } else { let args = serde_json::from_str::(&serde_json::to_string(&local_args)?)?; - // "Edit" on a linked step clears `agent` but keeps the host's `tool_inputs` until Save or - // Cancel folds them back, so overlay them here too: a flow persisted mid-edit must still - // bind its tools to this flow's context rather than the agent author's. + // "Edit" on a linked step clears `agent` but keeps the host's `tool_inputs`, so overlay them + // here too: a flow persisted mid-edit must still bind its tools to this flow's context + // rather than the agent author's. let mut tools = module_tools; overlay_tool_inputs(&mut tools, &tool_inputs); (args, tools) diff --git a/docs/ai-agent-evals.md b/docs/ai-agent-evals.md new file mode 100644 index 0000000000..53d6ccf272 --- /dev/null +++ b/docs/ai-agent-evals.md @@ -0,0 +1,482 @@ +# AI agent evals + +A reusable AI agent (`docs/reusable-ai-agents.md`) can be run on its own, against a curated set +of **cases** — the inputs it is expected to keep handling. + +Three words, and no fourth: + +- a **case** is one input the agent should handle, held in a **dataset**; +- a **run** (stored as an **experiment**) is one execution of a whole dataset: a single flow job + that answers every case, which is what the UI labels "Run N"; +- each case is answered as one **iteration** of that run. + +The surface is a dialog, two screens deep: **this agent's runs** across every dataset it has been +measured on (one row per run, a badge per scorer), and, on opening a run, **one table** — a row per +case, a column per scorer, the cell being that scorer's verdict — with a case's detail beside it. +Editing a dataset is a drawer over both. It opens from where an agent already is: the agent card at +the top of an AI agent step's inputs in the flow editor, and the `ai_agent` row on `/resources`. + +**Evals belong to a saved agent.** A dataset and its runs hang off an `ai_agent` resource, so they +outlive the step being renamed, copied or deleted, and two runs are comparable because they name +the same thing. A step whose agent is written inline has nothing to hang them on: what stands in +its place is **Save as reusable agent**, which is the only setup evals ask for. + +## What runs + +**A run is one flow**: a loop over the dataset's cases, each iteration answering its case and then +scoring the answer. Pushed as a `RawFlow`, so the agent step is the same vehicle +`ModuleTest.svelte` uses to test an agent step and a case exercises the production branch of +`ai_executor.rs` rather than a parallel one. + +One flow rather than one job per case, because **a run outlives the tab that started it**: a +dataset of two hundred cases against a slow provider takes long enough that nobody watches it, and +only a worker can notice that the last case finished. Scoring is therefore a step, not something +the client does afterwards, and the run is one thing to watch, cancel, or point a schedule at. Each +iteration is read back by node id (`get_result_and_success_by_id_from_flow`), so an answer is +fetched without walking the loop's status. + +The loop is `parallel` with a bounded `parallelism` and `skip_failures`: a dataset is a burst of +calls to one provider, and one case failing is one cell of the run rather than the end of it. + +The cases are the loop's **static iterator**, so they live in the flow's value, which is stored +once. Passing them as an argument would put a copy of the whole dataset in every iteration's +arguments. + +Whichever state of the agent is chosen — what is deployed, the edits in progress, or a past version +— its configuration is fixed once, when the run is opened, and inlined into the step every case +runs. A linked step would resolve the resource when each case reaches it, so a deploy part-way +through a run would be executed by the cases after it while every row still named the version the +run started against. One run measures one configuration; the cost is that a run does not exercise +the linked branch the production step takes. The edits and a past version could not be run any +other way: a reference resolves to what is deployed, which is exactly what neither of them is. + +A saved agent's and a past version's configuration are never taken from the request: both are read +from the workspace by the path they name, and a subject carrying one is refused. The edits in +progress are the one kind the request has to carry — they exist only in the editor — and the run +records what it was handed, inlined into its flow and hashed, so it is reproducible and +attributable to "this version plus these edits"; what the server cannot assert about them is that +they derive from that version. + +Each iteration is an agent step, then a **payload step**, then one step per scorer. The payload +step exists because the flow cannot see what it needs to: the agent's own result carries the +answer and every message, but each tool call's arguments, result, status, duration and schema +belong to the job that ran it. The step reads them back through `GET /ai_evals/run_payload`, whose +one argument is the iteration's own job id, and hands the scorers exactly what a scorer receives +anywhere else. A scorer therefore measures the agent's latency and not its own: the payload +reports the *agent step's* duration, never the iteration's. + +The edits are the transforms as authored, expressions included. One that reads `results..x` +or a `flow_input` the case does not supply resolves to nothing here, the same way it would in any +run of that step outside its flow. + +A linked agent is not fully self-contained: a host flow can override its tools' inputs through +the step's `tool_inputs`. A run does not reproduce that wiring — the agent runs with its own +authored defaults — so an agent whose behaviour depends on one flow's overrides is measured here +without them. + +A case carries **no conversation**: one question and the answer it should produce, so a run starts +from the agent's own memory configuration and nothing is replayed into it. + +## Where results live + +Results are jobs. A run's logs, trajectory, tool-call child jobs, permissions and retention are +already `v2_job` / `v2_job_completed` and the flow status's `agent_actions`, and none of that is +stored a second time. + +What the table itself is made of is the exception: each cell's answer, its outcome and every +scorer's verdict are copied into the run's own rows the first time they can be read. Jobs have +their own retention, and a recorded run is meant to still read as the run it was long after the +jobs that produced it are gone. + +The pane shows the **answer** and nothing else of a job; the trajectory is the run page's, and +`job_id` is the way there. For a recorded row the answer is read off the row rather than out of +the job, because `job_id` is the whole iteration — the agent and then the scorers that measured it +— and its result is the last scorer's verdict, not the answer. + +What makes a job findable again is stamped on it at push: + +- `runnable_path` is the agent's own path, so the existing `script_path_start` job filter + answers "every run of this agent" with no new state. +- `_eval` in the flow's args records `{subject: {kind, path, version}, dataset, experiment_id}`, + and every iteration inherits it, so a job opened cold from the runs page explains itself. Which + case an iteration ran is in its own `iter.value`, which is also how a cell finds its job again. + Extra flow inputs are inert — the agent step reads only `user_message`/`user_attachments`. + +## Versioning + +`subject.version` is the agent's version number: how many times it has been saved. It is counted +per resource rather than read off `resource_version.id`, which is one identity sequence for the +whole table — an agent saved nine times reads v4 … v24 under it, and the gaps count writes in +workspaces the reader cannot see. The id stays how a version is addressed, by the history routes +and by restore; the number is what a version is called, and what runs are named and compared by. + +The number is stored on the row rather than counted when read, because both ways of deleting +versions take the oldest: the monitor's trim past `MAX_RESOURCE_VERSIONS`, and clearing a history +down to its current value. Counting the survivors would renumber under either, so a run recorded +against v3 would later name a different version. + +For an `agent` run the version names the configuration the run read when it opened, which is the +one every case executes. Pinning an *older* version is a subject kind of its own — an +`agent_version` run says which version to read, where an `agent` run reads whatever is deployed at +the moment it starts. + +A version captures the resource, not its transitive closure. Two byte-identical versions can +behave differently because a `$var:`/`$res:` they reference changed underneath them, so a +recorded version is necessary for attribution but not sufficient. + +## Experiments + +Running a dataset produces an experiment: every case executed against one subject, with a row per +case. The experiment records the **exact case set it ran**, by value — a dataset keeps changing, +and a result set that cannot say which inputs produced it is not reproducible. + +### What a run is called + +An experiment is `Run N`: `run_number` is allocated per `(dataset, agent path)` when the run is +opened, once, and never reused. Stored rather than counted at read time, so a run keeps the name it +was given as history is pruned around it. There is no user-given label. + +Numbering is per agent and not per subject kind, so runs of what is deployed and runs of the edits +on top of it share one sequence: "Run 7" means one thing, and which of the two ran it is what the +run says beside its number. + +### A run is permanent + +Every run is written once and then only ever read: there is no writable experiment, no partial +rerun, and no cell that can be edited after the fact. A run in which some cells came from one +version and some from another would not be worth comparing, and running the dataset is the only +way a run appears. + +- **One experiment holds one subject, and one agent keeps one history.** The experiment list is + filtered to the agent the pane was opened on, across both kinds, so a dataset shared by two + agents never shows one agent's runs when the other is opened. +- **A version is per cell** (`eval_experiment_case.subject_version`), not per experiment. The + subject is resolved once when the run is opened and every cell is stamped from it, so the column + is uniform today; it is per cell so that a run which one day executes cell by cell can say so + rather than averaging two versions silently. +- **Edits are dated by their hash**, not by a version, because editing moves nothing a version + could record. Each cell carries `subject_draft_hash`, the hash of the configuration it ran + (canonicalised: key order is not meaningful and `serde_json` preserves insertion order). +- **An agent's unsaved edits are their own subject.** Their runs are keyed under `agent_draft`, so + a number produced by edits is never quietly read as the deployed agent's. The run dialog offers + them only when it was opened from the editing card, preselected there; from anywhere else the + agent is what is deployed. + +The table asks what version the agent is on when it opens and whenever the tab regains focus — a +small `subject_state` read — rather than polling for it; the results endpoint reports the same +version but collects the run as it goes, so it is polled only while a run is in flight, one pass +at a time. An agent saved in another tab while this one stays focused is noticed on the next focus +or the next run. + +A run of an older version is history and says so (`Run 14 · v23` beside an agent on v24); nothing +flags it, since that would flag every past run the moment anything is deployed. A run whose edits +were later deployed is a run of that version, and the results endpoint recognises and restamps it +(see "What a run says it ran"). The hash each run carries is recorded, not shown. + +In the flow editor, editing a linked agent forks the configuration into the step and clears the +link; the step is the only copy of the edits until Save changes, Cancel or Discard +(`docs/reusable-ai-agents.md`). Evals open from the agent card in both of its states: from the +editing card they run the edits as the step holds them when Run is pressed; from the linked card +they run the deployed agent, the same reading a linked step makes at run time. The agent's own +resource draft — the one the resource editor writes — is never read by evals. The card also names +the version a run is recorded against (`v24`, and `v24` beside an unsaved-changes badge for edits +on top of it), read from the resource's newest history entry since the resource itself does not +carry its version. + +## Scoring + +Scoring is not a second act with a button of its own: **Run** produces an answer and then scores +it. Each iteration of the run's flow scores its own answer as a step, and the numbers are harvested +into rows when results are read. + +A run's cells are therefore measured by the scorers as they stood when it ran, and never again. +A scorer edited or added afterwards has no cell in the runs that predate it: rescoring a run in +place would make a permanent run editable. The one thing read through the present is the pass line +— `pass_if` is applied when a score is read, so moving it re-reads every run with no model call. + +The columns themselves are the dataset's current scorers, so the table stays comparable across the +runs it lists rather than growing a column per run. Removing a scorer therefore takes its column +off the runs already recorded as well: the rows it produced are not deleted, but nothing renders +them, and adding the scorer back mints a new column that fills from the next run on. The removal +asks first, and says that. + +Two things are deliberately absent. Rescoring stored answers under edited scorers would need a run +of its own that reuses a parent run's answers and is attributed to the version that produced them, +so it never reads as the agent having answered again. A result cache keyed on (agent configuration, +case, scorer definition) would assert the agent is deterministic, which it is not, so it has to be +an explicit choice with its own answer to what a run means when half of it was computed last week. + +### A score is a number, and optionally a line through it + +Every scorer returns a number **between 0 and 1** — both templates say so, and the mean and the +pass rate read it as a fraction; a scorer returning anything outside that range has its result +recorded as an error rather than counted, and a `pass_if` threshold is held to the same range. Pass +or fail is not a second kind of score: a column carries an optional `pass_if`, and a case scoring at +or above it counts as a pass. A boolean scorer is one that returns 0 or 1 with the line at 0.5. + +A column with a threshold reports a **pass rate** beside its mean and marks each cell; a column +without one is a plain number and is not dressed up as a verdict. + +The line is deliberately outside the score's **definition** hash: where it sits is an +interpretation of a score rather than part of producing it, so moving it re-reads every run already +recorded with nothing re-run. It is set when the column is added and changed later under **Scorer +settings** in the dataset drawer, which is also where the column is named. + +The name is this dataset's own name for the scorer, seeded from the summary given when it was +added. It is a copy, not a link: the script or judge agent keeps whatever it is called, so renaming +a column here does not rename anything a second dataset shows. Reading it live from the runnable +would cost a fetch per column and leave a column blank for anyone who cannot read what it points +at. + +### What a scorer receives + +An agent is judged on its behaviour, so the final answer is the smaller half of the evidence. Every +scorer — a judge prompt or a script — is handed the same `EvalRun`, built from the job the run +already stored: + +| field | from | +|---|---| +| `input`, `expected` | the case as the experiment recorded it | +| `output` | the agent step's own result | +| `tool_calls` | every message carrying an `agent_action`, in order, with the arguments, result, error and duration of the job that call ran | +| `tools` | the tools that were called, with the schema of the script version that ran | +| `metrics` | `steps`, `duration_ms`, and the provider's `usage` when it reported any | + +Tool results are truncated at 4 KiB with `truncated: true`, so a large one cannot swamp a judge's +context, and a check that reads a truncated result can say so rather than failing on the missing +tail. A tool whose schema could not be resolved carries `null`, and a scorer validating arguments +must treat that as unchecked rather than as a failure. There is no cost field: Windmill keeps no +provider price table — the script template takes a rate as an argument instead. + +### The kinds + +Two, and both are runnables: + +| kind | is | receives | +|---|---|---| +| `agent` | an `ai_agent` resource used as a judge | the run, rendered as a message | +| `script` | a workspace script | `run`, with `input`, `output` and `expected` also spelled out | + +Keeping every scorer a runnable is what makes columns comparable: each has a path, a version, and +code you can open. There is no third kind stored as configuration on the dataset — a judge's model +and grading prompt live on the agent resource, so editing a judge is editing that agent, and the +column is not something you edit at all. Editing a column is editing the runnable it points at, so +the dataset drawer opens it in place: a script in the script editor, a judge in the resource +editor. + +Adding a scorer chooses the kind before the form opens: a judge is created next to the dataset +from the model you pick and a grading prompt that starts at the default; a script is created from +the template and opened in the editor. Both are named by a summary of what they score, which +becomes the column header and, prefixed with the dataset, the path. + +A `reason` is worth returning: it is what the cell shows on hover, together with the per-assertion +`checks`, so a number that looks wrong can be read rather than re-derived from the trajectory. + +A scorer may return a bare number, a boolean, or `{score, reason, checks}`; a judge's answer arrives +under `output`, sometimes as a string holding one of those, and often as a markdown code fence +around it, which is still read. `comment` is read as `reason`, so a scorer written for another +platform keeps its rationale. Anything with no number in it is left empty rather than guessed at, +and means skip the empty ones — a missing score counted as zero would read as a regression. + +`{score: null}` is the one exception, and it means the scorer read the case and had nothing to +measure on it: a column asking whether sources were cited has no verdict on a case with nothing to +cite. The cell shows `n/a` and is left out of the column's mean and pass rate, which is not the +same as the scorer failing — that is an error, and the column reports it as one. Written out +rather than merely absent, since a scorer that returns nothing at all is a scorer that is broken. + +### What a run says it ran + +A run is `v15` when it ran the deployed agent and `v15 + edits` when it ran that version with +undeployed changes on top. + +An `agent_draft` run records the version it is an edit of, because "the draft" is not attributable +without saying which deployed state it is a draft of. It also stops being a draft by itself: the +agent is hashed as deployed, in the same shape a draft is hashed in, so a run whose configuration +was later saved is recognised as the version it became. Edit, run, deploy, and the run you made +reads as `v16` rather than staying an edit of `v15` forever. + +That recognition is **written, not derived**. When the hashes match, the run's subject is rewritten +to `agent` at that version, once, keeping the hash it is founded on. Deriving it on every read +would make the answer expire: it would only ever mean "this ran what is deployed right now", so the +next deployment would send a run that already read `v16` back to `v15 + edits`. The write goes to +the unrestricted pool alongside the scores harvested in the same read, and nothing in it comes from +the caller — the hash is the proof, and a run of a configuration that was never deployed simply +stays an edit. + +It follows that the resolution needs someone to look: a run is stamped by the first results read +after its configuration is deployed. A run whose configuration was deployed and then replaced +without anyone opening the table keeps saying `+ edits`, which is the honest answer when the only +evidence is a hash that matches nothing deployed. + +### Reusing a scorer + +The add form lists the scorers this workspace already uses, most recently edited dataset first, +read out of the datasets' own `scorers` rather than stored anywhere new. It is filtered twice, +both times by what the caller can read: the datasets are read through `user_db`, so a scorer only +appears if the dataset carrying it does; then the runnables themselves are checked the same way, so +a script or agent the caller cannot open is never suggested. + +### A scorer is a column + +A scorer is stored on the dataset as `{id, name?, pass_if?, kind, path}`, with the `id` assigned +once and never reused: on a write, an incoming id is kept only when it names a column the dataset +already holds, and anything else is minted, so a column that was removed cannot come back under +its old id and inherit the scores recorded against it. That id is what makes a column the same +column across experiments when the scorer is renamed or its definition edited, and a delta is only +ever computed between two scores carrying the same id. Two scorers pointing at the same script are +two columns. + +A score is keyed `(experiment_id, ordinal, scorer_id)`, not baked into the experiment, so a frozen +experiment can gain a score without becoming mutable in any way that matters: what is frozen is +which runs are in it. + +Each score also records the **definition** that produced it — the kind, the path, and the script +hash or resource version that actually ran, so a path alone cannot hide an edit. When two scores +of one column carry different definitions the delta is still shown, marked: hiding the number +would force model calls just to see anything, and showing it unmarked would let a change of judge +read as a change of agent. + +### The surface + +Opening evals selects the dataset this agent was last worked in, remembered per agent in +`localStorage` and only restored while it still exists and is still readable; no run is opened for +you. The picker lists this agent's own datasets first and everyone else's below, sorted rather +than filtered, since running one dataset against a second agent is a comparison the picker exists +for. + +A dataset is named the way a script is: a **summary** of what the cases are for, from which the +path follows, prefixed with the agent so it sorts with the agent's own. With no summary the +fallback is `_dataset1`, taking the next free number. One path segment rather than a folder +under the agent, because a Windmill path is `//` and the picker that edits it +cannot express a deeper one. + +The dataset is edited in a drawer over the table: the summary and the path, the **scorers**, then +the cases in a grid. **Every way of managing a scorer is in that drawer** — adding, renaming, +moving its pass line, opening the runnable behind it, removing it; the column header over a run's +table reports and does not edit, since a run is permanent. Creating a dataset is the same drawer +with no cases yet, reached from the dataset named on a row of the runs list and from the run +dialog. Renaming moves the dataset, and its cases and its runs follow through the foreign keys. +The drawer edits a working copy and writes it in one request when **Save** is pressed — the +rename, the summary and the cases together — so a rename the server refuses leaves the cases as +they were, and a half-finished edit is never what the next run executes. A row's panel in the +results table is read-only and shows the case *as the run executed it*, not as the dataset holds +it now; deleting a case is in the drawer, and asks first. + +A case is its message, what it expects, and nothing else; the message is what identifies it. +`expected` is what a scorer compares an answer against: plain text, or JSON when the answer has +structure. + +The runs list is one row per run of this agent, newest first, whichever dataset it was of: the +run's number and what executed it (`v24`, `v24 + edits`, or a pinned `v18`), how many cases, one +badge per scorer, the dataset, and when. Each badge is the headline that column reports — a pass +rate where the column has a line, the mean where it does not — read through the thresholds as they +are **now**. A column that never scored a run reads `—`; a run still going spins. The badges are +named and resolved server-side: a list spanning datasets cannot hold every dataset's scorers to +look a column's name up, so the name and the kind ride along with the number, and the thresholds +are joined in per (run, column) — one grouped query over `eval_score` rather than a read of each +run's cells. A run whose scores are still in its flow is read out of it by the list itself, capped +per call and skipped for runs already collected, so the steady state is one query. + +**Run** asks two questions: which state of the agent (`v24 (latest deployed)` as it is saved when +you press Run; a past version as it was then; `v24 + edits (current)` running the step's edits as +they are when you press Run, offered and preselected only from the editing card), and which +dataset, with an edit button on the row and a way to start a new one without leaving. A pinned +version reproduces the configuration, not the world around it: `$var:` and `$res:` references +inside it still resolve at run time. The run that was just started opens straight away. + +The results table's rows are the dataset's cases, in dataset order, each carrying its result in +the selected experiment when it has one, so a dataset that has never been run is not an empty +table. A case the experiment ran but the dataset no longer holds keeps its row at the end: the run +happened, and deleting the case does not unmake it. Each column's mean sits under its header, with +its delta beside it when a baseline is selected. + +Picking a baseline adds a per-scorer delta to every cell and to each column's mean, and counts the +cells that regressed. Every delta names its scorer; there is no single number for a dataset, since +averaging a judge with an exact match would invent one. Rows are joined by case id, so a case added +after the baseline ran has no delta rather than counting as a change, and a column the baseline +was never scored with reports that rather than a difference that does not exist. + +## Storage + +Datasets, cases and experiments are rows: + +| table | holds | +|---|---| +| `eval_dataset` | one dataset, addressed by a workspace path, and the scorers that are its columns | +| `eval_case` | one case: its inputs and the answer it was expected to produce | +| `eval_experiment` | one run over a dataset, against one subject; written once, then only read | +| `eval_experiment_case` | the case set it executed, the job each case became, and the version or draft hash each ran against | +| `eval_score` | one scorer's verdict on one run, with the definition that produced it | + +An experiment records its cases by value instead of pointing at `eval_case`, because a dataset +keeps changing and a result set that cannot say which inputs produced it is not reproducible. For +the same reason `case_id` is a plain column rather than a foreign key: deleting a case must not +rewrite the history of the runs that used it. + +Deleting a dataset takes its cases, its experiments, their recorded case sets and every score with +it through the foreign keys. The jobs those experiments produced are left alone — they are jobs, +with their own retention. + +A case is text: a message and an expected answer. Attachments are S3 references rather than inline +bytes, so nothing in a case is meant to be large, and three caps keep it that way — 256 KiB per +case, 16 MiB and 1 000 cases per dataset — all refused at the API rather than truncated. A run +scores every case by every scorer, so a dataset also holds at most 20 scorers, refused the same +way. + +### Permissions + +A dataset is permissioned like any other path-addressed object: row-level security on +`eval_dataset` decides who may see it (readers of its folder, `u/`, a group, or an +`extra_perms` grant) and who may change it. Operators cannot write at all. Recording an experiment +counts as a write, since it persists into the dataset. + +Cases are the contents of a dataset rather than objects in their own right. `eval_case` carries a +read policy derived from its dataset (`see_parent_dataset`) and write policies that check the +dataset is *writable* — `eval_dataset_writable`, one function holding the same disjunction the +dataset's own write policies use, so a read-only grant can list a dataset's cases but not edit +them. A dataset and its cases therefore move in one `user_db` transaction, governed by the same +policies, and a rename is checked against the destination path the same way. The experiment tables +are the exception: their rows are written both by a launch (which holds dataset write) and by the +harvest (which holds only *read* of the run it copies onto its rows), so they carry read policies +only and are written on the unrestricted pool after the API has checked the right access. + +### Why an experiment is recorded before it is launched + +Launching picks the run job's id up front, writes the experiment, its case set and a pending score +per cell in one transaction, and only then queues the flow. Queueing first and recording afterwards +leaves a window in which a flow is running that no experiment accounts for, that nothing will +collect and that a retry would silently duplicate. In this order, a launch that dies before the +push leaves an experiment naming a job that never started — a run that did not run — and a push +that fails deletes it, because one failed push is the whole run. + +The dataset's foreign key guards a delete that races the assembly: the transaction fails, and at +that point nothing has been queued. It does not cover a delete that lands after this transaction +commits and before the flow is queued, which cascades the experiment away while the run still +starts. + +### How a cell finds its job, and its score + +The flow engine mints the iteration job ids, so a case is recorded before it has one. Three things +fill the gap, each copied out of the flow the first time it can be read: + +- **Which iteration ran which case.** The case is what the loop iterates over, so it is in the + iteration's own arguments by construction: `args -> 'iter' -> 'value' ->> 'case_id'` matches the + cell, whatever order the iterations finish in. +- **What the agent answered.** The agent step's result and outcome, copied onto the cell as soon + as that step is done — which is well before the iteration around it, since the scorers are still + reading it. +- **What the scorers returned.** Each scorer step's result is read out of the iteration's flow + status into the pending row that was written for it at launch. + +All three are written once, when they first become readable, and every later read is of the rows. +A job that was retained away before anything read it leaves the cell saying so, rather than +looking like a case still being answered. + +The flow itself cannot write them: it runs on workers that know nothing about these tables. So two +things call the collector. A run's flow ends with a step that calls `POST +/ai_evals/experiments/collect` on itself, which is what records a run nobody watched finish. +Reading a run collects it too, which covers the run whose flow never reached that step: one +cancelled part-way, or started while nothing served the `nativets` tag. + +That step is bookkeeping, so it is `continue_on_error`: a run whose every case answered and scored +does not become a failed job because the call did not land. diff --git a/docs/reusable-ai-agents.md b/docs/reusable-ai-agents.md index 8fa9facd74..8383bc2836 100644 --- a/docs/reusable-ai-agents.md +++ b/docs/reusable-ai-agents.md @@ -28,6 +28,11 @@ In the flow editor, the AI agent step's **Step Input** tab shows a single read-o (*linked to *, with the inherited brain + tools and an explanatory tooltip) plus *Edit* (fork into the editable step, Save changes upserts back and re-links) and *Unlink* (fork the resolved config — including any `tool_inputs` — back into the step as a one-off). +While editing, the step is the only copy of the edits: Cancel drops them and re-links (asking +first when there is something to drop), and the unsaved-changes badge opens a diff against the +deployed agent whose Discard changes is Cancel without the question. What a fork is an edit of, +and the deployed baseline the edits are judged against, live in `agentEditStore` (in memory), so +a reload brings the step back as a standalone agent with no path to save back to. A linked agent's tools appear as display-only graph tool nodes (clicking one selects the agent step); below the step's inputs, each tool gets a section with the standard schema-aware input editors (prop picker included) and a read-only view of its code — edits persist into @@ -62,6 +67,9 @@ as the reference, so two versions can be byte-identical while the agent behaves because the referenced variable changed underneath them. Anything comparing agent runs across versions has to account for that. +An eval run records the version its agent was at when the run was enqueued, which is what makes a +result attributable to a prompt state — see `docs/ai-agent-evals.md`. + A superseded value is retained for up to 100 versions. Values written through the UI keep their secrets in linked variables, but one pushed by `wmill` or written by `setResource` can hold an inline credential, and overwriting it no longer removes it from the database — anyone who can diff --git a/frontend/src/lib/aiStore.ts b/frontend/src/lib/aiStore.ts index 0f2e031a96..2d6a98c625 100644 --- a/frontend/src/lib/aiStore.ts +++ b/frontend/src/lib/aiStore.ts @@ -3,7 +3,13 @@ // import aiStore back, and such a cycle crashes the app once the bundler splits it across // chunks (docs/frontend-import-cycles.md; the build fails on the chunk cycle, not on this). import { writable, get } from 'svelte/store' -import { type AIProviderModel, type AIProvider, type AIConfig, type FreeTierInfo } from './gen' +import { + type AIProviderModel, + type AIProvider, + type AIConfig, + type FreeTierInfo, + type ModelPriceOverride +} from './gen' import { aiUserDisabled, COPILOT_SESSION_MODEL_SETTING_NAME, @@ -41,6 +47,8 @@ export const copilotInfo = writable<{ aiModels: AIProviderModel[] customPrompts?: Record maxTokensPerModel?: Record + /** Negotiated rates per `provider:model`, overriding the built-in price table. */ + modelPricing?: Record webSearchEnabledProviders?: Partial> // Set only when the workspace has no AI provider of its own and is running on // Windmill's free tier. `exhausted` means the grant is spent: there is no model, but @@ -54,6 +62,7 @@ export const copilotInfo = writable<{ aiModels: [], customPrompts: {}, maxTokensPerModel: {}, + modelPricing: {}, webSearchEnabledProviders: {} }) @@ -129,6 +138,7 @@ export function setCopilotInfo(aiConfig: AIConfig) { customPrompts: aiConfig.custom_prompts ?? {}, maxTokensPerModel: aiConfig.max_tokens_per_model ?? {}, webSearchEnabledProviders, + modelPricing: aiConfig.model_pricing ?? {}, freeTier: aiConfig.free_tier }) } else { @@ -143,6 +153,7 @@ export function setCopilotInfo(aiConfig: AIConfig) { customPrompts: {}, maxTokensPerModel: {}, webSearchEnabledProviders: {}, + modelPricing: {}, // An exhausted free grant lands here — no providers, but the reason AI is off // is "you used it up", not "you never set it up". freeTier: aiConfig.free_tier diff --git a/frontend/src/lib/components/Path.svelte b/frontend/src/lib/components/Path.svelte index 639081cc3d..37eef75bf5 100644 --- a/frontend/src/lib/components/Path.svelte +++ b/frontend/src/lib/components/Path.svelte @@ -89,6 +89,9 @@ * already have written there itself — a setup flow correcting its own failed attempt. * Every other existing path is still refused. */ allowedExistingPath?: string + /** Show the "moving may break other items" warning on a rename. Off for items nothing + * can reference by path and whose dependents move with them (eval datasets). */ + warnOnRename?: boolean } let { @@ -107,7 +110,8 @@ size = 'md', drawerOffset = 0, workspaceOverride = undefined, - allowedExistingPath = undefined + allowedExistingPath = undefined, + warnOnRename = true }: Props = $props() let ws = $derived(workspaceOverride ?? $workspaceStore) @@ -430,7 +434,8 @@ // rename. `checkInitialPathExistence` is what callers set when they are creating something, // which is the same question asked the other way round. let displayPathChangedWarning = $derived( - (['flow', 'script', 'resource', 'variable'] as PathKind[]).includes(kind) && + warnOnRename && + (['flow', 'script', 'resource', 'variable'] as PathKind[]).includes(kind) && !checkInitialPathExistence && initialPath && initialPath !== path diff --git a/frontend/src/lib/components/ResourceVersionHistory.svelte b/frontend/src/lib/components/ResourceVersionHistory.svelte index 90e41b74f0..9f73e7dda1 100644 --- a/frontend/src/lib/components/ResourceVersionHistory.svelte +++ b/frontend/src/lib/components/ResourceVersionHistory.svelte @@ -34,7 +34,11 @@ // moves and only reinstated once its own fetch lands, so the pane can never show one version's // JSON under another version's highlight. let selectedId = $state(undefined) - let loaded = $state<{ id: number; value: string; missing: string[] } | undefined>(undefined) + // `id` addresses the version, `version` is what it is called: the id is unique across every + // resource, so it is no indication of how many times this one has been saved. + let loaded = $state< + { id: number; version: number; value: string; missing: string[] } | undefined + >(undefined) // Undefined until the newest version's value arrives, which is what "Diff with current" needs. // Fetched without blocking the list, so an absent baseline disables the diff rather than // holding up the drawer everyone else opened to read. @@ -105,12 +109,21 @@ async function fetchVersion(id: number) { const version = await ResourceService.getResourceVersion({ workspace: effectiveWorkspace, - version: id + id }) - return { id, value: pretty(version.value), missing: version.missing_references ?? [] } + return { + id, + version: version.version, + value: pretty(version.value), + missing: version.missing_references ?? [] + } } - async function selectVersion(id: number | undefined, generation = loadGeneration) { + async function selectVersion( + id: number | undefined, + number: number | undefined, + generation = loadGeneration + ) { selectedId = id // Dropped up front rather than left in place while the new value is in flight: keeping it // would highlight the clicked row while the pane still rendered the previous version, and @@ -129,7 +142,7 @@ } catch (err) { if (selectedId === id && generation === loadGeneration) { selectedId = undefined - sendUserToast(`Could not load version ${id}`, true) + sendUserToast(`Could not load version ${number}`, true) } } } @@ -138,15 +151,15 @@ // loaded.id, never selectedId: restoring what the pane is showing. A selection whose value // has not arrived leaves `loaded` undefined, so this writes nothing rather than restoring a // version the user has not seen. - const id = loaded?.id - if (id === undefined) return + const target = loaded + if (target === undefined) return restoring = true try { await ResourceService.restoreResourceVersion({ workspace: effectiveWorkspace, - version: id + id: target.id }) - sendUserToast(`Restored ${path} to version ${id}`) + sendUserToast(`Restored ${path} to version ${target.version}`) onRestore?.() await loadVersions() } finally { @@ -222,14 +235,14 @@ {#each versions as version, index (version.id)} selectVersion(version.id)} + onclick={() => selectVersion(version.id, version.version)} >
{#if index === 0} {/if} - {index === 0 ? 'Current' : `Version ${version.id}`} + {index === 0 ? 'Current' : `Version ${version.version}`}
{displayDate(version.created_at)}{version.created_by diff --git a/frontend/src/lib/components/UserSettings.svelte b/frontend/src/lib/components/UserSettings.svelte index d3956f7f12..d5811e737b 100644 --- a/frontend/src/lib/components/UserSettings.svelte +++ b/frontend/src/lib/components/UserSettings.svelte @@ -9,6 +9,8 @@ import { createEventDispatcher } from 'svelte' import UserInfoSettings from './settings/UserInfoSettings.svelte' import AIUserSettings from './settings/AIUserSettings.svelte' + import AiUsagePanel from './workspaceSettings/AiUsagePanel.svelte' + import { copilotInfo, copilotWorkspace } from '$lib/aiStore' import { getDarkModeVariant, setDarkModeVariant, @@ -105,6 +107,17 @@
+ + {#if $copilotWorkspace} + + {/if} {/if}
diff --git a/frontend/src/lib/components/aiEvals/AddScorer.svelte b/frontend/src/lib/components/aiEvals/AddScorer.svelte new file mode 100644 index 0000000000..590a5ed573 --- /dev/null +++ b/frontend/src/lib/components/aiEvals/AddScorer.svelte @@ -0,0 +1,410 @@ + + +
+ {#if mode === 'new'} + {#if kind === 'agent'} + + An agent handed one whole run to grade. It is an ordinary AI agent resource: this creates it + with the prompt below, and editing the column later means editing that agent. + + {:else} + + A script handed the same run, returning a number, a boolean or {'{ score, reason, checks }'}. + The template scores the answer against the case's expected one, reports how the agent got + there as checks beside it, and leaves a case with no expected answer unmeasured. Helpers + below it cover exact and structural matches, which tools were called, arguments against each + tool's schema, repeated calls, step errors, latency and cost. + + {/if} + + + + + + + + {#if kind === 'agent'} + + + + + + {/if} + {:else} + {#if recent.length > 0} + + {#snippet children({ item })} + + + {/snippet} + + {/if} + + {#if usingRecent} +
+ {#each recent as scorer (scorer.path)} + {@const measures = datasetSummary(datasets, scorer.dataset)} + + {/each} +
+ {:else if kind === 'agent'} + + {:else} + + {/if} + {/if} +
diff --git a/frontend/src/lib/components/aiEvals/AgentEvalModal.svelte b/frontend/src/lib/components/aiEvals/AgentEvalModal.svelte new file mode 100644 index 0000000000..ff014f403a --- /dev/null +++ b/frontend/src/lib/components/aiEvals/AgentEvalModal.svelte @@ -0,0 +1,68 @@ + + + + + {#snippet titleBadge()} + Beta + {/snippet} +
+ {#if agentPath} + + {#key `${opWorkspace ?? ''}:${agentPath}`} + + {/key} + {:else} +
+ Evals run against a saved agent + + This agent is written into the flow step rather than saved as its own agent, so there is + nothing for a dataset and its runs to belong to. Save it as a reusable agent from the + step, and its evals start there. + +
+ {/if} +
+ diff --git a/frontend/src/lib/components/aiEvals/EvalCasesGrid.svelte b/frontend/src/lib/components/aiEvals/EvalCasesGrid.svelte new file mode 100644 index 0000000000..f7fec7a89f --- /dev/null +++ b/frontend/src/lib/components/aiEvals/EvalCasesGrid.svelte @@ -0,0 +1,132 @@ + + + + +
diff --git a/frontend/src/lib/components/aiEvals/EvalDatasetDrawer.svelte b/frontend/src/lib/components/aiEvals/EvalDatasetDrawer.svelte new file mode 100644 index 0000000000..3da84cf1ad --- /dev/null +++ b/frontend/src/lib/components/aiEvals/EvalDatasetDrawer.svelte @@ -0,0 +1,419 @@ + + + onClosed?.()}> + + (removingCase = undefined)} + on:confirmed={() => { + const target = removingCase + removingCase = undefined + if (target?.id) deleteCase(target.id) + }} + > + + {caseLabel(removingCase ?? { input: {} })} goes from the dataset. The runs that executed it keep + their results: a run that happened is not undone by curating the case away. + + + (removingDataset = false)} + on:confirmed={() => { + removingDataset = false + deleteDataset() + }} + > + + {datasetPath} goes with its cases and every run recorded against it. The jobs those runs produced + are kept. + + + drawer?.closeDrawer()} + > +
+ + {mode === 'edit' + ? 'The cases this agent is measured on. Editing them leaves the runs that already executed them as they were.' + : 'A set of cases to measure this agent on, and the scorers that read them.'} + + {#key formGeneration} + +
+ + +
+ {/key} + + +
+ (scorersWriting = w)} + /> +
+
+
+ Cases + {workingCases.length} +
+ +
+
+ (casesEditing = v)} + /> +
+
+
+ {#snippet actions()} + {#if mode === 'edit'} + + + {:else} + + {/if} + {/snippet} + + diff --git a/frontend/src/lib/components/aiEvals/EvalRunDialog.svelte b/frontend/src/lib/components/aiEvals/EvalRunDialog.svelte new file mode 100644 index 0000000000..d4d1714fd0 --- /dev/null +++ b/frontend/src/lib/components/aiEvals/EvalRunDialog.svelte @@ -0,0 +1,279 @@ + + + +
+ + + + + + +
+ {#snippet actions()} + + {/snippet} + diff --git a/frontend/src/lib/components/aiEvals/EvalRunsList.svelte b/frontend/src/lib/components/aiEvals/EvalRunsList.svelte new file mode 100644 index 0000000000..5d84ef88ce --- /dev/null +++ b/frontend/src/lib/components/aiEvals/EvalRunsList.svelte @@ -0,0 +1,170 @@ + + + + + + + + + + + + + Run + Dataset + Cases + Scores + When + + + + {#each experiments as experiment (experiment.id)} + onOpen(experiment)}> + +
+
+ {experimentName(experiment)} + + {subjectLabel(experiment, deployedHash, currentVersion)} + +
+ {experiment.created_by} +
+ + + {@const summary = datasetSummary(datasets, experiment.dataset)} + + + + {experiment.case_count} + + +
+ {#each experiment.scores ?? [] as score (score.scorer_id)} + {@const value = headline(score)} + + + {#if score.kind === 'agent'} + + {:else} + + {/if} + {score.name} + {#if value != undefined} + {value} + {:else if score.failed > 0} + failed + {:else if experiment.running} + + {:else} + — + {/if} + + + {/each} + {#if (experiment.scores ?? []).length === 0} + {#if experiment.running} + + + scoring + + {:else} + not scored + {/if} + {/if} +
+ + + + + + + + {/each} + {#if experiments.length === 0 && !loaded} + + + + + + {:else if experiments.length === 0} + + +
+ No runs yet + + A run answers every case of a dataset and scores the answers. Each one is kept, so the + next has something to be compared against. + + +
+ + + {/if} + + diff --git a/frontend/src/lib/components/aiEvals/EvalScorers.svelte b/frontend/src/lib/components/aiEvals/EvalScorers.svelte new file mode 100644 index 0000000000..68287b0fbc --- /dev/null +++ b/frontend/src/lib/components/aiEvals/EvalScorers.svelte @@ -0,0 +1,383 @@ + + +
+
+ Scorers + {scorers.length} +
+ openAdd('agent', 'new') }, + { + displayName: 'Existing AI judge', + icon: Bot, + action: () => openAdd('agent', 'existing') + }, + { displayName: 'New code scorer', icon: Code2, action: () => openAdd('script', 'new') }, + { + displayName: 'Existing code scorer', + icon: Code2, + action: () => openAdd('script', 'existing') + } + ]} + placement="bottom-end" + > + {#snippet buttonReplacement()} + + {/snippet} + +
+ +
+ {#if scorers.length === 0} +
+ A scorer reads one run and returns a number. Every run of this dataset is measured by all of + them, which is what makes two runs comparable. +
+ {:else} +
+ {#each scorers as scorer (scorer.id)} +
+ {#if scorer.kind === 'agent'} + + {:else} + + {/if} +
+ + {scorerLabel(scorer)} + + {scorer.path} +
+ {#if scorer.pass_if != undefined} + + ≥ {scorer.pass_if} + + {/if} +
+ {/each} +
+ {/if} +
+
+ + + scorerDrawer?.closeDrawer()} + > + {#if workspace && datasetPath} + {#key scorerFormGeneration} + + scriptEditorDrawer + ?.openDrawer(hash, onChanged) + .catch((e) => sendUserToast(`Failed to open the scorer: ${e}`, true))} + /> + {/key} + {/if} + {#snippet actions()} + {@const state = addScorerForm?.submitState()} + + {/snippet} + + + + + settingsDrawer?.closeDrawer()}> + {#if settingsScorer} +
+ + + +
+ {/if} + {#snippet actions()} + + {/snippet} + + + + + + + + (removingScorer = undefined)} + on:confirmed={async () => { + const target = removingScorer + removingScorer = undefined + if (!target) return + try { + await saveScorers(scorers.filter((s) => s.id !== target.id)) + } catch (e) { + sendUserToast(`Failed to remove the scorer: ${e}`, true) + } + }} +> + + The column goes from every run of this dataset, the ones already recorded included. Adding it + again starts a new column, which fills from the next run on. + + diff --git a/frontend/src/lib/components/aiEvals/EvalsPane.svelte b/frontend/src/lib/components/aiEvals/EvalsPane.svelte new file mode 100644 index 0000000000..d9e5b16fe4 --- /dev/null +++ b/frontend/src/lib/components/aiEvals/EvalsPane.svelte @@ -0,0 +1,978 @@ + + +
+
+ {#if viewingRun} + + {/if} +
+ {#if viewingRun && experiment?.run_job_id} + + Open the job + + + {/if} + {#if !viewingRun && loaded && datasets.length > 0} + + {#if experiments.length > 0} + + + {/if} + {/if} +
+ +
+ + +
+ {#if loaded && loadError} +
+ Could not load evals + + The datasets or runs could not be read. Check your access to this agent and reload. + +
+ {:else if loaded && datasets.length === 0} +
+ No dataset yet + + A dataset is the set of cases this agent is measured on. Runs are of a dataset, so + it is the first thing to make. + + +
+ {:else if !viewingRun || !loaded} + openRun(e.id)} + onEditDataset={async (path) => { + if (await useDataset(path)) datasetDrawer?.openDrawer('edit') + }} + onNew={() => (runDialogOpen = true)} + /> + {:else} + + + + + {#each scorers as scorer (scorer.id)} + + {/each} + + + + Case + Answer + {#each scorers as scorer, index (scorer.id)} + {@const mean = means.find((m) => m.scorer_id === scorer.id)} + {@const headline = columnHeadline(scorer, mean)} + + +
+ + {#if scorer.kind === 'agent'} + + {:else} + + {/if} + {scorerLabel(scorer)} + + + {#if headline} + + {headline.value} + + {#if headline.delta && headline.direction !== 0} + 0 ? 'text-green-500' : headline.direction < 0 ? 'text-red-500' : 'text-tertiary'}`} + > + {headline.delta} + + {/if} + {/if} + +
+ + {/each} + + + + {#each displayRows as row (row.case_id)} + {@const status = statusOf(row.status)} + openCase(row)} + > + + {caseLabel(row)} + + + + + {#if row.output != undefined} + {row.output} + {:else if status === STATUS.not_run} + not run + {:else} + {status.label.toLowerCase()} + {/if} + + + {#each scorers as scorer, index (scorer.id)} + {@const cell = row.scores.find((s) => s.scorer_id === scorer.id)} + + {#if cell?.pending} + + + + {:else if cell?.score != undefined} + + {#snippet text()} +
+ {#if cell.reason} + {cell.reason} + {/if} + {#each checksOf(cell) as check (check.name)} + + + {check.passed ? '✓' : '✗'} + + {check.name} + {#if check.detail} + {check.detail} + {/if} + + {/each} +
+ {/snippet} + + {#if cell.passed != undefined} + + {cell.passed ? '✓' : '✗'} + + {/if} + + {formatScore(cell.score)} + + {#if cell.baseline != undefined && cell.score !== cell.baseline} + {@const delta = cell.score - cell.baseline} + 0 ? 'text-green-500' : 'text-red-500'}`} + > + {formatDelta(delta)} + + {/if} + + + {:else if cell?.not_applicable} + + {#snippet text()} + {cell.reason} + {/snippet} + + n/a + + + {:else if cell?.error} + + {#snippet text()} + {cell.error} + {/snippet} + failed + + {:else} + — + {/if} + + {/each} + + {/each} + + + {/if} +
+ + {#if selectedRow} + {@const openRow = selectedRow} + +
+
+ + {openRow.input?.user_message ?? caseLabel(openRow)} + +
+ {#if openRow.job_id} + + Open the case job + + + {/if} +
+
+ {#if openRow.expected != undefined && openRow.expected !== ''} + + {/if} + {#if scorers.length > 0 && openRow.scores.length > 0} + + {/if} + {#if experiment && (openRow.job_id || openRow.output != undefined)} +
+
+ + Case result + +
+
+ {#if openRow.output != undefined} +
+ +
+ {:else if openRow.status === 'running'} + + + Running + + {:else} + {statusOf(openRow.status).label} + {/if} +
+
+ {/if} +
+
+ + {/if} + +
+
+ + { + if (await useDataset(path)) { + resumeRunDialog = true + datasetDrawer?.openDrawer('edit') + } + }} + onNewDataset={() => { + resumeRunDialog = true + datasetDrawer?.openDrawer('new') + }} +/> + + { + if (!resumeRunDialog) return + resumeRunDialog = false + // On the dataset the drawer was just in: the dialog opens on the pane's own, which + // creating or editing one has already moved to it. + runDialogOpen = true + }} +/> diff --git a/frontend/src/lib/components/aiEvals/evalUtils.test.ts b/frontend/src/lib/components/aiEvals/evalUtils.test.ts new file mode 100644 index 0000000000..d1f65e0d96 --- /dev/null +++ b/frontend/src/lib/components/aiEvals/evalUtils.test.ts @@ -0,0 +1,46 @@ +import { describe, expect, it } from 'vitest' +import type { EvalExperiment } from '$lib/gen' +import { parseThreshold, subjectLabel } from './evalUtils' + +describe('parseThreshold', () => { + it('keeps 0 as a threshold and reads only empty text as no threshold', () => { + expect(parseThreshold(0)).toEqual({ value: 0, error: false }) + expect(parseThreshold('0')).toEqual({ value: 0, error: false }) + expect(parseThreshold('')).toEqual({ error: false }) + expect(parseThreshold(' ')).toEqual({ error: false }) + expect(parseThreshold(null)).toEqual({ error: false }) + expect(parseThreshold(undefined)).toEqual({ error: false }) + }) + + it('refuses anything outside 0 to 1 or not a number', () => { + expect(parseThreshold('0.5')).toEqual({ value: 0.5, error: false }) + expect(parseThreshold('1')).toEqual({ value: 1, error: false }) + expect(parseThreshold('1.5')).toEqual({ error: true }) + expect(parseThreshold('-0.1')).toEqual({ error: true }) + expect(parseThreshold('abc')).toEqual({ error: true }) + }) +}) + +describe('subjectLabel', () => { + function run(subject: Record): EvalExperiment { + return { subject: { path: 'u/me/agent', ...subject } } as unknown as EvalExperiment + } + + it('names a deployed run and a pinned version by their number', () => { + expect(subjectLabel(run({ kind: 'agent', version: 4 }))).toBe('v4') + expect(subjectLabel(run({ kind: 'agent_version', version: 2 }))).toBe('v2') + }) + + it('says a draft run is edits on top of the version it was an edit of', () => { + expect(subjectLabel(run({ kind: 'agent_draft', version: 4, draft_hash: 'h1' }))).toBe( + 'v4 + edits' + ) + expect(subjectLabel(run({ kind: 'agent_draft', draft_hash: 'h1' }))).toBe('edits') + }) + + it('reads a draft whose configuration is now deployed as the current version', () => { + const draft = run({ kind: 'agent_draft', version: 4, draft_hash: 'h1' }) + expect(subjectLabel(draft, 'h1', 5)).toBe('v5') + expect(subjectLabel(draft, 'other', 5)).toBe('v4 + edits') + }) +}) diff --git a/frontend/src/lib/components/aiEvals/evalUtils.ts b/frontend/src/lib/components/aiEvals/evalUtils.ts new file mode 100644 index 0000000000..fcc5419f48 --- /dev/null +++ b/frontend/src/lib/components/aiEvals/evalUtils.ts @@ -0,0 +1,107 @@ +import type { + EvalCase, + EvalCaseInput, + EvalDataset, + EvalExperiment, + NewEvalCase, + Scorer +} from '$lib/gen' + +/** The case being edited in the drawer, before it is either run or saved to a dataset. */ +export type CaseDraft = NewEvalCase & { id?: string } + +/** A level the evals pane is on, and the way out of it. */ +export type EvalsLocation = { label: string; back: () => void } + +export type ScorerKind = Scorer['kind'] + +export function emptyCase(): CaseDraft { + return { input: { user_message: '' } } +} + +export function fromStoredCase(c: EvalCase): CaseDraft { + const { created_at: _created_at, created_by: _created_by, ...rest } = c + return rest +} + +export function caseLabel(c: { input?: EvalCaseInput }): string { + const message = c.input?.user_message?.trim() + if (message) return message.length > 60 ? message.slice(0, 60) + '…' : message + return 'Untitled case' +} + +export function experimentName(experiment: EvalExperiment): string { + return `Run ${experiment.run_number}` +} + +/** + * What ran: a deployed version, or a version with edits sitting on top of it. + * + * The list and the results endpoint restamp a draft run whose configuration was later deployed, so + * the kind is usually enough; `deployedHash` and `currentVersion` resolve the one still unstamped. + */ +export function subjectLabel( + experiment: EvalExperiment, + deployedHash?: string, + currentVersion?: number +): string { + if (experiment.subject.kind === 'agent_version') { + return experiment.subject.version ? `v${experiment.subject.version}` : 'a past version' + } + const deployed = + experiment.subject.kind === 'agent' || + (experiment.subject.draft_hash != undefined && experiment.subject.draft_hash === deployedHash) + if (deployed) { + const version = + experiment.subject.kind === 'agent' ? experiment.subject.version : currentVersion + return version ? `v${version}` : 'deployed' + } + return experiment.subject.version ? `v${experiment.subject.version} + edits` : 'edits' +} + +/** A scorer keeps its id when renamed, so its name is the column header and nothing else. */ +export function scorerLabel(scorer: Scorer): string { + return scorer.name || scorer.path.split('/').pop() || scorer.path +} + +export function kindLabel(kind: ScorerKind): string { + return kind === 'agent' ? 'Judge agent' : 'Script' +} + +export function formatScore(score: number | undefined): string { + return score == undefined ? '—' : score.toFixed(2) +} + +export function formatDelta(delta: number): string { + if (delta === 0) return '0.00' + return `${delta > 0 ? '+' : '−'}${Math.abs(delta).toFixed(2)}` +} + +/** What a dataset is for, where it says so: the path names it either way. */ +export function datasetSummary(datasets: EvalDataset[], path: unknown): string | undefined { + return datasets.find((d) => d.path === path)?.summary || undefined +} + +/** + * A pass threshold, as a field holds it. Empty is `''` or null, never a number: a number input + * coerces the text, so a valid threshold of 0 would otherwise read as empty and be dropped. The + * server refuses anything outside 0 to 1, caught here so the form blocks instead of the save. + */ +export function parseThreshold(text: string | number | null | undefined): { + value?: number + error: boolean +} { + const trimmed = typeof text === 'string' ? text.trim() : text + if (trimmed === '' || trimmed == undefined) return { error: false } + const value = Number(trimmed) + if (Number.isNaN(value) || value < 0 || value > 1) return { error: true } + return { value, error: false } +} + +export function summaryToName(summary: string): string { + return summary + .toLowerCase() + .replace(/[^a-z0-9_]/g, '_') + .replace(/_+/g, '_') + .replace(/^_|_$/g, '') +} diff --git a/frontend/src/lib/components/apps/components/display/table/multilineCellEditor.css b/frontend/src/lib/components/apps/components/display/table/multilineCellEditor.css new file mode 100644 index 0000000000..1ac9f8b31c --- /dev/null +++ b/frontend/src/lib/components/apps/components/display/table/multilineCellEditor.css @@ -0,0 +1,26 @@ +/* MultilineCellEditor: a popup positioned over the cell, so it has to paint the cell's own frame + rather than inherit it. */ +.ag-theme-alpine .wm-multiline-cell-editor, +.ag-theme-alpine-dark .wm-multiline-cell-editor { + background-color: var(--ag-background-color); +} +.ag-theme-alpine .wm-multiline-cell-editor textarea, +.ag-theme-alpine-dark .wm-multiline-cell-editor textarea { + display: block; + box-sizing: border-box; + /* Horizontal only: the vertical padding is set by the editor, which knows the height of the row + it is replacing. `line-height` here is what it computes against. */ + padding: 0 calc(var(--ag-cell-horizontal-padding) - 1px); + border: 1px solid var(--ag-input-focus-border-color); + border-radius: 3px; + outline: none; + resize: none; + /* Past this it scrolls rather than growing. */ + max-height: 40vh; + overflow-y: auto; + background-color: var(--ag-background-color); + color: var(--ag-foreground-color); + font: inherit; + line-height: 20px; + white-space: pre-wrap; +} diff --git a/frontend/src/lib/components/apps/components/display/table/multilineCellEditor.ts b/frontend/src/lib/components/apps/components/display/table/multilineCellEditor.ts new file mode 100644 index 0000000000..00975955c9 --- /dev/null +++ b/frontend/src/lib/components/apps/components/display/table/multilineCellEditor.ts @@ -0,0 +1,108 @@ +import type { ColDef, ICellEditorComp, ICellEditorParams } from 'ag-grid-community' +// Beside the editor rather than in the AgGrid theme: that file is the vendored theme, and a rule +// added to it is one the next copy of it drops. +import './multilineCellEditor.css' + +/** Kept in step with the `line-height` the stylesheet gives the textarea. */ +const LINE_HEIGHT = 20 + +/** + * A text cell editor that starts the height of the cell and grows as lines are added, for columns + * holding prose rather than a value. Enter commits, Shift+Enter adds a line, Escape cancels. + * + * Rendered as a popup positioned over the cell: an in-cell editor is clipped to the row height, so + * growing is only visible if the editor is allowed to paint outside it. + */ +export class MultilineCellEditor implements ICellEditorComp { + private eGui!: HTMLDivElement + private textarea!: HTMLTextAreaElement + private params!: ICellEditorParams + private wasEmpty = false + + init(params: ICellEditorParams) { + this.params = params + this.eGui = document.createElement('div') + this.eGui.className = 'wm-multiline-cell-editor' + + this.wasEmpty = params.value == undefined + + this.textarea = document.createElement('textarea') + this.textarea.rows = 1 + // A keystroke that opened the edit replaces the value, as it does in every other cell; F2 + // and double-click keep it to be edited. + this.textarea.value = params.eventKey?.length === 1 ? params.eventKey : (params.value ?? '') + this.textarea.style.width = `${params.column.getActualWidth() - 2}px` + // Padded so one line fills the cell it replaces and a second costs a line rather than a row. + // From the row rather than from `--ag-row-height`, which is the theme's figure and not + // necessarily this grid's. + const rowHeight = params.node.rowHeight ?? 28 + const padding = Math.max(0, (rowHeight - LINE_HEIGHT - 2) / 2) + this.textarea.style.paddingTop = `${padding}px` + this.textarea.style.paddingBottom = `${padding}px` + + this.textarea.addEventListener('input', () => this.resize()) + this.textarea.addEventListener('keydown', (e) => { + if (e.key === 'Escape') { + // Kept from whatever is around the grid: a grid in a drawer or a dialog is under a + // surface that closes on Escape, and leaving an edit is not asking to leave that. + e.preventDefault() + e.stopPropagation() + this.params.api.stopEditing(true) + return + } + if (e.key !== 'Enter' || e.isComposing) return + // Both branches keep the key from the grid, which ends the edit on Enter whether or not + // Shift is held: Shift+Enter falls through to the textarea's own newline, and plain Enter + // ends the edit here instead. + e.stopPropagation() + if (!e.shiftKey) { + e.preventDefault() + this.params.stopEditing() + } + }) + this.eGui.appendChild(this.textarea) + } + + private resize() { + this.textarea.style.height = 'auto' + this.textarea.style.height = `${this.textarea.scrollHeight}px` + } + + getGui() { + return this.eGui + } + + afterGuiAttached() { + this.resize() + this.textarea.focus() + // At the end rather than selected: a selection is a keystroke away from erasing the cell. + const end = this.textarea.value.length + this.textarea.setSelectionRange(end, end) + } + + getValue() { + // Nothing typed into a cell that held nothing is not an edit: returning '' here would write + // an empty string over a null, which the grid would see as a change and commit. + if (this.wasEmpty && this.textarea.value === '') return this.params.value + return this.textarea.value + } + + isPopup() { + return true + } + + getPopupPosition(): 'over' | 'under' { + return 'over' + } +} + +/** + * What a column of prose needs, ready to spread into a colDef. `suppressKeyboardEvent` as well as + * the editor: the grid ends an edit on Enter from a handler a popup editor's DOM does not sit + * under, so the editor cannot keep Shift+Enter for itself on its own. + */ +export const multilineCellColDef: Pick = { + cellEditor: MultilineCellEditor, + suppressKeyboardEvent: (p) => + p.editing && (p.event as KeyboardEvent).key === 'Enter' && (p.event as KeyboardEvent).shiftKey +} diff --git a/frontend/src/lib/components/common/drawer/Disposable.svelte b/frontend/src/lib/components/common/drawer/Disposable.svelte index 6a27357cf2..642d1366f6 100644 --- a/frontend/src/lib/components/common/drawer/Disposable.svelte +++ b/frontend/src/lib/components/common/drawer/Disposable.svelte @@ -94,6 +94,13 @@ return open } + /** Whether this is the overlay on top, i.e. the one a key press is for. Overlays that keep + * Escape for themselves (`preventEscape`) have to ask, or they answer keys aimed at whatever + * is stacked above them. Same condition the handler below arbitrates on. */ + export function isTopmost() { + return stack.val.length === 0 || stack.val[stack.val.length - 1] === id + } + function handleClickAway(e) { const last = stack.val[stack.val.length - 1] if (last === id) { diff --git a/frontend/src/lib/components/common/modal/Modal.svelte b/frontend/src/lib/components/common/modal/Modal.svelte index 20821a33f9..9cfa11b84e 100644 --- a/frontend/src/lib/components/common/modal/Modal.svelte +++ b/frontend/src/lib/components/common/modal/Modal.svelte @@ -1,5 +1,16 @@ + + {#if visible} diff --git a/frontend/src/lib/components/copilot/chat/anthropic.ts b/frontend/src/lib/components/copilot/chat/anthropic.ts index 52074a84c9..8d38f10916 100644 --- a/frontend/src/lib/components/copilot/chat/anthropic.ts +++ b/frontend/src/lib/components/copilot/chat/anthropic.ts @@ -195,7 +195,7 @@ export async function parseAnthropicCompletion( tools: Tool[], helpers: any, abortController?: AbortController, - options?: { workspace?: string } + options?: { workspace?: string; onTokenUsage?: (usage: ChatTokenUsage) => void } ): Promise { let toolCallsToProcess: ChatCompletionMessageFunctionToolCall[] = [] let error = null @@ -417,6 +417,7 @@ export async function parseAnthropicCompletion( const finalMessage = await completion.finalMessage() const tokenUsage = anthropicUsageToChatTokenUsage(finalMessage.usage) + options?.onTokenUsage?.(tokenUsage) // Process tool calls if any if (toolCallsToProcess.length > 0) { diff --git a/frontend/src/lib/components/copilot/chat/chatLoop.test.ts b/frontend/src/lib/components/copilot/chat/chatLoop.test.ts index 2c4eb466e9..a010f1369a 100644 --- a/frontend/src/lib/components/copilot/chat/chatLoop.test.ts +++ b/frontend/src/lib/components/copilot/chat/chatLoop.test.ts @@ -518,7 +518,7 @@ describe('runChatLoop lastIterationUsage', () => { expect(result.lastIterationUsage).toEqual({ prompt: 1200, completion: 80, total: 1280 }) // the aggregate keeps summing across iterations - expect(result.tokenUsage).toEqual({ prompt: 2200, completion: 130, total: 2330 }) + expect(result.tokenUsage).toMatchObject({ prompt: 2200, completion: 130, total: 2330 }) }) it('ignores empty usage reports and returns null when none are real', async () => { diff --git a/frontend/src/lib/components/copilot/chat/chatLoop.ts b/frontend/src/lib/components/copilot/chat/chatLoop.ts index cc963e4b25..91dd3ab29c 100644 --- a/frontend/src/lib/components/copilot/chat/chatLoop.ts +++ b/frontend/src/lib/components/copilot/chat/chatLoop.ts @@ -77,11 +77,16 @@ export interface ChatLoopConfig { helpers: any, modelProvider: ReasoningProviderModel ) => Promise + /** Fired for each completed provider response, before the loop continues. The + * loop can fail or be aborted at any iteration, so spend has to be handed over + * as it happens — a callback only at the end would discard everything the + * earlier iterations were already billed for. */ + onUsage?: (usage: ChatTokenUsage, modelProvider: ReasoningProviderModel) => void } export interface ChatLoopResult { addedMessages: ChatCompletionMessageParam[] - /** Sum of usage across all loop iterations (suitable for cost accounting). */ + /** Sum of usage across all loop iterations. */ tokenUsage: ChatTokenUsage lastIterationUsage: ChatTokenUsage | null hitMaxIterations: boolean @@ -328,6 +333,20 @@ export async function runChatLoop(config: ChatLoopConfig): Promise { + if (usage && iterationModel) { + config.onUsage?.(usage, iterationModel) + } + } const trackUsage = (usage: ChatTokenUsage | null | undefined) => { tokenUsage = addChatTokenUsage(tokenUsage, usage) @@ -351,6 +370,7 @@ export async function runChatLoop(config: ChatLoopConfig): Promise t.def) - const parseOptions = { workspace, provider: modelProvider.provider } + const parseOptions = { + workspace, + provider: modelProvider.provider, + onTokenUsage: reportUsage + } if (isOpenAI) { const reasoningSummaryCacheKey = getReasoningSummaryCacheKey(workspace, modelProvider) diff --git a/frontend/src/lib/components/copilot/chat/openai-responses.ts b/frontend/src/lib/components/copilot/chat/openai-responses.ts index 19efff2cd4..4118eb492d 100644 --- a/frontend/src/lib/components/copilot/chat/openai-responses.ts +++ b/frontend/src/lib/components/copilot/chat/openai-responses.ts @@ -392,7 +392,7 @@ export async function parseOpenAIResponsesCompletion( addedMessages: ChatCompletionMessageParam[], tools: Tool[], helpers: any, - options?: { workspace?: string } + options?: { workspace?: string; onTokenUsage?: (usage: ChatTokenUsage) => void } ): Promise { let toolCallsToProcess: ChatCompletionMessageFunctionToolCall[] = [] let error: OpenAIError | ResponseErrorEvent | null = null @@ -566,6 +566,7 @@ export async function parseOpenAIResponsesCompletion( const finalResponse = await runner.finalResponse() const tokenUsage = openAIResponsesUsageToChatTokenUsage(finalResponse.usage) + options?.onTokenUsage?.(tokenUsage) for (const item of finalResponse.output ?? []) { if (item.type === 'web_search_call' && !surfacedWebSearchCalls.has(item.id)) { diff --git a/frontend/src/lib/components/copilot/chat/tokenUsage.test.ts b/frontend/src/lib/components/copilot/chat/tokenUsage.test.ts new file mode 100644 index 0000000000..f23064f5a3 --- /dev/null +++ b/frontend/src/lib/components/copilot/chat/tokenUsage.test.ts @@ -0,0 +1,61 @@ +import { describe, expect, it } from 'vitest' +import { + anthropicUsageToChatTokenUsage, + billedTokens, + openAICompletionsUsageToChatTokenUsage +} from './tokenUsage' + +// The two providers report cache tokens under opposite conventions — Anthropic's +// input_tokens excludes them, OpenAI's includes them. Both are normalized so that +// `prompt` is the whole input, which is what makes `prompt - cached` the uncached +// share. Getting this backwards double-counts (or loses) the cached prefix, which +// is most of a long chat's input. +describe('billedTokens', () => { + it('derives uncached input under the Anthropic convention', () => { + const usage = anthropicUsageToChatTokenUsage({ + input_tokens: 1000, + output_tokens: 200, + cache_creation_input_tokens: 300, + cache_read_input_tokens: 5000 + }) + expect(usage.prompt).toBe(6300) + expect(billedTokens(usage)).toEqual({ + input: 1000, + cacheRead: 5000, + cacheWrite: 300, + output: 200 + }) + }) + + it('derives uncached input under the OpenAI convention', () => { + const usage = openAICompletionsUsageToChatTokenUsage({ + prompt_tokens: 6000, + completion_tokens: 200, + prompt_tokens_details: { cached_tokens: 5000 } + }) + expect(usage.prompt).toBe(6000) + expect(billedTokens(usage)).toEqual({ + input: 1000, + cacheRead: 5000, + cacheWrite: 0, + output: 200 + }) + }) + + // OpenRouter extends the OpenAI shape with cache-creation tokens, counted + // inside prompt_tokens like the reads beside them. Missing the field bills + // them as uncached input. + it('splits out OpenRouter cache-creation tokens', () => { + const usage = openAICompletionsUsageToChatTokenUsage({ + prompt_tokens: 6300, + completion_tokens: 200, + prompt_tokens_details: { cached_tokens: 5000, cache_write_tokens: 300 } + }) + expect(billedTokens(usage)).toEqual({ + input: 1000, + cacheRead: 5000, + cacheWrite: 300, + output: 200 + }) + }) +}) diff --git a/frontend/src/lib/components/copilot/chat/tokenUsage.ts b/frontend/src/lib/components/copilot/chat/tokenUsage.ts index 0091e11227..06ef5e71d8 100644 --- a/frontend/src/lib/components/copilot/chat/tokenUsage.ts +++ b/frontend/src/lib/components/copilot/chat/tokenUsage.ts @@ -1,7 +1,19 @@ +import type { PricedTokens } from '../modelPricing' + export interface ChatTokenUsage { prompt: number completion: number total: number + /** + * Subsets of `prompt`, split out because they are billed at different rates + * (a cached read is a fraction of an uncached one). `prompt` stays the whole + * input so the context gauge keeps measuring the whole request; uncached + * input is `prompt - cacheRead - cacheWrite`. + */ + cacheRead: number + cacheWrite: number + /** Cost in USD as billed, for the providers that report one. */ + cost?: number } /** @@ -28,7 +40,7 @@ export function normalizeContextUsage( } export function emptyChatTokenUsage(): ChatTokenUsage { - return { prompt: 0, completion: 0, total: 0 } + return { prompt: 0, completion: 0, total: 0, cacheRead: 0, cacheWrite: 0 } } export function addChatTokenUsage( @@ -39,10 +51,48 @@ export function addChatTokenUsage( return total } + const cost = + total.cost === undefined && usage.cost === undefined + ? undefined + : (total.cost ?? 0) + (usage.cost ?? 0) + return { prompt: total.prompt + usage.prompt, completion: total.completion + usage.completion, - total: total.total + usage.total + total: total.total + usage.total, + // `?? 0`: the cache split is newer than the field it lives on, so a usage + // object read back from storage may predate it. + cacheRead: (total.cacheRead ?? 0) + (usage.cacheRead ?? 0), + cacheWrite: (total.cacheWrite ?? 0) + (usage.cacheWrite ?? 0), + ...(cost === undefined ? {} : { cost }) + } +} + +/** Compact token count for readouts and tables (`1.2M`, `34k`, `567`). */ +export function formatTokenCount(tokens: number): string { + if (tokens >= 1_000_000) { + return `${(tokens / 1_000_000).toFixed(1).replace(/\.0$/, '')}M` + } + if (tokens >= 1000) { + return `${Math.round(tokens / 1000)}k` + } + return `${tokens}` +} + +/** + * Split a usage report into the four separately-billed token classes. `prompt` + * counts the whole input, so the uncached share is whatever the cached classes + * do not account for — which holds for both provider conventions below + * (Anthropic adds its cache counts into `prompt`, OpenAI's already includes them). + */ +export function billedTokens(usage: ChatTokenUsage): PricedTokens { + const cacheRead = usage.cacheRead ?? 0 + const cacheWrite = usage.cacheWrite ?? 0 + return { + input: Math.max(0, usage.prompt - cacheRead - cacheWrite), + cacheRead, + cacheWrite, + output: usage.completion } } @@ -57,16 +107,17 @@ export function anthropicUsageToChatTokenUsage( | null | undefined ): ChatTokenUsage { - const prompt = - (usage?.input_tokens ?? 0) + - (usage?.cache_creation_input_tokens ?? 0) + - (usage?.cache_read_input_tokens ?? 0) + const cacheWrite = usage?.cache_creation_input_tokens ?? 0 + const cacheRead = usage?.cache_read_input_tokens ?? 0 + const prompt = (usage?.input_tokens ?? 0) + cacheWrite + cacheRead const completion = usage?.output_tokens ?? 0 return { prompt, completion, - total: prompt + completion + total: prompt + completion, + cacheRead, + cacheWrite } } @@ -89,7 +140,11 @@ export function openAIResponsesUsageToChatTokenUsage( return { prompt, completion, - total: usage?.total_tokens ?? prompt + completion + total: usage?.total_tokens ?? prompt + completion, + cacheRead: usage?.input_tokens_details?.cached_tokens ?? 0, + // Automatic caching: nothing is billed for populating it, and no usage + // field reports it either. + cacheWrite: 0 } } @@ -101,7 +156,16 @@ export function openAICompletionsUsageToChatTokenUsage( prompt_tokens?: number | null completion_tokens?: number | null total_tokens?: number | null - prompt_tokens_details?: { cached_tokens?: number | null } | null + prompt_tokens_details?: { + cached_tokens?: number | null + /** Cache creation, reported by the providers that bill for it: OpenRouter + * passes Anthropic's through, and the Bedrock proxy folds + * `cacheWriteInputTokens` in here. OpenAI, whose caching is automatic and + * unbilled, reports no such field. */ + cache_write_tokens?: number | null + } | null + /** OpenRouter reports what it actually charged when the request opts in. */ + cost?: number | null } | null | undefined @@ -112,6 +176,9 @@ export function openAICompletionsUsageToChatTokenUsage( return { prompt, completion, - total: usage?.total_tokens ?? prompt + completion + total: usage?.total_tokens ?? prompt + completion, + cacheRead: usage?.prompt_tokens_details?.cached_tokens ?? 0, + cacheWrite: usage?.prompt_tokens_details?.cache_write_tokens ?? 0, + ...(typeof usage?.cost === 'number' ? { cost: usage.cost } : {}) } } diff --git a/frontend/src/lib/components/copilot/lib.ts b/frontend/src/lib/components/copilot/lib.ts index 09f34806c2..fc8a00727f 100644 --- a/frontend/src/lib/components/copilot/lib.ts +++ b/frontend/src/lib/components/copilot/lib.ts @@ -1089,6 +1089,23 @@ export async function getFimCompletion( } } +// A streamed OpenAI-compatible response carries no usage at all unless the request +// asks for it, so a provider missing from this set reports zero tokens — no context +// gauge, no cost. `stream_options.include_usage` is part of the OpenAI streaming +// spec and these providers document supporting it; `customai` is deliberately absent +// because it points at an arbitrary endpoint that may reject the field outright. +const STREAM_USAGE_PROVIDERS = new Set([ + 'openai', + 'azure_openai', + 'azure_foundry', + 'googleai', + 'openrouter', + 'groq', + 'deepseek', + 'mistral', + 'togetherai' +]) + export async function getCompletion( messages: ChatCompletionMessageParam[], abortController: AbortController, @@ -1132,17 +1149,17 @@ export async function getCompletion( // Use Completions API for other providers const client = options?.openaiClient ?? workspaceAIClients.getOpenaiClient() const completionConfig = applyReasoningToConfig( - (provider === 'openai' || - provider === 'azure_openai' || - provider === 'azure_foundry' || - provider === 'googleai') && - config.stream + config.stream && STREAM_USAGE_PROVIDERS.has(provider) ? { ...config, stream_options: { ...(config.stream_options ?? {}), include_usage: true - } + }, + // OpenRouter's own extension, on top of stream_options: it returns the + // credits actually charged next to the token counts, which is the one + // route by which the chat sees a real cost rather than an estimate. + ...(provider === 'openrouter' ? { usage: { include: true } } : {}) } : config, provider === 'deepseek' ? 'deepseek' : provider === 'mistral' ? 'mistral' : 'completions', @@ -1178,7 +1195,11 @@ export async function parseOpenAICompletion( tools: Tool[], helpers: any, _abortController?: AbortController, // unused, for signature compatibility with parseAnthropicCompletion - options?: { workspace?: string; provider?: string } + options?: { + workspace?: string + provider?: string + onTokenUsage?: (usage: ChatTokenUsage) => void + } ): Promise<{ shouldContinue: boolean; tokenUsage: ChatTokenUsage }> { const finalToolCalls: Record = {} // The tool call currently receiving argument deltas; when the stream moves on @@ -1328,6 +1349,7 @@ export async function parseOpenAICompletion( callbacks.onMessageEnd() + options?.onTokenUsage?.(tokenUsage) // Stream over: every parsed call is queued until its turn in processToolCall. for (const toolCall of Object.values(finalToolCalls)) { if (toolCall.id) { diff --git a/frontend/src/lib/components/copilot/modelConfig.ts b/frontend/src/lib/components/copilot/modelConfig.ts index a11fd5c034..cc45a23dac 100644 --- a/frontend/src/lib/components/copilot/modelConfig.ts +++ b/frontend/src/lib/components/copilot/modelConfig.ts @@ -123,21 +123,77 @@ function normalizeVersionSeparators(model: string): string { return model.replace(/\./g, '-') } -// An entry that ends on a version digit must not run into a longer version: -// `gpt-4.1` collapses to `gpt-4-1`, which would otherwise claim the 128K -// `gpt-4-1106-preview` as a 1M model. Suffixes that continue with a separator -// (`claude-opus-4-8` in `...-4-8-v1`, `gpt-5` in `gpt-5-mini`) still match. -// Family fallbacks ending on a letter get no such guard — a version welded -// straight onto the name (`llama3.1`) is exactly what they exist to catch. -const MODEL_CONTEXT_WINDOW_MATCHERS: [matcher: RegExp, contextWindow: number][] = - MODEL_CONTEXT_WINDOWS.map(([name, contextWindow]) => { +/** Suffixes that name a route to a model rather than a different model. */ +const DECORATIVE_SUFFIXES = ['latest', 'preview', 'beta', 'stable'] + +/** + * Compile a most-specific-first `[name, value]` table into matchers against the + * bare model id. Shared with the pricing table so both resolve the same set of + * ids — a model whose window is known but whose price is not (or vice versa) + * should be a gap in one table, never a difference in matching. + * + * An entry that ends on a version digit must not run into a longer version: + * `gpt-4.1` collapses to `gpt-4-1`, which would otherwise claim + * `gpt-4-1106-preview`. Suffixes that continue with a separator + * (`claude-opus-4-8` in `...-4-8-v1`, `gpt-5` in `gpt-5-mini`) still match. + * Family fallbacks ending on a letter get no such guard — a version welded + * straight onto the name (`llama3.1`) is exactly what they exist to catch. + */ +export function buildModelMatchers( + entries: [name: string, value: T][], + { strictVariants = false }: { strictVariants?: boolean } = {} +): [RegExp, T][] { + return entries.map(([name, value]) => { const pattern = normalizeVersionSeparators(name).replace(/[.*+?^${}()|[\]\\]/g, '\\$&') - return [new RegExp(/\d$/.test(pattern) ? `${pattern}(?!\\d)` : pattern), contextWindow] + const guards = [ + // An entry ending on a version digit must not run into a longer version. + /\d$/.test(pattern) ? '(?!\\d)' : '', + // A named sub-model (`gpt-5-pro`, `gpt-5-mini`) is a different model with + // its own price, not another route to this one — so under strictVariants an + // entry does not match when a further *name* segment follows. What follows + // is only a decoration when it is a date (`-20251101`), Bedrock's `-v1`, or + // one of the alias words below, at the very end of the id + // (`claude-3-5-haiku-latest` is the same model as `claude-3-5-haiku`, and is + // a shipped default; `gpt-5-preview-pro` would be a different one again). + // Off by default: for a context window an inherited value is a safe + // approximation, for a price it is a wrong number. + // A further revision segment (`gpt-5` vs `gpt-5-4-mini`) is a different model + // too, and the entry-ends-on-a-digit guard above does not catch it once the + // separator is normalized. Only a short segment: a date is digits as well + // (`-20251101`) and stays a decoration. + strictVariants ? '(?!-\\d{1,3}(?:$|-))' : '', + strictVariants + ? `(?!-(?!(?:v\\d|${DECORATIVE_SUFFIXES.join('|')})$)[a-z])` + : '' + ].join('') + return [new RegExp(pattern + guards), value] }) +} + +/** + * The `provider:model` key the workspace AI settings use for their per-model maps + * (`max_tokens_per_model`, `model_pricing`). A bare model id is not enough: the + * same id can be served by more than one provider at different rates. + * + * Matched exactly, unlike the fuzzy tables above. Those tables generalize across + * every route to one model on purpose; a per-model *setting* must not, or an + * admin could not give two variants of a family different values — and the key is + * built from the exact id the provider config lists, which is the same string the + * chat sends. + */ +export function modelKey(provider: AIProvider | string, model: string): string { + return `${provider}:${model}` +} + +export function matchModel(matchers: [RegExp, T][], model: string): T | undefined { + const id = normalizeVersionSeparators(parseModelId(model).base) + return matchers.find(([matcher]) => matcher.test(id))?.[1] +} + +const MODEL_CONTEXT_WINDOW_MATCHERS = buildModelMatchers(MODEL_CONTEXT_WINDOWS) export function getKnownModelContextWindow(model: string): number | undefined { - const id = normalizeVersionSeparators(parseModelId(model).base) - return MODEL_CONTEXT_WINDOW_MATCHERS.find(([matcher]) => matcher.test(id))?.[1] + return matchModel(MODEL_CONTEXT_WINDOW_MATCHERS, model) } export function getModelContextWindow(model: string) { diff --git a/frontend/src/lib/components/copilot/modelPricing.test.ts b/frontend/src/lib/components/copilot/modelPricing.test.ts new file mode 100644 index 0000000000..e295682187 --- /dev/null +++ b/frontend/src/lib/components/copilot/modelPricing.test.ts @@ -0,0 +1,187 @@ +import { describe, expect, it } from 'vitest' +import { billedTokens } from './chat/tokenUsage' +import { estimateCost, priceSpend, resolveModelPrice } from './modelPricing' + +describe('resolveModelPrice', () => { + it('resolves the same model across the routes that decorate its id', () => { + const direct = resolveModelPrice('anthropic', 'claude-opus-5', undefined) + expect(direct?.price.input).toBe(5) + // A gateway prefix, a dot-versioned id, a date suffix and a variant suffix + // must all land on the same entry — a miss here silently under-reports cost. + for (const id of [ + 'anthropic/claude-opus-5', + 'anthropic/claude-opus-4.8', + 'claude-opus-4-8-20260101', + 'anthropic/claude-opus-5:thinking' + ]) { + expect(resolveModelPrice('openrouter', id, undefined)?.price.input).toBe(5) + } + }) + + it('does not let a version-digit entry claim a longer version', () => { + expect(resolveModelPrice('openai', 'gpt-4.1', undefined)?.price.input).toBe(2) + expect(resolveModelPrice('openai', 'gpt-4-1106-preview', undefined)?.price.input).not.toBe(2) + }) + + it('prices flat-rate Gemini Flash while leaving the tiered Pro alone', () => { + expect(resolveModelPrice('googleai', 'gemini-2.5-flash', undefined)?.price.input).toBe(0.3) + expect(resolveModelPrice('googleai', 'gemini-2.5-flash-lite', undefined)?.price.input).toBe(0.1) + expect(resolveModelPrice('googleai', 'gemini-3.5-flash', undefined)?.price.output).toBe(9) + // Pro charges roughly double above a 200k prompt, which a per-model rate cannot + // express, so it must stay unpriced rather than be estimated at the low tier. + expect(resolveModelPrice('googleai', 'gemini-2.5-pro', undefined)).toBeUndefined() + expect(resolveModelPrice('googleai', 'gemini-3.1-pro', undefined)).toBeUndefined() + // Promotional rates carry an end date a timeless table cannot represent. + expect(resolveModelPrice('googleai', 'gemini-3.7-flash', undefined)).toBeUndefined() + }) + + it('reports an unknown model as unpriced rather than guessing', () => { + expect(resolveModelPrice('customai', 'some-in-house-model', undefined)).toBeUndefined() + }) + + it('does not let another model inherit a price through a shared prefix', () => { + // A sub-model (`-pro`) or a newer revision (`gpt-5.6` → `gpt-5-6`) is a + // different model at a different rate; inheriting `gpt-5`'s would be off by + // an order of magnitude, and silently so. + expect(resolveModelPrice('openai', 'gpt-5', undefined)?.price.input).toBe(1.25) + expect(resolveModelPrice('openai', 'gpt-5-mini', undefined)?.price.input).toBe(0.25) + expect(resolveModelPrice('openai', 'gpt-5-pro', undefined)).toBeUndefined() + expect(resolveModelPrice('openai', 'gpt-5.6', undefined)).toBeUndefined() + expect(resolveModelPrice('googleai', 'gemini-3.1', undefined)).toBeUndefined() + // A revision carrying a variant has to be caught by the matcher, not by an + // explicit entry: `gpt-5.4-mini` cannot match the `gpt-5.4` one (the `-mini` + // makes it a sub-model), so nothing but the guard stops it reaching `gpt-5`. + expect(resolveModelPrice('openai', 'gpt-5.4-mini', undefined)).toBeUndefined() + expect(resolveModelPrice('openai', 'gpt-5.5-pro', undefined)).toBeUndefined() + }) + + it('still resolves the route decorations that name the same model', () => { + // Dates, Bedrock's -v1 and floating aliases are ways of spelling one model, + // not sub-models. `claude-3-5-haiku-latest` is a shipped picker default, so + // unpricing it would silently disable cost tracking out of the box. + expect(resolveModelPrice('anthropic', 'claude-opus-4-5-20251101', undefined)?.price.input).toBe(5) + // The revision guard must not swallow a date, which is digits too. + expect(resolveModelPrice('openai', 'gpt-5-2026-01-01', undefined)?.price.input).toBe(1.25) + expect( + resolveModelPrice('bedrock', 'anthropic.claude-sonnet-4-6-20250101-v1:0', undefined)?.price + .input + ).toBe(3) + expect(resolveModelPrice('anthropic', 'claude-3-5-haiku-latest', undefined)?.price.input).toBe( + 0.8 + ) + // …while a genuine sub-model stays unpriced, including one hiding behind a + // decoration. + expect(resolveModelPrice('openai', 'gpt-5-pro', undefined)).toBeUndefined() + expect(resolveModelPrice('openai', 'gpt-5-preview-pro', undefined)).toBeUndefined() + // A family fallback must not price a model the table deliberately left out, + // nor the floating alias pointing at it. + expect(resolveModelPrice('anthropic', 'claude-sonnet-5', undefined)).toBeUndefined() + expect( + resolveModelPrice('openrouter', '~anthropic/claude-sonnet-latest', undefined) + ).toBeUndefined() + }) + + it('prefers a workspace override, keeping the model’s own cache ratios', () => { + const resolved = resolveModelPrice('anthropic', 'claude-opus-5', { + 'anthropic:claude-opus-5': { input: 2, output: 8 } + }) + expect(resolved?.source).toBe('override') + expect(resolved?.price.input).toBe(2) + // Anthropic reads a cached prefix at a tenth and writes at 1.25x. + expect(resolved?.price.cacheRead).toBeCloseTo(0.2) + expect(resolved?.price.cacheWrite).toBeCloseTo(2.5) + }) + + it('applies the overridden model’s own cache discount, not Anthropic’s', () => { + // gpt-4o discounts a cached read by half, not by a tenth — an override that + // only states input/output must not silently inherit the Anthropic ratio. + const resolved = resolveModelPrice('openai', 'gpt-4o', { + 'openai:gpt-4o': { input: 2, output: 8 } + }) + expect(resolved?.price.cacheRead).toBeCloseTo(1) + }) + + it('bills an unpriced model’s cached tokens at its input rate', () => { + // Gemini Pro is deliberately unpriced, so there is no ratio to inherit. Falling + // back to Anthropic's tenth would invent a discount the provider may not give; + // the admin states the cache rates explicitly or pays full input. + const resolved = resolveModelPrice('googleai', 'gemini-2.5-pro', { + 'googleai:gemini-2.5-pro': { input: 2, output: 8 } + }) + expect(resolved?.price.cacheRead).toBe(2) + expect(resolved?.price.cacheWrite).toBe(2) + + const stated = resolveModelPrice('googleai', 'gemini-2.5-pro', { + 'googleai:gemini-2.5-pro': { input: 2, output: 8, cache_read: 0.5, cache_write: 1 } + }) + expect(stated?.price.cacheRead).toBe(0.5) + expect(stated?.price.cacheWrite).toBe(1) + }) + + it('ignores an override whose rates could not be a price', () => { + for (const bad of [{ input: -1, output: 8 }, { input: 1e9, output: 8 }]) { + const resolved = resolveModelPrice('anthropic', 'claude-opus-5', { + 'anthropic:claude-opus-5': bad + }) + expect(resolved?.source).toBe('builtin') + } + }) +}) + +describe('estimateCost', () => { + it('bills each token class at its own rate', () => { + const cost = estimateCost( + { input: 1_000_000, cacheRead: 1_000_000, cacheWrite: 1_000_000, output: 1_000_000 }, + { input: 5, output: 25, cacheRead: 0.5, cacheWrite: 6.25 } + ) + expect(cost).toBeCloseTo(5 + 0.5 + 6.25 + 25) + }) + + it('charges a cached prefix less than an uncached one', () => { + const usage = { + prompt: 100_000, + completion: 0, + total: 100_000, + cacheRead: 90_000, + cacheWrite: 0 + } + const uncached = { ...usage, cacheRead: 0 } + const price = { input: 5, output: 25, cacheRead: 0.5, cacheWrite: 6.25 } + expect(estimateCost(billedTokens(usage), price)).toBeLessThan( + estimateCost(billedTokens(uncached), price) + ) + }) +}) + +describe('priceSpend', () => { + it('prefers a provider-reported cost over the estimate', () => { + const priced = priceSpend( + [ + { + provider: 'openrouter', + model: 'anthropic/claude-opus-5', + tokens: { input: 1_000_000, cacheRead: 0, cacheWrite: 0, output: 0 }, + reportedCostUsd: 0.42 + } + ], + undefined + ) + expect(priced.total).toBe(0.42) + expect(priced.hasReported).toBe(true) + }) + + it('flags an unpriced model instead of counting it as free', () => { + const priced = priceSpend( + [ + { + provider: 'customai', + model: 'some-in-house-model', + tokens: { input: 1_000_000, cacheRead: 0, cacheWrite: 0, output: 0 } + } + ], + undefined + ) + expect(priced.hasUnpriced).toBe(true) + expect(priced.rows[0].cost).toBeUndefined() + }) +}) diff --git a/frontend/src/lib/components/copilot/modelPricing.ts b/frontend/src/lib/components/copilot/modelPricing.ts new file mode 100644 index 0000000000..1e5b63247b --- /dev/null +++ b/frontend/src/lib/components/copilot/modelPricing.ts @@ -0,0 +1,288 @@ +import type { AIProvider, ModelPriceOverride } from '$lib/gen' +import { buildModelMatchers, matchModel, modelKey } from './modelConfig' + +/** Rates in USD per million tokens, one per billed token class. */ +export type ModelPrice = { + input: number + output: number + cacheRead: number + cacheWrite: number +} + +export type ModelPriceSource = 'override' | 'builtin' + +export type ResolvedModelPrice = { + price: ModelPrice + source: ModelPriceSource +} + +/** What a chat spent on one model, in tokens. */ +export type PricedTokens = { + input: number + cacheRead: number + cacheWrite: number + output: number +} + +// Fallbacks for entries that do not price their cache separately: Anthropic reads a +// cached prefix at a tenth of the input rate and writes one at 1.25x (5-minute TTL, +// the default the chat uses). The read ratio is NOT universal — OpenAI and Google +// discount a cached read far less — so every non-Anthropic entry below states its own +// `cacheRead` rather than inheriting this. Providers whose caching is automatic never +// report a cache write, so their write rate is unused. +const CACHE_READ_RATIO = 0.1 +const CACHE_WRITE_RATIO = 1.25 + +type PriceEntry = { + input: number + output: number + cacheRead?: number + cacheWrite?: number +} + +/** + * Published list prices, most specific entry first — the first name found in the + * bare model id wins, so vendor-namespaced and date-suffixed ids + * (anthropic/claude-opus-5, gpt-5-2026-01-01) still resolve. Matching is shared + * with the context-window table via `buildModelMatchers`. + * + * This is a best-effort snapshot: vendors change rates, ship models faster than + * this table is updated, and negotiated rates differ from list. A model that is + * not listed resolves to undefined and is reported as unpriced rather than + * guessed at, and any entry can be corrected per workspace from the AI settings. + * Providers whose catalogue turns over too quickly to track (DeepSeek, Mistral, + * Groq, TogetherAI, custom deployments) are deliberately absent. + * + * `null` marks a model that is known to exist but whose rates are not. Unpriced is + * a supported state (the UI says so and points at the override); a confidently + * wrong number is not — which is also why these matchers are built with + * `strictVariants`, so an unlisted sub-model (`gpt-5-pro`) reports no rate instead + * of inheriting its family's. + * + * One known gap the per-model shape cannot express: Anthropic's 1M-context beta + * charges more above a threshold. Usage is aggregated per model before pricing, so + * those requests are estimated at the standard tier and understate. An affected + * workspace can set the higher rate as its override. + */ +const MODEL_PRICES: [name: string, price: PriceEntry | null][] = [ + // Anthropic — Opus 4.1 and older bill at the pre-4.5 Opus rate, so the family + // fallback sits below the explicit entries rather than covering them. + ['claude-fable-5', { input: 10, output: 50 }], + ['claude-mythos-5', { input: 10, output: 50 }], + ['claude-opus-5', { input: 5, output: 25 }], + ['claude-opus-4-8', { input: 5, output: 25 }], + ['claude-opus-4-7', { input: 5, output: 25 }], + ['claude-opus-4-6', { input: 5, output: 25 }], + ['claude-opus-4-5', { input: 5, output: 25 }], + ['claude-opus-4-1', { input: 15, output: 75 }], + ['claude-opus-4', { input: 15, output: 75 }], + // Sonnet 5 runs a promotional rate with a published end date, and + // `claude-sonnet-latest` floats to it. Rates carry no date and apply at read + // time, so either figure misstates one side of that boundary — unpriced until + // the rate is a single number again. + ['claude-sonnet-5', null], + ['claude-sonnet-latest', null], + ['claude-sonnet-4-6', { input: 3, output: 15 }], + ['claude-sonnet-4-5', { input: 3, output: 15 }], + ['claude-sonnet-4', { input: 3, output: 15 }], + ['claude-haiku-4-5', { input: 1, output: 5 }], + ['claude-3-5-haiku', { input: 0.8, output: 4 }], + ['claude-opus', { input: 5, output: 25 }], + ['claude-sonnet', { input: 3, output: 15 }], + ['claude-haiku', { input: 1, output: 5 }], + // OpenAI — the cached-input discount varies by family (a tenth on gpt-5, a + // quarter on 4.1 and the o-series, half on 4o), so each entry carries its own + // rate. There is no charge for writing the cache and no usage field reporting + // one, so the write rate never applies. The -mini/-nano entries must precede + // the family entry, which would otherwise claim them. + // Revisions past gpt-5 are priced separately by OpenAI and are not tracked here. + // The matcher's revision guard already keeps them off the family rate; these + // entries stay so a revision the guard admits still resolves to no rate. + ['gpt-5.6', null], + ['gpt-5.5', null], + ['gpt-5.4', null], + ['gpt-5.2', null], + ['gpt-5.1', null], + ['gpt-5-mini', { input: 0.25, output: 2, cacheRead: 0.025 }], + ['gpt-5-nano', { input: 0.05, output: 0.4, cacheRead: 0.005 }], + ['gpt-5', { input: 1.25, output: 10, cacheRead: 0.125 }], + ['gpt-4.1-mini', { input: 0.4, output: 1.6, cacheRead: 0.1 }], + ['gpt-4.1-nano', { input: 0.1, output: 0.4, cacheRead: 0.025 }], + ['gpt-4.1', { input: 2, output: 8, cacheRead: 0.5 }], + ['gpt-4o-mini', { input: 0.15, output: 0.6, cacheRead: 0.075 }], + ['gpt-4o', { input: 2.5, output: 10, cacheRead: 1.25 }], + ['o4-mini', { input: 1.1, output: 4.4, cacheRead: 0.275 }], + ['o3-mini', { input: 1.1, output: 4.4, cacheRead: 0.55 }], + ['o3', { input: 2, output: 8, cacheRead: 0.5 }], + // Google — Flash takes a flat rate and is priced; Pro is not, because both its + // input and output roughly double above a 200k-token prompt and a per-model rate + // cannot express a threshold. Explicit context caching also bills storage per hour, + // which nothing here represents, so a workspace using it sees an underestimate. + // Gemini 3.7 and 3.6 Flash run a promotional rate with an end date, and stay + // unpriced for the same reason Sonnet 5 does. + ['gemini-2.5-flash-lite', { input: 0.1, output: 0.4, cacheRead: 0.01 }], + ['gemini-2.5-flash', { input: 0.3, output: 2.5, cacheRead: 0.03 }], + ['gemini-3.5-flash-lite', { input: 0.3, output: 2.5, cacheRead: 0.03 }], + ['gemini-3.5-flash', { input: 1.5, output: 9, cacheRead: 0.15 }], + ['gemini-3.7', null], + ['gemini-3.6', null], + ['gemini-3.1', null], + ['gemini-3', null], + ['gemini-2.5', null] +] + +const MODEL_PRICE_MATCHERS = buildModelMatchers( + MODEL_PRICES.map(([name, entry]): [string, ModelPrice | null] => [ + name, + entry && { + input: entry.input, + output: entry.output, + cacheRead: entry.cacheRead ?? entry.input * CACHE_READ_RATIO, + cacheWrite: entry.cacheWrite ?? entry.input * CACHE_WRITE_RATIO + } + ]), + { strictVariants: true } +) + +export function getKnownModelPrice(model: string): ModelPrice | undefined { + return matchModel(MODEL_PRICE_MATCHERS, model) ?? undefined +} + +/** + * Rates the API bounds on the way in — but an instance-level config is stored as an + * untyped settings blob that bypasses that handler, so the reader enforces the same + * bounds rather than rendering a negative, infinite or absurd total. + */ +const MAX_MODEL_RATE = 1000 + +function isUsableRate(rate: number | undefined): boolean { + return rate === undefined || (Number.isFinite(rate) && rate >= 0 && rate <= MAX_MODEL_RATE) +} + +/** What a cache rate falls back to when an override leaves it unset: the model's + * own published multiple of the input rate where the table has one, and the input + * rate itself where it does not, so an unstated discount is never borrowed from + * another vendor. Shared with the rates editor, which shows these as placeholders. */ +export function inheritedCacheRates( + model: string, + input: number +): { cacheRead: number; cacheWrite: number } { + const builtin = getKnownModelPrice(model) + return { + cacheRead: input * (builtin ? builtin.cacheRead / builtin.input : 1), + cacheWrite: input * (builtin ? builtin.cacheWrite / builtin.input : 1) + } +} + +/** + * The rate a workspace should be billed at for one model: its override when an + * admin set one, otherwise the published list price, otherwise nothing. An override + * that omits a cache rate takes it from `inheritedCacheRates`. + */ +export function resolveModelPrice( + provider: AIProvider | string, + model: string, + overrides: Record | undefined +): ResolvedModelPrice | undefined { + const builtin = getKnownModelPrice(model) + const candidate = overrides?.[modelKey(provider, model)] + const override = + candidate && + isUsableRate(candidate.input) && + isUsableRate(candidate.output) && + isUsableRate(candidate.cache_read) && + isUsableRate(candidate.cache_write) + ? candidate + : undefined + if (override) { + const inherited = inheritedCacheRates(model, override.input) + return { + source: 'override', + price: { + input: override.input, + output: override.output, + cacheRead: override.cache_read ?? inherited.cacheRead, + cacheWrite: override.cache_write ?? inherited.cacheWrite + } + } + } + return builtin ? { source: 'builtin', price: builtin } : undefined +} + +/** Cost in USD of `tokens` at `price`. */ +export function estimateCost(tokens: PricedTokens, price: ModelPrice): number { + return ( + (tokens.input * price.input + + tokens.cacheRead * price.cacheRead + + tokens.cacheWrite * price.cacheWrite + + tokens.output * price.output) / + 1_000_000 + ) +} + +/** Tokens spent on one model, from a chat's running totals or the usage API. */ +export type ModelSpend = { + provider: string + model: string + tokens: PricedTokens + /** What the provider billed, where it reports a figure. */ + reportedCostUsd?: number +} + +export type Priced = { + /** Undefined when no rate is known for the model — reported as unpriced, never guessed. */ + cost: number | undefined + source: ModelPriceSource | 'reported' | undefined +} + +export type PricedSpend = { + /** The input entries, each with its cost — callers carry their own fields through + * rather than zipping the result back against the input by index. */ + rows: (T & Priced)[] + total: number + /** True when at least one row has no rate, so `total` understates the truth. */ + hasUnpriced: boolean + /** True when at least one row is a figure the provider billed rather than an estimate. */ + hasReported: boolean +} + +/** + * Cost a set of per-model token counts. A provider-reported figure always wins: + * it is what was actually charged, where everything else is list price times + * tokens. `source` says which, so a view never presents an estimate as a bill. + */ +export function priceSpend( + spend: T[], + overrides: Record | undefined +): PricedSpend { + let total = 0 + let hasUnpriced = false + let hasReported = false + const rows = spend.map((entry): T & Priced => { + if (entry.reportedCostUsd !== undefined) { + hasReported = true + total += entry.reportedCostUsd + return { ...entry, cost: entry.reportedCostUsd, source: 'reported' } + } + const resolved = resolveModelPrice(entry.provider, entry.model, overrides) + if (!resolved) { + hasUnpriced = true + return { ...entry, cost: undefined, source: undefined } + } + const cost = estimateCost(entry.tokens, resolved.price) + total += cost + return { ...entry, cost, source: resolved.source } + }) + return { rows, total, hasUnpriced, hasReported } +} + +/** + * Money, at the precision the amount deserves: sub-cent spend is where a chat + * spends most of its life, and rounding it to `$0.00` would read as free. + */ +export function formatUsd(amount: number): string { + if (amount === 0) return '$0' + if (amount < 0.01) return `$${amount.toFixed(4)}` + if (amount < 1) return `$${amount.toFixed(3)}` + return `$${amount.toFixed(2)}` +} diff --git a/frontend/src/lib/components/flows/agentEditStore.svelte.ts b/frontend/src/lib/components/flows/agentEditStore.svelte.ts index 93b178af73..a44cda0792 100644 --- a/frontend/src/lib/components/flows/agentEditStore.svelte.ts +++ b/frontend/src/lib/components/flows/agentEditStore.svelte.ts @@ -1,6 +1,7 @@ -// The path an AI agent step is being edited-in-place against (set by "Edit" on a linked agent). -// Lives outside AgentResourceBar so the "Editing " mode survives that component unmounting -// when another node is selected. +// The agent an AI agent step is being edited-in-place against (set by "Edit" on a linked agent), +// with what the editor needs to judge the edits. Lives outside AgentResourceBar so the "Editing +// " mode survives that component unmounting — another node selected, or the step's own tab +// switched — while the session goes on. // // Entries are looked up by the forked step's `tools` array identity, not by any location key: // Edit assigns a fresh array (unique per fork, including nested editors reusing a module id), @@ -8,21 +9,42 @@ // objects that simply never match — a stale entry can't resurface as a phantom Editing banner. import type { FlowModule } from '$lib/gen' -type Entry = { path: string; marker: object } +export type AgentEdit = { + /** The agent the fork is an edit of, which "Save changes" writes back to. */ + path: string + /** The agent as deployed, in the form the editor compares edits in. Kept here rather than in + * the editor, which can unmount mid-session and would otherwise lose the baseline that tells + * an edit from the deployed value. */ + deployedConfig?: string +} + +type Entry = AgentEdit & { marker: object } // Capped: abandoned forks (editor closed without Save/Cancel) leave dead entries behind. const MAX_ENTRIES = 20 let entries = $state([]) -export function getAgentEditingPath(marker: object | undefined): string | undefined { - return marker ? entries.find((e) => e.marker === marker)?.path : undefined +export function getAgentEdit(marker: object | undefined): AgentEdit | undefined { + const entry = marker ? entries.find((e) => e.marker === marker) : undefined + if (!entry) return undefined + // Without the marker: a caller that hands the edit back to `setAgentEditingPath` for another + // marker must not smuggle the old one along. + return { path: entry.path, deployedConfig: entry.deployedConfig } } -export function setAgentEditingPath(marker: object | undefined, path: string | undefined) { +export function getAgentEditingPath(marker: object | undefined): string | undefined { + return getAgentEdit(marker)?.path +} + +export function setAgentEditingPath( + marker: object | undefined, + path: string | undefined, + details: Omit = {} +) { if (!marker) return const rest = entries.filter((e) => e.marker !== marker) - entries = path ? [...rest.slice(-(MAX_ENTRIES - 1)), { marker, path }] : rest + entries = path ? [...rest.slice(-(MAX_ENTRIES - 1)), { marker, path, ...details }] : rest } // Carry edit state across a wholesale clone of the flow: the clone replaces every `tools` array @@ -33,10 +55,10 @@ export function reanchorAgentEditsAcross( getModules: () => FlowModule[] | undefined, refresh: () => void ) { - const anchors: { key: string; path: string; marker: object }[] = [] + const anchors: { key: string; edit: AgentEdit; marker: object }[] = [] forEachAgentFork(getModules(), (key, tools) => { - const path = getAgentEditingPath(tools) - if (path) anchors.push({ key, path, marker: tools }) + const edit = getAgentEdit(tools) + if (edit) anchors.push({ key, edit, marker: tools }) }) refresh() if (anchors.length === 0) return @@ -45,7 +67,10 @@ export function reanchorAgentEditsAcross( for (const a of anchors) { setAgentEditingPath(a.marker, undefined) const tools = byKey.get(a.key) - if (tools) setAgentEditingPath(tools, a.path) + if (tools) { + const { path, ...details } = a.edit + setAgentEditingPath(tools, path, details) + } } } diff --git a/frontend/src/lib/components/flows/agentEditStore.test.ts b/frontend/src/lib/components/flows/agentEditStore.test.ts index 555496437e..7dc5c86203 100644 --- a/frontend/src/lib/components/flows/agentEditStore.test.ts +++ b/frontend/src/lib/components/flows/agentEditStore.test.ts @@ -1,6 +1,7 @@ import { describe, expect, it } from 'vitest' import type { FlowModule } from '$lib/gen' import { + getAgentEdit, getAgentEditingPath, setAgentEditingPath, reanchorAgentEditsAcross @@ -57,6 +58,24 @@ describe('reanchorAgentEditsAcross', () => { expect(getAgentEditingPath(newNested)).toBe('f/agents/nested') }) + it('carries the deployed baseline with the path', () => { + const tools: object[] = [tool('t1')] + let modules = [agentFork('a', tools)] + setAgentEditingPath(tools, 'f/agents/one', { deployedConfig: '{"d":1}' }) + + reanchorAgentEditsAcross( + () => modules, + () => { + modules = JSON.parse(JSON.stringify(modules)) + } + ) + + expect(getAgentEdit((modules[0].value as any).tools)).toEqual({ + path: 'f/agents/one', + deployedConfig: '{"d":1}' + }) + }) + it('drops the entry when the module is gone after the refresh', () => { const tools: object[] = [tool('t1')] let modules = [agentFork('a', tools)] diff --git a/frontend/src/lib/components/flows/agentResourceUtils.test.ts b/frontend/src/lib/components/flows/agentResourceUtils.test.ts index 63f4ee331c..75d4f9e0ff 100644 --- a/frontend/src/lib/components/flows/agentResourceUtils.test.ts +++ b/frontend/src/lib/components/flows/agentResourceUtils.test.ts @@ -1,6 +1,7 @@ import { describe, expect, it } from 'vitest' import { + agentConfigAsEdited, agentConfigToInputTransforms, flowLocalInputs, inputTransformsToAgentConfig, @@ -69,6 +70,24 @@ describe('inputTransformsToAgentConfig', () => { }) }) +describe('agentConfigAsEdited', () => { + it('keeps an expression as itself, where the saved config would drop it', () => { + const transforms = { + system_prompt: { type: 'static', value: 'hi' }, + temperature: { type: 'javascript', expr: 'flow_input.t' }, + max_iterations: { type: 'static', value: undefined }, + user_message: { type: 'static', value: 'hello' } + } as any + expect(agentConfigAsEdited(transforms, [])).toEqual({ + tools: [], + system_prompt: 'hi', + temperature: { type: 'javascript', expr: 'flow_input.t' } + }) + // The saved shape is blind to the expression: that is the difference this exists for. + expect(inputTransformsToAgentConfig(transforms, [])).toEqual({ tools: [], system_prompt: 'hi' }) + }) +}) + describe('agentConfigToInputTransforms', () => { it('wraps brain values as static transforms and round-trips through the config', () => { const config = { provider: { kind: 'openai' }, system_prompt: 'hi', tools: [{ id: 't1' }] } diff --git a/frontend/src/lib/components/flows/agentResourceUtils.ts b/frontend/src/lib/components/flows/agentResourceUtils.ts index 03d637cfe6..eeba76d271 100644 --- a/frontend/src/lib/components/flows/agentResourceUtils.ts +++ b/frontend/src/lib/components/flows/agentResourceUtils.ts @@ -94,6 +94,28 @@ export function inputTransformsToAgentConfig( return config } +/** + * The configuration as the step holds it, in the shape of the saved config: a static transform as + * its value, any other transform as itself. The saved config keeps static values only, so a + * comparison in that shape is blind to an expression — which the step runs, and linking strips. + */ +export function agentConfigAsEdited( + inputTransforms: Record | undefined, + tools: AgentTool[] | undefined +): Record { + const config: Record = { tools: tools ?? [] } + for (const key of AGENT_BRAIN_KEYS) { + const t = inputTransforms?.[key] as any + if (!t) continue + if (t.type === 'static') { + if (t.value !== undefined) config[key] = t.value + } else { + config[key] = t + } + } + return config +} + /** * Reduce the AI agent schema to only the flow-local inputs. Used when a step is linked to a saved * agent: the brain fields come from the resource, so only user_message/user_attachments stay editable. diff --git a/frontend/src/lib/components/flows/content/AgentResourceBar.svelte b/frontend/src/lib/components/flows/content/AgentResourceBar.svelte index 6393a75205..687dc9d98c 100644 --- a/frontend/src/lib/components/flows/content/AgentResourceBar.svelte +++ b/frontend/src/lib/components/flows/content/AgentResourceBar.svelte @@ -1,17 +1,22 @@ -
+
{#if agent} -
-
+
+ +
(showDetail = !showDetail)} + onkeydown={(e) => { + // Keys aimed at the buttons inside the row bubble through here; leave them theirs. + if (e.target !== e.currentTarget) return + if (e.key === 'Enter' || e.key === ' ') { + e.preventDefault() + showDetail = !showDetail + } + }} + > - Linked to - +
{agent} e.stopPropagation()}>{agent} - - {#snippet text()} - Read-only: the configuration comes from this saved agent, and only the message and - inputs are set in this flow. Edit changes the agent everywhere it's used. Unlink forks - an editable copy into just this step. - {/snippet} - - -
+ {#if version != undefined} + + v{version} + + {/if} +
+
+ {#if brainParams.length > 0 || inheritedTools.length > 0} + + {#if showDetail} + + {:else} + + {/if} + + {/if} +
- {#if brainParams.length > 0 || inheritedTools.length > 0} -
+ {#if showDetail && (brainParams.length > 0 || inheritedTools.length > 0)} +
{#each brainParams as param (param.label)}
{param.label}
@@ -478,12 +609,7 @@
Tools
{#each inheritedTools as tool (tool.id)} - - {toolLabel(tool)} - + {toolLabel(tool)} {/each}
@@ -502,14 +628,56 @@ {/if} {:else if editingPath}
- - Editing - {editingPath} -
+
+ +
+
+ {editingPath} + {#if version != undefined} + + v{version} + + {/if} + {#if edited} + + unsaved changes + + {/if} +
+
+ saving updates every flow using it + {#snippet text()} + The edits live in this step until you decide: Evals runs them as they are here, Save + changes writes them to the agent, Cancel drops them and re-links the step. + {/snippet} + +
+
+
+
+ + -
-

- Editing the saved agent. Save changes updates it and re-links this step — the update - propagates to every flow that links to it. Cancel keeps your edits here as a standalone step - instead. -

{#if providerSaveError} -

+

{providerSaveError}

{/if} {:else} -
-
- -
- or - -
+ {/if}
@@ -552,7 +711,8 @@

Save this AI agent's configuration and tools as a reusable resource. Other flows can then - link to it, and updates propagate automatically. + link to it, updates propagate automatically, and it gains a dataset of eval cases of its + own.

+ + + + + + + { + confirmCancel = false + const path = editingPath + if (path) relink(path) + }} + onCanceled={() => (confirmCancel = false)} +> + + The step goes back to {editingPath} as it is deployed, and the edits are not kept anywhere. Save + changes writes them to the agent instead. + + diff --git a/frontend/src/lib/components/flows/content/ScriptEditorDrawer.svelte b/frontend/src/lib/components/flows/content/ScriptEditorDrawer.svelte index 538bcf26c7..362b7bca37 100644 --- a/frontend/src/lib/components/flows/content/ScriptEditorDrawer.svelte +++ b/frontend/src/lib/components/flows/content/ScriptEditorDrawer.svelte @@ -175,52 +175,6 @@ let settingsDrawer: Drawer | undefined = $state() - { - unsavedModalOpen = false - }} - on:confirmed={() => { - console.log('confirmed') - closeAnyway = true - unsavedModalOpen = false - scriptEditorDrawer?.closeDrawer() - }} -> -
- Are you sure you want to discard the changes you have made? - -
- + + { + unsavedModalOpen = false + }} + on:confirmed={() => { + closeAnyway = true + unsavedModalOpen = false + scriptEditorDrawer?.closeDrawer() + }} + > +
+ Are you sure you want to discard the changes you have made? + +
+ { +export async function createAiAgent( + id: string, + agentPath?: string +): Promise<[FlowModule, FlowModuleState]> { const storedConfig = loadStoredConfig() const providerValue = storedConfig ?? { kind: 'openai', resource: '', model: '' } + // A step linked to a saved agent reads its brain and tools from the resource, so it carries only + // the flow-local inputs: seeding `provider`/`output_type` would leave transforms it never reads. const aiAgentFlowModules: FlowModule = { id, value: { type: 'aiagent', + ...(agentPath ? { agent: agentPath } : {}), tools: [], input_transforms: { - provider: { type: 'static', value: providerValue }, - output_type: { type: 'static', value: 'text' }, + ...(agentPath + ? {} + : { + provider: { type: 'static', value: providerValue }, + output_type: { type: 'static', value: 'text' } + }), user_message: { type: 'static', value: undefined } } } diff --git a/frontend/src/lib/components/flows/map/FlowModuleSchemaMap.svelte b/frontend/src/lib/components/flows/map/FlowModuleSchemaMap.svelte index c98c10b1a2..ef2dd38c39 100644 --- a/frontend/src/lib/components/flows/map/FlowModuleSchemaMap.svelte +++ b/frontend/src/lib/components/flows/map/FlowModuleSchemaMap.svelte @@ -165,7 +165,8 @@ kind: InsertKind, wsScript?: { path: string; summary: string; hash: string | undefined }, wsFlow?: { path: string; summary: string }, - inlineScript?: InlineScript + inlineScript?: InlineScript, + agentPath?: string ): Promise { let module = emptyModule(flowStateStore.val, flowStore.val, kind == 'flow') let state = emptyFlowModuleState() @@ -190,7 +191,7 @@ } else if (kind == 'branchall') { ;[module, state] = await createBranchAll(module.id) } else if (kind == 'aiagent') { - ;[module, state] = await createAiAgent(module.id) + ;[module, state] = await createAiAgent(module.id, agentPath) } else if (inlineScript) { const { language, kind, subkind, summary } = inlineScript ;[module, state] = await createInlineScriptModule(language, kind, subkind, module.id, summary) @@ -751,7 +752,8 @@ detail.kind as InsertKind, detail.script, detail.flow ? { path: detail.flow.path, summary: detail.flow.summary } : undefined, - detail.inlineScript + detail.inlineScript, + detail.agentPath ) const index = detail.index ?? 0 const extraModules: FlowModule[] = [module] diff --git a/frontend/src/lib/components/flows/map/InsertModuleInner.svelte b/frontend/src/lib/components/flows/map/InsertModuleInner.svelte index 0b3e77a848..3316463bbb 100644 --- a/frontend/src/lib/components/flows/map/InsertModuleInner.svelte +++ b/frontend/src/lib/components/flows/map/InsertModuleInner.svelte @@ -10,6 +10,11 @@ import ToggleHubWorkspaceQuick from '$lib/components/ToggleHubWorkspaceQuick.svelte' import TopLevelNode from '../pickers/TopLevelNode.svelte' import RefreshButton from '$lib/components/common/button/RefreshButton.svelte' + import Button from '$lib/components/common/button/Button.svelte' + import { ResourceService } from '$lib/gen' + import { workspaceStore } from '$lib/stores' + import type { FlowEditorContext } from '../types' + import { BotIcon, Loader2, Plus } from 'lucide-svelte' const dispatch = createEventDispatcher() interface Props { @@ -42,11 +47,44 @@ | 'approval' | 'flow' | 'failure' - | 'aisandbox' = $state(untrack(() => kind)) + | 'aisandbox' + | 'aiagent' = $state(untrack(() => kind)) let preFilter: 'all' | 'workspace' | 'hub' = $state('all') let loading = $state(false) let small = $derived(smallProp ?? (kind === 'preprocessor' || kind === 'failure')) + // Optional: this picker also renders outside the flow editor's context (the triggers wrapper). + const flowEditorContext = getContext('FlowEditorContext') + let ws = $derived(flowEditorContext?.opWorkspace?.() ?? $workspaceStore) + + let savedAgents = $state<{ path: string; description?: string }[]>([]) + let savedAgentsLoading = $state(false) + let savedAgentsWs: string | undefined = undefined + async function loadSavedAgents() { + if (!ws || savedAgentsWs === ws) { + return + } + savedAgentsLoading = true + try { + const rs = await ResourceService.listResource({ + workspace: ws, + resourceType: 'ai_agent', + perPage: 1000 + }) + savedAgents = rs.map((r) => ({ path: r.path, description: r.description })) + savedAgentsWs = ws + } catch { + savedAgents = [] + } finally { + savedAgentsLoading = false + } + } + let filteredAgents = $derived( + funcDesc + ? savedAgents.filter((a) => a.path.toLowerCase().includes(funcDesc.toLowerCase())) + : savedAgents + ) + let height = $state(0) let owners = $state([]) // Only the content-sized host (TriggersWrapper) grows past this. The fixed-height hosts top out @@ -81,6 +119,10 @@ {loading} onClick={() => { refreshCount.val += 1 + if (selectedKind === 'aiagent') { + savedAgentsWs = undefined + loadSavedAgents() + } }} />
@@ -184,9 +226,10 @@ {#if customUi?.aiAgent != false} { - dispatch('close') - dispatch('new', { kind: 'aiagent' }) + selectedKind = 'aiagent' + loadSavedAgents() }} /> {/if} @@ -203,7 +246,52 @@
{/if} - {#if selectedKind === 'aisandbox'} + {#if selectedKind === 'aiagent'} +
+ + {#if savedAgentsLoading} +
+ Loading saved agents +
+ {:else if filteredAgents.length > 0} +
Saved agents
+ {#each filteredAgents as agent (agent.path)} + + {/each} + {:else} +
+ {savedAgents.length > 0 + ? 'No saved agent matches this search' + : 'No saved agent in this workspace yet. Configure a blank one, then Save as reusable agent to reuse it.'} +
+ {/if} +
+ {:else if selectedKind === 'aisandbox'}
Promise diff --git a/frontend/src/lib/components/graph/graphBuilder.svelte.ts b/frontend/src/lib/components/graph/graphBuilder.svelte.ts index 450debbef4..3491cd3126 100644 --- a/frontend/src/lib/components/graph/graphBuilder.svelte.ts +++ b/frontend/src/lib/components/graph/graphBuilder.svelte.ts @@ -58,6 +58,8 @@ export type GraphEventHandlers = { inlineScript?: InlineScript script?: PathScript flow?: { path: string; summary: string } + /** Saved `ai_agent` resource the inserted agent step links to, for `kind: 'aiagent'`. */ + agentPath?: string isPreprocessor?: boolean }) => void deleteBranch: (detail: { id: string; index: number }, label: string) => void diff --git a/frontend/src/lib/components/graph/renderers/edges/BaseEdge.svelte b/frontend/src/lib/components/graph/renderers/edges/BaseEdge.svelte index 89e7c88c94..5678621062 100644 --- a/frontend/src/lib/components/graph/renderers/edges/BaseEdge.svelte +++ b/frontend/src/lib/components/graph/renderers/edges/BaseEdge.svelte @@ -192,7 +192,8 @@ branch: data.branch, index: data.index, kind: e.detail.kind, - inlineScript: e.detail.inlineScript + inlineScript: e.detail.inlineScript, + agentPath: e.detail.agentPath }) }} on:pickScript={(e) => { diff --git a/frontend/src/lib/components/select/SelectDropdown.svelte b/frontend/src/lib/components/select/SelectDropdown.svelte index d9fc2dcf70..8dd952e5bf 100644 --- a/frontend/src/lib/components/select/SelectDropdown.svelte +++ b/frontend/src/lib/components/select/SelectDropdown.svelte @@ -145,17 +145,21 @@ }} > {@render startSnippet?.({ item, close: () => (open = false) })} - - {item.label || '\xa0'} - + +
+ + {item.label || '\xa0'} + + {#if item.subtitle} +
{item.subtitle}
+ {/if} +
{#if item.__is_create} {:else} {@render endSnippet?.({ item, close: () => (open = false) })} {/if} - {#if item.subtitle} -
{item.subtitle}
- {/if} {/each} diff --git a/frontend/src/lib/components/workspaceSettings/AISettings.svelte b/frontend/src/lib/components/workspaceSettings/AISettings.svelte index 60d1cb77a0..5979e8a535 100644 --- a/frontend/src/lib/components/workspaceSettings/AISettings.svelte +++ b/frontend/src/lib/components/workspaceSettings/AISettings.svelte @@ -5,9 +5,11 @@ type AIConfig, type AIProvider, type GetCopilotSettingsStateResponse, - type InstanceAISummary + type InstanceAISummary, + type ModelPriceOverride } from '$lib/gen' import { workspaceStore } from '$lib/stores' + import { copilotInfo } from '$lib/aiStore' import { sendUserToast } from '$lib/toast' import { AI_PROVIDERS, fetchAvailableModels, providerSupportsWebSearch } from '../copilot/lib' import { supportsAutocomplete } from '../copilot/utils' @@ -25,6 +27,8 @@ import Badge from '../common/badge/Badge.svelte' import Tooltip from '../Tooltip.svelte' import ModelTokenLimits from './ModelTokenLimits.svelte' + import ModelPricing from './ModelPricing.svelte' + import AiUsagePanel from './AiUsagePanel.svelte' import { setCopilotInfo } from '$lib/aiStore' import AIPromptsModal from '../settings/AIPromptsModal.svelte' import { Settings } from 'lucide-svelte' @@ -73,6 +77,7 @@ let metadataModel: string | undefined = $state(undefined) let customPrompts: Record = $state({}) let maxTokensPerModel: Record = $state({}) + let modelPricing: Record = $state({}) let usingOpenaiClientCredentialsOauth = $state(false) let workspaceOverrideEditorOpened = $state(false) @@ -83,6 +88,7 @@ let initialMetadataModel: string | undefined = $state(undefined) let initialCustomPrompts: Record = $state({}) let initialMaxTokensPerModel: Record = $state({}) + let initialModelPricing: Record = $state({}) let initialPrompts: Record = $state({}) let lastLoadedConfigKey = $state(undefined) @@ -110,6 +116,7 @@ codeCompletionModel = config?.code_completion_model?.model customPrompts = clone(config?.custom_prompts ?? {}) maxTokensPerModel = clone(config?.max_tokens_per_model ?? {}) + modelPricing = clone(config?.model_pricing ?? {}) for (const mode of ['edit', 'fix', 'gen']) { if (!(mode in customPrompts)) { customPrompts[mode] = '' @@ -124,6 +131,7 @@ initialCodeCompletionModel = codeCompletionModel initialCustomPrompts = clone(customPrompts) initialMaxTokensPerModel = clone(maxTokensPerModel) + initialModelPricing = clone(modelPricing) initialPrompts = clone(customPrompts) } @@ -139,6 +147,7 @@ codeCompletionModel = initialCodeCompletionModel customPrompts = clone(initialCustomPrompts) maxTokensPerModel = clone(initialMaxTokensPerModel) + modelPricing = clone(initialModelPricing) } $effect(() => { @@ -172,7 +181,8 @@ metadataModel !== initialMetadataModel || codeCompletionModel !== initialCodeCompletionModel || JSON.stringify(customPrompts) !== JSON.stringify(initialCustomPrompts) || - JSON.stringify(maxTokensPerModel) !== JSON.stringify(initialMaxTokensPerModel) + JSON.stringify(maxTokensPerModel) !== JSON.stringify(initialMaxTokensPerModel) || + JSON.stringify(modelPricing) !== JSON.stringify(initialModelPricing) ) $effect(() => { @@ -285,7 +295,8 @@ metadata_model, custom_prompts: Object.keys(custom_prompts).length > 0 ? custom_prompts : undefined, max_tokens_per_model: - Object.keys(maxTokensPerModel).length > 0 ? maxTokensPerModel : undefined + Object.keys(maxTokensPerModel).length > 0 ? maxTokensPerModel : undefined, + model_pricing: Object.keys(modelPricing).length > 0 ? modelPricing : undefined } : {} } @@ -610,6 +621,24 @@ scope={promptScope} /> +{#if promptScope === 'workspace'} + + +{/if} + + +{#if showWorkspaceOverrideEditor} + +{/if} + {#if showWorkspaceOverrideEditor} + import { AiService, ApiError, type AITokenUsageBucket, type ModelPriceOverride } from '$lib/gen' + import { formatUsd, priceSpend, type ModelSpend } from '../copilot/modelPricing' + import { formatTokenCount } from '../copilot/chat/tokenUsage' + import SettingCard from '../instanceSettings/SettingCard.svelte' + import Select from '../select/Select.svelte' + import ToggleButtonGroup from '../common/toggleButton-v2/ToggleButtonGroup.svelte' + import ToggleButton from '../common/toggleButton-v2/ToggleButton.svelte' + import { resource } from 'runed' + import Tooltip from '../meltComponents/Tooltip.svelte' + import DataTable from '../table/DataTable.svelte' + import Head from '../table/Head.svelte' + import Cell from '../table/Cell.svelte' + + // Workspace and rates are both passed in rather than read from a store: the + // settings component that mounts this one also serves the instance scope, and + // the rates that priced a chat are the workspace's *effective* ones, which an + // inheriting workspace does not hold itself. + let { + workspace, + modelPricing, + scope = 'workspace' + }: { + workspace: string + modelPricing: Record + scope?: 'workspace' | 'self' + } = $props() + + type GroupBy = 'day' | 'user' | 'model' + + let days = $state(30) + let groupBy = $state('day') + + const rangeOptions = [ + { label: 'Last 7 days', value: 7 }, + { label: 'Last 30 days', value: 30 }, + { label: 'Last 90 days', value: 90 } + ] + + let usage = resource( + () => ({ workspace, days, groupBy, scope }), + async ({ workspace, days, groupBy, scope }) => + workspace ? await AiService.listAiUsage({ workspace, days, groupBy, scope }) : undefined + ) + + // The API groups by (dimension, provider, model) so every bucket resolves to a + // single rate; the table folds those back into one line per dimension value. + type Bucket = ModelSpend & { key: string; requests: number } + + function toSpend(bucket: AITokenUsageBucket): Bucket { + return { + // Grouping by model has no separate dimension — the model is the key. + key: groupBy === 'model' ? `${bucket.provider}/${bucket.model}` : bucket.key || '—', + requests: bucket.requests, + provider: bucket.provider, + model: bucket.model, + tokens: { + input: bucket.input_tokens, + cacheRead: bucket.cache_read_tokens, + cacheWrite: bucket.cache_write_tokens, + output: bucket.output_tokens + }, + reportedCostUsd: + bucket.reported_cost_nano_usd != undefined + ? bucket.reported_cost_nano_usd / 1_000_000_000 + : undefined + } + } + + let priced = $derived(priceSpend((usage.current?.buckets ?? []).map(toSpend), modelPricing)) + + type Row = { + key: string + cost: number | undefined + /** Every model behind this line was billed back by its provider, so the + * figure is an invoice rather than an estimate. A line mixing sources — or + * one holding a model with no rate, whose spend the figure omits entirely — + * makes the weaker claim. */ + reported: boolean + tokensIn: number + tokensOut: number + requests: number + } + + // Only a 403 on the workspace scope is a permission problem; reading your own + // usage is open to any member. Attributing every failure to permissions sends an + // admin looking for access they already hold, and buries the real cause of the + // far more common transient ones (an expired session, a database hiccup). + function usageError(error: unknown): string { + if (scope === 'workspace' && error instanceof ApiError && error.status === 403) { + return 'Only workspace admins can read workspace usage.' + } + return 'Could not load usage. Try again in a moment.' + } + + // The headline sums both kinds, so it only escapes the ~ when nothing under it + // was estimated. + let totalIsEstimated = $derived( + priced.rows.some((row) => row.cost !== undefined && row.source !== 'reported') + ) + + let rows = $derived.by(() => { + const byKey = new Map() + for (const row of priced.rows) { + const existing = byKey.get(row.key) ?? { + key: row.key, + cost: undefined, + reported: true, + tokensIn: 0, + tokensOut: 0, + requests: 0 + } + existing.tokensIn += row.tokens.input + row.tokens.cacheRead + row.tokens.cacheWrite + existing.tokensOut += row.tokens.output + existing.requests += row.requests + if (row.cost !== undefined) { + existing.cost = (existing.cost ?? 0) + row.cost + } + existing.reported &&= row.source === 'reported' + byKey.set(row.key, existing) + } + return [...byKey.values()].sort((a, b) => (b.cost ?? 0) - (a.cost ?? 0)) + }) + + + +
+
+
+
+