starred/windmill
1
0
Fork 0
mirror of https://github.com/windmill-labs/windmill.git synced 2026-09-10 08:07:03 +00:00
Code Issues Packages Projects Releases Wiki Activity

Merge branch 'main' into free-token-limit

Browse Source 
Catch up to main's latest 2 commits (AI agent evals #10633, AI token cost
tracking #10688), which the PR flagged as conflicting.

Resolutions, all keeping both sides:
- ai.rs AIConfig: keep our free_tier field alongside origin/main's model_pricing
  field + ModelPriceOverride struct/impl.
- aiStore.ts: import FreeTierInfo and ModelPriceOverride; carry both freeTier and
  modelPricing through copilotInfo.
- ContextUsageIndicator: keep our shared UsageMeter, import formatTokenCount from
  origin/main's new tokenUsage module.
- openapi.yaml / summarized_schema.txt: keep both the free-tier and the
  model-pricing / ai_token_usage schema additions.

Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>
This commit is contained in:
Diego Imbert
2026-08-24 15:05:04 +02:00
co-authored by Claude Opus 4.8
parent a0aac9bff9 9c557859c5
commit 94295613aa
158 changed files with 14357 additions and 298 deletions
Download Patch File Download Diff File Expand all files Collapse all files
backend/.sqlx/query-01bce88dd622f314d1a09c24cd12df5e7d3ff6a15a93e1c0c95e262f7b3d0ef1.json
Generated
+17
View File
@@ -0,0 +1,17 @@
{
"db_name": "PostgreSQL",
"query": "UPDATE eval_experiment\n SET subject = jsonb_set(\n jsonb_set(subject, '{kind}', '\"agent\"'),\n '{version}', to_jsonb($4::bigint))\n WHERE workspace_id = $1 AND dataset_path = $2 AND id = $3\n AND subject ->> 'kind' = 'agent_draft'",
"describe": {
"columns": [],
"parameters": {
"Left": [
"Text",
"Text",
"Uuid",
"Int8"
]
},
"nullable": []
},
"hash": "01bce88dd622f314d1a09c24cd12df5e7d3ff6a15a93e1c0c95e262f7b3d0ef1"
}
backend/.sqlx/query-0276e6030abb2eb00a68c568a9cc60f3e7c2af0331388c4b358035de865a121a.json
Generated
+35
View File
@@ -0,0 +1,35 @@
{
"db_name": "PostgreSQL",
"query": "SELECT path, value, version FROM resource_version WHERE workspace_id = $1 AND id = $2",
"describe": {
"columns": [
{
"ordinal": 0,
"name": "path",
"type_info": "Varchar"
},
{
"ordinal": 1,
"name": "value",
"type_info": "Jsonb"
},
{
"ordinal": 2,
"name": "version",
"type_info": "Int8"
}
],
"parameters": {
"Left": [
"Text",
"Int8"
]
},
"nullable": [
false,
true,
false
]
},
"hash": "0276e6030abb2eb00a68c568a9cc60f3e7c2af0331388c4b358035de865a121a"
}
backend/.sqlx/query-0335de6713de6678b9bf266121af23abc46d5db95da095bb15726c5a2db7ad2f.json
Generated
+24
View File
@@ -0,0 +1,24 @@
{
"db_name": "PostgreSQL",
"query": "SELECT coalesce(max(run_number), 0) + 1 FROM eval_experiment\n WHERE workspace_id = $1 AND dataset_path = $2 AND subject ->> 'path' = $3",
"describe": {
"columns": [
{
"ordinal": 0,
"name": "?column?",
"type_info": "Int4"
}
],
"parameters": {
"Left": [
"Text",
"Text",
"Text"
]
},
"nullable": [
null
]
},
"hash": "0335de6713de6678b9bf266121af23abc46d5db95da095bb15726c5a2db7ad2f"
}
backend/.sqlx/query-0aae275d9196e742b5783df4e67c72459d45e275bfeafa2952349cae259ac9f0.json
Generated
+16
View File
@@ -0,0 +1,16 @@
{
"db_name": "PostgreSQL",
"query": "UPDATE eval_dataset SET scorers = COALESCE((\n SELECT jsonb_agg(\n CASE WHEN elem->>'path' LIKE ('u/' || $2 || '/%')\n THEN jsonb_set(elem, '{path}', to_jsonb(REGEXP_REPLACE(elem->>'path', 'u/' || $2 || '/(.*)', $1 || '/\\1')))\n ELSE elem END)\n FROM jsonb_array_elements(scorers) elem), '[]'::jsonb)\n WHERE workspace_id = $3\n AND EXISTS (SELECT 1 FROM jsonb_array_elements(scorers) e WHERE e->>'path' LIKE ('u/' || $2 || '/%'))",
"describe": {
"columns": [],
"parameters": {
"Left": [
"Text",
"Text",
"Text"
]
},
"nullable": []
},
"hash": "0aae275d9196e742b5783df4e67c72459d45e275bfeafa2952349cae259ac9f0"
}
backend/.sqlx/query-0d6700ccffb8179e365bbc1f03398e474f23b013f642ba29ad6f68e6f047c1e5.json
Generated
+24
View File
@@ -0,0 +1,24 @@
{
"db_name": "PostgreSQL",
"query": "SELECT pg_advisory_xact_lock(hashtext('ai_eval_open:' || $1 || '/' || $2 || '/' || $3))",
"describe": {
"columns": [
{
"ordinal": 0,
"name": "pg_advisory_xact_lock",
"type_info": "Void"
}
],
"parameters": {
"Left": [
"Text",
"Text",
"Text"
]
},
"nullable": [
null
]
},
"hash": "0d6700ccffb8179e365bbc1f03398e474f23b013f642ba29ad6f68e6f047c1e5"
}
backend/.sqlx/query-1815730982dcaf7239ddcb22f88ae5c79794213cf6278167f8afdbca30b1b15c.json
Generated
+16
View File
@@ -0,0 +1,16 @@
{
"db_name": "PostgreSQL",
"query": "UPDATE eval_experiment SET subject = jsonb_set(subject, '{path}', to_jsonb(REGEXP_REPLACE(subject->>'path', 'u/' || $2 || '/(.*)', $1 || '/\\1'))) WHERE subject->>'path' LIKE ('u/' || $2 || '/%') AND workspace_id = $3",
"describe": {
"columns": [],
"parameters": {
"Left": [
"Text",
"Text",
"Text"
]
},
"nullable": []
},
"hash": "1815730982dcaf7239ddcb22f88ae5c79794213cf6278167f8afdbca30b1b15c"
}
backend/.sqlx/query-186c663249ffada82abf61ce214f52e2730501774a0ca4dc855380e6c6487917.json
Generated
+22
View File
@@ -0,0 +1,22 @@
{
"db_name": "PostgreSQL",
"query": "SELECT count(*) AS \"count!\" FROM eval_experiment_case\n WHERE experiment_id = $1 AND status IS NOT NULL",
"describe": {
"columns": [
{
"ordinal": 0,
"name": "count!",
"type_info": "Int8"
}
],
"parameters": {
"Left": [
"Uuid"
]
},
"nullable": [
null
]
},
"hash": "186c663249ffada82abf61ce214f52e2730501774a0ca4dc855380e6c6487917"
}
backend/.sqlx/query-196939257a334f7d37aa6d66153b251446d7701a893cb7852b81cf842c0fa228.json
Generated
+59
View File
@@ -0,0 +1,59 @@
{
"db_name": "PostgreSQL",
"query": "SELECT path, summary, scorers, created_at, created_by,\n edited_at, edited_by\n FROM eval_dataset WHERE workspace_id = $1 AND path = $2",
"describe": {
"columns": [
{
"ordinal": 0,
"name": "path",
"type_info": "Varchar"
},
{
"ordinal": 1,
"name": "summary",
"type_info": "Varchar"
},
{
"ordinal": 2,
"name": "scorers",
"type_info": "Jsonb"
},
{
"ordinal": 3,
"name": "created_at",
"type_info": "Timestamptz"
},
{
"ordinal": 4,
"name": "created_by",
"type_info": "Varchar"
},
{
"ordinal": 5,
"name": "edited_at",
"type_info": "Timestamptz"
},
{
"ordinal": 6,
"name": "edited_by",
"type_info": "Varchar"
}
],
"parameters": {
"Left": [
"Text",
"Text"
]
},
"nullable": [
false,
true,
false,
false,
false,
false,
false
]
},
"hash": "196939257a334f7d37aa6d66153b251446d7701a893cb7852b81cf842c0fa228"
}
backend/.sqlx/query-1b6e229545f6b877e72d21728257d1bddaba15ef0fbe72bb4f43b45140f184ce.json
Generated
+28
View File
@@ -0,0 +1,28 @@
{
"db_name": "PostgreSQL",
"query": "SELECT case_id, ordinal FROM eval_experiment_case WHERE experiment_id = $1",
"describe": {
"columns": [
{
"ordinal": 0,
"name": "case_id",
"type_info": "Uuid"
},
{
"ordinal": 1,
"name": "ordinal",
"type_info": "Int4"
}
],
"parameters": {
"Left": [
"Uuid"
]
},
"nullable": [
false,
false
]
},
"hash": "1b6e229545f6b877e72d21728257d1bddaba15ef0fbe72bb4f43b45140f184ce"
}
backend/.sqlx/query-1db80f3ba2c6c769a98424ebf9aaf168a4fa2c64e446a824038cf267236fe979.json
Generated
+23
View File
@@ -0,0 +1,23 @@
{
"db_name": "PostgreSQL",
"query": "SELECT DISTINCT path FROM script\n WHERE workspace_id = $1 AND path = ANY($2)\n AND deleted = false AND lock IS NOT NULL AND lock_error_logs IS NULL",
"describe": {
"columns": [
{
"ordinal": 0,
"name": "path",
"type_info": "Varchar"
}
],
"parameters": {
"Left": [
"Text",
"TextArray"
]
},
"nullable": [
false
]
},
"hash": "1db80f3ba2c6c769a98424ebf9aaf168a4fa2c64e446a824038cf267236fe979"
}
backend/.sqlx/query-242845c86084e010ab33c2197d44af9aeb181672a2f2330dbe65bfe586376450.json
Generated
+15
View File
@@ -0,0 +1,15 @@
{
"db_name": "PostgreSQL",
"query": "UPDATE eval_score SET error = 'The case did not run'\n WHERE experiment_id = $1 AND ordinal = ANY($2)\n AND score IS NULL AND error IS NULL AND NOT not_applicable",
"describe": {
"columns": [],
"parameters": {
"Left": [
"Uuid",
"Int4Array"
]
},
"nullable": []
},
"hash": "242845c86084e010ab33c2197d44af9aeb181672a2f2330dbe65bfe586376450"
}
backend/.sqlx/query-24fcc2b69f30953915f0cbf246e1c19b2310075a644d81c21784c991e52b4001.json
Generated
+24
View File
@@ -0,0 +1,24 @@
{
"db_name": "PostgreSQL",
"query": "INSERT INTO ai_token_usage (workspace_id, email, provider, model, session_id, input_tokens, cache_read_tokens, cache_write_tokens, output_tokens, reported_cost_nano_usd, requests)\n SELECT $1, $2, * FROM UNNEST($3::text[], $4::text[], $5::text[], $6::bigint[], $7::bigint[], $8::bigint[], $9::bigint[], $10::bigint[], $11::bigint[])\n ON CONFLICT (workspace_id, day, email, provider, model, session_id)\n DO UPDATE SET\n input_tokens = ai_token_usage.input_tokens + EXCLUDED.input_tokens,\n cache_read_tokens = ai_token_usage.cache_read_tokens + EXCLUDED.cache_read_tokens,\n cache_write_tokens = ai_token_usage.cache_write_tokens + EXCLUDED.cache_write_tokens,\n output_tokens = ai_token_usage.output_tokens + EXCLUDED.output_tokens,\n reported_cost_nano_usd = CASE\n WHEN EXCLUDED.reported_cost_nano_usd IS NULL\n THEN ai_token_usage.reported_cost_nano_usd\n ELSE COALESCE(ai_token_usage.reported_cost_nano_usd, 0)\n + EXCLUDED.reported_cost_nano_usd\n END,\n requests = ai_token_usage.requests + EXCLUDED.requests,\n updated_at = now()",
"describe": {
"columns": [],
"parameters": {
"Left": [
"Varchar",
"Varchar",
"TextArray",
"TextArray",
"TextArray",
"Int8Array",
"Int8Array",
"Int8Array",
"Int8Array",
"Int8Array",
"Int8Array"
]
},
"nullable": []
},
"hash": "24fcc2b69f30953915f0cbf246e1c19b2310075a644d81c21784c991e52b4001"
}
backend/.sqlx/query-2b41dc4d872af0e230c31bef1a496604c1562cdfb0bdb54bd3b0c6579d73e46d.json
Generated
+28
View File
@@ -0,0 +1,28 @@
{
"db_name": "PostgreSQL",
"query": "SELECT path, scorers FROM eval_dataset\n WHERE workspace_id = $1 ORDER BY edited_at DESC LIMIT 100",
"describe": {
"columns": [
{
"ordinal": 0,
"name": "path",
"type_info": "Varchar"
},
{
"ordinal": 1,
"name": "scorers",
"type_info": "Jsonb"
}
],
"parameters": {
"Left": [
"Text"
]
},
"nullable": [
false,
false
]
},
"hash": "2b41dc4d872af0e230c31bef1a496604c1562cdfb0bdb54bd3b0c6579d73e46d"
}
backend/.sqlx/query-307d5b797e51122dbf087e5dc9f33235a349731828d500c74a28b577469e2624.json
Generated
+23
View File
@@ -0,0 +1,23 @@
{
"db_name": "PostgreSQL",
"query": "SELECT path FROM resource WHERE workspace_id = $1 AND path = ANY($2) AND resource_type = 'ai_agent'",
"describe": {
"columns": [
{
"ordinal": 0,
"name": "path",
"type_info": "Varchar"
}
],
"parameters": {
"Left": [
"Text",
"TextArray"
]
},
"nullable": [
false
]
},
"hash": "307d5b797e51122dbf087e5dc9f33235a349731828d500c74a28b577469e2624"
}
backend/.sqlx/query-30b8590939bf3d6770cabca9f4ab3527e750d85d3b8cc0eadef55461de4a2687.json
Generated
+23
View File
@@ -0,0 +1,23 @@
{
"db_name": "PostgreSQL",
"query": "SELECT path FROM eval_dataset WHERE path LIKE $1 AND workspace_id = $2",
"describe": {
"columns": [
{
"ordinal": 0,
"name": "path",
"type_info": "Varchar"
}
],
"parameters": {
"Left": [
"Text",
"Text"
]
},
"nullable": [
false
]
},
"hash": "30b8590939bf3d6770cabca9f4ab3527e750d85d3b8cc0eadef55461de4a2687"
}
backend/.sqlx/query-316e7c86082b6ee2864b88674cfac5b5b34c5de851f1c6f832b868101edcb052.json
Generated
+23
View File
@@ -0,0 +1,23 @@
{
"db_name": "PostgreSQL",
"query": "SELECT version FROM resource_version WHERE workspace_id = $1 AND path = $2\n ORDER BY version DESC LIMIT 1",
"describe": {
"columns": [
{
"ordinal": 0,
"name": "version",
"type_info": "Int8"
}
],
"parameters": {
"Left": [
"Text",
"Text"
]
},
"nullable": [
false
]
},
"hash": "316e7c86082b6ee2864b88674cfac5b5b34c5de851f1c6f832b868101edcb052"
}
backend/.sqlx/query-34fbb2b141ad691e0cdc55bf2ebd6974e02a7c81d91217d3f74e03113edc9b0a.json
Generated
+16
View File
@@ -0,0 +1,16 @@
{
"db_name": "PostgreSQL",
"query": "DELETE FROM eval_case\n WHERE workspace_id = $1 AND dataset_path = $2 AND NOT (id = ANY($3))",
"describe": {
"columns": [],
"parameters": {
"Left": [
"Text",
"Text",
"UuidArray"
]
},
"nullable": []
},
"hash": "34fbb2b141ad691e0cdc55bf2ebd6974e02a7c81d91217d3f74e03113edc9b0a"
}
backend/.sqlx/query-3ca2f72d2917f48644cb79daba9858f472be9f1bb8e504cc3dc2eb998e3f7b91.json
Generated
+16
View File
@@ -0,0 +1,16 @@
{
"db_name": "PostgreSQL",
"query": "UPDATE eval_experiment SET created_by = $1 WHERE created_by = $2 AND workspace_id = $3",
"describe": {
"columns": [],
"parameters": {
"Left": [
"Varchar",
"Text",
"Text"
]
},
"nullable": []
},
"hash": "3ca2f72d2917f48644cb79daba9858f472be9f1bb8e504cc3dc2eb998e3f7b91"
}
backend/.sqlx/query-44a39475ba202bd5852b666335308360569aae717eff3fe9cc9253a261ae2fa7.json
Generated
+23
View File
@@ -0,0 +1,23 @@
{
"db_name": "PostgreSQL",
"query": "SELECT status::text AS \"status!\" FROM v2_job_completed WHERE id = $1 AND workspace_id = $2",
"describe": {
"columns": [
{
"ordinal": 0,
"name": "status!",
"type_info": "Text"
}
],
"parameters": {
"Left": [
"Uuid",
"Text"
]
},
"nullable": [
null
]
},
"hash": "44a39475ba202bd5852b666335308360569aae717eff3fe9cc9253a261ae2fa7"
}
backend/.sqlx/query-4550ae568abf23045259f95195bf1a35f5dc8f6828c2f3df51520683af48ebbd.json
Generated
+16
View File
@@ -0,0 +1,16 @@
{
"db_name": "PostgreSQL",
"query": "UPDATE eval_dataset SET path = REGEXP_REPLACE(path,'u/' || $2 || '/(.*)','u/' || $1 || '/\\1') WHERE path LIKE ('u/' || $2 || '/%') AND workspace_id = $3",
"describe": {
"columns": [],
"parameters": {
"Left": [
"Text",
"Text",
"Text"
]
},
"nullable": []
},
"hash": "4550ae568abf23045259f95195bf1a35f5dc8f6828c2f3df51520683af48ebbd"
}
backend/.sqlx/query-467ed4d282af003dd3b0d9542caec73feba3141f7345ffd1a6554fe44a6a9171.json
Generated
+24
View File
@@ -0,0 +1,24 @@
{
"db_name": "PostgreSQL",
"query": "WITH updated AS (\n UPDATE eval_dataset SET path = REGEXP_REPLACE(path, 'u/' || $2 || '/(.*)', $1 || '/\\1')\n WHERE path LIKE ('u/' || $2 || '/%') AND workspace_id = $3\n RETURNING 1\n ) SELECT COUNT(*) FROM updated",
"describe": {
"columns": [
{
"ordinal": 0,
"name": "count",
"type_info": "Int8"
}
],
"parameters": {
"Left": [
"Text",
"Text",
"Text"
]
},
"nullable": [
null
]
},
"hash": "467ed4d282af003dd3b0d9542caec73feba3141f7345ffd1a6554fe44a6a9171"
}
backend/.sqlx/query-479427dc09ebacb80cb20d553e93a577bb85b389ab056909a326212565a338bc.json
Generated
+15
View File
@@ -0,0 +1,15 @@
{
"db_name": "PostgreSQL",
"query": "INSERT INTO eval_dataset (workspace_id, path, summary, scorers, extra_perms, created_at, created_by, edited_at, edited_by)\n SELECT $2, path, summary, scorers, extra_perms, created_at, created_by, edited_at, edited_by\n FROM eval_dataset WHERE workspace_id = $1",
"describe": {
"columns": [],
"parameters": {
"Left": [
"Text",
"Varchar"
]
},
"nullable": []
},
"hash": "479427dc09ebacb80cb20d553e93a577bb85b389ab056909a326212565a338bc"
}
backend/.sqlx/query-4be42f447e10e420f2a909579398b01b5175e14eee47ca39fe63242a78da767e.json
Generated
+16
View File
@@ -0,0 +1,16 @@
{
"db_name": "PostgreSQL",
"query": "UPDATE eval_case SET created_by = $1 WHERE created_by = $2 AND workspace_id = $3",
"describe": {
"columns": [],
"parameters": {
"Left": [
"Varchar",
"Text",
"Text"
]
},
"nullable": []
},
"hash": "4be42f447e10e420f2a909579398b01b5175e14eee47ca39fe63242a78da767e"
}
backend/.sqlx/query-4bf7f1a0fd87e79bb789cee06a5da68e465ce7ce67e80770f22a36a9e8320657.json
Generated
+59
View File
@@ -0,0 +1,59 @@
{
"db_name": "PostgreSQL",
"query": "SELECT path, summary, scorers, created_at, created_by, edited_at, edited_by\n FROM eval_dataset WHERE workspace_id = $1 AND path = $2 FOR UPDATE",
"describe": {
"columns": [
{
"ordinal": 0,
"name": "path",
"type_info": "Varchar"
},
{
"ordinal": 1,
"name": "summary",
"type_info": "Varchar"
},
{
"ordinal": 2,
"name": "scorers",
"type_info": "Jsonb"
},
{
"ordinal": 3,
"name": "created_at",
"type_info": "Timestamptz"
},
{
"ordinal": 4,
"name": "created_by",
"type_info": "Varchar"
},
{
"ordinal": 5,
"name": "edited_at",
"type_info": "Timestamptz"
},
{
"ordinal": 6,
"name": "edited_by",
"type_info": "Varchar"
}
],
"parameters": {
"Left": [
"Text",
"Text"
]
},
"nullable": [
false,
true,
false,
false,
false,
false,
false
]
},
"hash": "4bf7f1a0fd87e79bb789cee06a5da68e465ce7ce67e80770f22a36a9e8320657"
}
backend/.sqlx/query-e74e283951aa87627a46aa8286819cf7aa4fecbde17bc7a67ee6f49c427cee9e.json → backend/.sqlx/query-4e4a9c7b6e95f81101a68a6cc6d29fe4424eaed36d5ac9e4cd0a2a9d8b878eeb.json
Generated
+5 -5
View File
@@ -1,6 +1,6 @@
{
"db_name": "PostgreSQL",
"query": "SELECT path, value FROM resource_version WHERE workspace_id = $1 AND id = $2",
"query": "SELECT path, scorers FROM eval_dataset WHERE workspace_id = $1 AND path = ANY($2)",
"describe": {
"columns": [
{
@@ -10,20 +10,20 @@
},
{
"ordinal": 1,
"name": "value",
"name": "scorers",
"type_info": "Jsonb"
}
],
"parameters": {
"Left": [
"Text",
"Int8"
"TextArray"
]
},
"nullable": [
false,
true
false
]
},
"hash": "e74e283951aa87627a46aa8286819cf7aa4fecbde17bc7a67ee6f49c427cee9e"
"hash": "4e4a9c7b6e95f81101a68a6cc6d29fe4424eaed36d5ac9e4cd0a2a9d8b878eeb"
}
backend/.sqlx/query-5329ce41bbdc36698ea059fec5f05ded7dfe1c64517289ad67028001417d30a3.json
Generated
+47
View File
@@ -0,0 +1,47 @@
{
"db_name": "PostgreSQL",
"query": "SELECT id, input, expected, created_at, created_by\n FROM eval_case\n WHERE workspace_id = $1 AND dataset_path = $2\n ORDER BY created_at, id",
"describe": {
"columns": [
{
"ordinal": 0,
"name": "id",
"type_info": "Uuid"
},
{
"ordinal": 1,
"name": "input",
"type_info": "Jsonb"
},
{
"ordinal": 2,
"name": "expected",
"type_info": "Jsonb"
},
{
"ordinal": 3,
"name": "created_at",
"type_info": "Timestamptz"
},
{
"ordinal": 4,
"name": "created_by",
"type_info": "Varchar"
}
],
"parameters": {
"Left": [
"Text",
"Text"
]
},
"nullable": [
false,
false,
true,
false,
false
]
},
"hash": "5329ce41bbdc36698ea059fec5f05ded7dfe1c64517289ad67028001417d30a3"
}
backend/.sqlx/query-5a219a2532517869578c4504ff3153c43903f929ae5d62fbba12610f89c36d55.json
Generated
+1 -1
View File
@@ -15,7 +15,7 @@
]
},
"nullable": [
true
null
]
},
"hash": "5a219a2532517869578c4504ff3153c43903f929ae5d62fbba12610f89c36d55"
backend/.sqlx/query-5d3560d7a42f86436fec362a790cea189be47e344d11871a90faa9220d509646.json
Generated
+16
View File
@@ -0,0 +1,16 @@
{
"db_name": "PostgreSQL",
"query": "UPDATE eval_experiment_case c SET job_id = j.id\n FROM v2_job j\n WHERE j.parent_job = $3 AND j.workspace_id = $2\n AND (j.args -> 'iter' -> 'value' ->> 'case_id')::uuid = c.case_id\n AND c.experiment_id = $1 AND c.job_id IS NULL",
"describe": {
"columns": [],
"parameters": {
"Left": [
"Uuid",
"Text",
"Uuid"
]
},
"nullable": []
},
"hash": "5d3560d7a42f86436fec362a790cea189be47e344d11871a90faa9220d509646"
}
backend/.sqlx/query-5d5186bb17092425664d8c4f92ca6bfb8b08420cc2b2a52e90064d6a12f86113.json
Generated
+76
View File
@@ -0,0 +1,76 @@
{
"db_name": "PostgreSQL",
"query": "SELECT ordinal, case_id, input, expected, job_id, subject_version,\n subject_draft_hash, output, answered, status\n FROM eval_experiment_case\n WHERE experiment_id = $1 ORDER BY ordinal",
"describe": {
"columns": [
{
"ordinal": 0,
"name": "ordinal",
"type_info": "Int4"
},
{
"ordinal": 1,
"name": "case_id",
"type_info": "Uuid"
},
{
"ordinal": 2,
"name": "input",
"type_info": "Jsonb"
},
{
"ordinal": 3,
"name": "expected",
"type_info": "Jsonb"
},
{
"ordinal": 4,
"name": "job_id",
"type_info": "Uuid"
},
{
"ordinal": 5,
"name": "subject_version",
"type_info": "Int8"
},
{
"ordinal": 6,
"name": "subject_draft_hash",
"type_info": "Varchar"
},
{
"ordinal": 7,
"name": "output",
"type_info": "Text"
},
{
"ordinal": 8,
"name": "answered",
"type_info": "Bool"
},
{
"ordinal": 9,
"name": "status",
"type_info": "Varchar"
}
],
"parameters": {
"Left": [
"Uuid"
]
},
"nullable": [
false,
false,
false,
true,
true,
true,
true,
true,
true,
true
]
},
"hash": "5d5186bb17092425664d8c4f92ca6bfb8b08420cc2b2a52e90064d6a12f86113"
}
backend/.sqlx/query-5dcaea907b1ebb2854becc44c00eac5838995dce3d2a696769e00fb74c34288f.json
Generated
+22
View File
@@ -0,0 +1,22 @@
{
"db_name": "PostgreSQL",
"query": "SELECT DISTINCT experiment_id FROM eval_score\n WHERE experiment_id = ANY($1) AND score IS NULL AND error IS NULL\n AND NOT not_applicable",
"describe": {
"columns": [
{
"ordinal": 0,
"name": "experiment_id",
"type_info": "Uuid"
}
],
"parameters": {
"Left": [
"UuidArray"
]
},
"nullable": [
false
]
},
"hash": "5dcaea907b1ebb2854becc44c00eac5838995dce3d2a696769e00fb74c34288f"
}
backend/.sqlx/query-61773dd5d5952607eddfacb1717126a87ee5dc4bd6664eaf31c2c0096fea08db.json
Generated
+26
View File
@@ -0,0 +1,26 @@
{
"db_name": "PostgreSQL",
"query": "UPDATE eval_case SET input = $4, expected = $5\n WHERE workspace_id = $1 AND dataset_path = $2 AND id = $3\n RETURNING id",
"describe": {
"columns": [
{
"ordinal": 0,
"name": "id",
"type_info": "Uuid"
}
],
"parameters": {
"Left": [
"Text",
"Text",
"Uuid",
"Jsonb",
"Jsonb"
]
},
"nullable": [
false
]
},
"hash": "61773dd5d5952607eddfacb1717126a87ee5dc4bd6664eaf31c2c0096fea08db"
}
backend/.sqlx/query-6a17a1dfeb75808e5d0726f1f8bd573168161abe350b047402cda4ac4a4d13e4.json
Generated
+74
View File
@@ -0,0 +1,74 @@
{
"db_name": "PostgreSQL",
"query": "SELECT\n (CASE $3::text\n WHEN 'day' THEN day::text\n WHEN 'user' THEN email\n ELSE ''\n END) AS \"key!\",\n provider AS \"provider!\",\n model AS \"model!\",\n SUM(input_tokens)::bigint AS \"input_tokens!\",\n SUM(cache_read_tokens)::bigint AS \"cache_read_tokens!\",\n SUM(cache_write_tokens)::bigint AS \"cache_write_tokens!\",\n SUM(output_tokens)::bigint AS \"output_tokens!\",\n SUM(reported_cost_nano_usd)::bigint AS \"reported_cost_nano_usd\",\n SUM(requests)::bigint AS \"requests!\"\n FROM ai_token_usage\n WHERE workspace_id = $1 AND day > CURRENT_DATE - $2::int\n AND ($5::text IS NULL OR email = $5)\n GROUP BY 1, provider, model\n ORDER BY SUM(input_tokens + cache_read_tokens + cache_write_tokens + output_tokens) DESC\n LIMIT $4",
"describe": {
"columns": [
{
"ordinal": 0,
"name": "key!",
"type_info": "Text"
},
{
"ordinal": 1,
"name": "provider!",
"type_info": "Varchar"
},
{
"ordinal": 2,
"name": "model!",
"type_info": "Varchar"
},
{
"ordinal": 3,
"name": "input_tokens!",
"type_info": "Int8"
},
{
"ordinal": 4,
"name": "cache_read_tokens!",
"type_info": "Int8"
},
{
"ordinal": 5,
"name": "cache_write_tokens!",
"type_info": "Int8"
},
{
"ordinal": 6,
"name": "output_tokens!",
"type_info": "Int8"
},
{
"ordinal": 7,
"name": "reported_cost_nano_usd",
"type_info": "Int8"
},
{
"ordinal": 8,
"name": "requests!",
"type_info": "Int8"
}
],
"parameters": {
"Left": [
"Text",
"Int4",
"Text",
"Int8",
"Text"
]
},
"nullable": [
null,
false,
false,
null,
null,
null,
null,
null,
null
]
},
"hash": "6a17a1dfeb75808e5d0726f1f8bd573168161abe350b047402cda4ac4a4d13e4"
}
backend/.sqlx/query-6bc9d682aabdf8e79beb693e0090d07f3514628a4970b5f25ddf3920ba5e9c53.json
Generated
+23
View File
@@ -0,0 +1,23 @@
{
"db_name": "PostgreSQL",
"query": "SELECT scorers FROM eval_dataset WHERE workspace_id = $1 AND path = $2 FOR UPDATE",
"describe": {
"columns": [
{
"ordinal": 0,
"name": "scorers",
"type_info": "Jsonb"
}
],
"parameters": {
"Left": [
"Text",
"Text"
]
},
"nullable": [
false
]
},
"hash": "6bc9d682aabdf8e79beb693e0090d07f3514628a4970b5f25ddf3920ba5e9c53"
}
backend/.sqlx/query-77db9fcef0d3f9c9eb9edba6c0a54469259621218616ef64a4923f3396b183a2.json
Generated
+14
View File
@@ -0,0 +1,14 @@
{
"db_name": "PostgreSQL",
"query": "DELETE FROM eval_experiment WHERE id = $1",
"describe": {
"columns": [],
"parameters": {
"Left": [
"Uuid"
]
},
"nullable": []
},
"hash": "77db9fcef0d3f9c9eb9edba6c0a54469259621218616ef64a4923f3396b183a2"
}
backend/.sqlx/query-790e90a3aca284fd060aac049fcf27d2e39781fccc4dd8419f934085d5453fc7.json
Generated
+20
View File
@@ -0,0 +1,20 @@
{
"db_name": "PostgreSQL",
"query": "INSERT INTO eval_experiment_case\n (experiment_id, ordinal, case_id, input, expected, subject_version,\n subject_draft_hash)\n SELECT $1, ordinal, case_id, input, expected, subject_version, subject_draft_hash\n FROM UNNEST($2::int[], $3::uuid[], $4::jsonb[], $5::jsonb[], $6::bigint[], $7::text[])\n AS t(ordinal, case_id, input, expected, subject_version, subject_draft_hash)",
"describe": {
"columns": [],
"parameters": {
"Left": [
"Uuid",
"Int4Array",
"UuidArray",
"JsonbArray",
"JsonbArray",
"Int8Array",
"TextArray"
]
},
"nullable": []
},
"hash": "790e90a3aca284fd060aac049fcf27d2e39781fccc4dd8419f934085d5453fc7"
}
backend/.sqlx/query-7a14c6815a7acc912fd8836191263a373b5c427a0b2f48ca775fb8f5a0096e18.json
Generated
+26
View File
@@ -0,0 +1,26 @@
{
"db_name": "PostgreSQL",
"query": "INSERT INTO eval_case\n (workspace_id, dataset_path, input, expected, created_by, created_at)\n VALUES ($1, $2, $3, $4, $5, clock_timestamp())\n RETURNING id",
"describe": {
"columns": [
{
"ordinal": 0,
"name": "id",
"type_info": "Uuid"
}
],
"parameters": {
"Left": [
"Varchar",
"Varchar",
"Jsonb",
"Jsonb",
"Varchar"
]
},
"nullable": [
false
]
},
"hash": "7a14c6815a7acc912fd8836191263a373b5c427a0b2f48ca775fb8f5a0096e18"
}
backend/.sqlx/query-7b040feaa84e85fff1a5ad1ddfc3d57b0f69b05f96ffafae0325cf3ff04f0f40.json
Generated
+60
View File
@@ -0,0 +1,60 @@
{
"db_name": "PostgreSQL",
"query": "SELECT s.experiment_id AS \"experiment_id!\", s.scorer_id AS \"scorer_id!\",\n avg(s.score) AS mean,\n count(s.score) AS \"scored!\",\n count(*) FILTER (WHERE s.error IS NOT NULL) AS \"failed!\",\n count(*) FILTER (WHERE t.pass_if IS NOT NULL AND s.score >= t.pass_if)\n AS \"passed!\",\n bool_or(t.pass_if IS NOT NULL) AS \"has_threshold!\"\n FROM eval_score s\n JOIN unnest($1::uuid[], $2::text[], $3::float8[])\n AS t(experiment_id, scorer_id, pass_if)\n ON t.experiment_id = s.experiment_id AND t.scorer_id = s.scorer_id\n GROUP BY s.experiment_id, s.scorer_id",
"describe": {
"columns": [
{
"ordinal": 0,
"name": "experiment_id!",
"type_info": "Uuid"
},
{
"ordinal": 1,
"name": "scorer_id!",
"type_info": "Varchar"
},
{
"ordinal": 2,
"name": "mean",
"type_info": "Float8"
},
{
"ordinal": 3,
"name": "scored!",
"type_info": "Int8"
},
{
"ordinal": 4,
"name": "failed!",
"type_info": "Int8"
},
{
"ordinal": 5,
"name": "passed!",
"type_info": "Int8"
},
{
"ordinal": 6,
"name": "has_threshold!",
"type_info": "Bool"
}
],
"parameters": {
"Left": [
"UuidArray",
"TextArray",
"Float8Array"
]
},
"nullable": [
false,
false,
null,
null,
null,
null,
null
]
},
"hash": "7b040feaa84e85fff1a5ad1ddfc3d57b0f69b05f96ffafae0325cf3ff04f0f40"
}
backend/.sqlx/query-7f373cf063907999580d0541a1a4a216e3887fdc0a9bab0747a447c01a4b37a5.json
Generated
+16
View File
@@ -0,0 +1,16 @@
{
"db_name": "PostgreSQL",
"query": "UPDATE eval_experiment_case\n SET subject_version = $3, subject_draft_hash = NULL\n WHERE experiment_id = $1 AND subject_draft_hash = $2",
"describe": {
"columns": [],
"parameters": {
"Left": [
"Uuid",
"Text",
"Int8"
]
},
"nullable": []
},
"hash": "7f373cf063907999580d0541a1a4a216e3887fdc0a9bab0747a447c01a4b37a5"
}
backend/.sqlx/query-80a5dd06cc5b9f7cb0bcd3b8cdd92c62d40a4ab97fab0f2dd330bd7629c0a632.json
Generated
+64
View File
@@ -0,0 +1,64 @@
{
"db_name": "PostgreSQL",
"query": "SELECT ordinal, scorer_id, score, reason, checks, error, not_applicable, definition\n FROM eval_score WHERE experiment_id = $1",
"describe": {
"columns": [
{
"ordinal": 0,
"name": "ordinal",
"type_info": "Int4"
},
{
"ordinal": 1,
"name": "scorer_id",
"type_info": "Varchar"
},
{
"ordinal": 2,
"name": "score",
"type_info": "Float8"
},
{
"ordinal": 3,
"name": "reason",
"type_info": "Text"
},
{
"ordinal": 4,
"name": "checks",
"type_info": "Jsonb"
},
{
"ordinal": 5,
"name": "error",
"type_info": "Text"
},
{
"ordinal": 6,
"name": "not_applicable",
"type_info": "Bool"
},
{
"ordinal": 7,
"name": "definition",
"type_info": "Varchar"
}
],
"parameters": {
"Left": [
"Uuid"
]
},
"nullable": [
false,
false,
true,
true,
true,
true,
false,
false
]
},
"hash": "80a5dd06cc5b9f7cb0bcd3b8cdd92c62d40a4ab97fab0f2dd330bd7629c0a632"
}
backend/.sqlx/query-e27ed86394e6568afdf7a1dd72edd7943c87fdf1d05fba193cbe575cb7281db5.json → backend/.sqlx/query-82b2e8383ae7e345e45fb8a1b2bb362439f5b12a3dd3290a8dab17ccf9c391f3.json
Generated
+9 -3
View File
@@ -1,6 +1,6 @@
{
"db_name": "PostgreSQL",
"query": "SELECT id, created_at, created_by FROM resource_version\n WHERE workspace_id = $1 AND path = $2 ORDER BY id DESC LIMIT $3",
"query": "SELECT id, version, created_at, created_by FROM resource_version\n WHERE workspace_id = $1 AND path = $2 ORDER BY id DESC LIMIT $3",
"describe": {
"columns": [
{
@@ -10,11 +10,16 @@
},
{
"ordinal": 1,
"name": "version",
"type_info": "Int8"
},
{
"ordinal": 2,
"name": "created_at",
"type_info": "Timestamptz"
},
{
"ordinal": 2,
"ordinal": 3,
"name": "created_by",
"type_info": "Varchar"
}
@@ -27,10 +32,11 @@
]
},
"nullable": [
false,
false,
false,
true
]
},
"hash": "e27ed86394e6568afdf7a1dd72edd7943c87fdf1d05fba193cbe575cb7281db5"
"hash": "82b2e8383ae7e345e45fb8a1b2bb362439f5b12a3dd3290a8dab17ccf9c391f3"
}
backend/.sqlx/query-858b5fe344d79913921d8c05e9e6219919f438e9a3ce0c3e996ea61d9b4f2ef8.json
Generated
+16
View File
@@ -0,0 +1,16 @@
{
"db_name": "PostgreSQL",
"query": "UPDATE eval_dataset SET edited_by = $1 WHERE edited_by = $2 AND workspace_id = $3",
"describe": {
"columns": [],
"parameters": {
"Left": [
"Varchar",
"Text",
"Text"
]
},
"nullable": []
},
"hash": "858b5fe344d79913921d8c05e9e6219919f438e9a3ce0c3e996ea61d9b4f2ef8"
}
backend/.sqlx/query-864184467477e73a45935bf9439ae60da7f9ecc12f8f98b926345a70f941845b.json
Generated
+24
View File
@@ -0,0 +1,24 @@
{
"db_name": "PostgreSQL",
"query": "SELECT value FROM resource_version\n WHERE version = $1 AND workspace_id = $2 AND path = $3",
"describe": {
"columns": [
{
"ordinal": 0,
"name": "value",
"type_info": "Jsonb"
}
],
"parameters": {
"Left": [
"Int8",
"Text",
"Text"
]
},
"nullable": [
true
]
},
"hash": "864184467477e73a45935bf9439ae60da7f9ecc12f8f98b926345a70f941845b"
}
backend/.sqlx/query-897e8da49e4a5c3efdfb2c36fc7aff3d53432da9ef57f8a775850efcd50a965c.json
Generated
+29
View File
@@ -0,0 +1,29 @@
{
"db_name": "PostgreSQL",
"query": "SELECT r.value AS \"value: sqlx::types::Json<serde_json::Value>\",\n (SELECT version FROM resource_version v\n WHERE v.workspace_id = r.workspace_id AND v.path = r.path\n ORDER BY v.version DESC LIMIT 1) AS version\n FROM resource r\n WHERE r.workspace_id = $1 AND r.path = $2 AND r.resource_type = 'ai_agent'",
"describe": {
"columns": [
{
"ordinal": 0,
"name": "value: sqlx::types::Json<serde_json::Value>",
"type_info": "Jsonb"
},
{
"ordinal": 1,
"name": "version",
"type_info": "Int8"
}
],
"parameters": {
"Left": [
"Text",
"Text"
]
},
"nullable": [
true,
null
]
},
"hash": "897e8da49e4a5c3efdfb2c36fc7aff3d53432da9ef57f8a775850efcd50a965c"
}
backend/.sqlx/query-8c319ac3eb2a289a6709ea1ea1c164f1a56e328902dd46fd9b68d801c16f4ffa.json
Generated
+16
View File
@@ -0,0 +1,16 @@
{
"db_name": "PostgreSQL",
"query": "UPDATE eval_dataset SET extra_perms = extra_perms - ('u/' || $2) || jsonb_build_object(('u/' || $1), extra_perms->('u/' || $2)) WHERE extra_perms ? ('u/' || $2) AND workspace_id = $3",
"describe": {
"columns": [],
"parameters": {
"Left": [
"Text",
"Text",
"Text"
]
},
"nullable": []
},
"hash": "8c319ac3eb2a289a6709ea1ea1c164f1a56e328902dd46fd9b68d801c16f4ffa"
}
backend/.sqlx/query-9071eff54395ca39809a5d55d1e3fedb64cfafa9fe74cd9446eb6ee526c2d9a0.json
Generated
+58
View File
@@ -0,0 +1,58 @@
{
"db_name": "PostgreSQL",
"query": "SELECT path, summary, scorers, created_at, created_by,\n edited_at, edited_by\n FROM eval_dataset WHERE workspace_id = $1 ORDER BY path",
"describe": {
"columns": [
{
"ordinal": 0,
"name": "path",
"type_info": "Varchar"
},
{
"ordinal": 1,
"name": "summary",
"type_info": "Varchar"
},
{
"ordinal": 2,
"name": "scorers",
"type_info": "Jsonb"
},
{
"ordinal": 3,
"name": "created_at",
"type_info": "Timestamptz"
},
{
"ordinal": 4,
"name": "created_by",
"type_info": "Varchar"
},
{
"ordinal": 5,
"name": "edited_at",
"type_info": "Timestamptz"
},
{
"ordinal": 6,
"name": "edited_by",
"type_info": "Varchar"
}
],
"parameters": {
"Left": [
"Text"
]
},
"nullable": [
false,
true,
false,
false,
false,
false,
false
]
},
"hash": "9071eff54395ca39809a5d55d1e3fedb64cfafa9fe74cd9446eb6ee526c2d9a0"
}
backend/.sqlx/query-97a701e16c0ce4b8c6a1394c2a71b8ae45e16c11c8003c5f64c9f53d0e7226aa.json
Generated
+23
View File
@@ -0,0 +1,23 @@
{
"db_name": "PostgreSQL",
"query": "SELECT COALESCE(s.flow_status, c.flow_status) -> 'modules' AS modules\n FROM v2_job j\n LEFT JOIN v2_job_status s ON s.id = j.id\n LEFT JOIN v2_job_completed c ON c.id = j.id\n WHERE j.id = $1 AND j.workspace_id = $2",
"describe": {
"columns": [
{
"ordinal": 0,
"name": "modules",
"type_info": "Jsonb"
}
],
"parameters": {
"Left": [
"Uuid",
"Text"
]
},
"nullable": [
null
]
},
"hash": "97a701e16c0ce4b8c6a1394c2a71b8ae45e16c11c8003c5f64c9f53d0e7226aa"
}
backend/.sqlx/query-9b1ad1bbf0c2dca3ce1cc9433c35db87e3101bcbcb0fa359885ca3807c1c9ca2.json
Generated
+23
View File
@@ -0,0 +1,23 @@
{
"db_name": "PostgreSQL",
"query": "SELECT j.id AS \"id!\" FROM v2_job j\n LEFT JOIN v2_job_completed c ON c.id = j.id AND c.workspace_id = $2\n WHERE j.id = ANY($1) AND j.workspace_id = $2 AND c.id IS NULL",
"describe": {
"columns": [
{
"ordinal": 0,
"name": "id!",
"type_info": "Uuid"
}
],
"parameters": {
"Left": [
"UuidArray",
"Text"
]
},
"nullable": [
false
]
},
"hash": "9b1ad1bbf0c2dca3ce1cc9433c35db87e3101bcbcb0fa359885ca3807c1c9ca2"
}
backend/.sqlx/query-9d3ca755b323330033eb891ac7162a53440a5c713b58e6c4a467f6f0d5d7e37c.json
Generated
+16
View File
@@ -0,0 +1,16 @@
{
"db_name": "PostgreSQL",
"query": "UPDATE eval_dataset SET scorers = COALESCE((\n SELECT jsonb_agg(\n CASE WHEN elem->>'path' LIKE ('u/' || $2 || '/%')\n THEN jsonb_set(elem, '{path}', to_jsonb(REGEXP_REPLACE(elem->>'path','u/' || $2 || '/(.*)','u/' || $1 || '/\\1')))\n ELSE elem END)\n FROM jsonb_array_elements(scorers) elem), '[]'::jsonb)\n WHERE workspace_id = $3\n AND EXISTS (SELECT 1 FROM jsonb_array_elements(scorers) e WHERE e->>'path' LIKE ('u/' || $2 || '/%'))",
"describe": {
"columns": [],
"parameters": {
"Left": [
"Text",
"Text",
"Text"
]
},
"nullable": []
},
"hash": "9d3ca755b323330033eb891ac7162a53440a5c713b58e6c4a467f6f0d5d7e37c"
}
backend/.sqlx/query-a4c842e395714346d5178190793d8ae66f891421b72fc6adfa8a38060ca2f3fa.json
Generated
+15
View File
@@ -0,0 +1,15 @@
{
"db_name": "PostgreSQL",
"query": "INSERT INTO eval_case (workspace_id, dataset_path, input, expected, created_at, created_by)\n SELECT $2, dataset_path, input, expected, created_at, created_by\n FROM eval_case WHERE workspace_id = $1",
"describe": {
"columns": [],
"parameters": {
"Left": [
"Text",
"Varchar"
]
},
"nullable": []
},
"hash": "a4c842e395714346d5178190793d8ae66f891421b72fc6adfa8a38060ca2f3fa"
}
backend/.sqlx/query-a5e8cf0e559742330d67d36d9ddb75ef7de6e0ad74e0cc6c4056b782155b86c7.json
Generated
+53
View File
@@ -0,0 +1,53 @@
{
"db_name": "PostgreSQL",
"query": "SELECT s.ordinal, s.scorer_id, c.job_id AS \"job_id!\", d.status::text AS status,\n c.answered, (j.id IS NOT NULL) AS \"job_exists!\"\n FROM eval_score s\n JOIN eval_experiment_case c\n ON c.experiment_id = s.experiment_id AND c.ordinal = s.ordinal\n LEFT JOIN v2_job j ON j.id = c.job_id AND j.workspace_id = $2\n LEFT JOIN v2_job_completed d ON d.id = c.job_id AND d.workspace_id = $2\n WHERE s.experiment_id = $1 AND s.score IS NULL AND s.error IS NULL\n AND NOT s.not_applicable AND c.job_id IS NOT NULL",
"describe": {
"columns": [
{
"ordinal": 0,
"name": "ordinal",
"type_info": "Int4"
},
{
"ordinal": 1,
"name": "scorer_id",
"type_info": "Varchar"
},
{
"ordinal": 2,
"name": "job_id!",
"type_info": "Uuid"
},
{
"ordinal": 3,
"name": "status",
"type_info": "Text"
},
{
"ordinal": 4,
"name": "answered",
"type_info": "Bool"
},
{
"ordinal": 5,
"name": "job_exists!",
"type_info": "Bool"
}
],
"parameters": {
"Left": [
"Uuid",
"Text"
]
},
"nullable": [
false,
false,
true,
null,
true,
null
]
},
"hash": "a5e8cf0e559742330d67d36d9ddb75ef7de6e0ad74e0cc6c4056b782155b86c7"
}
backend/.sqlx/query-a7b589b8d5cded97905bc24412ad8f3f0339a2a088243f045a212f9357205407.json
Generated
+27
View File
@@ -0,0 +1,27 @@
{
"db_name": "PostgreSQL",
"query": "UPDATE eval_dataset\n SET path = COALESCE($6, path), summary = COALESCE($3, summary),\n scorers = COALESCE($4, scorers), edited_at = now(), edited_by = $5\n WHERE workspace_id = $1 AND path = $2\n RETURNING path",
"describe": {
"columns": [
{
"ordinal": 0,
"name": "path",
"type_info": "Varchar"
}
],
"parameters": {
"Left": [
"Text",
"Text",
"Varchar",
"Jsonb",
"Varchar",
"Varchar"
]
},
"nullable": [
false
]
},
"hash": "a7b589b8d5cded97905bc24412ad8f3f0339a2a088243f045a212f9357205407"
}
backend/.sqlx/query-b2db4f32c615a99db7af23729682340328da39ac763bb26b6bcf82027fad48e5.json
Generated
+29
View File
@@ -0,0 +1,29 @@
{
"db_name": "PostgreSQL",
"query": "SELECT id, run_job_id FROM eval_experiment WHERE workspace_id = $1 AND id = $2",
"describe": {
"columns": [
{
"ordinal": 0,
"name": "id",
"type_info": "Uuid"
},
{
"ordinal": 1,
"name": "run_job_id",
"type_info": "Uuid"
}
],
"parameters": {
"Left": [
"Text",
"Uuid"
]
},
"nullable": [
false,
false
]
},
"hash": "b2db4f32c615a99db7af23729682340328da39ac763bb26b6bcf82027fad48e5"
}
backend/.sqlx/query-b8e732a03969666444f73397ac153d23ec2af084b2f93da24c920532c1916384.json
Generated
+3 -3
View File
@@ -98,12 +98,12 @@
null,
null,
null,
false,
true,
null,
null,
null,
false,
false
true,
true
]
},
"hash": "b8e732a03969666444f73397ac153d23ec2af084b2f93da24c920532c1916384"
backend/.sqlx/query-bac36542b16b687a823067c013e3728f1cabe3774f581401514f522ea7522de5.json
Generated
+29
View File
@@ -0,0 +1,29 @@
{
"db_name": "PostgreSQL",
"query": "SELECT status::text AS \"status!\", duration_ms FROM v2_job_completed\n WHERE id = $1 AND workspace_id = $2",
"describe": {
"columns": [
{
"ordinal": 0,
"name": "status!",
"type_info": "Text"
},
{
"ordinal": 1,
"name": "duration_ms",
"type_info": "Int8"
}
],
"parameters": {
"Left": [
"Uuid",
"Text"
]
},
"nullable": [
null,
false
]
},
"hash": "bac36542b16b687a823067c013e3728f1cabe3774f581401514f522ea7522de5"
}
backend/.sqlx/query-bbce2221f5724016543a5ac1db7fa74b8b35375dfb0f5d39b398369468ad0774.json
Generated
+54
View File
@@ -0,0 +1,54 @@
{
"db_name": "PostgreSQL",
"query": "SELECT e.subject, e.run_number, e.run_job_id, e.created_at,\n e.created_by,\n (SELECT count(*) FROM eval_experiment_case c WHERE c.experiment_id = e.id)\n AS \"case_count!\"\n FROM eval_experiment e\n WHERE e.workspace_id = $1 AND e.dataset_path = $2 AND e.id = $3",
"describe": {
"columns": [
{
"ordinal": 0,
"name": "subject",
"type_info": "Jsonb"
},
{
"ordinal": 1,
"name": "run_number",
"type_info": "Int4"
},
{
"ordinal": 2,
"name": "run_job_id",
"type_info": "Uuid"
},
{
"ordinal": 3,
"name": "created_at",
"type_info": "Timestamptz"
},
{
"ordinal": 4,
"name": "created_by",
"type_info": "Varchar"
},
{
"ordinal": 5,
"name": "case_count!",
"type_info": "Int8"
}
],
"parameters": {
"Left": [
"Text",
"Text",
"Uuid"
]
},
"nullable": [
false,
false,
false,
false,
false,
null
]
},
"hash": "bbce2221f5724016543a5ac1db7fa74b8b35375dfb0f5d39b398369468ad0774"
}
backend/.sqlx/query-bdb7185233941d1472a55c8ade3f61c82248f3f2c9a69c458bff3978ffd17379.json
Generated
+66
View File
@@ -0,0 +1,66 @@
{
"db_name": "PostgreSQL",
"query": "SELECT e.id, e.dataset_path, e.subject, e.run_number, e.run_job_id, e.created_at,\n e.created_by,\n (SELECT count(*) FROM eval_experiment_case c WHERE c.experiment_id = e.id)\n AS \"case_count!\"\n FROM eval_experiment e\n JOIN eval_dataset d ON d.workspace_id = e.workspace_id AND d.path = e.dataset_path\n WHERE e.workspace_id = $1\n AND ($3::text IS NULL OR e.subject ->> 'path' = $3)\n ORDER BY e.created_at DESC\n LIMIT $2",
"describe": {
"columns": [
{
"ordinal": 0,
"name": "id",
"type_info": "Uuid"
},
{
"ordinal": 1,
"name": "dataset_path",
"type_info": "Varchar"
},
{
"ordinal": 2,
"name": "subject",
"type_info": "Jsonb"
},
{
"ordinal": 3,
"name": "run_number",
"type_info": "Int4"
},
{
"ordinal": 4,
"name": "run_job_id",
"type_info": "Uuid"
},
{
"ordinal": 5,
"name": "created_at",
"type_info": "Timestamptz"
},
{
"ordinal": 6,
"name": "created_by",
"type_info": "Varchar"
},
{
"ordinal": 7,
"name": "case_count!",
"type_info": "Int8"
}
],
"parameters": {
"Left": [
"Text",
"Int8",
"Text"
]
},
"nullable": [
false,
false,
false,
false,
false,
false,
false,
null
]
},
"hash": "bdb7185233941d1472a55c8ade3f61c82248f3f2c9a69c458bff3978ffd17379"
}
backend/.sqlx/query-be17a65f144cc21e849c7f8cfbf9d7271b953dda42e4ba09864da7bda731e752.json
Generated
+54
View File
@@ -0,0 +1,54 @@
{
"db_name": "PostgreSQL",
"query": "SELECT j.id, j.args AS \"args: sqlx::types::Json<Box<RawValue>>\",\n c.result AS \"result: sqlx::types::Json<Box<RawValue>>\",\n c.status::text AS status, c.duration_ms,\n s.schema AS \"schema: sqlx::types::Json<Box<RawValue>>\"\n FROM v2_job j\n LEFT JOIN v2_job_completed c ON c.id = j.id\n LEFT JOIN script s ON s.workspace_id = j.workspace_id AND s.hash = j.runnable_id\n WHERE j.id = ANY($1) AND j.workspace_id = $2 AND j.parent_job = $3",
"describe": {
"columns": [
{
"ordinal": 0,
"name": "id",
"type_info": "Uuid"
},
{
"ordinal": 1,
"name": "args: sqlx::types::Json<Box<RawValue>>",
"type_info": "Jsonb"
},
{
"ordinal": 2,
"name": "result: sqlx::types::Json<Box<RawValue>>",
"type_info": "Jsonb"
},
{
"ordinal": 3,
"name": "status",
"type_info": "Text"
},
{
"ordinal": 4,
"name": "duration_ms",
"type_info": "Int8"
},
{
"ordinal": 5,
"name": "schema: sqlx::types::Json<Box<RawValue>>",
"type_info": "Json"
}
],
"parameters": {
"Left": [
"UuidArray",
"Text",
"Uuid"
]
},
"nullable": [
false,
true,
true,
null,
false,
true
]
},
"hash": "be17a65f144cc21e849c7f8cfbf9d7271b953dda42e4ba09864da7bda731e752"
}
backend/.sqlx/query-d10efb37765ac9a7f2e15f71dbbc02ec917b6ecb014f938e68ced3cc7fe9dc86.json
Generated
+16
View File
@@ -0,0 +1,16 @@
{
"db_name": "PostgreSQL",
"query": "UPDATE eval_experiment SET subject = jsonb_set(subject, '{path}', to_jsonb(REGEXP_REPLACE(subject->>'path','u/' || $2 || '/(.*)','u/' || $1 || '/\\1'))) WHERE subject->>'path' LIKE ('u/' || $2 || '/%') AND workspace_id = $3",
"describe": {
"columns": [],
"parameters": {
"Left": [
"Text",
"Text",
"Text"
]
},
"nullable": []
},
"hash": "d10efb37765ac9a7f2e15f71dbbc02ec917b6ecb014f938e68ced3cc7fe9dc86"
}
backend/.sqlx/query-d5cdb1121f2a414c0a70f4f7e4ae630bad0715bbbb3597cb1b14e41abe9fec14.json
Generated
+23
View File
@@ -0,0 +1,23 @@
{
"db_name": "PostgreSQL",
"query": "UPDATE eval_experiment_case SET status = $2, answered = false\n WHERE experiment_id = $1 AND job_id IS NULL AND status IS NULL\n RETURNING ordinal",
"describe": {
"columns": [
{
"ordinal": 0,
"name": "ordinal",
"type_info": "Int4"
}
],
"parameters": {
"Left": [
"Uuid",
"Varchar"
]
},
"nullable": [
false
]
},
"hash": "d5cdb1121f2a414c0a70f4f7e4ae630bad0715bbbb3597cb1b14e41abe9fec14"
}
backend/.sqlx/query-db19932d940b2467147eb2c13b059f2ec33b0c83b5c4a332a4b6e54d2591767c.json
Generated
+18
View File
@@ -0,0 +1,18 @@
{
"db_name": "PostgreSQL",
"query": "INSERT INTO eval_case\n (workspace_id, dataset_path, input, expected, created_by, created_at)\n VALUES ($1, $2, $3, $4, $5, clock_timestamp())",
"describe": {
"columns": [],
"parameters": {
"Left": [
"Varchar",
"Varchar",
"Jsonb",
"Jsonb",
"Varchar"
]
},
"nullable": []
},
"hash": "db19932d940b2467147eb2c13b059f2ec33b0c83b5c4a332a4b6e54d2591767c"
}
backend/.sqlx/query-db63d41718c50e264a949885d2d7c13719ae650e9c788ebf125627b785fb9ee0.json
Generated
+23
View File
@@ -0,0 +1,23 @@
{
"db_name": "PostgreSQL",
"query": "SELECT path FROM eval_dataset WHERE workspace_id = $1 AND path = $2",
"describe": {
"columns": [
{
"ordinal": 0,
"name": "path",
"type_info": "Varchar"
}
],
"parameters": {
"Left": [
"Text",
"Text"
]
},
"nullable": [
false
]
},
"hash": "db63d41718c50e264a949885d2d7c13719ae650e9c788ebf125627b785fb9ee0"
}
backend/.sqlx/query-e3c080b84f50622e0a74524111ae292c958e71293695ce18f9905af4dd940495.json
Generated
+49
View File
@@ -0,0 +1,49 @@
{
"db_name": "PostgreSQL",
"query": "SELECT id, input, expected, created_at, created_by\n FROM eval_case\n WHERE workspace_id = $1 AND dataset_path = $2\n ORDER BY created_at, id\n LIMIT $3 OFFSET $4",
"describe": {
"columns": [
{
"ordinal": 0,
"name": "id",
"type_info": "Uuid"
},
{
"ordinal": 1,
"name": "input",
"type_info": "Jsonb"
},
{
"ordinal": 2,
"name": "expected",
"type_info": "Jsonb"
},
{
"ordinal": 3,
"name": "created_at",
"type_info": "Timestamptz"
},
{
"ordinal": 4,
"name": "created_by",
"type_info": "Varchar"
}
],
"parameters": {
"Left": [
"Text",
"Text",
"Int8",
"Int8"
]
},
"nullable": [
false,
false,
true,
false,
false
]
},
"hash": "e3c080b84f50622e0a74524111ae292c958e71293695ce18f9905af4dd940495"
}
backend/.sqlx/query-e6d7e9779eaa6e584b613675ae8602c405fe620937a4563a17db80219a2930d2.json
Generated
+23
View File
@@ -0,0 +1,23 @@
{
"db_name": "PostgreSQL",
"query": "SELECT args AS \"args: sqlx::types::Json<Box<RawValue>>\" FROM v2_job\n WHERE id = $1 AND workspace_id = $2",
"describe": {
"columns": [
{
"ordinal": 0,
"name": "args: sqlx::types::Json<Box<RawValue>>",
"type_info": "Jsonb"
}
],
"parameters": {
"Left": [
"Uuid",
"Text"
]
},
"nullable": [
true
]
},
"hash": "e6d7e9779eaa6e584b613675ae8602c405fe620937a4563a17db80219a2930d2"
}
backend/.sqlx/query-e719e98cceef1383f882632c9398dd950ad0b1f30cde9c0392384c9a490290b9.json
Generated
+17
View File
@@ -0,0 +1,17 @@
{
"db_name": "PostgreSQL",
"query": "INSERT INTO eval_score (experiment_id, ordinal, scorer_id, definition)\n SELECT $1, ordinal, scorer_id, definition\n FROM UNNEST($2::int[], $3::text[], $4::text[]) AS t(ordinal, scorer_id, definition)\n ON CONFLICT (experiment_id, ordinal, scorer_id)\n DO UPDATE SET definition = EXCLUDED.definition, score = NULL, reason = NULL,\n checks = NULL, error = NULL, not_applicable = false",
"describe": {
"columns": [],
"parameters": {
"Left": [
"Uuid",
"Int4Array",
"TextArray",
"TextArray"
]
},
"nullable": []
},
"hash": "e719e98cceef1383f882632c9398dd950ad0b1f30cde9c0392384c9a490290b9"
}
backend/.sqlx/query-e72d71852b6c5998b8d46407d164fcee17400a0ffc172d2a257c3fc15617fbd9.json
Generated
+41
View File
@@ -0,0 +1,41 @@
{
"db_name": "PostgreSQL",
"query": "SELECT c.ordinal, c.job_id AS \"job_id!\", d.status::text AS status,\n (j.id IS NOT NULL) AS \"job_exists!\"\n FROM eval_experiment_case c\n LEFT JOIN v2_job j ON j.id = c.job_id AND j.workspace_id = $2\n LEFT JOIN v2_job_completed d ON d.id = c.job_id AND d.workspace_id = $2\n WHERE c.experiment_id = $1 AND c.job_id IS NOT NULL AND c.status IS NULL",
"describe": {
"columns": [
{
"ordinal": 0,
"name": "ordinal",
"type_info": "Int4"
},
{
"ordinal": 1,
"name": "job_id!",
"type_info": "Uuid"
},
{
"ordinal": 2,
"name": "status",
"type_info": "Text"
},
{
"ordinal": 3,
"name": "job_exists!",
"type_info": "Bool"
}
],
"parameters": {
"Left": [
"Uuid",
"Text"
]
},
"nullable": [
false,
true,
null,
null
]
},
"hash": "e72d71852b6c5998b8d46407d164fcee17400a0ffc172d2a257c3fc15617fbd9"
}
backend/.sqlx/query-e942a74104771b192c33ea03dbcaa409e03b6bfb7423929c201f39db4228cc36.json
Generated
+23
View File
@@ -0,0 +1,23 @@
{
"db_name": "PostgreSQL",
"query": "DELETE FROM eval_dataset WHERE workspace_id = $1 AND path = $2 RETURNING path",
"describe": {
"columns": [
{
"ordinal": 0,
"name": "path",
"type_info": "Varchar"
}
],
"parameters": {
"Left": [
"Text",
"Text"
]
},
"nullable": [
false
]
},
"hash": "e942a74104771b192c33ea03dbcaa409e03b6bfb7423929c201f39db4228cc36"
}
backend/.sqlx/query-eb0f25a10f4f1264e482674c06783204fddf0ccdca6d7c798a4ff6527548e96f.json
Generated
+20
View File
@@ -0,0 +1,20 @@
{
"db_name": "PostgreSQL",
"query": "INSERT INTO eval_experiment\n (id, workspace_id, dataset_path, subject, run_number, created_by, run_job_id)\n VALUES ($1, $2, $3, $4, $5, $6, $7)",
"describe": {
"columns": [],
"parameters": {
"Left": [
"Uuid",
"Varchar",
"Varchar",
"Jsonb",
"Int4",
"Varchar",
"Uuid"
]
},
"nullable": []
},
"hash": "eb0f25a10f4f1264e482674c06783204fddf0ccdca6d7c798a4ff6527548e96f"
}
backend/.sqlx/query-ee40e48afb5520b7ff84883204f064ce566cea6655e3cd11a9d7016cb08482cb.json
Generated
+21
View File
@@ -0,0 +1,21 @@
{
"db_name": "PostgreSQL",
"query": "UPDATE eval_score s\n SET score = t.score, reason = t.reason, checks = t.checks, error = t.error,\n not_applicable = t.not_applicable\n FROM UNNEST($2::int[], $3::text[], $4::double precision[], $5::text[], $6::jsonb[],\n $7::text[], $8::bool[])\n AS t(ordinal, scorer_id, score, reason, checks, error, not_applicable)\n WHERE s.experiment_id = $1 AND s.ordinal = t.ordinal AND s.scorer_id = t.scorer_id",
"describe": {
"columns": [],
"parameters": {
"Left": [
"Uuid",
"Int4Array",
"TextArray",
"Float8Array",
"TextArray",
"JsonbArray",
"TextArray",
"BoolArray"
]
},
"nullable": []
},
"hash": "ee40e48afb5520b7ff84883204f064ce566cea6655e3cd11a9d7016cb08482cb"
}
backend/.sqlx/query-f0e943244b125d0c42a9b472701ec172af9dd334786275c391e80f1c38bbb45b.json
Generated
+18
View File
@@ -0,0 +1,18 @@
{
"db_name": "PostgreSQL",
"query": "UPDATE eval_experiment_case c\n SET output = COALESCE(c.output, t.output), answered = COALESCE(c.answered, t.answered),\n status = COALESCE(c.status, t.status)\n FROM UNNEST($2::int[], $3::text[], $4::bool[], $5::text[])\n AS t(ordinal, output, answered, status)\n WHERE c.experiment_id = $1 AND c.ordinal = t.ordinal",
"describe": {
"columns": [],
"parameters": {
"Left": [
"Uuid",
"Int4Array",
"TextArray",
"BoolArray",
"TextArray"
]
},
"nullable": []
},
"hash": "f0e943244b125d0c42a9b472701ec172af9dd334786275c391e80f1c38bbb45b"
}
backend/.sqlx/query-f59afd524e3f216487ad0a780b1fda6341e215025dc6d8985891b91dad6d9dcb.json
Generated
+26
View File
@@ -0,0 +1,26 @@
{
"db_name": "PostgreSQL",
"query": "INSERT INTO eval_dataset\n (workspace_id, path, summary, scorers, created_by, edited_by)\n VALUES ($1, $2, $3, $4, $5, $5)\n ON CONFLICT (workspace_id, path) DO NOTHING\n RETURNING path",
"describe": {
"columns": [
{
"ordinal": 0,
"name": "path",
"type_info": "Varchar"
}
],
"parameters": {
"Left": [
"Varchar",
"Varchar",
"Varchar",
"Jsonb",
"Varchar"
]
},
"nullable": [
false
]
},
"hash": "f59afd524e3f216487ad0a780b1fda6341e215025dc6d8985891b91dad6d9dcb"
}
backend/.sqlx/query-f6c4c40b098ba06f4b3b057af1f5dce0711e90f7a4e1786d39a50d52400a3cb9.json
Generated
+16
View File
@@ -0,0 +1,16 @@
{
"db_name": "PostgreSQL",
"query": "UPDATE eval_dataset SET created_by = $1 WHERE created_by = $2 AND workspace_id = $3",
"describe": {
"columns": [],
"parameters": {
"Left": [
"Varchar",
"Text",
"Text"
]
},
"nullable": []
},
"hash": "f6c4c40b098ba06f4b3b057af1f5dce0711e90f7a4e1786d39a50d52400a3cb9"
}
backend/.sqlx/query-ef59abddc518f5213827e47a31aee49be917a61c46916c29d79c094438b1ff35.json → backend/.sqlx/query-f907114909b1064a3d5eb603e5929a6ffebdcadb20e4c6a68768a9ad42fe9328.json
Generated
+11 -5
View File
@@ -1,6 +1,6 @@
{
"db_name": "PostgreSQL",
"query": "SELECT id, path, created_at, created_by, value FROM resource_version\n WHERE workspace_id = $1 AND id = $2",
"query": "SELECT id, version, path, created_at, created_by, value FROM resource_version\n WHERE workspace_id = $1 AND id = $2",
"describe": {
"columns": [
{
@@ -10,21 +10,26 @@
},
{
"ordinal": 1,
"name": "version",
"type_info": "Int8"
},
{
"ordinal": 2,
"name": "path",
"type_info": "Varchar"
},
{
"ordinal": 2,
"ordinal": 3,
"name": "created_at",
"type_info": "Timestamptz"
},
{
"ordinal": 3,
"ordinal": 4,
"name": "created_by",
"type_info": "Varchar"
},
{
"ordinal": 4,
"ordinal": 5,
"name": "value",
"type_info": "Jsonb"
}
@@ -39,9 +44,10 @@
false,
false,
false,
false,
true,
true
]
},
"hash": "ef59abddc518f5213827e47a31aee49be917a61c46916c29d79c094438b1ff35"
"hash": "f907114909b1064a3d5eb603e5929a6ffebdcadb20e4c6a68768a9ad42fe9328"
}
backend/.sqlx/query-facbf7337d7ffa3f3e6287e2910ccd7ca7229f9d4e5c3af6c460ee7fe3c946e6.json
Generated
+23
View File
@@ -0,0 +1,23 @@
{
"db_name": "PostgreSQL",
"query": "SELECT path FROM eval_dataset WHERE workspace_id = $1 AND path = $2 FOR UPDATE",
"describe": {
"columns": [
{
"ordinal": 0,
"name": "path",
"type_info": "Varchar"
}
],
"parameters": {
"Left": [
"Text",
"Text"
]
},
"nullable": [
false
]
},
"hash": "facbf7337d7ffa3f3e6287e2910ccd7ca7229f9d4e5c3af6c460ee7fe3c946e6"
}
backend/.sqlx/query-fcf570337b2ceeb0f9dcc311144d8ada02aa4353feb2f63e7a1f520a4000ed70.json
Generated
+36
View File
@@ -0,0 +1,36 @@
{
"db_name": "PostgreSQL",
"query": "SELECT branch.parent_job AS \"case_job!\", scorer.flow_step_id AS \"module!\",\n done.result AS \"result: sqlx::types::Json<Box<RawValue>>\"\n FROM v2_job branch\n JOIN v2_job scorer ON scorer.parent_job = branch.id\n JOIN v2_job_completed done ON done.id = scorer.id\n WHERE branch.parent_job = ANY($1) AND branch.workspace_id = $2\n AND scorer.flow_step_id = ANY($3)",
"describe": {
"columns": [
{
"ordinal": 0,
"name": "case_job!",
"type_info": "Uuid"
},
{
"ordinal": 1,
"name": "module!",
"type_info": "Varchar"
},
{
"ordinal": 2,
"name": "result: sqlx::types::Json<Box<RawValue>>",
"type_info": "Jsonb"
}
],
"parameters": {
"Left": [
"UuidArray",
"Text",
"TextArray"
]
},
"nullable": [
true,
true,
true
]
},
"hash": "fcf570337b2ceeb0f9dcc311144d8ada02aa4353feb2f63e7a1f520a4000ed70"
}
backend/.sqlx/query-fd12bbe0605c80fced218a5a2e1288e6c04f4a1a76a2038679f8f538320cab1f.json
Generated
+24
View File
@@ -0,0 +1,24 @@
{
"db_name": "PostgreSQL",
"query": "SELECT path FROM eval_dataset\n WHERE scorers::text LIKE $1 AND NOT path LIKE $2 AND workspace_id = $3",
"describe": {
"columns": [
{
"ordinal": 0,
"name": "path",
"type_info": "Varchar"
}
],
"parameters": {
"Left": [
"Text",
"Text",
"Text"
]
},
"nullable": [
false
]
},
"hash": "fd12bbe0605c80fced218a5a2e1288e6c04f4a1a76a2038679f8f538320cab1f"
}
backend/migrations/20260812100659_ai_evals.down.sql
+6
View File
@@ -0,0 +1,6 @@
DROP TABLE IF EXISTS eval_score;
DROP TABLE IF EXISTS eval_experiment_case;
DROP TABLE IF EXISTS eval_experiment;
DROP TABLE IF EXISTS eval_case;
DROP TABLE IF EXISTS eval_dataset;
DROP FUNCTION IF EXISTS eval_dataset_writable(varchar, varchar);
backend/migrations/20260812100659_ai_evals.up.sql
+275
View File
@@ -0,0 +1,275 @@
-- Eval datasets and the cases they hold. Path-addressed like every other Windmill object, so the
-- folder a dataset is named by is what grants access to it.
CREATE TABLE eval_dataset (
workspace_id VARCHAR(50) NOT NULL,
path VARCHAR(255) NOT NULL,
summary VARCHAR(1000) NULL,
-- The scorers a dataset is scored by. One entry per column of the results table:
-- {id, name, kind, ...kind-specific config}. `id` is assigned once and never reused, so a
-- column stays the same column across experiments when it is renamed or its definition is
-- edited — which is what makes a delta between two experiments meaningful.
scorers JSONB NOT NULL DEFAULT '[]',
extra_perms JSONB NOT NULL DEFAULT '{}',
created_at TIMESTAMPTZ NOT NULL DEFAULT now(),
created_by VARCHAR(50) NOT NULL,
edited_at TIMESTAMPTZ NOT NULL DEFAULT now(),
edited_by VARCHAR(50) NOT NULL,
PRIMARY KEY (workspace_id, path),
FOREIGN KEY (workspace_id) REFERENCES workspace(id) ON DELETE CASCADE
);
-- A case is the input half of one evaluation: what the agent is fed, and what it was expected to
-- answer. The generated output, the trajectory and every scorer's return value are the job's, not
-- this table's.
--
-- ON UPDATE CASCADE so renaming a dataset carries its cases instead of stranding them.
CREATE TABLE eval_case (
id UUID PRIMARY KEY DEFAULT gen_random_uuid(),
workspace_id VARCHAR(50) NOT NULL,
dataset_path VARCHAR(255) NOT NULL,
-- {user_message, user_attachments}
input JSONB NOT NULL DEFAULT '{}',
expected JSONB NULL,
created_at TIMESTAMPTZ NOT NULL DEFAULT now(),
created_by VARCHAR(50) NOT NULL,
FOREIGN KEY (workspace_id, dataset_path) REFERENCES eval_dataset (workspace_id, path)
ON DELETE CASCADE ON UPDATE CASCADE
);
-- Serves the paginated case list, which is ordered oldest-first so a case keeps its position as
-- the dataset grows.
CREATE INDEX index_eval_case_dataset ON eval_case (workspace_id, dataset_path, created_at, id);
GRANT ALL ON eval_dataset TO windmill_user;
GRANT ALL ON eval_dataset TO windmill_admin;
GRANT ALL ON eval_case TO windmill_user;
GRANT ALL ON eval_case TO windmill_admin;
ALTER TABLE eval_dataset ENABLE ROW LEVEL SECURITY;
ALTER TABLE eval_case ENABLE ROW LEVEL SECURITY;
CREATE POLICY admin_policy ON eval_dataset FOR ALL TO windmill_admin USING (true);
CREATE POLICY admin_policy ON eval_case FOR ALL TO windmill_admin USING (true);
CREATE POLICY see_folder_extra_perms_user_select ON eval_dataset FOR SELECT TO windmill_user
USING (SPLIT_PART(eval_dataset.path, '/', 1) = 'f' AND SPLIT_PART(eval_dataset.path, '/', 2) = any((select regexp_split_to_array(current_setting('session.folders_read'), ','))::text[]));
CREATE POLICY see_folder_extra_perms_user_insert ON eval_dataset FOR INSERT TO windmill_user
WITH CHECK (SPLIT_PART(eval_dataset.path, '/', 1) = 'f' AND SPLIT_PART(eval_dataset.path, '/', 2) = any((select regexp_split_to_array(current_setting('session.folders_write'), ','))::text[]));
CREATE POLICY see_folder_extra_perms_user_update ON eval_dataset FOR UPDATE TO windmill_user
USING (SPLIT_PART(eval_dataset.path, '/', 1) = 'f' AND SPLIT_PART(eval_dataset.path, '/', 2) = any((select regexp_split_to_array(current_setting('session.folders_write'), ','))::text[]));
CREATE POLICY see_folder_extra_perms_user_delete ON eval_dataset FOR DELETE TO windmill_user
USING (SPLIT_PART(eval_dataset.path, '/', 1) = 'f' AND SPLIT_PART(eval_dataset.path, '/', 2) = any((select regexp_split_to_array(current_setting('session.folders_write'), ','))::text[]));
CREATE POLICY see_own ON eval_dataset FOR ALL TO windmill_user
USING (SPLIT_PART(eval_dataset.path, '/', 1) = 'u' AND SPLIT_PART(eval_dataset.path, '/', 2) = (select current_setting('session.user')));
CREATE POLICY see_member ON eval_dataset FOR ALL TO windmill_user
USING (SPLIT_PART(eval_dataset.path, '/', 1) = 'g' AND SPLIT_PART(eval_dataset.path, '/', 2) = any((select regexp_split_to_array(current_setting('session.groups'), ','))::text[]));
CREATE POLICY see_extra_perms_user_select ON eval_dataset FOR SELECT TO windmill_user
USING (extra_perms ? (select concat('u/', current_setting('session.user'))));
CREATE POLICY see_extra_perms_user_insert ON eval_dataset FOR INSERT TO windmill_user
WITH CHECK ((extra_perms ->> (select concat('u/', current_setting('session.user'))))::boolean);
CREATE POLICY see_extra_perms_user_update ON eval_dataset FOR UPDATE TO windmill_user
USING ((extra_perms ->> (select concat('u/', current_setting('session.user'))))::boolean);
CREATE POLICY see_extra_perms_user_delete ON eval_dataset FOR DELETE TO windmill_user
USING ((extra_perms ->> (select concat('u/', current_setting('session.user'))))::boolean);
CREATE POLICY see_extra_perms_groups_select ON eval_dataset FOR SELECT TO windmill_user
USING (extra_perms ?| (select regexp_split_to_array(current_setting('session.pgroups'), ','))::text[]);
CREATE POLICY see_extra_perms_groups_insert ON eval_dataset FOR INSERT TO windmill_user
WITH CHECK (exists(
SELECT key, value FROM jsonb_each_text(extra_perms)
WHERE SPLIT_PART(key, '/', 1) = 'g' AND key = ANY((select regexp_split_to_array(current_setting('session.pgroups'), ','))::text[])
AND value::boolean));
CREATE POLICY see_extra_perms_groups_update ON eval_dataset FOR UPDATE TO windmill_user
USING (exists(
SELECT key, value FROM jsonb_each_text(extra_perms)
WHERE SPLIT_PART(key, '/', 1) = 'g' AND key = ANY((select regexp_split_to_array(current_setting('session.pgroups'), ','))::text[])
AND value::boolean));
CREATE POLICY see_extra_perms_groups_delete ON eval_dataset FOR DELETE TO windmill_user
USING (exists(
SELECT key, value FROM jsonb_each_text(extra_perms)
WHERE SPLIT_PART(key, '/', 1) = 'g' AND key = ANY((select regexp_split_to_array(current_setting('session.pgroups'), ','))::text[])
AND value::boolean));
-- Whether the session may *write* the dataset at (_workspace_id, _path): the same disjunction the
-- dataset's own write policies use, in one place so the cases that hang off a dataset are governed
-- by exactly the rule the dataset is. A read grant is not enough — writing a case is writing the
-- dataset's contents — so this checks write, not merely visibility.
CREATE OR REPLACE FUNCTION eval_dataset_writable(_workspace_id varchar, _path varchar)
RETURNS boolean LANGUAGE sql STABLE AS $$
SELECT EXISTS (
SELECT 1 FROM eval_dataset d
WHERE d.workspace_id = _workspace_id AND d.path = _path
AND (
(SPLIT_PART(d.path, '/', 1) = 'f' AND SPLIT_PART(d.path, '/', 2) = any((select regexp_split_to_array(current_setting('session.folders_write'), ','))::text[]))
OR (SPLIT_PART(d.path, '/', 1) = 'u' AND SPLIT_PART(d.path, '/', 2) = (select current_setting('session.user')))
OR (SPLIT_PART(d.path, '/', 1) = 'g' AND SPLIT_PART(d.path, '/', 2) = any((select regexp_split_to_array(current_setting('session.groups'), ','))::text[]))
OR ((d.extra_perms ->> (select concat('u/', current_setting('session.user'))))::boolean)
OR EXISTS (
SELECT 1 FROM jsonb_each_text(d.extra_perms) ep
WHERE SPLIT_PART(ep.key, '/', 1) = 'g'
AND ep.key = ANY((select regexp_split_to_array(current_setting('session.pgroups'), ','))::text[])
AND ep.value::boolean)
)
);
$$;
-- Cases are the *contents* of a dataset, not independently addressable objects, so both their
-- visibility and who may change them are the parent's, stated once here instead of mirrored in the
-- API and left to drift. Read is the dataset's read (the subquery is itself subject to
-- eval_dataset's SELECT policies above); write is the dataset's write, which `eval_dataset_writable`
-- checks — so a read-only grant on a dataset can list its cases but not edit them. The whole edit
-- of a dataset and its cases therefore runs as one `user_db` transaction, governed by these
-- policies, rather than being split across the unrestricted pool after a hand-written check.
CREATE POLICY see_parent_dataset ON eval_case FOR SELECT TO windmill_user
USING (
EXISTS (
SELECT 1 FROM eval_dataset d
WHERE d.workspace_id = eval_case.workspace_id AND d.path = eval_case.dataset_path
)
);
CREATE POLICY write_parent_dataset_insert ON eval_case FOR INSERT TO windmill_user
WITH CHECK (eval_dataset_writable(eval_case.workspace_id, eval_case.dataset_path));
CREATE POLICY write_parent_dataset_update ON eval_case FOR UPDATE TO windmill_user
USING (eval_dataset_writable(eval_case.workspace_id, eval_case.dataset_path))
WITH CHECK (eval_dataset_writable(eval_case.workspace_id, eval_case.dataset_path));
CREATE POLICY write_parent_dataset_delete ON eval_case FOR DELETE TO windmill_user
USING (eval_dataset_writable(eval_case.workspace_id, eval_case.dataset_path));
-- One run of a dataset: written once when the dataset is run, and only ever read afterwards,
-- which is what makes it worth comparing against.
CREATE TABLE eval_experiment (
id UUID PRIMARY KEY,
workspace_id VARCHAR(50) NOT NULL,
dataset_path VARCHAR(255) NOT NULL,
-- {kind, path, version}: what was run, at the version it was at when the run was enqueued.
subject JSONB NOT NULL,
-- A run is named by the number it is: "Run 7" is stable, sorts, and survives history being
-- pruned, which a position computed at read time would not. Allocated per (dataset, subject
-- path) when the run is opened.
run_number INTEGER NOT NULL,
-- A run is one flow: a loop over the cases, each iteration answering and then scoring. This
-- is the job holding it, so the run can be watched, cancelled and rerun as the single thing
-- it is. Assigned before the flow is pushed, so a launch that dies partway leaves an
-- experiment naming a job that never started rather than a flow nothing accounts for.
run_job_id UUID NOT NULL,
created_at TIMESTAMPTZ NOT NULL DEFAULT now(),
created_by VARCHAR(50) NOT NULL,
FOREIGN KEY (workspace_id, dataset_path) REFERENCES eval_dataset (workspace_id, path)
ON DELETE CASCADE ON UPDATE CASCADE
);
CREATE INDEX index_eval_experiment_dataset ON eval_experiment (workspace_id, dataset_path, created_at DESC);
-- Serves the per-agent run list, which spans datasets: an agent's history is one list whichever
-- dataset each run was of.
CREATE INDEX index_eval_experiment_subject ON eval_experiment
(workspace_id, (subject ->> 'path'), created_at DESC);
-- The exact case set an experiment ran, by value: a dataset keeps changing, and a result set that
-- cannot say which inputs produced it is not reproducible. `case_id` is therefore deliberately not
-- a foreign key — deleting a case must not rewrite the history of the runs that used it.
CREATE TABLE eval_experiment_case (
experiment_id UUID NOT NULL REFERENCES eval_experiment (id) ON DELETE CASCADE,
ordinal INT NOT NULL,
case_id UUID NOT NULL,
input JSONB NOT NULL DEFAULT '{}',
expected JSONB NULL,
-- The iteration of the run's flow that answered this case. Minted by the flow engine, so the
-- case is recorded before it has one and the id is filled in once the iterations exist.
job_id UUID NULL,
-- What the run produced, copied out of the jobs once they have produced it. Jobs have their
-- own retention, and a recorded run has to still read as the run it was once they are gone.
-- `answered` is the agent step's own outcome, which is settled while the iteration around it
-- is still scoring; `status` is the iteration's, once it has one.
output TEXT NULL,
answered BOOLEAN NULL,
status VARCHAR(30) NULL,
-- The resource version the agent was at for this cell, and — for a draft, which has no
-- version to move — the hash of the configuration that actually ran: the only thing that can
-- say a row describes an agent that has since been edited.
subject_version BIGINT NULL,
subject_draft_hash VARCHAR(64) NULL,
started_at TIMESTAMPTZ NOT NULL DEFAULT now(),
PRIMARY KEY (experiment_id, ordinal),
-- A run holds each case once: the pair is what identifies a cell.
CONSTRAINT eval_experiment_case_unique_case UNIQUE (experiment_id, case_id)
);
-- One scorer's verdict on one run. Separate from the run because scoring is separate from running:
-- a scorer's verdict is stored per run and scorer, independent of the agent execution that
-- produced the answers.
CREATE TABLE eval_score (
experiment_id UUID NOT NULL,
ordinal INT NOT NULL,
scorer_id VARCHAR(64) NOT NULL,
-- NULL until the verdict has been read out of the run's flow, and when scoring failed.
score DOUBLE PRECISION NULL,
reason TEXT NULL,
-- [{name, passed, detail}], for scorers that report per-assertion results.
checks JSONB NULL,
error TEXT NULL,
-- The scorer read the run and said it had nothing to measure on this case. A verdict, not a
-- failure: the cell is left out of the column's mean and pass rate rather than counted as a
-- zero or reported as a scorer that produced nothing.
not_applicable BOOLEAN NOT NULL DEFAULT false,
-- Hash of the scorer configuration that produced this score, including the script hash or flow
-- version actually executed. Two scores of the same scorer whose definitions differ are still
-- compared, but the column says the scorer changed rather than letting it read as a change of
-- agent.
definition VARCHAR(64) NOT NULL,
created_at TIMESTAMPTZ NOT NULL DEFAULT now(),
PRIMARY KEY (experiment_id, ordinal, scorer_id),
FOREIGN KEY (experiment_id, ordinal) REFERENCES eval_experiment_case (experiment_id, ordinal)
ON DELETE CASCADE
);
GRANT ALL ON eval_experiment TO windmill_user;
GRANT ALL ON eval_experiment TO windmill_admin;
GRANT ALL ON eval_experiment_case TO windmill_user;
GRANT ALL ON eval_experiment_case TO windmill_admin;
GRANT ALL ON eval_score TO windmill_user;
GRANT ALL ON eval_score TO windmill_admin;
ALTER TABLE eval_experiment ENABLE ROW LEVEL SECURITY;
ALTER TABLE eval_experiment_case ENABLE ROW LEVEL SECURITY;
ALTER TABLE eval_score ENABLE ROW LEVEL SECURITY;
CREATE POLICY admin_policy ON eval_experiment FOR ALL TO windmill_admin USING (true);
CREATE POLICY admin_policy ON eval_experiment_case FOR ALL TO windmill_admin USING (true);
CREATE POLICY admin_policy ON eval_score FOR ALL TO windmill_admin USING (true);
-- Experiments are the *contents* of a dataset, not independently addressable objects, so their
-- visibility is the parent's: the subquery is itself subject to eval_dataset's policies, which
-- stay stated once instead of being mirrored here and left to drift.
--
-- SELECT only, deliberately. A `FOR ALL ... USING` would be reused as the INSERT/UPDATE/DELETE
-- check expression, and since the subquery is a SELECT it applies the dataset's *read* policies —
-- which would let someone with read-only access to a dataset forge an experiment row naming a job
-- they cannot otherwise read. Writes are done on the unrestricted pool after the API has checked
-- write access to the parent, and a stray `user_db` write to these tables is meant to fail rather
-- than silently succeed.
CREATE POLICY see_parent_dataset ON eval_experiment FOR SELECT TO windmill_user
USING (
EXISTS (
SELECT 1 FROM eval_dataset d
WHERE d.workspace_id = eval_experiment.workspace_id AND d.path = eval_experiment.dataset_path
)
);
CREATE POLICY see_parent_experiment ON eval_experiment_case FOR SELECT TO windmill_user
USING (
EXISTS (
SELECT 1 FROM eval_experiment e
WHERE e.id = eval_experiment_case.experiment_id
)
);
-- Visibility is the experiment's, which is the dataset's.
CREATE POLICY see_parent_experiment ON eval_score FOR SELECT TO windmill_user
USING (
EXISTS (
SELECT 1 FROM eval_experiment e
WHERE e.id = eval_score.experiment_id
)
);
backend/migrations/20260813164338_ai_token_usage.down.sql
+1
View File
@@ -0,0 +1 @@
DROP TABLE ai_token_usage;
backend/migrations/20260813164338_ai_token_usage.up.sql
+42
View File
@@ -0,0 +1,42 @@
-- Per-workspace AI token spend, accumulated from the chat client. Rows hold token
-- counts rather than money: prices live in the frontend price table plus the
-- workspace's `ai_config.model_pricing` overrides and are applied at read time, so
-- correcting a price also corrects the history. `reported_cost_nano_usd` is the
-- exception — a few providers (OpenRouter) return what they actually charged, and
-- that figure wins over the estimate.
--
-- Distinct from `feature_usage`, which is anonymous telemetry that leaves the
-- instance and is pruned after 60 days; spend is per-user and kept.
CREATE TABLE ai_token_usage (
workspace_id VARCHAR(50) NOT NULL REFERENCES workspace(id) ON DELETE CASCADE,
day DATE NOT NULL DEFAULT CURRENT_DATE,
email VARCHAR(255) NOT NULL,
provider VARCHAR(50) NOT NULL,
model VARCHAR(255) NOT NULL,
-- Empty for chats that are not attached to an AI session.
session_id VARCHAR(50) NOT NULL DEFAULT '',
-- Uncached input only; the two cache columns hold the rest of the prompt, so
-- each column maps to exactly one price and they never double-count.
input_tokens BIGINT NOT NULL DEFAULT 0,
cache_read_tokens BIGINT NOT NULL DEFAULT 0,
cache_write_tokens BIGINT NOT NULL DEFAULT 0,
output_tokens BIGINT NOT NULL DEFAULT 0,
reported_cost_nano_usd BIGINT,
requests BIGINT NOT NULL DEFAULT 0,
updated_at TIMESTAMPTZ NOT NULL DEFAULT now(),
PRIMARY KEY (workspace_id, day, email, provider, model, session_id)
);
-- The usage listing filters on workspace and a date range; the PK only reaches
-- `day` through `email`, so it cannot serve that on its own.
CREATE INDEX idx_ai_token_usage_ws_day ON ai_token_usage (workspace_id, day DESC);
GRANT ALL ON ai_token_usage TO windmill_admin;
GRANT ALL ON ai_token_usage TO windmill_user;
-- Both handlers go through the raw pool, so no policy is needed for them to work.
-- Enabling RLS with an admin-only policy is the backstop: a future query that
-- reaches this table through UserDB sees nothing rather than every user's spend.
ALTER TABLE ai_token_usage ENABLE ROW LEVEL SECURITY;
CREATE POLICY admin_policy ON ai_token_usage FOR ALL TO windmill_admin USING (true);
backend/migrations/20260819073729_resource_version_number.down.sql
+17
View File
@@ -0,0 +1,17 @@
-- Back to numbering versions by the table-wide identity sequence, so the function must stop
-- writing a column that is about to go.
CREATE OR REPLACE FUNCTION record_resource_version() RETURNS trigger AS $$
BEGIN
INSERT INTO resource_version (workspace_id, path, resource_type, value, created_by)
VALUES (
NEW.workspace_id, NEW.path, NEW.resource_type, NEW.value,
COALESCE(NULLIF(current_setting('session.user', true), ''), NEW.created_by)
);
RETURN NEW;
END;
$$ LANGUAGE plpgsql SECURITY DEFINER SET search_path FROM CURRENT;
DROP INDEX IF EXISTS index_resource_version_number;
ALTER TABLE resource_version DROP COLUMN version;
backend/migrations/20260819073729_resource_version_number.up.sql
+52
View File
@@ -0,0 +1,52 @@
-- `id` is one identity sequence for the whole table and stays how a version is addressed;
-- `version` is the resource's own count, which is what a version is presented by.
ALTER TABLE resource_version ADD COLUMN version BIGINT;
UPDATE resource_version rv SET version = ranked.rn
FROM (
SELECT id, row_number() OVER (PARTITION BY workspace_id, path ORDER BY id) AS rn
FROM resource_version
) ranked
WHERE rv.id = ranked.id;
ALTER TABLE resource_version ALTER COLUMN version SET NOT NULL;
-- The number is only meaningful within a path, so the triple is the natural key: it serves the
-- lookup by number and makes a duplicate a hard error rather than two rows claiming v7.
CREATE UNIQUE INDEX index_resource_version_number ON resource_version (workspace_id, path, version);
-- Numbering is assigned here rather than derived when read because both ways of deleting versions
-- take the oldest ones: the monitor's trim past MAX_RESOURCE_VERSIONS, and clearing a history down
-- to its current value. A number computed by counting the survivors would renumber under either,
-- so a run recorded against v3 would later name a different version.
CREATE OR REPLACE FUNCTION record_resource_version() RETURNS trigger AS $$
BEGIN
-- `session.user` is set by UserDB::begin for authed requests; worker and system writes fall
-- back to the row's own author. NULLIF because a transaction-local set_config resets the
-- placeholder to the empty string rather than unsetting it, so a pooled connection that
-- previously served an authed request reports '' here, not NULL.
--
-- MAX + 1 needs no lock of its own: this runs inside the transaction that wrote `resource`, and
-- a concurrent write to the same path blocks on that row's lock — or on the primary key, for an
-- insert — before its own trigger can run, so the maximum cannot be read stale. Deleting
-- versions never lowers it, since both deletions keep the newest row.
INSERT INTO resource_version (workspace_id, path, resource_type, value, created_by, version)
VALUES (
NEW.workspace_id, NEW.path, NEW.resource_type, NEW.value,
COALESCE(NULLIF(current_setting('session.user', true), ''), NEW.created_by),
(SELECT COALESCE(MAX(version), 0) + 1 FROM resource_version
WHERE workspace_id = NEW.workspace_id AND path = NEW.path)
);
-- The per-path cap is enforced by trim_resource_versions in the monitor, not here: trimming
-- on every write would tax a path `setResource` can drive in a loop, to keep a bound that
-- does not need to hold instantaneously.
RETURN NEW;
END;
-- SECURITY DEFINER so history is written on behalf of every writer without granting anyone direct
-- write access to the table, which users hold SELECT on only. `SET search_path FROM CURRENT` is the
-- injection hardening that goes with it, captured rather than hardcoded so installs running a
-- non-public PG_SCHEMA still resolve (see
-- 20260624103600_repair_folder_labels_search_path.up.sql).
$$ LANGUAGE plpgsql SECURITY DEFINER SET search_path FROM CURRENT;
backend/summarized_schema.txt
+2
View File
@@ -40,6 +40,8 @@ agent_token_blacklist: token(char), expires_at(ts), blacklisted_at(ts), blacklis
ai_agent_memory: workspace_id(char), conversation_id(uuid), step_id(char), messages(jsonb), created_at(ts), updated_at(ts)
ai_free_token_daily_usage: day(date), cost_nanos(bigint), updated_at(ts)
ai_free_token_usage: email(char), cost_nanos(bigint), updated_at(ts)
ai_token_usage: workspace_id(char), day(date), email(char), provider(char), model(char), session_id(char), input_tokens(bigint), cache_read_tokens(bigint), cache_write_tokens(bigint), output_tokens(bigint), reported_cost_nano_usd(bigint), requests(bigint), updated_at(ts)
FK: (workspace_id) -> workspace(id)
alerts: id(int), alert_type(char), message(text), created_at(ts), acknowledged(bool), workspace_id(text), acknowledged_workspace(bool), resource(text)
app: id(bigint), workspace_id(char), path(char), summary(char), policy(jsonb), versions(bigint[]), extra_perms(jsonb), draft_only(bool), custom_path(text), labels(text[])
FK: (workspace_id) -> workspace(id)
backend/windmill-ai/src/ai_google.rs
+119 -5
View File
@@ -277,7 +277,7 @@ pub struct GeminiSSECandidate {
}
/// Token usage from the `usageMetadata` field of a Gemini SSE event.
#[derive(Deserialize, Debug, Clone)]
#[derive(Deserialize, Debug, Clone, Default)]
pub struct GeminiUsageMetadata {
#[serde(rename = "promptTokenCount", default)]
pub prompt_token_count: Option<i32>,
@@ -285,6 +285,39 @@ pub struct GeminiUsageMetadata {
pub candidates_token_count: Option<i32>,
#[serde(rename = "totalTokenCount", default)]
pub total_token_count: Option<i32>,
/// Subset of `promptTokenCount` served from context cache, billed at a reduced
/// rate. Reported separately so the client can price it separately.
#[serde(rename = "cachedContentTokenCount", default)]
pub cached_content_token_count: Option<i32>,
/// Thinking tokens, billed as output but counted apart from `candidatesTokenCount`.
#[serde(rename = "thoughtsTokenCount", default)]
pub thoughts_token_count: Option<i32>,
/// Input tokens spent on tool-use prompts, counted apart from `promptTokenCount`
/// rather than within it.
#[serde(rename = "toolUsePromptTokenCount", default)]
pub tool_use_prompt_token_count: Option<i32>,
}
/// Input tokens as billed. Gemini reports tool-use prompts in their own field, and
/// they are disjoint from `promptTokenCount`: a live tool call returns 17 prompt +
/// 60 tool-use + 17 candidates + 52 thoughts against a `totalTokenCount` of 146, so
/// leaving them out under-reports the input of every tool-using turn. Cached tokens
/// are not added here, being already part of `promptTokenCount`.
fn gemini_prompt_tokens(usage: &GeminiUsageMetadata) -> i32 {
usage
.prompt_token_count
.unwrap_or(0)
.saturating_add(usage.tool_use_prompt_token_count.unwrap_or(0))
}
/// Output tokens as billed: Gemini counts thinking apart from `candidatesTokenCount`
/// but charges it at the output rate, so a reply that thought would otherwise be
/// reported as far cheaper than it was.
fn gemini_completion_tokens(usage: &GeminiUsageMetadata) -> i32 {
usage
.candidates_token_count
.unwrap_or(0)
.saturating_add(usage.thoughts_token_count.unwrap_or(0))
}
/// Top-level structure of one Gemini SSE event.
@@ -588,9 +621,12 @@ pub fn gemini_response_to_openai(parsed: &GeminiParsedEvent, model: &str) -> ser
let usage = parsed.usage.as_ref().map(|u| {
serde_json::json!({
"prompt_tokens": u.prompt_token_count.unwrap_or(0),
"completion_tokens": u.candidates_token_count.unwrap_or(0),
"prompt_tokens": gemini_prompt_tokens(u),
"completion_tokens": gemini_completion_tokens(u),
"total_tokens": u.total_token_count.unwrap_or(0),
"prompt_tokens_details": {
"cached_tokens": u.cached_content_token_count.unwrap_or(0)
},
})
});
@@ -680,8 +716,8 @@ pub fn gemini_event_to_openai_sse_chunks(
// OpenAI's `stream_options.include_usage` terminal chunk (top-level `usage`,
// empty `choices`) so the frontend's `'usage' in chunk` path records them.
if let Some(usage) = &parsed.usage {
let prompt_tokens = usage.prompt_token_count.unwrap_or(0);
let completion_tokens = usage.candidates_token_count.unwrap_or(0);
let prompt_tokens = gemini_prompt_tokens(usage);
let completion_tokens = gemini_completion_tokens(usage);
let total_tokens = usage
.total_token_count
.unwrap_or(prompt_tokens + completion_tokens);
@@ -694,6 +730,9 @@ pub fn gemini_event_to_openai_sse_chunks(
"prompt_tokens": prompt_tokens,
"completion_tokens": completion_tokens,
"total_tokens": total_tokens,
"prompt_tokens_details": {
"cached_tokens": usage.cached_content_token_count.unwrap_or(0)
},
}
});
chunks.push(format!("data: {}\n\n", chunk));
@@ -943,6 +982,7 @@ mod tests {
prompt_token_count: Some(12),
candidates_token_count: Some(7),
total_token_count: Some(19),
..Default::default()
}),
..Default::default()
};
@@ -969,6 +1009,79 @@ mod tests {
assert_eq!(usage_chunk["choices"], serde_json::json!([]));
}
#[test]
fn gemini_usage_chunk_splits_cached_and_bills_thoughts() {
let parsed = GeminiParsedEvent {
text: Some("the answer".to_string()),
usage: Some(GeminiUsageMetadata {
prompt_token_count: Some(1000),
candidates_token_count: Some(20),
total_token_count: Some(1120),
cached_content_token_count: Some(900),
thoughts_token_count: Some(100),
..Default::default()
}),
..Default::default()
};
let mut tool_call_index = 0;
let chunks = gemini_event_to_openai_sse_chunks(
&parsed,
"chatcmpl-test",
"gemini-3-flash-preview",
&mut tool_call_index,
);
let usage_chunk = chunks
.iter()
.map(|c| parse_sse_chunk(c))
.find(|v| v.get("usage").map(|u| !u.is_null()).unwrap_or(false))
.expect("a chunk should carry top-level usage");
// Gemini's prompt count already includes the cached tokens, so it passes
// through unchanged and the cached share is reported alongside it; thinking
// is billed as output but counted apart from the candidates.
assert_eq!(usage_chunk["usage"]["prompt_tokens"], 1000);
assert_eq!(usage_chunk["usage"]["prompt_tokens_details"]["cached_tokens"], 900);
assert_eq!(usage_chunk["usage"]["completion_tokens"], 120);
}
#[test]
fn gemini_usage_chunk_counts_tool_use_prompt_tokens() {
let parsed = GeminiParsedEvent {
text: Some("Canberra".to_string()),
usage: Some(GeminiUsageMetadata {
prompt_token_count: Some(17),
candidates_token_count: Some(17),
total_token_count: Some(146),
tool_use_prompt_token_count: Some(60),
thoughts_token_count: Some(52),
..Default::default()
}),
..Default::default()
};
let mut tool_call_index = 0;
let chunks = gemini_event_to_openai_sse_chunks(
&parsed,
"chatcmpl-test",
"gemini-2.5-flash",
&mut tool_call_index,
);
let usage_chunk = chunks
.iter()
.map(|c| parse_sse_chunk(c))
.find(|v| v.get("usage").map(|u| !u.is_null()).unwrap_or(false))
.expect("a chunk should carry top-level usage");
assert_eq!(usage_chunk["usage"]["prompt_tokens"], 77);
assert_eq!(usage_chunk["usage"]["completion_tokens"], 69);
assert_eq!(
usage_chunk["usage"]["prompt_tokens"].as_i64().unwrap()
+ usage_chunk["usage"]["completion_tokens"].as_i64().unwrap(),
146
);
}
#[test]
fn gemini_streaming_usage_total_falls_back_to_prompt_plus_completion() {
let parsed = GeminiParsedEvent {
@@ -976,6 +1089,7 @@ mod tests {
prompt_token_count: Some(5),
candidates_token_count: Some(3),
total_token_count: None,
..Default::default()
}),
..Default::default()
};
backend/windmill-ai/src/ai_types.rs
+54
View File
@@ -175,3 +175,57 @@ pub struct OpenAIMessage {
#[serde(skip_serializing_if = "Option::is_none")]
pub annotations: Option<Vec<UrlCitation>>,
}
// ============================================================================
// Model pricing
// ============================================================================
/// Far above any real per-million-token rate, so a value beyond it is a unit
/// mistake rather than a price. The floor matters more: a negative rate would make
/// spend subtract, and NaN/infinity would poison every total derived from it.
pub const MAX_MODEL_RATE: f64 = 1000.0;
/// Bound the `model_pricing` map of an AI config that is only available untyped —
/// the instance config is stored through the generic global-settings endpoint,
/// which never deserializes it into `AIConfig`, so the typed check on the
/// workspace path does not cover it.
pub fn validate_model_pricing_json(ai_config: &serde_json::Value) -> Result<(), String> {
// The container itself has to be checked too: a non-object `ai_config` persists
// here and then fails to deserialize as `AIConfig`, which drops the whole
// instance config back to its default for every workspace inheriting it.
if !ai_config.is_null() && !ai_config.is_object() {
return Err("ai_config must be an object".to_string());
}
let pricing = match ai_config.get("model_pricing") {
None | Some(serde_json::Value::Null) => return Ok(()),
// A present-but-wrong shape must be rejected, not skipped: it would persist
// and then fail to deserialize as `AIConfig`, which silently drops the whole
// instance config back to its default for every workspace inheriting it.
Some(v) => v
.as_object()
.ok_or_else(|| "model_pricing must be an object".to_string())?,
};
for (key, price) in pricing {
let Some(price) = price.as_object() else {
return Err(format!("Price override for {} is not an object", key));
};
for field in ["input", "output", "cache_read", "cache_write"] {
let Some(rate) = price.get(field) else { continue };
let rate = rate
.as_f64()
.filter(|r| r.is_finite() && *r >= 0.0 && *r <= MAX_MODEL_RATE);
if rate.is_none() {
return Err(format!(
"Price override for {}: {} must be between 0 and {}",
key, field, MAX_MODEL_RATE
));
}
}
for required in ["input", "output"] {
if !price.contains_key(required) {
return Err(format!("Price override for {} is missing {}", key, required));
}
}
}
Ok(())
}
backend/windmill-ai/src/providers/bedrock.rs
+72
View File
@@ -660,6 +660,41 @@ fn bedrock_sse_chunks_for_event(
chunks.push(Bytes::from(format!("data: {}\n\n", chunk)));
}
// Usage arrives only on the trailing Metadata event, and only this converter
// reaches the chat: without a chunk for it a Bedrock chat reports no tokens at
// all. Bedrock counts cache reads and writes apart from `inputTokens`, while the
// OpenAI shape the client parses treats `prompt_tokens` as the whole input, so
// they are folded in here and split back out through `prompt_tokens_details`.
if let aws_sdk_bedrockruntime::types::ConverseStreamOutput::Metadata(metadata) = event {
if let Some(token_usage) = metadata.usage() {
let cache_read = token_usage.cache_read_input_tokens().unwrap_or(0);
let cache_write = token_usage.cache_write_input_tokens().unwrap_or(0);
let prompt_tokens = token_usage
.input_tokens()
.saturating_add(cache_read)
.saturating_add(cache_write);
let chunk = serde_json::json!({
"id": state.id,
"object": "chat.completion.chunk",
"created": state.created,
"model": state.model,
"choices": [],
"usage": {
"prompt_tokens": prompt_tokens,
"completion_tokens": token_usage.output_tokens(),
"total_tokens": token_usage.total_tokens(),
"prompt_tokens_details": {
"cached_tokens": cache_read,
"cache_write_tokens": cache_write
}
}
});
chunks.push(Bytes::from(format!("data: {}\n\n", chunk)));
}
}
chunks
}
@@ -1190,6 +1225,43 @@ mod tests {
serde_json::from_str(payload).expect("chunk should contain JSON")
}
#[test]
fn metadata_event_emits_usage_chunk_with_cache_split() {
let mut state = BedrockSseStreamState::new("id".to_string(), "model".to_string(), 0);
let event = ConverseStreamOutput::Metadata(
aws_sdk_bedrockruntime::types::ConverseStreamMetadataEvent::builder()
.usage(
aws_sdk_bedrockruntime::types::TokenUsage::builder()
.input_tokens(10)
.output_tokens(7)
.total_tokens(1017)
.cache_read_input_tokens(900)
.cache_write_input_tokens(100)
.build()
.expect("usage"),
)
.build(),
);
let chunks = bedrock_sse_chunks_for_event(&event, &mut state);
let usage = chunks
.iter()
.map(sse_json)
.find(|v| v.get("usage").map(|u| !u.is_null()).unwrap_or(false))
.expect("the metadata event should carry usage");
// Bedrock reports cache reads and writes apart from `inputTokens`; the OpenAI
// shape the client parses treats `prompt_tokens` as the whole input, and
// recovers the uncached share by subtracting the details back out.
assert_eq!(usage["usage"]["prompt_tokens"], 1010);
assert_eq!(usage["usage"]["completion_tokens"], 7);
assert_eq!(usage["usage"]["prompt_tokens_details"]["cached_tokens"], 900);
assert_eq!(
usage["usage"]["prompt_tokens_details"]["cache_write_tokens"],
100
);
}
#[test]
fn determine_auth_config_prioritizes_bearer_token() {
let config = determine_auth_config(
backend/windmill-api-auth/src/scopes.rs
+3
View File
@@ -289,6 +289,7 @@ pub enum ScopeDomain {
OAuth,
AI,
AiSkills,
AiEvals, // AI agent eval datasets
Indexer,
Teams, // Microsoft Teams integration
@@ -349,6 +350,7 @@ impl ScopeDomain {
Self::OAuth => "oauth",
Self::AI => "ai",
Self::AiSkills => "ai_skills",
Self::AiEvals => "ai_evals",
Self::Capture => "capture",
Self::Drafts => "drafts",
Self::Favorites => "favorites",
@@ -404,6 +406,7 @@ impl ScopeDomain {
"oauth" => Some(Self::OAuth),
"ai" => Some(Self::AI),
"ai_skills" => Some(Self::AiSkills),
"ai_evals" => Some(Self::AiEvals),
"indexer" | "srch" => Some(Self::Indexer),
"teams" => Some(Self::Teams),
"native_triggers" => Some(Self::NativeTriggers),
backend/windmill-api-settings/src/lib.rs
+7
View File
@@ -891,6 +891,13 @@ async fn run_setting_pre_write_hook(
value: &serde_json::Value,
) -> error::Result<()> {
match key {
// The instance AI config is written as an untyped blob through this generic
// endpoint, so it never passes the typed check the workspace handler applies.
// Rates that reach a cost total unbounded would make it negative or infinite.
AI_CONFIG_SETTING => {
windmill_ai::ai_types::validate_model_pricing_json(value)
.map_err(error::Error::BadRequest)?;
}
AUTOMATE_USERNAME_CREATION_SETTING => {
if value.as_bool().unwrap_or(false) {
generate_instance_username_for_all_users(db)
backend/windmill-api-users/src/users.rs
+1
View File
@@ -2399,6 +2399,7 @@ pub async fn delete_workspace_user_internal(
"flow",
"app",
"resource",
"eval_dataset",
"variable",
"schedule",
"group_",
backend/windmill-api-workspaces/src/workspaces.rs
+32
View File
@@ -5471,6 +5471,8 @@ async fn clone_workspace_data(
// Clone scripts with new hashes
clone_scripts(tx, source_workspace_id, target_workspace_id).await?;
clone_eval_datasets(tx, source_workspace_id, target_workspace_id).await?;
// Clone the dbt graph sidecars. After `clone_scripts`, which keeps each
// script's hash: these key on it, and a static descriptor never re-ingests,
// so a fork without them shows dbt scripts with no models until someone
@@ -6005,6 +6007,36 @@ async fn clone_resources(
Ok(())
}
async fn clone_eval_datasets(
tx: &mut Transaction<'_, Postgres>,
source_workspace_id: &str,
target_workspace_id: &str,
) -> Result<()> {
// The authored evaluation data — datasets and their cases — travels with a fork like resources
// and scripts do; the runs (experiments) do not, since they name jobs the fork has no copy of.
sqlx::query!(
"INSERT INTO eval_dataset (workspace_id, path, summary, scorers, extra_perms, created_at, created_by, edited_at, edited_by)
SELECT $2, path, summary, scorers, extra_perms, created_at, created_by, edited_at, edited_by
FROM eval_dataset WHERE workspace_id = $1",
source_workspace_id,
target_workspace_id,
)
.execute(&mut **tx)
.await?;
// A new id per cloned case: `eval_case`'s primary key is the id alone, unique across the whole
// table, so copying it would collide with the source's own rows.
sqlx::query!(
"INSERT INTO eval_case (workspace_id, dataset_path, input, expected, created_at, created_by)
SELECT $2, dataset_path, input, expected, created_at, created_by
FROM eval_case WHERE workspace_id = $1",
source_workspace_id,
target_workspace_id,
)
.execute(&mut **tx)
.await?;
Ok(())
}
async fn clone_variables(
tx: &mut Transaction<'_, Postgres>,
db: &DB,
backend/windmill-api/openapi.yaml
+954 -4
View File
File diff suppressed because it is too large Load Diff
backend/windmill-api/src/ai.rs
+326 -5
View File
@@ -3,11 +3,16 @@ use crate::utils::check_scopes;
#[cfg(feature = "bedrock")]
use axum::routing::get;
#[cfg(feature = "bedrock")]
use axum::Json;
use axum::{body::Bytes, extract::Path, response::IntoResponse, routing::post, Extension, Router};
use axum::{
body::Bytes,
extract::{DefaultBodyLimit, Path, Query},
response::IntoResponse,
routing::post,
Extension, Router,
};
use futures::StreamExt;
use http::{HeaderMap, Method};
use http::{HeaderMap, Method, StatusCode};
use quick_cache::sync::Cache;
use reqwest::{Client, RequestBuilder};
use serde::{Deserialize, Serialize};
@@ -18,6 +23,7 @@ use windmill_ai::ai_cache::current_instance_ai_config_revision;
use windmill_ai::ai_providers::{
empty_string_as_none, AIPlatform, AIProvider, ProviderConfig, ProviderModel,
};
use windmill_ai::ai_types::MAX_MODEL_RATE;
use windmill_ai::credentials::ProviderCredentials;
#[cfg(feature = "bedrock")]
use windmill_ai::providers::bedrock::{
@@ -37,7 +43,7 @@ use windmill_ai::proxy::{
use windmill_audit::{audit_oss::audit_log, ActionKind};
use windmill_common::db::UserDB;
use windmill_common::error::{to_anyhow, Error, Result};
use windmill_common::utils::configure_client;
use windmill_common::utils::{configure_client, require_admin};
use windmill_common::variables::{get_variable_or_self, get_variable_or_self_as};
// AI timeout configuration constants
@@ -435,9 +441,54 @@ pub struct AIConfig {
/// free-tier marker. Only the server sets it, per-request.
#[serde(skip_serializing_if = "Option::is_none", skip_deserializing)]
pub free_tier: Option<FreeTierInfo>,
/// Per-model price overrides, keyed `provider:model` like `max_tokens_per_model`.
/// Only models whose rates differ from the built-in table are stored.
#[serde(skip_serializing_if = "Option::is_none")]
pub model_pricing: Option<HashMap<String, ModelPriceOverride>>,
}
/// Negotiated rates in USD per million tokens. An unset cache rate is read as the
/// provider's own multiple of the input rate where the model has a published one,
/// and as the input rate itself where it does not — an unstated discount is never
/// filled in from another vendor's.
#[derive(Serialize, Deserialize, Debug, Clone)]
pub struct ModelPriceOverride {
pub input: f64,
pub output: f64,
#[serde(skip_serializing_if = "Option::is_none")]
pub cache_read: Option<f64>,
#[serde(skip_serializing_if = "Option::is_none")]
pub cache_write: Option<f64>,
}
impl ModelPriceOverride {
pub fn validate(&self, key: &str) -> Result<()> {
for (field, rate) in [
("input", Some(self.input)),
("output", Some(self.output)),
("cache_read", self.cache_read),
("cache_write", self.cache_write),
] {
let Some(rate) = rate else { continue };
if !rate.is_finite() || rate < 0.0 || rate > MAX_MODEL_RATE {
return Err(Error::BadRequest(format!(
"Price override for {}: {} must be between 0 and {}",
key, field, MAX_MODEL_RATE
)));
}
}
Ok(())
}
}
impl AIConfig {
pub fn validate_model_pricing(&self) -> Result<()> {
for (key, price) in self.model_pricing.iter().flatten() {
price.validate(key)?;
}
Ok(())
}
pub fn has_providers(&self) -> bool {
self.providers
.as_ref()
@@ -450,7 +501,18 @@ pub fn global_service() -> Router {
}
pub fn workspaced_service() -> Router {
let router = Router::new().route("/proxy/{*ai}", post(proxy).get(proxy));
let router = Router::new()
.route("/proxy/{*ai}", post(proxy).get(proxy))
.route(
"/usage",
post(record_ai_usage)
.get(list_ai_usage)
// The handler caps how many events it *stores*, but Json deserializes
// the whole array first — without a body limit an authenticated member
// could make the server allocate and parse an arbitrarily large one.
// Sized well above a full batch of the shape below.
.layer(DefaultBodyLimit::max(AI_USAGE_BODY_LIMIT)),
);
#[cfg(feature = "bedrock")]
let router = router.route("/check_bedrock_credentials", get(check_bedrock_credentials));
@@ -458,6 +520,265 @@ pub fn workspaced_service() -> Router {
router
}
/// One provider request's worth of tokens, as counted by the chat client.
#[derive(Deserialize)]
struct AIUsageEvent {
provider: String,
model: String,
#[serde(default)]
session_id: String,
#[serde(default)]
input_tokens: i64,
#[serde(default)]
cache_read_tokens: i64,
#[serde(default)]
cache_write_tokens: i64,
#[serde(default)]
output_tokens: i64,
/// Only the providers that bill back an exact figure set this.
#[serde(default)]
reported_cost_nano_usd: Option<i64>,
#[serde(default)]
requests: Option<i64>,
}
#[derive(Deserialize)]
struct RecordAIUsagePayload {
events: Vec<AIUsageEvent>,
}
const MAX_AI_USAGE_EVENTS: usize = 50;
/// 64 KiB — a 50-event batch is a few kB even with the longest model ids.
const AI_USAGE_BODY_LIMIT: usize = 64 * 1024;
/// Well above any single conversation and far below an i64 overflow, so a client
/// bug caps out at one absurd row instead of poisoning the running total.
const MAX_TOKENS_PER_EVENT: i64 = 100_000_000;
/// $1000 in nano-USD.
const MAX_REPORTED_COST_PER_EVENT: i64 = 1_000_000_000_000;
/// Model ids carry vendor prefixes and variant suffixes (`anthropic/claude-opus-5:thinking`),
/// so the shape check is looser than an identifier but still excludes whitespace and
/// anything that would not be a model id.
fn is_model_shaped(s: &str, max_len: usize) -> bool {
!s.is_empty()
&& s.len() <= max_len
&& s.chars()
.all(|c| c.is_ascii_alphanumeric() || matches!(c, '_' | '-' | ':' | '.' | '/' | '~'))
}
/// Accumulate one workspace's AI token spend. Values are clamped and the caller's
/// email comes from the session, never the payload — the client is trusted to
/// report its own usage, not to attribute it to someone else.
async fn record_ai_usage(
authed: ApiAuthed,
Extension(db): Extension<DB>,
Path(w_id): Path<String>,
Json(payload): Json<RecordAIUsagePayload>,
) -> Result<StatusCode> {
// Pre-sum duplicate keys: two rows hitting the same conflict target in a single
// INSERT error out ("cannot affect row a second time").
let mut agg: HashMap<(String, String, String), AIUsageTotals> = HashMap::new();
for e in payload.events.into_iter().take(MAX_AI_USAGE_EVENTS) {
if AIProvider::try_from(e.provider.as_str()).is_err()
|| !is_model_shaped(&e.model, 255)
|| !(e.session_id.is_empty() || is_model_shaped(&e.session_id, 50))
{
continue;
}
let totals = agg
.entry((e.provider, e.model, e.session_id))
.or_insert_with(AIUsageTotals::default);
totals.input += e.input_tokens.clamp(0, MAX_TOKENS_PER_EVENT);
totals.cache_read += e.cache_read_tokens.clamp(0, MAX_TOKENS_PER_EVENT);
totals.cache_write += e.cache_write_tokens.clamp(0, MAX_TOKENS_PER_EVENT);
totals.output += e.output_tokens.clamp(0, MAX_TOKENS_PER_EVENT);
totals.requests += e.requests.unwrap_or(1).clamp(0, MAX_AI_USAGE_EVENTS as i64);
if let Some(cost) = e.reported_cost_nano_usd {
totals.reported_cost = Some(
totals.reported_cost.unwrap_or(0) + cost.clamp(0, MAX_REPORTED_COST_PER_EVENT),
);
}
}
if agg.is_empty() {
return Ok(StatusCode::NO_CONTENT);
}
let mut providers = Vec::with_capacity(agg.len());
let mut models = Vec::with_capacity(agg.len());
let mut session_ids = Vec::with_capacity(agg.len());
let mut inputs = Vec::with_capacity(agg.len());
let mut cache_reads = Vec::with_capacity(agg.len());
let mut cache_writes = Vec::with_capacity(agg.len());
let mut outputs = Vec::with_capacity(agg.len());
let mut reported_costs: Vec<Option<i64>> = Vec::with_capacity(agg.len());
let mut requests = Vec::with_capacity(agg.len());
for ((provider, model, session_id), totals) in agg {
providers.push(provider);
models.push(model);
session_ids.push(session_id);
inputs.push(totals.input);
cache_reads.push(totals.cache_read);
cache_writes.push(totals.cache_write);
outputs.push(totals.output);
reported_costs.push(totals.reported_cost);
requests.push(totals.requests);
}
sqlx::query!(
"INSERT INTO ai_token_usage (workspace_id, email, provider, model, session_id, \
input_tokens, cache_read_tokens, cache_write_tokens, output_tokens, \
reported_cost_nano_usd, requests)
SELECT $1, $2, * FROM UNNEST($3::text[], $4::text[], $5::text[], $6::bigint[], \
$7::bigint[], $8::bigint[], $9::bigint[], $10::bigint[], $11::bigint[])
ON CONFLICT (workspace_id, day, email, provider, model, session_id)
DO UPDATE SET
input_tokens = ai_token_usage.input_tokens + EXCLUDED.input_tokens,
cache_read_tokens = ai_token_usage.cache_read_tokens + EXCLUDED.cache_read_tokens,
cache_write_tokens = ai_token_usage.cache_write_tokens + EXCLUDED.cache_write_tokens,
output_tokens = ai_token_usage.output_tokens + EXCLUDED.output_tokens,
reported_cost_nano_usd = CASE
WHEN EXCLUDED.reported_cost_nano_usd IS NULL
THEN ai_token_usage.reported_cost_nano_usd
ELSE COALESCE(ai_token_usage.reported_cost_nano_usd, 0)
+ EXCLUDED.reported_cost_nano_usd
END,
requests = ai_token_usage.requests + EXCLUDED.requests,
updated_at = now()",
&w_id,
&authed.email,
&providers,
&models,
&session_ids,
&inputs,
&cache_reads,
&cache_writes,
&outputs,
&reported_costs as &[Option<i64>],
&requests
)
.execute(&db)
.await?;
Ok(StatusCode::NO_CONTENT)
}
#[derive(Default)]
struct AIUsageTotals {
input: i64,
cache_read: i64,
cache_write: i64,
output: i64,
reported_cost: Option<i64>,
requests: i64,
}
#[derive(Deserialize)]
struct ListAIUsageQuery {
days: Option<i32>,
group_by: Option<String>,
scope: Option<String>,
}
/// A bucket always carries its provider and model: the caller prices it from a
/// per-model rate table, which a bucket spanning several models could not be
/// resolved against.
#[derive(Serialize)]
struct AITokenUsageBucket {
key: String,
provider: String,
model: String,
input_tokens: i64,
cache_read_tokens: i64,
cache_write_tokens: i64,
output_tokens: i64,
reported_cost_nano_usd: Option<i64>,
requests: i64,
}
/// Grouping by day over a long range, or by model across many models, can produce
/// more buckets than a table is worth rendering, so the listing is capped.
/// `truncated` says so explicitly — a caller that sums the rows into a total must be
/// able to tell that the total is partial rather than silently under-reporting spend.
#[derive(Serialize)]
struct AITokenUsageListing {
buckets: Vec<AITokenUsageBucket>,
truncated: bool,
}
const AI_USAGE_MAX_BUCKETS: i64 = 1000;
async fn list_ai_usage(
authed: ApiAuthed,
Extension(db): Extension<DB>,
Path(w_id): Path<String>,
Query(query): Query<ListAIUsageQuery>,
) -> Result<Json<AITokenUsageListing>> {
// Reading the whole workspace's spend is an admin view; reading your own is
// not, so a member can see what they are costing without being shown their
// colleagues'. The filter is the session's email, never a parameter.
let own_email = match query.scope.as_deref().unwrap_or("workspace") {
"workspace" => {
require_admin(authed.is_admin, &authed.username)?;
None
}
"self" => Some(authed.email.clone()),
scope => return Err(Error::BadRequest(format!("Unsupported scope: {}", scope))),
};
let days = query.days.unwrap_or(30).clamp(1, 365);
let group_by = query.group_by.as_deref().unwrap_or("day");
// No `session`: a session is identified by a client-generated id whose name
// lives only in the browser that made it, so a bucket keyed on one is a label
// nobody can resolve. `session_id` is still stored, at the grain the client
// batches on, should sessions ever gain a server-side name.
if !matches!(group_by, "day" | "user" | "model") {
return Err(Error::BadRequest(format!(
"Unsupported group_by: {}",
group_by
)));
}
// Fetch one past the cap to detect truncation. Ordering is by token volume, not
// by cost: rates are applied by the caller, so this query cannot know what a
// bucket cost. Volume is the closest proxy available here, and the caller is told
// the listing was capped rather than being left to sum a partial set silently.
let mut rows = sqlx::query_as!(
AITokenUsageBucket,
r#"SELECT
(CASE $3::text
WHEN 'day' THEN day::text
WHEN 'user' THEN email
ELSE ''
END) AS "key!",
provider AS "provider!",
model AS "model!",
SUM(input_tokens)::bigint AS "input_tokens!",
SUM(cache_read_tokens)::bigint AS "cache_read_tokens!",
SUM(cache_write_tokens)::bigint AS "cache_write_tokens!",
SUM(output_tokens)::bigint AS "output_tokens!",
SUM(reported_cost_nano_usd)::bigint AS "reported_cost_nano_usd",
SUM(requests)::bigint AS "requests!"
FROM ai_token_usage
WHERE workspace_id = $1 AND day > CURRENT_DATE - $2::int
AND ($5::text IS NULL OR email = $5)
GROUP BY 1, provider, model
ORDER BY SUM(input_tokens + cache_read_tokens + cache_write_tokens + output_tokens) DESC
LIMIT $4"#,
&w_id,
days,
group_by,
AI_USAGE_MAX_BUCKETS + 1,
own_email.as_deref()
)
.fetch_all(&db)
.await?;
let truncated = rows.len() as i64 > AI_USAGE_MAX_BUCKETS;
rows.truncate(AI_USAGE_MAX_BUCKETS as usize);
Ok(Json(AITokenUsageListing { buckets: rows, truncated }))
}
/// Check if AWS Bedrock credentials are available from environment variables.
#[cfg(feature = "bedrock")]
async fn check_bedrock_credentials(
backend/windmill-api/src/ai_evals/datasets.rs
+473
View File
@@ -0,0 +1,473 @@
use super::*;
#[derive(Serialize, Deserialize, Debug, Clone)]
pub struct EvalDataset {
pub path: String,
#[serde(default, skip_serializing_if = "Option::is_none")]
pub summary: Option<String>,
/// The columns of the results table, in display order.
#[serde(default)]
pub scorers: Vec<Scorer>,
pub created_at: DateTime<Utc>,
pub created_by: String,
pub edited_at: DateTime<Utc>,
pub edited_by: String,
}
/// The agent-facing half of a case: exactly the inputs a standalone run feeds the agent.
#[derive(Serialize, Deserialize, Debug, Clone, Default)]
pub struct EvalCaseInput {
#[serde(default, skip_serializing_if = "Option::is_none")]
pub user_message: Option<String>,
#[serde(default, skip_serializing_if = "Option::is_none")]
pub user_attachments: Option<Box<RawValue>>,
}
#[derive(Serialize, Deserialize, Debug, Clone)]
pub struct EvalCase {
pub id: Uuid,
pub input: EvalCaseInput,
#[serde(default, skip_serializing_if = "Option::is_none")]
pub expected: Option<Box<RawValue>>,
pub created_at: DateTime<Utc>,
pub created_by: String,
}
/// The case fields a caller may set. `id`/`created_at`/`created_by` are assigned server-side so
/// a client cannot forge provenance or collide with an existing case.
#[derive(Deserialize, Debug)]
pub struct NewEvalCase {
#[serde(default)]
pub input: EvalCaseInput,
#[serde(default)]
pub expected: Option<Box<RawValue>>,
}
#[derive(Deserialize)]
pub struct CreateDataset {
pub path: String,
#[serde(default)]
pub summary: Option<String>,
#[serde(default)]
pub scorers: Vec<Scorer>,
/// The cases to create it holding. A case cannot be written before there is a dataset for it
/// to be a row of, so they are sent with it rather than added afterwards.
#[serde(default)]
pub cases: Vec<NewEvalCase>,
}
#[derive(Deserialize)]
pub struct EditDataset {
/// Renames the dataset. Its cases and experiments follow through the foreign keys.
#[serde(default)]
pub path: Option<String>,
/// Left out to keep the stored summary; sent as `""` to clear it.
#[serde(default)]
pub summary: Option<String>,
/// Left out to keep the dataset's columns as they are; sent to replace them wholesale.
#[serde(default)]
pub scorers: Option<Vec<Scorer>>,
/// The cases as they should stand afterwards: all of them, each carrying its `id` if the
/// dataset already has it. Sent with the rest of an edit so a rename the dataset refuses
/// refuses the case edits with it, rather than leaving them written under the old name.
#[serde(default)]
pub cases: Option<Vec<SaveCase>>,
}
#[derive(Deserialize)]
pub struct SaveCase {
#[serde(default)]
pub id: Option<Uuid>,
#[serde(default)]
pub input: EvalCaseInput,
#[serde(default)]
pub expected: Option<Box<RawValue>>,
}
#[derive(Serialize)]
pub struct ListCasesResponse {
pub cases: Vec<EvalCase>,
}
pub async fn list_datasets(
authed: ApiAuthed,
Extension(user_db): Extension<UserDB>,
Path(w_id): Path<String>,
) -> JsonResult<Vec<EvalDataset>> {
let mut tx = user_db.begin(&authed).await?;
let rows = sqlx::query!(
"SELECT path, summary, scorers, created_at, created_by,
edited_at, edited_by
FROM eval_dataset WHERE workspace_id = $1 ORDER BY path",
w_id
)
.fetch_all(&mut *tx)
.await?;
tx.commit().await?;
Ok(Json(
rows.into_iter()
.map(|row| {
dataset_from_row(
row.path,
row.summary,
row.scorers,
row.created_at,
row.created_by,
row.edited_at,
row.edited_by,
)
})
.collect::<Result<Vec<_>>>()?,
))
}
pub async fn create_dataset(
authed: ApiAuthed,
Extension(user_db): Extension<UserDB>,
Path(w_id): Path<String>,
Json(payload): Json<CreateDataset>,
) -> Result<String> {
check_proper_path(&payload.path)?;
check_summary(payload.summary.as_deref())?;
if authed.is_operator {
return Err(Error::NotAuthorized(
"Operators cannot create eval datasets".to_string(),
));
}
check_case_set(
payload
.cases
.iter()
.map(|case| (&case.input, case.expected.as_ref())),
)?;
let mut scorers = payload.scorers;
// A dataset being created has no columns yet, so every id is minted.
assign_scorer_ids(&mut scorers, &std::collections::HashSet::new())?;
let scorers = serde_json::to_value(&scorers)?;
// One `user_db` transaction: the row's insert policy gates the dataset, the cases' insert
// policy gates each case, and the two land together or not at all.
let mut tx = user_db.begin(&authed).await?;
// A path already taken returns no row; a path the caller may not create raises the insert
// policy, which `map_rls_denied` turns into an access error.
let created = sqlx::query_scalar!(
"INSERT INTO eval_dataset
(workspace_id, path, summary, scorers, created_by, edited_by)
VALUES ($1, $2, $3, $4, $5, $5)
ON CONFLICT (workspace_id, path) DO NOTHING
RETURNING path",
w_id,
payload.path,
payload.summary,
scorers,
authed.username,
)
.fetch_optional(&mut *tx)
.await
.map_err(|e| map_rls_denied(&payload.path, "create", e))?;
if created.is_none() {
return Err(Error::BadRequest(format!(
"Eval dataset {} already exists",
payload.path
)));
}
for case in &payload.cases {
sqlx::query!(
// clock_timestamp() (not the now() default, which is transaction-stable) so cases
// saved together get strictly increasing created_at and reload in insertion order;
// ORDER BY created_at, id would otherwise tie-break a same-transaction batch on the
// random uuid id.
"INSERT INTO eval_case
(workspace_id, dataset_path, input, expected, created_by, created_at)
VALUES ($1, $2, $3, $4, $5, clock_timestamp())",
w_id,
payload.path,
serde_json::to_value(&case.input)?,
opt_from_raw(case.expected.as_ref())?,
authed.username,
)
.execute(&mut *tx)
.await?;
}
tx.commit().await?;
Ok(format!("Created eval dataset {}", payload.path))
}
pub async fn get_dataset(
authed: ApiAuthed,
Extension(user_db): Extension<UserDB>,
Path((w_id, path)): Path<(String, String)>,
) -> JsonResult<EvalDataset> {
Ok(Json(read_dataset(&authed, &user_db, &w_id, &path).await?))
}
/// An edit is one transaction: the rename, the summary, the columns and the cases land together
/// or not at all.
pub async fn update_dataset(
authed: ApiAuthed,
Extension(user_db): Extension<UserDB>,
Path((w_id, path)): Path<(String, String)>,
Json(payload): Json<EditDataset>,
) -> Result<String> {
if authed.is_operator {
return Err(Error::NotAuthorized(
"Operators cannot modify eval datasets".to_string(),
));
}
check_summary(payload.summary.as_deref())?;
let new_path = match payload.path.filter(|p| *p != path) {
Some(new_path) => {
check_proper_path(&new_path)?;
// A rename is owner-only, as for every other renamable object. RLS write access is not
// enough: the UPDATE policies carry no explicit WITH CHECK, so Postgres reuses their
// USING, and the row's own extra_perms travels with the rename and would satisfy it
// for any destination.
windmill_api_auth::require_owner_of_path(&authed, &path)?;
Some(new_path)
}
None => None,
};
if let Some(cases) = &payload.cases {
check_cases(cases)?;
}
// One `user_db` transaction, governed by the row-level policies throughout. The row is read
// `FOR UPDATE` — its UPDATE policy decides who may — which also pins its cases, so a
// concurrent edit cannot restore a removed scorer's id or interleave with the case write.
let mut tx = user_db.clone().begin(&authed).await?;
let current = sqlx::query_scalar!(
"SELECT scorers FROM eval_dataset WHERE workspace_id = $1 AND path = $2 FOR UPDATE",
w_id,
path
)
.fetch_optional(&mut *tx)
.await?;
let Some(current) = current else {
drop(tx);
return Err(write_refused(&authed, &user_db, &w_id, &path).await);
};
let existing: std::collections::HashSet<String> =
parse_scorers(current)?.into_iter().map(|s| s.id).collect();
let scorers = match payload.scorers {
Some(mut scorers) => {
assign_scorer_ids(&mut scorers, &existing)?;
Some(serde_json::to_value(&scorers)?)
}
None => None,
};
let updated = sqlx::query_scalar!(
"UPDATE eval_dataset
SET path = COALESCE($6, path), summary = COALESCE($3, summary),
scorers = COALESCE($4, scorers), edited_at = now(), edited_by = $5
WHERE workspace_id = $1 AND path = $2
RETURNING path",
w_id,
path,
payload.summary,
scorers,
authed.username,
new_path.as_deref(),
)
.fetch_optional(&mut *tx)
.await
.map_err(|e| {
if e.as_database_error().and_then(|e| e.code()).as_deref() == Some("23505") {
Error::BadRequest(format!(
"Eval dataset {} already exists",
new_path.as_deref().unwrap_or(&path)
))
} else {
map_rls_denied(new_path.as_deref().unwrap_or(&path), "rename", e)
}
})?;
// No row updated: the caller cannot write this dataset (its UPDATE policy denied the row) or it
// is gone. A refused rename destination raises 42501 instead, handled just above.
let Some(updated) = updated else {
drop(tx);
return Err(write_refused(&authed, &user_db, &w_id, &path).await);
};
// Under the name the dataset now has: the cases followed the rename through the foreign key.
if let Some(cases) = &payload.cases {
write_cases(&mut tx, &w_id, &updated, cases, &authed.username).await?;
}
tx.commit().await?;
Ok(format!("Updated eval dataset {}", updated))
}
/// The cases, the experiments and their recorded case sets go with the dataset, through the
/// foreign keys. The jobs those experiments produced are not touched: they are jobs, with their
/// own retention.
pub async fn delete_dataset(
authed: ApiAuthed,
Extension(user_db): Extension<UserDB>,
Path((w_id, path)): Path<(String, String)>,
) -> Result<String> {
check_proper_path(&path)?;
if authed.is_operator {
return Err(Error::NotAuthorized(
"Operators cannot delete eval datasets".to_string(),
));
}
let mut tx = user_db.clone().begin(&authed).await?;
let deleted = sqlx::query_scalar!(
"DELETE FROM eval_dataset WHERE workspace_id = $1 AND path = $2 RETURNING path",
w_id,
path
)
.fetch_optional(&mut *tx)
.await?;
tx.commit().await?;
if deleted.is_none() {
return Err(write_refused(&authed, &user_db, &w_id, &path).await);
}
Ok(format!("Deleted eval dataset {}", path))
}
// -----------------------------------------------------------------------------------------------
// Cases
// -----------------------------------------------------------------------------------------------
async fn read_cases(
tx: &mut sqlx::Transaction<'_, sqlx::Postgres>,
w_id: &str,
dataset: &str,
per_page: usize,
offset: usize,
) -> Result<Vec<EvalCase>> {
let rows = sqlx::query!(
"SELECT id, input, expected, created_at, created_by
FROM eval_case
WHERE workspace_id = $1 AND dataset_path = $2
ORDER BY created_at, id
LIMIT $3 OFFSET $4",
w_id,
dataset,
per_page as i64,
offset as i64
)
.fetch_all(&mut **tx)
.await?;
rows.into_iter()
.map(|row| {
Ok(EvalCase {
id: row.id,
input: serde_json::from_value(row.input)?,
expected: opt_to_raw(row.expected)?,
created_at: row.created_at,
created_by: row.created_by,
})
})
.collect()
}
pub async fn list_cases(
authed: ApiAuthed,
Extension(user_db): Extension<UserDB>,
Path((w_id, path)): Path<(String, String)>,
Query(pagination): Query<Pagination>,
) -> JsonResult<ListCasesResponse> {
check_proper_path(&path)?;
let (per_page, offset) = paginate(pagination);
let mut tx = user_db.begin(&authed).await?;
// The dataset first, so an unknown or unreadable one is a 404 rather than an empty dataset:
// the case rows are invisible in both cases.
let dataset = sqlx::query_scalar!(
"SELECT path FROM eval_dataset WHERE workspace_id = $1 AND path = $2",
w_id,
path
)
.fetch_optional(&mut *tx)
.await?;
if dataset.is_none() {
return Err(Error::NotFound(format!("Eval dataset {} not found", path)));
}
let cases = read_cases(&mut tx, &w_id, &path, per_page, offset).await?;
tx.commit().await?;
Ok(Json(ListCasesResponse { cases }))
}
/// What a whole list of cases can be refused for, before any of it is written.
fn check_cases(cases: &[SaveCase]) -> Result<()> {
check_case_set(
cases
.iter()
.map(|case| (&case.input, case.expected.as_ref())),
)?;
// One row per id: the same id twice would write one row twice and return a list longer than
// the dataset it describes, and the save would read as having kept a case it dropped.
let mut ids: Vec<Uuid> = cases.iter().filter_map(|c| c.id).collect();
ids.sort();
let submitted = ids.len();
ids.dedup();
if ids.len() != submitted {
return Err(Error::BadRequest(
"A case id appears more than once in the dataset".to_string(),
));
}
Ok(())
}
/// Replace a dataset's cases with `cases`, in the caller's transaction: rows not in the list go,
/// rows carrying an id are updated, the rest are added. Returns one id per case, in order.
async fn write_cases(
tx: &mut sqlx::Transaction<'_, sqlx::Postgres>,
w_id: &str,
path: &str,
cases: &[SaveCase],
username: &str,
) -> Result<Vec<Uuid>> {
let kept: Vec<Uuid> = cases.iter().filter_map(|c| c.id).collect();
sqlx::query!(
"DELETE FROM eval_case
WHERE workspace_id = $1 AND dataset_path = $2 AND NOT (id = ANY($3))",
w_id,
path,
&kept
)
.execute(&mut **tx)
.await?;
let mut ids = Vec::with_capacity(cases.len());
for case in cases {
let input = serde_json::to_value(&case.input)?;
let expected = opt_from_raw(case.expected.as_ref())?;
let id = match case.id {
Some(id) => sqlx::query_scalar!(
"UPDATE eval_case SET input = $4, expected = $5
WHERE workspace_id = $1 AND dataset_path = $2 AND id = $3
RETURNING id",
w_id,
path,
id,
input,
expected,
)
.fetch_optional(&mut **tx)
.await?
.ok_or_else(|| Error::NotFound(format!("Eval case {} not found in {}", id, path)))?,
None => sqlx::query_scalar!(
// clock_timestamp() keeps a same-transaction batch in insertion order on reload.
"INSERT INTO eval_case
(workspace_id, dataset_path, input, expected, created_by, created_at)
VALUES ($1, $2, $3, $4, $5, clock_timestamp())
RETURNING id",
w_id,
path,
input,
expected,
username,
)
.fetch_one(&mut **tx)
.await
.map_err(|e| {
if is_missing_dataset(&e) {
Error::NotFound(format!("Eval dataset {} not found", path))
} else {
e.into()
}
})?,
};
ids.push(id);
}
Ok(ids)
}
backend/windmill-api/src/ai_evals/mod.rs
+371
View File
@@ -0,0 +1,371 @@
//! Eval datasets for reusable AI agents.
//!
//! Five tables: `eval_dataset` and the `eval_case` rows it holds are the curated inputs;
//! `eval_experiment`, `eval_experiment_case` and `eval_score` are one run of them, written once
//! and only ever read afterwards.
//!
//! Datasets and cases go through `user_db`, so row-level security is the only access authority:
//! `eval_case`'s policies derive from its dataset's (`eval_dataset_writable`, in the migration).
//! The experiment tables carry read policies only and are written on the unrestricted pool after
//! the API has checked access — see `run_experiment` and `collect_experiment`.
use axum::{
extract::{Path, Query},
routing::{get, post},
Extension, Json, Router,
};
use chrono::{DateTime, Utc};
use serde::{Deserialize, Serialize};
use serde_json::value::RawValue;
use uuid::Uuid;
use windmill_common::{
db::UserDB,
error::{Error, JsonResult, Result},
utils::{check_proper_path, paginate, Pagination},
};
use crate::db::{ApiAuthed, DB};
use windmill_api_auth::check_scopes;
pub(crate) mod datasets;
pub(crate) mod payload;
pub(crate) mod results;
pub(crate) mod run;
pub(crate) mod scorers;
pub(crate) mod scoring;
pub(crate) mod subject;
pub(crate) mod template;
pub(crate) use datasets::*;
pub(crate) use payload::*;
pub(crate) use results::*;
pub(crate) use run::*;
pub(crate) use scorers::*;
pub(crate) use scoring::*;
pub(crate) use subject::*;
pub(crate) use template::*;
pub fn workspaced_service() -> Router {
Router::new()
.route("/datasets/list", get(list_datasets))
.route("/datasets/create", post(create_dataset))
.route("/datasets/get/{*path}", get(get_dataset))
.route("/datasets/update/{*path}", post(update_dataset))
.route("/datasets/delete/{*path}", post(delete_dataset))
.route("/cases/list/{*path}", get(list_cases))
.route("/scorer_defaults", get(scorer_defaults))
.route("/run_payload", get(run_payload))
.route("/experiments/run", post(run_experiment))
.route("/experiments/collect", post(collect_experiment))
.route("/scorers/recent", get(recent_scorers))
.route("/subject_state", get(subject_state))
.route("/experiments/list_all", get(list_all_experiments))
.route("/experiments/results/{*path}", get(experiment_results))
}
/// Checked here rather than left to the column, whose own refusal comes back as an internal
/// database error naming no field.
const MAX_DATASET_SUMMARY_CHARS: usize = 1000;
fn check_summary(summary: Option<&str>) -> Result<()> {
match summary {
Some(summary) if summary.chars().count() > MAX_DATASET_SUMMARY_CHARS => {
Err(Error::BadRequest(format!(
"This dataset's summary is {} characters, over the {} the column holds.",
summary.chars().count(),
MAX_DATASET_SUMMARY_CHARS
)))
}
_ => Ok(()),
}
}
/// A case is text — attachments are S3 references rather than inline bytes.
const MAX_CASE_BYTES: usize = 256 * 1024;
/// The whole case set together, so cases at the per-case cap cannot add up to a dataset a listing
/// or a run must hold hundreds of megabytes of at once.
const MAX_DATASET_BYTES: usize = 16 * 1024 * 1024;
/// Also what a listing returns in one page, so a dataset is always read whole: the editor holds
/// every case at once and writes them together, and half a set on screen is a Save that drops the
/// rest.
const MAX_CASES_PER_DATASET: i64 = 1_000;
const MAX_EXPERIMENTS_LISTED: i64 = 100;
const MAX_RECENT_SCORERS: usize = 12;
/// A run's work is cases × scorers, so this bounds how far one request fans out.
const MAX_SCORERS_PER_DATASET: usize = 20;
/// The dataset a write was aimed at is gone. Raised from the foreign key rather than from a
/// preceding existence check, so a dataset deleted mid-request cannot slip between the two.
fn is_missing_dataset(e: &sqlx::Error) -> bool {
e.as_database_error().and_then(|d| d.code()).as_deref() == Some("23503")
}
/// A `user_db` write the row-level policies refused surfaces as SQLSTATE 42501, whose message
/// names the table and the policy. Turn it into one about access.
fn map_rls_denied(path: &str, action: &str, e: sqlx::Error) -> Error {
if e.as_database_error().and_then(|d| d.code()).as_deref() == Some("42501") {
return Error::NotAuthorized(format!("Not allowed to {} eval dataset {}", action, path));
}
e.into()
}
/// A write that matched no row is either a dataset that does not exist or one the caller can read
/// but not write. Row-level security cannot distinguish them — both are simply invisible to the
/// statement — so ask again with a plain read.
async fn write_refused(authed: &ApiAuthed, user_db: &UserDB, w_id: &str, path: &str) -> Error {
let visible = async {
let mut tx = user_db.clone().begin(authed).await?;
let found = sqlx::query_scalar!(
"SELECT path FROM eval_dataset WHERE workspace_id = $1 AND path = $2",
w_id,
path
)
.fetch_optional(&mut *tx)
.await?;
tx.commit().await?;
Ok::<_, Error>(found.is_some())
}
.await;
match visible {
Ok(true) => Error::NotAuthorized(format!(
"User {} does not have write access to eval dataset {}",
authed.username, path
)),
Ok(false) => Error::NotFound(format!("Eval dataset {} not found", path)),
Err(e) => e,
}
}
/// One `eval_dataset` row, from the columns every read of the table selects.
fn dataset_from_row(
path: String,
summary: Option<String>,
scorers: serde_json::Value,
created_at: DateTime<Utc>,
created_by: String,
edited_at: DateTime<Utc>,
edited_by: String,
) -> Result<EvalDataset> {
Ok(EvalDataset {
path,
summary,
scorers: parse_scorers(scorers)?,
created_at,
created_by,
edited_at,
edited_by,
})
}
/// A dataset's columns. Only this module writes them, through serde, so a value that does not
/// parse is corruption rather than input: defaulting to no columns would let the next save mint
/// fresh scorer ids and orphan every score already recorded.
pub(crate) fn parse_scorers(scorers: serde_json::Value) -> Result<Vec<Scorer>> {
serde_json::from_value(scorers)
.map_err(|e| Error::internal_err(format!("eval dataset scorers are not readable: {e}")))
}
/// Read the dataset the request names, through `user_db` so that a caller who cannot see it gets
/// the same answer as one asking for a dataset that does not exist.
async fn read_dataset(
authed: &ApiAuthed,
user_db: &UserDB,
w_id: &str,
path: &str,
) -> Result<EvalDataset> {
check_proper_path(path)?;
let mut tx = user_db.clone().begin(authed).await?;
let row = sqlx::query!(
"SELECT path, summary, scorers, created_at, created_by,
edited_at, edited_by
FROM eval_dataset WHERE workspace_id = $1 AND path = $2",
w_id,
path
)
.fetch_optional(&mut *tx)
.await?;
tx.commit().await?;
let row = row.ok_or_else(|| Error::NotFound(format!("Eval dataset {} not found", path)))?;
dataset_from_row(
row.path,
row.summary,
row.scorers,
row.created_at,
row.created_by,
row.edited_at,
row.edited_by,
)
}
/// The dataset and its cases as one snapshot, so a launch cannot record the cases from before an
/// edit beside the scorers from after it. One transaction is not enough: `user_db` runs at READ
/// COMMITTED, where each statement takes a fresh snapshot, so the row is taken `FOR UPDATE` —
/// which an edit's own `FOR UPDATE` and a case write's foreign-key lock both conflict with.
pub(crate) async fn read_dataset_and_cases(
authed: &ApiAuthed,
user_db: &UserDB,
w_id: &str,
path: &str,
) -> Result<(EvalDataset, Vec<EvalCase>)> {
check_proper_path(path)?;
let mut tx = user_db.clone().begin(authed).await?;
let row = sqlx::query!(
"SELECT path, summary, scorers, created_at, created_by, edited_at, edited_by
FROM eval_dataset WHERE workspace_id = $1 AND path = $2 FOR UPDATE",
w_id,
path
)
.fetch_optional(&mut *tx)
.await?;
let Some(row) = row else {
tx.commit().await?;
return Err(Error::NotFound(format!("Eval dataset {} not found", path)));
};
let case_rows = sqlx::query!(
"SELECT id, input, expected, created_at, created_by
FROM eval_case
WHERE workspace_id = $1 AND dataset_path = $2
ORDER BY created_at, id",
w_id,
path
)
.fetch_all(&mut *tx)
.await?;
tx.commit().await?;
let dataset = dataset_from_row(
row.path,
row.summary,
row.scorers,
row.created_at,
row.created_by,
row.edited_at,
row.edited_by,
)?;
let cases = case_rows
.into_iter()
.map(|row| {
Ok(EvalCase {
id: row.id,
input: serde_json::from_value(row.input)?,
expected: opt_to_raw(row.expected)?,
created_at: row.created_at,
created_by: row.created_by,
})
})
.collect::<Result<Vec<_>>>()?;
Ok((dataset, cases))
}
/// Whether this caller may write a dataset's contents: its cases, and the experiments that run
/// them.
///
/// `SELECT … FOR UPDATE` applies `eval_dataset`'s UPDATE policies on top of its SELECT policies,
/// so the row itself answers who may write it, and a grant in `extra_perms` is honoured without
/// being mirrored here.
async fn require_dataset_writable(
authed: &ApiAuthed,
user_db: &UserDB,
w_id: &str,
path: &str,
) -> Result<()> {
check_proper_path(path)?;
if authed.is_operator {
return Err(Error::NotAuthorized(
"Operators cannot modify eval datasets".to_string(),
));
}
let mut tx = user_db.clone().begin(authed).await?;
let writable = sqlx::query_scalar!(
"SELECT path FROM eval_dataset WHERE workspace_id = $1 AND path = $2 FOR UPDATE",
w_id,
path
)
.fetch_optional(&mut *tx)
.await?;
tx.commit().await?;
if writable.is_some() {
Ok(())
} else {
Err(write_refused(authed, user_db, w_id, path).await)
}
}
/// jsonb columns are read as `serde_json::Value` and handed on as `RawValue`: a case's `expected`
/// is arbitrary user JSON that this module never looks inside.
fn opt_to_raw(value: Option<serde_json::Value>) -> Result<Option<Box<RawValue>>> {
value
.map(|v| Ok(serde_json::value::to_raw_value(&v)?))
.transpose()
}
fn opt_from_raw(value: Option<&Box<RawValue>>) -> Result<Option<serde_json::Value>> {
value
.map(|v| Ok(serde_json::from_str(v.get())?))
.transpose()
}
fn check_case(input: &EvalCaseInput, expected: Option<&Box<RawValue>>) -> Result<()> {
// The shape the agent step reads its attachments in, checked when the case is written rather
// than when a run deserialises the step's arguments, which is after the case was queued.
if let Some(attachments) = &input.user_attachments {
if serde_json::from_str::<Vec<windmill_types::s3::S3Object>>(attachments.get()).is_err() {
return Err(Error::BadRequest(
"A case's user_attachments is a list of S3 objects, each with an `s3` key naming \
the file"
.to_string(),
));
}
}
check_case_size(input, expected)
}
/// The bytes one case weighs against its own and the dataset's caps.
fn case_bytes(input: &EvalCaseInput, expected: Option<&Box<RawValue>>) -> Result<usize> {
let mut bytes = serde_json::to_vec(input)?.len();
if let Some(expected) = expected {
bytes += expected.get().len();
}
Ok(bytes)
}
/// What a whole case set can be refused for, before any of it is written.
fn check_case_set<'a>(
cases: impl ExactSizeIterator<Item = (&'a EvalCaseInput, Option<&'a Box<RawValue>>)>,
) -> Result<()> {
if cases.len() as i64 > MAX_CASES_PER_DATASET {
return Err(Error::BadRequest(format!(
"An eval dataset holds at most {} cases. Split them into several datasets.",
MAX_CASES_PER_DATASET
)));
}
let mut total = 0usize;
for (input, expected) in cases {
check_case(input, expected)?;
total += case_bytes(input, expected)?;
}
if total > MAX_DATASET_BYTES {
return Err(Error::BadRequest(format!(
"This dataset is {} KiB of cases, over the {} KiB limit. Attachments belong in \
workspace storage and are referenced by a case, not stored inside it.",
total / 1024,
MAX_DATASET_BYTES / 1024
)));
}
Ok(())
}
fn check_case_size(input: &EvalCaseInput, expected: Option<&Box<RawValue>>) -> Result<()> {
let bytes = case_bytes(input, expected)?;
if bytes > MAX_CASE_BYTES {
return Err(Error::BadRequest(format!(
"This eval case is {} KiB, over the {} KiB limit. Attachments belong in workspace \
storage and are referenced by a case, not stored inside it.",
bytes / 1024,
MAX_CASE_BYTES / 1024
)));
}
Ok(())
}
backend/windmill-api/src/ai_evals/payload.rs
+410
View File
@@ -0,0 +1,410 @@
use super::*;
/// What every scorer is handed: the answer, and the calls the agent made to reach it.
///
/// Built from the job the run already stored, which is what lets a scorer added later score an
/// experiment that has already run.
#[derive(Serialize, Debug, Clone)]
pub struct EvalRunPayload {
pub input: EvalCaseInput,
#[serde(skip_serializing_if = "Option::is_none")]
pub output: Option<Box<RawValue>>,
#[serde(skip_serializing_if = "Option::is_none")]
pub expected: Option<Box<RawValue>>,
pub tool_calls: Vec<EvalToolCall>,
/// The tools that were actually called, with the schema they were called against. A tool
/// whose schema could not be resolved carries `null`, and a scorer validating arguments must
/// treat that as unchecked rather than as a failure.
pub tools: Vec<EvalToolDef>,
pub metrics: EvalMetrics,
pub status: String,
pub job_id: Uuid,
}
#[derive(Serialize, Debug, Clone)]
pub struct EvalToolCall {
pub name: String,
#[serde(skip_serializing_if = "Option::is_none")]
pub args: Option<Box<RawValue>>,
#[serde(skip_serializing_if = "Option::is_none")]
pub result: Option<Box<RawValue>>,
#[serde(skip_serializing_if = "Option::is_none")]
pub error: Option<String>,
#[serde(skip_serializing_if = "Option::is_none")]
pub duration_ms: Option<i64>,
/// Set when the result was too large to carry and was cut down.
#[serde(skip_serializing_if = "std::ops::Not::not")]
pub truncated: bool,
}
#[derive(Serialize, Debug, Clone)]
pub struct EvalToolDef {
pub name: String,
#[serde(skip_serializing_if = "Option::is_none")]
pub schema: Option<Box<RawValue>>,
}
#[derive(Serialize, Debug, Clone)]
pub struct EvalMetrics {
pub steps: usize,
#[serde(skip_serializing_if = "Option::is_none")]
pub duration_ms: Option<i64>,
/// The provider's token counts, when it reported any.
#[serde(skip_serializing_if = "Option::is_none")]
pub usage: Option<Box<RawValue>>,
}
/// A tool result large enough to swamp a judge's context is cut here. The scorer is told, so a
/// check reading a truncated result can say so instead of failing on the missing tail.
const MAX_TOOL_RESULT_BYTES: usize = 4 * 1024;
fn truncate_value(value: Box<RawValue>) -> (Box<RawValue>, bool) {
if value.get().len() <= MAX_TOOL_RESULT_BYTES {
return (value, false);
}
let text = value.get();
let mut end = MAX_TOOL_RESULT_BYTES;
while !text.is_char_boundary(end) {
end -= 1;
}
match serde_json::value::to_raw_value(&format!("{}… [truncated]", &text[..end])) {
Ok(v) => (v, true),
Err(_) => (value, false),
}
}
/// Assemble the payload from a completed case job: the agent step's own result carries the answer
/// and the message list, and every message that made a tool call names the job that ran it.
async fn build_run_payload(
db: &DB,
w_id: &str,
job_id: Uuid,
agent_job: Uuid,
input: EvalCaseInput,
expected: Option<Box<RawValue>>,
status: String,
duration_ms: Option<i64>,
) -> Result<EvalRunPayload> {
// A read that failed is not a run with no answer: handing the scorers an empty payload would
// have them grade the absence of evidence and record that verdict permanently.
let agent_result = agent_result(db, w_id, job_id).await?.map(|(r, _)| r);
let parsed: Option<serde_json::Value> = agent_result
.as_ref()
.and_then(|r| serde_json::from_str(r.get()).ok());
let output = parsed
.as_ref()
.and_then(|p| p.get("output"))
.map(|o| serde_json::value::to_raw_value(o))
.transpose()?;
let usage = parsed
.as_ref()
.and_then(|p| p.get("usage"))
.map(|u| serde_json::value::to_raw_value(u))
.transpose()?;
// Walk the messages in order: a tool call is an `agent_action` on the message that made it.
let mut calls: Vec<(String, Option<Uuid>, Option<Box<RawValue>>)> = vec![];
if let Some(messages) = parsed
.as_ref()
.and_then(|p| p.get("messages"))
.and_then(|m| m.as_array())
{
for message in messages {
let Some(action) = message.get("agent_action") else {
continue;
};
match action.get("type").and_then(|t| t.as_str()) {
Some("tool_call") => calls.push((
action
.get("function_name")
.and_then(|f| f.as_str())
.unwrap_or("tool")
.to_string(),
action
.get("job_id")
.and_then(|j| j.as_str())
.and_then(|j| Uuid::parse_str(j).ok()),
None,
)),
// An MCP call runs inside the agent rather than as a job, so its arguments are on
// the action itself. Its result lives in a later `role: "tool"` message rather
// than a child-job row, and is not surfaced to scorers yet.
Some("mcp_tool_call") => calls.push((
action
.get("function_name")
.and_then(|f| f.as_str())
.unwrap_or("tool")
.to_string(),
None,
action
.get("arguments")
.map(|a| serde_json::value::to_raw_value(a))
.transpose()?,
)),
_ => {}
}
}
}
let call_job_ids: Vec<Uuid> = calls.iter().filter_map(|(_, id, _)| *id).collect();
let mut jobs = std::collections::HashMap::new();
if !call_job_ids.is_empty() {
// Constrained to the agent step's own children rather than to the workspace: these ids
// come out of a job result, so a caller who can run a flow can put any id there. A tool
// call is pushed as a child of the agent that made it, which is what makes that the
// boundary.
let rows = sqlx::query!(
"SELECT j.id, j.args AS \"args: sqlx::types::Json<Box<RawValue>>\",
c.result AS \"result: sqlx::types::Json<Box<RawValue>>\",
c.status::text AS status, c.duration_ms,
s.schema AS \"schema: sqlx::types::Json<Box<RawValue>>\"
FROM v2_job j
LEFT JOIN v2_job_completed c ON c.id = j.id
LEFT JOIN script s ON s.workspace_id = j.workspace_id AND s.hash = j.runnable_id
WHERE j.id = ANY($1) AND j.workspace_id = $2 AND j.parent_job = $3",
&call_job_ids,
w_id,
agent_job
)
.fetch_all(db)
.await?;
for row in rows {
jobs.insert(row.id, row);
}
}
let mut tool_calls = Vec::with_capacity(calls.len());
let mut tools: Vec<EvalToolDef> = vec![];
for (name, call_job_id, inline_args) in calls {
let row = call_job_id.and_then(|id| jobs.get(&id));
let (result, truncated) = match row.and_then(|r| r.result.as_ref()) {
Some(result) => {
let (value, truncated) = truncate_value(result.0.clone());
(Some(value), truncated)
}
None => (None, false),
};
let failed = row
.and_then(|r| r.status.as_deref())
.map(|s| s != "success")
.unwrap_or(false);
if !tools.iter().any(|t| t.name == name) {
tools.push(EvalToolDef {
name: name.clone(),
schema: row.and_then(|r| r.schema.as_ref()).map(|s| s.0.clone()),
});
}
// The already truncated result restated. `render_tool_calls` shows `error` and not
// `result` for a failed call, so the judge's context carries the payload once and bounded;
// `result` stays on the raw call for a script scorer.
let error = failed
.then(|| result.as_ref().map(|r| r.get().to_string()))
.flatten();
tool_calls.push(EvalToolCall {
name,
args: inline_args.or_else(|| row.and_then(|r| r.args.as_ref()).map(|a| a.0.clone())),
result,
error,
duration_ms: row.map(|r| r.duration_ms),
truncated,
});
}
Ok(EvalRunPayload {
metrics: EvalMetrics { steps: tool_calls.len(), duration_ms, usage },
input,
output,
expected,
tool_calls,
tools,
status,
job_id,
})
}
#[derive(Deserialize)]
pub struct RunPayloadQuery {
/// The flow job that answered the case: an iteration of a run.
pub job_id: Uuid,
}
/// What the scorers of one iteration are handed.
#[derive(Serialize)]
pub struct RunPayloadResponse {
pub run: EvalRunPayload,
/// The same run as a judge reads it. Rendered once per case rather than once per judge.
pub rendered: String,
}
/// Assemble the payload for one answered case, for the step that feeds the scorers.
///
/// The case is read from the job's arguments rather than from the experiment, so this works for an
/// iteration whose row has not been filled in yet.
pub async fn run_payload(
authed: ApiAuthed,
Extension(db): Extension<DB>,
Extension(user_db): Extension<UserDB>,
Path(w_id): Path<String>,
Query(query): Query<RunPayloadQuery>,
) -> JsonResult<RunPayloadResponse> {
// `UserDB` enforces row permissions but not a token's scopes, so without this an
// `ai_evals:read` token would read job arguments, results and tool calls that `jobs:read`
// is what actually gates. Job tokens are unscoped, so the run flow's payload step passes.
check_scopes(&authed, || "jobs:read".to_string())?;
// Through `user_db`: the caller is a job token, and it reads what its runner can read.
let mut tx = user_db.begin(&authed).await?;
let args = sqlx::query_scalar!(
"SELECT args AS \"args: sqlx::types::Json<Box<RawValue>>\" FROM v2_job
WHERE id = $1 AND workspace_id = $2",
query.job_id,
w_id
)
.fetch_optional(&mut *tx)
.await?
.flatten()
.ok_or_else(|| Error::NotFound(format!("Job {} not found", query.job_id)))?;
tx.commit().await?;
let args: serde_json::Value = serde_json::from_str(args.0.get())?;
// An iteration carries its case; a run recorded one job per case carries the same input under
// the stamp that job was pushed with.
let case = args.get("iter").and_then(|i| i.get("value"));
let input = case
.and_then(|c| c.get("input"))
.or_else(|| args.get("_eval_input"))
.cloned()
.unwrap_or_else(|| serde_json::json!({}));
let expected = case
.and_then(|c| c.get("expected"))
.or_else(|| args.get("expected"))
.cloned();
// The agent step's own status and duration, never the iteration's: the iteration goes on to
// assemble this payload and run the scorers, so a scorer reading the iteration's duration
// would be measuring itself.
let agent_job = agent_step_job(&db, &w_id, query.job_id)
.await?
.unwrap_or(query.job_id);
let completed = sqlx::query!(
"SELECT status::text AS \"status!\", duration_ms FROM v2_job_completed
WHERE id = $1 AND workspace_id = $2",
agent_job,
w_id
)
.fetch_optional(&db)
.await?;
let run = build_run_payload(
&db,
&w_id,
query.job_id,
agent_job,
serde_json::from_value(input)?,
expected
.map(|e| serde_json::value::to_raw_value(&e))
.transpose()?,
completed
.as_ref()
.map(|c| c.status.clone())
// The iteration asking is itself still running: its agent step is what finished.
.unwrap_or_else(|| "success".to_string()),
completed.as_ref().map(|c| c.duration_ms),
)
.await?;
let rendered = render_run(&run);
Ok(Json(RunPayloadResponse { run, rendered }))
}
/// The job of the agent step inside a run's flow, from the flow status of either a running or a
/// finished one.
async fn agent_step_job(db: &DB, w_id: &str, flow_job: Uuid) -> Result<Option<Uuid>> {
let modules = sqlx::query_scalar!(
"SELECT COALESCE(s.flow_status, c.flow_status) -> 'modules' AS modules
FROM v2_job j
LEFT JOIN v2_job_status s ON s.id = j.id
LEFT JOIN v2_job_completed c ON c.id = j.id
WHERE j.id = $1 AND j.workspace_id = $2",
flow_job,
w_id
)
.fetch_optional(db)
.await?
.flatten();
Ok(modules
.as_ref()
.and_then(|m| m.as_array())
.and_then(|modules| {
modules
.iter()
.find(|m| m.get("id").and_then(|i| i.as_str()) == Some(AGENT_NODE_ID))
})
.and_then(|m| m.get("job"))
.and_then(|j| j.as_str())
.and_then(|j| Uuid::parse_str(j).ok()))
}
/// The system prompt a judge agent is created with. It is the agent's own, so editing a judge is
/// editing that resource — there is no second copy of the grading contract on the dataset.
pub const JUDGE_SYSTEM_PROMPT: &str = r#"You are grading one run of an AI agent.
Score how well the agent handled the request, from 0 to 1. Judge the whole trajectory, not only the
final answer. Penalise asking for information already in the request, calling a tool twice with the
same arguments, and tool errors left unrecovered.
Reply with JSON only, of the form {"score": <number between 0 and 1>, "reason": <one sentence>}."#;
fn render_json(value: Option<&RawValue>) -> String {
value
.map(|v| v.get().to_string())
.unwrap_or_else(|| "(none)".to_string())
}
/// Tool calls as the judge reads them: numbered, in order, with arguments, result and duration.
fn render_tool_calls(calls: &[EvalToolCall]) -> String {
if calls.is_empty() {
return "(none)".to_string();
}
calls
.iter()
.enumerate()
.map(|(index, call)| {
let args = call.args.as_ref().map(|a| a.get()).unwrap_or("{}");
let outcome = match (&call.error, &call.result) {
(Some(error), _) => format!("error: {}", error),
(None, Some(result)) => result.get().to_string(),
(None, None) => "(no result)".to_string(),
};
let timing = call
.duration_ms
.map(|ms| format!(" ({}ms)", ms))
.unwrap_or_default();
format!(
"{}. {}({}) -> {}{}",
index + 1,
call.name,
args,
outcome,
timing
)
})
.collect::<Vec<_>>()
.join("\n")
}
/// One run, as a judge is shown it.
fn render_run(run: &EvalRunPayload) -> String {
format!(
"Request: {}\nTool calls, in order:\n{}\nAnswer: {}\nExpected: {}",
run.input.user_message.as_deref().unwrap_or("(none)"),
render_tool_calls(&run.tool_calls),
render_json(run.output.as_deref()),
render_json(run.expected.as_deref()),
)
}
/// Module id of a scorer inside a scoring job. `assign_scorer_ids` keeps ids to
/// `[A-Za-z0-9_]`, so this is a valid identifier.
pub(crate) fn scorer_module_id(scorer_id: &str) -> String {
format!("s_{}", scorer_id)
}
backend/windmill-api/src/ai_evals/results.rs
+856
View File
@@ -0,0 +1,856 @@
use super::*;
/// One run of a dataset: written once when the dataset is run, and only ever read afterwards.
#[derive(Serialize, Deserialize, Debug, Clone)]
pub struct EvalExperiment {
pub id: Uuid,
pub dataset: String,
pub subject: EvalSubject,
/// This subject's nth run of this dataset, allocated once and never reused: "Run 7" survives
/// history being pruned, which a position computed when the list is read would not.
pub run_number: i32,
/// The flow executing the run: one job holding every case and its scores.
pub run_job_id: Uuid,
pub case_count: i64,
/// What the run scored, one entry per scorer that produced a number. Carried on the run so a
/// list can say what each one scored without reading every cell of every one of them.
#[serde(default, skip_serializing_if = "Vec::is_empty")]
pub scores: Vec<ExperimentScore>,
/// Whether the flow executing this run is still going. What makes a list of runs worth
/// watching rather than worth reloading.
#[serde(default, skip_serializing_if = "std::ops::Not::not")]
pub running: bool,
pub created_at: DateTime<Utc>,
pub created_by: String,
}
/// One scorer's headline for one run: the two numbers a column reports, over that run's cells.
#[derive(Serialize, Deserialize, Debug, Clone)]
pub struct ExperimentScore {
pub scorer_id: String,
/// What the column is called in the dataset that ran it, resolved here because a list of runs
/// spanning datasets cannot hold every dataset's scorers to look it up.
pub name: String,
/// `agent` or `script`, for the badge to say which kind of thing produced the number.
pub kind: String,
#[serde(skip_serializing_if = "Option::is_none")]
pub mean: Option<f64>,
/// The share of scored cells at or above the column's threshold, for a column that has one.
/// Absent where the column has no threshold and the mean is the whole headline.
#[serde(skip_serializing_if = "Option::is_none")]
pub pass_rate: Option<f64>,
pub scored: i64,
/// How many of this run's cells the column failed on. A column that failed on all of them
/// still ran, which is the difference between a headline of nothing and no headline at all.
pub failed: i64,
}
#[derive(Deserialize)]
pub struct ListExperimentsQuery {
/// Restrict to one agent's runs. Both what was deployed and what was drafted are that agent's
/// history, so this does not discriminate by kind.
#[serde(default)]
pub subject_path: Option<String>,
}
/// Every run of this agent, across every dataset it has been measured on.
///
/// Filtered by `user_db`: a run is visible exactly when the dataset it belongs to is.
pub async fn list_all_experiments(
authed: ApiAuthed,
Extension(db): Extension<DB>,
Extension(user_db): Extension<UserDB>,
Path(w_id): Path<String>,
Query(query): Query<ListExperimentsQuery>,
) -> JsonResult<Vec<EvalExperiment>> {
let mut tx = user_db.clone().begin(&authed).await?;
let rows = sqlx::query!(
"SELECT e.id, e.dataset_path, e.subject, e.run_number, e.run_job_id, e.created_at,
e.created_by,
(SELECT count(*) FROM eval_experiment_case c WHERE c.experiment_id = e.id)
AS \"case_count!\"
FROM eval_experiment e
JOIN eval_dataset d ON d.workspace_id = e.workspace_id AND d.path = e.dataset_path
WHERE e.workspace_id = $1
AND ($3::text IS NULL OR e.subject ->> 'path' = $3)
ORDER BY e.created_at DESC
LIMIT $2",
w_id,
MAX_EXPERIMENTS_LISTED,
query.subject_path,
)
.fetch_all(&mut *tx)
.await?;
tx.commit().await?;
let mut experiments = rows
.into_iter()
.map(|row| {
experiment_from_row(
row.id,
row.dataset_path,
row.subject,
row.run_number,
row.run_job_id,
row.case_count,
row.created_at,
row.created_by,
)
})
.collect::<Result<Vec<_>>>()?;
resolve_listed_drafts(&authed, &db, &user_db, &w_id, &mut experiments).await?;
let scorers_by_dataset = scorers_of_listed(&authed, &user_db, &w_id, &experiments).await?;
mark_running(&db, &w_id, &mut experiments).await?;
sync_listed_runs(&db, &w_id, &experiments).await?;
let mut scores = experiment_scores(&db, &experiments, &scorers_by_dataset).await?;
for experiment in experiments.iter_mut() {
experiment.scores = scores.remove(&experiment.id).unwrap_or_default();
}
Ok(Json(experiments))
}
/// Which listed runs are still going, read from the flows executing them. A run whose flow is no
/// longer there at all is over: jobs have their own retention, and reading a missing one as
/// unfinished would leave every run older than it spinning.
async fn mark_running(db: &DB, w_id: &str, experiments: &mut [EvalExperiment]) -> Result<()> {
let job_ids: Vec<Uuid> = experiments.iter().map(|e| e.run_job_id).collect();
if job_ids.is_empty() {
return Ok(());
}
let unfinished: std::collections::HashSet<Uuid> = sqlx::query_scalar!(
"SELECT j.id AS \"id!\" FROM v2_job j
LEFT JOIN v2_job_completed c ON c.id = j.id AND c.workspace_id = $2
WHERE j.id = ANY($1) AND j.workspace_id = $2 AND c.id IS NULL",
&job_ids,
w_id
)
.fetch_all(db)
.await?
.into_iter()
.collect();
for experiment in experiments.iter_mut() {
experiment.running = unfinished.contains(&experiment.run_job_id);
}
Ok(())
}
/// A run of a draft whose edits have since been deployed is a run of that version. Resolved once
/// per subject rather than once per run, because a listing is usually one agent's history.
async fn resolve_listed_drafts(
authed: &ApiAuthed,
db: &DB,
user_db: &UserDB,
w_id: &str,
experiments: &mut [EvalExperiment],
) -> Result<()> {
let drafted: std::collections::HashSet<String> = experiments
.iter()
.filter(|e| e.subject.kind == EvalSubjectKind::AgentDraft)
.map(|e| e.subject.path.clone())
.collect();
if drafted.is_empty() {
return Ok(());
}
// Read each subject as the caller (see experiment_results): an agent the caller cannot read
// yields no hash or version, so its config fingerprint never leaks through the list either.
let mut deployed = std::collections::HashMap::new();
for path in drafted {
let (hash, version) = match readable_agent_state(authed, user_db, w_id, &path).await? {
Some((config, version)) => (Some(draft_hash(&config)), Some(version)),
None => (None, None),
};
deployed.insert(path.clone(), (hash, version));
}
for experiment in experiments.iter_mut() {
let Some((hash, version)) = deployed.get(&experiment.subject.path) else {
continue;
};
// Each run's own dataset: the list may span them, and the update is keyed on both.
let dataset = experiment.dataset.clone();
resolve_deployed_draft(db, w_id, &dataset, experiment, hash.as_deref(), *version).await?;
}
Ok(())
}
/// How many listed runs one list call reads out of their flows. A run's scores live in its flow
/// until something reads them into `eval_score`, so an unopened run has nothing to report; the cap
/// keeps a long history from turning one list call into a hundred flow reads.
const MAX_RUNS_SYNCED_PER_LIST: usize = 10;
/// Read the flows of listed runs that still have scores to collect. Runs already collected are
/// skipped, so the steady-state cost of listing is one query rather than one read per run.
async fn sync_listed_runs(db: &DB, w_id: &str, experiments: &[EvalExperiment]) -> Result<()> {
if experiments.is_empty() {
return Ok(());
}
let ids: Vec<Uuid> = experiments.iter().map(|e| e.id).collect();
let unread = sqlx::query_scalar!(
"SELECT DISTINCT experiment_id FROM eval_score
WHERE experiment_id = ANY($1) AND score IS NULL AND error IS NULL
AND NOT not_applicable",
&ids
)
.fetch_all(db)
.await?
.into_iter()
.collect::<std::collections::HashSet<_>>();
for experiment in experiments
.iter()
.filter(|e| unread.contains(&e.id))
.take(MAX_RUNS_SYNCED_PER_LIST)
{
// Best-effort, for the same reason reading one run is: this is the home screen, and one
// run with an unreadable cell must not cost the list of every other run.
if let Err(e) = sync_run(db, w_id, experiment.id, experiment.run_job_id, false).await {
tracing::warn!("could not collect eval run {}: {e:#}", experiment.id);
}
}
Ok(())
}
/// Every listed run's per-scorer headline, in one grouped query.
///
/// Thresholds come from each run's own dataset as its scorers are *now*, joined per (run, scorer)
/// rather than per scorer: a list spanning datasets is a list of runs whose columns are not the
/// same columns.
async fn experiment_scores(
db: &DB,
experiments: &[EvalExperiment],
scorers_by_dataset: &std::collections::HashMap<String, Vec<Scorer>>,
) -> Result<std::collections::HashMap<Uuid, Vec<ExperimentScore>>> {
let mut by_experiment: std::collections::HashMap<Uuid, Vec<ExperimentScore>> =
Default::default();
// One entry per (run, column) it could have scored, which is what carries the threshold and
// the column's order into the query.
let mut ids: Vec<Uuid> = vec![];
let mut scorer_ids: Vec<String> = vec![];
let mut thresholds: Vec<Option<f64>> = vec![];
for experiment in experiments {
for scorer in scorers_by_dataset
.get(&experiment.dataset)
.map(|s| s.as_slice())
.unwrap_or(&[])
{
ids.push(experiment.id);
scorer_ids.push(scorer.id.clone());
thresholds.push(scorer.pass_if);
}
}
if ids.is_empty() {
return Ok(by_experiment);
}
let rows = sqlx::query!(
"SELECT s.experiment_id AS \"experiment_id!\", s.scorer_id AS \"scorer_id!\",
avg(s.score) AS mean,
count(s.score) AS \"scored!\",
count(*) FILTER (WHERE s.error IS NOT NULL) AS \"failed!\",
count(*) FILTER (WHERE t.pass_if IS NOT NULL AND s.score >= t.pass_if)
AS \"passed!\",
bool_or(t.pass_if IS NOT NULL) AS \"has_threshold!\"
FROM eval_score s
JOIN unnest($1::uuid[], $2::text[], $3::float8[])
AS t(experiment_id, scorer_id, pass_if)
ON t.experiment_id = s.experiment_id AND t.scorer_id = s.scorer_id
GROUP BY s.experiment_id, s.scorer_id",
&ids,
&scorer_ids,
&thresholds as &[Option<f64>],
)
.fetch_all(db)
.await?;
let mut headline: std::collections::HashMap<
(Uuid, String),
(Option<f64>, i64, i64, i64, bool),
> = Default::default();
for row in rows {
headline.insert(
(row.experiment_id, row.scorer_id),
(
row.mean,
row.scored,
row.failed,
row.passed,
row.has_threshold,
),
);
}
// Emitted in the dataset's column order rather than the query's, so the badges on a row read
// left to right the way that dataset's table does.
for experiment in experiments {
for scorer in scorers_by_dataset
.get(&experiment.dataset)
.map(|s| s.as_slice())
.unwrap_or(&[])
{
// A column with no cells at all on this run is one added after it. A column that has
// cells is reported even where none produced a number, which is what a column that
// failed throughout looks like.
let Some((mean, scored, failed, passed, has_threshold)) =
headline.get(&(experiment.id, scorer.id.clone()))
else {
continue;
};
by_experiment
.entry(experiment.id)
.or_default()
.push(ExperimentScore {
scorer_id: scorer.id.clone(),
name: scorer_name(scorer),
kind: scorer.def.kind_str().to_string(),
mean: *mean,
pass_rate: (*has_threshold && *scored > 0)
.then(|| *passed as f64 / *scored as f64),
scored: *scored,
failed: *failed,
});
}
}
Ok(by_experiment)
}
/// The scorers of every dataset named by a listed run, read through `user_db` so a run of a
/// dataset the caller cannot read contributes nothing.
async fn scorers_of_listed(
authed: &ApiAuthed,
user_db: &UserDB,
w_id: &str,
experiments: &[EvalExperiment],
) -> Result<std::collections::HashMap<String, Vec<Scorer>>> {
let paths: Vec<String> = experiments
.iter()
.map(|e| e.dataset.clone())
.collect::<std::collections::HashSet<_>>()
.into_iter()
.collect();
if paths.is_empty() {
return Ok(Default::default());
}
let mut tx = user_db.clone().begin(authed).await?;
let rows = sqlx::query!(
"SELECT path, scorers FROM eval_dataset WHERE workspace_id = $1 AND path = ANY($2)",
w_id,
&paths
)
.fetch_all(&mut *tx)
.await?;
tx.commit().await?;
rows.into_iter()
.map(|row| Ok((row.path, parse_scorers(row.scorers)?)))
.collect()
}
#[derive(Deserialize)]
pub struct ExperimentRef {
pub id: Uuid,
/// The experiment every column is compared against. A delta is only ever computed between two
/// scores of the same scorer id.
#[serde(default)]
pub baseline: Option<Uuid>,
}
/// One scorer's verdict on one run, and how it compares with the baseline.
#[derive(Serialize)]
pub struct CellScore {
pub scorer_id: String,
#[serde(skip_serializing_if = "Option::is_none")]
pub score: Option<f64>,
#[serde(skip_serializing_if = "Option::is_none")]
pub reason: Option<String>,
#[serde(skip_serializing_if = "Option::is_none")]
pub checks: Option<Box<RawValue>>,
#[serde(skip_serializing_if = "Option::is_none")]
pub error: Option<String>,
/// The scorer read this case and had nothing to measure on it. Left out of the column's mean
/// and pass rate rather than counted as a zero.
#[serde(skip_serializing_if = "std::ops::Not::not")]
pub not_applicable: bool,
/// A scoring job is still running for this cell.
pub pending: bool,
/// Which side of the scorer's threshold the score fell on, when it has one.
#[serde(skip_serializing_if = "Option::is_none")]
pub passed: Option<bool>,
#[serde(skip_serializing_if = "Option::is_none")]
pub baseline: Option<f64>,
/// The baseline's score for this scorer was produced by a different definition of it, so the
/// delta is a change of scorer as much as a change of agent.
pub definition_changed: bool,
}
/// One row per case: what it was asked, what the agent answered, and each scorer's cell.
#[derive(Serialize)]
pub struct ExperimentRow {
pub case_id: Uuid,
pub input: EvalCaseInput,
#[serde(skip_serializing_if = "Option::is_none")]
pub expected: Option<Box<RawValue>>,
/// The iteration that ran this case. Absent between a run being recorded and its flow
/// reaching this case, which reads as a case still to run.
#[serde(skip_serializing_if = "Option::is_none")]
pub job_id: Option<Uuid>,
/// What happened to the answer: the iteration's own `success`/`failure`/`canceled`/`skipped`
/// once it has finished, and until then the agent step's, since the answer is written before
/// the scorers that keep the iteration running have read it. `unavailable` for a case whose
/// job was retained away before anything read what it produced.
pub status: String,
/// The agent's answer, which is what a table cell shows. The whole trajectory stays
/// reachable through `job_id`, so the row carries the text rather than the result object.
#[serde(skip_serializing_if = "Option::is_none")]
pub output: Option<String>,
/// The agent version this cell ran against. Cells of one experiment can differ, which is what
/// the table says instead of averaging two versions silently.
#[serde(skip_serializing_if = "Option::is_none")]
pub subject_version: Option<i64>,
/// For a run of unsaved edits, the hash of the configuration this cell ran: edits move without
/// a version changing, and `resolve_deployed_draft` matches this against what is deployed.
#[serde(skip_serializing_if = "Option::is_none")]
pub subject_draft_hash: Option<String>,
/// One entry per scorer of the dataset, in column order.
pub scores: Vec<CellScore>,
}
/// A column's summary. There is no single number for a dataset: averaging a judge with an exact
/// match would invent one.
#[derive(Serialize)]
pub struct ScorerMean {
pub scorer_id: String,
#[serde(skip_serializing_if = "Option::is_none")]
pub mean: Option<f64>,
#[serde(skip_serializing_if = "Option::is_none")]
pub baseline_mean: Option<f64>,
/// The share of scored cells that passed, for a column with a threshold. Reported beside the
/// mean rather than instead of it: neither number answers the other's question.
#[serde(skip_serializing_if = "Option::is_none")]
pub pass_rate: Option<f64>,
#[serde(skip_serializing_if = "Option::is_none")]
pub baseline_pass_rate: Option<f64>,
pub scored: usize,
/// Cells the baseline has no score for, reported so a column the baseline never ran shows as
/// unscored rather than as a spurious difference.
pub missing_in_baseline: usize,
pub definition_changed: bool,
}
#[derive(Serialize)]
pub struct ExperimentResults {
pub experiment: EvalExperiment,
#[serde(skip_serializing_if = "Option::is_none")]
pub baseline: Option<EvalExperiment>,
/// The columns, which belong to the dataset rather than to the experiment.
pub scorers: Vec<Scorer>,
pub rows: Vec<ExperimentRow>,
pub means: Vec<ScorerMean>,
/// Cells scoring lower than the baseline, across every column.
pub regressed: usize,
/// The version the subject is on now. A row that ran against an earlier one describes an
/// agent that no longer exists.
#[serde(skip_serializing_if = "Option::is_none")]
pub subject_current_version: Option<i64>,
/// What the agent hashes to as deployed. A run of unsaved edits carrying this hash ran exactly
/// what is deployed now — the edits were saved — so it is a run of that version.
#[serde(skip_serializing_if = "Option::is_none")]
pub subject_deployed_hash: Option<String>,
}
/// The agent's own result is `{output, messages}`; the answer is its `output`.
pub(crate) fn agent_answer(result: &RawValue) -> Option<String> {
let parsed: serde_json::Value = serde_json::from_str(result.get()).ok()?;
match parsed.get("output") {
Some(serde_json::Value::String(s)) => Some(s.clone()),
Some(other) => Some(other.to_string()),
None => None,
}
}
struct ScoreRow {
score: Option<f64>,
reason: Option<String>,
checks: Option<serde_json::Value>,
error: Option<String>,
not_applicable: bool,
definition: String,
}
/// Every score of one experiment, keyed by the cell and the scorer that produced it.
async fn load_scores(
db: &DB,
experiment_id: Uuid,
) -> Result<std::collections::HashMap<(i32, String), ScoreRow>> {
Ok(sqlx::query!(
"SELECT ordinal, scorer_id, score, reason, checks, error, not_applicable, definition
FROM eval_score WHERE experiment_id = $1",
experiment_id
)
.fetch_all(db)
.await?
.into_iter()
.map(|r| {
(
(r.ordinal, r.scorer_id),
ScoreRow {
score: r.score,
reason: r.reason,
checks: r.checks,
error: r.error,
not_applicable: r.not_applicable,
definition: r.definition,
},
)
})
.collect())
}
async fn read_experiment(db: &DB, w_id: &str, dataset: &str, id: Uuid) -> Result<EvalExperiment> {
let row = sqlx::query!(
"SELECT e.subject, e.run_number, e.run_job_id, e.created_at,
e.created_by,
(SELECT count(*) FROM eval_experiment_case c WHERE c.experiment_id = e.id)
AS \"case_count!\"
FROM eval_experiment e
WHERE e.workspace_id = $1 AND e.dataset_path = $2 AND e.id = $3",
w_id,
dataset,
id
)
.fetch_optional(db)
.await?
.ok_or_else(|| {
Error::NotFound(format!(
"Experiment {} not found in eval dataset {}",
id, dataset
))
})?;
experiment_from_row(
id,
dataset.to_string(),
row.subject,
row.run_number,
row.run_job_id,
row.case_count,
row.created_at,
row.created_by,
)
}
/// Recognise a draft run that has since been deployed, and record it as the version it became.
///
/// Written once rather than derived per read: derived against what is deployed *now*, the next
/// deployment would send a run that already read `v21` back to `v18 + edits`.
async fn resolve_deployed_draft(
db: &DB,
w_id: &str,
dataset: &str,
experiment: &mut EvalExperiment,
deployed_hash: Option<&str>,
deployed_version: Option<i64>,
) -> Result<()> {
if experiment.subject.kind != EvalSubjectKind::AgentDraft {
return Ok(());
}
let (Some(hash), Some(deployed_hash), Some(version)) = (
experiment.subject.draft_hash.as_deref(),
deployed_hash,
deployed_version,
) else {
return Ok(());
};
if hash != deployed_hash {
return Ok(());
}
// The hash stays: it is what identifies the configuration, and what this resolution rests on.
experiment.subject.kind = EvalSubjectKind::Agent;
experiment.subject.version = Some(version);
// Both writes in one transaction: a failure between them would leave the experiment promoted
// to a version while its cells stayed a draft's, a split no later read repairs since the
// experiment is no longer a draft.
let mut tx = db.begin().await?;
sqlx::query!(
"UPDATE eval_experiment
SET subject = jsonb_set(
jsonb_set(subject, '{kind}', '\"agent\"'),
'{version}', to_jsonb($4::bigint))
WHERE workspace_id = $1 AND dataset_path = $2 AND id = $3
AND subject ->> 'kind' = 'agent_draft'",
w_id,
dataset,
experiment.id,
version,
)
.execute(&mut *tx)
.await?;
// The cells that ran that configuration are dated by the version too; leaving their hash would
// make the run go on reading as a draft's after the next deployment.
sqlx::query!(
"UPDATE eval_experiment_case
SET subject_version = $3, subject_draft_hash = NULL
WHERE experiment_id = $1 AND subject_draft_hash = $2",
experiment.id,
hash,
version,
)
.execute(&mut *tx)
.await?;
tx.commit().await?;
Ok(())
}
/// Record what a run produced, from inside the run: the last step of a run's own flow calls this.
///
/// Gated on reading the run rather than on writing its dataset, unlike everything else here: it is
/// the same harvest `experiment_results` performs behind the same check, over the run's own cells,
/// and it reports a count rather than any of what it read.
pub async fn collect_experiment(
authed: ApiAuthed,
Extension(db): Extension<DB>,
Extension(user_db): Extension<UserDB>,
Path(w_id): Path<String>,
Query(query): Query<ExperimentId>,
) -> JsonResult<usize> {
// Through `user_db`, so the run is one the caller can see. The row carries the job to read it
// out of, so nothing that is read afterwards is caller-supplied.
let mut tx = user_db.begin(&authed).await?;
let experiment = sqlx::query!(
"SELECT id, run_job_id FROM eval_experiment WHERE workspace_id = $1 AND id = $2",
w_id,
query.id
)
.fetch_optional(&mut *tx)
.await?;
tx.commit().await?;
let experiment =
experiment.ok_or_else(|| Error::NotFound(format!("Eval run {} not found", query.id)))?;
sync_run(&db, &w_id, experiment.id, experiment.run_job_id, true).await?;
let recorded = sqlx::query_scalar!(
"SELECT count(*) AS \"count!\" FROM eval_experiment_case
WHERE experiment_id = $1 AND status IS NOT NULL",
experiment.id
)
.fetch_one(&db)
.await?;
Ok(Json(recorded as usize))
}
#[derive(Deserialize)]
pub struct ExperimentId {
pub id: Uuid,
}
/// Collect a run for a reader, without letting the collection decide whether the read succeeds.
/// `collect_experiment` propagates instead: it is the run reporting on itself, and a failure there
/// is worth surfacing to the step that called it.
async fn collect_quietly(db: &DB, w_id: &str, experiment_id: Uuid, run_job_id: Uuid) {
if let Err(e) = sync_run(db, w_id, experiment_id, run_job_id, true).await {
tracing::warn!("could not collect eval run {}: {e:#}", experiment_id);
}
}
/// The rows a results table is built from. The job ids come out of `eval_experiment_case`, which
/// only this module writes, so they can be read on the unrestricted pool once the dataset read
/// below has established the caller's access.
pub async fn experiment_results(
authed: ApiAuthed,
Extension(db): Extension<DB>,
Extension(user_db): Extension<UserDB>,
Path((w_id, dataset)): Path<(String, String)>,
Query(query): Query<ExperimentRef>,
) -> JsonResult<ExperimentResults> {
// The rows carry what the run's jobs produced, which `jobs:read` gates. `UserDB` settles who
// may see the dataset; a token's scopes are a separate question.
check_scopes(&authed, || "jobs:read".to_string())?;
let dataset_row = read_dataset(&authed, &user_db, &w_id, &dataset).await?;
let scorers = dataset_row.scorers;
let mut experiment = read_experiment(&db, &w_id, &dataset, query.id).await?;
// Best-effort: collecting is what the run's own step is for, and a cell that could not be read
// — a job retained away between the iteration and its children — must not take the whole table
// down with it. The rows already recorded are still the run.
collect_quietly(&db, &w_id, query.id, experiment.run_job_id).await;
let scores = load_scores(&db, query.id).await?;
let baseline = match query.baseline.filter(|id| *id != query.id) {
Some(id) => {
let baseline = read_experiment(&db, &w_id, &dataset, id).await?;
collect_quietly(&db, &w_id, id, baseline.run_job_id).await;
Some((baseline, load_scores(&db, id).await?))
}
None => None,
};
// The baseline is compared case by case, so its cells are keyed by the case they ran.
let baseline_ordinals = match &baseline {
Some((baseline, _)) => sqlx::query!(
"SELECT case_id, ordinal FROM eval_experiment_case WHERE experiment_id = $1",
baseline.id
)
.fetch_all(&db)
.await?
.into_iter()
.map(|r| (r.case_id, r.ordinal))
.collect::<std::collections::HashMap<_, _>>(),
None => Default::default(),
};
let case_rows = sqlx::query!(
"SELECT ordinal, case_id, input, expected, job_id, subject_version,
subject_draft_hash, output, answered, status
FROM eval_experiment_case
WHERE experiment_id = $1 ORDER BY ordinal",
query.id
)
.fetch_all(&db)
.await?;
let mut sums = vec![(0.0f64, 0usize); scorers.len()];
let mut baseline_sums = vec![(0.0f64, 0usize); scorers.len()];
let mut passes = vec![0usize; scorers.len()];
let mut baseline_passes = vec![0usize; scorers.len()];
let mut missing_in_baseline = vec![0usize; scorers.len()];
let mut definition_changed = vec![false; scorers.len()];
let mut regressed = 0usize;
let mut rows = Vec::with_capacity(case_rows.len());
for case in case_rows {
let mut cells = Vec::with_capacity(scorers.len());
for (index, scorer) in scorers.iter().enumerate() {
let current = scores.get(&(case.ordinal, scorer.id.clone()));
let baseline_score = baseline.as_ref().and_then(|(_, baseline_scores)| {
baseline_ordinals
.get(&case.case_id)
.and_then(|ordinal| baseline_scores.get(&(*ordinal, scorer.id.clone())))
});
if let Some(score) = current.and_then(|c| c.score) {
sums[index].0 += score;
sums[index].1 += 1;
if scorer.passed(Some(score)) == Some(true) {
passes[index] += 1;
}
}
if let Some(score) = baseline_score.and_then(|b| b.score) {
baseline_sums[index].0 += score;
baseline_sums[index].1 += 1;
if scorer.passed(Some(score)) == Some(true) {
baseline_passes[index] += 1;
}
} else if baseline.is_some() {
missing_in_baseline[index] += 1;
}
let changed = match (current, baseline_score) {
(Some(current), Some(baseline)) => current.definition != baseline.definition,
_ => false,
};
if changed {
definition_changed[index] = true;
}
if let (Some(score), Some(previous)) = (
current.and_then(|c| c.score),
baseline_score.and_then(|b| b.score),
) {
if score < previous {
regressed += 1;
}
}
cells.push(CellScore {
scorer_id: scorer.id.clone(),
score: current.and_then(|c| c.score),
reason: current.and_then(|c| c.reason.clone()),
checks: current
.and_then(|c| c.checks.clone())
.map(|c| serde_json::value::to_raw_value(&c))
.transpose()?,
error: current.and_then(|c| c.error.clone()),
not_applicable: current.map(|c| c.not_applicable).unwrap_or(false),
// A row exists because the run was launched with this scorer, so an empty one is a
// score still to come, unless the scorer has already said this case is not one it
// measures.
pending: current
.map(|c| c.score.is_none() && c.error.is_none() && !c.not_applicable)
.unwrap_or(false),
passed: scorer.passed(current.and_then(|c| c.score)),
baseline: baseline_score.and_then(|b| b.score),
definition_changed: changed,
});
}
rows.push(ExperimentRow {
case_id: case.case_id,
input: serde_json::from_value(case.input)?,
expected: opt_to_raw(case.expected)?,
// The iteration's verdict once it has one. While it is still running, the agent step's:
// the answer is written before the scorers read it, and a spinner beside an answer
// already there reads as an answer still being written.
status: case
.status
.or_else(|| {
case.answered
.map(|ok| if ok { "success" } else { "failure" }.to_string())
})
.unwrap_or_else(|| "running".to_string()),
output: case.output,
subject_version: case.subject_version,
subject_draft_hash: case.subject_draft_hash,
job_id: case.job_id,
scores: cells,
});
}
let means = scorers
.iter()
.enumerate()
.map(|(index, scorer)| ScorerMean {
scorer_id: scorer.id.clone(),
mean: (sums[index].1 > 0).then(|| sums[index].0 / sums[index].1 as f64),
baseline_mean: (baseline_sums[index].1 > 0)
.then(|| baseline_sums[index].0 / baseline_sums[index].1 as f64),
pass_rate: (scorer.pass_if.is_some() && sums[index].1 > 0)
.then(|| passes[index] as f64 / sums[index].1 as f64),
baseline_pass_rate: (scorer.pass_if.is_some() && baseline_sums[index].1 > 0)
.then(|| baseline_passes[index] as f64 / baseline_sums[index].1 as f64),
scored: sums[index].1,
missing_in_baseline: missing_in_baseline[index],
definition_changed: definition_changed[index],
})
.collect();
// Read as the caller, so a viewer who can see the dataset but not the agent gets neither: the
// agent's version and configuration fingerprint must not leak past its own read permission.
let (subject_deployed_hash, subject_current_version) =
match readable_agent_state(&authed, &user_db, &w_id, &experiment.subject.path).await? {
Some((config, version)) => (Some(draft_hash(&config)), Some(version)),
None => (None, None),
};
// A run of unsaved edits whose configuration has since been deployed is a run of that version.
let mut baseline = baseline.map(|(baseline, _)| baseline);
resolve_deployed_draft(
&db,
&w_id,
&dataset,
&mut experiment,
subject_deployed_hash.as_deref(),
subject_current_version,
)
.await?;
if let Some(baseline) = baseline.as_mut() {
// The compare-to list holds this agent's runs, but the id is the caller's: a run of another
// agent must not be stamped with this one's version.
if baseline.subject.path == experiment.subject.path {
resolve_deployed_draft(
&db,
&w_id,
&dataset,
baseline,
subject_deployed_hash.as_deref(),
subject_current_version,
)
.await?;
}
}
Ok(Json(ExperimentResults {
experiment,
baseline,
scorers,
rows,
means,
regressed,
subject_current_version,
subject_deployed_hash,
}))
}
backend/windmill-api/src/ai_evals/run.rs
+950
View File
@@ -0,0 +1,950 @@
use super::*;
/// Node id of the agent step. The answer is read back by this id, so it is part of the stored
/// shape rather than an implementation detail.
pub const AGENT_NODE_ID: &str = "a";
/// Node id of the step that assembles what the scorers are handed.
const PAYLOAD_NODE_ID: &str = "p";
/// Node id of the loop over the dataset's cases.
const CASES_NODE_ID: &str = "cases";
/// The branch holding every scorer of a case, so they measure it at the same time.
const SCORERS_NODE_ID: &str = "scores";
/// In-flight iterations. A dataset is a burst of calls to one provider, so answering every case at
/// once is a run that spends its time being rate-limited.
const RUN_PARALLELISM: u16 = 8;
/// What each iteration is handed: the case, small enough to sit in every iteration's arguments.
#[derive(Serialize)]
struct CaseIteration {
case_id: Uuid,
ordinal: i32,
input: EvalCaseInput,
#[serde(skip_serializing_if = "Option::is_none")]
expected: Option<Box<RawValue>>,
}
/// Assembles the payload the scorers read.
///
/// A step rather than an input transform: every tool call is enriched with the arguments, result,
/// status and duration of the job that ran it, none of which the flow can see.
const PAYLOAD_SCRIPT: &str = r#"//native
// Generated by Windmill: reads the run this iteration answered.
export async function main() {
const id = process.env.WM_FLOW_JOB_ID
const base = process.env.BASE_URL || process.env.BASE_INTERNAL_URL
const res = await fetch(
`${base}/api/w/${process.env.WM_WORKSPACE}/ai_evals/run_payload?job_id=${id}`,
{ headers: { Authorization: `Bearer ${process.env.WM_TOKEN}` } }
)
if (!res.ok) {
throw new Error(`could not read the run of job ${id}: ${res.status} ${await res.text()}`)
}
return await res.json()
}
"#;
fn payload_module() -> serde_json::Value {
serde_json::json!({
"id": PAYLOAD_NODE_ID,
"summary": "Assemble the run the scorers read",
"value": {
"type": "rawscript",
// `bunnative` (tag `nativets`), matching the `//native` the script carries. That tag
// belongs to the `native` worker group rather than the default one, so a queued
// iteration never starts when nothing serves it.
"language": "bunnative",
"content": PAYLOAD_SCRIPT,
"lock": EMPTY_BUN_LOCK,
"input_transforms": {}
}
})
}
/// Node id of the step that records what the run produced.
const COLLECT_NODE_ID: &str = "collect";
/// Copies the run's answers and scores into its own rows, from inside the run.
///
/// The tables know nothing about the flow, so without this a run started and left is only ever
/// recorded by someone looking at it — after its jobs have been retained away, there is nothing
/// left to record.
const COLLECT_SCRIPT: &str = r#"//native
// Generated by Windmill: records what this run produced, so it outlives the jobs that produced it.
export async function main(experiment_id: string) {
const base = process.env.BASE_URL || process.env.BASE_INTERNAL_URL
const res = await fetch(
`${base}/api/w/${process.env.WM_WORKSPACE}/ai_evals/experiments/collect?id=${experiment_id}`,
{ method: 'POST', headers: { Authorization: `Bearer ${process.env.WM_TOKEN}` } }
)
if (!res.ok) {
throw new Error(`could not record run ${experiment_id}: ${res.status} ${await res.text()}`)
}
return await res.json()
}
"#;
fn collect_module(experiment_id: Uuid) -> serde_json::Value {
serde_json::json!({
"id": COLLECT_NODE_ID,
"summary": "Record what the run produced",
// Bookkeeping, so it does not decide whether the run succeeded. What it would have written
// is written again by the first read of the run.
"continue_on_error": true,
"value": {
"type": "rawscript",
"language": "bunnative",
"content": COLLECT_SCRIPT,
"lock": EMPTY_BUN_LOCK,
"input_transforms": {
"experiment_id": {
"type": "static",
"value": experiment_id.to_string(),
},
}
}
})
}
/// The script imports nothing, so its lockfile is the empty one, spelled the way the bun executor
/// splits it. Without a lock a worker running this as bun would resolve dependencies every time.
const EMPTY_BUN_LOCK: &str = "{\n \"dependencies\": {}\n}\n//bun.lock\n<empty>";
/// What a judge is asked about the case: the run as it reads it, and the case's own attachments,
/// handed to it as they were handed to the agent.
fn judge_case_transforms() -> serde_json::Map<String, serde_json::Value> {
let mut transforms = serde_json::Map::new();
transforms.insert(
"user_message".to_string(),
serde_json::json!({
"type": "javascript",
"expr": format!("results.{}.rendered", PAYLOAD_NODE_ID),
}),
);
transforms.insert(
"user_attachments".to_string(),
serde_json::json!({
"type": "javascript",
"expr": format!("results.{}.run.input.user_attachments", PAYLOAD_NODE_ID),
}),
);
transforms
}
/// The scorer steps of one iteration, reading the payload the step before them assembled.
///
/// Each scorer is baked in as `resolve_scorer` resolved it at launch, never linked by path: a
/// linked step resolves the resource when the case reaches it, so a scorer edited mid-run would
/// grade the later cases while every score still names the definition recorded at launch.
fn scorer_modules(scorers: &[(&Scorer, ResolvedScorer)]) -> Vec<serde_json::Value> {
scorers
.iter()
.map(|(scorer, resolved)| {
let value = match resolved {
// A judge is an agent handed the run as its message; its own system prompt is the
// grading contract, which is why editing a judge means editing that agent.
ResolvedScorer::Judge { config } => {
let mut transforms = match &config.input_transforms {
serde_json::Value::Object(map) => map.clone(),
_ => serde_json::Map::new(),
};
transforms.extend(judge_case_transforms());
serde_json::json!({
"type": "aiagent",
"tools": config.tools,
"input_transforms": serde_json::Value::Object(transforms),
})
}
// `run` is the whole payload; `input`, `output` and `expected` are the same values
// spelled out, so a three-line scorer does not have to reach into it.
ResolvedScorer::Script { hash } => serde_json::json!({
"type": "script",
"path": scorer.def.path(),
// Serialized as `ScriptHash` (a hex string), which is the only shape a flow
// module's `hash` field deserializes from — a bare number fails in the worker.
"hash": windmill_common::scripts::ScriptHash(*hash),
"input_transforms": {
"run": {
"type": "javascript",
"expr": format!("results.{}.run", PAYLOAD_NODE_ID),
},
"input": {
"type": "javascript",
"expr": format!("results.{}.run.input", PAYLOAD_NODE_ID),
},
"output": {
"type": "javascript",
"expr": format!("results.{}.run.output", PAYLOAD_NODE_ID),
},
"expected": {
"type": "javascript",
"expr": format!("results.{}.run.expected", PAYLOAD_NODE_ID),
},
}
}),
};
serde_json::json!({ "id": scorer_module_id(&scorer.id), "value": value })
})
.collect()
}
/// The flow a whole run is: one loop over the dataset's cases, each iteration answering the case
/// and then scoring the answer.
///
/// One job rather than one per case: a run outlives the tab that started it, and only a worker can
/// notice that the last case finished. The cases live in the flow's value, stored once, rather
/// than in its arguments, which every iteration inherits a copy of.
fn build_run_flow(
config: &AgentDraft,
cases: &[CaseIteration],
scorers: &[(&Scorer, ResolvedScorer)],
experiment_id: Uuid,
) -> Result<windmill_common::flows::FlowValue> {
let mut modules: Vec<serde_json::Value> = vec![agent_module(config)?];
if !scorers.is_empty() {
modules.push(payload_module());
// One branch each, run together: scorers read the answer and never each other. Each branch
// keeps its own failure, so a judge that errors costs its own column and no other.
modules.push(serde_json::json!({
"id": SCORERS_NODE_ID,
"value": {
"type": "branchall",
"parallel": true,
"branches": scorers
.iter()
.zip(scorer_modules(scorers))
.map(|((scorer, _), module)| serde_json::json!({
// Named for the column it produces: the graph of a run is read to see which
// scorer did what, and a module id is not what a scorer is called.
"summary": scorer_name(scorer),
"skip_failure": true,
"modules": [module],
}))
.collect::<Vec<_>>(),
}
}));
}
Ok(serde_json::from_value(serde_json::json!({
"modules": [
{
"id": CASES_NODE_ID,
"value": {
"type": "forloopflow",
"iterator": { "type": "static", "value": cases },
"parallel": true,
"parallelism": RUN_PARALLELISM,
// One case failing is one cell of the run, not the end of it.
"skip_failures": true,
"modules": modules,
}
},
// After the loop, so every case has both answered and been scored by the time it runs.
collect_module(experiment_id),
]
}))?)
}
/// The agent step, reading its case from the iteration rather than from the flow's arguments.
fn agent_module(config: &AgentDraft) -> Result<serde_json::Value> {
let flow = build_case_flow(config)?;
let mut value = serde_json::to_value(&flow.modules[0].value)?;
if let Some(map) = value.as_object_mut() {
let transforms = map
.entry("input_transforms")
.or_insert_with(|| serde_json::json!({}));
if let Some(transforms) = transforms.as_object_mut() {
for key in ["user_message", "user_attachments"] {
transforms.insert(
key.to_string(),
serde_json::json!({
"type": "javascript",
"expr": format!("flow_input.iter.value.input.{}", key),
}),
);
}
}
}
Ok(serde_json::json!({ "id": AGENT_NODE_ID, "value": value }))
}
/// The agent step as a one-module flow, so the module shape is validated by deserializing
/// through `FlowValue` rather than trusted as raw JSON.
fn build_case_flow(config: &AgentDraft) -> Result<windmill_common::flows::FlowValue> {
// The configuration runs exactly as authored: its own brain transforms are the module's, and
// the case supplies the message and the attachments over the top.
let mut input_transforms = match &config.input_transforms {
serde_json::Value::Object(map) => map.clone(),
_ => serde_json::Map::new(),
};
for key in ["user_message", "user_attachments"] {
input_transforms.insert(
key.to_string(),
serde_json::json!({ "type": "javascript", "expr": format!("flow_input.{}", key) }),
);
}
// Always inlined, never a link to the resource: a linked step would resolve the agent when
// each case runs, which is the one thing a run of a named version must not do.
let mut agent_value = serde_json::Map::new();
agent_value.insert("type".to_string(), serde_json::json!("aiagent"));
agent_value.insert("tools".to_string(), serde_json::json!(config.tools));
agent_value.insert(
"input_transforms".to_string(),
serde_json::Value::Object(input_transforms),
);
Ok(serde_json::from_value(serde_json::json!({
"modules": [{ "id": AGENT_NODE_ID, "value": serde_json::Value::Object(agent_value) }]
}))?)
}
/// How many times the agent has been saved, not the identity of the row holding that value: runs
/// are named by it and compared by it, so it has to be the resource's own count rather than a
/// sequence the whole instance shares.
pub(crate) async fn current_resource_version(
db: &DB,
w_id: &str,
path: &str,
) -> Result<Option<i64>> {
let version = sqlx::query_scalar!(
"SELECT version FROM resource_version WHERE workspace_id = $1 AND path = $2
ORDER BY version DESC LIMIT 1",
w_id,
path
)
.fetch_optional(db)
.await?;
Ok(version)
}
/// Read the agent through `user_db` so a caller who cannot read the resource cannot run it.
pub(crate) async fn require_agent(
authed: &ApiAuthed,
user_db: &UserDB,
w_id: &str,
agent_path: &str,
) -> Result<()> {
let mut tx = user_db.clone().begin(authed).await?;
let resource_type = sqlx::query_scalar!(
"SELECT resource_type FROM resource WHERE workspace_id = $1 AND path = $2",
w_id,
agent_path
)
.fetch_optional(&mut *tx)
.await?;
tx.commit().await?;
match resource_type.as_deref() {
Some("ai_agent") => Ok(()),
Some(other) => Err(Error::BadRequest(format!(
"Resource {} is a {}, not an ai_agent",
agent_path, other
))),
None => Err(Error::NotFound(format!("Agent {} not found", agent_path))),
}
}
/// An `ai_agent` value as the configuration to run it with: its brain becomes the module's input
/// transforms, its tools the module's tools. The same conversion for a draft and for what is
/// deployed, so the two hash comparably — which is what lets a draft run be recognised as the
/// version it became.
fn config_to_draft(value: serde_json::Value) -> Result<AgentDraft> {
let mut config = match value {
serde_json::Value::Object(map) => map,
_ => return Err(Error::BadRequest("The agent is not an object".to_string())),
};
let tools = match config.remove("tools") {
Some(serde_json::Value::Array(tools)) => tools,
_ => vec![],
};
// Every brain key becomes a static transform: `$res:`/`$var:` in them are resolved by the
// same argument machinery a linked step's resource goes through.
let input_transforms = config
.into_iter()
.map(|(key, value)| (key, serde_json::json!({ "type": "static", "value": value })))
.collect::<serde_json::Map<_, _>>();
Ok(AgentDraft { input_transforms: serde_json::Value::Object(input_transforms), tools })
}
/// An agent's deployed value and the version that names it, in the shape a step runs. `None` when
/// the caller cannot see the resource, or it is not a usable agent.
///
/// Both from one read: a deploy landing between two reads would pair one version's configuration
/// with another's number, and what a run records of its subject is permanent.
pub(crate) async fn readable_agent_state(
authed: &ApiAuthed,
user_db: &UserDB,
w_id: &str,
path: &str,
) -> Result<Option<(AgentDraft, i64)>> {
let mut tx = user_db.clone().begin(authed).await?;
let row = sqlx::query!(
"SELECT r.value AS \"value: sqlx::types::Json<serde_json::Value>\",
(SELECT version FROM resource_version v
WHERE v.workspace_id = r.workspace_id AND v.path = r.path
ORDER BY v.version DESC LIMIT 1) AS version
FROM resource r
WHERE r.workspace_id = $1 AND r.path = $2 AND r.resource_type = 'ai_agent'",
w_id,
path
)
.fetch_optional(&mut *tx)
.await?;
tx.commit().await?;
let Some(row) = row else {
return Ok(None);
};
let (Some(value), Some(version)) = (row.value, row.version) else {
return Ok(None);
};
// A resource's value isn't validated against its type on write, so an ai_agent whose value
// isn't a valid config is no usable state rather than an error: one bad row would otherwise
// 400 the whole results page or run list.
match config_to_draft(value.0) {
Ok(config) => Ok(Some((config, version))),
Err(_) => Ok(None),
}
}
/// Fill in what the client cannot: the version a saved agent is at, or the configuration a past
/// version held.
///
/// Returns the configuration the run executes, read once here. Every case then executes that one
/// configuration: resolved per case instead, an agent deployed mid-run would be executed by the
/// cases after it while every row still names the version the run started against.
async fn resolve_subject(
authed: &ApiAuthed,
db: &DB,
user_db: &UserDB,
w_id: &str,
subject: &mut EvalSubject,
draft: Option<AgentDraft>,
) -> Result<AgentDraft> {
Ok(match subject.kind {
EvalSubjectKind::Agent => {
let Some((config, version)) =
readable_agent_state(authed, user_db, w_id, &subject.path).await?
else {
return Err(Error::BadRequest(format!(
"Agent {} is not a readable ai_agent resource",
subject.path
)));
};
subject.version = Some(version);
config
}
EvalSubjectKind::AgentDraft => {
// The edits live nowhere the server can read them, so the request carries them. The
// agent is still read, so a run can only be filed under one the caller can see.
require_agent(authed, user_db, w_id, &subject.path).await?;
// The version the edits are an edit of, as of now: edits record no version of their
// own, so "v15 plus unsaved edits" means the edits and whatever was deployed when the
// run started.
subject.version = current_resource_version(db, w_id, &subject.path).await?;
draft.ok_or_else(|| Error::BadRequest(DRAFT_REQUIRED.to_string()))?
}
EvalSubjectKind::AgentVersion => {
let Some(version) = subject.version else {
return Err(Error::BadRequest(
"A run of a past version must say which version".to_string(),
));
};
let config =
agent_version_config(authed, user_db, db, w_id, &subject.path, version).await?;
subject.draft = Some(config.clone());
config
}
})
}
/// One version of an agent out of its history, in the shape a step runs.
///
/// Read through `user_db` for the agent itself first: a version is the resource as it was, so
/// seeing one is seeing the resource.
async fn agent_version_config(
authed: &ApiAuthed,
user_db: &UserDB,
db: &DB,
w_id: &str,
agent_path: &str,
version: i64,
) -> Result<AgentDraft> {
require_agent(authed, user_db, w_id, agent_path).await?;
let value = sqlx::query_scalar!(
"SELECT value FROM resource_version
WHERE version = $1 AND workspace_id = $2 AND path = $3",
version,
w_id,
agent_path
)
.fetch_optional(db)
.await?
.flatten()
.ok_or_else(|| Error::NotFound(format!("Agent {} has no version {}", agent_path, version)))?;
config_to_draft(value).map_err(|_| {
Error::BadRequest(format!(
"Version {} of {} is not an object",
version, agent_path
))
})
}
const DRAFT_REQUIRED: &str = "A run of unsaved edits must carry the configuration being edited";
/// The configuration the request may carry, taken out of the subject it belongs to.
///
/// A saved agent and a past version are read from the workspace by the path they name, so a
/// request carrying a configuration for them would run something other than the agent it claims to
/// be a run of. Unsaved edits are the one kind the request has to carry: they exist only in the
/// editor.
fn validate_subject(subject: &EvalSubject) -> Result<Option<AgentDraft>> {
if subject.path.trim().is_empty() {
return Err(Error::BadRequest(
"The subject needs a path: it is the agent a run is filed under".to_string(),
));
}
match (&subject.draft, &subject.kind) {
(Some(draft), EvalSubjectKind::AgentDraft) => Ok(Some(draft.clone())),
(Some(_), _) => Err(Error::BadRequest(
"A saved agent's configuration is read from the workspace; remove it from the request"
.to_string(),
)),
(None, EvalSubjectKind::AgentDraft) => Err(Error::BadRequest(DRAFT_REQUIRED.to_string())),
(None, _) => Ok(None),
}
}
// -----------------------------------------------------------------------------------------------
// Experiments
// -----------------------------------------------------------------------------------------------
#[derive(Deserialize)]
pub struct RunExperiment {
pub dataset: String,
pub subject: EvalSubject,
}
/// Open a run of this dataset.
///
/// Runs are numbered per (dataset, agent) pair, and the deployed agent and its draft share that
/// numbering: they are the same agent, so "Run 7" of a dataset means one thing whether it ran the
/// deployed value or the edits waiting on top of it.
async fn new_run(
tx: &mut sqlx::Transaction<'_, sqlx::Postgres>,
w_id: &str,
dataset: &str,
subject: &EvalSubject,
username: &str,
run_job_id: Uuid,
id: Uuid,
) -> Result<Uuid> {
// Two runs starting together would otherwise read the same run number. Held for the rest of
// this transaction, which pushes no jobs.
sqlx::query!(
"SELECT pg_advisory_xact_lock(hashtext('ai_eval_open:' || $1 || '/' || $2 || '/' || $3))",
w_id,
dataset,
subject.path,
)
.execute(&mut **tx)
.await?;
let run_number = sqlx::query_scalar!(
"SELECT coalesce(max(run_number), 0) + 1 FROM eval_experiment
WHERE workspace_id = $1 AND dataset_path = $2 AND subject ->> 'path' = $3",
w_id,
dataset,
subject.path,
)
.fetch_one(&mut **tx)
.await?
.unwrap_or(1);
sqlx::query!(
"INSERT INTO eval_experiment
(id, workspace_id, dataset_path, subject, run_number, created_by, run_job_id)
VALUES ($1, $2, $3, $4, $5, $6, $7)",
id,
w_id,
dataset,
serde_json::to_value(subject.stamp())?,
run_number,
username,
run_job_id,
)
.execute(&mut **tx)
.await
.map_err(|e| {
if is_missing_dataset(&e) {
Error::NotFound(format!("Eval dataset {} not found", dataset))
} else {
e.into()
}
})?;
Ok(id)
}
pub async fn run_experiment(
authed: ApiAuthed,
Extension(db): Extension<DB>,
Extension(user_db): Extension<UserDB>,
Path(w_id): Path<String>,
Json(payload): Json<RunExperiment>,
) -> Result<String> {
check_scopes(&authed, || "jobs:run".to_string())?;
// A write, not a read: it persists an experiment into the dataset.
require_dataset_writable(&authed, &user_db, &w_id, &payload.dataset).await?;
let mut subject = payload.subject;
let draft = validate_subject(&subject)?;
let config = resolve_subject(&authed, &db, &user_db, &w_id, &mut subject, draft).await?;
// One snapshot of the dataset: the scorers and the cases a run records must be the same
// revision, or a Save landing between two reads files a run under cases and columns that never
// stood together.
let (dataset, cases) =
read_dataset_and_cases(&authed, &user_db, &w_id, &payload.dataset).await?;
if cases.is_empty() {
return Err(Error::BadRequest(format!(
"Eval dataset {} has no case to run",
payload.dataset
)));
}
let case_count = cases.len();
// Resolved through the caller's own db, so a run executes only runnables the caller may read,
// and what resolving pinned is baked into the flow. Recorded per cell at launch rather than
// when the score comes back, so a scorer edited mid-run reads as the change of scorer it is.
let mut definitions = Vec::with_capacity(dataset.scorers.len());
let mut scorers: Vec<(&Scorer, ResolvedScorer)> = Vec::with_capacity(dataset.scorers.len());
for scorer in &dataset.scorers {
let (definition, resolved) = resolve_scorer(&user_db, &authed, &w_id, scorer).await?;
definitions.push(definition);
scorers.push((scorer, resolved));
}
let iterations = cases
.iter()
.enumerate()
.map(|(index, case)| CaseIteration {
case_id: case.id,
ordinal: index as i32,
input: case.input.clone(),
expected: case.expected.clone(),
})
.collect::<Vec<_>>();
// Both ids are chosen here: the run's own collect step is handed the experiment id, and the
// experiment names its job before that job exists.
let experiment_id = Uuid::new_v4();
let run_job_id = Uuid::new_v4();
let flow_value = build_run_flow(&config, &iterations, &scorers, experiment_id)?;
// Recorded before the job is queued, so a launch that dies partway leaves an experiment naming
// a job that never started rather than a flow no experiment accounts for and nothing collects.
let mut tx = db.begin().await?;
let experiment_id = new_run(
&mut tx,
&w_id,
&payload.dataset,
&subject,
&authed.username,
run_job_id,
experiment_id,
)
.await?;
let ordinals = (0..case_count as i32).collect::<Vec<_>>();
let case_ids = cases.iter().map(|c| c.id).collect::<Vec<_>>();
let inputs = cases
.iter()
.map(|c| serde_json::to_value(&c.input))
.collect::<std::result::Result<Vec<_>, _>>()?;
let expecteds = cases
.iter()
.map(|c| opt_from_raw(c.expected.as_ref()))
.collect::<Result<Vec<_>>>()?;
let versions = vec![subject.version; case_count];
let hashes = vec![subject.draft.as_ref().map(draft_hash); case_count];
// No job id: the iteration that answers a case is minted by the flow engine, and the case is
// matched back to it once it exists.
sqlx::query!(
"INSERT INTO eval_experiment_case
(experiment_id, ordinal, case_id, input, expected, subject_version,
subject_draft_hash)
SELECT $1, ordinal, case_id, input, expected, subject_version, subject_draft_hash
FROM UNNEST($2::int[], $3::uuid[], $4::jsonb[], $5::jsonb[], $6::bigint[], $7::text[])
AS t(ordinal, case_id, input, expected, subject_version, subject_draft_hash)",
experiment_id,
&ordinals,
&case_ids,
&inputs,
&expecteds as &[Option<serde_json::Value>],
&versions as &[Option<i64>],
&hashes as &[Option<String>],
)
.execute(&mut *tx)
.await?;
insert_pending_scores(&mut tx, experiment_id, &ordinals, &scorers, &definitions).await?;
// The foreign key makes a delete racing this assembly fail the commit, so nothing is queued.
// A delete landing between this commit and the push below still cascades the experiment away
// while the flow queues; that launch/delete race is a known beta limitation.
tx.commit().await?;
if let Err(e) = push_run_flow(
&authed,
&db,
&user_db,
&w_id,
&payload.dataset,
&subject,
experiment_id,
run_job_id,
flow_value,
)
.await
{
// Nothing ran, so there is nothing to keep: one failed push is the whole run.
sqlx::query!("DELETE FROM eval_experiment WHERE id = $1", experiment_id)
.execute(&db)
.await?;
return Err(e);
}
Ok(experiment_id.to_string())
}
/// The cells a run will fill in, written at launch. A pending row is what the table reads as a
/// score still being produced, and it is where the definition that produced it is recorded.
async fn insert_pending_scores(
tx: &mut sqlx::Transaction<'_, sqlx::Postgres>,
experiment_id: Uuid,
ordinals: &[i32],
scorers: &[(&Scorer, ResolvedScorer)],
definitions: &[String],
) -> Result<()> {
if scorers.is_empty() || ordinals.is_empty() {
return Ok(());
}
let mut rows_ordinal = vec![];
let mut rows_scorer = vec![];
let mut rows_definition = vec![];
for ordinal in ordinals {
for ((scorer, _), definition) in scorers.iter().zip(definitions.iter()) {
rows_ordinal.push(*ordinal);
rows_scorer.push(scorer.id.clone());
rows_definition.push(definition.clone());
}
}
sqlx::query!(
"INSERT INTO eval_score (experiment_id, ordinal, scorer_id, definition)
SELECT $1, ordinal, scorer_id, definition
FROM UNNEST($2::int[], $3::text[], $4::text[]) AS t(ordinal, scorer_id, definition)
ON CONFLICT (experiment_id, ordinal, scorer_id)
DO UPDATE SET definition = EXCLUDED.definition, score = NULL, reason = NULL,
checks = NULL, error = NULL, not_applicable = false",
experiment_id,
&rows_ordinal,
&rows_scorer,
&rows_definition,
)
.execute(&mut **tx)
.await?;
Ok(())
}
/// Queue the flow a run is. Its id is chosen by the caller, so the experiment can name it before
/// it exists.
async fn push_run_flow(
authed: &ApiAuthed,
db: &DB,
user_db: &UserDB,
w_id: &str,
dataset: &str,
subject: &EvalSubject,
experiment_id: Uuid,
run_job_id: Uuid,
flow_value: windmill_common::flows::FlowValue,
) -> Result<Uuid> {
use windmill_common::{jobs::JobPayload, users::username_to_permissioned_as};
use windmill_queue::{push, PushArgs, PushIsolationLevel};
let mut args = std::collections::HashMap::new();
// So the job says what it was evaluating when opened cold from the runs page. Every iteration
// inherits these, so they are the stamp and nothing bulkier.
args.insert(
"_eval".to_string(),
serde_json::value::to_raw_value(&serde_json::json!({
"subject": subject.stamp(),
"dataset": dataset,
"experiment_id": experiment_id,
}))?,
);
let path = subject.path.clone();
let tx = PushIsolationLevel::Isolated(user_db.clone(), authed.clone().into());
let (uuid, tx) = push(
db,
tx,
w_id,
JobPayload::RawFlow { value: flow_value, path: Some(path), restarted_from: None },
PushArgs::from(&args),
authed.display_username(),
&authed.email,
username_to_permissioned_as(&authed.username),
authed.token_prefix.as_deref(),
authed.username_override.as_deref(),
None,
None,
None,
None,
None,
Some(run_job_id),
false,
false,
None,
true,
None,
None,
None,
None,
Some(&authed.clone().into()),
false,
None,
authed.trigger_or_fallback(None),
None,
)
.await?;
tx.commit().await?;
Ok(uuid)
}
pub(crate) fn experiment_from_row(
id: Uuid,
dataset: String,
subject: serde_json::Value,
run_number: i32,
run_job_id: Uuid,
case_count: i64,
created_at: DateTime<Utc>,
created_by: String,
) -> Result<EvalExperiment> {
Ok(EvalExperiment {
id,
dataset,
subject: serde_json::from_value(subject)?,
run_number,
run_job_id,
case_count,
// Filled in by the list, which reads every listed run's scores in one query.
scores: vec![],
running: false,
created_at,
created_by,
})
}
#[cfg(test)]
mod tests {
use super::*;
fn agent_config() -> AgentDraft {
AgentDraft { input_transforms: serde_json::json!({}), tools: vec![] }
}
fn scorer(kind: ScorerDef) -> Scorer {
Scorer { id: "s1".to_string(), name: None, pass_if: None, def: kind }
}
fn subject(kind: EvalSubjectKind, draft: Option<AgentDraft>) -> EvalSubject {
EvalSubject { kind, path: "u/me/agent".to_string(), version: None, draft, draft_hash: None }
}
/// The whole argument for accepting a configuration from the request is that it is accepted
/// for exactly one kind: the edits in progress, which exist nowhere the server can read. A
/// saved agent or a past version carrying one would run something other than what it names.
#[test]
fn a_configuration_is_required_for_edits_and_refused_for_anything_saved() {
assert!(
validate_subject(&subject(EvalSubjectKind::AgentDraft, Some(agent_config()))).is_ok()
);
assert!(validate_subject(&subject(EvalSubjectKind::Agent, None)).is_ok());
assert!(validate_subject(&subject(EvalSubjectKind::AgentDraft, None)).is_err());
assert!(validate_subject(&subject(EvalSubjectKind::Agent, Some(agent_config()))).is_err());
assert!(validate_subject(&subject(
EvalSubjectKind::AgentVersion,
Some(agent_config())
))
.is_err());
}
/// Where the collect step sits is load-bearing twice over: inside the loop it would run once
/// per case, and `backfill_case_jobs` matches a case to any child of the run carrying an
/// `iter` argument, which the collect job must therefore never be.
#[test]
fn the_collect_step_runs_once_after_the_loop() {
let experiment = Uuid::new_v4();
let flow = build_run_flow(&agent_config(), &[], &[], experiment).unwrap();
let value = serde_json::to_value(&flow).unwrap();
let modules = value["modules"].as_array().unwrap();
assert_eq!(
modules
.iter()
.map(|m| m["id"].as_str().unwrap())
.collect::<Vec<_>>(),
vec![CASES_NODE_ID, COLLECT_NODE_ID]
);
let collect = &modules[1];
// The run it records is baked in rather than read from the iteration around it, which is
// what makes it a step of the run and not of a case.
assert_eq!(
collect["value"]["input_transforms"]["experiment_id"]["value"]
.as_str()
.unwrap(),
experiment.to_string()
);
assert!(collect["value"]["input_transforms"]["iter"].is_null());
// A failed record must not fail a run whose cases all answered.
assert_eq!(collect["continue_on_error"].as_bool(), Some(true));
}
/// A code scorer pins the deployed hash it resolved to, and that hash must reach the flow as a
/// `ScriptHash` (a hex string), not the bare number it is in the database.
#[test]
fn a_script_scorer_pins_its_resolved_hash_as_a_hex_string() {
let s = scorer(ScorerDef::Script { path: "f/e/scorer".to_string() });
let scorers = vec![(&s, ResolvedScorer::Script { hash: 8816320759749465854i64 })];
let modules = scorer_modules(&scorers);
// A flow module's `hash` deserializes only from a `ScriptHash` (a hex string); emitted as a
// bare number it fails in the worker and every code-scorer column breaks at runtime.
assert!(
modules[0]["value"]["hash"].is_string(),
"the pinned scorer hash must serialize as a hex string, not a number"
);
}
/// A judge is pinned by inlining the configuration resolved at launch rather than linked by
/// path, which is the difference between a run that grades against one definition and one that
/// resolves the judge per case.
#[test]
fn a_judge_is_inlined_rather_than_linked() {
let judge = scorer(ScorerDef::Agent { path: "f/e/judge".to_string() });
let scorers = vec![(
&judge,
ResolvedScorer::Judge {
config: AgentDraft {
input_transforms: serde_json::json!({
"system_prompt": { "type": "static", "value": "grade it" }
}),
tools: vec![],
},
},
)];
let pinned = scorer_modules(&scorers);
let value = &pinned[0]["value"];
assert!(value["agent"].is_null());
assert_eq!(
value["input_transforms"]["system_prompt"]["value"].as_str(),
Some("grade it")
);
// The case reaches the judge alongside the judge's own transforms.
assert!(value["input_transforms"]["user_message"]["expr"].is_string());
assert!(value["input_transforms"]["user_attachments"]["expr"].is_string());
}
}
backend/windmill-api/src/ai_evals/scorers.rs
+285
View File
@@ -0,0 +1,285 @@
use super::*;
/// A scorer is a column of the results table.
///
/// `id` is assigned when the scorer is added to a dataset and never reused: it is what makes a
/// column the same column across experiments when the scorer is renamed or its definition is
/// edited, and a delta is only ever computed between two scores carrying the same id.
#[derive(Serialize, Deserialize, Debug, Clone)]
pub struct Scorer {
/// Assigned on write when a new scorer arrives without one, so a client cannot collide two
/// columns onto one id.
#[serde(default)]
pub id: String,
/// The column header. Defaults to the kind, or the last segment of the path.
#[serde(default, skip_serializing_if = "Option::is_none")]
pub name: Option<String>,
/// A score at or above this counts as a pass. Deliberately outside `definition`: where the
/// line sits interprets a score rather than produces it, so moving it re-reads every score
/// already recorded instead of invalidating them.
#[serde(default, skip_serializing_if = "Option::is_none")]
pub pass_if: Option<f64>,
#[serde(flatten)]
pub def: ScorerDef,
}
/// A judge is an `ai_agent` resource sent the run to grade; a script receives the run as an
/// argument. Both are runnables, so every column has a path, a version and code you can open.
#[derive(Serialize, Deserialize, Debug, Clone, PartialEq)]
#[serde(tag = "kind", rename_all = "snake_case")]
pub enum ScorerDef {
Script { path: String },
Agent { path: String },
}
impl ScorerDef {
pub fn path(&self) -> &str {
match self {
ScorerDef::Script { path } | ScorerDef::Agent { path } => path,
}
}
/// The wire name of the kind, as the client sends it.
pub(crate) fn kind_str(&self) -> &'static str {
match self {
ScorerDef::Script { .. } => "script",
ScorerDef::Agent { .. } => "agent",
}
}
fn kind_label(&self) -> &'static str {
match self {
ScorerDef::Script { .. } => "Script",
ScorerDef::Agent { .. } => "Judge agent",
}
}
}
impl Scorer {
/// Whether a score counts as a pass. `None` when the column has no threshold, which keeps a
/// column of plain numbers from being rendered as if it had one.
pub fn passed(&self, score: Option<f64>) -> Option<bool> {
match (self.pass_if, score) {
(Some(threshold), Some(score)) => Some(score >= threshold),
_ => None,
}
}
/// What produced a score, recorded with it so a comparison can say the scorer changed instead
/// of letting that read as a difference between two agents. `resolved` is the script hash or
/// resource version that actually ran, which the path alone does not pin.
pub fn definition(&self, resolved: Option<&str>) -> String {
use sha2::{Digest, Sha256};
let mut hasher = Sha256::new();
hasher.update(self.def.kind_label().as_bytes());
hasher.update(b":");
hasher.update(self.def.path().as_bytes());
if let Some(resolved) = resolved {
hasher.update(b"@");
hasher.update(resolved.as_bytes());
}
hex::encode(hasher.finalize())[..32].to_string()
}
}
const MAX_SCORER_NAME_CHARS: usize = 120;
/// Ids are assigned here rather than trusted from the client: an id is kept only when it names a
/// column the dataset already has, so a removed column cannot come back and inherit the scores
/// recorded against it. Anything else is minted as a valid flow module identifier, which the
/// scoring flows it is baked into require (see `scorer_module_id`).
pub(crate) fn assign_scorer_ids(
scorers: &mut Vec<Scorer>,
existing: &std::collections::HashSet<String>,
) -> Result<()> {
if scorers.len() > MAX_SCORERS_PER_DATASET {
return Err(Error::BadRequest(format!(
"An eval dataset holds at most {} scorers",
MAX_SCORERS_PER_DATASET
)));
}
let mut seen = std::collections::HashSet::new();
for scorer in scorers.iter_mut() {
if !existing.contains(&scorer.id) || !seen.insert(scorer.id.clone()) {
scorer.id = Uuid::new_v4().simple().to_string();
seen.insert(scorer.id.clone());
}
if let Some(name) = &scorer.name {
if name.chars().count() > MAX_SCORER_NAME_CHARS {
return Err(Error::BadRequest(format!(
"Scorer name {} is too long, {} characters at most",
name, MAX_SCORER_NAME_CHARS
)));
}
}
// A score is 0 to 1, so a threshold outside that range would pass everything or nothing
// regardless of what the scorer measured.
if let Some(pass_if) = scorer.pass_if {
if !(0.0..=1.0).contains(&pass_if) {
return Err(Error::BadRequest(format!(
"Scorer pass threshold {} must be between 0 and 1",
pass_if
)));
}
}
check_proper_path(scorer.def.path())?;
}
Ok(())
}
/// What a column is called: the dataset's own name for it, or the last segment of what it points
/// at. The same fallback the column header uses.
pub(crate) fn scorer_name(scorer: &Scorer) -> String {
scorer
.name
.clone()
.filter(|n| !n.trim().is_empty())
.unwrap_or_else(|| {
let path = scorer.def.path();
path.rsplit('/').next().unwrap_or(path).to_string()
})
}
#[derive(Serialize)]
pub struct RecentScorer {
#[serde(flatten)]
pub scorer: Scorer,
/// The dataset it is a column of, which is where the user last saw it.
pub dataset: String,
}
#[derive(Deserialize)]
pub struct RecentScorersQuery {
/// Only scorers of this kind, which is the one the add form was opened for.
#[serde(default)]
pub kind: Option<String>,
}
/// The scorers already in use in this workspace, most recently edited dataset first.
///
/// Filtered twice through `user_db`: a scorer appears only if its dataset does, and the runnable
/// is checked the same way, so the list is scorers the caller could actually run.
pub async fn recent_scorers(
authed: ApiAuthed,
Extension(user_db): Extension<UserDB>,
Path(w_id): Path<String>,
Query(query): Query<RecentScorersQuery>,
) -> JsonResult<Vec<RecentScorer>> {
let mut tx = user_db.begin(&authed).await?;
let datasets = sqlx::query!(
"SELECT path, scorers FROM eval_dataset
WHERE workspace_id = $1 ORDER BY edited_at DESC LIMIT 100",
w_id
)
.fetch_all(&mut *tx)
.await?;
let mut seen = std::collections::HashSet::new();
let mut recent: Vec<RecentScorer> = vec![];
for row in datasets {
for scorer in parse_scorers(row.scorers)? {
if query
.kind
.as_deref()
.is_some_and(|kind| kind != scorer.def.kind_str())
{
continue;
}
let key = (scorer.def.kind_str(), scorer.def.path().to_string());
if seen.insert(key) {
recent.push(RecentScorer { scorer, dataset: row.path.clone() });
}
}
}
// Readability is resolved over every candidate, then the list is cut: an unreadable scorer must
// not take a slot a readable one further down would have filled.
let script_paths = recent
.iter()
.filter(|r| matches!(r.scorer.def, ScorerDef::Script { .. }))
.map(|r| r.scorer.def.path().to_string())
.collect::<Vec<_>>();
let agent_paths = recent
.iter()
.filter(|r| matches!(r.scorer.def, ScorerDef::Agent { .. }))
.map(|r| r.scorer.def.path().to_string())
.collect::<Vec<_>>();
// Same deployed-version predicate as get_latest_script_hash: a script with no successfully
// locked version can't be resolved at launch, so it must not offer itself as a scorer here.
let readable_scripts = sqlx::query_scalar!(
"SELECT DISTINCT path FROM script
WHERE workspace_id = $1 AND path = ANY($2)
AND deleted = false AND lock IS NOT NULL AND lock_error_logs IS NULL",
w_id,
&script_paths
)
.fetch_all(&mut *tx)
.await?
.into_iter()
.collect::<std::collections::HashSet<_>>();
let readable_agents = sqlx::query_scalar!(
"SELECT path FROM resource WHERE workspace_id = $1 AND path = ANY($2) AND resource_type = 'ai_agent'",
w_id,
&agent_paths
)
.fetch_all(&mut *tx)
.await?
.into_iter()
.collect::<std::collections::HashSet<_>>();
tx.commit().await?;
recent.retain(|r| match &r.scorer.def {
ScorerDef::Script { path } => readable_scripts.contains(path),
ScorerDef::Agent { path } => readable_agents.contains(path),
});
recent.truncate(MAX_RECENT_SCORERS);
Ok(Json(recent))
}
#[cfg(test)]
mod tests {
use super::*;
/// The definition hash is what tells a comparison that the scorer changed; the path alone
/// would miss an edit to the script itself.
#[test]
fn definition_moves_with_the_runnable_and_not_with_its_name() {
let script = |path: &str, name: Option<&str>| Scorer {
id: "s1".to_string(),
name: name.map(|n| n.to_string()),
pass_if: None,
def: ScorerDef::Script { path: path.to_string() },
};
// Renaming a column is not a change of scorer: same runnable, same version.
assert_eq!(
script("f/e/s", None).definition(Some("1234")),
script("f/e/s", Some("Tool discipline")).definition(Some("1234"))
);
// Same script, newly deployed: the column says the scorer changed.
assert_ne!(
script("f/e/s", None).definition(Some("1234")),
script("f/e/s", None).definition(Some("5678"))
);
// A judge agent and a script sharing a path are not the same column.
let agent = Scorer {
id: "s1".to_string(),
name: None,
pass_if: None,
def: ScorerDef::Agent { path: "f/e/s".to_string() },
};
assert_ne!(
agent.definition(Some("1")),
script("f/e/s", None).definition(Some("1"))
);
// If the pass line entered the hash, setting a threshold would mark every score already
// recorded as coming from a different scorer.
let mut thresholded = script("f/e/s", None);
thresholded.pass_if = Some(0.7);
assert_eq!(
thresholded.definition(Some("1234")),
script("f/e/s", None).definition(Some("1234"))
);
assert_eq!(thresholded.passed(Some(0.7)), Some(true));
assert_eq!(thresholded.passed(Some(0.69)), Some(false));
assert_eq!(script("f/e/s", None).passed(Some(0.1)), None);
}
}
backend/windmill-api/src/ai_evals/scoring.rs
+712
View File
@@ -0,0 +1,712 @@
use super::*;
/// What a scorer resolves to, alongside the definition to record: a script by its pinned hash, or
/// a judge by the configuration to inline.
pub(crate) enum ResolvedScorer {
Script { hash: i64 },
Judge { config: AgentDraft },
}
/// The runnable a scorer names, resolved through the caller's *own* database so a run can only
/// execute code the caller may read: a scorer is added with a bare path and nothing checks read
/// access there.
///
/// Returns the definition to record and what to run: a script by its deployed hash to pin, or a
/// judge by the configuration to inline, so a redeploy midway through a run cannot swap the code
/// out from under a score labelled with the old version.
pub(crate) async fn resolve_scorer(
user_db: &UserDB,
authed: &ApiAuthed,
w_id: &str,
scorer: &Scorer,
) -> Result<(String, ResolvedScorer)> {
match &scorer.def {
ScorerDef::Script { path } => {
// The latest *deployed* hash (no draft, no failed deploy), through the canonical helper
// so the version a scorer pins is the one everything else runs.
let mut tx = user_db.clone().begin(authed).await?;
let hash = windmill_common::get_latest_script_hash(&mut *tx, path, w_id).await?;
tx.commit().await?;
let Some(hash) = hash else {
return Err(Error::BadRequest(format!(
"Scorer script {} is not deployed or not readable",
path
)));
};
Ok((
scorer.definition(Some(&hash.to_string())),
ResolvedScorer::Script { hash },
))
}
ScorerDef::Agent { path } => {
let Some((config, version)) = readable_agent_state(authed, user_db, w_id, path).await?
else {
return Err(Error::BadRequest(format!(
"Judge scorer {} is not a readable ai_agent resource",
path
)));
};
Ok((
scorer.definition(Some(&version.to_string())),
ResolvedScorer::Judge { config },
))
}
}
}
/// Bring a run's record up to date with the flow that executed it: which iteration answered which
/// case, what the agent answered, and what its scorers returned.
///
/// `answers` is what separates the two callers: a listing reports each run's score aggregates and
/// never shows an answer, so harvesting them there reads a column of every case of every listed
/// run to display none of it.
pub(crate) async fn sync_run(
db: &DB,
w_id: &str,
experiment_id: Uuid,
run_job_id: Uuid,
answers: bool,
) -> Result<()> {
backfill_case_jobs(db, w_id, experiment_id, run_job_id).await?;
settle_unspawned_cases(db, w_id, experiment_id, run_job_id).await?;
if answers {
record_case_answers(db, w_id, experiment_id).await?;
}
harvest_flow_scores(db, w_id, experiment_id).await?;
Ok(())
}
/// Give a terminal status to cases the run never spawned an iteration for: with no `job_id` there
/// is nothing to read an answer or a score out of, so they would report "running" indefinitely.
async fn settle_unspawned_cases(
db: &DB,
w_id: &str,
experiment_id: Uuid,
run_job_id: Uuid,
) -> Result<()> {
// Only a run that has reached `v2_job_completed` is settled from here. A job absent from the
// tables is as likely mid-launch — the experiment is committed before its job is pushed — as
// aged out, and settling then would cancel the cases of a run about to start. A cancelled run
// lands in `v2_job_completed`, so a cancel before an iteration spawned is still covered.
let Some(terminal_status) = sqlx::query_scalar!(
"SELECT status::text AS \"status!\" FROM v2_job_completed WHERE id = $1 AND workspace_id = $2",
run_job_id,
w_id
)
.fetch_optional(db)
.await?
else {
return Ok(());
};
let settled = sqlx::query_scalar!(
"UPDATE eval_experiment_case SET status = $2, answered = false
WHERE experiment_id = $1 AND job_id IS NULL AND status IS NULL
RETURNING ordinal",
experiment_id,
terminal_status
)
.fetch_all(db)
.await?;
// The score cells of a case that never ran have no job to read a verdict out of either.
if !settled.is_empty() {
sqlx::query!(
"UPDATE eval_score SET error = 'The case did not run'
WHERE experiment_id = $1 AND ordinal = ANY($2)
AND score IS NULL AND error IS NULL AND NOT not_applicable",
experiment_id,
&settled
)
.execute(db)
.await?;
}
Ok(())
}
/// In-flight reads of what a case's agent step produced. Each is several queries and a run holds
/// up to `MAX_CASES_PER_DATASET` cases, so they go a few at a time.
const HARVEST_CONCURRENCY: usize = 8;
/// Cases whose scorer results are read in one query: every scorer of every case in the batch, so
/// the batch bounds how much of a run's worth of judge conversations is held at once.
const HARVEST_BATCH_CASES: usize = 100;
/// Copy what each iteration produced into its row: the agent's answer, whether producing it
/// succeeded, and how the iteration ended.
///
/// Written once, when it becomes readable, rather than read back out of the jobs whenever the
/// table is displayed — jobs have their own retention, and a run whose rows are kept has to still
/// read as the run it was after they have aged out.
async fn record_case_answers(db: &DB, w_id: &str, experiment_id: Uuid) -> Result<()> {
let unrecorded = sqlx::query!(
"SELECT c.ordinal, c.job_id AS \"job_id!\", d.status::text AS status,
(j.id IS NOT NULL) AS \"job_exists!\"
FROM eval_experiment_case c
LEFT JOIN v2_job j ON j.id = c.job_id AND j.workspace_id = $2
LEFT JOIN v2_job_completed d ON d.id = c.job_id AND d.workspace_id = $2
WHERE c.experiment_id = $1 AND c.job_id IS NOT NULL AND c.status IS NULL",
experiment_id,
w_id
)
.fetch_all(db)
.await?;
use futures::StreamExt;
let answers = futures::stream::iter(unrecorded.into_iter().map(|row| async move {
// The job was retained away before anything read it: nothing to read, and nothing more
// will ever be there to read.
if !row.job_exists {
return Ok((row.ordinal, None, None, Some("unavailable".to_string())));
}
// The agent step's own result, never the iteration's: the iteration goes on to score the
// answer, so the answer is settled long before the iteration is.
let agent = agent_result(db, w_id, row.job_id).await?;
// An iteration that ended without an answer — skipped, cancelled, or an agent that failed
// outright — produced none, and saying so is what stops this re-reading it.
let answered = agent
.as_ref()
.map(|(_, success)| *success)
.or_else(|| row.status.is_some().then_some(false));
let output = agent.as_ref().and_then(|(result, _)| agent_answer(result));
Ok::<_, Error>((row.ordinal, output, answered, row.status))
}))
.buffered(HARVEST_CONCURRENCY)
.collect::<Vec<_>>()
.await
.into_iter()
.collect::<Result<Vec<_>>>()?;
// One statement for the whole run: the run's own collect step reaches every case at once, and
// a thousand of them one at a time is a thousand round trips.
let mut ordinals = vec![];
let mut outputs = vec![];
let mut answered = vec![];
let mut statuses = vec![];
for (ordinal, output, was_answered, status) in answers {
// Nothing to record yet, and the iteration may still produce it.
if was_answered.is_none() && status.is_none() {
continue;
}
ordinals.push(ordinal);
outputs.push(output);
answered.push(was_answered);
statuses.push(status);
}
if ordinals.is_empty() {
return Ok(());
}
sqlx::query!(
"UPDATE eval_experiment_case c
SET output = COALESCE(c.output, t.output), answered = COALESCE(c.answered, t.answered),
status = COALESCE(c.status, t.status)
FROM UNNEST($2::int[], $3::text[], $4::bool[], $5::text[])
AS t(ordinal, output, answered, status)
WHERE c.experiment_id = $1 AND c.ordinal = t.ordinal",
experiment_id,
&ordinals,
&outputs as &[Option<String>],
&answered as &[Option<bool>],
&statuses as &[Option<String>],
)
.execute(db)
.await?;
Ok(())
}
/// The agent step's result, with "there is none" kept apart from "it could not be read": a lookup
/// that failed for any other reason must not be recorded as a case that produced no answer,
/// because nothing reads that row again.
pub(crate) async fn agent_result(
db: &DB,
w_id: &str,
job_id: Uuid,
) -> Result<Option<(Box<RawValue>, bool)>> {
match windmill_queue::get_result_and_success_by_id_from_flow(
db,
w_id,
&job_id,
AGENT_NODE_ID,
None,
)
.await
{
Ok(found) => Ok(Some(found)),
Err(Error::NotFound(_)) => Ok(None),
Err(e) => Err(e),
}
}
/// Match each case to the iteration that ran it. The flow engine mints those job ids, so the case
/// they belong to is read back from the iteration's own arguments, which survives iterations
/// finishing in any order.
async fn backfill_case_jobs(
db: &DB,
w_id: &str,
experiment_id: Uuid,
run_job_id: Uuid,
) -> Result<()> {
sqlx::query!(
"UPDATE eval_experiment_case c SET job_id = j.id
FROM v2_job j
WHERE j.parent_job = $3 AND j.workspace_id = $2
AND (j.args -> 'iter' -> 'value' ->> 'case_id')::uuid = c.case_id
AND c.experiment_id = $1 AND c.job_id IS NULL",
experiment_id,
w_id,
run_job_id
)
.execute(db)
.await?;
Ok(())
}
/// Read the scores a run's own flow produced into `eval_score`, so a score outlives the flow
/// that produced it and the retention on its jobs.
async fn harvest_flow_scores(db: &DB, w_id: &str, experiment_id: Uuid) -> Result<()> {
let pending = sqlx::query!(
// Left-joined, so an iteration still running is read too: a scorer runs after the agent
// within that iteration, so its verdict is there to be read as soon as its own step is
// done, and waiting for the iteration to end would hold every column of a case back until
// the last of them finished.
"SELECT s.ordinal, s.scorer_id, c.job_id AS \"job_id!\", d.status::text AS status,
c.answered, (j.id IS NOT NULL) AS \"job_exists!\"
FROM eval_score s
JOIN eval_experiment_case c
ON c.experiment_id = s.experiment_id AND c.ordinal = s.ordinal
LEFT JOIN v2_job j ON j.id = c.job_id AND j.workspace_id = $2
LEFT JOIN v2_job_completed d ON d.id = c.job_id AND d.workspace_id = $2
WHERE s.experiment_id = $1 AND s.score IS NULL AND s.error IS NULL
AND NOT s.not_applicable AND c.job_id IS NOT NULL",
experiment_id,
w_id
)
.fetch_all(db)
.await?;
if pending.is_empty() {
return Ok(());
}
// The job tree is walked in SQL rather than once per cell: a live run is read every couple of
// seconds and a full one is up to MAX_CASES_PER_DATASET × MAX_SCORERS_PER_DATASET cells. The
// shape is `build_run_flow`'s: a scorer is the one module of its own branch of the scoring
// step, so its job's parent is that branch and the branch's parent is the case.
let mut case_jobs: Vec<Uuid> = pending.iter().map(|row| row.job_id).collect();
case_jobs.sort();
case_jobs.dedup();
let mut modules: Vec<String> = pending
.iter()
.map(|row| scorer_module_id(&row.scorer_id))
.collect();
modules.sort();
modules.dedup();
let mut verdicts: Vec<(i32, String, Option<(Verdict, Option<String>)>)> =
Vec::with_capacity(pending.len());
for batch in case_jobs.chunks(HARVEST_BATCH_CASES) {
let results: std::collections::HashMap<(Uuid, String), Box<RawValue>> = sqlx::query!(
"SELECT branch.parent_job AS \"case_job!\", scorer.flow_step_id AS \"module!\",
done.result AS \"result: sqlx::types::Json<Box<RawValue>>\"
FROM v2_job branch
JOIN v2_job scorer ON scorer.parent_job = branch.id
JOIN v2_job_completed done ON done.id = scorer.id
WHERE branch.parent_job = ANY($1) AND branch.workspace_id = $2
AND scorer.flow_step_id = ANY($3)",
batch,
w_id,
&modules
)
.fetch_all(db)
.await?
.into_iter()
.map(|row| {
let result = row
.result
.map(|json| json.0)
.unwrap_or_else(|| RawValue::from_string("null".to_string()).expect("a literal"));
((row.case_job, row.module), result)
})
.collect();
let in_batch: std::collections::HashSet<Uuid> = batch.iter().copied().collect();
for row in pending.iter().filter(|row| in_batch.contains(&row.job_id)) {
// Nothing left to read the verdict out of. Settled here, since a cell left pending is
// one every later listing would go back to this same absent job for.
if !row.job_exists {
verdicts.push((
row.ordinal,
row.scorer_id.clone(),
Some((
Verdict::default(),
Some("The run that produced this score is no longer available".to_string()),
)),
));
continue;
}
// What to say when the job is over and this scorer left nothing. Only
// `record_case_answers` tells the two states apart and a listing syncs without it, so
// `None` withholds the sentence — not the harvest: a scorer that returned a number is
// read and recorded either way.
let missing = row.answered.map(|answered| {
if answered {
"This scorer did not run for the case"
} else {
"The case produced no answer to score"
}
});
let result = results
.get(&(row.job_id, scorer_module_id(&row.scorer_id)))
.map(|r| r.as_ref());
let verdict = settle_verdict(result, row.status.as_deref(), missing);
verdicts.push((row.ordinal, row.scorer_id.clone(), verdict));
}
}
// One statement for every cell read, for the same reason the answers are written that way.
let mut ordinals = vec![];
let mut scorer_ids = vec![];
let mut scores = vec![];
let mut reasons = vec![];
let mut checks = vec![];
let mut errors = vec![];
let mut not_applicable = vec![];
for (ordinal, scorer_id, read) in verdicts {
// Still to come: a scorer whose own step has not run yet.
let Some((verdict, error)) = read else {
continue;
};
ordinals.push(ordinal);
scorer_ids.push(scorer_id);
scores.push(verdict.score);
reasons.push(verdict.reason);
checks.push(verdict.checks);
errors.push(error);
not_applicable.push(verdict.not_applicable);
}
if ordinals.is_empty() {
return Ok(());
}
sqlx::query!(
"UPDATE eval_score s
SET score = t.score, reason = t.reason, checks = t.checks, error = t.error,
not_applicable = t.not_applicable
FROM UNNEST($2::int[], $3::text[], $4::double precision[], $5::text[], $6::jsonb[],
$7::text[], $8::bool[])
AS t(ordinal, scorer_id, score, reason, checks, error, not_applicable)
WHERE s.experiment_id = $1 AND s.ordinal = t.ordinal AND s.scorer_id = t.scorer_id",
experiment_id,
&ordinals,
&scorer_ids,
&scores as &[Option<f64>],
&reasons as &[Option<String>],
&checks as &[Option<serde_json::Value>],
&errors as &[Option<String>],
&not_applicable,
)
.execute(db)
.await?;
Ok(())
}
/// One scorer's verdict, from the result of the step that produced it, inside a job that may
/// still be running: a scorer's own step can be done while the iteration around it is not. `None`
/// while the result is not readable yet, which is a state to wait through rather than to record
/// as a failure; `Some` with an error is a scorer that produced nothing, worded by where it ran.
fn settle_verdict(
result: Option<&RawValue>,
job_status: Option<&str>,
// What to record when the job is over and this scorer produced nothing. A different statement
// depending on where the scorer ran: its own job failed, or the case it was to score never
// produced an answer. `None` when the caller cannot yet tell those apart, which leaves the
// cell pending for a read that can, rather than settling it on the wrong one of the two.
missing_error: Option<&str>,
) -> Option<(Verdict, Option<String>)> {
Some(match result {
Some(value) => {
let verdict = extract_verdict(value);
match verdict {
// A score is a fraction: the mean and the pass rate read it as one, so a number
// outside that range is recorded as an error rather than a value that would
// quietly skew the column.
Verdict { score: Some(score), .. } if !(0.0..=1.0).contains(&score) => (
Verdict::default(),
Some(format!(
"The scorer returned {}, outside the 0 to 1 range a score must be in",
score
)),
),
// A number in range, or the scorer saying this case is not one it measures. Both
// are answers, so neither is an error.
Verdict { score: Some(_), .. } | Verdict { not_applicable: true, .. } => {
(verdict, None)
}
// The job around this scorer is still going, so a module with no number in it is
// one that has not run yet. Recording a failure here would make it permanent.
_ if job_status.is_none() => return None,
_ if job_status == Some("success") => (
verdict,
Some("The scorer returned no number to plot".to_string()),
),
_ => match missing_error {
Some(missing) => (verdict, Some(missing.to_string())),
None => return None,
},
}
}
// The iteration is over, so a scorer step with no readable result produced nothing and
// never will; left pending it would be re-read on every listing.
None if job_status == Some("success") => (
Verdict::default(),
Some("The scorer step produced no result".to_string()),
),
// The job holding this scorer has not finished, so a module with nothing in it yet is a
// step that has not run rather than one that produced nothing.
None if job_status.is_none() => return None,
None => match missing_error {
Some(missing) => (Verdict::default(), Some(missing.to_string())),
None => return None,
},
})
}
/// The score and reason read straight out of text that failed to parse as JSON. Deliberately not a
/// second JSON parser: it looks for the two keys and takes what follows, which is what survives a
/// model writing an unescaped quote in the middle of a sentence.
fn salvage_verdict(text: &str) -> (Option<f64>, Option<String>) {
fn after_key<'a>(text: &'a str, key: &str) -> Option<&'a str> {
let start = text.find(key)? + key.len();
Some(text[start..].trim_start().strip_prefix(':')?.trim_start())
}
let score = after_key(text, "\"score\"").and_then(|rest| {
if rest.starts_with("true") {
return Some(1.0);
}
if rest.starts_with("false") {
return Some(0.0);
}
let end = rest
.find(|c: char| !matches!(c, '0'..='9' | '.' | '-' | '+' | 'e' | 'E'))
.unwrap_or(rest.len());
rest[..end].parse::<f64>().ok()
});
// To the last quote of the object, so an unescaped one inside the sentence stays part of it.
let reason = after_key(text, "\"reason\"")
.and_then(|rest| rest.strip_prefix('"'))
.and_then(|rest| {
let body = match rest.rfind('}') {
Some(brace) => &rest[..brace],
None => rest,
};
let end = body.rfind('"')?;
Some(body[..end].to_string())
})
.filter(|reason| !reason.is_empty());
(score, reason)
}
/// A fenced code block as the model wrote it, reduced to what is inside the fence. The opening
/// fence carries a language tag often enough that the first line goes with it.
fn unfence(text: &str) -> &str {
let trimmed = text.trim();
let Some(rest) = trimmed.strip_prefix("```") else {
return trimmed;
};
let inner = match rest.split_once('\n') {
Some((_language, body)) => body,
None => rest,
};
inner.trim_end().trim_end_matches("```").trim()
}
/// What a scorer said about one run. `not_applicable` is the scorer declining to measure this
/// case: an explicit `{"score": null}`. A bare `null` stays an error, since a scorer that forgot
/// to return is indistinguishable from one that returned nothing on purpose.
#[derive(Default)]
struct Verdict {
score: Option<f64>,
reason: Option<String>,
checks: Option<serde_json::Value>,
not_applicable: bool,
}
impl Verdict {
fn scored(score: f64) -> Self {
Verdict { score: Some(score), ..Default::default() }
}
}
/// A scorer may return a bare number, a boolean, or `{score, reason, checks}`; an agent wraps its
/// answer in `output`, sometimes as a string holding any of those. Anything with no number in it
/// is left empty rather than guessed at.
fn extract_verdict(value: &RawValue) -> Verdict {
let Ok(parsed) = serde_json::from_str::<serde_json::Value>(value.get()) else {
return Verdict::default();
};
fn as_number(value: &serde_json::Value) -> Option<f64> {
match value {
serde_json::Value::Number(n) => n.as_f64(),
serde_json::Value::Bool(b) => Some(if *b { 1.0 } else { 0.0 }),
_ => None,
}
}
if let Some(number) = as_number(&parsed) {
return Verdict::scored(number);
}
let serde_json::Value::Object(map) = &parsed else {
// A judge often answers with JSON inside a string, and often fences it as markdown even
// when told to reply with JSON only.
if let serde_json::Value::String(text) = &parsed {
let text = unfence(text);
if let Ok(inner) = serde_json::from_str::<serde_json::Value>(text) {
if let Ok(raw) = serde_json::value::to_raw_value(&inner) {
return extract_verdict(&raw);
}
}
// Nearly JSON: a judge that quotes the agent inside its own reason writes those quotes
// unescaped, which is invalid and also the most ordinary thing for it to say. The
// number is what the column plots, so it is read out of the text rather than lost with
// the object around it.
let (score, reason) = salvage_verdict(text);
return Verdict { score, reason, checks: None, not_applicable: false };
}
return Verdict::default();
};
let reason = || {
map.get("reason")
.or_else(|| map.get("comment"))
.and_then(|r| r.as_str())
.map(|r| r.to_string())
};
if let Some(score) = map.get("score").and_then(as_number) {
return Verdict {
score: Some(score),
reason: reason(),
checks: map.get("checks").cloned(),
not_applicable: false,
};
}
// Written out rather than merely absent, which is what separates it from a scorer that
// returned an object with no verdict in it at all.
if map.get("score").is_some_and(|s| s.is_null()) {
return Verdict {
score: None,
reason: reason(),
checks: map.get("checks").cloned(),
not_applicable: true,
};
}
match map.get("output") {
Some(output) => match serde_json::value::to_raw_value(output) {
Ok(raw) => extract_verdict(&raw),
Err(_) => Verdict::default(),
},
None => Verdict::default(),
}
}
#[cfg(test)]
mod tests {
use super::*;
fn raw(json: &str) -> Box<RawValue> {
serde_json::from_str(json).unwrap()
}
/// A scorer's answer arrives in whatever shape its runnable returns: a script's bare value or
/// object, or a judge's answer wrapped in `output` and often stringified. A shape that goes
/// unrecognised is a silently empty cell rather than an error.
#[test]
fn extract_verdict_reads_every_documented_scorer_shape() {
let score = |json: &str| extract_verdict(&raw(json)).score;
assert_eq!(score("0.75"), Some(0.75));
assert_eq!(score("true"), Some(1.0));
assert_eq!(score(r#"{"score": 0.5}"#), Some(0.5));
assert_eq!(score(r#"{"score": false}"#), Some(0.0));
// judges and agent scorers: the answer is under `output`, sometimes as a string
assert_eq!(score(r#"{"output": 0.25}"#), Some(0.25));
assert_eq!(score(r#"{"output": "0.9"}"#), Some(0.9));
assert_eq!(score(r#"{"output": {"score": 0.8}}"#), Some(0.8));
assert_eq!(score(r#"{"output": "{\"score\": 0.4}"}"#), Some(0.4));
// a judge told to reply with JSON only, replying with JSON only, in a code fence
assert_eq!(
score("{\"output\": \"```json\\n{\\\"score\\\": 0.15}\\n```\"}"),
Some(0.15)
);
assert_eq!(score("{\"output\": \"```\\n0.6\\n```\"}"), Some(0.6));
// A judge quoting the agent inside its own reason, which is invalid JSON.
let quoted = extract_verdict(&raw(
r#"{"output": "{\"score\": 0.8, \"reason\": \"invented context (\"stop asking me\", never said) here\"}"}"#,
));
assert_eq!(quoted.score, Some(0.8));
assert_eq!(
quoted.reason.as_deref(),
Some(r#"invented context ("stop asking me", never said) here"#)
);
// nothing numeric to plot: left empty rather than guessed at
assert_eq!(score(r#"{"output": "not a score"}"#), None);
assert_eq!(score(r#"{"verdict": "good"}"#), None);
let full = extract_verdict(&raw(
r#"{"score": 0.5, "reason": "half", "checks": [{"name": "a"}]}"#,
));
assert_eq!(
(full.score, full.reason),
(Some(0.5), Some("half".to_string()))
);
assert!(full.checks.is_some());
assert!(!full.not_applicable);
// `comment` as the rationale, which is what a scorer written for LangSmith or Langfuse
// returns. Read rather than dropped, since the number arrives either way.
assert_eq!(
extract_verdict(&raw(r#"{"score": 1, "comment": "fine"}"#))
.reason
.as_deref(),
Some("fine")
);
}
/// A score is a fraction: anything outside 0..=1 (a scorer that returned a count, say) is
/// recorded as an error naming the value rather than plotted as a bogus point.
#[test]
fn an_out_of_range_score_is_recorded_as_an_error_not_a_value() {
// In range: recorded as the score it is.
let (v, e) = settle_verdict(Some(&raw("0.5")), Some("success"), None).unwrap();
assert_eq!(v.score, Some(0.5));
assert!(e.is_none());
// Out of range (a scorer returning a count, say): no score, an error naming the value.
let (v, e) = settle_verdict(Some(&raw("100")), Some("success"), None).unwrap();
assert_eq!(v.score, None);
assert!(e.unwrap().contains("100"));
let (v, _) = settle_verdict(Some(&raw("-5")), Some("success"), None).unwrap();
assert_eq!(v.score, None);
// No result at all once the iteration is over: an error, not a cell pending forever.
let (v, e) = settle_verdict(None, Some("success"), None).unwrap();
assert_eq!(v.score, None);
assert!(e.is_some());
// Still running: nothing to settle yet.
assert!(settle_verdict(None, None, None).is_none());
}
/// A scorer saying it has nothing to measure on a case is a verdict rather than a failure: the
/// cell is left out of the mean instead of counted as a zero. Spelled out, so a scorer that
/// returns nothing at all is still an error rather than silently excused.
#[test]
fn an_explicit_null_score_is_not_applicable_rather_than_missing() {
let na = extract_verdict(&raw(r#"{"score": null, "reason": "no sources to cite"}"#));
assert!(na.not_applicable);
assert_eq!(na.score, None);
assert_eq!(na.reason.as_deref(), Some("no sources to cite"));
// Through a judge's wrapper, as any other verdict is.
assert!(extract_verdict(&raw(r#"{"output": {"score": null}}"#)).not_applicable);
assert!(extract_verdict(&raw(r#"{"output": "{\"score\": null}"}"#)).not_applicable);
// Not the same as a scorer that returned nothing, or an object with no verdict in it.
assert!(!extract_verdict(&raw("null")).not_applicable);
assert!(!extract_verdict(&raw(r#"{"verdict": "good"}"#)).not_applicable);
}
}
backend/windmill-api/src/ai_evals/subject.rs
+135
View File
@@ -0,0 +1,135 @@
use super::*;
/// What a run is executed against. Kept as `(kind, path, version)` rather than a bare agent
/// path so flow-scoped evaluation is a later superset instead of a rewrite.
#[derive(Serialize, Deserialize, Debug, Clone)]
pub struct EvalSubject {
#[serde(default = "default_subject_kind")]
pub kind: EvalSubjectKind,
/// The agent resource under test.
pub path: String,
/// Which version of the agent, counted per path: how many times it had been saved. The
/// request's to choose for a pinned run, and otherwise the version the run was enqueued
/// against.
#[serde(default, skip_serializing_if = "Option::is_none")]
pub version: Option<i64>,
/// The agent's unsaved edits, as the editor holds them. Present exactly when `kind` is
/// `agent_draft`, since the edits exist nowhere else.
#[serde(default, skip_serializing_if = "Option::is_none")]
pub draft: Option<AgentDraft>,
/// Hash of that configuration. A draft moves without the version moving, so this is the only
/// thing that can say a run describes an agent that has since been edited. Stamped
/// server-side.
#[serde(default, skip_serializing_if = "Option::is_none")]
pub draft_hash: Option<String>,
}
/// Key order is not meaningful and `serde_json` preserves insertion order here, so it is sorted
/// away before hashing: the same configuration must hash the same however it was assembled.
fn canonical_json(value: &serde_json::Value) -> String {
match value {
serde_json::Value::Object(map) => {
let sorted = map
.iter()
.collect::<std::collections::BTreeMap<_, _>>()
.into_iter()
.map(|(k, v)| {
format!(
"{}:{}",
serde_json::to_string(k).unwrap_or_default(),
canonical_json(v)
)
})
.collect::<Vec<_>>()
.join(",");
format!("{{{}}}", sorted)
}
serde_json::Value::Array(items) => format!(
"[{}]",
items
.iter()
.map(canonical_json)
.collect::<Vec<_>>()
.join(",")
),
other => other.to_string(),
}
}
pub(crate) fn draft_hash(draft: &AgentDraft) -> String {
use sha2::{Digest, Sha256};
let mut hasher = Sha256::new();
hasher.update(canonical_json(&draft.input_transforms).as_bytes());
hasher.update(b"|");
hasher.update(canonical_json(&serde_json::Value::Array(draft.tools.clone())).as_bytes());
hex::encode(hasher.finalize())[..32].to_string()
}
fn default_subject_kind() -> EvalSubjectKind {
EvalSubjectKind::Agent
}
#[derive(Serialize, Deserialize, Debug, Clone, PartialEq)]
#[serde(rename_all = "snake_case")]
pub enum EvalSubjectKind {
Agent,
/// A saved agent's unsaved edits, carried by the request and inlined: a linked step resolves
/// the resource live and so would run what the edits replace.
AgentDraft,
/// One past version of a saved agent, inlined for the same reason. `version` says which, and
/// it is the request's to choose rather than the server's.
AgentVersion,
}
/// The brain and tools of an agent, as the flow editor holds them.
#[derive(Serialize, Deserialize, Debug, Clone)]
pub struct AgentDraft {
/// The agent's input transforms: provider, system prompt, output type and the rest. The
/// message and attachments are supplied by the case and override anything named here.
#[serde(default)]
pub input_transforms: serde_json::Value,
#[serde(default)]
pub tools: Vec<serde_json::Value>,
}
impl EvalSubject {
/// What is recorded of a subject: enough to say what ran, without the configuration itself.
pub(crate) fn stamp(&self) -> EvalSubject {
EvalSubject {
kind: self.kind.clone(),
path: self.path.clone(),
version: self.version,
draft: None,
// Only ever derived from the draft this request carries: a hash the client supplies on
// its own could relabel a run as the deployed version.
draft_hash: self.draft.as_ref().map(draft_hash),
}
}
}
#[derive(Deserialize)]
pub struct SubjectStateQuery {
pub path: String,
}
#[derive(Serialize)]
pub struct SubjectState {
/// The version the agent is on now.
#[serde(skip_serializing_if = "Option::is_none")]
pub version: Option<i64>,
}
/// The version the agent is deployed at. Small on purpose: the results endpoint reports the same
/// thing, but it harvests scores and reads every job to do it.
pub async fn subject_state(
authed: ApiAuthed,
Extension(user_db): Extension<UserDB>,
Path(w_id): Path<String>,
Query(query): Query<SubjectStateQuery>,
) -> JsonResult<SubjectState> {
let Some((_, version)) = readable_agent_state(&authed, &user_db, &w_id, &query.path).await?
else {
return Err(Error::NotFound(format!("Agent {} not found", query.path)));
};
Ok(Json(SubjectState { version: Some(version) }))
}
backend/windmill-api/src/ai_evals/template.rs
+132
View File
@@ -0,0 +1,132 @@
use super::*;
/// What a script scorer starts from.
pub const SCORER_SCRIPT_TEMPLATE: &str = r#"// A scorer receives one run and returns a number between 0 and 1, a boolean, or
// { score, reason, checks } — checks show up in the case detail.
// Return { score: null } for a case this scorer has nothing to measure on: the cell
// is left out of the column's mean and pass rate rather than counted as a zero.
//
// The run is also handed to you spelled out, so a short scorer can skip the type below
// entirely: export async function main(output: unknown, expected: unknown) { ... }
type ToolCall = {
name: string
args?: Record<string, unknown>
result?: unknown
error?: string
duration_ms?: number
truncated?: boolean
}
type EvalRun = {
input: { user_message?: string; user_attachments?: unknown[] }
output?: unknown
expected?: unknown
tool_calls: ToolCall[]
tools: { name: string; schema?: Record<string, unknown> }[]
metrics: { steps: number; duration_ms?: number; usage?: Record<string, unknown> }
status: string
job_id: string
}
export async function main(run: EvalRun) {
// How the agent got to its answer. Reported rather than scored: checks render in the case
// detail either way, so they explain the number without being averaged into it.
const checks = [
check('arguments match the schema', args_schema_valid(run)),
check('no repeated calls', no_repeated_calls(run)),
check('no failed tool calls', no_step_errors(run)),
check('under 6 steps', run.metrics.steps <= 6, `${run.metrics.steps} steps`),
check('under 30 seconds', under_ms(run, 30_000), `${run.metrics.duration_ms ?? '?'} ms`)
]
// Nothing to compare the answer against, so this column has no verdict on this case rather
// than a failing one. The cell reads n/a and the column's mean is of the cases it measured.
if (run.expected == undefined) {
return { score: null, reason: 'this case has no expected answer', checks }
}
// One question per column, and this column's question is whether the answer is right.
// Deliberately not the share of checks above that passed: a right answer that was slow and a
// wrong answer that was fast would score the same, and the column could not say which it was.
const correct = contains(run.output, text(run.expected))
return {
score: correct ? 1 : 0,
reason: correct ? undefined : `expected ${text(run.expected)}`,
checks
}
}
// Helpers. Edit or delete freely.
function check(name: string, passed: boolean, detail?: string) {
return { name, passed, detail }
}
function text(value: unknown): string {
return typeof value === 'string' ? value : JSON.stringify(value ?? '')
}
function contains(output: unknown, needle: string): boolean {
return needle.trim().length > 0 && text(output).toLowerCase().includes(needle.trim().toLowerCase())
}
// Every call validated against the schema of the tool it called. A tool whose schema could not be
// resolved is not checked rather than failed.
function args_schema_valid(run: EvalRun): boolean {
return run.tool_calls.every((call) => {
const schema = run.tools.find((tool) => tool.name === call.name)?.schema as
| { properties?: Record<string, { type?: string }>; required?: string[] }
| undefined
if (!schema?.properties) return true
const args = call.args ?? {}
for (const key of schema.required ?? []) {
if (args[key] === undefined || args[key] === null) return false
}
for (const [key, value] of Object.entries(args)) {
const expected = schema.properties[key]?.type
if (!expected) continue
const actual = Array.isArray(value) ? 'array' : value === null ? 'null' : typeof value
if (expected === 'integer' ? !Number.isInteger(value) : expected !== actual) return false
}
return true
})
}
// The same tool called twice with the same arguments.
function no_repeated_calls(run: EvalRun): boolean {
const seen = new Set<string>()
for (const call of run.tool_calls) {
const key = `${call.name}:${JSON.stringify(call.args ?? {})}`
if (seen.has(key)) return false
seen.add(key)
}
return true
}
function no_step_errors(run: EvalRun): boolean {
return run.status === 'success' && run.tool_calls.every((call) => !call.error)
}
// A run with no recorded duration is not under the limit: a check that could not be evaluated
// should not report as one that passed.
function under_ms(run: EvalRun, max: number): boolean {
const ms = run.metrics.duration_ms
return ms != undefined && ms <= max
}
"#;
#[derive(Serialize)]
pub struct ScorerDefaults {
/// The system prompt a judge agent is created with. It lives on that agent afterwards.
pub judge_prompt: String,
/// The starting point for a script scorer, held here so the shape a scorer is handed and the
/// template that reads it cannot drift apart.
pub script_template: String,
}
pub async fn scorer_defaults() -> JsonResult<ScorerDefaults> {
Ok(Json(ScorerDefaults {
judge_prompt: JUDGE_SYSTEM_PROMPT.to_string(),
script_template: SCORER_SCRIPT_TEMPLATE.to_string(),
}))
}

Some files were not shown because too many files have changed in this diff Show More