windmill/ai_evals/history/app.jsonl

{"createdAt":"2026-04-10T14:24:42.248Z","gitSha":"8f8b487be517a0bdd318c36857c1d46d5ab0723a","mode":"app","runs":1,"runModel":"anthropic:claude-haiku-4-5-20251001","judgeModel":"claude-sonnet-4-6","caseCount":9,"attemptCount":9,"passedAttempts":7,"passRate":0.7777777777777778,"averageDurationMs":25680.777777777777,"averageJudgeScore":76.55555555555556,"averageTokenUsagePerAttempt":{"prompt":53989.22222222222,"completion":2629.222222222222,"total":56618.444444444445},"failedCaseIds":["app-test8-inventory-tracker-create","app-test9-recipe-book-create"],"cases":[{"id":"app-test1-counter-create","attemptCount":1,"passedAttempts":1,"passRate":1,"averageDurationMs":11071,"averageJudgeScore":100,"averageTokenUsagePerAttempt":{"prompt":17912,"completion":1079,"total":18991}},{"id":"app-test2-counter-reset","attemptCount":1,"passedAttempts":1,"passRate":1,"averageDurationMs":12121,"averageJudgeScore":97,"averageTokenUsagePerAttempt":{"prompt":19088,"completion":833,"total":19921}},{"id":"app-test3-shopping-cart-quantity","attemptCount":1,"passedAttempts":1,"passRate":1,"averageDurationMs":25852,"averageJudgeScore":98,"averageTokenUsagePerAttempt":{"prompt":58834,"completion":2446,"total":61280}},{"id":"app-test4-shopping-cart-discount","attemptCount":1,"passedAttempts":1,"passRate":1,"averageDurationMs":42350,"averageJudgeScore":97,"averageTokenUsagePerAttempt":{"prompt":90882,"completion":4984,"total":95866}},{"id":"app-test5-file-manager-search","attemptCount":1,"passedAttempts":1,"passRate":1,"averageDurationMs":29129,"averageJudgeScore":100,"averageTokenUsagePerAttempt":{"prompt":81980,"completion":2817,"total":84797}},{"id":"app-test6-file-manager-inline-rename","attemptCount":1,"passedAttempts":1,"passRate":1,"averageDurationMs":51576,"averageJudgeScore":100,"averageTokenUsagePerAttempt":{"prompt":110023,"completion":6328,"total":116351}},{"id":"app-test7-file-manager-select-all","attemptCount":1,"passedAttempts":1,"passRate":1,"averageDurationMs":39256,"averageJudgeScore":97,"averageTokenUsagePerAttempt":{"prompt":72006,"completion":4188,"total":76194}},{"id":"app-test8-inventory-tracker-create","attemptCount":1,"passedAttempts":0,"passRate":0,"averageDurationMs":10514,"averageJudgeScore":0,"averageTokenUsagePerAttempt":{"prompt":17600,"completion":511,"total":18111}},{"id":"app-test9-recipe-book-create","attemptCount":1,"passedAttempts":0,"passRate":0,"averageDurationMs":9258,"averageJudgeScore":0,"averageTokenUsagePerAttempt":{"prompt":17578,"completion":477,"total":18055}}]}
{"createdAt":"2026-04-10T14:27:49.271Z","gitSha":"8f8b487be517a0bdd318c36857c1d46d5ab0723a","mode":"app","runs":1,"runModel":"anthropic:claude-opus-4-6","judgeModel":"claude-sonnet-4-6","caseCount":9,"attemptCount":9,"passedAttempts":6,"passRate":0.6666666666666666,"averageDurationMs":57285.666666666664,"averageJudgeScore":82.55555555555556,"averageTokenUsagePerAttempt":{"prompt":54435.77777777778,"completion":3668.6666666666665,"total":58104.444444444445},"failedCaseIds":["app-test7-file-manager-select-all","app-test8-inventory-tracker-create","app-test9-recipe-book-create"],"cases":[{"id":"app-test1-counter-create","attemptCount":1,"passedAttempts":1,"passRate":1,"averageDurationMs":17930,"averageJudgeScore":100,"averageTokenUsagePerAttempt":{"prompt":17620,"completion":743,"total":18363}},{"id":"app-test2-counter-reset","attemptCount":1,"passedAttempts":1,"passRate":1,"averageDurationMs":17852,"averageJudgeScore":100,"averageTokenUsagePerAttempt":{"prompt":18887,"completion":701,"total":19588}},{"id":"app-test3-shopping-cart-quantity","attemptCount":1,"passedAttempts":1,"passRate":1,"averageDurationMs":43501,"averageJudgeScore":97,"averageTokenUsagePerAttempt":{"prompt":38855,"completion":2692,"total":41547}},{"id":"app-test4-shopping-cart-discount","attemptCount":1,"passedAttempts":1,"passRate":1,"averageDurationMs":60820,"averageJudgeScore":98,"averageTokenUsagePerAttempt":{"prompt":61707,"completion":3420,"total":65127}},{"id":"app-test5-file-manager-search","attemptCount":1,"passedAttempts":1,"passRate":1,"averageDurationMs":45253,"averageJudgeScore":100,"averageTokenUsagePerAttempt":{"prompt":67244,"completion":3031,"total":70275}},{"id":"app-test6-file-manager-inline-rename","attemptCount":1,"passedAttempts":1,"passRate":1,"averageDurationMs":104837,"averageJudgeScore":98,"averageTokenUsagePerAttempt":{"prompt":116979,"completion":6834,"total":123813}},{"id":"app-test7-file-manager-select-all","attemptCount":1,"passedAttempts":0,"passRate":0,"averageDurationMs":73325,"averageJudgeScore":78,"averageTokenUsagePerAttempt":{"prompt":76351,"completion":5239,"total":81590}},{"id":"app-test8-inventory-tracker-create","attemptCount":1,"passedAttempts":0,"passRate":0,"averageDurationMs":133705,"averageJudgeScore":72,"averageTokenUsagePerAttempt":{"prompt":68546,"completion":9881,"total":78427}},{"id":"app-test9-recipe-book-create","attemptCount":1,"passedAttempts":0,"passRate":0,"averageDurationMs":18348,"averageJudgeScore":0,"averageTokenUsagePerAttempt":{"prompt":23733,"completion":477,"total":24210}}]}
{"createdAt":"2026-04-10T14:29:28.396Z","gitSha":"8f8b487be517a0bdd318c36857c1d46d5ab0723a","mode":"app","runs":1,"runModel":"openai:gpt-4o","judgeModel":"claude-sonnet-4-6","caseCount":9,"attemptCount":9,"passedAttempts":5,"passRate":0.5555555555555556,"averageDurationMs":31682.555555555555,"averageJudgeScore":73.11111111111111,"averageTokenUsagePerAttempt":{"prompt":27221.222222222223,"completion":1564.6666666666667,"total":28785.88888888889},"failedCaseIds":["app-test6-file-manager-inline-rename","app-test7-file-manager-select-all","app-test8-inventory-tracker-create","app-test9-recipe-book-create"],"cases":[{"id":"app-test1-counter-create","attemptCount":1,"passedAttempts":1,"passRate":1,"averageDurationMs":9911,"averageJudgeScore":100,"averageTokenUsagePerAttempt":{"prompt":8116,"completion":525,"total":8641}},{"id":"app-test2-counter-reset","attemptCount":1,"passedAttempts":1,"passRate":1,"averageDurationMs":15146,"averageJudgeScore":97,"averageTokenUsagePerAttempt":{"prompt":13096,"completion":576,"total":13672}},{"id":"app-test3-shopping-cart-quantity","attemptCount":1,"passedAttempts":1,"passRate":1,"averageDurationMs":31146,"averageJudgeScore":97,"averageTokenUsagePerAttempt":{"prompt":42424,"completion":1691,"total":44115}},{"id":"app-test4-shopping-cart-discount","attemptCount":1,"passedAttempts":1,"passRate":1,"averageDurationMs":49382,"averageJudgeScore":92,"averageTokenUsagePerAttempt":{"prompt":35785,"completion":3345,"total":39130}},{"id":"app-test5-file-manager-search","attemptCount":1,"passedAttempts":1,"passRate":1,"averageDurationMs":62963,"averageJudgeScore":100,"averageTokenUsagePerAttempt":{"prompt":46902,"completion":3590,"total":50492}},{"id":"app-test6-file-manager-inline-rename","attemptCount":1,"passedAttempts":0,"passRate":0,"averageDurationMs":24203,"averageJudgeScore":100,"averageTokenUsagePerAttempt":{"prompt":33121,"completion":498,"total":33619}},{"id":"app-test7-file-manager-select-all","attemptCount":1,"passedAttempts":0,"passRate":0,"averageDurationMs":74058,"averageJudgeScore":72,"averageTokenUsagePerAttempt":{"prompt":46026,"completion":3591,"total":49617}},{"id":"app-test8-inventory-tracker-create","attemptCount":1,"passedAttempts":0,"passRate":0,"averageDurationMs":6757,"averageJudgeScore":0,"averageTokenUsagePerAttempt":{"prompt":7770,"completion":165,"total":7935}},{"id":"app-test9-recipe-book-create","attemptCount":1,"passedAttempts":0,"passRate":0,"averageDurationMs":11577,"averageJudgeScore":0,"averageTokenUsagePerAttempt":{"prompt":11751,"completion":101,"total":11852}}]}
{"createdAt":"2026-04-17T10:12:15.586Z","gitSha":"4bda600729f907514f3f58728f2e592d4d1495ed","mode":"app","runs":1,"runModel":"anthropic:claude-haiku-4-5-20251001","judgeModel":"claude-sonnet-4-6","caseCount":9,"attemptCount":9,"passedAttempts":7,"passRate":0.7777777777777778,"averageDurationMs":24964.555555555555,"averageJudgeScore":78.55555555555556,"averageTokenUsagePerAttempt":{"prompt":57243.88888888889,"completion":2763.6666666666665,"total":60007.555555555555},"failedCaseIds":["app-test8-inventory-tracker-create","app-test9-recipe-book-create"],"cases":[{"id":"app-test1-counter-create","attemptCount":1,"passedAttempts":1,"passRate":1,"averageDurationMs":10529,"averageJudgeScore":100,"averageTokenUsagePerAttempt":{"prompt":17912,"completion":1082,"total":18994}},{"id":"app-test2-counter-reset","attemptCount":1,"passedAttempts":1,"passRate":1,"averageDurationMs":10884,"averageJudgeScore":95,"averageTokenUsagePerAttempt":{"prompt":19088,"completion":833,"total":19921}},{"id":"app-test3-shopping-cart-quantity","attemptCount":1,"passedAttempts":1,"passRate":1,"averageDurationMs":22053,"averageJudgeScore":98,"averageTokenUsagePerAttempt":{"prompt":58812,"completion":2454,"total":61266}},{"id":"app-test4-shopping-cart-discount","attemptCount":1,"passedAttempts":1,"passRate":1,"averageDurationMs":26512,"averageJudgeScore":90,"averageTokenUsagePerAttempt":{"prompt":61312,"completion":2904,"total":64216}},{"id":"app-test5-file-manager-search","attemptCount":1,"passedAttempts":1,"passRate":1,"averageDurationMs":26169,"averageJudgeScore":100,"averageTokenUsagePerAttempt":{"prompt":82004,"completion":2831,"total":84835}},{"id":"app-test6-file-manager-inline-rename","attemptCount":1,"passedAttempts":1,"passRate":1,"averageDurationMs":48964,"averageJudgeScore":97,"averageTokenUsagePerAttempt":{"prompt":140937,"completion":6017,"total":146954}},{"id":"app-test7-file-manager-select-all","attemptCount":1,"passedAttempts":1,"passRate":1,"averageDurationMs":32094,"averageJudgeScore":97,"averageTokenUsagePerAttempt":{"prompt":72006,"completion":4192,"total":76198}},{"id":"app-test8-inventory-tracker-create","attemptCount":1,"passedAttempts":0,"passRate":0,"averageDurationMs":8630,"averageJudgeScore":0,"averageTokenUsagePerAttempt":{"prompt":17600,"completion":503,"total":18103}},{"id":"app-test9-recipe-book-create","attemptCount":1,"passedAttempts":0,"passRate":0,"averageDurationMs":38846,"averageJudgeScore":30,"averageTokenUsagePerAttempt":{"prompt":45524,"completion":4057,"total":49581}}]}
{"createdAt":"2026-04-17T14:08:50.784Z","gitSha":"71b0e6f9650e1259f14eddded7c00ab2feaf52b3","mode":"app","runs":1,"runModel":"anthropic:claude-haiku-4-5-20251001","judgeModel":"claude-sonnet-4-6","caseCount":9,"attemptCount":9,"passedAttempts":9,"passRate":1,"averageDurationMs":35878.77777777778,"averageJudgeScore":97.77777777777777,"averageTokenUsagePerAttempt":{"prompt":89905.22222222222,"completion":3967.5555555555557,"total":93872.77777777778},"failedCaseIds":[],"cases":[{"id":"app-test1-counter-create","attemptCount":1,"passedAttempts":1,"passRate":1,"averageDurationMs":11084,"averageJudgeScore":100,"averageTokenUsagePerAttempt":{"prompt":17912,"completion":1079,"total":18991}},{"id":"app-test2-counter-reset","attemptCount":1,"passedAttempts":1,"passRate":1,"averageDurationMs":13165,"averageJudgeScore":97,"averageTokenUsagePerAttempt":{"prompt":26438,"completion":920,"total":27358}},{"id":"app-test3-shopping-cart-quantity","attemptCount":1,"passedAttempts":1,"passRate":1,"averageDurationMs":23328,"averageJudgeScore":100,"averageTokenUsagePerAttempt":{"prompt":59255,"completion":2437,"total":61692}},{"id":"app-test4-shopping-cart-discount","attemptCount":1,"passedAttempts":1,"passRate":1,"averageDurationMs":29138,"averageJudgeScore":97,"averageTokenUsagePerAttempt":{"prompt":62296,"completion":2969,"total":65265}},{"id":"app-test5-file-manager-search","attemptCount":1,"passedAttempts":1,"passRate":1,"averageDurationMs":26743,"averageJudgeScore":98,"averageTokenUsagePerAttempt":{"prompt":82542,"completion":2928,"total":85470}},{"id":"app-test6-file-manager-inline-rename","attemptCount":1,"passedAttempts":1,"passRate":1,"averageDurationMs":75172,"averageJudgeScore":97,"averageTokenUsagePerAttempt":{"prompt":238398,"completion":9612,"total":248010}},{"id":"app-test7-file-manager-select-all","attemptCount":1,"passedAttempts":1,"passRate":1,"averageDurationMs":41074,"averageJudgeScore":97,"averageTokenUsagePerAttempt":{"prompt":94998,"completion":4353,"total":99351}},{"id":"app-test8-inventory-tracker-search-delete","attemptCount":1,"passedAttempts":1,"passRate":1,"averageDurationMs":56640,"averageJudgeScore":97,"averageTokenUsagePerAttempt":{"prompt":110728,"completion":6817,"total":117545}},{"id":"app-test9-recipe-book-search-delete","attemptCount":1,"passedAttempts":1,"passRate":1,"averageDurationMs":46565,"averageJudgeScore":97,"averageTokenUsagePerAttempt":{"prompt":116580,"completion":4593,"total":121173}}]}
{"createdAt":"2026-04-17T14:13:13.640Z","gitSha":"71b0e6f9650e1259f14eddded7c00ab2feaf52b3","mode":"app","runs":1,"runModel":"anthropic:claude-opus-4-6","judgeModel":"claude-sonnet-4-6","caseCount":9,"attemptCount":9,"passedAttempts":9,"passRate":1,"averageDurationMs":88346.11111111111,"averageJudgeScore":97.88888888888889,"averageTokenUsagePerAttempt":{"prompt":104798.77777777778,"completion":7132.444444444444,"total":111931.22222222222},"failedCaseIds":[],"cases":[{"id":"app-test1-counter-create","attemptCount":1,"passedAttempts":1,"passRate":1,"averageDurationMs":28005,"averageJudgeScore":100,"averageTokenUsagePerAttempt":{"prompt":25364,"completion":1414,"total":26778}},{"id":"app-test2-counter-reset","attemptCount":1,"passedAttempts":1,"passRate":1,"averageDurationMs":17380,"averageJudgeScore":100,"averageTokenUsagePerAttempt":{"prompt":18883,"completion":680,"total":19563}},{"id":"app-test3-shopping-cart-quantity","attemptCount":1,"passedAttempts":1,"passRate":1,"averageDurationMs":47370,"averageJudgeScore":100,"averageTokenUsagePerAttempt":{"prompt":39144,"completion":2688,"total":41832}},{"id":"app-test4-shopping-cart-discount","attemptCount":1,"passedAttempts":1,"passRate":1,"averageDurationMs":65811,"averageJudgeScore":98,"averageTokenUsagePerAttempt":{"prompt":66308,"completion":4148,"total":70456}},{"id":"app-test5-file-manager-search","attemptCount":1,"passedAttempts":1,"passRate":1,"averageDurationMs":59691,"averageJudgeScore":100,"averageTokenUsagePerAttempt":{"prompt":87497,"completion":4335,"total":91832}},{"id":"app-test6-file-manager-inline-rename","attemptCount":1,"passedAttempts":1,"passRate":1,"averageDurationMs":235728,"averageJudgeScore":98,"averageTokenUsagePerAttempt":{"prompt":290327,"completion":22188,"total":312515}},{"id":"app-test7-file-manager-select-all","attemptCount":1,"passedAttempts":1,"passRate":1,"averageDurationMs":77130,"averageJudgeScore":90,"averageTokenUsagePerAttempt":{"prompt":115688,"completion":5309,"total":120997}},{"id":"app-test8-inventory-tracker-search-delete","attemptCount":1,"passedAttempts":1,"passRate":1,"averageDurationMs":137948,"averageJudgeScore":97,"averageTokenUsagePerAttempt":{"prompt":152604,"completion":12560,"total":165164}},{"id":"app-test9-recipe-book-search-delete","attemptCount":1,"passedAttempts":1,"passRate":1,"averageDurationMs":126052,"averageJudgeScore":98,"averageTokenUsagePerAttempt":{"prompt":147374,"completion":10870,"total":158244}}]}
{"createdAt":"2026-04-17T14:15:46.134Z","gitSha":"71b0e6f9650e1259f14eddded7c00ab2feaf52b3","mode":"app","runs":1,"runModel":"openai:gpt-4o","judgeModel":"claude-sonnet-4-6","caseCount":9,"attemptCount":9,"passedAttempts":8,"passRate":0.8888888888888888,"averageDurationMs":54153.666666666664,"averageJudgeScore":90.77777777777777,"averageTokenUsagePerAttempt":{"prompt":38286.11111111111,"completion":2335.4444444444443,"total":40621.555555555555},"failedCaseIds":["app-test6-file-manager-inline-rename"],"cases":[{"id":"app-test1-counter-create","attemptCount":1,"passedAttempts":1,"passRate":1,"averageDurationMs":17177,"averageJudgeScore":85,"averageTokenUsagePerAttempt":{"prompt":8116,"completion":515,"total":8631}},{"id":"app-test2-counter-reset","attemptCount":1,"passedAttempts":1,"passRate":1,"averageDurationMs":17352,"averageJudgeScore":97,"averageTokenUsagePerAttempt":{"prompt":18184,"completion":578,"total":18762}},{"id":"app-test3-shopping-cart-quantity","attemptCount":1,"passedAttempts":1,"passRate":1,"averageDurationMs":47796,"averageJudgeScore":97,"averageTokenUsagePerAttempt":{"prompt":43405,"completion":1801,"total":45206}},{"id":"app-test4-shopping-cart-discount","attemptCount":1,"passedAttempts":1,"passRate":1,"averageDurationMs":51772,"averageJudgeScore":82,"averageTokenUsagePerAttempt":{"prompt":42880,"completion":1727,"total":44607}},{"id":"app-test5-file-manager-search","attemptCount":1,"passedAttempts":1,"passRate":1,"averageDurationMs":74300,"averageJudgeScore":95,"averageTokenUsagePerAttempt":{"prompt":46961,"completion":3570,"total":50531}},{"id":"app-test6-file-manager-inline-rename","attemptCount":1,"passedAttempts":0,"passRate":0,"averageDurationMs":16454,"averageJudgeScore":100,"averageTokenUsagePerAttempt":{"prompt":22703,"completion":359,"total":23062}},{"id":"app-test7-file-manager-select-all","attemptCount":1,"passedAttempts":1,"passRate":1,"averageDurationMs":103369,"averageJudgeScore":82,"averageTokenUsagePerAttempt":{"prompt":77450,"completion":5413,"total":82863}},{"id":"app-test8-inventory-tracker-search-delete","attemptCount":1,"passedAttempts":1,"passRate":1,"averageDurationMs":64880,"averageJudgeScore":97,"averageTokenUsagePerAttempt":{"prompt":42976,"completion":3628,"total":46604}},{"id":"app-test9-recipe-book-search-delete","attemptCount":1,"passedAttempts":1,"passRate":1,"averageDurationMs":94283,"averageJudgeScore":82,"averageTokenUsagePerAttempt":{"prompt":41900,"completion":3428,"total":45328}}]}