From 5fd94111c2c6415ee9a3ef83ca29e5ebd6aa2ded Mon Sep 17 00:00:00 2001 From: Sai Nikhil Date: Tue, 27 Oct 2026 17:47:24 +0000 Subject: [PATCH] feat(llm-evals): redesign the quality page on the new quality apis (#15244) Co-authored-by: udev Co-authored-by: github-actions[bot] <41898282+github-actions[bot]@users.noreply.github.com> --- scripts/translations/.translation_state.json | 236 +++--- src/api/http/src/handler/http/router/mod.rs | 7 +- src/api/management/src/models/mod.rs | 2 + src/api/management/src/models/quality.rs | 110 +++ src/api/management/src/request/mod.rs | 2 + src/api/management/src/request/quality.rs | 270 +++++++ .../src/meta/self_reporting/evaluator.rs | 4 + .../src/meta/self_reporting/llm_scores.rs | 26 + src/core/src/llm_evaluations/mod.rs | 4 +- src/core/src/pipeline/batch_execution.rs | 2 + .../src/self_reporting/evaluator_schema.rs | 6 + web/eslint.config.js | 4 - .../enterprise/components/OnlineEvals.spec.ts | 365 ++++++++++ web/src/enterprise/components/OnlineEvals.vue | 405 +++++------ .../onlineEvals/QualityPage.spec.ts | 432 ++++++++--- .../components/onlineEvals/QualityPage.vue | 685 ++++++++---------- .../composables/useQualityConfigDetail.ts | 395 ---------- .../onlineEvals/composables/useQualityData.ts | 395 ---------- .../composables/useQualityDetail.spec.ts | 320 ++++++++ .../composables/useQualityDetail.ts | 191 +++++ .../composables/useQualityDetailCharts.ts | 335 --------- .../composables/useQualityDetailScope.spec.ts | 108 --- .../composables/useQualityList.spec.ts | 170 +++++ .../onlineEvals/composables/useQualityList.ts | 120 +++ .../composables/useQualityRuns.spec.ts | 263 ------- .../onlineEvals/composables/useQualityRuns.ts | 466 ------------ .../useQualityScoreConfigs.spec.ts | 158 ---- .../composables/useQualityScoreConfigs.ts | 379 ---------- .../composables/useScoreFullText.ts | 83 +++ .../quality/QualityBooleanBarsChart.vue | 86 --- .../quality/QualityBooleanTrendChart.vue | 123 ---- .../quality/QualityCategoryBarsChart.vue | 106 --- .../quality/QualityConfigDetail.spec.ts | 511 +++++++++++++ .../quality/QualityConfigDetail.vue | 479 ++++++++++++ .../quality/QualityConfigsTable.spec.ts | 345 +++++++++ .../quality/QualityConfigsTable.vue | 396 ++++++++++ .../quality/QualityDetailPanel.spec.ts | 114 --- .../quality/QualityDetailPanel.vue | 440 ----------- .../quality/QualityDistributionChart.spec.ts | 69 ++ .../quality/QualityDistributionChart.vue | 267 ++++--- .../quality/QualityKpiCard.spec.ts | 79 -- .../onlineEvals/quality/QualityKpiCard.vue | 196 ----- .../quality/QualityKpiSkeleton.vue | 34 - .../quality/QualityRunsTable.spec.ts | 157 ---- .../onlineEvals/quality/QualityRunsTable.vue | 310 -------- .../quality/QualityScoreConfigsTable.spec.ts | 161 ---- .../quality/QualityScoreConfigsTable.vue | 399 ---------- .../quality/QualityScorePane.spec.ts | 513 +++++++++++++ .../onlineEvals/quality/QualityScorePane.vue | 334 +++++++++ .../quality/QualityTrendChart.spec.ts | 82 +++ .../onlineEvals/quality/QualityTrendChart.vue | 289 +++++--- .../onlineEvals/utils/agentFilterSql.spec.ts | 34 +- .../onlineEvals/utils/agentFilterSql.ts | 28 +- .../onlineEvals/utils/latestScoreSql.spec.ts | 16 +- .../onlineEvals/utils/latestScoreSql.ts | 52 -- .../onlineEvals/utils/qualityFormat.spec.ts | 588 +++++++++++++++ .../onlineEvals/utils/qualityFormat.ts | 498 +++++++++++++ .../onlineEvals/utils/qualityScope.spec.ts | 26 - .../onlineEvals/utils/qualityScope.ts | 43 -- .../onlineEvals/utils/qualitySummary.spec.ts | 118 --- .../onlineEvals/utils/qualitySummary.ts | 114 +-- .../onlineEvals/utils/scoreThreshold.ts | 150 +--- .../composables/useAgentScope.spec.ts | 84 +++ .../enterprise/composables/useAgentScope.ts | 47 +- .../composables/useAiDateRange.spec.ts | 2 +- .../enterprise/composables/useAiDateRange.ts | 14 +- .../AIObservability/ReviewContentBox.spec.ts | 31 + .../AIObservability/ReviewContentBox.vue | 25 +- web/src/lib/core/Badge/badgeGroups.ts | 9 +- web/src/lib/core/Button/OButton.spec.ts | 14 + web/src/lib/core/Button/OButton.types.ts | 4 +- web/src/lib/core/Button/OButton.vue | 9 + web/src/lib/styles/tokens/semantic.css | 4 +- .../vue-shortcut-manager/shortcutRegistry.ts | 11 + web/src/locales/languages/ar-SA.json | 234 +++--- web/src/locales/languages/de-DE.json | 236 +++--- web/src/locales/languages/en-US.json | 236 +++--- web/src/locales/languages/es-ES.json | 234 +++--- web/src/locales/languages/fr-FR.json | 236 +++--- web/src/locales/languages/it-IT.json | 238 +++--- web/src/locales/languages/ja-JP.json | 234 +++--- web/src/locales/languages/ko-KR.json | 236 +++--- web/src/locales/languages/nl-NL.json | 236 +++--- web/src/locales/languages/pl-PL.json | 238 +++--- web/src/locales/languages/pt-PT.json | 236 +++--- web/src/locales/languages/ru-RU.json | 238 +++--- web/src/locales/languages/tr-TR.json | 238 +++--- web/src/locales/languages/vi-VN.json | 236 +++--- web/src/locales/languages/zh-CN.json | 232 +++--- web/src/locales/languages/zh-TW.json | 234 +++--- .../services/online-evals.service.queries.ts | 65 +- .../online-evals.service.querykeys.ts | 11 + web/src/services/online-evals.service.spec.ts | 63 ++ web/src/services/online-evals.service.ts | 136 ++++ 94 files changed, 8961 insertions(+), 8440 deletions(-) create mode 100644 src/api/management/src/models/quality.rs create mode 100644 src/api/management/src/request/quality.rs create mode 100644 web/src/enterprise/components/OnlineEvals.spec.ts delete mode 100644 web/src/enterprise/components/onlineEvals/composables/useQualityConfigDetail.ts delete mode 100644 web/src/enterprise/components/onlineEvals/composables/useQualityData.ts create mode 100644 web/src/enterprise/components/onlineEvals/composables/useQualityDetail.spec.ts create mode 100644 web/src/enterprise/components/onlineEvals/composables/useQualityDetail.ts delete mode 100644 web/src/enterprise/components/onlineEvals/composables/useQualityDetailCharts.ts delete mode 100644 web/src/enterprise/components/onlineEvals/composables/useQualityDetailScope.spec.ts create mode 100644 web/src/enterprise/components/onlineEvals/composables/useQualityList.spec.ts create mode 100644 web/src/enterprise/components/onlineEvals/composables/useQualityList.ts delete mode 100644 web/src/enterprise/components/onlineEvals/composables/useQualityRuns.spec.ts delete mode 100644 web/src/enterprise/components/onlineEvals/composables/useQualityRuns.ts delete mode 100644 web/src/enterprise/components/onlineEvals/composables/useQualityScoreConfigs.spec.ts delete mode 100644 web/src/enterprise/components/onlineEvals/composables/useQualityScoreConfigs.ts create mode 100644 web/src/enterprise/components/onlineEvals/composables/useScoreFullText.ts delete mode 100644 web/src/enterprise/components/onlineEvals/quality/QualityBooleanBarsChart.vue delete mode 100644 web/src/enterprise/components/onlineEvals/quality/QualityBooleanTrendChart.vue delete mode 100644 web/src/enterprise/components/onlineEvals/quality/QualityCategoryBarsChart.vue create mode 100644 web/src/enterprise/components/onlineEvals/quality/QualityConfigDetail.spec.ts create mode 100644 web/src/enterprise/components/onlineEvals/quality/QualityConfigDetail.vue create mode 100644 web/src/enterprise/components/onlineEvals/quality/QualityConfigsTable.spec.ts create mode 100644 web/src/enterprise/components/onlineEvals/quality/QualityConfigsTable.vue delete mode 100644 web/src/enterprise/components/onlineEvals/quality/QualityDetailPanel.spec.ts delete mode 100644 web/src/enterprise/components/onlineEvals/quality/QualityDetailPanel.vue create mode 100644 web/src/enterprise/components/onlineEvals/quality/QualityDistributionChart.spec.ts delete mode 100644 web/src/enterprise/components/onlineEvals/quality/QualityKpiCard.spec.ts delete mode 100644 web/src/enterprise/components/onlineEvals/quality/QualityKpiCard.vue delete mode 100644 web/src/enterprise/components/onlineEvals/quality/QualityKpiSkeleton.vue delete mode 100644 web/src/enterprise/components/onlineEvals/quality/QualityRunsTable.spec.ts delete mode 100644 web/src/enterprise/components/onlineEvals/quality/QualityRunsTable.vue delete mode 100644 web/src/enterprise/components/onlineEvals/quality/QualityScoreConfigsTable.spec.ts delete mode 100644 web/src/enterprise/components/onlineEvals/quality/QualityScoreConfigsTable.vue create mode 100644 web/src/enterprise/components/onlineEvals/quality/QualityScorePane.spec.ts create mode 100644 web/src/enterprise/components/onlineEvals/quality/QualityScorePane.vue create mode 100644 web/src/enterprise/components/onlineEvals/quality/QualityTrendChart.spec.ts create mode 100644 web/src/enterprise/components/onlineEvals/utils/qualityFormat.spec.ts create mode 100644 web/src/enterprise/components/onlineEvals/utils/qualityFormat.ts delete mode 100644 web/src/enterprise/components/onlineEvals/utils/qualityScope.spec.ts delete mode 100644 web/src/enterprise/components/onlineEvals/utils/qualityScope.ts delete mode 100644 web/src/enterprise/components/onlineEvals/utils/qualitySummary.spec.ts diff --git a/scripts/translations/.translation_state.json b/scripts/translations/.translation_state.json index f74ce37ae0..7588c6ffe7 100644 --- a/scripts/translations/.translation_state.json +++ b/scripts/translations/.translation_state.json @@ -8037,11 +8037,10 @@ "paused": "e159b06187d369a0" }, "qualityStatus": { + "attention": "c1ebc7817870e5be", "healthy": "7f1e323b6272c620", - "nodata": "af75117dfbb0edb7", - "nothreshold": "e343e1b48e909484", - "unhealthy": "317b1fbc1e13beeb", - "warn": "8448814dc5caff5c" + "nodata": "dcb61373fe73eca0", + "unset": "237cddc737b07bf1" }, "queryStatus": { "canceled": "13ca2ee2499310d9", @@ -18324,7 +18323,6 @@ "testConnection": "c02977b07ec93816" }, "catalogScoreConfigNotFound": "f40f5ef1f9371929", - "count": "37bac4952becf21a", "deleteConfirmMessage": "db7e1f9dabc0cbc7", "deleteError": "4e1252dd62c17a4b", "deleteTitle": "0f81c0d86606f518", @@ -18335,7 +18333,6 @@ "failedToImportScorer": "657edceb184dad48", "failedToLoadCatalog": "81454f428d2eca65", "failedToRunScorerTest": "8efda6002ab8a147", - "healthyRate": "cfbdb2419025378c", "import": { "alreadyExistsSkipped": "7138074e5680ecc5", "createdSuccessfully": "a669346209687818", @@ -18727,143 +18724,122 @@ "typeRequired": "2d973f48294309ac" }, "quality": { - "agentLabel": "11b39c93777e8f1f", - "agentPlaceholder": "54c32d3e2cfa1cc8", "allAgents": "2ab73ea5b9f48140", - "averageWithHealthyRate": "457b958204b8415f", - "dataTypes": { - "booleanShort": "49f411f0a1a7f719", - "categoricalShort": "48735c4fae42d150", - "numericShort": "03732d19e64b727a" - }, "detail": { - "categoryDistributionTitle": "4a0f40cef45fc3b8", - "distributionTitle": "204e97ff526e88bc", - "empty": { - "description": "33a0c0ddb152c545", - "descriptionScoped": "040bea517fb3dd81", - "title": "c28eabc50c6172de", - "titleScoped": "0f6ba020af4d8734" - }, - "health": { - "noThreshold": "757555170371d25d", - "summary": "e4ef8942e7382c26", - "thresholdOnly": "f7e60f4f5bf81dcd", - "title": "bb2dc38ee25dd468" - }, - "healthyRateTitle": "fb401c8635c7b907", - "healthyThreshold": "870d9a17c5faf64e", - "kpis": { - "average": "b38ae2c5b84529e8", - "coverage": "523487a5de21adb5", - "falseRate": "9274b4e0211a61dd", - "healthy": "7f1e323b6272c620", - "p50": "fa4cedf31b643396", - "p95": "aebf8ec133e867b7", - "targetsScored": "5e1eea4fa8c36200", - "trueRate": "d5c5a6de7f1f2849", - "unhealthy": "317b1fbc1e13beeb" - }, + "averageValue": "b0b2248bc9e5f488", + "back": "d1da712ff756faed", + "backToAll": "66a0c37b52f215ed", + "barButton": "02b069f89d756025", + "chartTooltip": "5a613891de20bb60", + "chartTooltipCount": "06b2297283d5afa0", + "clearFilter": "ba59e2d4a5e39a42", + "distributionNumeric": "204e97ff526e88bc", + "distributionValues": "22197a16c5a80914", + "editConfig": "3f140981c2a04443", + "emptyDescription": "828c3fc50f4b18ef", + "emptyTitle": "c28eabc50c6172de", + "emptyTitleScoped": "0f6ba020af4d8734", + "filterAny": "004f62d2d5b5c5be", + "filterAtLeast": "9bb325debb8a823f", + "filterBelow": "e574e3d7f92935e9", + "filterByBar": "90d0c8c1d694680e", + "filterRange": "ba318cc195ceff48", "legendAvg": "41835ff14ac6b8c7", - "legendFalseCount": "20ee2de2332893c8", - "legendHealthy": "87695fdac81728b9", - "legendHealthyRate": "cfbdb2419025378c", - "legendThreshold": "e5920260ed27825f", - "legendTrueCount": "dad221927b66d136", - "legendTrueRate": "d5c5a6de7f1f2849", - "legendUnhealthy": "85826072c3d4c3a4", - "loading": "f3051ab55cc051a4", + "legendHealthy": "7f1e323b6272c620", + "legendScores": "c5f09715dba8604d", + "legendThreshold": "0bfbdd46a5674de7", + "legendUnhealthy": "317b1fbc1e13beeb", "markLineHealthy": "bcd6d4cd56e53ff0", - "noData": "2887c63c9003316c", - "ofScoreResults": "6979fb5bfdf392ec", - "range": "9c6a5df138917ac8", - "trendTitle": "9e217716c4e03eee", - "trueFalseTitle": "427cbdc3dde45771", - "trueRateTitle": "7fd6f91f0c0ad748" - }, - "kpis": { - "evaluationCost": { - "title": "dd5b14b252fc637d" + "notFound": "034503d3e44e3bb1", + "notFoundDescription": "67392d34bc6a3ea6", + "trendLegend": { + "count": "c5f09715dba8604d", + "healthyRate": "cfbdb2419025378c", + "trueRate": "d5c5a6de7f1f2849" }, - "latencyP95": { - "title": "dddc206eaf9f7ec2" - }, - "noData": "bda050585a00f0f6", - "noPriorData": "e80c62b92be4c15a", - "scoreResults": { - "title": "db1a23cbdc30e12d" - }, - "scorerFailures": { - "title": "4997bcc2bd10ef67" - }, - "scorerSuccess": { - "title": "ff716aa8a234d071" - }, - "vsPrev": "1c2453c2978f9985" - }, - "kpisAriaLabel": "49409b7c87cbb436", - "overview": { - "columns": { - "health": "55898449eb74fb2e", - "quality": "1b2c08a8733d7ff1", - "scopeMix": "55479bfa00b783b8", - "scoreConfig": "84039a14d5ad9a94", - "totalScores": "fe5f9ade2071a6eb", - "type": "baaddf70fb5d432b", - "updated": "3a5ecca188c0579c", - "volumeTrend": "180920ae5ef94340" - }, - "health": { - "noDataHint": "78737bac13daa445", - "noThresholdHint": "a708e7ac6bddb316", - "thresholdSummary": "5154cd5d6e1c2a4a" - }, - "relativeTime": { - "daysAgo": "33a682af5904321a", - "monthsAgo": "7289f8e6f0ed365e" - }, - "searchPlaceholder": "8bd11a03fc493153", - "status": { - "healthy": "7f1e323b6272c620", - "noData": "dcb61373fe73eca0", - "noThreshold": "237cddc737b07bf1", - "unhealthy": "c1ebc7817870e5be" + "trendTitle": { + "count": "43169ca648570cd2", + "healthyRate": "fb401c8635c7b907", + "score": "9e217716c4e03eee", + "trueRate": "7fd6f91f0c0ad748" } }, - "runs": { - "columns": { - "agent": "11b39c93777e8f1f", - "reasoning": "d8211e24e83d1600", - "result": "38e5a46cbc5ad328", - "target": "978354db0c00fc78", - "time": "ca533a85286317b4" - }, - "empty": "20c547d2d24ce30e", - "emptyUnhealthy": "cc13d51eb9e8b753", - "filterLabel": "8659d92fb947e840", - "filters": { - "all": "7d87c534d08f768a", - "unhealthy": "317b1fbc1e13beeb" - }, - "health": { - "healthy": "7f1e323b6272c620", - "unhealthy": "317b1fbc1e13beeb" - }, - "hint": "814307c2d705e314", + "healthyIf": "61314bb9a51b80bf", + "noHealthyValue": "140bedbf9c3f6d56", + "noScoresInWindow": "78737bac13daa445", + "outOf": "61dd1ffbd6954439", + "overview": { + "relativeTime": { + "daysAgo": "33a682af5904321a" + } + }, + "pane": { + "addToDataset": "159d93ec1948b77b", + "datasetNoStream": "bb078f15b2d4f118", + "datasetSession": "37fd1c0485ec875b", + "evaluatorTrace": "fe5c37f83b4c578d", + "fullTextUnavailable": "ba96812e4442db16", + "healthy": "7f1e323b6272c620", + "next": "1ddf974477c58115", "noReasoning": "b20ff56cb508dc0c", - "noThresholdHint": "e40d1f44676c2c45", - "queryError": "8ab4ae56eee26833", + "noThreshold": "2079685ba275f6cf", + "noneSelected": "80974ccb9819eda0", + "open": { + "session": "b205bb47f81a3096", + "span": "968c142311163240", + "trace": "c331a5916f7f017b" + }, + "previous": "5fc351495fd548d4", + "reasoning": "d8211e24e83d1600", + "unhealthy": "317b1fbc1e13beeb" + }, + "review": { + "all": "a52ace420f2175d0", + "inputNotCaptured": "6dfe0059fc5db462", + "noMatch": "6601ab7c6987bf11", + "outputNotCaptured": "45106b5442854c38", "title": "66055b66d4d610b4", - "unknownAgent": "342b4ab21e2eb31a", - "unknownScope": "6246e12e56bc3aa2" + "unhealthy": "317b1fbc1e13beeb", + "unhealthyDisabled": "64c27eff26d9a562" + }, + "scopeCount": "5411544954ed8f59", + "scopeNouns": { + "session": "3f3af1ecebbd1410", + "span": "37a0025aa1f2e29f", + "trace": "eafe895eb8119e6e" }, "scopes": { "all": "a52ace420f2175d0", - "filterHint": "395e2b0c8bb6b19e", - "filterLabel": "503647ae568c573b", "session": "6959b4159575d8dd", "span": "5b0e83c19eb41845", "trace": "7e14c3d7dd12bbd0" + }, + "setThreshold": "a708e7ac6bddb316", + "table": { + "averageRange": "2318f0d9be0999c5", + "columns": { + "health": "55898449eb74fb2e", + "scoreConfig": "84039a14d5ad9a94", + "scores": "c5f09715dba8604d", + "typicalScore": "21242c16652268a4", + "unhealthy": "317b1fbc1e13beeb", + "updated": "3a5ecca188c0579c" + }, + "configLink": "87e89abb4c1c551f", + "mostCommon": "e1f0b823d6de14a9", + "openUnhealthy": "47f8f7ac3b996ddb", + "searchPlaceholder": "8bd11a03fc493153" + }, + "tiles": { + "allGiveResult": "b43db3198ee5d6f8", + "attention": "c705641cc5b4fd62", + "attentionNote": "cab88e281cc40392", + "attentionTooltip": "12adcafc7502d45d", + "noScores": "c48b40434b3bfa84", + "noThreshold": "259270af3152ab51", + "ofTotal": "43db84087c320671", + "withoutResult": "496e670f4873446a", + "withoutResultTooltip": "00cec2e30b91d0bd" } }, "saved": "1e37a2e5e2cd6f23", @@ -19267,7 +19243,6 @@ "scorers": "d8598427499abb65" }, "title": "b25cc6989b4f2bc5", - "topCategory": "24ddf469eeb218d4", "traceScoreChip": { "allScores": "1e99f88511c40151", "moreCount": "ecccea94c62457a7", @@ -19275,7 +19250,6 @@ "reasoning": "d3ae14a50bfe463f", "scoredAt": "49f475a142832d91" }, - "trueRate": "d5c5a6de7f1f2849", "unknownError": "27c2ccd962c2b8dc", "validation": { "mustBeNumberBetweenZeroAndOne": "3afc12fc4cc6dca0", @@ -23052,6 +23026,7 @@ "pipelinesAdd": "f08dbc4665fafa23", "pipelinesImport": "d065d3e3aba4359c", "pipelinesRefresh": "3cf8d66227384dc2", + "qualityBack": "66a0c37b52f215ed", "regexPatternsRefresh": "3cf8d66227384dc2", "reportsAdd": "529b9d4aa1cf7301", "reportsRefresh": "3cf8d66227384dc2", @@ -23153,6 +23128,7 @@ "panelEditor": "2f8a354915ba91ad", "pipelineDestinations": "c973c318806d3c92", "pipelines": "d1a8fffeb935847e", + "quality": "3d1c3c527d4e499a", "regexPatterns": "83a58bf362610444", "reports": "dacca3cba3f346a4", "rumErrors": "52dd701cb2cf1fd2", diff --git a/src/api/http/src/handler/http/router/mod.rs b/src/api/http/src/handler/http/router/mod.rs index 1fa65487fb..67abd19f6c 100644 --- a/src/api/http/src/handler/http/router/mod.rs +++ b/src/api/http/src/handler/http/router/mod.rs @@ -63,8 +63,8 @@ use { openobserve_api_management::request::{ ai, alert_calibration, annotation_queues, annotations, anomaly_detection, datasets, discovery, domain_management, eval_jobs, experiments, fleet, gen_ai, keys, license, oncall, - playground, prompts, providers, remote_tasks, score_configs, scorers, service_streams, - workflows, + playground, prompts, providers, quality, remote_tasks, score_configs, scorers, + service_streams, workflows, }, openobserve_api_pipelines::request::re_pattern, openobserve_api_search::search::patterns, @@ -1763,6 +1763,9 @@ pub fn service_routes() -> Router { .route("/{org_id}/tasks/{entity_id}", get(remote_tasks::get_remote_task).put(remote_tasks::save_remote_task_draft).delete(remote_tasks::delete_remote_task)) .route("/{org_id}/score_configs", get(score_configs::list_score_configs).post(score_configs::create_score_config).put(score_configs::ensure_score_config)) .route("/{org_id}/score_configs/{entity_id}/versions", get(score_configs::list_score_config_versions)) + .route("/{org_id}/score_configs/quality", get(quality::list_quality_summaries)) + .route("/{org_id}/score_configs/{entity_id}/quality", get(quality::list_quality_scores)) + .route("/{org_id}/score_configs/{entity_id}/quality/timeline", get(quality::get_quality_timeline)) .route("/{org_id}/score_configs/{entity_id}", get(score_configs::get_score_config).put(score_configs::update_score_config).delete(score_configs::delete_score_config)) // Scorers (Online Eval Phase 2) diff --git a/src/api/management/src/models/mod.rs b/src/api/management/src/models/mod.rs index 25f279af47..04d2d107c8 100644 --- a/src/api/management/src/models/mod.rs +++ b/src/api/management/src/models/mod.rs @@ -42,6 +42,8 @@ pub mod prompts; #[cfg(feature = "enterprise")] pub mod providers; #[cfg(feature = "enterprise")] +pub mod quality; +#[cfg(feature = "enterprise")] pub mod remote_tasks; pub mod reports; #[cfg(feature = "enterprise")] diff --git a/src/api/management/src/models/quality.rs b/src/api/management/src/models/quality.rs new file mode 100644 index 0000000000..3932fddd5f --- /dev/null +++ b/src/api/management/src/models/quality.rs @@ -0,0 +1,110 @@ +// Copyright 2026 OpenObserve Inc. +// +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// This program is distributed in the hope that it will be useful +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. +// +// You should have received a copy of the GNU Affero General Public License +// along with this program. If not, see . + +use config::meta::self_reporting::llm_scores::LlmScoreTargetScope; +use openobserve_core::llm_evaluations::quality::{ + DEFAULT_PAGE_SIZE, ListQualityScores, QualityFilter, +}; +use serde::Deserialize; +use utoipa::IntoParams; + +#[derive(Clone, Debug, Deserialize, IntoParams)] +#[serde(deny_unknown_fields)] +#[into_params(parameter_in = Query)] +pub struct ListQualityQuery { + /// Inclusive Score write-time lower bound, in microseconds. + pub start_time: i64, + /// Exclusive Score write-time upper bound, in microseconds. + pub end_time: i64, + /// `span`, `trace` or `session`. Omit for all scopes. + #[param(value_type = Option, example = "trace")] + pub scope: Option, + /// Only Scores of this agent ID. + pub agent_id: Option, + /// Only Scores of this agent name. + pub agent_name: Option, + /// Only Scores of this agent environment. + pub agent_env: Option, + /// Only Scores of this agent version. + pub agent_version: Option, +} + +impl From for QualityFilter { + fn from(value: ListQualityQuery) -> Self { + Self { + start_time: value.start_time, + end_time: value.end_time, + scope: value.scope, + agent_id: value.agent_id, + agent_name: value.agent_name, + agent_env: value.agent_env, + agent_version: value.agent_version, + } + } +} + +#[derive(Clone, Debug, Deserialize, IntoParams)] +#[serde(deny_unknown_fields)] +#[into_params(parameter_in = Query)] +pub struct ListQualityScoresQuery { + /// Inclusive Score write-time lower bound, in microseconds. + pub start_time: i64, + /// Exclusive Score write-time upper bound, in microseconds. + pub end_time: i64, + /// `span`, `trace` or `session`. Omit for all scopes. + #[param(value_type = Option, example = "trace")] + pub scope: Option, + pub agent_id: Option, + pub agent_name: Option, + pub agent_env: Option, + pub agent_version: Option, + /// Only Scores that fail the config's healthy threshold. + pub unhealthy_only: Option, + /// Numeric configs: first distribution bucket index to include. + pub bucket_from: Option, + /// Numeric configs: last distribution bucket index to include, `bucket_from` when omitted. + pub bucket_to: Option, + /// Boolean and categorical configs: exact value to include. + pub value: Option, + /// Zero-based result offset. Defaults to 0. + pub from: Option, + /// Page size from 1 through 100. Defaults to 20. + pub size: Option, +} + +impl From for ListQualityScores { + fn from(value: ListQualityScoresQuery) -> Self { + Self { + filter: QualityFilter { + start_time: value.start_time, + end_time: value.end_time, + scope: value.scope, + agent_id: value.agent_id, + agent_name: value.agent_name, + agent_env: value.agent_env, + agent_version: value.agent_version, + }, + unhealthy_only: value.unhealthy_only.unwrap_or(false), + buckets: match (value.bucket_from, value.bucket_to) { + (Some(from), to) => Some((from, to.unwrap_or(from))), + (None, Some(to)) => Some((to, to)), + (None, None) => None, + }, + value: value.value, + from: value.from.unwrap_or(0), + size: value.size.unwrap_or(DEFAULT_PAGE_SIZE), + } + } +} diff --git a/src/api/management/src/request/mod.rs b/src/api/management/src/request/mod.rs index 19ee1cd456..f50bb6e105 100644 --- a/src/api/management/src/request/mod.rs +++ b/src/api/management/src/request/mod.rs @@ -66,6 +66,8 @@ pub mod prompts; pub mod providers; pub mod public_dashboards; pub mod public_rate_limit; +#[cfg(feature = "enterprise")] +pub mod quality; pub mod query_history; #[cfg(feature = "enterprise")] pub mod remote_tasks; diff --git a/src/api/management/src/request/quality.rs b/src/api/management/src/request/quality.rs new file mode 100644 index 0000000000..2e8823431a --- /dev/null +++ b/src/api/management/src/request/quality.rs @@ -0,0 +1,270 @@ +// Copyright 2026 OpenObserve Inc. +// +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// This program is distributed in the hope that it will be useful +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. +// +// You should have received a copy of the GNU Affero General Public License +// along with this program. If not, see . + +use axum::{ + extract::{Path, Query}, + response::Response, +}; +use infra::table::score_configs::ScoreConfig; +use openobserve_api_common::extractors::Headers; +use openobserve_core::{ + auth::{UserEmail, check_permissions, is_ofga_object_visible}, + http::map_error_to_http_response, + llm_evaluations::quality::{ + self, QualityConfigList, QualityError, QualityScorePage, QualityTimeline, + }, +}; + +use crate::{ + common::meta::http::HttpResponse as MetaHttpResponse, + models::quality::{ListQualityQuery, ListQualityScoresQuery}, +}; + +/// Same visibility as ListScoreConfigs; the routes only check the org-level trace grant. +async fn visible_configs(org_id: &str, user_id: &str) -> Result bool, Response> { + let permitted_objects = openobserve_api_common::auth::validator::list_objects_for_user( + org_id, + user_id, + "GET", + "score_config", + ) + .await + .map_err(|error| MetaHttpResponse::forbidden(error.to_string()))?; + let org_id = org_id.to_string(); + Ok(move |entity_id: &str| config_visible(&org_id, entity_id, permitted_objects.as_deref())) +} + +fn config_visible(org_id: &str, entity_id: &str, permitted_objects: Option<&[String]>) -> bool { + is_ofga_object_visible(org_id, "score_config", entity_id, permitted_objects) +} + +/// The per-config grant GetScoreConfig gets; the route itself only checks the org trace grant. +async fn check_config_access(org_id: &str, entity_id: &str, user_id: &str) -> Result<(), Response> { + let allowed = check_permissions( + entity_id, + org_id, + user_id, + "score_configs", + "GET", + None, + false, + false, + true, + ) + .await; + require_visible(allowed, entity_id).map_err(quality_error_response) +} + +/// A hidden config answers like a missing one, so the 404 does not reveal that it exists. +fn require_visible(allowed: bool, entity_id: &str) -> Result<(), QualityError> { + if allowed { + Ok(()) + } else { + Err(QualityError::NotFound(entity_id.to_string())) + } +} + +fn quality_error_response(error: QualityError) -> Response { + match error { + QualityError::NotFound(_) => MetaHttpResponse::not_found(error), + error @ (QualityError::InvalidTimeRange + | QualityError::InvalidPageSize + | QualityError::UnsupportedScope + | QualityError::InvalidValueFilter(_)) => MetaHttpResponse::bad_request(error), + QualityError::Infra(error) => { + log::error!("[Quality] score-config database error: {error}"); + MetaHttpResponse::internal_error("Internal server error") + } + QualityError::Search(error) => { + log::error!("[Quality] {error}"); + match error.downcast_ref::() { + Some(error) => map_error_to_http_response(error, None), + None => MetaHttpResponse::internal_error("Quality query failed"), + } + } + error @ QualityError::MalformedSearchResponse(_) => { + log::error!("[Quality] {error}"); + MetaHttpResponse::internal_error("Quality query failed") + } + } +} + +/// ListScoreConfigQuality +#[utoipa::path( + get, + path = "/{org_id}/score_configs/quality", + context_path = "/api", + tag = "ScoreConfigs", + operation_id = "ListScoreConfigQuality", + summary = "Quality summary for every Score Config", + description = "Reads the latest Score per evaluation for each active Score Config the caller can see, in the time window, and returns its health status, totals, average, per-scope counts and most frequent value. The scope filter applies to everything except the per-scope counts.", + security(("Authorization" = [])), + params( + ("org_id" = String, Path, description = "Organization name"), + ListQualityQuery, + ), + responses( + (status = 200, body = inline(QualityConfigList)), + (status = 400, description = "Invalid request or search query", content_type = "application/json", body = MetaHttpResponse), + (status = 403, description = "Forbidden", content_type = "application/json", body = MetaHttpResponse), + (status = 500, description = "Quality query failed", content_type = "application/json", body = MetaHttpResponse), + ), + extensions(("x-o2-ratelimit" = json!({"module": "ScoreConfigs", "operation": "list"}))), +)] +pub async fn list_quality_summaries( + Path(org_id): Path, + Query(query): Query, + Headers(user_email): Headers, +) -> Response { + let is_visible = match visible_configs(&org_id, &user_email.user_id).await { + Ok(is_visible) => is_visible, + Err(response) => return response, + }; + match quality::list_summaries(&org_id, query.into(), |config: &ScoreConfig| { + is_visible(&config.entity_id) + }) + .await + { + Ok(list) => MetaHttpResponse::json(list), + Err(error) => quality_error_response(error), + } +} + +/// ListScoreConfigQualityScores +#[utoipa::path( + get, + path = "/{org_id}/score_configs/{entity_id}/quality", + context_path = "/api", + tag = "ScoreConfigs", + operation_id = "ListScoreConfigQualityScores", + summary = "Value distribution and latest Scores for one Score Config", + description = "Pages the latest Score per evaluation for the config, newest first. Filters by scope, agent, health, numeric distribution bucket or exact value. The total counts every Score that matches the filters. The value distribution follows only the scope and agent filters, so it stays the same while the page drills into one bucket or value.", + security(("Authorization" = [])), + params( + ("org_id" = String, Path, description = "Organization name"), + ("entity_id" = String, Path, description = "Score Config entity ID"), + ListQualityScoresQuery, + ), + responses( + (status = 200, body = inline(QualityScorePage)), + (status = 400, description = "Invalid request or search query", content_type = "application/json", body = MetaHttpResponse), + (status = 403, description = "Forbidden", content_type = "application/json", body = MetaHttpResponse), + (status = 404, description = "Score Config not found", content_type = "application/json", body = MetaHttpResponse), + (status = 500, description = "Quality query failed", content_type = "application/json", body = MetaHttpResponse), + ), + extensions(("x-o2-ratelimit" = json!({"module": "ScoreConfigs", "operation": "list"}))), +)] +pub async fn list_quality_scores( + Path((org_id, entity_id)): Path<(String, String)>, + Query(query): Query, + Headers(user_email): Headers, +) -> Response { + if let Err(response) = check_config_access(&org_id, &entity_id, &user_email.user_id).await { + return response; + } + match quality::list_scores(&org_id, &entity_id, query.into()).await { + Ok(page) => MetaHttpResponse::json(page), + Err(error) => quality_error_response(error), + } +} + +/// GetScoreConfigQualityTimeline +#[utoipa::path( + get, + path = "/{org_id}/score_configs/{entity_id}/quality/timeline", + context_path = "/api", + tag = "ScoreConfigs", + operation_id = "GetScoreConfigQualityTimeline", + summary = "Quality trend for one Score Config", + description = "Splits the time window into buckets and returns, per bucket, the count, unhealthy count, true count, average and p95 of the latest Score per evaluation for the config. Follows only the scope and agent filters, so paging or drilling into the Scores does not change it.", + security(("Authorization" = [])), + params( + ("org_id" = String, Path, description = "Organization name"), + ("entity_id" = String, Path, description = "Score Config entity ID"), + ListQualityQuery, + ), + responses( + (status = 200, body = inline(QualityTimeline)), + (status = 400, description = "Invalid request or search query", content_type = "application/json", body = MetaHttpResponse), + (status = 403, description = "Forbidden", content_type = "application/json", body = MetaHttpResponse), + (status = 404, description = "Score Config not found", content_type = "application/json", body = MetaHttpResponse), + (status = 500, description = "Quality query failed", content_type = "application/json", body = MetaHttpResponse), + ), + extensions(("x-o2-ratelimit" = json!({"module": "ScoreConfigs", "operation": "list"}))), +)] +pub async fn get_quality_timeline( + Path((org_id, entity_id)): Path<(String, String)>, + Query(query): Query, + Headers(user_email): Headers, +) -> Response { + if let Err(response) = check_config_access(&org_id, &entity_id, &user_email.user_id).await { + return response; + } + match quality::timeline(&org_id, &entity_id, query.into()).await { + Ok(timeline) => MetaHttpResponse::json(timeline), + Err(error) => quality_error_response(error), + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn maps_client_errors() { + let status = |error| quality_error_response(error).status().as_u16(); + assert_eq!(status(QualityError::NotFound("cfg".to_string())), 404); + assert_eq!(status(QualityError::InvalidTimeRange), 400); + assert_eq!( + status(QualityError::InvalidValueFilter("x".to_string())), + 400 + ); + assert_eq!( + status(QualityError::Search(anyhow::anyhow!("private details"))), + 500 + ); + } + + #[test] + fn hidden_config_answers_404_like_a_missing_one() { + assert!(require_visible(true, "cfg").is_ok()); + let error = require_visible(false, "cfg").unwrap_err(); + assert!(matches!(&error, QualityError::NotFound(id) if id == "cfg")); + assert_eq!(quality_error_response(error).status().as_u16(), 404); + } + + #[test] + fn list_shows_only_permitted_configs() { + let grants = |ids: &[&str]| ids.iter().map(|id| id.to_string()).collect::>(); + + assert!(config_visible("org-1", "cfg-1", None)); + assert!(config_visible( + "org-1", + "cfg-1", + Some(grants(&["score_config:cfg-1"]).as_slice()) + )); + assert!(config_visible( + "org-1", + "cfg-1", + Some(grants(&["score_config:_all_org-1"]).as_slice()) + )); + assert!(!config_visible( + "org-1", + "cfg-1", + Some(grants(&["score_config:cfg-2", "score_config:_all_org-2"]).as_slice()) + )); + assert!(!config_visible("org-1", "cfg-1", Some(&[]))); + } +} diff --git a/src/config/src/meta/self_reporting/evaluator.rs b/src/config/src/meta/self_reporting/evaluator.rs index b3b9c7bf72..6e44c1407c 100644 --- a/src/config/src/meta/self_reporting/evaluator.rs +++ b/src/config/src/meta/self_reporting/evaluator.rs @@ -65,6 +65,8 @@ pub const ATTR_TARGET_STREAM: &str = "target_stream"; pub const ATTR_TARGET_STREAM_TYPE: &str = "target_stream_type"; pub const ATTR_TARGET_AGENT_NAME: &str = "target_agent_name"; pub const ATTR_TARGET_AGENT_ID: &str = "target_agent_id"; +pub const ATTR_TARGET_AGENT_ENV: &str = "target_agent_env"; +pub const ATTR_TARGET_AGENT_VERSION: &str = "target_agent_version"; // Identity of the evaluator pub const ATTR_SCORER_ID: &str = "scorer_id"; @@ -164,6 +166,8 @@ mod tests { ATTR_TARGET_STREAM_TYPE, ATTR_TARGET_AGENT_NAME, ATTR_TARGET_AGENT_ID, + ATTR_TARGET_AGENT_ENV, + ATTR_TARGET_AGENT_VERSION, ATTR_SCORER_ID, ATTR_SCORER_VERSION, ATTR_SCORER_TYPE, diff --git a/src/config/src/meta/self_reporting/llm_scores.rs b/src/config/src/meta/self_reporting/llm_scores.rs index 057b5597ea..5865d938a9 100644 --- a/src/config/src/meta/self_reporting/llm_scores.rs +++ b/src/config/src/meta/self_reporting/llm_scores.rs @@ -352,6 +352,18 @@ pub struct LlmScoreRecord { /// Justification for this individual Score dimension. #[serde(skip_serializing_if = "Option::is_none")] pub reasoning: Option, + /// Bounded excerpt of the target input the scorer evaluated. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub input_preview: Option, + /// Bounded excerpt of the target output the scorer evaluated. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub output_preview: Option, + /// Span the scorer read the target input from, when it came from one span. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub input_span_id: Option, + /// Span the scorer read the target output from, when it came from one span. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub output_span_id: Option, /// Attempts this scorer dimension has consumed, 1-based. Resuming the /// bounded sequence after a restart reads this and nothing else. #[serde(skip_serializing_if = "Option::is_none")] @@ -436,6 +448,10 @@ impl Default for LlmScoreRecord { job_id: None, job_version: None, reasoning: None, + input_preview: None, + output_preview: None, + input_span_id: None, + output_span_id: None, attempt_count: None, error_reason: None, annotation_id: None, @@ -484,6 +500,10 @@ impl LlmScoreRecord { job_id: Some(String::new()), job_version: Some(0), reasoning: Some(String::new()), + input_preview: Some(String::new()), + output_preview: Some(String::new()), + input_span_id: Some(String::new()), + output_span_id: Some(String::new()), skip_reason: Some(ExperimentSkipReason::NoReference), attempt_count: Some(0), error_reason: Some(LlmScoreErrorReason::AttemptFailed), @@ -629,6 +649,10 @@ mod tests { job_id: Some("job-1".to_string()), job_version: Some(1), reasoning: None, + input_preview: None, + output_preview: None, + input_span_id: None, + output_span_id: None, attempt_count: None, error_reason: None, annotation_id: None, @@ -771,6 +795,8 @@ mod tests { assert!(!obj.contains_key("target_agent_id")); assert!(obj.contains_key("job_id")); assert!(obj.contains_key("reasoning")); + assert!(obj.contains_key("input_span_id")); + assert!(obj.contains_key("output_span_id")); assert!(obj.contains_key("review_submission_comments")); assert!(obj.contains_key("scorer_metadata")); assert!(obj.contains_key("author")); diff --git a/src/core/src/llm_evaluations/mod.rs b/src/core/src/llm_evaluations/mod.rs index 5fa320f1bf..616c34e8ec 100644 --- a/src/core/src/llm_evaluations/mod.rs +++ b/src/core/src/llm_evaluations/mod.rs @@ -18,6 +18,6 @@ pub mod evaluator_trace_exporter; pub use o2_enterprise::enterprise::llm_evaluations::{ annotation_queues, annotations, datasets, discovery, evaluator_trace, experiments, playground, - prepared_scorers, providers, remote_tasks, score_configs, score_policy, score_writer, scorers, - secrets, sync_scoring, + prepared_scorers, providers, quality, remote_tasks, score_configs, score_policy, score_writer, + scorers, secrets, sync_scoring, }; diff --git a/src/core/src/pipeline/batch_execution.rs b/src/core/src/pipeline/batch_execution.rs index 18919bc3da..1042a02445 100644 --- a/src/core/src/pipeline/batch_execution.rs +++ b/src/core/src/pipeline/batch_execution.rs @@ -1605,6 +1605,8 @@ async fn process_llm_evaluation_node( target_stream_type: ctx.source_stream_type.clone(), target_agent_name: ctx.agent_name.clone(), target_agent_id: ctx.agent_id.clone(), + target_agent_env: ctx.agent_env.clone(), + target_agent_version: ctx.agent_version.clone(), scorer_id: None, scorer_version: None, scorer_type: None, diff --git a/src/core/src/self_reporting/evaluator_schema.rs b/src/core/src/self_reporting/evaluator_schema.rs index 36db473ef8..20ff26f7d5 100644 --- a/src/core/src/self_reporting/evaluator_schema.rs +++ b/src/core/src/self_reporting/evaluator_schema.rs @@ -73,6 +73,8 @@ struct EvaluatorSpanReflection { attributes_target_stream_type: String, attributes_target_agent_name: String, attributes_target_agent_id: String, + attributes_target_agent_env: String, + attributes_target_agent_version: String, attributes_scorer_id: String, attributes_scorer_version: String, attributes_scorer_type: String, @@ -129,6 +131,8 @@ impl EvaluatorSpanReflection { attributes_target_stream_type: String::new(), attributes_target_agent_name: String::new(), attributes_target_agent_id: String::new(), + attributes_target_agent_env: String::new(), + attributes_target_agent_version: String::new(), attributes_scorer_id: String::new(), attributes_scorer_version: String::new(), attributes_scorer_type: String::new(), @@ -262,6 +266,8 @@ mod tests { assert!(obj.contains_key("attributes_target_stream_type")); assert!(obj.contains_key("attributes_target_agent_name")); assert!(obj.contains_key("attributes_target_agent_id")); + assert!(obj.contains_key("attributes_target_agent_env")); + assert!(obj.contains_key("attributes_target_agent_version")); assert!(obj.contains_key("attributes_scorer_id")); assert!(obj.contains_key("attributes_scorer_version")); assert!(obj.contains_key("attributes_scorer_type")); diff --git a/web/eslint.config.js b/web/eslint.config.js index d5a3685b9f..9ee91d5650 100644 --- a/web/eslint.config.js +++ b/web/eslint.config.js @@ -369,10 +369,6 @@ const COLOR_ALLOWLIST = [ "web/src/components/slos/SloTimeSlicePreview.vue", "web/src/components/traces/FlameGraphView.vue", "web/src/enterprise/components/billings/usage.vue", - "web/src/enterprise/components/onlineEvals/quality/QualityBooleanBarsChart.vue", - "web/src/enterprise/components/onlineEvals/quality/QualityBooleanTrendChart.vue", - "web/src/enterprise/components/onlineEvals/quality/QualityCategoryBarsChart.vue", - "web/src/enterprise/components/onlineEvals/quality/QualityDistributionChart.vue", "web/src/lib/core/EmptyState/OEmptyState.vue", "web/src/lib/forms/Color/OColor.vue", "web/src/lib/overlay/Tooltip/OTooltip.vue", diff --git a/web/src/enterprise/components/OnlineEvals.spec.ts b/web/src/enterprise/components/OnlineEvals.spec.ts new file mode 100644 index 0000000000..b77445bbce --- /dev/null +++ b/web/src/enterprise/components/OnlineEvals.spec.ts @@ -0,0 +1,365 @@ +// @vitest-environment jsdom +import { defineComponent, h } from "vue"; +import { enableAutoUnmount, flushPromises, mount } from "@vue/test-utils"; +import { afterEach, beforeEach, describe, expect, it, vi } from "vitest"; +import store from "@/test/unit/helpers/store"; +import { queryClient } from "@/composables/query/queryClient"; +import { onlineEvalKeys } from "@/services/online-evals.service.querykeys"; +import type { QualityScoresParams } from "@/services/online-evals.service"; + +// An absolute range in the URL, so the window only moves when the test says so. +const { START, END, DAY, mocks, route, replace, picker } = await vi.hoisted(async () => { + const { reactive } = await import("vue"); + const START = 1_791_100_000_000_000; + return { + START, + END: START + 3_600_000_000, + DAY: 86_400_000_000, + mocks: { + list: vi.fn(), + scores: vi.fn(), + timeline: vi.fn(), + listAgents: vi.fn(), + }, + route: reactive({ name: "aiEvaluations", query: {} as Record }), + replace: vi.fn(() => Promise.resolve()), + picker: { window: { startTime: START, endTime: START + 3_600_000_000 } }, + }; +}); + +vi.mock("@/services/online-evals.service", async (importOriginal) => { + const actual: any = await importOriginal(); + const empty = () => Promise.resolve([]); + return { + ...actual, + default: { + ...actual.default, + providers: { ...actual.default.providers, list: empty }, + scoreConfigs: { ...actual.default.scoreConfigs, list: empty }, + scorers: { ...actual.default.scorers, list: empty }, + jobs: { ...actual.default.jobs, list: empty }, + quality: { list: mocks.list, scores: mocks.scores, timeline: mocks.timeline }, + }, + }; +}); + +vi.mock("@/services/gen-ai-agent-mapping.service", async (importOriginal) => { + const actual: any = await importOriginal(); + return { ...actual, default: { ...actual.default, listAgents: mocks.listAgents } }; +}); + +vi.mock("vue-router", () => ({ + useRoute: () => route, + useRouter: () => ({ + push: vi.fn(() => Promise.resolve()), + replace, + back: vi.fn(), + resolve: () => ({ href: "/" }), + }), +})); + +import OnlineEvals from "./OnlineEvals.vue"; +import { useAiDateRange } from "@/enterprise/composables/useAiDateRange"; + +const PickerStub = defineComponent({ + name: "DateTimePickerDashboard", + inheritAttrs: false, + props: { modelValue: { type: Object, default: null } }, + emits: ["update:modelValue"], + setup(props, { expose }) { + expose({ getConsumableDateTime: () => picker.window }); + return () => + h("div", { + "data-test": "picker-stub", + "data-type": props.modelValue?.valueType, + "data-period": props.modelValue?.relativeTimePeriod ?? "", + }); + }, +}); + +// Lets the test page the score list the way the real table's pager does. +const ScoresTableStub = defineComponent({ + name: "OTable", + inheritAttrs: false, + emits: ["update:current-page"], + setup(_, { emit }) { + return () => + h("button", { "data-test": "next-page", onClick: () => emit("update:current-page", 2) }); + }, +}); + +const PaneStub = defineComponent({ + name: "QualityScorePane", + props: ["score"], + setup(props) { + return () => h("div", { "data-test": "pane-stub" }, props.score?.id ?? ""); + }, +}); + +const scoreRow = (id: string) => ({ + id, + timestamp: 1, + refTimestamp: 1, + sourceType: "llm_judge", + targetScope: "trace", + targetId: id, + spanId: null, + traceId: id, + sessionId: null, + sourceStream: "default", + sourceStreamType: "traces", + value: "low", + unhealthy: false, + reasoning: null, + evaluatorTraceId: null, + taskId: null, + inputPreview: null, + outputPreview: null, + inputSpanId: null, + outputSpanId: null, +}); + +const PAGE = { + average: null, + distribution: [ + { key: "low", count: 10, unhealthy: 0 }, + { key: "medium", count: 15, unhealthy: 15 }, + ], + total: 25, +}; + +const scoreCalls = () => mocks.scores.mock.calls.map((call) => call[2] as QualityScoresParams); + +function mountPage() { + return mount(OnlineEvals, { + props: { hideTabBar: true }, + global: { + plugins: [store], + stubs: { + DateTimePickerDashboard: PickerStub, + AgentScopeCascade: true, + AiLastRefreshed: true, + OTable: ScoresTableStub, + QualityDistributionChart: true, + QualityScorePane: PaneStub, + QualityConfigsTable: true, + ScorerFormPage: true, + JobFormPage: true, + ImportScoreConfig: true, + ImportScorer: true, + ScoreConfigList: true, + ScorerList: true, + EvalJobList: true, + ScorerTypeDialog: true, + ScoreConfigDialog: true, + ScoreConfigDetail: true, + ScorerDetail: true, + EvalJobDetail: true, + ScoreConfigLibrary: true, + ScorerLibrary: true, + ConfirmDialog: true, + ODrawer: true, + }, + }, + }); +} + +/** Opens the toxicity detail on page 2, so page 1 stays in the cache as an earlier view. */ +async function openOnSecondPage() { + const wrapper = mountPage(); + await flushPromises(); + await wrapper.find('[data-test="next-page"]').trigger("click"); + await flushPromises(); + expect(scoreCalls().map((params) => params.from)).toEqual([0, 10]); + // The trend takes no page, so paging leaves it alone. + expect(mocks.timeline).toHaveBeenCalledTimes(1); + for (const mock of Object.values(mocks)) mock.mockClear(); + return wrapper; +} + +const cachedPage = (from: number) => + queryClient + .getQueryCache() + .findAll({ queryKey: onlineEvalKeys.quality("default") }) + .filter((query) => query.queryKey[4] === "scores" && (query.queryKey[6] as any).from === from); + +// Mounted pages read the shared reactive route, so each test unmounts its own. +enableAutoUnmount(afterEach); + +function resetMocks() { + picker.window = { startTime: START, endTime: END }; + replace.mockClear(); + mocks.list.mockReset().mockResolvedValue([]); + mocks.listAgents.mockReset().mockResolvedValue({ agents: [] }); + mocks.timeline.mockReset().mockResolvedValue({ timeline: [] }); + mocks.scores.mockReset().mockImplementation((_org, _id, params) => + Promise.resolve({ + ...PAGE, + list: [scoreRow(`s${params.from}`), scoreRow(`s${params.from + 1}`)], + from: params.from, + size: params.size, + }), + ); +} + +const listWindow = () => { + const params = mocks.list.mock.calls.at(-1)?.[1]; + return params ? params.end_time - params.start_time : null; +}; + +describe("OnlineEvals Quality range", () => { + beforeEach(() => { + route.query = { tab: "quality" }; + resetMocks(); + }); + + it("opens Quality at Past 24 Hours, while the shared AI range keeps its 15-minute default", async () => { + const wrapper = mountPage(); + await flushPromises(); + const pickerStub = wrapper.find('[data-test="picker-stub"]'); + expect(pickerStub.attributes("data-type")).toBe("relative"); + expect(pickerStub.attributes("data-period")).toBe("24h"); + expect(listWindow()).toBe(DAY); + expect(useAiDateRange().state.value.relativeTimePeriod).toBe("15m"); + }); + + it("keeps a chosen range in the URL and leaves the LLM Insights and Sessions range alone", async () => { + const shared = JSON.stringify(useAiDateRange().state.value); + const wrapper = mountPage(); + await flushPromises(); + picker.window = { startTime: START, endTime: START + 7 * DAY }; + wrapper.findComponent(PickerStub).vm.$emit("update:modelValue", { + valueType: "relative", + relativeTimePeriod: "7d", + startTime: START, + endTime: START + 7 * DAY, + }); + await flushPromises(); + expect(listWindow()).toBe(7 * DAY); + expect(replace).toHaveBeenLastCalledWith({ + query: expect.objectContaining({ tab: "quality", period: "7d" }), + }); + expect(JSON.stringify(useAiDateRange().state.value)).toBe(shared); + expect(localStorage.getItem("aiObservability:dateRange")).toBeNull(); + }); + + it("reopens at the range in the URL after a reload", async () => { + route.query = { tab: "quality", period: "7d" }; + const relative = mountPage(); + await flushPromises(); + expect(relative.find('[data-test="picker-stub"]').attributes("data-period")).toBe("7d"); + expect(listWindow()).toBe(7 * DAY); + + route.query = { tab: "quality", from: String(START), to: String(END) }; + const absolute = mountPage(); + await flushPromises(); + expect(absolute.find('[data-test="picker-stub"]').attributes("data-type")).toBe("absolute"); + expect(mocks.list).toHaveBeenLastCalledWith( + "default", + expect.objectContaining({ start_time: START, end_time: END }), + ); + }); +}); + +describe("OnlineEvals Quality detail header", () => { + beforeEach(() => { + route.query = { tab: "quality", config: "tox", from: String(START), to: String(END) }; + resetMocks(); + }); + + it("uses one header, the detail header, with the range, Refresh and the agent filters kept", async () => { + const wrapper = mountPage(); + await flushPromises(); + const headers = wrapper.findAll("header"); + expect(headers).toHaveLength(1); + expect(headers[0].find('[data-test="quality-detail-back"]').exists()).toBe(true); + expect(headers[0].find('[data-test="quality-detail-edit-config"]').exists()).toBe(true); + expect(headers[0].find('[data-test="picker-stub"]').exists()).toBe(true); + expect(headers[0].find('[data-test="quality-refresh-btn"]').exists()).toBe(true); + expect(wrapper.find('[data-test="quality-scope-bar"] agent-scope-cascade-stub').exists()).toBe( + true, + ); + }); +}); + +describe("OnlineEvals Quality round trip through a link", () => { + beforeEach(() => { + route.query = { tab: "quality", config: "tox", from: String(START), to: String(END) }; + resetMocks(); + }); + + it("writes the page and score with replace, then reopens them from the cache after Back", async () => { + const wrapper = await openOnSecondPage(); + expect(replace).toHaveBeenLastCalledWith({ + name: "aiEvaluations", + query: expect.objectContaining({ config: "tox", page: "2", score: "s10" }), + }); + wrapper.unmount(); + + // Back from the trace: the page remounts from the URL it left with. + route.query = { ...route.query, page: "2", score: "s11" }; + const restored = mountPage(); + await flushPromises(); + expect(restored.find('[data-test="pane-stub"]').text()).toBe("s11"); + expect(mocks.list).not.toHaveBeenCalled(); + expect(mocks.scores).not.toHaveBeenCalled(); + expect(mocks.timeline).not.toHaveBeenCalled(); + expect(mocks.listAgents).not.toHaveBeenCalled(); + }); +}); + +describe("OnlineEvals Quality Refresh", () => { + beforeEach(() => { + route.query = { tab: "quality", config: "tox", from: String(START), to: String(END) }; + resetMocks(); + }); + + it("fetches each read once, keeps the scores page, and marks earlier views stale", async () => { + const wrapper = await openOnSecondPage(); + await wrapper.find('[data-test="quality-refresh-btn"]').trigger("click"); + await flushPromises(); + + expect(mocks.list).toHaveBeenCalledTimes(1); + expect(mocks.listAgents).toHaveBeenCalledTimes(1); + expect(mocks.timeline).toHaveBeenCalledTimes(1); + expect(scoreCalls()).toEqual([expect.objectContaining({ from: 10, start_time: START })]); + expect(cachedPage(0).every((query) => query.state.isInvalidated)).toBe(true); + expect(cachedPage(10).every((query) => !query.state.isInvalidated)).toBe(true); + }); + + it("forces the score pane's cached spans too, like the other reads on the view", async () => { + const wrapper = await openOnSecondPage(); + // Loaded before the Refresh, so the "not fetched by this Refresh" rule applies. + const updatedAt = Date.now() - 1000; + const spansKey = onlineEvalKeys.qualitySpans("default", "default", ["span-1"]); + queryClient.setQueryData(spansKey, [], { updatedAt }); + const otherOrg = onlineEvalKeys.qualitySpans("other", "default", ["span-1"]); + queryClient.setQueryData(otherOrg, [], { updatedAt }); + await wrapper.find('[data-test="quality-refresh-btn"]').trigger("click"); + await flushPromises(); + expect(queryClient.getQueryState(spansKey)?.isInvalidated).toBe(true); + expect(queryClient.getQueryState(otherOrg)?.isInvalidated).toBe(false); + }); + + it("fetches each read once when the Refresh moves the window into a new minute", async () => { + const wrapper = await openOnSecondPage(); + picker.window = { startTime: START + 120_000_000, endTime: END + 120_000_000 }; + await wrapper.find('[data-test="quality-refresh-btn"]').trigger("click"); + await flushPromises(); + + expect(mocks.list).toHaveBeenCalledTimes(1); + expect(mocks.list).toHaveBeenCalledWith( + "default", + expect.objectContaining({ start_time: START + 120_000_000 }), + ); + expect(scoreCalls()).toEqual([ + expect.objectContaining({ from: 10, start_time: START + 120_000_000 }), + ]); + // The reads of the old window stay in the cache, marked stale. + const oldWindow = cachedPage(10).filter( + (query) => (query.queryKey[6] as any).start_time < START + 60_000_000, + ); + expect(oldWindow.length).toBe(1); + expect(oldWindow[0].state.isInvalidated).toBe(true); + }); +}); diff --git a/web/src/enterprise/components/OnlineEvals.vue b/web/src/enterprise/components/OnlineEvals.vue index f5930c548e..70858229e6 100644 --- a/web/src/enterprise/components/OnlineEvals.vue +++ b/web/src/enterprise/components/OnlineEvals.vue @@ -66,7 +66,7 @@ the Free Software Foundation, either version 3 of the License, or -