diff --git a/.vscode/launch.json b/.vscode/launch.json index 4fe372a2b9..ee903b4fe1 100644 --- a/.vscode/launch.json +++ b/.vscode/launch.json @@ -5,9 +5,10 @@ "version": "0.2.0", "configurations": [ { - "type": "lldb", + "type": "codelldb", "request": "launch", "name": "Debug executable 'openobserve' (server)", + "program": "${env:HOME}/.cargo/shared-target/debug/openobserve", "cargo": { "args": ["build", "--bin=openobserve", "--package=openobserve"], "filter": { @@ -17,15 +18,16 @@ }, "env": { "ZO_ROOT_USER_EMAIL": "root@example.com", - "ZO_ROOT_USER_PASSWORD": "Complexpass#123", + "ZO_ROOT_USER_PASSWORD": "Complexpass#123" }, "args": [], "cwd": "${workspaceFolder}" }, { - "type": "lldb", + "type": "codelldb", "request": "launch", "name": "Debug unit tests in library 'openobserve'", + "program": "${cargo:program}", "cargo": { "args": ["test", "--no-run", "--lib", "--package=openobserve"], "filter": { @@ -37,9 +39,10 @@ "cwd": "${workspaceFolder}" }, { - "type": "lldb", + "type": "codelldb", "request": "launch", "name": "Debug executable 'openobserve' (bin)", + "program": "${env:HOME}/.cargo/shared-target/debug/openobserve", "cargo": { "args": ["build", "--bin=openobserve", "--package=openobserve"], "filter": { @@ -51,9 +54,10 @@ "cwd": "${workspaceFolder}" }, { - "type": "lldb", + "type": "codelldb", "request": "launch", "name": "Debug unit tests in executable 'openobserve'", + "program": "${cargo:program}", "cargo": { "args": [ "test", @@ -68,6 +72,38 @@ }, "args": [], "cwd": "${workspaceFolder}" + }, + { + "type": "codelldb", + "request": "launch", + "name": "Debug executable 'openobserve' (enterprise)", + "program": "${env:HOME}/.cargo/shared-target/debug/openobserve", + "cargo": { + "args": ["build", "--bin=openobserve", "--package=openobserve"], + "filter": { + "name": "openobserve", + "kind": "bin" + } + }, + "envFile": "${workspaceFolder}/.env", + "args": [], + "cwd": "${workspaceFolder}" + }, + { + "type": "codelldb", + "request": "launch", + "name": "Debug executable 'openobserve' (cloud)", + "program": "${env:HOME}/.cargo/shared-target/debug/openobserve", + "cargo": { + "args": ["build", "--bin=openobserve", "--package=openobserve"], + "filter": { + "name": "openobserve", + "kind": "bin" + } + }, + "envFile": "${workspaceFolder}/.env.cloud", + "args": [], + "cwd": "${workspaceFolder}" } ] } diff --git a/src/common/utils/auth.rs b/src/common/utils/auth.rs index 505276de59..6fa47c9a0a 100644 --- a/src/common/utils/auth.rs +++ b/src/common/utils/auth.rs @@ -109,6 +109,7 @@ pub fn is_ofga_unsupported(name: &str) -> bool { RE_OFGA_UNSUPPORTED_NAME.is_match(name) } +#[cfg(feature = "enterprise")] pub(crate) fn is_ofga_object_visible( org_id: &str, object_type: &str, diff --git a/src/config/src/config.rs b/src/config/src/config.rs index 1dd3ab2853..33c2d69815 100644 --- a/src/config/src/config.rs +++ b/src/config/src/config.rs @@ -1364,12 +1364,6 @@ pub struct Common { help = "Enable user-defined model pricing. When true, uses DB pricing definitions and syncs from GitHub. When false, falls back to hardcoded built-in pricing only." )] pub model_pricing_enabled: bool, - #[env_config( - name = "ZO_ONLINE_EVALS_ENABLED", - default = true, - help = "Show the Online Evaluations UI (top-level Evaluations route) and the LLM Providers Settings page. When false, both are hidden. The backend endpoints remain reachable regardless — this flag only gates the frontend surface." - )] - pub online_evals_enabled: bool, #[env_config( name = "ZO_MODEL_PRICING_SOURCE_URL", default = "https://raw.githubusercontent.com/openobserve/sdr_patterns/refs/heads/main/llm_pricing.json", diff --git a/src/handler/http/models/mod.rs b/src/handler/http/models/mod.rs index a5b5f6ddde..73cf0d34a3 100644 --- a/src/handler/http/models/mod.rs +++ b/src/handler/http/models/mod.rs @@ -24,10 +24,14 @@ pub mod alerts; pub mod billings; pub mod dashboards; pub mod destinations; +#[cfg(feature = "enterprise")] pub mod eval_jobs; pub mod folders; pub mod pipelines; +#[cfg(feature = "enterprise")] pub mod providers; pub mod reports; +#[cfg(feature = "enterprise")] pub mod score_configs; +#[cfg(feature = "enterprise")] pub mod scorers; diff --git a/src/handler/http/request/eval_jobs/mod.rs b/src/handler/http/request/eval_jobs/mod.rs index cc9d38418b..ac6f19b306 100644 --- a/src/handler/http/request/eval_jobs/mod.rs +++ b/src/handler/http/request/eval_jobs/mod.rs @@ -34,9 +34,15 @@ use crate::{ impl From for Response { fn from(value: EvalJobError) -> Self { match value { - EvalJobError::InfraError(err) => MetaHttpResponse::internal_error(err), + EvalJobError::InfraError(err) => { + log::error!("[EvalJob] internal error: {err}"); + MetaHttpResponse::internal_error("Internal server error") + } EvalJobError::NotFound => MetaHttpResponse::not_found(value), - EvalJobError::ReconcilerError(_) => MetaHttpResponse::internal_error(value), + EvalJobError::ReconcilerError(err) => { + log::error!("[EvalJob] reconciler error: {err}"); + MetaHttpResponse::internal_error("Internal server error") + } EvalJobError::InvalidStatus(_) | EvalJobError::InvalidStatusTransition { .. } => { MetaHttpResponse::bad_request(value) } diff --git a/src/handler/http/request/mod.rs b/src/handler/http/request/mod.rs index cae6f47807..2f2c756e9e 100644 --- a/src/handler/http/request/mod.rs +++ b/src/handler/http/request/mod.rs @@ -32,6 +32,7 @@ pub mod dashboards; #[cfg(feature = "enterprise")] pub mod domain_management; pub mod enrichment_table; +#[cfg(feature = "enterprise")] pub mod eval_jobs; #[allow(deprecated)] pub mod folders; @@ -51,12 +52,15 @@ pub mod pipelines; #[cfg(feature = "profiling")] pub mod profiling; pub mod promql; +#[cfg(feature = "enterprise")] pub mod providers; pub mod ratelimit; #[cfg(feature = "enterprise")] pub mod re_pattern; pub mod rum; +#[cfg(feature = "enterprise")] pub mod score_configs; +#[cfg(feature = "enterprise")] pub mod scorers; pub mod search; pub mod service_accounts; diff --git a/src/handler/http/request/providers/mod.rs b/src/handler/http/request/providers/mod.rs index 57ba5c5bd8..50ee44b33b 100644 --- a/src/handler/http/request/providers/mod.rs +++ b/src/handler/http/request/providers/mod.rs @@ -30,7 +30,10 @@ use crate::{ impl From for Response { fn from(value: ProviderError) -> Self { match value { - ProviderError::InfraError(err) => MetaHttpResponse::internal_error(err), + ProviderError::InfraError(err) => { + log::error!("[Provider] internal error: {err}"); + MetaHttpResponse::internal_error("Internal server error") + } ProviderError::MissingName => { MetaHttpResponse::bad_request("Provider name cannot be empty") } diff --git a/src/handler/http/request/score_configs/mod.rs b/src/handler/http/request/score_configs/mod.rs index ecd55c6447..91c2a13a40 100644 --- a/src/handler/http/request/score_configs/mod.rs +++ b/src/handler/http/request/score_configs/mod.rs @@ -20,7 +20,10 @@ use crate::common::utils::auth::UserEmail; #[cfg(feature = "enterprise")] use crate::handler::http::extractors::Headers; use crate::{ - common::meta::http::HttpResponse as MetaHttpResponse, + common::{ + meta::{authz::Authz, http::HttpResponse as MetaHttpResponse}, + utils::auth::{is_ofga_object_visible, remove_ownership, set_ownership}, + }, handler::http::models::score_configs::{ ListScoreConfigVersionsResponseBody, ListScoreConfigsResponseBody, ScoreConfigRequestBody, ScoreConfigResponseBody, ScoreConfigUpdateRequestBody, @@ -28,19 +31,20 @@ use crate::{ service::llm_evaluations::score_configs::{self, ScoreConfigError}, }; -impl From for Response { - fn from(value: ScoreConfigError) -> Self { - match value { - ScoreConfigError::InfraError(err) => MetaHttpResponse::internal_error(err), - ScoreConfigError::MissingName => { - MetaHttpResponse::bad_request("Score config name cannot be empty") - } - ScoreConfigError::NotFound => MetaHttpResponse::not_found("Score config not found"), - ScoreConfigError::DuplicateName => { - MetaHttpResponse::conflict("Score config name already exists") - } - ScoreConfigError::InUseByScorer => MetaHttpResponse::conflict(value), +fn score_config_error_response(value: ScoreConfigError) -> Response { + match value { + ScoreConfigError::InfraError(err) => { + log::error!("[ScoreConfig] internal error: {err}"); + MetaHttpResponse::internal_error("Internal server error") } + ScoreConfigError::MissingName => { + MetaHttpResponse::bad_request("Score config name cannot be empty") + } + ScoreConfigError::NotFound => MetaHttpResponse::not_found("Score config not found"), + ScoreConfigError::DuplicateName => { + MetaHttpResponse::conflict("Score config name already exists") + } + ScoreConfigError::InUseByScorer => MetaHttpResponse::conflict(value), } } @@ -80,15 +84,23 @@ pub async fn list_score_configs( Err(e) => return MetaHttpResponse::forbidden(e.to_string()), } }; - #[cfg(not(feature = "enterprise"))] - let permitted_objects = None; - - match score_configs::list_score_configs(&org_id, permitted_objects).await { + match score_configs::list_score_configs(&org_id).await { Ok(list) => { + let list: Vec = list + .into_iter() + .filter(|config| { + is_ofga_object_visible( + &org_id, + "score_config", + &config.entity_id, + permitted_objects.as_deref(), + ) + }) + .collect(); let body: ListScoreConfigsResponseBody = list.into(); MetaHttpResponse::json(body) } - Err(err) => err.into(), + Err(err) => score_config_error_response(err), } } @@ -120,10 +132,11 @@ pub async fn create_score_config( let config: infra::table::score_configs::ScoreConfig = body.into(); match score_configs::save_score_config(&org_id, config).await { Ok(c) => { + set_ownership(&org_id, "score_configs", Authz::new(&c.entity_id)).await; let resp: ScoreConfigResponseBody = c.into(); MetaHttpResponse::json(resp) } - Err(err) => err.into(), + Err(err) => score_config_error_response(err), } } @@ -155,7 +168,7 @@ pub async fn get_score_config(Path((org_id, entity_id)): Path<(String, String)>) let resp: ScoreConfigResponseBody = c.into(); MetaHttpResponse::json(resp) } - Err(err) => err.into(), + Err(err) => score_config_error_response(err), } } @@ -188,7 +201,7 @@ pub async fn list_score_config_versions( let body: ListScoreConfigVersionsResponseBody = versions.into(); MetaHttpResponse::json(body) } - Err(err) => err.into(), + Err(err) => score_config_error_response(err), } } @@ -225,7 +238,7 @@ pub async fn update_score_config( let resp: ScoreConfigResponseBody = c.into(); MetaHttpResponse::json(resp) } - Err(err) => err.into(), + Err(err) => score_config_error_response(err), } } @@ -254,8 +267,11 @@ pub async fn update_score_config( )] pub async fn delete_score_config(Path((org_id, entity_id)): Path<(String, String)>) -> Response { match score_configs::delete_score_config(&org_id, &entity_id).await { - Ok(()) => MetaHttpResponse::ok("Score config deactivated"), - Err(err) => err.into(), + Ok(()) => { + remove_ownership(&org_id, "score_configs", Authz::new(&entity_id)).await; + MetaHttpResponse::ok("Score config deactivated") + } + Err(err) => score_config_error_response(err), } } @@ -272,7 +288,7 @@ mod tests { (ScoreConfigError::InUseByScorer, 409), ]; for (err, expected) in cases { - let resp: Response = err.into(); + let resp = score_config_error_response(err); assert_eq!(resp.status().as_u16(), expected); } } @@ -280,7 +296,7 @@ mod tests { #[test] fn test_score_config_error_infra_is_500() { let err = ScoreConfigError::InfraError(infra::errors::Error::Message("db".to_string())); - let resp: Response = err.into(); + let resp = score_config_error_response(err); assert_eq!(resp.status().as_u16(), 500); } } diff --git a/src/handler/http/request/scorers/mod.rs b/src/handler/http/request/scorers/mod.rs index f4fb74a0d8..7a3c3dbd03 100644 --- a/src/handler/http/request/scorers/mod.rs +++ b/src/handler/http/request/scorers/mod.rs @@ -23,7 +23,10 @@ use crate::common::utils::auth::UserEmail; #[cfg(feature = "enterprise")] use crate::handler::http::extractors::Headers; use crate::{ - common::meta::http::HttpResponse as MetaHttpResponse, + common::{ + meta::{authz::Authz, http::HttpResponse as MetaHttpResponse}, + utils::auth::{is_ofga_object_visible, remove_ownership, set_ownership}, + }, handler::http::models::scorers::{ ListScorerVersionsResponseBody, ListScorersQuery, ListScorersResponseBody, LlmJudgeOutputSchemaRequestBody, LlmJudgeOutputSchemaResponseBody, ScorerRequestBody, @@ -36,22 +39,21 @@ use crate::{ }, }; -impl From for Response { - fn from(value: ScorerError) -> Self { - match value { - ScorerError::InfraError(err) => MetaHttpResponse::internal_error(err), - ScorerError::MissingName => { - MetaHttpResponse::bad_request("Scorer name cannot be empty") - } - ScorerError::NotFound => MetaHttpResponse::not_found("Scorer not found"), - ScorerError::InvalidScorerType(_) - | ScorerError::ScorerTypeImmutable - | ScorerError::ProducesScoreConfigIdImmutable - | ScorerError::ScoreConfigVersionNotFound - | ScorerError::InvalidOutputSchema(_) => MetaHttpResponse::bad_request(value), - ScorerError::DuplicateName => MetaHttpResponse::conflict(value), - ScorerError::InUseByEvalJob => MetaHttpResponse::conflict(value), +fn scorer_error_response(value: ScorerError) -> Response { + match value { + ScorerError::InfraError(err) => { + log::error!("[Scorer] internal error: {err}"); + MetaHttpResponse::internal_error("Internal server error") } + ScorerError::MissingName => MetaHttpResponse::bad_request("Scorer name cannot be empty"), + ScorerError::NotFound => MetaHttpResponse::not_found("Scorer not found"), + ScorerError::InvalidScorerType(_) + | ScorerError::ScorerTypeImmutable + | ScorerError::ProducesScoreConfigIdImmutable + | ScorerError::ScoreConfigVersionNotFound + | ScorerError::InvalidOutputSchema(_) => MetaHttpResponse::bad_request(value), + ScorerError::DuplicateName => MetaHttpResponse::conflict(value), + ScorerError::InUseByEvalJob => MetaHttpResponse::conflict(value), } } @@ -95,15 +97,23 @@ pub async fn list_scorers( Err(e) => return MetaHttpResponse::forbidden(e.to_string()), } }; - #[cfg(not(feature = "enterprise"))] - let permitted_objects = None; - - match scorers::list_scorers(&org_id, query.scorer_type.as_ref(), permitted_objects).await { + match scorers::list_scorers(&org_id, query.scorer_type.as_ref()).await { Ok(list) => { + let list: Vec = list + .into_iter() + .filter(|scorer| { + is_ofga_object_visible( + &org_id, + "scorer", + &scorer.entity_id, + permitted_objects.as_deref(), + ) + }) + .collect(); let body: ListScorersResponseBody = list.into(); MetaHttpResponse::json(body) } - Err(err) => err.into(), + Err(err) => scorer_error_response(err), } } @@ -135,10 +145,11 @@ pub async fn create_scorer( let scorer: infra::table::scorers::Scorer = body.into(); match scorers::save_scorer(&org_id, scorer).await { Ok(s) => { + set_ownership(&org_id, "scorers", Authz::new(&s.entity_id)).await; let resp: ScorerResponseBody = s.into(); MetaHttpResponse::json(resp) } - Err(err) => err.into(), + Err(err) => scorer_error_response(err), } } @@ -170,7 +181,7 @@ pub async fn get_scorer(Path((org_id, entity_id)): Path<(String, String)>) -> Re let resp: ScorerResponseBody = s.into(); MetaHttpResponse::json(resp) } - Err(err) => err.into(), + Err(err) => scorer_error_response(err), } } @@ -201,7 +212,7 @@ pub async fn list_scorer_versions(Path((org_id, entity_id)): Path<(String, Strin let body: ListScorerVersionsResponseBody = versions.into(); MetaHttpResponse::json(body) } - Err(err) => err.into(), + Err(err) => scorer_error_response(err), } } @@ -238,7 +249,7 @@ pub async fn update_scorer( let resp: ScorerResponseBody = s.into(); MetaHttpResponse::json(resp) } - Err(err) => err.into(), + Err(err) => scorer_error_response(err), } } @@ -267,8 +278,11 @@ pub async fn update_scorer( )] pub async fn delete_scorer(Path((org_id, entity_id)): Path<(String, String)>) -> Response { match scorers::delete_scorer(&org_id, &entity_id).await { - Ok(()) => MetaHttpResponse::ok("Scorer deactivated"), - Err(err) => err.into(), + Ok(()) => { + remove_ownership(&org_id, "scorers", Authz::new(&entity_id)).await; + MetaHttpResponse::ok("Scorer deactivated") + } + Err(err) => scorer_error_response(err), } } @@ -299,7 +313,7 @@ pub async fn test_scorer( ) -> Response { match run_scorer_test(&org_id, &body).await { Ok(resp) => MetaHttpResponse::json(resp), - Err(err) => err.into(), + Err(err) => scorer_error_response(err), } } @@ -331,7 +345,7 @@ pub async fn preview_llm_judge_output_schema( Ok(output_schema) => { MetaHttpResponse::json(LlmJudgeOutputSchemaResponseBody { output_schema }) } - Err(err) => err.into(), + Err(err) => scorer_error_response(err), } } @@ -548,7 +562,7 @@ mod tests { (ScorerError::InUseByEvalJob, 409), ]; for (err, expected) in cases { - let resp: Response = err.into(); + let resp = scorer_error_response(err); assert_eq!(resp.status().as_u16(), expected); } } @@ -556,7 +570,7 @@ mod tests { #[test] fn test_scorer_error_infra_is_500() { let err = ScorerError::InfraError(infra::errors::Error::Message("db".to_string())); - let resp: Response = err.into(); + let resp = scorer_error_response(err); assert_eq!(resp.status().as_u16(), 500); } diff --git a/src/handler/http/request/status/mod.rs b/src/handler/http/request/status/mod.rs index e62beb58e4..deae7cf062 100644 --- a/src/handler/http/request/status/mod.rs +++ b/src/handler/http/request/status/mod.rs @@ -366,6 +366,7 @@ pub async fn zo_config() -> impl IntoResponse { let service_streams_enabled = enterprise_value!(false, o2cfg.service_streams.enabled); // Anomaly detection is always on when the enterprise feature is compiled in — no runtime flag. let anomaly_detection_enabled = enterprise_value!(false, true); + let online_evals_enabled = enterprise_value!(false, o2cfg.common.online_evals_enabled); #[cfg(all(feature = "cloud", not(feature = "enterprise")))] let build_type = "cloud"; @@ -473,7 +474,7 @@ pub async fn zo_config() -> impl IntoResponse { incidents_enabled, service_streams_enabled, model_pricing_enabled: cfg.common.model_pricing_enabled, - online_evals_enabled: cfg.common.online_evals_enabled, + online_evals_enabled, anomaly_detection_enabled, enable_cross_linking: cfg.common.enable_cross_linking, show_fts_field_values: cfg.common.show_fts_field_values, diff --git a/src/handler/http/router/mod.rs b/src/handler/http/router/mod.rs index abecc35be4..7161b28468 100644 --- a/src/handler/http/router/mod.rs +++ b/src/handler/http/router/mod.rs @@ -850,35 +850,41 @@ pub fn service_routes() -> Router { // sourcemaps .route("/{org_id}/sourcemaps",get(sourcemaps::list).post(sourcemaps::upload_maps).delete(sourcemaps::delete)) .route("/{org_id}/sourcemaps/values",get(sourcemaps::list_values)) - .route("/{org_id}/sourcemaps/stacktrace",post(sourcemaps::translate_stacktrace)) + .route("/{org_id}/sourcemaps/stacktrace",post(sourcemaps::translate_stacktrace)); - // LLM Providers (Online Eval Phase 2) - .route("/{org_id}/providers", get(providers::list_providers).post(providers::create_provider)) - .route("/{org_id}/providers/{provider_id}", get(providers::get_provider).put(providers::update_provider).delete(providers::delete_provider)) - .route("/{org_id}/providers/{provider_id}/test", post(providers::test_provider)) + #[cfg(feature = "enterprise")] + { + if get_o2_config().common.online_evals_enabled { + router = router + // LLM Providers (Online Eval Phase 2) + .route("/{org_id}/providers", get(providers::list_providers).post(providers::create_provider)) + .route("/{org_id}/providers/{provider_id}", get(providers::get_provider).put(providers::update_provider).delete(providers::delete_provider)) + .route("/{org_id}/providers/{provider_id}/test", post(providers::test_provider)) - // Score Configs (Online Eval Phase 2) - // NOTE: /{entity_id}/versions must precede /{entity_id} for routing correctness - .route("/{org_id}/score_configs", get(score_configs::list_score_configs).post(score_configs::create_score_config)) - .route("/{org_id}/score_configs/{entity_id}/versions", get(score_configs::list_score_config_versions)) - .route("/{org_id}/score_configs/{entity_id}", get(score_configs::get_score_config).put(score_configs::update_score_config).delete(score_configs::delete_score_config)) + // Score Configs (Online Eval Phase 2) + // NOTE: /{entity_id}/versions must precede /{entity_id} for routing correctness + .route("/{org_id}/score_configs", get(score_configs::list_score_configs).post(score_configs::create_score_config)) + .route("/{org_id}/score_configs/{entity_id}/versions", get(score_configs::list_score_config_versions)) + .route("/{org_id}/score_configs/{entity_id}", get(score_configs::get_score_config).put(score_configs::update_score_config).delete(score_configs::delete_score_config)) - // Scorers (Online Eval Phase 2) - // NOTE: static and nested routes must precede /{entity_id} - .route("/{org_id}/scorers", get(scorers::list_scorers).post(scorers::create_scorer)) - .route("/{org_id}/scorers/test", post(scorers::test_scorer)) - .route("/{org_id}/scorers/llm_judge/output_schema", post(scorers::preview_llm_judge_output_schema)) - .route("/{org_id}/scorers/{entity_id}/versions", get(scorers::list_scorer_versions)) - .route("/{org_id}/scorers/{entity_id}", get(scorers::get_scorer).put(scorers::update_scorer).delete(scorers::delete_scorer)) + // Scorers (Online Eval Phase 2) + // NOTE: static and nested routes must precede /{entity_id} + .route("/{org_id}/scorers", get(scorers::list_scorers).post(scorers::create_scorer)) + .route("/{org_id}/scorers/test", post(scorers::test_scorer)) + .route("/{org_id}/scorers/llm_judge/output_schema", post(scorers::preview_llm_judge_output_schema)) + .route("/{org_id}/scorers/{entity_id}/versions", get(scorers::list_scorer_versions)) + .route("/{org_id}/scorers/{entity_id}", get(scorers::get_scorer).put(scorers::update_scorer).delete(scorers::delete_scorer)) - // Online Eval Jobs (Online Eval Phase 2) - // NOTE: /activate, /pause, /resume, /archive must precede /{job_id} - .route("/{org_id}/eval_jobs", get(eval_jobs::list_eval_jobs).post(eval_jobs::create_eval_job)) - .route("/{org_id}/eval_jobs/{job_id}/activate", post(eval_jobs::activate_eval_job)) - .route("/{org_id}/eval_jobs/{job_id}/pause", post(eval_jobs::pause_eval_job)) - .route("/{org_id}/eval_jobs/{job_id}/resume", post(eval_jobs::resume_eval_job)) - .route("/{org_id}/eval_jobs/{job_id}/archive", post(eval_jobs::archive_eval_job)) - .route("/{org_id}/eval_jobs/{job_id}", get(eval_jobs::get_eval_job).put(eval_jobs::update_eval_job).delete(eval_jobs::delete_eval_job)); + // Online Eval Jobs (Online Eval Phase 2) + // NOTE: /activate, /pause, /resume, /archive must precede /{job_id} + .route("/{org_id}/eval_jobs", get(eval_jobs::list_eval_jobs).post(eval_jobs::create_eval_job)) + .route("/{org_id}/eval_jobs/{job_id}/activate", post(eval_jobs::activate_eval_job)) + .route("/{org_id}/eval_jobs/{job_id}/pause", post(eval_jobs::pause_eval_job)) + .route("/{org_id}/eval_jobs/{job_id}/resume", post(eval_jobs::resume_eval_job)) + .route("/{org_id}/eval_jobs/{job_id}/archive", post(eval_jobs::archive_eval_job)) + .route("/{org_id}/eval_jobs/{job_id}", get(eval_jobs::get_eval_job).put(eval_jobs::update_eval_job).delete(eval_jobs::delete_eval_job)); + } + } #[cfg(feature = "enterprise")] { diff --git a/src/service/db/pipeline.rs b/src/service/db/pipeline.rs index 366097029d..91bc3c96a7 100644 --- a/src/service/db/pipeline.rs +++ b/src/service/db/pipeline.rs @@ -40,7 +40,7 @@ use crate::{ pub enum PipelineError { // internal #[error("InfraError# {0}")] - InfraError(#[from] infra::errors::Error), + InfraError(infra::errors::Error), // not found #[error("Pipeline with ID {0} not found.")] NotFound(String), @@ -60,6 +60,17 @@ pub enum PipelineError { DeleteDerivedStream(String), } +impl From for PipelineError { + fn from(value: infra::errors::Error) -> Self { + match value { + infra::errors::Error::DbError(infra::errors::DbError::KeyNotExists(key)) => { + PipelineError::NotFound(key) + } + err => PipelineError::InfraError(err), + } + } +} + /// Stores a new pipeline to database. /// /// Pipeline validation should be handled by the caller. @@ -631,4 +642,13 @@ mod tests { // Clean up remove_scheduled_pipeline_from_cache(scheduled_id).await; } + + #[test] + fn test_key_not_exists_maps_to_not_found() { + let err = PipelineError::from(infra::errors::Error::DbError( + infra::errors::DbError::KeyNotExists("0".to_string()), + )); + + assert!(matches!(err, PipelineError::NotFound(id) if id == "0")); + } } diff --git a/src/service/llm_evaluations/eval_jobs/executor_runtime.rs b/src/service/llm_evaluations/eval_jobs/executor_runtime.rs deleted file mode 100644 index 1bc5b94f8c..0000000000 --- a/src/service/llm_evaluations/eval_jobs/executor_runtime.rs +++ /dev/null @@ -1,496 +0,0 @@ -// Copyright 2026 OpenObserve Inc. -// -// This program is free software: you can redistribute it and/or modify -// it under the terms of the GNU Affero General Public License as published by -// the Free Software Foundation, either version 3 of the License, or -// (at your option) any later version. -// -// This program is distributed in the hope that it will be useful -// but WITHOUT ANY WARRANTY; without even the implied warranty of -// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -// GNU Affero General Public License for more details. -// -// You should have received a copy of the GNU Affero General Public License -// along with this program. If not, see . - -//! Executor Runtime — orchestrates span evaluation through configured scorers. -//! -//! Glue between the pipeline's LlmEvaluation node and the individual scorer -//! executors (llm_judge, remote). Called from `batch_execution.rs` when the -//! pipeline node is an evaluation pipeline (job_id is set). -//! -//! For each span record: -//! 1. Resolves scorer configs from the database -//! 2. Dispatches to llm_judge or remote scorer -//! 3. Produces score records (forwarded through pipeline to _llm_scores) -//! 4. Produces evaluator traces (internally exported as OTLP spans) - -use std::collections::HashMap; - -use anyhow::Result; -use config::meta::{pipeline::components::ScorerRef, self_reporting::evaluator}; -use infra::table::{online_eval_jobs::JobInputMapping, scorers::ScorerType}; -use serde_json::Value; - -use crate::service::llm_evaluations::{ - evaluator_trace::{EvaluatorTrace, EvaluatorTraceInput, create_evaluator_trace}, - prepared_scorers::{PreparedScorer, ScorerExecutionError}, -}; - -#[derive(Debug)] -pub struct SpanEvalContext { - pub org_id: String, - pub span_id: String, - pub trace_id: String, - pub evaluator_trace_id: String, - pub session_id: Option, - pub source_stream: String, - pub job_id: Option, - pub eval_run_id: Option, - pub sampling_rate: Option, - pub sampled: Option, - pub attributes: HashMap, -} - -#[derive(Debug)] -pub struct EvalError { - pub scorer_id: String, - pub scorer_type: Option, - pub error_kind: String, - pub error_message: String, - pub latency_ms: i64, -} - -pub struct ExecutorOutput { - pub scores: Vec, - pub evaluator_traces: Vec, - pub errors: Vec, -} - -impl ExecutorOutput { - pub fn empty() -> Self { - Self { - scores: Vec::new(), - evaluator_traces: Vec::new(), - errors: Vec::new(), - } - } -} - -pub fn extract_context_from_span( - org_id: &str, - job_id: Option<&str>, - record: &Value, -) -> Option { - let span_id = record - .get("span_id") - .and_then(|v| v.as_str()) - .map(|s| s.to_string()) - .unwrap_or_default(); - - let trace_id = record - .get("trace_id") - .and_then(|v| v.as_str()) - .map(|s| s.to_string()) - .unwrap_or_default(); - - if span_id.is_empty() { - log::warn!("[EXECUTOR-RUNTIME] Skipping evaluation span with missing span_id"); - return None; - } - - if trace_id.is_empty() { - log::warn!( - "[EXECUTOR-RUNTIME] Skipping evaluation span '{}' with missing trace_id", - span_id - ); - return None; - } - - let session_id = record - .get("session_id") - .and_then(|v| v.as_str()) - .map(|s| s.to_string()); - - let source_stream = record - .get("pipeline_source_stream") - .and_then(|v| v.as_str()) - .unwrap_or("traces") - .to_string(); - - let mut attributes: HashMap = HashMap::new(); - - if let Some(attrs) = record.get("attributes").and_then(|v| v.as_object()) { - for (k, v) in attrs { - attributes.insert(k.clone(), v.clone()); - } - } - - for (key, value) in record.as_object().into_iter().flat_map(|o| o.iter()) { - if key == "span_id" - || key == "trace_id" - || key == "session_id" - || key == "attributes" - || key == "pipeline_source_stream" - { - continue; - } - attributes - .entry(key.clone()) - .or_insert_with(|| value.clone()); - } - - Some(SpanEvalContext { - org_id: org_id.to_string(), - span_id, - trace_id, - evaluator_trace_id: config::ider::generate_trace_id(), - session_id, - source_stream, - job_id: job_id.map(|s| s.to_string()), - eval_run_id: None, - sampling_rate: None, - sampled: None, - attributes, - }) -} - -fn resolve_scorer_input_variables( - job_input_mapping: &Option, - scorer_entity_id: &str, - span_attributes: &HashMap, -) -> HashMap { - let mut input_variables = HashMap::new(); - - let Some(mapping) = job_input_mapping - .as_ref() - .and_then(|m| m.get(scorer_entity_id)) - else { - return input_variables; - }; - - for (variable, pattern) in mapping { - input_variables.insert( - variable.clone(), - Value::String(render_mapping_value(pattern, span_attributes)), - ); - } - - input_variables -} - -fn render_mapping_value(pattern: &str, attrs: &HashMap) -> String { - if pattern.starts_with("{{") && pattern.ends_with("}}") && pattern.len() > 4 { - let path = pattern[2..pattern.len() - 2].trim(); - return get_attr(attrs, path); - } - pattern.to_string() -} - -fn get_attr(attrs: &HashMap, path: &str) -> String { - let parts: Vec<&str> = path.split('.').collect(); - let mut current = attrs.get(parts[0]); - for part in &parts[1..] { - current = current.and_then(|v| v.get(part)); - } - match current { - Some(Value::String(s)) => s.clone(), - Some(v) => v.to_string(), - None => attrs - .get(path) - .map(|v| match v { - Value::String(s) => s.clone(), - v => v.to_string(), - }) - .unwrap_or_default(), - } -} - -fn input_mapping_attributes(ctx: &SpanEvalContext) -> HashMap { - let mut attrs = ctx.attributes.clone(); - attrs.insert("span_id".to_string(), Value::String(ctx.span_id.clone())); - attrs.insert("trace_id".to_string(), Value::String(ctx.trace_id.clone())); - attrs.insert( - "pipeline_source_stream".to_string(), - Value::String(ctx.source_stream.clone()), - ); - if let Some(session_id) = &ctx.session_id { - attrs.insert("session_id".to_string(), Value::String(session_id.clone())); - } - attrs -} - -pub(crate) fn input_variables_with_context_defaults( - input_variables: &HashMap, - ctx: &SpanEvalContext, -) -> HashMap { - let mut variables = input_mapping_attributes(ctx); - for (key, value) in input_variables { - if !matches!(value, Value::String(s) if s.is_empty()) { - variables.insert(key.clone(), value.clone()); - } - } - variables -} - -/// Execute scorers on a batch of span records. -/// -/// This is the main entry point called from the LlmEvaluation pipeline node handler. -pub async fn execute_scorers( - ctx: &SpanEvalContext, - scorer_refs: &[ScorerRef], -) -> Result { - let mut output = ExecutorOutput::empty(); - let job_input_mapping = match ctx.job_id.as_deref() { - Some(job_id) => infra::table::online_eval_jobs::get_by_org(job_id, &ctx.org_id) - .await? - .and_then(|job| job.input_mapping), - None => None, - }; - - for scorer_ref in scorer_refs { - let mut resolved_scorer_type = None; - let scorer_lookup = match scorer_ref.version { - Some(v) => { - infra::table::scorers::get_by_entity_id_and_version(&ctx.org_id, &scorer_ref.id, v) - .await - } - None => infra::table::scorers::get_by_entity_id(&ctx.org_id, &scorer_ref.id).await, - }; - - let score_result = match scorer_lookup { - Ok(Some(scorer)) => { - resolved_scorer_type = Some(scorer.scorer_type); - let attrs = input_mapping_attributes(ctx); - let input_variables = - resolve_scorer_input_variables(&job_input_mapping, &scorer_ref.id, &attrs); - PreparedScorer::from_scorer(&ctx.org_id, scorer) - .await? - .execute(ctx, &input_variables) - .await - } - Ok(None) => { - log::warn!( - "[EXECUTOR-RUNTIME] Scorer entity '{}' not found in org '{}'", - scorer_ref.id, - ctx.org_id - ); - Ok(None) - } - Err(e) => Err(anyhow::anyhow!( - "Failed to look up scorer entity '{}': {e}", - scorer_ref.id - )), - }; - - match score_result { - Ok(Some(score_result)) => { - output.scores.push(score_result.score_json); - output.evaluator_traces.push(score_result.evaluator_trace); - } - Ok(None) => { - log::debug!( - "[EXECUTOR-RUNTIME] Scorer entity '{}' resolved but returned no result", - scorer_ref.id - ); - } - Err(e) => { - log::error!( - "[EXECUTOR-RUNTIME] Error executing scorer entity '{}': {e}", - scorer_ref.id - ); - let execution_error = e.downcast_ref::(); - let err_trace = create_evaluator_trace(EvaluatorTraceInput { - org_id: ctx.org_id.clone(), - evaluator_trace_id: ctx.evaluator_trace_id.clone(), - target_span_id: ctx.span_id.clone(), - target_trace_id: ctx.trace_id.clone(), - target_stream: ctx.source_stream.clone(), - scorer_id: Some(scorer_ref.id.clone()), - scorer_version: Some("?".to_string()), - scorer_type: resolved_scorer_type, - job_id: ctx.job_id.clone(), - score_config_id: None, - score_config_version: None, - eval_run_id: ctx.eval_run_id.clone(), - provider_id: execution_error.and_then(|e| e.provider_id.clone()), - provider_name: execution_error.and_then(|e| e.provider_name.clone()), - provider_type: execution_error.and_then(|e| e.provider_type.clone()), - model: execution_error.and_then(|e| e.model.clone()), - latency_ms: execution_error.map(|e| e.latency_ms).unwrap_or(0), - prompt_tokens: execution_error.and_then(|e| e.prompt_tokens), - completion_tokens: execution_error.and_then(|e| e.completion_tokens), - total_tokens: execution_error.and_then(|e| e.total_tokens), - sampling_rate: ctx.sampling_rate, - sampled: ctx.sampled, - status: evaluator::status::ERROR.to_string(), - error_kind: Some("execution_error".to_string()), - error_message: Some(format!("{e}")), - skip_reason: None, - prompt: execution_error.and_then(|e| e.scorer_input.clone()), - response: execution_error.and_then(|e| e.raw_response.clone()), - }); - output.evaluator_traces.push(err_trace); - output.errors.push(EvalError { - scorer_id: scorer_ref.id.clone(), - scorer_type: resolved_scorer_type, - error_kind: "execution_error".to_string(), - error_message: format!("{e}"), - latency_ms: execution_error.map(|e| e.latency_ms).unwrap_or(0), - }); - } - } - } - - Ok(output) -} - -#[cfg(test)] -mod tests { - use super::*; - - #[test] - fn test_extract_context_from_basic_span() { - let record = serde_json::json!({ - "span_id": "span-1", - "trace_id": "trace-1", - "span_name": "chat completion", - "attributes": { - "gen_ai.system": "openai", - "gen_ai.request.model": "gpt-4o" - } - }); - - let ctx = extract_context_from_span("org1", Some("job-1"), &record).unwrap(); - assert_eq!(ctx.org_id, "org1"); - assert_eq!(ctx.span_id, "span-1"); - assert_eq!(ctx.trace_id, "trace-1"); - assert_eq!(ctx.evaluator_trace_id.len(), 32); - assert_ne!(ctx.evaluator_trace_id, ctx.trace_id); - assert_eq!(ctx.job_id, Some("job-1".to_string())); - assert_eq!(ctx.attributes.get("span_name").unwrap(), "chat completion"); - assert_eq!(ctx.attributes.get("gen_ai.system").unwrap(), "openai"); - } - - #[test] - fn test_extract_context_no_span_id() { - let record = serde_json::json!({ - "trace_id": "trace-1", - "some_field": "value" - }); - assert!(extract_context_from_span("org1", None, &record).is_none()); - } - - #[test] - fn test_extract_context_with_session() { - let record = serde_json::json!({ - "span_id": "span-2", - "trace_id": "trace-2", - "session_id": "session-1", - "attributes": {} - }); - let ctx = extract_context_from_span("org2", None, &record).unwrap(); - assert_eq!(ctx.session_id, Some("session-1".to_string())); - assert_eq!(ctx.attributes.is_empty(), true); - } - - #[test] - fn test_resolve_scorer_input_variables_from_job_mapping() { - let mapping = Some( - serde_json::from_value(serde_json::json!({ - "scorer-1": { - "input": "{{gen_ai.prompt}}", - "output": "{{gen_ai.completion}}" - } - })) - .unwrap(), - ); - let attrs = HashMap::from([ - ("gen_ai.prompt".to_string(), serde_json::json!("question")), - ("gen_ai.completion".to_string(), serde_json::json!("answer")), - ]); - - let variables = resolve_scorer_input_variables(&mapping, "scorer-1", &attrs); - - assert_eq!(variables.get("input"), Some(&serde_json::json!("question"))); - assert_eq!(variables.get("output"), Some(&serde_json::json!("answer"))); - } - - #[test] - fn test_input_mapping_attributes_include_span_context_fields() { - let ctx = SpanEvalContext { - org_id: "org1".to_string(), - span_id: "span-1".to_string(), - trace_id: "trace-1".to_string(), - evaluator_trace_id: "11111111111111111111111111111111".to_string(), - session_id: Some("session-1".to_string()), - source_stream: "traces".to_string(), - job_id: Some("job-1".to_string()), - eval_run_id: None, - sampling_rate: None, - sampled: None, - attributes: HashMap::from([("input".to_string(), serde_json::json!("question"))]), - }; - let mapping = Some( - serde_json::from_value(serde_json::json!({ - "scorer-1": { - "trace_id": "{{trace_id}}", - "span_id": "{{span_id}}", - "session_id": "{{session_id}}", - "stream": "{{pipeline_source_stream}}", - "input": "{{input}}" - } - })) - .unwrap(), - ); - - let attrs = input_mapping_attributes(&ctx); - let variables = resolve_scorer_input_variables(&mapping, "scorer-1", &attrs); - - assert_eq!( - variables.get("trace_id"), - Some(&serde_json::json!("trace-1")) - ); - assert_eq!(variables.get("span_id"), Some(&serde_json::json!("span-1"))); - assert_eq!( - variables.get("session_id"), - Some(&serde_json::json!("session-1")) - ); - assert_eq!(variables.get("stream"), Some(&serde_json::json!("traces"))); - assert_eq!(variables.get("input"), Some(&serde_json::json!("question"))); - } - - #[test] - fn test_input_variables_with_context_defaults_override_empty_context_fields() { - let ctx = SpanEvalContext { - org_id: "org1".to_string(), - span_id: "span-1".to_string(), - trace_id: "trace-1".to_string(), - evaluator_trace_id: "11111111111111111111111111111111".to_string(), - session_id: Some("session-1".to_string()), - source_stream: "traces".to_string(), - job_id: Some("job-1".to_string()), - eval_run_id: None, - sampling_rate: None, - sampled: None, - attributes: HashMap::new(), - }; - let input_variables = HashMap::from([ - ("trace_id".to_string(), serde_json::json!("")), - ("input".to_string(), serde_json::json!("question")), - ]); - - let variables = input_variables_with_context_defaults(&input_variables, &ctx); - - assert_eq!( - variables.get("trace_id"), - Some(&serde_json::json!("trace-1")) - ); - assert_eq!(variables.get("span_id"), Some(&serde_json::json!("span-1"))); - assert_eq!( - variables.get("session_id"), - Some(&serde_json::json!("session-1")) - ); - assert_eq!(variables.get("input"), Some(&serde_json::json!("question"))); - } -} diff --git a/src/service/llm_evaluations/eval_jobs/mod.rs b/src/service/llm_evaluations/eval_jobs/mod.rs index b3f24529d8..c99a6c9a27 100644 --- a/src/service/llm_evaluations/eval_jobs/mod.rs +++ b/src/service/llm_evaluations/eval_jobs/mod.rs @@ -29,8 +29,8 @@ use crate::common::{ utils::auth::{is_ofga_object_visible, remove_ownership, set_ownership}, }; -pub mod executor_runtime; pub mod reconciler; +pub use o2_enterprise::enterprise::llm_evaluations::eval_jobs::executor_runtime; /// Errors that can occur when interacting with online eval jobs. #[derive(Debug, thiserror::Error)] diff --git a/src/service/llm_evaluations/evaluator_trace.rs b/src/service/llm_evaluations/evaluator_trace.rs deleted file mode 100644 index f1c4e0e804..0000000000 --- a/src/service/llm_evaluations/evaluator_trace.rs +++ /dev/null @@ -1,873 +0,0 @@ -// Copyright 2026 OpenObserve Inc. -// -// This program is free software: you can redistribute it and/or modify -// it under the terms of the GNU Affero General Public License as published by -// the Free Software Foundation, either version 3 of the License, or -// (at your option) any later version. -// -// This program is distributed in the hope that it will be useful -// but WITHOUT ANY WARRANTY; without even the implied warranty of -// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -// GNU Affero General Public License for more details. -// -// You should have received a copy of the GNU Affero General Public License -// along with this program. If not, see . - -//! Evaluator Trace — builds internal OTLP spans for online evaluation activity. -//! -//! Evaluator spans are exported to the org's `_evaluator` traces stream through -//! normal OTLP ingestion so the LLM processor can classify LLM-Judge calls and -//! calculate token/cost fields. The evaluator trace ID remains separate from the -//! target application trace; target trace/span identity is stored as evaluator -//! metadata. - -use chrono::Utc; -use config::{ider, meta::self_reporting::evaluator}; -use infra::table::scorers::ScorerType; -use opentelemetry_proto::tonic::{ - collector::trace::v1::ExportTraceServiceRequest, - common::v1::{AnyValue, InstrumentationScope, KeyValue, any_value}, - resource::v1::Resource, - trace::v1::{ResourceSpans, ScopeSpans, Span, Status, span::SpanKind, status::StatusCode}, -}; -use serde::Serialize; - -const SERVICE_NAME: &str = "online_eval"; -const SCOPE_NAME: &str = "openobserve.online_eval"; -const GEN_AI_CHAT_OPERATION: &str = "chat"; - -#[derive(Debug, Clone, Serialize)] -pub struct EvaluatorTraceInput { - pub org_id: String, - pub evaluator_trace_id: String, - pub target_span_id: String, - pub target_trace_id: String, - pub target_stream: String, - pub scorer_id: Option, - pub scorer_version: Option, - pub scorer_type: Option, - pub job_id: Option, - pub score_config_id: Option, - pub score_config_version: Option, - pub eval_run_id: Option, - pub provider_id: Option, - pub provider_name: Option, - pub provider_type: Option, - pub model: Option, - pub latency_ms: i64, - pub prompt_tokens: Option, - pub completion_tokens: Option, - pub total_tokens: Option, - pub sampling_rate: Option, - pub sampled: Option, - pub status: String, - pub error_kind: Option, - pub error_message: Option, - pub skip_reason: Option, - pub prompt: Option, - pub response: Option, -} - -#[derive(Debug, Clone)] -pub struct EvaluatorTrace { - pub span: Span, - pub is_error: bool, -} - -pub fn create_evaluator_trace(input: EvaluatorTraceInput) -> EvaluatorTrace { - let span_id = ider::generate_span_id(); - let trace_id = input.evaluator_trace_id.clone(); - let latency_ms = input.latency_ms.max(0) as u64; - let latency_nanos = latency_ms.saturating_mul(1_000_000); - let end_time = now_nanos(); - let start_time = end_time.saturating_sub(latency_nanos); - - let is_error = - input.status == evaluator::status::ERROR || input.status == evaluator::status::TIMEOUT; - let is_llm_judge = matches!(input.scorer_type.as_ref(), Some(ScorerType::LlmJudge)); - - let span_name = if input.status == evaluator::status::SKIPPED { - evaluator::SPAN_NAME_SPAN_SKIPPED - } else { - match input.scorer_type.as_ref() { - Some(ScorerType::LlmJudge) => evaluator::SPAN_NAME_LLM_JUDGE, - Some(ScorerType::Remote) => evaluator::SPAN_NAME_REMOTE_SCORER, - None => evaluator::SPAN_NAME_UNKNOWN_SCORER, - } - }; - - let mut attributes = Vec::new(); - - push_string_attr( - &mut attributes, - evaluator_attr_key(evaluator::ATTR_TARGET_SPAN_ID), - &input.target_span_id, - ); - push_string_attr( - &mut attributes, - evaluator_attr_key(evaluator::ATTR_TARGET_TRACE_ID), - &input.target_trace_id, - ); - push_string_attr( - &mut attributes, - evaluator_attr_key(evaluator::ATTR_TARGET_STREAM), - &input.target_stream, - ); - - if let Some(ref scorer_id) = input.scorer_id { - push_string_attr( - &mut attributes, - evaluator_attr_key(evaluator::ATTR_SCORER_ID), - scorer_id, - ); - } - if let Some(ref scorer_version) = input.scorer_version { - push_string_attr( - &mut attributes, - evaluator_attr_key(evaluator::ATTR_SCORER_VERSION), - scorer_version, - ); - } - if let Some(ref scorer_type) = input.scorer_type { - push_string_attr( - &mut attributes, - evaluator_attr_key(evaluator::ATTR_SCORER_TYPE), - scorer_type.to_string(), - ); - } - - push_optional_string_attr( - &mut attributes, - evaluator_attr_key(evaluator::ATTR_JOB_ID), - input.job_id.as_deref(), - ); - push_optional_string_attr( - &mut attributes, - evaluator_attr_key(evaluator::ATTR_SCORE_CONFIG_ID), - input.score_config_id.as_deref(), - ); - push_optional_string_attr( - &mut attributes, - evaluator_attr_key(evaluator::ATTR_SCORE_CONFIG_VERSION), - input.score_config_version.as_deref(), - ); - push_optional_string_attr( - &mut attributes, - evaluator_attr_key(evaluator::ATTR_EVAL_RUN_ID), - input.eval_run_id.as_deref(), - ); - - push_optional_string_attr( - &mut attributes, - evaluator_attr_key(evaluator::ATTR_PROVIDER_ID), - input.provider_id.as_deref(), - ); - push_optional_string_attr( - &mut attributes, - evaluator_attr_key(evaluator::ATTR_PROVIDER_NAME), - input.provider_name.as_deref(), - ); - push_optional_string_attr( - &mut attributes, - evaluator_attr_key(evaluator::ATTR_PROVIDER_TYPE), - input.provider_type.as_deref(), - ); - push_optional_string_attr( - &mut attributes, - evaluator_attr_key(evaluator::ATTR_MODEL), - input.model.as_deref(), - ); - - push_i64_attr( - &mut attributes, - evaluator_attr_key(evaluator::ATTR_LATENCY_MS), - input.latency_ms, - ); - push_optional_i64_attr( - &mut attributes, - evaluator_attr_key(evaluator::ATTR_PROMPT_TOKENS), - input.prompt_tokens, - ); - push_optional_i64_attr( - &mut attributes, - evaluator_attr_key(evaluator::ATTR_COMPLETION_TOKENS), - input.completion_tokens, - ); - push_optional_i64_attr( - &mut attributes, - evaluator_attr_key(evaluator::ATTR_TOTAL_TOKENS), - input.total_tokens, - ); - if let Some(sampling_rate) = input.sampling_rate { - push_f64_attr( - &mut attributes, - evaluator_attr_key(evaluator::ATTR_SAMPLING_RATE), - sampling_rate, - ); - } - if let Some(sampled) = input.sampled { - push_bool_attr( - &mut attributes, - evaluator_attr_key(evaluator::ATTR_SAMPLED), - sampled, - ); - } - - push_string_attr( - &mut attributes, - evaluator_attr_key(evaluator::ATTR_STATUS), - &input.status, - ); - push_optional_string_attr( - &mut attributes, - evaluator_attr_key(evaluator::ATTR_ERROR_KIND), - input.error_kind.as_deref(), - ); - push_optional_string_attr( - &mut attributes, - evaluator_attr_key(evaluator::ATTR_ERROR_MESSAGE), - input.error_message.as_deref(), - ); - push_optional_string_attr( - &mut attributes, - evaluator_attr_key(evaluator::ATTR_SKIP_REASON), - input.skip_reason.as_deref(), - ); - - let prompt = input.prompt.as_ref().map(|p| truncate_str(p, 4096)); - let response = input.response.as_ref().map(|r| truncate_str(r, 4096)); - - push_optional_string_attr( - &mut attributes, - evaluator_attr_key(evaluator::ATTR_PROMPT), - prompt.as_deref(), - ); - push_optional_string_attr( - &mut attributes, - evaluator_attr_key(evaluator::ATTR_RESPONSE), - response.as_deref(), - ); - - if is_llm_judge { - push_string_attr( - &mut attributes, - evaluator::GEN_AI_OPERATION_NAME, - GEN_AI_CHAT_OPERATION, - ); - push_optional_string_attr( - &mut attributes, - evaluator::GEN_AI_REQUEST_MODEL, - input.model.as_deref(), - ); - push_optional_string_attr( - &mut attributes, - evaluator::GEN_AI_RESPONSE_MODEL, - input.model.as_deref(), - ); - - let provider_for_gen_ai = input - .provider_type - .as_deref() - .or(input.provider_name.as_deref()); - push_optional_string_attr( - &mut attributes, - evaluator::GEN_AI_PROVIDER_NAME, - provider_for_gen_ai, - ); - push_optional_string_attr( - &mut attributes, - evaluator::GEN_AI_SYSTEM, - input.provider_type.as_deref(), - ); - - if let Some(ref prompt) = prompt { - push_string_attr( - &mut attributes, - evaluator::GEN_AI_INPUT_MESSAGES, - messages_json(prompt, "user"), - ); - } - if let Some(ref response) = response { - push_string_attr( - &mut attributes, - evaluator::GEN_AI_OUTPUT_MESSAGES, - messages_json(response, "assistant"), - ); - } - - push_optional_i64_attr( - &mut attributes, - evaluator::GEN_AI_USAGE_INPUT_TOKENS, - input.prompt_tokens, - ); - push_optional_i64_attr( - &mut attributes, - evaluator::GEN_AI_USAGE_OUTPUT_TOKENS, - input.completion_tokens, - ); - push_optional_i64_attr( - &mut attributes, - evaluator::GEN_AI_USAGE_TOTAL_TOKENS, - input.total_tokens, - ); - } - - let span = Span { - trace_id: decode_trace_id(&trace_id), - span_id: decode_span_id(&span_id), - trace_state: String::new(), - parent_span_id: vec![], - flags: 1, - name: span_name.to_string(), - kind: SpanKind::Internal as i32, - start_time_unix_nano: start_time, - end_time_unix_nano: end_time, - attributes, - dropped_attributes_count: 0, - events: vec![], - dropped_events_count: 0, - links: vec![], - dropped_links_count: 0, - status: Some(Status { - code: if is_error { - StatusCode::Error as i32 - } else { - StatusCode::Ok as i32 - }, - message: input.error_message.unwrap_or_default(), - }), - }; - - EvaluatorTrace { span, is_error } -} - -pub fn create_evaluator_trace_request(traces: Vec) -> ExportTraceServiceRequest { - let spans: Vec = traces.into_iter().map(|trace| trace.span).collect(); - if spans.is_empty() { - return ExportTraceServiceRequest { - resource_spans: vec![], - }; - } - - ExportTraceServiceRequest { - resource_spans: vec![ResourceSpans { - resource: Some(Resource { - attributes: vec![string_key_value("service.name", SERVICE_NAME)], - ..Default::default() - }), - scope_spans: vec![ScopeSpans { - scope: Some(InstrumentationScope { - name: SCOPE_NAME.to_string(), - version: config::VERSION.to_string(), - attributes: vec![], - dropped_attributes_count: 0, - }), - spans, - ..Default::default() - }], - ..Default::default() - }], - } -} - -fn now_nanos() -> u64 { - Utc::now() - .timestamp_nanos_opt() - .unwrap_or_else(|| Utc::now().timestamp_micros().saturating_mul(1000)) - .max(0) as u64 -} - -fn evaluator_attr_key(key: &str) -> String { - format!("attributes.{key}") -} - -fn string_key_value(key: impl Into, value: impl Into) -> KeyValue { - KeyValue { - key: key.into(), - value: Some(AnyValue { - value: Some(any_value::Value::StringValue(value.into())), - }), - ..Default::default() - } -} - -fn push_string_attr(attrs: &mut Vec, key: impl Into, value: impl Into) { - attrs.push(string_key_value(key, value)); -} - -fn push_optional_string_attr( - attrs: &mut Vec, - key: impl Into, - value: Option<&str>, -) { - if let Some(value) = value { - push_string_attr(attrs, key, value); - } -} - -fn push_i64_attr(attrs: &mut Vec, key: impl Into, value: i64) { - attrs.push(KeyValue { - key: key.into(), - value: Some(AnyValue { - value: Some(any_value::Value::IntValue(value)), - }), - ..Default::default() - }); -} - -fn push_optional_i64_attr(attrs: &mut Vec, key: impl Into, value: Option) { - if let Some(value) = value { - push_i64_attr(attrs, key, value); - } -} - -fn push_f64_attr(attrs: &mut Vec, key: impl Into, value: f64) { - attrs.push(KeyValue { - key: key.into(), - value: Some(AnyValue { - value: Some(any_value::Value::DoubleValue(value)), - }), - ..Default::default() - }); -} - -fn push_bool_attr(attrs: &mut Vec, key: impl Into, value: bool) { - attrs.push(KeyValue { - key: key.into(), - value: Some(AnyValue { - value: Some(any_value::Value::BoolValue(value)), - }), - ..Default::default() - }); -} - -fn decode_trace_id(trace_id: &str) -> Vec { - decode_hex_id(trace_id, 16).unwrap_or_else(|| { - decode_hex_id(&ider::generate_trace_id(), 16).unwrap_or_else(|| vec![1; 16]) - }) -} - -fn decode_span_id(span_id: &str) -> Vec { - decode_hex_id(span_id, 8).unwrap_or_else(|| { - decode_hex_id(&ider::generate_span_id(), 8).unwrap_or_else(|| vec![1; 8]) - }) -} - -fn decode_hex_id(id: &str, expected_len: usize) -> Option> { - let bytes = hex::decode(id).ok()?; - if bytes.len() == expected_len && bytes.iter().any(|byte| *byte != 0) { - Some(bytes) - } else { - None - } -} - -fn messages_json(content: &str, role: &str) -> String { - let trimmed = content.trim(); - if let Ok(value) = serde_json::from_str::(trimmed) - && value.is_array() - { - return trimmed.to_string(); - } - - serde_json::json!([{ "role": role, "content": content }]).to_string() -} - -fn truncate_str(s: &str, max_bytes: usize) -> String { - if s.len() <= max_bytes { - return s.to_string(); - } - // Walk back to the nearest char boundary so we never slice mid-codepoint. - let mut boundary = max_bytes; - while boundary > 0 && !s.is_char_boundary(boundary) { - boundary -= 1; - } - format!("{}...[truncated]", &s[..boundary]) -} - -#[cfg(test)] -mod tests { - use super::*; - - fn attr<'a>(span: &'a Span, key: &str) -> Option<&'a AnyValue> { - span.attributes - .iter() - .find(|attr| attr.key == key) - .and_then(|attr| attr.value.as_ref()) - } - - fn attr_str<'a>(span: &'a Span, key: &str) -> Option<&'a str> { - match attr(span, key)?.value.as_ref()? { - any_value::Value::StringValue(value) => Some(value.as_str()), - _ => None, - } - } - - fn attr_i64(span: &Span, key: &str) -> Option { - match attr(span, key)?.value.as_ref()? { - any_value::Value::IntValue(value) => Some(*value), - _ => None, - } - } - - fn attr_bool(span: &Span, key: &str) -> Option { - match attr(span, key)?.value.as_ref()? { - any_value::Value::BoolValue(value) => Some(*value), - _ => None, - } - } - - fn attr_f64(span: &Span, key: &str) -> Option { - match attr(span, key)?.value.as_ref()? { - any_value::Value::DoubleValue(value) => Some(*value), - _ => None, - } - } - - fn success_input() -> EvaluatorTraceInput { - EvaluatorTraceInput { - org_id: "org1".to_string(), - evaluator_trace_id: "11111111111111111111111111111111".to_string(), - target_span_id: "span-1".to_string(), - target_trace_id: "trace-1".to_string(), - target_stream: "traces".to_string(), - scorer_id: Some("sc-1".to_string()), - scorer_version: Some("1".to_string()), - scorer_type: Some(ScorerType::LlmJudge), - job_id: Some("job-1".to_string()), - score_config_id: Some("cfg-1".to_string()), - score_config_version: Some("1".to_string()), - eval_run_id: Some("run-1".to_string()), - provider_id: Some("prov-1".to_string()), - provider_name: Some("OpenAI Prod".to_string()), - provider_type: Some("openai".to_string()), - model: Some("gpt-4o".to_string()), - latency_ms: 500, - prompt_tokens: Some(100), - completion_tokens: Some(50), - total_tokens: Some(150), - sampling_rate: Some(0.25), - sampled: Some(true), - status: evaluator::status::SUCCESS.to_string(), - error_kind: None, - error_message: None, - skip_reason: None, - prompt: Some("Evaluate this text...".to_string()), - response: Some("{\"score\":0.9}".to_string()), - } - } - - #[test] - fn test_create_success_trace_as_otlp_span() { - let trace = create_evaluator_trace(success_input()); - assert!(!trace.is_error); - - let span = &trace.span; - assert_eq!(span.name, evaluator::SPAN_NAME_LLM_JUDGE); - assert_eq!(span.kind, SpanKind::Internal as i32); - assert_eq!(span.status.as_ref().unwrap().code, StatusCode::Ok as i32); - assert_eq!( - hex::encode(&span.trace_id), - "11111111111111111111111111111111" - ); - assert_eq!(span.span_id.len(), 8); - assert_eq!(span.parent_span_id.len(), 0); - assert_eq!( - span.end_time_unix_nano - span.start_time_unix_nano, - 500_000_000 - ); - - assert_eq!( - attr_str(span, &evaluator_attr_key(evaluator::ATTR_TARGET_SPAN_ID)), - Some("span-1") - ); - assert_eq!( - attr_str(span, &evaluator_attr_key(evaluator::ATTR_SCORER_ID)), - Some("sc-1") - ); - assert_eq!( - attr_i64(span, &evaluator_attr_key(evaluator::ATTR_LATENCY_MS)), - Some(500) - ); - assert_eq!( - attr_str(span, &evaluator_attr_key(evaluator::ATTR_PROVIDER_NAME)), - Some("OpenAI Prod") - ); - assert_eq!( - attr_str(span, &evaluator_attr_key(evaluator::ATTR_MODEL)), - Some("gpt-4o") - ); - assert_eq!( - attr_str(span, &evaluator_attr_key(evaluator::ATTR_STATUS)), - Some(evaluator::status::SUCCESS) - ); - assert_eq!( - attr_f64(span, &evaluator_attr_key(evaluator::ATTR_SAMPLING_RATE)), - Some(0.25) - ); - assert_eq!( - attr_bool(span, &evaluator_attr_key(evaluator::ATTR_SAMPLED)), - Some(true) - ); - - assert_eq!( - attr_str(span, evaluator::GEN_AI_OPERATION_NAME), - Some(GEN_AI_CHAT_OPERATION) - ); - assert_eq!( - attr_str(span, evaluator::GEN_AI_REQUEST_MODEL), - Some("gpt-4o") - ); - assert_eq!( - attr_str(span, evaluator::GEN_AI_RESPONSE_MODEL), - Some("gpt-4o") - ); - assert_eq!( - attr_str(span, evaluator::GEN_AI_PROVIDER_NAME), - Some("openai") - ); - assert_eq!( - attr_i64(span, evaluator::GEN_AI_USAGE_INPUT_TOKENS), - Some(100) - ); - assert_eq!( - attr_i64(span, evaluator::GEN_AI_USAGE_OUTPUT_TOKENS), - Some(50) - ); - assert_eq!( - attr_i64(span, evaluator::GEN_AI_USAGE_TOTAL_TOKENS), - Some(150) - ); - assert!( - attr_str(span, evaluator::GEN_AI_INPUT_MESSAGES) - .unwrap() - .contains("Evaluate this text") - ); - assert!( - attr_str(span, evaluator::GEN_AI_OUTPUT_MESSAGES) - .unwrap() - .contains("score") - ); - } - - #[test] - fn test_create_error_trace() { - let input = EvaluatorTraceInput { - org_id: "org1".to_string(), - evaluator_trace_id: "22222222222222222222222222222222".to_string(), - target_span_id: "span-2".to_string(), - target_trace_id: "trace-2".to_string(), - target_stream: "traces".to_string(), - scorer_id: Some("sc-2".to_string()), - scorer_version: Some("2".to_string()), - scorer_type: Some(ScorerType::LlmJudge), - job_id: None, - score_config_id: None, - score_config_version: None, - eval_run_id: None, - provider_id: None, - provider_name: None, - provider_type: None, - model: None, - latency_ms: 1000, - prompt_tokens: None, - completion_tokens: None, - total_tokens: None, - sampling_rate: None, - sampled: None, - status: evaluator::status::ERROR.to_string(), - error_kind: Some("timeout".to_string()), - error_message: Some("Request timed out".to_string()), - skip_reason: None, - prompt: None, - response: None, - }; - - let trace = create_evaluator_trace(input); - assert!(trace.is_error); - assert_eq!( - trace.span.status.as_ref().unwrap().code, - StatusCode::Error as i32 - ); - assert_eq!( - attr_str(&trace.span, &evaluator_attr_key(evaluator::ATTR_STATUS)), - Some(evaluator::status::ERROR) - ); - assert_eq!( - attr_str(&trace.span, &evaluator_attr_key(evaluator::ATTR_ERROR_KIND)), - Some("timeout") - ); - assert_eq!( - attr_str( - &trace.span, - &evaluator_attr_key(evaluator::ATTR_ERROR_MESSAGE) - ), - Some("Request timed out") - ); - } - - #[test] - fn test_create_remote_scorer_trace_has_no_gen_ai_attrs() { - let input = EvaluatorTraceInput { - org_id: "org1".to_string(), - evaluator_trace_id: "33333333333333333333333333333333".to_string(), - target_span_id: "span-3".to_string(), - target_trace_id: "trace-3".to_string(), - target_stream: "traces".to_string(), - scorer_id: Some("sc-3".to_string()), - scorer_version: Some("3".to_string()), - scorer_type: Some(ScorerType::Remote), - job_id: None, - score_config_id: None, - score_config_version: None, - eval_run_id: None, - provider_id: None, - provider_name: None, - provider_type: None, - model: None, - latency_ms: 200, - prompt_tokens: None, - completion_tokens: None, - total_tokens: None, - sampling_rate: None, - sampled: None, - status: evaluator::status::SUCCESS.to_string(), - error_kind: None, - error_message: None, - skip_reason: None, - prompt: None, - response: Some("{\"score\":0.9,\"reason\":\"good\"}".to_string()), - }; - - let trace = create_evaluator_trace(input); - assert!(!trace.is_error); - assert_eq!(trace.span.name, evaluator::SPAN_NAME_REMOTE_SCORER); - assert_eq!( - trace.span.status.as_ref().unwrap().code, - StatusCode::Ok as i32 - ); - assert_eq!( - attr_str(&trace.span, evaluator::GEN_AI_OPERATION_NAME), - None - ); - assert_eq!( - attr_str(&trace.span, &evaluator_attr_key(evaluator::ATTR_RESPONSE)), - Some("{\"score\":0.9,\"reason\":\"good\"}") - ); - } - - #[test] - fn test_create_skipped_trace() { - let input = EvaluatorTraceInput { - org_id: "org1".to_string(), - evaluator_trace_id: "55555555555555555555555555555555".to_string(), - target_span_id: "span-5".to_string(), - target_trace_id: "trace-5".to_string(), - target_stream: "traces".to_string(), - scorer_id: None, - scorer_version: None, - scorer_type: None, - job_id: Some("job-5".to_string()), - score_config_id: None, - score_config_version: None, - eval_run_id: Some("run-5".to_string()), - provider_id: None, - provider_name: None, - provider_type: None, - model: None, - latency_ms: 0, - prompt_tokens: None, - completion_tokens: None, - total_tokens: None, - sampling_rate: Some(0.1), - sampled: Some(false), - status: evaluator::status::SKIPPED.to_string(), - error_kind: None, - error_message: None, - skip_reason: Some("sampling".to_string()), - prompt: None, - response: None, - }; - - let trace = create_evaluator_trace(input); - assert!(!trace.is_error); - assert_eq!(trace.span.name, evaluator::SPAN_NAME_SPAN_SKIPPED); - assert_eq!( - trace.span.status.as_ref().unwrap().code, - StatusCode::Ok as i32 - ); - assert_eq!( - attr_str(&trace.span, &evaluator_attr_key(evaluator::ATTR_STATUS)), - Some(evaluator::status::SKIPPED) - ); - assert_eq!( - attr_str( - &trace.span, - &evaluator_attr_key(evaluator::ATTR_SKIP_REASON) - ), - Some("sampling") - ); - assert_eq!( - attr_bool(&trace.span, &evaluator_attr_key(evaluator::ATTR_SAMPLED)), - Some(false) - ); - assert_eq!( - attr_str(&trace.span, &evaluator_attr_key(evaluator::ATTR_SCORER_ID)), - None - ); - } - - #[test] - fn test_multiple_scorer_spans_share_evaluator_trace_id() { - let evaluator_trace_id = "11111111111111111111111111111111".to_string(); - - let make_input = |scorer_id: &str| EvaluatorTraceInput { - evaluator_trace_id: evaluator_trace_id.clone(), - scorer_id: Some(scorer_id.to_string()), - ..success_input() - }; - - let first = create_evaluator_trace(make_input("sc-1")); - let second = create_evaluator_trace(make_input("sc-2")); - - assert_eq!(hex::encode(&first.span.trace_id), evaluator_trace_id); - assert_eq!(first.span.trace_id, second.span.trace_id); - assert_ne!(first.span.span_id, second.span.span_id); - } - - #[test] - fn test_create_evaluator_trace_request_groups_spans_under_resource() { - let trace = create_evaluator_trace(success_input()); - let request = create_evaluator_trace_request(vec![trace]); - - assert_eq!(request.resource_spans.len(), 1); - let resource_spans = &request.resource_spans[0]; - assert_eq!( - resource_spans.resource.as_ref().unwrap().attributes[0].key, - "service.name" - ); - assert_eq!(resource_spans.scope_spans.len(), 1); - assert_eq!( - resource_spans.scope_spans[0].scope.as_ref().unwrap().name, - SCOPE_NAME - ); - assert_eq!(resource_spans.scope_spans[0].spans.len(), 1); - } - - #[test] - fn test_messages_json_wraps_plain_content_and_preserves_arrays() { - assert_eq!( - messages_json("hello", "user"), - serde_json::json!([{ "role": "user", "content": "hello" }]).to_string() - ); - - let messages = r#"[{"role":"system","content":"rules"}]"#; - assert_eq!(messages_json(messages, "user"), messages); - } - - #[test] - fn test_truncate_str() { - assert_eq!(truncate_str("hello", 10), "hello"); - assert_eq!(truncate_str("hello world", 5), "hello...[truncated]"); - } -} diff --git a/src/service/llm_evaluations/mod.rs b/src/service/llm_evaluations/mod.rs index 0a26eae34b..e5fca18147 100644 --- a/src/service/llm_evaluations/mod.rs +++ b/src/service/llm_evaluations/mod.rs @@ -14,9 +14,8 @@ // along with this program. If not, see . pub mod eval_jobs; -pub mod evaluator_trace; pub mod evaluator_trace_exporter; -pub mod prepared_scorers; -pub mod score_configs; -pub mod score_writer; -pub mod scorers; + +pub use o2_enterprise::enterprise::llm_evaluations::{ + evaluator_trace, prepared_scorers, score_configs, score_writer, scorers, +}; diff --git a/src/service/llm_evaluations/prepared_scorers.rs b/src/service/llm_evaluations/prepared_scorers.rs deleted file mode 100644 index 7dba27b344..0000000000 --- a/src/service/llm_evaluations/prepared_scorers.rs +++ /dev/null @@ -1,485 +0,0 @@ -// Copyright 2026 OpenObserve Inc. -// -// This program is free software: you can redistribute it and/or modify -// it under the terms of the GNU Affero General Public License as published by -// the Free Software Foundation, either version 3 of the License, or -// (at your option) any later version. -// -// This program is distributed in the hope that it will be useful -// but WITHOUT ANY WARRANTY; without even the implied warranty of -// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -// GNU Affero General Public License for more details. -// -// You should have received a copy of the GNU Affero General Public License -// along with this program. If not, see . - -use std::{collections::HashMap, fmt}; - -use anyhow::Result; -use config::meta::self_reporting::llm_scores::{LlmScoreDataSourceType, LlmScoreDataType}; -use infra::table::{score_configs::ScoreConfigDataType, scorers::ScorerType}; -use serde_json::Value; - -use crate::service::llm_evaluations::{ - eval_jobs::executor_runtime::{SpanEvalContext, input_variables_with_context_defaults}, - evaluator_trace::{EvaluatorTrace, EvaluatorTraceInput, create_evaluator_trace}, - score_writer::{ScoreWriterInput, create_score}, - scorers::{llm_judge, remote_client, schema_derivation::ExtraMetadataField}, -}; - -#[derive(Debug)] -pub(crate) struct ScorerExecutionError { - message: String, - pub(crate) raw_response: Option, - pub(crate) scorer_input: Option, - pub(crate) provider_id: Option, - pub(crate) provider_name: Option, - pub(crate) provider_type: Option, - pub(crate) model: Option, - pub(crate) prompt_tokens: Option, - pub(crate) completion_tokens: Option, - pub(crate) total_tokens: Option, - pub(crate) latency_ms: i64, -} - -impl fmt::Display for ScorerExecutionError { - fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { - f.write_str(&self.message) - } -} - -impl std::error::Error for ScorerExecutionError {} - -pub(crate) struct ScoreResult { - pub(crate) score_json: Value, - pub(crate) evaluator_trace: EvaluatorTrace, -} - -pub(crate) enum PreparedScorer { - LlmJudge(PreparedLlmJudgeScorer), - Remote(PreparedRemoteScorer), -} - -impl PreparedScorer { - pub(crate) async fn from_scorer( - org_id: &str, - scorer: infra::table::scorers::Scorer, - ) -> Result { - match scorer.scorer_type { - ScorerType::LlmJudge => Ok(Self::LlmJudge( - PreparedLlmJudgeScorer::prepare(org_id, &scorer).await?, - )), - ScorerType::Remote => Ok(Self::Remote( - PreparedRemoteScorer::prepare(org_id, &scorer).await?, - )), - } - } - - pub(crate) async fn execute( - &self, - span_ctx: &SpanEvalContext, - input_variables: &HashMap, - ) -> Result> { - match self { - Self::LlmJudge(scorer) => scorer.execute(span_ctx, input_variables).await, - Self::Remote(scorer) => scorer.execute(span_ctx, input_variables).await, - } - } -} - -pub struct PreparedLlmJudgeScorer { - scorer: infra::table::scorers::Scorer, - provider: infra::provider::PreparedProvider, - scorer_cfg: llm_judge::ScorerConfig, - score_cfg_info: llm_judge::ScoreConfigInfo, - extra_metadata_fields: Vec, - include_reasoning: bool, -} - -impl PreparedLlmJudgeScorer { - pub async fn prepare(org_id: &str, scorer: &infra::table::scorers::Scorer) -> Result { - Self::prepare_with_score_config_info(org_id, scorer, None).await - } - - pub async fn prepare_with_score_config_info( - org_id: &str, - scorer: &infra::table::scorers::Scorer, - score_config_info: Option, - ) -> Result { - use llm_judge::{LlmJudgeParams, ScoreConfigInfo, ScorerConfig}; - - let params: LlmJudgeParams = serde_json::from_value(scorer.params.clone()) - .map_err(|e| anyhow::anyhow!("Invalid LLM judge scorer params: {e}"))?; - - let provider_id = params.provider_id.as_str(); - - let provider = match infra::table::providers::get(provider_id).await { - Ok(Some(p)) => p, - Ok(None) => { - anyhow::bail!("Provider '{provider_id}' not found for LLM judge scorer"); - } - Err(e) => { - anyhow::bail!("Failed to look up provider '{provider_id}': {e}"); - } - }; - - let score_config_entity_id = scorer.produces_score_config_id.as_deref().unwrap_or(""); - - let score_cfg_info = if let Some(score_config_info) = score_config_info { - score_config_info - } else if !score_config_entity_id.is_empty() { - lookup_score_config_for_scorer(org_id, scorer) - .await - .as_ref() - .map(ScoreConfigInfo::from) - .unwrap_or_default() - } else { - ScoreConfigInfo::default() - }; - - let template = if scorer.template.trim().is_empty() { - "Evaluate the given content and produce a score.".to_string() - } else { - scorer.template.clone() - }; - - let include_reasoning = params.include_reasoning.unwrap_or(true); - let extra_metadata_fields = params.extra_metadata_fields.clone().unwrap_or_default(); - - let scorer_cfg = ScorerConfig { - scorer_entity_id: scorer.entity_id.clone(), - scorer_version: scorer.version.to_string(), - scorer_type: scorer.scorer_type, - template, - params, - }; - - let provider = infra::provider::PreparedProvider::parse((&provider).into())?; - - Ok(Self { - scorer: scorer.clone(), - provider, - scorer_cfg, - score_cfg_info, - extra_metadata_fields, - include_reasoning, - }) - } - - pub async fn run( - &self, - input_variables: &HashMap, - ) -> Result { - llm_judge::run_llm_judge( - &self.provider, - &self.scorer_cfg, - &self.score_cfg_info, - input_variables, - &self.extra_metadata_fields, - self.include_reasoning, - ) - .await - } - - async fn execute( - &self, - span_ctx: &SpanEvalContext, - input_variables: &HashMap, - ) -> Result> { - let start = std::time::Instant::now(); - - match self.run(input_variables).await { - Ok(llm_output) => { - let latency = start.elapsed(); - - let (value_numeric, value_categorical, value_boolean, data_type) = - match self.score_cfg_info.data_type { - ScoreConfigDataType::Numeric => ( - llm_output.value_numeric, - None, - None, - LlmScoreDataType::Numeric, - ), - ScoreConfigDataType::Categorical => ( - None, - llm_output.value_categorical, - None, - LlmScoreDataType::Categorical, - ), - ScoreConfigDataType::Boolean => ( - None, - None, - llm_output.value_boolean, - LlmScoreDataType::Boolean, - ), - }; - - let score_name = self - .score_cfg_info - .score_config_name - .clone() - .unwrap_or_else(|| self.scorer.name.clone()); - - let score = create_score(ScoreWriterInput { - span_id: span_ctx.span_id.clone(), - trace_id: span_ctx.trace_id.clone(), - session_id: span_ctx.session_id.clone(), - scorer_id: self.scorer_cfg.scorer_entity_id.clone(), - scorer_version: self.scorer_cfg.scorer_version.clone(), - score_config_id: self.score_cfg_info.score_config_entity_id.clone(), - score_config_version: self.score_cfg_info.score_config_version.clone(), - score_name: Some(score_name), - source_type: LlmScoreDataSourceType::LlmJudge, - source_stream: Some(span_ctx.source_stream.clone()), - job_id: span_ctx.job_id.clone(), - value_numeric, - value_categorical, - value_boolean, - data_type, - reasoning: llm_output.reasoning.clone(), - metadata: llm_output.metadata.clone(), - author: None, - }); - - let trace = create_evaluator_trace(EvaluatorTraceInput { - org_id: span_ctx.org_id.clone(), - evaluator_trace_id: span_ctx.evaluator_trace_id.clone(), - target_span_id: span_ctx.span_id.clone(), - target_trace_id: span_ctx.trace_id.clone(), - target_stream: span_ctx.source_stream.clone(), - scorer_id: Some(self.scorer_cfg.scorer_entity_id.clone()), - scorer_version: Some(self.scorer_cfg.scorer_version.clone()), - scorer_type: Some(self.scorer_cfg.scorer_type), - job_id: span_ctx.job_id.clone(), - score_config_id: self.score_cfg_info.score_config_entity_id.clone(), - score_config_version: self.score_cfg_info.score_config_version.clone(), - eval_run_id: span_ctx.eval_run_id.clone(), - provider_id: Some(self.provider.id.clone()), - provider_name: Some(self.provider.name.clone()), - provider_type: Some(self.provider.kind.as_str().to_string()), - model: Some(llm_output.model_used), - latency_ms: latency.as_millis() as i64, - prompt_tokens: llm_output.prompt_tokens, - completion_tokens: llm_output.completion_tokens, - total_tokens: llm_output.total_tokens, - sampling_rate: span_ctx.sampling_rate, - sampled: span_ctx.sampled, - status: config::meta::self_reporting::evaluator::status::SUCCESS.to_string(), - error_kind: None, - error_message: None, - skip_reason: None, - prompt: Some(llm_output.prompt_messages), - response: Some(llm_output.raw_response), - }); - - Ok(Some(ScoreResult { - score_json: score.json, - evaluator_trace: trace, - })) - } - Err(e) => { - let latency = start.elapsed(); - let structured_error = e.chain().find_map(|cause| { - cause.downcast_ref::() - }); - let prompt_tokens = structured_error.and_then(|e| e.prompt_tokens); - let completion_tokens = structured_error.and_then(|e| e.completion_tokens); - let total_tokens = structured_error - .and_then(|e| e.total_tokens) - .or_else(|| prompt_tokens.zip(completion_tokens).map(|(p, c)| p + c)); - - Err(ScorerExecutionError { - message: format!( - "LLM Judge scorer failed: {e} (latency: {}ms)", - latency.as_millis() - ), - raw_response: structured_error.map(|e| e.raw_response.clone()), - scorer_input: Some(llm_judge::build_prompt_messages_json( - &self.scorer_cfg, - input_variables, - )), - provider_id: Some(self.provider.id.clone()), - provider_name: Some(self.provider.name.clone()), - provider_type: Some(self.provider.kind.as_str().to_string()), - model: structured_error.map(|e| e.model_used.clone()).or_else(|| { - Some( - self.provider - .model_or_default(self.scorer_cfg.params.model.as_deref()), - ) - }), - prompt_tokens, - completion_tokens, - total_tokens, - latency_ms: structured_error - .map(|e| e.latency_ms) - .unwrap_or(latency.as_millis() as i64), - } - .into()) - } - } - } -} - -pub struct PreparedRemoteScorer { - scorer: infra::table::scorers::Scorer, - remote_cfg: remote_client::RemoteScorerConfig, - score_config: Option, -} - -impl PreparedRemoteScorer { - pub async fn prepare(org_id: &str, scorer: &infra::table::scorers::Scorer) -> Result { - let mut remote_cfg = remote_client::parse_remote_config(&scorer.params)?; - if !scorer.template.trim().is_empty() { - remote_cfg.request_body_template = Some(scorer.template.clone()); - } - - let score_config = match scorer.produces_score_config_id.as_deref() { - Some(entity_id) if !entity_id.is_empty() => { - lookup_score_config_for_scorer(org_id, scorer).await - } - _ => None, - }; - - Ok(Self { - scorer: scorer.clone(), - remote_cfg, - score_config, - }) - } - - pub async fn run( - &self, - input_variables: &HashMap, - ) -> Result { - remote_client::run_remote_scorer(&self.remote_cfg, input_variables).await - } - - async fn execute( - &self, - span_ctx: &SpanEvalContext, - input_variables: &HashMap, - ) -> Result> { - let start = std::time::Instant::now(); - let input_variables = input_variables_with_context_defaults(input_variables, span_ctx); - - match self.run(&input_variables).await { - Ok(remote_output) => { - let latency = start.elapsed(); - - let value = &remote_output.value; - let (value_numeric, value_categorical, value_boolean) = if value.is_number() { - (value.as_f64(), None, None) - } else if value.is_string() { - (None, value.as_str().map(|s| s.to_string()), None) - } else if value.is_boolean() { - (None, None, value.as_bool()) - } else { - (None, Some(value.to_string()), None) - }; - - let data_type = if value_numeric.is_some() { - LlmScoreDataType::Numeric - } else if value_boolean.is_some() { - LlmScoreDataType::Boolean - } else { - LlmScoreDataType::Categorical - }; - - let score_name = self - .score_config - .as_ref() - .map(|sc| sc.name.clone()) - .unwrap_or_else(|| self.scorer.name.clone()); - - let score = create_score(ScoreWriterInput { - span_id: span_ctx.span_id.clone(), - trace_id: span_ctx.trace_id.clone(), - session_id: span_ctx.session_id.clone(), - scorer_id: self.scorer.entity_id.clone(), - scorer_version: self.scorer.version.to_string(), - score_config_id: self.score_config.as_ref().map(|sc| sc.entity_id.clone()), - score_config_version: self - .score_config - .as_ref() - .map(|sc| sc.version.to_string()), - score_name: Some(score_name), - source_type: LlmScoreDataSourceType::Remote, - source_stream: Some(span_ctx.source_stream.clone()), - job_id: span_ctx.job_id.clone(), - value_numeric, - value_categorical, - value_boolean, - data_type, - reasoning: remote_output.reasoning.clone(), - metadata: remote_output.metadata.clone(), - author: None, - }); - - let trace = create_evaluator_trace(EvaluatorTraceInput { - org_id: span_ctx.org_id.clone(), - evaluator_trace_id: span_ctx.evaluator_trace_id.clone(), - target_span_id: span_ctx.span_id.clone(), - target_trace_id: span_ctx.trace_id.clone(), - target_stream: span_ctx.source_stream.clone(), - scorer_id: Some(self.scorer.entity_id.clone()), - scorer_version: Some(self.scorer.version.to_string()), - scorer_type: Some(self.scorer.scorer_type), - job_id: span_ctx.job_id.clone(), - score_config_id: self.score_config.as_ref().map(|sc| sc.entity_id.clone()), - score_config_version: self - .score_config - .as_ref() - .map(|sc| sc.version.to_string()), - eval_run_id: span_ctx.eval_run_id.clone(), - provider_id: None, - provider_name: None, - provider_type: None, - model: None, - latency_ms: latency.as_millis() as i64, - prompt_tokens: None, - completion_tokens: None, - total_tokens: None, - sampling_rate: span_ctx.sampling_rate, - sampled: span_ctx.sampled, - status: config::meta::self_reporting::evaluator::status::SUCCESS.to_string(), - error_kind: None, - error_message: None, - skip_reason: None, - prompt: None, - response: Some(remote_output.raw_response), - }); - - Ok(Some(ScoreResult { - score_json: score.json, - evaluator_trace: trace, - })) - } - Err(e) => { - let latency = start.elapsed(); - anyhow::bail!( - "Remote scorer failed: {e} (latency: {}ms)", - latency.as_millis() - ); - } - } - } -} - -async fn lookup_score_config_for_scorer( - org_id: &str, - scorer: &infra::table::scorers::Scorer, -) -> Option { - let entity_id = scorer.produces_score_config_id.as_deref()?; - match scorer.produces_score_config_version { - Some(version) => { - infra::table::score_configs::get_by_entity_id_and_version(org_id, entity_id, version) - .await - .ok() - .flatten() - } - None => infra::table::score_configs::get_by_entity_id(org_id, entity_id) - .await - .ok() - .flatten(), - } -} diff --git a/src/service/llm_evaluations/score_configs.rs b/src/service/llm_evaluations/score_configs.rs deleted file mode 100644 index 12a90fa0c7..0000000000 --- a/src/service/llm_evaluations/score_configs.rs +++ /dev/null @@ -1,250 +0,0 @@ -// Copyright 2026 OpenObserve Inc. -// -// This program is free software: you can redistribute it and/or modify -// it under the terms of the GNU Affero General Public License as published by -// the Free Software Foundation, either version 3 of the License, or -// (at your option) any later version. -// -// This program is distributed in the hope that it will be useful -// but WITHOUT ANY WARRANTY; without even the implied warranty of -// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -// GNU Affero General Public License for more details. -// -// You should have received a copy of the GNU Affero General Public License -// along with this program. If not, see . - -use chrono::Utc; -use config::ider; -use infra::table; - -use crate::common::{ - meta::authz::Authz, - utils::auth::{is_ofga_object_visible, remove_ownership, set_ownership}, -}; - -/// Errors that can occur when interacting with score configs. -#[derive(Debug, thiserror::Error)] -pub enum ScoreConfigError { - #[error("InfraError# Internal error")] - InfraError(#[from] infra::errors::Error), - - #[error("Score config name cannot be empty")] - MissingName, - - #[error("Score config not found")] - NotFound, - - #[error("Score config name already exists")] - DuplicateName, - - #[error("Score config cannot be deleted because it is used by one or more scorers")] - InUseByScorer, -} - -#[tracing::instrument(skip(config))] -pub async fn save_score_config( - org_id: &str, - mut config: table::score_configs::ScoreConfig, -) -> Result { - config.name = config.name.trim().to_string(); - if config.name.is_empty() { - return Err(ScoreConfigError::MissingName); - } - - if table::score_configs::get_by_active_name(org_id, &config.name) - .await? - .is_some() - { - return Err(ScoreConfigError::DuplicateName); - } - - config.id = ider::generate(); - config.entity_id = ider::generate(); - config.org_id = org_id.to_string(); - config.version = 1; - let now = Utc::now().timestamp_millis(); - config.created_at = now; - config.updated_at = now; - - table::score_configs::add(&config).await?; - set_ownership(org_id, "score_configs", Authz::new(&config.entity_id)).await; - publish_score_config_put(&config).await; - Ok(config) -} - -#[tracing::instrument(skip(config))] -pub async fn update_score_config( - org_id: &str, - entity_id: &str, - mut config: table::score_configs::ScoreConfig, -) -> Result { - // Get existing latest - let existing = table::score_configs::get_by_entity_id(org_id, entity_id) - .await? - .ok_or(ScoreConfigError::NotFound)?; - - let new_name = config.name.trim().to_string(); - let new_name = if new_name.is_empty() { - existing.name.clone() - } else { - new_name - }; - - if new_name != existing.name - && table::score_configs::get_by_active_name(org_id, &new_name) - .await? - .is_some() - { - return Err(ScoreConfigError::DuplicateName); - } - - let latest_version = table::score_configs::get_latest_version(org_id, entity_id).await?; - - config.id = ider::generate(); - config.entity_id = existing.entity_id; - config.org_id = org_id.to_string(); - config.version = latest_version + 1; - let now = Utc::now().timestamp_millis(); - config.created_at = now; - config.updated_at = now; - config.is_active = true; - - config.name = new_name; - // data_type is immutable - preserve from existing - config.data_type = existing.data_type; - - table::score_configs::delete(entity_id, org_id).await?; - table::score_configs::add(&config).await?; - publish_score_config_put(&config).await; - Ok(config) -} - -#[tracing::instrument()] -pub async fn list_score_configs( - org_id: &str, - permitted_objects: Option>, -) -> Result, ScoreConfigError> { - let configs = table::score_configs::get_all_by_org(org_id).await?; - Ok(configs - .into_iter() - .filter(|config| { - is_ofga_object_visible( - org_id, - "score_config", - &config.entity_id, - permitted_objects.as_deref(), - ) - }) - .collect()) -} - -#[tracing::instrument()] -pub async fn get_score_config( - org_id: &str, - entity_id: &str, -) -> Result { - let config = table::score_configs::get_by_entity_id(org_id, entity_id) - .await? - .ok_or(ScoreConfigError::NotFound)?; - Ok(config) -} - -#[tracing::instrument()] -pub async fn get_score_config_versions( - org_id: &str, - entity_id: &str, -) -> Result, ScoreConfigError> { - let versions = table::score_configs::get_versions(org_id, entity_id).await?; - Ok(versions) -} - -#[tracing::instrument()] -pub async fn delete_score_config(org_id: &str, entity_id: &str) -> Result<(), ScoreConfigError> { - table::score_configs::get_by_entity_id(org_id, entity_id) - .await? - .ok_or(ScoreConfigError::NotFound)?; - if table::scorers::has_active_by_score_config_id(org_id, entity_id).await? { - return Err(ScoreConfigError::InUseByScorer); - } - table::score_configs::delete(entity_id, org_id).await?; - remove_ownership(org_id, "score_configs", Authz::new(entity_id)).await; - publish_score_config_delete(org_id, entity_id).await; - Ok(()) -} - -#[cfg(feature = "enterprise")] -async fn publish_score_config_put(config: &table::score_configs::ScoreConfig) { - if !o2_enterprise::enterprise::common::config::get_config() - .super_cluster - .enabled - { - return; - } - - let key = format!("/eval/score_configs/{}/{}", config.org_id, config.entity_id); - let message = o2_enterprise::enterprise::super_cluster::queue::EvalScoreConfigMessage::Put { - config: config.clone(), - }; - match config::utils::json::to_vec(&message) { - Ok(value) => { - if let Err(e) = o2_enterprise::enterprise::super_cluster::queue::eval_score_config_put( - &key, - value.into(), - ) - .await - { - log::error!("[ScoreConfig] error publishing super cluster score config put: {e}"); - } - } - Err(e) => { - log::error!("[ScoreConfig] error serializing score config super cluster message: {e}") - } - } -} - -#[cfg(not(feature = "enterprise"))] -async fn publish_score_config_put(_config: &table::score_configs::ScoreConfig) {} - -#[cfg(feature = "enterprise")] -async fn publish_score_config_delete(org_id: &str, entity_id: &str) { - if !o2_enterprise::enterprise::common::config::get_config() - .super_cluster - .enabled - { - return; - } - - let key = format!("/eval/score_configs/{}/{}", org_id, entity_id); - if let Err(e) = - o2_enterprise::enterprise::super_cluster::queue::eval_score_config_delete(&key).await - { - log::error!("[ScoreConfig] error publishing super cluster score config delete: {e}"); - } -} - -#[cfg(not(feature = "enterprise"))] -async fn publish_score_config_delete(_org_id: &str, _entity_id: &str) {} - -#[cfg(test)] -mod tests { - use super::*; - - #[test] - fn test_score_config_error_variants() { - let infra_error = - ScoreConfigError::InfraError(infra::errors::Error::Message("test".to_string())); - assert!(matches!(infra_error, ScoreConfigError::InfraError(_))); - - let missing_name = ScoreConfigError::MissingName; - assert!(matches!(missing_name, ScoreConfigError::MissingName)); - - let not_found = ScoreConfigError::NotFound; - assert!(matches!(not_found, ScoreConfigError::NotFound)); - - let duplicate_name = ScoreConfigError::DuplicateName; - assert!(matches!(duplicate_name, ScoreConfigError::DuplicateName)); - - let in_use_by_scorer = ScoreConfigError::InUseByScorer; - assert!(matches!(in_use_by_scorer, ScoreConfigError::InUseByScorer)); - } -} diff --git a/src/service/llm_evaluations/score_writer.rs b/src/service/llm_evaluations/score_writer.rs deleted file mode 100644 index bc6fc89920..0000000000 --- a/src/service/llm_evaluations/score_writer.rs +++ /dev/null @@ -1,242 +0,0 @@ -// Copyright 2026 OpenObserve Inc. -// -// This program is free software: you can redistribute it and/or modify -// it under the terms of the GNU Affero General Public License as published by -// the Free Software Foundation, either version 3 of the License, or -// (at your option) any later version. -// -// This program is distributed in the hope that it will be useful -// but WITHOUT ANY WARRANTY; without even the implied warranty of -// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -// GNU Affero General Public License for more details. -// -// You should have received a copy of the GNU Affero General Public License -// along with this program. If not, see . - -//! Score Writer — writes evaluation results to the `_llm_scores` system stream. -//! -//! Produces `LlmScoreRecord` values from scorer execution outputs and -//! serializes them for ingestion through the pipeline's output node. - -use chrono::Utc; -use config::{ - ider, - meta::self_reporting::llm_scores::{ - LlmScoreDataLevel, LlmScoreDataSourceType, LlmScoreDataType, LlmScoreRecord, - }, -}; - -#[derive(Debug)] -pub struct ScoreWriterInput { - pub span_id: String, - pub trace_id: String, - pub session_id: Option, - pub scorer_id: String, - pub scorer_version: String, - pub score_config_id: Option, - pub score_config_version: Option, - pub score_name: Option, - pub source_type: LlmScoreDataSourceType, - pub source_stream: Option, - pub job_id: Option, - pub value_numeric: Option, - pub value_categorical: Option, - pub value_boolean: Option, - pub data_type: LlmScoreDataType, - pub reasoning: Option, - pub metadata: Option, - pub author: Option, -} - -pub struct WrittenScore { - pub record: LlmScoreRecord, - pub json: serde_json::Value, -} - -pub fn create_score(input: ScoreWriterInput) -> WrittenScore { - let now = Utc::now().timestamp_micros(); - let id = ider::generate(); - - let level = if input.span_id.is_empty() { - LlmScoreDataLevel::Trace - } else { - LlmScoreDataLevel::Span - }; - - let record = LlmScoreRecord { - id, - span_id: if input.span_id.is_empty() { - None - } else { - Some(input.span_id) - }, - trace_id: if input.trace_id.is_empty() { - None - } else { - Some(input.trace_id) - }, - session_id: input.session_id, - experiment_id: None, - level, - name: input.score_name.unwrap_or_else(|| "unknown".to_string()), - value_numeric: input.value_numeric, - value_categorical: input.value_categorical, - value_boolean: input.value_boolean, - data_type: input.data_type, - scorer_id: Some(input.scorer_id), - scorer_version: Some(input.scorer_version), - score_config_id: input.score_config_id, - score_config_version: input.score_config_version, - source_type: input.source_type, - source_stream: input.source_stream, - job_id: input.job_id, - reasoning: input.reasoning, - metadata: input.metadata, - author: input.author, - _timestamp: now, - }; - - let json = serde_json::to_value(&record) - .unwrap_or_else(|_| serde_json::json!({"error": "serialization_failed"})); - - WrittenScore { record, json } -} - -#[cfg(test)] -mod tests { - use super::*; - - #[test] - fn test_create_numeric_score() { - let input = ScoreWriterInput { - span_id: "span-1".to_string(), - trace_id: "trace-1".to_string(), - session_id: None, - scorer_id: "scorer-entity-1".to_string(), - scorer_version: "1".to_string(), - score_config_id: Some("cfg-entity-1".to_string()), - score_config_version: Some("1".to_string()), - score_name: Some("faithfulness".to_string()), - source_type: LlmScoreDataSourceType::LlmJudge, - source_stream: Some("traces".to_string()), - job_id: Some("job-1".to_string()), - value_numeric: Some(0.95), - value_categorical: None, - value_boolean: None, - data_type: LlmScoreDataType::Numeric, - reasoning: None, - metadata: None, - author: None, - }; - - let result = create_score(input); - assert_eq!(result.record.level, LlmScoreDataLevel::Span); - assert_eq!(result.record.name, "faithfulness"); - assert_eq!(result.record.value_numeric, Some(0.95)); - assert_eq!(result.record.source_type, LlmScoreDataSourceType::LlmJudge); - assert_eq!(result.record.scorer_id, Some("scorer-entity-1".to_string())); - assert_eq!(result.record.job_id, Some("job-1".to_string())); - assert_eq!( - result.record.score_config_id, - Some("cfg-entity-1".to_string()) - ); - - let json = &result.json; - assert_eq!(json["name"], "faithfulness"); - assert_eq!(json["level"], "span"); - assert_eq!(json["scorer_id"], "scorer-entity-1"); - assert_eq!(json["source_type"], "llm_judge"); - assert_eq!(json["score_config_id"], "cfg-entity-1"); - assert!(json.get("score_config_entity_id").is_none()); - } - - #[test] - fn test_create_categorical_score() { - let input = ScoreWriterInput { - span_id: "span-2".to_string(), - trace_id: "trace-2".to_string(), - session_id: None, - scorer_id: "sc-2".to_string(), - scorer_version: "2".to_string(), - score_config_id: None, - score_config_version: None, - score_name: Some("quality".to_string()), - source_type: LlmScoreDataSourceType::Remote, - source_stream: None, - job_id: None, - value_numeric: None, - value_categorical: Some("excellent".to_string()), - value_boolean: None, - data_type: LlmScoreDataType::Categorical, - reasoning: Some("good job".to_string()), - metadata: None, - author: None, - }; - - let result = create_score(input); - assert_eq!(result.record.data_type, LlmScoreDataType::Categorical); - assert_eq!( - result.record.value_categorical, - Some("excellent".to_string()) - ); - assert_eq!(result.record.source_type, LlmScoreDataSourceType::Remote); - assert_eq!(result.record.reasoning, Some("good job".to_string())); - } - - #[test] - fn test_create_boolean_score() { - let input = ScoreWriterInput { - span_id: String::new(), - trace_id: "trace-3".to_string(), - session_id: None, - scorer_id: "sc-3".to_string(), - scorer_version: "3".to_string(), - score_config_id: None, - score_config_version: None, - score_name: Some("pass_fail".to_string()), - source_type: LlmScoreDataSourceType::LlmJudge, - source_stream: None, - job_id: None, - value_numeric: None, - value_categorical: None, - value_boolean: Some(true), - data_type: LlmScoreDataType::Boolean, - reasoning: None, - metadata: None, - author: Some("admin".to_string()), - }; - - let result = create_score(input); - assert_eq!(result.record.level, LlmScoreDataLevel::Trace); - assert_eq!(result.record.value_boolean, Some(true)); - assert_eq!(result.record.author, Some("admin".to_string())); - } - - #[test] - fn test_create_score_with_metadata() { - let input = ScoreWriterInput { - span_id: "span-4".to_string(), - trace_id: "trace-4".to_string(), - session_id: Some("session-1".to_string()), - scorer_id: "sc-4".to_string(), - scorer_version: "4".to_string(), - score_config_id: None, - score_config_version: None, - score_name: Some("custom".to_string()), - source_type: LlmScoreDataSourceType::LlmJudge, - source_stream: Some("traces".to_string()), - job_id: None, - value_numeric: Some(0.5), - value_categorical: None, - value_boolean: None, - data_type: LlmScoreDataType::Numeric, - reasoning: None, - metadata: Some(serde_json::json!({"extra": "info"})), - author: None, - }; - - let result = create_score(input); - assert_eq!(result.record.session_id, Some("session-1".to_string())); - assert!(result.record.metadata.is_some()); - } -} diff --git a/src/service/llm_evaluations/scorers/llm_judge.rs b/src/service/llm_evaluations/scorers/llm_judge.rs deleted file mode 100644 index 4fb5af9333..0000000000 --- a/src/service/llm_evaluations/scorers/llm_judge.rs +++ /dev/null @@ -1,287 +0,0 @@ -// Copyright 2026 OpenObserve Inc. -// -// This program is free software: you can redistribute it and/or modify -// it under the terms of the GNU Affero General Public License as published by -// the Free Software Foundation, either version 3 of the License, or -// (at your option) any later version. -// -// This program is distributed in the hope that it will be useful -// but WITHOUT ANY WARRANTY; without even the implied warranty of -// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -// GNU Affero General Public License for more details. -// -// You should have received a copy of the GNU Affero General Public License -// along with this program. If not, see . - -//! LLM Judge scorer execution. - -use std::{collections::HashMap, time::Instant}; - -use anyhow::Result; -use infra::{ - provider::{PreparedProvider, ProviderRunParams}, - table::{score_configs::ScoreConfigDataType, scorers::ScorerType}, -}; -use serde::{Deserialize, Serialize}; -use serde_json::Value; - -use super::schema_derivation::{ExtraMetadataField, REASONING_FIELD, derive_output_schema}; - -#[derive(Debug, Clone, Serialize, Deserialize)] -pub struct LlmJudgeOutput { - pub value_numeric: Option, - pub value_categorical: Option, - pub value_boolean: Option, - pub reasoning: Option, - pub metadata: Option, - pub raw_response: String, - pub prompt_messages: String, - pub prompt_tokens: Option, - pub completion_tokens: Option, - pub total_tokens: Option, - pub model_used: String, - pub latency_ms: i64, -} - -#[derive(Debug, Clone)] -pub struct ScorerConfig { - pub scorer_entity_id: String, - pub scorer_version: String, - pub scorer_type: ScorerType, - pub template: String, - pub params: LlmJudgeParams, -} - -#[derive(Debug, Clone, Serialize, Deserialize, Default)] -pub struct LlmJudgeParams { - pub provider_id: String, - pub model: Option, - pub temperature: Option, - pub max_tokens: Option, - pub timeout_ms: Option, - pub output_parsing: Option, - pub include_reasoning: Option, - pub extra_metadata_fields: Option>, -} - -#[derive(Debug, Clone, Default)] -pub struct ScoreConfigInfo { - pub score_config_entity_id: Option, - pub score_config_version: Option, - pub score_config_name: Option, - pub data_type: ScoreConfigDataType, - pub numeric_range: Option, - pub categories: Option, -} - -impl From<&infra::table::score_configs::ScoreConfig> for ScoreConfigInfo { - fn from(score_config: &infra::table::score_configs::ScoreConfig) -> Self { - Self { - score_config_entity_id: Some(score_config.entity_id.clone()), - score_config_version: Some(score_config.version.to_string()), - score_config_name: Some(score_config.name.clone()), - data_type: score_config.data_type, - numeric_range: score_config.numeric_range.clone(), - categories: score_config.categories.clone(), - } - } -} - -pub async fn run_llm_judge( - provider: &PreparedProvider, - scorer: &ScorerConfig, - score_config: &ScoreConfigInfo, - input_variables: &HashMap, - extra_metadata_fields: &[ExtraMetadataField], - include_reasoning: bool, -) -> Result { - let system_prompt = evaluator_system_prompt(); - let user_prompt = render_template(&scorer.template, input_variables); - let prompt_messages = prompt_messages_json(&system_prompt, &user_prompt); - let output_schema = derive_output_schema( - score_config.data_type, - score_config.numeric_range.as_ref(), - score_config.categories.as_ref(), - extra_metadata_fields, - include_reasoning, - )?; - - let call_params = ProviderRunParams { - model: scorer.params.model.clone(), - system_prompt, - user_prompt, - temperature: scorer.params.temperature.unwrap_or(0.0), - max_tokens: scorer.params.max_tokens.unwrap_or(256), - response_schema: output_schema.json_schema, - timeout_ms: scorer.params.timeout_ms.unwrap_or(30_000), - }; - - let start = Instant::now(); - let result = provider.run(&call_params).await?; - let latency = start.elapsed(); - - let metadata = extract_metadata(&result.value, &output_schema.metadata_fields); - - let reasoning = result - .value - .get(REASONING_FIELD) - .and_then(|v| v.as_str()) - .map(|s| s.to_string()); - - let value_field = &output_schema.value_field; - let (value_numeric, value_categorical, value_boolean) = - extract_score_value(&result.value, score_config.data_type, value_field); - - Ok(LlmJudgeOutput { - value_numeric, - value_categorical, - value_boolean, - reasoning, - metadata, - raw_response: result.raw_response, - prompt_messages, - prompt_tokens: result.prompt_tokens, - completion_tokens: result.completion_tokens, - total_tokens: result.total_tokens, - model_used: result.model_used, - latency_ms: latency.as_millis() as i64, - }) -} - -pub(crate) fn build_prompt_messages_json( - scorer: &ScorerConfig, - input_variables: &HashMap, -) -> String { - let system_prompt = evaluator_system_prompt(); - let user_prompt = render_template(&scorer.template, input_variables); - prompt_messages_json(&system_prompt, &user_prompt) -} - -fn evaluator_system_prompt() -> String { - "You are an impartial LLM evaluator. Follow the instructions and output ONLY valid JSON." - .to_string() -} - -fn prompt_messages_json(system_prompt: &str, user_prompt: &str) -> String { - serde_json::json!([ - { "role": "system", "content": system_prompt }, - { "role": "user", "content": user_prompt } - ]) - .to_string() -} - -fn extract_metadata(value: &Value, metadata_fields: &[String]) -> Option { - if metadata_fields.is_empty() { - return None; - } - - let mut meta = serde_json::json!({}); - for field in metadata_fields { - if let Some(val) = value.get(field) { - meta[field] = val.clone(); - } - } - if meta.as_object().map(|o| o.is_empty()).unwrap_or(true) { - None - } else { - Some(meta) - } -} - -fn extract_score_value( - value: &Value, - data_type: ScoreConfigDataType, - value_field: &str, -) -> (Option, Option, Option) { - match data_type { - ScoreConfigDataType::Numeric => { - (value.get(value_field).and_then(|v| v.as_f64()), None, None) - } - ScoreConfigDataType::Categorical => ( - None, - value - .get(value_field) - .and_then(|v| v.as_str()) - .map(|s| s.to_string()), - None, - ), - ScoreConfigDataType::Boolean => { - (None, None, value.get(value_field).and_then(|v| v.as_bool())) - } - } -} - -pub(super) fn render_template(template: &str, input_variables: &HashMap) -> String { - let mut result = template.to_string(); - let mut start = 0; - while let Some(open) = result[start..].find("{{") { - let abs_open = start + open; - if let Some(close) = result[abs_open + 2..].find("}}") { - let abs_close = abs_open + 2 + close; - let variable = result[abs_open + 2..abs_close].trim(); - let value = get_nested_attr(input_variables, variable); - result.replace_range(abs_open..abs_close + 2, &value); - start = abs_open + value.len(); - } else { - break; - } - } - result -} - -pub(super) fn get_nested_attr(attrs: &HashMap, path: &str) -> String { - let parts: Vec<&str> = path.split('.').collect(); - let mut current = attrs.get(parts[0]); - for part in &parts[1..] { - current = current.and_then(|v| v.get(part)); - } - match current { - Some(Value::String(s)) => s.clone(), - Some(v) => v.to_string(), - None => attrs - .get(path) - .map(|v| match v { - Value::String(s) => s.clone(), - v => v.to_string(), - }) - .unwrap_or_default(), - } -} - -#[cfg(test)] -mod tests { - use super::*; - - #[test] - fn extracts_numeric_score_from_fixed_score_field() { - let value = serde_json::json!({ - "score": 0.75, - "reasoning": "grounded", - "failure_mode": "none" - }); - - let (numeric, categorical, boolean) = - extract_score_value(&value, ScoreConfigDataType::Numeric, "score"); - let metadata = extract_metadata(&value, &["failure_mode".to_string()]); - - assert_eq!(numeric, Some(0.75)); - assert_eq!(categorical, None); - assert_eq!(boolean, None); - assert_eq!(metadata, Some(serde_json::json!({"failure_mode": "none"}))); - } - - #[test] - fn extracts_categorical_and_boolean_scores_from_fixed_score_field() { - let categorical = serde_json::json!({"score": "high"}); - let boolean = serde_json::json!({"score": true}); - - assert_eq!( - extract_score_value(&categorical, ScoreConfigDataType::Categorical, "score"), - (None, Some("high".to_string()), None) - ); - assert_eq!( - extract_score_value(&boolean, ScoreConfigDataType::Boolean, "score"), - (None, None, Some(true)) - ); - } -} diff --git a/src/service/llm_evaluations/scorers/mod.rs b/src/service/llm_evaluations/scorers/mod.rs deleted file mode 100644 index 32aa5c6d9c..0000000000 --- a/src/service/llm_evaluations/scorers/mod.rs +++ /dev/null @@ -1,19 +0,0 @@ -// Copyright 2026 OpenObserve Inc. -// -// This program is free software: you can redistribute it and/or modify -// it under the terms of the GNU Affero General Public License as published by -// the Free Software Foundation, either version 3 of the License, or -// (at your option) any later version. -// -// This program is distributed in the hope that it will be useful -// but WITHOUT ANY WARRANTY; without even the implied warranty of -// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -// GNU Affero General Public License for more details. -// -// You should have received a copy of the GNU Affero General Public License -// along with this program. If not, see . - -pub mod llm_judge; -pub mod registry; -pub mod remote_client; -pub mod schema_derivation; diff --git a/src/service/llm_evaluations/scorers/registry.rs b/src/service/llm_evaluations/scorers/registry.rs deleted file mode 100644 index f710184248..0000000000 --- a/src/service/llm_evaluations/scorers/registry.rs +++ /dev/null @@ -1,304 +0,0 @@ -// Copyright 2026 OpenObserve Inc. -// -// This program is free software: you can redistribute it and/or modify -// it under the terms of the GNU Affero General Public License as published by -// the Free Software Foundation, either version 3 of the License, or -// (at your option) any later version. -// -// This program is distributed in the hope that it will be useful -// but WITHOUT ANY WARRANTY; without even the implied warranty of -// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -// GNU Affero General Public License for more details. -// -// You should have received a copy of the GNU Affero General Public License -// along with this program. If not, see . - -use chrono::Utc; -use config::ider; -use infra::table::{self, scorers::ScorerType}; - -use crate::common::{ - meta::authz::Authz, - utils::auth::{is_ofga_object_visible, remove_ownership, set_ownership}, -}; - -/// Errors that can occur when interacting with scorers. -#[derive(Debug, thiserror::Error)] -pub enum ScorerError { - #[error("InfraError# Internal error")] - InfraError(#[from] infra::errors::Error), - - #[error("Scorer name cannot be empty")] - MissingName, - - #[error("Scorer not found")] - NotFound, - - #[error("Invalid scorer_type '{0}'. Must be one of: llm_judge, remote")] - InvalidScorerType(String), - - #[error("scorer_type is immutable and cannot be changed on update")] - ScorerTypeImmutable, - - #[error("produces_score is immutable and cannot be changed on update")] - ProducesScoreConfigIdImmutable, - - #[error("Score config version not found")] - ScoreConfigVersionNotFound, - - #[error("Invalid LLM Judge output schema: {0}")] - InvalidOutputSchema(String), - - #[error("Scorer name already exists")] - DuplicateName, - - #[error("Scorer cannot be deleted because it is used by one or more eval jobs")] - InUseByEvalJob, -} - -#[tracing::instrument(skip(scorer))] -pub async fn save_scorer( - org_id: &str, - mut scorer: table::scorers::Scorer, -) -> Result { - scorer.name = scorer.name.trim().to_string(); - if scorer.name.is_empty() { - return Err(ScorerError::MissingName); - } - - if table::scorers::get_by_active_name(org_id, &scorer.name) - .await? - .is_some() - { - return Err(ScorerError::DuplicateName); - } - - scorer.id = ider::generate(); - scorer.entity_id = ider::generate(); - scorer.org_id = org_id.to_string(); - scorer.version = 1; - scorer.produces_score_config_version = resolve_score_config_version(org_id, &scorer).await?; - let now = Utc::now().timestamp_millis(); - scorer.created_at = now; - scorer.updated_at = now; - scorer.is_active = true; - - table::scorers::add(&scorer).await?; - set_ownership(org_id, "scorers", Authz::new(&scorer.entity_id)).await; - publish_scorer_put(&scorer).await; - Ok(scorer) -} - -#[tracing::instrument(skip(scorer))] -pub async fn update_scorer( - org_id: &str, - entity_id: &str, - mut scorer: table::scorers::Scorer, -) -> Result { - let existing = table::scorers::get_by_entity_id(org_id, entity_id) - .await? - .ok_or(ScorerError::NotFound)?; - - // scorer_type is immutable; produced score config can change with a new scorer version. - if scorer.scorer_type != existing.scorer_type { - return Err(ScorerError::ScorerTypeImmutable); - } - let produces_score_config_id = scorer - .produces_score_config_id - .clone() - .or_else(|| existing.produces_score_config_id.clone()); - - let new_name = scorer.name.trim().to_string(); - let new_name = if new_name.is_empty() { - existing.name.clone() - } else { - new_name - }; - - if new_name != existing.name - && table::scorers::get_by_active_name(org_id, &new_name) - .await? - .is_some() - { - return Err(ScorerError::DuplicateName); - } - - let latest_version = table::scorers::get_latest_version(org_id, entity_id).await?; - - scorer.id = ider::generate(); - scorer.entity_id = existing.entity_id; - scorer.org_id = org_id.to_string(); - scorer.name = new_name; - scorer.scorer_type = existing.scorer_type; - scorer.produces_score_config_id = produces_score_config_id; - scorer.produces_score_config_version = resolve_score_config_version(org_id, &scorer).await?; - scorer.version = latest_version + 1; - let now = Utc::now().timestamp_millis(); - scorer.created_at = now; - scorer.updated_at = now; - scorer.is_active = true; - - table::scorers::delete(entity_id, org_id).await?; - table::scorers::add(&scorer).await?; - publish_scorer_put(&scorer).await; - Ok(scorer) -} - -async fn resolve_score_config_version( - org_id: &str, - scorer: &table::scorers::Scorer, -) -> Result, ScorerError> { - match scorer.produces_score_config_id.as_deref() { - Some(entity_id) if !entity_id.is_empty() => { - if let Some(version) = scorer.produces_score_config_version { - let score_config = - table::score_configs::get_by_entity_id_and_version(org_id, entity_id, version) - .await?; - return match score_config { - Some(_) => Ok(Some(version)), - None => Err(ScorerError::ScoreConfigVersionNotFound), - }; - } - table::score_configs::get_by_entity_id(org_id, entity_id) - .await? - .map(|score_config| Some(score_config.version)) - .ok_or(ScorerError::ScoreConfigVersionNotFound) - } - _ => Ok(None), - } -} - -#[tracing::instrument()] -pub async fn list_scorers( - org_id: &str, - scorer_type: Option<&ScorerType>, - permitted_objects: Option>, -) -> Result, ScorerError> { - let scorers = match scorer_type { - Some(t) => table::scorers::get_by_type(org_id, t).await?, - None => table::scorers::get_all_by_org(org_id).await?, - }; - Ok(scorers - .into_iter() - .filter(|scorer| { - is_ofga_object_visible( - org_id, - "scorer", - &scorer.entity_id, - permitted_objects.as_deref(), - ) - }) - .collect()) -} - -#[tracing::instrument()] -pub async fn get_scorer( - org_id: &str, - entity_id: &str, -) -> Result { - table::scorers::get_by_entity_id(org_id, entity_id) - .await? - .ok_or(ScorerError::NotFound) -} - -#[tracing::instrument()] -pub async fn get_scorer_versions( - org_id: &str, - entity_id: &str, -) -> Result, ScorerError> { - let versions = table::scorers::get_versions(org_id, entity_id).await?; - Ok(versions) -} - -#[tracing::instrument()] -pub async fn delete_scorer(org_id: &str, entity_id: &str) -> Result<(), ScorerError> { - table::scorers::get_by_entity_id(org_id, entity_id) - .await? - .ok_or(ScorerError::NotFound)?; - if table::online_eval_jobs::has_non_archived_by_scorer_ref(org_id, entity_id).await? { - return Err(ScorerError::InUseByEvalJob); - } - table::scorers::delete(entity_id, org_id).await?; - remove_ownership(org_id, "scorers", Authz::new(entity_id)).await; - publish_scorer_delete(org_id, entity_id).await; - Ok(()) -} - -#[cfg(feature = "enterprise")] -async fn publish_scorer_put(scorer: &table::scorers::Scorer) { - if !o2_enterprise::enterprise::common::config::get_config() - .super_cluster - .enabled - { - return; - } - - let key = format!("/eval/scorers/{}/{}", scorer.org_id, scorer.entity_id); - let message = o2_enterprise::enterprise::super_cluster::queue::EvalScorerMessage::Put { - scorer: scorer.clone(), - }; - match config::utils::json::to_vec(&message) { - Ok(value) => { - if let Err(e) = - o2_enterprise::enterprise::super_cluster::queue::eval_scorer_put(&key, value.into()) - .await - { - log::error!("[Scorer] error publishing super cluster scorer put: {e}"); - } - } - Err(e) => log::error!("[Scorer] error serializing scorer super cluster message: {e}"), - } -} - -#[cfg(not(feature = "enterprise"))] -async fn publish_scorer_put(_scorer: &table::scorers::Scorer) {} - -#[cfg(feature = "enterprise")] -async fn publish_scorer_delete(org_id: &str, entity_id: &str) { - if !o2_enterprise::enterprise::common::config::get_config() - .super_cluster - .enabled - { - return; - } - - let key = format!("/eval/scorers/{}/{}", org_id, entity_id); - if let Err(e) = o2_enterprise::enterprise::super_cluster::queue::eval_scorer_delete(&key).await - { - log::error!("[Scorer] error publishing super cluster scorer delete: {e}"); - } -} - -#[cfg(not(feature = "enterprise"))] -async fn publish_scorer_delete(_org_id: &str, _entity_id: &str) {} - -#[cfg(test)] -mod tests { - use super::*; - - #[test] - fn test_scorer_error_variants() { - let cases = vec![ - ScorerError::MissingName, - ScorerError::NotFound, - ScorerError::InvalidScorerType("bad".to_string()), - ScorerError::ScorerTypeImmutable, - ScorerError::ProducesScoreConfigIdImmutable, - ScorerError::InvalidOutputSchema("bad".to_string()), - ScorerError::DuplicateName, - ScorerError::InUseByEvalJob, - ]; - for c in cases { - // each error should produce a non-empty display message - assert!(!c.to_string().is_empty()); - } - } - - #[test] - fn test_scorer_type_parse() { - assert_eq!("llm_judge".parse::(), Ok(ScorerType::LlmJudge)); - assert_eq!("remote".parse::(), Ok(ScorerType::Remote)); - assert!("code".parse::().is_err()); - assert!("".parse::().is_err()); - } -} diff --git a/src/service/llm_evaluations/scorers/remote_client.rs b/src/service/llm_evaluations/scorers/remote_client.rs deleted file mode 100644 index a7e0ac1035..0000000000 --- a/src/service/llm_evaluations/scorers/remote_client.rs +++ /dev/null @@ -1,424 +0,0 @@ -// Copyright 2026 OpenObserve Inc. -// -// This program is free software: you can redistribute it and/or modify -// it under the terms of the GNU Affero General Public License as published by -// the Free Software Foundation, either version 3 of the License, or -// (at your option) any later version. -// -// This program is distributed in the hope that it will be useful -// but WITHOUT ANY WARRANTY; without even the implied warranty of -// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -// GNU Affero General Public License for more details. -// -// You should have received a copy of the GNU Affero General Public License -// along with this program. If not, see . - -//! Remote scorer HTTP client. - -use std::{collections::HashMap, time::Duration}; - -use anyhow::Result; -use rand::RngExt; -use serde::{Deserialize, Serialize}; -use serde_json::{Map, Value}; - -#[derive(Debug, Clone, Serialize, Deserialize)] -pub struct RemoteScorerOutput { - pub value: Value, - pub reasoning: Option, - pub metadata: Option, - pub raw_response: String, - pub latency_ms: i64, -} - -#[derive(Debug, Clone)] -pub struct RemoteScorerConfig { - pub endpoint: String, - pub http_method: String, - pub auth_type: String, - pub auth_token: Option, - pub auth_username: Option, - pub auth_password: Option, - pub api_key_header: Option, - pub custom_headers: Vec<(String, String)>, - pub content_type: String, - pub request_body_template: Option, - pub timeout_ms: u64, - pub max_retries: u32, -} - -#[derive(Debug, Clone, Serialize, Deserialize)] -#[serde(rename_all = "snake_case", deny_unknown_fields)] -pub struct RemoteScorerParams { - pub endpoint: String, - pub http_method: Option, - pub auth: Option, - pub custom_headers: Option>, - pub content_type: Option, - pub timeout_ms: Option, - pub max_retries: Option, -} - -#[derive(Debug, Clone, Serialize, Deserialize)] -#[serde(tag = "type", rename_all = "snake_case", deny_unknown_fields)] -pub enum RemoteScorerAuth { - None, - Bearer { token: String }, - Basic { username: String, password: String }, - ApiKey { token: String, header_name: String }, -} - -#[derive(Debug, Clone, Serialize, Deserialize)] -#[serde(deny_unknown_fields)] -pub struct RemoteScorerHeader { - pub key: String, - pub value: String, -} - -pub fn parse_remote_config(params: &Value) -> Result { - let params: RemoteScorerParams = serde_json::from_value(params.clone()) - .map_err(|e| anyhow::anyhow!("Invalid remote scorer params: {e}"))?; - - let (auth_type, auth_token, auth_username, auth_password, api_key_header) = match params.auth { - Some(RemoteScorerAuth::Bearer { token }) => { - ("bearer".to_string(), Some(token), None, None, None) - } - Some(RemoteScorerAuth::Basic { username, password }) => ( - "basic".to_string(), - None, - Some(username), - Some(password), - None, - ), - Some(RemoteScorerAuth::ApiKey { token, header_name }) => ( - "api_key".to_string(), - Some(token), - None, - None, - Some(header_name), - ), - Some(RemoteScorerAuth::None) | None => ("none".to_string(), None, None, None, None), - }; - - Ok(RemoteScorerConfig { - endpoint: params.endpoint, - http_method: params - .http_method - .unwrap_or_else(|| "POST".to_string()) - .to_uppercase(), - auth_type, - auth_token, - auth_username, - auth_password, - api_key_header, - custom_headers: params - .custom_headers - .unwrap_or_default() - .into_iter() - .map(|h| (h.key, h.value)) - .collect(), - content_type: params - .content_type - .unwrap_or_else(|| "application/json".to_string()), - request_body_template: None, - timeout_ms: params.timeout_ms.unwrap_or(30_000), - max_retries: params.max_retries.unwrap_or(3), - }) -} - -pub async fn run_remote_scorer( - config: &RemoteScorerConfig, - span_attributes: &HashMap, -) -> Result { - let start = std::time::Instant::now(); - let body_template = config.request_body_template.as_deref().unwrap_or("{}"); - let rendered_body = render_template(body_template, span_attributes); - let mut last_error: Option = None; - - for attempt in 0..=config.max_retries { - if attempt > 0 { - tokio::time::sleep(Duration::from_millis(compute_backoff_ms(attempt))).await; - } - - match execute_request(config, &rendered_body).await { - Ok((output, raw_response)) => { - let latency = start.elapsed(); - let parsed = parse_remote_response(output)?; - return Ok(RemoteScorerOutput { - value: parsed.value, - reasoning: parsed.reason, - metadata: parsed.metadata, - raw_response, - latency_ms: latency.as_millis() as i64, - }); - } - Err(e) => { - log::warn!( - "[REMOTE-SCORER] Attempt {}/{} failed for {}: {}", - attempt + 1, - config.max_retries + 1, - config.endpoint, - e - ); - last_error = Some(e); - } - } - } - - Err(last_error.unwrap_or_else(|| anyhow::anyhow!("remote scorer request failed"))) -} - -fn render_template(template: &str, attrs: &HashMap) -> String { - let mut result = template.to_string(); - let mut start = 0; - while let Some(open) = result[start..].find("{{") { - let abs_open = start + open; - if let Some(close) = result[abs_open..].find("}}") { - let abs_close = abs_open + close + 2; - let path = result[abs_open + 2..abs_close - 2].trim(); - let mut value = get_attr(attrs, path); - if is_inside_string_literal(&result, abs_open) { - value = escape_json_string_content(&value); - } - result.replace_range(abs_open..abs_close, &value); - start = abs_open + value.len(); - } else { - break; - } - } - result -} - -fn get_attr(attrs: &HashMap, path: &str) -> String { - let parts: Vec<&str> = path.split('.').collect(); - let mut current = attrs.get(parts[0]); - for part in &parts[1..] { - current = current.and_then(|v| v.get(part)); - } - match current { - Some(Value::String(s)) => s.clone(), - Some(v) => v.to_string(), - None => path.to_string(), - } -} - -fn is_inside_string_literal(template: &str, pos: usize) -> bool { - let mut in_string = false; - let mut escaped = false; - - for ch in template[..pos].chars() { - if escaped { - escaped = false; - continue; - } - - match ch { - '\\' => escaped = true, - '"' => in_string = !in_string, - _ => {} - } - } - - in_string -} - -fn escape_json_string_content(value: &str) -> String { - serde_json::to_string(value) - .unwrap_or_else(|_| value.to_string()) - .trim_matches('"') - .to_string() -} - -async fn execute_request(config: &RemoteScorerConfig, body: &str) -> Result<(Value, String)> { - let client = reqwest::Client::builder() - .timeout(Duration::from_millis(config.timeout_ms)) - .build()?; - - let mut req = match config.http_method.as_str() { - "GET" => client.get(&config.endpoint), - "PUT" => client.put(&config.endpoint), - "PATCH" => client.patch(&config.endpoint), - "DELETE" => client.delete(&config.endpoint), - _ => client.post(&config.endpoint), - }; - - match config.auth_type.as_str() { - "bearer" => { - if let Some(token) = &config.auth_token { - req = req.header("Authorization", format!("Bearer {token}")); - } - } - "basic" => { - if let (Some(user), Some(pass)) = (&config.auth_username, &config.auth_password) { - let encoded = base64::Engine::encode( - &base64::engine::general_purpose::STANDARD, - format!("{user}:{pass}"), - ); - req = req.header("Authorization", format!("Basic {encoded}")); - } - } - "api_key" => { - if let (Some(header_name), Some(token)) = (&config.api_key_header, &config.auth_token) { - req = req.header(header_name, token); - } - } - _ => {} - } - - for (key, val) in &config.custom_headers { - req = req.header(key, val); - } - - if !body.is_empty() && config.http_method != "GET" { - req = req.header("Content-Type", &config.content_type); - req = req.body(body.to_string()); - } - - let resp = req.send().await?; - let status = resp.status(); - let raw_body = resp.text().await?; - let parsed = serde_json::from_str(&raw_body).map_err(|e| { - anyhow::anyhow!( - "Failed to parse remote scorer response as JSON (HTTP {}): {e}", - status - ) - })?; - Ok((parsed, raw_body)) -} - -#[derive(Debug)] -struct ParsedRemoteResponse { - value: Value, - reason: Option, - metadata: Option, -} - -fn parse_remote_response(response: Value) -> Result { - let mut obj = match response { - Value::Object(obj) => obj, - _ => anyhow::bail!("Remote scorer response must be a JSON object"), - }; - - let code = obj - .remove("code") - .and_then(|v| v.as_str().map(|s| s.to_string())) - .ok_or_else(|| anyhow::anyhow!("Remote scorer response missing string field 'code'"))?; - - let reason = obj - .remove("reason") - .and_then(|v| v.as_str().map(|s| s.to_string())); - - if code != "OK" { - anyhow::bail!( - "Remote scorer returned code '{}': {}", - code, - reason.as_deref().unwrap_or("no reason provided") - ); - } - - let value = obj.remove("value").ok_or_else(|| { - anyhow::anyhow!("Remote scorer response missing field 'value' for code OK") - })?; - - Ok(ParsedRemoteResponse { - value, - reason, - metadata: object_to_metadata(obj), - }) -} - -fn object_to_metadata(obj: Map) -> Option { - if obj.is_empty() { - None - } else { - Some(Value::Object(obj)) - } -} - -fn compute_backoff_ms(attempt: u32) -> u64 { - let base = 200u64.saturating_mul(1u64 << (attempt - 1).min(4)); - let mut rng = rand::rng(); - base + rng.random_range(0..=base / 2) -} - -#[cfg(test)] -mod tests { - use super::*; - - #[test] - fn test_parse_remote_response_ok_with_metadata() { - let parsed = parse_remote_response(serde_json::json!({ - "code": "OK", - "value": 0.85, - "reason": "good", - "confidence": "high" - })) - .unwrap(); - - assert_eq!(parsed.value, serde_json::json!(0.85)); - assert_eq!(parsed.reason.as_deref(), Some("good")); - assert_eq!(parsed.metadata.unwrap()["confidence"], "high"); - } - - #[test] - fn test_parse_remote_response_error_code_fails() { - let err = parse_remote_response(serde_json::json!({ - "code": "INPUT_TOO_LONG", - "reason": "too long" - })) - .unwrap_err(); - - assert!(err.to_string().contains("INPUT_TOO_LONG")); - } - - #[test] - fn test_parse_remote_response_ok_requires_value() { - let err = parse_remote_response(serde_json::json!({ - "code": "OK", - "reason": "missing value" - })) - .unwrap_err(); - - assert!(err.to_string().contains("missing field 'value'")); - } - - #[test] - fn test_render_template_escapes_values_inside_json_strings() { - let attrs = HashMap::from([ - ( - "input".to_string(), - serde_json::json!(r#"[{"content":"hello \"world\"","role":"user"}]"#), - ), - ("output".to_string(), serde_json::json!("line 1\nline 2")), - ]); - - let result = render_template(r#"{"input":"{{ input }}","output":"{{ output }}"}"#, &attrs); - let parsed: Value = serde_json::from_str(&result).unwrap(); - - assert_eq!(parsed["input"], attrs["input"]); - assert_eq!(parsed["output"], attrs["output"]); - } - - #[test] - fn test_render_template_preserves_free_text_values() { - let attrs = HashMap::from([( - "span".to_string(), - serde_json::json!({"name": "test-span", "kind": "internal"}), - )]); - - let result = render_template("Span: {{ span.name }} ({{ span.kind }})", &attrs); - - assert_eq!(result, "Span: test-span (internal)"); - } - - #[test] - fn test_remote_params_reject_response_paths() { - let err = parse_remote_config(&serde_json::json!({ - "endpoint": "http://localhost:8000/evaluate", - "response_value_path": "value" - })) - .unwrap_err(); - - assert!(err.to_string().contains("response_value_path")); - } -} diff --git a/src/service/llm_evaluations/scorers/schema_derivation.rs b/src/service/llm_evaluations/scorers/schema_derivation.rs deleted file mode 100644 index 6a7a1940e1..0000000000 --- a/src/service/llm_evaluations/scorers/schema_derivation.rs +++ /dev/null @@ -1,284 +0,0 @@ -// Copyright 2026 OpenObserve Inc. -// -// This program is free software: you can redistribute it and/or modify -// it under the terms of the GNU Affero General Public License as published by -// the Free Software Foundation, either version 3 of the License, or -// (at your option) any later version. -// -// This program is distributed in the hope that it will be useful -// but WITHOUT ANY WARRANTY; without even the implied warranty of -// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -// GNU Affero General Public License for more details. -// -// You should have received a copy of the GNU Affero General Public License -// along with this program. If not, see . - -//! Schema derivation for LLM Judge output. - -use anyhow::Result; -use infra::table::score_configs::ScoreConfigDataType; -use serde::{Deserialize, Serialize}; -use serde_json::Value; -use utoipa::ToSchema; - -#[derive(Debug, Clone)] -pub struct DerivedSchema { - pub json_schema: Value, - pub value_field: String, - pub metadata_fields: Vec, -} - -pub const SCORE_FIELD: &str = "score"; -pub const REASONING_FIELD: &str = "reasoning"; - -#[derive(Debug, Clone, Copy, Default, PartialEq, Eq, Serialize, Deserialize, ToSchema)] -#[serde(rename_all = "snake_case")] -pub enum ExtraMetadataFieldType { - #[default] - String, - Number, - Boolean, -} - -impl ExtraMetadataFieldType { - fn json_schema_type(self) -> &'static str { - match self { - Self::String => "string", - Self::Number => "number", - Self::Boolean => "boolean", - } - } -} - -#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize, ToSchema)] -#[serde(deny_unknown_fields)] -pub struct ExtraMetadataField { - pub name: String, - #[serde(rename = "type")] - pub data_type: ExtraMetadataFieldType, - #[serde(default, skip_serializing_if = "Option::is_none")] - pub description: Option, -} - -pub fn derive_output_schema( - data_type: ScoreConfigDataType, - numeric_range: Option<&Value>, - categories: Option<&Value>, - extra_metadata_fields: &[ExtraMetadataField], - include_reasoning: bool, -) -> Result { - let value_schema = match data_type { - ScoreConfigDataType::Numeric => { - let mut schema = serde_json::json!({ - "type": "number", - "description": "The numeric score value" - }); - if let Some(range) = numeric_range - && let (Some(min), Some(max)) = (range["min"].as_f64(), range["max"].as_f64()) - { - schema["minimum"] = serde_json::json!(min); - schema["maximum"] = serde_json::json!(max); - } - schema - } - ScoreConfigDataType::Categorical => { - let allowed: Vec<&str> = categories - .and_then(|c| c.as_array()) - .map(|arr| arr.iter().filter_map(|v| v.as_str()).collect()) - .unwrap_or_default(); - if allowed.is_empty() { - anyhow::bail!("categories must be provided for categorical data_type"); - } - serde_json::json!({ - "type": "string", - "enum": allowed, - "description": "The categorical score value" - }) - } - ScoreConfigDataType::Boolean => serde_json::json!({ - "type": "boolean", - "description": "The boolean pass/fail result" - }), - }; - - let mut properties = serde_json::json!({}); - properties[SCORE_FIELD] = value_schema; - - if include_reasoning { - properties[REASONING_FIELD] = serde_json::json!({ - "type": "string", - "description": "Brief explanation of the score" - }); - } - - let mut metadata_fields = Vec::new(); - for field in extra_metadata_fields { - let sanitized = field - .name - .chars() - .map(|c| { - if c.is_alphanumeric() || c == '_' { - c - } else { - '_' - } - }) - .collect::(); - if !sanitized.is_empty() && sanitized != SCORE_FIELD && sanitized != REASONING_FIELD { - metadata_fields.push(sanitized.clone()); - let description = field - .description - .as_deref() - .filter(|description| !description.trim().is_empty()) - .map(str::to_string) - .unwrap_or_else(|| format!("Additional metadata: {}", field.name)); - properties[&sanitized] = serde_json::json!({ - "type": field.data_type.json_schema_type(), - "description": description - }); - } - } - - let mut required_fields = vec![SCORE_FIELD.to_string()]; - if include_reasoning { - required_fields.push(REASONING_FIELD.to_string()); - } - - Ok(DerivedSchema { - json_schema: serde_json::json!({ - "type": "object", - "properties": properties, - "required": required_fields, - "additionalProperties": false - }), - value_field: SCORE_FIELD.to_string(), - metadata_fields, - }) -} - -#[cfg(test)] -mod tests { - use super::*; - - fn metadata_field( - name: &str, - data_type: ExtraMetadataFieldType, - description: Option<&str>, - ) -> ExtraMetadataField { - ExtraMetadataField { - name: name.to_string(), - data_type, - description: description.map(str::to_string), - } - } - - #[test] - fn derives_numeric_score_schema_from_range() { - let schema = derive_output_schema( - ScoreConfigDataType::Numeric, - Some(&serde_json::json!({"min": 0.0, "max": 1.0})), - None, - &[metadata_field( - "failure_mode", - ExtraMetadataFieldType::String, - Some("Reason for low score"), - )], - true, - ) - .unwrap(); - - assert_eq!(schema.value_field, "score"); - assert_eq!(schema.json_schema["properties"]["score"]["type"], "number"); - assert_eq!(schema.json_schema["properties"]["score"]["minimum"], 0.0); - assert_eq!(schema.json_schema["properties"]["score"]["maximum"], 1.0); - assert_eq!( - schema.json_schema["required"], - serde_json::json!(["score", "reasoning"]) - ); - assert_eq!(schema.metadata_fields, vec!["failure_mode".to_string()]); - assert_eq!( - schema.json_schema["properties"]["failure_mode"], - serde_json::json!({ - "type": "string", - "description": "Reason for low score" - }) - ); - } - - #[test] - fn derives_categorical_score_schema_from_categories() { - let schema = derive_output_schema( - ScoreConfigDataType::Categorical, - None, - Some(&serde_json::json!(["low", "medium", "high"])), - &[], - false, - ) - .unwrap(); - - assert_eq!(schema.json_schema["properties"]["score"]["type"], "string"); - assert_eq!( - schema.json_schema["properties"]["score"]["enum"], - serde_json::json!(["low", "medium", "high"]) - ); - assert_eq!(schema.json_schema["required"], serde_json::json!(["score"])); - } - - #[test] - fn derives_boolean_score_schema() { - let schema = derive_output_schema( - ScoreConfigDataType::Boolean, - None, - None, - &[ - metadata_field("score", ExtraMetadataFieldType::String, None), - metadata_field("reasoning", ExtraMetadataFieldType::String, None), - metadata_field("debug flag", ExtraMetadataFieldType::Boolean, None), - ], - true, - ) - .unwrap(); - - assert_eq!(schema.json_schema["properties"]["score"]["type"], "boolean"); - assert_eq!(schema.metadata_fields, vec!["debug_flag".to_string()]); - assert_eq!( - schema.json_schema["properties"]["debug_flag"], - serde_json::json!({ - "type": "boolean", - "description": "Additional metadata: debug flag" - }) - ); - } - - #[test] - fn rejects_string_metadata_field_entries() { - let result: Result = - serde_json::from_value(serde_json::json!("confidence")); - - assert!(result.is_err()); - } - - #[test] - fn derives_number_metadata_field_schema() { - let schema = derive_output_schema( - ScoreConfigDataType::Numeric, - None, - None, - &[metadata_field( - "confidence", - ExtraMetadataFieldType::Number, - Some("Judge confidence from 0.0 to 1.0"), - )], - true, - ) - .unwrap(); - - assert_eq!( - schema.json_schema["properties"]["confidence"], - serde_json::json!({ - "type": "number", - "description": "Judge confidence from 0.0 to 1.0" - }) - ); - } -} diff --git a/src/service/mod.rs b/src/service/mod.rs index 7446b68f13..5d11b63606 100644 --- a/src/service/mod.rs +++ b/src/service/mod.rs @@ -36,6 +36,7 @@ pub mod grpc; pub mod ingestion; pub mod ingestion_tokens; pub mod kv; +#[cfg(feature = "enterprise")] pub mod llm_evaluations; pub mod logs; pub mod metadata; @@ -48,6 +49,7 @@ pub mod org_usage; pub mod organization; pub mod pipeline; pub mod promql; +#[cfg(feature = "enterprise")] pub mod providers; #[cfg(feature = "enterprise")] pub mod ratelimit; diff --git a/src/service/pipeline/batch_execution.rs b/src/service/pipeline/batch_execution.rs index 0aa9b305c3..2c94e728f8 100644 --- a/src/service/pipeline/batch_execution.rs +++ b/src/service/pipeline/batch_execution.rs @@ -13,9 +13,10 @@ // You should have received a copy of the GNU Affero General Public License // along with this program. If not, see . +#[cfg(feature = "enterprise")] +use std::hash::{Hash, Hasher}; use std::{ collections::{HashMap, HashSet}, - hash::{Hash, Hasher}, sync::LazyLock as Lazy, }; @@ -65,6 +66,7 @@ struct BatchBuffer { last_write: Instant, } +#[cfg(feature = "enterprise")] fn should_sample_eval_record(record: &json::Value, idx: usize, sampling_rate: f64) -> bool { if sampling_rate >= 1.0 { return true; @@ -1341,7 +1343,22 @@ async fn process_node( ); } } + #[cfg(feature = "enterprise")] NodeData::LlmEvaluation(params) => { + if !o2_enterprise::enterprise::common::config::get_config() + .common + .online_evals_enabled + { + log::warn!( + "[Pipeline]: LLM evaluation node {node_idx} skipped because online evals are disabled" + ); + while receiver.recv().await.is_some() { + count += 1; + } + log::info!("[Pipeline]: LLM evaluation node {node_idx} skipped {count} records"); + return Ok(()); + } + log::info!("[Pipeline]: LLM evaluation node {node_idx} starts processing"); if let Err(e) = @@ -1470,6 +1487,19 @@ async fn process_node( "[Pipeline]: LLM evaluation node {node_idx} done processing {count} records" ); } + #[cfg(not(feature = "enterprise"))] + NodeData::LlmEvaluation(_) => { + log::warn!( + "[Pipeline]: LLM evaluation node {node_idx} skipped because online evals are enterprise-only" + ); + let mut skipped_count = 0usize; + while receiver.recv().await.is_some() { + skipped_count += 1; + } + log::info!( + "[Pipeline]: LLM evaluation node {node_idx} skipped {skipped_count} records" + ); + } } // all cloned senders dropped when function goes out of scope -> close the channel diff --git a/src/service/providers.rs b/src/service/providers.rs index 4704c513c3..f155f3eb9c 100644 --- a/src/service/providers.rs +++ b/src/service/providers.rs @@ -83,6 +83,9 @@ pub async fn update_provider( let existing = table::providers::get(provider_id) .await? .ok_or(ProviderError::NotFound)?; + if existing.org_id != org_id { + return Err(ProviderError::NotFound); + } // Check name uniqueness excluding self let all = table::providers::get_all_by_org(org_id).await?; diff --git a/src/service/self_reporting/mod.rs b/src/service/self_reporting/mod.rs index 3e99415daf..2625c84c00 100644 --- a/src/service/self_reporting/mod.rs +++ b/src/service/self_reporting/mod.rs @@ -42,17 +42,21 @@ use tokio::sync::oneshot; #[cfg(feature = "cloud")] pub mod cloud_events; +#[cfg(feature = "enterprise")] mod evaluator_schema; mod ingestion; +#[cfg(feature = "enterprise")] mod llm_scores_schema; mod queues; pub mod search; mod triggers_schema; mod usage_schema; +#[cfg(feature = "enterprise")] pub(crate) use evaluator_schema::ensure_evaluator_stream_initialized; #[cfg(feature = "cloud")] pub use ingestion::ingest_data_retention_usages; +#[cfg(feature = "enterprise")] pub(crate) use llm_scores_schema::ensure_llm_scores_stream_initialized; pub async fn run() { diff --git a/tests/api-testing/tests/test_online_eval.py b/tests/api-testing/tests/test_online_eval.py deleted file mode 100644 index aa9fdaef2b..0000000000 --- a/tests/api-testing/tests/test_online_eval.py +++ /dev/null @@ -1,1025 +0,0 @@ -"""Integration tests for Online Eval (Phase 2 LLM Observability) APIs. - -Covers completed tasks 6-9: - - Task 6: Provider CRUD - - Task 7: Score Config CRUD - - Task 8: Scorer CRUD - - Task 9: Online Eval Job CRUD + Reconciler - -Known server issues tracked: - - GH-ONLINE-EVAL-001: GET endpoints return CHAR-padded IDs (trailing whitespace) - - GH-ONLINE-EVAL-002: Update provider rejects same-name on self (duplicate check bug) - - GH-ONLINE-EVAL-003: Delete score_config/scorer returns 500 (SeaORM soft-delete bug) - - GH-ONLINE-EVAL-004: Update/activate/archive eval_job returns 500 (SeaORM "None updated") - - GH-ONLINE-EVAL-005: Missing required fields return 422 (axum) not 400 (handler) - -Prerequisites: - ZO_BASE_URL http://localhost:5080 - ZO_ROOT_USER_EMAIL root@example.com - ZO_ROOT_USER_PASSWORD Complexpass#123 - TEST_ORG_ID (optional, defaults to "default") -""" - -import pytest - - -# --------------------------------------------------------------------------- -# Helpers -# --------------------------------------------------------------------------- - -def _api(session, base_url, org_id, path, method="GET", json_data=None, - expected_status=200, label=""): - url = f"{base_url}api/{org_id}/{path.lstrip('/')}" - if method == "GET": - resp = session.get(url) - elif method == "POST": - resp = session.post(url, json=json_data) - elif method == "PUT": - resp = session.put(url, json=json_data) - elif method == "DELETE": - resp = session.delete(url) - else: - raise ValueError(method) - assert resp.status_code == expected_status, ( - f"{label or path} expected {expected_status}, got {resp.status_code}: " - f"{resp.text[:500]}" - ) - return resp - - -def _assert_ids_equal(a, b): - """Compare two IDs, handling CHAR-padded whitespace (GH-ONLINE-EVAL-001).""" - assert str(a).strip() == str(b).strip(), f"IDs don't match: {a!r} vs {b!r}" - - -# --------------------------------------------------------------------------- -# Providers -# --------------------------------------------------------------------------- - -class TestProviders: - - @pytest.fixture(autouse=True) - def setup(self, create_session, base_url, org_id, random_string): - self.s = create_session - self.base = base_url - self.org = org_id - self.rnd = random_string - self.pids = [] - yield - for pid in self.pids: - try: - self.s.delete(f"{self.base}api/{self.org}/providers/{pid}") - except Exception: - pass - - def _create(self, name=None, **overrides): - payload = { - "name": name or f"test-prov-{self.rnd()}", - "providerType": "openai", - "endpoint": "https://api.openai.com/v1", - "defaultModel": "gpt-4", - "availableModels": ["gpt-4", "gpt-4o"], - "authConfig": {"api_key": "sk-test-key"}, - "isDefault": False, - } - payload.update(overrides) - resp = _api(self.s, self.base, self.org, "providers", - method="POST", json_data=payload, label="create provider") - data = resp.json() - self.pids.append(data["id"]) - return data - - # -- CRUD ---------------------------------------------------------------- - - def test_01_list_empty(self): - resp = _api(self.s, self.base, self.org, "providers", label="list") - assert "list" in resp.json() - - def test_02_create(self): - data = self._create(name=f"crud-{self.rnd()}") - assert data["name"].startswith("crud-") - assert data["providerType"] == "openai" - assert data["authConfigMasked"] is True - - def test_03_get(self): - created = self._create(name=f"get-{self.rnd()}") - resp = _api(self.s, self.base, self.org, - f"providers/{created['id']}", label="get") - _assert_ids_equal(resp.json()["id"], created["id"]) - assert resp.json()["name"] == created["name"] - - def test_04_get_not_found(self): - _api(self.s, self.base, self.org, "providers/nonexistent", - expected_status=404, label="get missing") - - def test_05_update(self): - created = self._create(name=f"upd-{self.rnd()}") - payload = { - "name": created["name"], - "providerType": "openai", - "defaultModel": "gpt-4-turbo", - "authConfig": {"api_key": "sk-updated"}, - } - resp = _api(self.s, self.base, self.org, - f"providers/{created['id']}", method="PUT", - json_data=payload, label="update") - assert resp.json()["defaultModel"] == "gpt-4-turbo" - - def test_06_delete(self): - created = self._create(name=f"del-{self.rnd()}") - _api(self.s, self.base, self.org, f"providers/{created['id']}", - method="DELETE", label="delete") - self.pids.remove(created["id"]) - _api(self.s, self.base, self.org, f"providers/{created['id']}", - expected_status=404, label="get deleted") - - def test_07_delete_not_found(self): - _api(self.s, self.base, self.org, "providers/nonexistent", - method="DELETE", expected_status=404, label="delete missing") - - # -- Validation ---------------------------------------------------------- - - def test_08_missing_name(self): - """GH-ONLINE-EVAL-005: axum returns 422 for missing required field.""" - _api(self.s, self.base, self.org, "providers", method="POST", - json_data={"providerType": "openai", "defaultModel": "gpt-4", - "authConfig": {"api_key": "k"}}, - expected_status=422, label="create missing name") - - def test_09_duplicate_name(self): - name = f"dup-{self.rnd()}" - self._create(name=name) - _api(self.s, self.base, self.org, "providers", method="POST", - json_data={"name": name, "providerType": "anthropic", - "defaultModel": "claude-3", "authConfig": {"api_key": "k"}}, - expected_status=400, label="create duplicate") - - def test_10_test_endpoint(self): - created = self._create(name=f"tst-{self.rnd()}") - resp = _api(self.s, self.base, self.org, - f"providers/{created['id']}/test", method="POST", - label="test provider") - message = resp.json().get("message", "") - assert message - - def test_11_list_contains_created(self): - name = f"lst-{self.rnd()}" - created = self._create(name=name) - resp = _api(self.s, self.base, self.org, "providers", label="list") - names = [p["name"] for p in resp.json()["list"]] - assert created["name"] in names - - def test_12_update_not_found(self): - _api(self.s, self.base, self.org, "providers/nonexistent", - method="PUT", json_data={ - "name": "x", "providerType": "openai", "defaultModel": "gpt-4", - "authConfig": {"api_key": "k"}}, - expected_status=404, label="update missing") - - -# --------------------------------------------------------------------------- -# Score Configs -# --------------------------------------------------------------------------- - -class TestScoreConfigs: - - @pytest.fixture(autouse=True) - def setup(self, create_session, base_url, org_id, random_string): - self.s = create_session - self.base = base_url - self.org = org_id - self.rnd = random_string - self.ids = [] - yield - for entity_id in self.ids: - try: - self.s.delete(f"{self.base}api/{self.org}/score_configs/{entity_id}") - except Exception: - pass - - def _create(self, name=None, **overrides): - payload = { - "name": name or f"sc-{self.rnd()}", - "dataType": "numeric", - "description": "A numeric score config", - "numericRange": {"min": 0, "max": 100}, - "healthyThreshold": {"min": 70, "max": 100}, - } - payload.update(overrides) - resp = _api(self.s, self.base, self.org, "score_configs", - method="POST", json_data=payload, label="create score config") - data = resp.json() - self.ids.append(data["entityId"]) - return data - - def test_01_list_empty(self): - resp = _api(self.s, self.base, self.org, "score_configs", label="list") - assert "list" in resp.json() - - def test_02_create(self): - data = self._create(name=f"crud-{self.rnd()}") - assert data["name"].startswith("crud-") - assert data["dataType"] == "numeric" - assert data["version"] == 1 - - def test_03_get(self): - created = self._create(name=f"get-{self.rnd()}") - resp = _api(self.s, self.base, self.org, - f"score_configs/{created['entityId']}", label="get") - _assert_ids_equal(resp.json()["id"], created["id"]) - assert resp.json()["version"] == 1 - - def test_04_get_not_found(self): - _api(self.s, self.base, self.org, "score_configs/nope-xyz", - expected_status=404, label="get missing") - - def test_05_update_version_bump(self): - created = self._create(name=f"ver-{self.rnd()}") - assert created["version"] == 1 - resp = _api(self.s, self.base, self.org, - f"score_configs/{created['entityId']}", method="PUT", json_data={ - "name": created["name"], - "description": "v2", - "numericRange": {"min": 10, "max": 90}, - "healthyThreshold": {"min": 80, "max": 100}, - }, label="update score config") - data = resp.json() - assert data["version"] == 2 - assert data["description"] == "v2" - - def test_06_list_versions(self): - created = self._create(name=f"verlist-{self.rnd()}") - _api(self.s, self.base, self.org, f"score_configs/{created['entityId']}", - method="PUT", json_data={ - "name": created["name"], - "description": "v2", "numericRange": {"min": 0, "max": 50}, - "healthyThreshold": {"min": 25, "max": 50}, - }, label="bump v2") - resp = _api(self.s, self.base, self.org, - f"score_configs/{created['entityId']}/versions", - label="list versions") - versions = resp.json().get("versions", []) - assert len(versions) == 2 - assert versions[0]["version"] == 2 - def test_07_delete(self): - """Soft-delete (deactivate) a score config.""" - created = self._create(name=f"del-{self.rnd()}") - _api(self.s, self.base, self.org, f"score_configs/{created['entityId']}", - method="DELETE", label="delete score config") - self.ids.remove(created["entityId"]) - _api(self.s, self.base, self.org, f"score_configs/{created['entityId']}", - expected_status=404, label="get deleted") - - def test_08_missing_name(self): - _api(self.s, self.base, self.org, "score_configs", method="POST", - json_data={"dataType": "numeric"}, expected_status=422, - label="create missing name") - - def test_09_update_not_found(self): - _api(self.s, self.base, self.org, "score_configs/nope-xyz", - method="PUT", json_data={"name": "nope"}, - expected_status=404, label="update missing") - - -# --------------------------------------------------------------------------- -# Scorers -# --------------------------------------------------------------------------- - -class TestScorers: - - @pytest.fixture(autouse=True) - def setup(self, create_session, base_url, org_id, random_string): - self.s = create_session - self.base = base_url - self.org = org_id - self.rnd = random_string - self.ids = [] - self.score_config = _api(self.s, self.base, self.org, "score_configs", - method="POST", json_data={ - "name": f"scorer-sc-{self.rnd()}", - "dataType": "numeric", - "description": "Scorer test score config", - "numericRange": {"min": 0, "max": 100}, - "healthyThreshold": {"min": 70, "max": 100}, - }, label="create scorer score config").json() - yield - for entity_id in self.ids: - try: - self.s.delete(f"{self.base}api/{self.org}/scorers/{entity_id}") - except Exception: - pass - try: - self.s.delete( - f"{self.base}api/{self.org}/score_configs/{self.score_config['entityId']}" - ) - except Exception: - pass - - def _create(self, name=None, **overrides): - scorer_type = overrides.pop("scorerType", "llm_judge") - if scorer_type == "remote": - scorer = { - "type": "remote", - "producesScoreConfigId": self.score_config["entityId"], - "template": '{"input": "{{input}}"}', - "params": { - "endpoint": "http://127.0.0.1:1/evaluate", - "http_method": "POST", - "timeout_ms": 10, - "max_retries": 0, - }, - } - else: - scorer = { - "type": "llm_judge", - "producesScoreConfigId": self.score_config["entityId"], - "template": "Judge {{input}}", - "outputSchema": { - "type": "object", - "properties": {"value": {"type": "number"}}, - }, - "params": {"provider_id": "p1", "model": "gpt-4"}, - } - for key in ["producesScoreConfigId", "producesScoreConfigVersion", - "template", "outputSchema", "params"]: - if key in overrides: - scorer[key] = overrides.pop(key) - payload = { - "name": name or f"scorer-{self.rnd()}", - "description": "LLM Judge scorer", - "scorer": scorer, - } - payload.update(overrides) - resp = _api(self.s, self.base, self.org, "scorers", - method="POST", json_data=payload, label="create scorer") - data = resp.json() - self.ids.append(data["entityId"]) - return data - - def test_01_list_empty(self): - resp = _api(self.s, self.base, self.org, "scorers", label="list") - assert "list" in resp.json() - - def test_02_create(self): - data = self._create(name=f"crud-{self.rnd()}") - assert data["name"].startswith("crud-") - assert data["scorerType"] == "llm_judge" - assert data["producesScoreConfigId"] == self.score_config["entityId"] - assert data["version"] == 1 - - def test_03_get(self): - created = self._create(name=f"get-{self.rnd()}") - resp = _api(self.s, self.base, self.org, - f"scorers/{created['entityId']}", label="get") - _assert_ids_equal(resp.json()["id"], created["id"]) - assert resp.json()["name"] == created["name"] - - def test_04_get_not_found(self): - _api(self.s, self.base, self.org, "scorers/nope-xyz", - expected_status=404, label="get missing") - - def test_05_update_version_bump(self): - created = self._create(name=f"upd-{self.rnd()}") - assert created["version"] == 1 - resp = _api(self.s, self.base, self.org, - f"scorers/{created['entityId']}", method="PUT", json_data={ - "name": created["name"], - "description": "Updated scorer", - "scorer": { - "type": "llm_judge", - "template": "Judge {{input}}", - "params": {"provider_id": "p2", "model": "gpt-4-turbo"}, - }, - }, label="update scorer") - data = resp.json() - assert data["version"] == 2 - assert data["scorerType"] == "llm_judge" # immutable - assert data["producesScoreConfigId"] == self.score_config["entityId"] # immutable - - def test_06_list_versions(self): - created = self._create(name=f"verlist-{self.rnd()}") - _api(self.s, self.base, self.org, f"scorers/{created['entityId']}", - method="PUT", json_data={ - "name": created["name"], "description": "v2", - "scorer": { - "type": "llm_judge", - "template": "Judge {{input}}", - "params": created["params"], - }, - }, label="bump v2") - resp = _api(self.s, self.base, self.org, - f"scorers/{created['entityId']}/versions", - label="list versions") - versions = resp.json().get("versions", []) - assert len(versions) == 2 - def test_07_delete(self): - created = self._create(name=f"del-{self.rnd()}") - _api(self.s, self.base, self.org, f"scorers/{created['entityId']}", - method="DELETE", label="delete scorer") - self.ids.remove(created["entityId"]) - _api(self.s, self.base, self.org, f"scorers/{created['entityId']}", - expected_status=404, label="get deleted") - - def test_08_missing_name(self): - _api(self.s, self.base, self.org, "scorers", method="POST", - json_data={"scorer": {"type": "llm_judge", "template": "Judge {{input}}", - "params": {"provider_id": "p1"}}}, - expected_status=422, label="create missing name") - - def test_09_filter_by_scorer_type(self): - self._create(name=f"llm-{self.rnd()}", scorerType="llm_judge") - self._create(name=f"remote-{self.rnd()}", scorerType="remote") - resp = _api(self.s, self.base, self.org, - "scorers?scorerType=llm_judge", label="filter llm_judge") - for s in resp.json()["list"]: - assert s["scorerType"] == "llm_judge" - - def test_10_test_endpoint(self): - resp = _api(self.s, self.base, self.org, - "scorers/test", method="POST", json_data={ - "name": f"tst-{self.rnd()}", - "scorer": { - "type": "remote", - "template": '{"input": "{{input}}"}', - "params": { - "endpoint": "http://127.0.0.1:1/evaluate", - "http_method": "POST", - "timeout_ms": 10, - "max_retries": 0, - }, - }, - "inputVariables": {"input": "hello"}, - }, - label="test scorer") - data = resp.json() - assert data["success"] is False - assert data.get("error") - - -# --------------------------------------------------------------------------- -# Online Eval Jobs -# --------------------------------------------------------------------------- - -class TestEvalJobs: - - @pytest.fixture(autouse=True) - def setup(self, create_session, base_url, org_id, random_string): - self.s = create_session - self.base = base_url - self.org = org_id - self.rnd = random_string - self.jids = [] - yield - for jid in self.jids: - try: - self.s.delete(f"{self.base}api/{self.org}/eval_jobs/{jid}") - except Exception: - pass - - def _create(self, name=None, **overrides): - payload = { - "name": name or f"job-{self.rnd()}", - "description": "E2E test eval job", - "stream": "default", - "streamType": "logs", - "filterCondition": {"type": "condition", "field": "log", "operator": "exists"}, - "scorers": ["faithfulness"], - "inputMapping": None, - "samplingMode": "all", - "samplingValue": None, - } - payload.update(overrides) - resp = _api(self.s, self.base, self.org, "eval_jobs", - method="POST", json_data=payload, label="create eval job") - data = resp.json() - self.jids.append(data["id"]) - return data - - def _transition(self, job_id, action, expected_status=200): - return _api(self.s, self.base, self.org, - f"eval_jobs/{job_id}/{action}", method="POST", - expected_status=expected_status, - label=f"{action} job {job_id}") - - def test_01_list_empty(self): - resp = _api(self.s, self.base, self.org, "eval_jobs", label="list") - assert "list" in resp.json() - - def test_02_create_draft(self): - data = self._create(name=f"crud-{self.rnd()}") - assert data["name"].startswith("crud-") - assert data["status"] == "draft" - assert data["version"] == 1 - - def test_02b_create_preserves_multiple_input_mappings(self): - input_mapping = { - "scorer-1": { - "input": "{{gen_ai_input_messages}}", - "output": "{{gen_ai_output_messages}}", - "context": "{{gen_ai_system_instructions}}", - }, - "scorer-2": { - "input": "{{gen_ai_input_messages}}", - "output": "{{gen_ai_output_messages}}", - "expected": "{{gen_ai_system_instructions}}", - "trace_id": "{{trace_id}}", - }, - } - data = self._create( - name=f"mapping-{self.rnd()}", - scorers=["scorer-1", "scorer-2"], - inputMapping=input_mapping, - ) - - assert data["inputMapping"] == input_mapping - - resp = _api(self.s, self.base, self.org, - f"eval_jobs/{data['id']}", label="get eval job mappings") - assert resp.json()["inputMapping"] == input_mapping - - def test_03_get(self): - created = self._create(name=f"get-{self.rnd()}") - resp = _api(self.s, self.base, self.org, - f"eval_jobs/{created['id']}", label="get") - _assert_ids_equal(resp.json()["id"], created["id"]) - assert resp.json()["name"] == created["name"] - - def test_04_get_not_found(self): - _api(self.s, self.base, self.org, "eval_jobs/nonexistent", - expected_status=404, label="get missing") - def test_05_update(self): - created = self._create(name=f"upd-{self.rnd()}") - resp = _api(self.s, self.base, self.org, f"eval_jobs/{created['id']}", - method="PUT", json_data={ - "name": created["name"], - "description": "Updated", - "stream": "default", - "streamType": "logs", - "filterCondition": {"type": "condition", "field": "level", - "operator": "eq", "value": "error"}, - "scorers": ["faithfulness"], - "samplingMode": "rate", - "samplingValue": 0.5, - }, label="update eval job") - assert resp.json()["version"] == 2 - assert resp.json()["description"] == "Updated" - - def test_05b_update_adds_scorer_input_mapping(self): - initial_mapping = { - "scorer-1": { - "input": "{{gen_ai_input_messages}}", - "output": "{{gen_ai_output_messages}}", - "context": "{{gen_ai_system_instructions}}", - } - } - created = self._create( - name=f"upd-map-{self.rnd()}", - scorers=["scorer-1"], - inputMapping=initial_mapping, - ) - new_mapping = { - "scorer-2": { - "input": "{{gen_ai_input_messages}}", - "output": "{{gen_ai_output_messages}}", - "expected": "{{gen_ai_system_instructions}}", - "trace_id": "{{trace_id}}", - } - } - - resp = _api(self.s, self.base, self.org, f"eval_jobs/{created['id']}", - method="PUT", json_data={ - "name": created["name"], - "description": created["description"], - "stream": created["stream"], - "streamType": created["streamType"], - "filterCondition": created["filterCondition"], - "scorers": ["scorer-1", "scorer-2"], - "inputMapping": new_mapping, - "samplingMode": created["samplingMode"], - "samplingValue": created["samplingValue"], - }, label="update eval job mapping") - - expected_mapping = {**initial_mapping, **new_mapping} - assert resp.json()["scorers"] == ["scorer-1", "scorer-2"] - assert resp.json()["inputMapping"] == expected_mapping - - def test_06_delete(self): - created = self._create(name=f"del-{self.rnd()}") - _api(self.s, self.base, self.org, f"eval_jobs/{created['id']}", - method="DELETE", label="delete eval job") - self.jids.remove(created["id"]) - _api(self.s, self.base, self.org, f"eval_jobs/{created['id']}", - expected_status=404, label="get deleted") - def test_07_full_lifecycle(self): - """draft -> activate -> pause -> resume -> archive.""" - job = self._create(name=f"lifecycle-{self.rnd()}") - assert job["status"] == "draft" - - resp = self._transition(job["id"], "activate") - assert resp.json()["status"] == "active" - assert resp.json().get("pipelineId") is not None - - resp = self._transition(job["id"], "pause") - assert resp.json()["status"] == "paused" - - resp = self._transition(job["id"], "resume") - assert resp.json()["status"] == "active" - - resp = self._transition(job["id"], "archive") - assert resp.json()["status"] == "archived" - - def test_08_draft_to_archive(self): - job = self._create(name=f"d2a-{self.rnd()}") - resp = self._transition(job["id"], "archive") - assert resp.json()["status"] == "archived" - - def test_09_active_pause_resume(self): - job = self._create(name=f"apr-{self.rnd()}") - self._transition(job["id"], "activate") - resp = self._transition(job["id"], "pause") - assert resp.json()["status"] == "paused" - resp = self._transition(job["id"], "resume") - assert resp.json()["status"] == "active" - - def test_10_invalid_transition(self): - """Pausing a draft job should return 400.""" - job = self._create(name=f"inv-{self.rnd()}") - self._transition(job["id"], "pause", expected_status=400) - - def test_11_list_filter_by_status(self): - """Filter eval jobs by status -- tests the query param works.""" - self._create(name=f"draft-{self.rnd()}") - # List with filter (even if no active jobs, the endpoint should accept it) - resp = _api(self.s, self.base, self.org, "eval_jobs?status=draft", - label="filter draft") - assert "list" in resp.json() - - def test_12_missing_name(self): - _api(self.s, self.base, self.org, "eval_jobs", method="POST", - json_data={"stream": "default", "streamType": "logs", - "filterCondition": {}, "scorers": ["x"], - "samplingMode": "all"}, - expected_status=422, label="create missing name") - - def test_13_missing_stream(self): - _api(self.s, self.base, self.org, "eval_jobs", method="POST", - json_data={"name": f"nostream-{self.rnd()}", "streamType": "logs", - "filterCondition": {}, "scorers": ["x"], - "samplingMode": "all"}, - expected_status=422, label="create missing stream") - - -# --------------------------------------------------------------------------- -# End-to-End: full resource chain -# --------------------------------------------------------------------------- - -class TestOnlineEvalEndToEnd: - - @pytest.fixture(autouse=True) - def setup(self, create_session, base_url, org_id, random_string): - self.s = create_session - self.base = base_url - self.org = org_id - self.rnd = random_string - self.ids = {"providers": [], "score_configs": [], "scorers": [], "jobs": []} - yield - for jid in self.ids["jobs"]: - try: - self.s.delete(f"{self.base}api/{self.org}/eval_jobs/{jid}") - except Exception: - pass - for name in self.ids["scorers"]: - try: - self.s.delete(f"{self.base}api/{self.org}/scorers/{name}") - except Exception: - pass - for name in self.ids["score_configs"]: - try: - self.s.delete(f"{self.base}api/{self.org}/score_configs/{name}") - except Exception: - pass - for pid in self.ids["providers"]: - try: - self.s.delete(f"{self.base}api/{self.org}/providers/{pid}") - except Exception: - pass - - def test_01_create_all_resources(self): - """Create Provider -> Score Config -> Scorer -> Eval Job (draft). - This test covers the happy-path resource creation chain and verifies - all endpoints return expected response shapes. - """ - s, base, org = self.s, self.base, self.org - suffix = self.rnd() - - # Provider - prov = _api(s, base, org, "providers", method="POST", json_data={ - "name": f"e2e-prov-{suffix}", - "providerType": "openai", - "endpoint": "https://api.openai.com/v1", - "defaultModel": "gpt-4o", - "authConfig": {"api_key": "sk-e2e"}, - }, label="e2e: create provider").json() - self.ids["providers"].append(prov["id"]) - assert prov["providerType"] == "openai" - - # Score Config - sc = _api(s, base, org, "score_configs", method="POST", json_data={ - "name": f"e2e-sc-{suffix}", - "dataType": "categorical", - "description": "E2E score config", - "categories": {"Good": 85, "Fair": 50, "Poor": 15}, - }, label="e2e: create score config").json() - self.ids["score_configs"].append(sc["entityId"]) - assert sc["dataType"] == "categorical" - assert sc["version"] == 1 - - # Scorer (llm_judge) - scorer = _api(s, base, org, "scorers", method="POST", json_data={ - "name": f"e2e-scorer-{suffix}", - "description": "E2E LLM Judge", - "scorer": { - "type": "llm_judge", - "producesScoreConfigId": sc["entityId"], - "template": "Judge {{query}} and {{response}}", - "outputSchema": {"type": "object", - "properties": {"value": {"type": "number"}}}, - "params": {"provider_id": prov["id"], "model": "gpt-4o"}, - }, - }, label="e2e: create scorer").json() - self.ids["scorers"].append(scorer["entityId"]) - assert scorer["scorerType"] == "llm_judge" - assert scorer["version"] == 1 - - # Eval Job (draft) - job = _api(s, base, org, "eval_jobs", method="POST", json_data={ - "name": f"e2e-job-{suffix}", - "description": "E2E eval job", - "stream": "default", - "streamType": "logs", - "filterCondition": {"type": "condition", "field": "log", - "operator": "exists"}, - "scorers": [scorer["entityId"]], - "inputMapping": None, - "samplingMode": "all", - "samplingValue": None, - }, label="e2e: create eval job").json() - self.ids["jobs"].append(job["id"]) - assert job["status"] == "draft" - assert job["stream"] == "default" - - def test_02_activate_job_with_pipeline(self): - """Create job + activate -> verifies pipelineId is returned.""" - s, base, org = self.s, self.base, self.org - suffix = self.rnd() - - # Minimal resources needed for activation - scorer = _api(s, base, org, "scorers", method="POST", json_data={ - "name": f"e2e-scr-{suffix}", - "scorer": { - "type": "llm_judge", - "template": "Judge {{input}}", - "params": {"provider_id": "p", "model": "gpt-4"}, - }, - }, label="e2e: create scorer").json() - - job = _api(s, base, org, "eval_jobs", method="POST", json_data={ - "name": f"e2e-act-{suffix}", - "stream": "default", - "streamType": "logs", - "filterCondition": {"type": "condition", "field": "log", - "operator": "exists"}, - "scorers": [scorer["entityId"]], - "samplingMode": "all", - "samplingValue": None, - }, label="e2e: create job for activation").json() - self.ids["jobs"].append(job["id"]) - self.ids["scorers"].append(scorer["entityId"]) - - act = _api(s, base, org, f"eval_jobs/{job['id']}/activate", - method="POST", label="e2e: activate").json() - assert act["status"] == "active" - assert act.get("pipelineId") is not None - - def test_03_version_pinning_preserves_scorer_ref_and_versions(self): - """Pinned eval job scorer refs should survive scorer/config version bumps.""" - s, base, org = self.s, self.base, self.org - suffix = self.rnd() - - score_config = _api(s, base, org, "score_configs", method="POST", json_data={ - "name": f"pin-sc-{suffix}", - "dataType": "numeric", - "description": "Pinned score config v1", - "numericRange": {"min": 0, "max": 1}, - "healthyThreshold": {"min": 0.7, "max": 1}, - }, label="pinning: create score config").json() - self.ids["score_configs"].append(score_config["entityId"]) - - scorer = _api(s, base, org, "scorers", method="POST", json_data={ - "name": f"pin-scorer-{suffix}", - "description": "Pinned scorer v1", - "scorer": { - "type": "llm_judge", - "producesScoreConfigId": score_config["entityId"], - "template": "Judge v1 {{input}}", - "outputSchema": { - "type": "object", - "properties": {"value": {"type": "number"}}, - "required": ["value"], - }, - "params": {"provider_id": "pin-provider-v1", "model": "gpt-4o"}, - }, - }, label="pinning: create scorer").json() - self.ids["scorers"].append(scorer["entityId"]) - - assert score_config["version"] == 1 - assert scorer["version"] == 1 - - updated_score_config = _api( - s, base, org, f"score_configs/{score_config['entityId']}", - method="PUT", - json_data={ - "name": score_config["name"], - "description": "Pinned score config v2", - "numericRange": {"min": 0, "max": 100}, - "healthyThreshold": {"min": 70, "max": 100}, - }, - label="pinning: update score config", - ).json() - updated_scorer = _api( - s, base, org, f"scorers/{scorer['entityId']}", - method="PUT", - json_data={ - "name": scorer["name"], - "description": "Pinned scorer v2", - "scorer": { - "type": "llm_judge", - "template": "Judge v2 {{input}}", - "outputSchema": scorer["outputSchema"], - "params": {"provider_id": "pin-provider-v2", "model": "gpt-4o-mini"}, - }, - }, - label="pinning: update scorer", - ).json() - - assert updated_score_config["version"] == 2 - assert updated_scorer["version"] == 2 - assert updated_scorer["producesScoreConfigId"] == score_config["entityId"] - - job = _api(s, base, org, "eval_jobs", method="POST", json_data={ - "name": f"pin-job-{suffix}", - "description": "Eval job pinned to scorer v1", - "stream": "default", - "streamType": "logs", - "filterCondition": {"type": "condition", "field": "log", - "operator": "exists"}, - "scorers": [{"id": scorer["entityId"], "version": 1}], - "samplingMode": "all", - "samplingValue": None, - }, label="pinning: create eval job").json() - self.ids["jobs"].append(job["id"]) - - assert job["scorers"] == [{"id": scorer["entityId"], "version": 1}] - - scorer_versions = _api( - s, base, org, f"scorers/{scorer['entityId']}/versions", - label="pinning: list scorer versions", - ).json()["versions"] - score_config_versions = _api( - s, base, org, f"score_configs/{score_config['entityId']}/versions", - label="pinning: list score config versions", - ).json()["versions"] - - assert [v["version"] for v in scorer_versions[:2]] == [2, 1] - assert [v["version"] for v in score_config_versions[:2]] == [2, 1] - - def test_04_scorer_version_pins_score_config_version(self): - """A scorer version should retain the score config version it was created with.""" - s, base, org = self.s, self.base, self.org - suffix = self.rnd() - - score_config = _api(s, base, org, "score_configs", method="POST", json_data={ - "name": f"pin-sc-version-{suffix}", - "dataType": "numeric", - "description": "Score config version expected to be pinned", - "numericRange": {"min": 0, "max": 1}, - "healthyThreshold": {"min": 0.7, "max": 1}, - }, label="score_config pinning: create score config").json() - self.ids["score_configs"].append(score_config["entityId"]) - - scorer = _api(s, base, org, "scorers", method="POST", json_data={ - "name": f"pin-score-config-scorer-{suffix}", - "description": "Scorer should pin score config v1", - "scorer": { - "type": "llm_judge", - "producesScoreConfigId": score_config["entityId"], - "template": "Judge {{input}}", - "outputSchema": { - "type": "object", - "properties": {"value": {"type": "number"}}, - "required": ["value"], - }, - "params": {"provider_id": "pin-provider", "model": "gpt-4o"}, - }, - }, label="score_config pinning: create scorer").json() - self.ids["scorers"].append(scorer["entityId"]) - - _api( - s, base, org, f"score_configs/{score_config['entityId']}", - method="PUT", - json_data={ - "name": score_config["name"], - "description": "Score config v2 should not affect scorer v1", - "numericRange": {"min": 0, "max": 100}, - "healthyThreshold": {"min": 70, "max": 100}, - }, - label="score_config pinning: update score config", - ) - - scorer_versions = _api( - s, base, org, f"scorers/{scorer['entityId']}/versions", - label="score_config pinning: list scorer versions", - ).json()["versions"] - scorer_v1 = next(v for v in scorer_versions if v["version"] == 1) - - assert scorer_v1["producesScoreConfigId"] == score_config["entityId"] - assert scorer_v1["producesScoreConfigVersion"] == 1 - - def test_05_score_config_delete_blocked_when_used_by_scorer(self): - s, base, org = self.s, self.base, self.org - suffix = self.rnd() - - score_config = _api(s, base, org, "score_configs", method="POST", json_data={ - "name": f"block-sc-{suffix}", - "dataType": "numeric", - "description": "Score config used by a scorer", - "numericRange": {"min": 0, "max": 1}, - "healthyThreshold": {"min": 0.7, "max": 1}, - }, label="block delete: create score config").json() - self.ids["score_configs"].append(score_config["entityId"]) - - scorer = _api(s, base, org, "scorers", method="POST", json_data={ - "name": f"block-sc-scorer-{suffix}", - "description": "Scorer using score config", - "scorer": { - "type": "llm_judge", - "producesScoreConfigId": score_config["entityId"], - "template": "Judge {{input}}", - "params": {"provider_id": "block-provider", "model": "gpt-4o"}, - }, - }, label="block delete: create scorer").json() - self.ids["scorers"].append(scorer["entityId"]) - - resp = _api( - s, base, org, f"score_configs/{score_config['entityId']}", - method="DELETE", - expected_status=409, - label="block delete: delete score config in use", - ) - assert "used by one or more scorers" in resp.text - - _api(s, base, org, f"scorers/{scorer['entityId']}", - method="DELETE", label="block delete: delete scorer") - _api(s, base, org, f"score_configs/{score_config['entityId']}", - method="DELETE", label="block delete: delete score config after scorer") - - def test_06_scorer_delete_blocked_by_non_archived_eval_job_only(self): - s, base, org = self.s, self.base, self.org - suffix = self.rnd() - - scorer = _api(s, base, org, "scorers", method="POST", json_data={ - "name": f"block-scorer-{suffix}", - "description": "Scorer referenced by eval job", - "scorer": { - "type": "llm_judge", - "template": "Judge {{input}}", - "params": {"provider_id": "block-provider", "model": "gpt-4o"}, - }, - }, label="block scorer: create scorer").json() - self.ids["scorers"].append(scorer["entityId"]) - - job = _api(s, base, org, "eval_jobs", method="POST", json_data={ - "name": f"block-job-{suffix}", - "description": "Eval job referencing scorer", - "stream": "default", - "streamType": "logs", - "filterCondition": {"type": "condition", "field": "log", - "operator": "exists"}, - "scorers": [scorer["entityId"]], - "samplingMode": "all", - "samplingValue": None, - }, label="block scorer: create eval job").json() - self.ids["jobs"].append(job["id"]) - - resp = _api( - s, base, org, f"scorers/{scorer['entityId']}", - method="DELETE", - expected_status=409, - label="block scorer: delete scorer used by draft job", - ) - assert "used by one or more eval jobs" in resp.text - - _api(s, base, org, f"eval_jobs/{job['id']}/archive", - method="POST", label="block scorer: archive eval job") - _api(s, base, org, f"scorers/{scorer['entityId']}", - method="DELETE", label="block scorer: delete scorer after archive") diff --git a/web/src/layouts/MainLayout.vue b/web/src/layouts/MainLayout.vue index 8b47e5d410..d80ca3eaad 100644 --- a/web/src/layouts/MainLayout.vue +++ b/web/src/layouts/MainLayout.vue @@ -329,7 +329,7 @@ export default defineComponent({ }); // Backend `/config` flag `online_evals_enabled` — controlled by - // `ZO_ONLINE_EVALS_ENABLED`. Reactive so the menu picks it up regardless + // enterprise `O2_ONLINE_EVALS_ENABLED`. Reactive so the menu picks it up regardless // of whether the config response arrived before or after this component // mounted. const isOnlineEvalsEnabled = computed(() => {