mirror of
https://github.com/openobserve/openobserve.git
synced 2026-11-06 00:09:22 +00:00
feat(evaluation): eval scheduler metrics, window-split executor, completion-window defaults and guard rails (#13457)
Design at: docs/online-evaluation-scheduler-queue-review.zh-CN.md OSS-side half of the phase-2 eval scheduler rework (paired o2-enterprise PR linked in comments). The enterprise PR depends on the metrics and the executor contract below, so **this PR must merge first**. ## Scheduler metrics - `eval_scheduler_pending_targets` (gauge, org/stream) — in-flight pending targets tracked across scheduler passes - `eval_scheduler_pending_memory_bytes` (gauge, state=used|limit) — accounted pending memory vs the configured budget - `eval_scheduler_forced_ready_total` (counter, org/stream) — targets force-evaluated early by budget enforcement - `eval_scheduler_watermark_lag_seconds` (gauge, org/stream) — scan cursor vs committed (persisted) watermark Registration only; all emission sites live in the enterprise scheduler. ## Detection-search executor: window-split support The scheduler's search executor no longer errors on an oversized scan window. It reports `over_limit` against the shared 100k-row cap (`EVAL_DETECTION_MAX_ROWS`) so the scheduler can split the window in half and re-query, and fetches a capped result set only when the scheduler explicitly allows truncation at the minimum window width. This removes the previous failure mode where a high-throughput window could never be fetched and retried forever. ## Completion-window defaults, floor, and hard caps Eval jobs that don't set their own trace/session completion windows now take deployment-wide defaults, declared in the enterprise `LlmEvaluationConfig` (`O2_EVAL_TRACE_IDLE_TIMEOUT_SECS`=30, `O2_EVAL_TRACE_MAX_AGE_SECS`=1800, `O2_EVAL_SESSION_IDLE_TIMEOUT_SECS`=1800, `O2_EVAL_SESSION_MAX_AGE_SECS`=14400 — the design-spec baselines) and injected into infra at startup via `init_completion_window_defaults` (same registration pattern as the search executor). Sessions span user think-time between traces, hence the much wider session idle window. - **Floor**: the old 45s idle-window minimum is dropped to 1s. Readiness fires only once the observed ingest-time silence reaches the window, so sub-poll-interval windows are correct — their firing is merely quantized to the next scan pass. - **Hard caps** (`validate_trace_completion_window` / `validate_session_completion_window`, applied identically to job-supplied values and env defaults): trace idle ≤ 30m / max age ≤ 2h, session idle ≤ 4h / max age ≤ 24h. Max age bounds pending-target memory residency, committed-watermark lag, and restart rescan, so it is a guard rail, not an operator preference. The enterprise config parser calls the same validators and **fails startup** on out-of-range env values (0 still means built-in default). - **Frontend** mirrors all of it: per-scope defaults and limits in `completionWindow.ts`, zod schema validation, input `max` attributes, and validation messages across all 15 locales. ## Legacy eval-buffer cleanup (OSS side) Removes the pipeline-update eviction call into the legacy trace-buffering LLM evaluation node (`LlmEvaluationNode::remove_buffer`) — a no-op on an always-empty registry since span evaluation moved to the eval-task queue. The paired enterprise PR deletes the legacy module itself. ## Module split `infra::table::online_eval_jobs` splits into submodules — `completion_window.rs` (defaults, guard rails, trace/session configs) and `span_selector.rs` — with re-exports keeping every external path unchanged.
This commit is contained in:
@@ -958,7 +958,7 @@ pub fn service_routes() -> Router {
|
||||
.route("/{org_id}/settings/gen_ai/agent_registry", delete(gen_ai::clear_agent_registry))
|
||||
.route("/{org_id}/gen_ai/agents", get(gen_ai::list_scored_agents));
|
||||
|
||||
if get_o2_config().common.online_evals_enabled {
|
||||
if get_o2_config().llm_eval_config.enabled {
|
||||
router = router
|
||||
// LLM Providers (Online Eval Phase 2)
|
||||
.route("/{org_id}/providers", get(providers::list_providers).post(providers::create_provider))
|
||||
|
||||
@@ -518,7 +518,10 @@ mod tests {
|
||||
let job = infra::table::online_eval_jobs::OnlineEvalJob::try_from(body).unwrap();
|
||||
|
||||
assert_eq!(job.target_scope, TargetScope::Trace);
|
||||
assert_eq!(job.trace_config.unwrap().idle_window_secs, 120);
|
||||
assert_eq!(
|
||||
job.trace_config.unwrap().idle_window_secs,
|
||||
infra::table::online_eval_jobs::DEFAULT_TRACE_IDLE_WINDOW_SECS
|
||||
);
|
||||
assert!(job.session_config.is_none());
|
||||
}
|
||||
|
||||
@@ -539,7 +542,10 @@ mod tests {
|
||||
let job = infra::table::online_eval_jobs::OnlineEvalJob::try_from(body).unwrap();
|
||||
|
||||
assert_eq!(job.target_scope, TargetScope::Session);
|
||||
assert_eq!(job.session_config.unwrap().idle_window_secs, 120);
|
||||
assert_eq!(
|
||||
job.session_config.unwrap().idle_window_secs,
|
||||
infra::table::online_eval_jobs::DEFAULT_SESSION_IDLE_WINDOW_SECS
|
||||
);
|
||||
assert!(job.trace_config.is_none());
|
||||
}
|
||||
|
||||
@@ -712,13 +718,13 @@ mod tests {
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_eval_job_request_body_rejects_idle_window_below_scheduler_poll_interval() {
|
||||
fn test_eval_job_request_body_rejects_a_non_positive_idle_window() {
|
||||
let json = r#"{
|
||||
"name": "j",
|
||||
"stream": "traces",
|
||||
"streamType": "traces",
|
||||
"targetScope": "trace",
|
||||
"traceConfig": { "idleWindowSecs": 44, "maxAgeSecs": 1800 },
|
||||
"traceConfig": { "idleWindowSecs": 0, "maxAgeSecs": 1800 },
|
||||
"filterCondition": {},
|
||||
"scorers": ["scorer-1"],
|
||||
"samplingMode": "all",
|
||||
@@ -728,7 +734,7 @@ mod tests {
|
||||
let body: EvalJobRequestBody = serde_json::from_str(json).unwrap();
|
||||
let err = infra::table::online_eval_jobs::OnlineEvalJob::try_from(body).unwrap_err();
|
||||
|
||||
assert_eq!(err, "Completion idle window must be at least 45 seconds");
|
||||
assert_eq!(err, "Completion idle window must be at least 1 second");
|
||||
}
|
||||
|
||||
#[test]
|
||||
|
||||
@@ -364,7 +364,7 @@ pub async fn zo_config() -> impl IntoResponse {
|
||||
// Anomaly detection is on when the enterprise feature is compiled in, unless turned off at
|
||||
// runtime via O2_ANOMALY_DETECTION_DISABLED. When disabled the UI hides the anomaly tab.
|
||||
let anomaly_detection_enabled = enterprise_value!(false, !o2cfg.anomaly_detection.disabled);
|
||||
let online_evals_enabled = enterprise_value!(false, o2cfg.common.online_evals_enabled);
|
||||
let online_evals_enabled = enterprise_value!(false, o2cfg.llm_eval_config.enabled);
|
||||
let synthetics_enabled = enterprise_value!(false, o2cfg.synthetics.enabled);
|
||||
|
||||
#[cfg(all(feature = "cloud", not(feature = "enterprise")))]
|
||||
|
||||
@@ -784,7 +784,8 @@ pub async fn get_session_details(
|
||||
let use_cache = get_use_cache_from_request(&query);
|
||||
let user_id_opt = Some(user_id.to_string());
|
||||
|
||||
let query_sql = traces::session::trace_ids_sql(&stream_name, &session_id_columns, &session_id);
|
||||
let query_sql =
|
||||
traces::session::trace_ids_sql(&stream_name, &session_id_columns, &session_id, None);
|
||||
|
||||
let mut req = config::meta::search::Request {
|
||||
query: config::meta::search::Query {
|
||||
|
||||
@@ -74,7 +74,12 @@ pub mod session {
|
||||
.collect()
|
||||
}
|
||||
|
||||
pub fn trace_ids_sql(stream: &str, session_columns: &[String], session_id: &str) -> String {
|
||||
pub fn trace_ids_sql(
|
||||
stream: &str,
|
||||
session_columns: &[String],
|
||||
session_id: &str,
|
||||
ingest_cutoff_us: Option<i64>,
|
||||
) -> String {
|
||||
let escaped_session_id = escape_sql_string(session_id);
|
||||
let predicate = session_columns
|
||||
.iter()
|
||||
@@ -84,14 +89,27 @@ pub mod session {
|
||||
format!(
|
||||
"SELECT trace_id, min({}) as zo_sql_timestamp \
|
||||
FROM {} \
|
||||
WHERE ({predicate}) \
|
||||
WHERE ({predicate}){} \
|
||||
GROUP BY trace_id \
|
||||
ORDER BY zo_sql_timestamp DESC, trace_id ASC",
|
||||
quote_identifier(TIMESTAMP_COL_NAME),
|
||||
quote_identifier(stream),
|
||||
ingest_cutoff_predicate(ingest_cutoff_us),
|
||||
)
|
||||
}
|
||||
|
||||
/// An ingest-time upper bound appended to evidence queries: rows ingested
|
||||
/// at or after the cutoff are excluded, so hydration cannot mix evidence
|
||||
/// that arrived after an evaluation run's frozen boundary into that run.
|
||||
fn ingest_cutoff_predicate(ingest_cutoff_us: Option<i64>) -> String {
|
||||
ingest_cutoff_us.map_or_else(String::new, |cutoff_us| {
|
||||
format!(
|
||||
" AND {} < {cutoff_us}",
|
||||
quote_identifier(crate::O2_INGEST_TS_COL_NAME)
|
||||
)
|
||||
})
|
||||
}
|
||||
|
||||
pub fn trace_ids_from_hits(hits: &[Value]) -> Vec<String> {
|
||||
let mut seen = HashSet::with_capacity(hits.len());
|
||||
let mut trace_ids = Vec::with_capacity(hits.len());
|
||||
@@ -116,11 +134,16 @@ pub mod session {
|
||||
format!("{} IN ({values})", quote_identifier("trace_id"))
|
||||
}
|
||||
|
||||
pub fn span_rows_sql(stream: &str, trace_ids: &[String]) -> String {
|
||||
pub fn span_rows_sql(
|
||||
stream: &str,
|
||||
trace_ids: &[String],
|
||||
ingest_cutoff_us: Option<i64>,
|
||||
) -> String {
|
||||
format!(
|
||||
"SELECT * FROM {} WHERE {} ORDER BY {} ASC, _o2_ingest_ts ASC, trace_id ASC, span_id ASC",
|
||||
"SELECT * FROM {} WHERE {}{} ORDER BY {} ASC, _o2_ingest_ts ASC, trace_id ASC, span_id ASC",
|
||||
quote_identifier(stream),
|
||||
trace_id_predicate(trace_ids),
|
||||
ingest_cutoff_predicate(ingest_cutoff_us),
|
||||
quote_identifier(TIMESTAMP_COL_NAME),
|
||||
)
|
||||
}
|
||||
@@ -165,17 +188,37 @@ pub mod session {
|
||||
"llm_session_id".to_string(),
|
||||
],
|
||||
"session-'1",
|
||||
None,
|
||||
);
|
||||
|
||||
assert!(sql.contains("\"gen_ai_conversation_id\" = 'session-''1'"));
|
||||
assert!(!sql.contains("_o2_ingest_ts"));
|
||||
assert!(sql.contains("OR \"llm_session_id\" = 'session-''1'"));
|
||||
assert!(sql.contains("GROUP BY trace_id"));
|
||||
assert!(sql.contains("ORDER BY zo_sql_timestamp DESC, trace_id ASC"));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn evidence_queries_apply_the_frozen_ingest_cutoff() {
|
||||
let sql = trace_ids_sql(
|
||||
"traces",
|
||||
&["llm_session_id".to_string()],
|
||||
"session-1",
|
||||
Some(1_000_000),
|
||||
);
|
||||
assert!(sql.contains("AND \"_o2_ingest_ts\" < 1000000 GROUP BY"));
|
||||
|
||||
let sql = span_rows_sql("traces", &["abc-123".to_string()], Some(1_000_000));
|
||||
assert!(sql.contains("AND \"_o2_ingest_ts\" < 1000000 ORDER BY"));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn phase_two_selects_all_rows_for_discovered_traces() {
|
||||
let sql = span_rows_sql("traces", &["abc-123".to_string(), "def-456".to_string()]);
|
||||
let sql = span_rows_sql(
|
||||
"traces",
|
||||
&["abc-123".to_string(), "def-456".to_string()],
|
||||
None,
|
||||
);
|
||||
|
||||
assert!(sql.contains("SELECT *"));
|
||||
assert!(sql.contains("\"trace_id\" IN ('abc-123', 'def-456')"));
|
||||
|
||||
@@ -1952,6 +1952,71 @@ pub static QUEUE_OLDEST_MESSAGE_AGE_SECONDS: Lazy<IntGaugeVec> = Lazy::new(|| {
|
||||
.expect("Metric created")
|
||||
});
|
||||
|
||||
pub static EVAL_SCHEDULER_PENDING_TARGETS: Lazy<IntGaugeVec> = Lazy::new(|| {
|
||||
IntGaugeVec::new(
|
||||
Opts::new(
|
||||
"eval_scheduler_pending_targets",
|
||||
"Current in-flight (pending) evaluation targets tracked by the eval scheduler, summed across the organization's streams",
|
||||
)
|
||||
.namespace(NAMESPACE)
|
||||
.const_labels(create_const_labels()),
|
||||
&["organization"],
|
||||
)
|
||||
.expect("Metric created")
|
||||
});
|
||||
|
||||
pub static EVAL_SCHEDULER_PENDING_MEMORY_BYTES: Lazy<IntGaugeVec> = Lazy::new(|| {
|
||||
IntGaugeVec::new(
|
||||
Opts::new(
|
||||
"eval_scheduler_pending_memory_bytes",
|
||||
"Accounted bytes of pending evaluation targets, and the configured limit (state=used, limit)",
|
||||
)
|
||||
.namespace(NAMESPACE)
|
||||
.const_labels(create_const_labels()),
|
||||
&["state"],
|
||||
)
|
||||
.expect("Metric created")
|
||||
});
|
||||
|
||||
pub static EVAL_SCHEDULER_FORCED_READY_TOTAL: Lazy<IntCounterVec> = Lazy::new(|| {
|
||||
IntCounterVec::new(
|
||||
Opts::new(
|
||||
"eval_scheduler_forced_ready_total",
|
||||
"Total pending evaluation targets force-evaluated early because the pending memory budget was exceeded",
|
||||
)
|
||||
.namespace(NAMESPACE)
|
||||
.const_labels(create_const_labels()),
|
||||
&["organization"],
|
||||
)
|
||||
.expect("Metric created")
|
||||
});
|
||||
|
||||
pub static EVAL_SCHEDULER_EVICTED_EVIDENCE_TOTAL: Lazy<IntCounterVec> = Lazy::new(|| {
|
||||
IntCounterVec::new(
|
||||
Opts::new(
|
||||
"eval_scheduler_evicted_evidence_total",
|
||||
"Evaluation evidence dropped by pending-memory budget enforcement (kind=orphan: unbound session evidence whose session evaluation may be lost until a restart replays it; kind=binding: trace-to-session bindings). Sustained increases mean the budget does not match the workload",
|
||||
)
|
||||
.namespace(NAMESPACE)
|
||||
.const_labels(create_const_labels()),
|
||||
&["organization", "kind"],
|
||||
)
|
||||
.expect("Metric created")
|
||||
});
|
||||
|
||||
pub static EVAL_SCHEDULER_WATERMARK_LAG_SECONDS: Lazy<IntGaugeVec> = Lazy::new(|| {
|
||||
IntGaugeVec::new(
|
||||
Opts::new(
|
||||
"eval_scheduler_watermark_lag_seconds",
|
||||
"Lag in seconds between the eval scheduler scan cursor and the committed (persisted) watermark, worst case across the organization's streams",
|
||||
)
|
||||
.namespace(NAMESPACE)
|
||||
.const_labels(create_const_labels()),
|
||||
&["organization"],
|
||||
)
|
||||
.expect("Metric created")
|
||||
});
|
||||
|
||||
fn register_metrics(registry: &Registry) {
|
||||
// http latency
|
||||
registry
|
||||
@@ -2452,6 +2517,22 @@ fn register_metrics(registry: &Registry) {
|
||||
registry
|
||||
.register(Box::new(QUEUE_OLDEST_MESSAGE_AGE_SECONDS.clone()))
|
||||
.expect("Metric registered");
|
||||
// eval scheduler pending-target metrics
|
||||
registry
|
||||
.register(Box::new(EVAL_SCHEDULER_PENDING_TARGETS.clone()))
|
||||
.expect("Metric registered");
|
||||
registry
|
||||
.register(Box::new(EVAL_SCHEDULER_PENDING_MEMORY_BYTES.clone()))
|
||||
.expect("Metric registered");
|
||||
registry
|
||||
.register(Box::new(EVAL_SCHEDULER_FORCED_READY_TOTAL.clone()))
|
||||
.expect("Metric registered");
|
||||
registry
|
||||
.register(Box::new(EVAL_SCHEDULER_EVICTED_EVIDENCE_TOTAL.clone()))
|
||||
.expect("Metric registered");
|
||||
registry
|
||||
.register(Box::new(EVAL_SCHEDULER_WATERMARK_LAG_SECONDS.clone()))
|
||||
.expect("Metric registered");
|
||||
}
|
||||
|
||||
pub fn create_const_labels() -> HashMap<String, String> {
|
||||
|
||||
@@ -361,6 +361,8 @@ pub async fn manual_evaluate(
|
||||
o2_enterprise::enterprise::llm_evaluations::eval_jobs::tasks::EvaluationQueryWindow {
|
||||
start_us: body.start_time,
|
||||
end_us: body.end_time,
|
||||
// manual evaluations use the caller's window verbatim
|
||||
ingest_cutoff_us: None,
|
||||
},
|
||||
reason: body
|
||||
.reason
|
||||
|
||||
@@ -1344,8 +1344,8 @@ async fn process_llm_evaluation_node(
|
||||
return count;
|
||||
}
|
||||
if !o2_enterprise::enterprise::common::config::get_config()
|
||||
.common
|
||||
.online_evals_enabled
|
||||
.llm_eval_config
|
||||
.enabled
|
||||
{
|
||||
log::warn!(
|
||||
"[Pipeline] {} [inv={inv_id}]: LLM evaluation node {} skipped because online evals are disabled",
|
||||
|
||||
@@ -201,24 +201,6 @@ pub async fn update_pipeline(mut pipeline: Pipeline) -> Result<(), PipelineError
|
||||
|
||||
pipeline::update(&pipeline, prev_source_stream).await?;
|
||||
|
||||
// Evict any live LLM evaluation buffer tasks for this pipeline so they
|
||||
// are re-created with the updated config (e.g. scorer list) on
|
||||
// the next ingestion batch.
|
||||
#[cfg(feature = "enterprise")]
|
||||
{
|
||||
let pipeline_id = &pipeline.id;
|
||||
for node in &pipeline.nodes {
|
||||
if matches!(node.data, NodeData::LlmEvaluation(_)) {
|
||||
let buffer_key = format!("{}:{}", pipeline_id, node.id);
|
||||
o2_enterprise::enterprise::pipeline::llm_evaluation_node::LlmEvaluationNode::remove_buffer(&buffer_key).await;
|
||||
log::debug!(
|
||||
"[Pipeline] Evicted LLM evaluation buffer for key={} after pipeline update",
|
||||
buffer_key
|
||||
);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
Ok(())
|
||||
}
|
||||
|
||||
|
||||
@@ -0,0 +1,406 @@
|
||||
// Copyright 2026 OpenObserve Inc.
|
||||
//
|
||||
// This program is free software: you can redistribute it and/or modify
|
||||
// it under the terms of the GNU Affero General Public License as published by
|
||||
// the Free Software Foundation, either version 3 of the License, or
|
||||
// (at your option) any later version.
|
||||
//
|
||||
// This program is distributed in the hope that it will be useful
|
||||
// but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
// GNU Affero General Public License for more details.
|
||||
//
|
||||
// You should have received a copy of the GNU Affero General Public License
|
||||
// along with this program. If not, see <http://www.gnu.org/licenses/>.
|
||||
|
||||
//! Completion-window defaults, guard rails, and per-scope trace/session
|
||||
//! configuration for online eval jobs. A target completes once its observed
|
||||
//! ingest-time silence reaches the idle window, or unconditionally once
|
||||
//! `max_age` elapses from first ingest.
|
||||
|
||||
use std::sync::OnceLock;
|
||||
|
||||
use config::meta::pipeline::components::ConditionParams;
|
||||
use serde::{Deserialize, Serialize};
|
||||
|
||||
/// Any positive idle window is sound: readiness fires only once the observed
|
||||
/// ingest-time silence reaches the window, so a window shorter than one
|
||||
/// scheduler pass just gets its firing quantized to the next pass. The floor
|
||||
/// only rejects nonsensical (zero/negative) values; a window smaller than
|
||||
/// real gaps inside a trace splits it into resumed cycles, which is the
|
||||
/// user's own cost/latency tradeoff.
|
||||
pub const MIN_COMPLETION_IDLE_WINDOW_SECS: i64 = 1;
|
||||
pub const DEFAULT_TRACE_IDLE_WINDOW_SECS: i64 = 30;
|
||||
pub const DEFAULT_TRACE_MAX_AGE_SECS: i64 = 30 * 60;
|
||||
/// Sessions span user think-time between traces, so their idle window is far
|
||||
/// wider than a trace's.
|
||||
pub const DEFAULT_SESSION_IDLE_WINDOW_SECS: i64 = 30 * 60;
|
||||
pub const DEFAULT_SESSION_MAX_AGE_SECS: i64 = 4 * 60 * 60;
|
||||
|
||||
/// Per-scope guard rails for completion windows, applied identically to
|
||||
/// job-supplied values and env-override defaults. `max_age` bounds how long a
|
||||
/// pending target can occupy scheduler memory, how far the committed
|
||||
/// watermark may lag, and how much ingest time a restart has to rescan — so
|
||||
/// it gets a hard ceiling rather than trusting the operator.
|
||||
pub struct CompletionWindowLimits {
|
||||
pub max_idle_window_secs: i64,
|
||||
pub max_max_age_secs: i64,
|
||||
idle_error: &'static str,
|
||||
max_age_error: &'static str,
|
||||
}
|
||||
|
||||
pub const TRACE_COMPLETION_LIMITS: CompletionWindowLimits = CompletionWindowLimits {
|
||||
max_idle_window_secs: 30 * 60,
|
||||
max_max_age_secs: 2 * 60 * 60,
|
||||
idle_error: "Trace idle window cannot exceed 30 minutes",
|
||||
max_age_error: "Trace max age cannot exceed 2 hours",
|
||||
};
|
||||
|
||||
pub const SESSION_COMPLETION_LIMITS: CompletionWindowLimits = CompletionWindowLimits {
|
||||
max_idle_window_secs: 4 * 60 * 60,
|
||||
max_max_age_secs: 24 * 60 * 60,
|
||||
idle_error: "Session idle window cannot exceed 4 hours",
|
||||
max_age_error: "Session max age cannot exceed 24 hours",
|
||||
};
|
||||
|
||||
/// Deployment-wide completion-window defaults, applied whenever an eval job
|
||||
/// does not set its own values (no trace/session config at all, or a config
|
||||
/// missing the field).
|
||||
struct CompletionWindowDefaults {
|
||||
trace_idle_secs: i64,
|
||||
trace_max_age_secs: i64,
|
||||
session_idle_secs: i64,
|
||||
session_max_age_secs: i64,
|
||||
}
|
||||
|
||||
static COMPLETION_WINDOW_DEFAULTS: OnceLock<CompletionWindowDefaults> = OnceLock::new();
|
||||
|
||||
/// Install deployment-wide completion-window defaults from configuration
|
||||
/// (the enterprise `LlmEvaluationConfig` `O2_EVAL_TRACE_IDLE_TIMEOUT_SECS`,
|
||||
/// `O2_EVAL_TRACE_MAX_AGE_SECS`, `O2_EVAL_SESSION_IDLE_TIMEOUT_SECS` and
|
||||
/// `O2_EVAL_SESSION_MAX_AGE_SECS` values). Called once during startup wiring;
|
||||
/// values go through the same validation as job-supplied windows and fall
|
||||
/// back to the built-in constants (with a warning) when invalid. When never
|
||||
/// called — OSS builds, unit tests — the constants apply as-is.
|
||||
pub fn init_completion_window_defaults(
|
||||
trace_idle_secs: i64,
|
||||
trace_max_age_secs: i64,
|
||||
session_idle_secs: i64,
|
||||
session_max_age_secs: i64,
|
||||
) {
|
||||
let (trace_idle_secs, trace_max_age_secs) = resolve_window_defaults(
|
||||
"trace",
|
||||
trace_idle_secs,
|
||||
trace_max_age_secs,
|
||||
DEFAULT_TRACE_IDLE_WINDOW_SECS,
|
||||
DEFAULT_TRACE_MAX_AGE_SECS,
|
||||
&TRACE_COMPLETION_LIMITS,
|
||||
);
|
||||
let (session_idle_secs, session_max_age_secs) = resolve_window_defaults(
|
||||
"session",
|
||||
session_idle_secs,
|
||||
session_max_age_secs,
|
||||
DEFAULT_SESSION_IDLE_WINDOW_SECS,
|
||||
DEFAULT_SESSION_MAX_AGE_SECS,
|
||||
&SESSION_COMPLETION_LIMITS,
|
||||
);
|
||||
let defaults = CompletionWindowDefaults {
|
||||
trace_idle_secs,
|
||||
trace_max_age_secs,
|
||||
session_idle_secs,
|
||||
session_max_age_secs,
|
||||
};
|
||||
if COMPLETION_WINDOW_DEFAULTS.set(defaults).is_err() {
|
||||
log::warn!(
|
||||
"online eval completion-window defaults were already in use before initialization; keeping the earlier values"
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
fn completion_window_defaults() -> &'static CompletionWindowDefaults {
|
||||
COMPLETION_WINDOW_DEFAULTS.get_or_init(|| CompletionWindowDefaults {
|
||||
trace_idle_secs: DEFAULT_TRACE_IDLE_WINDOW_SECS,
|
||||
trace_max_age_secs: DEFAULT_TRACE_MAX_AGE_SECS,
|
||||
session_idle_secs: DEFAULT_SESSION_IDLE_WINDOW_SECS,
|
||||
session_max_age_secs: DEFAULT_SESSION_MAX_AGE_SECS,
|
||||
})
|
||||
}
|
||||
|
||||
/// Validate a trace completion window — job-supplied or configuration values.
|
||||
pub fn validate_trace_completion_window(
|
||||
idle_window_secs: i64,
|
||||
max_age_secs: i64,
|
||||
) -> Result<(), &'static str> {
|
||||
validate_completion_window(idle_window_secs, max_age_secs, &TRACE_COMPLETION_LIMITS)
|
||||
}
|
||||
|
||||
/// Validate a session completion window — job-supplied or configuration values.
|
||||
pub fn validate_session_completion_window(
|
||||
idle_window_secs: i64,
|
||||
max_age_secs: i64,
|
||||
) -> Result<(), &'static str> {
|
||||
validate_completion_window(idle_window_secs, max_age_secs, &SESSION_COMPLETION_LIMITS)
|
||||
}
|
||||
|
||||
fn resolve_window_defaults(
|
||||
scope: &str,
|
||||
idle_secs: i64,
|
||||
max_age_secs: i64,
|
||||
default_idle_secs: i64,
|
||||
default_max_age_secs: i64,
|
||||
limits: &CompletionWindowLimits,
|
||||
) -> (i64, i64) {
|
||||
match validate_completion_window(idle_secs, max_age_secs, limits) {
|
||||
Ok(()) => (idle_secs, max_age_secs),
|
||||
Err(error) => {
|
||||
log::warn!(
|
||||
"ignoring configured {scope} completion-window defaults (idle={idle_secs}s, max_age={max_age_secs}s): {error}; using built-in defaults idle={default_idle_secs}s, max_age={default_max_age_secs}s"
|
||||
);
|
||||
(default_idle_secs, default_max_age_secs)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Clone, Debug, PartialEq, Serialize, Deserialize)]
|
||||
#[serde(default, rename_all = "camelCase")]
|
||||
pub struct TraceEvalConfig {
|
||||
pub idle_window_secs: i64,
|
||||
pub max_age_secs: i64,
|
||||
pub end_signal: Option<serde_json::Value>,
|
||||
}
|
||||
|
||||
impl Default for TraceEvalConfig {
|
||||
fn default() -> Self {
|
||||
let defaults = completion_window_defaults();
|
||||
Self {
|
||||
idle_window_secs: defaults.trace_idle_secs,
|
||||
max_age_secs: defaults.trace_max_age_secs,
|
||||
end_signal: None,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
impl TraceEvalConfig {
|
||||
pub fn validate(&self) -> Result<(), &'static str> {
|
||||
validate_trace_completion_window(self.idle_window_secs, self.max_age_secs)?;
|
||||
validate_end_signal(self.end_signal.as_ref())
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Clone, Debug, PartialEq, Serialize, Deserialize)]
|
||||
#[serde(default, rename_all = "camelCase")]
|
||||
pub struct SessionEvalConfig {
|
||||
pub idle_window_secs: i64,
|
||||
pub max_age_secs: i64,
|
||||
pub end_signal: Option<serde_json::Value>,
|
||||
}
|
||||
|
||||
impl Default for SessionEvalConfig {
|
||||
fn default() -> Self {
|
||||
let defaults = completion_window_defaults();
|
||||
Self {
|
||||
idle_window_secs: defaults.session_idle_secs,
|
||||
max_age_secs: defaults.session_max_age_secs,
|
||||
end_signal: None,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
impl SessionEvalConfig {
|
||||
pub fn validate(&self) -> Result<(), &'static str> {
|
||||
validate_session_completion_window(self.idle_window_secs, self.max_age_secs)?;
|
||||
validate_end_signal(self.end_signal.as_ref())
|
||||
}
|
||||
}
|
||||
|
||||
fn validate_end_signal(end_signal: Option<&serde_json::Value>) -> Result<(), &'static str> {
|
||||
let Some(end_signal) = end_signal else {
|
||||
return Ok(());
|
||||
};
|
||||
|
||||
let condition = serde_json::from_value::<ConditionParams>(end_signal.clone())
|
||||
.map_err(|_| "End signal must be a valid condition")?;
|
||||
|
||||
match condition {
|
||||
ConditionParams::V1 { conditions } if conditions.has_conditions() => Ok(()),
|
||||
ConditionParams::V2 { conditions } if conditions.validate().is_ok() => Ok(()),
|
||||
_ => Err("End signal must contain at least one condition"),
|
||||
}
|
||||
}
|
||||
|
||||
fn validate_completion_window(
|
||||
idle_window_secs: i64,
|
||||
max_age_secs: i64,
|
||||
limits: &CompletionWindowLimits,
|
||||
) -> Result<(), &'static str> {
|
||||
if idle_window_secs < MIN_COMPLETION_IDLE_WINDOW_SECS {
|
||||
return Err("Completion idle window must be at least 1 second");
|
||||
}
|
||||
if max_age_secs <= 0 {
|
||||
return Err("Completion max age must be greater than zero");
|
||||
}
|
||||
if idle_window_secs > max_age_secs {
|
||||
return Err("Completion idle window cannot exceed max age");
|
||||
}
|
||||
if idle_window_secs > limits.max_idle_window_secs {
|
||||
return Err(limits.idle_error);
|
||||
}
|
||||
if max_age_secs > limits.max_max_age_secs {
|
||||
return Err(limits.max_age_error);
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
#[test]
|
||||
fn configured_window_defaults_apply_when_valid() {
|
||||
assert_eq!(
|
||||
resolve_window_defaults("trace", 300, 900, 30, 1800, &TRACE_COMPLETION_LIMITS),
|
||||
(300, 900)
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn configured_window_defaults_above_the_caps_fall_back() {
|
||||
// A 1-hour trace idle window is far beyond trace scale.
|
||||
assert_eq!(
|
||||
resolve_window_defaults(
|
||||
"trace",
|
||||
60 * 60,
|
||||
2 * 60 * 60,
|
||||
30,
|
||||
1800,
|
||||
&TRACE_COMPLETION_LIMITS
|
||||
),
|
||||
(30, 1800)
|
||||
);
|
||||
// A 10-day session would pin scheduler memory and the committed
|
||||
// watermark for 10 days.
|
||||
assert_eq!(
|
||||
resolve_window_defaults(
|
||||
"session",
|
||||
1800,
|
||||
10 * 24 * 60 * 60,
|
||||
1800,
|
||||
14_400,
|
||||
&SESSION_COMPLETION_LIMITS
|
||||
),
|
||||
(1800, 14_400)
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn invalid_configured_window_defaults_fall_back() {
|
||||
// Zero/negative idle is nonsensical.
|
||||
assert_eq!(
|
||||
resolve_window_defaults("trace", 0, 1800, 30, 1800, &TRACE_COMPLETION_LIMITS),
|
||||
(30, 1800)
|
||||
);
|
||||
// Idle exceeding max age.
|
||||
assert_eq!(
|
||||
resolve_window_defaults(
|
||||
"session",
|
||||
600,
|
||||
300,
|
||||
1800,
|
||||
14_400,
|
||||
&SESSION_COMPLETION_LIMITS
|
||||
),
|
||||
(1800, 14_400)
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn completion_windows_reject_values_above_the_scope_caps() {
|
||||
// A 1-hour trace idle window is far beyond trace scale.
|
||||
let trace = TraceEvalConfig {
|
||||
idle_window_secs: 60 * 60,
|
||||
max_age_secs: 2 * 60 * 60,
|
||||
end_signal: None,
|
||||
};
|
||||
assert_eq!(
|
||||
trace.validate(),
|
||||
Err("Trace idle window cannot exceed 30 minutes")
|
||||
);
|
||||
|
||||
let trace = TraceEvalConfig {
|
||||
idle_window_secs: 60,
|
||||
max_age_secs: 3 * 60 * 60,
|
||||
end_signal: None,
|
||||
};
|
||||
assert_eq!(trace.validate(), Err("Trace max age cannot exceed 2 hours"));
|
||||
|
||||
// A 10-day session would pin scheduler memory and the committed
|
||||
// watermark for 10 days.
|
||||
let session = SessionEvalConfig {
|
||||
idle_window_secs: 30 * 60,
|
||||
max_age_secs: 10 * 24 * 60 * 60,
|
||||
end_signal: None,
|
||||
};
|
||||
assert_eq!(
|
||||
session.validate(),
|
||||
Err("Session max age cannot exceed 24 hours")
|
||||
);
|
||||
|
||||
let session = SessionEvalConfig {
|
||||
idle_window_secs: 5 * 60 * 60,
|
||||
max_age_secs: 24 * 60 * 60,
|
||||
end_signal: None,
|
||||
};
|
||||
assert_eq!(
|
||||
session.validate(),
|
||||
Err("Session idle window cannot exceed 4 hours")
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_completion_config_rejects_a_non_positive_idle_window() {
|
||||
let below_minimum = TraceEvalConfig {
|
||||
idle_window_secs: MIN_COMPLETION_IDLE_WINDOW_SECS - 1,
|
||||
..TraceEvalConfig::default()
|
||||
};
|
||||
assert_eq!(
|
||||
below_minimum.validate(),
|
||||
Err("Completion idle window must be at least 1 second")
|
||||
);
|
||||
|
||||
// Sub-poll-interval windows are valid; the scheduler just quantizes
|
||||
// their firing to its next scan pass.
|
||||
let sub_pass_window = TraceEvalConfig {
|
||||
idle_window_secs: 10,
|
||||
..TraceEvalConfig::default()
|
||||
};
|
||||
assert!(sub_pass_window.validate().is_ok());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_completion_config_rejects_invalid_or_empty_end_signal() {
|
||||
let invalid = TraceEvalConfig {
|
||||
end_signal: Some(serde_json::json!({"field": "status"})),
|
||||
..TraceEvalConfig::default()
|
||||
};
|
||||
assert_eq!(
|
||||
invalid.validate(),
|
||||
Err("End signal must be a valid condition")
|
||||
);
|
||||
|
||||
let empty = SessionEvalConfig {
|
||||
end_signal: Some(serde_json::json!({
|
||||
"version": 2,
|
||||
"conditions": {
|
||||
"filterType": "group",
|
||||
"logicalOperator": "AND",
|
||||
"conditions": []
|
||||
}
|
||||
})),
|
||||
..SessionEvalConfig::default()
|
||||
};
|
||||
assert_eq!(
|
||||
empty.validate(),
|
||||
Err("End signal must contain at least one condition")
|
||||
);
|
||||
}
|
||||
}
|
||||
+15
-330
@@ -16,8 +16,7 @@
|
||||
use std::collections::BTreeMap;
|
||||
|
||||
use config::meta::{
|
||||
alerts::{ConditionGroup, ConditionList},
|
||||
pipeline::components::{ConditionParams, ScorerRef},
|
||||
pipeline::components::ScorerRef,
|
||||
self_reporting::{
|
||||
evaluator::EVALUATOR_STREAM, llm_scores::LLM_SCORES_STREAM,
|
||||
usage::is_reserved_self_reporting_stream,
|
||||
@@ -28,6 +27,12 @@ use sea_orm::{
|
||||
};
|
||||
use serde::{Deserialize, Serialize};
|
||||
|
||||
pub mod completion_window;
|
||||
pub mod span_selector;
|
||||
|
||||
pub use completion_window::*;
|
||||
pub use span_selector::*;
|
||||
|
||||
use super::get_lock;
|
||||
use crate::{
|
||||
db::{ORM_CLIENT, connect_to_orm},
|
||||
@@ -38,30 +43,8 @@ use crate::{
|
||||
/// Valid job states.
|
||||
pub const VALID_STATUSES: &[&str] = &["draft", "active", "paused", "degraded", "archived"];
|
||||
|
||||
/// Lowest configurable idle window, matching the default scheduler poll interval.
|
||||
pub const MIN_COMPLETION_IDLE_WINDOW_SECS: i64 = 45;
|
||||
pub const DEFAULT_TRACE_IDLE_WINDOW_SECS: i64 = 2 * 60;
|
||||
pub const DEFAULT_TRACE_MAX_AGE_SECS: i64 = 30 * 60;
|
||||
pub const DEFAULT_SESSION_IDLE_WINDOW_SECS: i64 = 2 * 60;
|
||||
pub const DEFAULT_SESSION_MAX_AGE_SECS: i64 = 4 * 60 * 60;
|
||||
|
||||
pub type ScorerInputMapping = BTreeMap<String, String>;
|
||||
pub type JobInputMapping = BTreeMap<String, ScorerInputMapping>;
|
||||
pub type SpanSelectorBindings = BTreeMap<String, String>;
|
||||
|
||||
pub const DEFAULT_SPAN_SELECTOR_MAXIMUM_SPANS: usize = 5;
|
||||
pub const SPAN_SELECTOR_FIELD_VALUE_MAX_CHARS: usize = 1_000;
|
||||
pub const SPAN_SELECTOR_OUTPUT_MAX_CHARS: usize = 40_000;
|
||||
pub const DEFAULT_SPAN_SELECTOR_FIELDS: [&str; 8] = [
|
||||
"name",
|
||||
"status",
|
||||
"gen_ai_tool_name",
|
||||
"gen_ai_tool_call_id",
|
||||
"gen_ai_tool_call_arguments",
|
||||
"gen_ai_tool_call_result",
|
||||
"gen_ai_input_messages",
|
||||
"gen_ai_output_messages",
|
||||
];
|
||||
|
||||
pub fn is_reserved_eval_source_stream(stream: &str) -> bool {
|
||||
let stream = stream.trim().to_ascii_lowercase();
|
||||
@@ -79,112 +62,6 @@ pub fn is_reserved_eval_source_stream(stream: &str) -> bool {
|
||||
)
|
||||
}
|
||||
|
||||
#[derive(Clone, Copy, Debug, Default, PartialEq, Eq, Serialize, Deserialize)]
|
||||
#[serde(rename_all = "snake_case")]
|
||||
pub enum SpanSelectorFieldMode {
|
||||
#[default]
|
||||
Default,
|
||||
Custom,
|
||||
}
|
||||
|
||||
#[derive(Clone, Debug, PartialEq, Serialize, Deserialize)]
|
||||
#[serde(default, rename_all = "camelCase")]
|
||||
pub struct SpanSelector {
|
||||
pub id: String,
|
||||
pub name: String,
|
||||
pub filter_condition: serde_json::Value,
|
||||
pub field_mode: SpanSelectorFieldMode,
|
||||
pub fields: Vec<String>,
|
||||
pub maximum_spans: usize,
|
||||
}
|
||||
|
||||
impl Default for SpanSelector {
|
||||
fn default() -> Self {
|
||||
Self {
|
||||
id: String::new(),
|
||||
name: String::new(),
|
||||
filter_condition: serde_json::json!({}),
|
||||
field_mode: SpanSelectorFieldMode::Default,
|
||||
fields: Vec::new(),
|
||||
maximum_spans: DEFAULT_SPAN_SELECTOR_MAXIMUM_SPANS,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
impl SpanSelector {
|
||||
pub fn field_count(&self) -> usize {
|
||||
match self.field_mode {
|
||||
SpanSelectorFieldMode::Default => DEFAULT_SPAN_SELECTOR_FIELDS.len(),
|
||||
SpanSelectorFieldMode::Custom => self.fields.len(),
|
||||
}
|
||||
}
|
||||
|
||||
pub fn validate(&self) -> Result<(), &'static str> {
|
||||
if self.id.trim().is_empty() {
|
||||
return Err("Span Selector id cannot be empty");
|
||||
}
|
||||
if self.name.trim().is_empty() {
|
||||
return Err("Span Selector name cannot be empty");
|
||||
}
|
||||
if self.maximum_spans == 0 {
|
||||
return Err("Span Selector maximumSpans must be greater than zero");
|
||||
}
|
||||
if !is_valid_selector_filter(&self.filter_condition) {
|
||||
return Err("Span Selector filterCondition is invalid");
|
||||
}
|
||||
|
||||
if matches!(self.field_mode, SpanSelectorFieldMode::Custom) {
|
||||
if self.fields.is_empty() {
|
||||
return Err("Custom Span Selector schema requires at least one field");
|
||||
}
|
||||
let mut fields = std::collections::BTreeSet::new();
|
||||
for field in &self.fields {
|
||||
let field = field.trim();
|
||||
if field.is_empty() {
|
||||
return Err("Span Selector field names cannot be empty");
|
||||
}
|
||||
if !fields.insert(field) {
|
||||
return Err("Span Selector fields must be unique");
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if self
|
||||
.maximum_spans
|
||||
.saturating_mul(self.field_count())
|
||||
.saturating_mul(SPAN_SELECTOR_FIELD_VALUE_MAX_CHARS)
|
||||
> SPAN_SELECTOR_OUTPUT_MAX_CHARS
|
||||
{
|
||||
return Err("Span Selector output budget exceeds 40000 characters");
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
}
|
||||
|
||||
fn is_valid_selector_filter(filter: &serde_json::Value) -> bool {
|
||||
if filter.is_null()
|
||||
|| filter.as_object().is_some_and(|value| value.is_empty())
|
||||
|| filter
|
||||
.get("type")
|
||||
.and_then(serde_json::Value::as_str)
|
||||
.is_some_and(|value| value.eq_ignore_ascii_case("all"))
|
||||
{
|
||||
return true;
|
||||
}
|
||||
|
||||
if let Ok(condition) = serde_json::from_value::<ConditionParams>(filter.clone()) {
|
||||
return match condition {
|
||||
ConditionParams::V1 { conditions } => conditions.has_conditions(),
|
||||
ConditionParams::V2 { conditions } => conditions.validate().is_ok(),
|
||||
};
|
||||
}
|
||||
if let Ok(conditions) = serde_json::from_value::<ConditionGroup>(filter.clone()) {
|
||||
return conditions.validate().is_ok();
|
||||
}
|
||||
serde_json::from_value::<ConditionList>(filter.clone())
|
||||
.is_ok_and(|conditions| conditions.has_conditions())
|
||||
}
|
||||
|
||||
#[derive(Clone, Copy, Debug, Default, PartialEq, Eq, Serialize, Deserialize)]
|
||||
#[serde(rename_all = "snake_case")]
|
||||
pub enum SamplingMode {
|
||||
@@ -269,87 +146,6 @@ impl std::fmt::Display for TargetScope {
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Clone, Debug, PartialEq, Serialize, Deserialize)]
|
||||
#[serde(default, rename_all = "camelCase")]
|
||||
pub struct TraceEvalConfig {
|
||||
pub idle_window_secs: i64,
|
||||
pub max_age_secs: i64,
|
||||
pub end_signal: Option<serde_json::Value>,
|
||||
}
|
||||
|
||||
impl Default for TraceEvalConfig {
|
||||
fn default() -> Self {
|
||||
Self {
|
||||
idle_window_secs: DEFAULT_TRACE_IDLE_WINDOW_SECS,
|
||||
max_age_secs: DEFAULT_TRACE_MAX_AGE_SECS,
|
||||
end_signal: None,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
impl TraceEvalConfig {
|
||||
pub fn validate(&self) -> Result<(), &'static str> {
|
||||
validate_completion_window(self.idle_window_secs, self.max_age_secs)?;
|
||||
validate_end_signal(self.end_signal.as_ref())
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Clone, Debug, PartialEq, Serialize, Deserialize)]
|
||||
#[serde(default, rename_all = "camelCase")]
|
||||
pub struct SessionEvalConfig {
|
||||
pub idle_window_secs: i64,
|
||||
pub max_age_secs: i64,
|
||||
pub end_signal: Option<serde_json::Value>,
|
||||
}
|
||||
|
||||
impl Default for SessionEvalConfig {
|
||||
fn default() -> Self {
|
||||
Self {
|
||||
idle_window_secs: DEFAULT_SESSION_IDLE_WINDOW_SECS,
|
||||
max_age_secs: DEFAULT_SESSION_MAX_AGE_SECS,
|
||||
end_signal: None,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
impl SessionEvalConfig {
|
||||
pub fn validate(&self) -> Result<(), &'static str> {
|
||||
validate_completion_window(self.idle_window_secs, self.max_age_secs)?;
|
||||
validate_end_signal(self.end_signal.as_ref())
|
||||
}
|
||||
}
|
||||
|
||||
fn validate_end_signal(end_signal: Option<&serde_json::Value>) -> Result<(), &'static str> {
|
||||
let Some(end_signal) = end_signal else {
|
||||
return Ok(());
|
||||
};
|
||||
|
||||
let condition = serde_json::from_value::<ConditionParams>(end_signal.clone())
|
||||
.map_err(|_| "End signal must be a valid condition")?;
|
||||
|
||||
match condition {
|
||||
ConditionParams::V1 { conditions } if conditions.has_conditions() => Ok(()),
|
||||
ConditionParams::V2 { conditions } if conditions.validate().is_ok() => Ok(()),
|
||||
_ => Err("End signal must contain at least one condition"),
|
||||
}
|
||||
}
|
||||
|
||||
fn validate_completion_window(
|
||||
idle_window_secs: i64,
|
||||
max_age_secs: i64,
|
||||
) -> Result<(), &'static str> {
|
||||
if idle_window_secs < MIN_COMPLETION_IDLE_WINDOW_SECS {
|
||||
return Err("Completion idle window must be at least 45 seconds");
|
||||
}
|
||||
if max_age_secs <= 0 {
|
||||
return Err("Completion max age must be greater than zero");
|
||||
}
|
||||
if idle_window_secs > max_age_secs {
|
||||
return Err("Completion idle window cannot exceed max age");
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// Valid state transitions. Maps from current state to allowed next states.
|
||||
pub fn is_valid_transition(from: &str, to: &str) -> bool {
|
||||
matches!(
|
||||
@@ -1080,7 +876,10 @@ mod tests {
|
||||
let job = OnlineEvalJob::from(model);
|
||||
|
||||
assert_eq!(job.target_scope, TargetScope::Trace);
|
||||
assert_eq!(job.trace_config.as_ref().unwrap().idle_window_secs, 120);
|
||||
assert_eq!(
|
||||
job.trace_config.as_ref().unwrap().idle_window_secs,
|
||||
DEFAULT_TRACE_IDLE_WINDOW_SECS
|
||||
);
|
||||
assert!(job.session_config.is_none());
|
||||
assert!(!job.uses_hidden_pipeline());
|
||||
}
|
||||
@@ -1121,7 +920,10 @@ mod tests {
|
||||
assert!(job.validate_target_scope().is_err());
|
||||
job.session_config = None;
|
||||
job.apply_target_scope_defaults();
|
||||
assert_eq!(job.session_config.as_ref().unwrap().idle_window_secs, 120);
|
||||
assert_eq!(
|
||||
job.session_config.as_ref().unwrap().idle_window_secs,
|
||||
DEFAULT_SESSION_IDLE_WINDOW_SECS
|
||||
);
|
||||
assert_eq!(
|
||||
job.session_config.as_ref().unwrap().max_age_secs,
|
||||
DEFAULT_SESSION_MAX_AGE_SECS
|
||||
@@ -1151,52 +953,6 @@ mod tests {
|
||||
assert!(job.session_config.is_none());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_completion_config_enforces_scheduler_poll_interval_minimum() {
|
||||
let below_minimum = TraceEvalConfig {
|
||||
idle_window_secs: MIN_COMPLETION_IDLE_WINDOW_SECS - 1,
|
||||
..TraceEvalConfig::default()
|
||||
};
|
||||
assert_eq!(
|
||||
below_minimum.validate(),
|
||||
Err("Completion idle window must be at least 45 seconds")
|
||||
);
|
||||
|
||||
let at_minimum = TraceEvalConfig {
|
||||
idle_window_secs: MIN_COMPLETION_IDLE_WINDOW_SECS,
|
||||
..TraceEvalConfig::default()
|
||||
};
|
||||
assert!(at_minimum.validate().is_ok());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_completion_config_rejects_invalid_or_empty_end_signal() {
|
||||
let invalid = TraceEvalConfig {
|
||||
end_signal: Some(serde_json::json!({"field": "status"})),
|
||||
..TraceEvalConfig::default()
|
||||
};
|
||||
assert_eq!(
|
||||
invalid.validate(),
|
||||
Err("End signal must be a valid condition")
|
||||
);
|
||||
|
||||
let empty = SessionEvalConfig {
|
||||
end_signal: Some(serde_json::json!({
|
||||
"version": 2,
|
||||
"conditions": {
|
||||
"filterType": "group",
|
||||
"logicalOperator": "AND",
|
||||
"conditions": []
|
||||
}
|
||||
})),
|
||||
..SessionEvalConfig::default()
|
||||
};
|
||||
assert_eq!(
|
||||
empty.validate(),
|
||||
Err("End signal must contain at least one condition")
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_non_span_scope_requires_trace_stream() {
|
||||
let mut model = make_model();
|
||||
@@ -1271,77 +1027,6 @@ mod tests {
|
||||
job
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_span_selector_default_schema_and_camel_case_wire_shape() {
|
||||
let selector = SpanSelector {
|
||||
id: "selector-1".to_string(),
|
||||
name: "default".to_string(),
|
||||
..SpanSelector::default()
|
||||
};
|
||||
|
||||
assert_eq!(selector.field_count(), DEFAULT_SPAN_SELECTOR_FIELDS.len());
|
||||
assert!(selector.validate().is_ok());
|
||||
|
||||
let value = serde_json::to_value(selector).unwrap();
|
||||
assert_eq!(value["fieldMode"], "default");
|
||||
assert_eq!(value["maximumSpans"], DEFAULT_SPAN_SELECTOR_MAXIMUM_SPANS);
|
||||
assert!(value.get("filterCondition").is_some());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_span_selector_rejects_invalid_custom_schema_and_output_budget() {
|
||||
let mut selector = SpanSelector {
|
||||
id: "selector-1".to_string(),
|
||||
name: "custom".to_string(),
|
||||
field_mode: SpanSelectorFieldMode::Custom,
|
||||
fields: Vec::new(),
|
||||
maximum_spans: 1,
|
||||
..SpanSelector::default()
|
||||
};
|
||||
|
||||
assert_eq!(
|
||||
selector.validate(),
|
||||
Err("Custom Span Selector schema requires at least one field")
|
||||
);
|
||||
|
||||
selector.fields = vec!["name".to_string(), "name".to_string()];
|
||||
assert_eq!(
|
||||
selector.validate(),
|
||||
Err("Span Selector fields must be unique")
|
||||
);
|
||||
|
||||
selector.fields = (0..9).map(|idx| format!("field_{idx}")).collect();
|
||||
selector.maximum_spans = 5;
|
||||
assert_eq!(
|
||||
selector.validate(),
|
||||
Err("Span Selector output budget exceeds 40000 characters")
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_span_selector_rejects_an_incomplete_filter_condition() {
|
||||
let selector = SpanSelector {
|
||||
id: "selector-1".to_string(),
|
||||
name: "invalid-filter".to_string(),
|
||||
filter_condition: serde_json::json!({
|
||||
"filterType": "group",
|
||||
"logicalOperator": "AND",
|
||||
"conditions": [{
|
||||
"filterType": "condition",
|
||||
"column": "span_status",
|
||||
"operator": "=",
|
||||
"logicalOperator": "AND"
|
||||
}]
|
||||
}),
|
||||
..SpanSelector::default()
|
||||
};
|
||||
|
||||
assert_eq!(
|
||||
selector.validate(),
|
||||
Err("Span Selector filterCondition is invalid")
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_trace_activation_requires_a_binding_only_for_scorers_using_spans() {
|
||||
let mut job = trace_job_with_selector();
|
||||
@@ -0,0 +1,223 @@
|
||||
// Copyright 2026 OpenObserve Inc.
|
||||
//
|
||||
// This program is free software: you can redistribute it and/or modify
|
||||
// it under the terms of the GNU Affero General Public License as published by
|
||||
// the Free Software Foundation, either version 3 of the License, or
|
||||
// (at your option) any later version.
|
||||
//
|
||||
// This program is distributed in the hope that it will be useful
|
||||
// but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
// GNU Affero General Public License for more details.
|
||||
//
|
||||
// You should have received a copy of the GNU Affero General Public License
|
||||
// along with this program. If not, see <http://www.gnu.org/licenses/>.
|
||||
|
||||
//! Span Selectors: named, filtered span-extraction schemas that trace eval
|
||||
//! jobs bind to scorers consuming `{{ spans }}`.
|
||||
|
||||
use std::collections::BTreeMap;
|
||||
|
||||
use config::meta::{
|
||||
alerts::{ConditionGroup, ConditionList},
|
||||
pipeline::components::ConditionParams,
|
||||
};
|
||||
use serde::{Deserialize, Serialize};
|
||||
|
||||
pub type SpanSelectorBindings = BTreeMap<String, String>;
|
||||
|
||||
pub const DEFAULT_SPAN_SELECTOR_MAXIMUM_SPANS: usize = 5;
|
||||
pub const SPAN_SELECTOR_FIELD_VALUE_MAX_CHARS: usize = 1_000;
|
||||
pub const SPAN_SELECTOR_OUTPUT_MAX_CHARS: usize = 40_000;
|
||||
pub const DEFAULT_SPAN_SELECTOR_FIELDS: [&str; 8] = [
|
||||
"name",
|
||||
"status",
|
||||
"gen_ai_tool_name",
|
||||
"gen_ai_tool_call_id",
|
||||
"gen_ai_tool_call_arguments",
|
||||
"gen_ai_tool_call_result",
|
||||
"gen_ai_input_messages",
|
||||
"gen_ai_output_messages",
|
||||
];
|
||||
|
||||
#[derive(Clone, Copy, Debug, Default, PartialEq, Eq, Serialize, Deserialize)]
|
||||
#[serde(rename_all = "snake_case")]
|
||||
pub enum SpanSelectorFieldMode {
|
||||
#[default]
|
||||
Default,
|
||||
Custom,
|
||||
}
|
||||
|
||||
#[derive(Clone, Debug, PartialEq, Serialize, Deserialize)]
|
||||
#[serde(default, rename_all = "camelCase")]
|
||||
pub struct SpanSelector {
|
||||
pub id: String,
|
||||
pub name: String,
|
||||
pub filter_condition: serde_json::Value,
|
||||
pub field_mode: SpanSelectorFieldMode,
|
||||
pub fields: Vec<String>,
|
||||
pub maximum_spans: usize,
|
||||
}
|
||||
|
||||
impl Default for SpanSelector {
|
||||
fn default() -> Self {
|
||||
Self {
|
||||
id: String::new(),
|
||||
name: String::new(),
|
||||
filter_condition: serde_json::json!({}),
|
||||
field_mode: SpanSelectorFieldMode::Default,
|
||||
fields: Vec::new(),
|
||||
maximum_spans: DEFAULT_SPAN_SELECTOR_MAXIMUM_SPANS,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
impl SpanSelector {
|
||||
pub fn field_count(&self) -> usize {
|
||||
match self.field_mode {
|
||||
SpanSelectorFieldMode::Default => DEFAULT_SPAN_SELECTOR_FIELDS.len(),
|
||||
SpanSelectorFieldMode::Custom => self.fields.len(),
|
||||
}
|
||||
}
|
||||
|
||||
pub fn validate(&self) -> Result<(), &'static str> {
|
||||
if self.id.trim().is_empty() {
|
||||
return Err("Span Selector id cannot be empty");
|
||||
}
|
||||
if self.name.trim().is_empty() {
|
||||
return Err("Span Selector name cannot be empty");
|
||||
}
|
||||
if self.maximum_spans == 0 {
|
||||
return Err("Span Selector maximumSpans must be greater than zero");
|
||||
}
|
||||
if !is_valid_selector_filter(&self.filter_condition) {
|
||||
return Err("Span Selector filterCondition is invalid");
|
||||
}
|
||||
|
||||
if matches!(self.field_mode, SpanSelectorFieldMode::Custom) {
|
||||
if self.fields.is_empty() {
|
||||
return Err("Custom Span Selector schema requires at least one field");
|
||||
}
|
||||
let mut fields = std::collections::BTreeSet::new();
|
||||
for field in &self.fields {
|
||||
let field = field.trim();
|
||||
if field.is_empty() {
|
||||
return Err("Span Selector field names cannot be empty");
|
||||
}
|
||||
if !fields.insert(field) {
|
||||
return Err("Span Selector fields must be unique");
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if self
|
||||
.maximum_spans
|
||||
.saturating_mul(self.field_count())
|
||||
.saturating_mul(SPAN_SELECTOR_FIELD_VALUE_MAX_CHARS)
|
||||
> SPAN_SELECTOR_OUTPUT_MAX_CHARS
|
||||
{
|
||||
return Err("Span Selector output budget exceeds 40000 characters");
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
}
|
||||
|
||||
fn is_valid_selector_filter(filter: &serde_json::Value) -> bool {
|
||||
if filter.is_null()
|
||||
|| filter.as_object().is_some_and(|value| value.is_empty())
|
||||
|| filter
|
||||
.get("type")
|
||||
.and_then(serde_json::Value::as_str)
|
||||
.is_some_and(|value| value.eq_ignore_ascii_case("all"))
|
||||
{
|
||||
return true;
|
||||
}
|
||||
|
||||
if let Ok(condition) = serde_json::from_value::<ConditionParams>(filter.clone()) {
|
||||
return match condition {
|
||||
ConditionParams::V1 { conditions } => conditions.has_conditions(),
|
||||
ConditionParams::V2 { conditions } => conditions.validate().is_ok(),
|
||||
};
|
||||
}
|
||||
if let Ok(conditions) = serde_json::from_value::<ConditionGroup>(filter.clone()) {
|
||||
return conditions.validate().is_ok();
|
||||
}
|
||||
serde_json::from_value::<ConditionList>(filter.clone())
|
||||
.is_ok_and(|conditions| conditions.has_conditions())
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
#[test]
|
||||
fn test_span_selector_default_schema_and_camel_case_wire_shape() {
|
||||
let selector = SpanSelector {
|
||||
id: "selector-1".to_string(),
|
||||
name: "default".to_string(),
|
||||
..SpanSelector::default()
|
||||
};
|
||||
|
||||
assert_eq!(selector.field_count(), DEFAULT_SPAN_SELECTOR_FIELDS.len());
|
||||
assert!(selector.validate().is_ok());
|
||||
|
||||
let value = serde_json::to_value(selector).unwrap();
|
||||
assert_eq!(value["fieldMode"], "default");
|
||||
assert_eq!(value["maximumSpans"], DEFAULT_SPAN_SELECTOR_MAXIMUM_SPANS);
|
||||
assert!(value.get("filterCondition").is_some());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_span_selector_rejects_invalid_custom_schema_and_output_budget() {
|
||||
let mut selector = SpanSelector {
|
||||
id: "selector-1".to_string(),
|
||||
name: "custom".to_string(),
|
||||
field_mode: SpanSelectorFieldMode::Custom,
|
||||
fields: Vec::new(),
|
||||
maximum_spans: 1,
|
||||
..SpanSelector::default()
|
||||
};
|
||||
|
||||
assert_eq!(
|
||||
selector.validate(),
|
||||
Err("Custom Span Selector schema requires at least one field")
|
||||
);
|
||||
|
||||
selector.fields = vec!["name".to_string(), "name".to_string()];
|
||||
assert_eq!(
|
||||
selector.validate(),
|
||||
Err("Span Selector fields must be unique")
|
||||
);
|
||||
|
||||
selector.fields = (0..9).map(|idx| format!("field_{idx}")).collect();
|
||||
selector.maximum_spans = 5;
|
||||
assert_eq!(
|
||||
selector.validate(),
|
||||
Err("Span Selector output budget exceeds 40000 characters")
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_span_selector_rejects_an_incomplete_filter_condition() {
|
||||
let selector = SpanSelector {
|
||||
id: "selector-1".to_string(),
|
||||
name: "invalid-filter".to_string(),
|
||||
filter_condition: serde_json::json!({
|
||||
"filterType": "group",
|
||||
"logicalOperator": "AND",
|
||||
"conditions": [{
|
||||
"filterType": "condition",
|
||||
"column": "span_status",
|
||||
"operator": "=",
|
||||
"logicalOperator": "AND"
|
||||
}]
|
||||
}),
|
||||
..SpanSelector::default()
|
||||
};
|
||||
|
||||
assert_eq!(
|
||||
selector.validate(),
|
||||
Err("Span Selector filterCondition is invalid")
|
||||
);
|
||||
}
|
||||
}
|
||||
+26
-38
@@ -27,20 +27,6 @@ use {
|
||||
|
||||
use crate::common::meta::user::{UserOrgRole, UserRequest};
|
||||
|
||||
#[cfg(feature = "enterprise")]
|
||||
fn eval_scheduler_fetch_size(total: usize, max_rows: usize) -> anyhow::Result<i64> {
|
||||
if total > max_rows {
|
||||
return Err(anyhow::anyhow!(
|
||||
"online eval scheduler interval has {total} rows, exceeding the safe limit of {max_rows}"
|
||||
));
|
||||
}
|
||||
let fetch_size = total
|
||||
.checked_add(1)
|
||||
.ok_or_else(|| anyhow::anyhow!("online eval scheduler result count exceeds search size"))?;
|
||||
i64::try_from(fetch_size)
|
||||
.map_err(|_| anyhow::anyhow!("online eval scheduler result count exceeds search size"))
|
||||
}
|
||||
|
||||
#[cfg(feature = "enterprise")]
|
||||
pub mod alert_grouping;
|
||||
mod alert_manager;
|
||||
@@ -699,6 +685,15 @@ pub async fn init() -> Result<(), anyhow::Error> {
|
||||
},
|
||||
);
|
||||
|
||||
let llm_eval_config =
|
||||
&o2_enterprise::enterprise::common::config::get_config().llm_eval_config;
|
||||
infra::table::online_eval_jobs::init_completion_window_defaults(
|
||||
llm_eval_config.trace_idle_timeout_secs,
|
||||
llm_eval_config.trace_max_age_secs,
|
||||
llm_eval_config.session_idle_timeout_secs,
|
||||
llm_eval_config.session_max_age_secs,
|
||||
);
|
||||
|
||||
o2_enterprise::enterprise::llm_evaluations::eval_jobs::async_executor::register_evaluator_trace_exporter(
|
||||
|org_id, traces, node_idx| {
|
||||
Box::pin(async move {
|
||||
@@ -718,7 +713,7 @@ pub async fn init() -> Result<(), anyhow::Error> {
|
||||
);
|
||||
|
||||
o2_enterprise::enterprise::llm_evaluations::eval_jobs::scheduler::register_search_executor(
|
||||
|org_id, stream_type, sql, start_time, end_time| {
|
||||
|org_id, stream_type, sql, start_time, end_time, truncate_over_limit| {
|
||||
Box::pin(async move {
|
||||
let count_req = config::meta::search::Request {
|
||||
query: config::meta::search::Query {
|
||||
@@ -742,7 +737,7 @@ pub async fn init() -> Result<(), anyhow::Error> {
|
||||
total: count_response.total,
|
||||
is_partial: count_response.is_partial,
|
||||
function_error: count_response.function_error,
|
||||
hits: Vec::new(),
|
||||
..Default::default()
|
||||
});
|
||||
}
|
||||
|
||||
@@ -750,12 +745,20 @@ pub async fn init() -> Result<(), anyhow::Error> {
|
||||
if total == 0 {
|
||||
return Ok(Default::default());
|
||||
}
|
||||
let max_rows = o2_enterprise::enterprise::common::config::get_config()
|
||||
.llm_eval_config
|
||||
.max_buffer_size
|
||||
.saturating_mul(10)
|
||||
.max(10_001);
|
||||
let size = eval_scheduler_fetch_size(total, max_rows)?;
|
||||
let max_rows = o2_enterprise::enterprise::llm_evaluations::eval_jobs::scheduler::EVAL_DETECTION_MAX_ROWS;
|
||||
let over_limit = total > max_rows;
|
||||
if over_limit && !truncate_over_limit {
|
||||
// The scheduler splits the window in half rather than
|
||||
// fetching an oversized result set.
|
||||
return Ok(o2_enterprise::enterprise::llm_evaluations::eval_jobs::scheduler::SchedulerSearchResult {
|
||||
total,
|
||||
over_limit: true,
|
||||
..Default::default()
|
||||
});
|
||||
}
|
||||
// total <= max_rows here; fetch one extra row so a count
|
||||
// that grew after the count query fails closed downstream.
|
||||
let size = (if over_limit { max_rows } else { total + 1 }) as i64;
|
||||
let data_req = config::meta::search::Request {
|
||||
query: config::meta::search::Query {
|
||||
sql,
|
||||
@@ -776,6 +779,7 @@ pub async fn init() -> Result<(), anyhow::Error> {
|
||||
total,
|
||||
is_partial: response.is_partial,
|
||||
function_error: response.function_error,
|
||||
over_limit,
|
||||
}
|
||||
})
|
||||
.map_err(|e| anyhow::anyhow!(e))
|
||||
@@ -1178,19 +1182,3 @@ pub async fn init_deferred() -> Result<(), anyhow::Error> {
|
||||
|
||||
Ok(())
|
||||
}
|
||||
|
||||
#[cfg(all(test, feature = "enterprise"))]
|
||||
mod tests {
|
||||
use super::eval_scheduler_fetch_size;
|
||||
|
||||
#[test]
|
||||
fn eval_scheduler_fetches_one_row_past_the_count() {
|
||||
assert_eq!(eval_scheduler_fetch_size(10_001, 100_000).unwrap(), 10_002);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn eval_scheduler_rejects_intervals_above_the_safe_limit() {
|
||||
let error = eval_scheduler_fetch_size(100_001, 100_000).unwrap_err();
|
||||
assert!(error.to_string().contains("exceeding the safe limit"));
|
||||
}
|
||||
}
|
||||
|
||||
@@ -22,6 +22,7 @@ import { z } from "zod";
|
||||
import {
|
||||
MIN_COMPLETION_IDLE_WINDOW_SECS,
|
||||
TRACE_COMPLETION_WINDOW_DEFAULTS,
|
||||
completionWindowLimitsForScope,
|
||||
} from "../utils/completionWindow";
|
||||
|
||||
const requiredText = (message: string) =>
|
||||
@@ -69,6 +70,34 @@ export const makeJobFormSchema = (t: (_key: string) => string) =>
|
||||
message: t("onlineEvals.job.validation.samplingValueRequired"),
|
||||
});
|
||||
}
|
||||
|
||||
// Per-scope hard ceilings, mirroring the backend guard rails.
|
||||
const limits = completionWindowLimitsForScope(val.targetScope);
|
||||
if (limits) {
|
||||
const isTrace = val.targetScope === "trace";
|
||||
if (val.idleWindowSecs > limits.maxIdleWindowSecs) {
|
||||
ctx.addIssue({
|
||||
code: z.ZodIssueCode.custom,
|
||||
path: ["idleWindowSecs"],
|
||||
message: t(
|
||||
isTrace
|
||||
? "onlineEvals.job.validation.idleWindowMaximumTrace"
|
||||
: "onlineEvals.job.validation.idleWindowMaximumSession",
|
||||
),
|
||||
});
|
||||
}
|
||||
if (val.maxAgeSecs > limits.maxAgeSecs) {
|
||||
ctx.addIssue({
|
||||
code: z.ZodIssueCode.custom,
|
||||
path: ["maxAgeSecs"],
|
||||
message: t(
|
||||
isTrace
|
||||
? "onlineEvals.job.validation.maxAgeMaximumTrace"
|
||||
: "onlineEvals.job.validation.maxAgeMaximumSession",
|
||||
),
|
||||
});
|
||||
}
|
||||
}
|
||||
});
|
||||
|
||||
export type JobForm = z.infer<ReturnType<typeof makeJobFormSchema>>;
|
||||
|
||||
@@ -266,6 +266,9 @@ describe("JobFormPage", () => {
|
||||
wrapper = createWrapper();
|
||||
|
||||
setField(wrapper, "targetScope", "trace");
|
||||
// Let the scope watcher apply its defaults before overriding them, so the
|
||||
// explicit values below are what the help text renders.
|
||||
await wrapper.vm.$nextTick();
|
||||
setField(wrapper, "idleWindowSecs", 120);
|
||||
setField(wrapper, "maxAgeSecs", 1800);
|
||||
await wrapper.vm.$nextTick();
|
||||
@@ -274,19 +277,19 @@ describe("JobFormPage", () => {
|
||||
expect(wrapper.find('[data-test="job-form-max-age-input"]').text()).toContain("30 min");
|
||||
});
|
||||
|
||||
it("defaults trace and session idle windows to 120 seconds", async () => {
|
||||
it("defaults idle windows per scope: 30s for traces, 30m for sessions", async () => {
|
||||
wrapper = createWrapper();
|
||||
|
||||
setField(wrapper, "targetScope", "trace");
|
||||
await wrapper.vm.$nextTick();
|
||||
expect(oform(wrapper).form.state.values.idleWindowSecs).toBe(120);
|
||||
expect(oform(wrapper).form.state.values.idleWindowSecs).toBe(30);
|
||||
|
||||
setField(wrapper, "targetScope", "session");
|
||||
await wrapper.vm.$nextTick();
|
||||
expect(oform(wrapper).form.state.values.idleWindowSecs).toBe(120);
|
||||
expect(oform(wrapper).form.state.values.idleWindowSecs).toBe(1800);
|
||||
});
|
||||
|
||||
it("rejects completion idle windows below 45 seconds", async () => {
|
||||
it("rejects non-positive completion idle windows", async () => {
|
||||
wrapper = createWrapper();
|
||||
setField(wrapper, "name", "trace-job");
|
||||
setField(wrapper, "stream", "default");
|
||||
@@ -294,7 +297,24 @@ describe("JobFormPage", () => {
|
||||
await wrapper.vm.$nextTick();
|
||||
setField(wrapper, "scorerIds", ["s1"]);
|
||||
setField(wrapper, "samplingMode", "all");
|
||||
setField(wrapper, "idleWindowSecs", 44);
|
||||
setField(wrapper, "idleWindowSecs", 0);
|
||||
|
||||
await submit(wrapper);
|
||||
|
||||
expect(oform(wrapper).form.state.isValid).toBe(false);
|
||||
expect(onlineEvalsService.jobs.create).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it("rejects completion windows above the per-scope hard caps", async () => {
|
||||
wrapper = createWrapper();
|
||||
setField(wrapper, "name", "trace-job");
|
||||
setField(wrapper, "stream", "default");
|
||||
setField(wrapper, "targetScope", "trace");
|
||||
await wrapper.vm.$nextTick();
|
||||
setField(wrapper, "scorerIds", ["s1"]);
|
||||
setField(wrapper, "samplingMode", "all");
|
||||
// A 1-hour trace idle window is beyond the 30-minute trace cap.
|
||||
setField(wrapper, "idleWindowSecs", 60 * 60);
|
||||
|
||||
await submit(wrapper);
|
||||
|
||||
|
||||
@@ -223,6 +223,7 @@
|
||||
:label="t('onlineEvals.job.idleWindowLabel')"
|
||||
type="number"
|
||||
:min="MIN_COMPLETION_IDLE_WINDOW_SECS"
|
||||
:max="completionLimits?.maxIdleWindowSecs"
|
||||
size="sm"
|
||||
:help-text="idleWindowHelp"
|
||||
data-test="job-form-idle-window-input"
|
||||
@@ -234,6 +235,7 @@
|
||||
:label="t('onlineEvals.job.maxAgeLabel')"
|
||||
type="number"
|
||||
min="1"
|
||||
:max="completionLimits?.maxAgeSecs"
|
||||
size="sm"
|
||||
:help-text="maxAgeHelp"
|
||||
data-test="job-form-max-age-input"
|
||||
@@ -366,6 +368,7 @@ import {
|
||||
buildCompletionConfigPayloads,
|
||||
completionWindowConfigFromJob,
|
||||
completionWindowDefaultsForScope,
|
||||
completionWindowLimitsForScope,
|
||||
durationPartsFromSecs,
|
||||
MIN_COMPLETION_IDLE_WINDOW_SECS,
|
||||
TRACE_COMPLETION_WINDOW_DEFAULTS,
|
||||
@@ -483,6 +486,10 @@ function humanizeSecs(value: string | number): string | null {
|
||||
|
||||
// NOTE: `formValues` is a store ref; script reads need `.value` (templates
|
||||
// auto-unwrap, script does not).
|
||||
const completionLimits = computed(() =>
|
||||
completionWindowLimitsForScope(formValues.value.targetScope),
|
||||
);
|
||||
|
||||
const idleWindowHelp = computed<string>(() => {
|
||||
const duration = humanizeSecs(formValues.value.idleWindowSecs);
|
||||
return duration
|
||||
|
||||
@@ -28,24 +28,24 @@ const endSignal = {
|
||||
};
|
||||
|
||||
describe("completionWindowDefaultsForScope", () => {
|
||||
it("uses the documented 2-minute / 30-minute trace window", () => {
|
||||
it("uses the documented 30-second / 30-minute trace window", () => {
|
||||
expect(completionWindowDefaultsForScope("trace")).toEqual({
|
||||
idleWindowSecs: 120,
|
||||
idleWindowSecs: 30,
|
||||
maxAgeSecs: 1800,
|
||||
});
|
||||
expect(TRACE_COMPLETION_WINDOW_DEFAULTS).toEqual({
|
||||
idleWindowSecs: 120,
|
||||
idleWindowSecs: 30,
|
||||
maxAgeSecs: 1800,
|
||||
});
|
||||
});
|
||||
|
||||
it("uses the documented 2-minute / 4-hour session window", () => {
|
||||
it("uses the documented 30-minute / 4-hour session window", () => {
|
||||
expect(completionWindowDefaultsForScope("session")).toEqual({
|
||||
idleWindowSecs: 120,
|
||||
idleWindowSecs: 1800,
|
||||
maxAgeSecs: 14400,
|
||||
});
|
||||
expect(SESSION_COMPLETION_WINDOW_DEFAULTS).toEqual({
|
||||
idleWindowSecs: 120,
|
||||
idleWindowSecs: 1800,
|
||||
maxAgeSecs: 14400,
|
||||
});
|
||||
});
|
||||
|
||||
@@ -11,15 +11,18 @@ export interface CompletionWindowDefaults {
|
||||
}
|
||||
|
||||
// LLM Observability Phase 2.6 §8 Parameter Baselines.
|
||||
export const MIN_COMPLETION_IDLE_WINDOW_SECS = 45;
|
||||
// Any positive idle window is valid: the scheduler fires once the observed
|
||||
// silence reaches the window, quantized to its scan pass. The floor only
|
||||
// rejects zero/negative values.
|
||||
export const MIN_COMPLETION_IDLE_WINDOW_SECS = 1;
|
||||
|
||||
export const TRACE_COMPLETION_WINDOW_DEFAULTS: CompletionWindowDefaults = {
|
||||
idleWindowSecs: 2 * 60,
|
||||
idleWindowSecs: 30,
|
||||
maxAgeSecs: 30 * 60,
|
||||
};
|
||||
|
||||
export const SESSION_COMPLETION_WINDOW_DEFAULTS: CompletionWindowDefaults = {
|
||||
idleWindowSecs: 2 * 60,
|
||||
idleWindowSecs: 30 * 60,
|
||||
maxAgeSecs: 4 * 60 * 60,
|
||||
};
|
||||
|
||||
@@ -31,6 +34,31 @@ export function completionWindowDefaultsForScope(
|
||||
return null;
|
||||
}
|
||||
|
||||
export interface CompletionWindowLimits {
|
||||
maxIdleWindowSecs: number;
|
||||
maxAgeSecs: number;
|
||||
}
|
||||
|
||||
// Hard per-scope ceilings, mirroring the backend guard rails: max age bounds
|
||||
// how long the scheduler holds a pending target in memory.
|
||||
export const TRACE_COMPLETION_WINDOW_LIMITS: CompletionWindowLimits = {
|
||||
maxIdleWindowSecs: 30 * 60,
|
||||
maxAgeSecs: 2 * 60 * 60,
|
||||
};
|
||||
|
||||
export const SESSION_COMPLETION_WINDOW_LIMITS: CompletionWindowLimits = {
|
||||
maxIdleWindowSecs: 4 * 60 * 60,
|
||||
maxAgeSecs: 24 * 60 * 60,
|
||||
};
|
||||
|
||||
export function completionWindowLimitsForScope(
|
||||
targetScope: EvalTargetScope,
|
||||
): CompletionWindowLimits | null {
|
||||
if (targetScope === "trace") return TRACE_COMPLETION_WINDOW_LIMITS;
|
||||
if (targetScope === "session") return SESSION_COMPLETION_WINDOW_LIMITS;
|
||||
return null;
|
||||
}
|
||||
|
||||
export interface DurationParts {
|
||||
hours: number;
|
||||
minutes: number;
|
||||
|
||||
@@ -365,7 +365,7 @@ export default defineComponent({
|
||||
);
|
||||
|
||||
// Backend `/config` flag `online_evals_enabled` — controlled by
|
||||
// enterprise `O2_ONLINE_EVALS_ENABLED`. Reactive so the menu picks it up regardless
|
||||
// enterprise `O2_EVAL_ENABLED`. Reactive so the menu picks it up regardless
|
||||
// of whether the config response arrived before or after this component
|
||||
// mounted.
|
||||
const isOnlineEvalsEnabled = computed(() => {
|
||||
|
||||
@@ -8871,7 +8871,11 @@
|
||||
"validation": {
|
||||
"nameRequired": "Job name is required",
|
||||
"streamRequired": "A stream is required",
|
||||
"idleWindowMinimum": "Leerlauffenster muss mindestens 45 Sekunden betragen",
|
||||
"idleWindowMinimum": "Leerlauffenster muss mindestens 1 Sekunde betragen",
|
||||
"idleWindowMaximumTrace": "Trace-Leerlauffenster darf 30 Minuten nicht überschreiten",
|
||||
"idleWindowMaximumSession": "Sitzungs-Leerlauffenster darf 4 Stunden nicht überschreiten",
|
||||
"maxAgeMaximumTrace": "Maximales Trace-Alter darf 2 Stunden nicht überschreiten",
|
||||
"maxAgeMaximumSession": "Maximales Sitzungsalter darf 24 Stunden nicht überschreiten",
|
||||
"samplingValueRequired": "A sampling value is required"
|
||||
}
|
||||
}
|
||||
|
||||
@@ -9043,7 +9043,11 @@
|
||||
"validation": {
|
||||
"nameRequired": "Job name is required",
|
||||
"streamRequired": "A stream is required",
|
||||
"idleWindowMinimum": "Idle window must be at least 45 seconds",
|
||||
"idleWindowMinimum": "Idle window must be at least 1 second",
|
||||
"idleWindowMaximumTrace": "Trace idle window cannot exceed 30 minutes",
|
||||
"idleWindowMaximumSession": "Session idle window cannot exceed 4 hours",
|
||||
"maxAgeMaximumTrace": "Trace max age cannot exceed 2 hours",
|
||||
"maxAgeMaximumSession": "Session max age cannot exceed 24 hours",
|
||||
"samplingValueRequired": "A sampling value is required"
|
||||
}
|
||||
}
|
||||
|
||||
@@ -8871,7 +8871,11 @@
|
||||
"validation": {
|
||||
"nameRequired": "Job name is required",
|
||||
"streamRequired": "A stream is required",
|
||||
"idleWindowMinimum": "La ventana de inactividad debe ser de al menos 45 segundos",
|
||||
"idleWindowMinimum": "La ventana de inactividad debe ser de al menos 1 segundo",
|
||||
"idleWindowMaximumTrace": "La ventana de inactividad de trace no puede superar los 30 minutos",
|
||||
"idleWindowMaximumSession": "La ventana de inactividad de sesión no puede superar las 4 horas",
|
||||
"maxAgeMaximumTrace": "La antigüedad máxima de trace no puede superar las 2 horas",
|
||||
"maxAgeMaximumSession": "La antigüedad máxima de sesión no puede superar las 24 horas",
|
||||
"samplingValueRequired": "A sampling value is required"
|
||||
}
|
||||
}
|
||||
|
||||
@@ -8871,7 +8871,11 @@
|
||||
"validation": {
|
||||
"nameRequired": "Job name is required",
|
||||
"streamRequired": "A stream is required",
|
||||
"idleWindowMinimum": "La fenêtre d'inactivité doit être d'au moins 45 secondes",
|
||||
"idleWindowMinimum": "La fenêtre d'inactivité doit être d'au moins 1 seconde",
|
||||
"idleWindowMaximumTrace": "La fenêtre d'inactivité de trace ne peut pas dépasser 30 minutes",
|
||||
"idleWindowMaximumSession": "La fenêtre d'inactivité de session ne peut pas dépasser 4 heures",
|
||||
"maxAgeMaximumTrace": "L'âge maximal de trace ne peut pas dépasser 2 heures",
|
||||
"maxAgeMaximumSession": "L'âge maximal de session ne peut pas dépasser 24 heures",
|
||||
"samplingValueRequired": "A sampling value is required"
|
||||
}
|
||||
}
|
||||
|
||||
@@ -8871,7 +8871,11 @@
|
||||
"validation": {
|
||||
"nameRequired": "Job name is required",
|
||||
"streamRequired": "A stream is required",
|
||||
"idleWindowMinimum": "La finestra di inattività deve essere di almeno 45 secondi",
|
||||
"idleWindowMinimum": "La finestra di inattività deve essere di almeno 1 secondo",
|
||||
"idleWindowMaximumTrace": "La finestra di inattività della trace non può superare i 30 minuti",
|
||||
"idleWindowMaximumSession": "La finestra di inattività della sessione non può superare le 4 ore",
|
||||
"maxAgeMaximumTrace": "L'età massima della trace non può superare le 2 ore",
|
||||
"maxAgeMaximumSession": "L'età massima della sessione non può superare le 24 ore",
|
||||
"samplingValueRequired": "A sampling value is required"
|
||||
}
|
||||
}
|
||||
|
||||
@@ -8871,7 +8871,11 @@
|
||||
"validation": {
|
||||
"nameRequired": "Job name is required",
|
||||
"streamRequired": "A stream is required",
|
||||
"idleWindowMinimum": "アイドルウィンドウは45秒以上である必要があります",
|
||||
"idleWindowMinimum": "アイドルウィンドウは1秒以上である必要があります",
|
||||
"idleWindowMaximumTrace": "トレースのアイドルウィンドウは30分を超えることはできません",
|
||||
"idleWindowMaximumSession": "セッションのアイドルウィンドウは4時間を超えることはできません",
|
||||
"maxAgeMaximumTrace": "トレースの最大保持時間は2時間を超えることはできません",
|
||||
"maxAgeMaximumSession": "セッションの最大保持時間は24時間を超えることはできません",
|
||||
"samplingValueRequired": "A sampling value is required"
|
||||
}
|
||||
}
|
||||
|
||||
@@ -8871,7 +8871,11 @@
|
||||
"validation": {
|
||||
"nameRequired": "Job name is required",
|
||||
"streamRequired": "A stream is required",
|
||||
"idleWindowMinimum": "유휴 창은 최소 45초 이상이어야 합니다.",
|
||||
"idleWindowMinimum": "유휴 창은 최소 1초 이상이어야 합니다.",
|
||||
"idleWindowMaximumTrace": "트레이스 유휴 창은 30분을 초과할 수 없습니다",
|
||||
"idleWindowMaximumSession": "세션 유휴 창은 4시간을 초과할 수 없습니다",
|
||||
"maxAgeMaximumTrace": "트레이스 최대 기간은 2시간을 초과할 수 없습니다",
|
||||
"maxAgeMaximumSession": "세션 최대 기간은 24시간을 초과할 수 없습니다",
|
||||
"samplingValueRequired": "A sampling value is required"
|
||||
}
|
||||
}
|
||||
|
||||
@@ -8871,7 +8871,11 @@
|
||||
"validation": {
|
||||
"nameRequired": "Job name is required",
|
||||
"streamRequired": "A stream is required",
|
||||
"idleWindowMinimum": "Inactief venster moet ten minste 45 seconden zijn",
|
||||
"idleWindowMinimum": "Inactief venster moet ten minste 1 seconde zijn",
|
||||
"idleWindowMaximumTrace": "Trace-inactiviteitsvenster mag niet langer zijn dan 30 minuten",
|
||||
"idleWindowMaximumSession": "Sessie-inactiviteitsvenster mag niet langer zijn dan 4 uur",
|
||||
"maxAgeMaximumTrace": "Maximale trace-leeftijd mag niet langer zijn dan 2 uur",
|
||||
"maxAgeMaximumSession": "Maximale sessieleeftijd mag niet langer zijn dan 24 uur",
|
||||
"samplingValueRequired": "A sampling value is required"
|
||||
}
|
||||
}
|
||||
|
||||
@@ -8871,7 +8871,11 @@
|
||||
"validation": {
|
||||
"nameRequired": "Nazwa zadania jest wymagana",
|
||||
"streamRequired": "Strumień jest wymagany",
|
||||
"idleWindowMinimum": "Okno bezczynności musi wynosić co najmniej 45 sekund",
|
||||
"idleWindowMinimum": "Okno bezczynności musi wynosić co najmniej 1 sekundę",
|
||||
"idleWindowMaximumTrace": "Okno bezczynności trace nie może przekraczać 30 minut",
|
||||
"idleWindowMaximumSession": "Okno bezczynności sesji nie może przekraczać 4 godzin",
|
||||
"maxAgeMaximumTrace": "Maksymalny wiek trace nie może przekraczać 2 godzin",
|
||||
"maxAgeMaximumSession": "Maksymalny wiek sesji nie może przekraczać 24 godzin",
|
||||
"samplingValueRequired": "Wartość próbkowania jest wymagana"
|
||||
}
|
||||
}
|
||||
|
||||
@@ -8871,7 +8871,11 @@
|
||||
"validation": {
|
||||
"nameRequired": "Job name is required",
|
||||
"streamRequired": "A stream is required",
|
||||
"idleWindowMinimum": "A janela ociosa deve ter pelo menos 45 segundos",
|
||||
"idleWindowMinimum": "A janela ociosa deve ter pelo menos 1 segundo",
|
||||
"idleWindowMaximumTrace": "A janela ociosa de trace não pode exceder 30 minutos",
|
||||
"idleWindowMaximumSession": "A janela ociosa de sessão não pode exceder 4 horas",
|
||||
"maxAgeMaximumTrace": "A idade máxima de trace não pode exceder 2 horas",
|
||||
"maxAgeMaximumSession": "A idade máxima de sessão não pode exceder 24 horas",
|
||||
"samplingValueRequired": "A sampling value is required"
|
||||
}
|
||||
}
|
||||
|
||||
@@ -8871,7 +8871,11 @@
|
||||
"validation": {
|
||||
"nameRequired": "Имя задачи обязательно",
|
||||
"streamRequired": "Необходимо указать поток",
|
||||
"idleWindowMinimum": "Окно бездействия должно быть не менее 45 секунд",
|
||||
"idleWindowMinimum": "Окно бездействия должно быть не менее 1 секунды",
|
||||
"idleWindowMaximumTrace": "Окно бездействия трейса не может превышать 30 минут",
|
||||
"idleWindowMaximumSession": "Окно бездействия сессии не может превышать 4 часа",
|
||||
"maxAgeMaximumTrace": "Максимальный возраст трейса не может превышать 2 часа",
|
||||
"maxAgeMaximumSession": "Максимальный возраст сессии не может превышать 24 часа",
|
||||
"samplingValueRequired": "Необходимо указать значение семплирования"
|
||||
}
|
||||
}
|
||||
|
||||
@@ -8871,7 +8871,11 @@
|
||||
"validation": {
|
||||
"nameRequired": "Job name is required",
|
||||
"streamRequired": "A stream is required",
|
||||
"idleWindowMinimum": "Boşta kalma süresi en az 45 saniye olmalıdır",
|
||||
"idleWindowMinimum": "Boşta kalma süresi en az 1 saniye olmalıdır",
|
||||
"idleWindowMaximumTrace": "Trace boşta kalma süresi 30 dakikayı aşamaz",
|
||||
"idleWindowMaximumSession": "Oturum boşta kalma süresi 4 saati aşamaz",
|
||||
"maxAgeMaximumTrace": "Maksimum trace süresi 2 saati aşamaz",
|
||||
"maxAgeMaximumSession": "Maksimum oturum süresi 24 saati aşamaz",
|
||||
"samplingValueRequired": "A sampling value is required"
|
||||
}
|
||||
}
|
||||
|
||||
@@ -8871,7 +8871,11 @@
|
||||
"validation": {
|
||||
"nameRequired": "Tên công việc là bắt buộc",
|
||||
"streamRequired": "Một luồng là bắt buộc",
|
||||
"idleWindowMinimum": "Thời gian chờ tối thiểu phải là 45 giây",
|
||||
"idleWindowMinimum": "Thời gian chờ tối thiểu phải là 1 giây",
|
||||
"idleWindowMaximumTrace": "Cửa sổ chờ của trace không được vượt quá 30 phút",
|
||||
"idleWindowMaximumSession": "Cửa sổ chờ của phiên không được vượt quá 4 giờ",
|
||||
"maxAgeMaximumTrace": "Tuổi tối đa của trace không được vượt quá 2 giờ",
|
||||
"maxAgeMaximumSession": "Tuổi tối đa của phiên không được vượt quá 24 giờ",
|
||||
"samplingValueRequired": "Một giá trị lấy mẫu là bắt buộc"
|
||||
}
|
||||
}
|
||||
|
||||
@@ -8871,7 +8871,11 @@
|
||||
"validation": {
|
||||
"nameRequired": "Job name is required",
|
||||
"streamRequired": "A stream is required",
|
||||
"idleWindowMinimum": "空闲窗口必须至少为45秒",
|
||||
"idleWindowMinimum": "空闲窗口必须至少为1秒",
|
||||
"idleWindowMaximumTrace": "Trace 空闲窗口不能超过30分钟",
|
||||
"idleWindowMaximumSession": "会话空闲窗口不能超过4小时",
|
||||
"maxAgeMaximumTrace": "Trace 最大时长不能超过2小时",
|
||||
"maxAgeMaximumSession": "会话最大时长不能超过24小时",
|
||||
"samplingValueRequired": "A sampling value is required"
|
||||
}
|
||||
}
|
||||
|
||||
@@ -8866,7 +8866,11 @@
|
||||
"validation": {
|
||||
"nameRequired": "Job name is required",
|
||||
"streamRequired": "A stream is required",
|
||||
"idleWindowMinimum": "閒置視窗必須至少45秒",
|
||||
"idleWindowMinimum": "閒置視窗必須至少1秒",
|
||||
"idleWindowMaximumTrace": "Trace 閒置視窗不能超過30分鐘",
|
||||
"idleWindowMaximumSession": "工作階段閒置視窗不能超過4小時",
|
||||
"maxAgeMaximumTrace": "Trace 最大時長不能超過2小時",
|
||||
"maxAgeMaximumSession": "工作階段最大時長不能超過24小時",
|
||||
"samplingValueRequired": "A sampling value is required"
|
||||
}
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user