mirror of
https://github.com/openobserve/openobserve.git
synced 2026-11-03 16:10:21 +00:00
fix(synthetics): ownership supervisor, status-page error policy, step outcomes and callback diagnostics (o2-enterprise#3190, #3191, #3140, #3165) (#16033)
Co-authored-by: github-actions[bot] <41898282+github-actions[bot]@users.noreply.github.com>
This commit is contained in:
co-authored by
github-actions[bot]
parent
2bbf08ce78
commit
fc7fdcdcec
@@ -8261,6 +8261,7 @@
|
||||
"critical": "427dd2969bd140be",
|
||||
"degraded": "a8494c12f2243903",
|
||||
"down": "b86d11af79188e58",
|
||||
"error": "54a0e8c17ebb21a1",
|
||||
"failed": "031a8f0f659df890",
|
||||
"healthy": "7f1e323b6272c620",
|
||||
"offline": "a1794783aab72d20",
|
||||
@@ -25154,6 +25155,7 @@
|
||||
"durationFast": "cfd210c0d957d454",
|
||||
"durationMid": "6a8fa448ca025881",
|
||||
"durationSlow": "03e0b960ac4d0d17",
|
||||
"error": "54a0e8c17ebb21a1",
|
||||
"failed": "031a8f0f659df890",
|
||||
"passed": "436fe71bb9561f05",
|
||||
"warning": "e981ddae45d8f4ca"
|
||||
@@ -25698,11 +25700,13 @@
|
||||
"back": "7e411e6d1fb73865",
|
||||
"checkName": "dcd1d5223f73b3a9",
|
||||
"checksTitle": "10014025dca36813",
|
||||
"loadFailed": "0715f69c63f61005",
|
||||
"noAgents": "53f30015179ba65a",
|
||||
"noChecks": "3709b844e6f621e3",
|
||||
"notFound": "8df056a568a44b88",
|
||||
"pool": "f13024272c385f73",
|
||||
"restartConfig": "39cd3ae99887b0c9",
|
||||
"retry": "942087cc2d41e013",
|
||||
"title": "57d84785ba9c4e19",
|
||||
"version": "dd167905de0defca"
|
||||
},
|
||||
@@ -25925,6 +25929,10 @@
|
||||
"attempts": "06e70139fcf5e5c7",
|
||||
"detailAction": "64cff1319d2fd2cb",
|
||||
"detailSelector": "f02a172a35354ca2",
|
||||
"detailSkipReason": "52bbd7b4a76b81ed",
|
||||
"detailStatus": "920e413c7d411b61",
|
||||
"detailStepId": "22e1beecfe5cf990",
|
||||
"detailTargetUrl": "5b341c960fb7166e",
|
||||
"detailUrl": "e7a241debad56609",
|
||||
"downloadTrace": "0d79fab080c1efe2",
|
||||
"errorFullscreenTitle": "024c4be2f89daed1",
|
||||
@@ -25969,8 +25977,11 @@
|
||||
"settleStaleNote": "0aee2b4a2b2be6a0",
|
||||
"showFullError": "4450d34301ab8d70",
|
||||
"showLess": "94ea9b1d33a02975",
|
||||
"skippedCount": "28541aa9135d134e",
|
||||
"startLoadLabel": "db96939121284345",
|
||||
"stepOf": "9bd8548d86fd91f5",
|
||||
"stepStatusSkipped": "12698ce1ea5cd4ab",
|
||||
"stepStatusUnknown": "b764cdc0eab71374",
|
||||
"stepsCount": "6e336a2b2b738463",
|
||||
"title": "95ccb018c85dce04",
|
||||
"viewFullError": "e0fcf523d4155c70",
|
||||
@@ -26075,7 +26086,7 @@
|
||||
"retriesOnFailure": "a2e021cdc7d91dbd",
|
||||
"retriesOnFailureSuffix": "ad5f0c777a2b470f",
|
||||
"retryDelay": "bbbdc7fae12b28e4",
|
||||
"retryDelaySuffix": "48e77c7c4e24e2f4",
|
||||
"retryDelaySuffix": "a2f9474ace454f9a",
|
||||
"schedule": "f4830a1dae298044",
|
||||
"scheduleLater": "058e3d9d22f8df88",
|
||||
"scheduleLaterTooltip": "946c56014f45896f",
|
||||
|
||||
@@ -526,6 +526,11 @@ pub const SYNTHETICS_RELOAD_CLASSES: &[(&str, SyntheticsReloadClass)] = &[
|
||||
"ZO_SYNTHETICS_ORPHAN_DETECTION_ENABLED",
|
||||
SyntheticsReloadClass::Hot,
|
||||
),
|
||||
("ZO_SYNTHETICS_LEASE_BATCH_SIZE", SyntheticsReloadClass::Hot),
|
||||
(
|
||||
"ZO_SYNTHETICS_MAX_INFLIGHT_PER_TICK",
|
||||
SyntheticsReloadClass::Hot,
|
||||
),
|
||||
// Were `HotViaLimits` while these lived in `o2_enterprise` and had to be
|
||||
// pushed across a seam into this crate. They are declared here now, so the
|
||||
// seam and its class are both gone and they are plainly `Hot`.
|
||||
@@ -580,6 +585,8 @@ pub(crate) fn synthetics_restart_required_changes(
|
||||
devices: _,
|
||||
scheduler_jitter_enabled: _,
|
||||
orphan_detection_enabled: _,
|
||||
lease_batch_size: _,
|
||||
max_inflight_per_tick: _,
|
||||
} = new;
|
||||
|
||||
let mut changed = Vec::new();
|
||||
@@ -1493,6 +1500,23 @@ pub struct Synthetics {
|
||||
help = "Kill switch for orphan detection, which reports enabled checks no scheduler is claiming."
|
||||
)]
|
||||
pub orphan_detection_enabled: bool,
|
||||
/// Jobs leased per pool in one dispatcher lease call. Must be > 0; a
|
||||
/// non-positive value falls back to the default. Read per tick.
|
||||
#[env_config(
|
||||
name = "ZO_SYNTHETICS_LEASE_BATCH_SIZE",
|
||||
default = 10,
|
||||
help = "Jobs the dispatcher leases per pool per lease call. Must be greater than 0."
|
||||
)]
|
||||
pub lease_batch_size: i64,
|
||||
/// Upper bound on jobs the dispatcher leases across all pools in one tick.
|
||||
/// While a pool's batch comes back full the dispatcher keeps leasing, up to
|
||||
/// this cap. Must be > 0; a non-positive value falls back to the default.
|
||||
#[env_config(
|
||||
name = "ZO_SYNTHETICS_MAX_INFLIGHT_PER_TICK",
|
||||
default = 100,
|
||||
help = "Cap on jobs the dispatcher leases across all pools in one 2s tick. Must be greater than 0."
|
||||
)]
|
||||
pub max_inflight_per_tick: i64,
|
||||
}
|
||||
|
||||
/// Feature 5 — SLO measurement (`alerts_2.md` §6b).
|
||||
@@ -5270,6 +5294,21 @@ fn check_synthetics_config(cfg: &mut Config) {
|
||||
cfg.synthetics.max_net_timeout_ms = crate::meta::synthetics::DEFAULT_MAX_NET_TIMEOUT_MS;
|
||||
}
|
||||
|
||||
if cfg.synthetics.lease_batch_size <= 0 {
|
||||
log::error!(
|
||||
"ZO_SYNTHETICS_LEASE_BATCH_SIZE ({}) must be greater than 0, using 10",
|
||||
cfg.synthetics.lease_batch_size
|
||||
);
|
||||
cfg.synthetics.lease_batch_size = 10;
|
||||
}
|
||||
if cfg.synthetics.max_inflight_per_tick <= 0 {
|
||||
log::error!(
|
||||
"ZO_SYNTHETICS_MAX_INFLIGHT_PER_TICK ({}) must be greater than 0, using 100",
|
||||
cfg.synthetics.max_inflight_per_tick
|
||||
);
|
||||
cfg.synthetics.max_inflight_per_tick = 100;
|
||||
}
|
||||
|
||||
// Every synthetics var now has exactly one name, in every build — which
|
||||
// means an OSS user can set a private-agent var and get silence back. One
|
||||
// line per var is the difference between "configured and ignored" and
|
||||
@@ -5450,6 +5489,8 @@ mod tests {
|
||||
"ZO_SYNTHETICS_DEVICES",
|
||||
"ZO_SYNTHETICS_SCHEDULER_JITTER_ENABLED",
|
||||
"ZO_SYNTHETICS_ORPHAN_DETECTION_ENABLED",
|
||||
"ZO_SYNTHETICS_LEASE_BATCH_SIZE",
|
||||
"ZO_SYNTHETICS_MAX_INFLIGHT_PER_TICK",
|
||||
];
|
||||
|
||||
fn synthetics_names_in_class(class: SyntheticsReloadClass) -> Vec<&'static str> {
|
||||
@@ -5469,8 +5510,8 @@ mod tests {
|
||||
fn synthetics_reload_classification_is_pinned() {
|
||||
assert_eq!(
|
||||
SYNTHETICS_RELOAD_CLASSES.len(),
|
||||
16,
|
||||
"Synthetics has 16 keys; every one needs a reload class"
|
||||
18,
|
||||
"Synthetics has 18 keys; every one needs a reload class"
|
||||
);
|
||||
|
||||
let mut classified: Vec<&str> = SYNTHETICS_RELOAD_CLASSES
|
||||
@@ -5502,7 +5543,9 @@ mod tests {
|
||||
"ZO_SYNTHETICS_JOB_LEASE_SECS",
|
||||
"ZO_SYNTHETICS_LAMBDA_BROWSER",
|
||||
"ZO_SYNTHETICS_LAMBDA_NET",
|
||||
"ZO_SYNTHETICS_LEASE_BATCH_SIZE",
|
||||
"ZO_SYNTHETICS_MAX_CHECK_BUDGET_SECS",
|
||||
"ZO_SYNTHETICS_MAX_INFLIGHT_PER_TICK",
|
||||
"ZO_SYNTHETICS_MAX_NET_TIMEOUT_MS",
|
||||
"ZO_SYNTHETICS_ORPHAN_DETECTION_ENABLED",
|
||||
"ZO_SYNTHETICS_RECORDER_EXTENSION_URL",
|
||||
@@ -5565,6 +5608,8 @@ mod tests {
|
||||
cfg.devices = "desktop:800:600".to_string();
|
||||
cfg.scheduler_jitter_enabled = !cfg.scheduler_jitter_enabled;
|
||||
cfg.orphan_detection_enabled = !cfg.orphan_detection_enabled;
|
||||
cfg.lease_batch_size += 1;
|
||||
cfg.max_inflight_per_tick += 1;
|
||||
}
|
||||
|
||||
#[test]
|
||||
|
||||
@@ -46,6 +46,10 @@ pub const CHECK_STATUS_UNKNOWN: i32 = 0;
|
||||
pub const CHECK_STATUS_UP: i32 = 1;
|
||||
pub const CHECK_STATUS_WARNING: i32 = 2;
|
||||
pub const CHECK_STATUS_DOWN: i32 = 3;
|
||||
/// The probe could not produce a verdict (dispatch dead-letter, expired lease,
|
||||
/// infra fault). Says nothing about the target: it never opens an incident and
|
||||
/// never counts toward recovery. Written by the dispatcher and the reaper.
|
||||
pub const CHECK_STATUS_ERROR: i32 = 4;
|
||||
|
||||
const MICROS_PER_MIN: i64 = 60 * 1_000_000;
|
||||
const MICROS_PER_DAY: i64 = 86_400 * 1_000_000;
|
||||
@@ -120,6 +124,10 @@ pub struct EngineCheck {
|
||||
pub first_fail_at: Option<i64>,
|
||||
/// When the streak started failing, for time-based confirmation.
|
||||
pub failing_since: Option<i64>,
|
||||
/// True only when this tick sees a run result it has not seen before.
|
||||
/// Recovery advances on distinct results, never on rebuild ticks, so the
|
||||
/// same UP observed over ten ticks counts once.
|
||||
pub fresh_result: bool,
|
||||
}
|
||||
|
||||
/// An auto-incident notice currently open (or just resolved, for the merge
|
||||
@@ -535,12 +543,33 @@ pub fn incident_actions(
|
||||
snoozed: &HashSet<String>,
|
||||
th: &Thresholds,
|
||||
now: i64,
|
||||
) -> Vec<EngineAction> {
|
||||
incident_actions_with(
|
||||
checks,
|
||||
open_by_check,
|
||||
recently_resolved,
|
||||
snoozed,
|
||||
|_| *th,
|
||||
now,
|
||||
)
|
||||
}
|
||||
|
||||
/// [`incident_actions`] with per-check thresholds (each check is judged by the
|
||||
/// confirmation settings of the page(s) it is mapped to).
|
||||
pub fn incident_actions_with(
|
||||
checks: &[EngineCheck],
|
||||
open_by_check: &HashMap<String, OpenAutoIncident>,
|
||||
recently_resolved: &HashMap<String, (String, i64)>,
|
||||
snoozed: &HashSet<String>,
|
||||
th_for: impl Fn(&str) -> Thresholds,
|
||||
now: i64,
|
||||
) -> Vec<EngineAction> {
|
||||
let mut actions = Vec::new();
|
||||
for check in checks {
|
||||
let th = th_for(&check.id);
|
||||
match open_by_check.get(&check.id) {
|
||||
Some(open) => handle_open(check, open, th, now, &mut actions),
|
||||
None => handle_closed(check, recently_resolved, snoozed, th, now, &mut actions),
|
||||
Some(open) => handle_open(check, open, &th, now, &mut actions),
|
||||
None => handle_closed(check, recently_resolved, snoozed, &th, now, &mut actions),
|
||||
}
|
||||
}
|
||||
actions
|
||||
@@ -568,7 +597,16 @@ fn handle_open(
|
||||
now: i64,
|
||||
actions: &mut Vec<EngineAction>,
|
||||
) {
|
||||
if check.last_check_status == CHECK_STATUS_UP {
|
||||
// UP and WARNING both mean the target answered: they advance recovery, but
|
||||
// only on a result this loop has not counted yet.
|
||||
let healthy = matches!(
|
||||
check.last_check_status,
|
||||
CHECK_STATUS_UP | CHECK_STATUS_WARNING
|
||||
);
|
||||
if healthy {
|
||||
if !check.fresh_result {
|
||||
return;
|
||||
}
|
||||
let streak = open.recovery_streak + 1;
|
||||
if streak >= th.confirm_recovery {
|
||||
actions.push(EngineAction::Resolve {
|
||||
@@ -581,8 +619,14 @@ fn handle_open(
|
||||
streak,
|
||||
});
|
||||
}
|
||||
} else if check.last_check_status == CHECK_STATUS_DOWN && open.recovery_streak > 0 {
|
||||
// Failed again mid-recovery: the streak restarts from zero.
|
||||
} else if matches!(
|
||||
check.last_check_status,
|
||||
CHECK_STATUS_DOWN | CHECK_STATUS_ERROR
|
||||
) && open.recovery_streak > 0
|
||||
{
|
||||
// Failed again mid-recovery: the streak restarts from zero. ERROR does
|
||||
// the same: an unknown verdict cannot prove recovery, so UP, ERROR, UP
|
||||
// is not two consecutive good results. The incident stays open.
|
||||
actions.push(EngineAction::RecoveryProgress {
|
||||
notice_id: open.notice_id.clone(),
|
||||
streak: 0,
|
||||
@@ -761,9 +805,119 @@ mod tests {
|
||||
consecutive_failures: streak,
|
||||
first_fail_at: None,
|
||||
failing_since: None,
|
||||
fresh_result: true,
|
||||
}
|
||||
}
|
||||
|
||||
fn open_inc(streak: i32) -> HashMap<String, OpenAutoIncident> {
|
||||
[(
|
||||
"c1".to_string(),
|
||||
OpenAutoIncident {
|
||||
notice_id: "n1".into(),
|
||||
check_id: "c1".into(),
|
||||
recovery_streak: streak,
|
||||
},
|
||||
)]
|
||||
.into()
|
||||
}
|
||||
|
||||
/// Replays results against an open incident, applying RecoveryProgress
|
||||
/// back into the streak; returns whether it resolved.
|
||||
fn replay(statuses: &[(i32, bool)], th: Thresholds) -> bool {
|
||||
let mut streak = 0;
|
||||
for &(status, fresh) in statuses {
|
||||
let mut c = check("c1", status, 0);
|
||||
c.fresh_result = fresh;
|
||||
let acts = incident_actions(
|
||||
&[c],
|
||||
&open_inc(streak),
|
||||
&HashMap::new(),
|
||||
&HashSet::new(),
|
||||
&th,
|
||||
0,
|
||||
);
|
||||
for a in acts {
|
||||
match a {
|
||||
EngineAction::Resolve { .. } => return true,
|
||||
EngineAction::RecoveryProgress { streak: s, .. } => streak = s,
|
||||
_ => panic!("unexpected {a:?}"),
|
||||
}
|
||||
}
|
||||
}
|
||||
false
|
||||
}
|
||||
|
||||
fn th(failures: i32, recovery: i32) -> Thresholds {
|
||||
Thresholds {
|
||||
confirm_failures: failures,
|
||||
confirm_recovery: recovery,
|
||||
confirm_after_micros: None,
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn up_error_up_does_not_resolve_with_recovery_two() {
|
||||
let s = CHECK_STATUS_UP;
|
||||
let e = CHECK_STATUS_ERROR;
|
||||
assert!(!replay(&[(s, true), (e, true), (s, true)], th(3, 2)));
|
||||
assert!(replay(
|
||||
&[(s, true), (e, true), (s, true), (s, true)],
|
||||
th(3, 2)
|
||||
));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn error_never_opens_an_incident() {
|
||||
let c = check("c1", CHECK_STATUS_ERROR, 50);
|
||||
let acts = incident_actions(
|
||||
&[c],
|
||||
&HashMap::new(),
|
||||
&HashMap::new(),
|
||||
&HashSet::new(),
|
||||
&th(1, 1),
|
||||
0,
|
||||
);
|
||||
assert!(acts.is_empty());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn error_keeps_an_open_incident_open() {
|
||||
let e = CHECK_STATUS_ERROR;
|
||||
assert!(!replay(&[(e, true); 10], th(3, 1)));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn warning_advances_recovery() {
|
||||
let w = CHECK_STATUS_WARNING;
|
||||
assert!(replay(&[(w, true), (w, true)], th(3, 2)));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn the_same_result_over_many_ticks_counts_once() {
|
||||
let mut ticks = vec![(CHECK_STATUS_UP, true)];
|
||||
ticks.extend(std::iter::repeat_n((CHECK_STATUS_UP, false), 20));
|
||||
assert!(!replay(&ticks, th(3, 2)));
|
||||
ticks.push((CHECK_STATUS_UP, true));
|
||||
assert!(replay(&ticks, th(3, 2)));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn seven_failures_four_recoveries() {
|
||||
let t = th(7, 4);
|
||||
let none = HashMap::new();
|
||||
let quiet = HashSet::new();
|
||||
let six = [check("c1", CHECK_STATUS_DOWN, 6)];
|
||||
assert!(incident_actions(&six, &none, &HashMap::new(), &quiet, &t, 0).is_empty());
|
||||
let seven = [check("c1", CHECK_STATUS_DOWN, 7)];
|
||||
assert_eq!(
|
||||
incident_actions(&seven, &none, &HashMap::new(), &quiet, &t, 0).len(),
|
||||
1
|
||||
);
|
||||
let up = CHECK_STATUS_UP;
|
||||
assert!(!replay(&[(up, true); 3], t));
|
||||
assert!(replay(&[(up, true); 4], t));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn union_counts_overlaps_once() {
|
||||
// Two checks confirm the same outage: 10:00-10:30 and 10:10-10:40.
|
||||
|
||||
@@ -31,7 +31,7 @@ pub struct Model {
|
||||
/// When the scheduler last fanned out this check (microseconds).
|
||||
pub last_triggered_at: i64,
|
||||
/// Denormalised status from the most recent completed check.
|
||||
/// 0=Unknown, 1=Up, 2=Warning, 3=Down
|
||||
/// 0=Unknown, 1=Up, 2=Warning, 3=Down, 4=Error (no verdict: dispatch/lease failure)
|
||||
pub last_check_status: i32,
|
||||
/// Runs that have failed back to back; reset to 0 by a pass. Compared
|
||||
/// against `settings.alert_if_fails` to decide whether to notify.
|
||||
|
||||
@@ -31,21 +31,26 @@ use super::entity::{
|
||||
status_page_audit_log, status_page_check_snoozes, status_page_component_checks,
|
||||
status_page_components, status_page_custom_domains, status_page_notice_components,
|
||||
status_page_notice_updates, status_page_notices, status_page_snapshots, status_pages,
|
||||
synthetics_checks,
|
||||
synthetics_checks, synthetics_runs,
|
||||
};
|
||||
use crate::{
|
||||
db::{get_orm_client_ro, get_orm_client_rw},
|
||||
errors,
|
||||
};
|
||||
|
||||
/// The three columns the auto-incident engine needs per mapped check —
|
||||
/// The columns the auto-incident engine needs per mapped check —
|
||||
/// deliberately not the whole row (config/secrets are KB each).
|
||||
#[derive(Clone, Debug)]
|
||||
pub struct CheckStateRow {
|
||||
pub id: String,
|
||||
pub org_id: String,
|
||||
pub last_check_status: i32,
|
||||
pub consecutive_failures: i32,
|
||||
pub last_triggered_at: i64,
|
||||
/// `completed_at` of the check's latest completed run, filled by
|
||||
/// [`latest_run_completions`]; `None` until read or when no run completed
|
||||
/// inside the lookback window.
|
||||
pub last_completed_at: Option<i64>,
|
||||
}
|
||||
|
||||
pub async fn get_page_by_slug<C: ConnectionTrait>(
|
||||
@@ -117,6 +122,7 @@ pub async fn get_check_states<C: ConnectionTrait>(
|
||||
let mut q = synthetics_checks::Entity::find()
|
||||
.select_only()
|
||||
.column(synthetics_checks::Column::Id)
|
||||
.column(synthetics_checks::Column::OrgId)
|
||||
.column(synthetics_checks::Column::LastCheckStatus)
|
||||
.column(synthetics_checks::Column::ConsecutiveFailures)
|
||||
.column(synthetics_checks::Column::LastTriggeredAt)
|
||||
@@ -126,22 +132,31 @@ pub async fn get_check_states<C: ConnectionTrait>(
|
||||
q = q.filter(synthetics_checks::Column::UpdatedAt.gt(watermark));
|
||||
}
|
||||
let rows = q
|
||||
.into_tuple::<(String, i32, i32, i64, i64)>()
|
||||
.into_tuple::<(String, String, i32, i32, i64, i64)>()
|
||||
.all(conn)
|
||||
.await?;
|
||||
let mut max_updated = since.unwrap_or(0);
|
||||
let out = rows
|
||||
.into_iter()
|
||||
.map(
|
||||
|(id, last_check_status, consecutive_failures, last_triggered_at, updated_at)| {
|
||||
|(
|
||||
id,
|
||||
org_id,
|
||||
last_check_status,
|
||||
consecutive_failures,
|
||||
last_triggered_at,
|
||||
updated_at,
|
||||
)| {
|
||||
if updated_at > max_updated {
|
||||
max_updated = updated_at;
|
||||
}
|
||||
CheckStateRow {
|
||||
id,
|
||||
org_id,
|
||||
last_check_status,
|
||||
consecutive_failures,
|
||||
last_triggered_at,
|
||||
last_completed_at: None,
|
||||
}
|
||||
},
|
||||
)
|
||||
@@ -149,6 +164,43 @@ pub async fn get_check_states<C: ConnectionTrait>(
|
||||
Ok((out, max_updated))
|
||||
}
|
||||
|
||||
/// Latest `synthetics_runs.completed_at` per check, for runs scheduled at or
|
||||
/// after `scheduled_since`. Keys are check ids; checks with no completed run in
|
||||
/// the window are absent.
|
||||
///
|
||||
/// Region-local like the runs table itself: it sees the runs this region's
|
||||
/// scheduler enqueued, which is the region that holds the job-cluster claim and
|
||||
/// therefore runs the incident engine. The `org_id` + `synthetics_id` +
|
||||
/// `scheduled_ts` predicates match `synthetics_runs_monitor_idx`, so the read
|
||||
/// is an index range per mapped check, not a table scan.
|
||||
pub async fn latest_run_completions<C: ConnectionTrait>(
|
||||
conn: &C,
|
||||
checks: &[(String, String)],
|
||||
scheduled_since: i64,
|
||||
) -> Result<std::collections::HashMap<String, i64>, errors::Error> {
|
||||
if checks.is_empty() {
|
||||
return Ok(Default::default());
|
||||
}
|
||||
let orgs: HashSet<&str> = checks.iter().map(|(o, _)| o.as_str()).collect();
|
||||
let ids: HashSet<&str> = checks.iter().map(|(_, id)| id.as_str()).collect();
|
||||
let rows = synthetics_runs::Entity::find()
|
||||
.select_only()
|
||||
.column(synthetics_runs::Column::SyntheticsId)
|
||||
.column_as(synthetics_runs::Column::CompletedAt.max(), "completed_at")
|
||||
.filter(synthetics_runs::Column::OrgId.is_in(orgs.into_iter().map(str::to_owned)))
|
||||
.filter(synthetics_runs::Column::SyntheticsId.is_in(ids.into_iter().map(str::to_owned)))
|
||||
.filter(synthetics_runs::Column::ScheduledTs.gte(scheduled_since))
|
||||
.filter(synthetics_runs::Column::CompletedAt.is_not_null())
|
||||
.group_by(synthetics_runs::Column::SyntheticsId)
|
||||
.into_tuple::<(String, Option<i64>)>()
|
||||
.all(conn)
|
||||
.await?;
|
||||
Ok(rows
|
||||
.into_iter()
|
||||
.filter_map(|(id, at)| at.map(|at| (id, at)))
|
||||
.collect())
|
||||
}
|
||||
|
||||
/// Notices the engine and snapshots need: not deleted, and open or resolved
|
||||
/// after `resolved_cutoff` (90d for history; the merge window rides within).
|
||||
pub async fn list_notices_since<C: ConnectionTrait>(
|
||||
|
||||
@@ -269,6 +269,17 @@ pub async fn count<C: ConnectionTrait>(
|
||||
org_id: &str,
|
||||
params: &ListSyntheticsParams,
|
||||
) -> Result<u64, errors::Error> {
|
||||
if has_json_filters(params) {
|
||||
// location/tag live in JSON text columns: filter in Rust so count and
|
||||
// page agree.
|
||||
let models = Entity::find()
|
||||
.filter(Column::OrgId.eq(org_id))
|
||||
.apply_filters(params)
|
||||
.order_by_asc(Column::Name)
|
||||
.all(conn)
|
||||
.await?;
|
||||
return Ok(filter_and_paginate(models, params).1);
|
||||
}
|
||||
let q = Entity::find()
|
||||
.filter(Column::OrgId.eq(org_id))
|
||||
.apply_filters(params);
|
||||
@@ -1040,6 +1051,50 @@ async fn get_model<C: ConnectionTrait>(
|
||||
.await
|
||||
}
|
||||
|
||||
fn has_json_filters(params: &ListSyntheticsParams) -> bool {
|
||||
params.location.is_some() || params.tag.is_some()
|
||||
}
|
||||
|
||||
fn json_array_contains(v: &serde_json::Value, needle: &str) -> bool {
|
||||
v.as_array()
|
||||
.is_some_and(|a| a.iter().any(|x| x.as_str() == Some(needle)))
|
||||
}
|
||||
|
||||
/// Applies the `location` / `tag` exact-match filters (JSON text columns, so
|
||||
/// no DB operators) and then page/page_size over the filtered set. Returns the
|
||||
/// requested page and the filtered total; `models` must already be ordered.
|
||||
fn filter_and_paginate(
|
||||
models: Vec<synthetics_checks::Model>,
|
||||
params: &ListSyntheticsParams,
|
||||
) -> (Vec<synthetics_checks::Model>, u64) {
|
||||
let filtered: Vec<_> = models
|
||||
.into_iter()
|
||||
.filter(|m| {
|
||||
params
|
||||
.location
|
||||
.as_deref()
|
||||
.is_none_or(|l| json_array_contains(&m.locations, l))
|
||||
&& params
|
||||
.tag
|
||||
.as_deref()
|
||||
.is_none_or(|t| json_array_contains(&m.tags, t))
|
||||
})
|
||||
.collect();
|
||||
let total = filtered.len() as u64;
|
||||
if let (Some(page_size), Some(page)) = (params.page_size, params.page)
|
||||
&& page_size > 0
|
||||
{
|
||||
let start = page.saturating_mul(page_size) as usize;
|
||||
let page_rows = filtered
|
||||
.into_iter()
|
||||
.skip(start)
|
||||
.take(page_size as usize)
|
||||
.collect();
|
||||
return (page_rows, total);
|
||||
}
|
||||
(filtered, total)
|
||||
}
|
||||
|
||||
async fn list_models<C: ConnectionTrait>(
|
||||
conn: &C,
|
||||
org_id: &str,
|
||||
@@ -1050,6 +1105,11 @@ async fn list_models<C: ConnectionTrait>(
|
||||
.apply_filters(params)
|
||||
.order_by_asc(Column::Name);
|
||||
|
||||
if has_json_filters(params) {
|
||||
let models = q.all(conn).await?;
|
||||
return Ok(filter_and_paginate(models, params).0);
|
||||
}
|
||||
|
||||
if let (Some(page_size), Some(page)) = (params.page_size, params.page)
|
||||
&& page_size > 0
|
||||
{
|
||||
@@ -1227,6 +1287,63 @@ pub(crate) mod tests {
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn filter_and_paginate_combined_filters_and_pages() {
|
||||
let mk = |id: &str, locs: &[&str], tags: &[&str]| {
|
||||
let mut m = make_model();
|
||||
m.id = id.to_string();
|
||||
m.locations = serde_json::json!(locs);
|
||||
m.tags = serde_json::json!(tags);
|
||||
m
|
||||
};
|
||||
let models = vec![
|
||||
mk("a", &["l1"], &["prod"]),
|
||||
mk("b", &["l1", "l2"], &["prod", "x"]),
|
||||
mk("c", &["l2"], &["prod"]),
|
||||
mk("d", &["l1"], &["dev"]),
|
||||
mk("e", &["l1"], &["prod"]),
|
||||
];
|
||||
let params = |loc: Option<&str>, tag: Option<&str>, page, size| ListSyntheticsParams {
|
||||
location: loc.map(String::from),
|
||||
tag: tag.map(String::from),
|
||||
page,
|
||||
page_size: size,
|
||||
..Default::default()
|
||||
};
|
||||
// location only
|
||||
let (rows, total) =
|
||||
filter_and_paginate(models.clone(), ¶ms(Some("l2"), None, None, None));
|
||||
assert_eq!((rows.len(), total), (2, 2));
|
||||
// exact match, not substring
|
||||
let (_, total) = filter_and_paginate(models.clone(), ¶ms(Some("l"), None, None, None));
|
||||
assert_eq!(total, 0);
|
||||
// combined
|
||||
let (rows, total) = filter_and_paginate(
|
||||
models.clone(),
|
||||
¶ms(Some("l1"), Some("prod"), None, None),
|
||||
);
|
||||
assert_eq!(total, 3);
|
||||
assert_eq!(
|
||||
rows.iter().map(|m| m.id.as_str()).collect::<Vec<_>>(),
|
||||
["a", "b", "e"]
|
||||
);
|
||||
// page boundaries: total stays 3, pages 2 + 1 + empty
|
||||
let (rows, total) = filter_and_paginate(
|
||||
models.clone(),
|
||||
¶ms(Some("l1"), Some("prod"), Some(0), Some(2)),
|
||||
);
|
||||
assert_eq!((rows.len(), total), (2, 3));
|
||||
let (rows, total) = filter_and_paginate(
|
||||
models.clone(),
|
||||
¶ms(Some("l1"), Some("prod"), Some(1), Some(2)),
|
||||
);
|
||||
assert_eq!((rows.len(), total), (1, 3));
|
||||
assert_eq!(rows[0].id, "e");
|
||||
let (rows, total) =
|
||||
filter_and_paginate(models, ¶ms(Some("l1"), Some("prod"), Some(2), Some(2)));
|
||||
assert_eq!((rows.len(), total), (0, 3));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_try_from_model() {
|
||||
let check = Synthetic::try_from(make_model()).unwrap();
|
||||
|
||||
@@ -0,0 +1,260 @@
|
||||
// Copyright 2026 OpenObserve Inc.
|
||||
//
|
||||
// This program is free software: you can redistribute it and/or modify
|
||||
// it under the terms of the GNU Affero General Public License as published by
|
||||
// the Free Software Foundation, either version 3 of the License, or
|
||||
// (at your option) any later version.
|
||||
//
|
||||
// This program is distributed in the hope that it will be useful
|
||||
// but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
// GNU Affero General Public License for more details.
|
||||
//
|
||||
// You should have received a copy of the GNU Affero General Public License
|
||||
// along with this program. If not, see <http://www.gnu.org/licenses/>.
|
||||
|
||||
//! The callback endpoint probes are told to call back on
|
||||
//! (`config::meta::synthetics::api_endpoint`), as it can safely appear in a
|
||||
//! log line or a dead-letter message, plus a startup reachability warning for
|
||||
//! Lambda venues (o2-enterprise#3165).
|
||||
|
||||
use std::{
|
||||
net::{IpAddr, Ipv4Addr, Ipv6Addr},
|
||||
time::Duration,
|
||||
};
|
||||
|
||||
/// Placeholder for an endpoint that does not parse, so no raw value (which may
|
||||
/// carry credentials) is ever echoed.
|
||||
pub const UNPARSEABLE_ENDPOINT: &str = "<unparseable callback endpoint>";
|
||||
|
||||
/// `scheme://host[:port]/path` of `raw`: no userinfo, no query, no fragment.
|
||||
/// `None` when `raw` is not an absolute URL with a host.
|
||||
pub fn sanitize_callback_url(raw: &str) -> Option<String> {
|
||||
let url = reqwest::Url::parse(raw.trim()).ok()?;
|
||||
let host = url.host_str()?;
|
||||
let port = url.port().map(|p| format!(":{p}")).unwrap_or_default();
|
||||
let path = url.path().trim_end_matches('/');
|
||||
Some(format!("{}://{host}{port}{path}", url.scheme()))
|
||||
}
|
||||
|
||||
/// The endpoint probes are currently told to call back on, sanitized.
|
||||
pub fn current_callback_endpoint() -> String {
|
||||
sanitize_callback_url(&config::meta::synthetics::api_endpoint())
|
||||
.unwrap_or_else(|| UNPARSEABLE_ENDPOINT.to_string())
|
||||
}
|
||||
|
||||
/// Address classes a Lambda function outside the deployment's network usually
|
||||
/// cannot reach.
|
||||
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
|
||||
pub enum AddrClass {
|
||||
Public,
|
||||
Loopback,
|
||||
Unspecified,
|
||||
LinkLocal,
|
||||
Private,
|
||||
}
|
||||
|
||||
pub fn classify_ip(ip: IpAddr) -> AddrClass {
|
||||
match ip {
|
||||
IpAddr::V4(v4) => classify_v4(v4),
|
||||
IpAddr::V6(v6) => {
|
||||
if let Some(v4) = v6.to_ipv4_mapped() {
|
||||
return classify_v4(v4);
|
||||
}
|
||||
classify_v6(v6)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
fn classify_v4(ip: Ipv4Addr) -> AddrClass {
|
||||
let o = ip.octets();
|
||||
if ip.is_loopback() {
|
||||
AddrClass::Loopback
|
||||
} else if ip.is_unspecified() {
|
||||
AddrClass::Unspecified
|
||||
} else if ip.is_link_local() {
|
||||
AddrClass::LinkLocal
|
||||
} else if ip.is_private() || (o[0] == 100 && (o[1] & 0xc0) == 64) {
|
||||
// RFC 1918 plus shared address space (100.64.0.0/10, carrier NAT).
|
||||
AddrClass::Private
|
||||
} else {
|
||||
AddrClass::Public
|
||||
}
|
||||
}
|
||||
|
||||
fn classify_v6(ip: Ipv6Addr) -> AddrClass {
|
||||
let first = ip.segments()[0];
|
||||
if ip.is_loopback() {
|
||||
AddrClass::Loopback
|
||||
} else if ip.is_unspecified() {
|
||||
AddrClass::Unspecified
|
||||
} else if (first & 0xffc0) == 0xfe80 {
|
||||
AddrClass::LinkLocal
|
||||
} else if (first & 0xfe00) == 0xfc00 {
|
||||
// Unique local addresses, fc00::/7.
|
||||
AddrClass::Private
|
||||
} else {
|
||||
AddrClass::Public
|
||||
}
|
||||
}
|
||||
|
||||
/// The host part of a parsed URL as an IP literal, IPv6 brackets stripped.
|
||||
fn host_ip(host: &str) -> Option<IpAddr> {
|
||||
host.trim_start_matches('[')
|
||||
.trim_end_matches(']')
|
||||
.parse()
|
||||
.ok()
|
||||
}
|
||||
|
||||
const DNS_TIMEOUT: Duration = Duration::from_secs(2);
|
||||
|
||||
/// Log-only check run when a Lambda venue is enabled: a Lambda function can
|
||||
/// call back only on an endpoint reachable from its own network. Never rejects
|
||||
/// anything, because a VPC-connected Lambda can legitimately reach private
|
||||
/// addresses. Logs only the sanitized endpoint.
|
||||
pub async fn warn_if_lambda_callback_may_be_unreachable() {
|
||||
let raw = config::meta::synthetics::api_endpoint();
|
||||
let Some(shown) = sanitize_callback_url(&raw) else {
|
||||
tracing::warn!(
|
||||
"[synthetics] Lambda venue enabled but the callback endpoint \
|
||||
(ZO_SYNTHETICS_API_ENDPOINT, else ZO_WEB_URL) is empty or not an absolute URL; \
|
||||
Lambda probes may be unable to report results"
|
||||
);
|
||||
return;
|
||||
};
|
||||
let Ok(url) = reqwest::Url::parse(raw.trim()) else {
|
||||
return;
|
||||
};
|
||||
let Some(host) = url.host_str() else { return };
|
||||
let advice = "a Lambda probe can report only if this endpoint is reachable from the \
|
||||
Lambda function's network (a VPC-connected function can reach private \
|
||||
addresses; otherwise use a public endpoint)";
|
||||
|
||||
if let Some(ip) = host_ip(host) {
|
||||
let class = classify_ip(ip);
|
||||
if class != AddrClass::Public {
|
||||
tracing::warn!(
|
||||
endpoint = %shown,
|
||||
address_class = ?class,
|
||||
"[synthetics] callback endpoint is a {class:?} address; {advice}"
|
||||
);
|
||||
}
|
||||
return;
|
||||
}
|
||||
if host.eq_ignore_ascii_case("localhost") || host.to_ascii_lowercase().ends_with(".localhost") {
|
||||
tracing::warn!(
|
||||
endpoint = %shown,
|
||||
"[synthetics] callback endpoint is localhost; {advice}"
|
||||
);
|
||||
return;
|
||||
}
|
||||
|
||||
let port = url.port_or_known_default().unwrap_or(443);
|
||||
let lookup = tokio::time::timeout(
|
||||
DNS_TIMEOUT,
|
||||
tokio::net::lookup_host((host.to_string(), port)),
|
||||
)
|
||||
.await;
|
||||
let addrs: Vec<IpAddr> = match lookup {
|
||||
Ok(Ok(it)) => it.map(|sa| sa.ip()).collect(),
|
||||
Ok(Err(e)) => {
|
||||
tracing::warn!(
|
||||
endpoint = %shown,
|
||||
"[synthetics] could not resolve the callback endpoint host from this node ({e}); \
|
||||
Lambda probes may be unable to report results if it does not resolve for them \
|
||||
either"
|
||||
);
|
||||
return;
|
||||
}
|
||||
Err(_) => {
|
||||
tracing::warn!(
|
||||
endpoint = %shown,
|
||||
"[synthetics] resolving the callback endpoint host timed out after {}s; \
|
||||
Lambda probes may be unable to report results if it does not resolve for them \
|
||||
either",
|
||||
DNS_TIMEOUT.as_secs()
|
||||
);
|
||||
return;
|
||||
}
|
||||
};
|
||||
let non_public = addrs
|
||||
.iter()
|
||||
.filter(|ip| classify_ip(**ip) != AddrClass::Public)
|
||||
.count();
|
||||
if non_public == 0 {
|
||||
return;
|
||||
}
|
||||
let kind = if non_public == addrs.len() {
|
||||
"only private, loopback or link-local addresses"
|
||||
} else {
|
||||
"a mix of public and private addresses"
|
||||
};
|
||||
tracing::warn!(
|
||||
endpoint = %shown,
|
||||
"[synthetics] callback endpoint host resolves to {kind}; {advice}"
|
||||
);
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
#[test]
|
||||
fn sanitize_strips_userinfo_query_and_fragment() {
|
||||
assert_eq!(
|
||||
sanitize_callback_url("https://user:secret@o2.example.com:8443/api/?token=abc#x"),
|
||||
Some("https://o2.example.com:8443/api".to_string())
|
||||
);
|
||||
assert_eq!(
|
||||
sanitize_callback_url("http://o2.example.com"),
|
||||
Some("http://o2.example.com".to_string())
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn sanitize_keeps_ipv6_brackets() {
|
||||
assert_eq!(
|
||||
sanitize_callback_url("http://[fd00::1]:5080/base?sig=1"),
|
||||
Some("http://[fd00::1]:5080/base".to_string())
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn sanitize_rejects_bad_urls() {
|
||||
assert_eq!(sanitize_callback_url(""), None);
|
||||
assert_eq!(sanitize_callback_url("not a url"), None);
|
||||
assert_eq!(sanitize_callback_url("o2.example.com/api"), None);
|
||||
assert_eq!(sanitize_callback_url("mailto:a@b.c"), None);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn classify_ipv4() {
|
||||
let c = |s: &str| classify_ip(s.parse().unwrap());
|
||||
assert_eq!(c("127.0.0.1"), AddrClass::Loopback);
|
||||
assert_eq!(c("0.0.0.0"), AddrClass::Unspecified);
|
||||
assert_eq!(c("169.254.169.254"), AddrClass::LinkLocal);
|
||||
assert_eq!(c("10.1.2.3"), AddrClass::Private);
|
||||
assert_eq!(c("172.16.0.1"), AddrClass::Private);
|
||||
assert_eq!(c("192.168.1.1"), AddrClass::Private);
|
||||
assert_eq!(c("100.64.0.1"), AddrClass::Private);
|
||||
assert_eq!(c("8.8.8.8"), AddrClass::Public);
|
||||
assert_eq!(c("172.32.0.1"), AddrClass::Public);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn classify_ipv6() {
|
||||
let c = |s: &str| classify_ip(s.parse().unwrap());
|
||||
assert_eq!(c("::1"), AddrClass::Loopback);
|
||||
assert_eq!(c("::"), AddrClass::Unspecified);
|
||||
assert_eq!(c("fe80::1"), AddrClass::LinkLocal);
|
||||
assert_eq!(c("fd12:3456::1"), AddrClass::Private);
|
||||
assert_eq!(c("::ffff:10.0.0.1"), AddrClass::Private);
|
||||
assert_eq!(c("2606:4700::1111"), AddrClass::Public);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn host_ip_parses_bracketed_ipv6() {
|
||||
assert_eq!(host_ip("[::1]"), Some("::1".parse().unwrap()));
|
||||
assert_eq!(host_ip("o2.example.com"), None);
|
||||
}
|
||||
}
|
||||
@@ -35,7 +35,22 @@ use infra::{
|
||||
};
|
||||
|
||||
const TICK: Duration = Duration::from_secs(2);
|
||||
const LEASE_BATCH_SIZE: i64 = 10;
|
||||
|
||||
/// Jobs per pool per lease call (`ZO_SYNTHETICS_LEASE_BATCH_SIZE`), read per tick.
|
||||
fn lease_batch_size() -> i64 {
|
||||
config::get_config().synthetics.lease_batch_size.max(1)
|
||||
}
|
||||
|
||||
/// Most jobs leased across all pools in one tick (`ZO_SYNTHETICS_MAX_INFLIGHT_PER_TICK`).
|
||||
fn max_inflight_per_tick() -> i64 {
|
||||
config::get_config().synthetics.max_inflight_per_tick.max(1)
|
||||
}
|
||||
|
||||
/// Size of the next lease call: a full batch, trimmed so the tick never leases
|
||||
/// past its cap. Zero means the tick is out of budget.
|
||||
fn next_batch_limit(batch: i64, leased_this_tick: i64, cap: i64) -> i64 {
|
||||
batch.min(cap - leased_this_tick).max(0)
|
||||
}
|
||||
/// How long a leased job may run before the reaper requeues it
|
||||
/// (`reaper::requeue_expired`).
|
||||
///
|
||||
@@ -95,39 +110,88 @@ static INSTANCE_ID: LazyLock<String> = LazyLock::new(|| {
|
||||
)
|
||||
});
|
||||
|
||||
pub async fn run() {
|
||||
static POOL_ROTATION: std::sync::atomic::AtomicUsize = std::sync::atomic::AtomicUsize::new(0);
|
||||
|
||||
pub async fn run(stop: crate::StopToken) {
|
||||
let instance_id = INSTANCE_ID.as_str();
|
||||
tracing::info!(instance_id = %instance_id, "[synthetics dispatcher] started");
|
||||
|
||||
// o2-enterprise#3165: a Lambda probe can only report to a callback endpoint
|
||||
// its own network reaches. Log-only; never blocks the dispatcher.
|
||||
{
|
||||
let cfg = config::get_config();
|
||||
if !cfg.synthetics.lambda_browser.is_empty() || !cfg.synthetics.lambda_net.is_empty() {
|
||||
crate::callback_endpoint::warn_if_lambda_callback_may_be_unreachable().await;
|
||||
}
|
||||
}
|
||||
|
||||
loop {
|
||||
tokio::time::sleep(TICK).await;
|
||||
if stop.sleep(TICK).await {
|
||||
tracing::info!("[synthetics dispatcher] stopped");
|
||||
return;
|
||||
}
|
||||
|
||||
let db = get_orm_client_rw().await;
|
||||
|
||||
let now_us = config::utils::time::now_micros();
|
||||
|
||||
for pool in dispatcher_pools().await {
|
||||
let rows = match synthetics_jobs::lease_batch(
|
||||
db,
|
||||
&pool,
|
||||
instance_id,
|
||||
LEASE_BATCH_SIZE,
|
||||
now_us,
|
||||
lease_secs(),
|
||||
None, // public pools are already type-segmented (net-* vs aws-browser)
|
||||
)
|
||||
.await
|
||||
{
|
||||
Ok(r) => r,
|
||||
Err(e) => {
|
||||
tracing::error!(pool = %pool, "[synthetics dispatcher] lease_batch: {e}");
|
||||
continue;
|
||||
// Round-robin across pools, one batch each per round. A pool whose batch
|
||||
// came back full may have more due, so it stays in for another round;
|
||||
// the per-tick cap bounds the total, and no pool gets a second batch
|
||||
// before every other active pool has had its turn.
|
||||
let batch = lease_batch_size();
|
||||
let cap = max_inflight_per_tick();
|
||||
let mut leased_this_tick: i64 = 0;
|
||||
let mut active = dispatcher_pools().await;
|
||||
// Rotate the starting pool every tick: with the cap reached, a fixed
|
||||
// order would let the first pools take the whole budget every tick.
|
||||
if !active.is_empty() {
|
||||
let start =
|
||||
POOL_ROTATION.fetch_add(1, std::sync::atomic::Ordering::Relaxed) % active.len();
|
||||
active.rotate_left(start);
|
||||
}
|
||||
while !active.is_empty() {
|
||||
let mut still_active = Vec::with_capacity(active.len());
|
||||
for pool in active {
|
||||
// Ownership lost mid-tick: lease nothing more.
|
||||
if stop.is_cancelled() {
|
||||
return;
|
||||
}
|
||||
};
|
||||
let limit = next_batch_limit(batch, leased_this_tick, cap);
|
||||
if limit == 0 {
|
||||
break;
|
||||
}
|
||||
let rows = match synthetics_jobs::lease_batch(
|
||||
db,
|
||||
&pool,
|
||||
instance_id,
|
||||
limit,
|
||||
now_us,
|
||||
lease_secs(),
|
||||
None, // public pools are already type-segmented (net-* vs aws-browser)
|
||||
)
|
||||
.await
|
||||
{
|
||||
Ok(r) => r,
|
||||
Err(e) => {
|
||||
tracing::error!(pool = %pool, "[synthetics dispatcher] lease_batch: {e}");
|
||||
continue;
|
||||
}
|
||||
};
|
||||
|
||||
for row in rows {
|
||||
tokio::spawn(invoke_probe(row));
|
||||
let full = rows.len() as i64 >= limit;
|
||||
leased_this_tick += rows.len() as i64;
|
||||
for row in rows {
|
||||
tokio::spawn(invoke_probe(row, stop.clone()));
|
||||
}
|
||||
if full {
|
||||
still_active.push(pool);
|
||||
}
|
||||
}
|
||||
if leased_this_tick >= cap {
|
||||
break;
|
||||
}
|
||||
active = still_active;
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -304,7 +368,13 @@ async fn mark_failure(row: &synthetics_jobs::LeasedRow, reason: Option<&str>) ->
|
||||
// nothing, while the first failure after a pass sends one message. Without
|
||||
// it the other regions' LIST would keep showing the last status they
|
||||
// themselves ran, or "Unknown".
|
||||
match synthetics_checks::update_last_check_status(db, &row.synthetics_id, 4).await {
|
||||
match synthetics_checks::update_last_check_status(
|
||||
db,
|
||||
&row.synthetics_id,
|
||||
config::meta::status_pages::CHECK_STATUS_ERROR,
|
||||
)
|
||||
.await
|
||||
{
|
||||
Ok(true) => {
|
||||
#[cfg(feature = "enterprise")]
|
||||
if o2_enterprise::enterprise::common::config::get_config()
|
||||
@@ -449,7 +519,13 @@ async fn report_to_stream(
|
||||
}
|
||||
|
||||
/// Invokes the browser probe Lambda for a leased check row.
|
||||
async fn invoke_probe(row: synthetics_jobs::LeasedRow) {
|
||||
async fn invoke_probe(row: synthetics_jobs::LeasedRow, stop: crate::StopToken) {
|
||||
// Leased just before ownership moved: do not start the probe here. The
|
||||
// lease expires and the new owner's reaper re-queues the row.
|
||||
if stop.is_cancelled() {
|
||||
tracing::info!(job_id = %row.id, "[synthetics dispatcher] ownership lost — not invoking");
|
||||
return;
|
||||
}
|
||||
// Derive AWS region from the job's location field.
|
||||
let region = match aws_region_from_location(&row.location) {
|
||||
Some(r) => r.to_owned(),
|
||||
@@ -691,3 +767,16 @@ async fn handle_dispatch_failure(
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod lease_batch_tests {
|
||||
use super::next_batch_limit;
|
||||
|
||||
#[test]
|
||||
fn batch_limit_is_trimmed_to_the_tick_cap() {
|
||||
assert_eq!(next_batch_limit(10, 0, 100), 10);
|
||||
assert_eq!(next_batch_limit(10, 95, 100), 5);
|
||||
assert_eq!(next_batch_limit(10, 100, 100), 0);
|
||||
assert_eq!(next_batch_limit(10, 120, 100), 0);
|
||||
}
|
||||
}
|
||||
|
||||
+248
-41
@@ -60,6 +60,7 @@ pub(crate) static CONFIG_SWAP_LOCK: tokio::sync::Mutex<()> = tokio::sync::Mutex:
|
||||
pub const BUILT_WITH_CLOUD: bool = cfg!(feature = "cloud");
|
||||
|
||||
pub mod alerting;
|
||||
pub mod callback_endpoint;
|
||||
pub mod dispatcher;
|
||||
pub mod job_api;
|
||||
pub mod pool;
|
||||
@@ -157,52 +158,200 @@ pub async fn init(step_pool: Option<pool::StepPoolHooks>) {
|
||||
// super-cluster KV client is built and no NATS read is issued on this path
|
||||
// (spec §14 — super cluster off must be byte-identical).
|
||||
#[cfg(not(feature = "enterprise"))]
|
||||
spawn_workers();
|
||||
drop(spawn_workers(StopToken::never()));
|
||||
|
||||
#[cfg(feature = "enterprise")]
|
||||
if !o2_enterprise::enterprise::common::config::get_config()
|
||||
.super_cluster
|
||||
.enabled
|
||||
{
|
||||
spawn_workers();
|
||||
drop(spawn_workers(StopToken::never()));
|
||||
return;
|
||||
}
|
||||
|
||||
#[cfg(feature = "enterprise")]
|
||||
tokio::spawn(async move {
|
||||
let mut announced = false;
|
||||
while !holds_job_cluster_claim().await {
|
||||
if !announced {
|
||||
// Once, not per poll: a region that legitimately does not hold
|
||||
// the claim is the steady state for the whole life of the
|
||||
// process, and an every-10s line would bury it.
|
||||
tracing::info!(
|
||||
"[synthetics] another cluster holds the job-cluster claim (or none is \
|
||||
registered yet) — workers not started, re-checking every {}s",
|
||||
JOB_CLUSTER_POLL.as_secs()
|
||||
);
|
||||
announced = true;
|
||||
}
|
||||
tokio::time::sleep(JOB_CLUSTER_POLL).await;
|
||||
}
|
||||
tracing::info!("[synthetics] this cluster holds the job-cluster claim — starting workers");
|
||||
spawn_workers();
|
||||
});
|
||||
tokio::spawn(ownership_supervisor());
|
||||
}
|
||||
|
||||
fn spawn_workers() {
|
||||
tokio::spawn(scheduler::run());
|
||||
tokio::spawn(dispatcher::run());
|
||||
tokio::spawn(reaper::run());
|
||||
// POC scope: single cluster. Multi-region needs this split — the
|
||||
// incident-engine half claim-gated, the snapshot half per region.
|
||||
// Status pages ships with synthetics — no separate toggle.
|
||||
tokio::spawn(status_pages::run());
|
||||
tokio::spawn(status_pages::run_domain_verifier());
|
||||
// Its own task, not a step on the reaper's tick: a pass is up to a thousand
|
||||
// rows of outbound HTTP, and the reaper's lease bookkeeping cannot wait
|
||||
// behind it. Its own kill switch too, checked per pass rather than here.
|
||||
tokio::spawn(reaper::orphan::run());
|
||||
/// Cooperative stop signal handed to every claim-gated worker.
|
||||
///
|
||||
/// A minimal `CancellationToken` on `tokio::sync::watch` (the crate has no
|
||||
/// `tokio-util` dependency, and adding one is not worth a manifest change).
|
||||
/// A dropped sender reads as cancelled, so a supervisor that goes away fails
|
||||
/// closed rather than leaving workers running unsupervised.
|
||||
#[derive(Clone)]
|
||||
pub struct StopToken {
|
||||
rx: tokio::sync::watch::Receiver<bool>,
|
||||
// Only set by [`StopToken::never`]: keeps the channel open forever.
|
||||
_keep: Option<std::sync::Arc<tokio::sync::watch::Sender<bool>>>,
|
||||
}
|
||||
|
||||
impl StopToken {
|
||||
/// A token plus the sender that cancels it.
|
||||
pub fn new() -> (tokio::sync::watch::Sender<bool>, Self) {
|
||||
let (tx, rx) = tokio::sync::watch::channel(false);
|
||||
(tx, Self { rx, _keep: None })
|
||||
}
|
||||
|
||||
/// A token that is never cancelled: the single-cluster path, where there
|
||||
/// is no ownership to lose.
|
||||
pub fn never() -> Self {
|
||||
let (tx, rx) = tokio::sync::watch::channel(false);
|
||||
Self {
|
||||
rx,
|
||||
_keep: Some(std::sync::Arc::new(tx)),
|
||||
}
|
||||
}
|
||||
|
||||
pub fn is_cancelled(&self) -> bool {
|
||||
*self.rx.borrow() || (self._keep.is_none() && self.rx.has_changed().is_err())
|
||||
}
|
||||
|
||||
pub async fn cancelled(&self) {
|
||||
let mut rx = self.rx.clone();
|
||||
// Err = sender dropped, which also means stop.
|
||||
let _ = rx.wait_for(|v| *v).await;
|
||||
}
|
||||
|
||||
/// Sleeps for `d`; returns `true` when cancelled first (the caller exits
|
||||
/// its loop at this tick boundary, never mid-pass).
|
||||
pub async fn sleep(&self, d: std::time::Duration) -> bool {
|
||||
tokio::select! {
|
||||
_ = self.cancelled() => true,
|
||||
_ = tokio::time::sleep(d) => self.is_cancelled(),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// Result of one read of the job-cluster claim. A failed read is `Unknown`,
|
||||
/// never "held": the supervisor fails closed.
|
||||
#[cfg_attr(not(feature = "enterprise"), allow(dead_code))]
|
||||
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
|
||||
enum ClaimRead {
|
||||
Held,
|
||||
NotHeld,
|
||||
Unknown,
|
||||
}
|
||||
|
||||
#[cfg_attr(not(feature = "enterprise"), allow(dead_code))]
|
||||
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
|
||||
enum SupervisorAction {
|
||||
Start,
|
||||
Stop,
|
||||
Hold,
|
||||
}
|
||||
|
||||
/// The supervisor's whole decision: a pure function of whether a worker
|
||||
/// generation is running and what the claim read returned. Workers run exactly
|
||||
/// while the claim is provably held here; `NotHeld` and `Unknown` both stop a
|
||||
/// running generation (same fail-closed rule as the anomaly supervisor in
|
||||
/// `jobs::job::mod.rs`).
|
||||
#[cfg_attr(not(feature = "enterprise"), allow(dead_code))]
|
||||
fn supervisor_action(running: bool, claim: ClaimRead) -> SupervisorAction {
|
||||
match (running, claim) {
|
||||
(false, ClaimRead::Held) => SupervisorAction::Start,
|
||||
(true, ClaimRead::NotHeld | ClaimRead::Unknown) => SupervisorAction::Stop,
|
||||
_ => SupervisorAction::Hold,
|
||||
}
|
||||
}
|
||||
|
||||
/// Start/Stop supervisor for the claim-gated workers (o2-enterprise#3190).
|
||||
///
|
||||
/// Replaces the old start-only latch, which spawned the workers once on
|
||||
/// gaining the claim and never stopped them when the claim moved away, so two
|
||||
/// regions could schedule, dispatch and reap the same checks. Each generation
|
||||
/// gets its own [`StopToken`]; Stop cancels it and JOINS every worker before
|
||||
/// the next Start can happen, so there are never two generations at once.
|
||||
#[cfg(feature = "enterprise")]
|
||||
async fn ownership_supervisor() {
|
||||
let mut generation: u64 = 0;
|
||||
let mut running: Option<(
|
||||
tokio::sync::watch::Sender<bool>,
|
||||
Vec<tokio::task::JoinHandle<()>>,
|
||||
)> = None;
|
||||
let mut last_claim: Option<ClaimRead> = None;
|
||||
loop {
|
||||
let claim = read_job_cluster_claim().await;
|
||||
if last_claim != Some(claim) && claim == ClaimRead::NotHeld && running.is_none() {
|
||||
tracing::info!(
|
||||
"[synthetics] another cluster holds the job-cluster claim (or none is registered \
|
||||
yet) — workers not running here, re-checking every {}s",
|
||||
JOB_CLUSTER_POLL.as_secs()
|
||||
);
|
||||
}
|
||||
last_claim = Some(claim);
|
||||
match supervisor_action(running.is_some(), claim) {
|
||||
SupervisorAction::Start => {
|
||||
generation += 1;
|
||||
tracing::info!(
|
||||
generation,
|
||||
"[synthetics] ownership acquired — starting workers"
|
||||
);
|
||||
let (tx, token) = StopToken::new();
|
||||
running = Some((tx, spawn_workers(token)));
|
||||
}
|
||||
SupervisorAction::Stop => {
|
||||
if claim == ClaimRead::Unknown {
|
||||
tracing::warn!(
|
||||
generation,
|
||||
"[synthetics] ownership unknown (claim read failed) — stopping workers"
|
||||
);
|
||||
} else {
|
||||
tracing::info!(generation, "[synthetics] ownership lost — stopping workers");
|
||||
}
|
||||
if let Some((tx, handles)) = running.take() {
|
||||
let _ = tx.send(true);
|
||||
for h in handles {
|
||||
if let Err(e) = h.await {
|
||||
tracing::error!(generation, "[synthetics] worker join failed: {e}");
|
||||
}
|
||||
}
|
||||
}
|
||||
tracing::info!(generation, "[synthetics] workers stopped");
|
||||
}
|
||||
SupervisorAction::Hold => {}
|
||||
}
|
||||
tokio::time::sleep(JOB_CLUSTER_POLL).await;
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(feature = "enterprise")]
|
||||
async fn read_job_cluster_claim() -> ClaimRead {
|
||||
match holds_job_cluster_claim().await {
|
||||
Some(true) => ClaimRead::Held,
|
||||
Some(false) => ClaimRead::NotHeld,
|
||||
None => ClaimRead::Unknown,
|
||||
}
|
||||
}
|
||||
|
||||
/// Spawns one generation of workers. Every worker takes `stop` and leaves its
|
||||
/// loop at the next tick boundary once it is cancelled.
|
||||
///
|
||||
/// All of them are claim-gated, as they were under the old latch: the status
|
||||
/// page task is still one loop (incident engine and snapshot together — the
|
||||
/// per-region snapshot split is future work), and the domain verifier writes
|
||||
/// shared verification state, so both stay owner-only.
|
||||
fn spawn_workers(stop: StopToken) -> Vec<tokio::task::JoinHandle<()>> {
|
||||
let verifier_stop = stop.clone();
|
||||
vec![
|
||||
tokio::spawn(scheduler::run(stop.clone())),
|
||||
tokio::spawn(dispatcher::run(stop.clone())),
|
||||
tokio::spawn(reaper::run(stop.clone())),
|
||||
// Status pages ships with synthetics — no separate toggle.
|
||||
tokio::spawn(status_pages::run(stop.clone())),
|
||||
// The verifier lives in the enterprise crate and takes no token, so it
|
||||
// is dropped at its next await point on cancel.
|
||||
tokio::spawn(async move {
|
||||
tokio::select! {
|
||||
_ = verifier_stop.cancelled() => {}
|
||||
_ = status_pages::run_domain_verifier() => {}
|
||||
}
|
||||
}),
|
||||
// Its own task, not a step on the reaper's tick: a pass is up to a
|
||||
// thousand rows of outbound HTTP, and the reaper's lease bookkeeping
|
||||
// cannot wait behind it. Its own kill switch too, checked per pass.
|
||||
tokio::spawn(reaper::orphan::run(stop)),
|
||||
]
|
||||
}
|
||||
|
||||
/// One evaluation of [`should_start`] against the super-cluster KV.
|
||||
@@ -217,7 +366,7 @@ fn spawn_workers() {
|
||||
/// increments `SYNTHETICS_ORPHAN_SCANS_TOTAL` anywhere, which is exactly the
|
||||
/// dead-region signal `reaper::orphan` documents.
|
||||
#[cfg(feature = "enterprise")]
|
||||
async fn holds_job_cluster_claim() -> bool {
|
||||
async fn holds_job_cluster_claim() -> Option<bool> {
|
||||
use o2_enterprise::enterprise::super_cluster::kv;
|
||||
|
||||
let local = config::get_cluster_name();
|
||||
@@ -225,27 +374,27 @@ async fn holds_job_cluster_claim() -> bool {
|
||||
Ok(name) => name,
|
||||
Err(e) => {
|
||||
tracing::error!("[synthetics] could not read the job cluster, not starting yet: {e}");
|
||||
return false;
|
||||
return None;
|
||||
}
|
||||
};
|
||||
|
||||
// Both settled without the cluster list, which is the more expensive read.
|
||||
// Must agree with `should_start`, which is where the rule is tested.
|
||||
if claim.is_empty() {
|
||||
return false;
|
||||
return Some(false);
|
||||
}
|
||||
if claim == local {
|
||||
return true;
|
||||
return Some(true);
|
||||
}
|
||||
|
||||
let live = match kv::cluster::list_by_role_group(None).await {
|
||||
Ok(clusters) => clusters.into_iter().map(|c| c.name).collect::<Vec<_>>(),
|
||||
Err(e) => {
|
||||
tracing::error!("[synthetics] could not list clusters, not starting yet: {e}");
|
||||
return false;
|
||||
return None;
|
||||
}
|
||||
};
|
||||
should_start(true, &claim, &local, &live)
|
||||
Some(should_start(true, &claim, &local, &live))
|
||||
}
|
||||
|
||||
/// Whether this cluster may start the synthetics workers.
|
||||
@@ -344,4 +493,62 @@ mod job_cluster_gate_tests {
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {}
|
||||
mod tests {
|
||||
use super::{ClaimRead::*, StopToken, SupervisorAction::*, supervisor_action};
|
||||
|
||||
#[test]
|
||||
fn starts_only_on_a_held_claim() {
|
||||
assert_eq!(supervisor_action(false, Held), Start);
|
||||
assert_eq!(supervisor_action(false, NotHeld), Hold);
|
||||
assert_eq!(supervisor_action(false, Unknown), Hold);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn a_running_generation_stops_on_loss_or_read_error() {
|
||||
assert_eq!(supervisor_action(true, NotHeld), Stop);
|
||||
assert_eq!(supervisor_action(true, Unknown), Stop);
|
||||
assert_eq!(supervisor_action(true, Held), Hold);
|
||||
}
|
||||
|
||||
/// Flapping claim: replay the reads through the decision function and
|
||||
/// check the generation never overlaps and every flip is acted on.
|
||||
#[test]
|
||||
fn flapping_claim_alternates_start_and_stop_without_overlap() {
|
||||
let reads = [Held, Held, NotHeld, Held, Unknown, Unknown, Held, NotHeld];
|
||||
let mut running = false;
|
||||
let mut starts = 0;
|
||||
let mut stops = 0;
|
||||
for r in reads {
|
||||
match supervisor_action(running, r) {
|
||||
Start => {
|
||||
assert!(!running, "never two generations");
|
||||
running = true;
|
||||
starts += 1;
|
||||
}
|
||||
Stop => {
|
||||
assert!(running);
|
||||
running = false;
|
||||
stops += 1;
|
||||
}
|
||||
Hold => {}
|
||||
}
|
||||
}
|
||||
assert_eq!((starts, stops), (3, 3));
|
||||
assert!(!running);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn stop_token_cancels_sleep_and_dropped_sender_fails_closed() {
|
||||
let (tx, token) = StopToken::new();
|
||||
assert!(!token.is_cancelled());
|
||||
tx.send(true).unwrap();
|
||||
assert!(token.sleep(std::time::Duration::from_secs(60)).await);
|
||||
|
||||
let (tx, token) = StopToken::new();
|
||||
drop(tx);
|
||||
assert!(token.sleep(std::time::Duration::from_secs(60)).await);
|
||||
|
||||
let never = StopToken::never();
|
||||
assert!(!never.sleep(std::time::Duration::from_millis(1)).await);
|
||||
}
|
||||
}
|
||||
|
||||
@@ -48,11 +48,14 @@ use infra::{
|
||||
use crate::{MAX_DISPATCH_ATTEMPTS as MAX_ATTEMPTS, alerting::ERROR_SOURCE_DISPATCH};
|
||||
const TICK: Duration = Duration::from_secs(30);
|
||||
|
||||
pub async fn run() {
|
||||
pub async fn run(stop: crate::StopToken) {
|
||||
tracing::info!("[synthetics reaper] started");
|
||||
|
||||
loop {
|
||||
tokio::time::sleep(TICK).await;
|
||||
if stop.sleep(TICK).await {
|
||||
tracing::info!("[synthetics reaper] stopped");
|
||||
return;
|
||||
}
|
||||
|
||||
let db = get_orm_client_rw().await;
|
||||
|
||||
@@ -181,7 +184,13 @@ async fn handle_dead_letter(
|
||||
// check that is already in Error dead-letters silently, which is the common
|
||||
// case when a private agent has gone away and every run of every check it
|
||||
// served is expiring.
|
||||
match synthetics_checks::update_last_check_status(db, &row.synthetics_id, 4).await {
|
||||
match synthetics_checks::update_last_check_status(
|
||||
db,
|
||||
&row.synthetics_id,
|
||||
config::meta::status_pages::CHECK_STATUS_ERROR,
|
||||
)
|
||||
.await
|
||||
{
|
||||
Ok(true) => {
|
||||
#[cfg(feature = "enterprise")]
|
||||
if o2_enterprise::enterprise::common::config::get_config()
|
||||
@@ -218,21 +227,16 @@ async fn handle_dead_letter(
|
||||
// points at a dead agent or a location nothing polls; the other two point at
|
||||
// a probe that took the job and stopped talking. They need different
|
||||
// responses, so they should not read identically in the record.
|
||||
let error_msg = match row.reason {
|
||||
synthetics_jobs::DeadLetterReason::AttemptsExhausted => format!(
|
||||
"probe did not respond after {} dispatch attempts",
|
||||
row.dispatch_attempts
|
||||
),
|
||||
synthetics_jobs::DeadLetterReason::Expired => format!(
|
||||
"probe held the check but did not respond before its schedule window closed \
|
||||
(after {} dispatch attempt(s)); the next scheduled run will retry",
|
||||
row.dispatch_attempts
|
||||
),
|
||||
synthetics_jobs::DeadLetterReason::NeverDispatched => {
|
||||
"no probe claimed the check for this location before its schedule window closed"
|
||||
.to_string()
|
||||
}
|
||||
};
|
||||
// Phase, attempt count and the callback endpoint (o2-enterprise#3165): a
|
||||
// timeout says the probe did not report, not why, so the message names
|
||||
// where it was told to report and stops short of a cause.
|
||||
//
|
||||
// The endpoint is the one resolved NOW, not at dispatch: the job row does
|
||||
// not store the dispatch-time endpoint. Capturing it needs a column
|
||||
// (deferred); until then a config change between dispatch and reaping
|
||||
// would show the new value.
|
||||
let callback = crate::callback_endpoint::current_callback_endpoint();
|
||||
let error_msg = dead_letter_message(row.reason, row.dispatch_attempts, &callback);
|
||||
|
||||
// Above the token lookup, which returns early: the triggers half needs no token.
|
||||
usage_reporting::publish_triggers_usage(dead_letter_trigger(row, now_us, &error_msg));
|
||||
@@ -264,6 +268,32 @@ async fn handle_dead_letter(
|
||||
write_results_stream(row, &ingest_token, &api_endpoint, now_us, &error_msg).await;
|
||||
}
|
||||
|
||||
/// The dead-letter text for one reason. `callback` must already be sanitized.
|
||||
fn dead_letter_message(
|
||||
reason: synthetics_jobs::DeadLetterReason,
|
||||
dispatch_attempts: i32,
|
||||
callback: &str,
|
||||
) -> String {
|
||||
match reason {
|
||||
synthetics_jobs::DeadLetterReason::AttemptsExhausted => format!(
|
||||
"probe did not respond after {dispatch_attempts} dispatch attempts \
|
||||
(phase: claimed, no result before the lease ran out); the probe may be unable to reach \
|
||||
the callback endpoint {callback}"
|
||||
),
|
||||
synthetics_jobs::DeadLetterReason::Expired => format!(
|
||||
"probe held the check but did not respond before its schedule window closed \
|
||||
(phase: claimed, lease expired without a result; after {dispatch_attempts} dispatch attempt(s)); \
|
||||
the probe may be unable to reach the callback endpoint {callback}; \
|
||||
the next scheduled run will retry"
|
||||
),
|
||||
synthetics_jobs::DeadLetterReason::NeverDispatched => format!(
|
||||
"no probe claimed the check for this location before its schedule window closed \
|
||||
(phase: never claimed; {dispatch_attempts} dispatch attempt(s)); a probe may be \
|
||||
unable to reach the callback endpoint {callback}"
|
||||
),
|
||||
}
|
||||
}
|
||||
|
||||
async fn write_results_stream(
|
||||
row: &synthetics_jobs::DeadLetteredRow,
|
||||
ingest_token: &str,
|
||||
@@ -393,4 +423,21 @@ mod tests {
|
||||
"an alert rule matches the SERIALIZED value, and this row writes `failed` today",
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn dead_letter_messages_name_phase_attempts_and_endpoint() {
|
||||
use infra::table::synthetics_jobs::DeadLetterReason as R;
|
||||
let ep = "https://o2.example.com/api";
|
||||
for (reason, phase) in [
|
||||
(R::AttemptsExhausted, "phase: claimed, no result"),
|
||||
(R::Expired, "phase: claimed, lease expired"),
|
||||
(R::NeverDispatched, "phase: never claimed"),
|
||||
] {
|
||||
let msg = super::dead_letter_message(reason, 3, ep);
|
||||
assert!(msg.contains(phase), "{msg}");
|
||||
assert!(msg.contains("3 dispatch attempt"), "{msg}");
|
||||
assert!(msg.contains(ep), "{msg}");
|
||||
assert!(msg.contains("may be unable to reach"), "{msg}");
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -296,7 +296,7 @@ async fn is_scan_leader() -> bool {
|
||||
/// slow scan stalls run accounting, and runs that never complete never notify.
|
||||
/// A separate loop cannot do that, and it is free to run at a cadence that
|
||||
/// suits the threshold instead of the lease timers.
|
||||
pub async fn run() {
|
||||
pub async fn run(stop: crate::StopToken) {
|
||||
tracing::info!("[synthetics orphan] started");
|
||||
if !orphan_detection_enabled() {
|
||||
tracing::info!(
|
||||
@@ -310,7 +310,10 @@ pub async fn run() {
|
||||
let mut last_reported: HashMap<String, i64> = HashMap::new();
|
||||
|
||||
loop {
|
||||
tokio::time::sleep(ORPHAN_TICK).await;
|
||||
if stop.sleep(ORPHAN_TICK).await {
|
||||
tracing::info!("[synthetics orphan] stopped");
|
||||
return;
|
||||
}
|
||||
|
||||
// Read per pass, not captured here: the operator turning this off is
|
||||
// reacting to something happening now.
|
||||
|
||||
@@ -279,11 +279,14 @@ pub fn trial_gate_decision(
|
||||
}
|
||||
}
|
||||
|
||||
pub async fn run() {
|
||||
pub async fn run(stop: crate::StopToken) {
|
||||
tracing::info!("[synthetics scheduler] started");
|
||||
|
||||
loop {
|
||||
tokio::time::sleep(TICK).await;
|
||||
if stop.sleep(TICK).await {
|
||||
tracing::info!("[synthetics scheduler] stopped");
|
||||
return;
|
||||
}
|
||||
|
||||
let db = get_orm_client_rw().await;
|
||||
|
||||
|
||||
@@ -30,6 +30,27 @@ enum Inner {
|
||||
Distributed(Option<infra::dist_lock::Locker>),
|
||||
}
|
||||
|
||||
/// Longest a save or delete waits for the org-wide composition lock locally.
|
||||
/// The existing lock-wait config (`ZO_NATS_LOCK_WAIT_TIMEOUT`, 3600s) belongs to
|
||||
/// NATS and is far too long for a request, so this is a fixed bound. A stuck
|
||||
/// holder then fails the request with a retryable error instead of hanging it.
|
||||
const LOCAL_LOCK_WAIT: std::time::Duration = std::time::Duration::from_secs(30);
|
||||
|
||||
async fn lock_local(
|
||||
key: &str,
|
||||
wait: std::time::Duration,
|
||||
) -> Result<tokio::sync::OwnedMutexGuard<bool>, infra::errors::Error> {
|
||||
let holder = infra::local_lock::lock(key).await?;
|
||||
tokio::time::timeout(wait, holder.lock_owned())
|
||||
.await
|
||||
.map_err(|_| {
|
||||
infra::errors::Error::Message(format!(
|
||||
"timed out after {}s waiting for the synthetics composition lock; retry shortly",
|
||||
wait.as_secs()
|
||||
))
|
||||
})
|
||||
}
|
||||
|
||||
pub async fn lock(org_id: &str) -> Result<CompositionGuard, infra::errors::Error> {
|
||||
#[cfg(test)]
|
||||
{
|
||||
@@ -41,8 +62,7 @@ pub async fn lock(org_id: &str) -> Result<CompositionGuard, infra::errors::Error
|
||||
}
|
||||
let key = format!("/synthetics/composition/{org_id}");
|
||||
let inner = if config::get_config().common.local_mode {
|
||||
let holder = infra::local_lock::lock(&key).await?;
|
||||
Inner::Local(holder.lock_owned().await)
|
||||
Inner::Local(lock_local(&key, LOCAL_LOCK_WAIT).await?)
|
||||
} else {
|
||||
Inner::Distributed(infra::dist_lock::lock(&key, 0).await?)
|
||||
};
|
||||
@@ -61,3 +81,22 @@ impl CompositionGuard {
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use std::time::Duration;
|
||||
|
||||
use super::*;
|
||||
|
||||
#[tokio::test]
|
||||
async fn local_lock_contention_times_out_then_recovers_on_release() {
|
||||
let key = "/synthetics/composition/lock-timeout-test";
|
||||
let held = lock_local(key, Duration::from_secs(1)).await.unwrap();
|
||||
let err = lock_local(key, Duration::from_millis(50))
|
||||
.await
|
||||
.unwrap_err();
|
||||
assert!(err.to_string().contains("retry"), "{err}");
|
||||
drop(held);
|
||||
lock_local(key, Duration::from_millis(50)).await.unwrap();
|
||||
}
|
||||
}
|
||||
|
||||
@@ -332,8 +332,14 @@ fn default_true() -> bool {
|
||||
#[derive(Debug, Deserialize)]
|
||||
pub struct UpdateLocationRequest {
|
||||
pub label: String,
|
||||
#[serde(default = "default_true")]
|
||||
pub enabled: bool,
|
||||
/// Omitted keeps the stored value; an explicit `false` disables.
|
||||
#[serde(default)]
|
||||
pub enabled: Option<bool>,
|
||||
}
|
||||
|
||||
/// The `enabled` value to store: the request's, else the row's current one.
|
||||
fn resolve_enabled(requested: Option<bool>, stored: bool) -> bool {
|
||||
requested.unwrap_or(stored)
|
||||
}
|
||||
|
||||
/// Legacy compat: old UIs sent bare AWS regions ("us-east-1") for public
|
||||
@@ -595,8 +601,9 @@ pub async fn update_location(
|
||||
id: &str,
|
||||
req: UpdateLocationRequest,
|
||||
) -> anyhow::Result<synthetics_locations::SyntheticsLocationRecord> {
|
||||
location_for_write(org_id, is_root, id).await?;
|
||||
synthetics_locations::update(id, &req.label, req.enabled).await?;
|
||||
let existing = location_for_write(org_id, is_root, id).await?;
|
||||
let enabled = resolve_enabled(req.enabled, existing.enabled);
|
||||
synthetics_locations::update(id, &req.label, enabled).await?;
|
||||
// The two columns the write touched, not the row: sending a row would carry
|
||||
// `down_notified_at` with it, and that flag is the claim on the right to
|
||||
// send one "location down" notification.
|
||||
@@ -606,9 +613,7 @@ pub async fn update_location(
|
||||
.enabled
|
||||
{
|
||||
o2_enterprise::enterprise::super_cluster::queue::synthetics_location_update(
|
||||
id,
|
||||
&req.label,
|
||||
req.enabled,
|
||||
id, &req.label, enabled,
|
||||
)
|
||||
.await?;
|
||||
}
|
||||
@@ -725,6 +730,20 @@ async fn private_location_install(
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
#[test]
|
||||
fn label_only_update_preserves_disabled() {
|
||||
let req: UpdateLocationRequest = serde_json::from_str(r#"{"label":"new"}"#).unwrap();
|
||||
assert_eq!(req.enabled, None);
|
||||
assert!(!resolve_enabled(req.enabled, false));
|
||||
assert!(resolve_enabled(req.enabled, true));
|
||||
let req: UpdateLocationRequest =
|
||||
serde_json::from_str(r#"{"label":"new","enabled":false}"#).unwrap();
|
||||
assert!(!resolve_enabled(req.enabled, true));
|
||||
let req: UpdateLocationRequest =
|
||||
serde_json::from_str(r#"{"label":"new","enabled":true}"#).unwrap();
|
||||
assert!(resolve_enabled(req.enabled, false));
|
||||
}
|
||||
|
||||
/// Moved from the enterprise config module with the reader it pins.
|
||||
///
|
||||
/// Decides whether a location shows Offline and whether the staleness
|
||||
|
||||
@@ -91,11 +91,20 @@ pub async fn preview_snapshot(
|
||||
.push(u);
|
||||
}
|
||||
|
||||
// Same state read as the rebuilder, so a preview of an all-Error component
|
||||
// shows NoData exactly as the published snapshot does.
|
||||
let preview_check_ids: Vec<String> = mappings
|
||||
.iter()
|
||||
.map(|m| m.synthetics_id.clone())
|
||||
.collect::<HashSet<_>>()
|
||||
.into_iter()
|
||||
.collect();
|
||||
let (states, _) = table::get_check_states(conn, &preview_check_ids, None).await?;
|
||||
let data = TickData {
|
||||
pages: vec![page.clone()],
|
||||
components,
|
||||
mappings,
|
||||
states: Vec::new(),
|
||||
states,
|
||||
snoozed: HashSet::new(),
|
||||
notices: notices.clone(),
|
||||
};
|
||||
@@ -146,7 +155,7 @@ pub async fn refresh_snapshot(org_id: &str, page_id: &str) -> Result<(), infra::
|
||||
.await
|
||||
}
|
||||
|
||||
pub async fn run() {
|
||||
pub async fn run(stop: crate::StopToken) {
|
||||
let interval = config::get_config()
|
||||
.synthetics
|
||||
.status_page_rebuild_interval
|
||||
@@ -164,7 +173,10 @@ pub async fn run() {
|
||||
// deletions and refreshes `last_triggered_at` for the dead-probe rule.
|
||||
let mut rt = RebuilderState::default();
|
||||
loop {
|
||||
tokio::time::sleep(std::time::Duration::from_secs(interval)).await;
|
||||
if stop.sleep(std::time::Duration::from_secs(interval)).await {
|
||||
tracing::info!("[status_pages] rebuilder stopped");
|
||||
return;
|
||||
}
|
||||
if let Err(e) = tick(&mut hashes, &mut down_streaks, &mut rt).await {
|
||||
tracing::error!("[status_pages] rebuild tick failed: {e}");
|
||||
}
|
||||
@@ -183,6 +195,106 @@ struct RebuilderState {
|
||||
/// Highest `updated_at` consumed so far (the delta watermark).
|
||||
watermark: i64,
|
||||
tick_count: u64,
|
||||
/// Per check, the `completed_at` of the last run this loop counted.
|
||||
seen_runs: HashMap<String, i64>,
|
||||
}
|
||||
|
||||
/// How far back (by `scheduled_ts`) the rebuilder looks for completed runs.
|
||||
/// Only completions that are new since the previous tick matter, and a probe
|
||||
/// run completes within its job timeout (minutes), so an hour covers every
|
||||
/// normal result with room for a delayed tick. A run that completes later than
|
||||
/// this — a long-expired dead letter — ends as Error, which never advances a
|
||||
/// streak anyway.
|
||||
const RUN_COMPLETION_LOOKBACK_MICROS: i64 = 3_600_000_000;
|
||||
|
||||
/// Records one tick's view of a check and says whether it is a new result.
|
||||
///
|
||||
/// Streaks count completed RUNS, not rebuild ticks: the rebuilder ticks far
|
||||
/// more often than most checks run, so counting ticks confirmed a single DOWN
|
||||
/// as three failures. The marker is the check's latest
|
||||
/// `synthetics_runs.completed_at`, so one completed run counts exactly once.
|
||||
/// (`last_triggered_at` moves at dispatch, before the result exists, so it
|
||||
/// counted the previous result again when the next run was dispatched.) No
|
||||
/// completed run in the lookback window means nothing new to count.
|
||||
fn observe_run(
|
||||
down_streaks: &mut HashMap<String, i32>,
|
||||
seen_runs: &mut HashMap<String, i64>,
|
||||
s: &table::CheckStateRow,
|
||||
) -> bool {
|
||||
let fresh = match s.last_completed_at {
|
||||
Some(at) => seen_runs.insert(s.id.clone(), at) != Some(at),
|
||||
None => false,
|
||||
};
|
||||
if s.last_check_status == engine::CHECK_STATUS_DOWN {
|
||||
if fresh {
|
||||
*down_streaks.entry(s.id.clone()).or_insert(0) += 1;
|
||||
}
|
||||
} else {
|
||||
down_streaks.remove(&s.id);
|
||||
}
|
||||
fresh
|
||||
}
|
||||
|
||||
/// Per-check thresholds from the pages the check is mapped to. A check on
|
||||
/// several pages uses the STRICTEST settings (max of each value), so one
|
||||
/// result stream never opens or resolves an org-wide notice earlier than any
|
||||
/// of its pages allows.
|
||||
fn thresholds_by_check(data: &TickData) -> HashMap<String, engine::Thresholds> {
|
||||
let page_of_component: HashMap<&str, &str> = data
|
||||
.components
|
||||
.iter()
|
||||
.map(|c| (c.id.as_str(), c.status_page_id.as_str()))
|
||||
.collect();
|
||||
let page_th: HashMap<&str, engine::Thresholds> = data
|
||||
.pages
|
||||
.iter()
|
||||
.map(|p| (p.id.as_str(), page_thresholds(p)))
|
||||
.collect();
|
||||
let mut out: HashMap<String, engine::Thresholds> = HashMap::new();
|
||||
for m in &data.mappings {
|
||||
let Some(th) = page_of_component
|
||||
.get(m.component_id.as_str())
|
||||
.and_then(|pid| page_th.get(pid))
|
||||
else {
|
||||
continue;
|
||||
};
|
||||
out.entry(m.synthetics_id.clone())
|
||||
.and_modify(|cur| *cur = strictest(*cur, *th))
|
||||
.or_insert(*th);
|
||||
}
|
||||
out
|
||||
}
|
||||
|
||||
fn page_thresholds(p: &infra::table::entity::status_pages::Model) -> engine::Thresholds {
|
||||
let d = engine::Thresholds::default();
|
||||
engine::Thresholds {
|
||||
confirm_failures: if p.confirm_failures > 0 {
|
||||
p.confirm_failures
|
||||
} else {
|
||||
d.confirm_failures
|
||||
},
|
||||
confirm_recovery: if p.confirm_recovery > 0 {
|
||||
p.confirm_recovery
|
||||
} else {
|
||||
d.confirm_recovery
|
||||
},
|
||||
confirm_after_micros: p
|
||||
.confirm_after_secs
|
||||
.filter(|s| *s > 0)
|
||||
.and_then(|s| (s as i64).checked_mul(1_000_000)),
|
||||
}
|
||||
}
|
||||
|
||||
fn strictest(a: engine::Thresholds, b: engine::Thresholds) -> engine::Thresholds {
|
||||
engine::Thresholds {
|
||||
confirm_failures: a.confirm_failures.max(b.confirm_failures),
|
||||
confirm_recovery: a.confirm_recovery.max(b.confirm_recovery),
|
||||
// Only time-confirm when every page allows it; then the longest wait.
|
||||
confirm_after_micros: match (a.confirm_after_micros, b.confirm_after_micros) {
|
||||
(Some(x), Some(y)) => Some(x.max(y)),
|
||||
_ => None,
|
||||
},
|
||||
}
|
||||
}
|
||||
|
||||
async fn tick(
|
||||
@@ -193,14 +305,15 @@ async fn tick(
|
||||
let conn = get_orm_client_rw().await;
|
||||
let now = config::utils::time::now_micros();
|
||||
let data = load(conn, now, rt).await?;
|
||||
let mut fresh: HashSet<String> = HashSet::new();
|
||||
rt.seen_runs
|
||||
.retain(|id, _| data.states.iter().any(|s| &s.id == id));
|
||||
for s in &data.states {
|
||||
if s.last_check_status == engine::CHECK_STATUS_DOWN {
|
||||
*down_streaks.entry(s.id.clone()).or_insert(0) += 1;
|
||||
} else {
|
||||
down_streaks.remove(&s.id);
|
||||
if observe_run(down_streaks, &mut rt.seen_runs, s) {
|
||||
fresh.insert(s.id.clone());
|
||||
}
|
||||
}
|
||||
apply_engine_actions(conn, &data, down_streaks, now).await?;
|
||||
apply_engine_actions(conn, &data, down_streaks, &fresh, now).await?;
|
||||
// Re-read notices only if actions changed them; a second bounded read per
|
||||
// tick is cheaper than tracking per-action deltas in the POC.
|
||||
let notices = table::list_notices_since(conn, now - RESOLVED_LOOKBACK_MICROS).await?;
|
||||
@@ -239,7 +352,13 @@ async fn load(
|
||||
// Delta by default; full sweep on the Nth tick. The sweep replaces the
|
||||
// cache wholesale (so deleted/unmapped checks drop out); the delta merges
|
||||
// only changed rows over the carried cache.
|
||||
let full_sweep = rt.tick_count.is_multiple_of(FULL_SWEEP_TICKS);
|
||||
// Always a full read of the mapped checks' state. The delta read keys on
|
||||
// `updated_at`, which only moves on a status TRANSITION; a check that keeps
|
||||
// failing (or keeps passing) advances only `last_triggered_at`, so the
|
||||
// run-based streaks in `observe_run` would see a repeated result only on
|
||||
// the periodic sweep and confirm many minutes late. The read is three
|
||||
// scalar columns of the mapped checks only.
|
||||
let full_sweep = FULL_SWEEP_TICKS > 0;
|
||||
rt.tick_count = rt.tick_count.wrapping_add(1);
|
||||
let since = if full_sweep { None } else { Some(rt.watermark) };
|
||||
let (changed, max_updated) = table::get_check_states(conn, &check_ids, since).await?;
|
||||
@@ -260,7 +379,16 @@ async fn load(
|
||||
rt.check_state.retain(|id, _| mapped.contains(id.as_str()));
|
||||
}
|
||||
rt.watermark = rt.watermark.max(max_updated);
|
||||
let states: Vec<table::CheckStateRow> = rt.check_state.values().cloned().collect();
|
||||
let mut states: Vec<table::CheckStateRow> = rt.check_state.values().cloned().collect();
|
||||
let keys: Vec<(String, String)> = states
|
||||
.iter()
|
||||
.map(|s| (s.org_id.clone(), s.id.clone()))
|
||||
.collect();
|
||||
let completions =
|
||||
table::latest_run_completions(conn, &keys, now - RUN_COMPLETION_LOOKBACK_MICROS).await?;
|
||||
for s in &mut states {
|
||||
s.last_completed_at = completions.get(&s.id).copied();
|
||||
}
|
||||
|
||||
let mut snoozed: HashSet<String> = table::list_active_snoozes(conn, now)
|
||||
.await?
|
||||
@@ -315,6 +443,7 @@ async fn apply_engine_actions(
|
||||
conn: &sea_orm::DatabaseConnection,
|
||||
data: &TickData,
|
||||
down_streaks: &HashMap<String, i32>,
|
||||
fresh: &HashSet<String>,
|
||||
now: i64,
|
||||
) -> Result<(), infra::errors::Error> {
|
||||
let checks: Vec<engine::EngineCheck> = data
|
||||
@@ -327,6 +456,7 @@ async fn apply_engine_actions(
|
||||
consecutive_failures: down_streaks.get(&s.id).copied().unwrap_or(0),
|
||||
first_fail_at: None,
|
||||
failing_since: None,
|
||||
fresh_result: fresh.contains(&s.id),
|
||||
})
|
||||
.collect();
|
||||
|
||||
@@ -358,12 +488,13 @@ async fn apply_engine_actions(
|
||||
}
|
||||
}
|
||||
|
||||
let actions = engine::incident_actions(
|
||||
let th_by_check = thresholds_by_check(data);
|
||||
let actions = engine::incident_actions_with(
|
||||
&checks,
|
||||
&open_by_check,
|
||||
&recently_resolved,
|
||||
&data.snoozed,
|
||||
&engine::Thresholds::default(),
|
||||
|id| th_by_check.get(id).copied().unwrap_or_default(),
|
||||
now,
|
||||
);
|
||||
for action in actions {
|
||||
@@ -616,8 +747,19 @@ fn build_page_snapshot(
|
||||
});
|
||||
}
|
||||
}
|
||||
let has_checks = data.mappings.iter().any(|m| m.component_id == component.id);
|
||||
if !has_checks {
|
||||
let mapped: Vec<&str> = data
|
||||
.mappings
|
||||
.iter()
|
||||
.filter(|m| m.component_id == component.id)
|
||||
.map(|m| m.synthetics_id.as_str())
|
||||
.collect();
|
||||
if mapped.is_empty() {
|
||||
status = engine::ComponentStatus::NoData;
|
||||
} else if status == engine::ComponentStatus::Operational
|
||||
&& all_checks_in_error(&mapped, &data.states)
|
||||
{
|
||||
// Every probe is failing to produce a verdict: we cannot say the
|
||||
// service is up, so show "no data" rather than plain Operational.
|
||||
status = engine::ComponentStatus::NoData;
|
||||
}
|
||||
current_components.push(engine::CurrentComponent {
|
||||
@@ -649,7 +791,24 @@ fn build_page_snapshot(
|
||||
generated_at: now,
|
||||
components: history_components,
|
||||
};
|
||||
(current, history, page_notice_ids.into_iter().collect())
|
||||
// Sorted: HashSet order is random per process, and this list feeds the
|
||||
// serialized snapshot whose hash decides whether to rewrite it.
|
||||
(current, history, sorted_ids(page_notice_ids))
|
||||
}
|
||||
|
||||
fn sorted_ids(set: HashSet<String>) -> Vec<String> {
|
||||
let mut ids: Vec<String> = set.into_iter().collect();
|
||||
ids.sort_unstable();
|
||||
ids
|
||||
}
|
||||
|
||||
fn all_checks_in_error(mapped: &[&str], states: &[table::CheckStateRow]) -> bool {
|
||||
mapped.iter().all(|id| {
|
||||
states
|
||||
.iter()
|
||||
.find(|s| s.id == *id)
|
||||
.is_some_and(|s| s.last_check_status == engine::CHECK_STATUS_ERROR)
|
||||
})
|
||||
}
|
||||
|
||||
fn public_notice(
|
||||
@@ -720,6 +879,116 @@ pub async fn run_domain_verifier() {
|
||||
#[cfg(not(feature = "enterprise"))]
|
||||
pub async fn run_domain_verifier() {}
|
||||
|
||||
#[cfg(test)]
|
||||
mod fast_pass_tests {
|
||||
use super::*;
|
||||
|
||||
fn row(id: &str, status: i32, completed: i64) -> table::CheckStateRow {
|
||||
row_at(id, status, Some(completed))
|
||||
}
|
||||
|
||||
fn row_at(id: &str, status: i32, completed: Option<i64>) -> table::CheckStateRow {
|
||||
table::CheckStateRow {
|
||||
id: id.into(),
|
||||
org_id: "org".into(),
|
||||
last_check_status: status,
|
||||
consecutive_failures: 0,
|
||||
last_triggered_at: 0,
|
||||
last_completed_at: completed,
|
||||
}
|
||||
}
|
||||
|
||||
/// The double count Codex found: dispatching the next run used to move the
|
||||
/// marker while the status still showed the previous result, so one DOWN
|
||||
/// counted twice and `confirm_failures = 2` opened on a single failure.
|
||||
#[test]
|
||||
fn dispatch_without_completion_does_not_count_again() {
|
||||
let (mut d, mut seen) = (HashMap::new(), HashMap::new());
|
||||
let down = engine::CHECK_STATUS_DOWN;
|
||||
assert!(observe_run(&mut d, &mut seen, &row("c", down, 100)));
|
||||
// Next run dispatched (last_triggered_at moves), not yet completed.
|
||||
let mut dispatched = row("c", down, 100);
|
||||
dispatched.last_triggered_at = 999;
|
||||
assert!(!observe_run(&mut d, &mut seen, &dispatched));
|
||||
assert_eq!(d["c"], 1);
|
||||
}
|
||||
|
||||
/// The latest completion falling out of the lookback window is not a new
|
||||
/// result, and its return is not one either.
|
||||
#[test]
|
||||
fn no_completion_in_window_is_not_fresh() {
|
||||
let (mut d, mut seen) = (HashMap::new(), HashMap::new());
|
||||
let down = engine::CHECK_STATUS_DOWN;
|
||||
assert!(!observe_run(&mut d, &mut seen, &row_at("c", down, None)));
|
||||
assert!(!d.contains_key("c"));
|
||||
assert!(observe_run(&mut d, &mut seen, &row("c", down, 100)));
|
||||
assert!(!observe_run(&mut d, &mut seen, &row_at("c", down, None)));
|
||||
assert!(!observe_run(&mut d, &mut seen, &row("c", down, 100)));
|
||||
assert_eq!(d["c"], 1);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn down_streak_counts_runs_not_ticks() {
|
||||
let (mut d, mut seen) = (HashMap::new(), HashMap::new());
|
||||
let down = engine::CHECK_STATUS_DOWN;
|
||||
assert!(observe_run(&mut d, &mut seen, &row("c", down, 100)));
|
||||
for _ in 0..10 {
|
||||
assert!(!observe_run(&mut d, &mut seen, &row("c", down, 100)));
|
||||
}
|
||||
assert_eq!(d["c"], 1);
|
||||
assert!(observe_run(&mut d, &mut seen, &row("c", down, 200)));
|
||||
assert_eq!(d["c"], 2);
|
||||
observe_run(
|
||||
&mut d,
|
||||
&mut seen,
|
||||
&row("c", engine::CHECK_STATUS_ERROR, 300),
|
||||
);
|
||||
assert!(!d.contains_key("c"));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn strictest_takes_max_and_requires_time_confirm_everywhere() {
|
||||
let a = engine::Thresholds {
|
||||
confirm_failures: 3,
|
||||
confirm_recovery: 4,
|
||||
confirm_after_micros: Some(10),
|
||||
};
|
||||
let b = engine::Thresholds {
|
||||
confirm_failures: 7,
|
||||
confirm_recovery: 2,
|
||||
confirm_after_micros: None,
|
||||
};
|
||||
let s = strictest(a, b);
|
||||
assert_eq!((s.confirm_failures, s.confirm_recovery), (7, 4));
|
||||
assert_eq!(s.confirm_after_micros, None);
|
||||
let c = engine::Thresholds {
|
||||
confirm_after_micros: Some(30),
|
||||
..a
|
||||
};
|
||||
assert_eq!(strictest(a, c).confirm_after_micros, Some(30));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn notice_ids_serialize_in_a_stable_order() {
|
||||
let ids = ["n3", "n1", "n2", "n9", "n0"];
|
||||
let fwd: HashSet<String> = ids.iter().map(|s| s.to_string()).collect();
|
||||
let rev: HashSet<String> = ids.iter().rev().map(|s| s.to_string()).collect();
|
||||
let a = serde_json::to_string(&sorted_ids(fwd)).unwrap();
|
||||
let b = serde_json::to_string(&sorted_ids(rev)).unwrap();
|
||||
assert_eq!(a, b);
|
||||
assert_eq!(a, r#"["n0","n1","n2","n3","n9"]"#);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn all_error_component_is_not_operational() {
|
||||
let e = engine::CHECK_STATUS_ERROR;
|
||||
let states = vec![row("a", e, 1), row("b", e, 1), row("c", 1, 1)];
|
||||
assert!(all_checks_in_error(&["a", "b"], &states));
|
||||
assert!(!all_checks_in_error(&["a", "c"], &states));
|
||||
assert!(!all_checks_in_error(&["a", "missing"], &states));
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use sea_orm::{DatabaseBackend, MockDatabase};
|
||||
|
||||
@@ -42,6 +42,7 @@ along with this program. If not, see <http://www.gnu.org/licenses/>.
|
||||
:options="providerOptions"
|
||||
:placeholder="t('synthetics.locations.providerPlaceholder')"
|
||||
required
|
||||
:disabled="isEdit"
|
||||
data-test="synthetics-location-provider-select"
|
||||
/>
|
||||
<OFormInput
|
||||
@@ -50,6 +51,7 @@ along with this program. If not, see <http://www.gnu.org/licenses/>.
|
||||
:label="t('synthetics.locations.customProviderLabel')"
|
||||
:placeholder="t('synthetics.locations.customProviderPlaceholder')"
|
||||
required
|
||||
:disabled="isEdit"
|
||||
data-test="synthetics-location-custom-provider-input"
|
||||
/>
|
||||
<OFormInput
|
||||
@@ -57,6 +59,7 @@ along with this program. If not, see <http://www.gnu.org/licenses/>.
|
||||
:label="t('synthetics.locations.region')"
|
||||
:placeholder="t('synthetics.locations.regionPlaceholder')"
|
||||
required
|
||||
:disabled="isEdit"
|
||||
data-test="synthetics-location-region-input"
|
||||
/>
|
||||
<OFormInput
|
||||
@@ -76,6 +79,7 @@ along with this program. If not, see <http://www.gnu.org/licenses/>.
|
||||
</label>
|
||||
<div
|
||||
class="text-text-secondary rounded-default border-border-default bg-surface-subtle border px-3 py-2 text-sm"
|
||||
data-test="synthetics-location-id-preview"
|
||||
>
|
||||
{{ derivedId || "-" }}
|
||||
</div>
|
||||
@@ -197,6 +201,13 @@ export default defineComponent({
|
||||
watch(
|
||||
[liveProvider, liveCustomProvider, liveRegion],
|
||||
([p, cp, r]: [string, string, string]) => {
|
||||
// Edit: the id is immutable, so show the persisted one, never a
|
||||
// value recomputed from (read-only) provider/region fields.
|
||||
const persistedId = props.isEdit ? (props.data as any)?.id : undefined;
|
||||
if (persistedId) {
|
||||
derivedId.value = persistedId;
|
||||
return;
|
||||
}
|
||||
const provider = p === "custom" ? cp || "custom" : p;
|
||||
derivedId.value = provider && r ? `${provider}-${r}` : "";
|
||||
},
|
||||
|
||||
@@ -632,14 +632,15 @@ type Mode = "all" | "browser";
|
||||
// noticing, not worth acting on this second, and washing it too would put most of
|
||||
// a wobbly estate in colour.
|
||||
const monitorRowClass = (row: any): string => {
|
||||
if (String(row?.status ?? "").toLowerCase() === "failed") return "!bg-status-error-bg";
|
||||
const st = String(row?.status ?? "").toLowerCase();
|
||||
if (st === "failed" || st === "error") return "!bg-status-error-bg";
|
||||
return row?.enabled === false ? "!bg-surface-panel" : "";
|
||||
};
|
||||
|
||||
const monitorRowStyle = (row: any): Record<string, string> => {
|
||||
const status = String(row?.status ?? "").toLowerCase();
|
||||
const color =
|
||||
status === "failed"
|
||||
status === "failed" || status === "error"
|
||||
? "var(--color-error-500)"
|
||||
: status === "warning"
|
||||
? "var(--color-warning-500)"
|
||||
|
||||
@@ -174,6 +174,18 @@ describe("CheckRetries", () => {
|
||||
const lastEmit = emitted![emitted!.length - 1][0] as BrowserCheck;
|
||||
expect(lastEmit.waitBeforeRetrySecs).toBe(30);
|
||||
});
|
||||
|
||||
it("round-trips 5 seconds unscaled (value is seconds, not milliseconds)", async () => {
|
||||
wrapper = mountCheckRetries({ waitBeforeRetrySecs: 5 });
|
||||
|
||||
const input = wrapper.find('[data-test="synthetics-check-retries-delay-input"]');
|
||||
expect(input.attributes("value")).toBe("5");
|
||||
await input.setValue("5");
|
||||
|
||||
const emitted = wrapper.emitted("update:check");
|
||||
const lastEmit = emitted![emitted!.length - 1][0] as BrowserCheck;
|
||||
expect(lastEmit.waitBeforeRetrySecs).toBe(5);
|
||||
});
|
||||
});
|
||||
|
||||
describe("validation error", () => {
|
||||
|
||||
@@ -60,7 +60,7 @@ const retries = computed({
|
||||
},
|
||||
});
|
||||
|
||||
const retryDelayMs = computed({
|
||||
const retryDelaySecs = computed({
|
||||
get: () => props.check.waitBeforeRetrySecs ?? 0,
|
||||
set: (v: string | number) =>
|
||||
emit("update:check", { ...props.check, waitBeforeRetrySecs: Number(v) }),
|
||||
@@ -111,7 +111,7 @@ const retryDelayMs = computed({
|
||||
t("synthetics.scheduleAlert.retryDelay")
|
||||
}}</label>
|
||||
<OInput
|
||||
v-model="retryDelayMs"
|
||||
v-model="retryDelaySecs"
|
||||
type="number"
|
||||
class="w-25!"
|
||||
:placeholder="raw('0')"
|
||||
|
||||
@@ -565,7 +565,7 @@ const startRowColspan = computed(
|
||||
>{{ stepProgress(row)!.done }}/{{ stepProgress(row)!.total }}</span
|
||||
>
|
||||
<OBadge
|
||||
v-else-if="mode === 'editor' && stepBadge(row)"
|
||||
v-else-if="mode !== 'preview' && stepBadge(row)"
|
||||
:variant="stepBadge(row)!.variant"
|
||||
size="sm"
|
||||
:data-test="`synthetics-journey-step-badge-${rowIndex(row)}`"
|
||||
|
||||
@@ -290,7 +290,8 @@ export interface RecordedStep {
|
||||
|
||||
export interface StepExecution {
|
||||
step_id: string;
|
||||
status: "ok" | "fail" | "skipped";
|
||||
/** `unknown`: a status string this UI does not recognise; never shown as a pass. */
|
||||
status: "ok" | "fail" | "skipped" | "unknown";
|
||||
duration_ms: number;
|
||||
error: string | null;
|
||||
start_time: number;
|
||||
@@ -1847,7 +1848,13 @@ export function mapRun(rawHit: Record<string, unknown>): SyntheticRun {
|
||||
|
||||
function normaliseStepStatus(status: unknown): StepExecution["status"] {
|
||||
if (status === "ok" || status === "passed") return "ok";
|
||||
return status === "skipped" ? "skipped" : "fail";
|
||||
if (status === "skipped") return "skipped";
|
||||
// An absent status keeps the historical reading (the step did not finish),
|
||||
// but a value we do not recognise is shown as unknown, not folded into fail.
|
||||
if (typeof status === "string" && status !== "" && status !== "fail" && status !== "failed") {
|
||||
return "unknown";
|
||||
}
|
||||
return "fail";
|
||||
}
|
||||
|
||||
/** `start_load` arrives nested on an attempt or as a JSON string on the record; both map here. */
|
||||
|
||||
@@ -535,6 +535,9 @@ export const BADGE_GROUPS = {
|
||||
offline: { variant: "error-soft", labelKey: "components.badge.serviceStatus.offline" },
|
||||
failed: { variant: "error-soft", labelKey: "components.badge.serviceStatus.failed" },
|
||||
down: { variant: "error-soft", labelKey: "components.badge.serviceStatus.down" },
|
||||
// Synthetics infrastructure error (backend status 4): the probe could not
|
||||
// run, which is not the same as the target failing.
|
||||
error: { variant: "error-soft", labelKey: "components.badge.serviceStatus.error" },
|
||||
unknown: { variant: "default-soft", labelKey: "common.unknown" },
|
||||
},
|
||||
},
|
||||
|
||||
@@ -19159,6 +19159,7 @@
|
||||
"probeError": "خطأ في المجس"
|
||||
},
|
||||
"filters": {
|
||||
"error": "Error",
|
||||
"actionClick": "نقر",
|
||||
"actionType": "النوع",
|
||||
"all": "الجميع",
|
||||
@@ -19874,6 +19875,13 @@
|
||||
"attemptN": "محاولة {n}",
|
||||
"detailAction": "الحدث",
|
||||
"detailSelector": "المختار",
|
||||
"detailStepId": "معرّف الخطوة",
|
||||
"detailTargetUrl": "عنوان URL الهدف",
|
||||
"detailStatus": "الحالة",
|
||||
"detailSkipReason": "سبب التخطي",
|
||||
"stepStatusSkipped": "تم التخطي",
|
||||
"stepStatusUnknown": "غير معروف",
|
||||
"skippedCount": "تم تخطي {n}",
|
||||
"downloadTrace": "تحميل {filename}",
|
||||
"errorFullscreenTitle": "الخطوة {id}: {action} - خطأ",
|
||||
"failedAtStartLoad": "فشل فتح عنوان URL الابتدائي",
|
||||
@@ -20023,7 +20031,7 @@
|
||||
"retriesMaxHint": "أقصى {max}. إعادة المحاولات تتم داخل نفس المهمة، لذا يجب أن يتناسب التسلسل كله مع ميزانية الفحص.",
|
||||
"retriesMaxBrowserHint": "أقصى {max} لفحص المتصفح. كل محاولة تعيد تشغيل الرحلة على كل جهاز، لذا سيتجاوز عدد الأجهزة مهلة الاستطلاع ويبلغ عن فشل لم يحدث في موقعك.",
|
||||
"retryDelay": "انتظر",
|
||||
"retryDelaySuffix": "MS بين محاولات.",
|
||||
"retryDelaySuffix": "ثوانٍ بين كل إعادة محاولة.",
|
||||
"schedule": "الجدول الزمني",
|
||||
"scheduleLater": "الجدول لاحقا",
|
||||
"scheduleLaterTooltip": "اختر تاريخا ووقتا مستقبليا لتفعيل الفحص.",
|
||||
@@ -20247,6 +20255,8 @@
|
||||
"restartConfig": "إعادة تشغيل الإعدادات — انسخ أمر إعداد هذا الوكيل لإعادة تشغيل التثبيت / إعادة تشغيل docker (مثلا بعد تدوير الرموز)",
|
||||
"noAgents": "لم يسجل أي وكلاء في هذا الموقع حتى الآن.",
|
||||
"agentsUnavailable": "وكلاء هذا الموقع مسجلون في مجموعة أخرى ولا يظهرون هنا.",
|
||||
"loadFailed": "Could not load this location. Check your connection and try again.",
|
||||
"retry": "Retry",
|
||||
"checksTitle": "فحوصات تجري من هذا الموقع",
|
||||
"checkName": "الاسم",
|
||||
"noChecks": "لم يتم تعيين أي فحوصات لهذا الموقع حتى الآن.",
|
||||
@@ -21137,6 +21147,7 @@
|
||||
"system": "النظام"
|
||||
},
|
||||
"serviceStatus": {
|
||||
"error": "Error",
|
||||
"up": "إلى الأعلى",
|
||||
"passed": "تم النجاح",
|
||||
"healthy": "سليم",
|
||||
|
||||
@@ -19159,6 +19159,7 @@
|
||||
"probeError": "Probe-Fehler"
|
||||
},
|
||||
"filters": {
|
||||
"error": "Error",
|
||||
"actionClick": "Klicken",
|
||||
"actionType": "Eingabe",
|
||||
"all": "Alle",
|
||||
@@ -19874,6 +19875,13 @@
|
||||
"attemptN": "Versuch {n}",
|
||||
"detailAction": "Aktion",
|
||||
"detailSelector": "Selektor",
|
||||
"detailStepId": "Schritt-ID",
|
||||
"detailTargetUrl": "Ziel-URL",
|
||||
"detailStatus": "Status",
|
||||
"detailSkipReason": "Grund für Überspringen",
|
||||
"stepStatusSkipped": "Übersprungen",
|
||||
"stepStatusUnknown": "Unbekannt",
|
||||
"skippedCount": "{n} übersprungen",
|
||||
"downloadTrace": "{filename} herunterladen",
|
||||
"errorFullscreenTitle": "Schritt {id}: {action} - Fehler",
|
||||
"failedAtStartLoad": "Öffnen der Start-URL fehlgeschlagen",
|
||||
@@ -20023,7 +20031,7 @@
|
||||
"retriesMaxHint": "Maximum {max}. Wiederholungen laufen im selben Job, daher muss die gesamte Sequenz in das Check-Budget passen.",
|
||||
"retriesMaxBrowserHint": "Maximum {max} für Browser-Checks. Jeder Versuch führt den Durchlauf auf jedem Gerät erneut aus, daher würde eine höhere Anzahl das Probe-Timeout überschreiten und einen Fehler melden, den Ihre Website nie hatte.",
|
||||
"retryDelay": "Zwischen den Wiederholungsversuchen",
|
||||
"retryDelaySuffix": "ms warten.",
|
||||
"retryDelaySuffix": "Sekunden zwischen den Wiederholungsversuchen.",
|
||||
"schedule": "Zeitplan",
|
||||
"scheduleLater": "Später planen",
|
||||
"scheduleLaterTooltip": "Wählen Sie ein Datum und eine Uhrzeit in der Zukunft, um den Scheck zu aktivieren.",
|
||||
@@ -20247,6 +20255,8 @@
|
||||
"restartConfig": "Neustartkonfiguration – kopieren Sie den Setup-Befehl dieses Agenten, um die Installation / den Docker-Neustart erneut auszuführen (z. B. nach einer Token-Rotation).",
|
||||
"noAgents": "Es haben sich noch keine Agenten für diesen Standort registriert.",
|
||||
"agentsUnavailable": "Die Agenten dieses Standorts sind bei einem anderen Cluster registriert und hier nicht sichtbar.",
|
||||
"loadFailed": "Could not load this location. Check your connection and try again.",
|
||||
"retry": "Retry",
|
||||
"checksTitle": "Von diesem Standort ausgeführte Prüfungen",
|
||||
"checkName": "Name",
|
||||
"noChecks": "Diesem Standort sind noch keine Prüfungen zugewiesen.",
|
||||
@@ -21137,6 +21147,7 @@
|
||||
"system": "System"
|
||||
},
|
||||
"serviceStatus": {
|
||||
"error": "Error",
|
||||
"up": "Verfügbar",
|
||||
"passed": "Bestanden",
|
||||
"healthy": "Gesund",
|
||||
|
||||
@@ -19157,6 +19157,7 @@
|
||||
"probeError": "Probe error"
|
||||
},
|
||||
"filters": {
|
||||
"error": "Error",
|
||||
"actionClick": "click",
|
||||
"actionType": "type",
|
||||
"all": "All",
|
||||
@@ -19872,6 +19873,13 @@
|
||||
"attemptN": "Attempt {n}",
|
||||
"detailAction": "Action",
|
||||
"detailSelector": "Selector",
|
||||
"detailStepId": "Step ID",
|
||||
"detailTargetUrl": "Target URL",
|
||||
"detailStatus": "Status",
|
||||
"detailSkipReason": "Skip reason",
|
||||
"stepStatusSkipped": "Skipped",
|
||||
"stepStatusUnknown": "Unknown",
|
||||
"skippedCount": "{n} skipped",
|
||||
"downloadTrace": "Download {filename}",
|
||||
"errorFullscreenTitle": "Step {id}: {action} - Error",
|
||||
"failedAtStartLoad": "Failed opening Starting URL",
|
||||
@@ -20021,7 +20029,7 @@
|
||||
"retriesMaxHint": "Maximum {max}. Retries run inside the same job, so the whole sequence must fit the check budget.",
|
||||
"retriesMaxBrowserHint": "Maximum {max} for browser checks. Each attempt reruns the journey on every device, so more would outrun the probe timeout and report a failure your site never had.",
|
||||
"retryDelay": "Wait for",
|
||||
"retryDelaySuffix": "ms in between retries.",
|
||||
"retryDelaySuffix": "seconds in between retries.",
|
||||
"schedule": "Schedule",
|
||||
"scheduleLater": "Schedule later",
|
||||
"scheduleLaterTooltip": "Choose a future date and time to activate the check.",
|
||||
@@ -20245,6 +20253,8 @@
|
||||
"restartConfig": "Restart config — copy this agent's setup command to re-run install / docker restart (e.g. after a token rotation)",
|
||||
"noAgents": "No agents have registered for this location yet.",
|
||||
"agentsUnavailable": "This location's agents registered with another cluster and are not visible here.",
|
||||
"loadFailed": "Could not load this location. Check your connection and try again.",
|
||||
"retry": "Retry",
|
||||
"checksTitle": "Checks running from this location",
|
||||
"checkName": "Name",
|
||||
"noChecks": "No checks are assigned to this location yet.",
|
||||
@@ -21135,6 +21145,7 @@
|
||||
"system": "system"
|
||||
},
|
||||
"serviceStatus": {
|
||||
"error": "Error",
|
||||
"up": "Up",
|
||||
"passed": "Passed",
|
||||
"healthy": "Healthy",
|
||||
|
||||
@@ -19159,6 +19159,7 @@
|
||||
"probeError": "Error de sonda"
|
||||
},
|
||||
"filters": {
|
||||
"error": "Error",
|
||||
"actionClick": "clic",
|
||||
"actionType": "escribir",
|
||||
"all": "Todos",
|
||||
@@ -19874,6 +19875,13 @@
|
||||
"attemptN": "Intento {n}",
|
||||
"detailAction": "Acción",
|
||||
"detailSelector": "Selector",
|
||||
"detailStepId": "ID del paso",
|
||||
"detailTargetUrl": "URL de destino",
|
||||
"detailStatus": "Estado",
|
||||
"detailSkipReason": "Motivo de omisión",
|
||||
"stepStatusSkipped": "Omitido",
|
||||
"stepStatusUnknown": "Desconocido",
|
||||
"skippedCount": "{n} omitidos",
|
||||
"downloadTrace": "Descargar {filename}",
|
||||
"errorFullscreenTitle": "Paso {id}: {action} - Error",
|
||||
"failedAtStartLoad": "Fallo al abrir la URL inicial",
|
||||
@@ -20023,7 +20031,7 @@
|
||||
"retriesMaxHint": "Máximo {max}. Los reintentos se ejecutan dentro del mismo trabajo, por lo que toda la secuencia debe caber en el presupuesto de la verificación.",
|
||||
"retriesMaxBrowserHint": "Máximo {max} para verificaciones de navegador. Cada intento vuelve a ejecutar el recorrido en cada dispositivo, por lo que más excedería el tiempo de espera de la sonda e informaría de un fallo que su sitio nunca tuvo.",
|
||||
"retryDelay": "Espera a",
|
||||
"retryDelaySuffix": "ms entre reintentos.",
|
||||
"retryDelaySuffix": "segundos entre reintentos.",
|
||||
"schedule": "Programación",
|
||||
"scheduleLater": "Programe más tarde",
|
||||
"scheduleLaterTooltip": "Elija una fecha y hora futuras para activar el cheque.",
|
||||
@@ -20247,6 +20255,8 @@
|
||||
"restartConfig": "Reiniciar configuración — copie el comando de configuración de este agente para volver a ejecutar la instalación / reinicio de Docker (p. ej., después de una rotación de token)",
|
||||
"noAgents": "Aún no se han registrado agentes para esta ubicación.",
|
||||
"agentsUnavailable": "Los agentes de esta ubicación se registraron en otro clúster y no son visibles aquí.",
|
||||
"loadFailed": "Could not load this location. Check your connection and try again.",
|
||||
"retry": "Retry",
|
||||
"checksTitle": "Verificaciones ejecutándose desde esta ubicación",
|
||||
"checkName": "Nombre",
|
||||
"noChecks": "Aún no se asignan verificaciones a esta ubicación.",
|
||||
@@ -21137,6 +21147,7 @@
|
||||
"system": "sistema"
|
||||
},
|
||||
"serviceStatus": {
|
||||
"error": "Error",
|
||||
"up": "Activo",
|
||||
"passed": "Aprobado",
|
||||
"healthy": "Saludable",
|
||||
|
||||
@@ -19159,6 +19159,7 @@
|
||||
"probeError": "Erreur de sonde"
|
||||
},
|
||||
"filters": {
|
||||
"error": "Error",
|
||||
"actionClick": "clic",
|
||||
"actionType": "saisie",
|
||||
"all": "Tous",
|
||||
@@ -19874,6 +19875,13 @@
|
||||
"attemptN": "Attempt {n}",
|
||||
"detailAction": "Action",
|
||||
"detailSelector": "Sélecteur",
|
||||
"detailStepId": "ID de l'étape",
|
||||
"detailTargetUrl": "URL cible",
|
||||
"detailStatus": "Statut",
|
||||
"detailSkipReason": "Raison de l'omission",
|
||||
"stepStatusSkipped": "Ignoré",
|
||||
"stepStatusUnknown": "Inconnu",
|
||||
"skippedCount": "{n} ignorés",
|
||||
"downloadTrace": "Télécharger {filename}",
|
||||
"errorFullscreenTitle": "Étape {id} : {action} - Erreur",
|
||||
"failedAtStartLoad": "Échec de l'ouverture de l'URL de départ",
|
||||
@@ -20023,7 +20031,7 @@
|
||||
"retriesMaxHint": "Maximum {max}. Les nouvelles tentatives s'exécutent dans le même travail, donc toute la séquence doit tenir dans le budget de vérification.",
|
||||
"retriesMaxBrowserHint": "Maximum {max} pour les vérifications de navigateur. Chaque tentative relance le parcours sur chaque appareil, donc un nombre plus élevé dépasserait le délai de la sonde et signalerait une défaillance que votre site n'a jamais connue.",
|
||||
"retryDelay": "Attendre",
|
||||
"retryDelaySuffix": "ms entre deux tentatives.",
|
||||
"retryDelaySuffix": "secondes entre les tentatives.",
|
||||
"schedule": "Programmation",
|
||||
"scheduleLater": "Planifier plus tard",
|
||||
"scheduleLaterTooltip": "Choisissez une date et une heure futures pour activer le chèque.",
|
||||
@@ -20247,6 +20255,8 @@
|
||||
"restartConfig": "Redémarrer la configuration — copiez la commande d'installation de cet agent pour relancer l'installation / le redémarrage Docker (par exemple après une rotation de jeton).",
|
||||
"noAgents": "Aucun agent ne s'est encore enregistré pour cet emplacement.",
|
||||
"agentsUnavailable": "Les agents de cet emplacement sont enregistrés auprès d'un autre cluster et ne sont pas visibles ici.",
|
||||
"loadFailed": "Could not load this location. Check your connection and try again.",
|
||||
"retry": "Retry",
|
||||
"checksTitle": "Vérifications en cours depuis cet emplacement",
|
||||
"checkName": "Nom",
|
||||
"noChecks": "Aucune vérification n'est encore assignée à cet emplacement.",
|
||||
@@ -21137,6 +21147,7 @@
|
||||
"system": "système"
|
||||
},
|
||||
"serviceStatus": {
|
||||
"error": "Error",
|
||||
"up": "Actif",
|
||||
"passed": "Réussi",
|
||||
"healthy": "Sain",
|
||||
|
||||
@@ -19159,6 +19159,7 @@
|
||||
"probeError": "Errore probe"
|
||||
},
|
||||
"filters": {
|
||||
"error": "Error",
|
||||
"actionClick": "clic",
|
||||
"actionType": "digita",
|
||||
"all": "Tutti",
|
||||
@@ -19874,6 +19875,13 @@
|
||||
"attemptN": "Tentativo {n}",
|
||||
"detailAction": "Azione",
|
||||
"detailSelector": "Selettore",
|
||||
"detailStepId": "ID passaggio",
|
||||
"detailTargetUrl": "URL di destinazione",
|
||||
"detailStatus": "Stato",
|
||||
"detailSkipReason": "Motivo del salto",
|
||||
"stepStatusSkipped": "Saltato",
|
||||
"stepStatusUnknown": "Sconosciuto",
|
||||
"skippedCount": "{n} saltati",
|
||||
"downloadTrace": "Scarica {filename}",
|
||||
"errorFullscreenTitle": "Step {id}: {action} - Errore",
|
||||
"failedAtStartLoad": "Apertura dell'URL iniziale non riuscita",
|
||||
@@ -20023,7 +20031,7 @@
|
||||
"retriesMaxHint": "Massimo {max}. I tentativi vengono eseguiti nello stesso job, quindi l'intera sequenza deve rientrare nel budget del controllo.",
|
||||
"retriesMaxBrowserHint": "Massimo {max} per i controlli browser. Ogni tentativo riesegue il percorso su ogni dispositivo, quindi un numero maggiore supererebbe il timeout della sonda e segnalerebbe un guasto che il tuo sito non ha mai avuto.",
|
||||
"retryDelay": "Aspetta",
|
||||
"retryDelaySuffix": "ms tra un tentativo e l'altro.",
|
||||
"retryDelaySuffix": "secondi tra un tentativo e l'altro.",
|
||||
"schedule": "Programmazione",
|
||||
"scheduleLater": "Pianifica più tardi",
|
||||
"scheduleLaterTooltip": "Scegli una data e un'ora future per attivare il controllo.",
|
||||
@@ -20247,6 +20255,8 @@
|
||||
"restartConfig": "Riavvio configurazione — copia il comando di configurazione di questo agente per rieseguire l'installazione / il riavvio di docker (ad es. dopo una rotazione del token)",
|
||||
"noAgents": "Nessun agente si è ancora registrato per questa posizione.",
|
||||
"agentsUnavailable": "Gli agenti di questa posizione sono registrati con un altro cluster e non sono visibili qui.",
|
||||
"loadFailed": "Could not load this location. Check your connection and try again.",
|
||||
"retry": "Retry",
|
||||
"checksTitle": "Controlli in esecuzione da questa posizione",
|
||||
"checkName": "Nome",
|
||||
"noChecks": "Nessun controllo è ancora assegnato a questa posizione.",
|
||||
@@ -21137,6 +21147,7 @@
|
||||
"system": "sistema"
|
||||
},
|
||||
"serviceStatus": {
|
||||
"error": "Error",
|
||||
"up": "Operativo",
|
||||
"passed": "Superato",
|
||||
"healthy": "Sano",
|
||||
|
||||
@@ -19159,6 +19159,7 @@
|
||||
"probeError": "プローブエラー"
|
||||
},
|
||||
"filters": {
|
||||
"error": "Error",
|
||||
"actionClick": "クリック",
|
||||
"actionType": "タイプ",
|
||||
"all": "すべて",
|
||||
@@ -19874,6 +19875,13 @@
|
||||
"attemptN": "試行 {n}",
|
||||
"detailAction": "アクション",
|
||||
"detailSelector": "セレクタ",
|
||||
"detailStepId": "ステップ ID",
|
||||
"detailTargetUrl": "ターゲット URL",
|
||||
"detailStatus": "ステータス",
|
||||
"detailSkipReason": "スキップ理由",
|
||||
"stepStatusSkipped": "スキップ",
|
||||
"stepStatusUnknown": "不明",
|
||||
"skippedCount": "{n} 件スキップ",
|
||||
"downloadTrace": "{filename}をダウンロード",
|
||||
"errorFullscreenTitle": "ステップ {id}: {action} - エラー",
|
||||
"failedAtStartLoad": "開始 URL を開けませんでした",
|
||||
@@ -20023,7 +20031,7 @@
|
||||
"retriesMaxHint": "最大 {max}。再試行は同じジョブ内で実行されるため、シーケンス全体がチェック予算内に収まる必要があります。",
|
||||
"retriesMaxBrowserHint": "ブラウザチェックの最大値は {max} です。各試行はすべてのデバイスでジャーニーを再実行するため、これ以上増やすとプローブのタイムアウトを超え、サイトには実際にはなかった障害が報告されます。",
|
||||
"retryDelay": "お待ちください",
|
||||
"retryDelaySuffix": "再試行の間隔はミリ秒です。",
|
||||
"retryDelaySuffix": "再試行の間隔(秒)。",
|
||||
"schedule": "スケジュール",
|
||||
"scheduleLater": "後でスケジュールする",
|
||||
"scheduleLaterTooltip": "チェックを有効にする将来の日付と時刻を選択してください。",
|
||||
@@ -20247,6 +20255,8 @@
|
||||
"restartConfig": "再起動設定 — このエージェントのセットアップコマンドをコピーして、インストール / Docker 再起動を再実行します (例: トークンローテーション後)",
|
||||
"noAgents": "このロケーションにまだエージェントが登録されていません。",
|
||||
"agentsUnavailable": "このロケーションのエージェントは別のクラスターに登録されているため、ここでは表示されません。",
|
||||
"loadFailed": "Could not load this location. Check your connection and try again.",
|
||||
"retry": "Retry",
|
||||
"checksTitle": "このロケーションから実行されるチェック",
|
||||
"checkName": "名前",
|
||||
"noChecks": "このロケーションにまだチェックが割り当てられていません。",
|
||||
@@ -21137,6 +21147,7 @@
|
||||
"system": "システム"
|
||||
},
|
||||
"serviceStatus": {
|
||||
"error": "Error",
|
||||
"up": "稼働中",
|
||||
"passed": "成功",
|
||||
"healthy": "正常",
|
||||
|
||||
@@ -19159,6 +19159,7 @@
|
||||
"probeError": "프로브 오류"
|
||||
},
|
||||
"filters": {
|
||||
"error": "Error",
|
||||
"actionClick": "클릭",
|
||||
"actionType": "입력",
|
||||
"all": "모두",
|
||||
@@ -19874,6 +19875,13 @@
|
||||
"attemptN": "시도 {n}",
|
||||
"detailAction": "작업",
|
||||
"detailSelector": "선택기",
|
||||
"detailStepId": "단계 ID",
|
||||
"detailTargetUrl": "대상 URL",
|
||||
"detailStatus": "상태",
|
||||
"detailSkipReason": "건너뛴 이유",
|
||||
"stepStatusSkipped": "건너뜀",
|
||||
"stepStatusUnknown": "알 수 없음",
|
||||
"skippedCount": "{n}개 건너뜀",
|
||||
"downloadTrace": "{filename} 다운로드",
|
||||
"errorFullscreenTitle": "단계 {id}: {action} - 오류",
|
||||
"failedAtStartLoad": "시작 URL을 여는 데 실패했습니다",
|
||||
@@ -20023,7 +20031,7 @@
|
||||
"retriesMaxHint": "최대 {max}. 재시도는 동일한 작업 내에서 실행되므로 전체 시퀀스가 검사 예산에 맞아야 합니다.",
|
||||
"retriesMaxBrowserHint": "브라우저 검사의 최대값은 {max}입니다. 각 시도는 모든 기기에서 여정을 다시 실행하므로, 더 많으면 프로브 시간 초과를 초과하여 사이트에 실제로 없던 실패를 보고하게 됩니다.",
|
||||
"retryDelay": "대기 시간",
|
||||
"retryDelaySuffix": "재시도 간격 (ms)",
|
||||
"retryDelaySuffix": "재시도 간격(초).",
|
||||
"schedule": "일정",
|
||||
"scheduleLater": "나중에 예약",
|
||||
"scheduleLaterTooltip": "확인을 활성화할 미래 날짜 및 시간을 선택합니다.",
|
||||
@@ -20247,6 +20255,8 @@
|
||||
"restartConfig": "구성 재시작 — 이 에이전트의 설정 명령을 복사하여 설치 / docker 재시작을 다시 실행합니다 (예: 토큰 회전 후)",
|
||||
"noAgents": "이 위치에 등록된 에이전트가 아직 없습니다.",
|
||||
"agentsUnavailable": "이 위치의 에이전트는 다른 클러스터에 등록되어 있으므로 여기서 볼 수 없습니다.",
|
||||
"loadFailed": "Could not load this location. Check your connection and try again.",
|
||||
"retry": "Retry",
|
||||
"checksTitle": "이 위치에서 실행 중인 검사",
|
||||
"checkName": "이름",
|
||||
"noChecks": "이 위치에 할당된 검사가 아직 없습니다.",
|
||||
@@ -21137,6 +21147,7 @@
|
||||
"system": "시스템"
|
||||
},
|
||||
"serviceStatus": {
|
||||
"error": "Error",
|
||||
"up": "정상",
|
||||
"passed": "통과",
|
||||
"healthy": "정상",
|
||||
|
||||
@@ -19159,6 +19159,7 @@
|
||||
"probeError": "Probe-fout"
|
||||
},
|
||||
"filters": {
|
||||
"error": "Error",
|
||||
"actionClick": "klik",
|
||||
"actionType": "type",
|
||||
"all": "Alle",
|
||||
@@ -19874,6 +19875,13 @@
|
||||
"attemptN": "Attempt {n}",
|
||||
"detailAction": "Actie",
|
||||
"detailSelector": "Selector",
|
||||
"detailStepId": "Stap-ID",
|
||||
"detailTargetUrl": "Doel-URL",
|
||||
"detailStatus": "Status",
|
||||
"detailSkipReason": "Reden voor overslaan",
|
||||
"stepStatusSkipped": "Overgeslagen",
|
||||
"stepStatusUnknown": "Onbekend",
|
||||
"skippedCount": "{n} overgeslagen",
|
||||
"downloadTrace": "{filename} downloaden",
|
||||
"errorFullscreenTitle": "Stap {id}: {action} - Fout",
|
||||
"failedAtStartLoad": "Openen van start-URL mislukt",
|
||||
@@ -20023,7 +20031,7 @@
|
||||
"retriesMaxHint": "Maximum {max}. Retries worden binnen dezelfde taak uitgevoerd, dus de hele reeks moet in het checkbudget passen.",
|
||||
"retriesMaxBrowserHint": "Maximum {max} voor browserchecks. Elke poging doorloopt het traject opnieuw op elk apparaat, dus meer zou de probe-time-out overschrijden en een fout melden die uw site nooit had.",
|
||||
"retryDelay": "Wacht tot",
|
||||
"retryDelaySuffix": "Ik zit tussen de nieuwe pogingen.",
|
||||
"retryDelaySuffix": "seconden tussen herpogingen.",
|
||||
"schedule": "Schema",
|
||||
"scheduleLater": "Later plannen",
|
||||
"scheduleLaterTooltip": "Kies een toekomstige datum en tijd om de cheque te activeren.",
|
||||
@@ -20247,6 +20255,8 @@
|
||||
"restartConfig": "Configuratie opnieuw starten — kopieer het installatiecommando van deze agent om de installatie opnieuw uit te voeren / docker opnieuw te starten (bijv. na een tokenrotatie)",
|
||||
"noAgents": "Er hebben zich nog geen agenten geregistreerd voor deze locatie.",
|
||||
"agentsUnavailable": "De agents van deze locatie zijn geregistreerd bij een ander cluster en zijn hier niet zichtbaar.",
|
||||
"loadFailed": "Could not load this location. Check your connection and try again.",
|
||||
"retry": "Retry",
|
||||
"checksTitle": "Controles die vanaf deze locatie worden uitgevoerd",
|
||||
"checkName": "Naam",
|
||||
"noChecks": "Er zijn nog geen controles toegewezen aan deze locatie.",
|
||||
@@ -21137,6 +21147,7 @@
|
||||
"system": "systeem"
|
||||
},
|
||||
"serviceStatus": {
|
||||
"error": "Error",
|
||||
"up": "Beschikbaar",
|
||||
"passed": "Geslaagd",
|
||||
"healthy": "Gezond",
|
||||
|
||||
@@ -19159,6 +19159,7 @@
|
||||
"probeError": "Błąd sondy"
|
||||
},
|
||||
"filters": {
|
||||
"error": "Error",
|
||||
"actionClick": "kliknij",
|
||||
"actionType": "wpisz",
|
||||
"all": "Wszystkie",
|
||||
@@ -19874,6 +19875,13 @@
|
||||
"attemptN": "Próba {n}",
|
||||
"detailAction": "Akcja",
|
||||
"detailSelector": "Selektor",
|
||||
"detailStepId": "ID kroku",
|
||||
"detailTargetUrl": "Docelowy URL",
|
||||
"detailStatus": "Status",
|
||||
"detailSkipReason": "Powód pominięcia",
|
||||
"stepStatusSkipped": "Pominięty",
|
||||
"stepStatusUnknown": "Nieznany",
|
||||
"skippedCount": "Pominięte: {n}",
|
||||
"downloadTrace": "Pobierz {filename}",
|
||||
"errorFullscreenTitle": "Krok {id}: {action} - Błąd",
|
||||
"failedAtStartLoad": "Nie udało się otworzyć początkowego URL",
|
||||
@@ -20023,7 +20031,7 @@
|
||||
"retriesMaxHint": "Maksimum {max}. Ponowienia są uruchamiane w ramach tego samego zadania, więc cała sekwencja musi zmieścić się w budżecie kontroli.",
|
||||
"retriesMaxBrowserHint": "Maksimum {max} dla kontroli przeglądarki. Każda próba ponownie uruchamia podróż na każdym urządzeniu, więc więcej przekroczyłoby limit czasu sondy i zgłosiło awarię, której Twoja witryna nigdy nie miała.",
|
||||
"retryDelay": "Poczekaj",
|
||||
"retryDelaySuffix": "ms między ponownymi próbami.",
|
||||
"retryDelaySuffix": "sekund między ponownymi próbami.",
|
||||
"schedule": "Harmonogram",
|
||||
"scheduleLater": "Zaplanuj później",
|
||||
"scheduleLaterTooltip": "Wybierz przyszłą datę i godzinę, aby aktywować sprawdzenie.",
|
||||
@@ -20247,6 +20255,8 @@
|
||||
"restartConfig": "Konfiguracja restartu — skopiuj polecenie konfiguracji tego agenta, aby ponownie uruchomić instalację / restart dockera (np. po rotacji tokenu)",
|
||||
"noAgents": "Żaden agent nie zarejestrował się jeszcze dla tej lokalizacji.",
|
||||
"agentsUnavailable": "Agenci tej lokalizacji zarejestrowali się w innym klastrze i nie są tutaj widoczni.",
|
||||
"loadFailed": "Could not load this location. Check your connection and try again.",
|
||||
"retry": "Retry",
|
||||
"checksTitle": "Kontrole uruchamiane z tej lokalizacji",
|
||||
"checkName": "Nazwa",
|
||||
"noChecks": "Żadne kontrole nie zostały jeszcze przypisane do tej lokalizacji.",
|
||||
@@ -21137,6 +21147,7 @@
|
||||
"system": "system"
|
||||
},
|
||||
"serviceStatus": {
|
||||
"error": "Error",
|
||||
"up": "Dostępny",
|
||||
"passed": "Zaliczony",
|
||||
"healthy": "Zdrowy",
|
||||
|
||||
@@ -19159,6 +19159,7 @@
|
||||
"probeError": "Erro de sonda"
|
||||
},
|
||||
"filters": {
|
||||
"error": "Error",
|
||||
"actionClick": "clique",
|
||||
"actionType": "introduzir",
|
||||
"all": "Todos",
|
||||
@@ -19874,6 +19875,13 @@
|
||||
"attemptN": "Tentativa {n}",
|
||||
"detailAction": "Ação",
|
||||
"detailSelector": "Seletor",
|
||||
"detailStepId": "ID do passo",
|
||||
"detailTargetUrl": "URL de destino",
|
||||
"detailStatus": "Estado",
|
||||
"detailSkipReason": "Motivo da omissão",
|
||||
"stepStatusSkipped": "Ignorado",
|
||||
"stepStatusUnknown": "Desconhecido",
|
||||
"skippedCount": "{n} ignorados",
|
||||
"downloadTrace": "Transferir {filename}",
|
||||
"errorFullscreenTitle": "Etapa {id}: {action} - Erro",
|
||||
"failedAtStartLoad": "Falha ao abrir o URL inicial",
|
||||
@@ -20023,7 +20031,7 @@
|
||||
"retriesMaxHint": "Máximo {max}. As tentativas são executadas no mesmo job, portanto toda a sequência deve caber no orçamento da verificação.",
|
||||
"retriesMaxBrowserHint": "Máximo {max} para verificações de navegador. Cada tentativa reexecuta a jornada em todos os dispositivos, por isso mais tentativas excederiam o tempo limite da sonda e reportariam uma falha que o seu site nunca teve.",
|
||||
"retryDelay": "Aguarde",
|
||||
"retryDelaySuffix": "ms entre as novas tentativas.",
|
||||
"retryDelaySuffix": "segundos entre tentativas.",
|
||||
"schedule": "Agendamento",
|
||||
"scheduleLater": "Agende mais tarde",
|
||||
"scheduleLaterTooltip": "Escolha uma data e hora futuras para ativar a verificação.",
|
||||
@@ -20247,6 +20255,8 @@
|
||||
"restartConfig": "Configuração de reinício — copie o comando de configuração deste agente para reexecutar a instalação / reinício do docker (por exemplo, após uma rotação de token)",
|
||||
"noAgents": "Nenhum agente se registou para este local ainda.",
|
||||
"agentsUnavailable": "Os agentes deste local registaram-se noutro cluster e não estão visíveis aqui.",
|
||||
"loadFailed": "Could not load this location. Check your connection and try again.",
|
||||
"retry": "Retry",
|
||||
"checksTitle": "Verificações em execução a partir deste local",
|
||||
"checkName": "Nome",
|
||||
"noChecks": "Nenhuma verificação está atribuída a este local ainda.",
|
||||
@@ -21137,6 +21147,7 @@
|
||||
"system": "sistema"
|
||||
},
|
||||
"serviceStatus": {
|
||||
"error": "Error",
|
||||
"up": "Ativo",
|
||||
"passed": "Aprovado",
|
||||
"healthy": "Saudável",
|
||||
|
||||
@@ -19159,6 +19159,7 @@
|
||||
"probeError": "Ошибка пробы"
|
||||
},
|
||||
"filters": {
|
||||
"error": "Error",
|
||||
"actionClick": "нажатие",
|
||||
"actionType": "ввод",
|
||||
"all": "Все",
|
||||
@@ -19874,6 +19875,13 @@
|
||||
"attemptN": "Попытка {n}",
|
||||
"detailAction": "Действие",
|
||||
"detailSelector": "Селектор",
|
||||
"detailStepId": "ID шага",
|
||||
"detailTargetUrl": "Целевой URL",
|
||||
"detailStatus": "Статус",
|
||||
"detailSkipReason": "Причина пропуска",
|
||||
"stepStatusSkipped": "Пропущен",
|
||||
"stepStatusUnknown": "Неизвестно",
|
||||
"skippedCount": "Пропущено: {n}",
|
||||
"downloadTrace": "Скачать {filename}",
|
||||
"errorFullscreenTitle": "Шаг {id}: {action} - Ошибка",
|
||||
"failedAtStartLoad": "Не удалось открыть начальный URL",
|
||||
@@ -20023,7 +20031,7 @@
|
||||
"retriesMaxHint": "Максимум {max}. Повторные попытки выполняются в рамках одного задания, поэтому вся последовательность должна укладываться в бюджет проверки.",
|
||||
"retriesMaxBrowserHint": "Максимум {max} для браузерных проверок. Каждая попытка повторно запускает сценарий на каждом устройстве, поэтому большее количество превысило бы тайм-аут зонда и сообщило бы о сбое, которого на вашем сайте не было.",
|
||||
"retryDelay": "Ждать",
|
||||
"retryDelaySuffix": "мс между повторными попытками.",
|
||||
"retryDelaySuffix": "секунд между повторами.",
|
||||
"schedule": "Расписание",
|
||||
"scheduleLater": "Запланировать позже",
|
||||
"scheduleLaterTooltip": "Выберите будущую дату и время для активации проверки.",
|
||||
@@ -20247,6 +20255,8 @@
|
||||
"restartConfig": "Перезапуск конфигурации — скопируйте команду настройки этого агента, чтобы повторно запустить установку / перезапуск docker (например, после ротации токена).",
|
||||
"noAgents": "Для этого местоположения еще не зарегистрировано ни одного агента.",
|
||||
"agentsUnavailable": "Агенты этой локации зарегистрированы в другом кластере и здесь не видны.",
|
||||
"loadFailed": "Could not load this location. Check your connection and try again.",
|
||||
"retry": "Retry",
|
||||
"checksTitle": "Проверки, выполняемые из этого местоположения",
|
||||
"checkName": "Имя",
|
||||
"noChecks": "Для этого местоположения еще не назначено ни одной проверки.",
|
||||
@@ -21137,6 +21147,7 @@
|
||||
"system": "система"
|
||||
},
|
||||
"serviceStatus": {
|
||||
"error": "Error",
|
||||
"up": "Работает",
|
||||
"passed": "Пройдено",
|
||||
"healthy": "Здоров",
|
||||
|
||||
@@ -19159,6 +19159,7 @@
|
||||
"probeError": "Prob hatası"
|
||||
},
|
||||
"filters": {
|
||||
"error": "Error",
|
||||
"actionClick": "tıklama",
|
||||
"actionType": "yazma",
|
||||
"all": "Tümü",
|
||||
@@ -19874,6 +19875,13 @@
|
||||
"attemptN": "Deneme {n}",
|
||||
"detailAction": "Eylem",
|
||||
"detailSelector": "Seçici",
|
||||
"detailStepId": "Adım kimliği",
|
||||
"detailTargetUrl": "Hedef URL",
|
||||
"detailStatus": "Durum",
|
||||
"detailSkipReason": "Atlanma nedeni",
|
||||
"stepStatusSkipped": "Atlandı",
|
||||
"stepStatusUnknown": "Bilinmiyor",
|
||||
"skippedCount": "{n} atlandı",
|
||||
"downloadTrace": "{filename} indir",
|
||||
"errorFullscreenTitle": "Adım {id}: {action} - Hata",
|
||||
"failedAtStartLoad": "Başlangıç URL'si açılamadı",
|
||||
@@ -20023,7 +20031,7 @@
|
||||
"retriesMaxHint": "Maksimum {max}. Yeniden denemeler aynı iş içinde çalışır, bu nedenle tüm sıra kontrol bütçesine sığmalıdır.",
|
||||
"retriesMaxBrowserHint": "Tarayıcı kontrolleri için maksimum {max}. Her deneme, her cihazda yolculuğu yeniden çalıştırır; bu nedenle daha fazlası denetim zaman aşımını aşar ve sitenizin hiç yaşamadığı bir hatayı raporlar.",
|
||||
"retryDelay": "Bekle",
|
||||
"retryDelaySuffix": "Tekrar denemeler arasında ms.",
|
||||
"retryDelaySuffix": "yeniden denemeler arasındaki saniye.",
|
||||
"schedule": "Zamanlama",
|
||||
"scheduleLater": "Daha sonra planlayın",
|
||||
"scheduleLaterTooltip": "Çeki etkinleştirmek için gelecekteki bir tarih ve saat seçin.",
|
||||
@@ -20247,6 +20255,8 @@
|
||||
"restartConfig": "Yeniden başlatma yapılandırması — bu aracının kurulum komutunu kopyalayarak kurulumu / docker yeniden başlatmayı tekrar çalıştırın (ör. bir token yenilemesinden sonra)",
|
||||
"noAgents": "Bu konum için henüz hiçbir agent kaydolmadı.",
|
||||
"agentsUnavailable": "Bu konumdaki ajanlar başka bir kümeye kayıtlı ve burada görünmüyorlar.",
|
||||
"loadFailed": "Could not load this location. Check your connection and try again.",
|
||||
"retry": "Retry",
|
||||
"checksTitle": "Bu konumdan çalışan kontroller",
|
||||
"checkName": "Ad",
|
||||
"noChecks": "Bu konuma henüz hiçbir kontrol atanmadı.",
|
||||
@@ -21137,6 +21147,7 @@
|
||||
"system": "sistem"
|
||||
},
|
||||
"serviceStatus": {
|
||||
"error": "Error",
|
||||
"up": "Aktif",
|
||||
"passed": "Geçti",
|
||||
"healthy": "Sağlıklı",
|
||||
|
||||
@@ -19159,6 +19159,7 @@
|
||||
"probeError": "Lỗi đầu dò"
|
||||
},
|
||||
"filters": {
|
||||
"error": "Error",
|
||||
"actionClick": "nhấp",
|
||||
"actionType": "gõ",
|
||||
"all": "Tất cả",
|
||||
@@ -19874,6 +19875,13 @@
|
||||
"attemptN": "Lần thử {n}",
|
||||
"detailAction": "Hành động",
|
||||
"detailSelector": "Bộ chọn",
|
||||
"detailStepId": "ID bước",
|
||||
"detailTargetUrl": "URL đích",
|
||||
"detailStatus": "Trạng thái",
|
||||
"detailSkipReason": "Lý do bỏ qua",
|
||||
"stepStatusSkipped": "Đã bỏ qua",
|
||||
"stepStatusUnknown": "Không xác định",
|
||||
"skippedCount": "Đã bỏ qua {n}",
|
||||
"downloadTrace": "Tải xuống {filename}",
|
||||
"errorFullscreenTitle": "Bước {id}: {action} - Lỗi",
|
||||
"failedAtStartLoad": "Không mở được URL bắt đầu",
|
||||
@@ -20023,7 +20031,7 @@
|
||||
"retriesMaxHint": "Tối đa {max}. Các lần thử lại chạy trong cùng một công việc, vì vậy toàn bộ chuỗi phải nằm trong ngân sách kiểm tra.",
|
||||
"retriesMaxBrowserHint": "Tối đa {max} cho kiểm tra trình duyệt. Mỗi lần thử sẽ chạy lại hành trình trên mọi thiết bị, nên nếu tăng thêm sẽ vượt quá thời gian chờ của probe và báo lỗi mà trang web của bạn không hề gặp phải.",
|
||||
"retryDelay": "Chờ",
|
||||
"retryDelaySuffix": "ms giữa các lần thử lại.",
|
||||
"retryDelaySuffix": "giây giữa các lần thử lại.",
|
||||
"schedule": "Lịch trình",
|
||||
"scheduleLater": "Lên lịch sau",
|
||||
"scheduleLaterTooltip": "Chọn ngày và giờ trong tương lai để kích hoạt kiểm tra.",
|
||||
@@ -20247,6 +20255,8 @@
|
||||
"restartConfig": "Cấu hình khởi động lại — sao chép lệnh thiết lập của agent này để chạy lại cài đặt / khởi động lại docker (ví dụ sau khi xoay vòng token)",
|
||||
"noAgents": "Chưa có agent nào đăng ký cho vị trí này.",
|
||||
"agentsUnavailable": "Các agent của vị trí này đã đăng ký với một cụm khác và không hiển thị ở đây.",
|
||||
"loadFailed": "Could not load this location. Check your connection and try again.",
|
||||
"retry": "Retry",
|
||||
"checksTitle": "Các kiểm tra đang chạy từ vị trí này",
|
||||
"checkName": "Tên",
|
||||
"noChecks": "Chưa có kiểm tra nào được gán cho vị trí này.",
|
||||
@@ -21137,6 +21147,7 @@
|
||||
"system": "hệ thống"
|
||||
},
|
||||
"serviceStatus": {
|
||||
"error": "Error",
|
||||
"up": "Hoạt động",
|
||||
"passed": "Đạt",
|
||||
"healthy": "Khỏe mạnh",
|
||||
|
||||
@@ -19159,6 +19159,7 @@
|
||||
"probeError": "探针错误"
|
||||
},
|
||||
"filters": {
|
||||
"error": "Error",
|
||||
"actionClick": "点击",
|
||||
"actionType": "输入",
|
||||
"all": "全部",
|
||||
@@ -19874,6 +19875,13 @@
|
||||
"attemptN": "Attempt {n}",
|
||||
"detailAction": "操作",
|
||||
"detailSelector": "选择器",
|
||||
"detailStepId": "步骤 ID",
|
||||
"detailTargetUrl": "目标 URL",
|
||||
"detailStatus": "状态",
|
||||
"detailSkipReason": "跳过原因",
|
||||
"stepStatusSkipped": "已跳过",
|
||||
"stepStatusUnknown": "未知",
|
||||
"skippedCount": "已跳过 {n} 个",
|
||||
"downloadTrace": "下载 {filename}",
|
||||
"errorFullscreenTitle": "步骤 {id}: {action} - 错误",
|
||||
"failedAtStartLoad": "打开起始 URL 失败",
|
||||
@@ -20023,7 +20031,7 @@
|
||||
"retriesMaxHint": "最大值为 {max}。重试在同一作业内运行,因此整个序列必须符合检查预算。",
|
||||
"retriesMaxBrowserHint": "浏览器检查的最大值为 {max}。每次尝试都会在每个设备上重新运行测试流程,因此更多尝试会超出探测超时,并报告您的网站从未发生过的故障。",
|
||||
"retryDelay": "等待",
|
||||
"retryDelaySuffix": "ms 在两次重试之间。",
|
||||
"retryDelaySuffix": "重试之间的秒数。",
|
||||
"schedule": "计划",
|
||||
"scheduleLater": "稍后安排",
|
||||
"scheduleLaterTooltip": "选择未来的日期和时间来激活支票。",
|
||||
@@ -20247,6 +20255,8 @@
|
||||
"restartConfig": "重启配置 — 复制此代理的设置命令以重新运行安装 / docker 重启(例如在令牌轮换后)",
|
||||
"noAgents": "尚无代理为此位置注册。",
|
||||
"agentsUnavailable": "此位置的代理已注册到另一个集群,因此在此处不可见。",
|
||||
"loadFailed": "Could not load this location. Check your connection and try again.",
|
||||
"retry": "Retry",
|
||||
"checksTitle": "从此位置运行的检查",
|
||||
"checkName": "名称",
|
||||
"noChecks": "尚未为此位置分配任何检查。",
|
||||
@@ -21137,6 +21147,7 @@
|
||||
"system": "系统"
|
||||
},
|
||||
"serviceStatus": {
|
||||
"error": "Error",
|
||||
"up": "正常",
|
||||
"passed": "通过",
|
||||
"healthy": "健康",
|
||||
|
||||
@@ -19154,6 +19154,7 @@
|
||||
"probeError": "探測錯誤"
|
||||
},
|
||||
"filters": {
|
||||
"error": "Error",
|
||||
"actionClick": "點擊",
|
||||
"actionType": "輸入",
|
||||
"all": "全部",
|
||||
@@ -19869,6 +19870,13 @@
|
||||
"attemptN": "Attempt {n}",
|
||||
"detailAction": "動作",
|
||||
"detailSelector": "選取器",
|
||||
"detailStepId": "步驟 ID",
|
||||
"detailTargetUrl": "目標 URL",
|
||||
"detailStatus": "狀態",
|
||||
"detailSkipReason": "略過原因",
|
||||
"stepStatusSkipped": "已略過",
|
||||
"stepStatusUnknown": "未知",
|
||||
"skippedCount": "已略過 {n} 個",
|
||||
"downloadTrace": "下載 {filename}",
|
||||
"errorFullscreenTitle": "步驟 {id}: {action} - 錯誤",
|
||||
"failedAtStartLoad": "開啟起始 URL 失敗",
|
||||
@@ -20018,7 +20026,7 @@
|
||||
"retriesMaxHint": "上限 {max}。重試在同一作業內執行,因此整個序列必須符合檢查預算。",
|
||||
"retriesMaxBrowserHint": "瀏覽器檢查的上限為 {max}。每次嘗試都會在每個裝置上重新執行旅程,因此更多嘗試會超出探查逾時,並回報您的網站從未發生的失敗。",
|
||||
"retryDelay": "等待",
|
||||
"retryDelaySuffix": "毫秒(重試間隔)。",
|
||||
"retryDelaySuffix": "重試之間間隔的秒數。",
|
||||
"schedule": "排程",
|
||||
"scheduleLater": "稍後排程",
|
||||
"scheduleLaterTooltip": "選擇未來的日期和時間以啟用檢查。",
|
||||
@@ -20242,6 +20250,8 @@
|
||||
"restartConfig": "重新啟動設定 — 複製此代理程式的設定命令,以重新執行安裝 / docker restart(例如在 Token 輪替之後)",
|
||||
"noAgents": "此位置尚未有代理程式註冊。",
|
||||
"agentsUnavailable": "此位置的代理程式已向另一個叢集註冊,因此在此處看不到它們。",
|
||||
"loadFailed": "Could not load this location. Check your connection and try again.",
|
||||
"retry": "Retry",
|
||||
"checksTitle": "從此位置執行的檢查",
|
||||
"checkName": "名稱",
|
||||
"noChecks": "此位置尚未分配任何檢查。",
|
||||
@@ -21132,6 +21142,7 @@
|
||||
"system": "系統"
|
||||
},
|
||||
"serviceStatus": {
|
||||
"error": "Error",
|
||||
"up": "運行中",
|
||||
"passed": "已通過",
|
||||
"healthy": "健康",
|
||||
|
||||
@@ -652,6 +652,19 @@ export interface BrowserCheck {
|
||||
tz_offset?: number;
|
||||
start?: number; // microseconds, top-level — matches reports pattern
|
||||
browserDevices?: { browser: string; device: string }[];
|
||||
/**
|
||||
* Supported browser config fields the form has no control for. Loaded from the
|
||||
* stored config and written back on edit so an unrelated edit cannot reset
|
||||
* them to defaults. Explicit allowlist, never a spread of the stored config
|
||||
* (which may hold secrets/headers). Absent on create: defaults apply there.
|
||||
*/
|
||||
browserConfigExtras?: {
|
||||
journey_budget_ms?: number;
|
||||
test_id_attr?: string;
|
||||
runtime?: string;
|
||||
timeout_ms?: number;
|
||||
video?: string;
|
||||
};
|
||||
// Alert / retry settings (top-level in API payload)
|
||||
retries?: number;
|
||||
waitBeforeRetrySecs?: number;
|
||||
|
||||
@@ -115,3 +115,68 @@ describe("environments round-trip", () => {
|
||||
expect(mapped.environments).toEqual(["env-1"]);
|
||||
});
|
||||
});
|
||||
|
||||
describe("browser config extras (UI-04)", () => {
|
||||
const stored = {
|
||||
name: "api made",
|
||||
target: "https://app.test",
|
||||
enabled: true,
|
||||
tags: [],
|
||||
locations: ["us-east"],
|
||||
frequency: { type: "minutes", interval: 5, cron: "" },
|
||||
config: {
|
||||
steps: [],
|
||||
timeout_ms: 90000,
|
||||
journey_budget_ms: 120000,
|
||||
test_id_attr: "data-qa",
|
||||
runtime: "playwright-1.50",
|
||||
capture: { screenshot: "always", trace: "on-fail", video: "on" },
|
||||
headers: [{ name: "Authorization", value: "Bearer secret" }],
|
||||
secret_blob: "do-not-spread",
|
||||
},
|
||||
};
|
||||
|
||||
it("uses defaults on create", () => {
|
||||
const config = buildCreateBrowserTestPayload(check()).config as Record<string, any>;
|
||||
expect(config.timeout_ms).toBe(30000);
|
||||
expect(config.capture.video).toBe("off");
|
||||
expect("journey_budget_ms" in config).toBe(false);
|
||||
expect("test_id_attr" in config).toBe(false);
|
||||
expect("runtime" in config).toBe(false);
|
||||
});
|
||||
|
||||
it("preserves API-set fields across an unrelated name edit", () => {
|
||||
const loaded = mapResponseToBrowserCheck(stored);
|
||||
const config = buildCreateBrowserTestPayload({ ...loaded, name: "renamed" }).config as Record<
|
||||
string,
|
||||
any
|
||||
>;
|
||||
expect(config.timeout_ms).toBe(90000);
|
||||
expect(config.journey_budget_ms).toBe(120000);
|
||||
expect(config.test_id_attr).toBe("data-qa");
|
||||
expect(config.runtime).toBe("playwright-1.50");
|
||||
expect(config.capture.video).toBe("on");
|
||||
});
|
||||
|
||||
it("does not leak unsupported config keys into the extras", () => {
|
||||
const loaded = mapResponseToBrowserCheck(stored);
|
||||
expect(Object.keys(loaded.browserConfigExtras ?? {}).sort()).toEqual([
|
||||
"journey_budget_ms",
|
||||
"runtime",
|
||||
"test_id_attr",
|
||||
"timeout_ms",
|
||||
"video",
|
||||
]);
|
||||
expect(JSON.stringify(loaded.browserConfigExtras)).not.toContain("do-not-spread");
|
||||
});
|
||||
|
||||
it("still applies intentional capture edits", () => {
|
||||
const loaded = mapResponseToBrowserCheck(stored);
|
||||
const config = buildCreateBrowserTestPayload({
|
||||
...loaded,
|
||||
capture: { screenshot: "off", trace: "off" },
|
||||
}).config as Record<string, any>;
|
||||
expect(config.capture.screenshot).toBe("off");
|
||||
expect(config.capture.video).toBe("on");
|
||||
});
|
||||
});
|
||||
|
||||
@@ -102,6 +102,7 @@ export function buildCreateBrowserTestPayload(check: BrowserCheck): Record<strin
|
||||
url,
|
||||
folder,
|
||||
browserDevices,
|
||||
browserConfigExtras: extras,
|
||||
retries,
|
||||
waitBeforeRetrySecs,
|
||||
cooldownMins,
|
||||
@@ -146,11 +147,17 @@ export function buildCreateBrowserTestPayload(check: BrowserCheck): Record<strin
|
||||
// throws here — the save gate is what stops it getting this far.
|
||||
steps: buildV2Steps(journey),
|
||||
browser_devices: browserDevices ?? [{ browser: "chromium", device: "desktop" }],
|
||||
timeout_ms: 30000,
|
||||
// Defaults apply on create only; on edit the stored values win.
|
||||
timeout_ms: extras?.timeout_ms ?? 30000,
|
||||
...(extras?.journey_budget_ms !== undefined && {
|
||||
journey_budget_ms: extras.journey_budget_ms,
|
||||
}),
|
||||
...(extras?.test_id_attr !== undefined && { test_id_attr: extras.test_id_attr }),
|
||||
...(extras?.runtime !== undefined && { runtime: extras.runtime }),
|
||||
capture: {
|
||||
screenshot: capture?.screenshot ?? "on-fail",
|
||||
trace: capture?.trace ?? "on-fail",
|
||||
video: "off",
|
||||
video: extras?.video ?? "off",
|
||||
},
|
||||
...(secrets?.length && { secrets: secrets.map(({ id: _id, ...s }) => s) }),
|
||||
...(headers?.length && { headers: headers.map(({ id: _id, ...h }) => h) }),
|
||||
@@ -348,6 +355,18 @@ function mapFrequencyToSchedule(freq: any, start?: number): BrowserCheck["schedu
|
||||
};
|
||||
}
|
||||
|
||||
function pickBrowserConfigExtras(config: any): BrowserCheck["browserConfigExtras"] {
|
||||
if (!config) return undefined;
|
||||
const out: NonNullable<BrowserCheck["browserConfigExtras"]> = {};
|
||||
if (typeof config.journey_budget_ms === "number")
|
||||
out.journey_budget_ms = config.journey_budget_ms;
|
||||
if (typeof config.test_id_attr === "string") out.test_id_attr = config.test_id_attr;
|
||||
if (typeof config.runtime === "string") out.runtime = config.runtime;
|
||||
if (typeof config.timeout_ms === "number") out.timeout_ms = config.timeout_ms;
|
||||
if (typeof config.capture?.video === "string") out.video = config.capture.video;
|
||||
return Object.keys(out).length ? out : undefined;
|
||||
}
|
||||
|
||||
export function mapResponseToBrowserCheck(data: Record<string, unknown>): BrowserCheck {
|
||||
const {
|
||||
config,
|
||||
@@ -397,6 +416,9 @@ export function mapResponseToBrowserCheck(data: Record<string, unknown>): Browse
|
||||
|
||||
browserDevices: config?.browser_devices,
|
||||
|
||||
// Allowlisted pick (not a spread of `config`, which can hold secrets).
|
||||
browserConfigExtras: pickBrowserConfigExtras(config),
|
||||
|
||||
// No `preserveWire`: a stored version-2 step is poorer than what
|
||||
// buildWireFromStep rebuilds from the UI fields, so keeping it would shadow
|
||||
// the correct reconstruction — which is how a reloaded `select` came to
|
||||
|
||||
@@ -640,6 +640,25 @@ function formatTimeAgo(microseconds: number): string {
|
||||
return t("synthetics.daysAgo", { count: Math.floor(h / 24) });
|
||||
}
|
||||
|
||||
/**
|
||||
* Backend check status -> UI status key. The API sends strings, but the
|
||||
* numeric enum (0 unknown, 1 passed, 2 warning, 3 failed, 4 error) may also
|
||||
* reach us; anything unmapped becomes "unknown" so it never renders blank.
|
||||
*/
|
||||
const MONITOR_STATUS_BY_NUMBER: Record<number, string> = {
|
||||
0: "unknown",
|
||||
1: "passed",
|
||||
2: "warning",
|
||||
3: "failed",
|
||||
4: "error",
|
||||
};
|
||||
const KNOWN_MONITOR_STATUSES = new Set(["passed", "warning", "failed", "error", "unknown"]);
|
||||
function normalizeMonitorStatus(status: unknown): string {
|
||||
if (typeof status === "number") return MONITOR_STATUS_BY_NUMBER[status] ?? "unknown";
|
||||
const s = String(status ?? "").toLowerCase();
|
||||
return KNOWN_MONITOR_STATUSES.has(s) ? s : "unknown";
|
||||
}
|
||||
|
||||
function mapMonitor(m: ApiMonitor) {
|
||||
return {
|
||||
id: m.id,
|
||||
@@ -649,7 +668,7 @@ function mapMonitor(m: ApiMonitor) {
|
||||
url: m.target,
|
||||
type: m.type.toUpperCase(),
|
||||
interval: formatFrequency(m.frequency),
|
||||
status: m.status,
|
||||
status: normalizeMonitorStatus(m.status),
|
||||
responseTime: m.last_response_ms !== null ? `${m.last_response_ms}ms` : null,
|
||||
locations: m.locations,
|
||||
lastCheck: m.last_check_at !== null ? formatTimeAgo(m.last_check_at) : "—",
|
||||
@@ -1354,6 +1373,15 @@ const summaryStats = computed<StatItem[]>(() => {
|
||||
max: share,
|
||||
dataTest: "synthetics-summary-failed",
|
||||
},
|
||||
{
|
||||
key: "error",
|
||||
label: t("synthetics.filters.error"),
|
||||
value: v(count("error")),
|
||||
icon: "error-outline",
|
||||
tone: "error",
|
||||
max: share,
|
||||
dataTest: "synthetics-summary-error",
|
||||
},
|
||||
{
|
||||
key: "warning",
|
||||
label: t("synthetics.filters.warning"),
|
||||
|
||||
@@ -349,4 +349,44 @@ describe("PrivateLocationDetail", () => {
|
||||
);
|
||||
});
|
||||
});
|
||||
|
||||
describe("load failures (UI-06)", () => {
|
||||
it("shows a retry state for a 500 and recovers on retry", async () => {
|
||||
mockServiceGetLocation.mockRejectedValueOnce({ response: { status: 500 } });
|
||||
wrapper = mountPage();
|
||||
await flushPromises();
|
||||
expect(wrapper.find('[data-test="synthetics-private-location-detail-error"]').exists()).toBe(
|
||||
true,
|
||||
);
|
||||
await wrapper
|
||||
.find('[data-test="synthetics-private-location-detail-retry-btn"]')
|
||||
.trigger("click");
|
||||
await flushPromises();
|
||||
expect(mockServiceGetLocation).toHaveBeenCalledTimes(2);
|
||||
expect(wrapper.find('[data-test="synthetics-private-location-detail-error"]').exists()).toBe(
|
||||
false,
|
||||
);
|
||||
});
|
||||
|
||||
it("shows the retry state for a network error (no response)", async () => {
|
||||
mockServiceGetLocation.mockRejectedValueOnce(new Error("Network Error"));
|
||||
wrapper = mountPage();
|
||||
await flushPromises();
|
||||
expect(wrapper.find('[data-test="synthetics-private-location-detail-error"]').exists()).toBe(
|
||||
true,
|
||||
);
|
||||
});
|
||||
|
||||
it("keeps the not-found state for a 404", async () => {
|
||||
mockServiceGetLocation.mockRejectedValueOnce({ response: { status: 404 } });
|
||||
wrapper = mountPage();
|
||||
await flushPromises();
|
||||
expect(wrapper.find('[data-test="synthetics-private-location-detail-error"]').exists()).toBe(
|
||||
false,
|
||||
);
|
||||
expect(wrapper.find('[data-test="synthetics-private-location-detail-empty"]').exists()).toBe(
|
||||
true,
|
||||
);
|
||||
});
|
||||
});
|
||||
});
|
||||
|
||||
@@ -204,6 +204,25 @@ along with this program. If not, see <http://www.gnu.org/licenses/>.
|
||||
</div>
|
||||
</div>
|
||||
|
||||
<div
|
||||
v-else-if="!loading && loadError"
|
||||
class="flex flex-col items-center gap-3 p-6"
|
||||
data-test="synthetics-private-location-detail-error"
|
||||
>
|
||||
<p class="text-status-error-text text-sm">
|
||||
{{ t("synthetics.privateLocations.detail.loadFailed") }}
|
||||
</p>
|
||||
<OButton
|
||||
variant="outline"
|
||||
size="sm"
|
||||
icon-left="refresh"
|
||||
data-test="synthetics-private-location-detail-retry-btn"
|
||||
@click="load"
|
||||
>
|
||||
{{ t("synthetics.privateLocations.detail.retry") }}
|
||||
</OButton>
|
||||
</div>
|
||||
|
||||
<div v-else-if="!loading" class="p-6">
|
||||
<OEmptyState
|
||||
size="block"
|
||||
@@ -269,6 +288,9 @@ const backTo = computed(() => syntheticsListRoute(navContext.value, { section: "
|
||||
|
||||
const detail = ref<SyntheticLocationDetail | null>(null);
|
||||
const loading = ref(false);
|
||||
/** True after a retryable failure (network error, timeout, 5xx). 403/404 keep
|
||||
* the not-found/forbidden handling instead. */
|
||||
const loadError = ref(false);
|
||||
const showSetup = ref(false);
|
||||
const agentSetup = ref<AgentSetup | null>(null);
|
||||
const setupAgentName = ref<string | null>(null);
|
||||
@@ -308,11 +330,15 @@ const agentsEmptyMessage = computed(() =>
|
||||
|
||||
async function load() {
|
||||
loading.value = true;
|
||||
loadError.value = false;
|
||||
try {
|
||||
const res = await syntheticsService.getLocation(orgIdentifier.value, String(route.params.id));
|
||||
detail.value = res.data as SyntheticLocationDetail;
|
||||
} catch (err) {
|
||||
detail.value = null;
|
||||
const status = (err as { response?: { status?: number } })?.response?.status;
|
||||
// No response (network/timeout) or 5xx is retryable; 403/404/other 4xx are not.
|
||||
loadError.value = status === undefined || status >= 500;
|
||||
console.error("[synthetics] failed to load location detail", err);
|
||||
} finally {
|
||||
loading.value = false;
|
||||
|
||||
@@ -795,3 +795,91 @@ describe("RunDetail — start load (row 0)", () => {
|
||||
w.unmount();
|
||||
});
|
||||
});
|
||||
|
||||
// ── Step outcomes and locators (ENT#3140 B2/B3) ──────────────────────────────
|
||||
describe("RunDetail — skipped / unknown steps and locators", () => {
|
||||
const detail = {
|
||||
...mockRunDetail,
|
||||
status: "failed",
|
||||
recordedSteps: [
|
||||
{ id: "s1", action: "navigate", name: "Open home", url: "https://app.test/" },
|
||||
{
|
||||
id: "s2",
|
||||
action: "click",
|
||||
name: "Dismiss banner",
|
||||
locator: { candidates: [{ kind: "role", value: "button[name=Close]" }] },
|
||||
},
|
||||
{ id: "s3", action: "click", name: "Sign in", selector: "#signin" },
|
||||
{ id: "s4", action: "wait", name: "Settle" },
|
||||
],
|
||||
lastAttemptSteps: [
|
||||
{ step_id: "s1", status: "ok", duration_ms: 10, error: "", screenshot_key: null },
|
||||
{
|
||||
step_id: "s2",
|
||||
status: "skipped",
|
||||
duration_ms: 5,
|
||||
error: "optional step: banner not present",
|
||||
screenshot_key: null,
|
||||
},
|
||||
{ step_id: "s3", status: "fail", duration_ms: 20, error: "timeout", screenshot_key: null },
|
||||
{ step_id: "s4", status: "weird", duration_ms: 1, error: "", screenshot_key: null },
|
||||
],
|
||||
};
|
||||
|
||||
afterEach(() => {
|
||||
mockRunDetailRef.value = { ...mockRunDetail };
|
||||
});
|
||||
|
||||
async function mountWith() {
|
||||
mockRunDetailRef.value = null;
|
||||
const w = mountComponent();
|
||||
await flushPromises();
|
||||
mockRunDetailRef.value = { ...detail };
|
||||
await flushPromises();
|
||||
return w;
|
||||
}
|
||||
|
||||
it("keeps pass, fail, skipped and unknown distinct", async () => {
|
||||
const w = await mountWith();
|
||||
const steps = (w.vm as any).steps;
|
||||
expect(steps.map((s: any) => s.status)).toEqual(["pass", "skipped", "fail", "unknown"]);
|
||||
expect((w.vm as any).stepDotState(steps[1])).toBe("skip");
|
||||
// Unknown must never take the pass colour.
|
||||
expect((w.vm as any).stepDotState(steps[3])).toBeUndefined();
|
||||
expect((w.vm as any).stepStatusBadge(steps[1])?.label).toBe(
|
||||
"synthetics.runDetail.stepStatusSkipped",
|
||||
);
|
||||
expect((w.vm as any).stepStatusBadge(steps[0])).toBeNull();
|
||||
w.unmount();
|
||||
});
|
||||
|
||||
it("shows the skipped count on the steps tab", async () => {
|
||||
const w = await mountWith();
|
||||
expect(w.find('[data-test="synthetics-run-detail-skipped-count"]').exists()).toBe(true);
|
||||
w.unmount();
|
||||
});
|
||||
|
||||
it("resolves the locator with effectiveSelector and never falls back to the step id", async () => {
|
||||
const w = await mountWith();
|
||||
const steps = (w.vm as any).steps;
|
||||
expect(steps[1].selector).toBe("button[name=Close]");
|
||||
expect(steps[2].selector).toBe("#signin");
|
||||
expect(steps[0].selector).toBeNull();
|
||||
expect(steps[3].selector).toBeNull();
|
||||
expect(steps[3].detail).toBe("");
|
||||
w.unmount();
|
||||
});
|
||||
|
||||
it("renders Selector and Step ID on the failed step, and the skip reason when expanded", async () => {
|
||||
const w = await mountWith();
|
||||
// The failed step auto-expands.
|
||||
expect(w.find('[data-test="synthetics-run-detail-step-selector"]').text()).toContain("#signin");
|
||||
expect(w.find('[data-test="synthetics-run-detail-step-id"]').text()).toContain("s3");
|
||||
(w.vm as any).handleUpdateExpanded(["2"]);
|
||||
await flushPromises();
|
||||
expect(w.find('[data-test="synthetics-run-detail-step-skip-reason"]').text()).toContain(
|
||||
"banner not present",
|
||||
);
|
||||
w.unmount();
|
||||
});
|
||||
});
|
||||
|
||||
@@ -182,6 +182,13 @@ along with this program. If not, see <http://www.gnu.org/licenses/>.
|
||||
<OTab name="steps" data-test="synthetics-run-detail-tab-steps">
|
||||
{{ t("synthetics.runs.tabSteps") }}
|
||||
<OBadge variant="default" size="sm">{{ steps.length }}</OBadge>
|
||||
<OBadge
|
||||
v-if="skippedStepCount > 0"
|
||||
variant="default"
|
||||
size="sm"
|
||||
data-test="synthetics-run-detail-skipped-count"
|
||||
>{{ t("synthetics.runDetail.skippedCount", { n: skippedStepCount }) }}</OBadge
|
||||
>
|
||||
</OTab>
|
||||
<OTab name="evidence" data-test="synthetics-run-detail-tab-evidence">
|
||||
{{ t("synthetics.runDetail.evidenceSection") }}
|
||||
@@ -332,6 +339,7 @@ along with this program. If not, see <http://www.gnu.org/licenses/>.
|
||||
detail-key="detail"
|
||||
icon-key="icon"
|
||||
:dot-state-fn="stepDotState"
|
||||
:step-badge-fn="stepStatusBadge"
|
||||
:expanded-ids="expandedStepIdsArr"
|
||||
@update:expanded-ids="handleUpdateExpanded"
|
||||
>
|
||||
@@ -435,20 +443,80 @@ along with this program. If not, see <http://www.gnu.org/licenses/>.
|
||||
{{ t("synthetics.runDetail.detailAction") }}
|
||||
</dt>
|
||||
<dd class="text-text-secondary">{{ row.action }}</dd>
|
||||
<dt
|
||||
class="text-text-secondary text-sm font-semibold tracking-wide capitalize"
|
||||
>
|
||||
{{ t("synthetics.runDetail.detailSelector") }}
|
||||
</dt>
|
||||
<dd class="text-text-secondary">{{ row.detail }}</dd>
|
||||
<dt
|
||||
class="text-text-secondary text-sm font-semibold tracking-wide capitalize"
|
||||
>
|
||||
{{ t("synthetics.runDetail.detailUrl") }}
|
||||
</dt>
|
||||
<OTruncatedText as="dd" class="text-text-secondary">
|
||||
{{ row.url || currentRun.url }}
|
||||
</OTruncatedText>
|
||||
<template v-if="row.selector">
|
||||
<dt
|
||||
class="text-text-secondary text-sm font-semibold tracking-wide capitalize"
|
||||
>
|
||||
{{ t("synthetics.runDetail.detailSelector") }}
|
||||
</dt>
|
||||
<OTruncatedText
|
||||
as="dd"
|
||||
class="text-text-secondary font-mono"
|
||||
data-test="synthetics-run-detail-step-selector"
|
||||
>
|
||||
{{ row.selector }}
|
||||
</OTruncatedText>
|
||||
</template>
|
||||
<!-- The configured navigation target, not the page the
|
||||
step observed: the probe does not report that yet. -->
|
||||
<template v-if="row.url">
|
||||
<dt
|
||||
class="text-text-secondary text-sm font-semibold tracking-wide capitalize"
|
||||
>
|
||||
{{ t("synthetics.runDetail.detailTargetUrl") }}
|
||||
</dt>
|
||||
<OTruncatedText
|
||||
as="dd"
|
||||
class="text-text-secondary"
|
||||
data-test="synthetics-run-detail-step-target-url"
|
||||
>
|
||||
{{ row.url }}
|
||||
</OTruncatedText>
|
||||
</template>
|
||||
<template v-if="row.stepId && row.id > 0">
|
||||
<dt
|
||||
class="text-text-secondary text-sm font-semibold tracking-wide capitalize"
|
||||
>
|
||||
{{ t("synthetics.runDetail.detailStepId") }}
|
||||
</dt>
|
||||
<OTruncatedText
|
||||
as="dd"
|
||||
class="text-text-secondary font-mono"
|
||||
data-test="synthetics-run-detail-step-id"
|
||||
>
|
||||
{{ row.stepId }}
|
||||
</OTruncatedText>
|
||||
</template>
|
||||
<template v-if="row.status === 'skipped' || row.status === 'unknown'">
|
||||
<dt
|
||||
class="text-text-secondary text-sm font-semibold tracking-wide capitalize"
|
||||
>
|
||||
{{ t("synthetics.runDetail.detailStatus") }}
|
||||
</dt>
|
||||
<dd
|
||||
class="text-text-secondary"
|
||||
data-test="synthetics-run-detail-step-status"
|
||||
>
|
||||
{{
|
||||
row.status === "skipped"
|
||||
? t("synthetics.runDetail.stepStatusSkipped")
|
||||
: t("synthetics.runDetail.stepStatusUnknown")
|
||||
}}
|
||||
</dd>
|
||||
</template>
|
||||
<template v-if="row.status === 'skipped' && row.error">
|
||||
<dt
|
||||
class="text-text-secondary text-sm font-semibold tracking-wide capitalize"
|
||||
>
|
||||
{{ t("synthetics.runDetail.detailSkipReason") }}
|
||||
</dt>
|
||||
<dd
|
||||
class="text-text-secondary break-words whitespace-pre-wrap"
|
||||
data-test="synthetics-run-detail-step-skip-reason"
|
||||
>
|
||||
{{ row.error }}
|
||||
</dd>
|
||||
</template>
|
||||
<dt
|
||||
class="text-text-secondary text-sm font-semibold tracking-wide capitalize"
|
||||
>
|
||||
@@ -631,7 +699,11 @@ import OTabs from "@/lib/navigation/Tabs/OTabs.vue";
|
||||
import OTab from "@/lib/navigation/Tabs/OTab.vue";
|
||||
import OTabPanels from "@/lib/navigation/Tabs/OTabPanels.vue";
|
||||
import OTabPanel from "@/lib/navigation/Tabs/OTabPanel.vue";
|
||||
import { buildAttemptViews, stepOwnDetail } from "@/composables/synthetics/syntheticResultsSchema";
|
||||
import {
|
||||
buildAttemptViews,
|
||||
effectiveSelector,
|
||||
stepOwnDetail,
|
||||
} from "@/composables/synthetics/syntheticResultsSchema";
|
||||
import type {
|
||||
AttemptView,
|
||||
EvidenceEvent,
|
||||
@@ -815,7 +887,13 @@ interface StepRow {
|
||||
* When the probe starts emitting `start_time`, this becomes a read of it.
|
||||
*/
|
||||
offsetMs: number;
|
||||
status: "pass" | "fail";
|
||||
status: StepRowStatus;
|
||||
/**
|
||||
* The locator the step acted on, from this run's recorded snapshot via
|
||||
* `effectiveSelector`. Null for actions without a locator (navigate, wait),
|
||||
* so the Selector row is omitted instead of showing the step ID.
|
||||
*/
|
||||
selector: string | null;
|
||||
icon: string;
|
||||
statusIcon: string;
|
||||
durStr: string;
|
||||
@@ -830,6 +908,27 @@ interface StepRow {
|
||||
bundleEvents: EvidenceEvent[];
|
||||
}
|
||||
|
||||
/**
|
||||
* A step's outcome as shown. Skipped and unknown stay distinct: a skipped
|
||||
* optional step is neutral, not passed, and a status the UI does not know
|
||||
* must never be shown as a pass.
|
||||
*/
|
||||
type StepRowStatus = "pass" | "fail" | "skipped" | "unknown";
|
||||
|
||||
function stepRowStatus(raw: unknown): StepRowStatus {
|
||||
if (raw === "ok" || raw === "passed" || raw === "pass") return "pass";
|
||||
if (raw === "fail" || raw === "failed") return "fail";
|
||||
if (raw === "skipped" || raw === "skip") return "skipped";
|
||||
return "unknown";
|
||||
}
|
||||
|
||||
const STEP_STATUS_ICON: Record<StepRowStatus, string> = {
|
||||
pass: "check-circle",
|
||||
fail: "cancel",
|
||||
skipped: "do-not-disturb-on",
|
||||
unknown: "help",
|
||||
};
|
||||
|
||||
function fmtDur(ms: number): string {
|
||||
return ms >= 1000 ? (ms / 1000).toFixed(1) + "s" : ms + "ms";
|
||||
}
|
||||
@@ -857,21 +956,25 @@ function buildSteps(
|
||||
let elapsedMs = 0;
|
||||
return steps.map((ex, idx) => {
|
||||
const recorded = recordedMap.get(ex.step_id);
|
||||
const isFail = ex.status === "fail";
|
||||
const status = stepRowStatus(ex.status);
|
||||
const isFail = status === "fail";
|
||||
const selector = recorded ? effectiveSelector(recorded) : null;
|
||||
const offsetMs = elapsedMs;
|
||||
elapsedMs += ex.duration_ms ?? 0;
|
||||
return {
|
||||
id: idx + 1,
|
||||
stepId: ex.step_id,
|
||||
action: recorded?.action ?? "step",
|
||||
name: recorded?.name || recorded?.selector || recorded?.url || ex.step_id.slice(0, 8),
|
||||
detail: recorded?.selector ?? recorded?.url ?? ex.step_id,
|
||||
name: recorded?.name || selector || recorded?.url || ex.step_id.slice(0, 8),
|
||||
// The step ID is shown only under "Step ID", never as the locator.
|
||||
detail: selector ?? recorded?.url ?? "",
|
||||
selector,
|
||||
url: recorded?.url ?? "",
|
||||
duration: ex.duration_ms,
|
||||
offsetMs,
|
||||
status: isFail ? ("fail" as const) : ("pass" as const),
|
||||
status,
|
||||
icon: recorded ? actionIcon(recorded.action) : "radio_button_checked",
|
||||
statusIcon: isFail ? "cancel" : "check-circle",
|
||||
statusIcon: STEP_STATUS_ICON[status],
|
||||
durStr: fmtDur(ex.duration_ms),
|
||||
durColor: isFail ? "var(--color-status-error-text)" : "var(--color-text-secondary)",
|
||||
error: ex.error,
|
||||
@@ -915,6 +1018,7 @@ function buildStartRow(
|
||||
action: "navigate",
|
||||
name: t("synthetics.runDetail.startLoadLabel", { url: load.url }),
|
||||
detail: load.url,
|
||||
selector: null,
|
||||
url: load.url,
|
||||
duration: load.duration_ms,
|
||||
offsetMs: 0,
|
||||
@@ -1256,7 +1360,28 @@ watch(
|
||||
);
|
||||
|
||||
function stepDotState(row: any): StepDotState | undefined {
|
||||
return row.status === "fail" ? "fail" : "pass";
|
||||
switch (row.status as StepRowStatus) {
|
||||
case "fail":
|
||||
return "fail";
|
||||
case "pass":
|
||||
return "pass";
|
||||
case "skipped":
|
||||
return "skip";
|
||||
default:
|
||||
// Unknown: the neutral outline, never the pass colour.
|
||||
return undefined;
|
||||
}
|
||||
}
|
||||
|
||||
/** Text label for the outcomes the dot colour alone cannot carry. */
|
||||
function stepStatusBadge(row: any): { label: string; variant: "default" | "error" } | null {
|
||||
if (row.status === "skipped") {
|
||||
return { label: t("synthetics.runDetail.stepStatusSkipped"), variant: "default" };
|
||||
}
|
||||
if (row.status === "unknown") {
|
||||
return { label: t("synthetics.runDetail.stepStatusUnknown"), variant: "default" };
|
||||
}
|
||||
return null;
|
||||
}
|
||||
|
||||
/**
|
||||
@@ -1360,6 +1485,8 @@ const steps = computed<StepRow[]>(() => {
|
||||
return [];
|
||||
});
|
||||
|
||||
const skippedStepCount = computed(() => steps.value.filter((s) => s.status === "skipped").length);
|
||||
|
||||
const startRow = computed<StepRow | null>(() => {
|
||||
const detail = synthetics.runDetail.value;
|
||||
if (!detail) return null;
|
||||
|
||||
Reference in New Issue
Block a user