From 45be37ef24126b3409a3120b42aace0a6e310b0c Mon Sep 17 00:00:00 2001 From: Subhra264 Date: Tue, 3 Nov 2026 14:37:30 +0000 Subject: [PATCH 1/2] fix(oncall): unsnooze a page so paging continues now (#16338) --- src/api/common/src/auth/delegation.rs | 2 + src/api/http/src/handler/http/router/mod.rs | 5 ++ .../http/src/handler/http/router/openapi.rs | 1 + src/api/management/src/request/oncall/mod.rs | 50 ++++++++++++++++++ src/config/src/meta/oncall/response.rs | 32 ++++++++++++ src/infra/src/table/oncall_responses.rs | 30 +++++++++++ web/src/locales/languages/en-US.json | 4 ++ web/src/services/oncall.queries.ts | 7 +++ web/src/services/oncall.ts | 12 +++++ .../views/OnCall/OnCallResponseDetail.spec.ts | 37 +++++++++++++ web/src/views/OnCall/OnCallResponseDetail.vue | 52 ++++++++++++++++++- web/src/views/OnCall/OnCallResponses.spec.ts | 24 +++++++++ web/src/views/OnCall/OnCallResponses.vue | 39 +++++++++++++- 13 files changed, 292 insertions(+), 3 deletions(-) diff --git a/src/api/common/src/auth/delegation.rs b/src/api/common/src/auth/delegation.rs index ce89ce4a0f..85fdbdb283 100644 --- a/src/api/common/src/auth/delegation.rs +++ b/src/api/common/src/auth/delegation.rs @@ -150,6 +150,7 @@ const ASSISTANT_SCOPE: &[ScopeRoute] = &[ v1(&[Param, Lit("oncall"), Lit("incidents"), Param, Lit("responses")], GET), v1(&[Param, Lit("oncall"), Lit("responses"), Param, Lit("acknowledge")], POST), v1(&[Param, Lit("oncall"), Lit("responses"), Param, Lit("snooze")], POST), + v1(&[Param, Lit("oncall"), Lit("responses"), Param, Lit("unsnooze")], POST), v1(&[Param, Lit("oncall"), Lit("responses"), Param, Lit("resolve")], POST), v1(&[Param, Lit("ai"), Lit("chat")], POST), v1(&[Param, Lit("ai"), Lit("chat_stream")], POST), @@ -1001,6 +1002,7 @@ mod tests { (Method::GET, "acme/oncall/teams/t1/on-call"), (Method::POST, "acme/oncall/responses/r1/acknowledge"), (Method::POST, "acme/oncall/responses/r1/snooze"), + (Method::POST, "acme/oncall/responses/r1/unsnooze"), (Method::POST, "acme/oncall/responses/r1/resolve"), (Method::POST, "acme/ai/chat_stream"), (Method::POST, "acme/ai/confirm/s1"), diff --git a/src/api/http/src/handler/http/router/mod.rs b/src/api/http/src/handler/http/router/mod.rs index 0cba32d8e9..b9bd7bb4b9 100644 --- a/src/api/http/src/handler/http/router/mod.rs +++ b/src/api/http/src/handler/http/router/mod.rs @@ -2416,6 +2416,10 @@ fn oncall_routes(router: Router) -> Router { "/{org_id}/oncall/responses/{response_id}/snooze", post(oncall::snooze_response), ) + .route( + "/{org_id}/oncall/responses/{response_id}/unsnooze", + post(oncall::unsnooze_response), + ) .route( "/{org_id}/oncall/responses/{response_id}/handoff", post(oncall::handoff_response), @@ -5213,6 +5217,7 @@ mod tests { "POST /{org_id}/oncall/responses/{response_id}/acknowledge", "POST /{org_id}/oncall/responses/{response_id}/resolve", "POST /{org_id}/oncall/responses/{response_id}/snooze", + "POST /{org_id}/oncall/responses/{response_id}/unsnooze", "POST /{org_id}/prometheus/api/v1/query", "POST /{org_id}/prometheus/api/v1/query_range", "POST /{org_id}/{stream_name}/_around", diff --git a/src/api/http/src/handler/http/router/openapi.rs b/src/api/http/src/handler/http/router/openapi.rs index 315af19a64..90c1408e5e 100644 --- a/src/api/http/src/handler/http/router/openapi.rs +++ b/src/api/http/src/handler/http/router/openapi.rs @@ -1328,6 +1328,7 @@ pub struct ApiDoc; openobserve_api_management::request::oncall::snooze_response, openobserve_api_management::request::oncall::test_team_slack_channel, openobserve_api_management::request::oncall::unacknowledge_response, + openobserve_api_management::request::oncall::unsnooze_response, openobserve_api_management::request::oncall::update_ownership_rule, openobserve_api_management::request::oncall::update_team, openobserve_api_management::request::oncall::who_is_on_call, diff --git a/src/api/management/src/request/oncall/mod.rs b/src/api/management/src/request/oncall/mod.rs index b43b970839..b8147dd692 100644 --- a/src/api/management/src/request/oncall/mod.rs +++ b/src/api/management/src/request/oncall/mod.rs @@ -3081,6 +3081,56 @@ pub async fn snooze_response( } } +#[utoipa::path( + post, + path = "/{org_id}/oncall/responses/{response_id}/unsnooze", + context_path = "/api", + tag = "OnCall", + operation_id = "UnsnoozeOnCallResponse", + summary = "End a snooze now, so paging continues from the current step", + security(("Authorization" = [])), + params( + ("org_id" = String, Path, description = "Organization name"), + ("response_id" = String, Path, description = "Response record ID"), + ), + responses((status = 200, description = "Success", content_type = "application/json", body = Object)), + extensions( + ("x-o2-ratelimit" = json!({"module": "OnCall", "operation": "create"})), + ("x-o2-mcp" = json!({"enabled": false})) + ) +)] +pub async fn unsnooze_response( + Path((org_id, response_id)): Path<(String, String)>, + #[cfg(feature = "enterprise")] Headers(user_email): Headers, +) -> Response { + #[cfg(feature = "enterprise")] + { + if !allowed(&org_id, &user_email.user_id, RESPONSES, "POST").await { + return MetaHttpResponse::forbidden("Forbidden"); + } + if let Err(resp) = page_problem(&org_id, &response_id, &user_email.user_id).await { + return resp; + } + match o2_enterprise::enterprise::oncall::escalation::unsnooze( + &org_id, + &response_id, + &user_email.user_id, + config::utils::time::now_micros(), + ) + .await + { + Ok(Some(r)) => MetaHttpResponse::json(r), + Ok(None) => MetaHttpResponse::not_found("Response record not found"), + Err(e) => to_response(e), + } + } + #[cfg(not(feature = "enterprise"))] + { + let _ = (org_id, response_id); + MetaHttpResponse::forbidden("Not Supported") + } +} + #[utoipa::path( post, path = "/{org_id}/oncall/responses/{response_id}/handoff", diff --git a/src/config/src/meta/oncall/response.rs b/src/config/src/meta/oncall/response.rs index d53a32eaae..afcf89cb0c 100644 --- a/src/config/src/meta/oncall/response.rs +++ b/src/config/src/meta/oncall/response.rs @@ -78,6 +78,17 @@ pub fn snoozed_ladder_anchor( existing_anchor.unwrap_or(opened_at) + (until - quiet_reaches).max(0) } +/// Where the ladder's clock sits when a snooze ends early at `now`: as if it had been that short. +pub fn unsnoozed_ladder_anchor( + existing_anchor: Option, + opened_at: i64, + snoozed_until: Option, + now: i64, +) -> i64 { + let unused = snoozed_until.map_or(0, |until| (until - now).max(0)); + existing_anchor.unwrap_or(opened_at) - unused +} + /// Lifecycle of a response record: `triggered → triaged → acknowledged → /// resolved`. `triaged` comes from the L0 agent and is skipped entirely when /// the agent is absent or disabled. @@ -1711,6 +1722,27 @@ mod tests { 17 * MIN ); } + + /// Ending a snooze early gives back the unused quiet, so the rungs keep their gaps from + /// where the ladder paused instead of waiting out the rest of the snooze. + #[test] + fn test_unsnoozing_leaves_the_clock_where_a_snooze_that_short_would() { + let snoozed = snoozed_ladder_anchor(None, 0, None, MIN, 61 * MIN); + assert_eq!( + unsnoozed_ladder_anchor(Some(snoozed), 0, Some(61 * MIN), 3 * MIN), + snoozed_ladder_anchor(None, 0, None, MIN, 3 * MIN), + ); + } + + /// A snooze that has already lapsed has no quiet left to give back. + #[test] + fn test_unsnoozing_a_lapsed_snooze_leaves_the_clock() { + let snoozed = snoozed_ladder_anchor(None, 0, None, MIN, 61 * MIN); + assert_eq!( + unsnoozed_ladder_anchor(Some(snoozed), 0, Some(61 * MIN), 90 * MIN), + snoozed + ); + } } #[test] diff --git a/src/infra/src/table/oncall_responses.rs b/src/infra/src/table/oncall_responses.rs index 8de6afda74..f3360b1b78 100644 --- a/src/infra/src/table/oncall_responses.rs +++ b/src/infra/src/table/oncall_responses.rs @@ -664,6 +664,36 @@ pub async fn snooze( .await } +/// Ends a snooze at `now`, giving the ladder's clock back the quiet that was not used. +pub async fn unsnooze(org_id: &str, id: &str, now: i64) -> Result, errors::Error> { + let client = get_orm_client_rw().await; + let Some(existing) = oncall_responses::Entity::find_by_id(id) + .filter(oncall_responses::Column::OrgId.eq(org_id)) + .one(client) + .await? + else { + return Ok(None); + }; + let anchor = config::meta::oncall::response::unsnoozed_ladder_anchor( + existing.ladder_anchor, + existing.opened_at, + existing.snoozed_until, + now, + ); + save( + org_id, + id, + vec![ + ( + oncall_responses::Column::SnoozedUntil, + Expr::value(Option::::None), + ), + (oncall_responses::Column::LadderAnchor, Expr::value(anchor)), + ], + ) + .await +} + /// Writes a new severity onto an open record. /// /// The severity is what SLO reporting, digests and the post-incident review diff --git a/web/src/locales/languages/en-US.json b/web/src/locales/languages/en-US.json index 882f44da23..c50582f9d2 100644 --- a/web/src/locales/languages/en-US.json +++ b/web/src/locales/languages/en-US.json @@ -25689,6 +25689,10 @@ "snoozed": "Snoozed", "snoozedUntil": "Quiet until {time} — still unassigned", "snoozeFailed": "Could not snooze the page", + "unsnooze": "Unsnooze", + "unsnoozeHint": "Ends the snooze now. Paging continues from the step it paused on.", + "unsnoozed": "Snooze ended · paging continues", + "unsnoozeFailed": "Could not end the snooze", "note": "Note", "notePlaceholder": "What did you find? What did you try?", "addNote": "Add note", diff --git a/web/src/services/oncall.queries.ts b/web/src/services/oncall.queries.ts index 4218ff6fc1..d025418b7c 100644 --- a/web/src/services/oncall.queries.ts +++ b/web/src/services/oncall.queries.ts @@ -721,6 +721,13 @@ export const snoozeResponseMutation = (org: string) => meta: { invalidates: responseWriteScopes(org), silentError: true }, }); +export const unsnoozeResponseMutation = (org: string) => + mutationOptions({ + mutationFn: (responseId: string) => + oncallService.unsnoozeResponse({ org_identifier: org, response_id: responseId }), + meta: { invalidates: responseWriteScopes(org), silentError: true }, + }); + export const resolveResponseMutation = (org: string) => mutationOptions({ mutationFn: ({ diff --git a/web/src/services/oncall.ts b/web/src/services/oncall.ts index 5341af330b..e5c223c653 100644 --- a/web/src/services/oncall.ts +++ b/web/src/services/oncall.ts @@ -511,6 +511,18 @@ const oncall = { { minutes }, ), + /// Ends a snooze now; the ladder continues from the step it paused on. + unsnoozeResponse: ({ + org_identifier, + response_id, + }: { + org_identifier: string; + response_id: string; + }) => + http().post( + `/api/${org_identifier}/oncall/responses/${encodeURIComponent(response_id)}/unsnooze`, + ), + addNote: ({ org_identifier, response_id, diff --git a/web/src/views/OnCall/OnCallResponseDetail.spec.ts b/web/src/views/OnCall/OnCallResponseDetail.spec.ts index 3660dd2a1c..915f1b95af 100644 --- a/web/src/views/OnCall/OnCallResponseDetail.spec.ts +++ b/web/src/views/OnCall/OnCallResponseDetail.spec.ts @@ -39,6 +39,7 @@ vi.mock("@/services/oncall", () => ({ unacknowledgeResponse: vi.fn(), confirmRecovery: vi.fn(), snoozeResponse: vi.fn(), + unsnoozeResponse: vi.fn(), addNote: vi.fn(), handoffResponse: vi.fn(), resolveResponse: vi.fn(), @@ -661,6 +662,29 @@ describe("OnCallResponseDetail", () => { expect(lapsed.find('[data-test="oncall-response-snoozed-banner"]').exists()).toBe(false); }); + /// Snooze means paused, then continues; while it is paused the same control continues it now. + it("swaps Snooze for Unsnooze while snoozed, and the banner offers it too", async () => { + const wrapper = await renderWith({ snoozed_until: (Date.now() + 30 * 60_000) * 1000 }); + service.unsnoozeResponse.mockResolvedValue({ data: {} } as any); + + expect(wrapper.find('[data-test="oncall-response-snooze-btn"]').exists()).toBe(false); + for (const button of [ + "oncall-response-unsnooze-btn", + "oncall-response-snoozed-banner-unsnooze", + ]) { + service.unsnoozeResponse.mockClear(); + toastSpy.mockClear(); + await wrapper.find(`[data-test="${button}"]`).trigger("click"); + await flushPromises(); + expect(service.unsnoozeResponse).toHaveBeenCalledWith( + expect.objectContaining({ response_id: "resp_1" }), + ); + expect(toastSpy).toHaveBeenCalledWith( + expect.objectContaining({ message: "Snooze ended · paging continues" }), + ); + } + }); + it("adds a note and clears the box", async () => { const wrapper = await renderWith(); service.addNote.mockResolvedValue({ data: {} } as any); @@ -1586,6 +1610,19 @@ describe("OnCallResponseDetail — what the payload already knew", () => { ); }); + it("unsnoozes from the menu while snoozed", async () => { + const wrapper = await renderWith({ snoozed_until: (Date.now() + 30 * 60_000) * 1000 }); + service.unsnoozeResponse.mockResolvedValue({ data: {} } as any); + + expect(wrapper.find('[data-test="oncall-response-snooze-30-menu"]').exists()).toBe(false); + await wrapper.find('[data-test="oncall-response-unsnooze-btn-menu"]').trigger("click"); + await flushPromises(); + + expect(service.unsnoozeResponse).toHaveBeenCalledWith( + expect.objectContaining({ response_id: "resp_1" }), + ); + }); + it("opens the hand-off drawer from the menu", async () => { const wrapper = await renderWith(); diff --git a/web/src/views/OnCall/OnCallResponseDetail.vue b/web/src/views/OnCall/OnCallResponseDetail.vue index f8ed0a3303..a8052312a7 100644 --- a/web/src/views/OnCall/OnCallResponseDetail.vue +++ b/web/src/views/OnCall/OnCallResponseDetail.vue @@ -146,7 +146,18 @@ - + + {{ t("oncall.unsnooze") }} + + + - + + {{ t("oncall.unsnooze") }} + + {{ t("oncall.snoozedUntil", { time: snoozedUntilLabel }) }} + - + + + {{ t("oncall.unsnooze") }} + + acknowledgeResponseMutation(orgId.value)); const snoozeWrite = useMutation(() => snoozeResponseMutation(orgId.value)); +const unsnoozeWrite = useMutation(() => unsnoozeResponseMutation(orgId.value)); const resolveWrite = useMutation(() => resolveResponseMutation(orgId.value)); const handoffWrite = useMutation(() => handoffResponseMutation(orgId.value)); const noteWrite = useMutation(() => addResponseNoteMutation(orgId.value)); @@ -1657,6 +1672,28 @@ async function snoozeRow(row: PageRow, minutes: number) { } } +function snoozedFirings(row: PageRow): OnCallResponse[] { + return row.escalating.filter((r) => isSnoozed(r)); +} + +/// Ends the snooze on every quiet firing the row stands for, so they all continue now. +async function unsnoozeRow(row: PageRow) { + busyId.value = row.rowKey; + try { + const results = await Promise.allSettled( + snoozedFirings(row).map((r) => unsnoozeWrite.mutateAsync(r.id)), + ); + if (results.some((r) => r.status === "rejected")) { + toast({ variant: "error", message: t("oncall.unsnoozeFailed") }); + } else { + toast({ variant: "success", message: t("oncall.unsnoozed") }); + } + await fetchResponses(); + } finally { + busyId.value = ""; + } +} + async function resolveRow(row: PageRow) { busyId.value = row.rowKey; try { From f595e0b7d65826c63d24b4029ab864883dfd4c51 Mon Sep 17 00:00:00 2001 From: Shrinath Rao Date: Tue, 3 Nov 2026 15:20:39 +0000 Subject: [PATCH 2/2] test(alerts): stop the two rca_turn unit-test flakes in the enterprise build (#16326) --- .cargo/config.toml | 4 ++++ src/core/src/alerts/rca_turn.rs | 9 ++++----- 2 files changed, 8 insertions(+), 5 deletions(-) diff --git a/.cargo/config.toml b/.cargo/config.toml index 9c9e20f089..5df8ea0c15 100644 --- a/.cargo/config.toml +++ b/.cargo/config.toml @@ -20,3 +20,7 @@ rustflags = ["-C", "target-feature=+aes,+sse2,+ssse3,+sse4.1,+sse4.2"] [target.x86_64-pc-windows-msvc] rustflags = ["-C", "target-feature=+aes,+sse2,+ssse3,+sse4.1,+sse4.2"] + +[env] +# test threads get the 16 MB stack of the server's runtime threads (THREAD_STACK_SIZE); debug async code outgrows 2 MB +RUST_MIN_STACK = "16777216" diff --git a/src/core/src/alerts/rca_turn.rs b/src/core/src/alerts/rca_turn.rs index cb2ed3c47c..4b05d007a8 100644 --- a/src/core/src/alerts/rca_turn.rs +++ b/src/core/src/alerts/rca_turn.rs @@ -3362,12 +3362,12 @@ mod tests { } } - /// Paused time that moves 100x faster than the wall clock. + /// 10x time: auto-advance fires the DB acquire timeout; faster makes CI stalls false backoffs. fn fast_clock() -> tokio::task::JoinHandle<()> { tokio::spawn(async { loop { std::thread::sleep(Duration::from_millis(1)); - tokio::time::advance(Duration::from_millis(100)).await; + tokio::time::advance(Duration::from_millis(10)).await; } }) } @@ -3491,9 +3491,9 @@ mod tests { clock.abort(); } - #[tokio::test(start_paused = true)] + // Real clock: even the 10x fast_clock turns a 500 ms CI stall into a false 5 s backoff. + #[tokio::test] async fn other_start_errors_fail_the_continuation_at_once() { - let clock = fast_clock(); let _db = db().await; for (error, can_retry, code) in [ (RcaTurnError::Busy, true, "busy"), @@ -3543,7 +3543,6 @@ mod tests { "{stored_error:?}" ); } - clock.abort(); } #[tokio::test]