Merge remote-tracking branch 'origin/main' into fix/synthetics-not-measured

.cargo/config.toml

@@ -20,3 +20,7 @@ rustflags = ["-C", "target-feature=+aes,+sse2,+ssse3,+sse4.1,+sse4.2"]
 
 [target.x86_64-pc-windows-msvc]
 rustflags = ["-C", "target-feature=+aes,+sse2,+ssse3,+sse4.1,+sse4.2"]
+
+[env]
+# test threads get the 16 MB stack of the server's runtime threads (THREAD_STACK_SIZE); debug async code outgrows 2 MB
+RUST_MIN_STACK = "16777216"

src/api/common/src/auth/delegation.rs

@@ -150,6 +150,7 @@ const ASSISTANT_SCOPE: &[ScopeRoute] = &[
     v1(&[Param, Lit("oncall"), Lit("incidents"), Param, Lit("responses")], GET),
     v1(&[Param, Lit("oncall"), Lit("responses"), Param, Lit("acknowledge")], POST),
     v1(&[Param, Lit("oncall"), Lit("responses"), Param, Lit("snooze")], POST),
+    v1(&[Param, Lit("oncall"), Lit("responses"), Param, Lit("unsnooze")], POST),
     v1(&[Param, Lit("oncall"), Lit("responses"), Param, Lit("resolve")], POST),
     v1(&[Param, Lit("ai"), Lit("chat")], POST),
     v1(&[Param, Lit("ai"), Lit("chat_stream")], POST),
@@ -1001,6 +1002,7 @@ mod tests {
             (Method::GET, "acme/oncall/teams/t1/on-call"),
             (Method::POST, "acme/oncall/responses/r1/acknowledge"),
             (Method::POST, "acme/oncall/responses/r1/snooze"),
+            (Method::POST, "acme/oncall/responses/r1/unsnooze"),
             (Method::POST, "acme/oncall/responses/r1/resolve"),
             (Method::POST, "acme/ai/chat_stream"),
             (Method::POST, "acme/ai/confirm/s1"),

src/api/http/src/handler/http/router/mod.rs

@@ -2417,6 +2417,10 @@ fn oncall_routes(router: Router) -> Router {
             "/{org_id}/oncall/responses/{response_id}/snooze",
             post(oncall::snooze_response),
         )
+        .route(
+            "/{org_id}/oncall/responses/{response_id}/unsnooze",
+            post(oncall::unsnooze_response),
+        )
         .route(
             "/{org_id}/oncall/responses/{response_id}/handoff",
             post(oncall::handoff_response),
@@ -5214,6 +5218,7 @@ mod tests {
             "POST /{org_id}/oncall/responses/{response_id}/acknowledge",
             "POST /{org_id}/oncall/responses/{response_id}/resolve",
             "POST /{org_id}/oncall/responses/{response_id}/snooze",
+            "POST /{org_id}/oncall/responses/{response_id}/unsnooze",
             "POST /{org_id}/prometheus/api/v1/query",
             "POST /{org_id}/prometheus/api/v1/query_range",
             "POST /{org_id}/{stream_name}/_around",

src/api/http/src/handler/http/router/openapi.rs

@@ -1329,6 +1329,7 @@ pub struct ApiDoc;
     openobserve_api_management::request::oncall::snooze_response,
     openobserve_api_management::request::oncall::test_team_slack_channel,
     openobserve_api_management::request::oncall::unacknowledge_response,
+    openobserve_api_management::request::oncall::unsnooze_response,
     openobserve_api_management::request::oncall::update_ownership_rule,
     openobserve_api_management::request::oncall::update_team,
     openobserve_api_management::request::oncall::who_is_on_call,

src/api/management/src/request/oncall/mod.rs

@@ -3081,6 +3081,56 @@ pub async fn snooze_response(
     }
 }
 
+#[utoipa::path(
+    post,
+    path = "/{org_id}/oncall/responses/{response_id}/unsnooze",
+    context_path = "/api",
+    tag = "OnCall",
+    operation_id = "UnsnoozeOnCallResponse",
+    summary = "End a snooze now, so paging continues from the current step",
+    security(("Authorization" = [])),
+    params(
+        ("org_id" = String, Path, description = "Organization name"),
+        ("response_id" = String, Path, description = "Response record ID"),
+    ),
+    responses((status = 200, description = "Success", content_type = "application/json", body = Object)),
+    extensions(
+        ("x-o2-ratelimit" = json!({"module": "OnCall", "operation": "create"})),
+        ("x-o2-mcp" = json!({"enabled": false}))
+    )
+)]
+pub async fn unsnooze_response(
+    Path((org_id, response_id)): Path<(String, String)>,
+    #[cfg(feature = "enterprise")] Headers(user_email): Headers<UserEmail>,
+) -> Response {
+    #[cfg(feature = "enterprise")]
+    {
+        if !allowed(&org_id, &user_email.user_id, RESPONSES, "POST").await {
+            return MetaHttpResponse::forbidden("Forbidden");
+        }
+        if let Err(resp) = page_problem(&org_id, &response_id, &user_email.user_id).await {
+            return resp;
+        }
+        match o2_enterprise::enterprise::oncall::escalation::unsnooze(
+            &org_id,
+            &response_id,
+            &user_email.user_id,
+            config::utils::time::now_micros(),
+        )
+        .await
+        {
+            Ok(Some(r)) => MetaHttpResponse::json(r),
+            Ok(None) => MetaHttpResponse::not_found("Response record not found"),
+            Err(e) => to_response(e),
+        }
+    }
+    #[cfg(not(feature = "enterprise"))]
+    {
+        let _ = (org_id, response_id);
+        MetaHttpResponse::forbidden("Not Supported")
+    }
+}
+
 #[utoipa::path(
     post,
     path = "/{org_id}/oncall/responses/{response_id}/handoff",

src/config/src/meta/oncall/response.rs

@@ -78,6 +78,17 @@ pub fn snoozed_ladder_anchor(
     existing_anchor.unwrap_or(opened_at) + (until - quiet_reaches).max(0)
 }
 
+/// Where the ladder's clock sits when a snooze ends early at `now`: as if it had been that short.
+pub fn unsnoozed_ladder_anchor(
+    existing_anchor: Option<i64>,
+    opened_at: i64,
+    snoozed_until: Option<i64>,
+    now: i64,
+) -> i64 {
+    let unused = snoozed_until.map_or(0, |until| (until - now).max(0));
+    existing_anchor.unwrap_or(opened_at) - unused
+}
+
 /// Lifecycle of a response record: `triggered → triaged → acknowledged →
 /// resolved`. `triaged` comes from the L0 agent and is skipped entirely when
 /// the agent is absent or disabled.
@@ -1711,6 +1722,27 @@ mod tests {
                 17 * MIN
             );
         }
+
+        /// Ending a snooze early gives back the unused quiet, so the rungs keep their gaps from
+        /// where the ladder paused instead of waiting out the rest of the snooze.
+        #[test]
+        fn test_unsnoozing_leaves_the_clock_where_a_snooze_that_short_would() {
+            let snoozed = snoozed_ladder_anchor(None, 0, None, MIN, 61 * MIN);
+            assert_eq!(
+                unsnoozed_ladder_anchor(Some(snoozed), 0, Some(61 * MIN), 3 * MIN),
+                snoozed_ladder_anchor(None, 0, None, MIN, 3 * MIN),
+            );
+        }
+
+        /// A snooze that has already lapsed has no quiet left to give back.
+        #[test]
+        fn test_unsnoozing_a_lapsed_snooze_leaves_the_clock() {
+            let snoozed = snoozed_ladder_anchor(None, 0, None, MIN, 61 * MIN);
+            assert_eq!(
+                unsnoozed_ladder_anchor(Some(snoozed), 0, Some(61 * MIN), 90 * MIN),
+                snoozed
+            );
+        }
     }
 
     #[test]

src/core/src/alerts/rca_turn.rs

@@ -3362,12 +3362,12 @@ mod tests {
         }
     }
 
-    /// Paused time that moves 100x faster than the wall clock.
+    /// 10x time: auto-advance fires the DB acquire timeout; faster makes CI stalls false backoffs.
     fn fast_clock() -> tokio::task::JoinHandle<()> {
         tokio::spawn(async {
             loop {
                 std::thread::sleep(Duration::from_millis(1));
-                tokio::time::advance(Duration::from_millis(100)).await;
+                tokio::time::advance(Duration::from_millis(10)).await;
             }
         })
     }
@@ -3491,9 +3491,9 @@ mod tests {
         clock.abort();
     }
 
-    #[tokio::test(start_paused = true)]
+    // Real clock: even the 10x fast_clock turns a 500 ms CI stall into a false 5 s backoff.
+    #[tokio::test]
     async fn other_start_errors_fail_the_continuation_at_once() {
-        let clock = fast_clock();
         let _db = db().await;
         for (error, can_retry, code) in [
             (RcaTurnError::Busy, true, "busy"),
@@ -3543,7 +3543,6 @@ mod tests {
                 "{stored_error:?}"
             );
         }
-        clock.abort();
     }
 
     #[tokio::test]

src/infra/src/table/oncall_responses.rs

@@ -664,6 +664,36 @@ pub async fn snooze(
     .await
 }
 
+/// Ends a snooze at `now`, giving the ladder's clock back the quiet that was not used.
+pub async fn unsnooze(org_id: &str, id: &str, now: i64) -> Result<Option<Response>, errors::Error> {
+    let client = get_orm_client_rw().await;
+    let Some(existing) = oncall_responses::Entity::find_by_id(id)
+        .filter(oncall_responses::Column::OrgId.eq(org_id))
+        .one(client)
+        .await?
+    else {
+        return Ok(None);
+    };
+    let anchor = config::meta::oncall::response::unsnoozed_ladder_anchor(
+        existing.ladder_anchor,
+        existing.opened_at,
+        existing.snoozed_until,
+        now,
+    );
+    save(
+        org_id,
+        id,
+        vec![
+            (
+                oncall_responses::Column::SnoozedUntil,
+                Expr::value(Option::<i64>::None),
+            ),
+            (oncall_responses::Column::LadderAnchor, Expr::value(anchor)),
+        ],
+    )
+    .await
+}
+
 /// Writes a new severity onto an open record.
 ///
 /// The severity is what SLO reporting, digests and the post-incident review

web/src/locales/languages/en-US.json

@@ -25717,6 +25717,10 @@
     "snoozed": "Snoozed",
     "snoozedUntil": "Quiet until {time} — still unassigned",
     "snoozeFailed": "Could not snooze the page",
+    "unsnooze": "Unsnooze",
+    "unsnoozeHint": "Ends the snooze now. Paging continues from the step it paused on.",
+    "unsnoozed": "Snooze ended · paging continues",
+    "unsnoozeFailed": "Could not end the snooze",
     "note": "Note",
     "notePlaceholder": "What did you find? What did you try?",
     "addNote": "Add note",

web/src/services/oncall.queries.ts

@@ -721,6 +721,13 @@ export const snoozeResponseMutation = (org: string) =>
     meta: { invalidates: responseWriteScopes(org), silentError: true },
   });
 
+export const unsnoozeResponseMutation = (org: string) =>
+  mutationOptions({
+    mutationFn: (responseId: string) =>
+      oncallService.unsnoozeResponse({ org_identifier: org, response_id: responseId }),
+    meta: { invalidates: responseWriteScopes(org), silentError: true },
+  });
+
 export const resolveResponseMutation = (org: string) =>
   mutationOptions({
     mutationFn: ({

web/src/services/oncall.ts

@@ -511,6 +511,18 @@ const oncall = {
       { minutes },
     ),
 
+  /// Ends a snooze now; the ladder continues from the step it paused on.
+  unsnoozeResponse: ({
+    org_identifier,
+    response_id,
+  }: {
+    org_identifier: string;
+    response_id: string;
+  }) =>
+    http().post<OnCallResponse>(
+      `/api/${org_identifier}/oncall/responses/${encodeURIComponent(response_id)}/unsnooze`,
+    ),
+
   addNote: ({
     org_identifier,
     response_id,

web/src/views/OnCall/OnCallResponseDetail.spec.ts

@@ -39,6 +39,7 @@ vi.mock("@/services/oncall", () => ({
     unacknowledgeResponse: vi.fn(),
     confirmRecovery: vi.fn(),
     snoozeResponse: vi.fn(),
+    unsnoozeResponse: vi.fn(),
     addNote: vi.fn(),
     handoffResponse: vi.fn(),
     resolveResponse: vi.fn(),
@@ -661,6 +662,29 @@ describe("OnCallResponseDetail", () => {
     expect(lapsed.find('[data-test="oncall-response-snoozed-banner"]').exists()).toBe(false);
   });
 
+  /// Snooze means paused, then continues; while it is paused the same control continues it now.
+  it("swaps Snooze for Unsnooze while snoozed, and the banner offers it too", async () => {
+    const wrapper = await renderWith({ snoozed_until: (Date.now() + 30 * 60_000) * 1000 });
+    service.unsnoozeResponse.mockResolvedValue({ data: {} } as any);
+
+    expect(wrapper.find('[data-test="oncall-response-snooze-btn"]').exists()).toBe(false);
+    for (const button of [
+      "oncall-response-unsnooze-btn",
+      "oncall-response-snoozed-banner-unsnooze",
+    ]) {
+      service.unsnoozeResponse.mockClear();
+      toastSpy.mockClear();
+      await wrapper.find(`[data-test="${button}"]`).trigger("click");
+      await flushPromises();
+      expect(service.unsnoozeResponse).toHaveBeenCalledWith(
+        expect.objectContaining({ response_id: "resp_1" }),
+      );
+      expect(toastSpy).toHaveBeenCalledWith(
+        expect.objectContaining({ message: "Snooze ended · paging continues" }),
+      );
+    }
+  });
+
   it("adds a note and clears the box", async () => {
     const wrapper = await renderWith();
     service.addNote.mockResolvedValue({ data: {} } as any);
@@ -1586,6 +1610,19 @@ describe("OnCallResponseDetail — what the payload already knew", () => {
       );
     });
 
+    it("unsnoozes from the menu while snoozed", async () => {
+      const wrapper = await renderWith({ snoozed_until: (Date.now() + 30 * 60_000) * 1000 });
+      service.unsnoozeResponse.mockResolvedValue({ data: {} } as any);
+
+      expect(wrapper.find('[data-test="oncall-response-snooze-30-menu"]').exists()).toBe(false);
+      await wrapper.find('[data-test="oncall-response-unsnooze-btn-menu"]').trigger("click");
+      await flushPromises();
+
+      expect(service.unsnoozeResponse).toHaveBeenCalledWith(
+        expect.objectContaining({ response_id: "resp_1" }),
+      );
+    });
+
     it("opens the hand-off drawer from the menu", async () => {
       const wrapper = await renderWith();
 

web/src/views/OnCall/OnCallResponseDetail.vue

@@ -146,7 +146,18 @@
         </template>
 
         <!-- A menu of durations, not a panel that pushes the page down. -->
-        <ODropdown v-if="canSnooze && lgUp">
+        <OButton
+          v-if="canUnsnooze && lgUp"
+          variant="outline"
+          size="sm-action"
+          :loading="snoozing"
+          data-test="oncall-response-unsnooze-btn"
+          @click="unsnoozeRecord"
+        >
+          {{ t("oncall.unsnooze") }}
+          <OTooltip :content="t('oncall.unsnoozeHint')" />
+        </OButton>
+        <ODropdown v-else-if="canSnooze && lgUp">
           <template #trigger>
             <OButton
               variant="outline"
@@ -253,7 +264,14 @@
             data-test="oncall-response-more-btn"
           />
         </template>
-        <ODropdownGroup v-if="canSnooze" :label="t('oncall.snoozeFor')">
+        <ODropdownItem
+          v-if="canUnsnooze"
+          data-test="oncall-response-unsnooze-btn-menu"
+          @select="unsnoozeRecord"
+        >
+          {{ t("oncall.unsnooze") }}
+        </ODropdownItem>
+        <ODropdownGroup v-else-if="canSnooze" :label="t('oncall.snoozeFor')">
           <ODropdownItem
             v-for="opt in snoozeOptions"
             :key="opt.minutes"
@@ -364,6 +382,17 @@
           data-test="oncall-response-snoozed-banner"
         >
           {{ t("oncall.snoozedUntil", { time: snoozedUntilLabel }) }}
+          <template v-if="canUnsnooze" #actions>
+            <OButton
+              variant="outline"
+              size="sm"
+              :loading="snoozing"
+              data-test="oncall-response-snoozed-banner-unsnooze"
+              @click="unsnoozeRecord"
+            >
+              {{ t("oncall.unsnooze") }}
+            </OButton>
+          </template>
         </OBanner>
 
         <!-- D-21. This record exists because somebody else's failure reaches
@@ -847,6 +876,7 @@ import {
   responseQuery,
   responseReportQuery,
   snoozeResponseMutation,
+  unsnoozeResponseMutation,
   teamMembersQuery,
   teamPolicyQuery,
   teamReachabilityQuery,
@@ -1088,6 +1118,7 @@ const ackWords = computed(() =>
   ),
 );
 const snoozeWrite = useMutation(() => snoozeResponseMutation(orgId.value));
+const unsnoozeWrite = useMutation(() => unsnoozeResponseMutation(orgId.value));
 const noteWrite = useMutation(() => addResponseNoteMutation(orgId.value));
 const handoffWrite = useMutation(() => handoffResponseMutation(orgId.value));
 const recoveryWrite = useMutation(() => confirmRecoveryMutation(orgId.value));
@@ -1299,6 +1330,7 @@ const snoozedUntilLabel = computed(() => {
   if (!r || !isSnoozed(r)) return "";
   return formatOnCallInstant(r.snoozed_until as number, nowMicros.value, store.state.timezone);
 });
+const canUnsnooze = computed(() => canSnooze.value && !!snoozedUntilLabel.value);
 
 const timeToAck = computed(() => {
   const r = response.value;
@@ -1519,6 +1551,22 @@ async function snoozeRecord(minutes: number) {
   }
 }
 
+async function unsnoozeRecord() {
+  snoozing.value = true;
+  try {
+    await unsnoozeWrite.mutateAsync(responseId.value);
+    toast({ variant: "success", message: t("oncall.unsnoozed") });
+    await fetchResponse();
+  } catch (err: any) {
+    toast({
+      variant: "error",
+      message: raw(err?.response?.data?.message) || t("oncall.unsnoozeFailed"),
+    });
+  } finally {
+    snoozing.value = false;
+  }
+}
+
 async function addNote() {
   const body = noteBody.value.trim();
   if (!body) return;

web/src/views/OnCall/OnCallResponses.spec.ts

@@ -43,6 +43,7 @@ vi.mock("@/services/oncall", () => ({
     acknowledgeResponse: vi.fn(),
     resolveResponse: vi.fn(),
     snoozeResponse: vi.fn(),
+    unsnoozeResponse: vi.fn(),
     getPolicy: vi.fn(),
     getSchedule: vi.fn(),
     escalationProgress: vi.fn(),
@@ -860,6 +861,29 @@ describe("OnCallResponses", () => {
       expect(handled.findAll('[data-test^="oncall-row-resolve-"]')).toHaveLength(1);
     });
 
+    it("offers unsnooze instead of snooze on a snoozed row", async () => {
+      const wrapper = await withPages([page({ snoozed_until: (Date.now() + 60_000) * 1000 })]);
+      service.unsnoozeResponse.mockResolvedValue({ data: {} } as any);
+
+      expect(wrapper.find('[data-test^="oncall-row-snooze-"]').exists()).toBe(false);
+      await wrapper.find('[data-test^="oncall-row-unsnooze-"]').trigger("click");
+      await flushPromises();
+
+      expect(service.unsnoozeResponse).toHaveBeenCalledWith(
+        expect.objectContaining({ response_id: "resp_1" }),
+      );
+    });
+
+    it("keeps snooze for the awake firing of a row whose other firing is snoozed", async () => {
+      const wrapper = await withPages([
+        page({ id: "a", snoozed_until: (Date.now() + 60_000) * 1000 }),
+        page({ id: "b" }),
+      ]);
+
+      expect(wrapper.find('[data-test^="oncall-row-unsnooze-"]').exists()).toBe(true);
+      expect(wrapper.find('[data-test^="oncall-row-snooze-"]').exists()).toBe(true);
+    });
+
     /// Snoozing only makes sense while the ladder is still climbing, so a
     /// handled row's menu must not offer it.
     it("offers no snooze once nothing is escalating", async () => {

web/src/views/OnCall/OnCallResponses.vue

@@ -815,7 +815,20 @@ along with this program.  If not, see <http://www.gnu.org/licenses/>.
 
               <!-- Grouped rather than four loose items: they are one decision
                    ("quiet it") asked at four lengths. -->
-              <ODropdownGroup v-if="canAcknowledge(row)" :label="t('oncall.snoozeFor')">
+              <ODropdownItem
+                v-if="canAcknowledge(row) && snoozedFirings(row).length"
+                :data-test="`oncall-row-unsnooze-${row.rowKey}`"
+                @select="unsnoozeRow(row)"
+              >
+                <template #icon-left>
+                  <OIcon name="play-circle-filled" size="sm" />
+                </template>
+                {{ t("oncall.unsnooze") }}
+              </ODropdownItem>
+              <ODropdownGroup
+                v-if="canAcknowledge(row) && snoozedFirings(row).length < row.escalating.length"
+                :label="t('oncall.snoozeFor')"
+              >
                 <ODropdownItem
                   v-for="option in snoozeOptions"
                   :key="option.minutes"
@@ -971,6 +984,7 @@ import {
   responseProgressQuery,
   responseQuery,
   snoozeResponseMutation,
+  unsnoozeResponseMutation,
   teamPolicyQuery,
   teamScheduleQuery,
   whoIsOnCallQuery,
@@ -1624,6 +1638,7 @@ function menuActions(row: PageRow): RowAction[] {
 /// One list writes to many rows, so the id rides in the variables, not the setup.
 const ackWrite = useMutation(() => acknowledgeResponseMutation(orgId.value));
 const snoozeWrite = useMutation(() => snoozeResponseMutation(orgId.value));
+const unsnoozeWrite = useMutation(() => unsnoozeResponseMutation(orgId.value));
 const resolveWrite = useMutation(() => resolveResponseMutation(orgId.value));
 const handoffWrite = useMutation(() => handoffResponseMutation(orgId.value));
 const noteWrite = useMutation(() => addResponseNoteMutation(orgId.value));
@@ -1657,6 +1672,28 @@ async function snoozeRow(row: PageRow, minutes: number) {
   }
 }
 
+function snoozedFirings(row: PageRow): OnCallResponse[] {
+  return row.escalating.filter((r) => isSnoozed(r));
+}
+
+/// Ends the snooze on every quiet firing the row stands for, so they all continue now.
+async function unsnoozeRow(row: PageRow) {
+  busyId.value = row.rowKey;
+  try {
+    const results = await Promise.allSettled(
+      snoozedFirings(row).map((r) => unsnoozeWrite.mutateAsync(r.id)),
+    );
+    if (results.some((r) => r.status === "rejected")) {
+      toast({ variant: "error", message: t("oncall.unsnoozeFailed") });
+    } else {
+      toast({ variant: "success", message: t("oncall.unsnoozed") });
+    }
+    await fetchResponses();
+  } finally {
+    busyId.value = "";
+  }
+}
+
 async function resolveRow(row: PageRow) {
   busyId.value = row.rowKey;
   try {