From fbe35aebf097429fe6d5cc30223c2d889f21ef2f Mon Sep 17 00:00:00 2001 From: Loakesh Indiran <66478092+Loaki07@users.noreply.github.com> Date: Tue, 13 Oct 2026 02:25:17 +0000 Subject: [PATCH] feat: alert downtimes with muted incidents and SLO corrections (#14875) Co-authored-by: github-actions[bot] <41898282+github-actions[bot]@users.noreply.github.com> --- scripts/translations/.translation_state.json | 533 +++ src/api/common/src/auth/validator.rs | 1 + src/api/http/src/handler/http/router/mod.rs | 21 +- .../http/src/handler/http/router/openapi.rs | 81 + .../management/src/models/alerts/responses.rs | 190 +- src/api/management/src/models/destinations.rs | 7 +- src/api/management/src/models/folders.rs | 11 + .../management/src/request/alerts/history.rs | 245 +- .../src/request/alerts/incidents.rs | 1 + src/api/management/src/request/alerts/mod.rs | 13 +- .../src/request/announcements/mod.rs | 17 +- .../src/request/anomaly_detection/mod.rs | 55 +- src/api/management/src/request/downtimes.rs | 576 ++++ src/api/management/src/request/folders.rs | 68 +- src/api/management/src/request/mod.rs | 1 + src/api/management/src/request/status/mod.rs | 41 +- .../src/request/synthetics/last_check.rs | 1 + .../management/src/request/synthetics/mod.rs | 15 +- src/common/src/meta/organization.rs | 3 +- src/config/src/config.rs | 4 +- src/config/src/meta/alerts/dispatch.rs | 159 +- src/config/src/meta/alerts/incidents.rs | 18 + src/config/src/meta/alerts/level.rs | 20 +- src/config/src/meta/alerts/mod.rs | 5 + src/config/src/meta/downtimes.rs | 942 ++++++ src/config/src/meta/folder.rs | 1 + src/config/src/meta/mod.rs | 1 + src/config/src/meta/oncall/routing.rs | 2 +- src/config/src/meta/self_reporting/mod.rs | 4 + src/config/src/meta/self_reporting/usage.rs | 40 + src/config/src/meta/slo/coverage.rs | 11 +- src/config/src/meta/slo/slice.rs | 3 + src/config/src/meta/slo/status_view.rs | 4 + src/config/src/meta/slo/stream.rs | 8 +- src/config/src/meta/synthetics.rs | 8 +- src/config/src/metrics/base.rs | 17 + src/core/src/alerts/alert.rs | 155 +- src/core/src/alerts/destination_usage.rs | 132 + src/core/src/alerts/downtimes.rs | 1176 +++++++ src/core/src/alerts/grouping.rs | 335 +- src/core/src/alerts/incidents.rs | 986 +++++- src/core/src/alerts/mod.rs | 1 + .../alerts/notifications/default_template.rs | 1 + src/core/src/alerts/prom_rule.rs | 1 + src/core/src/alerts/recovery.rs | 493 ++- src/core/src/alerts/scheduler/handlers.rs | 1844 ++++++++-- src/core/src/alerts/templates.rs | 175 +- src/core/src/anomaly_detection.rs | 47 + src/core/src/downtimes.rs | 3001 +++++++++++++++++ src/core/src/downtimes/access.rs | 390 +++ src/core/src/downtimes/inventory.rs | 294 ++ src/core/src/downtimes/listing.rs | 329 ++ src/core/src/downtimes/matching.rs | 469 +++ src/core/src/downtimes/notify.rs | 577 ++++ src/core/src/downtimes/resources.rs | 268 ++ src/core/src/downtimes/values.rs | 1253 +++++++ src/core/src/ingestion/mod.rs | 351 +- src/core/src/lib.rs | 2 + src/core/src/ofga/mod.rs | 167 +- src/core/src/org_cleanup.rs | 17 + .../src/self_reporting/triggers_schema.rs | 7 +- src/core/src/slo/backfill.rs | 1022 +++++- src/core/src/slo/corrections.rs | 1059 ++++++ src/core/src/slo/evaluate.rs | 72 +- src/core/src/slo/ingest.rs | 67 +- src/core/src/slo/job.rs | 644 +++- src/core/src/slo/mod.rs | 1 + src/core/src/slo/reconcile.rs | 214 +- src/core/src/slo/service.rs | 63 +- src/core/src/synthetics.rs | 325 +- src/db/src/alerts/alert.rs | 16 +- src/db/src/authz.rs | 19 + src/db/src/downtimes.rs | 486 +++ src/db/src/folders.rs | 47 +- src/db/src/lib.rs | 1 + src/infra/src/coordinator/downtimes.rs | 112 + src/infra/src/coordinator/mod.rs | 1 + src/infra/src/table/alert_incidents.rs | 437 ++- src/infra/src/table/alert_states.rs | 243 ++ src/infra/src/table/downtime_notifications.rs | 293 ++ src/infra/src/table/downtimes.rs | 1711 ++++++++++ src/infra/src/table/entity/alert_incidents.rs | 5 + src/infra/src/table/entity/alert_states.rs | 2 + .../table/entity/downtime_notifications.rs | 22 + src/infra/src/table/entity/downtimes.rs | 40 + src/infra/src/table/entity/mod.rs | 2 + .../src/table/entity/slo_backfill_jobs.rs | 3 + src/infra/src/table/folders.rs | 45 +- src/infra/src/table/incident_events.rs | 10 +- .../m20261009_000001_create_downtimes.rs | 292 ++ ...muted_by_downtime_id_to_alert_incidents.rs | 96 + ...03_add_last_downtime_id_to_alert_states.rs | 96 + ...09_000004_add_kind_to_slo_backfill_jobs.rs | 142 + ...09_000005_create_downtime_notifications.rs | 207 ++ src/infra/src/table/migration/mod.rs | 49 + src/infra/src/table/mod.rs | 2 + src/infra/src/table/slo.rs | 88 +- src/infra/src/table/slo_backfill_jobs.rs | 512 ++- src/infra/src/table/slos.rs | 27 +- src/infra/src/table/synthetics_locations.rs | 28 +- src/jobs/src/job/downtimes.rs | 577 ++++ src/jobs/src/job/incidents.rs | 61 +- src/jobs/src/job/mod.rs | 85 + src/super_cluster_queue/src/downtimes.rs | 1250 +++++++ src/super_cluster_queue/src/folders.rs | 138 +- src/super_cluster_queue/src/incidents.rs | 28 +- src/super_cluster_queue/src/lib.rs | 16 +- src/synthetics/Cargo.toml | 2 + src/synthetics/src/alerting.rs | 132 + src/synthetics/src/job_api.rs | 31 +- src/synthetics/src/service/checks.rs | 1 + src/synthetics/src/status_pages.rs | 56 +- src/usage_reporting/src/lib.rs | 1 + web/src/components/ConfirmDialog.vue | 6 +- web/src/components/alerts/AlertList.spec.ts | 19 + web/src/components/alerts/AlertList.vue | 88 +- .../alerts/FilterCondition.suggest.spec.ts | 128 + web/src/components/alerts/FilterCondition.vue | 51 +- web/src/components/alerts/ImportAlert.spec.ts | 21 + web/src/components/alerts/ImportAlert.vue | 30 +- .../alerts/IncidentDetailDrawer.vue | 56 + .../components/alerts/IncidentList.spec.ts | 60 + web/src/components/alerts/IncidentList.vue | 36 +- .../components/alerts/conditionOperators.ts | 76 + .../downtimes/AddDowntime.schema.spec.ts | 245 ++ .../alerts/downtimes/AddDowntime.schema.ts | 352 ++ .../alerts/downtimes/AddDowntime.spec.ts | 455 +++ .../alerts/downtimes/AddDowntime.vue | 850 +++++ .../downtimes/DowntimeConditionSection.vue | 199 ++ .../downtimes/DowntimeCountdown.spec.ts | 24 + .../alerts/downtimes/DowntimeCountdown.vue | 47 + .../DowntimeDetail.notifications.spec.ts | 194 ++ .../alerts/downtimes/DowntimeDetail.spec.ts | 261 ++ .../alerts/downtimes/DowntimeDetail.vue | 777 +++++ .../downtimes/DowntimeList.actions.spec.ts | 214 ++ .../alerts/downtimes/DowntimeList.spec.ts | 314 ++ .../alerts/downtimes/DowntimeList.vue | 1102 ++++++ .../downtimes/DowntimeNotifySection.vue | 138 + .../alerts/downtimes/DowntimePreviewPane.vue | 160 + .../downtimes/DowntimeResourcePicker.spec.ts | 152 + .../downtimes/DowntimeResourcePicker.vue | 316 ++ .../alerts/downtimes/DowntimeScheduleBand.vue | 128 + .../downtimes/DowntimeScheduleFields.spec.ts | 48 + .../downtimes/DowntimeScheduleFields.vue | 188 ++ .../alerts/downtimes/DowntimeSummary.vue | 102 + .../downtimes/DowntimeTargetCard.spec.ts | 128 + .../alerts/downtimes/DowntimeTargetCard.vue | 355 ++ .../alerts/downtimes/DowntimeTargetsCell.vue | 111 + .../downtimes/ExtendDowntimeDialog.schema.ts | 64 + .../downtimes/ExtendDowntimeDialog.spec.ts | 158 + .../alerts/downtimes/ExtendDowntimeDialog.vue | 133 + .../downtimes/ExtendDowntimeMenu.spec.ts | 65 + .../alerts/downtimes/ExtendDowntimeMenu.vue | 77 + .../alerts/downtimes/MuteMenuItems.vue | 60 + .../alerts/downtimes/MutedChip.spec.ts | 66 + .../components/alerts/downtimes/MutedChip.vue | 68 + .../downtimes/QuickMuteDialog.schema.ts | 79 + .../alerts/downtimes/QuickMuteDialog.spec.ts | 153 + .../alerts/downtimes/QuickMuteDialog.vue | 218 ++ .../alerts/steps/QueryConfig.spec.ts | 53 +- .../components/alerts/steps/QueryConfig.vue | 34 +- .../announcements/AnnouncementBanner.spec.ts | 111 + .../announcements/AnnouncementBanner.vue | 67 +- .../InlineSelectFolderDropdown.spec.ts | 34 +- .../sidebar/InlineSelectFolderDropdown.vue | 51 +- .../common/sidebar/MoveAcrossFolders.vue | 2 + .../flow/forms/ConditionBuilder.vue | 11 +- web/src/components/iam/roles/AppRoles.spec.ts | 8 + web/src/components/iam/roles/roleModules.ts | 4 + .../components/reports/CreateReport.spec.ts | 32 + web/src/components/reports/CreateReport.vue | 29 +- .../components/slos/SloBurndownChart.spec.ts | 46 + web/src/components/slos/SloBurndownChart.vue | 60 +- .../synthetic-monitoring/MonitorTable.vue | 51 + .../configure/CheckSchedule.spec.ts | 28 + .../synthetics/configure/CheckSchedule.vue | 72 +- .../alerts/useDependencyGraph.spec.ts | 16 + .../composables/alerts/useDependencyGraph.ts | 5 +- .../downtimes/useCountdown.spec.ts | 63 + web/src/composables/downtimes/useCountdown.ts | 51 + .../downtimes/useDefaultDowntimeFolder.ts | 75 + .../composables/downtimes/useDowntimeItems.ts | 86 + .../downtimes/useDowntimeLookup.spec.ts | 74 + .../downtimes/useDowntimeLookup.ts | 59 + .../downtimes/useDowntimeValueSuggest.spec.ts | 102 + .../downtimes/useDowntimeValueSuggest.ts | 75 + .../downtimes/useDowntimesEnabled.spec.ts | 118 + .../downtimes/useDowntimesEnabled.ts | 28 + .../downtimes/useExtendDowntime.spec.ts | 102 + .../downtimes/useExtendDowntime.ts | 96 + .../downtimes/useListBoundaryRefetch.spec.ts | 174 + .../downtimes/useListBoundaryRefetch.ts | 84 + .../downtimes/useNowMicros.spec.ts | 49 + web/src/composables/downtimes/useNowMicros.ts | 45 + web/src/composables/downtimes/useQuickMute.ts | 98 + web/src/composables/downtimes/useRowMute.ts | 44 + .../downtimes/useViewerTimezone.spec.ts | 94 + .../downtimes/useViewerTimezone.ts | 38 + .../iam/useRoleEntityLoaders.spec.ts | 44 + .../composables/iam/useRoleEntityLoaders.ts | 43 + web/src/composables/iam/useRoleJsonView.ts | 8 + web/src/composables/iam/useRoleScopes.spec.ts | 19 + web/src/composables/iam/useRoleScopes.ts | 11 +- .../composables/iam/useSavedGrantExpansion.ts | 23 + web/src/composables/shared/router.ts | 60 + .../useAnnouncementBanners.spec.ts | 108 +- web/src/composables/useAnnouncementBanners.ts | 77 +- .../composables/useTimezoneOptions.spec.ts | 88 + web/src/composables/useTimezoneOptions.ts | 75 + web/src/layouts/MainLayout.spec.ts | 1 + web/src/layouts/MainLayout.vue | 25 + web/src/lib/core/Badge/badgeGroups.spec.ts | 13 + web/src/lib/core/Badge/badgeGroups.ts | 32 + web/src/lib/core/EmptyState/presets.ts | 14 + web/src/lib/core/Icon/OIcon.icons.ts | 6 + web/src/lib/core/Navbar/ONavGroup.spec.ts | 12 + web/src/lib/core/Navbar/ONavbar.spec.ts | 4 +- web/src/lib/core/Navbar/ONavbar.types.ts | 2 + web/src/lib/core/Navbar/navGroups.spec.ts | 33 + web/src/lib/core/Navbar/navGroups.ts | 13 +- web/src/lib/core/Navbar/useNavGateContext.ts | 2 + web/src/lib/feedback/Banner/OBanner.spec.ts | 21 + web/src/lib/feedback/Banner/OBanner.vue | 9 + .../lib/forms/InlineEdit/OFormInlineEdit.vue | 3 +- .../lib/forms/InlineEdit/OInlineEdit.spec.ts | 21 + .../lib/forms/InlineEdit/OInlineEdit.types.ts | 2 + web/src/lib/forms/InlineEdit/OInlineEdit.vue | 7 +- .../forms/Select/OSelect.searchText.spec.ts | 76 + web/src/lib/forms/Select/OSelect.types.ts | 2 + web/src/lib/forms/Select/OSelect.vue | 11 +- web/src/lib/overlay/Dialog/ODialog.types.ts | 8 +- web/src/lib/overlay/Dialog/ODialog.vue | 4 +- .../ShortcutCheatsheet.vue | 1 + .../__tests__/shortcutRegistry.spec.ts | 14 + .../vue-shortcut-manager/shortcutRegistry.ts | 15 + web/src/locales/languages/ar-SA.json | 831 ++++- web/src/locales/languages/de-DE.json | 831 ++++- web/src/locales/languages/en-US.json | 835 ++++- web/src/locales/languages/es-ES.json | 831 ++++- web/src/locales/languages/fr-FR.json | 831 ++++- web/src/locales/languages/it-IT.json | 831 ++++- web/src/locales/languages/ja-JP.json | 831 ++++- web/src/locales/languages/ko-KR.json | 831 ++++- web/src/locales/languages/nl-NL.json | 831 ++++- web/src/locales/languages/pl-PL.json | 831 ++++- web/src/locales/languages/pt-PT.json | 831 ++++- web/src/locales/languages/ru-RU.json | 831 ++++- web/src/locales/languages/tr-TR.json | 831 ++++- web/src/locales/languages/vi-VN.json | 831 ++++- web/src/locales/languages/zh-CN.json | 831 ++++- web/src/locales/languages/zh-TW.json | 831 ++++- web/src/services/alerts.ts | 4 + web/src/services/announcements.querykeys.ts | 22 + web/src/services/common.queries.spec.ts | 54 + web/src/services/common.queries.ts | 31 + web/src/services/common.querykeys.ts | 2 + web/src/services/common.ts | 6 +- web/src/services/downtimes.queries.spec.ts | 48 + web/src/services/downtimes.queries.ts | 295 ++ web/src/services/downtimes.querykeys.ts | 29 + web/src/services/downtimes.ts | 290 ++ web/src/services/http.spec.ts | 48 + web/src/services/http.ts | 3 + web/src/services/incidents.ts | 2 + web/src/ts/interfaces/slo.ts | 4 + web/src/utils/downtimes/banner.spec.ts | 59 + web/src/utils/downtimes/banner.ts | 71 + .../utils/downtimes/conditionBridge.spec.ts | 182 + web/src/utils/downtimes/conditionBridge.ts | 194 ++ web/src/utils/downtimes/conditionRules.ts | 61 + web/src/utils/downtimes/downtimeForm.spec.ts | 416 +++ web/src/utils/downtimes/downtimeForm.ts | 496 +++ web/src/utils/downtimes/extend.ts | 36 + web/src/utils/downtimes/folderDefault.spec.ts | 48 + web/src/utils/downtimes/folderDefault.ts | 51 + web/src/utils/downtimes/listOrder.spec.ts | 48 + web/src/utils/downtimes/listOrder.ts | 64 + web/src/utils/downtimes/notifyTest.spec.ts | 122 + web/src/utils/downtimes/notifyTest.ts | 129 + web/src/utils/downtimes/quickMute.ts | 86 + web/src/utils/downtimes/schedule.spec.ts | 317 ++ web/src/utils/downtimes/schedule.ts | 263 ++ web/src/utils/downtimes/summary.spec.ts | 151 + web/src/utils/downtimes/summary.ts | 166 + web/src/utils/downtimes/suppressed.spec.ts | 47 + web/src/utils/downtimes/suppressed.ts | 50 + web/src/utils/downtimes/targetSummary.spec.ts | 116 + web/src/utils/downtimes/targetSummary.ts | 135 + web/src/utils/slos/burndownQuery.spec.ts | 45 + web/src/utils/slos/burndownQuery.ts | 48 +- web/src/utils/timezoneAliases.spec.ts | 41 + web/src/utils/timezoneAliases.ts | 70 + web/src/views/SyntheticMonitoring.vue | 3 + web/src/views/slos/SloDetail.vue | 49 + .../views/slos/SloDetailCorrections.spec.ts | 161 + web/src/views/slos/SloList.vue | 51 + 296 files changed, 52536 insertions(+), 3603 deletions(-) create mode 100644 src/api/management/src/request/downtimes.rs create mode 100644 src/config/src/meta/downtimes.rs create mode 100644 src/core/src/alerts/downtimes.rs create mode 100644 src/core/src/downtimes.rs create mode 100644 src/core/src/downtimes/access.rs create mode 100644 src/core/src/downtimes/inventory.rs create mode 100644 src/core/src/downtimes/listing.rs create mode 100644 src/core/src/downtimes/matching.rs create mode 100644 src/core/src/downtimes/notify.rs create mode 100644 src/core/src/downtimes/resources.rs create mode 100644 src/core/src/downtimes/values.rs create mode 100644 src/core/src/slo/corrections.rs create mode 100644 src/db/src/downtimes.rs create mode 100644 src/infra/src/coordinator/downtimes.rs create mode 100644 src/infra/src/table/downtime_notifications.rs create mode 100644 src/infra/src/table/downtimes.rs create mode 100644 src/infra/src/table/entity/downtime_notifications.rs create mode 100644 src/infra/src/table/entity/downtimes.rs create mode 100644 src/infra/src/table/migration/m20261009_000001_create_downtimes.rs create mode 100644 src/infra/src/table/migration/m20261009_000002_add_muted_by_downtime_id_to_alert_incidents.rs create mode 100644 src/infra/src/table/migration/m20261009_000003_add_last_downtime_id_to_alert_states.rs create mode 100644 src/infra/src/table/migration/m20261009_000004_add_kind_to_slo_backfill_jobs.rs create mode 100644 src/infra/src/table/migration/m20261009_000005_create_downtime_notifications.rs create mode 100644 src/jobs/src/job/downtimes.rs create mode 100644 src/super_cluster_queue/src/downtimes.rs create mode 100644 web/src/components/alerts/FilterCondition.suggest.spec.ts create mode 100644 web/src/components/alerts/conditionOperators.ts create mode 100644 web/src/components/alerts/downtimes/AddDowntime.schema.spec.ts create mode 100644 web/src/components/alerts/downtimes/AddDowntime.schema.ts create mode 100644 web/src/components/alerts/downtimes/AddDowntime.spec.ts create mode 100644 web/src/components/alerts/downtimes/AddDowntime.vue create mode 100644 web/src/components/alerts/downtimes/DowntimeConditionSection.vue create mode 100644 web/src/components/alerts/downtimes/DowntimeCountdown.spec.ts create mode 100644 web/src/components/alerts/downtimes/DowntimeCountdown.vue create mode 100644 web/src/components/alerts/downtimes/DowntimeDetail.notifications.spec.ts create mode 100644 web/src/components/alerts/downtimes/DowntimeDetail.spec.ts create mode 100644 web/src/components/alerts/downtimes/DowntimeDetail.vue create mode 100644 web/src/components/alerts/downtimes/DowntimeList.actions.spec.ts create mode 100644 web/src/components/alerts/downtimes/DowntimeList.spec.ts create mode 100644 web/src/components/alerts/downtimes/DowntimeList.vue create mode 100644 web/src/components/alerts/downtimes/DowntimeNotifySection.vue create mode 100644 web/src/components/alerts/downtimes/DowntimePreviewPane.vue create mode 100644 web/src/components/alerts/downtimes/DowntimeResourcePicker.spec.ts create mode 100644 web/src/components/alerts/downtimes/DowntimeResourcePicker.vue create mode 100644 web/src/components/alerts/downtimes/DowntimeScheduleBand.vue create mode 100644 web/src/components/alerts/downtimes/DowntimeScheduleFields.spec.ts create mode 100644 web/src/components/alerts/downtimes/DowntimeScheduleFields.vue create mode 100644 web/src/components/alerts/downtimes/DowntimeSummary.vue create mode 100644 web/src/components/alerts/downtimes/DowntimeTargetCard.spec.ts create mode 100644 web/src/components/alerts/downtimes/DowntimeTargetCard.vue create mode 100644 web/src/components/alerts/downtimes/DowntimeTargetsCell.vue create mode 100644 web/src/components/alerts/downtimes/ExtendDowntimeDialog.schema.ts create mode 100644 web/src/components/alerts/downtimes/ExtendDowntimeDialog.spec.ts create mode 100644 web/src/components/alerts/downtimes/ExtendDowntimeDialog.vue create mode 100644 web/src/components/alerts/downtimes/ExtendDowntimeMenu.spec.ts create mode 100644 web/src/components/alerts/downtimes/ExtendDowntimeMenu.vue create mode 100644 web/src/components/alerts/downtimes/MuteMenuItems.vue create mode 100644 web/src/components/alerts/downtimes/MutedChip.spec.ts create mode 100644 web/src/components/alerts/downtimes/MutedChip.vue create mode 100644 web/src/components/alerts/downtimes/QuickMuteDialog.schema.ts create mode 100644 web/src/components/alerts/downtimes/QuickMuteDialog.spec.ts create mode 100644 web/src/components/alerts/downtimes/QuickMuteDialog.vue create mode 100644 web/src/components/announcements/AnnouncementBanner.spec.ts create mode 100644 web/src/composables/downtimes/useCountdown.spec.ts create mode 100644 web/src/composables/downtimes/useCountdown.ts create mode 100644 web/src/composables/downtimes/useDefaultDowntimeFolder.ts create mode 100644 web/src/composables/downtimes/useDowntimeItems.ts create mode 100644 web/src/composables/downtimes/useDowntimeLookup.spec.ts create mode 100644 web/src/composables/downtimes/useDowntimeLookup.ts create mode 100644 web/src/composables/downtimes/useDowntimeValueSuggest.spec.ts create mode 100644 web/src/composables/downtimes/useDowntimeValueSuggest.ts create mode 100644 web/src/composables/downtimes/useDowntimesEnabled.spec.ts create mode 100644 web/src/composables/downtimes/useDowntimesEnabled.ts create mode 100644 web/src/composables/downtimes/useExtendDowntime.spec.ts create mode 100644 web/src/composables/downtimes/useExtendDowntime.ts create mode 100644 web/src/composables/downtimes/useListBoundaryRefetch.spec.ts create mode 100644 web/src/composables/downtimes/useListBoundaryRefetch.ts create mode 100644 web/src/composables/downtimes/useNowMicros.spec.ts create mode 100644 web/src/composables/downtimes/useNowMicros.ts create mode 100644 web/src/composables/downtimes/useQuickMute.ts create mode 100644 web/src/composables/downtimes/useRowMute.ts create mode 100644 web/src/composables/downtimes/useViewerTimezone.spec.ts create mode 100644 web/src/composables/downtimes/useViewerTimezone.ts create mode 100644 web/src/composables/useTimezoneOptions.spec.ts create mode 100644 web/src/composables/useTimezoneOptions.ts create mode 100644 web/src/lib/forms/Select/OSelect.searchText.spec.ts create mode 100644 web/src/services/announcements.querykeys.ts create mode 100644 web/src/services/common.queries.spec.ts create mode 100644 web/src/services/downtimes.queries.spec.ts create mode 100644 web/src/services/downtimes.queries.ts create mode 100644 web/src/services/downtimes.querykeys.ts create mode 100644 web/src/services/downtimes.ts create mode 100644 web/src/utils/downtimes/banner.spec.ts create mode 100644 web/src/utils/downtimes/banner.ts create mode 100644 web/src/utils/downtimes/conditionBridge.spec.ts create mode 100644 web/src/utils/downtimes/conditionBridge.ts create mode 100644 web/src/utils/downtimes/conditionRules.ts create mode 100644 web/src/utils/downtimes/downtimeForm.spec.ts create mode 100644 web/src/utils/downtimes/downtimeForm.ts create mode 100644 web/src/utils/downtimes/extend.ts create mode 100644 web/src/utils/downtimes/folderDefault.spec.ts create mode 100644 web/src/utils/downtimes/folderDefault.ts create mode 100644 web/src/utils/downtimes/listOrder.spec.ts create mode 100644 web/src/utils/downtimes/listOrder.ts create mode 100644 web/src/utils/downtimes/notifyTest.spec.ts create mode 100644 web/src/utils/downtimes/notifyTest.ts create mode 100644 web/src/utils/downtimes/quickMute.ts create mode 100644 web/src/utils/downtimes/schedule.spec.ts create mode 100644 web/src/utils/downtimes/schedule.ts create mode 100644 web/src/utils/downtimes/summary.spec.ts create mode 100644 web/src/utils/downtimes/summary.ts create mode 100644 web/src/utils/downtimes/suppressed.spec.ts create mode 100644 web/src/utils/downtimes/suppressed.ts create mode 100644 web/src/utils/downtimes/targetSummary.spec.ts create mode 100644 web/src/utils/downtimes/targetSummary.ts create mode 100644 web/src/utils/timezoneAliases.spec.ts create mode 100644 web/src/utils/timezoneAliases.ts create mode 100644 web/src/views/slos/SloDetailCorrections.spec.ts diff --git a/scripts/translations/.translation_state.json b/scripts/translations/.translation_state.json index a8111f4e6f..29f52d9edf 100644 --- a/scripts/translations/.translation_state.json +++ b/scripts/translations/.translation_state.json @@ -2202,6 +2202,7 @@ "countAnomalyDetection": "a1223bc8d82a5173", "countCompositeAlert": "4502d18cb5dcc5d8", "countDestinations": "124a72586a12ab90", + "countDowntime": "20e7e11d1e0112dd", "countIncidentIntegration": "7e70aef22e421de8", "countOncallPolicy": "537bc7a755773028", "countOncallTeamChannel": "fcce7a09b096a168", @@ -3244,6 +3245,500 @@ "destinationPreview": "4baa36d3165b6fb2", "destinationsRequiredLabel": "341c8a818453333b", "detectionFunction": "4625de26f0afff67", + "downtimes": { + "actions": { + "cancel": "1e83e5d555d4c816", + "delete": "e2d0a54968ead24e", + "duplicate": "02cdaabfca805d6a", + "edit": "464c4ffd019e1e96", + "endNow": "adf7eccb9562dfc7", + "extend": "496a37e3a10b9755", + "more": "f8d46c2570e70736", + "moveToFolder": "91d631e150a39784", + "muteForHour": "991443d657e1425d" + }, + "allFolders": "226fd0e51f770b0b", + "allFoldersTooltip": "26ab27ec8eb6fb82", + "autoName": { + "all": { + "alerts": "2ead279c901d5b24", + "anomaly_detections": "12be9cc50e1846dc", + "slos": "65918fb54f483e8f", + "synthetics": "53668a29191ec7da" + }, + "daily": "6f31304cdfd0a3a6", + "once": "200651a8b2d5912f", + "onceOn": "5d984d828da9096f", + "weekly": "8850eced986ec360" + }, + "band": { + "active": "92340695899bd2d8", + "aria": "e535bfb61874c02e", + "next": "1ff57a29d7c9d11b", + "none": "9bbf06c79d5f4e7e", + "now": "ed5eb9a37e2d8231", + "past": "b6a7c95e785f14dd", + "track": "d598026a9cbc6050" + }, + "banner": { + "count": "30d5ec4473fde4c6", + "ended": "e87ba7a0b779d8b7", + "endsIn": "e3f5f47cba745f88", + "endsInMinutes": "d27784cde45dbfb3", + "endsInUnderMinute": "89c26da748360a68", + "firstEndsIn": "4426c705d88473a7", + "firstEndsInMinutes": "a1aedbd917b68cee", + "firstEndsInUnderMinute": "243caa9b0d62c666", + "preview": "324b134f57c70c72", + "previewMessage": "4d171441614a497f", + "showBanner": "a0aadf239daf32df", + "showBannerCaption": "61b0d38e3acd9890", + "tooLong": "ab0d30dc42cc3897" + }, + "bannerOn": "4028dbeb549d24a8", + "columns": { + "createdBy": "374cd9d220fbeb7a", + "folder": "74ccd43303847f26", + "matched": "0d6346e060b47f1c", + "name": "dcd1d5223f73b3a9", + "schedule": "f4830a1dae298044", + "status": "920e413c7d411b61", + "targets": "27445f6ab61d9a5c", + "when": "9c4734aa9debfd11" + }, + "condition": { + "add": "9fd728c66c9a256b", + "caption": "51cd0322db6a8438", + "none": "9ef2d72a498f55b6", + "onePer": "f4143a38a449c698", + "remove": "f38bb5d8032f1358", + "title": "86d7b31ddd36de64" + }, + "confirmCancel": { + "bulkMessage": "60d844aa8e405faf", + "confirm": "8b96b7d5b52eb086", + "keep": "02650c8fee8a0034", + "message": "04f1f5c6087f69b5", + "title": "1e83e5d555d4c816" + }, + "confirmDelete": { + "message": "97caf93ccdae9977", + "title": "4dbee51e2f61627a" + }, + "confirmEndNow": { + "keep": "154949dbda2aa4ee", + "message": "ae2007fef3d5c18d", + "ok": "adf7eccb9562dfc7", + "title": "8b334dd1f9510dfd" + }, + "copyOf": "3d050ae2d5dce9e2", + "corrected": "eae618cf3750c786", + "correctedBy": "bc8dc94b765d2ab1", + "detail": { + "affected": "ec13ab31fa56f546", + "backToList": "7906410d8d99435c", + "banner": "b7f4d98f98d51f8e", + "bannerOff": "0e36b3598800ed29", + "bannerOn": "7ed5a29b16f8c7e6", + "condition": "39b36d38d6eb03c8", + "details": "45989de49fb7f66d", + "matchedBy": { + "folder": "74ccd43303847f26", + "id": "8605605ae0266484", + "org": "2065e7b8b629c45a", + "query": "b80a37564fbb3c07", + "source_alert": "36f9b601a7db30ad", + "tag": "1503916a2ab2b0fd" + }, + "matchedByHeader": "7693857969438ccf", + "muteHourName": "56f878e77cdc8ed3", + "nextWindow": "a34f4badd0f164ae", + "noSuppressed": "e1167a0d116e7a81", + "noTarget": "4b33d955f9a8e928", + "notFoundDescription": "0d01e9170c73286d", + "notFoundTitle": "f2439bef305161f7", + "occurrences": "6c58d39cec6d77d2", + "overview": "d4b1ea5708dd5329", + "sectionCount": "534026895b04bd45", + "source": "0e570ca6fabe24f9", + "suppressed": "d1d64da789198216", + "wouldHaveNotified": "4c8ca48245a44de6" + }, + "detailTitle": "5f8d6d8feb7b17a5", + "duration": { + "days": "d7066a9380ea87cf", + "hours": "b7fc364731644a66", + "minutes": "1829c58ba472520f" + }, + "editTitle": "cfc2b21f0c45e190", + "endsLabel": "b7a7513cd632f8a3", + "extend": { + "confirmUntil": "690456e9eb98fd77", + "dialogTitle": "e021a5a9e84bd4de", + "endAfterCurrent": "7f1ffb10f74c672c", + "followUp": "272c39e44abf882c", + "menu": "62ccf29b7b8fa2d2", + "newEnd": "f912cddaf06b6b08", + "notActive": "cfd20a4ba832b5f0" + }, + "filterAll": "a52ace420f2175d0", + "filterOnce": "ddd7eb99b3575ba4", + "filterRecurring": "c0c06e547b1b55e1", + "form": { + "activeBanner": "f14ff5aca8bc1213", + "addItems": "b4e00e800ef36726", + "addTags": "9c29db454e3dd0f5", + "alertFolders": "a6e2acee332f6d23", + "appliesTo": "6687458beee57a58", + "cancel": "19766ed6ccb2f4a3", + "cardCaption": "e6b6a06a6c7a08c2", + "changedMeanwhile": "0db609fe96ec22bc", + "checkSchedule": "028493139bd6e7ef", + "editMode": "cfc2b21f0c45e190", + "itemOption": "f42e5528f7e57da5", + "itemsHint": "ce436920ecd4c10c", + "matches": { + "alerts": "de053c9af3e30982", + "anomaly_detections": "5a79fd3822ad10a9", + "slos": "3f39dcd8f7f55215", + "synthetics": "2e0b2ccfdecd55b3" + }, + "modeInFolder": "d6720d82a0d1d94c", + "nameLabel": "febe01f8a740a5a5", + "namePlaceholder": "febe01f8a740a5a5", + "newMode": "9ac8385c2016de16", + "noAlertFolderAccess": "6a25f9dcb4ff03d0", + "noConditionForChecks": "a4c1b42fa688a959", + "noModule": "9edb12c59d679aab", + "noSyntheticsFolderAccess": "1b50049cc8068fc5", + "reason": "f81ab834de5f8491", + "reasonHelp": "6c97e754df814762", + "reload": "bdc090ec61e3fcfc", + "removeBlock": "c79fb1926d77892d", + "renameHint": "e1a4ecc1bed73c95", + "save": "1509f561f2416598", + "sloCountAsGood": "21a2e9a921f34a5f", + "sloCountAsGoodCaption": "8fd8cf618c755080", + "sloExclude": "98161f0604cca030", + "sloExcludeCaption": "95ff87a3a053f5b7", + "sloMode": "c45be405598f3b27", + "specificItems": "3d80f2bc129418ff", + "syntheticsFolders": "41a216ee0710900f", + "tabAdvanced": "9f088dbebd6c3c70", + "tabSchedule": "f4830a1dae298044", + "tabTargets": "27445f6ab61d9a5c", + "tagPlaceholder": "06dd3884aa86b226", + "tags": "1331275bc537b4c2", + "tagsHint": "742d12fb1a499bd2" + }, + "incident": { + "banner": "f86920189febdc1c", + "bannerUntil": "3df3d6cac2e6e42b", + "mutedBy": "d73aab45a5ad7e8f", + "notLoaded": "6c0d7037c5eac517", + "notLoadedTooltip": "80c17a2070f4bc00" + }, + "matched": { + "alerts": "42c0f3db6ef90f50", + "anomalies": "a16eb2c72faa6a50", + "slos": "1b068f9843631882", + "synthetics": "ba82fe00463aa525" + }, + "modules": { + "alerts": "f62e9267d196c761", + "anomaly_detections": "75648c85faa9cc80", + "slos": "46da04c2507b963f", + "synthetics": "1569d1eebf28e7da" + }, + "modulesShort": { + "alerts": "f62e9267d196c761", + "anomaly_detections": "79622828d2d8d988", + "slos": "46da04c2507b963f", + "synthetics": "f45c092981915635" + }, + "mute": { + "bulk": "8dd6857baf026850", + "confirm": "8dd6857baf026850", + "confirmUntil": "7daca9b07837f121", + "endsAt": "bfd9598957799ce4", + "endsIn": "fcd6155d5bfd1023", + "exclude": "b8d23549534c5189", + "fileIn": "3f41a5921c2d84a2", + "firingMuted": "390d295ccd85b699", + "firingMutedTooltip": "a822693833bb0415", + "menu": "c6fbfb2045dbb234", + "muteFor": "e1536152577fc598", + "muted": "bf23559e4c1d3e1a", + "mutedTooltip": "6c45db116b6b5b99", + "presets": { + "1h": "f4926b3c9a617ae1", + "2h": "f6f984a4124b10d7", + "30m": "f69fa291de134e3c", + "4h": "1e2e2db5fda936fe", + "custom": "494ca78f7374e46f" + }, + "subtitle": "34e5e8d2eaf713b0", + "title": { + "alerts": "9cfc47b5ccd4c45a", + "anomaly_detections": "2dd4a12b2f38f304", + "items": "61e86294348fc303", + "slos": "0bda858a9e03545c", + "synthetics": "e7cd617aeb4acacd" + }, + "until": "3a163441ec3b76df" + }, + "muteAllDialog": { + "cancel": "19766ed6ccb2f4a3", + "confirm": "323a7afbef2e99a7", + "every": { + "alerts": "69457d9768567d73", + "anomaly_detections": "ea14ac9376faea60", + "slos": "2339ae37b979bb83", + "synthetics": "c984c7128ca33c33" + }, + "intro": "711510f1db8e6c49", + "matchedNow": "dac4356c321e9613", + "outro": "54ff7c2a8df84805", + "title": "2d1aceb041aac8d1", + "titleOne": { + "alerts": "964452cbdf79d03c", + "anomaly_detections": "4199da9a414ddd8e", + "slos": "164014acfe372d7b", + "synthetics": "9d18595cabbed056" + }, + "window": "3821ac247d6f6913" + }, + "newDowntime": "9ac8385c2016de16", + "newTitle": "9ac8385c2016de16", + "nextLabel": "c6c1c9a9c8543f1e", + "noFolder": "4cc9fdc7cc253497", + "notify": { + "caption": "acfc3dd57675c82d", + "destinations": "72eb63f032e47e34", + "events": { + "cancelled": "f85f95dc4eb251a4", + "cancelledHelp": "d01cac5a93e637f5", + "ended": "866e3f08ac4997ab", + "endedHelp": "a3c7966b8c7e8468", + "ending_soon": "e1a42c73cde70002", + "ending_soonHelp": "ba03badd2504dc53", + "extended": "d3b637b34bcee97b", + "extendedHelp": "fc59d77e5ac7735c", + "started": "5fd61c66df5d29bc", + "startedHelp": "d82cc56f2c1eb890" + }, + "lead": "b615206b6c30e676", + "leadHelp": "6cf2909c369f818d", + "log": { + "destinations": "72eb63f032e47e34", + "empty": "7822537e59cf15cd", + "event": "4e1f49a9c8ae8a15", + "events": { + "cancelled": "d353a99eb4556847", + "ended": "7cdc804e6996997d", + "ending_soon": "584cf74c33fbf689", + "extended": "97b165f2f4e4ca54", + "started": "ecbc89cd37a03734" + }, + "ok": "c16bc82bf1f04ede", + "pending": "e595f17fac1d8384", + "result": "6e7d50e84f4731ef", + "time": "33b93476cf597a33" + }, + "regionNote": "f5ed2e60103bf71c", + "summary": { + "cancelled": "8add00b6023aedd3", + "ended": "70a8b18249455a09", + "endingSoon": "3092ca0ea8b2d0a0", + "extended": "87d13880323aa9c1", + "none": "8e4c3adb76279932", + "sentence": "588f7196ea82ef51", + "started": "b9486c7446183877" + }, + "tab": "788011833a5a0f22", + "test": { + "failed": "557f16c3c32eca01", + "send": "22d7b11d40c98b51", + "sent": "c2250e7ed00f3376" + }, + "title": "f4e2b7734cb9eb5a", + "validation": { + "eventRequired": "a00292a8f3334fa9", + "leadInvalid": "80cb65c2d61c1320", + "tooMany": "0f5c86778e1d01a8" + } + }, + "oneTime": "ddd7eb99b3575ba4", + "preview": { + "andMore": "1a2612705dd79784", + "failed": "eabd4ce51b31ceca", + "finishCondition": "dba8d9186a95c8dc", + "fireTime": "9ffb4b9d42e1dead", + "noMatch": "782a2819bed11cda", + "showAll": "739278fd42f862d7", + "showLess": "94ea9b1d33a02975", + "title": "324b134f57c70c72", + "undecidable": "f4b2da47e69a7dd1" + }, + "recurring": "c0c06e547b1b55e1", + "resources": { + "apply": "31e392d1c0378bec", + "footer": "b237ec336c28d269", + "fromRegistry": "090b22de22c1cc48", + "fromSearch": "571555b1c76a84aa", + "fromValues": "0e04d3e365c8fec4", + "items": "fb8e7a1a3cb7b25d", + "lastSeen": "21fd79c7de52e1cf", + "load": "0834d3c481792921", + "loadFailed": "5b93b635d84e129a", + "needsEq": "a10bf1f00e8cc7fe", + "noValues": "e65cd4655961fa39", + "notSeen": "5366b0bf1c3a68ef", + "partial": "47cb4ce27c491c52", + "refineBy": "e6e354bab807c9eb", + "streams": "bfd297afbf9e7430", + "tooMany": "e9d557f93756aa7f" + }, + "schedule": { + "dailySentence": "66de77c309efe205", + "fromNoEnd": "3c66035ce1832c87", + "fromUntil": "72722a4a51229dac", + "onceSentence": "afe78d183505636c", + "weeklySentence": "c5129b953bb604b5", + "windowRange": "4c9994e9d20628f5" + }, + "scheduleForm": { + "daily": "b36c2611dcdfa190", + "days": "e08c0aa8f558f39f", + "duration": "4fc52a3c4c558b51", + "durationHelp": "4ac1b55713c423f8", + "durationPlaceholder": "659ba3d3d1509f16", + "endDate": "14303aa0c4a08d39", + "endTime": "2e46006a5eebb262", + "nextWindow": "7028e10e8d2f9459", + "nextWindowWithUtc": "1c9fc89ad4e402de", + "noWindow": "03f3bde6d99598e3", + "once": "d88f6d8372f4d54e", + "startDate": "8169693101a4536c", + "startTime": "babe9dda85036fe4", + "startsAt": "faca355ff059d331", + "startsOn": "642efcf146d12253", + "timezone": "4ceca1d52cede44d", + "until": "a07f9029151d9729", + "untilPlaceholder": "6c43e8399a11d714", + "weekly": "2975132481a7a695" + }, + "search": "6ee57ed06e35bc0f", + "searchAcross": "169fb6354f56e3b4", + "sentence": { + "inFolders": "a1d8cf47d80193bb", + "named": "9819249ddc9d3ecc", + "nothingMuted": "2fa492fd21fcafd5", + "nouns": { + "alerts": "deb3e366c077c228", + "anomaly_detections": "bac36d00cf172630", + "slos": "46da04c2507b963f", + "synthetics": "c8996fb03050baee" + }, + "sloAsGood": "b7bc23b67e9a7e31", + "sloExcluded": "532e94dfac498031", + "sloUnchanged": "24e0f5b51dc1d32c", + "stopNotifying": "bff86a74d95fbba7", + "tagged": "6bdb3ce7ddd9c4b6", + "whenDaily": "57390f94bae3294b", + "whenOnce": "f2d856678e9059ba", + "whenWeekly": "78fd12a3264d7799", + "withCondition": "219a5cf3bd61463e" + }, + "sloCorrections": { + "item": "6092824c2221f9d1", + "shorterThanSlice": "b57f3450f4f65f03", + "title": "5f89642d51b55773" + }, + "stats": { + "active": "92340695899bd2d8", + "cancelled": "d353a99eb4556847", + "ended": "7cdc804e6996997d", + "endedEarly": "e5788800949e2d4b", + "recurring": "c0c06e547b1b55e1", + "scheduled": "4724f344c1c0e4a1", + "total": "c9b3c38247f744e1" + }, + "subtitle": "3e0611514c337da7", + "summary": { + "allFolders": "b0252c8176233332", + "asGood": "44b1f98a69bf000a", + "chip": "a83b2dfc38903aff", + "condition": "596d40a4ed24c02a", + "moreTargets": "ecadd87ff0e5d0aa", + "nFolders": "78817f6f060fbcd3", + "nItems": "736ad44ae2e3ff21", + "noIncidents": "6327a035265ffe1f", + "oneFolder": "3580adda4cac85b1", + "tags": "5fda8874e4efa952", + "valuesOfKey": "23c578cbdd9844ce" + }, + "summaryPane": { + "filedIn": "a7c8c465166a6e5e", + "inUtc": "a1ee288c8cd9a98b", + "keepRunning": "519b10ff113191bd", + "largeMatch": "c613fc285a3ee59f", + "mutedIncidents": "18a69c800c7a5674", + "nextWindow": "39bf594595913f03", + "noNotifications": "634459659657e0a0", + "title": "8e76a94ac8320d51", + "whatHappens": "70fbb54bd5e6ff77", + "window": "19734a1b58cabc22" + }, + "target": { + "incidentMode": { + "label": "a8b1ad111a575a71", + "muted": "93e35cb5a9ee9a88", + "mutedHelp": "863972b6ae1c5c6c", + "none": "1d1827745b69f605", + "noneHelp": "32e4187072a3f23c", + "summary": "812f3018948c6ae7" + } + }, + "thisFolder": "ab7db04c96526507", + "thisFolderTooltip": "68f67542bf19e9ea", + "title": "5d9972eaed771890", + "truncated": "c40514efed315298", + "validation": { + "conditionRequired": "280e70b10d98a7aa", + "durationInvalid": "29165cb4ad634336", + "emptyGroup": "894e52df67b6c6a0", + "endAfterStart": "bd36587e3752dbbf", + "endRequired": "15dad7a2f239b303", + "foldersRequired": "c05b06f380c0ffb0", + "idOutsideFolders": "335c70ce631befc8", + "idsRequired": "a52bb7da0a7907e7", + "innerStar": "47542d9f2383f355", + "loneStar": "38938b248a777d87", + "modulesRequired": "71be8a1a3353efab", + "nameTooLong": "5723cc6a278f8a15", + "needsEq": "d5fbb365a9adc50a", + "pairKey": "8bc8d9cbdab52f53", + "pairOperator": "c21e78625efae96e", + "pairValue": "d173a1e1be4a69fb", + "reasonTooLong": "ff76459b14c83daf", + "startDateRequired": "45b7f811a47b4731", + "startRequired": "9239343743437cf9", + "startTimeRequired": "76bff3e8f5489b42", + "tagsRequired": "2ba02e3a957fdd15", + "timezoneRequired": "d93e8be03fbc7096", + "tooDeep": "58e6544a95805305", + "tooManyFolders": "f02e3ac4ff364c69", + "tooManyIds": "8029ede589c9a568", + "tooManyPairs": "94defd0f155e5dae", + "tooManyTags": "b2406bf479c15337", + "untilAfterStart": "b733d94c0bf498e9", + "weekdaysRequired": "56f423871f48777c", + "windowOutlastsRepeat": "20ced4da0c23bb00", + "windowTooLong": "fed4b750a6b10414", + "windowTooShort": "36e5c0e9f8ba512e" + } + }, "duration": "4fc52a3c4c558b51", "edit": "464c4ffd019e1e96", "editAlertMode": "55d309d6da250466", @@ -5131,6 +5626,13 @@ "remotepipeline": "5003f204b0419ded", "webhook": "4814f62c108d82de" }, + "downtimeStatus": { + "active": "92340695899bd2d8", + "cancelled": "d353a99eb4556847", + "ended": "7cdc804e6996997d", + "endedEarly": "e5788800949e2d4b", + "scheduled": "4724f344c1c0e4a1" + }, "enrichmentJobStatus": { "completed": "22a970d2e5b1cc23", "failed": "031a8f0f659df890", @@ -8419,6 +8921,12 @@ "description": "58946c6145c03a73", "title": "29b0352f2dc58297" }, + "noDowntimes": { + "action": "928a7246a60c12cf", + "actionDesc": "b39e9d84e595a81e", + "description": "b7e99ffd675ba00b", + "title": "476efe6fb043a245" + }, "noEnrichmentTables": { "action": "55159b3215f09b6a", "actionDesc": "f848b7a436e73f0b", @@ -9207,6 +9715,7 @@ "rumPresetSeeded": "f9b03957a5cad1c5", "scopeAllOf": "c600e6cbe1bf6ad6", "scopeCoveredByParent": "f8e7e0b7e085d5a1", + "scopeDowntimeHint": "2d5e550e6c592398", "scopeEveryStream": "d60410af0eae5a3a", "scopeEveryStreamHint": "620ee9401c968689", "scopeIncludesFuture": "26c2ee5269db2d08", @@ -11681,6 +12190,7 @@ "data": "cec3a9b89b2e3913", "databases": "61d2afe2349913ce", "docs": "7af023c43013b9a5", + "downtimes": "5d9972eaed771890", "experience": "8eab0f09df013b1b", "help": "b79cac926e0b2e34", "home": "3a78695388b38b5c", @@ -17874,6 +18384,8 @@ "dashboardsListAdd": "c7c4cad42171aa5c", "dashboardsListImport": "001dd77b594f3299", "dashboardsListRefresh": "3cf8d66227384dc2", + "downtimesCreate": "9ac8385c2016de16", + "downtimesRefresh": "1a30422cd2d4bcbb", "enrichmentTablesRefresh": "3cf8d66227384dc2", "evalJobsRefresh": "3cf8d66227384dc2", "evalTemplatesRefresh": "3cf8d66227384dc2", @@ -18005,6 +18517,7 @@ "dashboards": "54fbf186c69e6863", "dashboardsList": "a53bcafb67d960df", "dashboardsPanel": "514929ee06065dad", + "downtimes": "5d9972eaed771890", "enrichmentTables": "ec8727ebf36ee78a", "evalJobs": "c4ee26b641cc79f8", "evalTemplates": "9f9c7ce511ab3377", @@ -18149,6 +18662,8 @@ "burnNote": "578c48ed7bb9cb09", "burnRate": "b1b28eda1dfed810", "burnTitle": "b1b28eda1dfed810", + "corrected": "3d1f4b168cc3d396", + "correctedTooltip": "c458f0fcb34b5443", "cumulativeHint": "e40ff1934f536079", "empty": "d73248c65643973e", "errorsAffordable": "7ea7143b543c36a4", @@ -20482,6 +20997,24 @@ "panelQueryHasNoStreamToAlertOn": "b7b3ff8012762728", "runningQueryCanceledSuccessfully": "acddcf9ab55be3b4" }, + "downtimes": { + "cancelFailed": "78e0fecb7090fdd5", + "cancelled": "5162ae07032ffd66", + "cancelledMany": "ec4d6ab0d94626cd", + "created": "1618d16b727812c6", + "deleteFailed": "e1002fbe55d51364", + "deleted": "58c3961b3d7538f7", + "extendFailed": "dca1772452d1059a", + "extended": "14f080c8e5ab88d0", + "extendedFollowUp": "0ee62b1e0263e140", + "itemsDropped": "802d6f7bb964f3fb", + "muteFailed": "ad98d850b7e3521e", + "muted": "776a13adfdfb580d", + "reload": "bdc090ec61e3fcfc", + "saveFailed": "7d7c72a4b7b82e93", + "updated": "7227232b174315f5", + "viewDowntime": "5080007542d648b7" + }, "functions": { "deletedSuccessfully": "b322151f8013dfed", "deletingFunctions": "1d77aee9c17e33cf", diff --git a/src/api/common/src/auth/validator.rs b/src/api/common/src/auth/validator.rs index 97271fce01..e2d9a69a52 100644 --- a/src/api/common/src/auth/validator.rs +++ b/src/api/common/src/auth/validator.rs @@ -922,6 +922,7 @@ async fn check_and_create_org(user_id: &str, method: &Method, path: &str) -> Res || path_columns[2].eq("reports") || path_columns[2].eq("synthetics") || path_columns[2].eq("incidents") + || path_columns[2].eq("downtimes") || path_columns[2].eq("workflows")) { path_columns[1] diff --git a/src/api/http/src/handler/http/router/mod.rs b/src/api/http/src/handler/http/router/mod.rs index 1b89eaf026..a6f7b6736e 100644 --- a/src/api/http/src/handler/http/router/mod.rs +++ b/src/api/http/src/handler/http/router/mod.rs @@ -31,8 +31,8 @@ use openobserve_api_management::request::cloud; #[cfg(feature = "profiling")] use openobserve_api_management::request::profiling; use openobserve_api_management::request::{ - alerts, announcements, authz, dashboards, db_monitoring, folders, ingest_rejections, kv, - metrics_usage, model_pricing, organization, query_history, rum_analytics, service_accounts, + alerts, announcements, authz, dashboards, db_monitoring, downtimes, folders, ingest_rejections, + kv, metrics_usage, model_pricing, organization, query_history, rum_analytics, service_accounts, short_url, slos, sourcemaps, status, status_pages, stream, synthetics, users, }; use openobserve_api_pipelines::request::{enrichment_table, functions, pipeline, pipelines}; @@ -876,7 +876,8 @@ pub fn service_routes() -> Router { #[cfg(not(feature = "enterprise"))] let server = cfg.common.instance_name_short.to_string(); - let mut router = Router::new(); + // Downtimes live in their own table to keep this function under the line limit. + let mut router = Router::new().merge(downtime_routes()); // Full UI configuration — authenticated counterpart of the unauthenticated // `/config` bootstrap in config_routes() router = router.route("/{org_id}/config", get(status::zo_config)); @@ -2093,6 +2094,20 @@ pub fn service_routes() -> Router { })) } +/// Downtimes routes; the literal segments stay before the {downtime_id} catch-all. +#[rustfmt::skip] +pub fn downtime_routes() -> Router { + Router::new() + .route("/v2/{org_id}/downtimes", get(downtimes::list_downtimes).post(downtimes::create_downtime)) + .route("/v2/{org_id}/downtimes/preview", post(downtimes::preview_downtime)) + .route("/v2/{org_id}/downtimes/resources", post(downtimes::downtime_resources)) + .route("/v2/{org_id}/downtimes/values", post(downtimes::downtime_values)) + .route("/v2/{org_id}/downtimes/move", patch(downtimes::move_downtimes)) + .route("/v2/{org_id}/downtimes/{downtime_id}", get(downtimes::get_downtime).put(downtimes::update_downtime).delete(downtimes::delete_downtime)) + .route("/v2/{org_id}/downtimes/{downtime_id}/cancel", post(downtimes::cancel_downtime)) + .route("/v2/{org_id}/downtimes/{downtime_id}/extend", post(downtimes::extend_downtime)) +} + /// Create other service routes (AWS, GCP, RUM) pub fn other_service_routes() -> Router { let aws_routes = Router::new() diff --git a/src/api/http/src/handler/http/router/openapi.rs b/src/api/http/src/handler/http/router/openapi.rs index e80e33489e..325c63d724 100644 --- a/src/api/http/src/handler/http/router/openapi.rs +++ b/src/api/http/src/handler/http/router/openapi.rs @@ -293,6 +293,17 @@ use crate::{ openobserve_api_management::request::slos::move_slos, openobserve_api_management::request::slos::list_slo_eligible_alerts, openobserve_api_management::request::slos::preview_alert_sli, + openobserve_api_management::request::downtimes::list_downtimes, + openobserve_api_management::request::downtimes::create_downtime, + openobserve_api_management::request::downtimes::preview_downtime, + openobserve_api_management::request::downtimes::downtime_resources, + openobserve_api_management::request::downtimes::downtime_values, + openobserve_api_management::request::downtimes::get_downtime, + openobserve_api_management::request::downtimes::update_downtime, + openobserve_api_management::request::downtimes::cancel_downtime, + openobserve_api_management::request::downtimes::extend_downtime, + openobserve_api_management::request::downtimes::move_downtimes, + openobserve_api_management::request::downtimes::delete_downtime, synthetics::list_synthetics, synthetics::create_synthetic, synthetics::get_synthetic, @@ -427,6 +438,44 @@ use crate::{ openobserve_api_management::request::alerts::incidents::IncidentSeverity, openobserve_api_management::request::alerts::incidents::IncidentStatus, config::meta::alerts::incidents::Incident, + config::meta::downtimes::Downtime, + config::meta::downtimes::DowntimeRequest, + config::meta::downtimes::DowntimeListItem, + config::meta::downtimes::DowntimeDetail, + config::meta::downtimes::ListDowntimesResponse, + config::meta::downtimes::PreviewRequest, + config::meta::downtimes::PreviewResponse, + config::meta::downtimes::ResourcesRequest, + config::meta::downtimes::ResourcesResponse, + config::meta::downtimes::ValuesRequest, + config::meta::downtimes::ValuesResponse, + config::meta::downtimes::ValueSuggestion, + config::meta::downtimes::MoveDowntimesRequest, + config::meta::downtimes::CreateDowntimeResponse, + config::meta::downtimes::ExtendDowntimeRequest, + config::meta::downtimes::ExtendDowntimeResponse, + config::meta::downtimes::DowntimeTarget, + config::meta::downtimes::TargetFolders, + config::meta::downtimes::DimensionCondition, + config::meta::downtimes::LogicalOp, + config::meta::downtimes::PairOperator, + config::meta::downtimes::TargetModule, + config::meta::downtimes::DowntimeSchedule, + config::meta::downtimes::Repeat, + config::meta::downtimes::DowntimeStatus, + config::meta::downtimes::DowntimeWindow, + config::meta::downtimes::ResourceValue, + config::meta::downtimes::SloCorrectionMode, + config::meta::downtimes::IncidentMode, + config::meta::downtimes::PreviewMatch, + config::meta::downtimes::ActiveDowntime, + config::meta::downtimes::CorrectionRef, + config::meta::downtimes::DowntimeStatusCounts, + config::meta::downtimes::AffectedItems, + config::meta::downtimes::DowntimeNotifications, + config::meta::downtimes::NotificationEvents, + config::meta::downtimes::NotificationEvent, + config::meta::downtimes::DowntimeNotificationLogEntry, config::meta::alerts::incidents::IncidentWithAlerts, config::meta::alerts::incidents::IncidentAlert, config::meta::alerts::incidents::IncidentStats, @@ -593,6 +642,7 @@ use crate::{ (name = "Patterns", description = "Log pattern extraction operations (enterprise)"), (name = "Service Streams", description = "Multi-signal correlation across logs, traces, and metrics (enterprise)"), (name = "Synthetics", description = "Synthetic monitoring — uptime and browser checks (enterprise)"), + (name = "Downtimes", description = "Scheduled windows that keep evaluating alerts, anomaly detections, synthetics checks and SLOs but notify nobody (enterprise)"), (name = "Announcements", description = "Operator-authored announcement banners shown across organizations (enterprise)"), ), info( @@ -1004,4 +1054,35 @@ mod tests { assert!(tags.contains(&"Product Analytics"), "{tags:?}"); assert!(!tags.contains(&"RUM"), "{tags:?}"); } + + #[test] + fn every_schema_ref_resolves() { + fn collect_refs<'a>(value: &'a serde_json::Value, refs: &mut Vec<&'a str>) { + match value { + serde_json::Value::Object(map) => { + if let Some(r) = map.get("$ref").and_then(|r| r.as_str()) { + refs.push(r); + } + map.values().for_each(|v| collect_refs(v, refs)); + } + serde_json::Value::Array(items) => items.iter().for_each(|v| collect_refs(v, refs)), + _ => {} + } + } + + let spec = serde_json::to_value(ApiDoc::openapi()).unwrap(); + let schemas = spec["components"]["schemas"].as_object().unwrap(); + let mut refs = Vec::new(); + collect_refs(&spec, &mut refs); + let mut unresolved = refs + .into_iter() + .filter(|r| { + r.strip_prefix("#/components/schemas/") + .is_some_and(|name| !schemas.contains_key(name)) + }) + .collect::>(); + unresolved.sort_unstable(); + unresolved.dedup(); + assert!(unresolved.is_empty(), "unresolved refs: {unresolved:?}"); + } } diff --git a/src/api/management/src/models/alerts/responses.rs b/src/api/management/src/models/alerts/responses.rs index 5727fc27f1..c972613469 100644 --- a/src/api/management/src/models/alerts/responses.rs +++ b/src/api/management/src/models/alerts/responses.rs @@ -13,7 +13,7 @@ // You should have received a copy of the GNU Affero General Public License // along with this program. If not, see . -use std::str::FromStr; +use std::{collections::HashMap, str::FromStr}; use config::{ meta::{alerts::alert as meta_alerts, folder as meta_folders, triggers::Trigger}, @@ -168,6 +168,12 @@ pub struct ListAlertsResponseBodyItem { // name or a KSUID through the summary. #[serde(skip_serializing_if = "Option::is_none")] pub expression_summary: Option, + /// The downtime muting this row now, for the Muted chip. Enterprise only. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub active_downtime: Option, + /// Anomaly rows only: the identity the anomaly scheduler matches downtimes on. + #[serde(skip)] + pub downtime_dimensions: HashMap, } #[derive(Clone, Debug, Deserialize, Serialize, ToSchema)] @@ -516,6 +522,8 @@ impl TryFrom<(meta_folders::Folder, meta_alerts::Alert, Option)> child_count: None, referenced_by_composite_count: None, expression_summary: None, + active_downtime: None, + downtime_dimensions: HashMap::new(), }) } } @@ -649,6 +657,135 @@ pub fn anomaly_config_to_list_item(v: &serde_json::Value) -> Option Vec { + // Only a downtime row turns the identity into a chip, so skip the SQL parsing without one. + let groups = if o2_enterprise::enterprise::common::config::get_config() + .downtimes + .enabled + && !db::downtimes::list_cached(org).is_empty() + { + Some(db::system_settings::get_semantic_field_groups(org).await) + } else { + None + }; + configs + .iter() + .filter_map(|v| { + let mut item = anomaly_config_to_list_item(v)?; + if let Some(groups) = &groups { + item.downtime_dimensions = anomaly_downtime_dimensions(v, &item.tags, groups); + } + Some(item) + }) + .collect() +} + +/// The Muted chip, from the cache or from the downtime the last firing recorded. +#[cfg(feature = "enterprise")] +pub async fn fill_active_downtimes(org: &str, list: &mut [ListAlertsResponseBodyItem]) { + if list.is_empty() + || !o2_enterprise::enterprise::common::config::get_config() + .downtimes + .enabled + { + return; + } + // No downtime rows, no chip: neither branch below can yield one, so skip the state query. + let rows = db::downtimes::list_cached(org); + if rows.is_empty() { + return; + } + let now = config::utils::time::now_micros(); + let ids: Vec = list.iter().map(|item| item.alert_id.to_string()).collect(); + let recorded = infra::table::alert_states::last_downtime_ids(&ids) + .await + .unwrap_or_default(); + for item in list.iter_mut() { + item.active_downtime = list_item_downtime(org, item, now).or_else(|| { + recorded + .get(&item.alert_id.to_string()) + .and_then(|id| still_active(&rows, id, now)) + }); + } +} + +#[cfg(not(feature = "enterprise"))] +pub async fn fill_active_downtimes(_org: &str, _list: &mut [ListAlertsResponseBodyItem]) {} + +#[cfg(feature = "enterprise")] +fn list_item_downtime( + org: &str, + item: &ListAlertsResponseBodyItem, + now: i64, +) -> Option { + use openobserve_core::alerts::downtimes::{active_for_alert, active_for_anomaly}; + + let id = item.alert_id.to_string(); + let dims = list_item_dimensions(item); + match item.alert_type.as_str() { + "anomaly_detection" => active_for_anomaly(org, &id, &item.folder_id, &dims, now), + _ => active_for_alert(org, &id, &item.folder_id, &dims, now), + } +} + +/// The identity a list row is matched on, the one its own scheduler matches. +#[cfg(feature = "enterprise")] +fn list_item_dimensions(item: &ListAlertsResponseBodyItem) -> HashMap { + use o2_enterprise::enterprise::downtimes::scope::composite_dimensions; + + match item.alert_type.as_str() { + "anomaly_detection" => item.downtime_dimensions.clone(), + "composite" => composite_dimensions(&item.tags), + _ => HashMap::new(), + } +} + +/// The `anomaly_dimensions` of a config row, read from the keys the anomaly scheduler reads. +#[cfg(feature = "enterprise")] +fn anomaly_downtime_dimensions( + v: &serde_json::Value, + tags: &[String], + semantic_groups: &[config::meta::correlation::FieldAlias], +) -> HashMap { + use openobserve_core::alerts::downtimes::{anomaly_filters, anomaly_sql}; + + let query_mode = v + .get("query_mode") + .and_then(|m| m.as_str()) + .unwrap_or_default(); + o2_enterprise::enterprise::downtimes::scope::anomaly_dimensions( + anomaly_sql(query_mode, v.get("custom_sql").and_then(|s| s.as_str())), + anomaly_filters(query_mode, v.get("filters")), + tags, + semantic_groups, + ) +} + +#[cfg(feature = "enterprise")] +fn still_active( + rows: &[config::meta::downtimes::Downtime], + downtime_id: &str, + now: i64, +) -> Option { + let row = rows + .iter() + .find(|row| row.id == downtime_id && row.cancelled_at.is_none())?; + let window = o2_enterprise::enterprise::downtimes::schedule::window_at(&row.schedule, now)?; + Some(config::meta::downtimes::ActiveDowntime { + id: row.id.clone(), + name: row.name.clone(), + ends_at: window.end, + incident_mode: Default::default(), }) } @@ -741,6 +878,8 @@ mod tests { child_count: None, referenced_by_composite_count: None, expression_summary: None, + active_downtime: None, + downtime_dimensions: HashMap::new(), }; let json = serde_json::to_value(&item).unwrap(); let obj = json.as_object().unwrap(); @@ -1171,4 +1310,53 @@ mod anomaly_priority_tag_tests { assert_eq!(item.level, None); assert_eq!(item.level_since, None); } + + #[cfg(feature = "enterprise")] + #[test] + fn an_anomaly_on_a_service_in_its_sql_shows_the_chip_of_that_service() { + use config::meta::{ + correlation::FieldAlias, + downtimes::{Downtime, TargetModule}, + }; + use o2_enterprise::enterprise::downtimes::scope::{TargetItem, matches, target_for}; + + let groups = [FieldAlias { + id: "service".to_string(), + display: "Service".to_string(), + group: None, + fields: vec!["service".to_string()], + is_workload_type: false, + }]; + let v = cfg(serde_json::json!({ + "query_mode": "custom_sql", + "custom_sql": "SELECT count(*) AS c FROM \"logs\" WHERE service = 'payments'", + })); + let mut item = anomaly_config_to_list_item(&v).expect("should parse"); + assert!(item.tags.is_empty()); + item.downtime_dimensions = anomaly_downtime_dimensions(&v, &item.tags, &groups); + + let downtime: Downtime = serde_json::from_value(serde_json::json!({ + "id": "dt-1", "org": "default", "name": "payments maintenance", + "condition": { "type": "pair", "key": "service", "operator": "=", "value": "payments" }, + "targets": [{ "module": "anomaly_detections", "folders": { "kind": "all" } }], + "schedule": { + "repeat": "none", "starts_at": 0, "timezone": "UTC", "duration_secs": 60 + }, + "created_by": "lin", "created_at": 0, "updated_by": "lin", "updated_at": 0 + })) + .unwrap(); + let target = target_for(&downtime.targets, TargetModule::AnomalyDetections).unwrap(); + let id = item.alert_id.to_string(); + let dims = list_item_dimensions(&item); + let row = TargetItem { + id: &id, + folder_id: &item.folder_id, + dimensions: &dims, + tags: &[], + }; + assert!( + matches(target, downtime.condition.as_ref(), &row), + "{dims:?}" + ); + } } diff --git a/src/api/management/src/models/destinations.rs b/src/api/management/src/models/destinations.rs index e87ba29f1e..c64e1b8a4c 100644 --- a/src/api/management/src/models/destinations.rs +++ b/src/api/management/src/models/destinations.rs @@ -148,7 +148,8 @@ impl From for Template { // Shared rule lives in `config::prebuilt_loader` so the HTTP DTO and // the service-layer guards can't drift apart. - let is_prebuilt = config::prebuilt_loader::is_prebuilt_template_name(&value.name); + let is_prebuilt = config::prebuilt_loader::is_prebuilt_template_name(&value.name) + || openobserve_core::alerts::templates::is_downtime_template_name(&value.name); Self { name: value.name, @@ -337,6 +338,8 @@ pub enum DestinationConsumerKind { AnomalyDetection, #[cfg(feature = "enterprise")] IncidentIntegration, + #[cfg(feature = "enterprise")] + Downtime, } impl From for DestinationConsumerKind { @@ -356,6 +359,8 @@ impl From for DestinationConsumerKind { destination_usage::DestinationConsumer::IncidentIntegration => { Self::IncidentIntegration } + #[cfg(feature = "enterprise")] + destination_usage::DestinationConsumer::Downtime => Self::Downtime, } } } diff --git a/src/api/management/src/models/folders.rs b/src/api/management/src/models/folders.rs index 7070d61c7a..861abfc5f5 100644 --- a/src/api/management/src/models/folders.rs +++ b/src/api/management/src/models/folders.rs @@ -57,6 +57,7 @@ pub enum FolderType { Synthetics, Workflows, Prompts, + Downtimes, } /// Common folder fields used in HTTP request and response bodies. @@ -111,6 +112,7 @@ impl From for config::meta::folder::FolderType { FolderType::Synthetics => Self::Synthetics, FolderType::Workflows => Self::Workflows, FolderType::Prompts => Self::Prompts, + FolderType::Downtimes => Self::Downtimes, } } } @@ -165,6 +167,15 @@ mod tests { ); } + #[test] + fn the_downtimes_path_segment_maps_to_the_downtimes_folder_type() { + let parsed: FolderType = serde_json::from_str("\"downtimes\"").unwrap(); + assert_eq!( + config::meta::folder::FolderType::from(parsed), + config::meta::folder::FolderType::Downtimes + ); + } + #[test] fn test_folder_from_config_preserves_fields() { let src = config::meta::folder::Folder { diff --git a/src/api/management/src/request/alerts/history.rs b/src/api/management/src/request/alerts/history.rs index 665a4f6809..b25d1302e1 100644 --- a/src/api/management/src/request/alerts/history.rs +++ b/src/api/management/src/request/alerts/history.rs @@ -13,6 +13,7 @@ // You should have received a copy of the GNU Affero General Public License // along with this program. If not, see . +use arrow_schema::Schema; use axum::{ extract::{Path, Query}, response::Response, @@ -40,6 +41,12 @@ use crate::common::{ meta::http::HttpResponse as MetaHttpResponse, utils::http::get_or_create_trace_id, }; +/// Every column the data query reads but `downtime_id`, which [history_columns] adds. +const HISTORY_COLUMNS: &str = "_timestamp, org, key, status, level, actual_value, \ + threshold_value, threshold_operator, group_label, value_is_lower_bound, is_realtime, \ + is_silenced, start_time, end_time, retries, delay_in_secs, evaluation_took_in_secs, \ + source_node, query_took, error"; + #[derive(Debug, Clone, Serialize, Deserialize)] pub struct AlertHistoryQuery { /// Filter by specific alert name @@ -61,6 +68,10 @@ pub struct AlertHistoryQuery { pub sort_by: Option, /// Sort order (asc or desc, default: desc) pub sort_order: Option, + /// Only runs this downtime suppressed; also reads synthetics and anomaly runs. + pub downtime_id: Option, + /// Filter by run outcome, for example `suppressed`. + pub status: Option, } #[derive(Debug, Clone, Serialize, Deserialize, ToSchema)] @@ -118,6 +129,9 @@ pub struct AlertHistoryEntry { /// §7.5) — the UI renders "≥ N". Absent = exact. #[serde(skip_serializing_if = "Option::is_none")] pub value_is_lower_bound: Option, + /// The downtime that suppressed this run. + #[serde(skip_serializing_if = "Option::is_none")] + pub downtime_id: Option, } #[derive(Debug, Clone, Serialize, Deserialize, ToSchema)] @@ -409,6 +423,7 @@ pub async fn get_alert_history( threshold_operator: None, group_label: None, value_is_lower_bound: None, + downtime_id: None, } }) .collect(); @@ -558,9 +573,36 @@ pub async fn get_alert_history( // Composites publish with module = "composite" and share the ordinary // alert outcome vocabulary, so include them in the same history read. + // Anomaly detections are the `alerts` resource, so the alert check above already gates them. + let synthetics_readable = + query.downtime_id.is_some() && reads_every_synthetic(&org_id, &user_email.user_id).await; + let modules = history_modules(query.downtime_id.is_some(), synthetics_readable); + let triggers_schema = + match triggers_schema(infra::schema::get(&org_id, TRIGGERS_STREAM, StreamType::Logs).await) + { + Ok(schema) => schema, + Err(response) => return response, + }; + if query + .downtime_id + .as_deref() + .is_some_and(|id| !id.is_empty()) + && triggers_schema.field_with_name("downtime_id").is_err() + { + return MetaHttpResponse::json(AlertHistoryResponse { + total: 0, + from, + size, + hits: vec![], + }); + } let mut where_clause = format!( - "module IN ('alert', 'composite') AND org = '{org_id}' AND _timestamp >= {start_time} AND _timestamp <= {end_time}" + "module IN ({modules}) AND org = '{org_id}' AND _timestamp >= {start_time} AND _timestamp <= {end_time}" ); + where_clause.push_str(&downtime_filters( + query.downtime_id.as_deref(), + query.status.as_deref(), + )); // Add alert ID filter if provided // The key field contains the alert ID in the format "alert_name/alert_id" @@ -657,13 +699,9 @@ pub async fn get_alert_history( // Step 2: Get the actual paginated results // Build data query with LIMIT/OFFSET for pagination + let columns = history_columns(&triggers_schema); let data_sql = format!( - "SELECT _timestamp, org, key, status, level, actual_value, threshold_value, \ - threshold_operator, group_label, value_is_lower_bound, is_realtime, is_silenced, \ - start_time, end_time, retries, \ - delay_in_secs, evaluation_took_in_secs, \ - source_node, query_took, error \ - FROM \"{TRIGGERS_STREAM}\" \ + "SELECT {columns} FROM \"{TRIGGERS_STREAM}\" \ WHERE {where_clause} \ ORDER BY {sort_column} {sort_order} LIMIT {size} OFFSET {from}" ); @@ -796,6 +834,11 @@ pub async fn get_alert_history( .filter(|s| !s.is_empty()) .map(String::from), value_is_lower_bound: hit.get("value_is_lower_bound").and_then(|v| v.as_bool()), + downtime_id: hit + .get("downtime_id") + .and_then(|v| v.as_str()) + .filter(|s| !s.is_empty()) + .map(String::from), }); } @@ -810,6 +853,109 @@ pub async fn get_alert_history( MetaHttpResponse::json(response) } +/// The modules a history read covers; a `?downtime_id=` read adds those a downtime can mute. +fn history_modules(by_downtime: bool, synthetics_readable: bool) -> String { + let mut modules = vec!["'alert'", "'composite'"]; + if by_downtime { + modules.push("'anomaly_detection'"); + if synthetics_readable { + modules.push("'synthetics'"); + } + } + modules.join(", ") +} + +/// Synthetics runs carry no per-check filter here, so only an org-wide synthetics read sees them. +#[cfg(feature = "enterprise")] +async fn reads_every_synthetic(org_id: &str, user_id: &str) -> bool { + if db::user::is_root_user(user_id) || !o2_openfga::config::get_config().enabled { + return true; + } + let object_type = o2_openfga::meta::mapping::OFGA_MODELS + .get("synthetics") + .map_or("synthetics", |model| model.key); + let listed = match openobserve_api_common::auth::validator::list_objects_for_user( + org_id, + user_id, + "GET", + object_type, + ) + .await + { + Ok(listed) => listed, + Err(e) => { + log::warn!("synthetics read of {user_id} in {org_id} unreadable: {e}"); + return false; + } + }; + every_synthetic_readable( + listed.as_deref(), + &format!("{object_type}:_all_{org_id}"), + async || { + openobserve_core::auth::check_permissions( + org_id, + org_id, + user_id, + "synthetics", + "GET", + None, + true, + false, + false, + ) + .await + }, + ) + .await +} + +#[cfg(not(feature = "enterprise"))] +async fn reads_every_synthetic(_org_id: &str, _user_id: &str) -> bool { + true +} + +/// `None` from ListObjects means no list filtering, not a grant, so the org-wide check decides. +#[cfg_attr(not(feature = "enterprise"), allow(dead_code))] +async fn every_synthetic_readable( + listed: Option<&[String]>, + all_key: &str, + org_wide_grant: impl AsyncFnOnce() -> bool, +) -> bool { + match listed { + Some(permitted) => permitted.iter().any(|object| object == all_key), + None => org_wide_grant().await, + } +} + +/// The triggers stream's schema; a failed read is a server error, never an empty stream. +fn triggers_schema(read: infra::errors::Result) -> Result { + read.map_err(|e| { + log::error!("Failed to read the alert history schema: {e}"); + MetaHttpResponse::internal_error(format!("Failed to read the alert history schema: {e}")) + }) +} + +/// [HISTORY_COLUMNS], with `downtime_id` once the stream has it: an upgraded stream learns it late. +fn history_columns(schema: &Schema) -> String { + if schema.field_with_name("downtime_id").is_ok() { + format!("{HISTORY_COLUMNS}, downtime_id") + } else { + HISTORY_COLUMNS.to_string() + } +} + +/// The downtime filters of the history read, quoted with the same single-quote escaping. +fn downtime_filters(downtime_id: Option<&str>, status: Option<&str>) -> String { + let mut clause = String::new(); + if let Some(id) = downtime_id.filter(|id| !id.is_empty()) { + clause.push_str(&format!(" AND downtime_id = '{}'", id.replace('\'', "''"))); + } + if let Some(status) = status.filter(|status| !status.is_empty()) { + clause.push_str(&format!(" AND status = '{}'", status.replace('\'', "''"))); + } + clause +} + /// Bulk anomaly history query — returns the most recent N hits per anomaly config in one request. /// /// Instead of the caller firing one request per config, this endpoint queries the triggers stream @@ -1042,6 +1188,8 @@ mod tests { size: None, sort_by: None, sort_order: None, + downtime_id: None, + status: None, }; assert!(query.alert_id.is_none()); @@ -1084,6 +1232,7 @@ mod tests { threshold_operator: None, group_label: None, value_is_lower_bound: None, + downtime_id: None, }; assert_eq!(entry.alert_name, "test_alert"); @@ -1141,6 +1290,7 @@ mod tests { threshold_operator: None, group_label: None, value_is_lower_bound: None, + downtime_id: None, }; let response = AlertHistoryResponse { @@ -1187,6 +1337,7 @@ mod tests { threshold_operator: None, group_label: None, value_is_lower_bound: None, + downtime_id: None, }; assert_eq!(entry.status, "error"); @@ -1208,6 +1359,8 @@ mod tests { size: Some(25), sort_by: None, sort_order: None, + downtime_id: None, + status: None, }; assert_eq!(query.from.unwrap(), 100); @@ -1243,6 +1396,8 @@ mod tests { size: None, sort_by: Some(field.to_string()), sort_order: Some("asc".to_string()), + downtime_id: None, + status: None, }; assert_eq!(query.sort_by, Some(field.to_string())); @@ -1262,6 +1417,8 @@ mod tests { size: None, sort_by: Some("timestamp".to_string()), sort_order: Some("asc".to_string()), + downtime_id: None, + status: None, }; assert_eq!(query_asc.sort_order, Some("asc".to_string())); @@ -1275,6 +1432,8 @@ mod tests { size: None, sort_by: Some("timestamp".to_string()), sort_order: Some("desc".to_string()), + downtime_id: None, + status: None, }; assert_eq!(query_desc.sort_order, Some("desc".to_string())); } @@ -1407,6 +1566,7 @@ mod tests { threshold_operator: None, group_label: None, value_is_lower_bound: None, + downtime_id: None, }; let json = serde_json::to_string(&entry).unwrap(); @@ -1450,6 +1610,7 @@ mod tests { threshold_operator: None, group_label: None, value_is_lower_bound: None, + downtime_id: None, }; let response = AlertHistoryResponse { @@ -1499,6 +1660,7 @@ mod tests { threshold_operator: None, group_label: None, value_is_lower_bound: None, + downtime_id: None, }; let json = serde_json::to_value(&entry).unwrap(); let obj = json.as_object().unwrap(); @@ -1541,6 +1703,7 @@ mod tests { threshold_operator: None, group_label: None, value_is_lower_bound: None, + downtime_id: None, }; let json = serde_json::to_value(&entry).unwrap(); let obj = json.as_object().unwrap(); @@ -1551,4 +1714,72 @@ mod tests { assert!(obj.contains_key("group_size")); assert!(obj.contains_key("anomaly_count")); } + + #[tokio::test] + async fn unfiltered_listing_is_not_a_synthetics_grant() { + let all = "synthetics:_all_acme"; + assert!(!every_synthetic_readable(None, all, async || false).await); + assert!(every_synthetic_readable(None, all, async || true).await); + let listed = [all.to_string()]; + assert!(every_synthetic_readable(Some(&listed), all, async || false).await); + let one = ["synthetics:c1".to_string()]; + assert!(!every_synthetic_readable(Some(&one), all, async || true).await); + } + + #[test] + fn a_user_without_synthetics_read_sees_no_synthetics_runs() { + assert_eq!( + history_modules(true, false), + "'alert', 'composite', 'anomaly_detection'" + ); + assert_eq!( + history_modules(true, true), + "'alert', 'composite', 'anomaly_detection', 'synthetics'" + ); + assert_eq!(history_modules(false, true), "'alert', 'composite'"); + } + + #[test] + fn downtime_filters_quote_and_combine() { + assert_eq!(downtime_filters(None, None), ""); + assert_eq!( + downtime_filters(Some("2f9K"), None), + " AND downtime_id = '2f9K'" + ); + assert_eq!( + downtime_filters(None, Some("suppressed")), + " AND status = 'suppressed'" + ); + assert_eq!( + downtime_filters(Some("a'b"), Some("suppressed")), + " AND downtime_id = 'a''b' AND status = 'suppressed'" + ); + assert_eq!(downtime_filters(Some(""), Some("")), ""); + } + + #[test] + fn a_failed_schema_read_is_a_server_error_not_an_empty_history() { + let failed = triggers_schema(Err(infra::errors::Error::Message("db down".to_string()))); + assert_eq!( + failed.map(|_| ()).unwrap_err().status(), + axum::http::StatusCode::INTERNAL_SERVER_ERROR + ); + let missing = triggers_schema(Ok(Schema::empty())).unwrap(); + assert!(missing.field_with_name("downtime_id").is_err()); + } + + #[test] + fn a_stream_from_before_downtimes_reads_without_downtime_id() { + use arrow_schema::{DataType, Field}; + + let field = |name: &str| Field::new(name, DataType::Utf8, true); + let upgraded: Vec = HISTORY_COLUMNS.split(", ").map(field).collect(); + let columns = history_columns(&Schema::new(upgraded.clone())); + assert!(!columns.contains("downtime_id"), "{columns}"); + assert_eq!(columns, HISTORY_COLUMNS); + + let mut current = upgraded; + current.push(field("downtime_id")); + assert!(history_columns(&Schema::new(current)).ends_with(", downtime_id")); + } } diff --git a/src/api/management/src/request/alerts/incidents.rs b/src/api/management/src/request/alerts/incidents.rs index 93eb0e4098..7e55a36362 100644 --- a/src/api/management/src/request/alerts/incidents.rs +++ b/src/api/management/src/request/alerts/incidents.rs @@ -1223,6 +1223,7 @@ mod tests { updated_at: 2000, group_values: serde_json::Value::Object(Default::default()), key_type: config::meta::alerts::incidents::KeyType::default(), + muted_by_downtime_id: None, }; let response = ListIncidentsResponse { diff --git a/src/api/management/src/request/alerts/mod.rs b/src/api/management/src/request/alerts/mod.rs index cdbcd87035..e4da6249c6 100644 --- a/src/api/management/src/request/alerts/mod.rs +++ b/src/api/management/src/request/alerts/mod.rs @@ -49,7 +49,7 @@ use { #[cfg(feature = "enterprise")] use crate::models::alerts::requests::UpdateAnomalyAlertFields; #[cfg(feature = "enterprise")] -use crate::models::alerts::responses::anomaly_config_to_list_item; +use crate::models::alerts::responses::anomaly_list_items; use crate::{ common::meta::http::HttpResponse as MetaHttpResponse, models::alerts::{ @@ -708,6 +708,8 @@ fn composite_list_item( child_count: None, referenced_by_composite_count: None, expression_summary: None, + active_downtime: None, + downtime_dimensions: Default::default(), }) } @@ -2983,9 +2985,9 @@ pub async fn list_alerts( // would return every anomaly config alongside the matching alerts. let before = list.len(); list.extend( - configs - .iter() - .filter_map(anomaly_config_to_list_item) + anomaly_list_items(&org_id, &configs) + .await + .into_iter() .filter(|item| enabled_filter.is_none_or(|enabled| item.enabled == enabled)) .filter(|item| match &priority_filter { None => true, @@ -3033,6 +3035,7 @@ pub async fn list_alerts( let mut list = list; enrich_with_run_state(&mut list).await; enrich_with_composite_metadata(&org_id, &visibility, &mut list).await; + crate::models::alerts::responses::fill_active_downtimes(&org_id, &mut list).await; // Feature-scoped fields: keep destinations/template off the default list path // (bytes + module-scoped names) unless the dependency view explicitly opted in. @@ -3665,7 +3668,7 @@ pub async fn trigger_alert( } let client = get_orm_client_ro().await; match alert::trigger_by_id(client, &org_id, alert_id).await { - Ok(_) => MetaHttpResponse::ok("Alert triggered"), + Ok(triggered) => MetaHttpResponse::ok(triggered.response_message()), Err(AlertError::AlertNotFound) => { if let Some(_composite) = openobserve_core::alerts::composite::get_composite(&org_id, &alert_id.to_string()) diff --git a/src/api/management/src/request/announcements/mod.rs b/src/api/management/src/request/announcements/mod.rs index 7b96de5127..63544bbc9d 100644 --- a/src/api/management/src/request/announcements/mod.rs +++ b/src/api/management/src/request/announcements/mod.rs @@ -163,10 +163,23 @@ pub async fn get_announcements(Path(org_id): Path) -> Response { } }; + let mut banners = announcements::resolve_for_org(&config, &org_id, now); + let mut next_boundary = next_boundary_for_org(&config, &org_id, now); + let downtimes = &o2_enterprise::enterprise::common::config::get_config().downtimes; + if downtimes.enabled && downtimes.banner_enabled { + match openobserve_core::downtimes::banners_for_org(&org_id, now).await { + Ok((generated, boundary)) => { + banners.extend(generated); + next_boundary = next_boundary.into_iter().chain(boundary).min(); + } + Err(e) => log::error!("Error building downtime banners for {org_id}: {e}"), + } + } + MetaHttpResponse::json(AnnouncementsResponse { - banners: announcements::resolve_for_org(&config, &org_id, now), + banners, now, - next_boundary: next_boundary_for_org(&config, &org_id, now), + next_boundary, }) } diff --git a/src/api/management/src/request/anomaly_detection/mod.rs b/src/api/management/src/request/anomaly_detection/mod.rs index a496bea2b9..050462dc46 100644 --- a/src/api/management/src/request/anomaly_detection/mod.rs +++ b/src/api/management/src/request/anomaly_detection/mod.rs @@ -83,7 +83,10 @@ pub async fn list_configs(Path(org_id): Path) -> Response { return resp; } match anomaly_service::list_configs(&org_id, None, None).await { - Ok(configs) => MetaHttpResponse::json(configs), + Ok(mut configs) => { + add_active_downtimes(&org_id, &mut configs).await; + MetaHttpResponse::json(configs) + } Err(e) => { tracing::error!("Failed to list anomaly configs: {}", e); MetaHttpResponse::error(StatusCode::INTERNAL_SERVER_ERROR.as_u16(), e.to_string()) @@ -92,6 +95,56 @@ pub async fn list_configs(Path(org_id): Path) -> Response { } } +/// Sets `active_downtime` on each config an active downtime's Anomaly detections target matches. +#[cfg(feature = "enterprise")] +async fn add_active_downtimes(org_id: &str, configs: &mut [serde_json::Value]) { + if !o2_enterprise::enterprise::common::config::get_config() + .downtimes + .enabled + || db::downtimes::list_cached(org_id).is_empty() + { + return; + } + let semantic_groups = db::system_settings::get_semantic_field_groups(org_id).await; + let now = config::utils::time::now_micros(); + for value in configs.iter_mut() { + let Some(obj) = value.as_object_mut() else { + continue; + }; + let text = |key: &str| obj.get(key).and_then(|v| v.as_str()).map(str::to_string); + let (Some(anomaly_id), Some(folder_id)) = (text("anomaly_id"), text("folder_id")) else { + continue; + }; + let tags: Vec = obj + .get("tags") + .and_then(|t| serde_json::from_value(t.clone()).ok()) + .unwrap_or_default(); + let query_mode = text("query_mode").unwrap_or_default(); + let dims = o2_enterprise::enterprise::downtimes::scope::anomaly_dimensions( + openobserve_core::alerts::downtimes::anomaly_sql( + &query_mode, + text("custom_sql").as_deref(), + ), + openobserve_core::alerts::downtimes::anomaly_filters(&query_mode, obj.get("filters")), + &tags, + &semantic_groups, + ); + if let Some(downtime) = openobserve_core::alerts::downtimes::active_for_anomaly( + org_id, + &anomaly_id, + &folder_id, + &dims, + now, + ) && let Ok(active) = serde_json::to_value(downtime) + { + obj.insert("active_downtime".to_string(), active); + } + } +} + +#[cfg(not(feature = "enterprise"))] +async fn add_active_downtimes(_org_id: &str, _configs: &mut [serde_json::Value]) {} + /// Get a specific anomaly detection configuration #[utoipa::path( get, diff --git a/src/api/management/src/request/downtimes.rs b/src/api/management/src/request/downtimes.rs new file mode 100644 index 0000000000..d7bffd98aa --- /dev/null +++ b/src/api/management/src/request/downtimes.rs @@ -0,0 +1,576 @@ +// Copyright 2026 OpenObserve Inc. +// +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// This program is distributed in the hope that it will be useful +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. +// +// You should have received a copy of the GNU Affero General Public License +// along with this program. If not, see . + +//! Downtime handlers: enterprise-only (D1), 403 in the OSS build and while the flag is off. + +use axum::{ + Json, + extract::{Path, Query}, + response::Response, +}; +use config::meta::downtimes::{ + CreateDowntimeResponse, Downtime, DowntimeDetail, DowntimeRequest, ExtendDowntimeRequest, + ExtendDowntimeResponse, ListDowntimesQuery, ListDowntimesResponse, MoveDowntimesRequest, + PreviewRequest, PreviewResponse, ResourcesRequest, ResourcesResponse, ValuesRequest, + ValuesResponse, +}; +use openobserve_api_common::extractors::Headers; +use serde::Deserialize; +use utoipa::IntoParams; + +use crate::{common::meta::http::HttpResponse as MetaHttpResponse, service::auth::UserEmail}; + +/// The destination folder of a create, when given in the query as the route check reads it. +#[derive(Debug, Default, Deserialize, IntoParams)] +#[into_params(parameter_in = Query)] +pub struct FolderQuery { + pub folder: Option, +} + +/// ListDowntimes +#[utoipa::path( + get, + path = "/v2/{org_id}/downtimes", + context_path = "/api", + tag = "Downtimes", + operation_id = "ListDowntimes", + summary = "List downtimes", + description = "Lists the downtimes in the folders the user may list, with their status, windows and match counts.", + security(("Authorization" = [])), + params(("org_id" = String, Path, description = "Organization name"), ListDowntimesQuery), + responses( + (status = 200, description = "Success", content_type = "application/json", body = ListDowntimesResponse), + (status = 403, description = "Forbidden or not enabled", content_type = "application/json", body = ()), + ), +)] +pub async fn list_downtimes( + Path(org_id): Path, + Query(query): Query, + Headers(user_email): Headers, +) -> Response { + #[cfg(feature = "enterprise")] + { + respond(openobserve_core::downtimes::list(&org_id, &user_email.user_id, &query).await) + } + #[cfg(not(feature = "enterprise"))] + { + let _ = (org_id, query, user_email); + not_supported() + } +} + +/// CreateDowntime +#[utoipa::path( + post, + path = "/v2/{org_id}/downtimes", + context_path = "/api", + tag = "Downtimes", + operation_id = "CreateDowntime", + summary = "Create a downtime", + description = "Creates a downtime. Quick mute uses this route with one target and specific ids.", + security(("Authorization" = [])), + params(("org_id" = String, Path, description = "Organization name"), FolderQuery), + request_body(content = DowntimeRequest, description = "Downtime definition", content_type = "application/json"), + responses( + (status = 201, description = "Created", content_type = "application/json", body = CreateDowntimeResponse), + (status = 400, description = "Invalid request", content_type = "application/json", body = ()), + (status = 403, description = "Forbidden or not enabled", content_type = "application/json", body = ()), + ), +)] +pub async fn create_downtime( + Path(org_id): Path, + Query(folder): Query, + Headers(user_email): Headers, + Json(req): Json, +) -> Response { + #[cfg(feature = "enterprise")] + { + let mut req = req; + if let Some(folder) = folder.folder.filter(|f| !f.is_empty()) { + req.folder_id = folder; + } + if req.folder_id.is_empty() { + req.folder_id = config::meta::folder::DEFAULT_FOLDER.to_string(); + } + if !crate::service::auth::check_folder_write_permissions( + &org_id, + &user_email.user_id, + "downtime_folders", + &req.folder_id, + ) + .await + { + return MetaHttpResponse::forbidden("You cannot create downtimes in this folder."); + } + match openobserve_core::downtimes::create(&org_id, &user_email.user_id, req).await { + Ok(downtime) => created(downtime.id), + Err(e) => error_response(e), + } + } + #[cfg(not(feature = "enterprise"))] + { + let _ = (org_id, folder, user_email, req); + not_supported() + } +} + +/// PreviewDowntime +#[utoipa::path( + post, + path = "/v2/{org_id}/downtimes/preview", + context_path = "/api", + tag = "Downtimes", + operation_id = "PreviewDowntime", + summary = "Preview what a downtime covers", + description = "Lists, per module, what the targets match today, the alerts decided at fire time, and the items a resource refinement cannot decide. Names are limited to what the user may list; totals count everything.", + security(("Authorization" = [])), + params(("org_id" = String, Path, description = "Organization name")), + request_body(content = PreviewRequest, description = "Condition and targets", content_type = "application/json"), + responses( + (status = 200, description = "Success", content_type = "application/json", body = PreviewResponse), + (status = 403, description = "Forbidden or not enabled", content_type = "application/json", body = ()), + ), +)] +pub async fn preview_downtime( + Path(org_id): Path, + Headers(user_email): Headers, + Json(req): Json, +) -> Response { + #[cfg(feature = "enterprise")] + { + respond(openobserve_core::downtimes::preview(&org_id, &user_email.user_id, &req).await) + } + #[cfg(not(feature = "enterprise"))] + { + let _ = (org_id, user_email, req); + not_supported() + } +} + +/// DowntimeResources +#[utoipa::path( + post, + path = "/v2/{org_id}/downtimes/resources", + context_path = "/api", + tag = "Downtimes", + operation_id = "DowntimeResources", + summary = "Resources behind a condition", + description = "The values of a finer dimension, such as host, that co-occur with the condition, for refining a downtime to some resources.", + security(("Authorization" = [])), + params(("org_id" = String, Path, description = "Organization name")), + request_body(content = ResourcesRequest, description = "Condition and the dimension to refine by", content_type = "application/json"), + responses( + (status = 200, description = "Success", content_type = "application/json", body = ResourcesResponse), + (status = 400, description = "Invalid request", content_type = "application/json", body = ()), + (status = 403, description = "Forbidden or not enabled", content_type = "application/json", body = ()), + ), +)] +pub async fn downtime_resources( + Path(org_id): Path, + Headers(user_email): Headers, + Json(req): Json, +) -> Response { + #[cfg(feature = "enterprise")] + { + respond(openobserve_core::downtimes::resources(&org_id, &user_email.user_id, &req).await) + } + #[cfg(not(feature = "enterprise"))] + { + let _ = (org_id, user_email, req); + not_supported() + } +} + +/// DowntimeValues +#[utoipa::path( + post, + path = "/v2/{org_id}/downtimes/values", + context_path = "/api", + tag = "Downtimes", + operation_id = "DowntimeValues", + summary = "Suggested values of a dimension", + description = "Values of one dimension for the condition value box and the resource picker, from the alerts, anomaly detections and SLOs, the service registry and, for a non-priority dimension, a one-hour search. `partial` is true when the search did not answer within 2 s.", + security(("Authorization" = [])), + params(("org_id" = String, Path, description = "Organization name")), + request_body(content = ValuesRequest, description = "The dimension, the typed prefix and the condition so far", content_type = "application/json"), + responses( + (status = 200, description = "Success", content_type = "application/json", body = ValuesResponse), + (status = 400, description = "Invalid request", content_type = "application/json", body = ()), + (status = 403, description = "Forbidden or not enabled", content_type = "application/json", body = ()), + ), +)] +pub async fn downtime_values( + Path(org_id): Path, + Headers(user_email): Headers, + Json(req): Json, +) -> Response { + #[cfg(feature = "enterprise")] + { + respond(openobserve_core::downtimes::values(&org_id, &user_email.user_id, &req).await) + } + #[cfg(not(feature = "enterprise"))] + { + let _ = (org_id, user_email, req); + not_supported() + } +} + +/// GetDowntime +#[utoipa::path( + get, + path = "/v2/{org_id}/downtimes/{downtime_id}", + context_path = "/api", + tag = "Downtimes", + operation_id = "GetDowntime", + summary = "Get a downtime", + security(("Authorization" = [])), + params( + ("org_id" = String, Path, description = "Organization name"), + ("downtime_id" = String, Path, description = "Downtime id"), + ), + responses( + (status = 200, description = "Success", content_type = "application/json", body = DowntimeDetail), + (status = 403, description = "Forbidden or not enabled", content_type = "application/json", body = ()), + (status = 404, description = "Not found", content_type = "application/json", body = ()), + ), +)] +pub async fn get_downtime( + Path((org_id, downtime_id)): Path<(String, String)>, + Headers(user_email): Headers, +) -> Response { + #[cfg(feature = "enterprise")] + { + respond(openobserve_core::downtimes::get(&org_id, &user_email.user_id, &downtime_id).await) + } + #[cfg(not(feature = "enterprise"))] + { + let _ = (org_id, downtime_id, user_email); + not_supported() + } +} + +/// UpdateDowntime +#[utoipa::path( + put, + path = "/v2/{org_id}/downtimes/{downtime_id}", + context_path = "/api", + tag = "Downtimes", + operation_id = "UpdateDowntime", + summary = "Update a downtime", + description = "Edits the condition, targets, schedule, reason and banner. The folder changes only through move.", + security(("Authorization" = [])), + params( + ("org_id" = String, Path, description = "Organization name"), + ("downtime_id" = String, Path, description = "Downtime id"), + ), + request_body(content = DowntimeRequest, description = "Downtime definition", content_type = "application/json"), + responses( + (status = 200, description = "Updated", content_type = "application/json", body = Downtime), + (status = 400, description = "Invalid request", content_type = "application/json", body = ()), + (status = 403, description = "Forbidden or not enabled", content_type = "application/json", body = ()), + (status = 404, description = "Not found", content_type = "application/json", body = ()), + ), +)] +pub async fn update_downtime( + Path((org_id, downtime_id)): Path<(String, String)>, + Headers(user_email): Headers, + Json(req): Json, +) -> Response { + #[cfg(feature = "enterprise")] + { + respond( + openobserve_core::downtimes::update(&org_id, &user_email.user_id, &downtime_id, req) + .await, + ) + } + #[cfg(not(feature = "enterprise"))] + { + let _ = (org_id, downtime_id, user_email, req); + not_supported() + } +} + +/// CancelDowntime +#[utoipa::path( + post, + path = "/v2/{org_id}/downtimes/{downtime_id}/cancel", + context_path = "/api", + tag = "Downtimes", + operation_id = "CancelDowntime", + summary = "End or cancel a downtime", + description = "Ends an active window now, or calls off a scheduled one. The row stays for history.", + security(("Authorization" = [])), + params( + ("org_id" = String, Path, description = "Organization name"), + ("downtime_id" = String, Path, description = "Downtime id"), + ), + responses( + (status = 200, description = "Cancelled", content_type = "application/json", body = Downtime), + (status = 403, description = "Forbidden or not enabled", content_type = "application/json", body = ()), + (status = 404, description = "Not found", content_type = "application/json", body = ()), + ), +)] +pub async fn cancel_downtime( + Path((org_id, downtime_id)): Path<(String, String)>, + Headers(user_email): Headers, +) -> Response { + #[cfg(feature = "enterprise")] + { + respond( + openobserve_core::downtimes::cancel(&org_id, &user_email.user_id, &downtime_id).await, + ) + } + #[cfg(not(feature = "enterprise"))] + { + let _ = (org_id, downtime_id, user_email); + not_supported() + } +} + +/// ExtendDowntime +#[utoipa::path( + post, + path = "/v2/{org_id}/downtimes/{downtime_id}/extend", + context_path = "/api", + tag = "Downtimes", + operation_id = "ExtendDowntime", + summary = "Extend an active downtime", + description = "Moves the end of the active window by `by_secs` or to `until`, exactly one of them. A recurring downtime keeps its schedule; a one-time downtime named \" (extended)\" is created from the end of the current window instead, and its id is returned as `created_id`.", + security(("Authorization" = [])), + params( + ("org_id" = String, Path, description = "Organization name"), + ("downtime_id" = String, Path, description = "Downtime id"), + ), + request_body(content = ExtendDowntimeRequest, description = "The extension", content_type = "application/json"), + responses( + (status = 200, description = "Extended", content_type = "application/json", body = ExtendDowntimeResponse), + (status = 400, description = "Not active, or an invalid extension", content_type = "application/json", body = ()), + (status = 403, description = "Forbidden or not enabled", content_type = "application/json", body = ()), + (status = 404, description = "Not found", content_type = "application/json", body = ()), + (status = 409, description = "The downtime changed since it was read", content_type = "application/json", body = ()), + ), +)] +pub async fn extend_downtime( + Path((org_id, downtime_id)): Path<(String, String)>, + Headers(user_email): Headers, + Json(req): Json, +) -> Response { + #[cfg(feature = "enterprise")] + { + respond( + openobserve_core::downtimes::extend(&org_id, &user_email.user_id, &downtime_id, &req) + .await, + ) + } + #[cfg(not(feature = "enterprise"))] + { + let _ = (org_id, downtime_id, user_email, req); + not_supported() + } +} + +/// MoveDowntimes +#[utoipa::path( + patch, + path = "/v2/{org_id}/downtimes/move", + context_path = "/api", + tag = "Downtimes", + operation_id = "MoveDowntimes", + summary = "Move downtimes to another folder", + security(("Authorization" = [])), + params(("org_id" = String, Path, description = "Organization name")), + request_body(content = MoveDowntimesRequest, description = "Ids and destination folder", content_type = "application/json"), + responses( + (status = 200, description = "Moved"), + (status = 400, description = "Invalid request", content_type = "application/json", body = ()), + (status = 403, description = "Forbidden or not enabled", content_type = "application/json", body = ()), + ), +)] +pub async fn move_downtimes( + Path(org_id): Path, + Headers(user_email): Headers, + Json(req): Json, +) -> Response { + #[cfg(feature = "enterprise")] + { + // The same rule as `resolve_folder`: an empty destination is the default folder, so the + // permission is checked on the folder the rows will land in. + let mut req = req; + if req.dst_folder_id.is_empty() { + req.dst_folder_id = config::meta::folder::DEFAULT_FOLDER.to_string(); + } + if !crate::service::auth::check_folder_write_permissions( + &org_id, + &user_email.user_id, + "downtime_folders", + &req.dst_folder_id, + ) + .await + { + return MetaHttpResponse::forbidden("You cannot move downtimes into this folder."); + } + match openobserve_core::downtimes::move_to_folder(&org_id, &user_email.user_id, &req).await + { + Ok(()) => MetaHttpResponse::ok("downtimes moved"), + Err(e) => error_response(e), + } + } + #[cfg(not(feature = "enterprise"))] + { + let _ = (org_id, user_email, req); + not_supported() + } +} + +/// DeleteDowntime +#[utoipa::path( + delete, + path = "/v2/{org_id}/downtimes/{downtime_id}", + context_path = "/api", + tag = "Downtimes", + operation_id = "DeleteDowntime", + summary = "Delete an ended or cancelled downtime", + security(("Authorization" = [])), + params( + ("org_id" = String, Path, description = "Organization name"), + ("downtime_id" = String, Path, description = "Downtime id"), + ), + responses( + (status = 200, description = "Deleted"), + (status = 403, description = "Forbidden or not enabled", content_type = "application/json", body = ()), + (status = 404, description = "Not found", content_type = "application/json", body = ()), + (status = 409, description = "Active, scheduled, or it corrected SLO slices", content_type = "application/json", body = ()), + ), +)] +pub async fn delete_downtime( + Path((org_id, downtime_id)): Path<(String, String)>, + Headers(user_email): Headers, +) -> Response { + #[cfg(feature = "enterprise")] + { + match openobserve_core::downtimes::delete(&org_id, &user_email.user_id, &downtime_id).await + { + Ok(()) => MetaHttpResponse::ok("downtime deleted"), + Err(e) => error_response(e), + } + } + #[cfg(not(feature = "enterprise"))] + { + let _ = (org_id, downtime_id, user_email); + not_supported() + } +} + +#[cfg(not(feature = "enterprise"))] +fn not_supported() -> Response { + MetaHttpResponse::forbidden("Not Supported") +} + +#[cfg(feature = "enterprise")] +fn respond( + result: Result, +) -> Response { + match result { + Ok(body) => MetaHttpResponse::json(body), + Err(e) => error_response(e), + } +} + +#[cfg(feature = "enterprise")] +fn created(id: String) -> Response { + use axum::response::IntoResponse; + ( + axum::http::StatusCode::CREATED, + Json(CreateDowntimeResponse { id }), + ) + .into_response() +} + +#[cfg(feature = "enterprise")] +fn error_response(e: openobserve_core::downtimes::DowntimeError) -> Response { + use openobserve_core::downtimes::DowntimeError; + match e { + DowntimeError::Disabled => MetaHttpResponse::forbidden("Downtimes are not enabled"), + DowntimeError::NotFound => MetaHttpResponse::not_found("Downtime not found"), + DowntimeError::BadRequest(m) => MetaHttpResponse::bad_request(m), + DowntimeError::Forbidden(m) => MetaHttpResponse::forbidden(m), + DowntimeError::Conflict(m) => MetaHttpResponse::conflict(m), + // The text can carry database errors, so it goes to the log only. + DowntimeError::Internal(m) => { + log::error!("[DOWNTIMES] {m}"); + MetaHttpResponse::internal_error("Something went wrong, see the server log") + } + } +} + +#[cfg(test)] +mod tests { + use axum::http::StatusCode; + + use super::*; + + #[cfg(not(feature = "enterprise"))] + fn user() -> Headers { + Headers(UserEmail { + user_id: "root@example.com".to_string(), + }) + } + + #[cfg(not(feature = "enterprise"))] + #[tokio::test] + async fn every_route_is_forbidden_in_the_oss_build() { + let org = || Path("default".to_string()); + let one = || Path(("default".to_string(), "d1".to_string())); + let body: DowntimeRequest = serde_json::from_value(serde_json::json!({ + "targets": [{ "module": "alerts", "folders": { "kind": "all" } }], + "schedule": { "repeat": "none", "starts_at": 1, "ends_at": 2, "timezone": "UTC", "duration_secs": 60 } + })) + .unwrap(); + let responses = vec![ + list_downtimes(org(), Query(ListDowntimesQuery::default()), user()).await, + create_downtime( + org(), + Query(FolderQuery::default()), + user(), + Json(body.clone()), + ) + .await, + get_downtime(one(), user()).await, + update_downtime(one(), user(), Json(body)).await, + cancel_downtime(one(), user()).await, + extend_downtime(one(), user(), Json(ExtendDowntimeRequest::default())).await, + delete_downtime(one(), user()).await, + ]; + for response in responses { + assert_eq!(response.status(), StatusCode::FORBIDDEN); + } + } + + #[cfg(feature = "enterprise")] + #[tokio::test] + async fn an_internal_error_answers_without_the_database_text() { + let db_text = "error returned from database: relation \"downtimes\" does not exist"; + let response = error_response(openobserve_core::downtimes::DowntimeError::Internal( + db_text.to_string(), + )); + assert_eq!(response.status(), StatusCode::INTERNAL_SERVER_ERROR); + let body = axum::body::to_bytes(response.into_body(), usize::MAX) + .await + .unwrap(); + let body = String::from_utf8_lossy(&body); + assert!(!body.contains("database"), "{body}"); + assert!(body.contains("see the server log"), "{body}"); + } +} diff --git a/src/api/management/src/request/folders.rs b/src/api/management/src/request/folders.rs index b6ab4e2f16..db3873e67d 100644 --- a/src/api/management/src/request/folders.rs +++ b/src/api/management/src/request/folders.rs @@ -52,6 +52,9 @@ fn folder_error_response(value: FolderError) -> Response { FolderError::DeleteWithWorkflows => MetaHttpResponse::bad_request( "Folder contains workflows, please move/delete workflows from folder", ), + FolderError::DeleteWithDowntimes => MetaHttpResponse::bad_request( + "Folder contains downtimes, please move/delete downtimes from folder", + ), FolderError::DeleteWithPrompts => MetaHttpResponse::bad_request( "Folder contains prompts, please move/delete prompts from folder", ), @@ -64,6 +67,25 @@ fn folder_error_response(value: FolderError) -> Response { } } +/// An old node never meets the downtimes folder type while the feature is off. +fn refused_folder_type( + folder_type: config::meta::folder::FolderType, + downtimes_enabled: bool, +) -> Option { + (folder_type == config::meta::folder::FolderType::Downtimes && !downtimes_enabled) + .then(|| MetaHttpResponse::forbidden("Downtimes are not enabled")) +} + +#[cfg(feature = "enterprise")] +fn downtimes_enabled() -> bool { + openobserve_core::downtimes::ensure_enabled().is_ok() +} + +#[cfg(not(feature = "enterprise"))] +fn downtimes_enabled() -> bool { + false +} + /// CreateFolder #[utoipa::path( post, @@ -102,8 +124,12 @@ pub async fn create_folder( Path((org_id, folder_type)): Path<(String, FolderType)>, axum::Json(body): axum::Json, ) -> Response { + let folder_type: config::meta::folder::FolderType = folder_type.into(); + if let Some(refused) = refused_folder_type(folder_type, downtimes_enabled()) { + return refused; + } let folder = body.into(); - match folders::save_folder(&org_id, folder, folder_type.into(), false).await { + match folders::save_folder(&org_id, folder, folder_type, false).await { Ok(folder) => { let body: CreateFolderResponseBody = folder.into(); MetaHttpResponse::json(body) @@ -196,8 +222,16 @@ pub async fn list_folders( #[cfg(feature = "enterprise")] let user_id = Some(user_email.user_id.as_str()); - match folders::list_folders(&org_id, user_id, folder_type.into()).await { + let folder_type = folder_type.into(); + match folders::list_folders(&org_id, user_id, folder_type).await { Ok(folders) => { + #[cfg(feature = "enterprise")] + let folders = if folder_type == config::meta::folder::FolderType::Downtimes { + openobserve_core::downtimes::listable_folders(&org_id, &user_email.user_id, folders) + .await + } else { + folders + }; let body: ListFoldersResponseBody = folders.into(); MetaHttpResponse::json(body) } @@ -269,11 +303,29 @@ pub async fn get_folder( ("x-o2-mcp" = json!({"description": "Get folder by name", "category": "folders"})) ), )] +#[allow(unused_variables)] pub async fn get_folder_by_name( Path((org_id, folder_type, folder_name)): Path<(String, FolderType, String)>, + #[cfg(feature = "enterprise")] Headers(user_email): Headers, ) -> Response { - match folders::get_folder_by_name(&org_id, &folder_name, folder_type.into()).await { + let folder_type = folder_type.into(); + match folders::get_folder_by_name(&org_id, &folder_name, folder_type).await { Ok(folder) => { + #[cfg(feature = "enterprise")] + let folder = if folder_type == config::meta::folder::FolderType::Downtimes { + let listed = openobserve_core::downtimes::listable_folders( + &org_id, + &user_email.user_id, + vec![folder], + ) + .await; + match listed.into_iter().next() { + Some(folder) => folder, + None => return MetaHttpResponse::forbidden("Unauthorized Access"), + } + } else { + folder + }; let body: CreateFolderResponseBody = folder.into(); MetaHttpResponse::json(body) } @@ -602,6 +654,16 @@ mod tests { } } + #[test] + fn a_downtimes_folder_is_forbidden_while_the_flag_is_off() { + use config::meta::folder::FolderType as Stored; + + let refused = refused_folder_type(Stored::Downtimes, false).expect("refused"); + assert_eq!(refused.status().as_u16(), 403); + assert!(refused_folder_type(Stored::Downtimes, true).is_none()); + assert!(refused_folder_type(Stored::Alerts, false).is_none()); + } + #[test] fn test_folder_error_infra_error_is_500() { let err = FolderError::InfraError(infra::errors::Error::Message("db down".to_string())); diff --git a/src/api/management/src/request/mod.rs b/src/api/management/src/request/mod.rs index 2e72a7f0b5..0d6a323781 100644 --- a/src/api/management/src/request/mod.rs +++ b/src/api/management/src/request/mod.rs @@ -34,6 +34,7 @@ pub mod db_monitoring; pub mod discovery; #[cfg(feature = "enterprise")] pub mod domain_management; +pub mod downtimes; #[cfg(feature = "enterprise")] pub mod eval_jobs; #[cfg(feature = "enterprise")] diff --git a/src/api/management/src/request/status/mod.rs b/src/api/management/src/request/status/mod.rs index f176f6cb3b..cb5069e1b5 100644 --- a/src/api/management/src/request/status/mod.rs +++ b/src/api/management/src/request/status/mod.rs @@ -217,6 +217,7 @@ struct ConfigResponse<'a> { query_values_default_num: i64, alert_preview_timerange_minutes: i64, incidents_enabled: bool, + downtimes_enabled: bool, service_streams_enabled: bool, model_pricing_enabled: bool, online_evals_enabled: bool, @@ -482,6 +483,7 @@ pub async fn zo_config( // AI needs o2-ai to answer; with no agent target its buttons could only fail, so report it off. let ai_enabled = enterprise_value!(false, o2cfg.ai.enabled && o2cfg.ai.has_agent_target()); let incidents_enabled = enterprise_value!(false, o2cfg.incidents.enabled); + let downtimes_enabled = enterprise_value!(false, o2cfg.downtimes.enabled); let service_streams_enabled = enterprise_value!(false, o2cfg.service_streams.enabled); // Anomaly detection is on when the enterprise feature is compiled in, unless turned off at // runtime via O2_ANOMALY_DETECTION_DISABLED. When disabled the UI hides the anomaly tab. @@ -614,6 +616,7 @@ pub async fn zo_config( query_values_default_num: cfg.limit.query_values_default_num, alert_preview_timerange_minutes: cfg.limit.alert_preview_timerange_minutes, incidents_enabled, + downtimes_enabled, service_streams_enabled, model_pricing_enabled: cfg.common.model_pricing_enabled, online_evals_enabled, @@ -1838,6 +1841,7 @@ const CACHE_MODULES: &[&str] = &[ "realtime_triggers", "org_users", "compact_retention", + "downtimes", ]; // Helper function to reload cache for a specific module @@ -1857,10 +1861,31 @@ async fn reload_module_cache(module: &str) -> Result<(), anyhow::Error> { "org_users" => db::org_users::cache().await, "org_ingestion_tokens" => db::org_ingestion_tokens::cache().await, "compact_retention" => db::compact::retention::cache().await, + "downtimes" => { + reload_downtimes(enterprise_value!(false, get_o2_config().downtimes.enabled)).await + } _ => Err(anyhow::anyhow!("unsupported module")), } } +/// `all` lists downtimes only with the flag on, so the reply and total match main otherwise. +fn all_cache_modules(downtimes_enabled: bool) -> Vec<&'static str> { + CACHE_MODULES + .iter() + .copied() + .filter(|module| downtimes_enabled || *module != "downtimes") + .collect() +} + +/// With the flag off the cache stays empty, so no evaluation or SLO pass reads a downtime. +async fn reload_downtimes(enabled: bool) -> Result<(), anyhow::Error> { + if enabled { + db::downtimes::cache().await + } else { + Ok(()) + } +} + pub async fn cache_reload( Query(query): Query>, ) -> Response { @@ -1880,7 +1905,8 @@ pub async fn cache_reload( // Expand "all" to all available modules if modules.contains(&"all") { - modules = CACHE_MODULES.to_vec(); + let downtimes_enabled = enterprise_value!(false, get_o2_config().downtimes.enabled); + modules = all_cache_modules(downtimes_enabled); } let total_modules = modules.len(); @@ -1932,6 +1958,19 @@ mod tests { use super::*; + #[test] + fn with_downtimes_off_all_names_the_modules_main_names() { + assert!(!all_cache_modules(false).contains(&"downtimes")); + assert_eq!(all_cache_modules(false).len(), CACHE_MODULES.len() - 1); + assert!(all_cache_modules(true).contains(&"downtimes")); + } + + #[tokio::test] + async fn with_downtimes_off_a_cache_reload_loads_no_downtime() { + reload_downtimes(false).await.unwrap(); + assert!(db::downtimes::all_cached().is_empty()); + } + #[cfg(feature = "enterprise")] #[tokio::test] async fn test_config_rbac_enabled_matches_enforcement_before_any_usage_report() { diff --git a/src/api/management/src/request/synthetics/last_check.rs b/src/api/management/src/request/synthetics/last_check.rs index 38507b6128..8408d5e294 100644 --- a/src/api/management/src/request/synthetics/last_check.rs +++ b/src/api/management/src/request/synthetics/last_check.rs @@ -194,6 +194,7 @@ mod tests { referenced_by: 0, references: None, reference_state: None, + active_downtime: None, } } diff --git a/src/api/management/src/request/synthetics/mod.rs b/src/api/management/src/request/synthetics/mod.rs index a7dbd8a0d9..92c97fa74d 100644 --- a/src/api/management/src/request/synthetics/mod.rs +++ b/src/api/management/src/request/synthetics/mod.rs @@ -447,6 +447,7 @@ pub async fn list_synthetics( match openobserve_synthetics::service::list_synthetics(&org_id, ¶ms).await { Ok(mut resp) => { last_check::enrich(&org_id, &mut resp.checks).await; + openobserve_core::synthetics::fill_active_downtimes(&org_id, &mut resp.checks); MetaHttpResponse::json(resp) } Err(e) => { @@ -1577,12 +1578,20 @@ async fn process_ack( key: format!("{}/{}", resp.synthetics_name, resp.synthetics_id), start_time: checked_at, end_time: checked_at, - status: config::meta::self_reporting::usage::RunOutcome::Succeeded, + status: if resp.suppressed_by.is_some() { + config::meta::self_reporting::usage::RunOutcome::Suppressed + } else { + config::meta::self_reporting::usage::RunOutcome::Succeeded + }, + downtime_id: resp.suppressed_by.clone(), success_response: Some(status.clone()), error: error.clone(), evaluation_took_in_secs: Some(response_time_ms / 1000.0), ..Default::default() }); + if resp.suppressed_by.is_some() { + openobserve_core::alerts::alert::count_suppressed_run(&resp.org_id, "synthetics"); + } // Notify once per run, not once per job ack — and only when the check's own // `alert_if_fails` / `cooldown_mins` settings say so. This used to fire on @@ -1600,7 +1609,8 @@ async fn process_ack( #[cfg(feature = "enterprise")] let degraded = matches!(resp.alert, AlertDecision::Degraded); #[cfg(feature = "enterprise")] - let should_notify = !matches!(resp.alert, AlertDecision::Silent); + let should_notify = + !matches!(resp.alert, AlertDecision::Silent) && resp.suppressed_by.is_none(); // Enterprise-gated because alert *destinations* are: the dispatch it ends in // (`alerts::alert::dispatch_notification`) is not built in OSS. An OSS build // runs the check, records the run and serves the result — it just has @@ -2281,6 +2291,7 @@ mod tests { passing_locations: Vec::new(), failing_environments: Vec::new(), usage_events, + suppressed_by: None, } } diff --git a/src/common/src/meta/organization.rs b/src/common/src/meta/organization.rs index d1ebd58fa7..3703ec4e35 100644 --- a/src/common/src/meta/organization.rs +++ b/src/common/src/meta/organization.rs @@ -334,7 +334,8 @@ impl TriggerStatus { usage::RunOutcome::Firing | usage::RunOutcome::Normal | usage::RunOutcome::Succeeded - | usage::RunOutcome::Pending => status.healthy += 1, + | usage::RunOutcome::Pending + | usage::RunOutcome::Suppressed => status.healthy += 1, usage::RunOutcome::Error | usage::RunOutcome::NotifyFailed => { status.failed += 1 } diff --git a/src/config/src/config.rs b/src/config/src/config.rs index aae85da732..8a1852a453 100644 --- a/src/config/src/config.rs +++ b/src/config/src/config.rs @@ -98,7 +98,9 @@ pub type RwBTreeMap = tokio::sync::RwLock>; // 98: key alert_dedup_state by (org_id, fingerprint). // 99: add name to rum_pa_tombstones. // 100: rule mode and raw transition values require new columns. -pub const DB_SCHEMA_VERSION: u64 = 100; +// 101: create downtimes; muted_by_downtime_id, last_downtime_id, slo_backfill_jobs kind/attempts. +// 102: create downtime_notifications. +pub const DB_SCHEMA_VERSION: u64 = 102; pub const DB_SCHEMA_KEY: &str = "/db_schema_version/"; // global version variables diff --git a/src/config/src/meta/alerts/dispatch.rs b/src/config/src/meta/alerts/dispatch.rs index 5cdc77c1c9..82936992fe 100644 --- a/src/config/src/meta/alerts/dispatch.rs +++ b/src/config/src/meta/alerts/dispatch.rs @@ -110,6 +110,8 @@ pub struct DispatchPlan { /// failures rather than routine outcomes: reported for logging, and /// deliberately NOT dispatched — see [`plan_dispatch`]. pub inconsistent: Vec, + /// Groups that would deliver but a downtime mutes; also counted in `suppressed`. + pub muted: Vec, } /// Decide which groups this evaluation notifies (MN-1, MN-8). @@ -132,6 +134,7 @@ pub struct DispatchPlan { /// key twice would otherwise notify the group twice and consume two slots of the MN-8 budget. /// * `max_sends` is the MN-8 knob: `0` = unlimited; otherwise it counts **qualifying** sends /// (applied after `delivery_decision`), worst-first. +/// * A deliverable group for which `muted` is true goes to `muted` instead, before the knob. pub fn plan_dispatch( classification: &GroupClassification, states: &HashMap, @@ -139,12 +142,14 @@ pub fn plan_dispatch( tc: &TriggerCondition, now: i64, max_sends: usize, + muted: impl Fn(&str) -> bool, ) -> DispatchPlan { let mut items = Vec::new(); let mut suppressed = 0usize; let mut pending = 0; let mut dropped_by_knob = Vec::new(); let mut inconsistent = Vec::new(); + let mut muted_keys = Vec::new(); // A key appearing twice in one classification is an invariant violation // (validation rejects the known source, MN-12). Counting first means both @@ -207,6 +212,12 @@ pub fn plan_dispatch( }; match decision { DeliveryDecision::Deliver | DeliveryDecision::DeliverEscalation => { + // A muted group is suppressed by its downtime before the knob, so it fills no slot. + if muted(key) { + suppressed += 1; + muted_keys.push(key.clone()); + continue; + } // The knob counts QUALIFYING sends, so a silenced group never // spends budget a deliverable one could have used. if max_sends > 0 && items.len() >= max_sends { @@ -223,9 +234,9 @@ pub fn plan_dispatch( row: row.clone(), }); } - DeliveryDecision::SuppressedBySilence | DeliveryDecision::SuppressedByWarningPolicy => { - suppressed += 1 - } + DeliveryDecision::SuppressedBySilence + | DeliveryDecision::SuppressedByWarningPolicy + | DeliveryDecision::SuppressedByDowntime(_) => suppressed += 1, DeliveryDecision::SuppressedByPending => pending += 1, DeliveryDecision::NotFiring => {} } @@ -237,6 +248,7 @@ pub fn plan_dispatch( pending, dropped_by_knob, inconsistent, + muted: muted_keys, } } @@ -722,7 +734,15 @@ mod tests { let c = classified(&[("a", 150.0), ("b", 75.0), ("c", 10.0)], 500); let states = states_for(&c, 1_000 * SEC, None, None); - let plan = plan_dispatch(&c, &states, &rows_for(&c), &tc(None), 1_000 * SEC, 0); + let plan = plan_dispatch( + &c, + &states, + &rows_for(&c), + &tc(None), + 1_000 * SEC, + 0, + |_| false, + ); assert_eq!( plan.items.len(), @@ -746,7 +766,7 @@ mod tests { let c = classified(&[("crit", 150.0), ("ok1", 1.0), ("ok2", 2.0)], 500); let states = states_for(&c, now, None, None); - let plan = plan_dispatch(&c, &states, &rows_for(&c), &tc(None), now, 0); + let plan = plan_dispatch(&c, &states, &rows_for(&c), &tc(None), now, 0, |_| false); assert_eq!(plan.items.len(), 1); assert_eq!( @@ -766,7 +786,7 @@ mod tests { let c = classified(&[("a", 150.0)], 500); let states = states_for(&c, now, None, None); - let plan = plan_dispatch(&c, &states, &rows_for(&c), &tc(None), now, 0); + let plan = plan_dispatch(&c, &states, &rows_for(&c), &tc(None), now, 0, |_| false); assert!( plan.items[0].escalation, @@ -782,7 +802,15 @@ mod tests { let c = classified(&[("a", 150.0), ("b", 75.0)], 500); let states = states_for(&c, 1_000 * SEC, None, None); - let plan = plan_dispatch(&c, &states, &rows_for(&c), &tc(None), 1_000 * SEC, 0); + let plan = plan_dispatch( + &c, + &states, + &rows_for(&c), + &tc(None), + 1_000 * SEC, + 0, + |_| false, + ); let b = plan .items .iter() @@ -799,7 +827,15 @@ mod tests { let c = classified(&[("warn", 75.0), ("crit2", 150.0), ("crit1", 150.0)], 500); let states = states_for(&c, 1_000 * SEC, None, None); - let plan = plan_dispatch(&c, &states, &rows_for(&c), &tc(None), 1_000 * SEC, 0); + let plan = plan_dispatch( + &c, + &states, + &rows_for(&c), + &tc(None), + 1_000 * SEC, + 0, + |_| false, + ); let keys: Vec<_> = plan.items.iter().map(|i| i.group_key.clone()).collect(); assert_eq!(plan.items[0].level, AlertLevel::Critical); @@ -820,7 +856,7 @@ mod tests { let c = classified(&[("a", 150.0)], 500); let states = states_for(&c, now, Some(AlertLevel::Critical), Some(now + 10 * MIN)); - let plan = plan_dispatch(&c, &states, &rows_for(&c), &tc(None), now, 0); + let plan = plan_dispatch(&c, &states, &rows_for(&c), &tc(None), now, 0, |_| false); assert!(plan.items.is_empty()); assert_eq!(plan.suppressed, 1); @@ -834,7 +870,7 @@ mod tests { let c = classified(&[("a", 150.0)], 500); let states = states_for(&c, now, Some(AlertLevel::Critical), Some(now - 1)); - let plan = plan_dispatch(&c, &states, &rows_for(&c), &tc(None), now, 0); + let plan = plan_dispatch(&c, &states, &rows_for(&c), &tc(None), now, 0, |_| false); assert_eq!(plan.items.len(), 1); assert!( @@ -851,7 +887,7 @@ mod tests { let c = classified(&[("a", 150.0)], 500); let states = states_for(&c, now, Some(AlertLevel::Warning), Some(now + 10 * MIN)); - let plan = plan_dispatch(&c, &states, &rows_for(&c), &tc(None), now, 0); + let plan = plan_dispatch(&c, &states, &rows_for(&c), &tc(None), now, 0, |_| false); assert_eq!(plan.items.len(), 1); assert!(plan.items[0].escalation); @@ -884,7 +920,7 @@ mod tests { assert!(row.is_firing(), "NotifyFailed is still a firing state"); let states = HashMap::from([(key.clone(), row)]); - let plan = plan_dispatch(&c, &states, &rows_for(&c), &tc(None), now, 0); + let plan = plan_dispatch(&c, &states, &rows_for(&c), &tc(None), now, 0, |_| false); assert_eq!( plan.items.len(), @@ -900,7 +936,15 @@ mod tests { let c = classified(&[("crit", 150.0), ("warn", 75.0)], 500); let states = states_for(&c, 1_000 * SEC, None, None); - let plan = plan_dispatch(&c, &states, &rows_for(&c), &tc(Some(false)), 1_000 * SEC, 0); + let plan = plan_dispatch( + &c, + &states, + &rows_for(&c), + &tc(Some(false)), + 1_000 * SEC, + 0, + |_| false, + ); assert_eq!(plan.items.len(), 1); assert_eq!(plan.items[0].group_key, key_of("crit")); @@ -940,7 +984,7 @@ mod tests { ]), ); - let plan = plan_dispatch(&c, &states, &rows, &tc(None), now, 0); + let plan = plan_dispatch(&c, &states, &rows, &tc(None), now, 0, |_| false); assert_eq!(plan.items.len(), 2, "only retained groups dispatch"); assert!( @@ -971,7 +1015,7 @@ mod tests { state(ROLLUP_GROUP_KEY, AlertLevel::Critical, now, None, None), ); - let plan = plan_dispatch(&c, &states, &rows_for(&c), &tc(None), now, 0); + let plan = plan_dispatch(&c, &states, &rows_for(&c), &tc(None), now, 0, |_| false); assert_eq!(plan.items.len(), 1, "one group fired, so one page"); assert_eq!(plan.items[0].group_key, key_of("a")); @@ -999,7 +1043,7 @@ mod tests { state(&gone, AlertLevel::Critical, now - 5 * MIN, None, None), ); - let plan = plan_dispatch(&c, &states, &rows_for(&c), &tc(None), now, 0); + let plan = plan_dispatch(&c, &states, &rows_for(&c), &tc(None), now, 0, |_| false); assert!(!plan.items.iter().any(|i| i.group_key == gone)); } @@ -1014,7 +1058,15 @@ mod tests { let now = 1_000 * SEC; let c = classified(&[("a", 150.0)], 500); - let plan = plan_dispatch(&c, &HashMap::new(), &rows_for(&c), &tc(None), now, 0); + let plan = plan_dispatch( + &c, + &HashMap::new(), + &rows_for(&c), + &tc(None), + now, + 0, + |_| false, + ); assert!(plan.items.is_empty(), "no durable row, no page"); assert_eq!( @@ -1033,7 +1085,7 @@ mod tests { let c = classified(&[("a", 150.0)], 500); let states = states_for(&c, now, None, None); - let plan = plan_dispatch(&c, &states, &HashMap::new(), &tc(None), now, 0); + let plan = plan_dispatch(&c, &states, &HashMap::new(), &tc(None), now, 0, |_| false); assert!(plan.items.is_empty()); assert_eq!(plan.inconsistent, vec![key_of("a")]); @@ -1060,7 +1112,7 @@ mod tests { assert_eq!(c.groups.len(), 3, "fixture: `a` is observed twice"); let states = states_for(&c, now, None, None); - let plan = plan_dispatch(&c, &states, &rows_for(&c), &tc(None), now, 0); + let plan = plan_dispatch(&c, &states, &rows_for(&c), &tc(None), now, 0, |_| false); assert!( !plan.items.iter().any(|i| i.group_key == key_of("a")), @@ -1089,7 +1141,7 @@ mod tests { assert_eq!(c.groups.len(), 3, "fixture: `a` observed at two levels"); let states = states_for(&c, now, None, None); - let plan = plan_dispatch(&c, &states, &rows_for(&c), &tc(None), now, 0); + let plan = plan_dispatch(&c, &states, &rows_for(&c), &tc(None), now, 0, |_| false); for item in &plan.items { assert_ne!( @@ -1119,7 +1171,7 @@ mod tests { ); let states = states_for(&c, now, None, None); - let plan = plan_dispatch(&c, &states, &rows_for(&c), &tc(None), now, 2); + let plan = plan_dispatch(&c, &states, &rows_for(&c), &tc(None), now, 2, |_| false); assert_eq!(plan.items.len(), 2, "`b` and `c` both page"); assert!(plan.dropped_by_knob.is_empty()); @@ -1134,7 +1186,7 @@ mod tests { let states = states_for(&c, now, None, None); let rows = rows_for(&c); - let plan = plan_dispatch(&c, &states, &rows, &tc(None), now, 0); + let plan = plan_dispatch(&c, &states, &rows, &tc(None), now, 0, |_| false); assert_eq!( &plan.items[0].row, @@ -1158,7 +1210,7 @@ mod tests { row.level_at = Some(now); // freshness advanced by a later same-level run let states = HashMap::from([(key.clone(), row)]); - let plan = plan_dispatch(&c, &states, &rows_for(&c), &tc(None), now, 0); + let plan = plan_dispatch(&c, &states, &rows_for(&c), &tc(None), now, 0, |_| false); assert_eq!( plan.items[0].episode, @@ -1176,7 +1228,15 @@ mod tests { fn test_knob_zero_is_unlimited() { let c = classified(&[("a", 150.0), ("b", 150.0), ("c", 150.0)], 500); let states = states_for(&c, 1_000 * SEC, None, None); - let plan = plan_dispatch(&c, &states, &rows_for(&c), &tc(None), 1_000 * SEC, 0); + let plan = plan_dispatch( + &c, + &states, + &rows_for(&c), + &tc(None), + 1_000 * SEC, + 0, + |_| false, + ); assert_eq!(plan.items.len(), 3); assert!(plan.dropped_by_knob.is_empty()); } @@ -1188,7 +1248,15 @@ mod tests { let c = classified(&[("crit", 150.0), ("warn1", 75.0), ("warn2", 75.0)], 500); let states = states_for(&c, 1_000 * SEC, None, None); - let plan = plan_dispatch(&c, &states, &rows_for(&c), &tc(None), 1_000 * SEC, 2); + let plan = plan_dispatch( + &c, + &states, + &rows_for(&c), + &tc(None), + 1_000 * SEC, + 2, + |_| false, + ); assert_eq!(plan.items.len(), 2); assert_eq!( @@ -1221,8 +1289,10 @@ mod tests { let c = build(); let states = states_for(&c, now, None, None); - let first = plan_dispatch(&c, &states, &rows_for(&c), &tc(None), now, 2); - let second = plan_dispatch(&build(), &states, &rows_for(&c), &tc(None), now, 2); + let first = plan_dispatch(&c, &states, &rows_for(&c), &tc(None), now, 2, |_| false); + let second = plan_dispatch(&build(), &states, &rows_for(&c), &tc(None), now, 2, |_| { + false + }); assert_eq!(first.items, second.items); assert_eq!(first.dropped_by_knob, second.dropped_by_knob); @@ -1254,7 +1324,7 @@ mod tests { ), ]); - let plan = plan_dispatch(&c, &states, &rows_for(&c), &tc(None), now, 1); + let plan = plan_dispatch(&c, &states, &rows_for(&c), &tc(None), now, 1, |_| false); assert_eq!(plan.items.len(), 1); assert_eq!(plan.items[0].group_key, key_b); @@ -1265,6 +1335,39 @@ mod tests { ); } + #[test] + fn test_a_muted_group_does_not_spend_the_knob() { + let now = 1_000 * SEC; + let c = classified(&[("a", 150.0), ("b", 75.0)], 500); + let (key_a, key_b) = (key_of("a"), key_of("b")); + let states = states_for(&c, now, None, None); + for _ in 0..2 { + let plan = plan_dispatch(&c, &states, &rows_for(&c), &tc(None), now, 1, |k| { + k == key_a + }); + assert_eq!(plan.items.len(), 1); + assert_eq!(plan.items[0].group_key, key_b); + assert_eq!(plan.suppressed, 1); + assert_eq!(plan.muted, vec![key_a.clone()]); + assert!(plan.dropped_by_knob.is_empty()); + } + } + + #[test] + fn test_a_muted_group_with_knob_zero_is_suppressed_and_the_rest_deliver() { + let now = 1_000 * SEC; + let c = classified(&[("a", 150.0), ("b", 75.0)], 500); + let (key_a, key_b) = (key_of("a"), key_of("b")); + let states = states_for(&c, now, None, None); + let plan = plan_dispatch(&c, &states, &rows_for(&c), &tc(None), now, 0, |k| { + k == key_a + }); + assert_eq!(plan.items.len(), 1); + assert_eq!(plan.items[0].group_key, key_b); + assert_eq!(plan.muted, vec![key_a]); + assert!(plan.dropped_by_knob.is_empty()); + } + // ═════════════════════════════════════════════════════════════════════ // delivery_success_update — MN-6 // ═════════════════════════════════════════════════════════════════════ diff --git a/src/config/src/meta/alerts/incidents.rs b/src/config/src/meta/alerts/incidents.rs index 1d88b7530c..2723ea540f 100644 --- a/src/config/src/meta/alerts/incidents.rs +++ b/src/config/src/meta/alerts/incidents.rs @@ -384,6 +384,10 @@ pub struct Incident { #[serde(skip_serializing_if = "Option::is_none")] pub topology_context: Option, + + /// Set while a downtime mutes this incident (D3). + #[serde(default, skip_serializing_if = "Option::is_none")] + pub muted_by_downtime_id: Option, } /// Alert info within an incident (junction table representation) @@ -1872,6 +1876,7 @@ mod tests { group_values: serde_json::Value::Null, key_type: KeyType::default(), topology_context, + muted_by_downtime_id: None, } } @@ -1886,6 +1891,19 @@ mod tests { assert!(!obj.contains_key("topology_context")); assert!(!obj.contains_key("acknowledged_by")); assert!(!obj.contains_key("acknowledged_at")); + assert!(!obj.contains_key("muted_by_downtime_id")); + } + + #[test] + fn a_muted_incident_names_its_downtime_and_old_rows_read_unmuted() { + let mut incident = make_incident(None, None, None, None); + incident.muted_by_downtime_id = Some("2f9K".to_string()); + let json = serde_json::to_value(&incident).unwrap(); + assert_eq!(json["muted_by_downtime_id"], "2f9K"); + let mut old = json.as_object().unwrap().clone(); + old.remove("muted_by_downtime_id"); + let back: Incident = serde_json::from_value(serde_json::Value::Object(old)).unwrap(); + assert_eq!(back.muted_by_downtime_id, None); } #[test] diff --git a/src/config/src/meta/alerts/level.rs b/src/config/src/meta/alerts/level.rs index 17e2417ede..a5c3cd40a7 100644 --- a/src/config/src/meta/alerts/level.rs +++ b/src/config/src/meta/alerts/level.rs @@ -348,7 +348,7 @@ pub fn level_for_completed_evaluation( } /// What a completed evaluation should do about notification (§7.1). -#[derive(Debug, Clone, Copy, PartialEq, Eq)] +#[derive(Debug, Clone, PartialEq, Eq)] pub enum DeliveryDecision { /// Deliver, and (re)start the silence window from now. Deliver, @@ -362,6 +362,8 @@ pub enum DeliveryDecision { NotFiring, /// Currently in pending state SuppressedByPending, + /// Condition matched inside a window of this downtime. + SuppressedByDowntime(String), } impl DeliveryDecision { @@ -373,6 +375,13 @@ impl DeliveryDecision { pub fn resets_silence(&self) -> bool { self.should_deliver() } + + pub fn downtime_id(&self) -> Option<&str> { + match self { + Self::SuppressedByDowntime(id) => Some(id), + _ => None, + } + } } /// Decide whether this evaluation delivers a notification. @@ -1096,6 +1105,15 @@ mod tests { assert!(!d.should_deliver()); } + #[test] + fn a_downtime_suppression_never_delivers_and_names_its_downtime() { + let d = DeliveryDecision::SuppressedByDowntime("2f9K".to_string()); + assert!(!d.should_deliver()); + assert!(!d.resets_silence()); + assert_eq!(d.downtime_id(), Some("2f9K")); + assert_eq!(DeliveryDecision::Deliver.downtime_id(), None); + } + // ── D11: notify_on_warning ────────────────────────────────────────────── #[test] diff --git a/src/config/src/meta/alerts/mod.rs b/src/config/src/meta/alerts/mod.rs index 58e3f0e9b0..b2b789995f 100644 --- a/src/config/src/meta/alerts/mod.rs +++ b/src/config/src/meta/alerts/mod.rs @@ -367,6 +367,11 @@ impl TriggerCondition { .max(REALTIME_MIN_SILENCE_SECS) .saturating_mul(1_000_000) } + + /// The realtime floor alone, without the alert's own silence. + pub fn min_silence_micros() -> i64 { + REALTIME_MIN_SILENCE_SECS * 1_000_000 + } } impl TriggerCondition { diff --git a/src/config/src/meta/downtimes.rs b/src/config/src/meta/downtimes.rs new file mode 100644 index 0000000000..4a549a00b1 --- /dev/null +++ b/src/config/src/meta/downtimes.rs @@ -0,0 +1,942 @@ +// Copyright 2026 OpenObserve Inc. +// +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// This program is distributed in the hope that it will be useful +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. +// +// You should have received a copy of the GNU Affero General Public License +// along with this program. If not, see . + +use std::collections::HashMap; + +use serde::{Deserialize, Serialize}; +use utoipa::{IntoParams, ToSchema}; + +use crate::meta::folder::DEFAULT_FOLDER; + +pub const DEFAULT_ENDING_SOON_LEAD_SECS: i64 = 600; + +#[derive(Clone, Debug, PartialEq, Serialize, Deserialize, ToSchema)] +pub struct Downtime { + pub id: String, + pub org: String, + #[serde(default = "default_folder")] + pub folder_id: String, + pub name: String, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub reason: Option, + /// One per downtime (D20). Applied to every target whose module has an identity. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub condition: Option, + pub targets: Vec, + pub schedule: DowntimeSchedule, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub cancelled_at: Option, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub cancelled_by: Option, + /// Show a warning banner to everyone in the org while a window is active (D19). + #[serde(default = "default_true")] + pub show_banner: bool, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub notifications: Option, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub origin_region: Option, + #[serde(default)] + pub version: i64, + pub created_by: String, + pub created_at: i64, + pub updated_by: String, + pub updated_at: i64, +} + +impl Downtime { + /// Same items covered: only the condition and the targets decide which items match. + pub fn same_coverage(&self, other: &Downtime) -> bool { + self.condition == other.condition && self.targets == other.targets + } +} + +/// At most one per module (D14); every present field narrows the module's items (D15). +#[derive(Clone, Debug, PartialEq, Serialize, Deserialize, ToSchema)] +pub struct DowntimeTarget { + pub module: TargetModule, + pub folders: TargetFolders, + /// Synthetics only. + #[serde(default, skip_serializing_if = "Vec::is_empty")] + pub tags: Vec, + #[serde(default, skip_serializing_if = "Vec::is_empty")] + pub ids: Vec, + /// Read only when `module == Slos` (D10). + #[serde(default, skip_serializing_if = "Option::is_none")] + pub slo_mode: Option, + /// Read only when `module == Alerts`; a row saved before the field reads as `Muted`. + #[serde(default, skip_serializing_if = "IncidentMode::is_muted")] + pub incident_mode: IncidentMode, +} + +#[derive(Clone, Debug, PartialEq, Serialize, Deserialize, ToSchema)] +#[serde(tag = "kind", rename_all = "snake_case")] +pub enum TargetFolders { + All, + Some { folder_ids: Vec }, +} + +/// A tree of dimension tests (D17); one AND group of `=` pairs is an on-call `OwnershipRule`. +#[derive(Clone, Debug, PartialEq, Serialize, Deserialize, ToSchema)] +#[serde(tag = "type", rename_all = "snake_case")] +pub enum DimensionCondition { + Group { + op: LogicalOp, + // An opaque item keeps the schema acyclic: the MCP tool generator rejects self-references. + #[schema(value_type = Vec)] + items: Vec, + }, + /// `value` may end in `*`, a prefix, as in ownership rules. + Pair { + key: String, + operator: PairOperator, + value: String, + }, +} + +#[derive(Clone, Copy, Debug, PartialEq, Eq, Serialize, Deserialize, ToSchema)] +#[serde(rename_all = "snake_case")] +pub enum LogicalOp { + And, + Or, +} + +#[derive(Clone, Copy, Debug, PartialEq, Eq, Serialize, Deserialize, ToSchema)] +pub enum PairOperator { + #[serde(rename = "=")] + Eq, + #[serde(rename = "!=")] + Ne, +} + +#[derive(Clone, Copy, Debug, PartialEq, Eq, Hash, Serialize, Deserialize, ToSchema)] +#[serde(rename_all = "snake_case")] +pub enum TargetModule { + Alerts, + AnomalyDetections, + Synthetics, + Slos, +} + +impl TargetModule { + pub fn has_identity(self) -> bool { + !matches!(self, Self::Synthetics) + } + + pub fn has_tags(self) -> bool { + matches!(self, Self::Synthetics) + } +} + +#[derive(Clone, Debug, PartialEq, Serialize, Deserialize, ToSchema)] +pub struct DowntimeSchedule { + pub repeat: Repeat, + /// Microseconds UTC. + pub starts_at: i64, + /// Microseconds UTC; a recurring row reads only its local date, an inclusive last day. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub ends_at: Option, + pub timezone: String, + /// `HH:MM` in `timezone`, recurring rows only. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub start_time_local: Option, + pub duration_secs: i64, + /// ISO weekdays, 1 is Monday. + #[serde(default)] + pub weekdays: Vec, +} + +#[derive(Clone, Copy, Debug, Default, PartialEq, Eq, Serialize, Deserialize, ToSchema)] +#[serde(rename_all = "snake_case")] +pub enum Repeat { + #[default] + None, + Daily, + Weekly, +} + +impl Repeat { + /// Stored in `downtimes.repeat`; never reorder, 3 is reserved for monthly. + pub fn to_i16(self) -> i16 { + match self { + Self::None => 0, + Self::Daily => 1, + Self::Weekly => 2, + } + } + + pub fn from_i16(value: i16) -> Option { + match value { + 0 => Some(Self::None), + 1 => Some(Self::Daily), + 2 => Some(Self::Weekly), + _ => None, + } + } +} + +#[derive(Clone, Copy, Debug, PartialEq, Eq, Serialize, Deserialize, ToSchema)] +#[serde(rename_all = "snake_case")] +pub enum DowntimeStatus { + Scheduled, + Active, + Ended, + Cancelled, + /// Cancelled inside a window, which it cut short. + EndedEarly, +} + +#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize, ToSchema)] +pub struct DowntimeWindow { + pub start: i64, + pub end: i64, +} + +#[derive(Clone, Debug, Serialize, ToSchema)] +pub struct DowntimeListItem { + #[serde(flatten)] + pub downtime: Downtime, + pub status: DowntimeStatus, + pub current_window: Option, + pub next_window: Option, + pub matched_alerts: usize, + pub matched_anomalies: usize, + pub matched_synthetics: usize, + pub matched_slos: usize, +} + +#[derive(Clone, Debug, Deserialize, ToSchema)] +pub struct DowntimeRequest { + #[serde(default = "default_folder")] + pub folder_id: String, + #[serde(default)] + pub name: Option, + #[serde(default)] + pub reason: Option, + #[serde(default)] + pub condition: Option, + pub targets: Vec, + pub schedule: DowntimeSchedule, + #[serde(default = "default_true")] + pub show_banner: bool, + #[serde(default)] + pub notifications: Option, + /// The version the edit was loaded at; a newer stored row makes the edit a 409. + #[serde(default)] + pub version: Option, +} + +#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize, ToSchema)] +pub struct DowntimeNotifications { + #[serde(default)] + pub destinations: Vec, + #[serde(default)] + pub events: NotificationEvents, + #[serde(default = "default_ending_soon_lead_secs")] + pub ending_soon_lead_secs: i64, + /// The id of the recurring downtime whose window this extension follow-up continues. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub continues: Option, +} + +impl Default for DowntimeNotifications { + fn default() -> Self { + Self { + destinations: vec![], + events: NotificationEvents::default(), + ending_soon_lead_secs: DEFAULT_ENDING_SOON_LEAD_SECS, + continues: None, + } + } +} + +#[derive(Clone, Copy, Debug, Default, PartialEq, Eq, Serialize, Deserialize, ToSchema)] +pub struct NotificationEvents { + #[serde(default)] + pub started: bool, + #[serde(default)] + pub ending_soon: bool, + #[serde(default)] + pub ended: bool, + #[serde(default)] + pub cancelled: bool, + #[serde(default)] + pub extended: bool, +} + +impl NotificationEvents { + pub fn any(&self) -> bool { + self.started || self.ending_soon || self.ended || self.cancelled || self.extended + } + + pub fn wants(&self, event: NotificationEvent) -> bool { + match event { + NotificationEvent::Started => self.started, + NotificationEvent::EndingSoon => self.ending_soon, + NotificationEvent::Ended => self.ended, + NotificationEvent::Cancelled => self.cancelled, + NotificationEvent::Extended => self.extended, + } + } +} + +#[derive(Clone, Copy, Debug, PartialEq, Eq, Hash, Serialize, Deserialize, ToSchema)] +#[serde(rename_all = "snake_case")] +pub enum NotificationEvent { + Started, + EndingSoon, + Ended, + Cancelled, + Extended, +} + +impl NotificationEvent { + /// Stored in `downtime_notifications.event`; never rename. + pub fn as_str(self) -> &'static str { + match self { + Self::Started => "started", + Self::EndingSoon => "ending_soon", + Self::Ended => "ended", + Self::Cancelled => "cancelled", + Self::Extended => "extended", + } + } +} + +#[derive(Clone, Debug, PartialEq, Eq, Serialize, ToSchema)] +pub struct DowntimeNotificationLogEntry { + pub window_start: i64, + pub event: String, + pub sent_at: i64, + pub destinations: Vec, + #[serde(skip_serializing_if = "Option::is_none")] + pub result: Option, +} + +#[derive(Clone, Debug, Deserialize, ToSchema)] +pub struct ResourcesRequest { + pub condition: DimensionCondition, + pub refine_by: String, +} + +#[derive(Clone, Debug, Serialize, ToSchema)] +pub struct ResourceValue { + pub value: String, + pub last_seen: i64, + /// Flattened from `ServiceStreams` as `/`. + pub streams: Vec, +} + +#[derive(Clone, Debug, Serialize, ToSchema)] +pub struct ResourcesResponse { + pub dimension: String, + pub values: Vec, + pub total: usize, + pub source: String, +} + +#[derive(Clone, Debug, Deserialize, ToSchema)] +pub struct ValuesRequest { + /// A semantic group id, as `refine_by` in `ResourcesRequest`. + pub key: String, + /// Case-insensitive prefix; empty lists the top values. + #[serde(default)] + pub prefix: String, + /// The condition typed so far; only its And-spine pairs narrow the sources. + #[serde(default)] + pub condition: Option, +} + +#[derive(Clone, Debug, PartialEq, Eq, Serialize, ToSchema)] +pub struct ValueSuggestion { + pub value: String, + /// `inventory`, `registry` or `search`. + pub source: String, + /// Alerts, anomaly detections and SLOs whose identity carries this value. + pub items: usize, + /// Newest sighting in microseconds, when a source knows one. + #[serde(skip_serializing_if = "Option::is_none")] + pub last_seen: Option, +} + +#[derive(Clone, Debug, PartialEq, Eq, Serialize, ToSchema)] +pub struct ValuesResponse { + pub values: Vec, + /// True when a source timed out; the list is what answered in time. + pub partial: bool, +} + +#[derive(Clone, Copy, Debug, Default, PartialEq, Eq, Serialize, Deserialize, ToSchema)] +#[serde(rename_all = "snake_case")] +pub enum SloCorrectionMode { + #[default] + Exclude, + CountAsGood, +} + +/// What a suppressed alert firing does to incidents during the window. +#[derive(Clone, Copy, Debug, Default, PartialEq, Eq, Serialize, Deserialize, ToSchema)] +#[serde(rename_all = "snake_case")] +pub enum IncidentMode { + #[default] + Muted, + None, +} + +impl IncidentMode { + pub fn is_muted(&self) -> bool { + matches!(self, Self::Muted) + } +} + +#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] +pub struct CorrectionWindow { + pub downtime_id: String, + pub start: i64, + pub end: i64, + pub mode: SloCorrectionMode, +} + +#[derive(Clone, Debug, Deserialize, ToSchema)] +pub struct PreviewRequest { + #[serde(default)] + pub condition: Option, + pub targets: Vec, +} + +#[derive(Clone, Debug, PartialEq, Serialize, ToSchema)] +pub struct PreviewMatch { + pub id: String, + pub name: String, + pub folder_id: String, + #[serde(skip_serializing_if = "Option::is_none")] + pub matched_by: Option, + #[serde(skip_serializing_if = "Option::is_none")] + pub missing: Option, + /// SLOs on the detail page: false when the current or next window holds no slice start. + #[serde(skip_serializing_if = "Option::is_none")] + pub applies: Option, +} + +#[derive(Clone, Debug, Default, Serialize, ToSchema)] +pub struct PreviewResponse { + pub alerts: Vec, + pub resolved_at_fire_time: Vec, + pub anomalies: Vec, + pub synthetics: Vec, + pub slos: Vec, + pub alerts_total: usize, + pub anomalies_total: usize, + pub synthetics_total: usize, + pub slos_total: usize, + pub undecidable: HashMap>, +} + +#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize, ToSchema)] +pub struct ActiveDowntime { + pub id: String, + pub name: String, + pub ends_at: i64, + #[serde(default, skip_serializing_if = "IncidentMode::is_muted")] + pub incident_mode: IncidentMode, +} + +#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize, ToSchema)] +pub struct CorrectionRef { + pub downtime_id: String, + pub name: String, + pub status: DowntimeStatus, + /// False when the window holds no slice start, so it corrects no minute of this SLO. + #[serde(default = "default_true")] + pub applies: bool, +} + +/// Query of `GET /v2/{org}/downtimes`. +#[derive(Clone, Debug, Default, Deserialize, IntoParams)] +#[into_params(parameter_in = Query)] +pub struct ListDowntimesQuery { + /// Absent means every folder the user may list. + pub folder_id: Option, + /// `scheduled`, `active`, `ended`, `ended_early` or `cancelled`. + pub status: Option, + /// `none`, `daily`, `weekly`, or `recurring` for both. + pub repeat: Option, + pub search: Option, + /// Rows whose alerts target names this alert, or whose condition matches its dimensions. + pub alert_id: Option, + pub page: Option, + pub page_size: Option, +} + +#[derive(Clone, Debug, Default, PartialEq, Eq, Serialize, ToSchema)] +pub struct DowntimeStatusCounts { + pub active: usize, + pub scheduled: usize, + pub recurring: usize, + pub ended: usize, + pub cancelled: usize, + pub ended_early: usize, +} + +#[derive(Clone, Debug, Serialize, ToSchema)] +pub struct ListDowntimesResponse { + pub items: Vec, + pub total: usize, + pub counts: DowntimeStatusCounts, +} + +/// `GET /v2/{org}/downtimes/{id}`: the list item and what it covers, as far as the user may see. +#[derive(Clone, Debug, Serialize, ToSchema)] +pub struct DowntimeDetail { + #[serde(flatten)] + pub item: DowntimeListItem, + pub affected: AffectedItems, + pub notification_log: Vec, +} + +#[derive(Clone, Debug, Default, Serialize, ToSchema)] +pub struct AffectedItems { + pub alerts: Vec, + pub resolved_at_fire_time: Vec, + pub anomalies: Vec, + pub synthetics: Vec, + pub slos: Vec, +} + +#[derive(Clone, Debug, Deserialize, ToSchema)] +pub struct MoveDowntimesRequest { + pub downtime_ids: Vec, + pub dst_folder_id: String, +} + +#[derive(Clone, Debug, Serialize, ToSchema)] +pub struct CreateDowntimeResponse { + pub id: String, +} + +/// `POST /v2/{org}/downtimes/{id}/extend`: exactly one of `by_secs` and `until`. +#[derive(Clone, Debug, Default, Deserialize, ToSchema)] +pub struct ExtendDowntimeRequest { + /// Added to the later of the current end and now. + #[serde(default)] + pub by_secs: Option, + /// The new end, microseconds UTC. + #[serde(default)] + pub until: Option, + /// The version the row was loaded at; a newer stored row makes the extension a 409. + #[serde(default)] + pub version: Option, +} + +/// The row that now ends later: the extended one-time row, or the follow-up of a recurring one. +#[derive(Clone, Debug, Serialize, ToSchema)] +pub struct ExtendDowntimeResponse { + #[serde(flatten)] + pub downtime: Downtime, + /// Set when a recurring row got a one-time follow-up instead of a longer window. + #[serde(skip_serializing_if = "Option::is_none")] + pub created_id: Option, +} + +pub fn default_folder() -> String { + DEFAULT_FOLDER.to_string() +} + +fn default_true() -> bool { + true +} + +fn default_ending_soon_lead_secs() -> i64 { + DEFAULT_ENDING_SOON_LEAD_SECS +} + +#[cfg(test)] +mod tests { + use super::*; + + fn minimal() -> Downtime { + Downtime { + id: "2f9KQe4b7Nq1vH3sT0mLzXpRcWd".to_string(), + org: "default".to_string(), + folder_id: DEFAULT_FOLDER.to_string(), + name: "Staging teardown".to_string(), + reason: None, + condition: None, + targets: vec![DowntimeTarget { + module: TargetModule::Alerts, + folders: TargetFolders::All, + tags: vec![], + ids: vec![], + slo_mode: None, + incident_mode: Default::default(), + }], + schedule: DowntimeSchedule { + repeat: Repeat::None, + starts_at: 1_789_855_200_000_000, + ends_at: Some(1_789_905_600_000_000), + timezone: "UTC".to_string(), + start_time_local: None, + duration_secs: 50_400, + weekdays: vec![], + }, + cancelled_at: None, + cancelled_by: None, + show_banner: true, + notifications: None, + origin_region: None, + version: 0, + created_by: "lin".to_string(), + created_at: 1, + updated_by: "lin".to_string(), + updated_at: 1, + } + } + + fn full() -> Downtime { + Downtime { + reason: Some("CHG-4471".to_string()), + condition: Some(DimensionCondition::Group { + op: LogicalOp::And, + items: vec![ + DimensionCondition::Pair { + key: "service".to_string(), + operator: PairOperator::Eq, + value: "payments".to_string(), + }, + DimensionCondition::Group { + op: LogicalOp::Or, + items: vec![DimensionCondition::Pair { + key: "host".to_string(), + operator: PairOperator::Ne, + value: "db-*".to_string(), + }], + }, + ], + }), + targets: vec![ + DowntimeTarget { + module: TargetModule::Synthetics, + folders: TargetFolders::Some { + folder_ids: vec!["4Kc1staging".to_string()], + }, + tags: vec!["service:payments".to_string()], + ids: vec!["syn_41".to_string()], + slo_mode: None, + incident_mode: Default::default(), + }, + DowntimeTarget { + module: TargetModule::Slos, + folders: TargetFolders::All, + tags: vec![], + ids: vec![], + slo_mode: Some(SloCorrectionMode::CountAsGood), + incident_mode: Default::default(), + }, + ], + schedule: DowntimeSchedule { + repeat: Repeat::Weekly, + starts_at: 1_789_603_200_000_000, + ends_at: None, + timezone: "Europe/Berlin".to_string(), + start_time_local: Some("02:00".to_string()), + duration_secs: 5400, + weekdays: vec![7], + }, + cancelled_at: Some(5), + cancelled_by: Some("lin".to_string()), + show_banner: false, + notifications: Some(DowntimeNotifications { + destinations: vec!["slack-oncall".to_string()], + events: NotificationEvents { + started: true, + ending_soon: true, + ..Default::default() + }, + ending_soon_lead_secs: 300, + continues: None, + }), + origin_region: Some("us-east".to_string()), + ..minimal() + } + } + + #[test] + fn a_downtime_round_trips_with_every_optional_field_absent() { + let d = minimal(); + let json = serde_json::to_value(&d).unwrap(); + for absent in [ + "reason", + "condition", + "cancelled_at", + "cancelled_by", + "notifications", + "origin_region", + ] { + assert!(json.get(absent).is_none(), "{absent} must be skipped"); + } + assert!(json["targets"][0].get("tags").is_none()); + assert!(json["targets"][0].get("slo_mode").is_none()); + assert!(json["schedule"].get("ends_at").is_some()); + assert_eq!(serde_json::from_value::(json).unwrap(), d); + } + + #[test] + fn a_downtime_round_trips_with_every_optional_field_present() { + let d = full(); + let json = serde_json::to_string(&d).unwrap(); + assert_eq!(serde_json::from_str::(&json).unwrap(), d); + } + + #[test] + fn stored_names_are_the_wire_contract() { + let json = serde_json::to_value(full()).unwrap(); + assert_eq!(json["condition"]["type"], "group"); + assert_eq!(json["condition"]["op"], "and"); + assert_eq!(json["condition"]["items"][0]["operator"], "="); + assert_eq!(json["condition"]["items"][1]["items"][0]["operator"], "!="); + assert_eq!(json["targets"][0]["module"], "synthetics"); + assert_eq!(json["targets"][0]["folders"]["kind"], "some"); + assert_eq!(json["targets"][1]["folders"]["kind"], "all"); + assert_eq!(json["targets"][1]["slo_mode"], "count_as_good"); + assert_eq!(json["schedule"]["repeat"], "weekly"); + assert_eq!( + serde_json::to_value(TargetModule::AnomalyDetections).unwrap(), + "anomaly_detections" + ); + } + + #[test] + fn missing_folder_and_banner_take_their_defaults() { + let mut json = serde_json::to_value(minimal()).unwrap(); + let obj = json.as_object_mut().unwrap(); + obj.remove("folder_id"); + obj.remove("show_banner"); + let d: Downtime = serde_json::from_value(json).unwrap(); + assert_eq!(d.folder_id, DEFAULT_FOLDER); + assert!(d.show_banner); + } + + #[test] + fn the_api_example_request_parses() { + let body = r#"{ + "folder_id": "planned-maintenance", + "name": "Payments failover · weekly", + "condition": { "type": "group", "op": "and", "items": [ + { "type": "pair", "key": "service", "operator": "=", "value": "payments" }, + { "type": "pair", "key": "env", "operator": "=", "value": "prod" } ] }, + "targets": [ + { "module": "alerts", "folders": { "kind": "all" } }, + { "module": "synthetics", "folders": { "kind": "all" }, "tags": ["service:payments"] }, + { "module": "slos", "folders": { "kind": "all" }, "slo_mode": "exclude" } + ], + "schedule": { "repeat": "weekly", "starts_at": 1789603200000000, "ends_at": null, + "timezone": "Europe/Berlin", "start_time_local": "02:00", + "duration_secs": 5400, "weekdays": [7] } + }"#; + let req: DowntimeRequest = serde_json::from_str(body).unwrap(); + assert_eq!(req.targets.len(), 3); + assert_eq!(req.schedule.ends_at, None); + assert!(req.show_banner); + assert_eq!(req.targets[2].slo_mode, Some(SloCorrectionMode::Exclude)); + + let quick: DowntimeRequest = serde_json::from_str( + r#"{ "targets": [ { "module": "anomaly_detections", "folders": { "kind": "all" }, "ids": ["ad_7c1"] } ], + "schedule": { "repeat": "none", "starts_at": 1, "ends_at": 2, "timezone": "UTC", "duration_secs": 60 }, + "show_banner": false }"#, + ) + .unwrap(); + assert_eq!(quick.folder_id, DEFAULT_FOLDER); + assert!(!quick.show_banner); + assert!(quick.schedule.weekdays.is_empty()); + } + + #[test] + fn notifications_take_their_defaults_and_keep_their_wire_names() { + let n: DowntimeNotifications = + serde_json::from_str(r#"{ "destinations": ["slack"] }"#).unwrap(); + assert_eq!(n.ending_soon_lead_secs, DEFAULT_ENDING_SOON_LEAD_SECS); + assert!(!n.events.any()); + + let json = serde_json::to_value(full()).unwrap(); + assert_eq!(json["notifications"]["destinations"][0], "slack-oncall"); + assert_eq!(json["notifications"]["events"]["ending_soon"], true); + assert_eq!(json["notifications"]["ending_soon_lead_secs"], 300); + assert_eq!(json["origin_region"], "us-east"); + assert!(json["notifications"].get("continues").is_none()); + let linked: DowntimeNotifications = + serde_json::from_str(r#"{ "destinations": [], "continues": "2f9K" }"#).unwrap(); + assert_eq!(linked.continues.as_deref(), Some("2f9K")); + + let req: DowntimeRequest = serde_json::from_str( + r#"{ "targets": [], "schedule": { "repeat": "none", "starts_at": 1, "ends_at": 2, + "timezone": "UTC", "duration_secs": 60 }, + "notifications": { "destinations": ["pd"], "events": { "ended": true } } }"#, + ) + .unwrap(); + let n = req.notifications.unwrap(); + assert!(n.events.wants(NotificationEvent::Ended)); + assert!(!n.events.wants(NotificationEvent::Started)); + } + + #[test] + fn notification_events_are_stored_by_their_snake_case_names() { + for (event, name) in [ + (NotificationEvent::Started, "started"), + (NotificationEvent::EndingSoon, "ending_soon"), + (NotificationEvent::Ended, "ended"), + (NotificationEvent::Cancelled, "cancelled"), + (NotificationEvent::Extended, "extended"), + ] { + assert_eq!(event.as_str(), name); + assert_eq!(serde_json::to_value(event).unwrap(), name); + } + } + + #[test] + fn repeat_round_trips_through_its_stored_number() { + for repeat in [Repeat::None, Repeat::Daily, Repeat::Weekly] { + assert_eq!(Repeat::from_i16(repeat.to_i16()), Some(repeat)); + } + assert_eq!(Repeat::None.to_i16(), 0); + assert_eq!(Repeat::Daily.to_i16(), 1); + assert_eq!(Repeat::Weekly.to_i16(), 2); + assert_eq!(Repeat::from_i16(3), None); + } + + #[test] + fn only_synthetics_has_tags_and_no_identity() { + for module in [ + TargetModule::Alerts, + TargetModule::AnomalyDetections, + TargetModule::Slos, + ] { + assert!(module.has_identity()); + assert!(!module.has_tags()); + } + assert!(!TargetModule::Synthetics.has_identity()); + assert!(TargetModule::Synthetics.has_tags()); + } + + #[test] + fn a_list_item_flattens_the_downtime() { + let item = DowntimeListItem { + downtime: minimal(), + status: DowntimeStatus::Scheduled, + current_window: None, + next_window: Some(DowntimeWindow { start: 1, end: 2 }), + matched_alerts: 7, + matched_anomalies: 0, + matched_synthetics: 2, + matched_slos: 3, + }; + let json = serde_json::to_value(item).unwrap(); + assert_eq!(json["id"], "2f9KQe4b7Nq1vH3sT0mLzXpRcWd"); + assert_eq!(json["status"], "scheduled"); + assert!(json["current_window"].is_null()); + assert_eq!(json["next_window"]["end"], 2); + } + + #[test] + fn ended_early_is_snake_case_on_the_wire() { + assert_eq!( + serde_json::to_value(DowntimeStatus::EndedEarly).unwrap(), + "ended_early" + ); + } + + #[test] + fn an_extend_body_takes_either_field_and_the_response_flattens_the_row() { + let by: ExtendDowntimeRequest = serde_json::from_str(r#"{ "by_secs": 1800 }"#).unwrap(); + assert_eq!((by.by_secs, by.until), (Some(1800), None)); + let until: ExtendDowntimeRequest = serde_json::from_str(r#"{ "until": 5 }"#).unwrap(); + assert_eq!((until.by_secs, until.until), (None, Some(5))); + let json = serde_json::to_value(ExtendDowntimeResponse { + downtime: minimal(), + created_id: None, + }) + .unwrap(); + assert_eq!(json["id"], "2f9KQe4b7Nq1vH3sT0mLzXpRcWd"); + assert!(json.get("created_id").is_none()); + } + + #[test] + fn preview_match_omits_unset_reasons() { + let m = PreviewMatch { + id: "a".to_string(), + name: "n".to_string(), + folder_id: "default".to_string(), + matched_by: None, + missing: Some("host".to_string()), + applies: None, + }; + let json = serde_json::to_value(m).unwrap(); + assert!(json.get("matched_by").is_none()); + assert_eq!(json["missing"], "host"); + } + + #[test] + fn only_the_condition_and_the_targets_change_the_coverage() { + let before = minimal(); + let cosmetic = Downtime { + name: "Renamed".to_string(), + reason: Some("typo".to_string()), + folder_id: "planned".to_string(), + show_banner: false, + cancelled_at: Some(5), + updated_at: 9, + ..before.clone() + }; + assert!(before.same_coverage(&cosmetic)); + + let narrowed = Downtime { + condition: Some(DimensionCondition::Pair { + key: "service".to_string(), + operator: PairOperator::Eq, + value: "payments".to_string(), + }), + ..before.clone() + }; + assert!(!before.same_coverage(&narrowed)); + + let mut other_module = before.clone(); + other_module.targets[0].module = TargetModule::Synthetics; + assert!(!before.same_coverage(&other_module)); + } + + #[test] + fn an_alerts_target_without_the_incident_mode_reads_as_muted() { + let old: DowntimeTarget = + serde_json::from_str(r#"{ "module": "alerts", "folders": { "kind": "all" } }"#) + .unwrap(); + assert_eq!(old.incident_mode, IncidentMode::Muted); + let json = serde_json::to_value(&old).unwrap(); + assert!( + json.get("incident_mode").is_none(), + "the default is not written" + ); + + let none: DowntimeTarget = serde_json::from_str( + r#"{ "module": "alerts", "folders": { "kind": "all" }, "incident_mode": "none" }"#, + ) + .unwrap(); + assert_eq!(none.incident_mode, IncidentMode::None); + let json = serde_json::to_value(&none).unwrap(); + assert_eq!(json["incident_mode"], "none"); + assert_eq!( + serde_json::from_value::(json).unwrap(), + none + ); + assert_eq!(serde_json::to_value(IncidentMode::Muted).unwrap(), "muted"); + } +} diff --git a/src/config/src/meta/folder.rs b/src/config/src/meta/folder.rs index b4c5ec4cea..4f4a34fd10 100644 --- a/src/config/src/meta/folder.rs +++ b/src/config/src/meta/folder.rs @@ -50,6 +50,7 @@ pub enum FolderType { Synthetics, Workflows, Prompts = 5, + Downtimes = 6, } pub const DEFAULT_FOLDER: &str = "default"; diff --git a/src/config/src/meta/mod.rs b/src/config/src/meta/mod.rs index e5dac063a7..5aa6dd4a4c 100644 --- a/src/config/src/meta/mod.rs +++ b/src/config/src/meta/mod.rs @@ -22,6 +22,7 @@ pub mod dashboards; pub mod db_monitoring; pub mod db_normalizer; pub mod destinations; +pub mod downtimes; pub mod enrichment_table; pub mod folder; pub mod function; diff --git a/src/config/src/meta/oncall/routing.rs b/src/config/src/meta/oncall/routing.rs index c87df3b28e..b4c5864cfd 100644 --- a/src/config/src/meta/oncall/routing.rs +++ b/src/config/src/meta/oncall/routing.rs @@ -138,7 +138,7 @@ impl std::error::Error for OwnershipError {} /// A trailing `*` is a prefix claim and nothing more: `host=db-*` owns `db-01` /// and `db-primary`. §7: "no regex, no boolean logic", because a routing table /// people cannot read at a glance is one they stop trusting. -fn pair_matches(want: &str, actual: &str) -> bool { +pub fn pair_matches(want: &str, actual: &str) -> bool { match want.strip_suffix('*') { Some(prefix) => actual.starts_with(prefix), None => want == actual, diff --git a/src/config/src/meta/self_reporting/mod.rs b/src/config/src/meta/self_reporting/mod.rs index c4910e93d0..c210b34c42 100644 --- a/src/config/src/meta/self_reporting/mod.rs +++ b/src/config/src/meta/self_reporting/mod.rs @@ -397,6 +397,7 @@ mod tests { value_is_lower_bound: None, synthetics_error_source: None, synthetics_location: None, + downtime_id: None, delivery_attempted: None, }; @@ -949,6 +950,7 @@ mod tests { value_is_lower_bound: None, synthetics_error_source: None, synthetics_location: None, + downtime_id: None, delivery_attempted: None, }; @@ -1032,6 +1034,7 @@ mod tests { value_is_lower_bound: None, synthetics_error_source: None, synthetics_location: None, + downtime_id: None, delivery_attempted: None, }; @@ -1100,6 +1103,7 @@ mod tests { value_is_lower_bound: None, synthetics_error_source: None, synthetics_location: None, + downtime_id: None, delivery_attempted: None, }; diff --git a/src/config/src/meta/self_reporting/usage.rs b/src/config/src/meta/self_reporting/usage.rs index 9f3e57232c..10bdc4d6ca 100644 --- a/src/config/src/meta/self_reporting/usage.rs +++ b/src/config/src/meta/self_reporting/usage.rs @@ -108,6 +108,9 @@ pub enum RunOutcome { /// transitioning to firing #[serde(rename = "pending")] Pending, + /// Condition matched inside a downtime window; nothing was delivered. + #[serde(rename = "suppressed")] + Suppressed, } impl RunOutcome { @@ -129,6 +132,7 @@ impl RunOutcome { Self::Skipped => 4, Self::NotifyFailed => 5, Self::Pending => 6, + Self::Suppressed => 7, } } @@ -141,6 +145,7 @@ impl RunOutcome { 4 => Some(Self::Skipped), 5 => Some(Self::NotifyFailed), 6 => Some(Self::Pending), + 7 => Some(Self::Suppressed), _ => None, } } @@ -154,6 +159,7 @@ impl RunOutcome { Self::Skipped => "skipped", Self::NotifyFailed => "notify_failed", Self::Pending => "pending", + Self::Suppressed => "suppressed", } } } @@ -196,6 +202,7 @@ pub fn normalize_outcome( "skipped" => Some(RunOutcome::Skipped), "notify_failed" => Some(RunOutcome::NotifyFailed), "pending" => Some(RunOutcome::Pending), + "suppressed" => Some(RunOutcome::Suppressed), // ── legacy vocabulary ── "condition_not_satisfied" => Some(RunOutcome::Normal), @@ -330,6 +337,8 @@ pub struct TriggerData { /// The venue the failed synthetics slot was scheduled for. #[serde(default, skip_serializing_if = "Option::is_none")] pub synthetics_location: Option, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub downtime_id: Option, /// Whether a firing tried a destination, workflow or incident notification; None on old rows. #[serde(default, skip_serializing_if = "Option::is_none")] pub delivery_attempted: Option, @@ -373,6 +382,7 @@ impl Default for TriggerData { value_is_lower_bound: None, synthetics_error_source: None, synthetics_location: None, + downtime_id: None, delivery_attempted: None, } } @@ -425,6 +435,7 @@ impl TriggerData { value_is_lower_bound: Some(false), synthetics_error_source: Some(String::new()), synthetics_location: Some(String::new()), + downtime_id: Some(String::new()), delivery_attempted: Some(true), } } @@ -1148,6 +1159,8 @@ mod run_outcome_tests { assert_eq!(RunOutcome::from_i32(3), Some(RunOutcome::Error)); assert_eq!(RunOutcome::from_i32(4), Some(RunOutcome::Skipped)); assert_eq!(RunOutcome::from_i32(5), Some(RunOutcome::NotifyFailed)); + assert_eq!(RunOutcome::Suppressed.to_i32(), 7); + assert_eq!(RunOutcome::from_i32(7), Some(RunOutcome::Suppressed)); } #[test] @@ -1159,6 +1172,8 @@ mod run_outcome_tests { RunOutcome::Error, RunOutcome::Skipped, RunOutcome::NotifyFailed, + RunOutcome::Pending, + RunOutcome::Suppressed, ] { let n = outcome.to_i32(); assert_eq!( @@ -1169,6 +1184,30 @@ mod run_outcome_tests { } } + #[test] + fn a_suppressed_run_carries_its_downtime_id_through_serde() { + assert_eq!( + serde_json::to_string(&RunOutcome::Suppressed).unwrap(), + "\"suppressed\"" + ); + assert_eq!( + normalize_outcome("suppressed", &TriggerDataType::Alert, None), + Some(RunOutcome::Suppressed) + ); + let td = TriggerData { + status: RunOutcome::Suppressed, + downtime_id: Some("2f9K".to_string()), + ..TriggerData::default() + }; + let json = serde_json::to_value(&td).unwrap(); + assert_eq!(json["status"], "suppressed"); + assert_eq!(json["downtime_id"], "2f9K"); + assert_eq!(serde_json::from_value::(json).unwrap(), td); + let bare = serde_json::to_value(TriggerData::default()).unwrap(); + assert!(bare.get("downtime_id").is_none()); + assert!(TriggerData::get_field_names().contains(&"downtime_id".to_string())); + } + #[test] fn test_run_outcome_from_i32_rejects_unknown() { assert_eq!(RunOutcome::from_i32(-1), None); @@ -1788,6 +1827,7 @@ mod tests { value_is_lower_bound: None, synthetics_error_source: None, synthetics_location: None, + downtime_id: None, delivery_attempted: None, }; diff --git a/src/config/src/meta/slo/coverage.rs b/src/config/src/meta/slo/coverage.rs index 1ac3f9356f..8ab64aff12 100644 --- a/src/config/src/meta/slo/coverage.rs +++ b/src/config/src/meta/slo/coverage.rs @@ -123,7 +123,11 @@ pub fn observe(read: WindowRead, coverage_floor: f64, watermark_stale: bool) -> pub fn evaluation_is_measured(outcome: &RunOutcome) -> bool { match outcome { // Computed a level — the same set §7.6 refreshes `level_at` for. - RunOutcome::Firing | RunOutcome::Normal | RunOutcome::NotifyFailed => true, + // A suppressed run computed its level; only the delivery was skipped. + RunOutcome::Firing + | RunOutcome::Normal + | RunOutcome::NotifyFailed + | RunOutcome::Suppressed => true, // Observed nothing: the query failed, the run was skipped, or the // outcome belongs to a non-condition module. RunOutcome::Error | RunOutcome::Skipped | RunOutcome::Succeeded | RunOutcome::Pending => { @@ -348,6 +352,11 @@ mod tests { assert!(!evaluation_is_measured(&RunOutcome::Skipped)); } + #[test] + fn a_suppressed_evaluation_is_a_measurement() { + assert!(evaluation_is_measured(&RunOutcome::Suppressed)); + } + /// An interval in which the source alert was **paused** has no records at /// all, so it must read as reduced coverage rather than as uptime. This is /// the case S-16 exists for: pausing an alert is a routine operator diff --git a/src/config/src/meta/slo/slice.rs b/src/config/src/meta/slo/slice.rs index e36bcfdacd..238ac6e324 100644 --- a/src/config/src/meta/slo/slice.rs +++ b/src/config/src/meta/slo/slice.rs @@ -64,6 +64,8 @@ pub struct SliceRow { /// Monotonic within the generation; higher wins on the same key. Exists /// for late-data re-emission, not for publication ordering. pub rev: i64, + /// The downtime whose window zeroed or filled this slice (D8). + pub corrected_by: Option, } /// Whether a row is visible to readers. @@ -271,6 +273,7 @@ mod tests { good, total, rev, + corrected_by: None, } } diff --git a/src/config/src/meta/slo/status_view.rs b/src/config/src/meta/slo/status_view.rs index b300e97d15..27a33e1217 100644 --- a/src/config/src/meta/slo/status_view.rs +++ b/src/config/src/meta/slo/status_view.rs @@ -29,6 +29,7 @@ use serde::{Deserialize, Serialize}; use utoipa::ToSchema; use super::math::{burn_rate, error_budget_remaining, sli, time_to_exhaust_secs}; +use crate::meta::downtimes::CorrectionRef; /// What the UI and the API render for one SLO or one group. #[derive(Debug, Clone, PartialEq, Serialize, Deserialize, ToSchema)] @@ -79,6 +80,8 @@ pub struct SloStatusView { /// not exist. #[serde(default)] pub measuring_since: Option, + #[serde(default, skip_serializing_if = "Vec::is_empty")] + pub corrections: Vec, } impl SloStatusView { @@ -144,6 +147,7 @@ impl SloStatusView { stale_watermark: false, watermark_end: None, measuring_since: None, + corrections: Vec::new(), } } diff --git a/src/config/src/meta/slo/stream.rs b/src/config/src/meta/slo/stream.rs index 9f23892356..3e9b5cbbbe 100644 --- a/src/config/src/meta/slo/stream.rs +++ b/src/config/src/meta/slo/stream.rs @@ -62,6 +62,9 @@ pub struct SloSliceRow { /// This is for **dedupe**, not publication ordering — there is no /// publication protocol to order (D64). pub rev: i64, + /// Id of the downtime that corrected this slice, or `""` when measured as is. + #[serde(default)] + pub corrected_by: String, } impl SloSliceRow { @@ -84,6 +87,7 @@ impl SloSliceRow { good: 1.0, total: 1.0, rev: 1, + corrected_by: "downtime".to_string(), } } } @@ -98,7 +102,7 @@ mod tests { // type and then conflict with the first real write. let v = serde_json::to_value(SloSliceRow::init_for_reflection()).unwrap(); let obj = v.as_object().unwrap(); - assert_eq!(obj.len(), 10, "a field was added without a sample value"); + assert_eq!(obj.len(), 11, "a field was added without a sample value"); for (k, val) in obj { assert!(!val.is_null(), "{k} is null in the reflection sample"); if let Some(s) = val.as_str() { @@ -130,6 +134,7 @@ mod tests { good: 98.0, total: 100.0, rev: 2, + corrected_by: String::new(), }; let json = serde_json::to_string(&row).unwrap(); assert_eq!(serde_json::from_str::(&json).unwrap(), row); @@ -151,6 +156,7 @@ mod tests { "good", "total", "rev", + "corrected_by", ] { assert!(v.get(field).is_some(), "field `{field}` was renamed"); } diff --git a/src/config/src/meta/synthetics.rs b/src/config/src/meta/synthetics.rs index e505951ad0..199fc1e6b8 100644 --- a/src/config/src/meta/synthetics.rs +++ b/src/config/src/meta/synthetics.rs @@ -19,7 +19,10 @@ use chrono::FixedOffset; use serde::{Deserialize, Serialize}; use utoipa::ToSchema; -use super::synthetics_variables::{placeholder_names, substitute_placeholders}; +use super::{ + downtimes::ActiveDowntime, + synthetics_variables::{placeholder_names, substitute_placeholders}, +}; // ── Frequency ───────────────────────────────────────────────────────────────── @@ -583,6 +586,8 @@ pub struct SyntheticListItem { /// Omitted for a check that holds no subtest reference. #[serde(default, skip_serializing_if = "Option::is_none")] pub reference_state: Option, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub active_downtime: Option, } /// Whether a parent's subtest references can run as stored. @@ -4656,6 +4661,7 @@ mod tests { referenced_by: 0, references: Some(0), reference_state: None, + active_downtime: None, }; let json = serde_json::to_value(&item).unwrap(); assert!(json.get("reference_state").is_none(), "{json}"); diff --git a/src/config/src/metrics/base.rs b/src/config/src/metrics/base.rs index d482af9e9d..6126d3067b 100644 --- a/src/config/src/metrics/base.rs +++ b/src/config/src/metrics/base.rs @@ -1394,6 +1394,19 @@ pub static ALERT_DEDUP_ERRORS_TOTAL: Lazy = Lazy::new(|| { .expect("Metric created") }); +pub static ALERT_RUNS_SUPPRESSED_TOTAL: Lazy = Lazy::new(|| { + IntCounterVec::new( + Opts::new( + "alert_runs_suppressed_total", + "Total number of runs whose notification a downtime suppressed", + ) + .namespace(NAMESPACE) + .const_labels(create_const_labels()), + &["org", "module"], + ) + .expect("Metric created") +}); + // Alert grouping/batching metrics pub static ALERT_GROUPING_BATCHES_PENDING: Lazy = Lazy::new(|| { IntGaugeVec::new( @@ -2886,6 +2899,9 @@ pub(crate) fn register(registry: &Registry) { registry .register(Box::new(ALERT_DEDUP_ERRORS_TOTAL.clone())) .expect("Metric registered"); + registry + .register(Box::new(ALERT_RUNS_SUPPRESSED_TOTAL.clone())) + .expect("Metric registered"); // alert grouping metrics registry @@ -3310,6 +3326,7 @@ mod tests { let _ = ALERT_DEDUP_SUPPRESSED_TOTAL.clone(); let _ = ALERT_DEDUP_PASSED_TOTAL.clone(); let _ = ALERT_DEDUP_ERRORS_TOTAL.clone(); + let _ = ALERT_RUNS_SUPPRESSED_TOTAL.clone(); let _ = ALERT_GROUPING_BATCHES_PENDING.clone(); let _ = ALERT_GROUPING_NOTIFICATIONS_SENT_TOTAL.clone(); let _ = ALERT_GROUPING_BATCH_SIZE.clone(); diff --git a/src/core/src/alerts/alert.rs b/src/core/src/alerts/alert.rs index 1855c14735..bc3b4b8fa6 100644 --- a/src/core/src/alerts/alert.rs +++ b/src/core/src/alerts/alert.rs @@ -359,6 +359,28 @@ pub enum AlertError { MultiAlertGroupingError(String), } +/// What a manual trigger did: sent, with its messages, or held by an active downtime. +#[derive(Debug, Clone, PartialEq)] +pub enum ManualTrigger { + Sent { + success_message: String, + error_message: String, + }, + Suppressed(config::meta::downtimes::ActiveDowntime), +} + +impl ManualTrigger { + /// The message the trigger route answers with. + pub fn response_message(&self) -> String { + match self { + Self::Sent { .. } => "Alert triggered".to_string(), + Self::Suppressed(downtime) => { + format!("Alert suppressed by downtime {}", downtime.name) + } + } + } +} + pub async fn save( org_id: &str, stream_name: &str, @@ -1147,6 +1169,25 @@ pub fn update_cron_expression(cron_exp: &str, now: u32) -> String { cron_exp } +/// `zo_alert_runs_suppressed_total`: one per run a downtime kept silent. +pub fn count_suppressed_run(org: &str, module: &str) { + config::metrics::ALERT_RUNS_SUPPRESSED_TOTAL + .with_label_values(&[org, module]) + .inc(); +} + +/// A run a downtime kept silent: `Suppressed` with its downtime id, no delivery attempt, counted. +pub fn record_suppressed_run( + data: &mut config::meta::self_reporting::usage::TriggerData, + module: &str, + downtime_id: String, +) { + data.status = config::meta::self_reporting::usage::RunOutcome::Suppressed; + data.downtime_id = Some(downtime_id); + data.delivery_attempted = Some(false); + count_suppressed_run(&data.org, module); +} + /// Runs the checks [`create`] (given a folder) or [`update`] makes on a copy, writing nothing. pub async fn validate_save( org_id: &str, @@ -2239,11 +2280,15 @@ pub async fn trigger_by_id( conn: &C, org_id: &str, alert_id: Ksuid, -) -> Result<(String, String), AlertError> { - let Some((_, alert)) = db::alerts::alert::get_by_id(conn, org_id, alert_id).await? else { +) -> Result { + let Some((_folder, alert)) = db::alerts::alert::get_by_id(conn, org_id, alert_id).await? else { return Err(AlertError::AlertNotFound); }; let now = Utc::now().timestamp_micros(); + #[cfg(feature = "enterprise")] + if let Some(downtime) = suppressed_manual_trigger(&alert, &_folder.folder_id, now).await { + return Ok(ManualTrigger::Suppressed(downtime)); + } // For creates_incident=true alerts the incident correlation path handles // the notification. For all other cases send the direct notification. @@ -2263,6 +2308,7 @@ pub async fn trigger_by_id( now, // Manual triggers evaluate nothing; no level to map. None, + None, ) .await { @@ -2331,9 +2377,12 @@ pub async fn trigger_by_id( None, ) .await?; - let err_message = with_incident_notify_error(outcome.error_message, incident_notify_error); + let error_message = with_incident_notify_error(outcome.error_message, incident_notify_error); - Ok((outcome.success_message, err_message)) + Ok(ManualTrigger::Sent { + success_message: outcome.success_message, + error_message, + }) } pub async fn trigger_by_name( @@ -2341,14 +2390,21 @@ pub async fn trigger_by_name( stream_type: StreamType, stream_name: &str, name: &str, -) -> Result<(String, String), AlertError> { - let alert = match db::alerts::alert::get_by_name(org_id, stream_type, stream_name, name).await { - Ok(Some(alert)) => alert, - _ => { - return Err(AlertError::AlertNotFound); - } - }; +) -> Result { + let (_folder, alert) = + match db::alerts::alert::get_by_name_with_folder(org_id, stream_type, stream_name, name) + .await + { + Ok(Some(found)) => found, + _ => { + return Err(AlertError::AlertNotFound); + } + }; let now = Utc::now().timestamp_micros(); + #[cfg(feature = "enterprise")] + if let Some(downtime) = suppressed_manual_trigger(&alert, &_folder.folder_id, now).await { + return Ok(ManualTrigger::Suppressed(downtime)); + } // For creates_incident=true alerts the incident correlation path handles // the notification. For all other cases send the direct notification. @@ -2368,6 +2424,7 @@ pub async fn trigger_by_name( now, // Manual triggers evaluate nothing; no level to map. None, + None, ) .await { @@ -2429,9 +2486,42 @@ pub async fn trigger_by_name( None, ) .await?; - let err_message = with_incident_notify_error(outcome.error_message, incident_notify_error); + let error_message = with_incident_notify_error(outcome.error_message, incident_notify_error); - Ok((outcome.success_message, err_message)) + Ok(ManualTrigger::Sent { + success_message: outcome.success_message, + error_message, + }) +} + +/// A manual trigger inside a window is suppressed like a scheduled run: no page, no un-mute. +#[cfg(feature = "enterprise")] +async fn suppressed_manual_trigger( + alert: &Alert, + folder_id: &str, + now: i64, +) -> Option { + let rows = [manual_trigger_row(alert)]; + let downtime = + crate::alerts::scheduler::handlers::downtime_decision(alert, folder_id, Some(&rows), now) + .await?; + log::info!( + "Manual trigger of alert {}/{} suppressed by downtime {}", + alert.org_id, + alert.name, + downtime.id + ); + crate::alerts::scheduler::handlers::correlate_muted_incident( + "manual_trigger", + alert, + &rows, + now, + None, + &downtime, + ) + .await; + count_suppressed_run(&alert.org_id, "alerts"); + Some(downtime) } /// Per-destination result of one notification attempt. @@ -8039,6 +8129,25 @@ mod tests { alert } + #[test] + fn a_suppressed_manual_trigger_answers_with_the_downtime_name() { + let suppressed = ManualTrigger::Suppressed(config::meta::downtimes::ActiveDowntime { + id: "dt-1".to_string(), + name: "db upgrade".to_string(), + ends_at: 0, + incident_mode: Default::default(), + }); + assert_eq!( + suppressed.response_message(), + "Alert suppressed by downtime db upgrade" + ); + let sent = ManualTrigger::Sent { + success_message: String::new(), + error_message: String::new(), + }; + assert_eq!(sent.response_message(), "Alert triggered"); + } + #[test] fn manual_trigger_row_carries_stream_alert_and_manual_marker() { let alert = manual_alert("my_alert", vec![]); @@ -8133,6 +8242,26 @@ mod tests { assert_eq!(workflow_alert_count(&alert, 25, None), json!(25u64)); } + #[test] + fn a_suppressed_run_is_recorded_with_its_downtime_and_counted() { + use config::meta::self_reporting::usage::{RunOutcome, TriggerData}; + + let org = "org_suppressed_run_record"; + let counter = + || config::metrics::ALERT_RUNS_SUPPRESSED_TOTAL.with_label_values(&[org, "alerts"]); + let before = counter().get(); + let mut data = TriggerData { + org: org.to_string(), + status: RunOutcome::Firing, + ..Default::default() + }; + record_suppressed_run(&mut data, "alerts", "dt-1".to_string()); + assert_eq!(data.status, RunOutcome::Suppressed); + assert_eq!(data.downtime_id.as_deref(), Some("dt-1")); + assert_eq!(data.delivery_attempted, Some(false)); + assert_eq!(counter().get(), before + 1); + } + /// A new alert with no id reaches the schedule checks before any DB read. async fn prepare_schedule(mutate: impl FnOnce(&mut Alert)) -> AlertError { let mut alert = Alert::default(); diff --git a/src/core/src/alerts/destination_usage.rs b/src/core/src/alerts/destination_usage.rs index 47c04b153d..e044b94d23 100644 --- a/src/core/src/alerts/destination_usage.rs +++ b/src/core/src/alerts/destination_usage.rs @@ -38,6 +38,8 @@ pub enum DestinationConsumer { AnomalyDetection, #[cfg(feature = "enterprise")] IncidentIntegration, + #[cfg(feature = "enterprise")] + Downtime, } impl DestinationConsumer { @@ -56,6 +58,8 @@ impl DestinationConsumer { Self::AnomalyDetection => ("anomaly detection config", "anomaly detection configs"), #[cfg(feature = "enterprise")] Self::IncidentIntegration => ("incident integration", "incident integrations"), + #[cfg(feature = "enterprise")] + Self::Downtime => ("downtime", "downtimes"), }; format!("{count} {}", if count == 1 { one } else { many }) } @@ -198,6 +202,8 @@ async fn usage_for( DestinationConsumer::AnomalyDetection => anomaly_detection_usage(org_id).await, #[cfg(feature = "enterprise")] DestinationConsumer::IncidentIntegration => incident_integration_usage(org_id).await, + #[cfg(feature = "enterprise")] + DestinationConsumer::Downtime => downtime_usage(org_id).await, } } @@ -401,6 +407,52 @@ async fn incident_integration_usage(org_id: &str) -> Result, Ok(uses) } +#[cfg(feature = "enterprise")] +async fn downtime_usage(org_id: &str) -> Result, DestinationError> { + let cached = db::downtimes::list_cached(org_id); + let stored = infra::table::downtimes::list(org_id, None).await?; + let ids: std::collections::HashSet<&str> = cached.iter().map(|d| d.id.as_str()).collect(); + let rows = cached + .iter() + .chain(stored.iter().filter(|d| !ids.contains(d.id.as_str()))); + Ok(downtime_uses(rows, config::utils::time::now_micros())) +} + +/// Only a row that can still notify holds its destinations; ended and cancelled rows never send. +#[cfg(feature = "enterprise")] +fn downtime_uses<'a>( + rows: impl Iterator, + now: i64, +) -> Vec { + use config::meta::downtimes::DowntimeStatus; + use o2_enterprise::enterprise::downtimes::schedule; + + let mut uses = Vec::new(); + let live = rows.filter(|row| { + matches!( + schedule::status(&row.schedule, row.cancelled_at, now), + DowntimeStatus::Scheduled | DowntimeStatus::Active + ) + }); + for row in live { + let names = row + .notifications + .as_ref() + .map(|n| n.destinations.clone()) + .unwrap_or_default(); + for destination_name in unique_names(names) { + uses.push(DestinationUse { + consumer: DestinationConsumer::Downtime, + id: row.id.clone(), + name: row.name.clone(), + folder_id: Some(row.folder_id.clone()), + destination_name, + }); + } + } + uses +} + #[cfg(test)] mod tests { use config::meta::{ @@ -879,6 +931,86 @@ mod tests { assert_eq!(uses[0].destination_name, name); } + #[cfg(feature = "enterprise")] + #[test] + fn test_downtime_arm_names_the_downtime_in_the_refusal() { + use config::meta::downtimes::{ + Downtime, DowntimeNotifications, DowntimeSchedule, NotificationEvents, Repeat, + }; + + let row = |id: &str, name: &str, destinations: &[&str]| Downtime { + id: id.to_string(), + org: "acme".to_string(), + folder_id: "default".to_string(), + name: name.to_string(), + reason: None, + condition: None, + targets: vec![], + schedule: DowntimeSchedule { + repeat: Repeat::None, + starts_at: 1, + ends_at: Some(2), + timezone: "UTC".to_string(), + start_time_local: None, + duration_secs: 60, + weekdays: vec![], + }, + cancelled_at: None, + cancelled_by: None, + show_banner: true, + notifications: Some(DowntimeNotifications { + destinations: destinations.iter().map(|d| d.to_string()).collect(), + events: NotificationEvents { + ended: true, + ..Default::default() + }, + ending_soon_lead_secs: 600, + continues: None, + }), + origin_region: None, + version: 0, + created_by: "lin".to_string(), + created_at: 1, + updated_by: "lin".to_string(), + updated_at: 1, + }; + let rows = [ + row("d1", "Nightly deploy", &["pagerduty", "pagerduty", "slack"]), + row("d2", "DB failover", &["slack"]), + Downtime { + notifications: None, + ..row("d3", "Quiet", &[]) + }, + ]; + let uses = downtime_uses(rows.iter(), 0); + let pagerduty: Vec = uses + .into_iter() + .filter(|u| u.destination_name == "pagerduty") + .collect(); + assert_eq!(pagerduty.len(), 1); + assert_eq!(pagerduty[0].consumer, DestinationConsumer::Downtime); + assert_eq!(pagerduty[0].folder_id.as_deref(), Some("default")); + assert_eq!( + usage_message("pagerduty", &pagerduty), + "'pagerduty' is used by 1 downtime (Nightly deploy)" + ); + let slack: Vec = downtime_uses(rows.iter(), 0) + .into_iter() + .filter(|u| u.destination_name == "slack") + .collect(); + assert_eq!( + usage_message("slack", &slack), + "'slack' is used by 2 downtimes (Nightly deploy, DB failover)" + ); + // Ended at 2, then cancelled before its start: neither holds `slack` any more. + assert!(downtime_uses(rows.iter(), 2).is_empty()); + let cancelled = Downtime { + cancelled_at: Some(0), + ..row("d4", "Called off", &["slack"]) + }; + assert!(downtime_uses([cancelled].iter(), 0).is_empty()); + } + #[test] fn test_unique_names_drops_a_duplicate_on_one_row() { let names = vec!["a".to_string(), "b".to_string(), "a".to_string()]; diff --git a/src/core/src/alerts/downtimes.rs b/src/core/src/alerts/downtimes.rs new file mode 100644 index 0000000000..39e6d4d206 --- /dev/null +++ b/src/core/src/alerts/downtimes.rs @@ -0,0 +1,1176 @@ +// Copyright 2026 OpenObserve Inc. +// +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// This program is distributed in the hope that it will be useful +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. +// +// You should have received a copy of the GNU Affero General Public License +// along with this program. If not, see . + +//! Whether an item is inside an active downtime now, and which windows correct an SLO. + +use std::collections::HashMap; + +use config::meta::{ + downtimes::{ActiveDowntime, CorrectionRef, CorrectionWindow}, + slo::Slo, +}; + +/// Whether `O2_DOWNTIMES_ENABLED` is on; always off in the OSS build. +#[cfg(feature = "enterprise")] +pub fn enabled() -> bool { + o2_enterprise::enterprise::common::config::get_config() + .downtimes + .enabled +} + +#[cfg(not(feature = "enterprise"))] +pub fn enabled() -> bool { + false +} + +/// Whether the org has an uncancelled downtime targeting `module`; else skip identity work. +#[cfg(feature = "enterprise")] +pub fn any_for(org: &str, module: config::meta::downtimes::TargetModule) -> bool { + o2_enterprise::enterprise::common::config::get_config() + .downtimes + .enabled + && enterprise::any_live_target( + &db::downtimes::list_cached(org), + module, + config::utils::time::now_micros(), + ) +} + +#[cfg(not(feature = "enterprise"))] +pub fn any_for(_org: &str, _module: config::meta::downtimes::TargetModule) -> bool { + false +} + +/// [any_for], counting also a row whose window ended after `since`, so the mute it left is closed. +#[cfg(feature = "enterprise")] +pub fn any_since(org: &str, module: config::meta::downtimes::TargetModule, since: i64) -> bool { + o2_enterprise::enterprise::common::config::get_config() + .downtimes + .enabled + && enterprise::any_target_since( + &db::downtimes::list_cached(org), + module, + since, + config::utils::time::now_micros(), + ) +} + +/// Whether an uncancelled row targets `module`, so a mute recorded under it can linger. +#[cfg(feature = "enterprise")] +pub fn any_row(org: &str, module: config::meta::downtimes::TargetModule) -> bool { + o2_enterprise::enterprise::common::config::get_config() + .downtimes + .enabled + && enterprise::any_row_in(&db::downtimes::list_cached(org), module) +} + +#[cfg(feature = "enterprise")] +pub fn active_for_alert( + org: &str, + alert_id: &str, + folder_id: &str, + dims: &HashMap, + now: i64, +) -> Option { + let item = enterprise::item(alert_id, folder_id, dims, &[]); + enterprise::active_in( + &db::downtimes::list_cached(org), + config::meta::downtimes::TargetModule::Alerts, + &item, + now, + ) +} + +#[cfg(not(feature = "enterprise"))] +pub fn active_for_alert( + _org: &str, + _alert_id: &str, + _folder_id: &str, + _dims: &HashMap, + _now: i64, +) -> Option { + None +} + +/// [active_for_alert] without row `except`, which a cancel or delete just took out of force. +#[cfg(feature = "enterprise")] +pub fn active_for_alert_except( + org: &str, + alert_id: &str, + folder_id: &str, + dims: &HashMap, + now: i64, + except: &str, +) -> Option { + let item = enterprise::item(alert_id, folder_id, dims, &[]); + enterprise::active_in_except( + &db::downtimes::list_cached(org), + config::meta::downtimes::TargetModule::Alerts, + &item, + now, + Some(except), + ) +} + +/// The recorded downtime's window at `at`; the muted firing that recorded it matched its scope. +#[cfg(feature = "enterprise")] +pub fn active_by_id(org: &str, id: &str, at: i64) -> Option { + enterprise::active_by_id_in( + &db::downtimes::list_cached(org), + config::meta::downtimes::TargetModule::Alerts, + id, + at, + ) +} + +#[cfg(not(feature = "enterprise"))] +pub fn active_by_id(_org: &str, _id: &str, _at: i64) -> Option { + None +} + +/// Whether the recorded downtime had a window in `[from, to)`, cut at its cancel. +#[cfg(feature = "enterprise")] +pub fn had_window(org: &str, id: &str, from: i64, to: i64) -> bool { + enterprise::had_window_in(&db::downtimes::list_cached(org), id, from, to) +} + +#[cfg(not(feature = "enterprise"))] +pub fn had_window(_org: &str, _id: &str, _from: i64, _to: i64) -> bool { + false +} + +#[cfg(feature = "enterprise")] +pub fn active_for_synthetic( + org: &str, + check_id: &str, + folder_id: &str, + tags: &[String], + now: i64, +) -> Option { + let no_identity = HashMap::new(); + let item = enterprise::item(check_id, folder_id, &no_identity, tags); + enterprise::active_in( + &db::downtimes::list_cached(org), + config::meta::downtimes::TargetModule::Synthetics, + &item, + now, + ) +} + +#[cfg(not(feature = "enterprise"))] +pub fn active_for_synthetic( + _org: &str, + _check_id: &str, + _folder_id: &str, + _tags: &[String], + _now: i64, +) -> Option { + None +} + +#[cfg(feature = "enterprise")] +pub fn active_for_anomaly( + org: &str, + anomaly_id: &str, + folder_id: &str, + dims: &HashMap, + now: i64, +) -> Option { + let item = enterprise::item(anomaly_id, folder_id, dims, &[]); + enterprise::active_in( + &db::downtimes::list_cached(org), + config::meta::downtimes::TargetModule::AnomalyDetections, + &item, + now, + ) +} + +#[cfg(not(feature = "enterprise"))] +pub fn active_for_anomaly( + _org: &str, + _anomaly_id: &str, + _folder_id: &str, + _dims: &HashMap, + _now: i64, +) -> Option { + None +} + +/// An anomaly config's filters count only in filters mode, where its query applies them. +pub fn anomaly_filters<'a>( + query_mode: &str, + filters: Option<&'a serde_json::Value>, +) -> Option<&'a serde_json::Value> { + filters.filter(|_| query_mode.eq_ignore_ascii_case("filters")) +} + +/// An anomaly config's `custom_sql` counts in every mode but filters, whose query ignores it. +pub fn anomaly_sql<'a>(query_mode: &str, custom_sql: Option<&'a str>) -> Option<&'a str> { + custom_sql.filter(|_| !query_mode.eq_ignore_ascii_case("filters")) +} + +/// The only async part: semantic groups and the source alert both come from in-memory caches. +#[cfg(feature = "enterprise")] +pub async fn dimensions_for_slo(slo: &Slo) -> HashMap { + let groups = db::system_settings::get_semantic_field_groups(&slo.org).await; + let source = match &slo.definition.sli_config { + config::meta::slo::SliConfig::Alert { alert_id } => { + db::alerts::alert::get_alert_from_cache(&slo.org, alert_id).await + } + _ => None, + }; + o2_enterprise::enterprise::downtimes::scope::slo_dimensions( + slo, + &groups, + source.as_ref().map(|(_, alert)| &alert.query_condition), + ) +} + +#[cfg(not(feature = "enterprise"))] +pub async fn dimensions_for_slo(_slo: &Slo) -> HashMap { + HashMap::new() +} + +/// Windows correcting this SLO in `[from, to)`; a cancelled row counts up to its cancel. +#[cfg(feature = "enterprise")] +pub fn corrections_for_slo( + slo: &Slo, + dims: &HashMap, + from: i64, + to: i64, +) -> Vec { + enterprise::corrections_in(&db::downtimes::list_cached(&slo.org), slo, dims, from, to) +} + +#[cfg(not(feature = "enterprise"))] +pub fn corrections_for_slo( + _slo: &Slo, + _dims: &HashMap, + _from: i64, + _to: i64, +) -> Vec { + Vec::new() +} + +/// The active and scheduled downtimes that correct this SLO, for its detail page. +#[cfg(feature = "enterprise")] +pub fn corrections_refs_for_slo( + slo: &Slo, + dims: &HashMap, + now: i64, +) -> Vec { + enterprise::refs_in(&db::downtimes::list_cached(&slo.org), slo, dims, now) +} + +#[cfg(not(feature = "enterprise"))] +pub fn corrections_refs_for_slo( + _slo: &Slo, + _dims: &HashMap, + _now: i64, +) -> Vec { + Vec::new() +} + +/// Pure over a slice of rows, so the rules are tested without the cache. +#[cfg(feature = "enterprise")] +pub(crate) mod enterprise { + use std::collections::HashMap; + + use config::meta::{ + downtimes::{ + ActiveDowntime, CorrectionRef, CorrectionWindow, Downtime, DowntimeStatus, + IncidentMode, Repeat, SloCorrectionMode, TargetModule, + }, + slo::{Slo, window::align_up}, + }; + use o2_enterprise::enterprise::downtimes::{ + schedule, + scope::{self, TargetItem}, + }; + + const MICROS: i64 = 1_000_000; + + pub(crate) fn item<'a>( + id: &'a str, + folder_id: &'a str, + dimensions: &'a HashMap, + tags: &'a [String], + ) -> TargetItem<'a> { + TargetItem { + id, + folder_id, + dimensions, + tags, + } + } + + /// An org whose windows are all over leaves the slow path, though its rows stay for history. + pub(crate) fn any_live_target(rows: &[Downtime], module: TargetModule, now: i64) -> bool { + rows.iter().any(|row| { + row.cancelled_at.is_none() + && !ended_by(row, now) + && scope::target_for(&row.targets, module).is_some() + && (schedule::window_at(&row.schedule, now).is_some() + || schedule::next_window(&row.schedule, now).is_some()) + }) + } + + /// [any_live_target], or a row with a window in `[since, now)`. + pub(crate) fn any_target_since( + rows: &[Downtime], + module: TargetModule, + since: i64, + now: i64, + ) -> bool { + any_live_target(rows, module, now) + || rows.iter().any(|row| { + !row.cancelled_at.is_some_and(|at| at <= since) + && !ended_by(row, since) + && scope::target_for(&row.targets, module).is_some() + && !schedule::windows_between(&row.schedule, row.cancelled_at, since, now) + .is_empty() + }) + } + + /// A cancel already forgot the mutes recorded under its row; an ended row's may still linger. + pub(crate) fn any_row_in(rows: &[Downtime], module: TargetModule) -> bool { + rows.iter().any(|row| { + row.cancelled_at.is_none() && scope::target_for(&row.targets, module).is_some() + }) + } + + /// A one-time row whose end is at or before `at` has no window from `at` on. + pub(crate) fn ended_by(row: &Downtime, at: i64) -> bool { + row.schedule.repeat == Repeat::None && row.schedule.ends_at.is_some_and(|end| end <= at) + } + + /// Every row live at `now` whose window and `module` target match, merged. + pub(crate) fn active_in( + rows: &[Downtime], + module: TargetModule, + item: &TargetItem<'_>, + now: i64, + ) -> Option { + active_in_except(rows, module, item, now, None) + } + + /// [active_in] without the row `except`. + pub(crate) fn active_in_except( + rows: &[Downtime], + module: TargetModule, + item: &TargetItem<'_>, + now: i64, + except: Option<&str>, + ) -> Option { + rows.iter() + .filter(|row| { + row.cancelled_at.is_none_or(|cancelled| cancelled > now) + && except != Some(row.id.as_str()) + }) + .filter_map(|row| { + let window = schedule::window_at(&row.schedule, now)?; + let target = scope::target_for(&row.targets, module)?; + scope::matches(target, row.condition.as_ref(), item).then(|| ActiveDowntime { + id: row.id.clone(), + name: row.name.clone(), + ends_at: window.end, + incident_mode: target.incident_mode, + }) + }) + .reduce(merged) + } + + /// Overlapping downtimes in any order: the latest end names the chip, any muted mode mutes. + pub(crate) fn merged(a: ActiveDowntime, b: ActiveDowntime) -> ActiveDowntime { + let incident_mode = if a.incident_mode.is_muted() || b.incident_mode.is_muted() { + IncidentMode::Muted + } else { + IncidentMode::None + }; + let latest = if b.ends_at > a.ends_at { b } else { a }; + ActiveDowntime { + incident_mode, + ..latest + } + } + + /// Whether row `id` had a window overlapping `[from, to)`, cut at its cancel. + pub(crate) fn had_window_in(rows: &[Downtime], id: &str, from: i64, to: i64) -> bool { + rows.iter().any(|row| { + row.id == id + && !schedule::windows_between(&row.schedule, row.cancelled_at, from, to).is_empty() + }) + } + + /// Row `id` if uncancelled at `at`, its window holds `at` and it still targets `module`. + pub(crate) fn active_by_id_in( + rows: &[Downtime], + module: TargetModule, + id: &str, + at: i64, + ) -> Option { + let row = rows.iter().find(|row| { + row.id == id + && row.cancelled_at.is_none_or(|cancelled| cancelled > at) + && scope::target_for(&row.targets, module).is_some() + })?; + let window = schedule::window_at(&row.schedule, at)?; + let target = scope::target_for(&row.targets, module)?; + Some(ActiveDowntime { + id: row.id.clone(), + name: row.name.clone(), + ends_at: window.end, + incident_mode: target.incident_mode, + }) + } + + /// The span of slice starts `[start, end)` corrects, in micros; `None` when it holds none. + pub(crate) fn aligned_span( + start: i64, + end: i64, + slice_interval_secs: i64, + ) -> Option<(i64, i64)> { + let ceil_secs = |us: i64| us.div_euclid(MICROS) + i64::from(us.rem_euclid(MICROS) != 0); + let start = align_up(ceil_secs(start), slice_interval_secs); + let end = align_up(ceil_secs(end), slice_interval_secs); + (start < end).then_some((start * MICROS, end * MICROS)) + } + + /// Aligned to the SLO's slices, so a window inside one slice is dropped; `Exclude` sorts first. + pub(crate) fn corrections_in( + rows: &[Downtime], + slo: &Slo, + dims: &HashMap, + from: i64, + to: i64, + ) -> Vec { + let item = slo_item(slo, dims); + let interval = slo.definition.slice_interval_secs; + let mut windows: Vec = rows + .iter() + .filter_map(|row| { + let target = scope::target_for(&row.targets, TargetModule::Slos)?; + scope::matches(target, row.condition.as_ref(), &item).then_some((row, target)) + }) + .flat_map(|(row, target)| { + let mode = target.slo_mode.unwrap_or_default(); + schedule::windows_between(&row.schedule, row.cancelled_at, from, to) + .into_iter() + .filter_map(move |w| { + let (start, end) = aligned_span(w.start, w.end, interval)?; + Some(CorrectionWindow { + downtime_id: row.id.clone(), + start, + end, + mode, + }) + }) + }) + .collect(); + windows.sort_by_key(|w| (w.mode != SloCorrectionMode::Exclude, w.start)); + windows + } + + pub(crate) fn refs_in( + rows: &[Downtime], + slo: &Slo, + dims: &HashMap, + now: i64, + ) -> Vec { + let item = slo_item(slo, dims); + rows.iter() + .filter(|row| covers(row, TargetModule::Slos, &item)) + .filter_map(|row| { + let status = schedule::status(&row.schedule, row.cancelled_at, now); + matches!(status, DowntimeStatus::Active | DowntimeStatus::Scheduled).then(|| { + CorrectionRef { + downtime_id: row.id.clone(), + name: row.name.clone(), + status, + applies: applies_to(row, slo.definition.slice_interval_secs, now), + } + }) + }) + .collect() + } + + /// Whether the current or next window holds a slice start; true when no window is known. + pub(crate) fn applies_to(row: &Downtime, slice_interval_secs: i64, now: i64) -> bool { + schedule::window_at(&row.schedule, now) + .or_else(|| schedule::next_window(&row.schedule, now)) + .is_none_or(|w| aligned_span(w.start, w.end, slice_interval_secs).is_some()) + } + + fn covers(row: &Downtime, module: TargetModule, item: &TargetItem<'_>) -> bool { + scope::target_for(&row.targets, module) + .is_some_and(|target| scope::matches(target, row.condition.as_ref(), item)) + } + + fn slo_item<'a>(slo: &'a Slo, dims: &'a HashMap) -> TargetItem<'a> { + item(&slo.id, &slo.folder_id, dims, &slo.tags) + } +} + +#[cfg(all(test, feature = "enterprise"))] +mod tests { + use config::meta::{ + downtimes::{ + DimensionCondition, Downtime, DowntimeSchedule, DowntimeTarget, IncidentMode, + PairOperator, Repeat, SloCorrectionMode, TargetFolders, TargetModule, + }, + slo::{CountSource, SliConfig, SloDefinition}, + }; + + use super::{enterprise::*, *}; + + const HOUR: i64 = 3_600_000_000; + + fn target(module: TargetModule) -> DowntimeTarget { + DowntimeTarget { + module, + folders: TargetFolders::All, + tags: vec![], + ids: vec![], + slo_mode: None, + incident_mode: Default::default(), + } + } + + fn row(id: &str, targets: Vec, start: i64, end: i64) -> Downtime { + Downtime { + id: id.to_string(), + org: "acme".to_string(), + folder_id: "default".to_string(), + name: id.to_string(), + reason: None, + condition: Some(DimensionCondition::Pair { + key: "service".to_string(), + operator: PairOperator::Eq, + value: "payments".to_string(), + }), + targets, + schedule: DowntimeSchedule { + repeat: Repeat::None, + starts_at: start, + ends_at: Some(end), + timezone: "UTC".to_string(), + start_time_local: None, + duration_secs: (end - start) / 1_000_000, + weekdays: vec![], + }, + cancelled_at: None, + cancelled_by: None, + show_banner: true, + notifications: None, + origin_region: None, + version: 0, + created_by: "lin".to_string(), + created_at: 0, + updated_by: "lin".to_string(), + updated_at: 0, + } + } + + fn payments() -> HashMap { + HashMap::from([("service".to_string(), "payments".to_string())]) + } + + fn slo() -> Slo { + Slo { + id: "slo1".to_string(), + org: "acme".to_string(), + folder_id: "default".to_string(), + name: "checkout".to_string(), + description: String::new(), + definition: SloDefinition { + sli_config: SliConfig::Count { + source: CountSource::SingleQuery { + stream: "requests".to_string(), + stream_type: "logs".to_string(), + scope: None, + good_expr: "status < 500".to_string(), + }, + }, + group_by: None, + window_secs: 30 * 86_400, + slice_interval_secs: 300, + }, + target: 99.9, + tags: vec![], + enabled: true, + owner: None, + definition_generation: 1, + groups_estimate: None, + groups_reserved: 1, + } + } + + #[test] + fn active_in_matches_the_module_target_inside_the_window_only() { + let rows = vec![row( + "d1", + vec![target(TargetModule::Alerts)], + 10 * HOUR, + 12 * HOUR, + )]; + let dims = payments(); + let alert = item("a1", "default", &dims, &[]); + let hit = active_in(&rows, TargetModule::Alerts, &alert, 11 * HOUR).unwrap(); + assert_eq!(hit.id, "d1"); + assert_eq!(hit.ends_at, 12 * HOUR); + assert!(active_in(&rows, TargetModule::Alerts, &alert, 13 * HOUR).is_none()); + assert!( + active_in(&rows, TargetModule::AnomalyDetections, &alert, 11 * HOUR).is_none(), + "a row without a target for the module is skipped" + ); + let other = HashMap::from([("service".to_string(), "checkout".to_string())]); + assert!( + active_in( + &rows, + TargetModule::Alerts, + &item("a1", "default", &other, &[]), + 11 * HOUR + ) + .is_none() + ); + } + + #[test] + fn only_an_uncancelled_row_with_a_target_of_the_module_asks_for_the_identity() { + let alerts = row("d1", vec![target(TargetModule::Alerts)], 0, HOUR); + let mut cancelled = row("d2", vec![target(TargetModule::Slos)], 0, HOUR); + cancelled.cancelled_at = Some(1); + let rows = [alerts, cancelled]; + assert!(any_live_target(&rows, TargetModule::Alerts, 1)); + assert!(!any_live_target(&rows, TargetModule::Slos, 1)); + assert!(!any_live_target(&rows, TargetModule::Synthetics, 1)); + assert!(!any_live_target(&[], TargetModule::Alerts, 1)); + } + + #[test] + fn a_row_whose_windows_are_over_takes_the_org_off_the_slow_path() { + const DAY: i64 = 24 * HOUR; + let ended_yesterday = row("d1", vec![target(TargetModule::Alerts)], 0, HOUR); + let now = DAY + 2 * HOUR; + assert!(!any_live_target( + std::slice::from_ref(&ended_yesterday), + TargetModule::Alerts, + now + )); + let scheduled = row( + "d2", + vec![target(TargetModule::Alerts)], + 2 * DAY, + 2 * DAY + HOUR, + ); + assert!(any_live_target(&[scheduled], TargetModule::Alerts, now)); + let mut daily = row("d3", vec![target(TargetModule::Alerts)], 0, HOUR); + daily.schedule = DowntimeSchedule { + repeat: Repeat::Daily, + starts_at: 0, + ends_at: None, + timezone: "UTC".to_string(), + start_time_local: Some("00:00".to_string()), + duration_secs: 3_600, + weekdays: vec![], + }; + assert!(any_live_target( + &[ended_yesterday, daily], + TargetModule::Alerts, + now + )); + } + + #[test] + fn a_window_that_ended_after_since_still_counts_for_closing_its_mute() { + let ended = row("d1", vec![target(TargetModule::Alerts)], 0, HOUR); + let rows = std::slice::from_ref(&ended); + let now = 2 * HOUR; + assert!(!any_live_target(rows, TargetModule::Alerts, now)); + assert!(any_target_since(rows, TargetModule::Alerts, HOUR / 2, now)); + assert!(!any_target_since(rows, TargetModule::Alerts, HOUR, now)); + assert!(!any_target_since(rows, TargetModule::Slos, 0, now)); + } + + #[test] + fn only_a_cancelled_row_leaves_no_mute_to_close() { + let mut cancelled = row("d1", vec![target(TargetModule::Alerts)], 0, 20 * HOUR); + cancelled.cancelled_at = Some(HOUR); + assert!(!any_row_in( + std::slice::from_ref(&cancelled), + TargetModule::Alerts + )); + // A clear run 25 h after a one-time window still finds the row and forgets its mute. + let now = 26 * HOUR; + let ended = row("d2", vec![target(TargetModule::Alerts)], 0, HOUR); + let rows = [cancelled, ended]; + assert!(!any_target_since( + &rows, + TargetModule::Alerts, + now - 24 * HOUR, + now + )); + assert!(any_row_in(&rows, TargetModule::Alerts)); + assert!(!any_row_in(&rows, TargetModule::Slos)); + } + + #[test] + fn rows_that_cannot_be_live_skip_the_window_math() { + let now = 1_000 * HOUR; + let ended: Vec = (0..500) + .map(|i| { + let start = i64::from(i) * HOUR; + row( + &format!("d{i}"), + vec![target(TargetModule::Alerts)], + start, + start + HOUR, + ) + }) + .collect(); + assert!(ended.iter().all(|r| ended_by(r, now))); + assert!(!any_live_target(&ended, TargetModule::Alerts, now)); + assert!(!any_target_since( + &ended, + TargetModule::Alerts, + now - HOUR, + now + )); + let mut cancelled = row("c", vec![target(TargetModule::Alerts)], now, now + HOUR); + cancelled.cancelled_at = Some(now - 2 * HOUR); + assert!(!any_target_since( + &[cancelled], + TargetModule::Alerts, + now - HOUR, + now + )); + let live = row( + "l", + vec![target(TargetModule::Alerts)], + now - HOUR, + now + HOUR, + ); + assert!(!ended_by(&live, now)); + assert!(any_live_target(&[live], TargetModule::Alerts, now)); + } + + #[test] + fn a_cancelled_downtime_leaves_the_alert_muted_by_the_other_until_it_ends() { + let dims = payments(); + let alert = item("a1", "default", &dims, &[]); + let d1 = row( + "d1", + vec![target(TargetModule::Alerts)], + 9 * HOUR, + 11 * HOUR, + ); + let d2 = row( + "d2", + vec![target(TargetModule::Alerts)], + 9 * HOUR, + 12 * HOUR, + ); + let rows = [d1, d2]; + let at = 10 * HOUR; + let still = active_in_except(&rows, TargetModule::Alerts, &alert, at, Some("d1")); + assert_eq!(still.map(|d| d.id).as_deref(), Some("d2")); + assert!( + active_in_except(&rows[..1], TargetModule::Alerts, &alert, at, Some("d1")).is_none(), + "with no other downtime the record closes" + ); + assert!( + active_in_except(&rows, TargetModule::Alerts, &alert, 12 * HOUR, Some("d1")).is_none(), + "once d2 ends the record closes" + ); + } + + #[test] + fn overlapping_downtimes_give_the_latest_end_and_mute_in_either_order() { + let dims = payments(); + let alert = item("a1", "default", &dims, &[]); + let a = row( + "a", + vec![DowntimeTarget { + incident_mode: IncidentMode::None, + ..target(TargetModule::Alerts) + }], + 9 * HOUR, + 10 * HOUR, + ); + let b = row("b", vec![target(TargetModule::Alerts)], 9 * HOUR, 12 * HOUR); + for rows in [[a.clone(), b.clone()], [b, a]] { + let hit = active_in(&rows, TargetModule::Alerts, &alert, 9 * HOUR + 1).unwrap(); + assert_eq!(hit.id, "b"); + assert_eq!(hit.ends_at, 12 * HOUR); + assert_eq!(hit.incident_mode, IncidentMode::Muted); + } + } + + #[test] + fn a_later_ending_none_mode_downtime_keeps_the_mute_of_the_earlier_one() { + let early = ActiveDowntime { + id: "a".to_string(), + name: "a".to_string(), + ends_at: 10, + incident_mode: IncidentMode::Muted, + }; + let late = ActiveDowntime { + id: "b".to_string(), + name: "b".to_string(), + ends_at: 12, + incident_mode: IncidentMode::None, + }; + for merged in [ + merged(early.clone(), late.clone()), + merged(late.clone(), early.clone()), + ] { + assert_eq!((merged.id.as_str(), merged.ends_at), ("b", 12)); + assert_eq!(merged.incident_mode, IncidentMode::Muted); + } + let none = ActiveDowntime { + incident_mode: IncidentMode::None, + ..early + }; + assert_eq!(merged(none, late).incident_mode, IncidentMode::None); + } + + #[test] + fn a_recorded_downtime_holds_only_while_its_window_overlaps_the_hold() { + let mut rows = vec![row( + "d1", + vec![target(TargetModule::Alerts)], + 10 * HOUR, + 12 * HOUR, + )]; + // A hold that started inside the window still overlaps it after the window ends. + assert!(had_window_in(&rows, "d1", 12 * HOUR - 10, 12 * HOUR + 20)); + assert!(!had_window_in(&rows, "d1", 13 * HOUR, 14 * HOUR)); + assert!(!had_window_in(&rows, "d1", 12 * HOUR, 12 * HOUR), "no hold"); + assert!(!had_window_in(&rows, "d2", 10 * HOUR, 11 * HOUR)); + rows[0].cancelled_at = Some(10 * HOUR); + assert!(!had_window_in(&rows, "d1", 11 * HOUR, 12 * HOUR)); + } + + #[test] + fn a_recorded_downtime_covers_a_recovery_its_own_scope_cannot_see() { + let rows = vec![row( + "d1", + vec![target(TargetModule::Alerts)], + 10 * HOUR, + 12 * HOUR, + )]; + // The firing row carried service=payments; the definition alone has no service. + let no_dims = HashMap::new(); + let definition = item("a1", "default", &no_dims, &[]); + assert!(active_in(&rows, TargetModule::Alerts, &definition, 11 * HOUR).is_none()); + let recorded = active_by_id_in(&rows, TargetModule::Alerts, "d1", 11 * HOUR).unwrap(); + assert_eq!(recorded.id, "d1"); + // Asked at the recovery instant, so a hold that ends after the window still finds it. + assert!(active_by_id_in(&rows, TargetModule::Alerts, "d1", 13 * HOUR).is_none()); + assert!(active_by_id_in(&rows, TargetModule::Slos, "d1", 11 * HOUR).is_none()); + assert!(active_by_id_in(&rows, TargetModule::Alerts, "d2", 11 * HOUR).is_none()); + let mut cancelled = rows.clone(); + cancelled[0].cancelled_at = Some(10 * HOUR); + assert!(active_by_id_in(&cancelled, TargetModule::Alerts, "d1", 11 * HOUR).is_none()); + } + + #[test] + fn active_in_carries_the_incident_mode_of_the_alerts_target() { + let dims = payments(); + let alert = item("a1", "default", &dims, &[]); + let muted = row("d1", vec![target(TargetModule::Alerts)], 0, HOUR); + let hit = active_in(&[muted], TargetModule::Alerts, &alert, 1).unwrap(); + assert_eq!(hit.incident_mode, IncidentMode::Muted); + let none = row( + "d2", + vec![DowntimeTarget { + incident_mode: IncidentMode::None, + ..target(TargetModule::Alerts) + }], + 0, + HOUR, + ); + let hit = active_in(&[none], TargetModule::Alerts, &alert, 1).unwrap(); + assert_eq!(hit.incident_mode, IncidentMode::None); + } + + #[test] + fn active_in_skips_a_cancelled_row() { + let mut cancelled = row( + "d1", + vec![target(TargetModule::Alerts)], + 10 * HOUR, + 12 * HOUR, + ); + cancelled.cancelled_at = Some(10 * HOUR + 1); + let dims = payments(); + let alert = item("a1", "default", &dims, &[]); + assert!(active_in(&[cancelled], TargetModule::Alerts, &alert, 11 * HOUR).is_none()); + } + + #[test] + fn corrections_ignore_rows_without_an_slos_target_and_rows_that_miss_the_slo() { + let mut other_service = row("d2", vec![target(TargetModule::Slos)], 0, 2 * HOUR); + other_service.condition = Some(DimensionCondition::Pair { + key: "service".to_string(), + operator: PairOperator::Eq, + value: "checkout".to_string(), + }); + let rows = vec![ + row("d1", vec![target(TargetModule::Alerts)], 0, 2 * HOUR), + other_service, + ]; + assert!(corrections_in(&rows, &slo(), &payments(), 0, 4 * HOUR).is_empty()); + } + + #[test] + fn corrections_clip_a_cancelled_row_and_put_exclude_first() { + let mut cancelled = row("d1", vec![target(TargetModule::Slos)], HOUR, 5 * HOUR); + cancelled.cancelled_at = Some(3 * HOUR); + let mut as_good = target(TargetModule::Slos); + as_good.slo_mode = Some(SloCorrectionMode::CountAsGood); + let rows = vec![row("d0", vec![as_good], 0, 2 * HOUR), cancelled]; + + let got = corrections_in( + &rows, + &slo(), + &payments(), + 2 * HOUR - 300_000_000, + 10 * HOUR, + ); + assert_eq!(got.len(), 2); + assert_eq!(got[0].downtime_id, "d1"); + assert_eq!(got[0].mode, SloCorrectionMode::Exclude); + assert_eq!( + (got[0].start, got[0].end), + (2 * HOUR - 300_000_000, 3 * HOUR) + ); + assert_eq!(got[1].downtime_id, "d0"); + assert_eq!(got[1].mode, SloCorrectionMode::CountAsGood); + } + + /// A slice is corrected when it starts inside the window, so the window is cut to those starts. + #[test] + fn corrections_are_aligned_to_the_slo_slices() { + let minute = 60_000_000; + let rows = vec![row( + "d1", + vec![target(TargetModule::Slos)], + 10 * HOUR + 10 * minute + 1, + 11 * HOUR + 2 * minute, + )]; + let got = corrections_in(&rows, &slo(), &payments(), 0, 24 * HOUR); + assert_eq!( + (got[0].start, got[0].end), + (10 * HOUR + 15 * minute, 11 * HOUR + 5 * minute) + ); + } + + #[test] + fn a_window_inside_one_slice_corrects_nothing() { + let minute = 60_000_000; + let mut hourly = slo(); + hourly.definition.slice_interval_secs = 3_600; + let rows = vec![row( + "d1", + vec![target(TargetModule::Slos)], + 10 * HOUR + 10 * minute, + 10 * HOUR + 40 * minute, + )]; + assert!(corrections_in(&rows, &hourly, &payments(), 0, 24 * HOUR).is_empty()); + } + + #[test] + fn a_30_minute_window_on_an_hourly_slo_does_not_apply() { + let minute = 60_000_000; + let mut hourly = slo(); + hourly.definition.slice_interval_secs = 3_600; + let rows = vec![ + row( + "short", + vec![target(TargetModule::Slos)], + 10 * HOUR + 10 * minute, + 10 * HOUR + 40 * minute, + ), + row( + "hour", + vec![target(TargetModule::Slos)], + 10 * HOUR, + 11 * HOUR, + ), + ]; + let refs = refs_in(&rows, &hourly, &payments(), 10 * HOUR + 20 * minute); + let applies: Vec<(&str, bool)> = refs + .iter() + .map(|r| (r.downtime_id.as_str(), r.applies)) + .collect(); + assert_eq!(applies, [("short", false), ("hour", true)]); + + let scheduled = refs_in(&rows, &hourly, &payments(), 9 * HOUR); + assert!(!scheduled[0].applies, "a scheduled window is checked too"); + } + + #[test] + fn refs_list_active_and_scheduled_rows_only() { + let mut cancelled = row("gone", vec![target(TargetModule::Slos)], 0, HOUR); + cancelled.cancelled_at = Some(1); + let rows = vec![ + row("active", vec![target(TargetModule::Slos)], 0, 2 * HOUR), + row( + "later", + vec![target(TargetModule::Slos)], + 5 * HOUR, + 6 * HOUR, + ), + row("past", vec![target(TargetModule::Slos)], 0, HOUR / 2), + cancelled, + ]; + let got: Vec = refs_in(&rows, &slo(), &payments(), HOUR) + .into_iter() + .map(|r| r.downtime_id) + .collect(); + assert_eq!(got, ["active", "later"]); + } + + #[test] + fn a_composite_tagged_with_the_service_is_suppressed_by_its_downtime() { + let rows = vec![row( + "d1", + vec![target(TargetModule::Alerts)], + 10 * HOUR, + 12 * HOUR, + )]; + let tags = ["service:payments".to_string(), "team:core".to_string()]; + let dims = o2_enterprise::enterprise::downtimes::scope::composite_dimensions(&tags); + let composite = item("c1", "default", &dims, &[]); + let hit = active_in(&rows, TargetModule::Alerts, &composite, 11 * HOUR); + assert_eq!(hit.map(|d| d.id).as_deref(), Some("d1")); + + let other = o2_enterprise::enterprise::downtimes::scope::composite_dimensions(&[ + "service:checkout".to_string(), + ]); + let unrelated = item("c2", "default", &other, &[]); + assert!(active_in(&rows, TargetModule::Alerts, &unrelated, 11 * HOUR).is_none()); + } + + #[test] + fn an_slo_alert_takes_the_identity_of_its_slo_query() { + let groups = [config::meta::correlation::FieldAlias { + id: "service".to_string(), + display: "Service".to_string(), + group: None, + fields: vec!["service".to_string(), "service_name".to_string()], + is_workload_type: false, + }]; + let mut measured = slo(); + measured.definition.sli_config = SliConfig::Count { + source: CountSource::SingleQuery { + stream: "requests".to_string(), + stream_type: "logs".to_string(), + scope: Some("service_name = 'payments'".to_string()), + good_expr: "status < 500".to_string(), + }, + }; + let dims = + o2_enterprise::enterprise::downtimes::scope::slo_dimensions(&measured, &groups, None); + assert_eq!(dims.get("service").map(String::as_str), Some("payments")); + + let rows = vec![row( + "d1", + vec![target(TargetModule::Alerts)], + 10 * HOUR, + 12 * HOUR, + )]; + let slo_alert = item("burn-alert", "default", &dims, &[]); + let hit = active_in(&rows, TargetModule::Alerts, &slo_alert, 11 * HOUR); + assert_eq!(hit.map(|d| d.id).as_deref(), Some("d1")); + } + + /// A cancel re-measures the pre-cancel windows, so later slices lose their correction. + #[test] + fn a_cancel_re_measures_the_windows_of_the_row_before_the_cancel() { + let before = row("d1", vec![target(TargetModule::Slos)], HOUR, 5 * HOUR); + let mut after = before.clone(); + after.cancelled_at = Some(3 * HOUR); + + let after_only = corrections_in( + std::slice::from_ref(&after), + &slo(), + &payments(), + 0, + 10 * HOUR, + ); + let span_after = crate::slo::corrections::remeasure_span(&after_only, 0, 36_000, 300); + assert_eq!(span_after, Some((3_600, 10_800))); + + let both = corrections_in(&[before, after], &slo(), &payments(), 0, 10 * HOUR); + let span = crate::slo::corrections::remeasure_span(&both, 0, 36_000, 300); + assert_eq!( + span, + Some((3_600, 18_000)), + "the cancelled tail is measured again" + ); + } + + #[test] + fn an_anomaly_config_filtered_on_a_service_is_muted_by_that_service() { + let groups = [config::meta::correlation::FieldAlias { + id: "service".to_string(), + display: "Service".to_string(), + group: None, + fields: vec!["service".to_string(), "service_name".to_string()], + is_workload_type: false, + }]; + let filters = serde_json::json!([ + {"field": "service", "operator": "=", "value": "payments"} + ]); + let dims = o2_enterprise::enterprise::downtimes::scope::anomaly_dimensions( + None, + Some(&filters), + &[], + &groups, + ); + assert_eq!(dims.get("service").map(String::as_str), Some("payments")); + + let rows = vec![row( + "d1", + vec![target(TargetModule::AnomalyDetections)], + 10 * HOUR, + 12 * HOUR, + )]; + let config = item("an1", "default", &dims, &[]); + let hit = active_in(&rows, TargetModule::AnomalyDetections, &config, 11 * HOUR); + assert_eq!(hit.map(|d| d.id).as_deref(), Some("d1")); + } + + #[test] + fn only_a_filters_mode_config_takes_its_filters_into_the_identity() { + let filters = serde_json::json!([ + {"field": "env", "operator": "=", "value": "prod"} + ]); + assert_eq!(anomaly_filters("filters", Some(&filters)), Some(&filters)); + assert_eq!(anomaly_filters("custom_sql", Some(&filters)), None); + assert_eq!(anomaly_filters("", Some(&filters)), None); + assert_eq!(anomaly_filters("filters", None), None); + } + + #[test] + fn a_filters_mode_config_ignores_a_leftover_custom_sql() { + let sql = Some("SELECT * FROM logs WHERE env = 'prod'"); + assert_eq!(anomaly_sql("filters", sql), None); + assert_eq!(anomaly_sql("custom_sql", sql), sql); + assert_eq!(anomaly_sql("", sql), sql, "a missing mode keeps the SQL"); + } +} diff --git a/src/core/src/alerts/grouping.rs b/src/core/src/alerts/grouping.rs index 47a212fe6c..7a1c3936fb 100644 --- a/src/core/src/alerts/grouping.rs +++ b/src/core/src/alerts/grouping.rs @@ -332,26 +332,11 @@ pub fn refused_row(trigger_data: &TriggerData) -> TriggerData { } } -/// Sends a batch, then publishes its members' rows at once so a sent page never loses them. +/// Sends a batch and publishes its members' rows; false when nothing was delivered. #[cfg(feature = "enterprise")] pub async fn flush_batch(trace_id: &str, batch: PendingBatch) -> bool { - let mut rows = flush_rows(&batch); - let delivered = match send_grouped_notification(trace_id, batch).await { - Ok(attempted) => { - for row in &mut rows { - row.delivery_attempted = Some(attempted); - } - true - } - Err(e) => { - stamp_flush_error(&mut rows, &e); - false - } - }; - for row in rows { - usage_reporting::publish_triggers_usage(row); - } - delivered + let decisions = entry_decisions(&batch).await; + flush_decided(trace_id, batch, decisions).await } /// Sends one batch's notification; `Ok` says whether any destination or workflow was tried. @@ -566,6 +551,149 @@ pub async fn send_grouped_notification( } } +/// A member muted at its own evaluation leaves with a `Suppressed` row and does not count. +#[cfg(feature = "enterprise")] +async fn flush_decided( + trace_id: &str, + mut batch: PendingBatch, + decisions: Vec>, +) -> bool { + drop_muted_entries(&mut batch, decisions); + if batch.alerts.is_empty() { + return false; + } + let mut rows = flush_rows(&batch); + let delivered = match send_grouped_notification(trace_id, batch).await { + Ok(attempted) => { + for row in &mut rows { + row.delivery_attempted = Some(attempted); + } + true + } + Err(e) => { + stamp_flush_error(&mut rows, &e); + false + } + }; + for row in rows { + usage_reporting::publish_triggers_usage(row); + } + delivered +} + +/// Judged at each entry's own timestamp, so a firing admitted before a window is still sent. +#[cfg(feature = "enterprise")] +async fn entry_decisions( + batch: &PendingBatch, +) -> Vec> { + if !crate::alerts::downtimes::any_since( + &batch.org_id, + config::meta::downtimes::TargetModule::Alerts, + oldest_entry(batch), + ) { + return vec![None; batch.alerts.len()]; + } + let mut decisions = Vec::with_capacity(batch.alerts.len()); + for (entry, at) in batch.alerts.iter().zip(judged_at(batch)) { + decisions.push(entry_downtime(entry, at).await); + } + decisions +} + +/// The instant each entry is judged at: its own evaluation, never the flush. +#[cfg(feature = "enterprise")] +fn judged_at(batch: &PendingBatch) -> Vec { + batch.alerts.iter().map(|entry| entry.timestamp).collect() +} + +/// The downtime gate looks back to here, so a window that ended before the flush still mutes. +#[cfg(feature = "enterprise")] +fn oldest_entry(batch: &PendingBatch) -> i64 { + judged_at(batch).into_iter().min().unwrap_or_default() +} + +#[cfg(feature = "enterprise")] +fn drop_muted_entries( + batch: &mut PendingBatch, + decisions: Vec>, +) { + for (entry, downtime) in take_muted(batch, decisions) { + log::info!( + "[alert_grouping_worker] alert {}/{} dropped from its batch by downtime {}", + entry.alert.org_id, + entry.alert.name, + downtime.id + ); + crate::alerts::alert::count_suppressed_run(&entry.alert.org_id, "alerts"); + usage_reporting::publish_triggers_usage(suppressed_entry_record(&entry, downtime)); + } +} + +/// Removes the muted entries from the batch and returns them with their downtime. +#[cfg(feature = "enterprise")] +fn take_muted( + batch: &mut PendingBatch, + decisions: Vec>, +) -> Vec<(BatchedAlert, config::meta::downtimes::ActiveDowntime)> { + let (kept, muted) = split_muted(std::mem::take(&mut batch.alerts), decisions); + batch.alerts = kept; + muted +} + +/// The entries no downtime covers, in their order, and the others with their downtime. +#[cfg(feature = "enterprise")] +fn split_muted( + entries: Vec, + decisions: Vec>, +) -> (Vec, Vec<(T, config::meta::downtimes::ActiveDowntime)>) { + let mut kept = Vec::with_capacity(entries.len()); + let mut muted = Vec::new(); + for (entry, decision) in entries.into_iter().zip(decisions) { + match decision { + Some(downtime) => muted.push((entry, downtime)), + None => kept.push(entry), + } + } + (kept, muted) +} + +#[cfg(feature = "enterprise")] +async fn entry_downtime( + entry: &BatchedAlert, + at: i64, +) -> Option { + let alert_id = entry.alert.id.as_ref()?.to_string(); + let (folder, _) = + crate::db::alerts::alert::get_alert_from_cache(&entry.alert.org_id, &alert_id).await?; + let identity = + crate::alerts::scheduler::handlers::downtime_identity(&entry.alert, &entry.rows).await; + crate::alerts::scheduler::handlers::muted_in_every_group(&identity, |dims| { + crate::alerts::downtimes::active_for_alert( + &entry.alert.org_id, + &alert_id, + &folder.folder_id, + dims, + at, + ) + }) +} + +/// The held-back evaluation row of a dropped entry, recorded as suppressed by its downtime. +#[cfg(feature = "enterprise")] +fn suppressed_entry_record( + entry: &BatchedAlert, + downtime: config::meta::downtimes::ActiveDowntime, +) -> TriggerData { + TriggerData { + status: RunOutcome::Suppressed, + downtime_id: Some(downtime.id), + delivery_attempted: Some(false), + error: None, + grouped: Some(true), + ..entry.trigger_data.clone() + } +} + #[cfg(test)] mod tests { use super::*; @@ -869,4 +997,175 @@ mod tests { PENDING_BATCHES.remove(&batch_key(busy, fp1)); PENDING_BATCHES.remove(&batch_key(busy, fp2)); } + + #[cfg(feature = "enterprise")] + fn downtime(id: &str) -> config::meta::downtimes::ActiveDowntime { + config::meta::downtimes::ActiveDowntime { + id: id.to_string(), + name: id.to_string(), + ends_at: 0, + incident_mode: Default::default(), + } + } + + #[cfg(feature = "enterprise")] + #[test] + fn a_flush_drops_the_muted_entries_and_keeps_the_rest_in_order() { + let (kept, muted) = split_muted( + vec!["a", "b", "c"], + vec![None, Some(downtime("dt-1")), None], + ); + assert_eq!(kept, ["a", "c"]); + assert_eq!(muted.len(), 1); + assert_eq!((muted[0].0, muted[0].1.id.as_str()), ("b", "dt-1")); + + let (kept, muted) = split_muted(vec!["a"], vec![Some(downtime("dt-1"))]); + assert!(kept.is_empty(), "an emptied batch sends nothing"); + assert_eq!(muted.len(), 1); + } + + #[cfg(feature = "enterprise")] + fn queued_batch(fp: &str, org: &str, timestamps: &[i64]) -> PendingBatch { + for (i, _) in timestamps.iter().enumerate() { + admit(fp, org, 10, evaluation_row(&format!("a/{i}"))); + } + let mut batch = PENDING_BATCHES.remove(&batch_key(org, fp)).unwrap().1; + for (entry, at) in batch.alerts.iter_mut().zip(timestamps) { + entry.timestamp = *at; + } + batch + } + + // Dropping an entry publishes its row, and the usage queue starts on a Tokio runtime. + #[cfg(feature = "enterprise")] + #[tokio::test] + async fn an_entry_queued_before_the_window_stays_in_the_batch_at_flush() { + let window_opens = 100; + let mut batch = queued_batch("grouping_test_before_window", "org-before", &[50, 150]); + let decisions = judged_at(&batch) + .into_iter() + .map(|at| (at >= window_opens).then(|| downtime("dt-1"))) + .collect(); + drop_muted_entries(&mut batch, decisions); + let kept: Vec<_> = batch.alerts.iter().map(|e| e.timestamp).collect(); + assert_eq!(kept, [50]); + } + + #[cfg(feature = "enterprise")] + #[test] + fn an_entry_inside_a_window_that_ended_before_the_flush_is_still_judged() { + use config::meta::downtimes::{ + Downtime, DowntimeSchedule, DowntimeTarget, Repeat, TargetFolders, TargetModule, + }; + + use crate::alerts::downtimes::enterprise::{any_live_target, any_target_since}; + + const MINUTE: i64 = 60_000_000; + let (opens, closes) = (540 * MINUTE, 600 * MINUTE); + let only_window = Downtime { + id: "dt-1".to_string(), + org: "org-ended".to_string(), + folder_id: "default".to_string(), + name: "dt-1".to_string(), + reason: None, + condition: None, + targets: vec![DowntimeTarget { + module: TargetModule::Alerts, + folders: TargetFolders::All, + tags: vec![], + ids: vec![], + slo_mode: None, + incident_mode: Default::default(), + }], + schedule: DowntimeSchedule { + repeat: Repeat::None, + starts_at: opens, + ends_at: Some(closes), + timezone: "UTC".to_string(), + start_time_local: None, + duration_secs: 3_600, + weekdays: vec![], + }, + cancelled_at: None, + cancelled_by: None, + show_banner: false, + notifications: None, + origin_region: None, + version: 1, + created_by: "lin".to_string(), + created_at: 0, + updated_by: "lin".to_string(), + updated_at: 0, + }; + let rows = std::slice::from_ref(&only_window); + let batch = queued_batch("grouping_test_ended_window", "org-ended", &[598 * MINUTE]); + let flush = 602 * MINUTE; + assert!(!any_live_target(rows, TargetModule::Alerts, flush)); + assert!(any_target_since( + rows, + TargetModule::Alerts, + oldest_entry(&batch), + flush + )); + } + + #[cfg(feature = "enterprise")] + #[tokio::test] + async fn an_emptied_batch_records_no_delivery() { + let batch = queued_batch("grouping_test_emptied", "org-emptied", &[150, 160]); + let decisions = vec![Some(downtime("dt-1")), Some(downtime("dt-1"))]; + assert!(!flush_decided("t", batch, decisions).await); + } + + #[cfg(feature = "enterprise")] + #[test] + fn a_dropped_entry_is_recorded_as_suppressed_with_its_downtime() { + let entry = BatchedAlert { + alert: make_alert(), + rows: vec![], + timestamp: 5, + trigger_data: TriggerData { + start_time: 5, + end_time: 9, + ..evaluation_row("a/1") + }, + }; + let record = suppressed_entry_record(&entry, downtime("dt-1")); + assert_eq!(record.key, "a/1"); + assert_eq!(record.status, RunOutcome::Suppressed); + assert_eq!(record.downtime_id.as_deref(), Some("dt-1")); + assert_eq!((record.start_time, record.end_time), (5, 9)); + assert_eq!(record.grouped, Some(true)); + assert_eq!(record.delivery_attempted, Some(false)); + } + + #[cfg(feature = "enterprise")] + #[test] + fn a_muted_entry_leaves_the_batch_before_its_rows_are_built() { + let (fp, org) = ("grouping_test_muted_flush", "org-muted-flush"); + assert_eq!( + admit(fp, org, 10, evaluation_row("a/1")), + BatchAdmission::Queued + ); + assert_eq!( + admit(fp, org, 10, evaluation_row("b/2")), + BatchAdmission::Queued + ); + let mut batch = PENDING_BATCHES.remove(&batch_key(org, fp)).unwrap().1; + + let muted = take_muted(&mut batch, vec![None, Some(downtime("dt-1"))]); + let suppressed: Vec<_> = muted + .into_iter() + .map(|(entry, downtime)| suppressed_entry_record(&entry, downtime)) + .collect(); + let sent = flush_rows(&batch); + + // One row per evaluation: the muted one only as Suppressed, the rest as the batch. + assert_eq!(suppressed.len(), 1); + assert_eq!(suppressed[0].key, "b/2"); + assert_eq!(suppressed[0].status, RunOutcome::Suppressed); + let keys: Vec<&str> = sent.iter().map(|row| row.key.as_str()).collect(); + assert_eq!(keys, ["a/1"]); + assert_eq!(sent[0].group_size, Some(1)); + } } diff --git a/src/core/src/alerts/incidents.rs b/src/core/src/alerts/incidents.rs index 88a2588397..1d981e9eac 100644 --- a/src/core/src/alerts/incidents.rs +++ b/src/core/src/alerts/incidents.rs @@ -39,6 +39,8 @@ use config::{ const UNKNOWN_SERVICE: &str = "unknown"; /// Window behind the alert in which a service-graph edge counts as a dependency. const INCIDENT_EDGE_RANGE_SECS: i64 = 3600; +/// How many of an alert's latest firings a Firing run after a window looks through for open pages. +const MUTED_RESOLVES_SCANNED: u64 = 10; /// Service Discovery correlation result struct ServiceDiscoveryResult { @@ -94,6 +96,14 @@ struct ParallelCorrelationResult { correlation_reason: String, } +/// What a firing does to the downtime mute of the incident it joins. +#[derive(Debug, PartialEq)] +enum MuteChange<'a> { + Keep, + Unmute(&'a str), + Retarget { from: &'a str, to: &'a str }, +} + /// Extract semantic dimensions using configured distinguish_by groups only /// This replaces the current fallback that uses ALL labels #[cfg(feature = "enterprise")] @@ -571,6 +581,7 @@ pub async fn correlate_alert_to_incident( // Warning → P3). None (manual triggers, single-level alerts) keeps the // enterprise default. eval_level: Option, + downtime: Option<&config::meta::downtimes::ActiveDowntime>, ) -> Result, anyhow::Error> { let mut labels = labels_from_row(result_row); @@ -679,6 +690,7 @@ pub async fn correlate_alert_to_incident( &correlation_reason, &service_name, eval_level, + downtime.map(|d| d.id.as_str()), ) .await?; @@ -688,6 +700,7 @@ pub async fn correlate_alert_to_incident( incident_id, service_name, } = &outcome + && downtime.is_none() && o2_enterprise::enterprise::oncall::is_enabled() { // Single-sourced with `scheduler::handlers`: the row, then the alert's own conditions. @@ -770,11 +783,10 @@ pub async fn correlate_alert_to_incident( } } - // Send incident notification unless rows are empty (manual trigger path) - // or the outcome is a repeated alert (suppressed by design). + // No notification for the manual-trigger path (no rows), a suppressed firing, or a repeat. let mut notify_error = None; let mut notify_attempted = false; - if !notify_rows.is_empty() { + if !notify_rows.is_empty() && downtime.is_none() { match &outcome { IncidentCorrelationOutcome::NewIncidentCreated { incident_id, .. } | IncidentCorrelationOutcome::NewAlertTypeJoined { incident_id, .. } @@ -899,6 +911,7 @@ pub async fn correlate_external_event( &correlation_reason, &service_name, None, // external events carry no evaluated alert level + None, ) .await?; @@ -942,6 +955,7 @@ pub async fn correlate_external_event( /// assigned incident is a human's to close. `auto_resolve_after_minutes` stays as the backstop. pub async fn resolve_alert_firing( event: &config::meta::alerts::recovery::RecoveryEvent, + muted: bool, ) -> Result<(), anyhow::Error> { let Some(incident_id) = event.incident_id.as_deref() else { return Ok(()); @@ -987,11 +1001,12 @@ pub async fn resolve_alert_firing( return Ok(()); } - update_status( + update_status_as( &event.org_id, incident_id, "resolved", "system@openobserve.ai", + muted, ) .await?; log::info!( @@ -1001,6 +1016,29 @@ pub async fn resolve_alert_firing( Ok(()) } +/// The on-call recovery a muted [resolve_alert_firing] skipped, over the recent resolved incidents. +pub async fn recover_muted_resolves(org_id: &str, alert_id: &str) -> Result<(), anyhow::Error> { + if !o2_enterprise::enterprise::oncall::is_enabled() { + return Ok(()); + } + let resolved = infra::table::alert_incidents::resolved_incidents_for_alert( + org_id, + alert_id, + MUTED_RESOLVES_SCANNED, + ) + .await?; + let mut records = HashMap::with_capacity(resolved.len()); + for incident_id in &resolved { + let held = infra::table::oncall_responses::list_for_incident(org_id, incident_id).await?; + records.insert(incident_id.clone(), held); + } + for incident_id in left_open(&resolved, &records) { + o2_enterprise::enterprise::oncall::escalation::recover_for_incident(org_id, incident_id) + .await?; + } + Ok(()) +} + /// Auto-resolve the open incident containing `external.id`, but only once every /// other `External`-kind alert already linked to that incident is also resolved /// in `external_alerts` — a single source clearing shouldn't close an incident @@ -1194,6 +1232,221 @@ fn spawn_topology_enrichment( }); } +/// Starts the first RCA of an incident: at creation, or when a downtime mute is lifted. +#[cfg(feature = "enterprise")] +async fn spawn_initial_rca(org_id: &str, incident_id: &str) { + if !rca_will_run(org_id, incident_id).await { + return; + } + #[cfg(feature = "cloud")] + let usage_permit = { + let usage_context = crate::trial_quota::AiUsageContext { + user_email: "system@openobserve.ai".to_string(), + incident_id: Some(incident_id.to_string()), + ..Default::default() + }; + match crate::trial_quota::authorize_ai_usage( + org_id, + crate::trial_quota::TrialQuotaFeature::NewIncident, + &usage_context, + ) + .await + { + Ok(permit) => Some(permit), + Err(error) => { + log::info!( + "[INCIDENTS::RCA] New incident {incident_id} retained without RCA: {error}" + ); + // §6: no verdict is coming, so nothing may go on holding a page for one. + o2_enterprise::enterprise::alerts::rca_service::skip_analysis_for_incident( + org_id, + incident_id, + ) + .await; + None + } + } + }; + #[cfg(not(feature = "cloud"))] + let usage_permit = None; + + if cfg!(feature = "cloud") && usage_permit.is_none() { + return; + } + if let Err(e) = crate::incidents::append_event( + org_id, + incident_id, + config::meta::alerts::incidents::IncidentEvent::ai_analysis_begin(), + ) + .await + { + log::error!( + "[INCIDENTS::RCA] Failed to emit AIAnalysisBegin for {org_id}/{incident_id}: {e}" + ); + } + + let org_id_rca = org_id.to_string(); + let incident_id_rca = incident_id.to_string(); + let handle = tokio::spawn(async move { + if let Err(e) = trigger_rca_for_incident( + org_id_rca.clone(), + incident_id_rca.clone(), + false, + true, + "system@openobserve.ai".to_string(), + // First analysis for this incident — there is nothing to build on. + false, + usage_permit, + ) + .await + { + log::debug!( + "[INCIDENTS::RCA] Immediate trigger failed for incident {incident_id_rca}: {e}" + ); + + emit_analysis_failure( + &org_id_rca, + &incident_id_rca, + config::meta::alerts::incidents::AnalysisTriggerType::AutomaticNewIncident, + "Background spawn failed", + Some(&e), + ) + .await; + } + unregister_rca_task(&org_id_rca, &incident_id_rca); + }); + register_rca_task(org_id, incident_id, handle.abort_handle()); +} + +#[cfg(not(feature = "enterprise"))] +async fn spawn_initial_rca(_org_id: &str, _incident_id: &str) {} + +/// Runs the incident-created workflows; an error must not block the incident flow. +async fn trigger_created_workflows(org_id: &str, incident_id: &str) { + if let Err(e) = + crate::incidents::send_incident_event_trigger(org_id, incident_id, IncidentEvent::created()) + .await + { + log::error!( + "error triggering workflow for incident created for {org_id} incident id {incident_id} : {e}" + ); + } +} + +/// Clears a downtime mute; `true` only for the compare-and-set winner, so "opened" goes once. +async fn unmute_for_firing( + org_id: &str, + incident: &infra::table::entity::alert_incidents::Model, + muted_by: Option<&str>, +) -> bool { + let downtime_id = match mute_change(muted_by, incident.muted_by_downtime_id.as_deref()) { + MuteChange::Keep => return false, + MuteChange::Retarget { from, to } => { + retarget_mute(org_id, &incident.id, from, to).await; + return false; + } + MuteChange::Unmute(downtime_id) => downtime_id, + }; + match infra::table::alert_incidents::clear_muted_by_downtime_id( + org_id, + &incident.id, + downtime_id, + ) + .await + { + Ok(true) => {} + Ok(false) => return false, + Err(e) => { + log::error!( + "[incidents] could not un-mute incident {}: {e}", + incident.id + ); + return false; + } + } + #[cfg(feature = "enterprise")] + if db::downtimes::replicates() + && let Err(e) = + o2_enterprise::enterprise::super_cluster::queue::incidents_unmute(org_id, &incident.id) + .await + { + log::error!("[SUPER_CLUSTER] Failed to publish incident unmute: {e}"); + } + log::info!( + "[incidents] Incident {} un-muted by a firing after downtime {downtime_id}", + incident.id + ); + true +} + +/// A firing outside every window un-mutes; one muted by another downtime moves the mute to it. +fn mute_change<'a>(muted_by: Option<&'a str>, recorded: Option<&'a str>) -> MuteChange<'a> { + match (muted_by, recorded) { + (None, Some(recorded)) => MuteChange::Unmute(recorded), + (Some(current), Some(recorded)) if current != recorded => MuteChange::Retarget { + from: recorded, + to: current, + }, + _ => MuteChange::Keep, + } +} + +/// Joins first, so a failed join leaves the mute for the next firing to clear and reopen. +async fn join_then_unmute( + join: impl std::future::Future>, + unmute: impl AsyncFnOnce() -> bool, +) -> Result<(T, bool), infra::errors::Error> { + let joined = join.await?; + Ok((joined, unmute().await)) +} + +/// The chip and the resolved comment name the downtime that mutes the incident now. +async fn retarget_mute(org_id: &str, incident_id: &str, from: &str, to: &str) { + match infra::table::alert_incidents::retarget_muted_by_downtime_id( + org_id, + incident_id, + from, + to, + ) + .await + { + Ok(true) => log::info!( + "[incidents] Incident {incident_id} now muted by downtime {to} instead of {from}" + ), + Ok(false) => {} + Err(e) => log::error!("[incidents] could not re-mute incident {incident_id} by {to}: {e}"), + } +} + +/// An un-muted incident runs as a fresh one: workflows, RCA, then the page and "opened". +async fn reopened_after_mute( + org_id: &str, + incident_id: String, + service_name: &str, +) -> IncidentCorrelationOutcome { + trigger_created_workflows(org_id, &incident_id).await; + spawn_initial_rca(org_id, &incident_id).await; + IncidentCorrelationOutcome::NewIncidentCreated { + incident_id, + service_name: service_name.to_string(), + } +} + +/// The un-mute already won, so a failed severity raise is logged and the "opened" page still goes. +async fn escalate_then_reopen( + org_id: &str, + incident_id: &str, + escalate: impl std::future::Future>, + reopen: impl std::future::Future, +) -> IncidentCorrelationOutcome { + if let Err(e) = escalate.await { + log::error!( + "[incidents] severity of reopened incident {org_id}/{incident_id} not raised: {e}" + ); + } + reopen.await +} + /// Create a brand new incident and set up its initial state. #[allow(clippy::too_many_arguments)] async fn create_new_incident( @@ -1205,6 +1458,7 @@ async fn create_new_incident( correlation_reason: &str, service_name: &str, eval_level: Option, + muted_by: Option<&str>, ) -> Result { // Pre-mapped severity from the subject (external events) wins; otherwise // T-8's evaluated level maps to incident severity — Critical opens a P2, @@ -1222,13 +1476,14 @@ async fn create_new_incident( let title = o2_enterprise::enterprise::alerts::incidents::generate_title(&subject.name, group_values); - let incident = infra::table::alert_incidents::create( + let incident = infra::table::alert_incidents::create_with_mute( org_id, &severity, serde_json::to_value(group_values)?, &key_type.to_string(), triggered_at, Some(title.clone()), + muted_by.map(str::to_string), ) .await?; @@ -1240,18 +1495,8 @@ async fn create_new_incident( ); } - // error in sending workflow trigger should not block the incident flow - if let Err(e) = crate::incidents::send_incident_event_trigger( - org_id, - &incident.id, - IncidentEvent::created(), - ) - .await - { - log::error!( - "error triggering workflow for incident created for {org_id} incident id {} : {e}", - incident.id - ); + if muted_by.is_none() { + trigger_created_workflows(org_id, &incident.id).await; } // Add the first alert to the incident @@ -1310,13 +1555,17 @@ async fn create_new_incident( .super_cluster .enabled && !config::get_config().common.local_mode - && let Err(e) = o2_enterprise::enterprise::super_cluster::queue::incidents_create( + && let Err(e) = o2_enterprise::enterprise::super_cluster::queue::incidents_create_with_mute( org_id, &key_type.to_string(), &severity, serde_json::to_value(group_values)?, triggered_at, Some(title), + // Without replication the other region never gets the un-mute, so it gets no mute. + muted_by + .filter(|_| db::downtimes::replicates()) + .map(str::to_string), ) .await { @@ -1334,90 +1583,8 @@ async fn create_new_incident( triggered_at, ); - // Trigger immediate RCA for new incident. - #[cfg(feature = "enterprise")] - { - if rca_will_run(org_id, &incident.id).await { - #[cfg(feature = "cloud")] - let usage_permit = { - let usage_context = crate::trial_quota::AiUsageContext { - user_email: "system@openobserve.ai".to_string(), - incident_id: Some(incident.id.clone()), - ..Default::default() - }; - match crate::trial_quota::authorize_ai_usage( - org_id, - crate::trial_quota::TrialQuotaFeature::NewIncident, - &usage_context, - ) - .await - { - Ok(permit) => Some(permit), - Err(error) => { - log::info!( - "[INCIDENTS::RCA] New incident {} retained without RCA: {error}", - incident.id - ); - // §6: no verdict is coming, so nothing may go on holding a page for one. - o2_enterprise::enterprise::alerts::rca_service::skip_analysis_for_incident( - org_id, - &incident.id, - ) - .await; - None - } - } - }; - #[cfg(not(feature = "cloud"))] - let usage_permit = None; - - if !cfg!(feature = "cloud") || usage_permit.is_some() { - if let Err(e) = crate::incidents::append_event( - org_id, - &incident.id, - config::meta::alerts::incidents::IncidentEvent::ai_analysis_begin(), - ) - .await - { - log::error!( - "[INCIDENTS::RCA] Failed to emit AIAnalysisBegin for {org_id}/{}: {e}", - incident.id - ); - } - - let org_id_rca = org_id.to_string(); - let incident_id_rca = incident.id.clone(); - let handle = tokio::spawn(async move { - if let Err(e) = trigger_rca_for_incident( - org_id_rca.clone(), - incident_id_rca.clone(), - false, - true, - "system@openobserve.ai".to_string(), - // First analysis for this incident — there is nothing to build on. - false, - usage_permit, - ) - .await - { - log::debug!( - "[INCIDENTS::RCA] Immediate trigger failed for incident {incident_id_rca}: {e}" - ); - - emit_analysis_failure( - &org_id_rca, - &incident_id_rca, - config::meta::alerts::incidents::AnalysisTriggerType::AutomaticNewIncident, - "Background spawn failed", - Some(&e), - ) - .await; - } - unregister_rca_task(&org_id_rca, &incident_id_rca); - }); - register_rca_task(org_id, &incident.id, handle.abort_handle()); - } - } + if muted_by.is_none() { + spawn_initial_rca(org_id, &incident.id).await; } Ok(IncidentCorrelationOutcome::NewIncidentCreated { @@ -1426,6 +1593,82 @@ async fn create_new_incident( }) } +/// A repeat at a more urgent level raises the severity; the alert's own priority beats the level. +async fn escalate_severity( + org_id: &str, + incident: &infra::table::entity::alert_incidents::Model, + subject: &CorrelationSubject, + eval_level: Option, +) -> Result { + let Some((current_severity, new_severity)) = + escalated_severity(&incident.severity, subject.severity, eval_level) + else { + return Ok(false); + }; + infra::table::alert_incidents::update_severity(org_id, &incident.id, &new_severity.to_string()) + .await?; + if let Err(e) = infra::table::incident_events::append( + org_id, + &incident.id, + IncidentEvent::severity_upgrade( + current_severity, + new_severity, + format!("alert '{}' escalated to {}", subject.name, new_severity), + ), + ) + .await + { + log::error!( + "[Incidents] Failed to record severity-upgrade event for incident {org_id}/{}: {e}", + incident.id + ); + } + log::info!( + "[Incidents] Incident {} escalated {current_severity} -> {new_severity} by alert '{}'", + incident.id, + subject.name + ); + Ok(true) +} + +/// The current and the raised severity when the firing's level is more urgent than the incident. +fn escalated_severity( + current: &str, + subject_severity: Option, + eval_level: Option, +) -> Option<( + config::meta::alerts::incidents::IncidentSeverity, + config::meta::alerts::incidents::IncidentSeverity, +)> { + use config::meta::alerts::incidents::IncidentSeverity; + let level_severity = subject_severity.or_else(|| { + eval_level.and_then(|l| match l { + config::meta::alerts::level::AlertLevel::Critical => Some(IncidentSeverity::P2), + config::meta::alerts::level::AlertLevel::Warning => Some(IncidentSeverity::P3), + _ => None, + }) + }); + // P1 is most urgent; higher urgency = escalation. + let urgency = |s: IncidentSeverity| match s { + IncidentSeverity::P1 => 4u8, + IncidentSeverity::P2 => 3, + IncidentSeverity::P3 => 2, + IncidentSeverity::P4 => 1, + }; + let new_severity = level_severity?; + let current_severity = current.parse::().ok()?; + (urgency(new_severity) > urgency(current_severity)).then_some((current_severity, new_severity)) +} + +/// A suppressed firing, or one that joins a still-muted incident, starts no workflow and no page. +fn firing_is_muted( + muted_by: Option<&str>, + incident: &infra::table::entity::alert_incidents::Model, + reopened: bool, +) -> bool { + muted_by.is_some() || (incident.muted_by_downtime_id.is_some() && !reopened) +} + /// Find an existing open incident or create a new one #[allow(clippy::too_many_arguments)] async fn find_or_create_incident( @@ -1437,6 +1680,7 @@ async fn find_or_create_incident( correlation_reason: &str, service_name: &str, eval_level: Option, + muted_by: Option<&str>, ) -> Result { use config::meta::alerts::incidents::{DimensionRelationship, KeyType}; @@ -1449,13 +1693,16 @@ async fn find_or_create_incident( infra::table::alert_incidents::find_open_incident_by_alert_id(org_id, &alert_id).await? { // Found existing AlertId incident for this alert - join it - let _ = infra::table::alert_incidents::add_alert_to_incident( - &incident.id, - &alert_id, - &subject.name, - subject.kind.as_str(), - triggered_at, - correlation_reason, + let (_, reopened) = join_then_unmute( + infra::table::alert_incidents::add_alert_to_incident( + &incident.id, + &alert_id, + &subject.name, + subject.kind.as_str(), + triggered_at, + correlation_reason, + ), + async || unmute_for_firing(org_id, &incident, muted_by).await, ) .await?; @@ -1474,12 +1721,14 @@ async fn find_or_create_incident( ); } - if let Err(e) = crate::incidents::send_incident_event_trigger( - org_id, - &incident.id, - IncidentEvent::alert(&alert_id, &subject.name, triggered_at), - ) - .await + if muted_by.is_none() + && !reopened + && let Err(e) = crate::incidents::send_incident_event_trigger( + org_id, + &incident.id, + IncidentEvent::alert(&alert_id, &subject.name, triggered_at), + ) + .await { log::error!( "error triggering workflow for alert added for {org_id} incident id {} : {e}", @@ -1495,66 +1744,21 @@ async fn find_or_create_incident( &subject.name, triggered_at, ); + if reopened { + // Recomputed from the un-muting firing, so "opened" carries the right level. + return Ok(escalate_then_reopen( + org_id, + &incident.id, + escalate_severity(org_id, &incident, subject, eval_level), + reopened_after_mute(org_id, incident.id.clone(), service_name), + ) + .await); + } - // T-8/§7.1: a repeat at HIGHER severity is an ESCALATION, not a - // repeat. A Warning-created P3 incident must upgrade to P2 and - // notify when the alert re-fires at Critical — the scheduler's - // silence layer explicitly let this delivery through, and - // suppressing it here would lose the only page for the - // escalation. - // - // B-29: `subject.severity` (alert.priority, when set) is the same - // precedence signal `create_new_incident` uses and takes the same - // priority here — otherwise a P1 alert repeating against an - // incident it didn't create could be capped at the eval_level - // default (P2) instead of escalating to the priority it's - // actually configured for. - use config::meta::alerts::incidents::{IncidentEvent, IncidentSeverity}; - let level_severity = subject.severity.or_else(|| { - eval_level.and_then(|l| match l { - config::meta::alerts::level::AlertLevel::Critical => Some(IncidentSeverity::P2), - config::meta::alerts::level::AlertLevel::Warning => Some(IncidentSeverity::P3), - _ => None, - }) - }); - // P1 is most urgent; higher urgency = escalation. - let urgency = |s: IncidentSeverity| match s { - IncidentSeverity::P1 => 4u8, - IncidentSeverity::P2 => 3, - IncidentSeverity::P3 => 2, - IncidentSeverity::P4 => 1, - }; - if let Some(new_severity) = level_severity - && let Ok(current_severity) = incident.severity.parse::() - && urgency(new_severity) > urgency(current_severity) + // A muted firing must not raise the severity silently; the un-muting firing does it. + if !firing_is_muted(muted_by, &incident, reopened) + && escalate_severity(org_id, &incident, subject, eval_level).await? { - infra::table::alert_incidents::update_severity( - org_id, - &incident.id, - &new_severity.to_string(), - ) - .await?; - if let Err(e) = infra::table::incident_events::append( - org_id, - &incident.id, - IncidentEvent::severity_upgrade( - current_severity, - new_severity, - format!("alert '{}' escalated to {}", subject.name, new_severity), - ), - ) - .await - { - log::error!( - "[Incidents] Failed to record severity-upgrade event for incident {org_id}/{}: {e}", - incident.id - ); - } - log::info!( - "[Incidents] Incident {} escalated {current_severity} -> {new_severity} by alert '{}'", - incident.id, - subject.name - ); return Ok(IncidentCorrelationOutcome::SeverityEscalated { incident_id: incident.id, service_name: service_name.to_string(), @@ -1599,13 +1803,16 @@ async fn find_or_create_incident( let mut merged_dims = existing_dims.clone(); let dimensions_changed = merge_dimensions(&mut merged_dims, group_values, &existing.id); - let is_new_alert_type = infra::table::alert_incidents::add_alert_to_incident( - &existing.id, - &subject.id, - &subject.name, - subject.kind.as_str(), - triggered_at, - correlation_reason, + let (is_new_alert_type, reopened) = join_then_unmute( + infra::table::alert_incidents::add_alert_to_incident( + &existing.id, + &subject.id, + &subject.name, + subject.kind.as_str(), + triggered_at, + correlation_reason, + ), + async || unmute_for_firing(org_id, &existing, muted_by).await, ) .await?; @@ -1625,6 +1832,8 @@ async fn find_or_create_incident( } if !is_new_alert_type + && muted_by.is_none() + && !reopened && let Err(e) = crate::incidents::send_incident_event_trigger( org_id, &existing.id, @@ -1658,13 +1867,14 @@ async fn find_or_create_incident( ) .await?; - if let Err(e) = crate::incidents::append_event( + if let Err(e) = append_event_unless_muted( org_id, &existing.id, config::meta::alerts::incidents::IncidentEvent::dimensions_upgraded( old_key_type_str, new_key_type_str, ), + firing_is_muted(muted_by, &existing, reopened), ) .await { @@ -1713,10 +1923,19 @@ async fn find_or_create_incident( &subject.name, triggered_at, ); + if reopened { + return Ok(escalate_then_reopen( + org_id, + &existing.id, + escalate_severity(org_id, &existing, subject, eval_level), + reopened_after_mute(org_id, existing.id.clone(), service_name), + ) + .await); + } if is_new_alert_type { #[cfg(feature = "enterprise")] - { + if muted_by.is_none() { let org_id_rca = org_id.to_string(); let incident_id_rca = existing.id.clone(); let cooldown = o2_enterprise::enterprise::common::config::get_config() @@ -1826,6 +2045,7 @@ async fn find_or_create_incident( correlation_reason, service_name, eval_level, + muted_by, ) .await } @@ -2699,7 +2919,48 @@ async fn model_to_incident( Ok(model_to_incident_with_topology(db_model, topology)) } +/// The resolved incidents whose on-call record a muted resolve left open, in scan order. +fn left_open<'a>( + resolved: &'a [String], + records: &HashMap>, +) -> Vec<&'a str> { + resolved + .iter() + .filter(|id| records.get(*id).is_some_and(|held| holds_open_record(held))) + .map(String::as_str) + .collect() +} + +/// A record still open on a resolved incident is the page a muted resolve left escalating. +fn holds_open_record(records: &[config::meta::oncall::Response]) -> bool { + records.iter().any(|record| !record.state.is_terminal()) +} + +/// Only a resolve can clear a mute, and only an org with downtimes can have one. +fn reads_mute_before(status: &str, downtimes_enabled: bool) -> bool { + downtimes_enabled && status == "resolved" +} + +/// A quiet change or a muted incident pages nobody and starts no workflow. +fn status_change_is_muted(quiet: bool, muted_by_downtime_id: Option<&str>) -> bool { + quiet || muted_by_downtime_id.is_some() +} + /// Convert database model to domain model with pre-fetched topology +/// A muted incident keeps its timeline but starts no workflow (D3). +async fn append_event_unless_muted( + org_id: &str, + incident_id: &str, + event: IncidentEvent, + muted: bool, +) -> Result<(), anyhow::Error> { + if muted { + infra::table::incident_events::append(org_id, incident_id, event).await?; + return Ok(()); + } + crate::incidents::append_event(org_id, incident_id, event).await +} + fn model_to_incident_with_topology( db_model: infra::table::entity::alert_incidents::Model, topology_context: Option, @@ -2722,20 +2983,67 @@ fn model_to_incident_with_topology( group_values: db_model.group_values, key_type: config::meta::alerts::incidents::KeyType::from_stored(&db_model.key_type), topology_context, + muted_by_downtime_id: db_model.muted_by_downtime_id, } } +/// Clears the mute before the events, so a failed append cannot leave it muted. +async fn record_muted_auto_resolve( + conn: &C, + org_id: &str, + incident_id: &str, + downtime_id: &str, +) -> Result<(), anyhow::Error> { + infra::table::alert_incidents::clear_muted_by_downtime_id_with( + conn, + org_id, + incident_id, + downtime_id, + ) + .await?; + let resolved = IncidentEvent::resolved(None); + infra::table::incident_events::append_with(conn, org_id, incident_id, resolved).await?; + let name = infra::table::downtimes::get_with(conn, org_id, downtime_id) + .await + .ok() + .flatten() + .map_or_else(|| downtime_id.to_string(), |d| d.name); + let comment = IncidentEvent::comment("system", format!("Resolved while muted by {name}")); + infra::table::incident_events::append_with(conn, org_id, incident_id, comment).await?; + Ok(()) +} + /// Update incident status pub async fn update_status( org_id: &str, incident_id: &str, status: &str, user_id: &str, +) -> Result { + update_status_as(org_id, incident_id, status, user_id, false).await +} + +/// [update_status]; `quiet` treats the incident as muted for this change only. +pub async fn update_status_as( + org_id: &str, + incident_id: &str, + status: &str, + user_id: &str, + quiet: bool, ) -> Result { // Acknowledging goes through a dedicated atomic path so the actor and // timestamp land on the row itself (not just the event log) and a // second/concurrent acknowledge can't silently overwrite the first // acknowledger — see `infra::table::alert_incidents::acknowledge`. + // Read before the resolve clears it, so resolving a muted incident still notifies nobody. + let muted_before = reads_mute_before( + status, + o2_enterprise::enterprise::common::config::get_config() + .downtimes + .enabled, + ) && infra::table::alert_incidents::get(org_id, incident_id) + .await? + .is_some_and(|m| m.muted_by_downtime_id.is_some()); let updated = if status == "acknowledged" { infra::table::alert_incidents::acknowledge(org_id, incident_id, user_id) .await? @@ -2744,9 +3052,14 @@ pub async fn update_status( infra::table::alert_incidents::update_status(org_id, incident_id, status).await? }; + let muted = status_change_is_muted( + quiet || muted_before, + updated.muted_by_downtime_id.as_deref(), + ); // Every resolution path lands here, so closing the record once covers all of them. #[cfg(feature = "enterprise")] if status == "resolved" + && !muted && o2_enterprise::enterprise::oncall::is_enabled() && let Err(e) = o2_enterprise::enterprise::oncall::escalation::recover_for_incident(org_id, incident_id) @@ -2764,7 +3077,7 @@ pub async fn update_status( _ => None, }; if let Some(evt) = event - && let Err(e) = crate::incidents::append_event(org_id, incident_id, evt).await + && let Err(e) = append_event_unless_muted(org_id, incident_id, evt, muted).await { log::error!("[Incidents] Failed to record status event: org_id: {org_id}, error: {e}"); } @@ -2772,7 +3085,7 @@ pub async fn update_status( // Trigger RCA reanalysis when incident is reopened — context is fresh, // cooldown is bypassed, but in-flight guard still applies. #[cfg(feature = "enterprise")] - if status == "open" { + if status == "open" && !muted { let org_id_rca = org_id.to_string(); let incident_id_rca = incident_id.to_string(); let cooldown = o2_enterprise::enterprise::common::config::get_config() @@ -2881,6 +3194,20 @@ pub async fn update_status( model_to_incident(updated).await } +/// Records the resolved event of the auto-resolve job; a muted incident notifies nobody (D3). +pub async fn record_auto_resolved( + org_id: &str, + incident_id: &str, + muted_by_downtime_id: Option<&str>, +) -> Result<(), anyhow::Error> { + let Some(downtime_id) = muted_by_downtime_id else { + return crate::incidents::append_event(org_id, incident_id, IncidentEvent::resolved(None)) + .await; + }; + let conn = infra::db::get_orm_client_rw().await; + record_muted_auto_resolve(conn, org_id, incident_id, downtime_id).await +} + /// Update incident title pub async fn update_title( org_id: &str, @@ -2933,8 +3260,9 @@ pub async fn update_severity( infra::table::alert_incidents::update_severity(org_id, incident_id, severity).await?; // Emit severity override event and notify only when the severity actually changed + let muted = updated.muted_by_downtime_id.is_some(); if from_severity != to_severity { - if let Err(e) = crate::incidents::append_event( + if let Err(e) = append_event_unless_muted( org_id, incident_id, config::meta::alerts::incidents::IncidentEvent::severity_override( @@ -2942,6 +3270,7 @@ pub async fn update_severity( to_severity, user_id, ), + muted, ) .await { @@ -2949,7 +3278,9 @@ pub async fn update_severity( "[Incidents] Failed to record severity event: org_id: {org_id}, error: {e}" ); } - send_incident_severity_notification(org_id, incident_id).await; + if !muted { + send_incident_severity_notification(org_id, incident_id).await; + } } model_to_incident(updated).await @@ -2963,6 +3294,16 @@ mod tests { /// alert, so an unset priority has to mean the same thing on both. They used /// to differ — P2 here, P3 in `scheduler::handlers` — which made /// `creates_incident` a hidden severity switch. + #[test] + fn a_resolve_caused_by_a_muted_recovery_is_quiet() { + assert!( + status_change_is_muted(true, None), + "an incident opened before the window" + ); + assert!(status_change_is_muted(false, Some("dt-1"))); + assert!(!status_change_is_muted(false, None)); + } + #[test] fn test_the_incident_path_uses_the_shared_default_priority() { assert_eq!( @@ -3223,4 +3564,277 @@ mod tests { "the guard compares against this exact value, so the two must agree" ); } + + fn incident(muted_by: Option<&str>) -> infra::table::entity::alert_incidents::Model { + infra::table::entity::alert_incidents::Model { + id: "inc-1".to_string(), + org_id: "default".to_string(), + status: "open".to_string(), + severity: "P2".to_string(), + group_values: serde_json::json!({}), + key_type: "alert_id".to_string(), + topology_context: None, + first_alert_at: 1, + last_alert_at: 1, + resolved_at: None, + alert_count: 1, + title: None, + assigned_to: None, + acknowledged_by: None, + acknowledged_at: None, + created_at: 1, + updated_at: 1, + muted_by_downtime_id: muted_by.map(str::to_string), + } + } + + #[test] + fn a_muted_firing_escalates_nothing_and_the_unmuting_firing_does() { + use config::meta::alerts::{incidents::IncidentSeverity, level::AlertLevel}; + let mut opened_at_p3 = incident(Some("dt-1")); + opened_at_p3.severity = "P3".to_string(); + assert!(firing_is_muted(Some("dt-1"), &opened_at_p3, false)); + assert!(firing_is_muted(None, &opened_at_p3, false)); + assert!(!firing_is_muted(None, &opened_at_p3, true)); + assert!(firing_is_muted(Some("dt-1"), &incident(None), false)); + assert!(!firing_is_muted(None, &incident(None), false)); + assert_eq!( + escalated_severity(&opened_at_p3.severity, None, Some(AlertLevel::Critical)), + Some((IncidentSeverity::P3, IncidentSeverity::P2)), + "the opened message after the window carries P2" + ); + assert_eq!( + escalated_severity("P3", None, Some(AlertLevel::Warning)), + None + ); + assert_eq!( + escalated_severity("P2", Some(IncidentSeverity::P1), Some(AlertLevel::Warning)), + Some((IncidentSeverity::P2, IncidentSeverity::P1)) + ); + assert_eq!(escalated_severity("P2", None, None), None); + } + + #[test] + fn a_suppressed_firing_never_unmutes_and_names_the_downtime_that_mutes_it_now() { + assert_eq!( + mute_change(Some("dt-2"), Some("dt-1")), + MuteChange::Retarget { + from: "dt-1", + to: "dt-2" + } + ); + assert_eq!(mute_change(Some("dt-1"), Some("dt-1")), MuteChange::Keep); + assert_eq!(mute_change(Some("dt-1"), None), MuteChange::Keep); + assert_eq!(mute_change(None, Some("dt-1")), MuteChange::Unmute("dt-1")); + assert_eq!(mute_change(None, None), MuteChange::Keep); + } + + fn oncall_record( + incident_id: &str, + state: config::meta::oncall::ResponseState, + ) -> config::meta::oncall::Response { + use config::meta::oncall::{ResponderRole, Response, SubjectRef, SubjectType}; + Response { + id: format!("resp_{incident_id}"), + org_id: "acme".into(), + subject: SubjectRef::new(SubjectType::Alert, "al_1", 1), + team_id: Some("team_1".into()), + title: None, + cause: None, + cause_note: None, + snoozed_until: None, + ladder_anchor: None, + ladder_run: None, + priority: 2, + responder_role: ResponderRole::Owner, + exhausted_at: None, + origin_response_id: None, + state, + opened_at: 0, + acked_by: None, + acked_at: None, + closed_at: None, + incident_id: Some(incident_id.into()), + updated_at: 0, + } + } + + #[test] + fn only_a_resolved_incident_with_an_open_record_is_recovered_after_the_window() { + use config::meta::oncall::ResponseState; + let record = |state| oncall_record("inc_1", state); + assert!(holds_open_record(&[record(ResponseState::Triggered)])); + assert!(holds_open_record(&[ + record(ResponseState::Resolved), + record(ResponseState::Acknowledged) + ])); + assert!(!holds_open_record(&[record(ResponseState::Resolved)])); + assert!(!holds_open_record(&[])); + } + + #[test] + fn a_resolve_reads_the_mute_only_with_downtimes_on() { + assert!(reads_mute_before("resolved", true)); + assert!( + !reads_mute_before("resolved", false), + "no extra get with the flag off" + ); + assert!(!reads_mute_before("acknowledged", true)); + assert!(!reads_mute_before("open", true)); + } + + #[tokio::test] + async fn a_failed_join_leaves_the_mute_so_the_next_firing_still_reopens() { + use std::sync::atomic::{AtomicBool, Ordering}; + + let muted = AtomicBool::new(true); + let failed = join_then_unmute( + async { Err::(infra::errors::Error::Message("db down".to_string())) }, + async || muted.swap(false, Ordering::SeqCst), + ) + .await; + assert!(failed.is_err()); + assert!( + muted.load(Ordering::SeqCst), + "the mute stays for the next firing" + ); + let (_, reopened) = join_then_unmute(async { Ok(false) }, async || { + muted.swap(false, Ordering::SeqCst) + }) + .await + .unwrap(); + assert!( + reopened, + "the next firing reopens, so it yields NewIncidentCreated" + ); + assert!(!muted.load(Ordering::SeqCst)); + } + + #[tokio::test] + async fn an_incident_that_is_not_muted_has_nothing_to_unmute() { + assert!(!unmute_for_firing("default", &incident(None), None).await); + } + + #[tokio::test] + async fn a_muted_auto_resolve_whose_event_append_fails_still_clears_the_mute() { + use infra::table::entity::alert_incidents; + use sea_orm::{ActiveModelTrait, ConnectionTrait, Database, EntityTrait, Schema}; + + // Only the incidents table exists, so both event appends fail. + let db = Database::connect("sqlite::memory:").await.unwrap(); + let backend = db.get_database_backend(); + let table = Schema::new(backend).create_table_from_entity(alert_incidents::Entity); + db.execute(backend.build(&table)).await.unwrap(); + let row: alert_incidents::ActiveModel = incident(Some("dt-1")).into(); + alert_incidents::Entity::insert(row.reset_all()) + .exec(&db) + .await + .unwrap(); + + let recorded = record_muted_auto_resolve(&db, "default", "inc-1", "dt-1").await; + assert!(recorded.is_err()); + let stored = alert_incidents::Entity::find_by_id("inc-1".to_string()) + .one(&db) + .await + .unwrap() + .unwrap(); + assert_eq!(stored.muted_by_downtime_id, None); + } + + #[tokio::test] + async fn a_newer_open_incident_does_not_hide_the_page_a_muted_resolve_left_open() { + use config::meta::oncall::ResponseState; + use infra::table::entity::{alert_incident_alerts, alert_incidents}; + use sea_orm::{ActiveModelTrait, ConnectionTrait, Database, EntityTrait, Schema}; + + let db = Database::connect("sqlite::memory:").await.unwrap(); + let backend = db.get_database_backend(); + let schema = Schema::new(backend); + for table in [ + schema.create_table_from_entity(alert_incidents::Entity), + schema.create_table_from_entity(alert_incident_alerts::Entity), + ] { + db.execute(backend.build(&table)).await.unwrap(); + } + // I1 resolved quietly under the ended window; the Firing run after it opened I2. + for (id, status, fired_at) in [("i1", "resolved", 1), ("i2", "open", 2)] { + let mut model = incident(None); + model.id = id.to_string(); + model.status = status.to_string(); + let row: alert_incidents::ActiveModel = model.into(); + alert_incidents::Entity::insert(row.reset_all()) + .exec(&db) + .await + .unwrap(); + let link = alert_incident_alerts::Model { + incident_id: id.to_string(), + alert_id: "a1".to_string(), + alert_fired_at: fired_at, + alert_name: "a1".to_string(), + alert_kind: "alert".to_string(), + correlation_reason: None, + created_at: fired_at, + resolved_at: None, + }; + let link: alert_incident_alerts::ActiveModel = link.into(); + alert_incident_alerts::Entity::insert(link.reset_all()) + .exec(&db) + .await + .unwrap(); + } + let resolved = infra::table::alert_incidents::resolved_incidents_for_alert_with( + &db, + "default", + "a1", + MUTED_RESOLVES_SCANNED, + ) + .await + .unwrap(); + assert_eq!(resolved, ["i1"]); + + let records = HashMap::from([ + ( + "i1".to_string(), + vec![oncall_record("i1", ResponseState::Triggered)], + ), + ( + "i2".to_string(), + vec![oncall_record("i2", ResponseState::Triggered)], + ), + ]); + assert_eq!( + left_open(&resolved, &records), + ["i1"], + "I1's page is closed; the live I2 keeps its own" + ); + let closed = HashMap::from([( + "i1".to_string(), + vec![oncall_record("i1", ResponseState::Resolved)], + )]); + assert!(left_open(&resolved, &closed).is_empty()); + } + + #[tokio::test] + async fn a_failed_severity_raise_after_the_unmute_still_sends_the_opened_page() { + let mut paged = 0; + let outcome = escalate_then_reopen( + "default", + "inc-1", + async { Err(anyhow::anyhow!("db down")) }, + async { + paged += 1; + IncidentCorrelationOutcome::NewIncidentCreated { + incident_id: "inc-1".to_string(), + service_name: "payments".to_string(), + } + }, + ) + .await; + assert!(matches!( + outcome, + IncidentCorrelationOutcome::NewIncidentCreated { ref incident_id, .. } + if incident_id == "inc-1" + )); + assert_eq!(paged, 1, "the delayed opened message goes out once"); + } } diff --git a/src/core/src/alerts/mod.rs b/src/core/src/alerts/mod.rs index 52f4e3fe03..2b01849195 100644 --- a/src/core/src/alerts/mod.rs +++ b/src/core/src/alerts/mod.rs @@ -49,6 +49,7 @@ pub mod deduplication; pub mod derived_streams; pub mod destination_usage; pub mod destinations; +pub mod downtimes; pub mod external_alerts; #[cfg(feature = "enterprise")] pub mod grouping; diff --git a/src/core/src/alerts/notifications/default_template.rs b/src/core/src/alerts/notifications/default_template.rs index 3bf1237e19..76d22afe94 100644 --- a/src/core/src/alerts/notifications/default_template.rs +++ b/src/core/src/alerts/notifications/default_template.rs @@ -80,6 +80,7 @@ pub fn compiled_default_content() -> ContentSpec { pub fn is_reserved_template_name(name: &str) -> bool { config::prebuilt_loader::is_prebuilt_template_name(name) || name == DEFAULT_CONTENT_TEMPLATE_NAME + || crate::alerts::templates::is_downtime_template_name(name) } /// Create-if-missing (and overwrite under the same `should_apply_prebuilt` diff --git a/src/core/src/alerts/prom_rule.rs b/src/core/src/alerts/prom_rule.rs index 266d4f8e93..78430bc2b5 100644 --- a/src/core/src/alerts/prom_rule.rs +++ b/src/core/src/alerts/prom_rule.rs @@ -445,6 +445,7 @@ mod tests { &TriggerCondition::default(), 600, 0, + |_| false, ); assert!(dispatch.inconsistent.is_empty()); assert_eq!(dispatch.items.len(), 2); diff --git a/src/core/src/alerts/recovery.rs b/src/core/src/alerts/recovery.rs index f1e1eca27c..a8b6b65d60 100644 --- a/src/core/src/alerts/recovery.rs +++ b/src/core/src/alerts/recovery.rs @@ -16,7 +16,34 @@ //! Subsystems used to decide independently that an alert had got better; they are consumers now. //! Best effort: the episode is already committed, so a failure costs a message, not the record. -use config::meta::alerts::{alert::Alert, recovery::RecoveryEvent}; +#[cfg(feature = "enterprise")] +use config::meta::downtimes::TargetModule; +use config::meta::{ + alerts::{alert::Alert, recovery::RecoveryEvent}, + self_reporting::usage::{TriggerData, TriggerDataType}, +}; + +/// A clear run this long after the last window still closes the mute that window recorded. +#[cfg(feature = "enterprise")] +const RECORDED_MUTE_GRACE_MICROS: i64 = 24 * 3_600 * 1_000_000; + +/// Which consumers one recovery reaches. +#[derive(Debug, PartialEq)] +struct RecoveryRoute { + oncall: bool, + destinations: bool, +} + +/// One clear run's on-call decision in the scheduler. +#[cfg(feature = "enterprise")] +#[derive(Debug, PartialEq)] +struct ClearRunOncall<'a> { + withheld: bool, + /// The downtime to record as the alert's mute. + record: Option<&'a str>, + /// The recorded mute is over, so the record goes and the escalations it held close. + forget: bool, +} /// Tell every consumer, then return. Never fails. pub async fn dispatch_recovery(alert: &Alert, event: &RecoveryEvent) { @@ -28,9 +55,37 @@ pub async fn dispatch_recovery(alert: &Alert, event: &RecoveryEvent) { event.duration_micros() ); + // A recovery inside a window is dropped, never delayed: the episode is already closed. + let downtime_id = recovery_downtime(alert, event).await; + let route = recovery_route(alert, event, downtime_id.is_some()); + if let Some(downtime_id) = downtime_id.as_deref() { + log::info!( + "[RECOVERY] {}/{} recovery suppressed by downtime {downtime_id}", + event.org_id, + event.alert_id + ); + // Keeps the per-run on-call recovery withheld through a hold, then lets it close the rest. + #[cfg(feature = "enterprise")] + if let Err(e) = + infra::table::alert_states::set_last_downtime_id(&event.alert_id, Some(downtime_id)) + .await + { + log::warn!( + "[RECOVERY] could not record the downtime of {}: {e}", + event.alert_id + ); + } + usage_reporting::publish_triggers_usage(suppressed_recovery_record( + alert, + event, + downtime_id.to_string(), + )); + } + // Once per episode, not once per evaluation: that is the whole fix. #[cfg(feature = "enterprise")] - if o2_enterprise::enterprise::oncall::is_enabled() + if route.oncall + && o2_enterprise::enterprise::oncall::is_enabled() && let Err(e) = o2_enterprise::enterprise::oncall::escalation::recover_for_alert( &event.org_id, &event.alert_id, @@ -43,17 +98,16 @@ pub async fn dispatch_recovery(alert: &Alert, event: &RecoveryEvent) { ); } + // The links close even inside a window; a resolve it causes then pages and notifies nobody. #[cfg(feature = "enterprise")] - if let Err(e) = crate::alerts::incidents::resolve_alert_firing(event).await { + if let Err(e) = crate::alerts::incidents::resolve_alert_firing(event, !route.oncall).await { log::error!( "[RECOVERY] incident consumer failed for {}: {e}", event.alert_id ); } - // Correlated alerts: the incident owns the resolve because it sent the trigger. - if alert.notify_on_recovery - && event.incident_id.is_none() + if route.destinations && let Err(e) = crate::alerts::alert::send_recovery_notification(alert, event).await { log::error!( @@ -64,3 +118,430 @@ pub async fn dispatch_recovery(alert: &Alert, event: &RecoveryEvent) { // TODO: a workflow consumer, once a link can carry AlertResolved (see that variant). } + +/// Whether the scheduler's per-run on-call recovery waits: a window now, or a hold it started. +#[cfg(feature = "enterprise")] +pub(crate) async fn oncall_recovery_withheld(alert: &Alert, folder_id: &str, now: i64) -> bool { + let Some(alert_id) = alert.id.map(|id| id.to_string()) else { + return false; + }; + let since = now.saturating_sub(RECORDED_MUTE_GRACE_MICROS); + if !crate::alerts::downtimes::any_since(&alert.org_id, TargetModule::Alerts, since) { + // A clear run more than the grace after the window still closes what the mute left open. + if crate::alerts::downtimes::any_row(&alert.org_id, TargetModule::Alerts) + && recorded_downtime(&alert_id).await.is_some() + { + forget_recorded_mute(&alert.org_id, &alert_id).await; + } + return false; + } + let recorded = recorded_downtime(&alert_id).await; + let muted_now = downtime_at(alert, folder_id, recorded.as_deref(), now).await; + let episode_open = (recorded.is_some() || muted_now.is_some()) && episode_open(&alert_id).await; + let held = recorded.as_deref().is_some_and(|id| { + let hold = alert.keep_firing_for.saturating_mul(1_000_000); + crate::alerts::downtimes::had_window(&alert.org_id, id, now.saturating_sub(hold), now) + }); + let decision = clear_run_oncall( + recorded.as_deref(), + muted_now.as_deref(), + episode_open, + held, + ); + if let Some(downtime_id) = decision.record { + crate::alerts::scheduler::handlers::record_last_downtime( + &alert.org_id, + &alert_id, + Some(downtime_id), + ) + .await; + } + if decision.forget { + forget_recorded_mute(&alert.org_id, &alert_id).await; + } + decision.withheld +} + +/// A recorded mute withholds only while its downtime still covers the `keep_firing_for` hold. +#[cfg(feature = "enterprise")] +fn clear_run_oncall<'a>( + recorded: Option<&str>, + muted_now: Option<&'a str>, + episode_open: bool, + recorded_held: bool, +) -> ClearRunOncall<'a> { + let withheld = muted_now.is_some() || (recorded.is_some() && episode_open && recorded_held); + ClearRunOncall { + withheld, + record: muted_now.filter(|_| recorded.is_none() && episode_open), + forget: recorded.is_some() && !withheld, + } +} + +/// Correlated alerts: the incident owns the resolve because it sent the trigger. +fn recovery_route(alert: &Alert, event: &RecoveryEvent, muted: bool) -> RecoveryRoute { + RecoveryRoute { + oncall: !muted, + destinations: !muted && alert.notify_on_recovery && event.incident_id.is_none(), + } +} + +/// The history record of a recovery a downtime kept silent. +fn suppressed_recovery_record( + alert: &Alert, + event: &RecoveryEvent, + downtime_id: String, +) -> TriggerData { + let mut data = TriggerData { + _timestamp: event.emitted_at, + org: event.org_id.clone(), + module: TriggerDataType::Alert, + key: format!("{}/{}", alert.name, event.alert_id), + start_time: event.recovered_at, + end_time: event.emitted_at, + is_realtime: alert.is_real_time, + ..Default::default() + }; + crate::alerts::alert::record_suppressed_run(&mut data, "alerts", downtime_id); + data +} + +/// Asked at `recovered_at`, so a `keep_firing_for` hold that outlasts the window still mutes. +#[cfg(feature = "enterprise")] +async fn recovery_downtime(alert: &Alert, event: &RecoveryEvent) -> Option { + // From `recovered_at`, so a hold that outlasts the org's last window still mutes. + if !crate::alerts::downtimes::any_since(&event.org_id, TargetModule::Alerts, event.recovered_at) + { + return None; + } + let alert_id = alert.id?; + let conn = infra::db::get_orm_client_ro().await; + let folder_id = match crate::alerts::alert::get_by_id(conn, &event.org_id, alert_id).await { + Ok((folder, _)) => folder.folder_id, + Err(e) => { + log::warn!( + "[RECOVERY] folder of {}/{} unreadable for the downtime check: {e}", + event.org_id, + event.alert_id + ); + return None; + } + }; + let recorded = recorded_downtime(&event.alert_id).await; + downtime_at(alert, &folder_id, recorded.as_deref(), event.recovered_at).await +} + +#[cfg(not(feature = "enterprise"))] +async fn recovery_downtime(_alert: &Alert, _event: &RecoveryEvent) -> Option { + None +} + +/// The muted firing's downtime carries its row identity; the clear run has no rows of its own. +#[cfg(feature = "enterprise")] +async fn downtime_at( + alert: &Alert, + folder_id: &str, + recorded: Option<&str>, + at: i64, +) -> Option { + if let Some(downtime) = + recorded.and_then(|id| crate::alerts::downtimes::active_by_id(&alert.org_id, id, at)) + { + return Some(downtime.id); + } + crate::alerts::scheduler::handlers::downtime_decision(alert, folder_id, Some(&[]), at) + .await + .map(|downtime| downtime.id) +} + +/// A delivered firing whose recovery is still due, possibly held by `keep_firing_for`. +#[cfg(feature = "enterprise")] +async fn episode_open(alert_id: &str) -> bool { + match infra::table::alert_states::get(alert_id, config::meta::alerts::state::ROLLUP_GROUP_KEY) + .await + { + Ok(state) => state.is_some_and(|state| state.episode_id.is_some()), + Err(e) => { + log::warn!("[RECOVERY] state of {alert_id} unreadable for the mute record: {e}"); + false + } + } +} + +/// Clears the record and closes the escalation of the incident a muted recovery resolved quietly. +#[cfg(feature = "enterprise")] +async fn forget_recorded_mute(org: &str, alert_id: &str) { + // Direct: the org may already be off the downtime path that `record_last_downtime` checks. + if let Err(e) = infra::table::alert_states::set_last_downtime_id(alert_id, None).await { + log::warn!("[RECOVERY] could not clear the recorded downtime of {alert_id}: {e}"); + } + if let Err(e) = crate::alerts::incidents::recover_muted_resolves(org, alert_id).await { + log::error!("[RECOVERY] incident on-call recovery failed for {org}/{alert_id}: {e}"); + } +} + +/// The downtime the alert's last muted firing recorded, until a delivered firing clears it. +#[cfg(feature = "enterprise")] +async fn recorded_downtime(alert_id: &str) -> Option { + match infra::table::alert_states::last_downtime_ids(&[alert_id.to_string()]).await { + Ok(mut recorded) => recorded.remove(alert_id), + Err(e) => { + log::warn!("[RECOVERY] recorded downtime of {alert_id} unreadable: {e}"); + None + } + } +} + +#[cfg(test)] +mod tests { + use config::meta::{ + alerts::{ + recovery::{EpisodeInput, apply_episode, recovery_event}, + state::AlertState, + }, + self_reporting::usage::RunOutcome, + }; + + use super::*; + + fn delivered() -> EpisodeInput { + EpisodeInput { + delivered: true, + incident_id: None, + keep_firing_for_secs: 0, + episode_id: None, + } + } + + #[test] + fn a_recovery_inside_a_window_sends_nothing_then_or_later() { + let mut alert = Alert::default(); + alert.notify_on_recovery = true; + let mut state = AlertState::empty("a1", ""); + apply_episode(&mut state, true, &delivered(), 10, || "ep_1".into()); + let closed = apply_episode(&mut state, false, &EpisodeInput::undelivered(0), 20, || { + "x".into() + }) + .expect("the delivered episode recovers inside the window"); + let event = recovery_event("default", &state, closed, 20); + + let muted = recovery_route(&alert, &event, true); + assert_eq!( + muted, + RecoveryRoute { + oncall: false, + destinations: false + } + ); + let record = suppressed_recovery_record(&alert, &event, "dt-1".to_string()); + assert_eq!(record.status, RunOutcome::Suppressed); + assert_eq!(record.downtime_id.as_deref(), Some("dt-1")); + // The episode closed when it recovered, so no later run re-emits it. + for at in [30, 40] { + assert!( + apply_episode(&mut state, false, &EpisodeInput::undelivered(0), at, || { + "x".into() + }) + .is_none() + ); + } + } + + #[test] + fn the_next_firing_after_the_window_notifies_its_recovery() { + let mut alert = Alert::default(); + alert.notify_on_recovery = true; + let mut state = AlertState::empty("a1", ""); + apply_episode(&mut state, true, &delivered(), 50, || "ep_2".into()); + let closed = apply_episode(&mut state, false, &EpisodeInput::undelivered(0), 60, || { + "x".into() + }) + .expect("the next episode recovers"); + let event = recovery_event("default", &state, closed, 60); + assert_eq!( + recovery_route(&alert, &event, false), + RecoveryRoute { + oncall: true, + destinations: true + } + ); + } + + #[cfg(feature = "enterprise")] + #[test] + fn a_hold_that_outlasts_the_window_keeps_the_oncall_recovery_withheld() { + // Delivered before the window; the first clear run falls inside it and starts the hold. + let first = clear_run_oncall(None, Some("dt-1"), true, false); + assert_eq!( + first, + ClearRunOncall { + withheld: true, + record: Some("dt-1"), + forget: false, + } + ); + // After the window, still inside the hold: the recorded mute keeps it withheld. + let after = clear_run_oncall(first.record, None, true, true); + assert!(after.withheld); + assert_eq!(after.record, None, "recorded once"); + assert!(!after.forget); + // No open episode: withheld inside the window, nothing recorded. + assert_eq!( + clear_run_oncall(None, Some("dt-1"), false, false), + ClearRunOncall { + withheld: true, + record: None, + forget: false, + } + ); + assert!(!clear_run_oncall(None, None, true, false).withheld); + } + + #[cfg(feature = "enterprise")] + #[test] + fn a_recovery_inside_a_window_closes_the_escalation_once_the_window_is_over() { + use config::meta::oncall::{PageDecision, ResponseState, page_decision}; + + // Paged at 09:00, the window opens at 09:05 and the alert recovers at 09:10. + let inside = clear_run_oncall(None, Some("dt-1"), true, false); + assert!(inside.withheld, "no recovery is sent inside the window"); + // The muted recovery closed the episode and recorded the downtime. + let still_inside = clear_run_oncall(inside.record, Some("dt-1"), false, false); + assert!(still_inside.withheld && !still_inside.forget); + // 10:01, the window is over: the clear run recovers and drops the record. + let after = clear_run_oncall(inside.record, None, false, false); + assert_eq!( + after, + ClearRunOncall { + withheld: false, + record: None, + forget: true, + } + ); + // A record left from a window long gone does not hold a firing episode either. + let stale = clear_run_oncall(Some("dt-old"), None, true, false); + assert!(!stale.withheld && stale.forget); + // Left open, the escalation would absorb the next firing; closed, the next firing pages. + let window = config::meta::oncall::DEFAULT_FLAP_DAMPENING_SECS * 1_000_000; + assert_eq!( + page_decision( + Some(&escalation(ResponseState::Triggered, None)), + 1_000, + window + ), + PageDecision::AlreadyOpen + ); + let closed = escalation(ResponseState::Resolved, Some(1_000)); + assert_eq!( + page_decision(Some(&closed), 1_000 + window + 1, window), + PageDecision::Page + ); + } + + #[cfg(feature = "enterprise")] + #[test] + fn a_recovery_before_the_cancel_of_its_window_stays_suppressed_when_emitted_after() { + use std::collections::HashMap; + + use config::meta::downtimes::{Downtime, TargetModule}; + + use crate::alerts::downtimes::enterprise::{ + active_by_id_in, active_in, any_target_since, item, + }; + + const MINUTE: i64 = 60_000_000; + let at = |minutes: i64| 9 * 60 * MINUTE + minutes * MINUTE; + // 09:00-10:00, cancelled at 09:30. + let row: Downtime = serde_json::from_value(serde_json::json!({ + "id": "dt-1", "org": "default", "name": "deploy", + "targets": [{ "module": "alerts", "folders": { "kind": "all" } }], + "schedule": { + "repeat": "none", "starts_at": at(0), "ends_at": at(60), + "timezone": "UTC", "duration_secs": 3_600 + }, + "cancelled_at": at(30), + "created_by": "lin", "created_at": 0, "updated_by": "lin", "updated_at": 0 + })) + .unwrap(); + let rows = [row]; + let mut alert = Alert::default(); + alert.notify_on_recovery = true; + // Recovered at 09:20, emitted at 09:40 once `keep_firing_for` ran out. + let event = RecoveryEvent { + org_id: "default".into(), + alert_id: "a1".into(), + group_key: String::new(), + episode_id: "ep_1".into(), + incident_id: None, + opened_at: at(5), + recovered_at: at(20), + emitted_at: at(40), + last_notified_level: None, + group_labels: None, + }; + + assert!(any_target_since( + &rows, + TargetModule::Alerts, + event.recovered_at, + event.emitted_at + )); + let dims = HashMap::new(); + let matched = active_in( + &rows, + TargetModule::Alerts, + &item("a1", "default", &dims, &[]), + event.recovered_at, + ); + assert_eq!(matched.map(|d| d.id).as_deref(), Some("dt-1")); + let downtime_id = + active_by_id_in(&rows, TargetModule::Alerts, "dt-1", event.recovered_at).map(|d| d.id); + assert_eq!(downtime_id.as_deref(), Some("dt-1")); + + assert_eq!( + recovery_route(&alert, &event, downtime_id.is_some()), + RecoveryRoute { + oncall: false, + destinations: false + } + ); + let record = suppressed_recovery_record(&alert, &event, downtime_id.unwrap()); + assert_eq!(record.status, RunOutcome::Suppressed); + assert_eq!(record.downtime_id.as_deref(), Some("dt-1")); + assert_eq!(record._timestamp, event.emitted_at); + + // A recovery after the cancel is not inside the window. + assert!(active_by_id_in(&rows, TargetModule::Alerts, "dt-1", at(35)).is_none()); + } + + #[cfg(feature = "enterprise")] + fn escalation( + state: config::meta::oncall::ResponseState, + closed_at: Option, + ) -> config::meta::oncall::Response { + use config::meta::oncall::{ResponderRole, SubjectRef, SubjectType}; + config::meta::oncall::Response { + id: "resp_1".into(), + org_id: "default".into(), + subject: SubjectRef::new(SubjectType::Alert, "al_1", 1), + team_id: Some("team_1".into()), + title: None, + cause: None, + cause_note: None, + snoozed_until: None, + ladder_anchor: None, + ladder_run: None, + priority: 2, + responder_role: ResponderRole::Owner, + exhausted_at: None, + origin_response_id: None, + state, + opened_at: 0, + acked_by: None, + acked_at: None, + closed_at, + incident_id: None, + updated_at: 0, + } + } +} diff --git a/src/core/src/alerts/scheduler/handlers.rs b/src/core/src/alerts/scheduler/handlers.rs index d679ca2124..eed3d0a4da 100644 --- a/src/core/src/alerts/scheduler/handlers.rs +++ b/src/core/src/alerts/scheduler/handlers.rs @@ -70,6 +70,8 @@ pub(crate) struct AnomalyRunRecord { pub(crate) error: Option, pub(crate) success_response: Option, pub(crate) gate_passed: bool, + /// The downtime that muted the run's alert. + pub(crate) downtime_id: Option, pub(crate) start_us: i64, pub(crate) end_us: i64, } @@ -87,6 +89,12 @@ async fn persist_alert_run_state( ) -> bool { use config::meta::alerts::state::{ROLLUP_GROUP_KEY, apply_outcome}; + // Every delivery path persists here, the grouped and deduplicated ones included. + #[cfg(feature = "enterprise")] + if clears_recorded_mute(outcome) { + clear_recorded_mute(&alert.org_id, alert_id).await; + } + // ── Per-group fan-out (M-1/M-2/M-3) ───────────────────────────────────── // Only reachable for an alert that opted in (M-9); everything else falls // through to the single-row path below, unchanged. @@ -180,11 +188,19 @@ async fn persist_alert_run_state( // the same evaluation, and two `now_micros()` calls would let the coverage // record and the freshness clock disagree about when it happened. let now = now_micros(); + // A suppressed run is a firing run to the state: the condition matched and only the + // notification was withheld. Writing `Suppressed` would reset "firing since" at both ends + // of a window and, through `may_age_groups`, freeze group aging inside it. The run record + // keeps `Suppressed`; only the state axis reads it as `Firing`. + let state_outcome = match outcome { + RunOutcome::Suppressed => RunOutcome::Firing, + other => other.clone(), + }; let mut update = if alert.query_condition.promql_rule_mode { config::meta::alerts::prom_rule::apply_rule_rollup_outcome( alert_id, prev.as_ref(), - outcome.clone(), + state_outcome, level, now, rule_value, @@ -194,7 +210,7 @@ async fn persist_alert_run_state( alert_id, ROLLUP_GROUP_KEY, prev.as_ref(), - outcome.clone(), + state_outcome, level, now, ) @@ -329,9 +345,22 @@ struct GroupDispatchOutcome { /// and the caller would advance the trigger as if the alert had been /// handled — no notification, no error recorded, no retry. state_failed: bool, + /// The first downtime that kept one of the groups silent. + downtime_id: Option, } impl GroupDispatchOutcome { + /// Any muted group names its downtime; only a run with every group muted is Suppressed. + fn record_downtime(&self, row: &mut TriggerData) { + let Some(downtime_id) = self.downtime_id.clone() else { + return; + }; + if self.delivered == 0 && self.failed == 0 && self.pending == 0 { + row.status = RunOutcome::Suppressed; + } + row.downtime_id = Some(downtime_id); + } + /// The evaluation's record: any undelivered destination is a delivery failure. fn history_status(&self) -> Option { if self.errors.is_empty() { @@ -369,6 +398,7 @@ impl GroupDispatchOutcome { #[allow(clippy::too_many_arguments)] async fn dispatch_per_group( alert: &config::meta::alerts::alert::Alert, + folder_id: &str, trace_id: &str, classification: Option<&config::meta::alerts::grouping::GroupClassification>, records: &[config::utils::json::Map], @@ -425,6 +455,7 @@ async fn dispatch_per_group( delivered_groups: Default::default(), attempted: false, state_failed: true, + downtime_id: None, }); } @@ -440,6 +471,7 @@ async fn dispatch_per_group( delivered_groups: Default::default(), attempted: false, state_failed: true, + downtime_id: None, }); } }; @@ -465,6 +497,7 @@ async fn dispatch_per_group( rows_by_group_key(records, &group_by) }; let cfg = get_config(); + let muted_groups = group_downtime_map(alert, folder_id, records).await; let plan = plan_dispatch( classification, @@ -473,6 +506,7 @@ async fn dispatch_per_group( &alert.trigger_condition, now_micros(), cfg.limit.alert_max_group_notifications_per_eval, + |key| muted_groups.contains_key(key), ); // Never silent: both the volume cap and any consistency failure are @@ -499,8 +533,26 @@ async fn dispatch_per_group( let (mut delivered, mut failed) = (0usize, 0usize); let mut errors: Vec = Vec::new(); let mut delivered_groups: std::collections::HashSet = Default::default(); + let (mut downtime_suppressed, mut downtime_id) = (0usize, None); let mut attempted = false; + // One group muted, the others notify: no send and no silence window for this group. + for key in &plan.muted { + let Some(downtime) = muted_groups.get(key) else { + continue; + }; + log::info!( + "[SCHEDULER trace_id {trace_id}] alert {alert_id} group {key}: suppressed by downtime {}", + downtime.id + ); + crate::alerts::alert::count_suppressed_run(&alert.org_id, "alerts"); + downtime_suppressed += 1; + downtime_id.get_or_insert(downtime.id.clone()); + } for item in &plan.items { + // The plan already moved muted groups out; this only guards a send. + if muted_groups.contains_key(&item.group_key) { + continue; + } // The group's OWN row, level and value — never the worst group's // (MN-3). One send per group is what makes host-a and host-b page // independently. @@ -593,7 +645,7 @@ async fn dispatch_per_group( log::info!( "[SCHEDULER trace_id {trace_id}] alert {alert_id}: per-group dispatch delivered={delivered} \ - pending={} failed={failed} suppressed={} candidates={}", + pending={} failed={failed} suppressed={} downtime={downtime_suppressed} candidates={}", plan.pending, plan.suppressed, plan.items.len() @@ -606,9 +658,29 @@ async fn dispatch_per_group( delivered_groups, attempted, state_failed: false, + downtime_id, }) } +/// The downtime of each muted group of a multi-alert, from one identity call over every row. +#[cfg(feature = "enterprise")] +async fn group_downtime_map( + alert: &config::meta::alerts::alert::Alert, + folder_id: &str, + rows: &[config::utils::json::Map], +) -> HashMap { + group_downtimes(alert, folder_id, rows, now_micros()).await +} + +#[cfg(not(feature = "enterprise"))] +async fn group_downtime_map( + _alert: &config::meta::alerts::alert::Alert, + _folder_id: &str, + _rows: &[config::utils::json::Map], +) -> HashMap { + HashMap::new() +} + /// Confirm this evaluation's dedup reservations once a notification landed /// (§5.5 MN-6). /// @@ -1040,6 +1112,12 @@ async fn handle_composite_alert_trigger( let mut delivery_retry_at = None; let mut delivery_error = None; + let downtime = if evaluated.result { + composite_downtime(&definition.definition, now) + } else { + None + }; + let suppressed_by = composite_suppressed_by(&outcome, downtime.as_ref()); if evaluated.result { scheduled_data.last_satisfied_at = Some(now); // Hoisted: correlation runs in the deliverable branch, but paging needs the answer. @@ -1048,6 +1126,8 @@ async fn handle_composite_alert_trigger( let delivery = if matches!(outcome, RunOutcome::Pending) { DeliveryDecision::SuppressedByPending + } else if let Some(downtime) = suppressed_by { + DeliveryDecision::SuppressedByDowntime(downtime.id.clone()) } else { config::meta::alerts::level::delivery_decision( evaluated.level, @@ -1059,18 +1139,25 @@ async fn handle_composite_alert_trigger( Some(true), ) }; - if delivery.should_deliver() - && (!definition + let has_targets = !definition + .definition + .destinations + .as_array() + .is_none_or(Vec::is_empty) + || !definition .definition - .destinations + .workflows .as_array() - .is_none_or(Vec::is_empty) - || !definition - .definition - .workflows - .as_array() - .is_none_or(Vec::is_empty)) - { + .is_none_or(Vec::is_empty); + #[cfg(feature = "enterprise")] + let opens_incident = composite_opens_incident( + has_targets, + definition.definition.creates_incident + && o2_enterprise::enterprise::common::config::get_config() + .incidents + .enabled, + ); + if delivery.should_deliver() && has_targets { let notification_alert = composite_notification_alert(&definition.definition); let notification_row = composite_notification_row( &definition.definition.expression, @@ -1080,17 +1167,14 @@ async fn handle_composite_alert_trigger( let rows = [notification_row]; #[cfg(feature = "enterprise")] - let (incident_handled, incident_notify_error) = if notification_alert.creates_incident - && o2_enterprise::enterprise::common::config::get_config() - .incidents - .enabled - { + let (incident_handled, incident_notify_error) = if opens_incident { match crate::alerts::incidents::correlate_alert_to_incident( ¬ification_alert, &rows[0], &rows, now, Some(evaluated.level), + None, ) .await { @@ -1175,9 +1259,34 @@ async fn handle_composite_alert_trigger( } } + #[cfg(feature = "enterprise")] + if let Some(downtime) = suppressed_by { + if opens_incident { + let notification_alert = composite_notification_alert(&definition.definition); + let rows = [composite_notification_row( + &definition.definition.expression, + evaluated.result, + &evaluated.children, + )]; + correlate_muted_incident( + trace_id, + ¬ification_alert, + &rows, + now, + Some(evaluated.level), + downtime, + ) + .await; + } + crate::alerts::alert::count_suppressed_run(&trigger.org, "alerts"); + } + // Outside the deliverable branch: a silenced composite never reaches correlation. #[cfg(feature = "enterprise")] - if o2_enterprise::enterprise::oncall::is_enabled() && !composite_incident_handled { + if o2_enterprise::enterprise::oncall::is_enabled() + && !composite_incident_handled + && downtime.is_none() + { let notification_alert = composite_notification_alert(&definition.definition); let rows = [composite_notification_row( &definition.definition.expression, @@ -1199,12 +1308,16 @@ async fn handle_composite_alert_trigger( "{}/{}", definition.definition.name, definition.definition.id ), - status: if delivery_error.is_some() { + status: if suppressed_by.is_some() { + RunOutcome::Suppressed + } else if delivery_error.is_some() { RunOutcome::NotifyFailed } else { outcome.clone() }, error: delivery_error, + downtime_id: suppressed_by.map(|d| d.id.clone()), + delivery_attempted: suppressed_by.map(|_| false), actual_value: Some(i32::from(evaluated.result) as f64), level: Some(evaluated.level.to_i32()), scheduler_trace_id: Some(trace_id.to_string()), @@ -1291,6 +1404,52 @@ fn should_dispatch_after_incident( !incident_destinations_handled || has_workflows } +/// A composite has no query and no row, so its `key:value` tags are its identity. +#[cfg(feature = "enterprise")] +fn composite_downtime( + definition: &infra::table::entity::alert_composites::Model, + now: i64, +) -> Option { + if !alert_downtimes_in(&definition.org) { + return None; + } + let tags: Vec = definition + .tags + .clone() + .and_then(|value| serde_json::from_value(value).ok()) + .unwrap_or_default(); + let dims = o2_enterprise::enterprise::downtimes::scope::composite_dimensions(&tags); + crate::alerts::downtimes::active_for_alert( + &definition.org, + &definition.id, + &definition.folder_id, + &dims, + now, + ) +} + +#[cfg(not(feature = "enterprise"))] +fn composite_downtime( + _definition: &infra::table::entity::alert_composites::Model, + _now: i64, +) -> Option { + None +} + +/// A Pending composite is held by its pending period, so no downtime suppresses or correlates it. +fn composite_suppressed_by<'a>( + outcome: &RunOutcome, + downtime: Option<&'a config::meta::downtimes::ActiveDowntime>, +) -> Option<&'a config::meta::downtimes::ActiveDowntime> { + downtime.filter(|_| !matches!(outcome, RunOutcome::Pending)) +} + +/// A composite with no destinations and no workflows delivers nothing, so it opens no incident. +#[cfg(feature = "enterprise")] +fn composite_opens_incident(has_targets: bool, creates_incident: bool) -> bool { + has_targets && creates_incident +} + /// Why a composite's send, or its incident's notification, left a destination unnotified. fn composite_delivery_error( delivery: &Result, @@ -1462,6 +1621,8 @@ async fn handle_anomaly_detection_triggers( // Run detection via enterprise and track outcome for the triggers stream. let run_start_us = now_micros(); + #[cfg_attr(not(feature = "enterprise"), allow(unused_mut))] + let mut downtime_id: Option = None; let (trigger_status, trigger_error, trigger_success_response, gate_passed) = { #[cfg(feature = "enterprise")] { @@ -1470,12 +1631,23 @@ async fn handle_anomaly_detection_triggers( ) .await { - Ok(run) => ( - anomaly_run_status(&run), - run.notify_error.clone(), - Some(serde_json::json!({ "anomalies_found": run.anomaly_count }).to_string()), - run.gate_passed, - ), + Ok(run) => { + if run.suppressed_by.is_some() { + crate::alerts::alert::count_suppressed_run( + &trigger.org, + "anomaly_detections", + ); + } + downtime_id = run.suppressed_by.clone(); + ( + anomaly_run_status(&run), + run.notify_error.clone(), + Some( + serde_json::json!({ "anomalies_found": run.anomaly_count }).to_string(), + ), + run.gate_passed, + ) + } Err(e) => { log::error!( "[anomaly_detection] detection failed for {}/{anomaly_id}: {e}", @@ -1504,6 +1676,7 @@ async fn handle_anomaly_detection_triggers( error: trigger_error, success_response: trigger_success_response, gate_passed, + downtime_id, start_us: run_start_us, end_us: run_end_us, }; @@ -1514,11 +1687,7 @@ async fn handle_anomaly_detection_triggers( // processed by the training scheduler yet, or processed but status not yet // flipped), move it to Active so the UI reflects the real state. #[cfg(feature = "enterprise")] - if matches!( - trigger_status, - RunOutcome::Firing | RunOutcome::NotifyFailed | RunOutcome::Normal - ) && config.is_trained - { + if detection_ran(&trigger_status) && config.is_trained { use o2_enterprise::enterprise::anomaly_detection::types::Status as AnomalyStatus; if config.status != AnomalyStatus::Active.to_i32() { use infra::table::entity::anomaly_detection_config as anomaly_entity; @@ -1557,6 +1726,15 @@ async fn handle_anomaly_detection_triggers( Ok(()) } +/// The model executed: anomalies found (delivered or not), none found, or muted by a downtime. +#[cfg(feature = "enterprise")] +fn detection_ran(outcome: &RunOutcome) -> bool { + matches!( + outcome, + RunOutcome::Firing | RunOutcome::NotifyFailed | RunOutcome::Normal | RunOutcome::Suppressed + ) +} + /// Stamp the run outcome onto the trigger, the only per-row record of it — /// anomaly detection writes no `alert_states` rollup the list could read. /// As scheduled alerts: the condition met is Firing even when cooldown silenced it. @@ -1566,6 +1744,8 @@ pub(crate) fn anomaly_run_status( ) -> RunOutcome { if run.claim_lost || run.ineligible { RunOutcome::Skipped + } else if run.suppressed_by.is_some() { + RunOutcome::Suppressed } else if run.notify_failed { RunOutcome::NotifyFailed } else if run.gate_passed { @@ -1597,6 +1777,7 @@ pub(crate) fn publish_anomaly_run( error: record.error.clone(), success_response: record.success_response.clone(), evaluation_took_in_secs: Some((record.end_us - record.start_us) as f64 / 1_000_000.0), + downtime_id: record.downtime_id.clone(), ..Default::default() }); } @@ -1828,30 +2009,19 @@ pub(crate) async fn page_blast_radius( } } -/// Shared by the scheduled-alert, composite and manual-trigger producers so `creates_incident` -/// cannot drift. +/// Stage A of on-call routing, one map per group; paging and the downtime check both read it. #[cfg(feature = "enterprise")] -pub(crate) async fn page_for_alert_firing( - trace_id: &str, +pub(crate) async fn alert_identity( alert: &config::meta::alerts::alert::Alert, rows: &[config::utils::json::Map], -) { - let (Some(first_row), Some(alert_id)) = (rows.first(), alert.id.as_ref()) else { - // Nothing fired, or the signal has no stable id to key a record on. - return; - }; - // Decided in the engine, on the key the record is stored under; the bare alert id is not it. +) -> Vec> { let semantic_groups = crate::db::system_settings::get_semantic_field_groups(&alert.org_id).await; + let empty_row = config::utils::json::Map::new(); + let first_row = rows.first().unwrap_or(&empty_row); // Row first, then the alert's conditions: an aggregating alert has no identity columns. - let dimensions = o2_enterprise::enterprise::oncall::routing::dimensions_for_alert( - &semantic_groups, - &alert.query_condition, - first_row, - ); - // One row per group key, as `dispatch_per_group` reduces: `rows.first()` woke one group. - let mut group_dimensions: Vec> = - if alert.query_condition.multi_alert_enabled() { + let mut identity: Vec> = + if alert.query_condition.multi_alert_enabled() && !rows.is_empty() { let group_by = alert .query_condition .aggregation @@ -1875,26 +2045,74 @@ pub(crate) async fn page_for_alert_firing( }) .collect() } else { - vec![dimensions.clone()] + vec![ + o2_enterprise::enterprise::oncall::routing::dimensions_for_alert( + &semantic_groups, + &alert.query_condition, + first_row, + ), + ] }; // Last resort, matching the incident path so a checkbox about incidents cannot reroute. - if group_dimensions.iter().all(|d| d.is_empty()) + if !rows.is_empty() + && identity.iter().all(|d| d.is_empty()) && let Some(service) = crate::alerts::incidents::correlated_service_for_routing(&alert.org_id, first_row).await { - for dims in &mut group_dimensions { + for dims in &mut identity { dims.insert( config::meta::oncall::SERVICE_DIMENSION.to_string(), service.clone(), ); } log::debug!( - "[SCHEDULER trace_id {trace_id}] {}/{}: no identity fields in the result row; routing \ - on the correlated service `{service}`", + "{}/{}: no identity fields in the result row; using the correlated service \ + `{service}`", alert.org_id, alert.name, ); } + identity +} + +/// One map per SQL group, PromQL series or row, so one muted row never decides for the others. +#[cfg(feature = "enterprise")] +pub(crate) async fn downtime_identities( + alert: &config::meta::alerts::alert::Alert, + rows: &[config::utils::json::Map], +) -> Vec> { + downtime_identities_by_key(alert, rows) + .await + .into_iter() + .map(|(_, dims)| dims) + .collect() +} + +/// Shared by the composite and manual-trigger producers so `creates_incident` cannot drift. +#[cfg(feature = "enterprise")] +pub(crate) async fn page_for_alert_firing( + trace_id: &str, + alert: &config::meta::alerts::alert::Alert, + rows: &[config::utils::json::Map], +) { + if rows.is_empty() || alert.id.is_none() { + // Nothing fired, or the signal has no stable id to key a record on. + return; + } + let group_dimensions = alert_identity(alert, rows).await; + page_for_alert_identity(trace_id, alert, &group_dimensions).await; +} + +/// Pages on maps `alert_identity` built, so paging and the downtime check read the same identity. +#[cfg(feature = "enterprise")] +pub(crate) async fn page_for_alert_identity( + trace_id: &str, + alert: &config::meta::alerts::alert::Alert, + group_dimensions: &[HashMap], +) { + let Some(alert_id) = alert.id.as_ref() else { + return; + }; // Single-sourced with the incident path: `creates_incident` must not change the severity. let priority = alert .priority @@ -1905,7 +2123,7 @@ pub(crate) async fn page_for_alert_firing( &alert.name, priority, alert.oncall_team.as_deref(), - &group_dimensions, + group_dimensions, ) .await { @@ -2101,6 +2319,528 @@ async fn release_alert_firing_hold( } } +#[cfg(feature = "enterprise")] +fn alert_downtimes_in(org: &str) -> bool { + crate::alerts::downtimes::any_for(org, config::meta::downtimes::TargetModule::Alerts) +} + +/// The downtime silencing a matched run (D2); a multi-alert only when every group is muted. +#[cfg(feature = "enterprise")] +pub(crate) async fn downtime_decision( + alert: &config::meta::alerts::alert::Alert, + folder_id: &str, + rows: Option<&[config::utils::json::Map]>, + now: i64, +) -> Option { + let rows = rows?; + if !crate::alerts::downtimes::enabled() { + return None; + } + let cached = db::downtimes::list_cached(&alert.org_id); + let wall = config::utils::time::now_micros(); + downtime_decision_in(&cached, alert, folder_id, rows, now, wall).await +} + +/// [downtime_decision] over `cached`; `now` is past when a delayed recovery asks at `recovered_at`. +#[cfg(feature = "enterprise")] +async fn downtime_decision_in( + cached: &[config::meta::downtimes::Downtime], + alert: &config::meta::alerts::alert::Alert, + folder_id: &str, + rows: &[config::utils::json::Map], + now: i64, + wall: i64, +) -> Option { + use config::meta::downtimes::TargetModule; + + use crate::alerts::downtimes::enterprise::{active_in, any_target_since, item}; + + if !any_target_since(cached, TargetModule::Alerts, now, wall) { + return None; + } + let alert_id = alert.id.as_ref()?.to_string(); + let identity = downtime_identity(alert, rows).await; + muted_in_every_group(&identity, |dims| { + active_in( + cached, + TargetModule::Alerts, + &item(&alert_id, folder_id, dims, &[]), + now, + ) + }) +} + +/// [downtime_decision] on the definition, ignoring row `except` the cache may still show live. +#[cfg(feature = "enterprise")] +pub(crate) async fn downtime_decision_except( + alert: &config::meta::alerts::alert::Alert, + folder_id: &str, + except: &str, + now: i64, +) -> Option { + if !alert_downtimes_in(&alert.org_id) { + return None; + } + let alert_id = alert.id.as_ref()?.to_string(); + let identity = downtime_identity(alert, &[]).await; + muted_in_every_group(&identity, |dims| { + crate::alerts::downtimes::active_for_alert_except( + &alert.org_id, + &alert_id, + folder_id, + dims, + now, + except, + ) + }) +} + +/// The identity a downtime is matched on, shared by evaluation and the grouped flush. +#[cfg(feature = "enterprise")] +pub(crate) async fn downtime_identity( + alert: &config::meta::alerts::alert::Alert, + rows: &[config::utils::json::Map], +) -> Vec> { + match identity_slo(alert) { + Some(slo_id) => vec![slo_dimensions(&alert.org_id, slo_id).await], + None => downtime_identities(alert, rows).await, + } +} + +/// Every group's downtimes merged when each group has one; else `dispatch_per_group` decides. +#[cfg(feature = "enterprise")] +pub(crate) fn muted_in_every_group( + identity: &[HashMap], + mut active: impl FnMut(&HashMap) -> Option, +) -> Option { + let mut decision: Option = None; + for dims in identity { + let downtime = active(dims)?; + decision = Some(match decision { + Some(merged) => crate::alerts::downtimes::enterprise::merged(merged, downtime), + None => downtime, + }); + } + decision +} + +/// An SLO alert runs no query, so its identity is its SLO's (D9). +#[cfg(feature = "enterprise")] +fn identity_slo(alert: &config::meta::alerts::alert::Alert) -> Option<&str> { + alert + .query_condition + .slo_condition + .as_ref() + .map(|slo_condition| slo_condition.slo_id.as_str()) +} + +/// [downtime_identities] keyed as `dispatch_per_group` keys its groups; a plain alert by row index. +#[cfg(feature = "enterprise")] +async fn downtime_identities_by_key( + alert: &config::meta::alerts::alert::Alert, + rows: &[config::utils::json::Map], +) -> Vec<(String, HashMap)> { + let semantic_groups = + crate::db::system_settings::get_semantic_field_groups(&alert.org_id).await; + let mut identities = identities_by_key(alert, rows, &semantic_groups); + // Once per run and only when no row has an identity, as in `alert_identity`. + if let Some(first_row) = rows.first() + && identities.iter().all(|(_, dims)| dims.is_empty()) + && let Some(service) = + crate::alerts::incidents::correlated_service_for_routing(&alert.org_id, first_row).await + { + for (_, dims) in &mut identities { + dims.insert( + config::meta::oncall::SERVICE_DIMENSION.to_string(), + service.clone(), + ); + } + } + identities +} + +/// The pure part of [downtime_identities_by_key], before the correlated-service fallback. +#[cfg(feature = "enterprise")] +pub(crate) fn identities_by_key( + alert: &config::meta::alerts::alert::Alert, + rows: &[config::utils::json::Map], + semantic_groups: &[config::meta::correlation::FieldAlias], +) -> Vec<(String, HashMap)> { + let dims = |row: &config::utils::json::Map| { + o2_enterprise::enterprise::oncall::routing::dimensions_for_alert( + semantic_groups, + &alert.query_condition, + row, + ) + }; + if rows.is_empty() { + // No row: the alert's own conditions decide, as in `alert_identity`. + return vec![(String::new(), dims(&config::utils::json::Map::new()))]; + } + if !alert.query_condition.multi_alert_enabled() { + return rows + .iter() + .enumerate() + .map(|(i, row)| (i.to_string(), dims(row))) + .collect(); + } + let mut by_key: Vec<_> = rows_by_downtime_key(alert, rows).into_iter().collect(); + by_key.sort_by(|a, b| a.0.cmp(&b.0)); + by_key + .into_iter() + .map(|(key, row)| { + let dims = dims(&row); + (key, dims) + }) + .collect() +} + +/// The rows of a multi-alert under the group keys `dispatch_per_group` uses. +#[cfg(feature = "enterprise")] +fn rows_by_downtime_key( + alert: &config::meta::alerts::alert::Alert, + rows: &[config::utils::json::Map], +) -> HashMap> { + if alert.query_condition.query_type == config::meta::alerts::QueryType::PromQL { + config::meta::alerts::dispatch::rows_by_series_key(rows) + } else { + config::meta::alerts::dispatch::rows_by_group_key(rows, &alert_group_by(alert)) + } +} + +/// The group key of one multi-alert row, the same key [rows_by_downtime_key] files it under. +#[cfg(feature = "enterprise")] +fn downtime_row_key( + alert: &config::meta::alerts::alert::Alert, + group_by: &[String], + row: &config::utils::json::Map, +) -> String { + use config::meta::alerts::{dispatch, grouping::group_key}; + if alert.query_condition.query_type == config::meta::alerts::QueryType::PromQL { + group_key(&dispatch::promql_series_labels(row)) + } else { + group_key(&dispatch::row_group_labels(row, group_by)) + } +} + +#[cfg(feature = "enterprise")] +fn alert_group_by(alert: &config::meta::alerts::alert::Alert) -> Vec { + alert + .query_condition + .aggregation + .as_ref() + .and_then(|a| a.group_by.clone()) + .unwrap_or_default() +} + +/// The downtime of each muted group of a multi-alert, by group key. +#[cfg(feature = "enterprise")] +async fn group_downtimes( + alert: &config::meta::alerts::alert::Alert, + folder_id: &str, + rows: &[config::utils::json::Map], + now: i64, +) -> HashMap { + let Some(alert_id) = alert.id.as_ref().map(|id| id.to_string()) else { + return HashMap::new(); + }; + if !alert_downtimes_in(&alert.org_id) { + return HashMap::new(); + } + downtime_identities_by_key(alert, rows) + .await + .into_iter() + .filter_map(|(key, dims)| { + crate::alerts::downtimes::active_for_alert( + &alert.org_id, + &alert_id, + folder_id, + &dims, + now, + ) + .map(|downtime| (key, downtime)) + }) + .collect() +} + +/// The Stage A maps of the unmuted groups, so a muted group of a multi-alert is not paged. +#[cfg(feature = "enterprise")] +async fn unmuted_identity( + alert: &config::meta::alerts::alert::Alert, + folder_id: &str, + rows: &[config::utils::json::Map], + now: i64, +) -> Vec> { + let on_call = alert_identity(alert, rows).await; + let Some(alert_id) = alert.id.as_ref().map(|id| id.to_string()) else { + return on_call; + }; + if !alert_downtimes_in(&alert.org_id) { + return on_call; + } + let identities = downtime_identities(alert, rows).await; + unmuted_maps(on_call, identities, |dims| { + crate::alerts::downtimes::active_for_alert(&alert.org_id, &alert_id, folder_id, dims, now) + .is_some() + }) +} + +/// The on-call maps while no group is muted, else the unmuted groups' own maps. +#[cfg(feature = "enterprise")] +fn unmuted_maps( + on_call: Vec>, + identities: Vec>, + mut muted: impl FnMut(&HashMap) -> bool, +) -> Vec> { + let total = identities.len(); + let unmuted: Vec<_> = identities.into_iter().filter(|dims| !muted(dims)).collect(); + if unmuted.len() == total { + on_call + } else { + unmuted + } +} + +/// The rows of a multi-alert's unmuted groups for incident correlation; `None` keeps every row. +#[cfg(feature = "enterprise")] +async fn unmuted_group_rows( + alert: &config::meta::alerts::alert::Alert, + folder_id: &str, + rows: &[config::utils::json::Map], + now: i64, +) -> Option>> { + alert.id.as_ref()?; + if !alert.creates_incident + || !alert.query_condition.multi_alert_enabled() + || !alert_downtimes_in(&alert.org_id) + { + return None; + } + let muted = group_downtimes(alert, folder_id, rows, now).await; + let group_by = alert_group_by(alert); + Some( + rows.iter() + .filter(|row| !muted.contains_key(&downtime_row_key(alert, &group_by, row))) + .cloned() + .collect(), + ) +} + +#[cfg(not(feature = "enterprise"))] +pub(crate) async fn downtime_decision( + _alert: &config::meta::alerts::alert::Alert, + _folder_id: &str, + _rows: Option<&[config::utils::json::Map]>, + _now: i64, +) -> Option { + None +} + +#[cfg(feature = "enterprise")] +async fn slo_dimensions(org: &str, slo_id: &str) -> HashMap { + match infra::table::slos::get(get_orm_client_ro().await, org, slo_id).await { + Ok(Some(slo)) => crate::alerts::downtimes::dimensions_for_slo(&slo).await, + Ok(None) => HashMap::new(), + Err(e) => { + log::warn!("[SCHEDULER] SLO {org}/{slo_id} unreadable for the downtime check: {e}"); + HashMap::new() + } + } +} + +/// A suppressed firing still correlates, into an incident that opens muted (D3). +#[cfg(feature = "enterprise")] +pub(crate) async fn correlate_muted_incident( + trace_id: &str, + alert: &config::meta::alerts::alert::Alert, + rows: &[config::utils::json::Map], + triggered_at: i64, + eval_level: Option, + downtime: &config::meta::downtimes::ActiveDowntime, +) { + let Some(first_row) = rows.first() else { + return; + }; + let incidents_enabled = o2_enterprise::enterprise::common::config::get_config() + .incidents + .enabled; + if !correlates_muted_firing(alert.creates_incident, incidents_enabled, downtime) { + return; + } + if let Err(e) = crate::alerts::incidents::correlate_alert_to_incident( + alert, + first_row, + rows, + triggered_at, + eval_level, + Some(downtime), + ) + .await + { + log::error!( + "[SCHEDULER trace_id {trace_id}] muted incident correlation failed for {}/{}: {e}", + alert.org_id, + alert.name + ); + } +} + +/// Whether a matched single-series run is still inside its pending period. +fn holds_pending( + alert: &config::meta::alerts::alert::Alert, + last: Option<&config::meta::alerts::state::AlertState>, + rows: Option<&[config::utils::json::Map]>, + now: i64, +) -> bool { + if alert.query_condition.multi_alert_enabled() + || alert.pending_period_sec <= 0 + || rows.is_none_or(|rows| rows.is_empty()) + { + return false; + } + match last.map(|state| (state.last_outcome.as_ref(), state.since)) { + None | Some((None | Some(RunOutcome::Normal), _)) => true, + Some((Some(RunOutcome::Pending), Some(since))) => { + now - since < alert.pending_period_sec.saturating_mul(1_000_000) + } + _ => false, + } +} + +/// A muted multi-alert correlates only the groups whose own downtime keeps incidents. +#[cfg(feature = "enterprise")] +async fn muted_incident_rows( + alert: &config::meta::alerts::alert::Alert, + folder_id: &str, + rows: &[config::utils::json::Map], + now: i64, + decision: &config::meta::downtimes::ActiveDowntime, +) -> Option<( + Vec>, + config::meta::downtimes::ActiveDowntime, +)> { + // A plain alert's decision is its first row's, and an SLO alert has one identity. + if !alert.query_condition.multi_alert_enabled() || alert.query_condition.slo_condition.is_some() + { + return keep_muted_incident_rows(rows, |_| Some(decision.clone())); + } + let by_group = group_downtimes(alert, folder_id, rows, now).await; + let group_by = alert_group_by(alert); + keep_muted_incident_rows(rows, |row| { + by_group + .get(&downtime_row_key(alert, &group_by, row)) + .cloned() + }) +} + +/// The rows whose downtime is `muted`, with the first such row's downtime for the incident. +#[cfg(feature = "enterprise")] +fn keep_muted_incident_rows( + rows: &[config::utils::json::Map], + row_downtime: impl Fn( + &config::utils::json::Map, + ) -> Option, +) -> Option<( + Vec>, + config::meta::downtimes::ActiveDowntime, +)> { + let mut downtime = None; + let mut kept = Vec::new(); + for row in rows { + let Some(row_downtime) = row_downtime(row).filter(|d| d.incident_mode.is_muted()) else { + continue; + }; + downtime.get_or_insert(row_downtime); + kept.push(row.clone()); + } + downtime.map(|downtime| (kept, downtime)) +} + +/// Incident mode `none` leaves the window's firings Suppressed with no incident to join later. +#[cfg(feature = "enterprise")] +fn correlates_muted_firing( + creates_incident: bool, + incidents_enabled: bool, + downtime: &config::meta::downtimes::ActiveDowntime, +) -> bool { + creates_incident && incidents_enabled && downtime.incident_mode.is_muted() +} + +/// D6: only a delivered firing opens the silence, so the run after a downtime delivers. +fn starts_silence_window( + condition_matched: bool, + suppressed_by_downtime: bool, + silence_minutes: i64, + evaluates_through_silence: bool, +) -> bool { + condition_matched + && !suppressed_by_downtime + && silence_minutes > 0 + && !evaluates_through_silence +} + +/// A pending run of an alert with silence stores no end time, as when silence was decided first. +fn pending_stores_end_time( + should_store_last_end_time: bool, + silence_minutes: i64, + evaluates_through_silence: bool, +) -> bool { + should_store_last_end_time + && !starts_silence_window(true, false, silence_minutes, evaluates_through_silence) +} + +/// Whether this run moves `last_notified_level` and the delivery silence window. +fn records_delivery_state(alert_level_delivery: bool, delivery: &DeliveryDecision) -> bool { + alert_level_delivery && delivery.resets_silence() +} + +/// A delivered firing ends the Muted chip; a pending run persists before the downtime decision. +#[cfg(feature = "enterprise")] +fn clears_recorded_mute(outcome: &RunOutcome) -> bool { + matches!(outcome, RunOutcome::Firing | RunOutcome::NotifyFailed) +} + +/// The Muted chip of a condition-scoped downtime, recorded or cleared at run time. +#[cfg(feature = "enterprise")] +pub(crate) async fn record_last_downtime(org: &str, alert_id: &str, downtime_id: Option<&str>) { + if !alert_downtimes_in(org) { + return; + } + if let Err(e) = infra::table::alert_states::set_last_downtime_id(alert_id, downtime_id).await { + log::warn!("[SCHEDULER] could not record the downtime of {alert_id}: {e}"); + } +} + +/// An ended one-time row is off the live path, but the mute it recorded can still hold a page open. +#[cfg(feature = "enterprise")] +fn mute_may_linger(rows: &[config::meta::downtimes::Downtime]) -> bool { + crate::alerts::downtimes::enterprise::any_row_in( + rows, + config::meta::downtimes::TargetModule::Alerts, + ) +} + +/// Drops the Muted chip; a record that existed means a muted resolve may have left a page open. +#[cfg(feature = "enterprise")] +async fn clear_recorded_mute(org: &str, alert_id: &str) { + if !crate::alerts::downtimes::enabled() || !mute_may_linger(&db::downtimes::list_cached(org)) { + return; + } + match infra::table::alert_states::set_last_downtime_id(alert_id, None).await { + Ok(0) => {} + Ok(_) => { + if let Err(e) = crate::alerts::incidents::recover_muted_resolves(org, alert_id).await { + log::error!( + "[SCHEDULER] on-call recovery after the mute failed for {alert_id}: {e}" + ); + } + } + Err(e) => { + log::warn!("[SCHEDULER] could not clear the recorded downtime of {alert_id}: {e}") + } + } +} + /// Records a failed incident page; the returned pre-failure outcome is what the alert state keeps. #[cfg(feature = "enterprise")] fn record_incident_notify_failure(row: &mut TriggerData, error: String) -> RunOutcome { @@ -2137,10 +2877,11 @@ async fn handle_alert_triggers( }; // here it can be alert id or alert name - let alert = if let Ok(alert_id) = svix_ksuid::Ksuid::from_str(&trigger.module_key) { + let (folder_id, alert) = if let Ok(alert_id) = svix_ksuid::Ksuid::from_str(&trigger.module_key) + { let client = get_orm_client_rw().await; match db::alerts::alert::get_by_id(client, &trigger.org, alert_id).await { - Ok(Some((_, alert))) => alert, + Ok(Some((folder, alert))) => (folder.folder_id, alert), Ok(None) => { log::error!( "[SCHEDULER trace_id {scheduler_trace_id}] Alert not found for module_key: {}, org_id: {}, deleting this trigger job", @@ -2617,6 +3358,7 @@ async fn handle_alert_triggers( if matched_level.is_none() && o2_enterprise::enterprise::oncall::is_enabled() && let Some(alert_id) = alert.id.as_ref() + && !crate::alerts::recovery::oncall_recovery_withheld(&alert, &folder_id, now).await && let Err(e) = o2_enterprise::enterprise::oncall::escalation::recover_for_alert( &alert.org_id, &alert_id.to_string(), @@ -2705,10 +3447,84 @@ async fn handle_alert_triggers( alert.query_condition.multi_alert_enabled(), multi_level, ); - if trigger_results.data.is_some() - && alert.trigger_condition.silence > 0 - && !evaluates_through_silence + // ── §7.1 delivery decision ────────────────────────────────────────────── + // Computed for multi-level alerts only; single-level alerts short-circuit + // to `Deliver` so their behaviour is bit-for-bit unchanged (G5). + // MN-1/MN-2: a multi-alert's suppression is per group, decided inside + // `dispatch_per_group` from each row's own `silenced_until`. The + // alert-level decision must not run in front of it — one window for the + // whole alert would mute every group at once, and a group that starts + // firing mid-window would never page at all. + let alert_level_delivery = config::meta::alerts::dispatch::alert_level_delivery_applies( + alert.query_condition.multi_alert_enabled(), + multi_level, + ); + let level_delivery = if alert_level_delivery { + config::meta::alerts::level::delivery_decision( + recorded_level, + trigger_data + .last_notified_level + .and_then(config::meta::alerts::level::AlertLevel::from_i32), + trigger_data.delivery_silenced_until, + triggered_at, + notify_on_warning, + ) + } else { + config::meta::alerts::level::DeliveryDecision::Deliver + }; + if trigger_results.data.is_some() { + trigger_data.last_satisfied_at = Some(triggered_at); + } + // Asked before the downtime, so a muted run still counts toward the pending period. + if level_delivery.should_deliver() + && holds_pending( + &alert, + last_states.get(config::meta::alerts::state::ROLLUP_GROUP_KEY), + trigger_results.data.as_deref(), + now, + ) { + trigger_data_stream.status = RunOutcome::Pending; + trigger_data.period_end_time = pending_stores_end_time( + should_store_last_end_time, + alert.trigger_condition.silence, + evaluates_through_silence, + ) + .then_some(trigger_results.end_time); + // A pending run was never delivered, so the next run keeps the cadence without silence. + new_trigger.next_run_at = + alert + .trigger_condition + .get_next_trigger_time(true, alert.tz_offset, false, None)?; + new_trigger.is_silenced = false; + trigger_data_stream.next_run_at = new_trigger.next_run_at; + new_trigger.data = json::to_string(&trigger_data).unwrap(); + db::scheduler::update_trigger(new_trigger, true, &query_trace_id).await?; + if let Some(alert_id) = alert.id.as_ref() { + let _ = persist_alert_run_state( + &alert, + &alert_id.to_string(), + &trigger_data_stream.status, + eval_level, + trigger_results.group_classification.as_ref(), + trigger_results.rule_value.as_deref(), + &config::meta::alerts::recovery::EpisodeInput::undelivered(alert.keep_firing_for), + ) + .await; + } + publish_triggers_usage(trigger_data_stream); + return Ok(()); + } + // Decided after the query, from the fired rows, and never from the definition alone. + let downtime = + downtime_decision(&alert, &folder_id, trigger_results.data.as_deref(), now).await; + // D6: a suppressed run starts no silence window, so the first run after the window delivers. + if starts_silence_window( + trigger_results.data.is_some(), + downtime.is_some(), + alert.trigger_condition.silence, + evaluates_through_silence, + ) { new_trigger.next_run_at = alert .trigger_condition @@ -2724,37 +3540,12 @@ async fn handle_alert_triggers( } trigger_data_stream.next_run_at = new_trigger.next_run_at; - if trigger_results.data.is_some() { - trigger_data.last_satisfied_at = Some(triggered_at); - } - // send notification - // ── §7.1 delivery decision ────────────────────────────────────────────── - // Computed for multi-level alerts only; single-level alerts short-circuit - // to `Deliver` so their behaviour is bit-for-bit unchanged (G5). - // MN-1/MN-2: a multi-alert's suppression is per group, decided inside - // `dispatch_per_group` from each row's own `silenced_until`. The - // alert-level decision must not run in front of it — one window for the - // whole alert would mute every group at once, and a group that starts - // firing mid-window would never page at all. - let alert_level_delivery = config::meta::alerts::dispatch::alert_level_delivery_applies( - alert.query_condition.multi_alert_enabled(), - multi_level, - ); - let delivery = if alert_level_delivery { - config::meta::alerts::level::delivery_decision( - recorded_level, - trigger_data - .last_notified_level - .and_then(config::meta::alerts::level::AlertLevel::from_i32), - trigger_data.delivery_silenced_until, - triggered_at, - notify_on_warning, - ) - } else { - config::meta::alerts::level::DeliveryDecision::Deliver + let delivery = match downtime.as_ref() { + Some(downtime) => DeliveryDecision::SuppressedByDowntime(downtime.id.clone()), + None => level_delivery, }; - if !delivery.should_deliver() && alert_level_delivery { + if !delivery.should_deliver() && (alert_level_delivery || downtime.is_some()) { log::info!( "[SCHEDULER trace_id {scheduler_trace_id}] delivery suppressed ({delivery:?}) for {}/{}", new_trigger.org, @@ -2776,7 +3567,7 @@ async fn handle_alert_triggers( // harmless now that nothing reads them for this path, but writing them // would resurrect the alert-level window the moment anything did. let record_delivery = |trigger_data: &mut ScheduledTriggerData| { - if alert_level_delivery && delivery.resets_silence() { + if records_delivery_state(alert_level_delivery, &delivery) { trigger_data.last_notified_level = eval_level.map(|l| l.to_i32()); trigger_data.delivery_silenced_until = if alert.trigger_condition.silence > 0 { Some( @@ -2800,6 +3591,23 @@ async fn handle_alert_triggers( let condition_matched = trigger_results.data.is_some(); let payload_empty = trigger_results.data.as_ref().is_none_or(|d| d.is_empty()); + #[cfg(feature = "enterprise")] + if let Some(decision) = downtime.as_ref() + && let Some(rows) = trigger_results.data.as_deref() + && let Some((rows, downtime)) = + muted_incident_rows(&alert, &folder_id, rows, now, decision).await + { + correlate_muted_incident( + &scheduler_trace_id, + &alert, + &rows, + triggered_at, + eval_level, + &downtime, + ) + .await; + } + // What this evaluation DELIVERED, not what it decided to: the episode opens on a landed send. let mut episode_delivered = false; // Only the enterprise correlation block assigns this. @@ -2857,137 +3665,6 @@ async fn handle_alert_triggers( false }; - // for non multi alert, we need to check if it should move to pending state or firing state - // this only applies if the pending period > 0, for 0 pending period, always immediately - // transition to firing etc. - if !is_multi_alert && alert.pending_period_sec > 0 { - if let Some(last_state) = last_states.get("") { - match (last_state.last_outcome.as_ref(), last_state.since) { - (None, _) | (Some(RunOutcome::Normal), _) => { - // last state not recorded, so maybe first firing, or normal - // so set to pending - trigger_data_stream.status = RunOutcome::Pending; - trigger_data.period_end_time = if should_store_last_end_time { - Some(trigger_results.end_time) - } else { - None - }; - // reset the next run time without silence, because this was never - // delivered, simply pending - new_trigger.next_run_at = alert.trigger_condition.get_next_trigger_time( - true, - alert.tz_offset, - false, - None, - )?; - new_trigger.is_silenced = false; - trigger_data_stream.next_run_at = new_trigger.next_run_at; - new_trigger.data = json::to_string(&trigger_data).unwrap(); - db::scheduler::update_trigger(new_trigger, true, &query_trace_id).await?; - // Condition matched; only the notification was - // deduplicated away. State must reflect the firing. - if let Some(alert_id) = alert.id.as_ref() { - let _ = persist_alert_run_state( - &alert, - &alert_id.to_string(), - &trigger_data_stream.status, - eval_level, - trigger_results.group_classification.as_ref(), - trigger_results.rule_value.as_deref(), - &config::meta::alerts::recovery::EpisodeInput::undelivered( - alert.keep_firing_for, - ), - ) - .await; - } - publish_triggers_usage(trigger_data_stream); - return Ok(()); - } - #[allow(clippy::collapsible_match)] - (Some(RunOutcome::Pending), Some(last)) => { - // last state was pending, so check if the the pending state exists for more - // than pending seconds or not. - if now - last < alert.pending_period_sec.saturating_mul(1_000_000) { - trigger_data_stream.status = RunOutcome::Pending; - trigger_data.period_end_time = if should_store_last_end_time { - Some(trigger_results.end_time) - } else { - None - }; - // reset the next run time without silence, because this was never - // delivered, simply pending - new_trigger.next_run_at = alert - .trigger_condition - .get_next_trigger_time(true, alert.tz_offset, false, None)?; - new_trigger.is_silenced = false; - trigger_data_stream.next_run_at = new_trigger.next_run_at; - new_trigger.data = json::to_string(&trigger_data).unwrap(); - db::scheduler::update_trigger(new_trigger, true, &query_trace_id) - .await?; - // Condition matched; only the notification was - // deduplicated away. State must reflect the firing. - if let Some(alert_id) = alert.id.as_ref() { - let _ = persist_alert_run_state( - &alert, - &alert_id.to_string(), - &trigger_data_stream.status, - eval_level, - trigger_results.group_classification.as_ref(), - trigger_results.rule_value.as_deref(), - &config::meta::alerts::recovery::EpisodeInput::undelivered( - alert.keep_firing_for, - ), - ) - .await; - } - publish_triggers_usage(trigger_data_stream); - return Ok(()); - } - } - // for all other states, continue processing - _ => {} - } - } else { - // last state not recorded, so maybe first firing, set it to pending - trigger_data_stream.status = RunOutcome::Pending; - trigger_data.period_end_time = if should_store_last_end_time { - Some(trigger_results.end_time) - } else { - None - }; - // reset the next run time without silence, because this was never delivered, - // simply pending - new_trigger.next_run_at = alert.trigger_condition.get_next_trigger_time( - true, - alert.tz_offset, - false, - None, - )?; - new_trigger.is_silenced = false; - trigger_data_stream.next_run_at = new_trigger.next_run_at; - new_trigger.data = json::to_string(&trigger_data).unwrap(); - db::scheduler::update_trigger(new_trigger, true, &query_trace_id).await?; - // Condition matched; only the notification was - // deduplicated away. State must reflect the firing. - if let Some(alert_id) = alert.id.as_ref() { - let _ = persist_alert_run_state( - &alert, - &alert_id.to_string(), - &trigger_data_stream.status, - eval_level, - trigger_results.group_classification.as_ref(), - trigger_results.rule_value.as_deref(), - &config::meta::alerts::recovery::EpisodeInput::undelivered( - alert.keep_firing_for, - ), - ) - .await; - } - publish_triggers_usage(trigger_data_stream); - return Ok(()); - } - } - if grouping_enabled { #[cfg(feature = "enterprise")] { @@ -3200,19 +3877,24 @@ async fn handle_alert_triggers( } }; + #[cfg(feature = "enterprise")] + let unmuted_rows = unmuted_group_rows(&alert, &folder_id, &data, now).await; + #[cfg(feature = "enterprise")] + let correlation_rows = unmuted_rows.as_deref().unwrap_or(data.as_slice()); #[cfg(feature = "enterprise")] let (incident_handled_notification, incident_notify_error) = if alert.creates_incident && o2_enterprise::enterprise::common::config::get_config() .incidents .enabled - && let Some(first_row) = data.first() + && let Some(first_row) = correlation_rows.first() { match crate::alerts::incidents::correlate_alert_to_incident( &alert, first_row, - &data, + correlation_rows, triggered_at, eval_level, + None, ) .await { @@ -3258,7 +3940,10 @@ async fn handle_alert_triggers( // Asked after correlation, not predicted: a prediction suppresses a page nobody made. #[cfg(feature = "enterprise")] if o2_enterprise::enterprise::oncall::is_enabled() && !incident_handled_notification { - page_for_alert_firing(&scheduler_trace_id, &alert, &data).await; + let identity = unmuted_identity(&alert, &folder_id, &data, now).await; + if !identity.is_empty() { + page_for_alert_identity(&scheduler_trace_id, &alert, &identity).await; + } } let vars = get_row_column_map(&data); @@ -3305,6 +3990,7 @@ async fn handle_alert_triggers( } else if !incident_handled_notification && let Some(dispatch) = dispatch_per_group( &alert, + &folder_id, &scheduler_trace_id, trigger_results.group_classification.as_ref(), &data, @@ -3358,6 +4044,8 @@ async fn handle_alert_triggers( if !dispatch.errors.is_empty() { trigger_data_stream.error = Some(dispatch.errors.join("; ")); } + + dispatch.record_downtime(&mut trigger_data_stream); // The rollup row was committed before anything was sent. if let Some(outcome) = dispatch.rollup_rewrite() && let Some(alert_id) = alert.id.as_ref() @@ -3636,6 +4324,13 @@ async fn handle_alert_triggers( new_trigger.module_key ); trigger_data_stream.delivery_attempted = Some(false); + if let Some(downtime) = downtime.as_ref() { + crate::alerts::alert::record_suppressed_run( + &mut trigger_data_stream, + "alerts", + downtime.id.clone(), + ); + } } else if trigger_results.frozen { // Frozen is not Normal: nothing was measured (§7.6). `Skipped` is // the outcome `should_persist` drops entirely, so BOTH state axes @@ -3702,6 +4397,15 @@ async fn handle_alert_triggers( ) .await; } + // A partial mute names its downtime in history, but the alert still fired, so no Muted chip. + #[cfg(feature = "enterprise")] + if let (Some(alert_id), Some(downtime_id), RunOutcome::Suppressed) = ( + alert.id.as_ref(), + trigger_data_stream.downtime_id.as_deref(), + &trigger_data_stream.status, + ) { + record_last_downtime(&alert.org_id, &alert_id.to_string(), Some(downtime_id)).await; + } log::debug!( "[SCHEDULER trace_id {scheduler_trace_id}] publish_triggers_usage for alert: {}", @@ -6264,7 +6968,25 @@ async fn handle_slo_backfill_triggers( .await?; return Ok(()); } + Ok(crate::slo::backfill::ChunkOutcome::Failed) => { + log::error!( + "[slo] backfill job of {}/{slo_id} failed {} times to finish; marked failed", + trigger.org, + crate::slo::backfill::MAX_FINISH_ATTEMPTS + ); + db::scheduler::delete( + &trigger.org, + db::scheduler::TriggerModule::SloBackfill, + &slo_id, + ) + .await?; + return Ok(()); + } Ok(crate::slo::backfill::ChunkOutcome::More) => {} + // The trigger stays, so the next tick reloads the job that changed under this chunk. + Ok(crate::slo::backfill::ChunkOutcome::Superseded) => { + log::info!("[slo] backfill job of {slo_id} changed during the chunk; reloading"); + } Err(e) => { log::error!( "[slo] backfill chunk failed for {}/{slo_id}: {e}", @@ -6320,6 +7042,35 @@ mod tests { use super::*; + #[test] + fn a_pending_composite_inside_a_window_stays_pending_and_correlates_nothing() { + let downtime = config::meta::downtimes::ActiveDowntime { + id: "dt-1".to_string(), + name: "dt-1".to_string(), + ends_at: 0, + incident_mode: Default::default(), + }; + // The muted-incident correlation and the Suppressed status both read this answer. + assert_eq!( + composite_suppressed_by(&RunOutcome::Pending, Some(&downtime)), + None + ); + assert_eq!( + composite_suppressed_by(&RunOutcome::Firing, Some(&downtime)), + Some(&downtime) + ); + assert_eq!(composite_suppressed_by(&RunOutcome::Firing, None), None); + } + + #[cfg(feature = "enterprise")] + #[test] + fn a_composite_with_no_destinations_opens_no_incident_with_or_without_a_downtime() { + assert!(!composite_opens_incident(false, true)); + assert!(composite_opens_incident(true, true)); + assert!(!composite_opens_incident(true, false)); + assert!(!composite_opens_incident(false, false)); + } + #[test] fn incident_destination_delivery_does_not_suppress_attached_workflows() { assert!(should_dispatch_after_incident(true, true)); @@ -6341,9 +7092,40 @@ mod tests { delivered_groups: std::collections::HashSet::new(), attempted: false, state_failed: false, + downtime_id: None, } } + #[test] + fn a_partial_mute_keeps_the_firing_record_and_names_the_downtime() { + let mut partial = dispatch_counts(1, 0, 0, &[]); + partial.downtime_id = Some("dt-1".to_string()); + let mut row = TriggerData { + status: RunOutcome::Firing, + ..Default::default() + }; + partial.record_downtime(&mut row); + assert_eq!(row.status, RunOutcome::Firing); + assert_eq!(row.downtime_id.as_deref(), Some("dt-1")); + + let mut all_muted = dispatch_counts(0, 0, 0, &[]); + all_muted.downtime_id = Some("dt-1".to_string()); + let mut row = TriggerData { + status: RunOutcome::Firing, + ..Default::default() + }; + all_muted.record_downtime(&mut row); + assert_eq!(row.status, RunOutcome::Suppressed); + assert_eq!(row.downtime_id.as_deref(), Some("dt-1")); + + let mut row = TriggerData { + status: RunOutcome::Firing, + ..Default::default() + }; + dispatch_counts(0, 0, 0, &[]).record_downtime(&mut row); + assert_eq!((row.status, row.downtime_id), (RunOutcome::Firing, None)); + } + #[test] fn test_group_dispatch_status_diverges_from_rollup_only_on_partial_send() { use RunOutcome::{NotifyFailed, Pending}; @@ -7639,6 +8421,350 @@ mod tests { } } + #[cfg(feature = "enterprise")] + #[tokio::test] + async fn a_recovery_emitted_after_its_downtime_was_cancelled_is_still_muted() { + use config::meta::downtimes::Downtime; + + const MINUTE: i64 = 60_000_000; + let at = |minutes: i64| 9 * 60 * MINUTE + minutes * MINUTE; + // The org's only downtime, 09:00-10:00, cancelled at 09:30; the cancel cleared the record. + let row: Downtime = serde_json::from_value(serde_json::json!({ + "id": "dt-1", "org": "recovery_after_cancel", "name": "deploy", + "targets": [{ "module": "alerts", "folders": { "kind": "all" } }], + "schedule": { + "repeat": "none", "starts_at": at(0), "ends_at": at(60), + "timezone": "UTC", "duration_secs": 3_600 + }, + "cancelled_at": at(30), + "created_by": "lin", "created_at": 0, "updated_by": "lin", "updated_at": 0 + })) + .unwrap(); + let mut alert = config::meta::alerts::alert::Alert::default(); + alert.org_id = "recovery_after_cancel".to_string(); + alert.id = Some(::new( + None, None, + )); + + // Recovered at 09:20, asked at 09:40 when `keep_firing_for` released it. + let decision = downtime_decision_in( + std::slice::from_ref(&row), + &alert, + "default", + &[], + at(20), + at(40), + ) + .await; + assert_eq!(decision.map(|d| d.id).as_deref(), Some("dt-1")); + // A recovery after the cancel is not muted. + assert!( + downtime_decision_in(&[row], &alert, "default", &[], at(35), at(40)) + .await + .is_none() + ); + } + + #[tokio::test] + async fn a_run_that_matched_nothing_takes_no_downtime_decision() { + let alert = config::meta::alerts::alert::Alert::default(); + assert!( + downtime_decision(&alert, "default", None, now_micros()) + .await + .is_none() + ); + } + + #[test] + fn a_downtime_suppression_never_starts_the_silence_window() { + let decision = DeliveryDecision::SuppressedByDowntime("dt-1".to_string()); + assert!(!decision.should_deliver()); + assert!(!decision.resets_silence()); + assert_eq!(decision.downtime_id(), Some("dt-1")); + } + + #[test] + fn a_suppressed_run_opens_no_silence_window_and_keeps_its_cadence() { + assert!(starts_silence_window(true, false, 10, false)); + assert!( + !starts_silence_window(true, true, 10, false), + "a suppressed run must leave next_run_at at now + frequency" + ); + assert!(!starts_silence_window(false, false, 10, false)); + assert!(!starts_silence_window(true, false, 10, true)); + } + + #[test] + fn a_pending_run_with_silence_stores_no_period_end_time() { + // Silence 10, pending 2, no downtime: the end time is re-derived, as on main. + assert!(!pending_stores_end_time(true, 10, false)); + assert!(pending_stores_end_time(true, 0, false)); + assert!( + pending_stores_end_time(true, 10, true), + "it evaluates through silence" + ); + assert!(!pending_stores_end_time(false, 0, false)); + } + + #[test] + fn a_suppressed_run_leaves_the_last_notified_level_alone() { + let muted = DeliveryDecision::SuppressedByDowntime("dt-1".to_string()); + assert!(!records_delivery_state(true, &muted)); + assert!(records_delivery_state(true, &DeliveryDecision::Deliver)); + assert!(!records_delivery_state(false, &DeliveryDecision::Deliver)); + } + + #[test] + fn the_first_run_after_the_window_delivers_without_a_cooldown() { + use config::meta::alerts::level::{AlertLevel, delivery_decision}; + + const MINUTE: i64 = 60_000_000; + // Delivered at t=0, silence 10 min; suppressed runs to t=2h wrote no delivery state. + let last_notified = Some(AlertLevel::Critical); + let silenced_until = Some(10 * MINUTE); + let after_window = 121 * MINUTE; + let decision = delivery_decision( + AlertLevel::Critical, + last_notified, + silenced_until, + after_window, + None, + ); + assert_eq!(decision, DeliveryDecision::Deliver); + let never_notified = + delivery_decision(AlertLevel::Critical, None, None, after_window, None); + assert!(never_notified.should_deliver()); + } + + #[cfg(feature = "enterprise")] + #[test] + fn a_multi_alert_is_muted_at_alert_level_only_when_every_group_is() { + let group = |host: &str| HashMap::from([("host".to_string(), host.to_string())]); + let downtime = |id: &str| config::meta::downtimes::ActiveDowntime { + id: id.to_string(), + name: id.to_string(), + ends_at: 1, + incident_mode: Default::default(), + }; + let identity = [group("a"), group("b")]; + let only_a = muted_in_every_group(&identity, |dims| { + (dims["host"] == "a").then(|| downtime("d1")) + }); + assert!(only_a.is_none(), "host b still notifies, per group"); + let both = muted_in_every_group(&identity, |dims| { + Some(downtime(if dims["host"] == "a" { "d1" } else { "d2" })) + }); + assert_eq!(both.map(|d| d.id).as_deref(), Some("d1")); + assert!(muted_in_every_group(&[], |_| Some(downtime("d1"))).is_none()); + } + + #[cfg(feature = "enterprise")] + #[test] + fn the_muted_mode_and_end_of_a_multi_alert_do_not_depend_on_the_first_group() { + use config::meta::downtimes::{ActiveDowntime, IncidentMode}; + let group = |host: &str| HashMap::from([("host".to_string(), host.to_string())]); + let per_host = |dims: &HashMap| { + Some(if dims["host"] == "a" { + ActiveDowntime { + id: "a".to_string(), + name: "a".to_string(), + ends_at: 10, + incident_mode: IncidentMode::None, + } + } else { + ActiveDowntime { + id: "b".to_string(), + name: "b".to_string(), + ends_at: 12, + incident_mode: IncidentMode::Muted, + } + }) + }; + for identity in [[group("a"), group("b")], [group("b"), group("a")]] { + let decision = muted_in_every_group(&identity, per_host).unwrap(); + assert_eq!((decision.id.as_str(), decision.ends_at), ("b", 12)); + assert_eq!(decision.incident_mode, IncidentMode::Muted); + } + } + + #[cfg(feature = "enterprise")] + #[test] + fn incident_mode_none_correlates_no_muted_firing() { + use config::meta::downtimes::{ActiveDowntime, IncidentMode}; + let downtime = |incident_mode| ActiveDowntime { + id: "dt-1".to_string(), + name: "Deploy".to_string(), + ends_at: 1, + incident_mode, + }; + assert!(correlates_muted_firing( + true, + true, + &downtime(IncidentMode::Muted) + )); + assert!(!correlates_muted_firing( + true, + true, + &downtime(IncidentMode::None) + )); + assert!(!correlates_muted_firing( + false, + true, + &downtime(IncidentMode::Muted) + )); + assert!(!correlates_muted_firing( + true, + false, + &downtime(IncidentMode::Muted) + )); + assert_eq!( + IncidentMode::default(), + IncidentMode::Muted, + "an Alerts target without the field keeps today's muted incident" + ); + } + + #[cfg(feature = "enterprise")] + #[test] + fn only_an_unsuppressed_matched_run_clears_the_recorded_mute() { + assert!(clears_recorded_mute(&RunOutcome::Firing)); + assert!(clears_recorded_mute(&RunOutcome::NotifyFailed)); + assert!(!clears_recorded_mute(&RunOutcome::Pending)); + assert!(!clears_recorded_mute(&RunOutcome::Suppressed)); + assert!(!clears_recorded_mute(&RunOutcome::Normal)); + assert!(!clears_recorded_mute(&RunOutcome::Skipped)); + } + + #[cfg(feature = "enterprise")] + #[test] + fn a_pending_run_inside_a_window_keeps_the_recorded_mute() { + // Suppressed records the mute, Normal withholds the recovery, then the alert matches again. + let mut recorded = None; + let mut escalations_closed = 0; + for outcome in [ + RunOutcome::Suppressed, + RunOutcome::Normal, + RunOutcome::Pending, + ] { + if matches!(outcome, RunOutcome::Suppressed) { + recorded = Some("d1"); + } + // As `clear_recorded_mute`: a cleared record recovers the muted on-call escalations. + if clears_recorded_mute(&outcome) && recorded.take().is_some() { + escalations_closed += 1; + } + } + assert_eq!(recorded, Some("d1")); + assert_eq!(escalations_closed, 0); + } + + #[cfg(feature = "enterprise")] + #[test] + fn a_firing_after_an_ended_one_time_window_still_clears_its_mute() { + let ended: config::meta::downtimes::Downtime = serde_json::from_value(serde_json::json!({ + "id": "d1", + "org": "acme", + "name": "deploy", + "targets": [{"module": "alerts", "folders": {"kind": "all"}}], + "schedule": { + "repeat": "none", + "starts_at": 0, + "ends_at": 3_600_000_000_i64, + "timezone": "UTC", + "duration_secs": 3600, + "weekdays": [] + }, + "created_by": "lin", + "created_at": 0, + "updated_by": "lin", + "updated_at": 0 + })) + .unwrap(); + let rows = std::slice::from_ref(&ended); + let now = 2 * 3_600_000_000_i64; + assert!(!crate::alerts::downtimes::enterprise::any_live_target( + rows, + config::meta::downtimes::TargetModule::Alerts, + now + )); + assert!(mute_may_linger(rows)); + let mut cancelled = ended; + cancelled.cancelled_at = Some(1); + assert!(!mute_may_linger(&[cancelled])); + } + + #[cfg(feature = "enterprise")] + #[test] + fn a_muted_anomaly_run_still_counts_as_a_detection_that_ran() { + assert!(detection_ran(&RunOutcome::Suppressed)); + assert!(detection_ran(&RunOutcome::Firing)); + assert!(detection_ran(&RunOutcome::NotifyFailed)); + assert!(detection_ran(&RunOutcome::Normal)); + assert!(!detection_ran(&RunOutcome::Error)); + } + + #[test] + fn a_muted_run_still_counts_toward_the_pending_period() { + use config::meta::alerts::state::{ROLLUP_GROUP_KEY, apply_outcome}; + + const MINUTE: i64 = 60_000_000; + let mut alert = config::meta::alerts::alert::Alert::default(); + alert.pending_period_sec = 120; + let rows = [json::json!({ "value": 1 }).as_object().cloned().unwrap()]; + let run = |prev: Option<&config::meta::alerts::state::AlertState>, at: i64| { + let pending = holds_pending(&alert, prev, Some(&rows), at); + let outcome = if pending { + RunOutcome::Pending + } else { + RunOutcome::Firing + }; + let state = apply_outcome("a1", ROLLUP_GROUP_KEY, prev, outcome, None, at).state; + (pending, state) + }; + // Run 1 falls inside the window; the pending check runs before the downtime decision. + let (pending, state) = run(None, 0); + assert!(pending, "run 1 is pending, muted or not"); + let (pending, state) = run(state.as_ref(), MINUTE); + assert!(pending, "run 2, after the window, still waits"); + let (pending, _) = run(state.as_ref(), 2 * MINUTE); + assert!(!pending, "run 3 pages"); + + let fired = apply_outcome("a1", ROLLUP_GROUP_KEY, None, RunOutcome::Firing, None, 0).state; + assert!(!holds_pending(&alert, fired.as_ref(), Some(&rows), MINUTE)); + assert!(!holds_pending(&alert, None, None, 0)); + alert.pending_period_sec = 0; + assert!(!holds_pending(&alert, None, Some(&rows), 0)); + } + + #[test] + fn a_composite_with_downtimes_off_is_never_suppressed() { + let definition = infra::table::entity::alert_composites::Model { + id: "2f9K".to_string(), + org: "default".to_string(), + folder_id: "default".to_string(), + name: "c".to_string(), + description: None, + expression: "{a}".to_string(), + warning_counts_as_firing: false, + stale_child_policy: 0, + destinations: serde_json::json!([]), + template: None, + context_attributes: None, + enabled: true, + silence_seconds: 0, + creates_incident: false, + workflows: serde_json::json!([]), + priority: None, + tags: Some(serde_json::json!(["service:payments"])), + owner: None, + last_edited_by: None, + updated_at: None, + evaluation_generation: 0, + pending_period_sec: 0, + }; + assert!(composite_downtime(&definition, now_micros()).is_none()); + } + /// A lost claim and an ineligible row judged nothing, so neither can read as Normal. #[cfg(feature = "enterprise")] #[test] @@ -7651,6 +8777,7 @@ mod tests { notify_error: None, claim_lost, ineligible, + suppressed_by: None, }; assert_eq!( anomaly_run_status(&run(true, true, true, false)), @@ -7672,6 +8799,275 @@ mod tests { anomaly_run_status(&run(false, false, false, false)), RunOutcome::Normal ); + let muted = DetectionRunOutcome { + suppressed_by: Some("dt-1".to_string()), + ..run(true, false, false, false) + }; + assert_eq!(anomaly_run_status(&muted), RunOutcome::Suppressed); + } + + #[cfg(feature = "enterprise")] + fn service_groups() -> Vec { + vec![config::meta::correlation::FieldAlias { + id: "service".to_string(), + display: "Service".to_string(), + group: None, + fields: vec!["service".to_string()], + is_workload_type: false, + }] + } + + #[cfg(feature = "enterprise")] + fn service_row(service: &str, value: i64) -> config::utils::json::Map { + json::json!({ "service": service, "value": value }) + .as_object() + .cloned() + .unwrap() + } + + /// The decision `downtime_decision`, `realtime_downtime` and `entry_downtime` take. + #[cfg(feature = "enterprise")] + fn run_downtime( + alert: &config::meta::alerts::alert::Alert, + rows: &[config::utils::json::Map], + muted: &[&str], + ) -> Option { + let identity: Vec> = + identities_by_key(alert, rows, &service_groups()) + .into_iter() + .map(|(_, dims)| dims) + .collect(); + muted_in_every_group(&identity, |dims| { + let service = dims.get("service")?; + muted + .contains(&service.as_str()) + .then(|| config::meta::downtimes::ActiveDowntime { + id: format!("dt-{service}"), + name: service.clone(), + ends_at: 1, + incident_mode: Default::default(), + }) + }) + .map(|d| d.id) + } + + #[cfg(feature = "enterprise")] + #[test] + fn a_batch_with_one_unmuted_service_is_not_suppressed_in_either_order() { + let alert = config::meta::alerts::alert::Alert::default(); + let payments_first = [service_row("payments", 1), service_row("checkout", 1)]; + let checkout_first = [service_row("checkout", 1), service_row("payments", 1)]; + assert_eq!(run_downtime(&alert, &payments_first, &["payments"]), None); + assert_eq!(run_downtime(&alert, &checkout_first, &["payments"]), None); + } + + #[cfg(feature = "enterprise")] + #[test] + fn a_batch_whose_every_service_is_muted_is_suppressed() { + let alert = config::meta::alerts::alert::Alert::default(); + let rows = [service_row("payments", 1), service_row("checkout", 1)]; + assert_eq!( + run_downtime(&alert, &rows, &["payments", "checkout"]).as_deref(), + Some("dt-payments") + ); + assert_eq!( + run_downtime(&alert, &rows[..1], &["payments"]).as_deref(), + Some("dt-payments") + ); + } + + #[cfg(feature = "enterprise")] + #[test] + fn a_promql_multi_alert_with_one_unmuted_series_is_not_suppressed() { + let mut alert = config::meta::alerts::alert::Alert::default(); + alert.query_condition.query_type = config::meta::alerts::QueryType::PromQL; + alert.query_condition.promql_multi_alert = true; + let rows = [service_row("payments", 1), service_row("checkout", 2)]; + assert_eq!(identities_by_key(&alert, &rows, &service_groups()).len(), 2); + assert_eq!(run_downtime(&alert, &rows, &["payments"]), None); + assert_eq!(run_downtime(&alert, &rows, &["checkout"]), None); + assert!(run_downtime(&alert, &rows, &["payments", "checkout"]).is_some()); + } + + #[cfg(feature = "enterprise")] + #[test] + fn a_promql_multi_alert_with_a_muted_first_series_pages_only_the_unmuted_series() { + let mut alert = config::meta::alerts::alert::Alert::default(); + alert.query_condition.query_type = config::meta::alerts::QueryType::PromQL; + alert.query_condition.promql_multi_alert = true; + let rows = [service_row("checkout", 1), service_row("payments", 2)]; + let identities = || -> Vec> { + identities_by_key(&alert, &rows, &service_groups()) + .into_iter() + .map(|(_, dims)| dims) + .collect() + }; + let first_row = vec![identities()[0].clone()]; + let muted_first = first_row[0]["service"].clone(); + let paged = unmuted_maps(first_row.clone(), identities(), |dims| { + dims["service"] == muted_first + }); + assert_eq!(paged.len(), 1); + assert_ne!(paged[0]["service"], muted_first); + let unmuted = unmuted_maps(first_row.clone(), identities(), |_| false); + assert_eq!(unmuted, first_row, "nothing muted keeps the on-call maps"); + assert!(unmuted_maps(first_row, identities(), |_| true).is_empty()); + } + + /// `entry_downtime` reads every row of a grouped-flush entry, not the first. + #[cfg(feature = "enterprise")] + #[test] + fn a_grouped_flush_entry_with_mixed_rows_is_not_dropped() { + let mut alert = config::meta::alerts::alert::Alert::default(); + alert.is_real_time = true; + let rows = vec![ + service_row("payments", 1), + service_row("payments", 2), + service_row("checkout", 3), + ]; + assert_eq!(run_downtime(&alert, &rows, &["payments"]), None); + } + + #[cfg(feature = "enterprise")] + fn sql_multi_alert() -> config::meta::alerts::alert::Alert { + let mut sql = config::meta::alerts::alert::Alert::default(); + sql.query_condition.aggregation = Some(config::meta::alerts::Aggregation { + group_by: Some(vec!["service".to_string()]), + function: config::meta::alerts::AggFunction::Avg, + having: config::meta::alerts::Condition { + column: "value".to_string(), + operator: config::meta::alerts::Operator::GreaterThan, + value: json::json!(0), + ignore_case: false, + }, + warning_value: None, + multi_alert: true, + }); + sql + } + + /// `dispatch_per_group` and `unmuted_group_rows` look a row's group up under this key. + #[cfg(feature = "enterprise")] + #[test] + fn every_multi_alert_row_finds_its_group_identity() { + let mut promql = config::meta::alerts::alert::Alert::default(); + promql.query_condition.query_type = config::meta::alerts::QueryType::PromQL; + promql.query_condition.promql_multi_alert = true; + let sql = sql_multi_alert(); + let rows = [service_row("payments", 1), service_row("checkout", 2)]; + for alert in [&promql, &sql] { + let identities: HashMap> = + identities_by_key(alert, &rows, &service_groups()) + .into_iter() + .collect(); + let group_by = alert_group_by(alert); + for row in &rows { + let dims = &identities[&downtime_row_key(alert, &group_by, row)]; + assert_eq!( + dims.get("service"), + row["service"].as_str().map(str::to_string).as_ref() + ); + } + } + } + + /// The grouped flush calls [downtime_identity] too, so both points judge an SLO alert alike. + #[cfg(feature = "enterprise")] + #[test] + fn an_slo_alert_is_matched_on_its_slo_dimensions_not_its_rows() { + use config::meta::{ + alerts::Operator, + slo::condition::{SloAlertKind, SloCondition}, + }; + let mut alert = config::meta::alerts::alert::Alert::default(); + alert.org_id = "acme".to_string(); + alert.query_condition.slo_condition = Some(SloCondition { + slo_id: "slo-1".to_string(), + kind: SloAlertKind::ErrorBudget, + operator: Operator::GreaterThan, + critical: 10.0, + warning: None, + long_window_secs: None, + short_window_secs: None, + multi_alert: false, + }); + assert_eq!(identity_slo(&alert), Some("slo-1")); + alert.query_condition.slo_condition = None; + assert_eq!(identity_slo(&alert), None); + } + + /// Result order opposite to group-key order must not hand one group's mode to another. + #[cfg(feature = "enterprise")] + #[test] + fn each_muted_group_correlates_under_its_own_incident_mode() { + use config::meta::downtimes::{ActiveDowntime, IncidentMode}; + let alert = sql_multi_alert(); + let group_by = alert_group_by(&alert); + let mut rows = [service_row("payments", 1), service_row("checkout", 2)]; + let key = |row: &config::utils::json::Map| { + downtime_row_key(&alert, &group_by, row) + }; + if key(&rows[0]) < key(&rows[1]) { + rows.swap(0, 1); + } + let keys: Vec = identities_by_key(&alert, &rows, &service_groups()) + .into_iter() + .map(|(key, _)| key) + .collect(); + assert_eq!( + keys[0], + key(&rows[1]), + "the first result row is the last group" + ); + let first = rows[0]["service"].as_str().unwrap().to_string(); + let downtime = |service: &str, incident_mode| ActiveDowntime { + id: format!("dt-{service}"), + name: service.to_string(), + ends_at: 1, + incident_mode, + }; + let modes = |first_mode, other_mode| { + let first = first.clone(); + move |row: &config::utils::json::Map| { + let service = row["service"].as_str().unwrap(); + let mode = if service == first { + first_mode + } else { + other_mode + }; + Some(downtime(service, mode)) + } + }; + let other = rows[1]["service"].as_str().unwrap(); + + let (kept, chosen) = + keep_muted_incident_rows(&rows, modes(IncidentMode::None, IncidentMode::Muted)) + .unwrap(); + assert_eq!( + kept, + vec![rows[1].clone()], + "the none group opens no incident" + ); + assert_eq!(chosen.id, format!("dt-{other}")); + + let (kept, chosen) = + keep_muted_incident_rows(&rows, modes(IncidentMode::Muted, IncidentMode::None)) + .unwrap(); + assert_eq!( + kept, + vec![rows[0].clone()], + "the muted group still correlates" + ); + assert_eq!(chosen.id, format!("dt-{first}")); + + assert!( + keep_muted_incident_rows(&rows, modes(IncidentMode::None, IncidentMode::None)) + .is_none() + ); + let (kept, _) = + keep_muted_incident_rows(&rows, modes(IncidentMode::Muted, IncidentMode::Muted)) + .unwrap(); + assert_eq!(kept.len(), 2); } #[cfg(feature = "enterprise")] diff --git a/src/core/src/alerts/templates.rs b/src/core/src/alerts/templates.rs index fd0f53cab5..252d3281a2 100644 --- a/src/core/src/alerts/templates.rs +++ b/src/core/src/alerts/templates.rs @@ -39,6 +39,13 @@ use crate::{ common::meta::authz::Authz, }; +pub const DOWNTIME_TEMPLATE: &str = "prebuilt_downtime"; +pub const DOWNTIME_EMAIL_TEMPLATE: &str = "prebuilt_downtime_email"; +pub const DOWNTIME_EMAIL_TITLE: &str = "OpenObserve downtime {downtime_name}: {downtime_event}"; + +const DOWNTIME_HTTP_BODY: &str = r#"{"text": "Downtime {downtime_name}: {downtime_event}. {downtime_starts_at} to {downtime_ends_at}, muting {downtime_counts}. {downtime_url}", "downtime": {"name": "{downtime_name}", "event": "{downtime_event}", "reason": "{downtime_reason}", "starts_at": "{downtime_starts_at}", "ends_at": "{downtime_ends_at}", "targets": "{downtime_targets}", "counts": "{downtime_counts}", "url": "{downtime_url}", "org": "{org_name}"}}"#; +const DOWNTIME_EMAIL_BODY: &str = r#"

Downtime {downtime_name}: {downtime_event}

Window: {downtime_starts_at} to {downtime_ends_at}

Covers {downtime_targets}, muting {downtime_counts}.

Reason: {downtime_reason}

Organization: {org_name}

Open downtimes in OpenObserve

"#; + /// Sticky-kind rule (design §6.2): create defaults absent→Custom; update /// preserves the existing kind when the client didn't specify one. pub(crate) fn resolve_kind( @@ -167,6 +174,32 @@ pub async fn delete(org_id: &str, name: &str, is_root: bool) -> Result<(), Templ pub(crate) const PREBUILT_REVISION_KEY: &str = "prebuilt_templates_revision"; +pub fn downtime_templates() -> [Template; 2] { + let template = |name: &str, body: &str, template_type| Template { + id: None, + org_id: DEFAULT_ORG.to_string(), + name: name.to_string(), + is_default: true, + template_type, + body: body.to_string(), + kind: TemplateKind::Custom, + }; + [ + template(DOWNTIME_TEMPLATE, DOWNTIME_HTTP_BODY, TemplateType::Http), + template( + DOWNTIME_EMAIL_TEMPLATE, + DOWNTIME_EMAIL_BODY, + TemplateType::Email { + title: DOWNTIME_EMAIL_TITLE.to_string(), + }, + ), + ] +} + +pub fn is_downtime_template_name(name: &str) -> bool { + name == DOWNTIME_TEMPLATE || name == DOWNTIME_EMAIL_TEMPLATE +} + /// Reseed gate (design §6.3): overwrite a stored prebuilt template when the /// shipped revision is newer, or when revisions are equal but the stored /// body/type drifted — equal-revision drift is the old-binary-revert @@ -220,85 +253,86 @@ pub async fn ensure_system_templates() -> Result<(), anyhow::Error> { let mut updated_count = 0; let mut skipped_count = 0; - for prebuilt_type in prebuilt_types { - if let Some(mut template) = get_prebuilt_template(prebuilt_type) { - // Set org_id to DEFAULT_ORG for global visibility - template.org_id = DEFAULT_ORG.to_string(); + let prebuilt = prebuilt_types + .into_iter() + .filter_map(get_prebuilt_template) + .chain(seeded_downtime_templates()); + for mut template in prebuilt { + // Set org_id to DEFAULT_ORG for global visibility + template.org_id = DEFAULT_ORG.to_string(); - // Check if template already exists - match db::alerts::templates::get(DEFAULT_ORG, &template.name).await { - Ok(existing) => { - // System templates are protected from user edits, so the - // prebuilt definition is the source of truth. Refresh the - // stored copy when it has drifted (e.g. a shipped fix to the - // body/title) so existing installs pick up the correction — - // but only when the reseed gate says to (revision-gated, see - // `should_apply_prebuilt`). - let drifted = existing.body != template.body - || existing.template_type != template.template_type; - if should_apply_prebuilt(shipped_rev, applied_rev, drifted) && drifted { - // Preserve the stored id so this is an update, not an insert. - template.id = existing.id; - match db::alerts::templates::set(template.clone()).await { - Ok(_) => { - updated_count += 1; - log::info!( - "[TEMPLATES] Updated system template '{}' in {}", - template.name, - DEFAULT_ORG - ); - } - Err(e) => { - log::error!( - "[TEMPLATES] Failed to update system template '{}' in {}: {}", - template.name, - DEFAULT_ORG, - e - ); - } - } - } else { - skipped_count += 1; - log::debug!( - "[TEMPLATES] System template '{}' already up to date in {}", - template.name, - DEFAULT_ORG - ); - } - } - Err(TemplateError::NotFound) => { - // Template doesn't exist, create it + // Check if template already exists + match db::alerts::templates::get(DEFAULT_ORG, &template.name).await { + Ok(existing) => { + // System templates are protected from user edits, so the + // prebuilt definition is the source of truth. Refresh the + // stored copy when it has drifted (e.g. a shipped fix to the + // body/title) so existing installs pick up the correction — + // but only when the reseed gate says to (revision-gated, see + // `should_apply_prebuilt`). + let drifted = existing.body != template.body + || existing.template_type != template.template_type; + if should_apply_prebuilt(shipped_rev, applied_rev, drifted) && drifted { + // Preserve the stored id so this is an update, not an insert. + template.id = existing.id; match db::alerts::templates::set(template.clone()).await { Ok(_) => { - created_count += 1; + updated_count += 1; log::info!( - "[TEMPLATES] Created system template '{}' in {}", + "[TEMPLATES] Updated system template '{}' in {}", template.name, DEFAULT_ORG ); } Err(e) => { log::error!( - "[TEMPLATES] Failed to create system template '{}' in {}: {}", + "[TEMPLATES] Failed to update system template '{}' in {}: {}", template.name, DEFAULT_ORG, e ); } } - } - Err(e) => { - log::error!( - "[TEMPLATES] Error checking system template '{}' in {}: {}", + } else { + skipped_count += 1; + log::debug!( + "[TEMPLATES] System template '{}' already up to date in {}", template.name, - DEFAULT_ORG, - e + DEFAULT_ORG ); } } + Err(TemplateError::NotFound) => { + // Template doesn't exist, create it + match db::alerts::templates::set(template.clone()).await { + Ok(_) => { + created_count += 1; + log::info!( + "[TEMPLATES] Created system template '{}' in {}", + template.name, + DEFAULT_ORG + ); + } + Err(e) => { + log::error!( + "[TEMPLATES] Failed to create system template '{}' in {}: {}", + template.name, + DEFAULT_ORG, + e + ); + } + } + } + Err(e) => { + log::error!( + "[TEMPLATES] Error checking system template '{}' in {}: {}", + template.name, + DEFAULT_ORG, + e + ); + } } } - // Seed the compiled-in default content template's DB copy — same // dist_lock + revision-gate scope as the prebuilt loop above, so it // converges on the same startup pass rather than racing it. @@ -348,10 +382,37 @@ pub async fn ensure_system_templates() -> Result<(), anyhow::Error> { Ok(()) } +fn seeded_downtime_templates() -> Vec diff --git a/web/src/components/alerts/downtimes/DowntimeConditionSection.vue b/web/src/components/alerts/downtimes/DowntimeConditionSection.vue new file mode 100644 index 0000000000..93168af371 --- /dev/null +++ b/web/src/components/alerts/downtimes/DowntimeConditionSection.vue @@ -0,0 +1,199 @@ + + + + + diff --git a/web/src/components/alerts/downtimes/DowntimeCountdown.spec.ts b/web/src/components/alerts/downtimes/DowntimeCountdown.spec.ts new file mode 100644 index 0000000000..a21d9a30df --- /dev/null +++ b/web/src/components/alerts/downtimes/DowntimeCountdown.spec.ts @@ -0,0 +1,24 @@ +// Copyright 2026 OpenObserve Inc. + +import { afterEach, describe, expect, it, vi } from "vitest"; +import { mount } from "@vue/test-utils"; +import DowntimeCountdown from "./DowntimeCountdown.vue"; + +const NOW = Date.parse("2026-09-17T14:00:00Z"); + +describe("DowntimeCountdown", () => { + afterEach(() => vi.useRealTimers()); + + it("counts down live against the given clock", async () => { + vi.useFakeTimers(); + vi.setSystemTime(NOW); + const wrapper = mount(DowntimeCountdown, { + props: { endsAt: (NOW + 2 * 60_000 + 5_000) * 1000, now: () => Date.now() + 60_000 }, + }); + expect(wrapper.text()).toBe("ends in 1 minute"); + vi.advanceTimersByTime(5_100); + await wrapper.vm.$nextTick(); + expect(wrapper.text()).toBe("ends in less than a minute"); + wrapper.unmount(); + }); +}); diff --git a/web/src/components/alerts/downtimes/DowntimeCountdown.vue b/web/src/components/alerts/downtimes/DowntimeCountdown.vue new file mode 100644 index 0000000000..32b357a34e --- /dev/null +++ b/web/src/components/alerts/downtimes/DowntimeCountdown.vue @@ -0,0 +1,47 @@ + + + + + diff --git a/web/src/components/alerts/downtimes/DowntimeDetail.notifications.spec.ts b/web/src/components/alerts/downtimes/DowntimeDetail.notifications.spec.ts new file mode 100644 index 0000000000..ca4939e24c --- /dev/null +++ b/web/src/components/alerts/downtimes/DowntimeDetail.notifications.spec.ts @@ -0,0 +1,194 @@ +// Copyright 2026 OpenObserve Inc. + +import { describe, it, expect, vi, beforeEach } from "vitest"; +import { flushPromises, mount } from "@vue/test-utils"; +import { createMemoryHistory, createRouter } from "vue-router"; +import store from "@/test/unit/helpers/store"; +import DowntimeDetail from "./DowntimeDetail.vue"; +import downtimes, { type DowntimeDetail as Detail } from "@/services/downtimes"; +import destination from "@/services/alert_destination"; +import template from "@/services/alert_templates"; +import { queryClient } from "@/composables/query/queryClient"; +import OTabs from "@/lib/navigation/Tabs/OTabs.vue"; + +vi.mock("@/services/downtimes", () => ({ + default: { + list: vi.fn(), + get: vi.fn(), + create: vi.fn(), + update: vi.fn(), + cancel: vi.fn(), + extend: vi.fn(), + remove: vi.fn(), + move: vi.fn(), + preview: vi.fn(), + resources: vi.fn(), + }, +})); + +vi.mock("@/services/alert_destination", () => ({ + default: { list: vi.fn(), test: vi.fn() }, +})); + +vi.mock("@/services/alert_templates", () => ({ + default: { get_by_name: vi.fn() }, +})); + +vi.mock("@/services/alerts", () => ({ + default: { getHistory: vi.fn(() => Promise.resolve({ data: { hits: [], total: 0 } })) }, +})); + +vi.mock("@/utils/commons", async (importOriginal) => ({ + ...(await importOriginal>()), + getFoldersListByType: vi.fn(() => Promise.resolve([])), +})); + +vi.mock("@/services/common", async (importOriginal) => { + const actual = await importOriginal<{ default: Record }>(); + return { + ...actual, + default: { + ...actual.default, + list_Folders: vi.fn(() => Promise.resolve({ data: { list: [] } })), + }, + }; +}); + +const HOUR = 3_600_000_000; +const NOW = Date.now() * 1000; + +const DETAIL = { + id: "2f9K", + org: "default", + folder_id: "default", + name: "Nightly deploy", + targets: [{ module: "alerts", folders: { kind: "all" } }], + schedule: { + repeat: "none", + starts_at: NOW + HOUR, + ends_at: NOW + 2 * HOUR, + timezone: "UTC", + duration_secs: 3600, + weekdays: [], + }, + show_banner: false, + notifications: { + destinations: ["slack-oncall"], + events: { started: true, ending_soon: true, ended: true, cancelled: false, extended: false }, + ending_soon_lead_secs: 600, + }, + created_by: "lin@example.com", + created_at: 0, + updated_by: "lin@example.com", + updated_at: 0, + status: "scheduled", + current_window: null, + next_window: { start: NOW + HOUR, end: NOW + 2 * HOUR }, + matched_alerts: 3, + matched_anomalies: 0, + matched_synthetics: 0, + matched_slos: 0, + affected: { alerts: [], anomalies: [], synthetics: [], slos: [] }, + notification_log: [ + { + window_start: NOW - 24 * HOUR, + event: "ended", + sent_at: NOW - 23 * HOUR, + destinations: ["slack-oncall"], + result: "slack-oncall: 500 Internal Server Error", + }, + { + window_start: NOW - 24 * HOUR, + event: "started", + sent_at: NOW - 24 * HOUR, + destinations: ["slack-oncall"], + result: "ok", + }, + ], +} as unknown as Detail; + +const makeRouter = () => + createRouter({ + history: createMemoryHistory(), + routes: [ + { path: "/downtimes", name: "downtimes", component: { template: "
" } }, + { path: "/downtimes/:id", name: "downtimeDetail", component: DowntimeDetail }, + { path: "/downtimes/:id/edit", name: "editDowntime", component: { template: "
" } }, + ], + }); + +const mountDetail = async () => { + const router = makeRouter(); + await router.push({ + name: "downtimeDetail", + params: { id: "2f9K" }, + query: { org_identifier: "default" }, + }); + await router.isReady(); + const wrapper = mount(DowntimeDetail, { + global: { plugins: [store, router] }, + attachTo: document.body, + }); + await flushPromises(); + wrapper.findComponent(OTabs).vm.$emit("update:modelValue", "notifications"); + await flushPromises(); + return wrapper; +}; + +describe("DowntimeDetail notifications tab", () => { + beforeEach(() => { + queryClient.clear(); + vi.mocked(downtimes.get).mockResolvedValue({ data: DETAIL } as any); + vi.mocked(destination.list).mockResolvedValue({ + data: [ + { name: "slack-oncall", type: "http", url: "https://hooks.example.com", method: "post" }, + ], + } as any); + vi.mocked(template.get_by_name).mockResolvedValue({ + data: { body: '{"text": "{downtime_name}: {downtime_event}"}' }, + } as any); + vi.mocked(destination.test).mockReset(); + vi.mocked(destination.test).mockResolvedValue({ data: { success: true } } as any); + store.state.selectedOrganization = { + ...store.state.selectedOrganization, + identifier: "default", + }; + }); + + it("lists the sends newest first with their event, destinations and result", async () => { + const wrapper = await mountDetail(); + const tab = wrapper.get('[data-test="downtime-detail-notifications"]'); + expect(tab.text()).toContain("Notifies slack-oncall: when it starts, 10 min before it ends"); + const text = wrapper.get('[data-test="downtime-detail-notifications-table"]').text(); + expect(text).toContain("Ended"); + expect(text).toContain("slack-oncall: 500 Internal Server Error"); + expect(text).toContain("Started"); + expect(text).toContain("Sent"); + expect(text.indexOf("Ended")).toBeLessThan(text.indexOf("Started")); + wrapper.unmount(); + }); + + it("sends the started message through the destination test path", async () => { + const wrapper = await mountDetail(); + await wrapper.get('[data-test="downtime-detail-send-test"]').trigger("click"); + await flushPromises(); + expect(template.get_by_name).toHaveBeenCalledWith({ + org_identifier: "default", + template_name: "prebuilt_downtime", + }); + expect(destination.test).toHaveBeenCalledTimes(1); + const [{ data }] = vi.mocked(destination.test).mock.calls[0] as any; + expect(data).toMatchObject({ type: "http", url: "https://hooks.example.com" }); + expect(JSON.parse(data.body)).toEqual({ text: "Nightly deploy: started" }); + wrapper.unmount(); + }); + + it("shows an empty log for a row that has sent nothing", async () => { + vi.mocked(downtimes.get).mockResolvedValue({ + data: { ...DETAIL, notification_log: [] }, + } as any); + const wrapper = await mountDetail(); + expect(wrapper.find('[data-test="downtime-detail-notifications-empty"]').exists()).toBe(true); + wrapper.unmount(); + }); +}); diff --git a/web/src/components/alerts/downtimes/DowntimeDetail.spec.ts b/web/src/components/alerts/downtimes/DowntimeDetail.spec.ts new file mode 100644 index 0000000000..daba3f9b45 --- /dev/null +++ b/web/src/components/alerts/downtimes/DowntimeDetail.spec.ts @@ -0,0 +1,261 @@ +// Copyright 2026 OpenObserve Inc. +// +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// This program is distributed in the hope that it will be useful +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. +// +// You should have received a copy of the GNU Affero General Public License +// along with this program. If not, see . + +import { afterEach, beforeEach, describe, expect, it, vi } from "vitest"; +import { flushPromises, mount } from "@vue/test-utils"; +import { createMemoryHistory, createRouter } from "vue-router"; +import store from "@/test/unit/helpers/store"; +import DowntimeDetail from "./DowntimeDetail.vue"; +import downtimes from "@/services/downtimes"; +import type { DowntimeDetail as DowntimeDetailData } from "@/services/downtimes"; +import alerts from "@/services/alerts"; +import { queryClient } from "@/composables/query/queryClient"; +import OTimeCell from "@/lib/core/Table/cells/OTimeCell.vue"; + +vi.mock("@/services/downtimes", () => ({ + default: { + list: vi.fn(), + get: vi.fn(), + create: vi.fn(), + update: vi.fn(), + cancel: vi.fn(), + remove: vi.fn(), + move: vi.fn(), + preview: vi.fn(), + resources: vi.fn(), + }, +})); + +vi.mock("@/services/alerts", async (importOriginal) => { + const actual = await importOriginal<{ default: Record }>(); + return { ...actual, default: { ...actual.default, getHistory: vi.fn() } }; +}); + +vi.mock("@/services/common", async (importOriginal) => { + const actual = await importOriginal<{ default: Record }>(); + return { + ...actual, + default: { + ...actual.default, + list_Folders: vi.fn(() => + Promise.resolve({ data: { list: [{ folderId: "default", name: "default" }] } }), + ), + }, + }; +}); + +const NOW_MICROS = 1_770_000_000_000_000; +const HOUR_MICROS = 3600 * 1_000_000; + +const detail: DowntimeDetailData = { + id: "dt-1", + org: "default", + folder_id: "default", + name: "Kafka upgrade", + targets: [{ module: "alerts", folders: { kind: "all" } }], + schedule: { + repeat: "none", + starts_at: NOW_MICROS - HOUR_MICROS, + ends_at: NOW_MICROS + HOUR_MICROS, + timezone: "UTC", + duration_secs: 7200, + weekdays: [], + }, + show_banner: false, + created_by: "root@example.com", + created_at: NOW_MICROS - 2 * HOUR_MICROS, + updated_by: "root@example.com", + updated_at: NOW_MICROS - 2 * HOUR_MICROS, + status: "active", + current_window: { start: NOW_MICROS - HOUR_MICROS, end: NOW_MICROS + HOUR_MICROS }, + next_window: null, + matched_alerts: 2, + matched_anomalies: 0, + matched_synthetics: 0, + matched_slos: 0, + affected: { alerts: [], anomalies: [], synthetics: [], slos: [] }, +}; + +const mountDetail = async () => { + const router = createRouter({ + history: createMemoryHistory(), + routes: [ + { path: "/downtimes", name: "downtimes", component: { template: "
" } }, + { path: "/downtimes/:id", name: "downtimeDetail", component: DowntimeDetail }, + { path: "/downtimes/:id/edit", name: "editDowntime", component: { template: "
" } }, + ], + }); + await router.push({ + name: "downtimeDetail", + params: { id: "dt-1" }, + query: { org_identifier: "default", folder: "default" }, + }); + await router.isReady(); + const wrapper = mount(DowntimeDetail, { + global: { plugins: [store, router] }, + attachTo: document.body, + }); + await flushPromises(); + return { wrapper, router }; +}; + +describe("DowntimeDetail", () => { + beforeEach(() => { + queryClient.clear(); + vi.mocked(downtimes.get).mockReset(); + vi.mocked(alerts.getHistory).mockReset(); + vi.mocked(alerts.getHistory).mockResolvedValue({ data: { hits: [], total: 0 } } as never); + store.state.selectedOrganization = { + ...store.state.selectedOrganization, + identifier: "default", + }; + }); + + afterEach(() => { + document.body.innerHTML = ""; + }); + + it("renders the Suppressed tab from a 200 with two hits", async () => { + vi.mocked(downtimes.get).mockResolvedValue({ data: detail } as never); + vi.mocked(alerts.getHistory).mockResolvedValue({ + data: { + total: 2, + from: 0, + size: 500, + hits: [ + { timestamp: NOW_MICROS - 60_000_000, alert_name: "kafka-lag", status: "suppressed" }, + { timestamp: NOW_MICROS - 30_000_000, alert_name: "kafka-isr", status: "suppressed" }, + ], + }, + } as never); + + const { wrapper } = await mountDetail(); + const tab = wrapper.get('[data-test="downtime-detail-tab-suppressed"]'); + expect(tab.text()).toContain("2"); + await tab.trigger("mousedown", { button: 0 }); + await flushPromises(); + + const table = '[data-test="downtime-detail-suppressed-table"]'; + await vi.waitFor(() => expect(wrapper.get(table).text()).toContain("kafka-lag")); + expect(wrapper.get(table).text()).toContain("kafka-isr"); + expect(wrapper.get(table).attributes("data-test-loading")).not.toBe("true"); + expect(alerts.getHistory).toHaveBeenCalledWith( + "default", + expect.objectContaining({ downtime_id: "dt-1", status: "suppressed" }), + ); + wrapper.unmount(); + }); + + it("shows suppressed times in the app's zone, the zone every downtime view uses", async () => { + store.state.timezone = "Pacific/Chatham"; + vi.mocked(downtimes.get).mockResolvedValue({ data: detail } as never); + vi.mocked(alerts.getHistory).mockResolvedValue({ + data: { + total: 1, + from: 0, + size: 500, + hits: [ + { timestamp: NOW_MICROS - 60_000_000, alert_name: "kafka-lag", status: "suppressed" }, + ], + }, + } as never); + const { wrapper } = await mountDetail(); + await wrapper.get('[data-test="downtime-detail-tab-suppressed"]').trigger("mousedown", { + button: 0, + }); + await flushPromises(); + await vi.waitFor(() => expect(wrapper.findAllComponents(OTimeCell).length).toBeGreaterThan(0)); + const zones = wrapper.findAllComponents(OTimeCell).map((c) => c.props("timezone")); + expect(zones.every((z) => z === "Pacific/Chatham")).toBe(true); + store.state.timezone = "UTC"; + wrapper.unmount(); + }); + + it("notes an SLO whose window holds no slice start, and no other", async () => { + const slo = (id: string, applies?: boolean) => ({ + id, + name: `${id}-name`, + folder_id: "default", + ...(applies === undefined ? {} : { applies }), + }); + vi.mocked(downtimes.get).mockResolvedValue({ + data: { + ...detail, + targets: [{ module: "slos", folders: { kind: "all" } }], + matched_alerts: 0, + matched_slos: 3, + affected: { + ...detail.affected, + slos: [slo("hourly", false), slo("minutely", true), slo("unknown")], + }, + }, + } as never); + + const { wrapper } = await mountDetail(); + await wrapper + .get('[data-test="downtime-detail-tab-affected"]') + .trigger("mousedown", { button: 0 }); + await flushPromises(); + + const section = '[data-test="downtime-detail-affected-slos"]'; + await vi.waitFor(() => expect(wrapper.get(section).text()).toContain("hourly-name")); + const inert = (id: string) => + wrapper.find(`[data-test="downtime-detail-affected-slos-${id}-inert"]`); + expect(inert("hourly").text()).toContain("shorter than the SLO's slice"); + expect(inert("minutely").exists()).toBe(false); + expect(inert("unknown").exists()).toBe(false); + wrapper.unmount(); + }); + + it("reloads the status when the open window ends, without a refresh", async () => { + const endsSoon = Date.now() * 1000 + 300_000; + vi.mocked(downtimes.get) + .mockResolvedValueOnce({ + data: { ...detail, current_window: { start: NOW_MICROS, end: endsSoon } }, + } as never) + .mockResolvedValue({ + data: { ...detail, status: "ended", current_window: null }, + } as never); + + const { wrapper } = await mountDetail(); + expect(downtimes.get).toHaveBeenCalledTimes(1); + await vi.waitFor(() => expect(downtimes.get).toHaveBeenCalledTimes(2), { timeout: 5_000 }); + wrapper.unmount(); + }); + + it("shows a not-found state on a 404", async () => { + vi.mocked(downtimes.get).mockRejectedValue({ response: { status: 404 } }); + + const { wrapper } = await mountDetail(); + await vi.waitFor(() => + expect(wrapper.find('[data-test="downtime-detail-not-found"]').exists()).toBe(true), + ); + expect(wrapper.text()).toContain("Downtime not found"); + expect(wrapper.find('[data-test="downtime-detail-error"]').exists()).toBe(false); + wrapper.unmount(); + }); + + it("shows an error state on any other failure", async () => { + vi.mocked(downtimes.get).mockRejectedValue({ response: { status: 500 } }); + + const { wrapper } = await mountDetail(); + await vi.waitFor( + () => expect(wrapper.find('[data-test="downtime-detail-error"]').exists()).toBe(true), + { timeout: 10_000 }, + ); + expect(wrapper.find('[data-test="downtime-detail-not-found"]').exists()).toBe(false); + wrapper.unmount(); + }); +}); diff --git a/web/src/components/alerts/downtimes/DowntimeDetail.vue b/web/src/components/alerts/downtimes/DowntimeDetail.vue new file mode 100644 index 0000000000..2e66d64452 --- /dev/null +++ b/web/src/components/alerts/downtimes/DowntimeDetail.vue @@ -0,0 +1,777 @@ + + + + + diff --git a/web/src/components/alerts/downtimes/DowntimeList.actions.spec.ts b/web/src/components/alerts/downtimes/DowntimeList.actions.spec.ts new file mode 100644 index 0000000000..a53b34b909 --- /dev/null +++ b/web/src/components/alerts/downtimes/DowntimeList.actions.spec.ts @@ -0,0 +1,214 @@ +// Copyright 2026 OpenObserve Inc. + +import { describe, it, expect, vi, beforeEach } from "vitest"; +import { flushPromises, mount } from "@vue/test-utils"; +import { createMemoryHistory, createRouter } from "vue-router"; +import store from "@/test/unit/helpers/store"; +import DowntimeList from "./DowntimeList.vue"; +import downtimes, { type DowntimeListItem } from "@/services/downtimes"; +import { queryClient } from "@/composables/query/queryClient"; + +// The route guard opens this page only with downtimes on. +vi.mock("@/composables/downtimes/useDowntimesEnabled", async () => { + const { computed } = await import("vue"); + return { useDowntimesEnabled: () => computed(() => true) }; +}); + +vi.mock("@/services/downtimes", () => ({ + default: { + list: vi.fn(), + get: vi.fn(), + create: vi.fn(), + update: vi.fn(), + cancel: vi.fn(), + extend: vi.fn(), + remove: vi.fn(), + move: vi.fn(), + preview: vi.fn(), + resources: vi.fn(), + }, +})); + +vi.mock("@/utils/commons", async (importOriginal) => ({ + ...(await importOriginal>()), + getFoldersListByType: vi.fn(() => Promise.resolve([])), +})); + +vi.mock("@/services/common", async (importOriginal) => { + const actual = await importOriginal<{ default: Record }>(); + return { + ...actual, + default: { + ...actual.default, + list_Folders: vi.fn(() => Promise.resolve({ data: { list: [] } })), + }, + }; +}); + +const HOUR = 3_600_000_000; +const NOW = Date.now() * 1000; + +const row = ( + id: string, + status: DowntimeListItem["status"], + over: Partial = {}, +) => + ({ + id, + org: "default", + folder_id: "default", + name: `${id} window`, + targets: [{ module: "alerts", folders: { kind: "all" } }], + schedule: { + repeat: "none", + starts_at: NOW - HOUR, + ends_at: NOW + HOUR, + timezone: "UTC", + duration_secs: 7200, + weekdays: [], + }, + show_banner: false, + created_by: "lin@example.com", + created_at: 0, + updated_by: "lin@example.com", + updated_at: 0, + status, + current_window: status === "active" ? { start: NOW - HOUR, end: NOW + HOUR } : null, + next_window: null, + matched_alerts: 1, + matched_anomalies: 0, + matched_synthetics: 0, + matched_slos: 0, + ...over, + }) as DowntimeListItem; + +const ROWS = [ + row("ended", "ended"), + row("early", "ended_early", { cancelled_at: NOW - HOUR / 2 }), + row("cancelled", "cancelled", { cancelled_at: NOW - 2 * HOUR }), + row("scheduled", "scheduled", { next_window: { start: NOW + 2 * HOUR, end: NOW + 3 * HOUR } }), + row("active", "active"), +]; + +const mountList = async () => { + const router = createRouter({ + history: createMemoryHistory(), + routes: [ + { path: "/downtimes", name: "downtimes", component: DowntimeList }, + { path: "/downtimes/add", name: "addDowntime", component: { template: "
" } }, + { path: "/downtimes/:id/edit", name: "editDowntime", component: { template: "
" } }, + { path: "/downtimes/:id", name: "downtimeDetail", component: { template: "
" } }, + ], + }); + await router.push({ name: "downtimes", query: { org_identifier: "default", folder: "default" } }); + await router.isReady(); + const wrapper = mount(DowntimeList, { + global: { plugins: [store, router] }, + attachTo: document.body, + }); + await flushPromises(); + await new Promise((resolve) => setTimeout(resolve, 0)); + await flushPromises(); + return wrapper; +}; + +const cell = (wrapper: ReturnType, id: string) => + wrapper.get(`[data-test="downtime-list-${id}-actions"]`); + +const dialog = () => document.body.querySelector('[data-test="confirm-dialog"]'); + +describe("DowntimeList row actions", () => { + beforeEach(() => { + queryClient.clear(); + vi.mocked(downtimes.list).mockResolvedValue({ + data: { + items: ROWS, + total: ROWS.length, + counts: { active: 1, scheduled: 1, recurring: 0, ended: 1, cancelled: 1, ended_early: 1 }, + }, + } as any); + vi.mocked(downtimes.cancel) + .mockReset() + .mockResolvedValue({ data: {} } as any); + store.state.selectedOrganization = { + ...store.state.selectedOrganization, + identifier: "default", + }; + }); + + it("keeps five action slots on every row, disabling the ones that do not apply", async () => { + const wrapper = await mountList(); + for (const r of ROWS) { + const slots = cell(wrapper, r.id).element.children; + expect(slots.length, r.id).toBe(5); + } + expect( + cell(wrapper, "active").find('[data-test="downtime-list-active-extend"]').attributes(), + ).not.toHaveProperty("disabled"); + expect( + cell(wrapper, "ended").get('[data-test="downtime-list-ended-extend"]').attributes("disabled"), + ).toBeDefined(); + expect( + cell(wrapper, "ended") + .get('[data-test="downtime-list-ended-end-now"]') + .attributes("disabled"), + ).toBeDefined(); + expect( + cell(wrapper, "scheduled").find('[data-test="downtime-list-scheduled-cancel"]').exists(), + ).toBe(true); + wrapper.unmount(); + }); + + it("hides Edit on finished rows and keeps Duplicate", async () => { + const wrapper = await mountList(); + for (const id of ["ended", "early", "cancelled"]) { + expect(wrapper.find(`[data-test="downtime-list-${id}-edit"]`).exists(), id).toBe(false); + expect(wrapper.find(`[data-test="downtime-list-${id}-duplicate"]`).exists(), id).toBe(true); + } + for (const id of ["active", "scheduled"]) { + expect(wrapper.find(`[data-test="downtime-list-${id}-edit"]`).exists(), id).toBe(true); + } + wrapper.unmount(); + }); + + it("asks before ending a downtime now, and Keep running leaves it", async () => { + const wrapper = await mountList(); + await wrapper.get('[data-test="downtime-list-active-end-now"]').trigger("click"); + await flushPromises(); + expect(dialog()?.textContent).toContain("End this downtime now?"); + expect(dialog()?.textContent).toContain("Keep running"); + dialog()!.querySelector('[data-test="o-dialog-secondary-btn"]')!.click(); + await flushPromises(); + expect(downtimes.cancel).not.toHaveBeenCalled(); + + await wrapper.get('[data-test="downtime-list-active-end-now"]').trigger("click"); + await flushPromises(); + const ok = dialog()!.querySelector('[data-test="o-dialog-primary-btn"]')!; + expect(ok.textContent).toContain("End now"); + ok.click(); + await flushPromises(); + expect(downtimes.cancel).toHaveBeenCalledWith("default", "active", "default"); + wrapper.unmount(); + }); + + it("shows the Ended early chip and its own stat tile", async () => { + const wrapper = await mountList(); + expect( + wrapper.get('[data-test="downtime-list-early-name"]').element.closest("tr")?.textContent, + ).toContain("Ended early"); + const tile = wrapper.get('[data-test="downtime-summary-ended-early"]'); + expect(tile.text()).toContain("1"); + wrapper.unmount(); + }); + + it("orders rows by status, then start, then id", async () => { + const wrapper = await mountList(); + const order = [ + ...wrapper.element.querySelectorAll('[data-test^="downtime-list-"][data-test$="-actions"]'), + ].map((el) => + el.getAttribute("data-test")!.replace("downtime-list-", "").replace("-actions", ""), + ); + expect(order).toEqual(["active", "scheduled", "ended", "early", "cancelled"]); + wrapper.unmount(); + }); +}); diff --git a/web/src/components/alerts/downtimes/DowntimeList.spec.ts b/web/src/components/alerts/downtimes/DowntimeList.spec.ts new file mode 100644 index 0000000000..4ddd0607b9 --- /dev/null +++ b/web/src/components/alerts/downtimes/DowntimeList.spec.ts @@ -0,0 +1,314 @@ +// Copyright 2026 OpenObserve Inc. +// +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// This program is distributed in the hope that it will be useful +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. +// +// You should have received a copy of the GNU Affero General Public License +// along with this program. If not, see . + +import { afterEach, beforeEach, describe, expect, it, vi } from "vitest"; +import { flushPromises, mount } from "@vue/test-utils"; +import { createMemoryHistory, createRouter } from "vue-router"; +import store from "@/test/unit/helpers/store"; +import DowntimeList from "./DowntimeList.vue"; +import downtimes from "@/services/downtimes"; +import type { DowntimeListItem, DowntimeListResponse } from "@/services/downtimes"; +import common from "@/services/common"; +import { queryClient } from "@/composables/query/queryClient"; +import OTimeCell from "@/lib/core/Table/cells/OTimeCell.vue"; + +// The route guard opens this page only with downtimes on. +vi.mock("@/composables/downtimes/useDowntimesEnabled", async () => { + const { computed } = await import("vue"); + return { useDowntimesEnabled: () => computed(() => true) }; +}); + +vi.mock("@/services/downtimes", () => ({ + default: { + list: vi.fn(), + get: vi.fn(), + create: vi.fn(), + update: vi.fn(), + cancel: vi.fn(), + remove: vi.fn(), + move: vi.fn(), + preview: vi.fn(), + resources: vi.fn(), + }, +})); + +vi.mock("@/utils/commons", async (importOriginal) => ({ + ...(await importOriginal>()), + getFoldersListByType: vi.fn(() => Promise.resolve([])), +})); + +vi.mock("@/services/common", async (importOriginal) => { + const actual = await importOriginal<{ default: Record }>(); + return { + ...actual, + default: { + ...actual.default, + list_Folders: vi.fn(() => + Promise.resolve({ data: { list: [{ folderId: "default", name: "default" }] } }), + ), + }, + }; +}); + +const NOW_MS = 1_770_000_000_000; +const SECOND_MICROS = 1_000_000; +const NOW_MICROS = NOW_MS * 1000; + +const row = (overrides: Partial = {}): DowntimeListItem => ({ + id: "dt-1", + org: "default", + folder_id: "default", + name: "Kafka upgrade", + targets: [{ module: "alerts", folders: { kind: "all" } }], + schedule: { + repeat: "none", + starts_at: NOW_MICROS + 2 * SECOND_MICROS, + ends_at: NOW_MICROS + 3600 * SECOND_MICROS, + timezone: "UTC", + duration_secs: 3600, + weekdays: [], + }, + show_banner: false, + created_by: "root@example.com", + created_at: NOW_MICROS, + updated_by: "root@example.com", + updated_at: NOW_MICROS, + status: "scheduled", + current_window: null, + next_window: { start: NOW_MICROS + 2 * SECOND_MICROS, end: NOW_MICROS + 3600 * SECOND_MICROS }, + matched_alerts: 3, + matched_anomalies: 0, + matched_synthetics: 0, + matched_slos: 0, + ...overrides, +}); + +const page = (items: DowntimeListItem[]): { data: DowntimeListResponse } => ({ + data: { + items, + total: items.length, + counts: { active: 0, scheduled: 0, recurring: 0, ended: 0, cancelled: 0 }, + }, +}); + +const mountList = async (query: Record = {}) => { + const router = createRouter({ + history: createMemoryHistory(), + routes: [ + { path: "/downtimes", name: "downtimes", component: DowntimeList }, + { path: "/downtimes/add", name: "addDowntime", component: { template: "
" } }, + { path: "/downtimes/:id", name: "downtimeDetail", component: { template: "
" } }, + { path: "/downtimes/:id/edit", name: "editDowntime", component: { template: "
" } }, + ], + }); + await router.push({ + name: "downtimes", + query: { org_identifier: "default", folder: "default", ...query }, + }); + await router.isReady(); + const wrapper = mount(DowntimeList, { + global: { + plugins: [store, router], + stubs: { FolderList: true, MoveAcrossFolders: true }, + }, + attachTo: document.body, + }); + await flushPromises(); + return Object.assign(wrapper, { router }); +}; + +describe("DowntimeList", () => { + beforeEach(() => { + queryClient.clear(); + vi.mocked(downtimes.list).mockReset(); + vi.mocked(common.list_Folders).mockClear(); + store.state.selectedOrganization = { + ...store.state.selectedOrganization, + identifier: "default", + }; + }); + + afterEach(() => { + vi.useRealTimers(); + document.body.innerHTML = ""; + }); + + it("flips a row to Active within a second of its start, without a reload", async () => { + vi.useFakeTimers({ toFake: ["setTimeout", "clearTimeout", "Date"] }); + vi.setSystemTime(NOW_MS); + const started = row({ + status: "active", + current_window: { + start: NOW_MICROS + 2 * SECOND_MICROS, + end: NOW_MICROS + 3600 * SECOND_MICROS, + }, + next_window: null, + }); + vi.mocked(downtimes.list) + .mockResolvedValueOnce(page([row()]) as never) + .mockResolvedValue(page([started]) as never); + + const wrapper = await mountList(); + const activeCount = () => wrapper.get('[data-test="downtime-summary-active"]').text(); + expect(activeCount()).toContain("0"); + expect(downtimes.list).toHaveBeenCalledTimes(1); + + // Just before the start (plus the one second of slack) nothing has refetched. + await vi.advanceTimersByTimeAsync(2_900); + expect(downtimes.list).toHaveBeenCalledTimes(1); + + await vi.advanceTimersByTimeAsync(200); + await flushPromises(); + expect(downtimes.list).toHaveBeenCalledTimes(2); + expect(activeCount()).toContain("1"); + wrapper.unmount(); + }); + + it("shows the When column in the app's zone, the zone every downtime view uses", async () => { + store.state.timezone = "Pacific/Chatham"; + vi.mocked(downtimes.list).mockResolvedValue(page([row()]) as never); + const wrapper = await mountList(); + await vi.waitFor(() => expect(wrapper.findAllComponents(OTimeCell).length).toBeGreaterThan(0)); + const cells = wrapper.findAllComponents(OTimeCell); + expect(cells.every((c) => c.props("timezone") === "Pacific/Chatham")).toBe(true); + store.state.timezone = "UTC"; + wrapper.unmount(); + }); + + const cutList = () => + vi.mocked(downtimes.list).mockResolvedValue({ + data: { + items: [ + row({ id: "a1", folder_id: "team-a", status: "active" }), + row({ id: "a2", folder_id: "team-a", status: "active" }), + row({ id: "a3", folder_id: "team-a", status: "active" }), + row({ id: "d1", folder_id: "default", status: "active" }), + ], + total: 600, + counts: { + active: 120, + scheduled: 300, + recurring: 50, + ended: 100, + cancelled: 80, + ended_early: 0, + }, + }, + } as never); + + it("says when the list is cut and keeps a folder's tiles on the rows it shows", async () => { + cutList(); + const wrapper = await mountList({ folder: "team-a" }); + expect(wrapper.get('[data-test="downtime-list-truncated"]').text()).toContain( + "Showing the first 4 of 600 downtimes", + ); + expect(wrapper.get('[data-test="downtime-summary-active"]').text()).toContain("3"); + expect(wrapper.get('[data-test="downtime-summary-active"]').text()).not.toContain("120"); + expect(wrapper.get('[data-test="downtime-summary-total"]').text()).toContain("3"); + expect(wrapper.get('[data-test="downtime-summary-total"]').text()).not.toContain("600"); + wrapper.unmount(); + }); + + it("takes the tiles from the server counts when a cut list is viewed across the org", async () => { + cutList(); + const wrapper = await mountList({ scope: "all", status: "active" }); + expect(wrapper.get('[data-test="downtime-summary-active"]').text()).toContain("120"); + expect(wrapper.get('[data-test="downtime-summary-total"]').text()).toContain("600"); + wrapper.unmount(); + }); + + it("shows no cut notice when one page holds the org", async () => { + vi.mocked(downtimes.list).mockResolvedValue(page([row()]) as never); + const wrapper = await mountList(); + expect(wrapper.find('[data-test="downtime-list-truncated"]').exists()).toBe(false); + wrapper.unmount(); + }); + + it("shows a load error instead of the first-run empty state", async () => { + vi.mocked(downtimes.list).mockRejectedValue({ response: { status: 500 } }); + + const wrapper = await mountList(); + await vi.waitFor(() => + expect(wrapper.find('[data-test="downtime-list-error"]').exists()).toBe(true), + ); + expect(wrapper.find('[data-test="downtime-list-empty"]').exists()).toBe(false); + wrapper.unmount(); + }); + + it("keeps the targets on one row and folds the rest behind +N", async () => { + vi.mocked(downtimes.list).mockResolvedValue( + page([ + row({ + targets: [ + { module: "alerts", folders: { kind: "all" } }, + { module: "anomaly_detections", folders: { kind: "all" } }, + { module: "synthetics", folders: { kind: "all" } }, + { module: "slos", folders: { kind: "all" } }, + ], + }), + ]) as never, + ); + + const wrapper = await mountList(); + const more = '[data-test="downtime-list-dt-1-targets-more"]'; + await vi.waitFor(() => expect(wrapper.find(more).exists()).toBe(true)); + expect(wrapper.get(more).text()).toContain("+2"); + wrapper.unmount(); + }); + + it("keeps the user's folder narrowing when the status filter changes after a banner link", async () => { + vi.mocked(downtimes.list).mockResolvedValue( + page([ + row({ id: "dt-1", status: "active" }), + row({ id: "dt-2", folder_id: "ops", status: "scheduled" }), + ]) as never, + ); + const wrapper = await mountList({ status: "active", scope: "all" }); + const total = () => wrapper.get('[data-test="downtime-summary-total"]').text(); + await vi.waitFor(() => expect(total()).toContain("2")); + + await wrapper.get('[data-test="downtime-list-scope-this"]').trigger("click"); + await flushPromises(); + expect(total()).toContain("1"); + expect(wrapper.router.currentRoute.value.query.scope).toBeUndefined(); + + await wrapper.get('[data-test="downtime-summary-scheduled"]').trigger("click"); + await flushPromises(); + expect(wrapper.router.currentRoute.value.query.status).toBe("scheduled"); + expect(total()).toContain("1"); + wrapper.unmount(); + }); + + it("filters to the ended-early rows when the URL changes to that status after mount", async () => { + vi.mocked(downtimes.list).mockResolvedValue( + page([ + row({ id: "dt-1", status: "active" }), + row({ id: "dt-2", status: "ended_early" }), + ]) as never, + ); + const wrapper = await mountList(); + const shown = (id: string) => wrapper.find(`[data-test="downtime-list-${id}-name"]`).exists(); + await vi.waitFor(() => expect(shown("dt-1")).toBe(true)); + expect(shown("dt-2")).toBe(true); + + await wrapper.router.push({ + query: { ...wrapper.router.currentRoute.value.query, status: "ended_early" }, + }); + await flushPromises(); + expect(shown("dt-1")).toBe(false); + expect(shown("dt-2")).toBe(true); + wrapper.unmount(); + }); +}); diff --git a/web/src/components/alerts/downtimes/DowntimeList.vue b/web/src/components/alerts/downtimes/DowntimeList.vue new file mode 100644 index 0000000000..91c2761cb6 --- /dev/null +++ b/web/src/components/alerts/downtimes/DowntimeList.vue @@ -0,0 +1,1102 @@ + + + + + diff --git a/web/src/components/alerts/downtimes/DowntimeNotifySection.vue b/web/src/components/alerts/downtimes/DowntimeNotifySection.vue new file mode 100644 index 0000000000..d0b8619437 --- /dev/null +++ b/web/src/components/alerts/downtimes/DowntimeNotifySection.vue @@ -0,0 +1,138 @@ + + + + + diff --git a/web/src/components/alerts/downtimes/DowntimePreviewPane.vue b/web/src/components/alerts/downtimes/DowntimePreviewPane.vue new file mode 100644 index 0000000000..2d37c65be8 --- /dev/null +++ b/web/src/components/alerts/downtimes/DowntimePreviewPane.vue @@ -0,0 +1,160 @@ + + + + + diff --git a/web/src/components/alerts/downtimes/DowntimeResourcePicker.spec.ts b/web/src/components/alerts/downtimes/DowntimeResourcePicker.spec.ts new file mode 100644 index 0000000000..a7298c3006 --- /dev/null +++ b/web/src/components/alerts/downtimes/DowntimeResourcePicker.spec.ts @@ -0,0 +1,152 @@ +// Copyright 2026 OpenObserve Inc. + +import { describe, it, expect, vi, beforeEach } from "vitest"; +import { flushPromises, mount } from "@vue/test-utils"; +import store from "@/test/unit/helpers/store"; +import downtimes from "@/services/downtimes"; +import serviceStreams from "@/services/service_streams"; +import { queryClient } from "@/composables/query/queryClient"; +import type { DimensionCondition } from "@/services/downtimes"; +import DowntimeResourcePicker from "./DowntimeResourcePicker.vue"; +import OSelect from "@/lib/forms/Select/OSelect.vue"; + +vi.mock("@/services/downtimes", () => ({ default: { values: vi.fn(), resources: vi.fn() } })); + +vi.mock("@/services/service_streams", async (importOriginal) => { + const actual = await importOriginal<{ default: Record }>(); + return { + ...actual, + default: { + ...actual.default, + getSemanticGroups: vi.fn(), + getDimensionAnalytics: vi.fn(), + }, + }; +}); + +const group = (id: string) => ({ id, display: id, fields: [id] }); + +const CONDITION: DimensionCondition = { + type: "pair", + key: "service", + operator: "=", + value: "payments-api", +}; + +const mountPicker = async () => { + const wrapper = mount(DowntimeResourcePicker, { + props: { condition: CONDITION, folder: "planned" }, + global: { plugins: [store] }, + }); + await flushPromises(); + await flushPromises(); + return wrapper; +}; + +describe("DowntimeResourcePicker", () => { + beforeEach(() => { + queryClient.clear(); + store.state.selectedOrganization = { + ...store.state.selectedOrganization, + identifier: "acme", + }; + vi.mocked(downtimes.values).mockReset(); + vi.mocked(downtimes.resources).mockReset(); + vi.mocked(serviceStreams.getDimensionAnalytics).mockReset(); + vi.mocked(downtimes.values).mockResolvedValue({ + data: { + values: [ + { value: "node-1", source: "inventory", items: 1, last_seen: 1791500000000000 }, + { value: "node-2", source: "registry", items: 0 }, + ], + partial: false, + }, + } as any); + }); + + it("refines by host by default and lists its values with their item counts", async () => { + vi.mocked(serviceStreams.getSemanticGroups).mockResolvedValue({ + data: [group("aws-ecs-cluster"), group("service"), group("host")], + } as any); + const wrapper = await mountPicker(); + expect(downtimes.values).toHaveBeenCalledWith( + "acme", + { + key: "host", + prefix: "", + condition: { + type: "group", + op: "and", + items: [{ type: "pair", key: "service", operator: "=", value: "payments-api" }], + }, + }, + "planned", + ); + expect(serviceStreams.getDimensionAnalytics).not.toHaveBeenCalled(); + const table = wrapper.get('[data-test="downtime-resource-picker-table"]'); + expect(table.text()).toContain("node-1"); + expect(table.text()).toContain("node-2"); + expect(table.text()).toContain("Items"); + expect(downtimes.resources).not.toHaveBeenCalled(); + expect(wrapper.find('[data-test="downtime-resource-picker-load"]').exists()).toBe(true); + }); + + it("falls back to the first priority dimension when the org has no host group", async () => { + vi.mocked(serviceStreams.getSemanticGroups).mockResolvedValue({ + data: [group("aws-ecs-cluster"), group("service"), group("k8s-pod")], + } as any); + vi.mocked(serviceStreams.getDimensionAnalytics).mockResolvedValue({ + data: { recommended_priority_dimensions: ["unknown", "k8s-pod", "service"] }, + } as any); + await mountPicker(); + const keys = vi.mocked(downtimes.values).mock.calls.map((call) => call[1].key); + expect(keys[keys.length - 1]).toBe("k8s-pod"); + }); + + it("keeps Load resources for the full list with streams", async () => { + vi.mocked(serviceStreams.getSemanticGroups).mockResolvedValue({ + data: [group("service"), group("host")], + } as any); + vi.mocked(downtimes.resources).mockResolvedValue({ + data: { + dimension: "host", + values: [{ value: "node-9", last_seen: 1, streams: ["logs/app_logs"] }], + total: 1, + source: "search", + }, + } as any); + const wrapper = await mountPicker(); + await wrapper.get('[data-test="downtime-resource-picker-load"]').trigger("click"); + await flushPromises(); + expect(downtimes.resources).toHaveBeenCalledWith( + "acme", + { condition: CONDITION, refine_by: "host" }, + "planned", + ); + const table = wrapper.get('[data-test="downtime-resource-picker-table"]'); + expect(table.text()).toContain("node-9"); + expect(table.text()).toContain("logs/app_logs"); + }); + it("never offers host rows under service while the service values are still loading", async () => { + vi.mocked(serviceStreams.getSemanticGroups).mockResolvedValue({ + data: [group("service"), group("host")], + } as any); + const wrapper = await mountPicker(); + expect(wrapper.get('[data-test="downtime-resource-picker-table"]').text()).toContain("node-1"); + let answerService: (value: unknown) => void = () => {}; + vi.mocked(downtimes.values).mockImplementation( + () => new Promise((resolve) => (answerService = resolve)) as any, + ); + wrapper.findComponent(OSelect).vm.$emit("update:modelValue", "service"); + await flushPromises(); + expect(wrapper.find('[data-test="downtime-resource-picker-table"]').exists()).toBe(false); + expect(wrapper.find('[data-test="downtime-resource-picker-apply"]').exists()).toBe(false); + answerService({ + data: { values: [{ value: "payments-api", source: "inventory", items: 3 }], partial: false }, + }); + await flushPromises(); + const table = wrapper.get('[data-test="downtime-resource-picker-table"]'); + expect(table.text()).toContain("payments-api"); + expect(table.text()).not.toContain("node-1"); + }); +}); diff --git a/web/src/components/alerts/downtimes/DowntimeResourcePicker.vue b/web/src/components/alerts/downtimes/DowntimeResourcePicker.vue new file mode 100644 index 0000000000..a0b19f8120 --- /dev/null +++ b/web/src/components/alerts/downtimes/DowntimeResourcePicker.vue @@ -0,0 +1,316 @@ + + + + + diff --git a/web/src/components/alerts/downtimes/DowntimeScheduleBand.vue b/web/src/components/alerts/downtimes/DowntimeScheduleBand.vue new file mode 100644 index 0000000000..d8e72869d1 --- /dev/null +++ b/web/src/components/alerts/downtimes/DowntimeScheduleBand.vue @@ -0,0 +1,128 @@ + + + + + diff --git a/web/src/components/alerts/downtimes/DowntimeScheduleFields.spec.ts b/web/src/components/alerts/downtimes/DowntimeScheduleFields.spec.ts new file mode 100644 index 0000000000..30f65304c5 --- /dev/null +++ b/web/src/components/alerts/downtimes/DowntimeScheduleFields.spec.ts @@ -0,0 +1,48 @@ +// Copyright 2026 OpenObserve Inc. +// +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// This program is distributed in the hope that it will be useful +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. +// +// You should have received a copy of the GNU Affero General Public License +// along with this program. If not, see . + +import { describe, it, expect } from "vitest"; +import { flushPromises, mount } from "@vue/test-utils"; +import { defineComponent } from "vue"; +import store from "@/test/unit/helpers/store"; +import OForm from "@/lib/forms/Form/OForm.vue"; +import { defaultDowntimeValues } from "@/utils/downtimes/downtimeForm"; +import DowntimeScheduleFields from "./DowntimeScheduleFields.vue"; + +const mountFields = async (timezone: string) => { + const Host = defineComponent({ + components: { OForm, DowntimeScheduleFields }, + setup: () => ({ defaultValues: defaultDowntimeValues(Date.now(), timezone) }), + template: ``, + }); + const wrapper = mount(Host, { global: { plugins: [store] } }); + await flushPromises(); + return wrapper; +}; + +describe("DowntimeScheduleFields", () => { + it("shows the next window once for a stored legacy UTC alias", async () => { + const wrapper = await mountFields("Etc/UTC"); + const text = wrapper.text(); + expect(text).toMatch(/Next window: [^(]*\. A window can cross midnight\./); + wrapper.unmount(); + }); + + it("adds the UTC time for any other zone", async () => { + const wrapper = await mountFields("Asia/Tokyo"); + expect(wrapper.text()).toMatch(/Next window: .*\(.*\)\. A window can cross midnight\./); + wrapper.unmount(); + }); +}); diff --git a/web/src/components/alerts/downtimes/DowntimeScheduleFields.vue b/web/src/components/alerts/downtimes/DowntimeScheduleFields.vue new file mode 100644 index 0000000000..2d0fc47d22 --- /dev/null +++ b/web/src/components/alerts/downtimes/DowntimeScheduleFields.vue @@ -0,0 +1,188 @@ + + + + + diff --git a/web/src/components/alerts/downtimes/DowntimeSummary.vue b/web/src/components/alerts/downtimes/DowntimeSummary.vue new file mode 100644 index 0000000000..34a0f9b6df --- /dev/null +++ b/web/src/components/alerts/downtimes/DowntimeSummary.vue @@ -0,0 +1,102 @@ + + + + + diff --git a/web/src/components/alerts/downtimes/DowntimeTargetCard.spec.ts b/web/src/components/alerts/downtimes/DowntimeTargetCard.spec.ts new file mode 100644 index 0000000000..d1d01b7eff --- /dev/null +++ b/web/src/components/alerts/downtimes/DowntimeTargetCard.spec.ts @@ -0,0 +1,128 @@ +// Copyright 2026 OpenObserve Inc. + +import { describe, it, expect, vi } from "vitest"; +import { defineComponent, h, ref } from "vue"; +import { flushPromises, mount } from "@vue/test-utils"; +import OForm from "@/lib/forms/Form/OForm.vue"; +import { useOForm } from "@/lib/forms/Form/useOForm"; +import { ALL_FOLDERS, defaultDowntimeValues } from "@/utils/downtimes/downtimeForm"; +import store from "@/test/unit/helpers/store"; +import common from "@/services/common"; +import DowntimeTargetCard from "./DowntimeTargetCard.vue"; + +const toast = vi.fn(); +vi.mock("@/lib/feedback/Toast/useToast", () => ({ useToast: () => ({ toast }) })); + +vi.mock("@/composables/downtimes/useDowntimeItems", () => ({ + useDowntimeItems: () => ({ + items: ref([ + { id: "a1", name: "payments-db-replica-lag", folderId: "payments", tags: [] }, + { id: "a2", name: "ops-disk", folderId: "ops", tags: [] }, + ]), + query: { isPending: ref(false) }, + }), +})); + +vi.mock("@/services/common", async (importOriginal) => { + const actual = await importOriginal<{ default: Record }>(); + return { + ...actual, + default: { + ...actual.default, + list_Folders: vi.fn(() => Promise.resolve({ data: { list: [] } })), + }, + }; +}); + +const mountCard = (module: "alerts" | "synthetics" | "slos" = "alerts", idsOpen = true) => { + let form!: ReturnType>; + const Host = defineComponent({ + setup() { + const values = defaultDowntimeValues(Date.parse("2026-09-17T12:00:00Z"), "UTC"); + values.targets.alerts.ids_open = idsOpen; + values.targets.alerts.ids = idsOpen ? ["a1", "a2"] : []; + form = useOForm({ defaultValues: values }); + return () => h(OForm, { form }, () => h(DowntimeTargetCard, { module })); + }, + }); + const wrapper = mount(Host, { global: { plugins: [store] } }); + return { wrapper, form: () => form }; +}; + +describe("DowntimeTargetCard", () => { + it("clears All folders when a folder is picked, and drops items outside it", async () => { + const { wrapper, form } = mountCard(); + form().setFieldValue("targets.alerts.folders", [ALL_FOLDERS, "payments"]); + await flushPromises(); + expect(form().state.values.targets.alerts.folders).toEqual(["payments"]); + expect(form().state.values.targets.alerts.ids).toEqual(["a1"]); + expect(toast).toHaveBeenCalledWith( + expect.objectContaining({ + variant: "info", + message: expect.stringContaining("1 chosen item"), + }), + ); + wrapper.unmount(); + }); + + it("clears the chosen folders when All folders is picked", async () => { + const { wrapper, form } = mountCard(); + form().setFieldValue("targets.alerts.folders", ["payments"]); + await flushPromises(); + form().setFieldValue("targets.alerts.folders", ["payments", ALL_FOLDERS]); + await flushPromises(); + expect(form().state.values.targets.alerts.folders).toEqual([ALL_FOLDERS]); + wrapper.unmount(); + }); + + it("keeps the folder picker usable when the module's folders load", async () => { + const { wrapper } = mountCard("alerts", false); + await flushPromises(); + const picker = wrapper.find('[data-test="downtime-target-alerts-folders"]'); + expect(picker.find("[disabled]").exists()).toBe(false); + expect(wrapper.text()).not.toContain("You don't have access to alert folders."); + expect(wrapper.find('[data-test="downtime-target-alerts-add-items"]').exists()).toBe(true); + wrapper.unmount(); + }); + + it("offers the incident mode on the alerts card only, muted by default", async () => { + const { wrapper, form } = mountCard("alerts", false); + await flushPromises(); + const muted = wrapper.find('[data-test="downtime-target-alerts-incident-mode-muted"]'); + const none = wrapper.find('[data-test="downtime-target-alerts-incident-mode-none"]'); + expect(muted.attributes("data-state")).toBe("checked"); + expect(none.attributes("data-state")).toBe("unchecked"); + expect(wrapper.text()).toContain("During the window, incidents are"); + expect(wrapper.text()).toContain("Created and muted (default)"); + expect(wrapper.text()).toContain("Not created"); + await none.trigger("click"); + await flushPromises(); + expect(form().state.values.targets.alerts.incident_mode).toBe("none"); + wrapper.unmount(); + + for (const module of ["synthetics", "slos"] as const) { + const other = mountCard(module, false); + await flushPromises(); + expect( + other.wrapper.find('[data-test="downtime-target-alerts-incident-mode-block"]').exists(), + ).toBe(false); + other.wrapper.unmount(); + } + }); + + it("disables the folder picker with a hint when the module's folders answer 403", async () => { + vi.mocked(common.list_Folders).mockRejectedValueOnce({ response: { status: 403 } }); + const { wrapper, form } = mountCard("synthetics", false); + await flushPromises(); + const picker = wrapper.find('[data-test="downtime-target-synthetics-folders"]'); + expect(picker.find("[disabled]").exists()).toBe(true); + expect(wrapper.text()).toContain("You don't have access to synthetics folders."); + expect(wrapper.find('[data-test="downtime-target-synthetics-add-items"]').exists()).toBe(false); + expect(wrapper.find('[data-test="downtime-target-synthetics-add-tags"]').exists()).toBe(false); + expect(form().state.values.targets.synthetics.folders).toEqual([ALL_FOLDERS]); + expect(common.list_Folders).toHaveBeenCalledWith(expect.any(String), "synthetics", { + skipAccessToast: true, + }); + wrapper.unmount(); + }); +}); diff --git a/web/src/components/alerts/downtimes/DowntimeTargetCard.vue b/web/src/components/alerts/downtimes/DowntimeTargetCard.vue new file mode 100644 index 0000000000..a6f111d996 --- /dev/null +++ b/web/src/components/alerts/downtimes/DowntimeTargetCard.vue @@ -0,0 +1,355 @@ + + + + + diff --git a/web/src/components/alerts/downtimes/DowntimeTargetsCell.vue b/web/src/components/alerts/downtimes/DowntimeTargetsCell.vue new file mode 100644 index 0000000000..924a1b57e2 --- /dev/null +++ b/web/src/components/alerts/downtimes/DowntimeTargetsCell.vue @@ -0,0 +1,111 @@ + + + + + diff --git a/web/src/components/alerts/downtimes/ExtendDowntimeDialog.schema.ts b/web/src/components/alerts/downtimes/ExtendDowntimeDialog.schema.ts new file mode 100644 index 0000000000..76d66eb6ec --- /dev/null +++ b/web/src/components/alerts/downtimes/ExtendDowntimeDialog.schema.ts @@ -0,0 +1,64 @@ +// Copyright 2026 OpenObserve Inc. +// +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// This program is distributed in the hope that it will be useful +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. +// +// You should have received a copy of the GNU Affero General Public License +// along with this program. If not, see . + +import { z } from "zod"; +import type { TranslateFn } from "@/types/i18n"; +import { + MAX_DURATION_SECS, + MIN_DURATION_SECS, + localToUtcMicros, + utcMicrosToLocal, +} from "@/utils/downtimes/schedule"; + +export interface ExtendForm extends Record { + end_date: string; + end_time: string; +} + +/** The bounds an "Until…" end is checked against, in microseconds UTC. */ +export interface ExtendBounds { + /** The current end of the window. */ + currentEnd: number; + /** Where the extended window starts, so its length stays within the 7-day limit. */ + start: number; +} + +export const extendUntil = (v: ExtendForm, tz: string): number | null => + localToUtcMicros(v.end_date, v.end_time, tz); + +export const makeExtendSchema = (t: TranslateFn, tz: string, bounds: () => ExtendBounds | null) => + z.object({ end_date: z.string(), end_time: z.string() }).superRefine((raw, ctx) => { + const b = bounds(); + if (!b) return; + const until = extendUntil(raw as ExtendForm, tz); + const secs = until === null ? null : (until - b.start) / 1_000_000; + const message = + until === null + ? t("alerts.downtimes.validation.endRequired") + : until <= b.currentEnd + ? t("alerts.downtimes.extend.endAfterCurrent") + : secs !== null && secs < MIN_DURATION_SECS + ? t("alerts.downtimes.validation.windowTooShort") + : secs !== null && secs > MAX_DURATION_SECS + ? t("alerts.downtimes.validation.windowTooLong") + : null; + if (message) ctx.addIssue({ code: "custom", path: ["end_date"], message }); + }); + +/** An hour past the current end, in the viewer's zone. */ +export const extendDefaults = (currentEnd: number, tz: string): ExtendForm => { + const end = utcMicrosToLocal(currentEnd + 3_600_000_000, tz); + return { end_date: end.date, end_time: end.time }; +}; diff --git a/web/src/components/alerts/downtimes/ExtendDowntimeDialog.spec.ts b/web/src/components/alerts/downtimes/ExtendDowntimeDialog.spec.ts new file mode 100644 index 0000000000..6ea02a63d6 --- /dev/null +++ b/web/src/components/alerts/downtimes/ExtendDowntimeDialog.spec.ts @@ -0,0 +1,158 @@ +// Copyright 2026 OpenObserve Inc. + +import { describe, it, expect, vi, beforeEach } from "vitest"; +import { flushPromises, mount } from "@vue/test-utils"; +import { createMemoryHistory, createRouter } from "vue-router"; +import { gt } from "@/types/i18n"; +import store from "@/test/unit/helpers/store"; +import downtimes, { type DowntimeListItem } from "@/services/downtimes"; +import { queryClient } from "@/composables/query/queryClient"; +import { dismissAll, toastRecords } from "@/lib/feedback/Toast/useToast"; +import ExtendDowntimeDialog from "./ExtendDowntimeDialog.vue"; +import { makeExtendSchema, type ExtendForm } from "./ExtendDowntimeDialog.schema"; + +vi.mock("@/services/downtimes", () => ({ default: { extend: vi.fn() } })); + +const HOUR = 3_600_000_000; +const START = Date.parse("2026-09-17T14:00:00Z") * 1000; + +const row = (repeat: "none" | "daily"): DowntimeListItem => + ({ + id: "d1", + org: "default", + folder_id: "planned", + name: "Nightly deploy", + targets: [{ module: "alerts", folders: { kind: "all" } }], + schedule: { + repeat, + starts_at: START, + ends_at: repeat === "none" ? START + 2 * HOUR : null, + timezone: "UTC", + start_time_local: repeat === "none" ? null : "14:00", + duration_secs: 7200, + weekdays: [], + }, + show_banner: false, + created_by: "lin", + created_at: 0, + updated_by: "lin", + updated_at: 0, + status: "active", + current_window: { start: START, end: START + 2 * HOUR }, + next_window: null, + matched_alerts: 1, + matched_anomalies: 0, + matched_synthetics: 0, + matched_slos: 0, + }) as DowntimeListItem; + +const mountDialog = async (downtime: DowntimeListItem) => { + const router = createRouter({ + history: createMemoryHistory(), + routes: [{ path: "/", name: "downtimes", component: { template: "
" } }], + }); + await router.push("/"); + const wrapper = mount(ExtendDowntimeDialog, { + props: { open: false, downtime }, + global: { plugins: [store, router] }, + attachTo: document.body, + }); + await wrapper.setProps({ open: true }); + await flushPromises(); + return wrapper; +}; + +const dialog = () => + document.body.querySelector('[data-test="extend-downtime-dialog"]'); + +describe("ExtendDowntimeDialog", () => { + beforeEach(() => { + queryClient.clear(); + vi.mocked(downtimes.extend).mockReset(); + store.state.selectedOrganization = { + ...store.state.selectedOrganization, + identifier: "default", + }; + }); + + it("names the follow-up downtime a recurring extension creates", async () => { + const wrapper = await mountDialog(row("daily")); + const note = document.body.querySelector('[data-test="extend-downtime-follow-up"]'); + expect(note?.textContent).toContain("Nightly deploy (extended)"); + wrapper.unmount(); + }); + + it("says nothing about a follow-up for a one-time downtime", async () => { + const wrapper = await mountDialog(row("none")); + expect(dialog()).not.toBeNull(); + expect(document.body.querySelector('[data-test="extend-downtime-follow-up"]')).toBeNull(); + wrapper.unmount(); + }); + + it("sends the default end, an hour past the current one, as until", async () => { + vi.mocked(downtimes.extend).mockResolvedValue({ + data: { ...row("none"), schedule: { ...row("none").schedule, ends_at: START + 3 * HOUR } }, + } as any); + const wrapper = await mountDialog(row("none")); + dialog()!.querySelector('[data-test="o-dialog-primary-btn"]')!.click(); + await flushPromises(); + await new Promise((resolve) => setTimeout(resolve, 0)); + await flushPromises(); + const call = vi.mocked(downtimes.extend).mock.calls[0]; + expect(call[1]).toBe("d1"); + expect(call[3]).toBe("planned"); + expect((call[2] as { until: number }).until).toBe(START + 3 * HOUR); + expect(wrapper.emitted("update:open")?.at(-1)).toEqual([false]); + wrapper.unmount(); + }); + + it("states the new end in the app's zone, not the browser's", async () => { + const resolved = Intl.DateTimeFormat.prototype.resolvedOptions; + const spy = vi + .spyOn(Intl.DateTimeFormat.prototype, "resolvedOptions") + .mockImplementation(function (this: Intl.DateTimeFormat) { + return { ...resolved.call(this), timeZone: "Asia/Tokyo" }; + }); + store.state.timezone = "UTC"; + dismissAll(); + vi.mocked(downtimes.extend).mockResolvedValue({ + data: { ...row("none"), schedule: { ...row("none").schedule, ends_at: START + 3 * HOUR } }, + } as any); + try { + const wrapper = await mountDialog(row("none")); + dialog()!.querySelector('[data-test="o-dialog-primary-btn"]')!.click(); + await flushPromises(); + await new Promise((resolve) => setTimeout(resolve, 0)); + await flushPromises(); + const success = toastRecords.find((r) => r.variant === "success"); + expect(String(success?.message)).toBe("Extended until 17 Sep 17:00 UTC"); + wrapper.unmount(); + } finally { + spy.mockRestore(); + dismissAll(); + } + }); +}); + +describe("extend form", () => { + const bounds = { currentEnd: START + 2 * HOUR, start: START }; + const schema = makeExtendSchema(gt, "UTC", () => bounds); + const check = (end_time: string, end_date = "2026-09-17") => + schema.safeParse({ end_date, end_time } satisfies ExtendForm); + + it("accepts an end past the current one", () => { + expect(check("17:00").success).toBe(true); + }); + + it("rejects an end at or before the current one", () => { + const result = check("16:00"); + expect(result.success).toBe(false); + expect(result.error?.issues[0].message).toBe("The new end must be later than the current end."); + }); + + it("rejects a window longer than 7 days", () => { + const result = check("15:00", "2026-09-24"); + expect(result.success).toBe(false); + expect(result.error?.issues[0].message).toBe("A window can last at most 7 days."); + }); +}); diff --git a/web/src/components/alerts/downtimes/ExtendDowntimeDialog.vue b/web/src/components/alerts/downtimes/ExtendDowntimeDialog.vue new file mode 100644 index 0000000000..01b1bf5639 --- /dev/null +++ b/web/src/components/alerts/downtimes/ExtendDowntimeDialog.vue @@ -0,0 +1,133 @@ + + + + + diff --git a/web/src/components/alerts/downtimes/ExtendDowntimeMenu.spec.ts b/web/src/components/alerts/downtimes/ExtendDowntimeMenu.spec.ts new file mode 100644 index 0000000000..07528fe950 --- /dev/null +++ b/web/src/components/alerts/downtimes/ExtendDowntimeMenu.spec.ts @@ -0,0 +1,65 @@ +// Copyright 2026 OpenObserve Inc. + +import { describe, it, expect, vi } from "vitest"; +import { defineComponent, h } from "vue"; +import { flushPromises, mount } from "@vue/test-utils"; +import ODropdown from "@/lib/overlay/Dropdown/ODropdown.vue"; +import ExtendDowntimeMenu from "./ExtendDowntimeMenu.vue"; +import MuteMenuItems from "./MuteMenuItems.vue"; + +vi.mock("reka-ui", async (importOriginal) => { + const actual = await importOriginal(); + return { ...actual, DropdownMenuPortal: actual.DropdownMenuContent }; +}); + +const openMenu = (labels: "mute" | "extend", onPreset: (secs: number) => void) => + mount( + defineComponent({ + setup: () => () => + h( + ODropdown, + { open: true }, + { + trigger: () => h("button", "open"), + default: () => + h(MuteMenuItems, { labels, dataTestPrefix: "menu", onPreset, onUntil: () => {} }), + }, + ), + }), + { attachTo: document.body }, + ); + +describe("Extend menu", () => { + it("offers the quick-mute presets and Until under an Extend by heading", async () => { + const picked: number[] = []; + const wrapper = openMenu("extend", (secs) => picked.push(secs)); + await flushPromises(); + const text = wrapper.text(); + expect(text).toContain("Extend by"); + for (const label of ["30 min", "1 h", "2 h", "4 h", "Until…"]) expect(text).toContain(label); + await wrapper.get('[data-test="menu-2h"]').trigger("click"); + await flushPromises(); + expect(picked).toEqual([7200]); + wrapper.unmount(); + }); + + it("keeps the mute heading for the mute label set", async () => { + const wrapper = openMenu("mute", () => {}); + await flushPromises(); + expect(wrapper.text()).toContain("Mute notifications"); + expect(wrapper.text()).not.toContain("Extend by"); + wrapper.unmount(); + }); + + it("renders a disabled slot when the row cannot be extended", () => { + const wrapper = mount(ExtendDowntimeMenu, { + props: { enabled: false, compact: true, dataTest: "row-extend" }, + }); + expect(wrapper.get('[data-test="row-extend"]').attributes("disabled")).toBeDefined(); + }); + + it("renders nothing for a header button on a row that cannot be extended", () => { + const wrapper = mount(ExtendDowntimeMenu, { props: { enabled: false } }); + expect(wrapper.find("button").exists()).toBe(false); + }); +}); diff --git a/web/src/components/alerts/downtimes/ExtendDowntimeMenu.vue b/web/src/components/alerts/downtimes/ExtendDowntimeMenu.vue new file mode 100644 index 0000000000..adcbc2399f --- /dev/null +++ b/web/src/components/alerts/downtimes/ExtendDowntimeMenu.vue @@ -0,0 +1,77 @@ + + + + + diff --git a/web/src/components/alerts/downtimes/MuteMenuItems.vue b/web/src/components/alerts/downtimes/MuteMenuItems.vue new file mode 100644 index 0000000000..0d05680789 --- /dev/null +++ b/web/src/components/alerts/downtimes/MuteMenuItems.vue @@ -0,0 +1,60 @@ + + + + + + diff --git a/web/src/components/alerts/downtimes/MutedChip.spec.ts b/web/src/components/alerts/downtimes/MutedChip.spec.ts new file mode 100644 index 0000000000..fc8d509e7f --- /dev/null +++ b/web/src/components/alerts/downtimes/MutedChip.spec.ts @@ -0,0 +1,66 @@ +// Copyright 2026 OpenObserve Inc. + +import { describe, it, expect, vi, afterEach } from "vitest"; +import { mount } from "@vue/test-utils"; +import { defineComponent, h } from "vue"; +import store from "@/test/unit/helpers/store"; +import announcements from "@/services/announcements"; +import { useAnnouncementBanners } from "@/composables/useAnnouncementBanners"; +import MutedChip from "./MutedChip.vue"; + +vi.mock("@/aws-exports", async (importOriginal) => { + const actual = await importOriginal<{ default: Record }>(); + return { default: { ...actual.default, isEnterprise: "true" } }; +}); + +const NOW = Date.parse("2026-09-17T14:10:00Z"); + +describe("MutedChip", () => { + afterEach(() => vi.useRealTimers()); + + it("says how long the mute has left, rounded up to the minute", () => { + vi.useFakeTimers(); + vi.setSystemTime(NOW); + const wrapper = mount(MutedChip, { + props: { + downtime: { id: "d1", name: "INC-231", ends_at: (NOW + 72 * 60_000 - 30_000) * 1000 }, + }, + }); + expect(wrapper.text()).toContain("Muted · ends in 1 h 12 min"); + }); + + it("renders nothing once the mute has ended", () => { + vi.useFakeTimers(); + vi.setSystemTime(NOW); + const wrapper = mount(MutedChip, { + props: { downtime: { id: "d1", name: "INC-231", ends_at: (NOW - 1000) * 1000 } }, + }); + expect(wrapper.find('[data-test="muted-chip"]').exists()).toBe(false); + }); + + it("counts down on the banner's skew-corrected clock, not the browser's", async () => { + vi.useFakeTimers(); + vi.setSystemTime(NOW); + vi.spyOn(announcements, "getActive").mockResolvedValue({ + // Server runs 30 minutes ahead of this browser. + data: { banners: [], now: (NOW + 30 * 60_000) * 1000 }, + } as never); + const host = mount( + defineComponent({ + setup() { + const { refresh } = useAnnouncementBanners(); + return { refresh }; + }, + render: () => h("div"), + }), + { global: { plugins: [store] } }, + ); + await (host.vm as unknown as { refresh: () => Promise }).refresh(); + + const wrapper = mount(MutedChip, { + props: { downtime: { id: "d1", name: "INC-231", ends_at: (NOW + 60 * 60_000) * 1000 } }, + }); + expect(wrapper.text()).toContain("Muted · ends in 30 min"); + host.unmount(); + }); +}); diff --git a/web/src/components/alerts/downtimes/MutedChip.vue b/web/src/components/alerts/downtimes/MutedChip.vue new file mode 100644 index 0000000000..bac03b0781 --- /dev/null +++ b/web/src/components/alerts/downtimes/MutedChip.vue @@ -0,0 +1,68 @@ + + + + + diff --git a/web/src/components/alerts/downtimes/QuickMuteDialog.schema.ts b/web/src/components/alerts/downtimes/QuickMuteDialog.schema.ts new file mode 100644 index 0000000000..95d2d826bf --- /dev/null +++ b/web/src/components/alerts/downtimes/QuickMuteDialog.schema.ts @@ -0,0 +1,79 @@ +// Copyright 2026 OpenObserve Inc. +// +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// This program is distributed in the hope that it will be useful +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. +// +// You should have received a copy of the GNU Affero General Public License +// along with this program. If not, see . + +import { z } from "zod"; +import type { TranslateFn } from "@/types/i18n"; +import { + MAX_DURATION_SECS, + MIN_DURATION_SECS, + localToUtcMicros, + utcMicrosToLocal, +} from "@/utils/downtimes/schedule"; +import { + QUICK_MUTE_PRESETS, + presetSeconds, + type QuickMutePreset, +} from "@/utils/downtimes/quickMute"; + +export interface QuickMuteForm extends Record { + preset: QuickMutePreset | "custom"; + end_date: string; + end_time: string; + reason: string; + folder_id: string; +} + +/** The end of the mute in microseconds UTC, or null while a custom end is incomplete. */ +export function quickMuteEndsAt(v: QuickMuteForm, nowMicros: number, tz: string): number | null { + if (v.preset !== "custom") return nowMicros + presetSeconds(v.preset) * 1_000_000; + return localToUtcMicros(v.end_date, v.end_time, tz); +} + +// The toggle offers Custom plus the presets, so the schema accepts exactly those keys. +const PRESET_KEYS: [QuickMuteForm["preset"], ...QuickMuteForm["preset"][]] = [ + "custom", + ...QUICK_MUTE_PRESETS.map((p) => p.key), +]; + +export const makeQuickMuteSchema = (t: TranslateFn, tz: () => string, now: () => number) => + z + .object({ + preset: z.enum(PRESET_KEYS), + end_date: z.string(), + end_time: z.string(), + reason: z.string(), + folder_id: z.string().min(1), + }) + .superRefine((raw, ctx) => { + const v = raw as QuickMuteForm; + if (v.preset !== "custom") return; + const nowMicros = now(); + const ends = quickMuteEndsAt(v, nowMicros, tz()); + const secs = ends === null ? null : (ends - nowMicros) / 1_000_000; + const message = + secs === null + ? t("alerts.downtimes.validation.endRequired") + : secs < MIN_DURATION_SECS + ? t("alerts.downtimes.validation.endAfterStart") + : secs > MAX_DURATION_SECS + ? t("alerts.downtimes.validation.windowTooLong") + : null; + if (message) ctx.addIssue({ code: "custom", path: ["end_date"], message }); + }); + +export const quickMuteDefaults = (nowMs: number, tz: string, folderId: string): QuickMuteForm => { + const end = utcMicrosToLocal((nowMs + 2 * 3_600_000) * 1000, tz); + return { preset: "2h", end_date: end.date, end_time: end.time, reason: "", folder_id: folderId }; +}; diff --git a/web/src/components/alerts/downtimes/QuickMuteDialog.spec.ts b/web/src/components/alerts/downtimes/QuickMuteDialog.spec.ts new file mode 100644 index 0000000000..afbf92e6eb --- /dev/null +++ b/web/src/components/alerts/downtimes/QuickMuteDialog.spec.ts @@ -0,0 +1,153 @@ +// Copyright 2026 OpenObserve Inc. + +import { describe, it, expect, vi } from "vitest"; +import { flushPromises, mount } from "@vue/test-utils"; +import { createStore } from "vuex"; +import { gt } from "@/types/i18n"; +import common from "@/services/common"; +import downtimes from "@/services/downtimes"; +import { queryClient } from "@/composables/query/queryClient"; +import QuickMuteDialog from "./QuickMuteDialog.vue"; +import { + QUICK_MUTE_PRESETS, + buildQuickMuteRequest, + groupSelection, + presetSeconds, +} from "@/utils/downtimes/quickMute"; +import { makeQuickMuteSchema, quickMuteEndsAt, type QuickMuteForm } from "./QuickMuteDialog.schema"; + +vi.mock("@/aws-exports", () => ({ default: { isEnterprise: "true", isCloud: "false" } })); +vi.mock("@/services/common", () => ({ default: { list_Folders: vi.fn() } })); +vi.mock("@/services/downtimes", () => ({ default: { quickMute: vi.fn(), create: vi.fn() } })); +vi.mock("vue-router", () => ({ useRouter: () => ({ push: vi.fn() }) })); + +const NOW = Date.parse("2026-09-17T14:10:00Z") * 1000; + +describe("quick mute request body", () => { + it("sends one target with all folders and the ids, filed in default, with no banner", () => { + const body = buildQuickMuteRequest( + [{ module: "alerts", ids: ["a1", "a2"] }], + NOW, + NOW + presetSeconds("2h") * 1_000_000, + "UTC", + " Working INC-231 ", + ); + expect(body).toEqual({ + folder_id: "default", + reason: "Working INC-231", + targets: [{ module: "alerts", folders: { kind: "all" }, ids: ["a1", "a2"] }], + schedule: { + repeat: "none", + starts_at: NOW, + ends_at: NOW + 7200 * 1_000_000, + timezone: "UTC", + duration_secs: 7200, + weekdays: [], + }, + show_banner: false, + }); + }); + + it("sends one target per module when alert and anomaly rows are muted together", () => { + const rows = [ + { id: "a1", anomaly: false }, + { id: "ad_7c1", anomaly: true }, + { id: "a2", anomaly: false }, + ]; + const selection = groupSelection( + rows, + (r) => (r.anomaly ? "anomaly_detections" : "alerts"), + (r) => r.id, + ); + const body = buildQuickMuteRequest(selection, NOW, NOW + 1, "UTC"); + expect(body.targets).toEqual([ + { module: "alerts", folders: { kind: "all" }, ids: ["a1", "a2"] }, + { module: "anomaly_detections", folders: { kind: "all" }, ids: ["ad_7c1"] }, + ]); + expect(body).not.toHaveProperty("reason"); + }); + + it("files the mute in the folder the user picked", () => { + const body = buildQuickMuteRequest( + [{ module: "alerts", ids: ["a1"] }], + NOW, + NOW + 1, + "UTC", + undefined, + "payments-maintenance", + ); + expect(body.folder_id).toBe("payments-maintenance"); + }); +}); + +describe("quick mute form", () => { + const schema = makeQuickMuteSchema( + gt, + () => "UTC", + () => NOW, + ); + const values = (over: Partial): QuickMuteForm => ({ + preset: "custom", + end_date: "2026-09-17", + end_time: "18:00", + reason: "", + folder_id: "payments", + ...over, + }); + + it("ends a preset at now plus its length", () => { + expect(quickMuteEndsAt(values({ preset: "30m" }), NOW, "UTC")).toBe(NOW + 1800 * 1_000_000); + }); + + it("accepts a custom end in the future and refuses one in the past", () => { + expect(schema.safeParse(values({})).success).toBe(true); + const past = schema.safeParse(values({ end_time: "12:00" })); + expect(past.success).toBe(false); + expect(past.error?.issues[0].message).toBe("The end must be after the start."); + }); + + it("accepts exactly the offered presets and Custom", () => { + for (const { key } of QUICK_MUTE_PRESETS) { + expect(schema.safeParse(values({ preset: key })).success).toBe(true); + } + expect(schema.safeParse(values({ preset: "8h" as QuickMuteForm["preset"] })).success).toBe( + false, + ); + }); + + it("refuses a custom end more than 7 days away", () => { + const far = schema.safeParse(values({ end_date: "2026-09-30" })); + expect(far.error?.issues[0].message).toBe("A window can last at most 7 days."); + }); +}); + +describe("quick mute with no folder to file in", () => { + it("says so and disables Mute when the permitted folder list is empty", async () => { + queryClient.clear(); + vi.mocked(common.list_Folders).mockResolvedValue({ data: { list: [] } } as never); + const store = createStore({ + state: { + timezone: "UTC", + selectedOrganization: { identifier: "acme" }, + zoConfig: { downtimes_enabled: true }, + }, + }); + const wrapper = mount(QuickMuteDialog, { + props: { open: false, selection: [{ module: "alerts", ids: ["a1"] }] }, + global: { plugins: [store] }, + attachTo: document.body, + }); + await wrapper.setProps({ open: true }); + await flushPromises(); + await vi.waitFor(() => + expect(document.body.querySelector('[data-test="quick-mute-no-folder"]')).not.toBeNull(), + ); + expect(document.body.textContent).toContain("No folder you can file a downtime in"); + const mute = document.body.querySelector( + '[data-test="o-dialog-primary-btn"]', + ); + expect(mute?.disabled).toBe(true); + expect(downtimes.create).not.toHaveBeenCalled(); + wrapper.unmount(); + }); +}); diff --git a/web/src/components/alerts/downtimes/QuickMuteDialog.vue b/web/src/components/alerts/downtimes/QuickMuteDialog.vue new file mode 100644 index 0000000000..e7db58d50e --- /dev/null +++ b/web/src/components/alerts/downtimes/QuickMuteDialog.vue @@ -0,0 +1,218 @@ + + + + + diff --git a/web/src/components/alerts/steps/QueryConfig.spec.ts b/web/src/components/alerts/steps/QueryConfig.spec.ts index 9a433fc170..1641dde392 100644 --- a/web/src/components/alerts/steps/QueryConfig.spec.ts +++ b/web/src/components/alerts/steps/QueryConfig.spec.ts @@ -265,6 +265,7 @@ describe("QueryConfig.vue", () => { qcOverrides: Record = {}, formOverrides: Record = {}, storeOverride?: any, + onSubmit: (values: any) => void = () => {}, ) { const props = reactive({ ...baseQCProps(), ...qcOverrides }); const theStore = storeOverride ?? mockStore; @@ -274,9 +275,10 @@ describe("QueryConfig.vue", () => { schema: addAlertSchema, defaultValues: hostDefaults(formOverrides), qcProps: props, + onSubmit, }), template: ` - + `, @@ -351,6 +353,55 @@ describe("QueryConfig.vue", () => { await nextTick(); expect(wrapper.vm.cronTimezone).toBe("America/Los_Angeles"); }); + + it("keeps a loaded Asia/Calcutta through mount and an unchanged save, shown as Asia/Kolkata", async () => { + host.unmount(); + const submitted: any[] = []; + const triggerCondition = { + ...hostDefaults().trigger_condition, + frequency_type: "cron", + cron: "0 */5 * * * *", + timezone: "Asia/Calcutta", + }; + const { h } = mountHost( + { triggerCondition }, + { trigger_condition: triggerCondition }, + undefined, + (values) => submitted.push(values), + ); + host = h; + wrapper = host.findComponent(QueryConfig) as unknown as VueWrapper; + await flushPromises(); + expect(hostForm().getFieldValue("trigger_condition.timezone")).toBe("Asia/Calcutta"); + + const options = wrapper.vm.timezoneSelectOptions as { + label: string; + value: string; + searchText?: string; + }[]; + const stored = options.find((o) => o.value === "Asia/Calcutta"); + expect(stored?.label).toBe("Asia/Kolkata"); + expect(stored?.searchText).toContain("Calcutta"); + expect(options.filter((o) => o.label === "Asia/Kolkata")).toHaveLength(1); + + const picker = host + .findAllComponents(OFormSelect) + .find((c: any) => c.props("name") === "trigger_condition.timezone"); + expect(picker?.findComponent(OSelect).props("modelValue")).toBe("Asia/Calcutta"); + expect(picker?.text()).toContain("Asia/Kolkata"); + + await hostForm().handleSubmit(); + await flushPromises(); + expect(hostForm().state.errors).toEqual([]); + expect(submitted).toHaveLength(1); + expect(submitted[0].trigger_condition.timezone).toBe("Asia/Calcutta"); + }); + + it("onCronTimezoneChange stores a legacy zone as picked, as main did", async () => { + wrapper.vm.onCronTimezoneChange("Asia/Calcutta"); + await nextTick(); + expect(wrapper.vm.cronTimezone).toBe("Asia/Calcutta"); + }); }); describe("Initialization", () => { diff --git a/web/src/components/alerts/steps/QueryConfig.vue b/web/src/components/alerts/steps/QueryConfig.vue index b87db5230d..7d0ef3bbe5 100644 --- a/web/src/components/alerts/steps/QueryConfig.vue +++ b/web/src/components/alerts/steps/QueryConfig.vue @@ -1813,6 +1813,8 @@ import { useSqlEditorDiagnostics } from "@/composables/useSqlEditorDiagnostics"; import { useVrlPlaceholder } from "@/composables/useVrlPlaceholder"; import { useQueryPlaceholder } from "@/components/logs/useQueryPlaceholder"; import useStreams from "@/composables/useStreams"; +import { useTimezoneOptions } from "@/composables/useTimezoneOptions"; +import { savedBrowserTimezone } from "@/utils/timezoneAliases"; import { useTypewriterPlaceholder } from "@/components/ai-assistant/welcome/useTypewriterPlaceholder"; import { alertPromqlSamples } from "@/utils/alerts/promqlSamples"; import AlertQueryPreview from "@/components/alerts/AlertQueryPreview.vue"; @@ -2934,8 +2936,6 @@ export default defineComponent({ const cronDescription = computed(() => describeCron(t, cronExpression.value, cronTimezone.value), ); - const filteredTimezones = ref([]); - // Initialize timezone // Populate the timezone OPTIONS only. This must NOT seed the timezone value: // `cronTimezone` is now form-owned, so writing here would push a browser @@ -2943,31 +2943,9 @@ export default defineComponent({ // display ref, leaving the stored value untouched until the user entered cron // mode (onFrequencyUnitChange still seeds it there). defaultAlertValue() // already seeds `timezone: "UTC"`, so the control is never blank anyway. - const browserTz = Intl.DateTimeFormat().resolvedOptions().timeZone || "UTC"; - const browserTime = raw("Browser Time (" + browserTz + ")"); - const initTimezones = () => { - try { - // @ts-ignore - const zones: string[] = - typeof Intl !== "undefined" && typeof Intl.supportedValuesOf === "function" - ? // @ts-ignore - Intl.supportedValuesOf("timeZone") - : [cronTimezone.value || "UTC"]; - // Convenience shortcuts first (matching the reports picker), then every - // IANA zone. This only populates OPTIONS — it must not seed cronTimezone. - filteredTimezones.value = [browserTime, "UTC", ...zones]; - } catch { - filteredTimezones.value = ["UTC"]; - } - }; - initTimezones(); - - const timezoneSelectOptions = computed(() => - filteredTimezones.value.map((tz: string) => - tz === browserTime - ? { label: t("common.browserTimeWithZone", { zone: browserTz }), value: tz } - : { label: raw(tz), value: tz }, - ), + // The stored zone is an option under its own name, so a legacy `Asia/Calcutta` saves back unchanged. + const { zones: filteredTimezones, timezoneOptions: timezoneSelectOptions } = useTimezoneOptions( + { browserEntry: true, current: cronTimezone }, ); const validateCron = () => { @@ -3023,7 +3001,7 @@ export default defineComponent({ // Entering cron mode is where a timezone first gets SAVED (parity: // pre-migration seeded it here, not at mount). if (!cronTimezone.value) { - cronTimezone.value = Intl.DateTimeFormat().resolvedOptions().timeZone; + cronTimezone.value = savedBrowserTimezone(); } } validateCron(); diff --git a/web/src/components/announcements/AnnouncementBanner.spec.ts b/web/src/components/announcements/AnnouncementBanner.spec.ts new file mode 100644 index 0000000000..6288c27119 --- /dev/null +++ b/web/src/components/announcements/AnnouncementBanner.spec.ts @@ -0,0 +1,111 @@ +// Copyright 2026 OpenObserve Inc. +// +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// This program is distributed in the hope that it will be useful +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. +// +// You should have received a copy of the GNU Affero General Public License +// along with this program. If not, see . + +import { ref } from "vue"; +import { afterEach, beforeEach, describe, expect, it, vi } from "vitest"; +import { flushPromises, mount } from "@vue/test-utils"; +import { createMemoryHistory, createRouter } from "vue-router"; +import store from "@/test/unit/helpers/store"; +import { raw } from "@/types/i18n"; +import type { Banner } from "@/composables/useAnnouncementBanners"; + +const NOW_MS = 1_770_000_000_000; +const banners = ref([]); + +vi.mock("@/composables/useAnnouncementBanners", () => ({ + useAnnouncementBanners: () => ({ + banners, + dismiss: vi.fn(), + start: vi.fn(), + serverNowMs: () => NOW_MS, + }), +})); + +import AnnouncementBanner from "./AnnouncementBanner.vue"; + +const downtimeBanner = (url: string): Banner => ({ + id: "downtime:dt-1:1", + message: raw("Kafka upgrade is active."), + variant: "warning", + ends_at: (NOW_MS + 20 * 60_000) * 1000, + dismissible: true, + cta: { text: raw("View downtimes"), url }, +}); + +const mountBanner = async () => { + const router = createRouter({ + history: createMemoryHistory(), + routes: [ + { path: "/", component: { template: "
" } }, + { path: "/downtimes", name: "downtimes", component: { template: "
" } }, + ], + }); + await router.push("/"); + await router.isReady(); + const wrapper = mount(AnnouncementBanner, { global: { plugins: [store, router] } }); + await flushPromises(); + return { wrapper, router }; +}; + +describe("AnnouncementBanner", () => { + beforeEach(() => { + store.state.selectedOrganization = { + ...store.state.selectedOrganization, + identifier: "acme", + }; + }); + + afterEach(() => { + banners.value = []; + }); + + it("opens an in-app link in place, on the current org", async () => { + banners.value = [downtimeBanner("/web/downtimes?status=active&scope=all")]; + const { wrapper, router } = await mountBanner(); + + const cta = wrapper.get('[data-test="announcement-banner-cta-downtime:dt-1:1"]'); + expect(cta.attributes("target")).toBeUndefined(); + await cta.trigger("click"); + await flushPromises(); + + expect(router.currentRoute.value.path).toBe("/downtimes"); + expect(router.currentRoute.value.query).toEqual({ + status: "active", + scope: "all", + org_identifier: "acme", + }); + }); + + it("keeps an external link in a new tab", async () => { + banners.value = [downtimeBanner("https://status.example.com/incident")]; + const { wrapper } = await mountBanner(); + + const cta = wrapper.get('[data-test="announcement-banner-cta-downtime:dt-1:1"]'); + expect(cta.attributes("href")).toBe("https://status.example.com/incident"); + expect(cta.attributes("target")).toBe("_blank"); + }); + + it("stacks the countdown under the message on a phone", async () => { + banners.value = [downtimeBanner("/web/downtimes")]; + const { wrapper } = await mountBanner(); + + const body = wrapper.get('[data-test="announcement-banner-body-downtime:dt-1:1"]'); + expect(body.classes()).toContain("max-sm:flex-col"); + expect(body.text()).toContain("Kafka upgrade is active."); + expect(body.find('[data-test="announcement-banner-countdown-downtime:dt-1:1"]').exists()).toBe( + true, + ); + }); +}); diff --git a/web/src/components/announcements/AnnouncementBanner.vue b/web/src/components/announcements/AnnouncementBanner.vue index ad5029df00..739a30c67a 100644 --- a/web/src/components/announcements/AnnouncementBanner.vue +++ b/web/src/components/announcements/AnnouncementBanner.vue @@ -23,12 +23,47 @@ along with this program. If not, see . :icon="bannerIcon(banner)" :data-test="`announcement-banner-${banner.variant}`" > - {{ banner.message }} +
+ {{ banner.message }} + +
+ + +