starred/openobserve
1
0
Fork 0
mirror of https://github.com/openobserve/openobserve.git synced 2026-11-04 00:08:55 +00:00
Code Issues Packages Projects Releases Wiki Activity

feat: alert downtimes with muted incidents and SLO corrections (#14875)

Browse Source 
Co-authored-by: github-actions[bot] <41898282+github-actions[bot]@users.noreply.github.com>
This commit is contained in:
Loakesh Indiran
2026-10-13 02:25:17 +00:00
committed by GitHub
co-authored by github-actions[bot]
parent c99cb0d516
commit fbe35aebf0
296 changed files with 52536 additions and 3603 deletions
Download Patch File Download Diff File Expand all files Collapse all files
scripts/translations/.translation_state.json
+533
View File
@@ -2202,6 +2202,7 @@
"countAnomalyDetection": "a1223bc8d82a5173",
"countCompositeAlert": "4502d18cb5dcc5d8",
"countDestinations": "124a72586a12ab90",
"countDowntime": "20e7e11d1e0112dd",
"countIncidentIntegration": "7e70aef22e421de8",
"countOncallPolicy": "537bc7a755773028",
"countOncallTeamChannel": "fcce7a09b096a168",
@@ -3244,6 +3245,500 @@
"destinationPreview": "4baa36d3165b6fb2",
"destinationsRequiredLabel": "341c8a818453333b",
"detectionFunction": "4625de26f0afff67",
"downtimes": {
"actions": {
"cancel": "1e83e5d555d4c816",
"delete": "e2d0a54968ead24e",
"duplicate": "02cdaabfca805d6a",
"edit": "464c4ffd019e1e96",
"endNow": "adf7eccb9562dfc7",
"extend": "496a37e3a10b9755",
"more": "f8d46c2570e70736",
"moveToFolder": "91d631e150a39784",
"muteForHour": "991443d657e1425d"
},
"allFolders": "226fd0e51f770b0b",
"allFoldersTooltip": "26ab27ec8eb6fb82",
"autoName": {
"all": {
"alerts": "2ead279c901d5b24",
"anomaly_detections": "12be9cc50e1846dc",
"slos": "65918fb54f483e8f",
"synthetics": "53668a29191ec7da"
},
"daily": "6f31304cdfd0a3a6",
"once": "200651a8b2d5912f",
"onceOn": "5d984d828da9096f",
"weekly": "8850eced986ec360"
},
"band": {
"active": "92340695899bd2d8",
"aria": "e535bfb61874c02e",
"next": "1ff57a29d7c9d11b",
"none": "9bbf06c79d5f4e7e",
"now": "ed5eb9a37e2d8231",
"past": "b6a7c95e785f14dd",
"track": "d598026a9cbc6050"
},
"banner": {
"count": "30d5ec4473fde4c6",
"ended": "e87ba7a0b779d8b7",
"endsIn": "e3f5f47cba745f88",
"endsInMinutes": "d27784cde45dbfb3",
"endsInUnderMinute": "89c26da748360a68",
"firstEndsIn": "4426c705d88473a7",
"firstEndsInMinutes": "a1aedbd917b68cee",
"firstEndsInUnderMinute": "243caa9b0d62c666",
"preview": "324b134f57c70c72",
"previewMessage": "4d171441614a497f",
"showBanner": "a0aadf239daf32df",
"showBannerCaption": "61b0d38e3acd9890",
"tooLong": "ab0d30dc42cc3897"
},
"bannerOn": "4028dbeb549d24a8",
"columns": {
"createdBy": "374cd9d220fbeb7a",
"folder": "74ccd43303847f26",
"matched": "0d6346e060b47f1c",
"name": "dcd1d5223f73b3a9",
"schedule": "f4830a1dae298044",
"status": "920e413c7d411b61",
"targets": "27445f6ab61d9a5c",
"when": "9c4734aa9debfd11"
},
"condition": {
"add": "9fd728c66c9a256b",
"caption": "51cd0322db6a8438",
"none": "9ef2d72a498f55b6",
"onePer": "f4143a38a449c698",
"remove": "f38bb5d8032f1358",
"title": "86d7b31ddd36de64"
},
"confirmCancel": {
"bulkMessage": "60d844aa8e405faf",
"confirm": "8b96b7d5b52eb086",
"keep": "02650c8fee8a0034",
"message": "04f1f5c6087f69b5",
"title": "1e83e5d555d4c816"
},
"confirmDelete": {
"message": "97caf93ccdae9977",
"title": "4dbee51e2f61627a"
},
"confirmEndNow": {
"keep": "154949dbda2aa4ee",
"message": "ae2007fef3d5c18d",
"ok": "adf7eccb9562dfc7",
"title": "8b334dd1f9510dfd"
},
"copyOf": "3d050ae2d5dce9e2",
"corrected": "eae618cf3750c786",
"correctedBy": "bc8dc94b765d2ab1",
"detail": {
"affected": "ec13ab31fa56f546",
"backToList": "7906410d8d99435c",
"banner": "b7f4d98f98d51f8e",
"bannerOff": "0e36b3598800ed29",
"bannerOn": "7ed5a29b16f8c7e6",
"condition": "39b36d38d6eb03c8",
"details": "45989de49fb7f66d",
"matchedBy": {
"folder": "74ccd43303847f26",
"id": "8605605ae0266484",
"org": "2065e7b8b629c45a",
"query": "b80a37564fbb3c07",
"source_alert": "36f9b601a7db30ad",
"tag": "1503916a2ab2b0fd"
},
"matchedByHeader": "7693857969438ccf",
"muteHourName": "56f878e77cdc8ed3",
"nextWindow": "a34f4badd0f164ae",
"noSuppressed": "e1167a0d116e7a81",
"noTarget": "4b33d955f9a8e928",
"notFoundDescription": "0d01e9170c73286d",
"notFoundTitle": "f2439bef305161f7",
"occurrences": "6c58d39cec6d77d2",
"overview": "d4b1ea5708dd5329",
"sectionCount": "534026895b04bd45",
"source": "0e570ca6fabe24f9",
"suppressed": "d1d64da789198216",
"wouldHaveNotified": "4c8ca48245a44de6"
},
"detailTitle": "5f8d6d8feb7b17a5",
"duration": {
"days": "d7066a9380ea87cf",
"hours": "b7fc364731644a66",
"minutes": "1829c58ba472520f"
},
"editTitle": "cfc2b21f0c45e190",
"endsLabel": "b7a7513cd632f8a3",
"extend": {
"confirmUntil": "690456e9eb98fd77",
"dialogTitle": "e021a5a9e84bd4de",
"endAfterCurrent": "7f1ffb10f74c672c",
"followUp": "272c39e44abf882c",
"menu": "62ccf29b7b8fa2d2",
"newEnd": "f912cddaf06b6b08",
"notActive": "cfd20a4ba832b5f0"
},
"filterAll": "a52ace420f2175d0",
"filterOnce": "ddd7eb99b3575ba4",
"filterRecurring": "c0c06e547b1b55e1",
"form": {
"activeBanner": "f14ff5aca8bc1213",
"addItems": "b4e00e800ef36726",
"addTags": "9c29db454e3dd0f5",
"alertFolders": "a6e2acee332f6d23",
"appliesTo": "6687458beee57a58",
"cancel": "19766ed6ccb2f4a3",
"cardCaption": "e6b6a06a6c7a08c2",
"changedMeanwhile": "0db609fe96ec22bc",
"checkSchedule": "028493139bd6e7ef",
"editMode": "cfc2b21f0c45e190",
"itemOption": "f42e5528f7e57da5",
"itemsHint": "ce436920ecd4c10c",
"matches": {
"alerts": "de053c9af3e30982",
"anomaly_detections": "5a79fd3822ad10a9",
"slos": "3f39dcd8f7f55215",
"synthetics": "2e0b2ccfdecd55b3"
},
"modeInFolder": "d6720d82a0d1d94c",
"nameLabel": "febe01f8a740a5a5",
"namePlaceholder": "febe01f8a740a5a5",
"newMode": "9ac8385c2016de16",
"noAlertFolderAccess": "6a25f9dcb4ff03d0",
"noConditionForChecks": "a4c1b42fa688a959",
"noModule": "9edb12c59d679aab",
"noSyntheticsFolderAccess": "1b50049cc8068fc5",
"reason": "f81ab834de5f8491",
"reasonHelp": "6c97e754df814762",
"reload": "bdc090ec61e3fcfc",
"removeBlock": "c79fb1926d77892d",
"renameHint": "e1a4ecc1bed73c95",
"save": "1509f561f2416598",
"sloCountAsGood": "21a2e9a921f34a5f",
"sloCountAsGoodCaption": "8fd8cf618c755080",
"sloExclude": "98161f0604cca030",
"sloExcludeCaption": "95ff87a3a053f5b7",
"sloMode": "c45be405598f3b27",
"specificItems": "3d80f2bc129418ff",
"syntheticsFolders": "41a216ee0710900f",
"tabAdvanced": "9f088dbebd6c3c70",
"tabSchedule": "f4830a1dae298044",
"tabTargets": "27445f6ab61d9a5c",
"tagPlaceholder": "06dd3884aa86b226",
"tags": "1331275bc537b4c2",
"tagsHint": "742d12fb1a499bd2"
},
"incident": {
"banner": "f86920189febdc1c",
"bannerUntil": "3df3d6cac2e6e42b",
"mutedBy": "d73aab45a5ad7e8f",
"notLoaded": "6c0d7037c5eac517",
"notLoadedTooltip": "80c17a2070f4bc00"
},
"matched": {
"alerts": "42c0f3db6ef90f50",
"anomalies": "a16eb2c72faa6a50",
"slos": "1b068f9843631882",
"synthetics": "ba82fe00463aa525"
},
"modules": {
"alerts": "f62e9267d196c761",
"anomaly_detections": "75648c85faa9cc80",
"slos": "46da04c2507b963f",
"synthetics": "1569d1eebf28e7da"
},
"modulesShort": {
"alerts": "f62e9267d196c761",
"anomaly_detections": "79622828d2d8d988",
"slos": "46da04c2507b963f",
"synthetics": "f45c092981915635"
},
"mute": {
"bulk": "8dd6857baf026850",
"confirm": "8dd6857baf026850",
"confirmUntil": "7daca9b07837f121",
"endsAt": "bfd9598957799ce4",
"endsIn": "fcd6155d5bfd1023",
"exclude": "b8d23549534c5189",
"fileIn": "3f41a5921c2d84a2",
"firingMuted": "390d295ccd85b699",
"firingMutedTooltip": "a822693833bb0415",
"menu": "c6fbfb2045dbb234",
"muteFor": "e1536152577fc598",
"muted": "bf23559e4c1d3e1a",
"mutedTooltip": "6c45db116b6b5b99",
"presets": {
"1h": "f4926b3c9a617ae1",
"2h": "f6f984a4124b10d7",
"30m": "f69fa291de134e3c",
"4h": "1e2e2db5fda936fe",
"custom": "494ca78f7374e46f"
},
"subtitle": "34e5e8d2eaf713b0",
"title": {
"alerts": "9cfc47b5ccd4c45a",
"anomaly_detections": "2dd4a12b2f38f304",
"items": "61e86294348fc303",
"slos": "0bda858a9e03545c",
"synthetics": "e7cd617aeb4acacd"
},
"until": "3a163441ec3b76df"
},
"muteAllDialog": {
"cancel": "19766ed6ccb2f4a3",
"confirm": "323a7afbef2e99a7",
"every": {
"alerts": "69457d9768567d73",
"anomaly_detections": "ea14ac9376faea60",
"slos": "2339ae37b979bb83",
"synthetics": "c984c7128ca33c33"
},
"intro": "711510f1db8e6c49",
"matchedNow": "dac4356c321e9613",
"outro": "54ff7c2a8df84805",
"title": "2d1aceb041aac8d1",
"titleOne": {
"alerts": "964452cbdf79d03c",
"anomaly_detections": "4199da9a414ddd8e",
"slos": "164014acfe372d7b",
"synthetics": "9d18595cabbed056"
},
"window": "3821ac247d6f6913"
},
"newDowntime": "9ac8385c2016de16",
"newTitle": "9ac8385c2016de16",
"nextLabel": "c6c1c9a9c8543f1e",
"noFolder": "4cc9fdc7cc253497",
"notify": {
"caption": "acfc3dd57675c82d",
"destinations": "72eb63f032e47e34",
"events": {
"cancelled": "f85f95dc4eb251a4",
"cancelledHelp": "d01cac5a93e637f5",
"ended": "866e3f08ac4997ab",
"endedHelp": "a3c7966b8c7e8468",
"ending_soon": "e1a42c73cde70002",
"ending_soonHelp": "ba03badd2504dc53",
"extended": "d3b637b34bcee97b",
"extendedHelp": "fc59d77e5ac7735c",
"started": "5fd61c66df5d29bc",
"startedHelp": "d82cc56f2c1eb890"
},
"lead": "b615206b6c30e676",
"leadHelp": "6cf2909c369f818d",
"log": {
"destinations": "72eb63f032e47e34",
"empty": "7822537e59cf15cd",
"event": "4e1f49a9c8ae8a15",
"events": {
"cancelled": "d353a99eb4556847",
"ended": "7cdc804e6996997d",
"ending_soon": "584cf74c33fbf689",
"extended": "97b165f2f4e4ca54",
"started": "ecbc89cd37a03734"
},
"ok": "c16bc82bf1f04ede",
"pending": "e595f17fac1d8384",
"result": "6e7d50e84f4731ef",
"time": "33b93476cf597a33"
},
"regionNote": "f5ed2e60103bf71c",
"summary": {
"cancelled": "8add00b6023aedd3",
"ended": "70a8b18249455a09",
"endingSoon": "3092ca0ea8b2d0a0",
"extended": "87d13880323aa9c1",
"none": "8e4c3adb76279932",
"sentence": "588f7196ea82ef51",
"started": "b9486c7446183877"
},
"tab": "788011833a5a0f22",
"test": {
"failed": "557f16c3c32eca01",
"send": "22d7b11d40c98b51",
"sent": "c2250e7ed00f3376"
},
"title": "f4e2b7734cb9eb5a",
"validation": {
"eventRequired": "a00292a8f3334fa9",
"leadInvalid": "80cb65c2d61c1320",
"tooMany": "0f5c86778e1d01a8"
}
},
"oneTime": "ddd7eb99b3575ba4",
"preview": {
"andMore": "1a2612705dd79784",
"failed": "eabd4ce51b31ceca",
"finishCondition": "dba8d9186a95c8dc",
"fireTime": "9ffb4b9d42e1dead",
"noMatch": "782a2819bed11cda",
"showAll": "739278fd42f862d7",
"showLess": "94ea9b1d33a02975",
"title": "324b134f57c70c72",
"undecidable": "f4b2da47e69a7dd1"
},
"recurring": "c0c06e547b1b55e1",
"resources": {
"apply": "31e392d1c0378bec",
"footer": "b237ec336c28d269",
"fromRegistry": "090b22de22c1cc48",
"fromSearch": "571555b1c76a84aa",
"fromValues": "0e04d3e365c8fec4",
"items": "fb8e7a1a3cb7b25d",
"lastSeen": "21fd79c7de52e1cf",
"load": "0834d3c481792921",
"loadFailed": "5b93b635d84e129a",
"needsEq": "a10bf1f00e8cc7fe",
"noValues": "e65cd4655961fa39",
"notSeen": "5366b0bf1c3a68ef",
"partial": "47cb4ce27c491c52",
"refineBy": "e6e354bab807c9eb",
"streams": "bfd297afbf9e7430",
"tooMany": "e9d557f93756aa7f"
},
"schedule": {
"dailySentence": "66de77c309efe205",
"fromNoEnd": "3c66035ce1832c87",
"fromUntil": "72722a4a51229dac",
"onceSentence": "afe78d183505636c",
"weeklySentence": "c5129b953bb604b5",
"windowRange": "4c9994e9d20628f5"
},
"scheduleForm": {
"daily": "b36c2611dcdfa190",
"days": "e08c0aa8f558f39f",
"duration": "4fc52a3c4c558b51",
"durationHelp": "4ac1b55713c423f8",
"durationPlaceholder": "659ba3d3d1509f16",
"endDate": "14303aa0c4a08d39",
"endTime": "2e46006a5eebb262",
"nextWindow": "7028e10e8d2f9459",
"nextWindowWithUtc": "1c9fc89ad4e402de",
"noWindow": "03f3bde6d99598e3",
"once": "d88f6d8372f4d54e",
"startDate": "8169693101a4536c",
"startTime": "babe9dda85036fe4",
"startsAt": "faca355ff059d331",
"startsOn": "642efcf146d12253",
"timezone": "4ceca1d52cede44d",
"until": "a07f9029151d9729",
"untilPlaceholder": "6c43e8399a11d714",
"weekly": "2975132481a7a695"
},
"search": "6ee57ed06e35bc0f",
"searchAcross": "169fb6354f56e3b4",
"sentence": {
"inFolders": "a1d8cf47d80193bb",
"named": "9819249ddc9d3ecc",
"nothingMuted": "2fa492fd21fcafd5",
"nouns": {
"alerts": "deb3e366c077c228",
"anomaly_detections": "bac36d00cf172630",
"slos": "46da04c2507b963f",
"synthetics": "c8996fb03050baee"
},
"sloAsGood": "b7bc23b67e9a7e31",
"sloExcluded": "532e94dfac498031",
"sloUnchanged": "24e0f5b51dc1d32c",
"stopNotifying": "bff86a74d95fbba7",
"tagged": "6bdb3ce7ddd9c4b6",
"whenDaily": "57390f94bae3294b",
"whenOnce": "f2d856678e9059ba",
"whenWeekly": "78fd12a3264d7799",
"withCondition": "219a5cf3bd61463e"
},
"sloCorrections": {
"item": "6092824c2221f9d1",
"shorterThanSlice": "b57f3450f4f65f03",
"title": "5f89642d51b55773"
},
"stats": {
"active": "92340695899bd2d8",
"cancelled": "d353a99eb4556847",
"ended": "7cdc804e6996997d",
"endedEarly": "e5788800949e2d4b",
"recurring": "c0c06e547b1b55e1",
"scheduled": "4724f344c1c0e4a1",
"total": "c9b3c38247f744e1"
},
"subtitle": "3e0611514c337da7",
"summary": {
"allFolders": "b0252c8176233332",
"asGood": "44b1f98a69bf000a",
"chip": "a83b2dfc38903aff",
"condition": "596d40a4ed24c02a",
"moreTargets": "ecadd87ff0e5d0aa",
"nFolders": "78817f6f060fbcd3",
"nItems": "736ad44ae2e3ff21",
"noIncidents": "6327a035265ffe1f",
"oneFolder": "3580adda4cac85b1",
"tags": "5fda8874e4efa952",
"valuesOfKey": "23c578cbdd9844ce"
},
"summaryPane": {
"filedIn": "a7c8c465166a6e5e",
"inUtc": "a1ee288c8cd9a98b",
"keepRunning": "519b10ff113191bd",
"largeMatch": "c613fc285a3ee59f",
"mutedIncidents": "18a69c800c7a5674",
"nextWindow": "39bf594595913f03",
"noNotifications": "634459659657e0a0",
"title": "8e76a94ac8320d51",
"whatHappens": "70fbb54bd5e6ff77",
"window": "19734a1b58cabc22"
},
"target": {
"incidentMode": {
"label": "a8b1ad111a575a71",
"muted": "93e35cb5a9ee9a88",
"mutedHelp": "863972b6ae1c5c6c",
"none": "1d1827745b69f605",
"noneHelp": "32e4187072a3f23c",
"summary": "812f3018948c6ae7"
}
},
"thisFolder": "ab7db04c96526507",
"thisFolderTooltip": "68f67542bf19e9ea",
"title": "5d9972eaed771890",
"truncated": "c40514efed315298",
"validation": {
"conditionRequired": "280e70b10d98a7aa",
"durationInvalid": "29165cb4ad634336",
"emptyGroup": "894e52df67b6c6a0",
"endAfterStart": "bd36587e3752dbbf",
"endRequired": "15dad7a2f239b303",
"foldersRequired": "c05b06f380c0ffb0",
"idOutsideFolders": "335c70ce631befc8",
"idsRequired": "a52bb7da0a7907e7",
"innerStar": "47542d9f2383f355",
"loneStar": "38938b248a777d87",
"modulesRequired": "71be8a1a3353efab",
"nameTooLong": "5723cc6a278f8a15",
"needsEq": "d5fbb365a9adc50a",
"pairKey": "8bc8d9cbdab52f53",
"pairOperator": "c21e78625efae96e",
"pairValue": "d173a1e1be4a69fb",
"reasonTooLong": "ff76459b14c83daf",
"startDateRequired": "45b7f811a47b4731",
"startRequired": "9239343743437cf9",
"startTimeRequired": "76bff3e8f5489b42",
"tagsRequired": "2ba02e3a957fdd15",
"timezoneRequired": "d93e8be03fbc7096",
"tooDeep": "58e6544a95805305",
"tooManyFolders": "f02e3ac4ff364c69",
"tooManyIds": "8029ede589c9a568",
"tooManyPairs": "94defd0f155e5dae",
"tooManyTags": "b2406bf479c15337",
"untilAfterStart": "b733d94c0bf498e9",
"weekdaysRequired": "56f423871f48777c",
"windowOutlastsRepeat": "20ced4da0c23bb00",
"windowTooLong": "fed4b750a6b10414",
"windowTooShort": "36e5c0e9f8ba512e"
}
},
"duration": "4fc52a3c4c558b51",
"edit": "464c4ffd019e1e96",
"editAlertMode": "55d309d6da250466",
@@ -5131,6 +5626,13 @@
"remotepipeline": "5003f204b0419ded",
"webhook": "4814f62c108d82de"
},
"downtimeStatus": {
"active": "92340695899bd2d8",
"cancelled": "d353a99eb4556847",
"ended": "7cdc804e6996997d",
"endedEarly": "e5788800949e2d4b",
"scheduled": "4724f344c1c0e4a1"
},
"enrichmentJobStatus": {
"completed": "22a970d2e5b1cc23",
"failed": "031a8f0f659df890",
@@ -8419,6 +8921,12 @@
"description": "58946c6145c03a73",
"title": "29b0352f2dc58297"
},
"noDowntimes": {
"action": "928a7246a60c12cf",
"actionDesc": "b39e9d84e595a81e",
"description": "b7e99ffd675ba00b",
"title": "476efe6fb043a245"
},
"noEnrichmentTables": {
"action": "55159b3215f09b6a",
"actionDesc": "f848b7a436e73f0b",
@@ -9207,6 +9715,7 @@
"rumPresetSeeded": "f9b03957a5cad1c5",
"scopeAllOf": "c600e6cbe1bf6ad6",
"scopeCoveredByParent": "f8e7e0b7e085d5a1",
"scopeDowntimeHint": "2d5e550e6c592398",
"scopeEveryStream": "d60410af0eae5a3a",
"scopeEveryStreamHint": "620ee9401c968689",
"scopeIncludesFuture": "26c2ee5269db2d08",
@@ -11681,6 +12190,7 @@
"data": "cec3a9b89b2e3913",
"databases": "61d2afe2349913ce",
"docs": "7af023c43013b9a5",
"downtimes": "5d9972eaed771890",
"experience": "8eab0f09df013b1b",
"help": "b79cac926e0b2e34",
"home": "3a78695388b38b5c",
@@ -17874,6 +18384,8 @@
"dashboardsListAdd": "c7c4cad42171aa5c",
"dashboardsListImport": "001dd77b594f3299",
"dashboardsListRefresh": "3cf8d66227384dc2",
"downtimesCreate": "9ac8385c2016de16",
"downtimesRefresh": "1a30422cd2d4bcbb",
"enrichmentTablesRefresh": "3cf8d66227384dc2",
"evalJobsRefresh": "3cf8d66227384dc2",
"evalTemplatesRefresh": "3cf8d66227384dc2",
@@ -18005,6 +18517,7 @@
"dashboards": "54fbf186c69e6863",
"dashboardsList": "a53bcafb67d960df",
"dashboardsPanel": "514929ee06065dad",
"downtimes": "5d9972eaed771890",
"enrichmentTables": "ec8727ebf36ee78a",
"evalJobs": "c4ee26b641cc79f8",
"evalTemplates": "9f9c7ce511ab3377",
@@ -18149,6 +18662,8 @@
"burnNote": "578c48ed7bb9cb09",
"burnRate": "b1b28eda1dfed810",
"burnTitle": "b1b28eda1dfed810",
"corrected": "3d1f4b168cc3d396",
"correctedTooltip": "c458f0fcb34b5443",
"cumulativeHint": "e40ff1934f536079",
"empty": "d73248c65643973e",
"errorsAffordable": "7ea7143b543c36a4",
@@ -20482,6 +20997,24 @@
"panelQueryHasNoStreamToAlertOn": "b7b3ff8012762728",
"runningQueryCanceledSuccessfully": "acddcf9ab55be3b4"
},
"downtimes": {
"cancelFailed": "78e0fecb7090fdd5",
"cancelled": "5162ae07032ffd66",
"cancelledMany": "ec4d6ab0d94626cd",
"created": "1618d16b727812c6",
"deleteFailed": "e1002fbe55d51364",
"deleted": "58c3961b3d7538f7",
"extendFailed": "dca1772452d1059a",
"extended": "14f080c8e5ab88d0",
"extendedFollowUp": "0ee62b1e0263e140",
"itemsDropped": "802d6f7bb964f3fb",
"muteFailed": "ad98d850b7e3521e",
"muted": "776a13adfdfb580d",
"reload": "bdc090ec61e3fcfc",
"saveFailed": "7d7c72a4b7b82e93",
"updated": "7227232b174315f5",
"viewDowntime": "5080007542d648b7"
},
"functions": {
"deletedSuccessfully": "b322151f8013dfed",
"deletingFunctions": "1d77aee9c17e33cf",
src/api/common/src/auth/validator.rs
+1
View File
@@ -922,6 +922,7 @@ async fn check_and_create_org(user_id: &str, method: &Method, path: &str) -> Res
|| path_columns[2].eq("reports")
|| path_columns[2].eq("synthetics")
|| path_columns[2].eq("incidents")
|| path_columns[2].eq("downtimes")
|| path_columns[2].eq("workflows"))
{
path_columns[1]
src/api/http/src/handler/http/router/mod.rs
+18 -3
View File
@@ -31,8 +31,8 @@ use openobserve_api_management::request::cloud;
#[cfg(feature = "profiling")]
use openobserve_api_management::request::profiling;
use openobserve_api_management::request::{
alerts, announcements, authz, dashboards, db_monitoring, folders, ingest_rejections, kv,
metrics_usage, model_pricing, organization, query_history, rum_analytics, service_accounts,
alerts, announcements, authz, dashboards, db_monitoring, downtimes, folders, ingest_rejections,
kv, metrics_usage, model_pricing, organization, query_history, rum_analytics, service_accounts,
short_url, slos, sourcemaps, status, status_pages, stream, synthetics, users,
};
use openobserve_api_pipelines::request::{enrichment_table, functions, pipeline, pipelines};
@@ -876,7 +876,8 @@ pub fn service_routes() -> Router {
#[cfg(not(feature = "enterprise"))]
let server = cfg.common.instance_name_short.to_string();
let mut router = Router::new();
// Downtimes live in their own table to keep this function under the line limit.
let mut router = Router::new().merge(downtime_routes());
// Full UI configuration — authenticated counterpart of the unauthenticated
// `/config` bootstrap in config_routes()
router = router.route("/{org_id}/config", get(status::zo_config));
@@ -2093,6 +2094,20 @@ pub fn service_routes() -> Router {
}))
}
/// Downtimes routes; the literal segments stay before the {downtime_id} catch-all.
#[rustfmt::skip]
pub fn downtime_routes() -> Router {
Router::new()
.route("/v2/{org_id}/downtimes", get(downtimes::list_downtimes).post(downtimes::create_downtime))
.route("/v2/{org_id}/downtimes/preview", post(downtimes::preview_downtime))
.route("/v2/{org_id}/downtimes/resources", post(downtimes::downtime_resources))
.route("/v2/{org_id}/downtimes/values", post(downtimes::downtime_values))
.route("/v2/{org_id}/downtimes/move", patch(downtimes::move_downtimes))
.route("/v2/{org_id}/downtimes/{downtime_id}", get(downtimes::get_downtime).put(downtimes::update_downtime).delete(downtimes::delete_downtime))
.route("/v2/{org_id}/downtimes/{downtime_id}/cancel", post(downtimes::cancel_downtime))
.route("/v2/{org_id}/downtimes/{downtime_id}/extend", post(downtimes::extend_downtime))
}
/// Create other service routes (AWS, GCP, RUM)
pub fn other_service_routes() -> Router {
let aws_routes = Router::new()
src/api/http/src/handler/http/router/openapi.rs
+81
View File
@@ -293,6 +293,17 @@ use crate::{
openobserve_api_management::request::slos::move_slos,
openobserve_api_management::request::slos::list_slo_eligible_alerts,
openobserve_api_management::request::slos::preview_alert_sli,
openobserve_api_management::request::downtimes::list_downtimes,
openobserve_api_management::request::downtimes::create_downtime,
openobserve_api_management::request::downtimes::preview_downtime,
openobserve_api_management::request::downtimes::downtime_resources,
openobserve_api_management::request::downtimes::downtime_values,
openobserve_api_management::request::downtimes::get_downtime,
openobserve_api_management::request::downtimes::update_downtime,
openobserve_api_management::request::downtimes::cancel_downtime,
openobserve_api_management::request::downtimes::extend_downtime,
openobserve_api_management::request::downtimes::move_downtimes,
openobserve_api_management::request::downtimes::delete_downtime,
synthetics::list_synthetics,
synthetics::create_synthetic,
synthetics::get_synthetic,
@@ -427,6 +438,44 @@ use crate::{
openobserve_api_management::request::alerts::incidents::IncidentSeverity,
openobserve_api_management::request::alerts::incidents::IncidentStatus,
config::meta::alerts::incidents::Incident,
config::meta::downtimes::Downtime,
config::meta::downtimes::DowntimeRequest,
config::meta::downtimes::DowntimeListItem,
config::meta::downtimes::DowntimeDetail,
config::meta::downtimes::ListDowntimesResponse,
config::meta::downtimes::PreviewRequest,
config::meta::downtimes::PreviewResponse,
config::meta::downtimes::ResourcesRequest,
config::meta::downtimes::ResourcesResponse,
config::meta::downtimes::ValuesRequest,
config::meta::downtimes::ValuesResponse,
config::meta::downtimes::ValueSuggestion,
config::meta::downtimes::MoveDowntimesRequest,
config::meta::downtimes::CreateDowntimeResponse,
config::meta::downtimes::ExtendDowntimeRequest,
config::meta::downtimes::ExtendDowntimeResponse,
config::meta::downtimes::DowntimeTarget,
config::meta::downtimes::TargetFolders,
config::meta::downtimes::DimensionCondition,
config::meta::downtimes::LogicalOp,
config::meta::downtimes::PairOperator,
config::meta::downtimes::TargetModule,
config::meta::downtimes::DowntimeSchedule,
config::meta::downtimes::Repeat,
config::meta::downtimes::DowntimeStatus,
config::meta::downtimes::DowntimeWindow,
config::meta::downtimes::ResourceValue,
config::meta::downtimes::SloCorrectionMode,
config::meta::downtimes::IncidentMode,
config::meta::downtimes::PreviewMatch,
config::meta::downtimes::ActiveDowntime,
config::meta::downtimes::CorrectionRef,
config::meta::downtimes::DowntimeStatusCounts,
config::meta::downtimes::AffectedItems,
config::meta::downtimes::DowntimeNotifications,
config::meta::downtimes::NotificationEvents,
config::meta::downtimes::NotificationEvent,
config::meta::downtimes::DowntimeNotificationLogEntry,
config::meta::alerts::incidents::IncidentWithAlerts,
config::meta::alerts::incidents::IncidentAlert,
config::meta::alerts::incidents::IncidentStats,
@@ -593,6 +642,7 @@ use crate::{
(name = "Patterns", description = "Log pattern extraction operations (enterprise)"),
(name = "Service Streams", description = "Multi-signal correlation across logs, traces, and metrics (enterprise)"),
(name = "Synthetics", description = "Synthetic monitoring — uptime and browser checks (enterprise)"),
(name = "Downtimes", description = "Scheduled windows that keep evaluating alerts, anomaly detections, synthetics checks and SLOs but notify nobody (enterprise)"),
(name = "Announcements", description = "Operator-authored announcement banners shown across organizations (enterprise)"),
),
info(
@@ -1004,4 +1054,35 @@ mod tests {
assert!(tags.contains(&"Product Analytics"), "{tags:?}");
assert!(!tags.contains(&"RUM"), "{tags:?}");
}
#[test]
fn every_schema_ref_resolves() {
fn collect_refs<'a>(value: &'a serde_json::Value, refs: &mut Vec<&'a str>) {
match value {
serde_json::Value::Object(map) => {
if let Some(r) = map.get("$ref").and_then(|r| r.as_str()) {
refs.push(r);
}
map.values().for_each(|v| collect_refs(v, refs));
}
serde_json::Value::Array(items) => items.iter().for_each(|v| collect_refs(v, refs)),
_ => {}
}
}
let spec = serde_json::to_value(ApiDoc::openapi()).unwrap();
let schemas = spec["components"]["schemas"].as_object().unwrap();
let mut refs = Vec::new();
collect_refs(&spec, &mut refs);
let mut unresolved = refs
.into_iter()
.filter(|r| {
r.strip_prefix("#/components/schemas/")
.is_some_and(|name| !schemas.contains_key(name))
})
.collect::<Vec<_>>();
unresolved.sort_unstable();
unresolved.dedup();
assert!(unresolved.is_empty(), "unresolved refs: {unresolved:?}");
}
}
src/api/management/src/models/alerts/responses.rs
+189 -1
View File
@@ -13,7 +13,7 @@
// You should have received a copy of the GNU Affero General Public License
// along with this program. If not, see <http://www.gnu.org/licenses/>.
use std::str::FromStr;
use std::{collections::HashMap, str::FromStr};
use config::{
meta::{alerts::alert as meta_alerts, folder as meta_folders, triggers::Trigger},
@@ -168,6 +168,12 @@ pub struct ListAlertsResponseBodyItem {
// name or a KSUID through the summary.
#[serde(skip_serializing_if = "Option::is_none")]
pub expression_summary: Option<String>,
/// The downtime muting this row now, for the Muted chip. Enterprise only.
#[serde(default, skip_serializing_if = "Option::is_none")]
pub active_downtime: Option<config::meta::downtimes::ActiveDowntime>,
/// Anomaly rows only: the identity the anomaly scheduler matches downtimes on.
#[serde(skip)]
pub downtime_dimensions: HashMap<String, String>,
}
#[derive(Clone, Debug, Deserialize, Serialize, ToSchema)]
@@ -516,6 +522,8 @@ impl TryFrom<(meta_folders::Folder, meta_alerts::Alert, Option<Trigger>)>
child_count: None,
referenced_by_composite_count: None,
expression_summary: None,
active_downtime: None,
downtime_dimensions: HashMap::new(),
})
}
}
@@ -649,6 +657,135 @@ pub fn anomaly_config_to_list_item(v: &serde_json::Value) -> Option<ListAlertsRe
child_count: None,
referenced_by_composite_count: None,
expression_summary: None,
active_downtime: None,
downtime_dimensions: HashMap::new(),
})
}
/// The anomaly configs as list items, each with the identity its scheduler mutes it by.
#[cfg(feature = "enterprise")]
pub async fn anomaly_list_items(
org: &str,
configs: &[serde_json::Value],
) -> Vec<ListAlertsResponseBodyItem> {
// Only a downtime row turns the identity into a chip, so skip the SQL parsing without one.
let groups = if o2_enterprise::enterprise::common::config::get_config()
.downtimes
.enabled
&& !db::downtimes::list_cached(org).is_empty()
{
Some(db::system_settings::get_semantic_field_groups(org).await)
} else {
None
};
configs
.iter()
.filter_map(|v| {
let mut item = anomaly_config_to_list_item(v)?;
if let Some(groups) = &groups {
item.downtime_dimensions = anomaly_downtime_dimensions(v, &item.tags, groups);
}
Some(item)
})
.collect()
}
/// The Muted chip, from the cache or from the downtime the last firing recorded.
#[cfg(feature = "enterprise")]
pub async fn fill_active_downtimes(org: &str, list: &mut [ListAlertsResponseBodyItem]) {
if list.is_empty()
|| !o2_enterprise::enterprise::common::config::get_config()
.downtimes
.enabled
{
return;
}
// No downtime rows, no chip: neither branch below can yield one, so skip the state query.
let rows = db::downtimes::list_cached(org);
if rows.is_empty() {
return;
}
let now = config::utils::time::now_micros();
let ids: Vec<String> = list.iter().map(|item| item.alert_id.to_string()).collect();
let recorded = infra::table::alert_states::last_downtime_ids(&ids)
.await
.unwrap_or_default();
for item in list.iter_mut() {
item.active_downtime = list_item_downtime(org, item, now).or_else(|| {
recorded
.get(&item.alert_id.to_string())
.and_then(|id| still_active(&rows, id, now))
});
}
}
#[cfg(not(feature = "enterprise"))]
pub async fn fill_active_downtimes(_org: &str, _list: &mut [ListAlertsResponseBodyItem]) {}
#[cfg(feature = "enterprise")]
fn list_item_downtime(
org: &str,
item: &ListAlertsResponseBodyItem,
now: i64,
) -> Option<config::meta::downtimes::ActiveDowntime> {
use openobserve_core::alerts::downtimes::{active_for_alert, active_for_anomaly};
let id = item.alert_id.to_string();
let dims = list_item_dimensions(item);
match item.alert_type.as_str() {
"anomaly_detection" => active_for_anomaly(org, &id, &item.folder_id, &dims, now),
_ => active_for_alert(org, &id, &item.folder_id, &dims, now),
}
}
/// The identity a list row is matched on, the one its own scheduler matches.
#[cfg(feature = "enterprise")]
fn list_item_dimensions(item: &ListAlertsResponseBodyItem) -> HashMap<String, String> {
use o2_enterprise::enterprise::downtimes::scope::composite_dimensions;
match item.alert_type.as_str() {
"anomaly_detection" => item.downtime_dimensions.clone(),
"composite" => composite_dimensions(&item.tags),
_ => HashMap::new(),
}
}
/// The `anomaly_dimensions` of a config row, read from the keys the anomaly scheduler reads.
#[cfg(feature = "enterprise")]
fn anomaly_downtime_dimensions(
v: &serde_json::Value,
tags: &[String],
semantic_groups: &[config::meta::correlation::FieldAlias],
) -> HashMap<String, String> {
use openobserve_core::alerts::downtimes::{anomaly_filters, anomaly_sql};
let query_mode = v
.get("query_mode")
.and_then(|m| m.as_str())
.unwrap_or_default();
o2_enterprise::enterprise::downtimes::scope::anomaly_dimensions(
anomaly_sql(query_mode, v.get("custom_sql").and_then(|s| s.as_str())),
anomaly_filters(query_mode, v.get("filters")),
tags,
semantic_groups,
)
}
#[cfg(feature = "enterprise")]
fn still_active(
rows: &[config::meta::downtimes::Downtime],
downtime_id: &str,
now: i64,
) -> Option<config::meta::downtimes::ActiveDowntime> {
let row = rows
.iter()
.find(|row| row.id == downtime_id && row.cancelled_at.is_none())?;
let window = o2_enterprise::enterprise::downtimes::schedule::window_at(&row.schedule, now)?;
Some(config::meta::downtimes::ActiveDowntime {
id: row.id.clone(),
name: row.name.clone(),
ends_at: window.end,
incident_mode: Default::default(),
})
}
@@ -741,6 +878,8 @@ mod tests {
child_count: None,
referenced_by_composite_count: None,
expression_summary: None,
active_downtime: None,
downtime_dimensions: HashMap::new(),
};
let json = serde_json::to_value(&item).unwrap();
let obj = json.as_object().unwrap();
@@ -1171,4 +1310,53 @@ mod anomaly_priority_tag_tests {
assert_eq!(item.level, None);
assert_eq!(item.level_since, None);
}
#[cfg(feature = "enterprise")]
#[test]
fn an_anomaly_on_a_service_in_its_sql_shows_the_chip_of_that_service() {
use config::meta::{
correlation::FieldAlias,
downtimes::{Downtime, TargetModule},
};
use o2_enterprise::enterprise::downtimes::scope::{TargetItem, matches, target_for};
let groups = [FieldAlias {
id: "service".to_string(),
display: "Service".to_string(),
group: None,
fields: vec!["service".to_string()],
is_workload_type: false,
}];
let v = cfg(serde_json::json!({
"query_mode": "custom_sql",
"custom_sql": "SELECT count(*) AS c FROM \"logs\" WHERE service = 'payments'",
}));
let mut item = anomaly_config_to_list_item(&v).expect("should parse");
assert!(item.tags.is_empty());
item.downtime_dimensions = anomaly_downtime_dimensions(&v, &item.tags, &groups);
let downtime: Downtime = serde_json::from_value(serde_json::json!({
"id": "dt-1", "org": "default", "name": "payments maintenance",
"condition": { "type": "pair", "key": "service", "operator": "=", "value": "payments" },
"targets": [{ "module": "anomaly_detections", "folders": { "kind": "all" } }],
"schedule": {
"repeat": "none", "starts_at": 0, "timezone": "UTC", "duration_secs": 60
},
"created_by": "lin", "created_at": 0, "updated_by": "lin", "updated_at": 0
}))
.unwrap();
let target = target_for(&downtime.targets, TargetModule::AnomalyDetections).unwrap();
let id = item.alert_id.to_string();
let dims = list_item_dimensions(&item);
let row = TargetItem {
id: &id,
folder_id: &item.folder_id,
dimensions: &dims,
tags: &[],
};
assert!(
matches(target, downtime.condition.as_ref(), &row),
"{dims:?}"
);
}
}
src/api/management/src/models/destinations.rs
+6 -1
View File
@@ -148,7 +148,8 @@ impl From<meta_dest::Template> for Template {
// Shared rule lives in `config::prebuilt_loader` so the HTTP DTO and
// the service-layer guards can't drift apart.
let is_prebuilt = config::prebuilt_loader::is_prebuilt_template_name(&value.name);
let is_prebuilt = config::prebuilt_loader::is_prebuilt_template_name(&value.name)
|| openobserve_core::alerts::templates::is_downtime_template_name(&value.name);
Self {
name: value.name,
@@ -337,6 +338,8 @@ pub enum DestinationConsumerKind {
AnomalyDetection,
#[cfg(feature = "enterprise")]
IncidentIntegration,
#[cfg(feature = "enterprise")]
Downtime,
}
impl From<destination_usage::DestinationConsumer> for DestinationConsumerKind {
@@ -356,6 +359,8 @@ impl From<destination_usage::DestinationConsumer> for DestinationConsumerKind {
destination_usage::DestinationConsumer::IncidentIntegration => {
Self::IncidentIntegration
}
#[cfg(feature = "enterprise")]
destination_usage::DestinationConsumer::Downtime => Self::Downtime,
}
}
}
src/api/management/src/models/folders.rs
+11
View File
@@ -57,6 +57,7 @@ pub enum FolderType {
Synthetics,
Workflows,
Prompts,
Downtimes,
}
/// Common folder fields used in HTTP request and response bodies.
@@ -111,6 +112,7 @@ impl From<FolderType> for config::meta::folder::FolderType {
FolderType::Synthetics => Self::Synthetics,
FolderType::Workflows => Self::Workflows,
FolderType::Prompts => Self::Prompts,
FolderType::Downtimes => Self::Downtimes,
}
}
}
@@ -165,6 +167,15 @@ mod tests {
);
}
#[test]
fn the_downtimes_path_segment_maps_to_the_downtimes_folder_type() {
let parsed: FolderType = serde_json::from_str("\"downtimes\"").unwrap();
assert_eq!(
config::meta::folder::FolderType::from(parsed),
config::meta::folder::FolderType::Downtimes
);
}
#[test]
fn test_folder_from_config_preserves_fields() {
let src = config::meta::folder::Folder {
src/api/management/src/request/alerts/history.rs
+238 -7
View File
@@ -13,6 +13,7 @@
// You should have received a copy of the GNU Affero General Public License
// along with this program. If not, see <http://www.gnu.org/licenses/>.
use arrow_schema::Schema;
use axum::{
extract::{Path, Query},
response::Response,
@@ -40,6 +41,12 @@ use crate::common::{
meta::http::HttpResponse as MetaHttpResponse, utils::http::get_or_create_trace_id,
};
/// Every column the data query reads but `downtime_id`, which [history_columns] adds.
const HISTORY_COLUMNS: &str = "_timestamp, org, key, status, level, actual_value, \
threshold_value, threshold_operator, group_label, value_is_lower_bound, is_realtime, \
is_silenced, start_time, end_time, retries, delay_in_secs, evaluation_took_in_secs, \
source_node, query_took, error";
#[derive(Debug, Clone, Serialize, Deserialize)]
pub struct AlertHistoryQuery {
/// Filter by specific alert name
@@ -61,6 +68,10 @@ pub struct AlertHistoryQuery {
pub sort_by: Option<String>,
/// Sort order (asc or desc, default: desc)
pub sort_order: Option<String>,
/// Only runs this downtime suppressed; also reads synthetics and anomaly runs.
pub downtime_id: Option<String>,
/// Filter by run outcome, for example `suppressed`.
pub status: Option<String>,
}
#[derive(Debug, Clone, Serialize, Deserialize, ToSchema)]
@@ -118,6 +129,9 @@ pub struct AlertHistoryEntry {
/// §7.5) — the UI renders "≥ N". Absent = exact.
#[serde(skip_serializing_if = "Option::is_none")]
pub value_is_lower_bound: Option<bool>,
/// The downtime that suppressed this run.
#[serde(skip_serializing_if = "Option::is_none")]
pub downtime_id: Option<String>,
}
#[derive(Debug, Clone, Serialize, Deserialize, ToSchema)]
@@ -409,6 +423,7 @@ pub async fn get_alert_history(
threshold_operator: None,
group_label: None,
value_is_lower_bound: None,
downtime_id: None,
}
})
.collect();
@@ -558,9 +573,36 @@ pub async fn get_alert_history(
// Composites publish with module = "composite" and share the ordinary
// alert outcome vocabulary, so include them in the same history read.
// Anomaly detections are the `alerts` resource, so the alert check above already gates them.
let synthetics_readable =
query.downtime_id.is_some() && reads_every_synthetic(&org_id, &user_email.user_id).await;
let modules = history_modules(query.downtime_id.is_some(), synthetics_readable);
let triggers_schema =
match triggers_schema(infra::schema::get(&org_id, TRIGGERS_STREAM, StreamType::Logs).await)
{
Ok(schema) => schema,
Err(response) => return response,
};
if query
.downtime_id
.as_deref()
.is_some_and(|id| !id.is_empty())
&& triggers_schema.field_with_name("downtime_id").is_err()
{
return MetaHttpResponse::json(AlertHistoryResponse {
total: 0,
from,
size,
hits: vec![],
});
}
let mut where_clause = format!(
"module IN ('alert', 'composite') AND org = '{org_id}' AND _timestamp >= {start_time} AND _timestamp <= {end_time}"
"module IN ({modules}) AND org = '{org_id}' AND _timestamp >= {start_time} AND _timestamp <= {end_time}"
);
where_clause.push_str(&downtime_filters(
query.downtime_id.as_deref(),
query.status.as_deref(),
));
// Add alert ID filter if provided
// The key field contains the alert ID in the format "alert_name/alert_id"
@@ -657,13 +699,9 @@ pub async fn get_alert_history(
// Step 2: Get the actual paginated results
// Build data query with LIMIT/OFFSET for pagination
let columns = history_columns(&triggers_schema);
let data_sql = format!(
"SELECT _timestamp, org, key, status, level, actual_value, threshold_value, \
threshold_operator, group_label, value_is_lower_bound, is_realtime, is_silenced, \
start_time, end_time, retries, \
delay_in_secs, evaluation_took_in_secs, \
source_node, query_took, error \
FROM \"{TRIGGERS_STREAM}\" \
"SELECT {columns} FROM \"{TRIGGERS_STREAM}\" \
WHERE {where_clause} \
ORDER BY {sort_column} {sort_order} LIMIT {size} OFFSET {from}"
);
@@ -796,6 +834,11 @@ pub async fn get_alert_history(
.filter(|s| !s.is_empty())
.map(String::from),
value_is_lower_bound: hit.get("value_is_lower_bound").and_then(|v| v.as_bool()),
downtime_id: hit
.get("downtime_id")
.and_then(|v| v.as_str())
.filter(|s| !s.is_empty())
.map(String::from),
});
}
@@ -810,6 +853,109 @@ pub async fn get_alert_history(
MetaHttpResponse::json(response)
}
/// The modules a history read covers; a `?downtime_id=` read adds those a downtime can mute.
fn history_modules(by_downtime: bool, synthetics_readable: bool) -> String {
let mut modules = vec!["'alert'", "'composite'"];
if by_downtime {
modules.push("'anomaly_detection'");
if synthetics_readable {
modules.push("'synthetics'");
}
}
modules.join(", ")
}
/// Synthetics runs carry no per-check filter here, so only an org-wide synthetics read sees them.
#[cfg(feature = "enterprise")]
async fn reads_every_synthetic(org_id: &str, user_id: &str) -> bool {
if db::user::is_root_user(user_id) || !o2_openfga::config::get_config().enabled {
return true;
}
let object_type = o2_openfga::meta::mapping::OFGA_MODELS
.get("synthetics")
.map_or("synthetics", |model| model.key);
let listed = match openobserve_api_common::auth::validator::list_objects_for_user(
org_id,
user_id,
"GET",
object_type,
)
.await
{
Ok(listed) => listed,
Err(e) => {
log::warn!("synthetics read of {user_id} in {org_id} unreadable: {e}");
return false;
}
};
every_synthetic_readable(
listed.as_deref(),
&format!("{object_type}:_all_{org_id}"),
async || {
openobserve_core::auth::check_permissions(
org_id,
org_id,
user_id,
"synthetics",
"GET",
None,
true,
false,
false,
)
.await
},
)
.await
}
#[cfg(not(feature = "enterprise"))]
async fn reads_every_synthetic(_org_id: &str, _user_id: &str) -> bool {
true
}
/// `None` from ListObjects means no list filtering, not a grant, so the org-wide check decides.
#[cfg_attr(not(feature = "enterprise"), allow(dead_code))]
async fn every_synthetic_readable(
listed: Option<&[String]>,
all_key: &str,
org_wide_grant: impl AsyncFnOnce() -> bool,
) -> bool {
match listed {
Some(permitted) => permitted.iter().any(|object| object == all_key),
None => org_wide_grant().await,
}
}
/// The triggers stream's schema; a failed read is a server error, never an empty stream.
fn triggers_schema(read: infra::errors::Result<Schema>) -> Result<Schema, Response> {
read.map_err(|e| {
log::error!("Failed to read the alert history schema: {e}");
MetaHttpResponse::internal_error(format!("Failed to read the alert history schema: {e}"))
})
}
/// [HISTORY_COLUMNS], with `downtime_id` once the stream has it: an upgraded stream learns it late.
fn history_columns(schema: &Schema) -> String {
if schema.field_with_name("downtime_id").is_ok() {
format!("{HISTORY_COLUMNS}, downtime_id")
} else {
HISTORY_COLUMNS.to_string()
}
}
/// The downtime filters of the history read, quoted with the same single-quote escaping.
fn downtime_filters(downtime_id: Option<&str>, status: Option<&str>) -> String {
let mut clause = String::new();
if let Some(id) = downtime_id.filter(|id| !id.is_empty()) {
clause.push_str(&format!(" AND downtime_id = '{}'", id.replace('\'', "''")));
}
if let Some(status) = status.filter(|status| !status.is_empty()) {
clause.push_str(&format!(" AND status = '{}'", status.replace('\'', "''")));
}
clause
}
/// Bulk anomaly history query — returns the most recent N hits per anomaly config in one request.
///
/// Instead of the caller firing one request per config, this endpoint queries the triggers stream
@@ -1042,6 +1188,8 @@ mod tests {
size: None,
sort_by: None,
sort_order: None,
downtime_id: None,
status: None,
};
assert!(query.alert_id.is_none());
@@ -1084,6 +1232,7 @@ mod tests {
threshold_operator: None,
group_label: None,
value_is_lower_bound: None,
downtime_id: None,
};
assert_eq!(entry.alert_name, "test_alert");
@@ -1141,6 +1290,7 @@ mod tests {
threshold_operator: None,
group_label: None,
value_is_lower_bound: None,
downtime_id: None,
};
let response = AlertHistoryResponse {
@@ -1187,6 +1337,7 @@ mod tests {
threshold_operator: None,
group_label: None,
value_is_lower_bound: None,
downtime_id: None,
};
assert_eq!(entry.status, "error");
@@ -1208,6 +1359,8 @@ mod tests {
size: Some(25),
sort_by: None,
sort_order: None,
downtime_id: None,
status: None,
};
assert_eq!(query.from.unwrap(), 100);
@@ -1243,6 +1396,8 @@ mod tests {
size: None,
sort_by: Some(field.to_string()),
sort_order: Some("asc".to_string()),
downtime_id: None,
status: None,
};
assert_eq!(query.sort_by, Some(field.to_string()));
@@ -1262,6 +1417,8 @@ mod tests {
size: None,
sort_by: Some("timestamp".to_string()),
sort_order: Some("asc".to_string()),
downtime_id: None,
status: None,
};
assert_eq!(query_asc.sort_order, Some("asc".to_string()));
@@ -1275,6 +1432,8 @@ mod tests {
size: None,
sort_by: Some("timestamp".to_string()),
sort_order: Some("desc".to_string()),
downtime_id: None,
status: None,
};
assert_eq!(query_desc.sort_order, Some("desc".to_string()));
}
@@ -1407,6 +1566,7 @@ mod tests {
threshold_operator: None,
group_label: None,
value_is_lower_bound: None,
downtime_id: None,
};
let json = serde_json::to_string(&entry).unwrap();
@@ -1450,6 +1610,7 @@ mod tests {
threshold_operator: None,
group_label: None,
value_is_lower_bound: None,
downtime_id: None,
};
let response = AlertHistoryResponse {
@@ -1499,6 +1660,7 @@ mod tests {
threshold_operator: None,
group_label: None,
value_is_lower_bound: None,
downtime_id: None,
};
let json = serde_json::to_value(&entry).unwrap();
let obj = json.as_object().unwrap();
@@ -1541,6 +1703,7 @@ mod tests {
threshold_operator: None,
group_label: None,
value_is_lower_bound: None,
downtime_id: None,
};
let json = serde_json::to_value(&entry).unwrap();
let obj = json.as_object().unwrap();
@@ -1551,4 +1714,72 @@ mod tests {
assert!(obj.contains_key("group_size"));
assert!(obj.contains_key("anomaly_count"));
}
#[tokio::test]
async fn unfiltered_listing_is_not_a_synthetics_grant() {
let all = "synthetics:_all_acme";
assert!(!every_synthetic_readable(None, all, async || false).await);
assert!(every_synthetic_readable(None, all, async || true).await);
let listed = [all.to_string()];
assert!(every_synthetic_readable(Some(&listed), all, async || false).await);
let one = ["synthetics:c1".to_string()];
assert!(!every_synthetic_readable(Some(&one), all, async || true).await);
}
#[test]
fn a_user_without_synthetics_read_sees_no_synthetics_runs() {
assert_eq!(
history_modules(true, false),
"'alert', 'composite', 'anomaly_detection'"
);
assert_eq!(
history_modules(true, true),
"'alert', 'composite', 'anomaly_detection', 'synthetics'"
);
assert_eq!(history_modules(false, true), "'alert', 'composite'");
}
#[test]
fn downtime_filters_quote_and_combine() {
assert_eq!(downtime_filters(None, None), "");
assert_eq!(
downtime_filters(Some("2f9K"), None),
" AND downtime_id = '2f9K'"
);
assert_eq!(
downtime_filters(None, Some("suppressed")),
" AND status = 'suppressed'"
);
assert_eq!(
downtime_filters(Some("a'b"), Some("suppressed")),
" AND downtime_id = 'a''b' AND status = 'suppressed'"
);
assert_eq!(downtime_filters(Some(""), Some("")), "");
}
#[test]
fn a_failed_schema_read_is_a_server_error_not_an_empty_history() {
let failed = triggers_schema(Err(infra::errors::Error::Message("db down".to_string())));
assert_eq!(
failed.map(|_| ()).unwrap_err().status(),
axum::http::StatusCode::INTERNAL_SERVER_ERROR
);
let missing = triggers_schema(Ok(Schema::empty())).unwrap();
assert!(missing.field_with_name("downtime_id").is_err());
}
#[test]
fn a_stream_from_before_downtimes_reads_without_downtime_id() {
use arrow_schema::{DataType, Field};
let field = |name: &str| Field::new(name, DataType::Utf8, true);
let upgraded: Vec<Field> = HISTORY_COLUMNS.split(", ").map(field).collect();
let columns = history_columns(&Schema::new(upgraded.clone()));
assert!(!columns.contains("downtime_id"), "{columns}");
assert_eq!(columns, HISTORY_COLUMNS);
let mut current = upgraded;
current.push(field("downtime_id"));
assert!(history_columns(&Schema::new(current)).ends_with(", downtime_id"));
}
}
src/api/management/src/request/alerts/incidents.rs
+1
View File
@@ -1223,6 +1223,7 @@ mod tests {
updated_at: 2000,
group_values: serde_json::Value::Object(Default::default()),
key_type: config::meta::alerts::incidents::KeyType::default(),
muted_by_downtime_id: None,
};
let response = ListIncidentsResponse {
src/api/management/src/request/alerts/mod.rs
+8 -5
View File
@@ -49,7 +49,7 @@ use {
#[cfg(feature = "enterprise")]
use crate::models::alerts::requests::UpdateAnomalyAlertFields;
#[cfg(feature = "enterprise")]
use crate::models::alerts::responses::anomaly_config_to_list_item;
use crate::models::alerts::responses::anomaly_list_items;
use crate::{
common::meta::http::HttpResponse as MetaHttpResponse,
models::alerts::{
@@ -708,6 +708,8 @@ fn composite_list_item(
child_count: None,
referenced_by_composite_count: None,
expression_summary: None,
active_downtime: None,
downtime_dimensions: Default::default(),
})
}
@@ -2983,9 +2985,9 @@ pub async fn list_alerts(
// would return every anomaly config alongside the matching alerts.
let before = list.len();
list.extend(
configs
.iter()
.filter_map(anomaly_config_to_list_item)
anomaly_list_items(&org_id, &configs)
.await
.into_iter()
.filter(|item| enabled_filter.is_none_or(|enabled| item.enabled == enabled))
.filter(|item| match &priority_filter {
None => true,
@@ -3033,6 +3035,7 @@ pub async fn list_alerts(
let mut list = list;
enrich_with_run_state(&mut list).await;
enrich_with_composite_metadata(&org_id, &visibility, &mut list).await;
crate::models::alerts::responses::fill_active_downtimes(&org_id, &mut list).await;
// Feature-scoped fields: keep destinations/template off the default list path
// (bytes + module-scoped names) unless the dependency view explicitly opted in.
@@ -3665,7 +3668,7 @@ pub async fn trigger_alert(
}
let client = get_orm_client_ro().await;
match alert::trigger_by_id(client, &org_id, alert_id).await {
Ok(_) => MetaHttpResponse::ok("Alert triggered"),
Ok(triggered) => MetaHttpResponse::ok(triggered.response_message()),
Err(AlertError::AlertNotFound) => {
if let Some(_composite) =
openobserve_core::alerts::composite::get_composite(&org_id, &alert_id.to_string())
src/api/management/src/request/announcements/mod.rs
+15 -2
View File
@@ -163,10 +163,23 @@ pub async fn get_announcements(Path(org_id): Path<String>) -> Response {
}
};
let mut banners = announcements::resolve_for_org(&config, &org_id, now);
let mut next_boundary = next_boundary_for_org(&config, &org_id, now);
let downtimes = &o2_enterprise::enterprise::common::config::get_config().downtimes;
if downtimes.enabled && downtimes.banner_enabled {
match openobserve_core::downtimes::banners_for_org(&org_id, now).await {
Ok((generated, boundary)) => {
banners.extend(generated);
next_boundary = next_boundary.into_iter().chain(boundary).min();
}
Err(e) => log::error!("Error building downtime banners for {org_id}: {e}"),
}
}
MetaHttpResponse::json(AnnouncementsResponse {
banners: announcements::resolve_for_org(&config, &org_id, now),
banners,
now,
next_boundary: next_boundary_for_org(&config, &org_id, now),
next_boundary,
})
}
src/api/management/src/request/anomaly_detection/mod.rs
+54 -1
View File
@@ -83,7 +83,10 @@ pub async fn list_configs(Path(org_id): Path<String>) -> Response {
return resp;
}
match anomaly_service::list_configs(&org_id, None, None).await {
Ok(configs) => MetaHttpResponse::json(configs),
Ok(mut configs) => {
add_active_downtimes(&org_id, &mut configs).await;
MetaHttpResponse::json(configs)
}
Err(e) => {
tracing::error!("Failed to list anomaly configs: {}", e);
MetaHttpResponse::error(StatusCode::INTERNAL_SERVER_ERROR.as_u16(), e.to_string())
@@ -92,6 +95,56 @@ pub async fn list_configs(Path(org_id): Path<String>) -> Response {
}
}
/// Sets `active_downtime` on each config an active downtime's Anomaly detections target matches.
#[cfg(feature = "enterprise")]
async fn add_active_downtimes(org_id: &str, configs: &mut [serde_json::Value]) {
if !o2_enterprise::enterprise::common::config::get_config()
.downtimes
.enabled
|| db::downtimes::list_cached(org_id).is_empty()
{
return;
}
let semantic_groups = db::system_settings::get_semantic_field_groups(org_id).await;
let now = config::utils::time::now_micros();
for value in configs.iter_mut() {
let Some(obj) = value.as_object_mut() else {
continue;
};
let text = |key: &str| obj.get(key).and_then(|v| v.as_str()).map(str::to_string);
let (Some(anomaly_id), Some(folder_id)) = (text("anomaly_id"), text("folder_id")) else {
continue;
};
let tags: Vec<String> = obj
.get("tags")
.and_then(|t| serde_json::from_value(t.clone()).ok())
.unwrap_or_default();
let query_mode = text("query_mode").unwrap_or_default();
let dims = o2_enterprise::enterprise::downtimes::scope::anomaly_dimensions(
openobserve_core::alerts::downtimes::anomaly_sql(
&query_mode,
text("custom_sql").as_deref(),
),
openobserve_core::alerts::downtimes::anomaly_filters(&query_mode, obj.get("filters")),
&tags,
&semantic_groups,
);
if let Some(downtime) = openobserve_core::alerts::downtimes::active_for_anomaly(
org_id,
&anomaly_id,
&folder_id,
&dims,
now,
) && let Ok(active) = serde_json::to_value(downtime)
{
obj.insert("active_downtime".to_string(), active);
}
}
}
#[cfg(not(feature = "enterprise"))]
async fn add_active_downtimes(_org_id: &str, _configs: &mut [serde_json::Value]) {}
/// Get a specific anomaly detection configuration
#[utoipa::path(
get,
src/api/management/src/request/downtimes.rs
+576
View File
@@ -0,0 +1,576 @@
// Copyright 2026 OpenObserve Inc.
//
// This program is free software: you can redistribute it and/or modify
// it under the terms of the GNU Affero General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
//
// This program is distributed in the hope that it will be useful
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU Affero General Public License for more details.
//
// You should have received a copy of the GNU Affero General Public License
// along with this program. If not, see <http://www.gnu.org/licenses/>.
//! Downtime handlers: enterprise-only (D1), 403 in the OSS build and while the flag is off.
use axum::{
Json,
extract::{Path, Query},
response::Response,
};
use config::meta::downtimes::{
CreateDowntimeResponse, Downtime, DowntimeDetail, DowntimeRequest, ExtendDowntimeRequest,
ExtendDowntimeResponse, ListDowntimesQuery, ListDowntimesResponse, MoveDowntimesRequest,
PreviewRequest, PreviewResponse, ResourcesRequest, ResourcesResponse, ValuesRequest,
ValuesResponse,
};
use openobserve_api_common::extractors::Headers;
use serde::Deserialize;
use utoipa::IntoParams;
use crate::{common::meta::http::HttpResponse as MetaHttpResponse, service::auth::UserEmail};
/// The destination folder of a create, when given in the query as the route check reads it.
#[derive(Debug, Default, Deserialize, IntoParams)]
#[into_params(parameter_in = Query)]
pub struct FolderQuery {
pub folder: Option<String>,
}
/// ListDowntimes
#[utoipa::path(
get,
path = "/v2/{org_id}/downtimes",
context_path = "/api",
tag = "Downtimes",
operation_id = "ListDowntimes",
summary = "List downtimes",
description = "Lists the downtimes in the folders the user may list, with their status, windows and match counts.",
security(("Authorization" = [])),
params(("org_id" = String, Path, description = "Organization name"), ListDowntimesQuery),
responses(
(status = 200, description = "Success", content_type = "application/json", body = ListDowntimesResponse),
(status = 403, description = "Forbidden or not enabled", content_type = "application/json", body = ()),
),
)]
pub async fn list_downtimes(
Path(org_id): Path<String>,
Query(query): Query<ListDowntimesQuery>,
Headers(user_email): Headers<UserEmail>,
) -> Response {
#[cfg(feature = "enterprise")]
{
respond(openobserve_core::downtimes::list(&org_id, &user_email.user_id, &query).await)
}
#[cfg(not(feature = "enterprise"))]
{
let _ = (org_id, query, user_email);
not_supported()
}
}
/// CreateDowntime
#[utoipa::path(
post,
path = "/v2/{org_id}/downtimes",
context_path = "/api",
tag = "Downtimes",
operation_id = "CreateDowntime",
summary = "Create a downtime",
description = "Creates a downtime. Quick mute uses this route with one target and specific ids.",
security(("Authorization" = [])),
params(("org_id" = String, Path, description = "Organization name"), FolderQuery),
request_body(content = DowntimeRequest, description = "Downtime definition", content_type = "application/json"),
responses(
(status = 201, description = "Created", content_type = "application/json", body = CreateDowntimeResponse),
(status = 400, description = "Invalid request", content_type = "application/json", body = ()),
(status = 403, description = "Forbidden or not enabled", content_type = "application/json", body = ()),
),
)]
pub async fn create_downtime(
Path(org_id): Path<String>,
Query(folder): Query<FolderQuery>,
Headers(user_email): Headers<UserEmail>,
Json(req): Json<DowntimeRequest>,
) -> Response {
#[cfg(feature = "enterprise")]
{
let mut req = req;
if let Some(folder) = folder.folder.filter(|f| !f.is_empty()) {
req.folder_id = folder;
}
if req.folder_id.is_empty() {
req.folder_id = config::meta::folder::DEFAULT_FOLDER.to_string();
}
if !crate::service::auth::check_folder_write_permissions(
&org_id,
&user_email.user_id,
"downtime_folders",
&req.folder_id,
)
.await
{
return MetaHttpResponse::forbidden("You cannot create downtimes in this folder.");
}
match openobserve_core::downtimes::create(&org_id, &user_email.user_id, req).await {
Ok(downtime) => created(downtime.id),
Err(e) => error_response(e),
}
}
#[cfg(not(feature = "enterprise"))]
{
let _ = (org_id, folder, user_email, req);
not_supported()
}
}
/// PreviewDowntime
#[utoipa::path(
post,
path = "/v2/{org_id}/downtimes/preview",
context_path = "/api",
tag = "Downtimes",
operation_id = "PreviewDowntime",
summary = "Preview what a downtime covers",
description = "Lists, per module, what the targets match today, the alerts decided at fire time, and the items a resource refinement cannot decide. Names are limited to what the user may list; totals count everything.",
security(("Authorization" = [])),
params(("org_id" = String, Path, description = "Organization name")),
request_body(content = PreviewRequest, description = "Condition and targets", content_type = "application/json"),
responses(
(status = 200, description = "Success", content_type = "application/json", body = PreviewResponse),
(status = 403, description = "Forbidden or not enabled", content_type = "application/json", body = ()),
),
)]
pub async fn preview_downtime(
Path(org_id): Path<String>,
Headers(user_email): Headers<UserEmail>,
Json(req): Json<PreviewRequest>,
) -> Response {
#[cfg(feature = "enterprise")]
{
respond(openobserve_core::downtimes::preview(&org_id, &user_email.user_id, &req).await)
}
#[cfg(not(feature = "enterprise"))]
{
let _ = (org_id, user_email, req);
not_supported()
}
}
/// DowntimeResources
#[utoipa::path(
post,
path = "/v2/{org_id}/downtimes/resources",
context_path = "/api",
tag = "Downtimes",
operation_id = "DowntimeResources",
summary = "Resources behind a condition",
description = "The values of a finer dimension, such as host, that co-occur with the condition, for refining a downtime to some resources.",
security(("Authorization" = [])),
params(("org_id" = String, Path, description = "Organization name")),
request_body(content = ResourcesRequest, description = "Condition and the dimension to refine by", content_type = "application/json"),
responses(
(status = 200, description = "Success", content_type = "application/json", body = ResourcesResponse),
(status = 400, description = "Invalid request", content_type = "application/json", body = ()),
(status = 403, description = "Forbidden or not enabled", content_type = "application/json", body = ()),
),
)]
pub async fn downtime_resources(
Path(org_id): Path<String>,
Headers(user_email): Headers<UserEmail>,
Json(req): Json<ResourcesRequest>,
) -> Response {
#[cfg(feature = "enterprise")]
{
respond(openobserve_core::downtimes::resources(&org_id, &user_email.user_id, &req).await)
}
#[cfg(not(feature = "enterprise"))]
{
let _ = (org_id, user_email, req);
not_supported()
}
}
/// DowntimeValues
#[utoipa::path(
post,
path = "/v2/{org_id}/downtimes/values",
context_path = "/api",
tag = "Downtimes",
operation_id = "DowntimeValues",
summary = "Suggested values of a dimension",
description = "Values of one dimension for the condition value box and the resource picker, from the alerts, anomaly detections and SLOs, the service registry and, for a non-priority dimension, a one-hour search. `partial` is true when the search did not answer within 2 s.",
security(("Authorization" = [])),
params(("org_id" = String, Path, description = "Organization name")),
request_body(content = ValuesRequest, description = "The dimension, the typed prefix and the condition so far", content_type = "application/json"),
responses(
(status = 200, description = "Success", content_type = "application/json", body = ValuesResponse),
(status = 400, description = "Invalid request", content_type = "application/json", body = ()),
(status = 403, description = "Forbidden or not enabled", content_type = "application/json", body = ()),
),
)]
pub async fn downtime_values(
Path(org_id): Path<String>,
Headers(user_email): Headers<UserEmail>,
Json(req): Json<ValuesRequest>,
) -> Response {
#[cfg(feature = "enterprise")]
{
respond(openobserve_core::downtimes::values(&org_id, &user_email.user_id, &req).await)
}
#[cfg(not(feature = "enterprise"))]
{
let _ = (org_id, user_email, req);
not_supported()
}
}
/// GetDowntime
#[utoipa::path(
get,
path = "/v2/{org_id}/downtimes/{downtime_id}",
context_path = "/api",
tag = "Downtimes",
operation_id = "GetDowntime",
summary = "Get a downtime",
security(("Authorization" = [])),
params(
("org_id" = String, Path, description = "Organization name"),
("downtime_id" = String, Path, description = "Downtime id"),
),
responses(
(status = 200, description = "Success", content_type = "application/json", body = DowntimeDetail),
(status = 403, description = "Forbidden or not enabled", content_type = "application/json", body = ()),
(status = 404, description = "Not found", content_type = "application/json", body = ()),
),
)]
pub async fn get_downtime(
Path((org_id, downtime_id)): Path<(String, String)>,
Headers(user_email): Headers<UserEmail>,
) -> Response {
#[cfg(feature = "enterprise")]
{
respond(openobserve_core::downtimes::get(&org_id, &user_email.user_id, &downtime_id).await)
}
#[cfg(not(feature = "enterprise"))]
{
let _ = (org_id, downtime_id, user_email);
not_supported()
}
}
/// UpdateDowntime
#[utoipa::path(
put,
path = "/v2/{org_id}/downtimes/{downtime_id}",
context_path = "/api",
tag = "Downtimes",
operation_id = "UpdateDowntime",
summary = "Update a downtime",
description = "Edits the condition, targets, schedule, reason and banner. The folder changes only through move.",
security(("Authorization" = [])),
params(
("org_id" = String, Path, description = "Organization name"),
("downtime_id" = String, Path, description = "Downtime id"),
),
request_body(content = DowntimeRequest, description = "Downtime definition", content_type = "application/json"),
responses(
(status = 200, description = "Updated", content_type = "application/json", body = Downtime),
(status = 400, description = "Invalid request", content_type = "application/json", body = ()),
(status = 403, description = "Forbidden or not enabled", content_type = "application/json", body = ()),
(status = 404, description = "Not found", content_type = "application/json", body = ()),
),
)]
pub async fn update_downtime(
Path((org_id, downtime_id)): Path<(String, String)>,
Headers(user_email): Headers<UserEmail>,
Json(req): Json<DowntimeRequest>,
) -> Response {
#[cfg(feature = "enterprise")]
{
respond(
openobserve_core::downtimes::update(&org_id, &user_email.user_id, &downtime_id, req)
.await,
)
}
#[cfg(not(feature = "enterprise"))]
{
let _ = (org_id, downtime_id, user_email, req);
not_supported()
}
}
/// CancelDowntime
#[utoipa::path(
post,
path = "/v2/{org_id}/downtimes/{downtime_id}/cancel",
context_path = "/api",
tag = "Downtimes",
operation_id = "CancelDowntime",
summary = "End or cancel a downtime",
description = "Ends an active window now, or calls off a scheduled one. The row stays for history.",
security(("Authorization" = [])),
params(
("org_id" = String, Path, description = "Organization name"),
("downtime_id" = String, Path, description = "Downtime id"),
),
responses(
(status = 200, description = "Cancelled", content_type = "application/json", body = Downtime),
(status = 403, description = "Forbidden or not enabled", content_type = "application/json", body = ()),
(status = 404, description = "Not found", content_type = "application/json", body = ()),
),
)]
pub async fn cancel_downtime(
Path((org_id, downtime_id)): Path<(String, String)>,
Headers(user_email): Headers<UserEmail>,
) -> Response {
#[cfg(feature = "enterprise")]
{
respond(
openobserve_core::downtimes::cancel(&org_id, &user_email.user_id, &downtime_id).await,
)
}
#[cfg(not(feature = "enterprise"))]
{
let _ = (org_id, downtime_id, user_email);
not_supported()
}
}
/// ExtendDowntime
#[utoipa::path(
post,
path = "/v2/{org_id}/downtimes/{downtime_id}/extend",
context_path = "/api",
tag = "Downtimes",
operation_id = "ExtendDowntime",
summary = "Extend an active downtime",
description = "Moves the end of the active window by `by_secs` or to `until`, exactly one of them. A recurring downtime keeps its schedule; a one-time downtime named \"<name> (extended)\" is created from the end of the current window instead, and its id is returned as `created_id`.",
security(("Authorization" = [])),
params(
("org_id" = String, Path, description = "Organization name"),
("downtime_id" = String, Path, description = "Downtime id"),
),
request_body(content = ExtendDowntimeRequest, description = "The extension", content_type = "application/json"),
responses(
(status = 200, description = "Extended", content_type = "application/json", body = ExtendDowntimeResponse),
(status = 400, description = "Not active, or an invalid extension", content_type = "application/json", body = ()),
(status = 403, description = "Forbidden or not enabled", content_type = "application/json", body = ()),
(status = 404, description = "Not found", content_type = "application/json", body = ()),
(status = 409, description = "The downtime changed since it was read", content_type = "application/json", body = ()),
),
)]
pub async fn extend_downtime(
Path((org_id, downtime_id)): Path<(String, String)>,
Headers(user_email): Headers<UserEmail>,
Json(req): Json<ExtendDowntimeRequest>,
) -> Response {
#[cfg(feature = "enterprise")]
{
respond(
openobserve_core::downtimes::extend(&org_id, &user_email.user_id, &downtime_id, &req)
.await,
)
}
#[cfg(not(feature = "enterprise"))]
{
let _ = (org_id, downtime_id, user_email, req);
not_supported()
}
}
/// MoveDowntimes
#[utoipa::path(
patch,
path = "/v2/{org_id}/downtimes/move",
context_path = "/api",
tag = "Downtimes",
operation_id = "MoveDowntimes",
summary = "Move downtimes to another folder",
security(("Authorization" = [])),
params(("org_id" = String, Path, description = "Organization name")),
request_body(content = MoveDowntimesRequest, description = "Ids and destination folder", content_type = "application/json"),
responses(
(status = 200, description = "Moved"),
(status = 400, description = "Invalid request", content_type = "application/json", body = ()),
(status = 403, description = "Forbidden or not enabled", content_type = "application/json", body = ()),
),
)]
pub async fn move_downtimes(
Path(org_id): Path<String>,
Headers(user_email): Headers<UserEmail>,
Json(req): Json<MoveDowntimesRequest>,
) -> Response {
#[cfg(feature = "enterprise")]
{
// The same rule as `resolve_folder`: an empty destination is the default folder, so the
// permission is checked on the folder the rows will land in.
let mut req = req;
if req.dst_folder_id.is_empty() {
req.dst_folder_id = config::meta::folder::DEFAULT_FOLDER.to_string();
}
if !crate::service::auth::check_folder_write_permissions(
&org_id,
&user_email.user_id,
"downtime_folders",
&req.dst_folder_id,
)
.await
{
return MetaHttpResponse::forbidden("You cannot move downtimes into this folder.");
}
match openobserve_core::downtimes::move_to_folder(&org_id, &user_email.user_id, &req).await
{
Ok(()) => MetaHttpResponse::ok("downtimes moved"),
Err(e) => error_response(e),
}
}
#[cfg(not(feature = "enterprise"))]
{
let _ = (org_id, user_email, req);
not_supported()
}
}
/// DeleteDowntime
#[utoipa::path(
delete,
path = "/v2/{org_id}/downtimes/{downtime_id}",
context_path = "/api",
tag = "Downtimes",
operation_id = "DeleteDowntime",
summary = "Delete an ended or cancelled downtime",
security(("Authorization" = [])),
params(
("org_id" = String, Path, description = "Organization name"),
("downtime_id" = String, Path, description = "Downtime id"),
),
responses(
(status = 200, description = "Deleted"),
(status = 403, description = "Forbidden or not enabled", content_type = "application/json", body = ()),
(status = 404, description = "Not found", content_type = "application/json", body = ()),
(status = 409, description = "Active, scheduled, or it corrected SLO slices", content_type = "application/json", body = ()),
),
)]
pub async fn delete_downtime(
Path((org_id, downtime_id)): Path<(String, String)>,
Headers(user_email): Headers<UserEmail>,
) -> Response {
#[cfg(feature = "enterprise")]
{
match openobserve_core::downtimes::delete(&org_id, &user_email.user_id, &downtime_id).await
{
Ok(()) => MetaHttpResponse::ok("downtime deleted"),
Err(e) => error_response(e),
}
}
#[cfg(not(feature = "enterprise"))]
{
let _ = (org_id, downtime_id, user_email);
not_supported()
}
}
#[cfg(not(feature = "enterprise"))]
fn not_supported() -> Response {
MetaHttpResponse::forbidden("Not Supported")
}
#[cfg(feature = "enterprise")]
fn respond<T: serde::Serialize>(
result: Result<T, openobserve_core::downtimes::DowntimeError>,
) -> Response {
match result {
Ok(body) => MetaHttpResponse::json(body),
Err(e) => error_response(e),
}
}
#[cfg(feature = "enterprise")]
fn created(id: String) -> Response {
use axum::response::IntoResponse;
(
axum::http::StatusCode::CREATED,
Json(CreateDowntimeResponse { id }),
)
.into_response()
}
#[cfg(feature = "enterprise")]
fn error_response(e: openobserve_core::downtimes::DowntimeError) -> Response {
use openobserve_core::downtimes::DowntimeError;
match e {
DowntimeError::Disabled => MetaHttpResponse::forbidden("Downtimes are not enabled"),
DowntimeError::NotFound => MetaHttpResponse::not_found("Downtime not found"),
DowntimeError::BadRequest(m) => MetaHttpResponse::bad_request(m),
DowntimeError::Forbidden(m) => MetaHttpResponse::forbidden(m),
DowntimeError::Conflict(m) => MetaHttpResponse::conflict(m),
// The text can carry database errors, so it goes to the log only.
DowntimeError::Internal(m) => {
log::error!("[DOWNTIMES] {m}");
MetaHttpResponse::internal_error("Something went wrong, see the server log")
}
}
}
#[cfg(test)]
mod tests {
use axum::http::StatusCode;
use super::*;
#[cfg(not(feature = "enterprise"))]
fn user() -> Headers<UserEmail> {
Headers(UserEmail {
user_id: "root@example.com".to_string(),
})
}
#[cfg(not(feature = "enterprise"))]
#[tokio::test]
async fn every_route_is_forbidden_in_the_oss_build() {
let org = || Path("default".to_string());
let one = || Path(("default".to_string(), "d1".to_string()));
let body: DowntimeRequest = serde_json::from_value(serde_json::json!({
"targets": [{ "module": "alerts", "folders": { "kind": "all" } }],
"schedule": { "repeat": "none", "starts_at": 1, "ends_at": 2, "timezone": "UTC", "duration_secs": 60 }
}))
.unwrap();
let responses = vec![
list_downtimes(org(), Query(ListDowntimesQuery::default()), user()).await,
create_downtime(
org(),
Query(FolderQuery::default()),
user(),
Json(body.clone()),
)
.await,
get_downtime(one(), user()).await,
update_downtime(one(), user(), Json(body)).await,
cancel_downtime(one(), user()).await,
extend_downtime(one(), user(), Json(ExtendDowntimeRequest::default())).await,
delete_downtime(one(), user()).await,
];
for response in responses {
assert_eq!(response.status(), StatusCode::FORBIDDEN);
}
}
#[cfg(feature = "enterprise")]
#[tokio::test]
async fn an_internal_error_answers_without_the_database_text() {
let db_text = "error returned from database: relation \"downtimes\" does not exist";
let response = error_response(openobserve_core::downtimes::DowntimeError::Internal(
db_text.to_string(),
));
assert_eq!(response.status(), StatusCode::INTERNAL_SERVER_ERROR);
let body = axum::body::to_bytes(response.into_body(), usize::MAX)
.await
.unwrap();
let body = String::from_utf8_lossy(&body);
assert!(!body.contains("database"), "{body}");
assert!(body.contains("see the server log"), "{body}");
}
}
src/api/management/src/request/folders.rs
+65 -3
View File
@@ -52,6 +52,9 @@ fn folder_error_response(value: FolderError) -> Response {
FolderError::DeleteWithWorkflows => MetaHttpResponse::bad_request(
"Folder contains workflows, please move/delete workflows from folder",
),
FolderError::DeleteWithDowntimes => MetaHttpResponse::bad_request(
"Folder contains downtimes, please move/delete downtimes from folder",
),
FolderError::DeleteWithPrompts => MetaHttpResponse::bad_request(
"Folder contains prompts, please move/delete prompts from folder",
),
@@ -64,6 +67,25 @@ fn folder_error_response(value: FolderError) -> Response {
}
}
/// An old node never meets the downtimes folder type while the feature is off.
fn refused_folder_type(
folder_type: config::meta::folder::FolderType,
downtimes_enabled: bool,
) -> Option<Response> {
(folder_type == config::meta::folder::FolderType::Downtimes && !downtimes_enabled)
.then(|| MetaHttpResponse::forbidden("Downtimes are not enabled"))
}
#[cfg(feature = "enterprise")]
fn downtimes_enabled() -> bool {
openobserve_core::downtimes::ensure_enabled().is_ok()
}
#[cfg(not(feature = "enterprise"))]
fn downtimes_enabled() -> bool {
false
}
/// CreateFolder
#[utoipa::path(
post,
@@ -102,8 +124,12 @@ pub async fn create_folder(
Path((org_id, folder_type)): Path<(String, FolderType)>,
axum::Json(body): axum::Json<CreateFolderRequestBody>,
) -> Response {
let folder_type: config::meta::folder::FolderType = folder_type.into();
if let Some(refused) = refused_folder_type(folder_type, downtimes_enabled()) {
return refused;
}
let folder = body.into();
match folders::save_folder(&org_id, folder, folder_type.into(), false).await {
match folders::save_folder(&org_id, folder, folder_type, false).await {
Ok(folder) => {
let body: CreateFolderResponseBody = folder.into();
MetaHttpResponse::json(body)
@@ -196,8 +222,16 @@ pub async fn list_folders(
#[cfg(feature = "enterprise")]
let user_id = Some(user_email.user_id.as_str());
match folders::list_folders(&org_id, user_id, folder_type.into()).await {
let folder_type = folder_type.into();
match folders::list_folders(&org_id, user_id, folder_type).await {
Ok(folders) => {
#[cfg(feature = "enterprise")]
let folders = if folder_type == config::meta::folder::FolderType::Downtimes {
openobserve_core::downtimes::listable_folders(&org_id, &user_email.user_id, folders)
.await
} else {
folders
};
let body: ListFoldersResponseBody = folders.into();
MetaHttpResponse::json(body)
}
@@ -269,11 +303,29 @@ pub async fn get_folder(
("x-o2-mcp" = json!({"description": "Get folder by name", "category": "folders"}))
),
)]
#[allow(unused_variables)]
pub async fn get_folder_by_name(
Path((org_id, folder_type, folder_name)): Path<(String, FolderType, String)>,
#[cfg(feature = "enterprise")] Headers(user_email): Headers<UserEmail>,
) -> Response {
match folders::get_folder_by_name(&org_id, &folder_name, folder_type.into()).await {
let folder_type = folder_type.into();
match folders::get_folder_by_name(&org_id, &folder_name, folder_type).await {
Ok(folder) => {
#[cfg(feature = "enterprise")]
let folder = if folder_type == config::meta::folder::FolderType::Downtimes {
let listed = openobserve_core::downtimes::listable_folders(
&org_id,
&user_email.user_id,
vec![folder],
)
.await;
match listed.into_iter().next() {
Some(folder) => folder,
None => return MetaHttpResponse::forbidden("Unauthorized Access"),
}
} else {
folder
};
let body: CreateFolderResponseBody = folder.into();
MetaHttpResponse::json(body)
}
@@ -602,6 +654,16 @@ mod tests {
}
}
#[test]
fn a_downtimes_folder_is_forbidden_while_the_flag_is_off() {
use config::meta::folder::FolderType as Stored;
let refused = refused_folder_type(Stored::Downtimes, false).expect("refused");
assert_eq!(refused.status().as_u16(), 403);
assert!(refused_folder_type(Stored::Downtimes, true).is_none());
assert!(refused_folder_type(Stored::Alerts, false).is_none());
}
#[test]
fn test_folder_error_infra_error_is_500() {
let err = FolderError::InfraError(infra::errors::Error::Message("db down".to_string()));
src/api/management/src/request/mod.rs
+1
View File
@@ -34,6 +34,7 @@ pub mod db_monitoring;
pub mod discovery;
#[cfg(feature = "enterprise")]
pub mod domain_management;
pub mod downtimes;
#[cfg(feature = "enterprise")]
pub mod eval_jobs;
#[cfg(feature = "enterprise")]
src/api/management/src/request/status/mod.rs
+40 -1
View File
@@ -217,6 +217,7 @@ struct ConfigResponse<'a> {
query_values_default_num: i64,
alert_preview_timerange_minutes: i64,
incidents_enabled: bool,
downtimes_enabled: bool,
service_streams_enabled: bool,
model_pricing_enabled: bool,
online_evals_enabled: bool,
@@ -482,6 +483,7 @@ pub async fn zo_config(
// AI needs o2-ai to answer; with no agent target its buttons could only fail, so report it off.
let ai_enabled = enterprise_value!(false, o2cfg.ai.enabled && o2cfg.ai.has_agent_target());
let incidents_enabled = enterprise_value!(false, o2cfg.incidents.enabled);
let downtimes_enabled = enterprise_value!(false, o2cfg.downtimes.enabled);
let service_streams_enabled = enterprise_value!(false, o2cfg.service_streams.enabled);
// Anomaly detection is on when the enterprise feature is compiled in, unless turned off at
// runtime via O2_ANOMALY_DETECTION_DISABLED. When disabled the UI hides the anomaly tab.
@@ -614,6 +616,7 @@ pub async fn zo_config(
query_values_default_num: cfg.limit.query_values_default_num,
alert_preview_timerange_minutes: cfg.limit.alert_preview_timerange_minutes,
incidents_enabled,
downtimes_enabled,
service_streams_enabled,
model_pricing_enabled: cfg.common.model_pricing_enabled,
online_evals_enabled,
@@ -1838,6 +1841,7 @@ const CACHE_MODULES: &[&str] = &[
"realtime_triggers",
"org_users",
"compact_retention",
"downtimes",
];
// Helper function to reload cache for a specific module
@@ -1857,10 +1861,31 @@ async fn reload_module_cache(module: &str) -> Result<(), anyhow::Error> {
"org_users" => db::org_users::cache().await,
"org_ingestion_tokens" => db::org_ingestion_tokens::cache().await,
"compact_retention" => db::compact::retention::cache().await,
"downtimes" => {
reload_downtimes(enterprise_value!(false, get_o2_config().downtimes.enabled)).await
}
_ => Err(anyhow::anyhow!("unsupported module")),
}
}
/// `all` lists downtimes only with the flag on, so the reply and total match main otherwise.
fn all_cache_modules(downtimes_enabled: bool) -> Vec<&'static str> {
CACHE_MODULES
.iter()
.copied()
.filter(|module| downtimes_enabled || *module != "downtimes")
.collect()
}
/// With the flag off the cache stays empty, so no evaluation or SLO pass reads a downtime.
async fn reload_downtimes(enabled: bool) -> Result<(), anyhow::Error> {
if enabled {
db::downtimes::cache().await
} else {
Ok(())
}
}
pub async fn cache_reload(
Query(query): Query<std::collections::HashMap<String, String>>,
) -> Response {
@@ -1880,7 +1905,8 @@ pub async fn cache_reload(
// Expand "all" to all available modules
if modules.contains(&"all") {
modules = CACHE_MODULES.to_vec();
let downtimes_enabled = enterprise_value!(false, get_o2_config().downtimes.enabled);
modules = all_cache_modules(downtimes_enabled);
}
let total_modules = modules.len();
@@ -1932,6 +1958,19 @@ mod tests {
use super::*;
#[test]
fn with_downtimes_off_all_names_the_modules_main_names() {
assert!(!all_cache_modules(false).contains(&"downtimes"));
assert_eq!(all_cache_modules(false).len(), CACHE_MODULES.len() - 1);
assert!(all_cache_modules(true).contains(&"downtimes"));
}
#[tokio::test]
async fn with_downtimes_off_a_cache_reload_loads_no_downtime() {
reload_downtimes(false).await.unwrap();
assert!(db::downtimes::all_cached().is_empty());
}
#[cfg(feature = "enterprise")]
#[tokio::test]
async fn test_config_rbac_enabled_matches_enforcement_before_any_usage_report() {
src/api/management/src/request/synthetics/last_check.rs
+1
View File
@@ -194,6 +194,7 @@ mod tests {
referenced_by: 0,
references: None,
reference_state: None,
active_downtime: None,
}
}
src/api/management/src/request/synthetics/mod.rs
+13 -2
View File
@@ -447,6 +447,7 @@ pub async fn list_synthetics(
match openobserve_synthetics::service::list_synthetics(&org_id, &params).await {
Ok(mut resp) => {
last_check::enrich(&org_id, &mut resp.checks).await;
openobserve_core::synthetics::fill_active_downtimes(&org_id, &mut resp.checks);
MetaHttpResponse::json(resp)
}
Err(e) => {
@@ -1577,12 +1578,20 @@ async fn process_ack(
key: format!("{}/{}", resp.synthetics_name, resp.synthetics_id),
start_time: checked_at,
end_time: checked_at,
status: config::meta::self_reporting::usage::RunOutcome::Succeeded,
status: if resp.suppressed_by.is_some() {
config::meta::self_reporting::usage::RunOutcome::Suppressed
} else {
config::meta::self_reporting::usage::RunOutcome::Succeeded
},
downtime_id: resp.suppressed_by.clone(),
success_response: Some(status.clone()),
error: error.clone(),
evaluation_took_in_secs: Some(response_time_ms / 1000.0),
..Default::default()
});
if resp.suppressed_by.is_some() {
openobserve_core::alerts::alert::count_suppressed_run(&resp.org_id, "synthetics");
}
// Notify once per run, not once per job ack — and only when the check's own
// `alert_if_fails` / `cooldown_mins` settings say so. This used to fire on
@@ -1600,7 +1609,8 @@ async fn process_ack(
#[cfg(feature = "enterprise")]
let degraded = matches!(resp.alert, AlertDecision::Degraded);
#[cfg(feature = "enterprise")]
let should_notify = !matches!(resp.alert, AlertDecision::Silent);
let should_notify =
!matches!(resp.alert, AlertDecision::Silent) && resp.suppressed_by.is_none();
// Enterprise-gated because alert *destinations* are: the dispatch it ends in
// (`alerts::alert::dispatch_notification`) is not built in OSS. An OSS build
// runs the check, records the run and serves the result — it just has
@@ -2281,6 +2291,7 @@ mod tests {
passing_locations: Vec::new(),
failing_environments: Vec::new(),
usage_events,
suppressed_by: None,
}
}
src/common/src/meta/organization.rs
+2 -1
View File
@@ -334,7 +334,8 @@ impl TriggerStatus {
usage::RunOutcome::Firing
| usage::RunOutcome::Normal
| usage::RunOutcome::Succeeded
| usage::RunOutcome::Pending => status.healthy += 1,
| usage::RunOutcome::Pending
| usage::RunOutcome::Suppressed => status.healthy += 1,
usage::RunOutcome::Error | usage::RunOutcome::NotifyFailed => {
status.failed += 1
}
src/config/src/config.rs
+3 -1
View File
@@ -98,7 +98,9 @@ pub type RwBTreeMap<K, V> = tokio::sync::RwLock<BTreeMap<K, V>>;
// 98: key alert_dedup_state by (org_id, fingerprint).
// 99: add name to rum_pa_tombstones.
// 100: rule mode and raw transition values require new columns.
pub const DB_SCHEMA_VERSION: u64 = 100;
// 101: create downtimes; muted_by_downtime_id, last_downtime_id, slo_backfill_jobs kind/attempts.
// 102: create downtime_notifications.
pub const DB_SCHEMA_VERSION: u64 = 102;
pub const DB_SCHEMA_KEY: &str = "/db_schema_version/";
// global version variables
src/config/src/meta/alerts/dispatch.rs
+131 -28
View File
@@ -110,6 +110,8 @@ pub struct DispatchPlan {
/// failures rather than routine outcomes: reported for logging, and
/// deliberately NOT dispatched — see [`plan_dispatch`].
pub inconsistent: Vec<String>,
/// Groups that would deliver but a downtime mutes; also counted in `suppressed`.
pub muted: Vec<String>,
}
/// Decide which groups this evaluation notifies (MN-1, MN-8).
@@ -132,6 +134,7 @@ pub struct DispatchPlan {
/// key twice would otherwise notify the group twice and consume two slots of the MN-8 budget.
/// * `max_sends` is the MN-8 knob: `0` = unlimited; otherwise it counts **qualifying** sends
/// (applied after `delivery_decision`), worst-first.
/// * A deliverable group for which `muted` is true goes to `muted` instead, before the knob.
pub fn plan_dispatch(
classification: &GroupClassification,
states: &HashMap<String, AlertState>,
@@ -139,12 +142,14 @@ pub fn plan_dispatch(
tc: &TriggerCondition,
now: i64,
max_sends: usize,
muted: impl Fn(&str) -> bool,
) -> DispatchPlan {
let mut items = Vec::new();
let mut suppressed = 0usize;
let mut pending = 0;
let mut dropped_by_knob = Vec::new();
let mut inconsistent = Vec::new();
let mut muted_keys = Vec::new();
// A key appearing twice in one classification is an invariant violation
// (validation rejects the known source, MN-12). Counting first means both
@@ -207,6 +212,12 @@ pub fn plan_dispatch(
};
match decision {
DeliveryDecision::Deliver | DeliveryDecision::DeliverEscalation => {
// A muted group is suppressed by its downtime before the knob, so it fills no slot.
if muted(key) {
suppressed += 1;
muted_keys.push(key.clone());
continue;
}
// The knob counts QUALIFYING sends, so a silenced group never
// spends budget a deliverable one could have used.
if max_sends > 0 && items.len() >= max_sends {
@@ -223,9 +234,9 @@ pub fn plan_dispatch(
row: row.clone(),
});
}
DeliveryDecision::SuppressedBySilence | DeliveryDecision::SuppressedByWarningPolicy => {
suppressed += 1
}
DeliveryDecision::SuppressedBySilence
| DeliveryDecision::SuppressedByWarningPolicy
| DeliveryDecision::SuppressedByDowntime(_) => suppressed += 1,
DeliveryDecision::SuppressedByPending => pending += 1,
DeliveryDecision::NotFiring => {}
}
@@ -237,6 +248,7 @@ pub fn plan_dispatch(
pending,
dropped_by_knob,
inconsistent,
muted: muted_keys,
}
}
@@ -722,7 +734,15 @@ mod tests {
let c = classified(&[("a", 150.0), ("b", 75.0), ("c", 10.0)], 500);
let states = states_for(&c, 1_000 * SEC, None, None);
let plan = plan_dispatch(&c, &states, &rows_for(&c), &tc(None), 1_000 * SEC, 0);
let plan = plan_dispatch(
&c,
&states,
&rows_for(&c),
&tc(None),
1_000 * SEC,
0,
|_| false,
);
assert_eq!(
plan.items.len(),
@@ -746,7 +766,7 @@ mod tests {
let c = classified(&[("crit", 150.0), ("ok1", 1.0), ("ok2", 2.0)], 500);
let states = states_for(&c, now, None, None);
let plan = plan_dispatch(&c, &states, &rows_for(&c), &tc(None), now, 0);
let plan = plan_dispatch(&c, &states, &rows_for(&c), &tc(None), now, 0, |_| false);
assert_eq!(plan.items.len(), 1);
assert_eq!(
@@ -766,7 +786,7 @@ mod tests {
let c = classified(&[("a", 150.0)], 500);
let states = states_for(&c, now, None, None);
let plan = plan_dispatch(&c, &states, &rows_for(&c), &tc(None), now, 0);
let plan = plan_dispatch(&c, &states, &rows_for(&c), &tc(None), now, 0, |_| false);
assert!(
plan.items[0].escalation,
@@ -782,7 +802,15 @@ mod tests {
let c = classified(&[("a", 150.0), ("b", 75.0)], 500);
let states = states_for(&c, 1_000 * SEC, None, None);
let plan = plan_dispatch(&c, &states, &rows_for(&c), &tc(None), 1_000 * SEC, 0);
let plan = plan_dispatch(
&c,
&states,
&rows_for(&c),
&tc(None),
1_000 * SEC,
0,
|_| false,
);
let b = plan
.items
.iter()
@@ -799,7 +827,15 @@ mod tests {
let c = classified(&[("warn", 75.0), ("crit2", 150.0), ("crit1", 150.0)], 500);
let states = states_for(&c, 1_000 * SEC, None, None);
let plan = plan_dispatch(&c, &states, &rows_for(&c), &tc(None), 1_000 * SEC, 0);
let plan = plan_dispatch(
&c,
&states,
&rows_for(&c),
&tc(None),
1_000 * SEC,
0,
|_| false,
);
let keys: Vec<_> = plan.items.iter().map(|i| i.group_key.clone()).collect();
assert_eq!(plan.items[0].level, AlertLevel::Critical);
@@ -820,7 +856,7 @@ mod tests {
let c = classified(&[("a", 150.0)], 500);
let states = states_for(&c, now, Some(AlertLevel::Critical), Some(now + 10 * MIN));
let plan = plan_dispatch(&c, &states, &rows_for(&c), &tc(None), now, 0);
let plan = plan_dispatch(&c, &states, &rows_for(&c), &tc(None), now, 0, |_| false);
assert!(plan.items.is_empty());
assert_eq!(plan.suppressed, 1);
@@ -834,7 +870,7 @@ mod tests {
let c = classified(&[("a", 150.0)], 500);
let states = states_for(&c, now, Some(AlertLevel::Critical), Some(now - 1));
let plan = plan_dispatch(&c, &states, &rows_for(&c), &tc(None), now, 0);
let plan = plan_dispatch(&c, &states, &rows_for(&c), &tc(None), now, 0, |_| false);
assert_eq!(plan.items.len(), 1);
assert!(
@@ -851,7 +887,7 @@ mod tests {
let c = classified(&[("a", 150.0)], 500);
let states = states_for(&c, now, Some(AlertLevel::Warning), Some(now + 10 * MIN));
let plan = plan_dispatch(&c, &states, &rows_for(&c), &tc(None), now, 0);
let plan = plan_dispatch(&c, &states, &rows_for(&c), &tc(None), now, 0, |_| false);
assert_eq!(plan.items.len(), 1);
assert!(plan.items[0].escalation);
@@ -884,7 +920,7 @@ mod tests {
assert!(row.is_firing(), "NotifyFailed is still a firing state");
let states = HashMap::from([(key.clone(), row)]);
let plan = plan_dispatch(&c, &states, &rows_for(&c), &tc(None), now, 0);
let plan = plan_dispatch(&c, &states, &rows_for(&c), &tc(None), now, 0, |_| false);
assert_eq!(
plan.items.len(),
@@ -900,7 +936,15 @@ mod tests {
let c = classified(&[("crit", 150.0), ("warn", 75.0)], 500);
let states = states_for(&c, 1_000 * SEC, None, None);
let plan = plan_dispatch(&c, &states, &rows_for(&c), &tc(Some(false)), 1_000 * SEC, 0);
let plan = plan_dispatch(
&c,
&states,
&rows_for(&c),
&tc(Some(false)),
1_000 * SEC,
0,
|_| false,
);
assert_eq!(plan.items.len(), 1);
assert_eq!(plan.items[0].group_key, key_of("crit"));
@@ -940,7 +984,7 @@ mod tests {
]),
);
let plan = plan_dispatch(&c, &states, &rows, &tc(None), now, 0);
let plan = plan_dispatch(&c, &states, &rows, &tc(None), now, 0, |_| false);
assert_eq!(plan.items.len(), 2, "only retained groups dispatch");
assert!(
@@ -971,7 +1015,7 @@ mod tests {
state(ROLLUP_GROUP_KEY, AlertLevel::Critical, now, None, None),
);
let plan = plan_dispatch(&c, &states, &rows_for(&c), &tc(None), now, 0);
let plan = plan_dispatch(&c, &states, &rows_for(&c), &tc(None), now, 0, |_| false);
assert_eq!(plan.items.len(), 1, "one group fired, so one page");
assert_eq!(plan.items[0].group_key, key_of("a"));
@@ -999,7 +1043,7 @@ mod tests {
state(&gone, AlertLevel::Critical, now - 5 * MIN, None, None),
);
let plan = plan_dispatch(&c, &states, &rows_for(&c), &tc(None), now, 0);
let plan = plan_dispatch(&c, &states, &rows_for(&c), &tc(None), now, 0, |_| false);
assert!(!plan.items.iter().any(|i| i.group_key == gone));
}
@@ -1014,7 +1058,15 @@ mod tests {
let now = 1_000 * SEC;
let c = classified(&[("a", 150.0)], 500);
let plan = plan_dispatch(&c, &HashMap::new(), &rows_for(&c), &tc(None), now, 0);
let plan = plan_dispatch(
&c,
&HashMap::new(),
&rows_for(&c),
&tc(None),
now,
0,
|_| false,
);
assert!(plan.items.is_empty(), "no durable row, no page");
assert_eq!(
@@ -1033,7 +1085,7 @@ mod tests {
let c = classified(&[("a", 150.0)], 500);
let states = states_for(&c, now, None, None);
let plan = plan_dispatch(&c, &states, &HashMap::new(), &tc(None), now, 0);
let plan = plan_dispatch(&c, &states, &HashMap::new(), &tc(None), now, 0, |_| false);
assert!(plan.items.is_empty());
assert_eq!(plan.inconsistent, vec![key_of("a")]);
@@ -1060,7 +1112,7 @@ mod tests {
assert_eq!(c.groups.len(), 3, "fixture: `a` is observed twice");
let states = states_for(&c, now, None, None);
let plan = plan_dispatch(&c, &states, &rows_for(&c), &tc(None), now, 0);
let plan = plan_dispatch(&c, &states, &rows_for(&c), &tc(None), now, 0, |_| false);
assert!(
!plan.items.iter().any(|i| i.group_key == key_of("a")),
@@ -1089,7 +1141,7 @@ mod tests {
assert_eq!(c.groups.len(), 3, "fixture: `a` observed at two levels");
let states = states_for(&c, now, None, None);
let plan = plan_dispatch(&c, &states, &rows_for(&c), &tc(None), now, 0);
let plan = plan_dispatch(&c, &states, &rows_for(&c), &tc(None), now, 0, |_| false);
for item in &plan.items {
assert_ne!(
@@ -1119,7 +1171,7 @@ mod tests {
);
let states = states_for(&c, now, None, None);
let plan = plan_dispatch(&c, &states, &rows_for(&c), &tc(None), now, 2);
let plan = plan_dispatch(&c, &states, &rows_for(&c), &tc(None), now, 2, |_| false);
assert_eq!(plan.items.len(), 2, "`b` and `c` both page");
assert!(plan.dropped_by_knob.is_empty());
@@ -1134,7 +1186,7 @@ mod tests {
let states = states_for(&c, now, None, None);
let rows = rows_for(&c);
let plan = plan_dispatch(&c, &states, &rows, &tc(None), now, 0);
let plan = plan_dispatch(&c, &states, &rows, &tc(None), now, 0, |_| false);
assert_eq!(
&plan.items[0].row,
@@ -1158,7 +1210,7 @@ mod tests {
row.level_at = Some(now); // freshness advanced by a later same-level run
let states = HashMap::from([(key.clone(), row)]);
let plan = plan_dispatch(&c, &states, &rows_for(&c), &tc(None), now, 0);
let plan = plan_dispatch(&c, &states, &rows_for(&c), &tc(None), now, 0, |_| false);
assert_eq!(
plan.items[0].episode,
@@ -1176,7 +1228,15 @@ mod tests {
fn test_knob_zero_is_unlimited() {
let c = classified(&[("a", 150.0), ("b", 150.0), ("c", 150.0)], 500);
let states = states_for(&c, 1_000 * SEC, None, None);
let plan = plan_dispatch(&c, &states, &rows_for(&c), &tc(None), 1_000 * SEC, 0);
let plan = plan_dispatch(
&c,
&states,
&rows_for(&c),
&tc(None),
1_000 * SEC,
0,
|_| false,
);
assert_eq!(plan.items.len(), 3);
assert!(plan.dropped_by_knob.is_empty());
}
@@ -1188,7 +1248,15 @@ mod tests {
let c = classified(&[("crit", 150.0), ("warn1", 75.0), ("warn2", 75.0)], 500);
let states = states_for(&c, 1_000 * SEC, None, None);
let plan = plan_dispatch(&c, &states, &rows_for(&c), &tc(None), 1_000 * SEC, 2);
let plan = plan_dispatch(
&c,
&states,
&rows_for(&c),
&tc(None),
1_000 * SEC,
2,
|_| false,
);
assert_eq!(plan.items.len(), 2);
assert_eq!(
@@ -1221,8 +1289,10 @@ mod tests {
let c = build();
let states = states_for(&c, now, None, None);
let first = plan_dispatch(&c, &states, &rows_for(&c), &tc(None), now, 2);
let second = plan_dispatch(&build(), &states, &rows_for(&c), &tc(None), now, 2);
let first = plan_dispatch(&c, &states, &rows_for(&c), &tc(None), now, 2, |_| false);
let second = plan_dispatch(&build(), &states, &rows_for(&c), &tc(None), now, 2, |_| {
false
});
assert_eq!(first.items, second.items);
assert_eq!(first.dropped_by_knob, second.dropped_by_knob);
@@ -1254,7 +1324,7 @@ mod tests {
),
]);
let plan = plan_dispatch(&c, &states, &rows_for(&c), &tc(None), now, 1);
let plan = plan_dispatch(&c, &states, &rows_for(&c), &tc(None), now, 1, |_| false);
assert_eq!(plan.items.len(), 1);
assert_eq!(plan.items[0].group_key, key_b);
@@ -1265,6 +1335,39 @@ mod tests {
);
}
#[test]
fn test_a_muted_group_does_not_spend_the_knob() {
let now = 1_000 * SEC;
let c = classified(&[("a", 150.0), ("b", 75.0)], 500);
let (key_a, key_b) = (key_of("a"), key_of("b"));
let states = states_for(&c, now, None, None);
for _ in 0..2 {
let plan = plan_dispatch(&c, &states, &rows_for(&c), &tc(None), now, 1, |k| {
k == key_a
});
assert_eq!(plan.items.len(), 1);
assert_eq!(plan.items[0].group_key, key_b);
assert_eq!(plan.suppressed, 1);
assert_eq!(plan.muted, vec![key_a.clone()]);
assert!(plan.dropped_by_knob.is_empty());
}
}
#[test]
fn test_a_muted_group_with_knob_zero_is_suppressed_and_the_rest_deliver() {
let now = 1_000 * SEC;
let c = classified(&[("a", 150.0), ("b", 75.0)], 500);
let (key_a, key_b) = (key_of("a"), key_of("b"));
let states = states_for(&c, now, None, None);
let plan = plan_dispatch(&c, &states, &rows_for(&c), &tc(None), now, 0, |k| {
k == key_a
});
assert_eq!(plan.items.len(), 1);
assert_eq!(plan.items[0].group_key, key_b);
assert_eq!(plan.muted, vec![key_a]);
assert!(plan.dropped_by_knob.is_empty());
}
// ═════════════════════════════════════════════════════════════════════
// delivery_success_update — MN-6
// ═════════════════════════════════════════════════════════════════════
src/config/src/meta/alerts/incidents.rs
+18
View File
@@ -384,6 +384,10 @@ pub struct Incident {
#[serde(skip_serializing_if = "Option::is_none")]
pub topology_context: Option<IncidentTopology>,
/// Set while a downtime mutes this incident (D3).
#[serde(default, skip_serializing_if = "Option::is_none")]
pub muted_by_downtime_id: Option<String>,
}
/// Alert info within an incident (junction table representation)
@@ -1872,6 +1876,7 @@ mod tests {
group_values: serde_json::Value::Null,
key_type: KeyType::default(),
topology_context,
muted_by_downtime_id: None,
}
}
@@ -1886,6 +1891,19 @@ mod tests {
assert!(!obj.contains_key("topology_context"));
assert!(!obj.contains_key("acknowledged_by"));
assert!(!obj.contains_key("acknowledged_at"));
assert!(!obj.contains_key("muted_by_downtime_id"));
}
#[test]
fn a_muted_incident_names_its_downtime_and_old_rows_read_unmuted() {
let mut incident = make_incident(None, None, None, None);
incident.muted_by_downtime_id = Some("2f9K".to_string());
let json = serde_json::to_value(&incident).unwrap();
assert_eq!(json["muted_by_downtime_id"], "2f9K");
let mut old = json.as_object().unwrap().clone();
old.remove("muted_by_downtime_id");
let back: Incident = serde_json::from_value(serde_json::Value::Object(old)).unwrap();
assert_eq!(back.muted_by_downtime_id, None);
}
#[test]
src/config/src/meta/alerts/level.rs
+19 -1
View File
@@ -348,7 +348,7 @@ pub fn level_for_completed_evaluation(
}
/// What a completed evaluation should do about notification (§7.1).
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
#[derive(Debug, Clone, PartialEq, Eq)]
pub enum DeliveryDecision {
/// Deliver, and (re)start the silence window from now.
Deliver,
@@ -362,6 +362,8 @@ pub enum DeliveryDecision {
NotFiring,
/// Currently in pending state
SuppressedByPending,
/// Condition matched inside a window of this downtime.
SuppressedByDowntime(String),
}
impl DeliveryDecision {
@@ -373,6 +375,13 @@ impl DeliveryDecision {
pub fn resets_silence(&self) -> bool {
self.should_deliver()
}
pub fn downtime_id(&self) -> Option<&str> {
match self {
Self::SuppressedByDowntime(id) => Some(id),
_ => None,
}
}
}
/// Decide whether this evaluation delivers a notification.
@@ -1096,6 +1105,15 @@ mod tests {
assert!(!d.should_deliver());
}
#[test]
fn a_downtime_suppression_never_delivers_and_names_its_downtime() {
let d = DeliveryDecision::SuppressedByDowntime("2f9K".to_string());
assert!(!d.should_deliver());
assert!(!d.resets_silence());
assert_eq!(d.downtime_id(), Some("2f9K"));
assert_eq!(DeliveryDecision::Deliver.downtime_id(), None);
}
// ── D11: notify_on_warning ──────────────────────────────────────────────
#[test]
src/config/src/meta/alerts/mod.rs
+5
View File
@@ -367,6 +367,11 @@ impl TriggerCondition {
.max(REALTIME_MIN_SILENCE_SECS)
.saturating_mul(1_000_000)
}
/// The realtime floor alone, without the alert's own silence.
pub fn min_silence_micros() -> i64 {
REALTIME_MIN_SILENCE_SECS * 1_000_000
}
}
impl TriggerCondition {
src/config/src/meta/downtimes.rs
+942
View File
@@ -0,0 +1,942 @@
// Copyright 2026 OpenObserve Inc.
//
// This program is free software: you can redistribute it and/or modify
// it under the terms of the GNU Affero General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
//
// This program is distributed in the hope that it will be useful
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU Affero General Public License for more details.
//
// You should have received a copy of the GNU Affero General Public License
// along with this program. If not, see <http://www.gnu.org/licenses/>.
use std::collections::HashMap;
use serde::{Deserialize, Serialize};
use utoipa::{IntoParams, ToSchema};
use crate::meta::folder::DEFAULT_FOLDER;
pub const DEFAULT_ENDING_SOON_LEAD_SECS: i64 = 600;
#[derive(Clone, Debug, PartialEq, Serialize, Deserialize, ToSchema)]
pub struct Downtime {
pub id: String,
pub org: String,
#[serde(default = "default_folder")]
pub folder_id: String,
pub name: String,
#[serde(default, skip_serializing_if = "Option::is_none")]
pub reason: Option<String>,
/// One per downtime (D20). Applied to every target whose module has an identity.
#[serde(default, skip_serializing_if = "Option::is_none")]
pub condition: Option<DimensionCondition>,
pub targets: Vec<DowntimeTarget>,
pub schedule: DowntimeSchedule,
#[serde(default, skip_serializing_if = "Option::is_none")]
pub cancelled_at: Option<i64>,
#[serde(default, skip_serializing_if = "Option::is_none")]
pub cancelled_by: Option<String>,
/// Show a warning banner to everyone in the org while a window is active (D19).
#[serde(default = "default_true")]
pub show_banner: bool,
#[serde(default, skip_serializing_if = "Option::is_none")]
pub notifications: Option<DowntimeNotifications>,
#[serde(default, skip_serializing_if = "Option::is_none")]
pub origin_region: Option<String>,
#[serde(default)]
pub version: i64,
pub created_by: String,
pub created_at: i64,
pub updated_by: String,
pub updated_at: i64,
}
impl Downtime {
/// Same items covered: only the condition and the targets decide which items match.
pub fn same_coverage(&self, other: &Downtime) -> bool {
self.condition == other.condition && self.targets == other.targets
}
}
/// At most one per module (D14); every present field narrows the module's items (D15).
#[derive(Clone, Debug, PartialEq, Serialize, Deserialize, ToSchema)]
pub struct DowntimeTarget {
pub module: TargetModule,
pub folders: TargetFolders,
/// Synthetics only.
#[serde(default, skip_serializing_if = "Vec::is_empty")]
pub tags: Vec<String>,
#[serde(default, skip_serializing_if = "Vec::is_empty")]
pub ids: Vec<String>,
/// Read only when `module == Slos` (D10).
#[serde(default, skip_serializing_if = "Option::is_none")]
pub slo_mode: Option<SloCorrectionMode>,
/// Read only when `module == Alerts`; a row saved before the field reads as `Muted`.
#[serde(default, skip_serializing_if = "IncidentMode::is_muted")]
pub incident_mode: IncidentMode,
}
#[derive(Clone, Debug, PartialEq, Serialize, Deserialize, ToSchema)]
#[serde(tag = "kind", rename_all = "snake_case")]
pub enum TargetFolders {
All,
Some { folder_ids: Vec<String> },
}
/// A tree of dimension tests (D17); one AND group of `=` pairs is an on-call `OwnershipRule`.
#[derive(Clone, Debug, PartialEq, Serialize, Deserialize, ToSchema)]
#[serde(tag = "type", rename_all = "snake_case")]
pub enum DimensionCondition {
Group {
op: LogicalOp,
// An opaque item keeps the schema acyclic: the MCP tool generator rejects self-references.
#[schema(value_type = Vec<Object>)]
items: Vec<DimensionCondition>,
},
/// `value` may end in `*`, a prefix, as in ownership rules.
Pair {
key: String,
operator: PairOperator,
value: String,
},
}
#[derive(Clone, Copy, Debug, PartialEq, Eq, Serialize, Deserialize, ToSchema)]
#[serde(rename_all = "snake_case")]
pub enum LogicalOp {
And,
Or,
}
#[derive(Clone, Copy, Debug, PartialEq, Eq, Serialize, Deserialize, ToSchema)]
pub enum PairOperator {
#[serde(rename = "=")]
Eq,
#[serde(rename = "!=")]
Ne,
}
#[derive(Clone, Copy, Debug, PartialEq, Eq, Hash, Serialize, Deserialize, ToSchema)]
#[serde(rename_all = "snake_case")]
pub enum TargetModule {
Alerts,
AnomalyDetections,
Synthetics,
Slos,
}
impl TargetModule {
pub fn has_identity(self) -> bool {
!matches!(self, Self::Synthetics)
}
pub fn has_tags(self) -> bool {
matches!(self, Self::Synthetics)
}
}
#[derive(Clone, Debug, PartialEq, Serialize, Deserialize, ToSchema)]
pub struct DowntimeSchedule {
pub repeat: Repeat,
/// Microseconds UTC.
pub starts_at: i64,
/// Microseconds UTC; a recurring row reads only its local date, an inclusive last day.
#[serde(default, skip_serializing_if = "Option::is_none")]
pub ends_at: Option<i64>,
pub timezone: String,
/// `HH:MM` in `timezone`, recurring rows only.
#[serde(default, skip_serializing_if = "Option::is_none")]
pub start_time_local: Option<String>,
pub duration_secs: i64,
/// ISO weekdays, 1 is Monday.
#[serde(default)]
pub weekdays: Vec<u8>,
}
#[derive(Clone, Copy, Debug, Default, PartialEq, Eq, Serialize, Deserialize, ToSchema)]
#[serde(rename_all = "snake_case")]
pub enum Repeat {
#[default]
None,
Daily,
Weekly,
}
impl Repeat {
/// Stored in `downtimes.repeat`; never reorder, 3 is reserved for monthly.
pub fn to_i16(self) -> i16 {
match self {
Self::None => 0,
Self::Daily => 1,
Self::Weekly => 2,
}
}
pub fn from_i16(value: i16) -> Option<Self> {
match value {
0 => Some(Self::None),
1 => Some(Self::Daily),
2 => Some(Self::Weekly),
_ => None,
}
}
}
#[derive(Clone, Copy, Debug, PartialEq, Eq, Serialize, Deserialize, ToSchema)]
#[serde(rename_all = "snake_case")]
pub enum DowntimeStatus {
Scheduled,
Active,
Ended,
Cancelled,
/// Cancelled inside a window, which it cut short.
EndedEarly,
}
#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize, ToSchema)]
pub struct DowntimeWindow {
pub start: i64,
pub end: i64,
}
#[derive(Clone, Debug, Serialize, ToSchema)]
pub struct DowntimeListItem {
#[serde(flatten)]
pub downtime: Downtime,
pub status: DowntimeStatus,
pub current_window: Option<DowntimeWindow>,
pub next_window: Option<DowntimeWindow>,
pub matched_alerts: usize,
pub matched_anomalies: usize,
pub matched_synthetics: usize,
pub matched_slos: usize,
}
#[derive(Clone, Debug, Deserialize, ToSchema)]
pub struct DowntimeRequest {
#[serde(default = "default_folder")]
pub folder_id: String,
#[serde(default)]
pub name: Option<String>,
#[serde(default)]
pub reason: Option<String>,
#[serde(default)]
pub condition: Option<DimensionCondition>,
pub targets: Vec<DowntimeTarget>,
pub schedule: DowntimeSchedule,
#[serde(default = "default_true")]
pub show_banner: bool,
#[serde(default)]
pub notifications: Option<DowntimeNotifications>,
/// The version the edit was loaded at; a newer stored row makes the edit a 409.
#[serde(default)]
pub version: Option<i64>,
}
#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize, ToSchema)]
pub struct DowntimeNotifications {
#[serde(default)]
pub destinations: Vec<String>,
#[serde(default)]
pub events: NotificationEvents,
#[serde(default = "default_ending_soon_lead_secs")]
pub ending_soon_lead_secs: i64,
/// The id of the recurring downtime whose window this extension follow-up continues.
#[serde(default, skip_serializing_if = "Option::is_none")]
pub continues: Option<String>,
}
impl Default for DowntimeNotifications {
fn default() -> Self {
Self {
destinations: vec![],
events: NotificationEvents::default(),
ending_soon_lead_secs: DEFAULT_ENDING_SOON_LEAD_SECS,
continues: None,
}
}
}
#[derive(Clone, Copy, Debug, Default, PartialEq, Eq, Serialize, Deserialize, ToSchema)]
pub struct NotificationEvents {
#[serde(default)]
pub started: bool,
#[serde(default)]
pub ending_soon: bool,
#[serde(default)]
pub ended: bool,
#[serde(default)]
pub cancelled: bool,
#[serde(default)]
pub extended: bool,
}
impl NotificationEvents {
pub fn any(&self) -> bool {
self.started || self.ending_soon || self.ended || self.cancelled || self.extended
}
pub fn wants(&self, event: NotificationEvent) -> bool {
match event {
NotificationEvent::Started => self.started,
NotificationEvent::EndingSoon => self.ending_soon,
NotificationEvent::Ended => self.ended,
NotificationEvent::Cancelled => self.cancelled,
NotificationEvent::Extended => self.extended,
}
}
}
#[derive(Clone, Copy, Debug, PartialEq, Eq, Hash, Serialize, Deserialize, ToSchema)]
#[serde(rename_all = "snake_case")]
pub enum NotificationEvent {
Started,
EndingSoon,
Ended,
Cancelled,
Extended,
}
impl NotificationEvent {
/// Stored in `downtime_notifications.event`; never rename.
pub fn as_str(self) -> &'static str {
match self {
Self::Started => "started",
Self::EndingSoon => "ending_soon",
Self::Ended => "ended",
Self::Cancelled => "cancelled",
Self::Extended => "extended",
}
}
}
#[derive(Clone, Debug, PartialEq, Eq, Serialize, ToSchema)]
pub struct DowntimeNotificationLogEntry {
pub window_start: i64,
pub event: String,
pub sent_at: i64,
pub destinations: Vec<String>,
#[serde(skip_serializing_if = "Option::is_none")]
pub result: Option<String>,
}
#[derive(Clone, Debug, Deserialize, ToSchema)]
pub struct ResourcesRequest {
pub condition: DimensionCondition,
pub refine_by: String,
}
#[derive(Clone, Debug, Serialize, ToSchema)]
pub struct ResourceValue {
pub value: String,
pub last_seen: i64,
/// Flattened from `ServiceStreams` as `<type>/<name>`.
pub streams: Vec<String>,
}
#[derive(Clone, Debug, Serialize, ToSchema)]
pub struct ResourcesResponse {
pub dimension: String,
pub values: Vec<ResourceValue>,
pub total: usize,
pub source: String,
}
#[derive(Clone, Debug, Deserialize, ToSchema)]
pub struct ValuesRequest {
/// A semantic group id, as `refine_by` in `ResourcesRequest`.
pub key: String,
/// Case-insensitive prefix; empty lists the top values.
#[serde(default)]
pub prefix: String,
/// The condition typed so far; only its And-spine pairs narrow the sources.
#[serde(default)]
pub condition: Option<DimensionCondition>,
}
#[derive(Clone, Debug, PartialEq, Eq, Serialize, ToSchema)]
pub struct ValueSuggestion {
pub value: String,
/// `inventory`, `registry` or `search`.
pub source: String,
/// Alerts, anomaly detections and SLOs whose identity carries this value.
pub items: usize,
/// Newest sighting in microseconds, when a source knows one.
#[serde(skip_serializing_if = "Option::is_none")]
pub last_seen: Option<i64>,
}
#[derive(Clone, Debug, PartialEq, Eq, Serialize, ToSchema)]
pub struct ValuesResponse {
pub values: Vec<ValueSuggestion>,
/// True when a source timed out; the list is what answered in time.
pub partial: bool,
}
#[derive(Clone, Copy, Debug, Default, PartialEq, Eq, Serialize, Deserialize, ToSchema)]
#[serde(rename_all = "snake_case")]
pub enum SloCorrectionMode {
#[default]
Exclude,
CountAsGood,
}
/// What a suppressed alert firing does to incidents during the window.
#[derive(Clone, Copy, Debug, Default, PartialEq, Eq, Serialize, Deserialize, ToSchema)]
#[serde(rename_all = "snake_case")]
pub enum IncidentMode {
#[default]
Muted,
None,
}
impl IncidentMode {
pub fn is_muted(&self) -> bool {
matches!(self, Self::Muted)
}
}
#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)]
pub struct CorrectionWindow {
pub downtime_id: String,
pub start: i64,
pub end: i64,
pub mode: SloCorrectionMode,
}
#[derive(Clone, Debug, Deserialize, ToSchema)]
pub struct PreviewRequest {
#[serde(default)]
pub condition: Option<DimensionCondition>,
pub targets: Vec<DowntimeTarget>,
}
#[derive(Clone, Debug, PartialEq, Serialize, ToSchema)]
pub struct PreviewMatch {
pub id: String,
pub name: String,
pub folder_id: String,
#[serde(skip_serializing_if = "Option::is_none")]
pub matched_by: Option<String>,
#[serde(skip_serializing_if = "Option::is_none")]
pub missing: Option<String>,
/// SLOs on the detail page: false when the current or next window holds no slice start.
#[serde(skip_serializing_if = "Option::is_none")]
pub applies: Option<bool>,
}
#[derive(Clone, Debug, Default, Serialize, ToSchema)]
pub struct PreviewResponse {
pub alerts: Vec<PreviewMatch>,
pub resolved_at_fire_time: Vec<PreviewMatch>,
pub anomalies: Vec<PreviewMatch>,
pub synthetics: Vec<PreviewMatch>,
pub slos: Vec<PreviewMatch>,
pub alerts_total: usize,
pub anomalies_total: usize,
pub synthetics_total: usize,
pub slos_total: usize,
pub undecidable: HashMap<String, Vec<PreviewMatch>>,
}
#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize, ToSchema)]
pub struct ActiveDowntime {
pub id: String,
pub name: String,
pub ends_at: i64,
#[serde(default, skip_serializing_if = "IncidentMode::is_muted")]
pub incident_mode: IncidentMode,
}
#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize, ToSchema)]
pub struct CorrectionRef {
pub downtime_id: String,
pub name: String,
pub status: DowntimeStatus,
/// False when the window holds no slice start, so it corrects no minute of this SLO.
#[serde(default = "default_true")]
pub applies: bool,
}
/// Query of `GET /v2/{org}/downtimes`.
#[derive(Clone, Debug, Default, Deserialize, IntoParams)]
#[into_params(parameter_in = Query)]
pub struct ListDowntimesQuery {
/// Absent means every folder the user may list.
pub folder_id: Option<String>,
/// `scheduled`, `active`, `ended`, `ended_early` or `cancelled`.
pub status: Option<String>,
/// `none`, `daily`, `weekly`, or `recurring` for both.
pub repeat: Option<String>,
pub search: Option<String>,
/// Rows whose alerts target names this alert, or whose condition matches its dimensions.
pub alert_id: Option<String>,
pub page: Option<usize>,
pub page_size: Option<usize>,
}
#[derive(Clone, Debug, Default, PartialEq, Eq, Serialize, ToSchema)]
pub struct DowntimeStatusCounts {
pub active: usize,
pub scheduled: usize,
pub recurring: usize,
pub ended: usize,
pub cancelled: usize,
pub ended_early: usize,
}
#[derive(Clone, Debug, Serialize, ToSchema)]
pub struct ListDowntimesResponse {
pub items: Vec<DowntimeListItem>,
pub total: usize,
pub counts: DowntimeStatusCounts,
}
/// `GET /v2/{org}/downtimes/{id}`: the list item and what it covers, as far as the user may see.
#[derive(Clone, Debug, Serialize, ToSchema)]
pub struct DowntimeDetail {
#[serde(flatten)]
pub item: DowntimeListItem,
pub affected: AffectedItems,
pub notification_log: Vec<DowntimeNotificationLogEntry>,
}
#[derive(Clone, Debug, Default, Serialize, ToSchema)]
pub struct AffectedItems {
pub alerts: Vec<PreviewMatch>,
pub resolved_at_fire_time: Vec<PreviewMatch>,
pub anomalies: Vec<PreviewMatch>,
pub synthetics: Vec<PreviewMatch>,
pub slos: Vec<PreviewMatch>,
}
#[derive(Clone, Debug, Deserialize, ToSchema)]
pub struct MoveDowntimesRequest {
pub downtime_ids: Vec<String>,
pub dst_folder_id: String,
}
#[derive(Clone, Debug, Serialize, ToSchema)]
pub struct CreateDowntimeResponse {
pub id: String,
}
/// `POST /v2/{org}/downtimes/{id}/extend`: exactly one of `by_secs` and `until`.
#[derive(Clone, Debug, Default, Deserialize, ToSchema)]
pub struct ExtendDowntimeRequest {
/// Added to the later of the current end and now.
#[serde(default)]
pub by_secs: Option<i64>,
/// The new end, microseconds UTC.
#[serde(default)]
pub until: Option<i64>,
/// The version the row was loaded at; a newer stored row makes the extension a 409.
#[serde(default)]
pub version: Option<i64>,
}
/// The row that now ends later: the extended one-time row, or the follow-up of a recurring one.
#[derive(Clone, Debug, Serialize, ToSchema)]
pub struct ExtendDowntimeResponse {
#[serde(flatten)]
pub downtime: Downtime,
/// Set when a recurring row got a one-time follow-up instead of a longer window.
#[serde(skip_serializing_if = "Option::is_none")]
pub created_id: Option<String>,
}
pub fn default_folder() -> String {
DEFAULT_FOLDER.to_string()
}
fn default_true() -> bool {
true
}
fn default_ending_soon_lead_secs() -> i64 {
DEFAULT_ENDING_SOON_LEAD_SECS
}
#[cfg(test)]
mod tests {
use super::*;
fn minimal() -> Downtime {
Downtime {
id: "2f9KQe4b7Nq1vH3sT0mLzXpRcWd".to_string(),
org: "default".to_string(),
folder_id: DEFAULT_FOLDER.to_string(),
name: "Staging teardown".to_string(),
reason: None,
condition: None,
targets: vec![DowntimeTarget {
module: TargetModule::Alerts,
folders: TargetFolders::All,
tags: vec![],
ids: vec![],
slo_mode: None,
incident_mode: Default::default(),
}],
schedule: DowntimeSchedule {
repeat: Repeat::None,
starts_at: 1_789_855_200_000_000,
ends_at: Some(1_789_905_600_000_000),
timezone: "UTC".to_string(),
start_time_local: None,
duration_secs: 50_400,
weekdays: vec![],
},
cancelled_at: None,
cancelled_by: None,
show_banner: true,
notifications: None,
origin_region: None,
version: 0,
created_by: "lin".to_string(),
created_at: 1,
updated_by: "lin".to_string(),
updated_at: 1,
}
}
fn full() -> Downtime {
Downtime {
reason: Some("CHG-4471".to_string()),
condition: Some(DimensionCondition::Group {
op: LogicalOp::And,
items: vec![
DimensionCondition::Pair {
key: "service".to_string(),
operator: PairOperator::Eq,
value: "payments".to_string(),
},
DimensionCondition::Group {
op: LogicalOp::Or,
items: vec![DimensionCondition::Pair {
key: "host".to_string(),
operator: PairOperator::Ne,
value: "db-*".to_string(),
}],
},
],
}),
targets: vec![
DowntimeTarget {
module: TargetModule::Synthetics,
folders: TargetFolders::Some {
folder_ids: vec!["4Kc1staging".to_string()],
},
tags: vec!["service:payments".to_string()],
ids: vec!["syn_41".to_string()],
slo_mode: None,
incident_mode: Default::default(),
},
DowntimeTarget {
module: TargetModule::Slos,
folders: TargetFolders::All,
tags: vec![],
ids: vec![],
slo_mode: Some(SloCorrectionMode::CountAsGood),
incident_mode: Default::default(),
},
],
schedule: DowntimeSchedule {
repeat: Repeat::Weekly,
starts_at: 1_789_603_200_000_000,
ends_at: None,
timezone: "Europe/Berlin".to_string(),
start_time_local: Some("02:00".to_string()),
duration_secs: 5400,
weekdays: vec![7],
},
cancelled_at: Some(5),
cancelled_by: Some("lin".to_string()),
show_banner: false,
notifications: Some(DowntimeNotifications {
destinations: vec!["slack-oncall".to_string()],
events: NotificationEvents {
started: true,
ending_soon: true,
..Default::default()
},
ending_soon_lead_secs: 300,
continues: None,
}),
origin_region: Some("us-east".to_string()),
..minimal()
}
}
#[test]
fn a_downtime_round_trips_with_every_optional_field_absent() {
let d = minimal();
let json = serde_json::to_value(&d).unwrap();
for absent in [
"reason",
"condition",
"cancelled_at",
"cancelled_by",
"notifications",
"origin_region",
] {
assert!(json.get(absent).is_none(), "{absent} must be skipped");
}
assert!(json["targets"][0].get("tags").is_none());
assert!(json["targets"][0].get("slo_mode").is_none());
assert!(json["schedule"].get("ends_at").is_some());
assert_eq!(serde_json::from_value::<Downtime>(json).unwrap(), d);
}
#[test]
fn a_downtime_round_trips_with_every_optional_field_present() {
let d = full();
let json = serde_json::to_string(&d).unwrap();
assert_eq!(serde_json::from_str::<Downtime>(&json).unwrap(), d);
}
#[test]
fn stored_names_are_the_wire_contract() {
let json = serde_json::to_value(full()).unwrap();
assert_eq!(json["condition"]["type"], "group");
assert_eq!(json["condition"]["op"], "and");
assert_eq!(json["condition"]["items"][0]["operator"], "=");
assert_eq!(json["condition"]["items"][1]["items"][0]["operator"], "!=");
assert_eq!(json["targets"][0]["module"], "synthetics");
assert_eq!(json["targets"][0]["folders"]["kind"], "some");
assert_eq!(json["targets"][1]["folders"]["kind"], "all");
assert_eq!(json["targets"][1]["slo_mode"], "count_as_good");
assert_eq!(json["schedule"]["repeat"], "weekly");
assert_eq!(
serde_json::to_value(TargetModule::AnomalyDetections).unwrap(),
"anomaly_detections"
);
}
#[test]
fn missing_folder_and_banner_take_their_defaults() {
let mut json = serde_json::to_value(minimal()).unwrap();
let obj = json.as_object_mut().unwrap();
obj.remove("folder_id");
obj.remove("show_banner");
let d: Downtime = serde_json::from_value(json).unwrap();
assert_eq!(d.folder_id, DEFAULT_FOLDER);
assert!(d.show_banner);
}
#[test]
fn the_api_example_request_parses() {
let body = r#"{
"folder_id": "planned-maintenance",
"name": "Payments failover · weekly",
"condition": { "type": "group", "op": "and", "items": [
{ "type": "pair", "key": "service", "operator": "=", "value": "payments" },
{ "type": "pair", "key": "env", "operator": "=", "value": "prod" } ] },
"targets": [
{ "module": "alerts", "folders": { "kind": "all" } },
{ "module": "synthetics", "folders": { "kind": "all" }, "tags": ["service:payments"] },
{ "module": "slos", "folders": { "kind": "all" }, "slo_mode": "exclude" }
],
"schedule": { "repeat": "weekly", "starts_at": 1789603200000000, "ends_at": null,
"timezone": "Europe/Berlin", "start_time_local": "02:00",
"duration_secs": 5400, "weekdays": [7] }
}"#;
let req: DowntimeRequest = serde_json::from_str(body).unwrap();
assert_eq!(req.targets.len(), 3);
assert_eq!(req.schedule.ends_at, None);
assert!(req.show_banner);
assert_eq!(req.targets[2].slo_mode, Some(SloCorrectionMode::Exclude));
let quick: DowntimeRequest = serde_json::from_str(
r#"{ "targets": [ { "module": "anomaly_detections", "folders": { "kind": "all" }, "ids": ["ad_7c1"] } ],
"schedule": { "repeat": "none", "starts_at": 1, "ends_at": 2, "timezone": "UTC", "duration_secs": 60 },
"show_banner": false }"#,
)
.unwrap();
assert_eq!(quick.folder_id, DEFAULT_FOLDER);
assert!(!quick.show_banner);
assert!(quick.schedule.weekdays.is_empty());
}
#[test]
fn notifications_take_their_defaults_and_keep_their_wire_names() {
let n: DowntimeNotifications =
serde_json::from_str(r#"{ "destinations": ["slack"] }"#).unwrap();
assert_eq!(n.ending_soon_lead_secs, DEFAULT_ENDING_SOON_LEAD_SECS);
assert!(!n.events.any());
let json = serde_json::to_value(full()).unwrap();
assert_eq!(json["notifications"]["destinations"][0], "slack-oncall");
assert_eq!(json["notifications"]["events"]["ending_soon"], true);
assert_eq!(json["notifications"]["ending_soon_lead_secs"], 300);
assert_eq!(json["origin_region"], "us-east");
assert!(json["notifications"].get("continues").is_none());
let linked: DowntimeNotifications =
serde_json::from_str(r#"{ "destinations": [], "continues": "2f9K" }"#).unwrap();
assert_eq!(linked.continues.as_deref(), Some("2f9K"));
let req: DowntimeRequest = serde_json::from_str(
r#"{ "targets": [], "schedule": { "repeat": "none", "starts_at": 1, "ends_at": 2,
"timezone": "UTC", "duration_secs": 60 },
"notifications": { "destinations": ["pd"], "events": { "ended": true } } }"#,
)
.unwrap();
let n = req.notifications.unwrap();
assert!(n.events.wants(NotificationEvent::Ended));
assert!(!n.events.wants(NotificationEvent::Started));
}
#[test]
fn notification_events_are_stored_by_their_snake_case_names() {
for (event, name) in [
(NotificationEvent::Started, "started"),
(NotificationEvent::EndingSoon, "ending_soon"),
(NotificationEvent::Ended, "ended"),
(NotificationEvent::Cancelled, "cancelled"),
(NotificationEvent::Extended, "extended"),
] {
assert_eq!(event.as_str(), name);
assert_eq!(serde_json::to_value(event).unwrap(), name);
}
}
#[test]
fn repeat_round_trips_through_its_stored_number() {
for repeat in [Repeat::None, Repeat::Daily, Repeat::Weekly] {
assert_eq!(Repeat::from_i16(repeat.to_i16()), Some(repeat));
}
assert_eq!(Repeat::None.to_i16(), 0);
assert_eq!(Repeat::Daily.to_i16(), 1);
assert_eq!(Repeat::Weekly.to_i16(), 2);
assert_eq!(Repeat::from_i16(3), None);
}
#[test]
fn only_synthetics_has_tags_and_no_identity() {
for module in [
TargetModule::Alerts,
TargetModule::AnomalyDetections,
TargetModule::Slos,
] {
assert!(module.has_identity());
assert!(!module.has_tags());
}
assert!(!TargetModule::Synthetics.has_identity());
assert!(TargetModule::Synthetics.has_tags());
}
#[test]
fn a_list_item_flattens_the_downtime() {
let item = DowntimeListItem {
downtime: minimal(),
status: DowntimeStatus::Scheduled,
current_window: None,
next_window: Some(DowntimeWindow { start: 1, end: 2 }),
matched_alerts: 7,
matched_anomalies: 0,
matched_synthetics: 2,
matched_slos: 3,
};
let json = serde_json::to_value(item).unwrap();
assert_eq!(json["id"], "2f9KQe4b7Nq1vH3sT0mLzXpRcWd");
assert_eq!(json["status"], "scheduled");
assert!(json["current_window"].is_null());
assert_eq!(json["next_window"]["end"], 2);
}
#[test]
fn ended_early_is_snake_case_on_the_wire() {
assert_eq!(
serde_json::to_value(DowntimeStatus::EndedEarly).unwrap(),
"ended_early"
);
}
#[test]
fn an_extend_body_takes_either_field_and_the_response_flattens_the_row() {
let by: ExtendDowntimeRequest = serde_json::from_str(r#"{ "by_secs": 1800 }"#).unwrap();
assert_eq!((by.by_secs, by.until), (Some(1800), None));
let until: ExtendDowntimeRequest = serde_json::from_str(r#"{ "until": 5 }"#).unwrap();
assert_eq!((until.by_secs, until.until), (None, Some(5)));
let json = serde_json::to_value(ExtendDowntimeResponse {
downtime: minimal(),
created_id: None,
})
.unwrap();
assert_eq!(json["id"], "2f9KQe4b7Nq1vH3sT0mLzXpRcWd");
assert!(json.get("created_id").is_none());
}
#[test]
fn preview_match_omits_unset_reasons() {
let m = PreviewMatch {
id: "a".to_string(),
name: "n".to_string(),
folder_id: "default".to_string(),
matched_by: None,
missing: Some("host".to_string()),
applies: None,
};
let json = serde_json::to_value(m).unwrap();
assert!(json.get("matched_by").is_none());
assert_eq!(json["missing"], "host");
}
#[test]
fn only_the_condition_and_the_targets_change_the_coverage() {
let before = minimal();
let cosmetic = Downtime {
name: "Renamed".to_string(),
reason: Some("typo".to_string()),
folder_id: "planned".to_string(),
show_banner: false,
cancelled_at: Some(5),
updated_at: 9,
..before.clone()
};
assert!(before.same_coverage(&cosmetic));
let narrowed = Downtime {
condition: Some(DimensionCondition::Pair {
key: "service".to_string(),
operator: PairOperator::Eq,
value: "payments".to_string(),
}),
..before.clone()
};
assert!(!before.same_coverage(&narrowed));
let mut other_module = before.clone();
other_module.targets[0].module = TargetModule::Synthetics;
assert!(!before.same_coverage(&other_module));
}
#[test]
fn an_alerts_target_without_the_incident_mode_reads_as_muted() {
let old: DowntimeTarget =
serde_json::from_str(r#"{ "module": "alerts", "folders": { "kind": "all" } }"#)
.unwrap();
assert_eq!(old.incident_mode, IncidentMode::Muted);
let json = serde_json::to_value(&old).unwrap();
assert!(
json.get("incident_mode").is_none(),
"the default is not written"
);
let none: DowntimeTarget = serde_json::from_str(
r#"{ "module": "alerts", "folders": { "kind": "all" }, "incident_mode": "none" }"#,
)
.unwrap();
assert_eq!(none.incident_mode, IncidentMode::None);
let json = serde_json::to_value(&none).unwrap();
assert_eq!(json["incident_mode"], "none");
assert_eq!(
serde_json::from_value::<DowntimeTarget>(json).unwrap(),
none
);
assert_eq!(serde_json::to_value(IncidentMode::Muted).unwrap(), "muted");
}
}
src/config/src/meta/folder.rs
+1
View File
@@ -50,6 +50,7 @@ pub enum FolderType {
Synthetics,
Workflows,
Prompts = 5,
Downtimes = 6,
}
pub const DEFAULT_FOLDER: &str = "default";
src/config/src/meta/mod.rs
+1
View File
@@ -22,6 +22,7 @@ pub mod dashboards;
pub mod db_monitoring;
pub mod db_normalizer;
pub mod destinations;
pub mod downtimes;
pub mod enrichment_table;
pub mod folder;
pub mod function;
src/config/src/meta/oncall/routing.rs
+1 -1
View File
@@ -138,7 +138,7 @@ impl std::error::Error for OwnershipError {}
/// A trailing `*` is a prefix claim and nothing more: `host=db-*` owns `db-01`
/// and `db-primary`. §7: "no regex, no boolean logic", because a routing table
/// people cannot read at a glance is one they stop trusting.
fn pair_matches(want: &str, actual: &str) -> bool {
pub fn pair_matches(want: &str, actual: &str) -> bool {
match want.strip_suffix('*') {
Some(prefix) => actual.starts_with(prefix),
None => want == actual,
src/config/src/meta/self_reporting/mod.rs
+4
View File
@@ -397,6 +397,7 @@ mod tests {
value_is_lower_bound: None,
synthetics_error_source: None,
synthetics_location: None,
downtime_id: None,
delivery_attempted: None,
};
@@ -949,6 +950,7 @@ mod tests {
value_is_lower_bound: None,
synthetics_error_source: None,
synthetics_location: None,
downtime_id: None,
delivery_attempted: None,
};
@@ -1032,6 +1034,7 @@ mod tests {
value_is_lower_bound: None,
synthetics_error_source: None,
synthetics_location: None,
downtime_id: None,
delivery_attempted: None,
};
@@ -1100,6 +1103,7 @@ mod tests {
value_is_lower_bound: None,
synthetics_error_source: None,
synthetics_location: None,
downtime_id: None,
delivery_attempted: None,
};
src/config/src/meta/self_reporting/usage.rs
+40
View File
@@ -108,6 +108,9 @@ pub enum RunOutcome {
/// transitioning to firing
#[serde(rename = "pending")]
Pending,
/// Condition matched inside a downtime window; nothing was delivered.
#[serde(rename = "suppressed")]
Suppressed,
}
impl RunOutcome {
@@ -129,6 +132,7 @@ impl RunOutcome {
Self::Skipped => 4,
Self::NotifyFailed => 5,
Self::Pending => 6,
Self::Suppressed => 7,
}
}
@@ -141,6 +145,7 @@ impl RunOutcome {
4 => Some(Self::Skipped),
5 => Some(Self::NotifyFailed),
6 => Some(Self::Pending),
7 => Some(Self::Suppressed),
_ => None,
}
}
@@ -154,6 +159,7 @@ impl RunOutcome {
Self::Skipped => "skipped",
Self::NotifyFailed => "notify_failed",
Self::Pending => "pending",
Self::Suppressed => "suppressed",
}
}
}
@@ -196,6 +202,7 @@ pub fn normalize_outcome(
"skipped" => Some(RunOutcome::Skipped),
"notify_failed" => Some(RunOutcome::NotifyFailed),
"pending" => Some(RunOutcome::Pending),
"suppressed" => Some(RunOutcome::Suppressed),
// ── legacy vocabulary ──
"condition_not_satisfied" => Some(RunOutcome::Normal),
@@ -330,6 +337,8 @@ pub struct TriggerData {
/// The venue the failed synthetics slot was scheduled for.
#[serde(default, skip_serializing_if = "Option::is_none")]
pub synthetics_location: Option<String>,
#[serde(default, skip_serializing_if = "Option::is_none")]
pub downtime_id: Option<String>,
/// Whether a firing tried a destination, workflow or incident notification; None on old rows.
#[serde(default, skip_serializing_if = "Option::is_none")]
pub delivery_attempted: Option<bool>,
@@ -373,6 +382,7 @@ impl Default for TriggerData {
value_is_lower_bound: None,
synthetics_error_source: None,
synthetics_location: None,
downtime_id: None,
delivery_attempted: None,
}
}
@@ -425,6 +435,7 @@ impl TriggerData {
value_is_lower_bound: Some(false),
synthetics_error_source: Some(String::new()),
synthetics_location: Some(String::new()),
downtime_id: Some(String::new()),
delivery_attempted: Some(true),
}
}
@@ -1148,6 +1159,8 @@ mod run_outcome_tests {
assert_eq!(RunOutcome::from_i32(3), Some(RunOutcome::Error));
assert_eq!(RunOutcome::from_i32(4), Some(RunOutcome::Skipped));
assert_eq!(RunOutcome::from_i32(5), Some(RunOutcome::NotifyFailed));
assert_eq!(RunOutcome::Suppressed.to_i32(), 7);
assert_eq!(RunOutcome::from_i32(7), Some(RunOutcome::Suppressed));
}
#[test]
@@ -1159,6 +1172,8 @@ mod run_outcome_tests {
RunOutcome::Error,
RunOutcome::Skipped,
RunOutcome::NotifyFailed,
RunOutcome::Pending,
RunOutcome::Suppressed,
] {
let n = outcome.to_i32();
assert_eq!(
@@ -1169,6 +1184,30 @@ mod run_outcome_tests {
}
}
#[test]
fn a_suppressed_run_carries_its_downtime_id_through_serde() {
assert_eq!(
serde_json::to_string(&RunOutcome::Suppressed).unwrap(),
"\"suppressed\""
);
assert_eq!(
normalize_outcome("suppressed", &TriggerDataType::Alert, None),
Some(RunOutcome::Suppressed)
);
let td = TriggerData {
status: RunOutcome::Suppressed,
downtime_id: Some("2f9K".to_string()),
..TriggerData::default()
};
let json = serde_json::to_value(&td).unwrap();
assert_eq!(json["status"], "suppressed");
assert_eq!(json["downtime_id"], "2f9K");
assert_eq!(serde_json::from_value::<TriggerData>(json).unwrap(), td);
let bare = serde_json::to_value(TriggerData::default()).unwrap();
assert!(bare.get("downtime_id").is_none());
assert!(TriggerData::get_field_names().contains(&"downtime_id".to_string()));
}
#[test]
fn test_run_outcome_from_i32_rejects_unknown() {
assert_eq!(RunOutcome::from_i32(-1), None);
@@ -1788,6 +1827,7 @@ mod tests {
value_is_lower_bound: None,
synthetics_error_source: None,
synthetics_location: None,
downtime_id: None,
delivery_attempted: None,
};
src/config/src/meta/slo/coverage.rs
+10 -1
View File
@@ -123,7 +123,11 @@ pub fn observe(read: WindowRead, coverage_floor: f64, watermark_stale: bool) ->
pub fn evaluation_is_measured(outcome: &RunOutcome) -> bool {
match outcome {
// Computed a level — the same set §7.6 refreshes `level_at` for.
RunOutcome::Firing | RunOutcome::Normal | RunOutcome::NotifyFailed => true,
// A suppressed run computed its level; only the delivery was skipped.
RunOutcome::Firing
| RunOutcome::Normal
| RunOutcome::NotifyFailed
| RunOutcome::Suppressed => true,
// Observed nothing: the query failed, the run was skipped, or the
// outcome belongs to a non-condition module.
RunOutcome::Error | RunOutcome::Skipped | RunOutcome::Succeeded | RunOutcome::Pending => {
@@ -348,6 +352,11 @@ mod tests {
assert!(!evaluation_is_measured(&RunOutcome::Skipped));
}
#[test]
fn a_suppressed_evaluation_is_a_measurement() {
assert!(evaluation_is_measured(&RunOutcome::Suppressed));
}
/// An interval in which the source alert was **paused** has no records at
/// all, so it must read as reduced coverage rather than as uptime. This is
/// the case S-16 exists for: pausing an alert is a routine operator
src/config/src/meta/slo/slice.rs
+3
View File
@@ -64,6 +64,8 @@ pub struct SliceRow {
/// Monotonic within the generation; higher wins on the same key. Exists
/// for late-data re-emission, not for publication ordering.
pub rev: i64,
/// The downtime whose window zeroed or filled this slice (D8).
pub corrected_by: Option<String>,
}
/// Whether a row is visible to readers.
@@ -271,6 +273,7 @@ mod tests {
good,
total,
rev,
corrected_by: None,
}
}
src/config/src/meta/slo/status_view.rs
+4
View File
@@ -29,6 +29,7 @@ use serde::{Deserialize, Serialize};
use utoipa::ToSchema;
use super::math::{burn_rate, error_budget_remaining, sli, time_to_exhaust_secs};
use crate::meta::downtimes::CorrectionRef;
/// What the UI and the API render for one SLO or one group.
#[derive(Debug, Clone, PartialEq, Serialize, Deserialize, ToSchema)]
@@ -79,6 +80,8 @@ pub struct SloStatusView {
/// not exist.
#[serde(default)]
pub measuring_since: Option<i64>,
#[serde(default, skip_serializing_if = "Vec::is_empty")]
pub corrections: Vec<CorrectionRef>,
}
impl SloStatusView {
@@ -144,6 +147,7 @@ impl SloStatusView {
stale_watermark: false,
watermark_end: None,
measuring_since: None,
corrections: Vec::new(),
}
}
src/config/src/meta/slo/stream.rs
+7 -1
View File
@@ -62,6 +62,9 @@ pub struct SloSliceRow {
/// This is for **dedupe**, not publication ordering — there is no
/// publication protocol to order (D64).
pub rev: i64,
/// Id of the downtime that corrected this slice, or `""` when measured as is.
#[serde(default)]
pub corrected_by: String,
}
impl SloSliceRow {
@@ -84,6 +87,7 @@ impl SloSliceRow {
good: 1.0,
total: 1.0,
rev: 1,
corrected_by: "downtime".to_string(),
}
}
}
@@ -98,7 +102,7 @@ mod tests {
// type and then conflict with the first real write.
let v = serde_json::to_value(SloSliceRow::init_for_reflection()).unwrap();
let obj = v.as_object().unwrap();
assert_eq!(obj.len(), 10, "a field was added without a sample value");
assert_eq!(obj.len(), 11, "a field was added without a sample value");
for (k, val) in obj {
assert!(!val.is_null(), "{k} is null in the reflection sample");
if let Some(s) = val.as_str() {
@@ -130,6 +134,7 @@ mod tests {
good: 98.0,
total: 100.0,
rev: 2,
corrected_by: String::new(),
};
let json = serde_json::to_string(&row).unwrap();
assert_eq!(serde_json::from_str::<SloSliceRow>(&json).unwrap(), row);
@@ -151,6 +156,7 @@ mod tests {
"good",
"total",
"rev",
"corrected_by",
] {
assert!(v.get(field).is_some(), "field `{field}` was renamed");
}
src/config/src/meta/synthetics.rs
+7 -1
View File
@@ -19,7 +19,10 @@ use chrono::FixedOffset;
use serde::{Deserialize, Serialize};
use utoipa::ToSchema;
use super::synthetics_variables::{placeholder_names, substitute_placeholders};
use super::{
downtimes::ActiveDowntime,
synthetics_variables::{placeholder_names, substitute_placeholders},
};
// ── Frequency ─────────────────────────────────────────────────────────────────
@@ -583,6 +586,8 @@ pub struct SyntheticListItem {
/// Omitted for a check that holds no subtest reference.
#[serde(default, skip_serializing_if = "Option::is_none")]
pub reference_state: Option<ReferenceState>,
#[serde(default, skip_serializing_if = "Option::is_none")]
pub active_downtime: Option<ActiveDowntime>,
}
/// Whether a parent's subtest references can run as stored.
@@ -4656,6 +4661,7 @@ mod tests {
referenced_by: 0,
references: Some(0),
reference_state: None,
active_downtime: None,
};
let json = serde_json::to_value(&item).unwrap();
assert!(json.get("reference_state").is_none(), "{json}");
src/config/src/metrics/base.rs
+17
View File
@@ -1394,6 +1394,19 @@ pub static ALERT_DEDUP_ERRORS_TOTAL: Lazy<IntCounterVec> = Lazy::new(|| {
.expect("Metric created")
});
pub static ALERT_RUNS_SUPPRESSED_TOTAL: Lazy<IntCounterVec> = Lazy::new(|| {
IntCounterVec::new(
Opts::new(
"alert_runs_suppressed_total",
"Total number of runs whose notification a downtime suppressed",
)
.namespace(NAMESPACE)
.const_labels(create_const_labels()),
&["org", "module"],
)
.expect("Metric created")
});
// Alert grouping/batching metrics
pub static ALERT_GROUPING_BATCHES_PENDING: Lazy<IntGaugeVec> = Lazy::new(|| {
IntGaugeVec::new(
@@ -2886,6 +2899,9 @@ pub(crate) fn register(registry: &Registry) {
registry
.register(Box::new(ALERT_DEDUP_ERRORS_TOTAL.clone()))
.expect("Metric registered");
registry
.register(Box::new(ALERT_RUNS_SUPPRESSED_TOTAL.clone()))
.expect("Metric registered");
// alert grouping metrics
registry
@@ -3310,6 +3326,7 @@ mod tests {
let _ = ALERT_DEDUP_SUPPRESSED_TOTAL.clone();
let _ = ALERT_DEDUP_PASSED_TOTAL.clone();
let _ = ALERT_DEDUP_ERRORS_TOTAL.clone();
let _ = ALERT_RUNS_SUPPRESSED_TOTAL.clone();
let _ = ALERT_GROUPING_BATCHES_PENDING.clone();
let _ = ALERT_GROUPING_NOTIFICATIONS_SENT_TOTAL.clone();
let _ = ALERT_GROUPING_BATCH_SIZE.clone();
src/core/src/alerts/alert.rs
+142 -13
View File
@@ -359,6 +359,28 @@ pub enum AlertError {
MultiAlertGroupingError(String),
}
/// What a manual trigger did: sent, with its messages, or held by an active downtime.
#[derive(Debug, Clone, PartialEq)]
pub enum ManualTrigger {
Sent {
success_message: String,
error_message: String,
},
Suppressed(config::meta::downtimes::ActiveDowntime),
}
impl ManualTrigger {
/// The message the trigger route answers with.
pub fn response_message(&self) -> String {
match self {
Self::Sent { .. } => "Alert triggered".to_string(),
Self::Suppressed(downtime) => {
format!("Alert suppressed by downtime {}", downtime.name)
}
}
}
}
pub async fn save(
org_id: &str,
stream_name: &str,
@@ -1147,6 +1169,25 @@ pub fn update_cron_expression(cron_exp: &str, now: u32) -> String {
cron_exp
}
/// `zo_alert_runs_suppressed_total`: one per run a downtime kept silent.
pub fn count_suppressed_run(org: &str, module: &str) {
config::metrics::ALERT_RUNS_SUPPRESSED_TOTAL
.with_label_values(&[org, module])
.inc();
}
/// A run a downtime kept silent: `Suppressed` with its downtime id, no delivery attempt, counted.
pub fn record_suppressed_run(
data: &mut config::meta::self_reporting::usage::TriggerData,
module: &str,
downtime_id: String,
) {
data.status = config::meta::self_reporting::usage::RunOutcome::Suppressed;
data.downtime_id = Some(downtime_id);
data.delivery_attempted = Some(false);
count_suppressed_run(&data.org, module);
}
/// Runs the checks [`create`] (given a folder) or [`update`] makes on a copy, writing nothing.
pub async fn validate_save(
org_id: &str,
@@ -2239,11 +2280,15 @@ pub async fn trigger_by_id<C: ConnectionTrait>(
conn: &C,
org_id: &str,
alert_id: Ksuid,
) -> Result<(String, String), AlertError> {
let Some((_, alert)) = db::alerts::alert::get_by_id(conn, org_id, alert_id).await? else {
) -> Result<ManualTrigger, AlertError> {
let Some((_folder, alert)) = db::alerts::alert::get_by_id(conn, org_id, alert_id).await? else {
return Err(AlertError::AlertNotFound);
};
let now = Utc::now().timestamp_micros();
#[cfg(feature = "enterprise")]
if let Some(downtime) = suppressed_manual_trigger(&alert, &_folder.folder_id, now).await {
return Ok(ManualTrigger::Suppressed(downtime));
}
// For creates_incident=true alerts the incident correlation path handles
// the notification. For all other cases send the direct notification.
@@ -2263,6 +2308,7 @@ pub async fn trigger_by_id<C: ConnectionTrait>(
now,
// Manual triggers evaluate nothing; no level to map.
None,
None,
)
.await
{
@@ -2331,9 +2377,12 @@ pub async fn trigger_by_id<C: ConnectionTrait>(
None,
)
.await?;
let err_message = with_incident_notify_error(outcome.error_message, incident_notify_error);
let error_message = with_incident_notify_error(outcome.error_message, incident_notify_error);
Ok((outcome.success_message, err_message))
Ok(ManualTrigger::Sent {
success_message: outcome.success_message,
error_message,
})
}
pub async fn trigger_by_name(
@@ -2341,14 +2390,21 @@ pub async fn trigger_by_name(
stream_type: StreamType,
stream_name: &str,
name: &str,
) -> Result<(String, String), AlertError> {
let alert = match db::alerts::alert::get_by_name(org_id, stream_type, stream_name, name).await {
Ok(Some(alert)) => alert,
_ => {
return Err(AlertError::AlertNotFound);
}
};
) -> Result<ManualTrigger, AlertError> {
let (_folder, alert) =
match db::alerts::alert::get_by_name_with_folder(org_id, stream_type, stream_name, name)
.await
{
Ok(Some(found)) => found,
_ => {
return Err(AlertError::AlertNotFound);
}
};
let now = Utc::now().timestamp_micros();
#[cfg(feature = "enterprise")]
if let Some(downtime) = suppressed_manual_trigger(&alert, &_folder.folder_id, now).await {
return Ok(ManualTrigger::Suppressed(downtime));
}
// For creates_incident=true alerts the incident correlation path handles
// the notification. For all other cases send the direct notification.
@@ -2368,6 +2424,7 @@ pub async fn trigger_by_name(
now,
// Manual triggers evaluate nothing; no level to map.
None,
None,
)
.await
{
@@ -2429,9 +2486,42 @@ pub async fn trigger_by_name(
None,
)
.await?;
let err_message = with_incident_notify_error(outcome.error_message, incident_notify_error);
let error_message = with_incident_notify_error(outcome.error_message, incident_notify_error);
Ok((outcome.success_message, err_message))
Ok(ManualTrigger::Sent {
success_message: outcome.success_message,
error_message,
})
}
/// A manual trigger inside a window is suppressed like a scheduled run: no page, no un-mute.
#[cfg(feature = "enterprise")]
async fn suppressed_manual_trigger(
alert: &Alert,
folder_id: &str,
now: i64,
) -> Option<config::meta::downtimes::ActiveDowntime> {
let rows = [manual_trigger_row(alert)];
let downtime =
crate::alerts::scheduler::handlers::downtime_decision(alert, folder_id, Some(&rows), now)
.await?;
log::info!(
"Manual trigger of alert {}/{} suppressed by downtime {}",
alert.org_id,
alert.name,
downtime.id
);
crate::alerts::scheduler::handlers::correlate_muted_incident(
"manual_trigger",
alert,
&rows,
now,
None,
&downtime,
)
.await;
count_suppressed_run(&alert.org_id, "alerts");
Some(downtime)
}
/// Per-destination result of one notification attempt.
@@ -8039,6 +8129,25 @@ mod tests {
alert
}
#[test]
fn a_suppressed_manual_trigger_answers_with_the_downtime_name() {
let suppressed = ManualTrigger::Suppressed(config::meta::downtimes::ActiveDowntime {
id: "dt-1".to_string(),
name: "db upgrade".to_string(),
ends_at: 0,
incident_mode: Default::default(),
});
assert_eq!(
suppressed.response_message(),
"Alert suppressed by downtime db upgrade"
);
let sent = ManualTrigger::Sent {
success_message: String::new(),
error_message: String::new(),
};
assert_eq!(sent.response_message(), "Alert triggered");
}
#[test]
fn manual_trigger_row_carries_stream_alert_and_manual_marker() {
let alert = manual_alert("my_alert", vec![]);
@@ -8133,6 +8242,26 @@ mod tests {
assert_eq!(workflow_alert_count(&alert, 25, None), json!(25u64));
}
#[test]
fn a_suppressed_run_is_recorded_with_its_downtime_and_counted() {
use config::meta::self_reporting::usage::{RunOutcome, TriggerData};
let org = "org_suppressed_run_record";
let counter =
|| config::metrics::ALERT_RUNS_SUPPRESSED_TOTAL.with_label_values(&[org, "alerts"]);
let before = counter().get();
let mut data = TriggerData {
org: org.to_string(),
status: RunOutcome::Firing,
..Default::default()
};
record_suppressed_run(&mut data, "alerts", "dt-1".to_string());
assert_eq!(data.status, RunOutcome::Suppressed);
assert_eq!(data.downtime_id.as_deref(), Some("dt-1"));
assert_eq!(data.delivery_attempted, Some(false));
assert_eq!(counter().get(), before + 1);
}
/// A new alert with no id reaches the schedule checks before any DB read.
async fn prepare_schedule(mutate: impl FnOnce(&mut Alert)) -> AlertError {
let mut alert = Alert::default();
src/core/src/alerts/destination_usage.rs
+132
View File
@@ -38,6 +38,8 @@ pub enum DestinationConsumer {
AnomalyDetection,
#[cfg(feature = "enterprise")]
IncidentIntegration,
#[cfg(feature = "enterprise")]
Downtime,
}
impl DestinationConsumer {
@@ -56,6 +58,8 @@ impl DestinationConsumer {
Self::AnomalyDetection => ("anomaly detection config", "anomaly detection configs"),
#[cfg(feature = "enterprise")]
Self::IncidentIntegration => ("incident integration", "incident integrations"),
#[cfg(feature = "enterprise")]
Self::Downtime => ("downtime", "downtimes"),
};
format!("{count} {}", if count == 1 { one } else { many })
}
@@ -198,6 +202,8 @@ async fn usage_for(
DestinationConsumer::AnomalyDetection => anomaly_detection_usage(org_id).await,
#[cfg(feature = "enterprise")]
DestinationConsumer::IncidentIntegration => incident_integration_usage(org_id).await,
#[cfg(feature = "enterprise")]
DestinationConsumer::Downtime => downtime_usage(org_id).await,
}
}
@@ -401,6 +407,52 @@ async fn incident_integration_usage(org_id: &str) -> Result<Vec<DestinationUse>,
Ok(uses)
}
#[cfg(feature = "enterprise")]
async fn downtime_usage(org_id: &str) -> Result<Vec<DestinationUse>, DestinationError> {
let cached = db::downtimes::list_cached(org_id);
let stored = infra::table::downtimes::list(org_id, None).await?;
let ids: std::collections::HashSet<&str> = cached.iter().map(|d| d.id.as_str()).collect();
let rows = cached
.iter()
.chain(stored.iter().filter(|d| !ids.contains(d.id.as_str())));
Ok(downtime_uses(rows, config::utils::time::now_micros()))
}
/// Only a row that can still notify holds its destinations; ended and cancelled rows never send.
#[cfg(feature = "enterprise")]
fn downtime_uses<'a>(
rows: impl Iterator<Item = &'a config::meta::downtimes::Downtime>,
now: i64,
) -> Vec<DestinationUse> {
use config::meta::downtimes::DowntimeStatus;
use o2_enterprise::enterprise::downtimes::schedule;
let mut uses = Vec::new();
let live = rows.filter(|row| {
matches!(
schedule::status(&row.schedule, row.cancelled_at, now),
DowntimeStatus::Scheduled | DowntimeStatus::Active
)
});
for row in live {
let names = row
.notifications
.as_ref()
.map(|n| n.destinations.clone())
.unwrap_or_default();
for destination_name in unique_names(names) {
uses.push(DestinationUse {
consumer: DestinationConsumer::Downtime,
id: row.id.clone(),
name: row.name.clone(),
folder_id: Some(row.folder_id.clone()),
destination_name,
});
}
}
uses
}
#[cfg(test)]
mod tests {
use config::meta::{
@@ -879,6 +931,86 @@ mod tests {
assert_eq!(uses[0].destination_name, name);
}
#[cfg(feature = "enterprise")]
#[test]
fn test_downtime_arm_names_the_downtime_in_the_refusal() {
use config::meta::downtimes::{
Downtime, DowntimeNotifications, DowntimeSchedule, NotificationEvents, Repeat,
};
let row = |id: &str, name: &str, destinations: &[&str]| Downtime {
id: id.to_string(),
org: "acme".to_string(),
folder_id: "default".to_string(),
name: name.to_string(),
reason: None,
condition: None,
targets: vec![],
schedule: DowntimeSchedule {
repeat: Repeat::None,
starts_at: 1,
ends_at: Some(2),
timezone: "UTC".to_string(),
start_time_local: None,
duration_secs: 60,
weekdays: vec![],
},
cancelled_at: None,
cancelled_by: None,
show_banner: true,
notifications: Some(DowntimeNotifications {
destinations: destinations.iter().map(|d| d.to_string()).collect(),
events: NotificationEvents {
ended: true,
..Default::default()
},
ending_soon_lead_secs: 600,
continues: None,
}),
origin_region: None,
version: 0,
created_by: "lin".to_string(),
created_at: 1,
updated_by: "lin".to_string(),
updated_at: 1,
};
let rows = [
row("d1", "Nightly deploy", &["pagerduty", "pagerduty", "slack"]),
row("d2", "DB failover", &["slack"]),
Downtime {
notifications: None,
..row("d3", "Quiet", &[])
},
];
let uses = downtime_uses(rows.iter(), 0);
let pagerduty: Vec<DestinationUse> = uses
.into_iter()
.filter(|u| u.destination_name == "pagerduty")
.collect();
assert_eq!(pagerduty.len(), 1);
assert_eq!(pagerduty[0].consumer, DestinationConsumer::Downtime);
assert_eq!(pagerduty[0].folder_id.as_deref(), Some("default"));
assert_eq!(
usage_message("pagerduty", &pagerduty),
"'pagerduty' is used by 1 downtime (Nightly deploy)"
);
let slack: Vec<DestinationUse> = downtime_uses(rows.iter(), 0)
.into_iter()
.filter(|u| u.destination_name == "slack")
.collect();
assert_eq!(
usage_message("slack", &slack),
"'slack' is used by 2 downtimes (Nightly deploy, DB failover)"
);
// Ended at 2, then cancelled before its start: neither holds `slack` any more.
assert!(downtime_uses(rows.iter(), 2).is_empty());
let cancelled = Downtime {
cancelled_at: Some(0),
..row("d4", "Called off", &["slack"])
};
assert!(downtime_uses([cancelled].iter(), 0).is_empty());
}
#[test]
fn test_unique_names_drops_a_duplicate_on_one_row() {
let names = vec!["a".to_string(), "b".to_string(), "a".to_string()];
src/core/src/alerts/downtimes.rs
+1176
View File
File diff suppressed because it is too large Load Diff
src/core/src/alerts/grouping.rs
+317 -18
View File
@@ -332,26 +332,11 @@ pub fn refused_row(trigger_data: &TriggerData) -> TriggerData {
}
}
/// Sends a batch, then publishes its members' rows at once so a sent page never loses them.
/// Sends a batch and publishes its members' rows; false when nothing was delivered.
#[cfg(feature = "enterprise")]
pub async fn flush_batch(trace_id: &str, batch: PendingBatch) -> bool {
let mut rows = flush_rows(&batch);
let delivered = match send_grouped_notification(trace_id, batch).await {
Ok(attempted) => {
for row in &mut rows {
row.delivery_attempted = Some(attempted);
}
true
}
Err(e) => {
stamp_flush_error(&mut rows, &e);
false
}
};
for row in rows {
usage_reporting::publish_triggers_usage(row);
}
delivered
let decisions = entry_decisions(&batch).await;
flush_decided(trace_id, batch, decisions).await
}
/// Sends one batch's notification; `Ok` says whether any destination or workflow was tried.
@@ -566,6 +551,149 @@ pub async fn send_grouped_notification(
}
}
/// A member muted at its own evaluation leaves with a `Suppressed` row and does not count.
#[cfg(feature = "enterprise")]
async fn flush_decided(
trace_id: &str,
mut batch: PendingBatch,
decisions: Vec<Option<config::meta::downtimes::ActiveDowntime>>,
) -> bool {
drop_muted_entries(&mut batch, decisions);
if batch.alerts.is_empty() {
return false;
}
let mut rows = flush_rows(&batch);
let delivered = match send_grouped_notification(trace_id, batch).await {
Ok(attempted) => {
for row in &mut rows {
row.delivery_attempted = Some(attempted);
}
true
}
Err(e) => {
stamp_flush_error(&mut rows, &e);
false
}
};
for row in rows {
usage_reporting::publish_triggers_usage(row);
}
delivered
}
/// Judged at each entry's own timestamp, so a firing admitted before a window is still sent.
#[cfg(feature = "enterprise")]
async fn entry_decisions(
batch: &PendingBatch,
) -> Vec<Option<config::meta::downtimes::ActiveDowntime>> {
if !crate::alerts::downtimes::any_since(
&batch.org_id,
config::meta::downtimes::TargetModule::Alerts,
oldest_entry(batch),
) {
return vec![None; batch.alerts.len()];
}
let mut decisions = Vec::with_capacity(batch.alerts.len());
for (entry, at) in batch.alerts.iter().zip(judged_at(batch)) {
decisions.push(entry_downtime(entry, at).await);
}
decisions
}
/// The instant each entry is judged at: its own evaluation, never the flush.
#[cfg(feature = "enterprise")]
fn judged_at(batch: &PendingBatch) -> Vec<i64> {
batch.alerts.iter().map(|entry| entry.timestamp).collect()
}
/// The downtime gate looks back to here, so a window that ended before the flush still mutes.
#[cfg(feature = "enterprise")]
fn oldest_entry(batch: &PendingBatch) -> i64 {
judged_at(batch).into_iter().min().unwrap_or_default()
}
#[cfg(feature = "enterprise")]
fn drop_muted_entries(
batch: &mut PendingBatch,
decisions: Vec<Option<config::meta::downtimes::ActiveDowntime>>,
) {
for (entry, downtime) in take_muted(batch, decisions) {
log::info!(
"[alert_grouping_worker] alert {}/{} dropped from its batch by downtime {}",
entry.alert.org_id,
entry.alert.name,
downtime.id
);
crate::alerts::alert::count_suppressed_run(&entry.alert.org_id, "alerts");
usage_reporting::publish_triggers_usage(suppressed_entry_record(&entry, downtime));
}
}
/// Removes the muted entries from the batch and returns them with their downtime.
#[cfg(feature = "enterprise")]
fn take_muted(
batch: &mut PendingBatch,
decisions: Vec<Option<config::meta::downtimes::ActiveDowntime>>,
) -> Vec<(BatchedAlert, config::meta::downtimes::ActiveDowntime)> {
let (kept, muted) = split_muted(std::mem::take(&mut batch.alerts), decisions);
batch.alerts = kept;
muted
}
/// The entries no downtime covers, in their order, and the others with their downtime.
#[cfg(feature = "enterprise")]
fn split_muted<T>(
entries: Vec<T>,
decisions: Vec<Option<config::meta::downtimes::ActiveDowntime>>,
) -> (Vec<T>, Vec<(T, config::meta::downtimes::ActiveDowntime)>) {
let mut kept = Vec::with_capacity(entries.len());
let mut muted = Vec::new();
for (entry, decision) in entries.into_iter().zip(decisions) {
match decision {
Some(downtime) => muted.push((entry, downtime)),
None => kept.push(entry),
}
}
(kept, muted)
}
#[cfg(feature = "enterprise")]
async fn entry_downtime(
entry: &BatchedAlert,
at: i64,
) -> Option<config::meta::downtimes::ActiveDowntime> {
let alert_id = entry.alert.id.as_ref()?.to_string();
let (folder, _) =
crate::db::alerts::alert::get_alert_from_cache(&entry.alert.org_id, &alert_id).await?;
let identity =
crate::alerts::scheduler::handlers::downtime_identity(&entry.alert, &entry.rows).await;
crate::alerts::scheduler::handlers::muted_in_every_group(&identity, |dims| {
crate::alerts::downtimes::active_for_alert(
&entry.alert.org_id,
&alert_id,
&folder.folder_id,
dims,
at,
)
})
}
/// The held-back evaluation row of a dropped entry, recorded as suppressed by its downtime.
#[cfg(feature = "enterprise")]
fn suppressed_entry_record(
entry: &BatchedAlert,
downtime: config::meta::downtimes::ActiveDowntime,
) -> TriggerData {
TriggerData {
status: RunOutcome::Suppressed,
downtime_id: Some(downtime.id),
delivery_attempted: Some(false),
error: None,
grouped: Some(true),
..entry.trigger_data.clone()
}
}
#[cfg(test)]
mod tests {
use super::*;
@@ -869,4 +997,175 @@ mod tests {
PENDING_BATCHES.remove(&batch_key(busy, fp1));
PENDING_BATCHES.remove(&batch_key(busy, fp2));
}
#[cfg(feature = "enterprise")]
fn downtime(id: &str) -> config::meta::downtimes::ActiveDowntime {
config::meta::downtimes::ActiveDowntime {
id: id.to_string(),
name: id.to_string(),
ends_at: 0,
incident_mode: Default::default(),
}
}
#[cfg(feature = "enterprise")]
#[test]
fn a_flush_drops_the_muted_entries_and_keeps_the_rest_in_order() {
let (kept, muted) = split_muted(
vec!["a", "b", "c"],
vec![None, Some(downtime("dt-1")), None],
);
assert_eq!(kept, ["a", "c"]);
assert_eq!(muted.len(), 1);
assert_eq!((muted[0].0, muted[0].1.id.as_str()), ("b", "dt-1"));
let (kept, muted) = split_muted(vec!["a"], vec![Some(downtime("dt-1"))]);
assert!(kept.is_empty(), "an emptied batch sends nothing");
assert_eq!(muted.len(), 1);
}
#[cfg(feature = "enterprise")]
fn queued_batch(fp: &str, org: &str, timestamps: &[i64]) -> PendingBatch {
for (i, _) in timestamps.iter().enumerate() {
admit(fp, org, 10, evaluation_row(&format!("a/{i}")));
}
let mut batch = PENDING_BATCHES.remove(&batch_key(org, fp)).unwrap().1;
for (entry, at) in batch.alerts.iter_mut().zip(timestamps) {
entry.timestamp = *at;
}
batch
}
// Dropping an entry publishes its row, and the usage queue starts on a Tokio runtime.
#[cfg(feature = "enterprise")]
#[tokio::test]
async fn an_entry_queued_before_the_window_stays_in_the_batch_at_flush() {
let window_opens = 100;
let mut batch = queued_batch("grouping_test_before_window", "org-before", &[50, 150]);
let decisions = judged_at(&batch)
.into_iter()
.map(|at| (at >= window_opens).then(|| downtime("dt-1")))
.collect();
drop_muted_entries(&mut batch, decisions);
let kept: Vec<_> = batch.alerts.iter().map(|e| e.timestamp).collect();
assert_eq!(kept, [50]);
}
#[cfg(feature = "enterprise")]
#[test]
fn an_entry_inside_a_window_that_ended_before_the_flush_is_still_judged() {
use config::meta::downtimes::{
Downtime, DowntimeSchedule, DowntimeTarget, Repeat, TargetFolders, TargetModule,
};
use crate::alerts::downtimes::enterprise::{any_live_target, any_target_since};
const MINUTE: i64 = 60_000_000;
let (opens, closes) = (540 * MINUTE, 600 * MINUTE);
let only_window = Downtime {
id: "dt-1".to_string(),
org: "org-ended".to_string(),
folder_id: "default".to_string(),
name: "dt-1".to_string(),
reason: None,
condition: None,
targets: vec![DowntimeTarget {
module: TargetModule::Alerts,
folders: TargetFolders::All,
tags: vec![],
ids: vec![],
slo_mode: None,
incident_mode: Default::default(),
}],
schedule: DowntimeSchedule {
repeat: Repeat::None,
starts_at: opens,
ends_at: Some(closes),
timezone: "UTC".to_string(),
start_time_local: None,
duration_secs: 3_600,
weekdays: vec![],
},
cancelled_at: None,
cancelled_by: None,
show_banner: false,
notifications: None,
origin_region: None,
version: 1,
created_by: "lin".to_string(),
created_at: 0,
updated_by: "lin".to_string(),
updated_at: 0,
};
let rows = std::slice::from_ref(&only_window);
let batch = queued_batch("grouping_test_ended_window", "org-ended", &[598 * MINUTE]);
let flush = 602 * MINUTE;
assert!(!any_live_target(rows, TargetModule::Alerts, flush));
assert!(any_target_since(
rows,
TargetModule::Alerts,
oldest_entry(&batch),
flush
));
}
#[cfg(feature = "enterprise")]
#[tokio::test]
async fn an_emptied_batch_records_no_delivery() {
let batch = queued_batch("grouping_test_emptied", "org-emptied", &[150, 160]);
let decisions = vec![Some(downtime("dt-1")), Some(downtime("dt-1"))];
assert!(!flush_decided("t", batch, decisions).await);
}
#[cfg(feature = "enterprise")]
#[test]
fn a_dropped_entry_is_recorded_as_suppressed_with_its_downtime() {
let entry = BatchedAlert {
alert: make_alert(),
rows: vec![],
timestamp: 5,
trigger_data: TriggerData {
start_time: 5,
end_time: 9,
..evaluation_row("a/1")
},
};
let record = suppressed_entry_record(&entry, downtime("dt-1"));
assert_eq!(record.key, "a/1");
assert_eq!(record.status, RunOutcome::Suppressed);
assert_eq!(record.downtime_id.as_deref(), Some("dt-1"));
assert_eq!((record.start_time, record.end_time), (5, 9));
assert_eq!(record.grouped, Some(true));
assert_eq!(record.delivery_attempted, Some(false));
}
#[cfg(feature = "enterprise")]
#[test]
fn a_muted_entry_leaves_the_batch_before_its_rows_are_built() {
let (fp, org) = ("grouping_test_muted_flush", "org-muted-flush");
assert_eq!(
admit(fp, org, 10, evaluation_row("a/1")),
BatchAdmission::Queued
);
assert_eq!(
admit(fp, org, 10, evaluation_row("b/2")),
BatchAdmission::Queued
);
let mut batch = PENDING_BATCHES.remove(&batch_key(org, fp)).unwrap().1;
let muted = take_muted(&mut batch, vec![None, Some(downtime("dt-1"))]);
let suppressed: Vec<_> = muted
.into_iter()
.map(|(entry, downtime)| suppressed_entry_record(&entry, downtime))
.collect();
let sent = flush_rows(&batch);
// One row per evaluation: the muted one only as Suppressed, the rest as the batch.
assert_eq!(suppressed.len(), 1);
assert_eq!(suppressed[0].key, "b/2");
assert_eq!(suppressed[0].status, RunOutcome::Suppressed);
let keys: Vec<&str> = sent.iter().map(|row| row.key.as_str()).collect();
assert_eq!(keys, ["a/1"]);
assert_eq!(sent[0].group_size, Some(1));
}
}
src/core/src/alerts/incidents.rs
+800 -186
View File
File diff suppressed because it is too large Load Diff
src/core/src/alerts/mod.rs
+1
View File
@@ -49,6 +49,7 @@ pub mod deduplication;
pub mod derived_streams;
pub mod destination_usage;
pub mod destinations;
pub mod downtimes;
pub mod external_alerts;
#[cfg(feature = "enterprise")]
pub mod grouping;
src/core/src/alerts/notifications/default_template.rs
+1
View File
@@ -80,6 +80,7 @@ pub fn compiled_default_content() -> ContentSpec {
pub fn is_reserved_template_name(name: &str) -> bool {
config::prebuilt_loader::is_prebuilt_template_name(name)
|| name == DEFAULT_CONTENT_TEMPLATE_NAME
|| crate::alerts::templates::is_downtime_template_name(name)
}
/// Create-if-missing (and overwrite under the same `should_apply_prebuilt`
src/core/src/alerts/prom_rule.rs
+1
View File
@@ -445,6 +445,7 @@ mod tests {
&TriggerCondition::default(),
600,
0,
|_| false,
);
assert!(dispatch.inconsistent.is_empty());
assert_eq!(dispatch.items.len(), 2);
src/core/src/alerts/recovery.rs
+487 -6
View File
@@ -16,7 +16,34 @@
//! Subsystems used to decide independently that an alert had got better; they are consumers now.
//! Best effort: the episode is already committed, so a failure costs a message, not the record.
use config::meta::alerts::{alert::Alert, recovery::RecoveryEvent};
#[cfg(feature = "enterprise")]
use config::meta::downtimes::TargetModule;
use config::meta::{
alerts::{alert::Alert, recovery::RecoveryEvent},
self_reporting::usage::{TriggerData, TriggerDataType},
};
/// A clear run this long after the last window still closes the mute that window recorded.
#[cfg(feature = "enterprise")]
const RECORDED_MUTE_GRACE_MICROS: i64 = 24 * 3_600 * 1_000_000;
/// Which consumers one recovery reaches.
#[derive(Debug, PartialEq)]
struct RecoveryRoute {
oncall: bool,
destinations: bool,
}
/// One clear run's on-call decision in the scheduler.
#[cfg(feature = "enterprise")]
#[derive(Debug, PartialEq)]
struct ClearRunOncall<'a> {
withheld: bool,
/// The downtime to record as the alert's mute.
record: Option<&'a str>,
/// The recorded mute is over, so the record goes and the escalations it held close.
forget: bool,
}
/// Tell every consumer, then return. Never fails.
pub async fn dispatch_recovery(alert: &Alert, event: &RecoveryEvent) {
@@ -28,9 +55,37 @@ pub async fn dispatch_recovery(alert: &Alert, event: &RecoveryEvent) {
event.duration_micros()
);
// A recovery inside a window is dropped, never delayed: the episode is already closed.
let downtime_id = recovery_downtime(alert, event).await;
let route = recovery_route(alert, event, downtime_id.is_some());
if let Some(downtime_id) = downtime_id.as_deref() {
log::info!(
"[RECOVERY] {}/{} recovery suppressed by downtime {downtime_id}",
event.org_id,
event.alert_id
);
// Keeps the per-run on-call recovery withheld through a hold, then lets it close the rest.
#[cfg(feature = "enterprise")]
if let Err(e) =
infra::table::alert_states::set_last_downtime_id(&event.alert_id, Some(downtime_id))
.await
{
log::warn!(
"[RECOVERY] could not record the downtime of {}: {e}",
event.alert_id
);
}
usage_reporting::publish_triggers_usage(suppressed_recovery_record(
alert,
event,
downtime_id.to_string(),
));
}
// Once per episode, not once per evaluation: that is the whole fix.
#[cfg(feature = "enterprise")]
if o2_enterprise::enterprise::oncall::is_enabled()
if route.oncall
&& o2_enterprise::enterprise::oncall::is_enabled()
&& let Err(e) = o2_enterprise::enterprise::oncall::escalation::recover_for_alert(
&event.org_id,
&event.alert_id,
@@ -43,17 +98,16 @@ pub async fn dispatch_recovery(alert: &Alert, event: &RecoveryEvent) {
);
}
// The links close even inside a window; a resolve it causes then pages and notifies nobody.
#[cfg(feature = "enterprise")]
if let Err(e) = crate::alerts::incidents::resolve_alert_firing(event).await {
if let Err(e) = crate::alerts::incidents::resolve_alert_firing(event, !route.oncall).await {
log::error!(
"[RECOVERY] incident consumer failed for {}: {e}",
event.alert_id
);
}
// Correlated alerts: the incident owns the resolve because it sent the trigger.
if alert.notify_on_recovery
&& event.incident_id.is_none()
if route.destinations
&& let Err(e) = crate::alerts::alert::send_recovery_notification(alert, event).await
{
log::error!(
@@ -64,3 +118,430 @@ pub async fn dispatch_recovery(alert: &Alert, event: &RecoveryEvent) {
// TODO: a workflow consumer, once a link can carry AlertResolved (see that variant).
}
/// Whether the scheduler's per-run on-call recovery waits: a window now, or a hold it started.
#[cfg(feature = "enterprise")]
pub(crate) async fn oncall_recovery_withheld(alert: &Alert, folder_id: &str, now: i64) -> bool {
let Some(alert_id) = alert.id.map(|id| id.to_string()) else {
return false;
};
let since = now.saturating_sub(RECORDED_MUTE_GRACE_MICROS);
if !crate::alerts::downtimes::any_since(&alert.org_id, TargetModule::Alerts, since) {
// A clear run more than the grace after the window still closes what the mute left open.
if crate::alerts::downtimes::any_row(&alert.org_id, TargetModule::Alerts)
&& recorded_downtime(&alert_id).await.is_some()
{
forget_recorded_mute(&alert.org_id, &alert_id).await;
}
return false;
}
let recorded = recorded_downtime(&alert_id).await;
let muted_now = downtime_at(alert, folder_id, recorded.as_deref(), now).await;
let episode_open = (recorded.is_some() || muted_now.is_some()) && episode_open(&alert_id).await;
let held = recorded.as_deref().is_some_and(|id| {
let hold = alert.keep_firing_for.saturating_mul(1_000_000);
crate::alerts::downtimes::had_window(&alert.org_id, id, now.saturating_sub(hold), now)
});
let decision = clear_run_oncall(
recorded.as_deref(),
muted_now.as_deref(),
episode_open,
held,
);
if let Some(downtime_id) = decision.record {
crate::alerts::scheduler::handlers::record_last_downtime(
&alert.org_id,
&alert_id,
Some(downtime_id),
)
.await;
}
if decision.forget {
forget_recorded_mute(&alert.org_id, &alert_id).await;
}
decision.withheld
}
/// A recorded mute withholds only while its downtime still covers the `keep_firing_for` hold.
#[cfg(feature = "enterprise")]
fn clear_run_oncall<'a>(
recorded: Option<&str>,
muted_now: Option<&'a str>,
episode_open: bool,
recorded_held: bool,
) -> ClearRunOncall<'a> {
let withheld = muted_now.is_some() || (recorded.is_some() && episode_open && recorded_held);
ClearRunOncall {
withheld,
record: muted_now.filter(|_| recorded.is_none() && episode_open),
forget: recorded.is_some() && !withheld,
}
}
/// Correlated alerts: the incident owns the resolve because it sent the trigger.
fn recovery_route(alert: &Alert, event: &RecoveryEvent, muted: bool) -> RecoveryRoute {
RecoveryRoute {
oncall: !muted,
destinations: !muted && alert.notify_on_recovery && event.incident_id.is_none(),
}
}
/// The history record of a recovery a downtime kept silent.
fn suppressed_recovery_record(
alert: &Alert,
event: &RecoveryEvent,
downtime_id: String,
) -> TriggerData {
let mut data = TriggerData {
_timestamp: event.emitted_at,
org: event.org_id.clone(),
module: TriggerDataType::Alert,
key: format!("{}/{}", alert.name, event.alert_id),
start_time: event.recovered_at,
end_time: event.emitted_at,
is_realtime: alert.is_real_time,
..Default::default()
};
crate::alerts::alert::record_suppressed_run(&mut data, "alerts", downtime_id);
data
}
/// Asked at `recovered_at`, so a `keep_firing_for` hold that outlasts the window still mutes.
#[cfg(feature = "enterprise")]
async fn recovery_downtime(alert: &Alert, event: &RecoveryEvent) -> Option<String> {
// From `recovered_at`, so a hold that outlasts the org's last window still mutes.
if !crate::alerts::downtimes::any_since(&event.org_id, TargetModule::Alerts, event.recovered_at)
{
return None;
}
let alert_id = alert.id?;
let conn = infra::db::get_orm_client_ro().await;
let folder_id = match crate::alerts::alert::get_by_id(conn, &event.org_id, alert_id).await {
Ok((folder, _)) => folder.folder_id,
Err(e) => {
log::warn!(
"[RECOVERY] folder of {}/{} unreadable for the downtime check: {e}",
event.org_id,
event.alert_id
);
return None;
}
};
let recorded = recorded_downtime(&event.alert_id).await;
downtime_at(alert, &folder_id, recorded.as_deref(), event.recovered_at).await
}
#[cfg(not(feature = "enterprise"))]
async fn recovery_downtime(_alert: &Alert, _event: &RecoveryEvent) -> Option<String> {
None
}
/// The muted firing's downtime carries its row identity; the clear run has no rows of its own.
#[cfg(feature = "enterprise")]
async fn downtime_at(
alert: &Alert,
folder_id: &str,
recorded: Option<&str>,
at: i64,
) -> Option<String> {
if let Some(downtime) =
recorded.and_then(|id| crate::alerts::downtimes::active_by_id(&alert.org_id, id, at))
{
return Some(downtime.id);
}
crate::alerts::scheduler::handlers::downtime_decision(alert, folder_id, Some(&[]), at)
.await
.map(|downtime| downtime.id)
}
/// A delivered firing whose recovery is still due, possibly held by `keep_firing_for`.
#[cfg(feature = "enterprise")]
async fn episode_open(alert_id: &str) -> bool {
match infra::table::alert_states::get(alert_id, config::meta::alerts::state::ROLLUP_GROUP_KEY)
.await
{
Ok(state) => state.is_some_and(|state| state.episode_id.is_some()),
Err(e) => {
log::warn!("[RECOVERY] state of {alert_id} unreadable for the mute record: {e}");
false
}
}
}
/// Clears the record and closes the escalation of the incident a muted recovery resolved quietly.
#[cfg(feature = "enterprise")]
async fn forget_recorded_mute(org: &str, alert_id: &str) {
// Direct: the org may already be off the downtime path that `record_last_downtime` checks.
if let Err(e) = infra::table::alert_states::set_last_downtime_id(alert_id, None).await {
log::warn!("[RECOVERY] could not clear the recorded downtime of {alert_id}: {e}");
}
if let Err(e) = crate::alerts::incidents::recover_muted_resolves(org, alert_id).await {
log::error!("[RECOVERY] incident on-call recovery failed for {org}/{alert_id}: {e}");
}
}
/// The downtime the alert's last muted firing recorded, until a delivered firing clears it.
#[cfg(feature = "enterprise")]
async fn recorded_downtime(alert_id: &str) -> Option<String> {
match infra::table::alert_states::last_downtime_ids(&[alert_id.to_string()]).await {
Ok(mut recorded) => recorded.remove(alert_id),
Err(e) => {
log::warn!("[RECOVERY] recorded downtime of {alert_id} unreadable: {e}");
None
}
}
}
#[cfg(test)]
mod tests {
use config::meta::{
alerts::{
recovery::{EpisodeInput, apply_episode, recovery_event},
state::AlertState,
},
self_reporting::usage::RunOutcome,
};
use super::*;
fn delivered() -> EpisodeInput {
EpisodeInput {
delivered: true,
incident_id: None,
keep_firing_for_secs: 0,
episode_id: None,
}
}
#[test]
fn a_recovery_inside_a_window_sends_nothing_then_or_later() {
let mut alert = Alert::default();
alert.notify_on_recovery = true;
let mut state = AlertState::empty("a1", "");
apply_episode(&mut state, true, &delivered(), 10, || "ep_1".into());
let closed = apply_episode(&mut state, false, &EpisodeInput::undelivered(0), 20, || {
"x".into()
})
.expect("the delivered episode recovers inside the window");
let event = recovery_event("default", &state, closed, 20);
let muted = recovery_route(&alert, &event, true);
assert_eq!(
muted,
RecoveryRoute {
oncall: false,
destinations: false
}
);
let record = suppressed_recovery_record(&alert, &event, "dt-1".to_string());
assert_eq!(record.status, RunOutcome::Suppressed);
assert_eq!(record.downtime_id.as_deref(), Some("dt-1"));
// The episode closed when it recovered, so no later run re-emits it.
for at in [30, 40] {
assert!(
apply_episode(&mut state, false, &EpisodeInput::undelivered(0), at, || {
"x".into()
})
.is_none()
);
}
}
#[test]
fn the_next_firing_after_the_window_notifies_its_recovery() {
let mut alert = Alert::default();
alert.notify_on_recovery = true;
let mut state = AlertState::empty("a1", "");
apply_episode(&mut state, true, &delivered(), 50, || "ep_2".into());
let closed = apply_episode(&mut state, false, &EpisodeInput::undelivered(0), 60, || {
"x".into()
})
.expect("the next episode recovers");
let event = recovery_event("default", &state, closed, 60);
assert_eq!(
recovery_route(&alert, &event, false),
RecoveryRoute {
oncall: true,
destinations: true
}
);
}
#[cfg(feature = "enterprise")]
#[test]
fn a_hold_that_outlasts_the_window_keeps_the_oncall_recovery_withheld() {
// Delivered before the window; the first clear run falls inside it and starts the hold.
let first = clear_run_oncall(None, Some("dt-1"), true, false);
assert_eq!(
first,
ClearRunOncall {
withheld: true,
record: Some("dt-1"),
forget: false,
}
);
// After the window, still inside the hold: the recorded mute keeps it withheld.
let after = clear_run_oncall(first.record, None, true, true);
assert!(after.withheld);
assert_eq!(after.record, None, "recorded once");
assert!(!after.forget);
// No open episode: withheld inside the window, nothing recorded.
assert_eq!(
clear_run_oncall(None, Some("dt-1"), false, false),
ClearRunOncall {
withheld: true,
record: None,
forget: false,
}
);
assert!(!clear_run_oncall(None, None, true, false).withheld);
}
#[cfg(feature = "enterprise")]
#[test]
fn a_recovery_inside_a_window_closes_the_escalation_once_the_window_is_over() {
use config::meta::oncall::{PageDecision, ResponseState, page_decision};
// Paged at 09:00, the window opens at 09:05 and the alert recovers at 09:10.
let inside = clear_run_oncall(None, Some("dt-1"), true, false);
assert!(inside.withheld, "no recovery is sent inside the window");
// The muted recovery closed the episode and recorded the downtime.
let still_inside = clear_run_oncall(inside.record, Some("dt-1"), false, false);
assert!(still_inside.withheld && !still_inside.forget);
// 10:01, the window is over: the clear run recovers and drops the record.
let after = clear_run_oncall(inside.record, None, false, false);
assert_eq!(
after,
ClearRunOncall {
withheld: false,
record: None,
forget: true,
}
);
// A record left from a window long gone does not hold a firing episode either.
let stale = clear_run_oncall(Some("dt-old"), None, true, false);
assert!(!stale.withheld && stale.forget);
// Left open, the escalation would absorb the next firing; closed, the next firing pages.
let window = config::meta::oncall::DEFAULT_FLAP_DAMPENING_SECS * 1_000_000;
assert_eq!(
page_decision(
Some(&escalation(ResponseState::Triggered, None)),
1_000,
window
),
PageDecision::AlreadyOpen
);
let closed = escalation(ResponseState::Resolved, Some(1_000));
assert_eq!(
page_decision(Some(&closed), 1_000 + window + 1, window),
PageDecision::Page
);
}
#[cfg(feature = "enterprise")]
#[test]
fn a_recovery_before_the_cancel_of_its_window_stays_suppressed_when_emitted_after() {
use std::collections::HashMap;
use config::meta::downtimes::{Downtime, TargetModule};
use crate::alerts::downtimes::enterprise::{
active_by_id_in, active_in, any_target_since, item,
};
const MINUTE: i64 = 60_000_000;
let at = |minutes: i64| 9 * 60 * MINUTE + minutes * MINUTE;
// 09:00-10:00, cancelled at 09:30.
let row: Downtime = serde_json::from_value(serde_json::json!({
"id": "dt-1", "org": "default", "name": "deploy",
"targets": [{ "module": "alerts", "folders": { "kind": "all" } }],
"schedule": {
"repeat": "none", "starts_at": at(0), "ends_at": at(60),
"timezone": "UTC", "duration_secs": 3_600
},
"cancelled_at": at(30),
"created_by": "lin", "created_at": 0, "updated_by": "lin", "updated_at": 0
}))
.unwrap();
let rows = [row];
let mut alert = Alert::default();
alert.notify_on_recovery = true;
// Recovered at 09:20, emitted at 09:40 once `keep_firing_for` ran out.
let event = RecoveryEvent {
org_id: "default".into(),
alert_id: "a1".into(),
group_key: String::new(),
episode_id: "ep_1".into(),
incident_id: None,
opened_at: at(5),
recovered_at: at(20),
emitted_at: at(40),
last_notified_level: None,
group_labels: None,
};
assert!(any_target_since(
&rows,
TargetModule::Alerts,
event.recovered_at,
event.emitted_at
));
let dims = HashMap::new();
let matched = active_in(
&rows,
TargetModule::Alerts,
&item("a1", "default", &dims, &[]),
event.recovered_at,
);
assert_eq!(matched.map(|d| d.id).as_deref(), Some("dt-1"));
let downtime_id =
active_by_id_in(&rows, TargetModule::Alerts, "dt-1", event.recovered_at).map(|d| d.id);
assert_eq!(downtime_id.as_deref(), Some("dt-1"));
assert_eq!(
recovery_route(&alert, &event, downtime_id.is_some()),
RecoveryRoute {
oncall: false,
destinations: false
}
);
let record = suppressed_recovery_record(&alert, &event, downtime_id.unwrap());
assert_eq!(record.status, RunOutcome::Suppressed);
assert_eq!(record.downtime_id.as_deref(), Some("dt-1"));
assert_eq!(record._timestamp, event.emitted_at);
// A recovery after the cancel is not inside the window.
assert!(active_by_id_in(&rows, TargetModule::Alerts, "dt-1", at(35)).is_none());
}
#[cfg(feature = "enterprise")]
fn escalation(
state: config::meta::oncall::ResponseState,
closed_at: Option<i64>,
) -> config::meta::oncall::Response {
use config::meta::oncall::{ResponderRole, SubjectRef, SubjectType};
config::meta::oncall::Response {
id: "resp_1".into(),
org_id: "default".into(),
subject: SubjectRef::new(SubjectType::Alert, "al_1", 1),
team_id: Some("team_1".into()),
title: None,
cause: None,
cause_note: None,
snoozed_until: None,
ladder_anchor: None,
ladder_run: None,
priority: 2,
responder_role: ResponderRole::Owner,
exhausted_at: None,
origin_response_id: None,
state,
opened_at: 0,
acked_by: None,
acked_at: None,
closed_at,
incident_id: None,
updated_at: 0,
}
}
}
src/core/src/alerts/scheduler/handlers.rs
+1620 -224
View File
File diff suppressed because it is too large Load Diff
src/core/src/alerts/templates.rs
+118 -57
View File
@@ -39,6 +39,13 @@ use crate::{
common::meta::authz::Authz,
};
pub const DOWNTIME_TEMPLATE: &str = "prebuilt_downtime";
pub const DOWNTIME_EMAIL_TEMPLATE: &str = "prebuilt_downtime_email";
pub const DOWNTIME_EMAIL_TITLE: &str = "OpenObserve downtime {downtime_name}: {downtime_event}";
const DOWNTIME_HTTP_BODY: &str = r#"{"text": "Downtime {downtime_name}: {downtime_event}. {downtime_starts_at} to {downtime_ends_at}, muting {downtime_counts}. {downtime_url}", "downtime": {"name": "{downtime_name}", "event": "{downtime_event}", "reason": "{downtime_reason}", "starts_at": "{downtime_starts_at}", "ends_at": "{downtime_ends_at}", "targets": "{downtime_targets}", "counts": "{downtime_counts}", "url": "{downtime_url}", "org": "{org_name}"}}"#;
const DOWNTIME_EMAIL_BODY: &str = r#"<h3>Downtime {downtime_name}: {downtime_event}</h3><p>Window: {downtime_starts_at} to {downtime_ends_at}</p><p>Covers {downtime_targets}, muting {downtime_counts}.</p><p>Reason: {downtime_reason}</p><p>Organization: {org_name}</p><p><a href="{downtime_url}">Open downtimes in OpenObserve</a></p>"#;
/// Sticky-kind rule (design §6.2): create defaults absent→Custom; update
/// preserves the existing kind when the client didn't specify one.
pub(crate) fn resolve_kind(
@@ -167,6 +174,32 @@ pub async fn delete(org_id: &str, name: &str, is_root: bool) -> Result<(), Templ
pub(crate) const PREBUILT_REVISION_KEY: &str = "prebuilt_templates_revision";
pub fn downtime_templates() -> [Template; 2] {
let template = |name: &str, body: &str, template_type| Template {
id: None,
org_id: DEFAULT_ORG.to_string(),
name: name.to_string(),
is_default: true,
template_type,
body: body.to_string(),
kind: TemplateKind::Custom,
};
[
template(DOWNTIME_TEMPLATE, DOWNTIME_HTTP_BODY, TemplateType::Http),
template(
DOWNTIME_EMAIL_TEMPLATE,
DOWNTIME_EMAIL_BODY,
TemplateType::Email {
title: DOWNTIME_EMAIL_TITLE.to_string(),
},
),
]
}
pub fn is_downtime_template_name(name: &str) -> bool {
name == DOWNTIME_TEMPLATE || name == DOWNTIME_EMAIL_TEMPLATE
}
/// Reseed gate (design §6.3): overwrite a stored prebuilt template when the
/// shipped revision is newer, or when revisions are equal but the stored
/// body/type drifted — equal-revision drift is the old-binary-revert
@@ -220,85 +253,86 @@ pub async fn ensure_system_templates() -> Result<(), anyhow::Error> {
let mut updated_count = 0;
let mut skipped_count = 0;
for prebuilt_type in prebuilt_types {
if let Some(mut template) = get_prebuilt_template(prebuilt_type) {
// Set org_id to DEFAULT_ORG for global visibility
template.org_id = DEFAULT_ORG.to_string();
let prebuilt = prebuilt_types
.into_iter()
.filter_map(get_prebuilt_template)
.chain(seeded_downtime_templates());
for mut template in prebuilt {
// Set org_id to DEFAULT_ORG for global visibility
template.org_id = DEFAULT_ORG.to_string();
// Check if template already exists
match db::alerts::templates::get(DEFAULT_ORG, &template.name).await {
Ok(existing) => {
// System templates are protected from user edits, so the
// prebuilt definition is the source of truth. Refresh the
// stored copy when it has drifted (e.g. a shipped fix to the
// body/title) so existing installs pick up the correction —
// but only when the reseed gate says to (revision-gated, see
// `should_apply_prebuilt`).
let drifted = existing.body != template.body
|| existing.template_type != template.template_type;
if should_apply_prebuilt(shipped_rev, applied_rev, drifted) && drifted {
// Preserve the stored id so this is an update, not an insert.
template.id = existing.id;
match db::alerts::templates::set(template.clone()).await {
Ok(_) => {
updated_count += 1;
log::info!(
"[TEMPLATES] Updated system template '{}' in {}",
template.name,
DEFAULT_ORG
);
}
Err(e) => {
log::error!(
"[TEMPLATES] Failed to update system template '{}' in {}: {}",
template.name,
DEFAULT_ORG,
e
);
}
}
} else {
skipped_count += 1;
log::debug!(
"[TEMPLATES] System template '{}' already up to date in {}",
template.name,
DEFAULT_ORG
);
}
}
Err(TemplateError::NotFound) => {
// Template doesn't exist, create it
// Check if template already exists
match db::alerts::templates::get(DEFAULT_ORG, &template.name).await {
Ok(existing) => {
// System templates are protected from user edits, so the
// prebuilt definition is the source of truth. Refresh the
// stored copy when it has drifted (e.g. a shipped fix to the
// body/title) so existing installs pick up the correction —
// but only when the reseed gate says to (revision-gated, see
// `should_apply_prebuilt`).
let drifted = existing.body != template.body
|| existing.template_type != template.template_type;
if should_apply_prebuilt(shipped_rev, applied_rev, drifted) && drifted {
// Preserve the stored id so this is an update, not an insert.
template.id = existing.id;
match db::alerts::templates::set(template.clone()).await {
Ok(_) => {
created_count += 1;
updated_count += 1;
log::info!(
"[TEMPLATES] Created system template '{}' in {}",
"[TEMPLATES] Updated system template '{}' in {}",
template.name,
DEFAULT_ORG
);
}
Err(e) => {
log::error!(
"[TEMPLATES] Failed to create system template '{}' in {}: {}",
"[TEMPLATES] Failed to update system template '{}' in {}: {}",
template.name,
DEFAULT_ORG,
e
);
}
}
}
Err(e) => {
log::error!(
"[TEMPLATES] Error checking system template '{}' in {}: {}",
} else {
skipped_count += 1;
log::debug!(
"[TEMPLATES] System template '{}' already up to date in {}",
template.name,
DEFAULT_ORG,
e
DEFAULT_ORG
);
}
}
Err(TemplateError::NotFound) => {
// Template doesn't exist, create it
match db::alerts::templates::set(template.clone()).await {
Ok(_) => {
created_count += 1;
log::info!(
"[TEMPLATES] Created system template '{}' in {}",
template.name,
DEFAULT_ORG
);
}
Err(e) => {
log::error!(
"[TEMPLATES] Failed to create system template '{}' in {}: {}",
template.name,
DEFAULT_ORG,
e
);
}
}
}
Err(e) => {
log::error!(
"[TEMPLATES] Error checking system template '{}' in {}: {}",
template.name,
DEFAULT_ORG,
e
);
}
}
}
// Seed the compiled-in default content template's DB copy — same
// dist_lock + revision-gate scope as the prebuilt loop above, so it
// converges on the same startup pass rather than racing it.
@@ -348,10 +382,37 @@ pub async fn ensure_system_templates() -> Result<(), anyhow::Error> {
Ok(())
}
fn seeded_downtime_templates() -> Vec<Template> {
#[cfg(feature = "enterprise")]
if o2_enterprise::enterprise::common::config::get_config()
.downtimes
.enabled
{
return downtime_templates().into();
}
vec![]
}
#[cfg(test)]
mod tests {
use super::*;
#[test]
fn test_downtime_templates_are_reserved_and_use_only_downtime_variables() {
let [http, email] = downtime_templates();
assert_eq!(http.template_type, TemplateType::Http);
assert!(matches!(email.template_type, TemplateType::Email { .. }));
assert!(serde_json::from_str::<serde_json::Value>(&http.body).is_ok());
for t in [&http, &email] {
assert!(is_downtime_template_name(&t.name));
assert!(
super::super::notifications::default_template::is_reserved_template_name(&t.name)
);
assert!(!t.body.contains("{alert_"));
}
assert!(!is_downtime_template_name("prebuilt_slack_downtime"));
}
#[test]
fn test_should_apply_prebuilt_gate() {
// shipped newer than applied: always apply
src/core/src/anomaly_detection.rs
+47
View File
@@ -1417,6 +1417,7 @@ pub async fn detect_anomalies(org_id: &str, anomaly_id: &str) -> Result<serde_js
error: Some(e.to_string()),
success_response: None,
gate_passed: false,
downtime_id: None,
start_us,
end_us,
};
@@ -1433,6 +1434,7 @@ pub async fn detect_anomalies(org_id: &str, anomaly_id: &str) -> Result<serde_js
serde_json::json!({ "anomalies_found": outcome.anomaly_count }).to_string(),
),
gate_passed: outcome.gate_passed,
downtime_id: outcome.suppressed_by.clone(),
start_us,
end_us,
};
@@ -3167,6 +3169,51 @@ fn anomaly_alert_payload(
/// Send an anomaly alert (or recovery message) to the configured destination.
///
/// The downtime check the anomaly scheduler asks before it sends, registered next to the sender.
#[cfg(feature = "enterprise")]
pub fn downtime_is_muted(
org_id: &str,
anomaly_id: &str,
) -> std::pin::Pin<Box<dyn std::future::Future<Output = Option<String>> + Send>> {
let (org_id, anomaly_id) = (org_id.to_string(), anomaly_id.to_string());
Box::pin(async move { muted_by_downtime(&org_id, &anomaly_id).await })
}
/// `custom_sql` equalities, then plain `=` filters, then `key:value` tags, on the Anomaly target.
#[cfg(feature = "enterprise")]
async fn muted_by_downtime(org_id: &str, anomaly_id: &str) -> Option<String> {
if !crate::alerts::downtimes::any_for(
org_id,
config::meta::downtimes::TargetModule::AnomalyDetections,
) {
return None;
}
let row = anomaly_config_table::get_by_id(get_orm_client_rw().await, org_id, anomaly_id)
.await
.ok()
.flatten()?;
let folder_id = pk_to_name(Some(&row.folder_id)).await;
let tags: Vec<String> = row
.tags
.and_then(|tags| serde_json::from_value(tags).ok())
.unwrap_or_default();
let semantic_groups = crate::db::system_settings::get_semantic_field_groups(org_id).await;
let dims = o2_enterprise::enterprise::downtimes::scope::anomaly_dimensions(
crate::alerts::downtimes::anomaly_sql(&row.query_mode, row.custom_sql.as_deref()),
crate::alerts::downtimes::anomaly_filters(&row.query_mode, row.filters.as_ref()),
&tags,
&semantic_groups,
);
crate::alerts::downtimes::active_for_anomaly(
org_id,
anomaly_id,
&folder_id,
&dims,
config::utils::time::now_micros(),
)
.map(|downtime| downtime.id)
}
/// Called by the enterprise scheduler when anomalies are detected and alert_enabled=true.
/// Looks up the destination by name and POSTs a JSON payload to its webhook URL.
/// Non-HTTP destinations (email, SNS) are skipped with a warning — a known, parked gap
src/core/src/downtimes.rs
+3001
View File
File diff suppressed because it is too large Load Diff
src/core/src/downtimes/access.rs
+390
View File
@@ -0,0 +1,390 @@
// Copyright 2026 OpenObserve Inc.
//
// This program is free software: you can redistribute it and/or modify
// it under the terms of the GNU Affero General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
//
// This program is distributed in the hope that it will be useful
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU Affero General Public License for more details.
//
// You should have received a copy of the GNU Affero General Public License
// along with this program. If not, see <http://www.gnu.org/licenses/>.
//! The extra rules for a feature that can silence what a user cannot see (WP6).
use std::collections::HashSet;
use config::meta::{
downtimes::{Downtime, DowntimeTarget, TargetFolders, TargetModule},
folder::FolderType,
};
use o2_openfga::authorizer::authz::{get_ofga_type, remove_parent_relation, set_parent_relation};
use super::{
DowntimeError,
matching::{Inventory, Visibility},
};
use crate::auth::check_permissions;
/// Get, update, cancel and delete are checked against the row's own folder.
pub async fn authorize_row(
org: &str,
user_id: &str,
row: &Downtime,
method: &str,
) -> Result<(), DowntimeError> {
if check_permissions(
&row.id,
org,
user_id,
"downtimes",
method,
Some(&row.folder_id),
false,
true,
false,
)
.await
{
Ok(())
} else {
Err(DowntimeError::Forbidden(format!(
"You are not allowed to {} this downtime.",
verb(method)
)))
}
}
/// Folders must exist and be listable; ids must exist, lie in the chosen folders and be readable.
pub async fn check_targets(
org: &str,
user_id: &str,
targets: &[DowntimeTarget],
inventory: &Inventory,
) -> Result<(), DowntimeError> {
for target in targets {
check_target_folders(org, user_id, target).await?;
check_target_ids(org, user_id, target, inventory).await?;
}
Ok(())
}
/// The folders whose items the user may see, per folder type.
pub async fn visibility(org: &str, user_id: &str) -> Result<Visibility, DowntimeError> {
Ok(Visibility {
alert_folders: Some(listable_folders(org, user_id, FolderType::Alerts).await?),
synthetics_folders: Some(listable_folders(org, user_id, FolderType::Synthetics).await?),
})
}
/// The downtime folders the user may list; the list route is org-level, so each folder is checked.
pub async fn listable_downtime_folders(
org: &str,
user_id: &str,
) -> Result<HashSet<String>, DowntimeError> {
let candidates = listable_folders(org, user_id, FolderType::Downtimes).await?;
Ok(retain_listable_folders(org, user_id, candidates).await)
}
/// Of rows outside the listable folders, those granted one by one; one OpenFGA ListObjects call.
pub async fn individually_readable(
org: &str,
user_id: &str,
rows: Vec<(String, String)>,
) -> HashSet<String> {
if rows.is_empty() {
return HashSet::new();
}
let ofga_type = get_ofga_type("downtimes");
match crate::authz::list_objects_for_user(org, user_id, "GET_INDIVIDUAL_FROM_ROLE", &ofga_type)
.await
{
Ok(Some(objects)) => granted_ids(&ofga_type, org, &objects, rows),
// No list filtering configured: the per-row check the GET route runs decides.
Ok(None) => {
let checks = rows.into_iter().map(|(id, folder_id)| async move {
check_permissions(
&id,
org,
user_id,
"downtimes",
"GET",
Some(&folder_id),
false,
true,
false,
)
.await
.then_some(id)
});
futures::future::join_all(checks)
.await
.into_iter()
.flatten()
.collect()
}
Err(e) => {
log::warn!("[downtimes] individual grants of {user_id} in {org} unreadable: {e}");
HashSet::new()
}
}
}
/// Keeps the downtime folders the user may LIST, each checked on its own.
pub async fn retain_listable_folders(
org: &str,
user_id: &str,
folder_ids: impl IntoIterator<Item = String>,
) -> HashSet<String> {
let checks = folder_ids.into_iter().map(|folder_id| async move {
check_permissions(
&folder_id,
org,
user_id,
"downtime_folders",
"LIST",
None,
false,
false,
true,
)
.await
.then_some(folder_id)
});
futures::future::join_all(checks)
.await
.into_iter()
.flatten()
.collect()
}
/// Moves the folder tuple; writing a tuple that exists fails, so an unchanged folder is skipped.
pub async fn reparent(id: &str, from: &str, to: &str) {
if from == to || !o2_openfga::config::get_config().enabled {
return;
}
let downtime = get_ofga_type("downtimes");
let folder = get_ofga_type("downtime_folders");
set_parent_relation(id, &downtime, to, &folder).await;
remove_parent_relation(id, &downtime, from, &folder).await;
}
async fn listable_folders(
org: &str,
user_id: &str,
folder_type: FolderType,
) -> Result<HashSet<String>, DowntimeError> {
Ok(db::folders::list_folders(org, Some(user_id), folder_type)
.await
.map_err(|e| DowntimeError::Internal(e.to_string()))?
.into_iter()
.map(|f| f.folder_id)
.collect())
}
async fn check_target_folders(
org: &str,
user_id: &str,
target: &DowntimeTarget,
) -> Result<(), DowntimeError> {
if !needs_folder_check(target) {
return Ok(());
}
let (folder_type, ofga_type) = folder_kind(target.module);
let TargetFolders::Some { folder_ids } = &target.folders else {
// All needs the role-wide LIST grant on every folder of the module.
let all = format!("_all_{org}");
if check_permissions(
&all, org, user_id, ofga_type, "LIST", None, false, false, false,
)
.await
{
return Ok(());
}
return Err(DowntimeError::Forbidden(all_folders_denied(target.module)));
};
for folder_id in folder_ids {
if !infra::table::folders::exists(org, folder_id, folder_type).await? {
return Err(DowntimeError::BadRequest(format!(
"Folder {folder_id} does not exist for {}.",
module_label(target.module)
)));
}
if !check_permissions(
folder_id, org, user_id, ofga_type, "LIST", None, false, false, true,
)
.await
{
return Err(DowntimeError::Forbidden(format!(
"You cannot list folder {folder_id}, so you cannot silence it."
)));
}
}
Ok(())
}
async fn check_target_ids(
org: &str,
user_id: &str,
target: &DowntimeTarget,
inventory: &Inventory,
) -> Result<(), DowntimeError> {
let object_type = match target.module {
TargetModule::Synthetics => "synthetics",
_ => "alerts",
};
for id in &target.ids {
let Some(item) = inventory.find(target.module, id) else {
return Err(DowntimeError::BadRequest(format!(
"{id} is not one of the {} of this organization.",
module_label(target.module)
)));
};
if let TargetFolders::Some { folder_ids } = &target.folders
&& !folder_ids.contains(&item.folder_id)
{
return Err(DowntimeError::BadRequest(format!(
"{id} is not in the chosen folders. Choose its folder or remove it."
)));
}
if !check_permissions(
id,
org,
user_id,
object_type,
"GET",
Some(&item.folder_id),
false,
true,
false,
)
.await
{
return Err(DowntimeError::Forbidden(format!(
"You cannot read {id}, so you cannot mute it."
)));
}
}
Ok(())
}
/// Named ids narrow All to those items, and [check_target_ids] reads each of them.
fn needs_folder_check(target: &DowntimeTarget) -> bool {
!matches!(target.folders, TargetFolders::All) || target.ids.is_empty()
}
/// The `(id, folder)` rows a ListObjects answer grants; a role-wide grant answers `_all_{org}`.
fn granted_ids(
ofga_type: &str,
org: &str,
objects: &[String],
rows: Vec<(String, String)>,
) -> HashSet<String> {
let objects: HashSet<&str> = objects.iter().map(String::as_str).collect();
let all = objects.contains(format!("{ofga_type}:_all_{org}").as_str());
rows.into_iter()
.map(|(id, _)| id)
.filter(|id| all || objects.contains(format!("{ofga_type}:{id}").as_str()))
.collect()
}
fn all_folders_denied(module: TargetModule) -> String {
let folders = match module {
TargetModule::Synthetics => "synthetics",
_ => "alert",
};
format!("You cannot list every {folders} folder, so you cannot silence all of them.")
}
/// SLOs and anomaly detections live in alert folders and use the `alerts` resource.
fn folder_kind(module: TargetModule) -> (FolderType, &'static str) {
match module {
TargetModule::Synthetics => (FolderType::Synthetics, "synthetic_folder"),
_ => (FolderType::Alerts, "alert_folders"),
}
}
pub(super) fn module_label(module: TargetModule) -> &'static str {
match module {
TargetModule::Alerts => "alerts",
TargetModule::AnomalyDetections => "anomaly detections",
TargetModule::Synthetics => "synthetics checks",
TargetModule::Slos => "SLOs",
}
}
fn verb(method: &str) -> &'static str {
match method {
"GET" => "read",
"DELETE" => "delete",
_ => "change",
}
}
#[cfg(test)]
mod tests {
use super::*;
fn rows() -> Vec<(String, String)> {
[("d1", "ops"), ("d2", "ops"), ("d3", "db")]
.map(|(id, folder)| (id.to_string(), folder.to_string()))
.to_vec()
}
#[test]
fn a_single_downtime_grant_lists_only_that_row() {
let objects = ["downtime:d2".to_string()];
let granted = granted_ids("downtime", "acme", &objects, rows());
assert_eq!(granted, HashSet::from(["d2".to_string()]));
}
#[test]
fn all_folders_without_the_org_wide_list_right_names_the_folders_it_needs() {
assert_eq!(
all_folders_denied(TargetModule::Alerts),
"You cannot list every alert folder, so you cannot silence all of them."
);
assert_eq!(
all_folders_denied(TargetModule::Slos),
all_folders_denied(TargetModule::AnomalyDetections)
);
assert_eq!(
all_folders_denied(TargetModule::Synthetics),
"You cannot list every synthetics folder, so you cannot silence all of them."
);
}
fn target(folders: TargetFolders, ids: &[&str]) -> DowntimeTarget {
DowntimeTarget {
module: TargetModule::Alerts,
folders,
tags: vec![],
ids: ids.iter().map(|id| id.to_string()).collect(),
slo_mode: None,
incident_mode: Default::default(),
}
}
#[test]
fn a_quick_mute_of_named_ids_needs_no_org_wide_list() {
assert!(!needs_folder_check(&target(TargetFolders::All, &["a1"])));
assert!(needs_folder_check(&target(TargetFolders::All, &[])));
let some = TargetFolders::Some {
folder_ids: vec!["ops".to_string()],
};
assert!(needs_folder_check(&target(some.clone(), &[])));
assert!(needs_folder_check(&target(some, &["a1"])));
}
#[test]
fn a_role_wide_grant_lists_every_row_and_none_lists_nothing() {
let all = ["downtime:_all_acme".to_string()];
assert_eq!(granted_ids("downtime", "acme", &all, rows()).len(), 3);
let other_org = ["downtime:_all_other".to_string()];
assert!(granted_ids("downtime", "acme", &other_org, rows()).is_empty());
assert!(granted_ids("downtime", "acme", &[], rows()).is_empty());
}
}
src/core/src/downtimes/inventory.rs
+294
View File
@@ -0,0 +1,294 @@
// Copyright 2026 OpenObserve Inc.
//
// This program is free software: you can redistribute it and/or modify
// it under the terms of the GNU Affero General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
//
// This program is distributed in the hope that it will be useful
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU Affero General Public License for more details.
//
// You should have received a copy of the GNU Affero General Public License
// along with this program. If not, see <http://www.gnu.org/licenses/>.
//! The org's items a downtime can cover, with identities read from their definitions.
use std::{
collections::HashMap,
sync::{Arc, LazyLock, RwLock},
time::{Duration, Instant},
};
use config::meta::{
alerts::alert::Alert,
correlation::FieldAlias,
folder::Folder,
slo::{SliConfig, Slo},
synthetics::ListSyntheticsParams,
};
use infra::table::entity::alert_composites;
use o2_enterprise::enterprise::{
downtimes::scope::{anomaly_dimensions, composite_dimensions, slo_dimensions},
oncall::routing::dimensions_for_alert,
};
use super::matching::{Inventory, Item};
/// Counts and banners read the inventory at most this stale.
const INVENTORY_TTL: Duration = Duration::from_secs(60);
/// `org -> (loaded at, inventory)`.
type InventoryCache = RwLock<HashMap<String, (Instant, Arc<Inventory>)>>;
static INVENTORIES: LazyLock<InventoryCache> = LazyLock::new(Default::default);
/// The inventory of an org, reloaded when older than a minute.
pub async fn cached(org: &str) -> Result<Arc<Inventory>, anyhow::Error> {
let fresh = INVENTORIES
.read()
.unwrap_or_else(|e| e.into_inner())
.get(org)
.filter(|(at, _)| at.elapsed() < INVENTORY_TTL)
.map(|(_, inventory)| inventory.clone());
if let Some(inventory) = fresh {
return Ok(inventory);
}
let inventory = Arc::new(load(org).await?);
INVENTORIES
.write()
.unwrap_or_else(|e| e.into_inner())
.insert(org.to_string(), (Instant::now(), inventory.clone()));
Ok(inventory)
}
pub async fn load(org: &str) -> Result<Inventory, anyhow::Error> {
let groups = db::system_settings::get_semantic_field_groups(org).await;
let alerts = cached_alerts(org).await;
let conn = infra::db::get_orm_client_ro().await;
let slos = infra::table::slos::list(conn, org, None).await?;
let composites = infra::table::alert_composites::list_by_org(conn, org).await?;
let mut alert_items = alert_items(&alerts, &slos, &groups);
alert_items.extend(composites.iter().map(composite_item));
Ok(Inventory {
alerts: alert_items,
anomalies: anomaly_items(org, &groups).await?,
synthetics: synthetic_items(org).await?,
slos: slos
.iter()
.map(|slo| slo_item(slo, &groups, &alerts))
.collect(),
})
}
async fn cached_alerts(org: &str) -> Vec<(Folder, Alert)> {
let prefix = format!("{org}/");
common::infra::config::ALERTS
.read()
.await
.iter()
.filter(|(key, _)| key.starts_with(&prefix))
.map(|(_, value)| value.clone())
.collect()
}
/// An SLO alert runs no query, so it takes the identity of its SLO (WP4 item 3a).
fn alert_items(alerts: &[(Folder, Alert)], slos: &[Slo], groups: &[FieldAlias]) -> Vec<Item> {
alerts
.iter()
.filter_map(|(folder, alert)| {
let id = alert.id?.to_string();
let dims = match &alert.query_condition.slo_condition {
Some(condition) => slos
.iter()
.find(|slo| slo.id == condition.slo_id)
.map(|slo| slo_item(slo, groups, alerts).dims)
.unwrap_or_default(),
None => dimensions_for_alert(groups, &alert.query_condition, &Default::default()),
};
Some(Item {
id,
name: alert.name.clone(),
folder_id: folder.folder_id.clone(),
dims,
tags: alert.tags.clone(),
..Default::default()
})
})
.collect()
}
/// Composites are not in `ALERTS`; their `key:value` tags are their identity, as at run time.
fn composite_item(composite: &alert_composites::Model) -> Item {
let tags: Vec<String> = composite
.tags
.clone()
.and_then(|value| serde_json::from_value(value).ok())
.unwrap_or_default();
Item {
id: composite.id.clone(),
name: composite.name.clone(),
folder_id: composite.folder_id.clone(),
dims: composite_dimensions(&tags),
tags,
..Default::default()
}
}
fn slo_item(slo: &Slo, groups: &[FieldAlias], alerts: &[(Folder, Alert)]) -> Item {
let source = match &slo.definition.sli_config {
SliConfig::Alert { alert_id } => alerts
.iter()
.find(|(_, alert)| alert.id.is_some_and(|id| id.to_string() == *alert_id))
.map(|(_, alert)| &alert.query_condition),
_ => None,
};
let untagged = Slo {
tags: vec![],
..slo.clone()
};
Item {
id: slo.id.clone(),
name: slo.name.clone(),
folder_id: slo.folder_id.clone(),
dims: slo_dimensions(slo, groups, source),
tags: slo.tags.clone(),
measured: slo_dimensions(&untagged, groups, source),
measured_by: if matches!(slo.definition.sli_config, SliConfig::Alert { .. }) {
"source_alert"
} else {
"query"
},
slice_interval_secs: slo.definition.slice_interval_secs,
}
}
async fn anomaly_items(org: &str, groups: &[FieldAlias]) -> Result<Vec<Item>, anyhow::Error> {
let conn = infra::db::get_orm_client_ro().await;
let configs = infra::table::anomaly_detection::config::list_by_org(conn, org).await?;
let mut slugs: HashMap<String, String> = HashMap::new();
let mut items = Vec::with_capacity(configs.len());
for config in configs {
let folder_id = match slugs.get(&config.folder_id) {
Some(slug) => slug.clone(),
None => {
let slug = infra::table::folders::get_name_by_pk(&config.folder_id)
.await?
.unwrap_or_else(|| config.folder_id.clone());
slugs.insert(config.folder_id.clone(), slug.clone());
slug
}
};
let tags: Vec<String> = config
.tags
.and_then(|t| serde_json::from_value(t).ok())
.unwrap_or_default();
items.push(Item {
dims: anomaly_dimensions(
crate::alerts::downtimes::anomaly_sql(
&config.query_mode,
config.custom_sql.as_deref(),
),
crate::alerts::downtimes::anomaly_filters(
&config.query_mode,
config.filters.as_ref(),
),
&tags,
groups,
),
id: config.anomaly_id,
name: config.name,
folder_id,
tags,
..Default::default()
});
}
Ok(items)
}
async fn synthetic_items(org: &str) -> Result<Vec<Item>, anyhow::Error> {
let checks =
openobserve_synthetics::service::list_synthetics(org, &ListSyntheticsParams::default())
.await?
.checks;
Ok(checks
.into_iter()
.map(|check| Item {
id: check.id,
name: check.name,
folder_id: check.folder_id,
tags: check.tags,
..Default::default()
})
.collect())
}
#[cfg(test)]
mod tests {
use config::meta::downtimes::{DowntimeTarget, TargetFolders, TargetModule};
use super::*;
use crate::downtimes::matching::{Visibility, match_all, to_preview};
fn composite(id: &str, folder_id: &str, tags: &[&str]) -> alert_composites::Model {
alert_composites::Model {
id: id.to_string(),
org: "acme".to_string(),
folder_id: folder_id.to_string(),
name: format!("{id} name"),
description: None,
expression: "a AND b".to_string(),
warning_counts_as_firing: false,
stale_child_policy: 0,
destinations: serde_json::json!([]),
template: None,
context_attributes: None,
enabled: true,
silence_seconds: 0,
creates_incident: false,
workflows: serde_json::json!([]),
priority: None,
tags: Some(serde_json::json!(tags)),
owner: None,
last_edited_by: None,
updated_at: None,
evaluation_generation: 0,
pending_period_sec: 0,
}
}
#[test]
fn a_composite_is_an_alert_item_whose_tags_are_its_identity() {
let item = composite_item(&composite("c1", "ops", &["service:payments", "critical"]));
assert_eq!(item.id, "c1");
assert_eq!(item.folder_id, "ops");
assert_eq!(item.tags, ["service:payments", "critical"]);
assert_eq!(item.dims, composite_dimensions(&item.tags));
assert!(!item.dims.is_empty());
}
#[test]
fn a_composite_named_by_id_is_found_and_previewed_under_its_folder() {
let inventory = Inventory {
alerts: vec![composite_item(&composite("c1", "ops", &[]))],
..Default::default()
};
assert!(inventory.find(TargetModule::Alerts, "c1").is_some());
let targets = [DowntimeTarget {
module: TargetModule::Alerts,
folders: TargetFolders::All,
tags: vec![],
ids: vec!["c1".to_string()],
slo_mode: None,
incident_mode: Default::default(),
}];
let preview = to_preview(
&match_all(&inventory, None, &targets),
&Visibility::default(),
);
assert_eq!(preview.alerts_total, 1);
assert_eq!(preview.alerts[0].id, "c1");
assert_eq!(preview.alerts[0].folder_id, "ops");
}
}
src/core/src/downtimes/listing.rs
+329
View File
@@ -0,0 +1,329 @@
// Copyright 2026 OpenObserve Inc.
//
// This program is free software: you can redistribute it and/or modify
// it under the terms of the GNU Affero General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
//
// This program is distributed in the hope that it will be useful
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU Affero General Public License for more details.
//
// You should have received a copy of the GNU Affero General Public License
// along with this program. If not, see <http://www.gnu.org/licenses/>.
//! The list endpoint's filters, counts and paging. Pure.
use config::meta::downtimes::{Downtime, DowntimeListItem, DowntimeStatus, Repeat, TargetModule};
pub use config::meta::downtimes::{
DowntimeStatusCounts as StatusCounts, ListDowntimesQuery as ListQuery,
ListDowntimesResponse as ListResponse,
};
use o2_enterprise::enterprise::downtimes::scope::{self, TargetItem};
pub const DEFAULT_PAGE_SIZE: usize = 50;
pub const MAX_PAGE_SIZE: usize = 500;
/// Filters, counts and pages already-authorized rows. `alert` is the item behind `alert_id`.
pub fn list_page(
items: Vec<DowntimeListItem>,
query: &ListQuery,
alert: Option<&TargetItem<'_>>,
) -> ListResponse {
let mut kept: Vec<DowntimeListItem> = items
.into_iter()
.filter(|item| keep(item, query, alert))
.collect();
let counts = count(&kept);
kept.sort_by_key(|item| std::cmp::Reverse(item.downtime.updated_at));
let total = kept.len();
let size = query
.page_size
.unwrap_or(DEFAULT_PAGE_SIZE)
.clamp(1, MAX_PAGE_SIZE);
let page = query.page.unwrap_or(1).max(1);
let items = kept
.into_iter()
.skip((page - 1).saturating_mul(size))
.take(size)
.collect();
ListResponse {
items,
total,
counts,
}
}
fn keep(item: &DowntimeListItem, query: &ListQuery, alert: Option<&TargetItem<'_>>) -> bool {
let d = &item.downtime;
query.folder_id.as_ref().is_none_or(|f| *f == d.folder_id)
&& query
.status
.as_deref()
.is_none_or(|s| status_name(item.status) == s)
&& query
.repeat
.as_deref()
.is_none_or(|r| repeat_matches(d.schedule.repeat, r))
&& query
.search
.as_deref()
.filter(|s| !s.trim().is_empty())
.is_none_or(|s| mentions(d, s))
&& (query.alert_id.is_none() || alert.is_some_and(|a| names_alert(d, a)))
}
fn count(items: &[DowntimeListItem]) -> StatusCounts {
let mut counts = StatusCounts::default();
for item in items {
match item.status {
DowntimeStatus::Active => counts.active += 1,
DowntimeStatus::Scheduled => counts.scheduled += 1,
DowntimeStatus::Ended => counts.ended += 1,
DowntimeStatus::Cancelled => counts.cancelled += 1,
DowntimeStatus::EndedEarly => counts.ended_early += 1,
}
if item.downtime.schedule.repeat != Repeat::None {
counts.recurring += 1;
}
}
counts
}
fn status_name(status: DowntimeStatus) -> &'static str {
match status {
DowntimeStatus::Scheduled => "scheduled",
DowntimeStatus::Active => "active",
DowntimeStatus::Ended => "ended",
DowntimeStatus::Cancelled => "cancelled",
DowntimeStatus::EndedEarly => "ended_early",
}
}
fn repeat_matches(repeat: Repeat, wanted: &str) -> bool {
match wanted {
"recurring" => repeat != Repeat::None,
"none" | "once" => repeat == Repeat::None,
"daily" => repeat == Repeat::Daily,
"weekly" => repeat == Repeat::Weekly,
_ => false,
}
}
fn mentions(d: &Downtime, search: &str) -> bool {
let needle = search.trim().to_lowercase();
d.name.to_lowercase().contains(&needle)
|| d.reason
.as_deref()
.is_some_and(|r| r.to_lowercase().contains(&needle))
}
/// The send path's rule: folders, tags, ids and the condition over the alert's dimensions.
fn names_alert(d: &Downtime, alert: &TargetItem<'_>) -> bool {
scope::target_for(&d.targets, TargetModule::Alerts)
.is_some_and(|target| scope::matches(target, d.condition.as_ref(), alert))
}
#[cfg(test)]
mod tests {
use std::collections::HashMap;
use config::meta::downtimes::{
DimensionCondition, DowntimeSchedule, DowntimeTarget, PairOperator, TargetFolders,
};
use super::*;
fn item(id: &str, status: DowntimeStatus, repeat: Repeat, updated_at: i64) -> DowntimeListItem {
DowntimeListItem {
downtime: Downtime {
id: id.to_string(),
org: "acme".to_string(),
folder_id: if id.starts_with('p') {
"planned"
} else {
"default"
}
.to_string(),
name: format!("{id} maintenance"),
reason: Some("CHG-4471".to_string()),
condition: Some(DimensionCondition::Pair {
key: "service".to_string(),
operator: PairOperator::Eq,
value: "payments".to_string(),
}),
targets: vec![DowntimeTarget {
module: TargetModule::Alerts,
folders: TargetFolders::All,
tags: vec![],
ids: vec![],
slo_mode: None,
incident_mode: Default::default(),
}],
schedule: DowntimeSchedule {
repeat,
starts_at: 0,
ends_at: None,
timezone: "UTC".to_string(),
start_time_local: None,
duration_secs: 60,
weekdays: vec![],
},
cancelled_at: None,
cancelled_by: None,
show_banner: true,
notifications: None,
origin_region: None,
version: 0,
created_by: "lin".to_string(),
created_at: 0,
updated_by: "lin".to_string(),
updated_at,
},
status,
current_window: None,
next_window: None,
matched_alerts: 0,
matched_anomalies: 0,
matched_synthetics: 0,
matched_slos: 0,
}
}
fn rows() -> Vec<DowntimeListItem> {
vec![
item("a", DowntimeStatus::Active, Repeat::None, 5),
item("p1", DowntimeStatus::Scheduled, Repeat::Weekly, 4),
item("p2", DowntimeStatus::Scheduled, Repeat::None, 3),
item("e", DowntimeStatus::Ended, Repeat::None, 2),
item("c", DowntimeStatus::Cancelled, Repeat::Daily, 1),
item("x", DowntimeStatus::EndedEarly, Repeat::None, 0),
]
}
fn ids(resp: &ListResponse) -> Vec<&str> {
resp.items.iter().map(|i| i.downtime.id.as_str()).collect()
}
#[test]
fn counts_cover_the_filtered_rows_and_recurring_overlaps() {
let resp = list_page(rows(), &ListQuery::default(), None);
assert_eq!(resp.total, 6);
assert_eq!(
resp.counts,
StatusCounts {
active: 1,
scheduled: 2,
recurring: 2,
ended: 1,
cancelled: 1,
ended_early: 1,
}
);
assert_eq!(ids(&resp), ["a", "p1", "p2", "e", "c", "x"]);
}
#[test]
fn filters_by_folder_status_repeat_and_search() {
let q = |f: fn(&mut ListQuery)| {
let mut query = ListQuery::default();
f(&mut query);
list_page(rows(), &query, None)
};
assert_eq!(
ids(&q(|q| q.folder_id = Some("planned".into()))),
["p1", "p2"]
);
assert_eq!(
ids(&q(|q| q.status = Some("scheduled".into()))),
["p1", "p2"]
);
assert_eq!(ids(&q(|q| q.status = Some("ended_early".into()))), ["x"]);
assert_eq!(ids(&q(|q| q.status = Some("cancelled".into()))), ["c"]);
assert_eq!(
ids(&q(|q| q.repeat = Some("recurring".into()))),
["p1", "c"]
);
assert_eq!(
ids(&q(|q| q.repeat = Some("none".into()))),
["a", "p2", "e", "x"]
);
assert_eq!(ids(&q(|q| q.search = Some("P1 MAINT".into()))), ["p1"]);
assert_eq!(q(|q| q.search = Some("chg-4471".into())).total, 6);
}
#[test]
fn pages_are_one_based_and_capped() {
let query = ListQuery {
page: Some(2),
page_size: Some(2),
..Default::default()
};
let resp = list_page(rows(), &query, None);
assert_eq!(ids(&resp), ["p2", "e"]);
assert_eq!(resp.total, 6);
let huge = ListQuery {
page_size: Some(10_000),
..Default::default()
};
assert_eq!(list_page(rows(), &huge, None).items.len(), 6);
}
#[test]
fn the_alert_filter_reads_ids_and_the_condition() {
let dims = HashMap::from([("service".to_string(), "payments".to_string())]);
let alert = TargetItem {
id: "al1",
folder_id: "default",
dimensions: &dims,
tags: &[],
};
let query = ListQuery {
alert_id: Some("al1".into()),
..Default::default()
};
assert_eq!(list_page(rows(), &query, Some(&alert)).total, 6);
let other = HashMap::from([("service".to_string(), "checkout".to_string())]);
let unrelated = TargetItem {
dimensions: &other,
..alert
};
assert_eq!(list_page(rows(), &query, Some(&unrelated)).total, 0);
assert_eq!(list_page(rows(), &query, None).total, 0);
}
#[test]
fn a_folder_only_downtime_is_listed_for_an_alert_in_that_folder() {
let no_dims = HashMap::new();
let in_planned = TargetItem {
id: "al1",
folder_id: "planned",
dimensions: &no_dims,
tags: &[],
};
let elsewhere = TargetItem {
folder_id: "default",
..in_planned
};
let folder_only = |folders: TargetFolders| {
let mut row = item("f", DowntimeStatus::Active, Repeat::None, 1);
row.downtime.condition = None;
row.downtime.targets[0].folders = folders;
row
};
let query = ListQuery {
alert_id: Some("al1".into()),
..Default::default()
};
let planned = TargetFolders::Some {
folder_ids: vec!["planned".to_string()],
};
let rows = vec![folder_only(planned)];
assert_eq!(list_page(rows.clone(), &query, Some(&in_planned)).total, 1);
assert_eq!(list_page(rows, &query, Some(&elsewhere)).total, 0);
let all = vec![folder_only(TargetFolders::All)];
assert_eq!(list_page(all.clone(), &query, Some(&in_planned)).total, 1);
assert_eq!(list_page(all, &query, Some(&elsewhere)).total, 1);
}
}
src/core/src/downtimes/matching.rs
+469
View File
@@ -0,0 +1,469 @@
// Copyright 2026 OpenObserve Inc.
//
// This program is free software: you can redistribute it and/or modify
// it under the terms of the GNU Affero General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
//
// This program is distributed in the hope that it will be useful
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU Affero General Public License for more details.
//
// You should have received a copy of the GNU Affero General Public License
// along with this program. If not, see <http://www.gnu.org/licenses/>.
//! What a downtime covers today, forecast from definitions; pure over preloaded items.
use std::collections::{HashMap, HashSet};
use config::meta::downtimes::{
DimensionCondition, DowntimeTarget, LogicalOp, PairOperator, PreviewMatch, PreviewResponse,
TargetFolders, TargetModule,
};
use o2_enterprise::enterprise::downtimes::scope::{self, TargetItem};
/// Preview lists are capped at this many names per module; totals count them all.
pub const PREVIEW_CAP: usize = 50;
/// One alert, anomaly detection, check or SLO, as the forecast sees it.
#[derive(Clone, Debug, Default, PartialEq)]
pub struct Item {
pub id: String,
pub name: String,
/// Public folder id.
pub folder_id: String,
pub dims: HashMap<String, String>,
pub tags: Vec<String>,
/// SLOs only: the dimensions of what the SLO measures, without its tags (D9).
pub measured: HashMap<String, String>,
/// SLOs only: `"query"` or `"source_alert"`.
pub measured_by: &'static str,
/// SLOs only: the slice a correction window must hold a start of.
pub slice_interval_secs: i64,
}
impl Item {
fn view(&self) -> TargetItem<'_> {
TargetItem {
id: &self.id,
folder_id: &self.folder_id,
dimensions: &self.dims,
tags: &self.tags,
}
}
fn preview(&self) -> PreviewMatch {
PreviewMatch {
id: self.id.clone(),
name: self.name.clone(),
folder_id: self.folder_id.clone(),
matched_by: None,
missing: None,
applies: None,
}
}
}
/// Every item of an org that a downtime can cover, per module.
#[derive(Clone, Debug, Default)]
pub struct Inventory {
pub alerts: Vec<Item>,
pub anomalies: Vec<Item>,
pub synthetics: Vec<Item>,
pub slos: Vec<Item>,
}
impl Inventory {
pub fn items(&self, module: TargetModule) -> &[Item] {
match module {
TargetModule::Alerts => &self.alerts,
TargetModule::AnomalyDetections => &self.anomalies,
TargetModule::Synthetics => &self.synthetics,
TargetModule::Slos => &self.slos,
}
}
pub fn find(&self, module: TargetModule, id: &str) -> Option<&Item> {
self.items(module).iter().find(|item| item.id == id)
}
}
/// What one target matches in its module.
#[derive(Clone, Debug, Default, PartialEq)]
pub struct ModuleMatch {
pub matched: Vec<PreviewMatch>,
/// Alerts whose definition names no dimension: decided when they fire (section 7.6).
pub at_fire_time: Vec<PreviewMatch>,
/// Items whose identity lacks a key the condition tests inside an `Or` group (D16).
pub undecidable: Vec<PreviewMatch>,
}
#[derive(Clone, Debug, Default, PartialEq)]
pub struct Matches {
pub alerts: ModuleMatch,
pub anomalies: ModuleMatch,
pub synthetics: ModuleMatch,
pub slos: ModuleMatch,
}
impl Matches {
pub fn module(&self, module: TargetModule) -> &ModuleMatch {
match module {
TargetModule::Alerts => &self.alerts,
TargetModule::AnomalyDetections => &self.anomalies,
TargetModule::Synthetics => &self.synthetics,
TargetModule::Slos => &self.slos,
}
}
}
/// The folders a user may list, per folder type; `None` means every folder.
#[derive(Clone, Debug, Default)]
pub struct Visibility {
pub alert_folders: Option<HashSet<String>>,
pub synthetics_folders: Option<HashSet<String>>,
}
impl Visibility {
pub fn sees(&self, module: TargetModule, folder_id: &str) -> bool {
let folders = match module {
TargetModule::Synthetics => &self.synthetics_folders,
_ => &self.alert_folders,
};
folders.as_ref().is_none_or(|f| f.contains(folder_id))
}
}
pub fn match_all(
inventory: &Inventory,
condition: Option<&DimensionCondition>,
targets: &[DowntimeTarget],
) -> Matches {
let run = |module| {
scope::target_for(targets, module)
.map(|target| match_module(target, condition, inventory.items(module)))
.unwrap_or_default()
};
Matches {
alerts: run(TargetModule::Alerts),
anomalies: run(TargetModule::AnomalyDetections),
synthetics: run(TargetModule::Synthetics),
slos: run(TargetModule::Slos),
}
}
pub fn match_module(
target: &DowntimeTarget,
condition: Option<&DimensionCondition>,
items: &[Item],
) -> ModuleMatch {
let condition = condition.filter(|_| target.module.has_identity());
let mut out = ModuleMatch::default();
for item in items {
if !scope::matches(target, None, &item.view()) {
continue;
}
let Some(cond) = condition else {
out.matched.push(matched(target, None, item));
continue;
};
if target.module == TargetModule::Alerts && item.dims.is_empty() {
out.at_fire_time.push(item.preview());
} else if scope::eval_condition(cond, &item.dims) {
out.matched.push(matched(target, Some(cond), item));
} else if !ruled_out(cond, &item.dims)
&& let Some(key) = scope::undecidable_key(cond, &item.dims)
{
out.undecidable.push(PreviewMatch {
missing: Some(key),
..item.preview()
});
}
}
out
}
/// The preview a user sees: names filtered to what the user may list, capped, totals complete.
pub fn to_preview(matches: &Matches, visibility: &Visibility) -> PreviewResponse {
let visible = |module: TargetModule, list: &[PreviewMatch], cap: usize| -> Vec<PreviewMatch> {
list.iter()
.filter(|m| visibility.sees(module, &m.folder_id))
.take(cap)
.cloned()
.collect()
};
let mut undecidable = HashMap::new();
for (key, module) in [
("alerts", TargetModule::Alerts),
("anomalies", TargetModule::AnomalyDetections),
("slos", TargetModule::Slos),
] {
let list = visible(module, &matches.module(module).undecidable, PREVIEW_CAP);
if !list.is_empty() {
undecidable.insert(key.to_string(), list);
}
}
PreviewResponse {
alerts: visible(TargetModule::Alerts, &matches.alerts.matched, PREVIEW_CAP),
resolved_at_fire_time: visible(
TargetModule::Alerts,
&matches.alerts.at_fire_time,
PREVIEW_CAP,
),
anomalies: visible(
TargetModule::AnomalyDetections,
&matches.anomalies.matched,
PREVIEW_CAP,
),
synthetics: visible(
TargetModule::Synthetics,
&matches.synthetics.matched,
PREVIEW_CAP,
),
slos: visible(TargetModule::Slos, &matches.slos.matched, PREVIEW_CAP),
alerts_total: matches.alerts.matched.len(),
anomalies_total: matches.anomalies.matched.len(),
synthetics_total: matches.synthetics.matched.len(),
slos_total: matches.slos.matched.len(),
undecidable,
}
}
/// An `=` pair every match must hold that the item's own value fails, whatever the `Or` keys say.
fn ruled_out(cond: &DimensionCondition, dims: &HashMap<String, String>) -> bool {
match cond {
DimensionCondition::Group {
op: LogicalOp::And,
items,
} => items.iter().any(|c| ruled_out(c, dims)),
DimensionCondition::Group { .. } => false,
DimensionCondition::Pair {
key,
operator: PairOperator::Eq,
..
} => dims.contains_key(key) && !scope::eval_condition(cond, dims),
DimensionCondition::Pair { .. } => false,
}
}
fn matched(
target: &DowntimeTarget,
condition: Option<&DimensionCondition>,
item: &Item,
) -> PreviewMatch {
PreviewMatch {
matched_by: (target.module == TargetModule::Slos)
.then(|| slo_matched_by(target, condition, item).to_string()),
..item.preview()
}
}
/// Why an SLO is in the list, so the form can say it.
fn slo_matched_by(
target: &DowntimeTarget,
condition: Option<&DimensionCondition>,
item: &Item,
) -> &'static str {
if !target.ids.is_empty() {
return "id";
}
match (condition, &target.folders) {
(Some(cond), _) if scope::eval_condition(cond, &item.measured) => item.measured_by,
(Some(_), _) => "tag",
(None, TargetFolders::Some { .. }) => "folder",
(None, TargetFolders::All) => "org",
}
}
#[cfg(test)]
mod tests {
use super::*;
fn dims(pairs: &[(&str, &str)]) -> HashMap<String, String> {
pairs
.iter()
.map(|(k, v)| (k.to_string(), v.to_string()))
.collect()
}
fn item(id: &str, folder: &str, pairs: &[(&str, &str)]) -> Item {
Item {
id: id.to_string(),
name: format!("{id}-name"),
folder_id: folder.to_string(),
dims: dims(pairs),
..Default::default()
}
}
fn eq(key: &str, value: &str) -> DimensionCondition {
DimensionCondition::Pair {
key: key.to_string(),
operator: PairOperator::Eq,
value: value.to_string(),
}
}
fn target(module: TargetModule) -> DowntimeTarget {
DowntimeTarget {
module,
folders: TargetFolders::All,
tags: vec![],
ids: vec![],
slo_mode: None,
incident_mode: Default::default(),
}
}
fn ids(list: &[PreviewMatch]) -> Vec<&str> {
list.iter().map(|m| m.id.as_str()).collect()
}
#[test]
fn alerts_without_dimensions_resolve_at_fire_time() {
let items = vec![
item("a1", "default", &[("service", "payments")]),
item("a2", "default", &[]),
item("a3", "default", &[("service", "checkout")]),
];
let got = match_module(
&target(TargetModule::Alerts),
Some(&eq("service", "payments")),
&items,
);
assert_eq!(ids(&got.matched), ["a1"]);
assert_eq!(ids(&got.at_fire_time), ["a2"]);
assert!(got.undecidable.is_empty());
}
#[test]
fn undecidable_names_the_key_an_alert_cannot_answer() {
let cond = DimensionCondition::Group {
op: LogicalOp::And,
items: vec![
eq("service", "payments"),
DimensionCondition::Group {
op: LogicalOp::Or,
items: vec![eq("host", "db-1"), eq("host", "db-2")],
},
],
};
let items = vec![
item(
"a1",
"default",
&[("service", "payments"), ("host", "db-1")],
),
item("a2", "default", &[("service", "payments")]),
];
let got = match_module(&target(TargetModule::Alerts), Some(&cond), &items);
assert_eq!(ids(&got.matched), ["a1"]);
assert_eq!(ids(&got.undecidable), ["a2"]);
assert_eq!(got.undecidable[0].missing.as_deref(), Some("host"));
}
#[test]
fn an_alert_the_and_pairs_rule_out_is_not_undecidable() {
let cond = DimensionCondition::Group {
op: LogicalOp::And,
items: vec![
eq("service", "payments-api"),
DimensionCondition::Group {
op: LogicalOp::Or,
items: vec![eq("host", "node-1")],
},
],
};
let items = vec![
item("web", "default", &[("service", "web-server")]),
item("pay", "default", &[("service", "payments-api")]),
];
let got = match_module(&target(TargetModule::Alerts), Some(&cond), &items);
assert!(got.matched.is_empty());
assert_eq!(ids(&got.undecidable), ["pay"]);
}
#[test]
fn a_module_without_a_target_matches_nothing() {
let inventory = Inventory {
alerts: vec![item("a1", "default", &[("service", "payments")])],
..Default::default()
};
let got = match_all(&inventory, None, &[target(TargetModule::Slos)]);
assert!(got.alerts.matched.is_empty());
}
#[test]
fn synthetics_ignore_the_condition_and_use_tags() {
let mut check = item("c1", "syn", &[]);
check.tags = vec!["service:payments".to_string()];
let mut t = target(TargetModule::Synthetics);
t.tags = vec!["service:payments".to_string()];
let got = match_module(&t, Some(&eq("service", "checkout")), &[check]);
assert_eq!(ids(&got.matched), ["c1"]);
}
#[test]
fn slo_matches_say_why() {
let mut by_query = item("s1", "default", &[("service", "payments")]);
by_query.measured = dims(&[("service", "payments")]);
by_query.measured_by = "query";
let mut by_tag = item("s2", "default", &[("service", "payments")]);
by_tag.measured_by = "query";
let items = vec![by_query, by_tag];
let got = match_module(
&target(TargetModule::Slos),
Some(&eq("service", "payments")),
&items,
);
let why: Vec<_> = got
.matched
.iter()
.map(|m| m.matched_by.as_deref())
.collect();
assert_eq!(why, [Some("query"), Some("tag")]);
let mut by_id = target(TargetModule::Slos);
by_id.ids = vec!["s1".to_string()];
assert_eq!(
match_module(&by_id, None, &items).matched[0]
.matched_by
.as_deref(),
Some("id")
);
assert_eq!(
match_module(&target(TargetModule::Slos), None, &items).matched[0]
.matched_by
.as_deref(),
Some("org")
);
}
#[test]
fn the_preview_hides_names_the_user_cannot_list_and_keeps_the_totals() {
let inventory = Inventory {
alerts: (0..60)
.map(|i| {
let folder = if i % 2 == 0 { "mine" } else { "theirs" };
item(&format!("a{i}"), folder, &[])
})
.collect(),
..Default::default()
};
let matches = match_all(&inventory, None, &[target(TargetModule::Alerts)]);
let visibility = Visibility {
alert_folders: Some(HashSet::from(["mine".to_string()])),
synthetics_folders: None,
};
let preview = to_preview(&matches, &visibility);
assert_eq!(preview.alerts_total, 60);
assert_eq!(preview.alerts.len(), 30);
assert!(preview.alerts.iter().all(|m| m.folder_id == "mine"));
let everyone = to_preview(&matches, &Visibility::default());
assert_eq!(everyone.alerts.len(), PREVIEW_CAP);
assert_eq!(everyone.slos_total, 0);
}
}
src/core/src/downtimes/notify.rs
+577
View File
@@ -0,0 +1,577 @@
// Copyright 2026 OpenObserve Inc.
//
// This program is free software: you can redistribute it and/or modify
// it under the terms of the GNU Affero General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
//
// This program is distributed in the hope that it will be useful
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU Affero General Public License for more details.
//
// You should have received a copy of the GNU Affero General Public License
// along with this program. If not, see <http://www.gnu.org/licenses/>.
use config::{
get_config,
meta::{
destinations::{DestinationType, Module, Template, TemplateType},
downtimes::{
Downtime, DowntimeNotificationLogEntry, DowntimeWindow, NotificationEvent, TargetModule,
},
},
utils::time::now_micros,
};
use infra::table::downtime_notifications::{self as log_table, DowntimeNotification};
use o2_enterprise::enterprise::common::config::get_config as get_o2_config;
use super::{DowntimeError, MatchCounts};
use crate::alerts::templates::{DOWNTIME_EMAIL_TITLE, downtime_templates};
pub const LOG_LIMIT: u64 = 50;
/// How often the notifier ticks; a start one tick back is already behind every healthy checkpoint.
pub const TICK_INTERVAL_SECS: u64 = 30;
pub const RESULT_OK: &str = "ok";
const DOWNTIME_VARIABLE_PREFIX: &str = "{downtime_";
#[derive(Clone, Debug, PartialEq, Eq)]
pub struct DueEvent {
pub event: NotificationEvent,
pub window: DowntimeWindow,
pub key: i64,
}
impl DueEvent {
pub fn of_window(event: NotificationEvent, window: DowntimeWindow) -> Self {
Self {
event,
key: window.start,
window,
}
}
}
#[derive(Clone, Copy, Debug, PartialEq, Eq)]
enum Escape {
Json,
Html,
Plain,
}
impl Escape {
fn apply(self, value: &str) -> String {
match self {
Self::Json => {
let quoted = serde_json::Value::String(value.to_string()).to_string();
quoted[1..quoted.len() - 1].to_string()
}
Self::Html => value
.replace('&', "&amp;")
.replace('<', "&lt;")
.replace('>', "&gt;")
.replace('"', "&quot;")
.replace('\'', "&#39;"),
Self::Plain => value.to_string(),
}
}
}
pub fn this_region() -> String {
let o2 = get_o2_config();
if o2.super_cluster.enabled {
o2.super_cluster.region.clone()
} else {
String::new()
}
}
/// Only the origin region sends, so replicated rows do not notify once per region.
pub fn sends_here(row: &Downtime, region: &str) -> bool {
row.origin_region
.as_deref()
.is_none_or(|origin| origin.is_empty() || origin == region)
}
pub fn wants(row: &Downtime, event: NotificationEvent) -> bool {
row.notifications
.as_ref()
.is_some_and(|n| !n.destinations.is_empty() && n.events.wants(event))
}
pub async fn deliver(row: &Downtime, due: &DueEvent) -> Result<bool, DowntimeError> {
let Some(record) = record_for(row, due, now_micros()) else {
return Ok(false);
};
if !log_table::insert_if_absent(&record).await? {
return Ok(false);
}
let destinations = destinations_of(row);
let result = match send(row, due, &destinations).await {
Ok(()) => RESULT_OK.to_string(),
Err(e) => {
log::warn!(
"[DOWNTIMES] {} notification of {}/{} failed: {e}",
due.event.as_str(),
row.org,
row.id
);
e
}
};
log_table::set_result(&record.id, &result).await?;
Ok(true)
}
pub fn deliver_in_background(row: Downtime, due: DueEvent) {
if !wants(&row, due.event) {
return;
}
tokio::spawn(async move {
if let Err(e) = deliver(&row, &due).await {
log::warn!(
"[DOWNTIMES] could not record the {} notification of {}/{}: {e}",
due.event.as_str(),
row.org,
row.id
);
}
});
}
pub async fn log_for(
org: &str,
downtime_id: &str,
) -> Result<Vec<DowntimeNotificationLogEntry>, DowntimeError> {
Ok(log_table::list_for_downtime(org, downtime_id, LOG_LIMIT)
.await?
.into_iter()
.map(log_entry)
.collect())
}
fn record_for(row: &Downtime, due: &DueEvent, now: i64) -> Option<DowntimeNotification> {
if !wants(row, due.event) {
return None;
}
Some(DowntimeNotification {
id: infra::table::downtimes::new_id(),
org: row.org.clone(),
downtime_id: row.id.clone(),
window_start: due.key,
event: due.event.as_str().to_string(),
sent_at: now,
destinations: serde_json::json!(destinations_of(row)),
result: None,
})
}
fn destinations_of(row: &Downtime) -> Vec<String> {
row.notifications
.as_ref()
.map(|n| n.destinations.clone())
.unwrap_or_default()
}
fn log_entry(row: DowntimeNotification) -> DowntimeNotificationLogEntry {
DowntimeNotificationLogEntry {
window_start: row.window_start,
event: row.event,
sent_at: row.sent_at,
destinations: serde_json::from_value(row.destinations).unwrap_or_default(),
result: row.result,
}
}
async fn send(row: &Downtime, due: &DueEvent, destinations: &[String]) -> Result<(), String> {
let counts = super::match_counts(row).await.ok();
let vars = variables(
row,
due.event,
&due.window,
counts.as_ref(),
&get_config().common.web_url,
);
let mut errors = Vec::new();
for name in destinations {
if let Err(e) = send_one(&row.org, name, &vars).await {
errors.push(format!("{name}: {e}"));
}
}
if errors.is_empty() {
Ok(())
} else {
Err(errors.join("; "))
}
}
async fn send_one(org: &str, name: &str, vars: &[(&str, String)]) -> Result<(), String> {
let dest = crate::alerts::destinations::get(org, name)
.await
.map_err(|e| e.to_string())?;
let Module::Alert {
template,
destination_type,
} = dest.module
else {
return Err("not an alert destination".to_string());
};
let own = match template {
Some(template) => db::alerts::templates::get(org, &template).await.ok(),
None => None,
};
let template = template_for(own, &destination_type);
let escape = if matches!(destination_type, DestinationType::Email(_)) {
Escape::Html
} else {
Escape::Json
};
let body = render(&template.body, vars, escape);
let subject = match &template.template_type {
TemplateType::Email { title } if !title.is_empty() => render(title, vars, Escape::Plain),
_ => render(DOWNTIME_EMAIL_TITLE, vars, Escape::Plain),
};
crate::alerts::alert::dispatch_notification(&destination_type, &subject, body)
.await
.map(|_| ())
.map_err(|e| e.to_string())
}
fn template_for(own: Option<Template>, destination_type: &DestinationType) -> Template {
if let Some(own) = own.filter(|t| t.body.contains(DOWNTIME_VARIABLE_PREFIX)) {
return own;
}
let [http, email] = downtime_templates();
if matches!(destination_type, DestinationType::Email(_)) {
email
} else {
http
}
}
fn variables(
row: &Downtime,
event: NotificationEvent,
window: &DowntimeWindow,
counts: Option<&MatchCounts>,
web_url: &str,
) -> Vec<(&'static str, String)> {
vec![
("downtime_name", row.name.clone()),
("downtime_event", event_phrase(event).to_string()),
("downtime_reason", row.reason.clone().unwrap_or_default()),
("downtime_starts_at", utc(window.start)),
("downtime_ends_at", utc(window.end)),
("downtime_targets", targets_phrase(row)),
("downtime_counts", counts_phrase(row, counts)),
(
"downtime_url",
format!("{web_url}/web/downtimes?org_identifier={}", row.org),
),
("org_name", row.org.clone()),
]
}
/// One pass, so a value that itself reads like `{downtime_url}` is not substituted again.
fn render(body: &str, vars: &[(&str, String)], escape: Escape) -> String {
let mut out = String::with_capacity(body.len());
let mut rest = body;
while let Some(open) = rest.find('{') {
out.push_str(&rest[..open]);
let tail = &rest[open..];
let hit = tail.find('}').and_then(|close| {
let name = &tail[1..close];
vars.iter()
.find(|(n, _)| *n == name)
.map(|(_, value)| (close, value))
});
match hit {
Some((close, value)) => {
out.push_str(&escape.apply(value));
rest = &tail[close + 1..];
}
None => {
out.push('{');
rest = &tail[1..];
}
}
}
out.push_str(rest);
out
}
fn event_phrase(event: NotificationEvent) -> &'static str {
match event {
NotificationEvent::Started => "started",
NotificationEvent::EndingSoon => "ends soon",
NotificationEvent::Ended => "ended",
NotificationEvent::Cancelled => "was cancelled",
NotificationEvent::Extended => "was extended",
}
}
fn utc(micros: i64) -> String {
chrono::DateTime::from_timestamp_micros(micros)
.map(|at| at.format("%Y-%m-%d %H:%M UTC").to_string())
.unwrap_or_default()
}
fn targets_phrase(row: &Downtime) -> String {
row.targets
.iter()
.map(|t| module_words(t.module).1)
.collect::<Vec<_>>()
.join(", ")
}
fn counts_phrase(row: &Downtime, counts: Option<&MatchCounts>) -> String {
let Some(counts) = counts else {
return String::new();
};
row.targets
.iter()
.map(|t| {
let n = counts.of(t.module);
let (one, many) = module_words(t.module);
format!("{n} {}", if n == 1 { one } else { many })
})
.collect::<Vec<_>>()
.join(", ")
}
fn module_words(module: TargetModule) -> (&'static str, &'static str) {
match module {
TargetModule::Alerts => ("alert", "alerts"),
TargetModule::AnomalyDetections => ("anomaly detection", "anomaly detections"),
TargetModule::Synthetics => ("synthetics check", "synthetics checks"),
TargetModule::Slos => ("SLO", "SLOs"),
}
}
#[cfg(test)]
mod tests {
use config::meta::{
destinations::{Email, Endpoint},
downtimes::{
DowntimeNotifications, DowntimeSchedule, DowntimeTarget, NotificationEvents, Repeat,
TargetFolders,
},
};
use infra::table::entity::downtime_notifications::Entity as LogEntity;
use sea_orm::{ConnectionTrait, Database, DatabaseConnection, Schema};
use super::*;
const HOUR: i64 = 3_600_000_000;
fn row(origin: Option<&str>) -> Downtime {
Downtime {
id: "2f9KQe4b7Nq1vH3sT0mLzXpRcWd".to_string(),
org: "acme".to_string(),
folder_id: "default".to_string(),
name: "Nightly \"deploy\" <db>".to_string(),
reason: Some("CHG-1".to_string()),
condition: None,
targets: [TargetModule::Alerts, TargetModule::Synthetics]
.into_iter()
.map(|module| DowntimeTarget {
module,
folders: TargetFolders::All,
tags: vec![],
ids: vec![],
slo_mode: None,
incident_mode: Default::default(),
})
.collect(),
schedule: DowntimeSchedule {
repeat: Repeat::None,
starts_at: 10 * HOUR,
ends_at: Some(12 * HOUR),
timezone: "UTC".to_string(),
start_time_local: None,
duration_secs: 7200,
weekdays: vec![],
},
cancelled_at: None,
cancelled_by: None,
show_banner: true,
notifications: Some(DowntimeNotifications {
destinations: vec!["slack".to_string(), "pagerduty".to_string()],
events: NotificationEvents {
started: true,
ended: true,
..Default::default()
},
ending_soon_lead_secs: 600,
continues: None,
}),
origin_region: origin.map(str::to_string),
version: 0,
created_by: "lin".to_string(),
created_at: 1,
updated_by: "lin".to_string(),
updated_at: 1,
}
}
fn window() -> DowntimeWindow {
DowntimeWindow {
start: 10 * HOUR,
end: 12 * HOUR,
}
}
async fn log_db() -> DatabaseConnection {
let db = Database::connect("sqlite::memory:").await.unwrap();
let backend = db.get_database_backend();
let stmt = Schema::new(backend).create_table_from_entity(LogEntity);
db.execute(backend.build(&stmt)).await.unwrap();
db.execute_unprepared(
"CREATE UNIQUE INDEX downtime_notifications_window_event_idx \
ON downtime_notifications (downtime_id, window_start, event)",
)
.await
.unwrap();
db
}
#[test]
fn a_row_without_an_origin_sends_from_every_region_and_one_with_it_only_from_there() {
for region in ["", "us-east", "eu-west"] {
assert!(sends_here(&row(None), region));
assert!(sends_here(&row(Some("")), region));
}
assert!(sends_here(&row(Some("us-east")), "us-east"));
assert!(!sends_here(&row(Some("us-east")), "eu-west"));
assert!(!sends_here(&row(Some("us-east")), ""));
}
#[test]
fn only_a_wanted_event_with_destinations_gets_a_log_row() {
let d = row(None);
let due = DueEvent::of_window(NotificationEvent::Started, window());
let record = record_for(&d, &due, 7).unwrap();
assert_eq!(record.window_start, 10 * HOUR);
assert_eq!(record.event, "started");
assert_eq!(record.sent_at, 7);
assert_eq!(
record.destinations,
serde_json::json!(["slack", "pagerduty"])
);
assert_eq!(record.result, None);
let reminder = DueEvent::of_window(NotificationEvent::EndingSoon, window());
assert!(record_for(&d, &reminder, 7).is_none());
let mut silent = row(None);
silent.notifications.as_mut().unwrap().destinations.clear();
assert!(record_for(&silent, &due, 7).is_none());
silent.notifications = None;
assert!(record_for(&silent, &due, 7).is_none());
}
#[tokio::test]
async fn a_second_node_finds_the_claim_and_sends_nothing() {
let db = log_db().await;
let d = row(None);
let due = DueEvent::of_window(NotificationEvent::Started, window());
let first = record_for(&d, &due, 1).unwrap();
let second = record_for(&d, &due, 2).unwrap();
assert_ne!(first.id, second.id);
assert!(log_table::insert_if_absent_with(&db, &first).await.unwrap());
assert!(
!log_table::insert_if_absent_with(&db, &second)
.await
.unwrap()
);
let ended = DueEvent::of_window(NotificationEvent::Ended, window());
let ended = record_for(&d, &ended, 3).unwrap();
assert!(log_table::insert_if_absent_with(&db, &ended).await.unwrap());
let rows = log_table::list_for_downtime_with(&db, "acme", &d.id, LOG_LIMIT)
.await
.unwrap();
let entries: Vec<_> = rows.into_iter().map(log_entry).collect();
assert_eq!(entries.len(), 2);
assert_eq!(entries[0].event, "ended");
assert_eq!(entries[0].destinations, ["slack", "pagerduty"]);
}
#[test]
fn variables_fill_both_prebuilt_templates_escaped_for_their_format() {
let counts = MatchCounts {
alerts: 2,
synthetics: 1,
..Default::default()
};
let vars = variables(
&row(None),
NotificationEvent::EndingSoon,
&window(),
Some(&counts),
"https://o2.example.com",
);
let [http, email] = downtime_templates();
let body = render(&http.body, &vars, Escape::Json);
let json: serde_json::Value = serde_json::from_str(&body).unwrap();
assert_eq!(json["downtime"]["name"], "Nightly \"deploy\" <db>");
assert_eq!(json["downtime"]["event"], "ends soon");
assert_eq!(json["downtime"]["starts_at"], "1970-01-01 10:00 UTC");
assert_eq!(json["downtime"]["ends_at"], "1970-01-01 12:00 UTC");
assert_eq!(json["downtime"]["targets"], "alerts, synthetics checks");
assert_eq!(json["downtime"]["counts"], "2 alerts, 1 synthetics check");
assert_eq!(
json["downtime"]["url"],
"https://o2.example.com/web/downtimes?org_identifier=acme"
);
assert_eq!(json["downtime"]["org"], "acme");
assert_eq!(json["downtime"]["reason"], "CHG-1");
let html = render(&email.body, &vars, Escape::Html);
assert!(html.contains("Nightly &quot;deploy&quot; &lt;db&gt;"));
assert!(!html.contains("{downtime_"));
assert!(!html.contains("{org_name}"));
}
#[test]
fn render_substitutes_once_and_leaves_unknown_braces_alone() {
let vars = vec![
("downtime_name", "{downtime_url}".to_string()),
("downtime_url", "https://x".to_string()),
];
assert_eq!(
render(
"{downtime_name} {alert_name} {downtime_url} {",
&vars,
Escape::Plain
),
"{downtime_url} {alert_name} https://x {"
);
assert_eq!(Escape::Json.apply("a\"b\nc"), "a\\\"b\\nc");
}
#[test]
fn a_destination_template_is_used_only_when_written_for_downtimes() {
let http = DestinationType::Http(Endpoint::default());
let email = DestinationType::Email(Email::default());
let own = |body: &str| Template {
name: "mine".to_string(),
body: body.to_string(),
..Default::default()
};
assert_eq!(
template_for(Some(own("{downtime_name} is {downtime_event}")), &http).name,
"mine"
);
assert_eq!(
template_for(Some(own("[{alert_status}] {alert_name}")), &http).name,
"prebuilt_downtime"
);
assert_eq!(template_for(None, &email).name, "prebuilt_downtime_email");
}
#[test]
fn counts_are_empty_when_unknown() {
assert_eq!(counts_phrase(&row(None), None), "");
}
}
src/core/src/downtimes/resources.rs
+268
View File
@@ -0,0 +1,268 @@
// Copyright 2026 OpenObserve Inc.
//
// This program is free software: you can redistribute it and/or modify
// it under the terms of the GNU Affero General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
//
// This program is distributed in the hope that it will be useful
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU Affero General Public License for more details.
//
// You should have received a copy of the GNU Affero General Public License
// along with this program. If not, see <http://www.gnu.org/licenses/>.
//! The values of a finer dimension that co-occur with a condition (D16, section 7.1).
use std::time::Duration;
use config::meta::downtimes::{ResourceValue, ResourcesRequest, ResourcesResponse};
use o2_enterprise::enterprise::{downtimes::scope::and_eq_pairs, oncall::routing::normalize_value};
use super::{
DowntimeError,
values::{self, Found, SearchPlan},
};
pub const MAX_RESOURCES: usize = 500;
const SEARCH_WINDOW_MICROS: i64 = 24 * 3_600 * 1_000_000;
const SEARCH_DEADLINE: Duration = Duration::from_secs(5);
pub async fn resources(
org: &str,
user_id: &str,
req: &ResourcesRequest,
) -> Result<ResourcesResponse, DowntimeError> {
let groups = db::system_settings::get_semantic_field_groups(org).await;
values::ensure_dimension(&groups, &req.refine_by)?;
let pairs = lookup_pairs(req)?;
let records = values::registry_records(org, &pairs).await?;
if values::is_priority(org, &req.refine_by).await? {
let readable = values::searchable_records(org, user_id, &records).await;
let found = values::registry_values(&readable, &req.refine_by, &pairs);
return Ok(respond(&req.refine_by, found, "registry"));
}
let plan = SearchPlan {
key: &req.refine_by,
pairs: &pairs,
prefix: "",
window_micros: SEARCH_WINDOW_MICROS,
deadline: SEARCH_DEADLINE,
rows_per_stream: MAX_RESOURCES,
};
let streams = values::stream_columns(org, &groups, &plan).await;
let queries = values::stream_queries(&records, &groups, &streams, &plan);
let (found, timed_out) = values::search_values(org, user_id, queries, &plan).await;
if timed_out {
log::warn!(
"[DOWNTIMES] resources: search for {} in {org} passed its deadline; answering with what returned",
req.refine_by
);
}
Ok(respond(&req.refine_by, found, "search"))
}
/// The `=` pairs of the And spine, values normalized as stored; none is a 400.
pub fn lookup_pairs(req: &ResourcesRequest) -> Result<Vec<(String, String)>, DowntimeError> {
let pairs: Vec<(String, String)> = and_eq_pairs(&req.condition)
.into_iter()
.map(|(k, v)| (k, normalize_value(&v)))
.collect();
if pairs.is_empty() {
return Err(DowntimeError::BadRequest(
"Add at least one = condition before refining by resource.".to_string(),
));
}
Ok(pairs)
}
/// One row per value, newest first, capped at `MAX_RESOURCES` with the full count kept.
pub fn respond(dimension: &str, found: Vec<Found>, source: &str) -> ResourcesResponse {
let mut values: Vec<ResourceValue> = values::merge(found)
.into_iter()
.map(|f| ResourceValue {
value: f.value,
last_seen: f.last_seen.unwrap_or(0),
streams: f.streams,
})
.collect();
values.sort_by(|a, b| b.last_seen.cmp(&a.last_seen).then(a.value.cmp(&b.value)));
let total = values.len();
values.truncate(MAX_RESOURCES);
ResourcesResponse {
dimension: dimension.to_string(),
values,
total,
source: source.to_string(),
}
}
#[cfg(test)]
mod tests {
use std::collections::{BTreeMap, BTreeSet, HashMap, HashSet};
use config::meta::{
downtimes::{DimensionCondition, LogicalOp, PairOperator},
service_streams::StreamInfo,
stream::StreamType,
};
use o2_enterprise::enterprise::service_streams::meta::{ServiceMetadata, ServiceStreams};
use super::{values::Source, *};
fn eq(key: &str, value: &str) -> DimensionCondition {
DimensionCondition::Pair {
key: key.to_string(),
operator: PairOperator::Eq,
value: value.to_string(),
}
}
fn record(host: &str, last_seen: i64, stream: &str) -> ServiceMetadata {
ServiceMetadata {
service_name: "payments".to_string(),
streams: ServiceStreams {
logs: HashSet::from([StreamInfo {
stream_name: stream.to_string(),
stream_type: StreamType::Logs,
filters: HashMap::new(),
dropped_dimensions: vec![],
}]),
..Default::default()
},
last_seen,
disambiguation: HashMap::new(),
set_id: "default".to_string(),
all_dimensions: HashMap::from([
("service".to_string(), "payments".to_string()),
("host".to_string(), host.to_string()),
]),
field_name_mapping: HashMap::from([
("service".to_string(), "service_name".to_string()),
("host".to_string(), "hostname".to_string()),
]),
}
}
fn plan<'a>(pairs: &'a [(String, String)], key: &'a str) -> SearchPlan<'a> {
SearchPlan {
key,
pairs,
prefix: "",
window_micros: SEARCH_WINDOW_MICROS,
deadline: SEARCH_DEADLINE,
rows_per_stream: MAX_RESOURCES,
}
}
#[test]
fn a_tree_reduces_to_its_and_eq_pairs_and_one_without_any_is_refused() {
let req = ResourcesRequest {
condition: DimensionCondition::Group {
op: LogicalOp::And,
items: vec![
eq("service", "Payments"),
DimensionCondition::Group {
op: LogicalOp::Or,
items: vec![eq("host", "db-1")],
},
],
},
refine_by: "host".to_string(),
};
assert_eq!(
lookup_pairs(&req).unwrap(),
vec![("service".to_string(), "payments".to_string())]
);
let none = ResourcesRequest {
condition: DimensionCondition::Pair {
key: "env".to_string(),
operator: PairOperator::Ne,
value: "prod".to_string(),
},
refine_by: "host".to_string(),
};
assert!(matches!(
lookup_pairs(&none),
Err(DowntimeError::BadRequest(_))
));
}
#[tokio::test]
async fn a_registry_stream_the_user_cannot_search_is_absent_from_the_resources() {
let records = vec![
record("node-1", 1, "app_logs"),
record("node-2", 2, "secret_logs"),
];
let readable = values::readable_records(&records, |_, stream: String| async move {
stream != "secret_logs"
})
.await;
let resp = respond(
"host",
values::registry_values(&readable, "host", &[]),
"registry",
);
let hosts: Vec<&str> = resp.values.iter().map(|v| v.value.as_str()).collect();
assert_eq!(hosts, ["node-1"]);
}
#[test]
fn registry_values_are_distinct_and_newest_wins() {
let records = vec![
record("db-1", 10, "payments"),
record("db-1", 30, "payments_audit"),
record("db-2", 20, "payments"),
];
let found = values::registry_values(&records, "host", &[]);
let resp = respond("host", found, "registry");
assert_eq!(resp.total, 2);
assert_eq!(resp.values[0].value, "db-1");
assert_eq!(resp.values[0].last_seen, 30);
assert_eq!(
resp.values[0].streams.iter().collect::<BTreeSet<_>>(),
BTreeSet::from([
&"logs/payments".to_string(),
&"logs/payments_audit".to_string()
])
);
assert_eq!(resp.source, "registry");
}
#[test]
fn stream_queries_map_aliases_to_raw_columns_per_stream() {
let pairs = vec![("service".to_string(), "pay*".to_string())];
let records = [record("db-1", 1, "payments")];
let queries =
values::stream_queries(&records, &[], &BTreeMap::new(), &plan(&pairs, "host"));
assert_eq!(queries.len(), 1);
assert_eq!(queries[0].stream, "payments");
assert_eq!(
queries[0].sql,
"SELECT \"hostname\" AS value, max(_timestamp) AS last_seen FROM \"payments\" WHERE \"service_name\" LIKE 'pay%' GROUP BY \"hostname\" LIMIT 500"
);
assert!(
values::stream_queries(&records, &[], &BTreeMap::new(), &plan(&pairs, "k8s-pod"))
.is_empty(),
"a stream without a column for refine_by is skipped"
);
}
#[test]
fn the_answer_is_capped_at_500_and_says_how_many_there_were() {
let found = (0..600)
.map(|i| Found {
value: format!("h{i}"),
source: Source::Search,
items: 0,
last_seen: Some(i),
streams: vec![],
})
.collect();
let resp = respond("host", found, "search");
assert_eq!(resp.total, 600);
assert_eq!(resp.values.len(), MAX_RESOURCES);
assert_eq!(resp.values[0].value, "h599");
}
}
src/core/src/downtimes/values.rs
+1253
View File
File diff suppressed because it is too large Load Diff
src/core/src/ingestion/mod.rs
+312 -39
View File
@@ -17,8 +17,8 @@ use std::{
collections::{HashMap, HashSet},
io::Write,
sync::{
Arc,
atomic::{AtomicUsize, Ordering},
Arc, LazyLock,
atomic::{AtomicU64, AtomicUsize, Ordering},
},
};
@@ -29,7 +29,7 @@ use config::{
cluster::{LOCAL_NODE, LOCAL_NODE_ID},
ider::SnowflakeIdGenerator,
meta::{
alerts::alert::Alert,
alerts::{TriggerCondition, alert::Alert},
promql::HASH_LABEL,
self_reporting::usage::{RequestStats, RunOutcome, TriggerData, TriggerDataType},
stream::{PartitionTimeLevel, StreamParams, StreamPartition, StreamType},
@@ -42,6 +42,7 @@ use config::{
time::{DAY_MICRO_SECS, HOUR_MICRO_SECS},
},
};
use dashmap::{DashMap, mapref::entry::Entry};
use db::{
self,
alerts::{alert::scheduler_key, realtime_triggers::REALTIME_ALERT_TRIGGERS},
@@ -79,6 +80,12 @@ pub type TriggerAlertData = Vec<(Alert, Vec<Map<String, Value>>)>;
/// This ensures even distribution of ingestion load across multiple buckets in axum.
static REQUEST_COUNTER: AtomicUsize = AtomicUsize::new(0);
/// `org/alert -> until`, taken before the downtime check so concurrent requests run it once.
static REALTIME_FLOOR_CLAIMS: LazyLock<DashMap<String, (i64, u64)>> = LazyLock::new(DashMap::new);
/// Tells claims of one key apart, so a request releases only its own.
static REALTIME_FLOOR_TOKENS: AtomicU64 = AtomicU64::new(0);
/// Memoizes the write partition per time bucket while no partition key is enabled.
pub struct PartitionMemo<'a> {
partition_keys: &'a Vec<StreamPartition>,
@@ -185,10 +192,7 @@ pub async fn get_stream_alerts(
.filter(|alert| alert.enabled && alert.is_real_time)
.filter(|alert| {
let key = format!("{}/{}", stream.org_id, alert.id.as_ref().unwrap());
match triggers_cache.get(&key) {
Some(v) => !v.is_silenced,
None => true,
}
realtime_alert_is_live(triggers_cache.get(&key))
})
.collect::<Vec<_>>();
if alerts.is_empty() {
@@ -236,40 +240,60 @@ pub async fn evaluate_trigger(triggers: TriggerAlertData) {
alert.org_id,
alert.name
);
let outcome = alert
.send_notification(&trace_id, val, now, None, now, None, None, None, &[], None)
.await;
record_realtime_delivery(&mut trigger_data_stream, &outcome);
match outcome {
Err(e) => {
// Only an org with an alert downtime pays for the identity work the claim guards.
let floor_key = format!("{}/{module_key}", alert.org_id);
let floor_claim = if crate::alerts::downtimes::any_for(
&alert.org_id,
config::meta::downtimes::TargetModule::Alerts,
) {
let Some(token) =
claim_realtime_floor(&floor_key, now, TriggerCondition::min_silence_micros())
else {
log::debug!(
"Realtime alert {}/{} already evaluated inside its floor on this node",
alert.org_id,
alert.name
);
continue;
};
Some(token)
} else {
None
};
let delivered = if let Some(downtime) = realtime_downtime(alert, val, now).await {
log::info!(
"Realtime alert {}/{} suppressed by downtime {}",
alert.org_id,
alert.name,
downtime.id
);
crate::alerts::alert::record_suppressed_run(
&mut trigger_data_stream,
"alerts",
downtime.id,
);
false
} else {
let outcome = alert
.send_notification(&trace_id, val, now, None, now, None, None, None, &[], None)
.await;
record_realtime_delivery(&mut trigger_data_stream, &outcome);
if let Err(e) = &outcome {
log::error!("Failed to send notification: {e}");
}
Ok(_) => {
// enforce a minimum silence floor so a high-volume stream cannot
// fire (and write to the db) once per matching request
let silence_micros = alert.trigger_condition.effective_silence_micros();
if silence_micros > 0 {
log::debug!(
"Realtime alert {}/{}/{}/{} triggered successfully, hence applying silence period",
alert.org_id,
alert.stream_type,
alert.stream_name,
alert.name
);
let next_run_at = Utc::now().timestamp_micros() + silence_micros;
// After the notification is sent successfully, we need to update
// the silence period of the trigger
if let Err(e) = db::scheduler::update_trigger(
silenced_realtime_trigger(&alert.org_id, module_key, next_run_at, now),
false,
"",
)
.await
{
log::error!("Failed to update trigger: {e}");
}
trigger_data_stream.next_run_at = next_run_at;
outcome.is_ok()
};
#[cfg(feature = "enterprise")]
record_realtime_mute(alert, &trigger_data_stream).await;
match realtime_silence_micros(alert, &trigger_data_stream, delivered) {
Some(silence_micros) => {
trigger_data_stream.next_run_at =
apply_realtime_silence(alert, module_key, silence_micros, now).await;
}
// A failed send keeps no floor, so the next matching request retries it.
None => {
if let Some(token) = floor_claim {
release_realtime_floor(&floor_key, token);
}
}
}
@@ -898,6 +922,126 @@ pub fn refactor_map(
new_map
}
/// The downtime that silences a real-time firing, only when every ingested row is muted.
#[cfg(feature = "enterprise")]
async fn realtime_downtime(
alert: &Alert,
rows: &[Map<String, Value>],
now: i64,
) -> Option<config::meta::downtimes::ActiveDowntime> {
if !crate::alerts::downtimes::any_for(
&alert.org_id,
config::meta::downtimes::TargetModule::Alerts,
) {
return None;
}
let alert_id = alert.id.as_ref()?.to_string();
let (folder, _) = alert::get_alert_from_cache(&alert.org_id, &alert_id).await?;
let identity = crate::alerts::scheduler::handlers::downtime_identities(alert, rows).await;
crate::alerts::scheduler::handlers::muted_in_every_group(&identity, |dims| {
crate::alerts::downtimes::active_for_alert(
&alert.org_id,
&alert_id,
&folder.folder_id,
dims,
now,
)
})
}
#[cfg(not(feature = "enterprise"))]
async fn realtime_downtime(
_alert: &Alert,
_rows: &[Map<String, Value>],
_now: i64,
) -> Option<config::meta::downtimes::ActiveDowntime> {
None
}
/// The Muted chip of a real-time run, set by a suppressed run and cleared by a send.
#[cfg(feature = "enterprise")]
async fn record_realtime_mute(alert: &Alert, run: &TriggerData) {
if let Some(alert_id) = alert.id.as_ref() {
crate::alerts::scheduler::handlers::record_last_downtime(
&alert.org_id,
&alert_id.to_string(),
realtime_last_downtime(run),
)
.await;
}
}
/// A suppressed run's downtime; any other real-time run reached a send and clears the chip.
#[cfg(feature = "enterprise")]
fn realtime_last_downtime(run: &TriggerData) -> Option<&str> {
match run.status {
RunOutcome::Suppressed => run.downtime_id.as_deref(),
_ => None,
}
}
/// The token of the one caller that may evaluate `key` until `now + floor_micros` on this node.
fn claim_realtime_floor(key: &str, now: i64, floor_micros: i64) -> Option<u64> {
let token = REALTIME_FLOOR_TOKENS.fetch_add(1, Ordering::Relaxed);
match REALTIME_FLOOR_CLAIMS.entry(key.to_string()) {
Entry::Occupied(held) if held.get().0 > now => None,
Entry::Occupied(mut expired) => {
expired.insert((now + floor_micros, token));
Some(token)
}
Entry::Vacant(free) => {
free.insert((now + floor_micros, token));
Some(token)
}
}
}
/// Drops the claim only while `token` still holds it, never a later request's.
fn release_realtime_floor(key: &str, token: u64) {
REALTIME_FLOOR_CLAIMS.remove_if(key, |_, (_, held)| *held == token);
}
/// A realtime alert is evaluated unless its trigger is inside a silence.
fn realtime_alert_is_live(trigger: Option<&db::scheduler::Trigger>) -> bool {
trigger.is_none_or(|t| !t.is_silenced)
}
/// A suppressed run keeps the rate-limit floor only, so the first firing after the window pages.
fn realtime_silence_micros(alert: &Alert, run: &TriggerData, delivered: bool) -> Option<i64> {
match run.status {
RunOutcome::Suppressed => Some(TriggerCondition::min_silence_micros()),
_ if delivered => Some(alert.trigger_condition.effective_silence_micros()),
_ => None,
}
}
/// Silences the trigger for `silence_micros` and returns the wakeup time it wrote.
async fn apply_realtime_silence(
alert: &Alert,
module_key: String,
silence_micros: i64,
fired_at: i64,
) -> i64 {
log::debug!(
"Realtime alert {}/{}/{}/{} silenced for {silence_micros}us",
alert.org_id,
alert.stream_type,
alert.stream_name,
alert.name
);
let next_run_at = Utc::now().timestamp_micros() + silence_micros;
if let Err(e) = db::scheduler::update_trigger(
silenced_realtime_trigger(&alert.org_id, module_key, next_run_at, fired_at),
false,
"",
)
.await
{
log::error!("Failed to update trigger: {e}");
}
next_run_at
}
fn silenced_realtime_trigger(
org: &str,
module_key: String,
@@ -1012,6 +1156,65 @@ mod tests {
assert_eq!(data.last_satisfied_at, Some(1_000));
}
#[tokio::test(flavor = "multi_thread", worker_threads = 4)]
async fn test_overlapping_requests_inside_the_floor_record_one_suppressed_row() {
let key = "org1/overlapping-floor";
let floor = TriggerCondition::min_silence_micros();
// The coordinator never answers here, so only the claim stands between the requests.
let runs = (0..16).map(|_| {
tokio::spawn(async move {
tokio::task::yield_now().await;
claim_realtime_floor(key, 1_000_000, floor).is_some()
})
});
let admitted = futures::future::join_all(runs)
.await
.into_iter()
.filter(|claimed| *claimed.as_ref().unwrap())
.count();
assert_eq!(admitted, 1, "one Suppressed row per floor period");
assert!(claim_realtime_floor(key, 1_000_000 + floor - 1, floor).is_none());
assert!(claim_realtime_floor(key, 1_000_000 + floor, floor).is_some());
}
#[test]
fn test_a_failed_send_releases_the_floor() {
let key = "org1/released-floor";
let floor = TriggerCondition::min_silence_micros();
let token = claim_realtime_floor(key, 0, floor).unwrap();
release_realtime_floor(key, token);
assert!(claim_realtime_floor(key, 1, floor).is_some());
}
#[test]
fn test_a_slow_failed_request_cannot_release_a_newer_claim() {
let key = "org1/slow-failed-floor";
let floor = TriggerCondition::min_silence_micros();
let a = claim_realtime_floor(key, 0, floor).unwrap();
// A outlives its floor, so B takes the expired claim before A fails.
let b = claim_realtime_floor(key, floor, floor).unwrap();
release_realtime_floor(key, a);
assert!(claim_realtime_floor(key, floor + 1, floor).is_none());
release_realtime_floor(key, b);
assert!(claim_realtime_floor(key, floor + 2, floor).is_some());
}
#[test]
fn test_only_a_send_or_a_suppression_silences_a_realtime_alert() {
let mut alert = Alert::default();
alert.trigger_condition.silence = 60;
let failed = TriggerData {
status: RunOutcome::NotifyFailed,
..Default::default()
};
assert_eq!(realtime_silence_micros(&alert, &failed, false), None);
assert_eq!(
realtime_silence_micros(&alert, &failed, true),
Some(3_600_000_000)
);
assert!(realtime_alert_is_live(None));
}
#[test]
fn test_format_partition_key() {
assert_eq!(format_partition_key("default/olympics"), "defaultolympics");
@@ -1656,6 +1859,76 @@ mod tests {
);
}
/// The identity `realtime_downtime` decides on: one map per ingested row.
#[cfg(feature = "enterprise")]
#[test]
fn a_realtime_batch_is_muted_only_when_every_row_is() {
use crate::alerts::scheduler::handlers::{identities_by_key, muted_in_every_group};
let groups = [config::meta::correlation::FieldAlias {
id: "service".to_string(),
display: "Service".to_string(),
group: None,
fields: vec!["service".to_string()],
is_workload_type: false,
}];
let row = |service: &str| {
serde_json::json!({ "service": service })
.as_object()
.cloned()
.unwrap()
};
let mut alert = Alert::default();
alert.is_real_time = true;
let decide = |rows: &[Map<String, Value>], muted: &[&str]| {
let identity: Vec<_> = identities_by_key(&alert, rows, &groups)
.into_iter()
.map(|(_, dims)| dims)
.collect();
muted_in_every_group(&identity, |dims| {
let service = dims.get("service")?;
muted
.contains(&service.as_str())
.then(|| config::meta::downtimes::ActiveDowntime {
id: format!("dt-{service}"),
name: service.clone(),
ends_at: 1,
incident_mode: Default::default(),
})
})
.map(|d| d.id)
};
let (payments, checkout) = (row("payments"), row("checkout"));
let mixed = [payments.clone(), checkout.clone()];
let reversed = [checkout, payments];
assert_eq!(decide(&mixed, &["payments"]), None);
assert_eq!(decide(&reversed, &["payments"]), None);
assert_eq!(
decide(&mixed, &["payments", "checkout"]).as_deref(),
Some("dt-payments")
);
}
#[cfg(feature = "enterprise")]
#[test]
fn a_suppressed_realtime_run_sets_the_muted_chip_and_the_next_send_clears_it() {
let mut muted = realtime_row();
crate::alerts::alert::record_suppressed_run(&mut muted, "alerts", "dt-1".to_string());
assert_eq!(realtime_last_downtime(&muted), Some("dt-1"));
assert_eq!(muted.delivery_attempted, Some(false));
let mut delivered = realtime_row();
record_realtime_delivery(&mut delivered, &Ok(NotificationOutcome::default()));
assert_eq!(realtime_last_downtime(&delivered), None);
let mut failed = realtime_row();
let error = Err(AlertError::SendNotificationError {
error_message: "http 500".to_string(),
});
record_realtime_delivery(&mut failed, &error);
assert_eq!(realtime_last_downtime(&failed), None);
}
fn realtime_row() -> TriggerData {
TriggerData {
status: RunOutcome::Firing,
src/core/src/lib.rs
+2
View File
@@ -28,6 +28,8 @@ pub mod bootstrap;
pub mod cache;
pub mod dashboards;
pub mod db_monitoring;
#[cfg(feature = "enterprise")]
pub mod downtimes;
use ::common;
use ::db;
pub mod error_suggest;
src/core/src/ofga/mod.rs
+95 -72
View File
@@ -75,6 +75,7 @@ struct PendingMigrations {
prompts: bool,
prompt_folders: bool,
query_history: bool,
downtimes: bool,
}
pub async fn init() -> Result<(), anyhow::Error> {
@@ -339,78 +340,7 @@ pub async fn init() -> Result<(), anyhow::Error> {
);
}
}
if pending.alert_folders {
match migrations::migrate_alert_folders().await {
Ok(_) => {
log::info!("[OFGA:Local] Alert folders migrated to openfga");
}
Err(e) => {
log::error!(
"[OFGA:Local] Error migrating alert folders to openfga: {e}"
);
}
}
}
if pending.workflow_folders {
match migrations::migrate_workflow_folders().await {
Ok(_) => {
log::info!("[OFGA:Local] Workflow folders migrated to openfga");
}
Err(e) => {
log::error!(
"[OFGA:Local] Error migrating workflow folders to openfga: {e}"
);
}
}
}
if pending.prompt_folders {
match migrations::migrate_prompt_folders().await {
Ok(_) => {
log::info!("[OFGA:Local] Prompt folders migrated to openfga");
}
Err(e) => {
log::error!(
"[OFGA:Local] Error migrating prompt folders to openfga: {e}"
);
}
}
}
if pending.report_folders {
match migrations::migrate_report_folders().await {
Ok(_) => {
log::info!("[OFGA:Local] Report folders migrated to openfga");
}
Err(e) => {
log::error!(
"[OFGA:Local] Error migrating report folders to openfga: {e}"
);
}
}
}
if pending.anomaly_detection {
match migrations::migrate_anomaly_detection().await {
Ok(_) => {
log::info!("[OFGA:Local] Anomaly detection migrated to openfga");
}
Err(e) => {
log::error!(
"[OFGA:Local] Error migrating anomaly detection to openfga: {e}"
);
}
}
}
if pending.stream_names {
match migrations::migrate_stream_names().await {
Ok(_) => {
log::info!("[OFGA:Local] Stream names migrated to openfga");
}
Err(e) => {
log::error!(
"[OFGA:Local] Error migrating stream names to openfga: {e}"
);
}
}
}
run_folder_and_data_migrations(&pending).await;
}
// Check if there are init ofga tuples that needs to be added now
@@ -446,6 +376,72 @@ pub async fn init() -> Result<(), anyhow::Error> {
Ok(())
}
/// Database sweeps a pending back-fill needs beyond its ownership tuples.
///
/// Kept out of `init` so that function stays within the cognitive-complexity limit.
async fn run_folder_and_data_migrations(pending: &PendingMigrations) {
if pending.alert_folders {
match migrations::migrate_alert_folders().await {
Ok(_) => {
log::info!("[OFGA:Local] Alert folders migrated to openfga");
}
Err(e) => {
log::error!("[OFGA:Local] Error migrating alert folders to openfga: {e}");
}
}
}
if pending.workflow_folders {
match migrations::migrate_workflow_folders().await {
Ok(_) => {
log::info!("[OFGA:Local] Workflow folders migrated to openfga");
}
Err(e) => {
log::error!("[OFGA:Local] Error migrating workflow folders to openfga: {e}");
}
}
}
if pending.prompt_folders {
match migrations::migrate_prompt_folders().await {
Ok(_) => {
log::info!("[OFGA:Local] Prompt folders migrated to openfga");
}
Err(e) => {
log::error!("[OFGA:Local] Error migrating prompt folders to openfga: {e}");
}
}
}
if pending.report_folders {
match migrations::migrate_report_folders().await {
Ok(_) => {
log::info!("[OFGA:Local] Report folders migrated to openfga");
}
Err(e) => {
log::error!("[OFGA:Local] Error migrating report folders to openfga: {e}");
}
}
}
if pending.anomaly_detection {
match migrations::migrate_anomaly_detection().await {
Ok(_) => {
log::info!("[OFGA:Local] Anomaly detection migrated to openfga");
}
Err(e) => {
log::error!("[OFGA:Local] Error migrating anomaly detection to openfga: {e}");
}
}
}
if pending.stream_names {
match migrations::migrate_stream_names().await {
Ok(_) => {
log::info!("[OFGA:Local] Stream names migrated to openfga");
}
Err(e) => {
log::error!("[OFGA:Local] Error migrating stream names to openfga: {e}");
}
}
}
}
/// The resources a pending back-fill needs nothing but an `_all_` org tuple for.
///
/// Data rather than control flow. The three that also need a per-object tuple or
@@ -539,6 +535,9 @@ fn all_org_ownership_keys(pending: &PendingMigrations) -> Vec<&'static str> {
if pending.query_history {
keys.push("query_history");
}
if pending.downtimes {
keys.extend(["downtime_folders", "downtimes"]);
}
keys
}
@@ -579,6 +578,7 @@ fn pending_migrations(latest: &str, existing: &str) -> PendingMigrations {
let v0_0_50 = version_compare::Version::from("0.0.50").unwrap();
let v0_0_51 = version_compare::Version::from("0.0.51").unwrap();
let v0_0_53 = version_compare::Version::from("0.0.53").unwrap();
let v0_0_54 = version_compare::Version::from("0.0.54").unwrap();
if meta_version > v0_0_5 && existing_model_version < v0_0_6 {
pending.pipeline = true;
@@ -698,6 +698,11 @@ fn pending_migrations(latest: &str, existing: &str) -> PendingMigrations {
log::info!("[OFGA:Local] query history permissions migration needed");
pending.query_history = true;
}
// Gated on the latest model so a 0.0.53 model never back-fills types it lacks.
if meta_version >= v0_0_54 && existing_model_version < v0_0_54 {
log::info!("[OFGA:Local] downtimes permissions migration needed");
pending.downtimes = true;
}
pending
}
@@ -717,4 +722,22 @@ mod tests {
assert!(!pending.query_history);
assert!(all_org_ownership_keys(&pending).is_empty());
}
#[test]
fn test_downtimes_back_fill_below_0_0_54() {
let pending = pending_migrations("0.0.54", "0.0.53");
assert!(pending.downtimes);
assert!(!pending.query_history);
assert_eq!(
all_org_ownership_keys(&pending),
vec!["downtime_folders", "downtimes"]
);
let pending = pending_migrations("0.0.54", "0.0.52");
assert!(pending.downtimes && pending.query_history);
let pending = pending_migrations("0.0.54", "0.0.54");
assert!(!pending.downtimes);
assert!(all_org_ownership_keys(&pending).is_empty());
}
}
src/core/src/org_cleanup.rs
+17
View File
@@ -373,6 +373,11 @@ async fn step_delete_file_list(org_id: &str) -> Result<(), anyhow::Error> {
Ok(())
}
/// Delete every downtime through the db layer, so caches, regions and OpenFGA tuples drop it.
async fn delete_org_downtimes(org_id: &str) -> Result<(), anyhow::Error> {
crate::db::downtimes::delete_by_org(org_id).await
}
/// Delete every alert in an org through the service layer so the ALERTS /
/// STREAM_ALERTS caches evict cluster-wide (via the coordinator delete event) and
/// each alert's scheduler trigger + OFGA ownership is removed — none of which a
@@ -492,6 +497,10 @@ async fn step_delete_db_resources(org_id: &str) -> Result<(), anyhow::Error> {
delete_org_alerts(org_id)
.await
.map_err(|e| anyhow::anyhow!("step_delete_db_resources/alerts: {e}"))?;
// downtimes.folder_id has a FK (downtimes_folder_fk) → folders.id.
delete_org_downtimes(org_id)
.await
.map_err(|e| anyhow::anyhow!("step_delete_db_resources/downtimes: {e}"))?;
// timed_annotation_panels cascade from timed_annotations; both are deleted here
// via the three-hop join: folders.org → dashboards.folder_id → timed_annotations.dashboard_id
// Must run BEFORE dashboards::delete_by_org or the join finds no rows.
@@ -1238,6 +1247,14 @@ mod tests {
}
}
#[test]
fn test_downtimes_are_deleted_before_their_folders() {
assert!(
position_of("delete_org_downtimes(org_id)")
< position_of("folders::delete_by_org(org_id)")
);
}
#[test]
fn test_slos_are_deleted_before_alerts() {
// An SLO reads from a source alert, and a later PR refuses to delete an
src/core/src/self_reporting/triggers_schema.rs
+6 -1
View File
@@ -183,7 +183,12 @@ mod tests {
assert!(field_names.contains(&"synthetics_location".to_string()));
assert!(field_names.contains(&"delivery_attempted".to_string()));
assert_eq!(field_names.len(), 36);
// The history filter and the Suppressed tab of a downtime read this column.
assert!(field_names.contains(&"downtime_id".to_string()));
// Verify count matches struct fields
// 37: 21 original + 5 dedup + 6 value + 2 synthetics + downtime + delivery + rule_value
assert_eq!(field_names.len(), 37);
// Verify no duplicate fields
let unique_count = field_names
src/core/src/slo/backfill.rs
+982 -40
View File
File diff suppressed because it is too large Load Diff
src/core/src/slo/corrections.rs
+1059
View File
File diff suppressed because it is too large Load Diff
src/core/src/slo/evaluate.rs
+71 -1
View File
@@ -150,7 +150,7 @@ fn frozen(reason: UnobservedReason, group_key: &str, slo: &Slo) -> SloEvalResult
}
}
fn evaluate_row(
pub(super) fn evaluate_row(
slo: &Slo,
cond: &SloCondition,
row: &infra::table::entity::slo_status::Model,
@@ -445,6 +445,76 @@ mod tests {
assert_eq!(r.actual_value, Some(1.0), "burn rate at exactly 1.0");
}
/// D8: a burn window inside an `Exclude` downtime freezes; `CountAsGood` reads zero burn.
#[test]
fn a_burn_window_fully_inside_a_downtime_reads_zero_total() {
use config::meta::{
downtimes::{CorrectionWindow, SloCorrectionMode},
slo::{SliType, burn, slice::SliceRow},
};
const SLICE: i64 = 300;
const WM: i64 = 1_000_000;
let params = crate::slo::ingest::PassParams {
slo_id: "slo1".into(),
definition_generation: 1,
range_start: WM - 3600,
range_end: WM,
slice_interval_secs: SLICE,
rev: 7,
max_groups: 500,
};
let windows_for = |mode| {
let window = CorrectionWindow {
downtime_id: "dt".into(),
start: (WM - 3600) * 1_000_000,
end: WM * 1_000_000,
mode,
};
let mut slices: Vec<SliceRow> = (1..=12)
.map(|i| SliceRow {
slo_id: "slo1".into(),
definition_generation: 1,
group_key: String::new(),
slice_start: WM - i * SLICE,
good: 50.0,
total: 100.0,
rev: 7,
corrected_by: None,
})
.collect();
crate::slo::corrections::apply(
&mut slices,
&[window],
&Default::default(),
SliType::Count,
&params,
);
let buf = burn::fold_trailing(
burn::TrailingSlices::new(),
slices.iter().map(|s| (s.slice_start, s.good, s.total)),
WM,
3600,
);
let mut row = status(Some(burn::burn_windows_json(&buf, &[3600, 300], WM, SLICE)));
row.watermark_end = Some(WM);
let c = SloCondition {
long_window_secs: Some(3600),
short_window_secs: Some(300),
..cond(SloAlertKind::BurnRate)
};
burn_windows(&row, &c, 0.9, false)
};
let (long, short) = windows_for(SloCorrectionMode::Exclude);
assert_eq!(long, Observation::Unobserved(UnobservedReason::ZeroTotal));
assert_eq!(short, Observation::Unobserved(UnobservedReason::ZeroTotal));
let (long, short) = windows_for(SloCorrectionMode::CountAsGood);
assert_eq!(long.sli(), Some(100.0));
assert_eq!(short.sli(), Some(100.0));
}
/// The negative half of the contract: a window the ingest pass did NOT
/// precompute must still freeze. Proves the round-trip test above passes
/// for the right reason rather than because everything reads as observed.
src/core/src/slo/ingest.rs
+64 -3
View File
@@ -190,6 +190,7 @@ pub fn build_slices(sli: &SliConfig, rows: Vec<QueryRow>, params: &PassParams) -
good,
total,
rev: params.rev,
corrected_by: None,
});
}
@@ -247,6 +248,7 @@ pub fn fill_missing(sli: &SliConfig, present: &[SliceRow], params: &PassParams)
good: 0.0,
total: fill_total,
rev: params.rev,
corrected_by: None,
});
}
t += params.slice_interval_secs;
@@ -261,19 +263,22 @@ pub fn fill_missing(sli: &SliConfig, present: &[SliceRow], params: &PassParams)
/// ratios weights a 3-event group the same as a 30,000-event one, which is how
/// a tiny group's bad minute can swamp the real number.
pub fn exact_rollup(slices: &[SliceRow], params: &PassParams) -> Vec<SliceRow> {
let mut by_slice: std::collections::BTreeMap<i64, (f64, f64)> = Default::default();
let mut by_slice: std::collections::BTreeMap<i64, (f64, f64, Option<&String>)> =
Default::default();
for s in slices {
// Guard against a caller passing rollup rows back in.
if s.group_key.is_empty() {
continue;
}
let e = by_slice.entry(s.slice_start).or_insert((0.0, 0.0));
let e = by_slice.entry(s.slice_start).or_insert((0.0, 0.0, None));
e.0 += s.good;
e.1 += s.total;
// The chart reads the rollup, so it carries the first group row's correction.
e.2 = e.2.or(s.corrected_by.as_ref());
}
by_slice
.into_iter()
.map(|(slice_start, (good, total))| SliceRow {
.map(|(slice_start, (good, total, corrected_by))| SliceRow {
slo_id: params.slo_id.clone(),
definition_generation: params.definition_generation,
group_key: String::new(),
@@ -281,6 +286,7 @@ pub fn exact_rollup(slices: &[SliceRow], params: &PassParams) -> Vec<SliceRow> {
good,
total,
rev: params.rev,
corrected_by: corrected_by.cloned(),
})
.collect()
}
@@ -488,6 +494,7 @@ mod tests {
good: 5.0,
total: 5.0,
rev: 7,
corrected_by: None,
}];
let filled = fill_missing(&count_sli(), &present, &params());
assert_eq!(filled.len(), 2, "buckets 300 and 600");
@@ -525,6 +532,7 @@ mod tests {
good: 1.0,
total: 1.0,
rev: 7,
corrected_by: None,
},
SliceRow {
slo_id: SLO.into(),
@@ -534,6 +542,7 @@ mod tests {
good: 1.0,
total: 1.0,
rev: 7,
corrected_by: None,
},
];
let filled = fill_missing(&count_sli(), &present, &params());
@@ -555,6 +564,7 @@ mod tests {
good: 30_000.0,
total: 30_000.0,
rev: 7,
corrected_by: None,
},
SliceRow {
slo_id: SLO.into(),
@@ -564,6 +574,7 @@ mod tests {
good: 0.0,
total: 3.0,
rev: 7,
corrected_by: None,
},
];
let rollup = exact_rollup(&slices, &params());
@@ -587,6 +598,7 @@ mod tests {
good,
total,
rev: 7,
corrected_by: None,
};
let rollup = exact_rollup(&[mk(0, 1.0, 2.0), mk(300, 3.0, 4.0)], &params());
assert_eq!(rollup.len(), 2);
@@ -605,11 +617,59 @@ mod tests {
good,
total,
rev: 7,
corrected_by: None,
};
let rollup = exact_rollup(&[mk("region=eu", 1.0, 2.0), mk("", 1.0, 2.0)], &params());
assert_eq!((rollup[0].good, rollup[0].total), (1.0, 2.0));
}
#[test]
fn a_grouped_slo_with_a_correction_shows_it_on_the_rollup_row() {
let mk = |group: &str, slice_start, corrected_by: Option<&str>| SliceRow {
slo_id: SLO.into(),
definition_generation: 1,
group_key: group.to_string(),
slice_start,
good: 0.0,
total: 0.0,
rev: 7,
corrected_by: corrected_by.map(str::to_string),
};
let rollup = exact_rollup(
&[
mk("region=eu", 0, Some("dt")),
mk("region=us", 0, Some("dt")),
mk("region=eu", 300, None),
mk("region=us", 300, Some("other")),
mk("region=eu", 600, None),
],
&params(),
);
assert_eq!(rollup[0].corrected_by.as_deref(), Some("dt"));
assert_eq!(
rollup[1].corrected_by.as_deref(),
Some("other"),
"a slice any group row corrected shows the band"
);
assert_eq!(rollup[2].corrected_by, None);
}
#[test]
fn group_rows_with_different_corrections_give_the_rollup_the_first() {
let mk = |group: &str, corrected_by: &str| SliceRow {
slo_id: SLO.into(),
definition_generation: 1,
group_key: group.to_string(),
slice_start: 0,
good: 0.0,
total: 0.0,
rev: 7,
corrected_by: Some(corrected_by.to_string()),
};
let rollup = exact_rollup(&[mk("region=eu", "a"), mk("region=us", "b")], &params());
assert_eq!(rollup[0].corrected_by.as_deref(), Some("a"));
}
// ===================== group cap ======================================
#[test]
@@ -680,6 +740,7 @@ mod absent_is_bad_fill_tests {
good: 300.0,
total: 300.0,
rev: 7,
corrected_by: None,
}
}
src/core/src/slo/job.rs
+538 -106
View File
@@ -29,6 +29,7 @@
use config::{
get_config,
meta::{
downtimes::{CorrectionWindow, TargetModule},
search::{Query, Request, RequestEncoding, SearchEventContext},
slo::{
CountSource, QueryLanguage, SliConfig, Slo,
@@ -70,6 +71,57 @@ pub enum PassOutcome {
Fenced { expected: i32, found: i32 },
}
/// A rollup burn cache rebuilt from the slices after a re-measure, written as the job finishes.
#[derive(Debug, Clone, PartialEq)]
pub struct BurnRebuild {
/// The watermark the slices were read up to.
pub watermark_end: i64,
pub trailing: config::meta::slo::burn::TrailingSlices,
pub durations: Vec<i64>,
pub slice_interval_secs: i64,
}
impl BurnRebuild {
pub fn from_slices(
rollup: Vec<(i64, f64, f64)>,
watermark_end: i64,
durations: Vec<i64>,
slice_interval_secs: i64,
) -> Self {
use config::meta::slo::burn;
let trailing = burn::fold_trailing(
burn::TrailingSlices::new(),
rollup,
watermark_end,
burn::retain_secs(&durations),
);
Self {
watermark_end,
trailing,
durations,
slice_interval_secs,
}
}
/// The trailing buffer and windows at `watermark_end`; slices a pass folded since stay.
pub fn merged(
&self,
stored: config::meta::slo::burn::TrailingSlices,
watermark_end: i64,
) -> (serde_json::Value, serde_json::Value) {
let mut buf = self.trailing.clone();
buf.extend(stored.range(self.watermark_end..).map(|(k, v)| (*k, *v)));
fold_burn_cache(
buf,
std::iter::empty(),
watermark_end,
&self.durations,
self.slice_interval_secs,
)
}
}
/// Run one SLI ingest pass for `slo`.
pub async fn run_pass(slo: &Slo, now_secs: i64) -> Result<PassOutcome, anyhow::Error> {
let cfg = get_config();
@@ -153,6 +205,15 @@ pub async fn run_pass(slo: &Slo, now_secs: i64) -> Result<PassOutcome, anyhow::E
// would be missing from the exact overall row.
let filled = fill_missing(&slo.definition.sli_config, &result.slices, &params);
result.slices.extend(filled);
// Before the empty-result return: a downtime over a silent service finds nothing.
let windows = correction_windows(slo, range.start, range.end).await;
super::corrections::apply(
&mut result.slices,
&windows,
&Default::default(),
slo.definition.sli_config.sli_type(),
&params,
);
if !group_by.is_empty() {
let rollup = exact_rollup(&result.slices, &params);
@@ -797,6 +858,7 @@ async fn write_slices(org: &str, slices: &[SliceRow], now_secs: i64) -> Result<(
good: s.good,
total: s.total,
rev: s.rev,
corrected_by: s.corrected_by.clone().unwrap_or_default(),
})
.unwrap_or(json::Value::Null)
})
@@ -813,56 +875,23 @@ async fn commit_status(
) -> Result<slo_table::WriteOutcome, anyhow::Error> {
// SQLite opens the read-only pool with read_only(true), and this path always writes.
let db = get_orm_client_rw().await;
let mut by_group: std::collections::BTreeMap<String, (f64, f64, i32)> = Default::default();
for s in &result.slices {
let e = by_group.entry(s.group_key.clone()).or_insert((0.0, 0.0, 0));
e.0 += s.good;
e.1 += s.total;
e.2 += 1;
}
let deltas = by_group
.into_iter()
.map(
|(group_key, (good, total, covered))| slo_table::GroupDelta {
group_key,
good_delta: good,
total_delta: total,
covered_slices_delta: covered,
},
)
.collect();
// The burn-window cache (§6b.4c). Computed here rather than at alert time
// so five alerts on one SLO cost zero extra scans (§6b.9). A failure to
// build it must NOT fail the pass: the running aggregate is the primary
// product, and a missing burn window freezes the burn-rate alerts (safe)
// rather than losing the measurement (not).
let (trailing_slices, burn_windows) =
match build_burn_cache(db, slo, result, watermark_end).await {
Ok(v) => v,
Err(e) => {
log::warn!(
"[slo] could not build burn windows for {}: {e} — the pass still publishes",
slo.id
);
(None, None)
}
};
Ok(slo_table::apply_status(
db,
&slo_table::StatusWrite {
slo_id: slo.id.clone(),
definition_generation: slo.definition_generation,
writer: config::meta::slo::slice::Writer::Incremental,
deltas,
watermark_end: Some(watermark_end),
trailing_slices,
burn_windows,
computed_at: now_secs,
},
)
.await?)
let durations = match burn_durations(db, slo).await {
Ok(v) => v,
Err(e) => {
log::warn!(
"[slo] could not build burn windows for {}: {e} — the pass still publishes",
slo.id
);
Vec::new()
}
};
write_pass_status(db, slo, result, watermark_end, &durations, now_secs).await
}
/// Build the trailing buffer and the burn-window aggregates for this pass.
@@ -873,31 +902,25 @@ async fn commit_status(
///
/// Returns `(None, None)` when no enabled alert asks for a burn window — an
/// SLO nobody alerts on burn-rate over pays nothing for the machinery.
async fn build_burn_cache(
db: &sea_orm::DatabaseConnection,
fn build_burn_cache(
slo: &Slo,
rollup_row: Option<&infra::table::entity::slo_status::Model>,
result: &PassResult,
watermark_end: i64,
) -> Result<(Option<serde_json::Value>, Option<serde_json::Value>), anyhow::Error> {
durations: &[i64],
) -> (Option<serde_json::Value>, Option<serde_json::Value>) {
use config::meta::slo::burn;
let cfg = get_config();
// `None`: the ingest pass must see EVERY enabled alert's pair. Excluding
// anything here would stop precomputing a window some alert still needs.
let pairs =
infra::table::alerts::list_slo_burn_window_pairs(db, &slo.org, &slo.id, None).await?;
let durations = burn::durations_for_pairs(&pairs, cfg.slo.max_burn_window_pairs as usize);
if durations.is_empty() {
return Ok((None, None));
return (None, None);
}
// The previous buffer, from the rollup row. Absent on the first pass of a
// generation, and absent (rather than fatal) if it cannot be parsed.
let prev = slo_table::load_status(db, &slo.id, "")
.await?
let prev = rollup_row
.filter(|row| row.definition_generation == slo.definition_generation)
.and_then(|row| row.trailing_slices);
let buf = burn::parse_trailing(prev.as_ref());
.and_then(|row| row.trailing_slices.as_ref());
let buf = burn::parse_trailing(prev);
// Only the rollup series feeds the buffer.
let rollup = result
@@ -906,14 +929,103 @@ async fn build_burn_cache(
.filter(|s| s.group_key.is_empty())
.map(|s| (s.slice_start, s.good, s.total));
let buf = burn::fold_trailing(buf, rollup, watermark_end, burn::retain_secs(&durations));
let windows = burn::burn_windows_json(
&buf,
&durations,
let (trailing, windows) = fold_burn_cache(
buf,
rollup,
watermark_end,
durations,
slo.definition.slice_interval_secs,
);
Ok((Some(burn::trailing_to_json(&buf)), Some(windows)))
(Some(trailing), Some(windows))
}
/// Every window duration an enabled burn-rate alert on the SLO reads.
async fn burn_durations(
db: &sea_orm::DatabaseConnection,
slo: &Slo,
) -> Result<Vec<i64>, anyhow::Error> {
// `None`: the ingest pass must see EVERY enabled alert's pair. Excluding
// anything here would stop precomputing a window some alert still needs.
let pairs =
infra::table::alerts::list_slo_burn_window_pairs(db, &slo.org, &slo.id, None).await?;
Ok(config::meta::slo::burn::durations_for_pairs(
&pairs,
get_config().slo.max_burn_window_pairs as usize,
))
}
/// Folds rollup slices into `buf` and computes the windows ending at `watermark_end`.
fn fold_burn_cache(
buf: config::meta::slo::burn::TrailingSlices,
rollup: impl IntoIterator<Item = (i64, f64, f64)>,
watermark_end: i64,
durations: &[i64],
slice_interval_secs: i64,
) -> (serde_json::Value, serde_json::Value) {
use config::meta::slo::burn;
let buf = burn::fold_trailing(buf, rollup, watermark_end, burn::retain_secs(durations));
let windows = burn::burn_windows_json(&buf, durations, watermark_end, slice_interval_secs);
(burn::trailing_to_json(&buf), windows)
}
/// The rollup row under its lock, read only for a burn cache; an SLO without burn alerts skips it.
async fn rollup_for_burn<C: sea_orm::ConnectionTrait>(
txn: &C,
slo_id: &str,
durations: &[i64],
) -> Result<Option<infra::table::entity::slo_status::Model>, infra::errors::Error> {
if durations.is_empty() {
return Ok(None);
}
// Read under the row lock, so a re-measure's burn rebuild cannot land before the write.
slo_table::load_rollup_for_update(txn, slo_id).await
}
/// The pass's status write, with the burn cache folded onto the rollup row it locks first.
pub(super) async fn write_pass_status(
db: &sea_orm::DatabaseConnection,
slo: &Slo,
result: &PassResult,
watermark_end: i64,
durations: &[i64],
now_secs: i64,
) -> Result<slo_table::WriteOutcome, anyhow::Error> {
use sea_orm::TransactionTrait;
let txn = db.begin().await?;
let rollup = match rollup_for_burn(&txn, &slo.id, durations).await {
Ok(row) => row,
Err(e) => {
let _ = txn.rollback().await;
return Err(e.into());
}
};
let (trailing_slices, burn_windows) =
build_burn_cache(slo, rollup.as_ref(), result, watermark_end, durations);
let write = slo_table::StatusWrite {
slo_id: slo.id.clone(),
definition_generation: slo.definition_generation,
writer: config::meta::slo::slice::Writer::Incremental,
deltas: group_deltas(&result.slices),
watermark_end: Some(watermark_end),
trailing_slices,
burn_windows,
computed_at: now_secs,
};
let outcome = match slo_table::apply_status_in_txn(&txn, &write).await {
Ok(outcome) => outcome,
Err(e) => {
let _ = txn.rollback().await;
return Err(e.into());
}
};
if matches!(outcome, slo_table::WriteOutcome::FencedByGeneration { .. }) {
let _ = txn.rollback().await;
return Ok(outcome);
}
txn.commit().await?;
Ok(outcome)
}
/// Measure an explicit `[start, end)` and publish it.
@@ -929,55 +1041,14 @@ pub async fn run_range(
end: i64,
writer: config::meta::slo::slice::Writer,
) -> Result<usize, anyhow::Error> {
let cfg = get_config();
let db = get_orm_client_rw().await;
let group_by = slo.definition.group_by.clone().unwrap_or_default();
let params = PassParams {
slo_id: slo.id.clone(),
definition_generation: slo.definition_generation,
range_start: start,
range_end: end,
slice_interval_secs: slo.definition.slice_interval_secs,
rev: end,
max_groups: cfg.slo.max_groups,
};
let range = config::meta::slo::window::IngestRange { start, end };
let (rows, query_rejects) = fetch_rows(slo, &group_by, &range, &params).await?;
let mut result = build_slices(&slo.definition.sli_config, rows, &params);
result.rejected.extend(query_rejects);
let filled = fill_missing(&slo.definition.sli_config, &result.slices, &params);
result.slices.extend(filled);
if !group_by.is_empty() {
let rollup = exact_rollup(&result.slices, &params);
result.slices.extend(rollup);
}
let result = measure_range(slo, start, end, end, &Default::default()).await?;
if result.slices.is_empty() {
return Ok(0);
}
let now_secs = now_micros() / 1_000_000;
write_slices(&slo.org, &result.slices, now_secs).await?;
let mut by_group: std::collections::BTreeMap<String, (f64, f64, i32)> = Default::default();
for s in &result.slices {
let e = by_group.entry(s.group_key.clone()).or_insert((0.0, 0.0, 0));
e.0 += s.good;
e.1 += s.total;
e.2 += 1;
}
let deltas = by_group
.into_iter()
.map(
|(group_key, (good, total, covered))| slo_table::GroupDelta {
group_key,
good_delta: good,
total_delta: total,
covered_slices_delta: covered,
},
)
.collect();
let deltas = group_deltas(&result.slices);
slo_table::apply_status(
db,
@@ -1001,6 +1072,221 @@ pub async fn run_range(
Ok(result.slices.len())
}
/// [`run_range`] with `rev = now` to win the dedupe, and no delta: `reconcile` sets totals.
pub async fn remeasure_range(slo: &Slo, start: i64, end: i64) -> Result<usize, anyhow::Error> {
// A cancelled or shrunk window leaves its filler rows behind unless they are written over.
let corrected_before = super::corrections::corrected_keys(slo, start, end).await?;
let rev = now_micros() / 1_000_000;
Ok(measure_range(slo, start, end, rev, &corrected_before)
.await?
.slices
.len())
}
/// The rollup's burn cache rebuilt from the slices, for a re-measure's finish.
pub async fn rebuild_burn_cache(slo: &Slo) -> Result<Option<BurnRebuild>, anyhow::Error> {
rebuild_burn_cache_with(get_orm_client_rw().await, slo).await
}
/// [rebuild_burn_cache] on `db`; an unreadable alert list skips the cache, as a pass does.
pub async fn rebuild_burn_cache_with(
db: &sea_orm::DatabaseConnection,
slo: &Slo,
) -> Result<Option<BurnRebuild>, anyhow::Error> {
let Some(status) = slo_table::load_status(db, &slo.id, "").await? else {
return Ok(None);
};
let Some(watermark_end) = status.watermark_end else {
return Ok(None);
};
if status.definition_generation != slo.definition_generation {
return Ok(None);
}
let durations = match burn_durations(db, slo).await {
Ok(durations) => durations,
Err(e) => {
log::warn!(
"[slo] could not read the burn windows of {} for its re-measure finish: {e}",
slo.id
);
return Ok(None);
}
};
if durations.is_empty() {
return Ok(None);
}
let from = watermark_end - config::meta::slo::burn::retain_secs(&durations);
let sql = trailing_rollup_sql(
&slo.id,
slo.definition_generation,
from,
watermark_end,
super::reconcile::tombstones_possible(slo).await,
);
let hits = super::reconcile::search_slices(&slo.org, sql, from).await?;
let rollup: Vec<(i64, f64, f64)> = hits
.iter()
.filter_map(|h| {
Some((
h.get("slice_start").and_then(json::Value::as_i64)?,
h.get("good").and_then(json::Value::as_f64)?,
h.get("total").and_then(json::Value::as_f64)?,
))
})
.collect();
Ok(Some(BurnRebuild::from_slices(
rollup,
watermark_end,
durations,
slo.definition.slice_interval_secs,
)))
}
/// Writes a rebuilt burn cache under the current watermark, keeping what the pass folded since.
pub async fn write_burn_rebuild<C: sea_orm::ConnectionTrait>(
conn: &C,
slo_id: &str,
generation: i32,
rebuilt: &BurnRebuild,
) -> Result<bool, anyhow::Error> {
// Locked like the pass's write, so neither writer folds onto a buffer the other replaced.
let Some(row) = slo_table::load_rollup_for_update(conn, slo_id).await? else {
return Ok(false);
};
let Some(watermark_end) = row.watermark_end else {
return Ok(false);
};
if row.definition_generation != generation || watermark_end < rebuilt.watermark_end {
return Ok(false);
}
let stored = config::meta::slo::burn::parse_trailing(row.trailing_slices.as_ref());
let (trailing, windows) = rebuilt.merged(stored, watermark_end);
Ok(
slo_table::replace_burn_cache(conn, slo_id, generation, watermark_end, trailing, windows)
.await?,
)
}
/// The latest rollup slice per start in `[from, to)`, minus withdrawn-correction tombstones.
pub fn trailing_rollup_sql(
slo_id: &str,
generation: i32,
from: i64,
to: i64,
tombstones: bool,
) -> String {
let (columns, measured) = if tombstones {
(
", corrected_by",
format!(" AND {}", super::reconcile::MEASURED_SQL),
)
} else {
("", String::new())
};
format!(
"SELECT slice_start, good, total FROM ( \
SELECT slice_start, good, total{columns}, \
ROW_NUMBER() OVER (PARTITION BY slice_start ORDER BY rev DESC) AS zo_rn \
FROM {SLO_SLICES_STREAM} \
WHERE slo_id = '{slo_id}' \
AND definition_generation = {generation} \
AND group_key = '' \
AND slice_start >= {from} AND slice_start < {to} \
) WHERE zo_rn = 1{measured}"
)
}
/// Fetch, build, fill, correct, roll up and write `[start, end)` under `rev`.
async fn measure_range(
slo: &Slo,
start: i64,
end: i64,
rev: i64,
corrected_before: &std::collections::HashSet<super::corrections::SliceKey>,
) -> Result<PassResult, anyhow::Error> {
let cfg = get_config();
let group_by = slo.definition.group_by.clone().unwrap_or_default();
let params = PassParams {
slo_id: slo.id.clone(),
definition_generation: slo.definition_generation,
range_start: start,
range_end: end,
slice_interval_secs: slo.definition.slice_interval_secs,
rev,
max_groups: cfg.slo.max_groups,
};
let range = config::meta::slo::window::IngestRange { start, end };
let (rows, query_rejects) = fetch_rows(slo, &group_by, &range, &params).await?;
let mut result = build_slices(&slo.definition.sli_config, rows, &params);
result.rejected.extend(query_rejects);
let filled = fill_missing(&slo.definition.sli_config, &result.slices, &params);
result.slices.extend(filled);
let windows = correction_windows(slo, start, end).await;
super::corrections::apply(
&mut result.slices,
&windows,
corrected_before,
slo.definition.sli_config.sli_type(),
&params,
);
if !group_by.is_empty() {
let rollup = exact_rollup(&result.slices, &params);
result.slices.extend(rollup);
}
if !result.slices.is_empty() {
write_slices(&slo.org, &result.slices, now_micros() / 1_000_000).await?;
}
Ok(result)
}
/// The downtime windows over `[start, end)`; none without a cached downtime that targets SLOs.
async fn correction_windows(slo: &Slo, start: i64, end: i64) -> Vec<CorrectionWindow> {
let rows = db::downtimes::list_cached(&slo.org);
if !corrects_slos(crate::alerts::downtimes::enabled(), &rows) {
return Vec::new();
}
let dims = crate::alerts::downtimes::dimensions_for_slo(slo).await;
crate::alerts::downtimes::corrections_for_slo(
slo,
&dims,
start.saturating_mul(1_000_000),
end.saturating_mul(1_000_000),
)
}
/// With downtimes off, rows left in the cache correct nothing, as before the feature existed.
fn corrects_slos(enabled: bool, rows: &[config::meta::downtimes::Downtime]) -> bool {
enabled
&& rows.iter().any(|row| {
row.targets
.iter()
.any(|target| target.module == TargetModule::Slos)
})
}
/// One delta per group; a corrected slice counts as covered and adds `0 / 0`.
fn group_deltas(slices: &[SliceRow]) -> Vec<slo_table::GroupDelta> {
let mut by_group: std::collections::BTreeMap<String, (f64, f64, i32)> = Default::default();
for s in slices {
let e = by_group.entry(s.group_key.clone()).or_insert((0.0, 0.0, 0));
e.0 += s.good;
e.1 += s.total;
e.2 += 1;
}
by_group
.into_iter()
.map(
|(group_key, (good, total, covered))| slo_table::GroupDelta {
group_key,
good_delta: good,
total_delta: total,
covered_slices_delta: covered,
},
)
.collect()
}
/// Schedule the next pass for an SLO.
pub fn next_run_at(slice_interval_secs: i64) -> i64 {
now_micros() + slice_interval_secs * 1_000_000
@@ -2052,3 +2338,149 @@ mod promql_value_rows_tests {
);
}
}
#[cfg(test)]
mod downtime_correction_tests {
use config::meta::{
downtimes::{CorrectionWindow, SloCorrectionMode},
slo::SliType,
};
use super::*;
const SLICE: i64 = 300;
fn pass_params() -> PassParams {
PassParams {
slo_id: "slo1".to_string(),
definition_generation: 1,
range_start: 0,
range_end: 4 * SLICE,
slice_interval_secs: SLICE,
rev: 7,
max_groups: 500,
}
}
fn slice(start: i64, good: f64, total: f64) -> SliceRow {
SliceRow {
slo_id: "slo1".to_string(),
definition_generation: 1,
group_key: String::new(),
slice_start: start,
good,
total,
rev: 7,
corrected_by: None,
}
}
/// The service is down from 300 to 900: one bucket is bad, the next has no data at all.
#[test]
fn a_pass_with_a_window_writes_corrected_rows_that_stay_covered_and_weigh_nothing() {
let mut slices = vec![
slice(0, 300.0, 300.0),
slice(SLICE, 0.0, 300.0),
slice(3 * SLICE, 300.0, 300.0),
];
let windows = [CorrectionWindow {
downtime_id: "dt".to_string(),
start: SLICE * 1_000_000,
end: 3 * SLICE * 1_000_000,
mode: SloCorrectionMode::Exclude,
}];
let added = crate::slo::corrections::apply(
&mut slices,
&windows,
&Default::default(),
SliType::TimeSlice,
&pass_params(),
);
assert_eq!(added, 1, "the empty bucket inside the window gets a row");
for s in slices
.iter()
.filter(|s| (SLICE..3 * SLICE).contains(&s.slice_start))
{
assert_eq!(s.corrected_by.as_deref(), Some("dt"));
assert_eq!((s.good, s.total), (0.0, 0.0));
}
assert_eq!(
group_deltas(&slices),
vec![slo_table::GroupDelta {
group_key: String::new(),
good_delta: 600.0,
total_delta: 600.0,
covered_slices_delta: 4,
}]
);
}
#[test]
fn the_trailing_read_drops_tombstones_only_where_the_column_exists() {
let with = trailing_rollup_sql("slo1", 1, 0, 3_600, true);
assert!(
with.contains("zo_rn = 1 AND (total > 0 OR corrected_by <> '')"),
"{with}"
);
let without = trailing_rollup_sql("slo1", 1, 0, 3_600, false);
assert!(!without.contains("corrected_by"), "{without}");
assert!(without.contains("group_key = ''"), "{without}");
}
#[test]
fn with_downtimes_off_a_cached_slo_downtime_corrects_nothing() {
use config::meta::downtimes::{
Downtime, DowntimeSchedule, DowntimeTarget, Repeat, TargetFolders,
};
let row = Downtime {
id: "d1".to_string(),
org: "acme".to_string(),
folder_id: "default".to_string(),
name: "d1".to_string(),
reason: None,
condition: None,
targets: vec![DowntimeTarget {
module: TargetModule::Slos,
folders: TargetFolders::All,
tags: vec![],
ids: vec![],
slo_mode: None,
incident_mode: Default::default(),
}],
schedule: DowntimeSchedule {
repeat: Repeat::None,
starts_at: 0,
ends_at: Some(3_600_000_000),
timezone: "UTC".to_string(),
start_time_local: None,
duration_secs: 3_600,
weekdays: vec![],
},
cancelled_at: None,
cancelled_by: None,
show_banner: false,
notifications: None,
origin_region: None,
version: 1,
created_by: "lin".to_string(),
created_at: 0,
updated_by: "lin".to_string(),
updated_at: 0,
};
let rows = [row];
assert!(!corrects_slos(false, &rows));
assert!(corrects_slos(true, &rows));
assert!(!corrects_slos(true, &[]));
}
#[tokio::test]
async fn a_pass_without_burn_alerts_locks_no_rollup_row() {
// No status table exists, so any read of the rollup row fails.
let db = sea_orm::Database::connect("sqlite::memory:").await.unwrap();
assert!(rollup_for_burn(&db, "slo1", &[]).await.unwrap().is_none());
assert!(
rollup_for_burn(&db, "slo1", &[3600]).await.is_err(),
"a burn cache reads the row"
);
}
}
src/core/src/slo/mod.rs
+1
View File
@@ -31,6 +31,7 @@
//! * [`job`] is the IO shell: load, search, write, commit, reschedule.
pub mod backfill;
pub mod corrections;
pub mod evaluate;
pub mod ingest;
pub mod job;
src/core/src/slo/reconcile.rs
+190 -24
View File
@@ -31,12 +31,19 @@
use config::{
get_config,
meta::{
slo::{Slo, stream::SLO_SLICES_STREAM, window::read_window},
slo::{SliType, Slo, stream::SLO_SLICES_STREAM, window::read_window},
stream::StreamType,
},
utils::json,
};
use infra::{db::get_orm_client_rw, table::slo as slo_table};
use sea_orm::ConnectionTrait;
/// What a measured slice meets; a time-slice or alert `0 / 0` row nobody corrected is a tombstone.
pub const MEASURED_SQL: &str = "(total > 0 OR corrected_by <> '')";
/// Rows per request when every hit of a slices query is needed.
const SLICE_PAGE_SIZE: i64 = 10_000;
/// The per-group figures a rebuild produced.
#[derive(Debug, Clone, PartialEq)]
@@ -47,38 +54,61 @@ pub struct Rebuilt {
pub covered_slices: i32,
}
/// A rebuild with the watermark it read up to, so a guarded write can tell if a pass moved it.
#[derive(Debug, Clone, Default, PartialEq)]
pub struct Rebuild {
pub watermark_end: Option<i64>,
pub rows: Vec<Rebuilt>,
}
/// Rebuild every group's aggregate for one SLO and write the results.
pub async fn reconcile(slo: &Slo) -> Result<Vec<Rebuilt>, anyhow::Error> {
let rows = rebuild(slo).await?.rows;
write_rebuilt(get_orm_client_rw().await, &slo.id, &rows).await?;
Ok(rows)
}
/// The figures [reconcile] would write, read without writing, so a caller can write them guarded.
pub async fn rebuild(slo: &Slo) -> Result<Rebuild, anyhow::Error> {
let db = get_orm_client_rw().await;
let Some(status) = slo_table::load_status(db, &slo.id, "").await? else {
return Ok(Vec::new());
return Ok(Rebuild::default());
};
// Nothing has been measured under this generation yet, so there is no
// cache to repair — and no watermark to bound the read by.
let Some(watermark) = status.watermark_end else {
return Ok(Vec::new());
return Ok(Rebuild::default());
};
// A rebuild computed under a superseded generation would write the old
// definition's arithmetic into the new epoch. Reconciliation is exactly
// where that mistake is easy to make: it looks like a pure repair.
if status.definition_generation != slo.definition_generation {
return Ok(Vec::new());
return Ok(Rebuild::default());
}
let (from, to) = read_window(watermark, slo.definition.window_secs);
let rows = read_aggregate(slo, from, to).await?;
Ok(Rebuild {
watermark_end: Some(watermark),
rows: read_aggregate(slo, from, to).await?,
})
}
for r in &rows {
pub async fn write_rebuilt<C: ConnectionTrait>(
conn: &C,
slo_id: &str,
rows: &[Rebuilt],
) -> Result<(), anyhow::Error> {
for r in rows {
slo_table::reconcile_from_slices(
db,
&slo.id,
conn,
slo_id,
&r.group_key,
(r.good, r.total, r.covered_slices),
)
.await?;
}
Ok(rows)
Ok(())
}
/// The SQL that rebuilds one SLO's window.
@@ -87,14 +117,29 @@ pub async fn reconcile(slo: &Slo) -> Result<Vec<Rebuilt>, anyhow::Error> {
/// revision per `(group_key, slice_start)` before aggregating. The §6b.4a
/// spike confirmed the engine supports it; the alternative — `MAX(good)` — is
/// wrong for the reason in the module note.
pub fn reconcile_sql(slo_id: &str, generation: i32, from: i64, to: i64) -> String {
pub fn reconcile_sql(
slo_id: &str,
generation: i32,
from: i64,
to: i64,
tombstones: bool,
) -> String {
// Only a stream that can hold tombstones is asked for `corrected_by`, which older streams lack.
let (covered, columns) = if tombstones {
(
format!("SUM(CASE WHEN {MEASURED_SQL} THEN 1 ELSE 0 END)"),
", corrected_by",
)
} else {
("COUNT(*)".to_string(), "")
};
format!(
"SELECT group_key, \
SUM(good) AS good, \
SUM(total) AS total, \
COUNT(*) AS covered_slices \
{covered} AS covered_slices \
FROM ( \
SELECT group_key, slice_start, good, total, \
SELECT group_key, slice_start, good, total{columns}, \
ROW_NUMBER() OVER ( \
PARTITION BY group_key, slice_start ORDER BY rev DESC \
) AS zo_rn \
@@ -107,13 +152,72 @@ pub fn reconcile_sql(slo_id: &str, generation: i32, from: i64, to: i64) -> Strin
)
}
async fn read_aggregate(slo: &Slo, from: i64, to: i64) -> Result<Vec<Rebuilt>, anyhow::Error> {
let sql = reconcile_sql(&slo.id, slo.definition_generation, from, to);
/// Whether a slice of this SLO can be a tombstone: a time-slice or alert SLI with `corrected_by`.
pub async fn tombstones_possible(slo: &Slo) -> bool {
slo.definition.sli_config.sli_type() != SliType::Count
&& slices_carry_corrections(&slo.org).await
}
/// False when the org's slices stream has no `corrected_by` column, so no slice was ever corrected.
pub async fn slices_carry_corrections(org: &str) -> bool {
match infra::schema::get(org, SLO_SLICES_STREAM, StreamType::Logs).await {
Ok(schema) => has_corrected_by(&schema),
// Unknown: the query that follows reports the real error and the job retries.
Err(_) => true,
}
}
pub fn has_corrected_by(schema: &arrow_schema::Schema) -> bool {
schema.field_with_name("corrected_by").is_ok()
}
/// Every hit of `sql`, page by page until an empty page, so a result limit cannot drop rows.
pub async fn search_all_slices(
org: &str,
sql: &str,
from: i64,
) -> Result<Vec<json::Value>, anyhow::Error> {
collect_pages(|offset| search_slices_page(org, sql.to_string(), from, offset, SLICE_PAGE_SIZE))
.await
}
/// Calls `fetch` with the running offset until it returns no hits; pages may be shorter than asked.
pub async fn collect_pages<F, Fut>(mut fetch: F) -> Result<Vec<json::Value>, anyhow::Error>
where
F: FnMut(i64) -> Fut,
Fut: std::future::Future<Output = Result<Vec<json::Value>, anyhow::Error>>,
{
let mut all = Vec::new();
loop {
let page = fetch(all.len() as i64).await?;
if page.is_empty() {
return Ok(all);
}
all.extend(page);
}
}
/// The first 100,000 hits of `sql` over the slices stream, scanning from `from` minus the horizon.
pub async fn search_slices(
org: &str,
sql: String,
from: i64,
) -> Result<Vec<json::Value>, anyhow::Error> {
search_slices_page(org, sql, from, 0, 100_000).await
}
async fn search_slices_page(
org: &str,
sql: String,
from: i64,
offset: i64,
size: i64,
) -> Result<Vec<json::Value>, anyhow::Error> {
let req = config::meta::search::Request {
query: config::meta::search::Query {
sql,
from: 0,
size: 100_000,
from: offset,
size,
// Slices are written when measured, so a slice for `from` can
// have been written well after it. Widen the scan by the
// horizon rather than the window, or a late-written slice would
@@ -133,7 +237,7 @@ async fn read_aggregate(slo: &Slo, from: i64, to: i64) -> Result<Vec<Rebuilt>, a
let trace_id = config::ider::generate_trace_id();
let resp = crate::search::grpc_search::grpc_search(
&trace_id,
&slo.org,
org,
StreamType::Logs,
None,
&req,
@@ -146,9 +250,19 @@ async fn read_aggregate(slo: &Slo, from: i64, to: i64) -> Result<Vec<Rebuilt>, a
// to repair, because the next pass's deltas accumulate on top of it.
anyhow::bail!("SLO reconciliation returned a partial result; leaving the cache alone");
}
Ok(resp.hits)
}
Ok(resp
.hits
async fn read_aggregate(slo: &Slo, from: i64, to: i64) -> Result<Vec<Rebuilt>, anyhow::Error> {
let sql = reconcile_sql(
&slo.id,
slo.definition_generation,
from,
to,
tombstones_possible(slo).await,
);
let hits = search_slices(&slo.org, sql, from).await?;
Ok(hits
.iter()
.filter_map(|h| {
let o = h.as_object()?;
@@ -175,7 +289,7 @@ mod tests {
/// only ever over-reports uptime.
#[test]
fn the_rebuild_dedupes_by_latest_revision_not_by_max() {
let sql = reconcile_sql("slo1", 3, 0, 900);
let sql = reconcile_sql("slo1", 3, 0, 900, false);
assert!(sql.contains("ROW_NUMBER() OVER"), "{sql}");
assert!(sql.contains("ORDER BY rev DESC"), "{sql}");
assert!(sql.contains("zo_rn = 1"), "{sql}");
@@ -190,14 +304,14 @@ mod tests {
/// consistency does not repair.
#[test]
fn the_rebuild_is_scoped_to_one_generation() {
let sql = reconcile_sql("slo1", 3, 0, 900);
let sql = reconcile_sql("slo1", 3, 0, 900, false);
assert!(sql.contains("definition_generation = 3"), "{sql}");
assert!(sql.contains("slo_id = 'slo1'"), "{sql}");
}
#[test]
fn the_rebuild_is_bounded_by_the_read_window() {
let sql = reconcile_sql("slo1", 1, 100, 900);
let sql = reconcile_sql("slo1", 1, 100, 900, false);
assert!(sql.contains("slice_start >= 100"), "{sql}");
assert!(sql.contains("slice_start < 900"), "{sql}");
}
@@ -206,13 +320,65 @@ mod tests {
/// the whole window to one slice per group.
#[test]
fn dedupe_partitions_by_group_and_slice_together() {
let sql = reconcile_sql("slo1", 1, 0, 900);
let sql = reconcile_sql("slo1", 1, 0, 900, false);
assert!(sql.contains("PARTITION BY group_key, slice_start"), "{sql}");
}
#[test]
fn a_withdrawn_correction_does_not_count_as_covered_for_a_time_slice_sli() {
let sql = reconcile_sql("slo1", 1, 0, 900, true);
assert!(
sql.contains("SUM(CASE WHEN (total > 0 OR corrected_by <> '') THEN 1 ELSE 0 END)"),
"{sql}"
);
}
/// A count SLI's `0 / 0` is a quiet bucket; a stream without `corrected_by` has no tombstone.
#[test]
fn without_tombstones_every_slice_counts_as_covered() {
let sql = reconcile_sql("slo1", 1, 0, 900, false);
assert!(sql.contains("COUNT(*) AS covered_slices"), "{sql}");
assert!(
!sql.contains("corrected_by"),
"an old stream lacks the column: {sql}"
);
}
#[test]
fn a_stream_from_before_corrections_has_no_corrected_by() {
use arrow_schema::{DataType, Field, Schema};
let old = Schema::new(vec![
Field::new("slice_start", DataType::Int64, false),
Field::new("good", DataType::Float64, false),
]);
assert!(!has_corrected_by(&old));
let new = Schema::new(vec![Field::new("corrected_by", DataType::Utf8, true)]);
assert!(has_corrected_by(&new));
assert!(
!has_corrected_by(&Schema::empty()),
"a stream not yet created"
);
}
/// Pages stop only on an empty page, so a server cap below the asked size loses nothing.
#[tokio::test]
async fn paging_reads_every_hit_past_the_first_100k() {
let total = 250_001_usize;
let all = collect_pages(|offset| async move {
let offset = offset as usize;
let end = (offset + 7_000).min(total);
Ok((offset..end).map(|i| json::json!({ "i": i })).collect())
})
.await
.unwrap();
assert_eq!(all.len(), total);
assert_eq!(all[total - 1]["i"], total - 1);
}
#[test]
fn the_rebuild_groups_by_group_key() {
let sql = reconcile_sql("slo1", 1, 0, 900);
let sql = reconcile_sql("slo1", 1, 0, 900, false);
assert!(sql.contains("GROUP BY group_key"), "{sql}");
assert!(sql.contains("COUNT(*) AS covered_slices"), "{sql}");
}
src/core/src/slo/service.rs
+52 -11
View File
@@ -1060,6 +1060,7 @@ pub async fn group_status(
cfg.slo.min_coverage,
now_secs,
cfg.slo.recompute_slices.max(1),
Vec::new(),
)
})
.collect())
@@ -1086,6 +1087,7 @@ async fn rollup_view(
cfg.slo.min_coverage,
now_secs,
cfg.slo.recompute_slices.max(1),
correction_refs(slo).await,
);
// Only the rollup carries this, like the watermark: it describes the SLO's
// whole window, not one group's slice of it.
@@ -1095,6 +1097,15 @@ async fn rollup_view(
)))
}
/// The active and scheduled downtimes that correct this SLO, for its detail page (WP11).
async fn correction_refs(slo: &Slo) -> Vec<config::meta::downtimes::CorrectionRef> {
if db::downtimes::list_cached(&slo.org).is_empty() {
return Vec::new();
}
let dims = crate::alerts::downtimes::dimensions_for_slo(slo).await;
crate::alerts::downtimes::corrections_refs_for_slo(slo, &dims, now_micros())
}
/// The earliest instant this SLO's window is actually measured from, epoch
/// seconds — the clamp PR 4's backfill was queued under, read back.
///
@@ -1102,13 +1113,30 @@ async fn rollup_view(
/// range that was really filled, so the banner cannot disagree with the data.
/// `None` for every non-alert SLI, and for an alert SLI whose job row is gone —
/// in both cases there is nothing to explain.
async fn measurement_floor(db: &sea_orm::DatabaseConnection, slo: &Slo) -> Option<i64> {
pub(crate) async fn measurement_floor(db: &sea_orm::DatabaseConnection, slo: &Slo) -> Option<i64> {
source_alert_id(slo)?;
backfill_jobs::get(db, &slo.id, slo.definition_generation)
let job = backfill_jobs::get(db, &slo.id, slo.definition_generation)
.await
.ok()
.flatten()
.map(|job| job.range_start)
.flatten()?;
if job.kind == backfill_jobs::KIND_BACKFILL {
return Some(job.range_start);
}
// A downtime re-measure reuses the row and its range, so the clamp is computed again.
let reset_time = slos_table::generation_reset_time(db, &slo.id)
.await
.ok()
.flatten()?;
let (start, end) = super::backfill::backfill_range(
slo.definition.window_secs,
reset_time,
slo.definition.slice_interval_secs,
);
Some(alert_source_floor(db, slo).await.clamp_start(
start,
end,
slo.definition.slice_interval_secs,
))
}
fn view_of(
@@ -1117,13 +1145,14 @@ fn view_of(
coverage_floor: f64,
now_secs: i64,
stale_k: i64,
corrections: Vec<config::meta::downtimes::CorrectionRef>,
) -> config::meta::slo::SloStatusView {
let expected = config::meta::slo::window::expected_slices(
0,
slo.definition.window_secs,
slo.definition.slice_interval_secs,
);
let view = config::meta::slo::SloStatusView::derive(
let mut view = config::meta::slo::SloStatusView::derive(
row.group_key.clone(),
row.good,
row.total,
@@ -1134,6 +1163,7 @@ fn view_of(
coverage_floor,
row.computed_at,
);
view.corrections = corrections;
// The watermark lives ONLY on the rollup row (`apply_status_in_txn`), so a
// group row's absent watermark says nothing about staleness — reporting it
// as stale would flag every group of every healthy SLO.
@@ -1268,6 +1298,17 @@ async fn alert_source_floor(db: &sea_orm::DatabaseConnection, slo: &Slo) -> Sour
}
}
/// Wakes the backfill lane for a downtime re-measure (WP11).
pub(crate) async fn push_backfill_trigger(slo: &Slo) {
push_trigger(
&slo.org,
&slo.id,
crate::db::scheduler::TriggerModule::SloBackfill,
now_micros(),
)
.await;
}
/// Add or remove the ingest trigger to match `enabled`.
async fn sync_ingest_trigger(slo: &Slo) {
let module = crate::db::scheduler::TriggerModule::Slo;
@@ -2836,7 +2877,7 @@ mod tests {
let slo = alert_slo();
// K=3 at 300s slices = 900s of tolerance; this is 4100s past it.
let row = covered_row(Some(100_000));
let view = view_of(&slo, &row, 0.9, 105_000, 3);
let view = view_of(&slo, &row, 0.9, 105_000, 3, Vec::new());
assert!(
!view.no_data,
"coverage is full, so the floor is not tripped"
@@ -2849,7 +2890,7 @@ mod tests {
fn a_moving_watermark_is_not_a_freeze() {
let slo = alert_slo();
let row = covered_row(Some(100_000));
let view = view_of(&slo, &row, 0.9, 100_600, 3);
let view = view_of(&slo, &row, 0.9, 100_600, 3, Vec::new());
assert!(!view.stale_watermark);
assert!(!view.no_data);
}
@@ -2859,7 +2900,7 @@ mod tests {
#[test]
fn an_absent_watermark_reads_as_stale() {
let slo = alert_slo();
let view = view_of(&slo, &covered_row(None), 0.9, 105_000, 3);
let view = view_of(&slo, &covered_row(None), 0.9, 105_000, 3, Vec::new());
assert!(view.stale_watermark);
assert_eq!(view.watermark_end, None);
}
@@ -2872,7 +2913,7 @@ mod tests {
let slo = alert_slo();
let mut row = covered_row(Some(100_000));
row.covered_slices = Some(1);
let view = view_of(&slo, &row, 0.9, 100_600, 3);
let view = view_of(&slo, &row, 0.9, 100_600, 3, Vec::new());
assert!(view.no_data);
assert!(!view.stale_watermark);
}
@@ -2885,7 +2926,7 @@ mod tests {
let slo = alert_slo();
let mut row = covered_row(None);
row.group_key = "region=eu".into();
let view = view_of(&slo, &row, 0.9, 105_000, 3);
let view = view_of(&slo, &row, 0.9, 105_000, 3, Vec::new());
assert!(!view.stale_watermark);
assert_eq!(view.watermark_end, None);
}
@@ -2898,7 +2939,7 @@ mod tests {
let slo = alert_slo();
let mut row = covered_row(Some(100_000));
row.covered_slices = Some(1);
let view = view_of(&slo, &row, 0.9, 105_000, 3);
let view = view_of(&slo, &row, 0.9, 105_000, 3, Vec::new());
assert!(view.no_data);
assert!(view.stale_watermark);
}
src/core/src/synthetics.rs
+285 -40
View File
@@ -472,6 +472,112 @@ pub async fn create_default_probe_token(
Ok(())
}
/// The synthetics mute check over the downtimes cache, for `register_mute_check`.
pub fn downtime_mute_check(
org: &str,
check_id: &str,
folder_id: &str,
tags: &[String],
) -> openobserve_synthetics::alerting::BoxFuture<'static, Option<String>> {
let (org, check_id, folder_id, tags) = (
org.to_string(),
check_id.to_string(),
folder_id.to_string(),
tags.to_vec(),
);
Box::pin(async move {
if !crate::alerts::downtimes::any_for(
&org,
config::meta::downtimes::TargetModule::Synthetics,
) {
return None;
}
let folder_id = public_folder_id(&folder_id).await;
crate::alerts::downtimes::active_for_synthetic(
&org,
&check_id,
&folder_id,
&tags,
config::utils::time::now_micros(),
)
.map(|downtime| downtime.id)
})
}
/// Whether the org can mute a check now, from the downtime cache, for `register_mute_check`.
pub fn downtime_may_mute(org: &str) -> bool {
crate::alerts::downtimes::any_for(org, config::meta::downtimes::TargetModule::Synthetics)
}
/// Sets the Muted chip on each listed check a downtime silences now.
#[cfg(feature = "enterprise")]
pub fn fill_active_downtimes(
org_id: &str,
items: &mut [config::meta::synthetics::SyntheticListItem],
) {
if !crate::alerts::downtimes::any_for(org_id, config::meta::downtimes::TargetModule::Synthetics)
{
return;
}
let now = config::utils::time::now_micros();
for item in items.iter_mut() {
item.active_downtime = crate::alerts::downtimes::active_for_synthetic(
org_id,
&item.id,
&item.folder_id,
&item.tags,
now,
);
}
}
#[cfg(not(feature = "enterprise"))]
pub fn fill_active_downtimes(
_org_id: &str,
_items: &mut [config::meta::synthetics::SyntheticListItem],
) {
}
/// Stored checks hold the folder primary key, while downtime targets name the public id.
async fn public_folder_id(folder_id: &str) -> String {
infra::table::folders::get_name_by_pk(folder_id)
.await
.ok()
.flatten()
.unwrap_or_else(|| folder_id.to_string())
}
/// Destinations of the unmuted checks; without `may_mute` all count and no folder is read.
#[cfg(feature = "enterprise")]
async fn unmuted_destinations(
org_id: &str,
checks: &[config::meta::synthetics::Synthetic],
may_mute: bool,
) -> Vec<String> {
let now = config::utils::time::now_micros();
let mut destinations = Vec::new();
for check in checks {
if may_mute {
let folder_id = public_folder_id(&check.folder_id).await;
if crate::alerts::downtimes::active_for_synthetic(
org_id,
&check.id,
&folder_id,
&check.tags,
now,
)
.is_some()
{
continue;
}
}
destinations.extend(check.destinations.iter().cloned());
}
destinations.sort();
destinations.dedup();
destinations
}
// ── Private-location staleness watcher ────────────────────────────────────────
/// Ticks every 60s on scheduler nodes. A private location whose registered
@@ -521,54 +627,78 @@ pub async fn location_staleness_watcher() {
// Nothing runs here — stay quiet, re-evaluate next tick.
continue;
}
// Claim before dispatch so a location without destinations is still
// one-shot (no per-tick log spam / retry storm), AND so that only one
// scheduler node speaks. This watcher runs on every scheduler node, so
// the suppression flag cannot live in this process's memory — N nodes
// would each believe they had not notified yet and send N pages for
// one outage. The CAS in `try_claim_down_notification` makes exactly
// one node the winner.
match infra::table::synthetics_locations::try_claim_down_notification(&loc.id, now)
.await
{
Ok(true) => {}
Ok(false) => continue, // another node is sending it
Err(e) => {
log::error!(
"[synthetics] staleness watcher: claim down notification for {}: {e}",
loc.id
);
continue;
}
let may_mute = downtime_may_mute(&org_id);
let destinations = unmuted_destinations(&org_id, &checks, may_mute).await;
if may_mute && destinations.is_empty() {
log::debug!(
"[synthetics] private location down: {} ({}) org={} affected_checks={}, every check muted",
loc.label,
loc.id,
org_id,
checks.len()
);
}
let mut destinations: Vec<String> =
checks.iter().flat_map(|c| c.destinations.clone()).collect();
destinations.sort();
destinations.dedup();
log::warn!(
"[synthetics] private location down: {} ({}) org={} affected_checks={} destinations={}",
loc.label,
loc.id,
org_id,
checks.len(),
destinations.len()
);
if destinations.is_empty() {
continue;
}
notify_location_down(
&org_id,
&loc,
checks.len(),
window_us / 1_000_000,
report_location_down(
&loc.id,
&destinations,
may_mute,
async || {
infra::table::synthetics_locations::try_claim_down_notification(&loc.id, now)
.await
},
async |destinations| {
log::warn!(
"[synthetics] private location down: {} ({}) org={} affected_checks={} destinations={}",
loc.label,
loc.id,
org_id,
checks.len(),
destinations.len()
);
if !destinations.is_empty() {
notify_location_down(
&org_id,
&loc,
checks.len(),
window_us / 1_000_000,
destinations,
)
.await
}
},
)
.await;
}
}
}
/// One notification per outage from the CAS winner; only a mute emptying the list keeps the claim.
#[cfg(feature = "enterprise")]
async fn report_location_down(
loc_id: &str,
destinations: &[String],
may_mute: bool,
claim: impl AsyncFnOnce() -> Result<bool, infra::errors::Error>,
notify: impl AsyncFnOnce(&[String]),
) -> bool {
if may_mute && destinations.is_empty() {
return false;
}
match claim().await {
Ok(true) => {}
Ok(false) => return false,
Err(e) => {
log::error!(
"[synthetics] staleness watcher: claim down notification for {loc_id}: {e}"
);
return false;
}
}
// The winner reports even an empty list, so the outage is logged once as on main.
notify(destinations).await;
!destinations.is_empty()
}
#[cfg(all(test, feature = "enterprise"))]
mod tests {
use super::*;
@@ -769,6 +899,121 @@ mod tests {
assert!(line.starts_with("2 of 3: "), "{line}");
assert!(!line.contains("aws-eu-central-1"), "{line}");
}
async fn locations_db() -> sea_orm::DatabaseConnection {
use infra::table::entity::synthetics_locations;
use sea_orm::{ConnectionTrait, Database, EntityTrait, Schema, Set};
let db = Database::connect("sqlite::memory:").await.unwrap();
let backend = db.get_database_backend();
let table = Schema::new(backend).create_table_from_entity(synthetics_locations::Entity);
db.execute(backend.build(&table)).await.unwrap();
synthetics_locations::Entity::insert(synthetics_locations::ActiveModel {
id: Set("loc-1".to_string()),
org_id: Set(Some("org1".to_string())),
kind: Set("private".to_string()),
provider: Set("custom".to_string()),
region: Set("corp-hq".to_string()),
label: Set("Corp HQ".to_string()),
pool: Set("private-org1-corp-hq".to_string()),
enabled: Set(true),
down_notified_at: Set(0),
created_at: Set(1),
updated_at: Set(1),
})
.exec(&db)
.await
.unwrap();
db
}
async fn down_notified_at(db: &sea_orm::DatabaseConnection) -> i64 {
use sea_orm::EntityTrait;
infra::table::entity::synthetics_locations::Entity::find_by_id("loc-1")
.one(db)
.await
.unwrap()
.unwrap()
.down_notified_at
}
/// One watcher tick for loc-1, with the destinations its unmuted checks left.
async fn tick(
db: &sea_orm::DatabaseConnection,
destinations: &[String],
now: i64,
sent: &std::sync::atomic::AtomicUsize,
) -> bool {
tick_with(db, destinations, true, now, sent).await
}
async fn tick_with(
db: &sea_orm::DatabaseConnection,
destinations: &[String],
may_mute: bool,
now: i64,
sent: &std::sync::atomic::AtomicUsize,
) -> bool {
report_location_down(
"loc-1",
destinations,
may_mute,
async || {
infra::table::synthetics_locations::try_claim_down_notification_with(
db, "loc-1", now,
)
.await
},
async |destinations| {
if !destinations.is_empty() {
sent.fetch_add(1, std::sync::atomic::Ordering::SeqCst);
}
},
)
.await
}
#[tokio::test]
async fn a_location_whose_every_check_is_muted_keeps_its_down_claim() {
let db = locations_db().await;
let sent = std::sync::atomic::AtomicUsize::new(0);
assert!(!tick(&db, &[], 100, &sent).await);
assert_eq!(
down_notified_at(&db).await,
0,
"a muted tick must not use up the claim"
);
assert_eq!(sent.load(std::sync::atomic::Ordering::SeqCst), 0);
}
#[tokio::test]
async fn with_no_downtime_a_location_without_destinations_uses_its_claim_as_before() {
let db = locations_db().await;
let sent = std::sync::atomic::AtomicUsize::new(0);
assert!(!tick_with(&db, &[], false, 100, &sent).await);
assert_eq!(
down_notified_at(&db).await,
100,
"the claim is one-shot on main"
);
assert!(!tick_with(&db, &["pager".to_string()], false, 200, &sent).await);
assert_eq!(sent.load(std::sync::atomic::Ordering::SeqCst), 0);
}
#[tokio::test]
async fn the_first_tick_after_the_mute_claims_and_notifies_once() {
let db = locations_db().await;
let sent = std::sync::atomic::AtomicUsize::new(0);
let destinations = ["pager".to_string()];
assert!(!tick(&db, &[], 100, &sent).await);
assert!(tick(&db, &destinations, 200, &sent).await);
assert!(!tick(&db, &destinations, 300, &sent).await);
assert_eq!(down_notified_at(&db).await, 200);
assert_eq!(sent.load(std::sync::atomic::Ordering::SeqCst), 1);
}
}
/// Sends the "location down" notification to each destination, matching the
src/db/src/alerts/alert.rs
+12 -4
View File
@@ -63,11 +63,19 @@ pub async fn get_by_name(
stream_name: &str,
name: &str,
) -> Result<Option<Alert>, infra::errors::Error> {
let alert = get_by_name_with_folder(org_id, stream_type, stream_name, name).await?;
Ok(alert.map(|(_f, a)| a))
}
/// [get_by_name] with the alert's parent folder, read from the table.
pub async fn get_by_name_with_folder(
org_id: &str,
stream_type: StreamType,
stream_name: &str,
name: &str,
) -> Result<Option<(Folder, Alert)>, infra::errors::Error> {
let client = get_orm_client_ro().await;
let alert =
table::get_by_name(client, org_id, "default", stream_type, stream_name, name).await?;
let value = alert.map(|(_f, a)| a);
Ok(value)
table::get_by_name(client, org_id, "default", stream_type, stream_name, name).await
}
pub async fn set(org_id: &str, alert: Alert, create: bool) -> Result<Alert, infra::errors::Error> {
src/db/src/authz.rs
+19
View File
@@ -47,6 +47,25 @@ pub async fn set_ownership(org_id: &str, obj_type: &str, obj: Authz) {
#[cfg(not(feature = "enterprise"))]
pub async fn set_ownership(_org_id: &str, _obj_type: &str, _obj: Authz) {}
/// The tuples `set_ownership` would write for `obj`, for a caller that batches many objects
/// into one write. The folder existence check of `set_ownership` is not repeated here.
#[cfg(feature = "enterprise")]
pub fn ownership_tuples(
org_id: &str,
obj_type: &str,
obj: &Authz,
) -> Vec<o2_openfga::authorizer::authz::TupleKey> {
use o2_openfga::{authorizer, meta::mapping::OFGA_MODELS};
let obj_str = format!("{}:{}", OFGA_MODELS.get(obj_type).unwrap().key, obj.obj_id);
let parent_type = if obj.parent_type.is_empty() {
""
} else {
OFGA_MODELS.get(obj.parent_type.as_str()).unwrap().key
};
authorizer::authz::ownership_tuples(org_id, &obj_str, &obj.parent, parent_type)
}
#[cfg(feature = "enterprise")]
pub async fn remove_ownership(org_id: &str, obj_type: &str, obj: Authz) {
if o2_openfga::config::get_config().enabled {
src/db/src/downtimes.rs
+486
View File
@@ -0,0 +1,486 @@
// Copyright 2026 OpenObserve Inc.
//
// This program is free software: you can redistribute it and/or modify
// it under the terms of the GNU Affero General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
//
// This program is distributed in the hope that it will be useful
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU Affero General Public License for more details.
//
// You should have received a copy of the GNU Affero General Public License
// along with this program. If not, see <http://www.gnu.org/licenses/>.
//! Downtime rows and the per-node cache every send point reads synchronously.
use std::{
collections::HashMap,
sync::{Arc, LazyLock, RwLock},
};
use common::meta::authz::Authz;
use config::meta::downtimes::Downtime;
use infra::{
coordinator::downtimes as coordinator,
table::{
downtimes::{self as table, Guard},
slo_backfill_jobs::Remeasure,
},
};
use sea_orm::ConnectionTrait;
#[cfg(feature = "enterprise")]
pub use super_cluster::replicates;
/// `org -> rows`. Replaced whole per org, so a reader never sees a partial org.
static DOWNTIMES: LazyLock<RwLock<HashMap<String, Arc<Vec<Downtime>>>>> =
LazyLock::new(Default::default);
/// Writes the row with its SLO re-measures, tells this region's nodes, then the other regions.
pub async fn set(downtime: &Downtime, remeasures: &[Remeasure]) -> Result<(), anyhow::Error> {
write(downtime, Guard::None, remeasures).await?;
reload_org(&downtime.org).await?;
Ok(())
}
/// [set] for an edit: writes only if the stored row still has `expected_updated_at`.
pub async fn set_if_unchanged(
downtime: &Downtime,
expected_updated_at: i64,
remeasures: &[Remeasure],
) -> Result<bool, anyhow::Error> {
if !write(downtime, Guard::Unchanged(expected_updated_at), remeasures).await? {
return Ok(false);
}
reload_org(&downtime.org).await?;
Ok(true)
}
/// [set_if_unchanged] without this node's reload, for a batch that calls [reload_org] once after.
pub async fn write_if_unchanged(
downtime: &Downtime,
expected_updated_at: i64,
) -> Result<bool, anyhow::Error> {
write(downtime, Guard::Unchanged(expected_updated_at), &[]).await
}
/// [set] for a follow-up row, written only while `parent_id` still has `expected_updated_at`.
pub async fn set_if_parent_unchanged(
downtime: &Downtime,
parent_id: &str,
expected_updated_at: i64,
remeasures: &[Remeasure],
) -> Result<bool, anyhow::Error> {
let guard = Guard::ParentUnchanged {
parent_id,
updated_at: expected_updated_at,
};
if !write(downtime, guard, remeasures).await? {
return Ok(false);
}
reload_org(&downtime.org).await?;
Ok(true)
}
/// [set_if_unchanged] for a live follow-up, with its parent bumped in the same transaction.
pub async fn set_follow_up_if_unchanged(
downtime: &Downtime,
expected_updated_at: i64,
parent_id: &str,
parent_updated_at: i64,
remeasures: &[Remeasure],
) -> Result<bool, anyhow::Error> {
let guard = Guard::FollowUpUnchanged {
updated_at: expected_updated_at,
parent_id,
parent_updated_at,
};
if !write(downtime, guard, remeasures).await? {
return Ok(false);
}
reload_org(&downtime.org).await?;
replicate(&downtime.org, parent_id).await;
Ok(true)
}
/// Sends the stored row to the other regions, for a write that changed it as a side effect.
pub async fn replicate(org: &str, id: &str) {
#[cfg(feature = "enterprise")]
super_cluster::emit_stored(org, id).await;
#[cfg(not(feature = "enterprise"))]
let _ = (org, id);
}
/// Deletes the row only while it is still at `stored`; false when another write moved it first.
pub async fn delete(org: &str, id: &str, stored: table::RowVersion) -> Result<bool, anyhow::Error> {
let Some(tombstone) = table::delete_if_stored(org, id, stored).await? else {
return Ok(false);
};
coordinator::emit_delete_event(org, id).await?;
remove_cached(org, id);
// Other regions order by the tombstone's own version and time, so both travel as written.
#[cfg(feature = "enterprise")]
super_cluster::emit_delete(org, id, tombstone.version, tombstone.updated_at).await;
#[cfg(not(feature = "enterprise"))]
let _ = tombstone;
Ok(true)
}
/// Org deletion: every row, every node cache, every region and every OpenFGA tuple.
pub async fn delete_by_org(org: &str) -> Result<(), anyhow::Error> {
let rows = table::list(org, None).await?;
let ids = table::delete_by_org(org).await?;
for id in &ids {
coordinator::emit_delete_event(org, id).await?;
// Org deletion removes the rows outright, so no region may keep a tombstone for them.
#[cfg(feature = "enterprise")]
super_cluster::emit_delete(org, id, 0, 0).await;
}
remove_org(org);
for row in &rows {
crate::authz::remove_ownership(org, "downtimes", ownership(row)).await;
}
Ok(())
}
/// Retention sweep for this region only; soft-deleted rows lost their tuple at delete time.
pub async fn delete_ended_before(cutoff: i64) -> Result<u64, anyhow::Error> {
let ended = table::list_ended_before(cutoff).await?;
let removed = table::delete_ended_before(cutoff).await?;
// The listed rows, not this node's cache, carry the folder each OpenFGA tuple hangs off.
for row in &ended {
coordinator::emit_delete_event(&row.org, &row.id).await?;
crate::authz::remove_ownership(&row.org, "downtimes", ownership(row)).await;
remove_cached(&row.org, &row.id);
}
Ok(removed)
}
/// Loads every row of every org, replacing the cache.
pub async fn cache() -> Result<(), anyhow::Error> {
let mut by_org: HashMap<String, Vec<Downtime>> = HashMap::new();
for downtime in table::list_all().await? {
by_org
.entry(downtime.org.clone())
.or_default()
.push(downtime);
}
let fresh = by_org
.into_iter()
.map(|(org, rows)| (org, Arc::new(rows)))
.collect();
*DOWNTIMES.write().unwrap_or_else(|e| e.into_inner()) = fresh;
Ok(())
}
pub async fn watch() -> Result<(), anyhow::Error> {
coordinator::watch_events(on_put, on_delete).await
}
#[cfg(feature = "enterprise")]
pub async fn backfill_ownership() -> Result<usize, anyhow::Error> {
let tuples = ownership_backfill(&all_cached());
let writes: Vec<_> = tuples
.iter()
.flat_map(|(org, authz)| crate::authz::ownership_tuples(org, "downtimes", authz))
.collect();
if !writes.is_empty() {
o2_openfga::authorizer::authz::update_tuples(writes, vec![]).await?;
}
Ok(tuples.len())
}
#[cfg(not(feature = "enterprise"))]
pub async fn backfill_ownership() -> Result<usize, anyhow::Error> {
Ok(ownership_backfill(&all_cached()).len())
}
/// The cached rows of an org; cheap to clone, never partial.
pub fn list_cached(org: &str) -> Arc<Vec<Downtime>> {
DOWNTIMES
.read()
.unwrap_or_else(|e| e.into_inner())
.get(org)
.cloned()
.unwrap_or_default()
}
pub fn all_cached() -> Vec<Arc<Vec<Downtime>>> {
DOWNTIMES
.read()
.unwrap_or_else(|e| e.into_inner())
.values()
.cloned()
.collect()
}
/// The OpenFGA object of a downtime, parented by its folder so folder grants reach it.
pub fn ownership(downtime: &Downtime) -> Authz {
Authz {
obj_id: downtime.id.clone(),
parent_type: "downtime_folders".to_string(),
parent: downtime.folder_id.clone(),
}
}
/// Reloads one org from the table so the cache never holds half a row.
pub async fn reload_org(org: &str) -> Result<(), anyhow::Error> {
let rows = table::list(org, None).await?;
replace_org(org, rows);
Ok(())
}
/// [reload_org] through `conn`, so a caller can read the primary instead of a replica.
pub async fn reload_org_with<C: ConnectionTrait>(conn: &C, org: &str) -> Result<(), anyhow::Error> {
let rows = table::list_with(conn, org, None).await?;
replace_org(org, rows);
Ok(())
}
/// The table write, then this region's nodes, then the other regions; false if `guard` failed.
async fn write(
downtime: &Downtime,
guard: Guard<'_>,
remeasures: &[Remeasure],
) -> Result<bool, anyhow::Error> {
if !table::write(downtime, guard, remeasures).await? {
return Ok(false);
}
coordinator::emit_put_event(&downtime.org, &downtime.id).await?;
#[cfg(feature = "enterprise")]
super_cluster::emit_put(downtime).await;
Ok(true)
}
async fn on_put(org: String, _id: String) -> Result<(), anyhow::Error> {
reload_org(&org).await
}
async fn on_delete(org: String, id: String) -> Result<(), anyhow::Error> {
remove_cached(&org, &id);
Ok(())
}
/// `(org, tuple)` for every row, each parented by the folder it is filed in.
fn ownership_backfill(orgs: &[Arc<Vec<Downtime>>]) -> Vec<(String, Authz)> {
orgs.iter()
.flat_map(|rows| rows.iter())
.map(|row| (row.org.clone(), ownership(row)))
.collect()
}
fn replace_org(org: &str, rows: Vec<Downtime>) {
let mut cache = DOWNTIMES.write().unwrap_or_else(|e| e.into_inner());
if rows.is_empty() {
cache.remove(org);
} else {
cache.insert(org.to_string(), Arc::new(rows));
}
}
fn remove_cached(org: &str, id: &str) {
let mut cache = DOWNTIMES.write().unwrap_or_else(|e| e.into_inner());
let Some(rows) = cache.get(org) else {
return;
};
if !rows.iter().any(|d| d.id == id) {
return;
}
let kept: Vec<Downtime> = rows.iter().filter(|d| d.id != id).cloned().collect();
if kept.is_empty() {
cache.remove(org);
} else {
cache.insert(org.to_string(), Arc::new(kept));
}
}
fn remove_org(org: &str) {
DOWNTIMES
.write()
.unwrap_or_else(|e| e.into_inner())
.remove(org);
}
/// Best effort: the local write happened, and the next edit repairs a lost message (`put` upserts).
#[cfg(feature = "enterprise")]
mod super_cluster {
use config::{get_config, meta::downtimes::Downtime};
use o2_enterprise::enterprise::{
common::config::{O2Config, get_config as get_o2_config},
super_cluster::queue,
};
/// A region without downtimes cannot read bytes 93 and 94, so sending waits for its own switch.
pub fn replicates() -> bool {
replicates_with(&get_o2_config(), get_config().common.local_mode)
}
pub(super) async fn emit_put(downtime: &Downtime) {
if !replicates() {
return;
}
if let Err(e) = queue::downtimes_put(&downtime.org, downtime).await {
log::error!(
"[DOWNTIMES] super cluster put {}/{} failed: {e}",
downtime.org,
downtime.id
);
}
}
pub(super) async fn emit_stored(org: &str, id: &str) {
if !replicates() {
return;
}
match super::table::get_rw(org, id).await {
Ok(Some(row)) => emit_put(&row).await,
Ok(None) => {}
Err(e) => log::error!("[DOWNTIMES] super cluster put {org}/{id} failed: {e}"),
}
}
pub(super) async fn emit_delete(org: &str, id: &str, version: i64, deleted_at: i64) {
if !replicates() {
return;
}
if let Err(e) = queue::downtimes_delete(org, id, version, deleted_at).await {
log::error!("[DOWNTIMES] super cluster delete {org}/{id} failed: {e}");
}
}
pub(super) fn replicates_with(o2: &O2Config, local_mode: bool) -> bool {
o2.super_cluster.enabled
&& o2.downtimes.enabled
&& o2.downtimes.replication_enabled
&& !local_mode
}
}
#[cfg(test)]
mod tests {
use config::meta::downtimes::{DowntimeSchedule, Repeat};
use super::*;
fn downtime(org: &str, id: &str) -> Downtime {
Downtime {
id: id.to_string(),
org: org.to_string(),
folder_id: "default".to_string(),
name: id.to_string(),
reason: None,
condition: None,
targets: vec![],
schedule: DowntimeSchedule {
repeat: Repeat::None,
starts_at: 1,
ends_at: Some(2),
timezone: "UTC".to_string(),
start_time_local: None,
duration_secs: 60,
weekdays: vec![],
},
cancelled_at: None,
cancelled_by: None,
show_banner: true,
notifications: None,
origin_region: None,
version: 0,
created_by: "lin".to_string(),
created_at: 1,
updated_by: "lin".to_string(),
updated_at: 1,
}
}
fn ids(org: &str) -> Vec<String> {
list_cached(org).iter().map(|d| d.id.clone()).collect()
}
#[test]
fn a_reload_replaces_the_org_list_and_a_delete_removes_one_id() {
let org = "db_downtimes_cache_test";
replace_org(org, vec![downtime(org, "a"), downtime(org, "b")]);
assert_eq!(ids(org), ["a", "b"]);
replace_org(org, vec![downtime(org, "c")]);
assert_eq!(ids(org), ["c"], "a reload replaces, never merges");
replace_org(org, vec![downtime(org, "c"), downtime(org, "d")]);
remove_cached(org, "c");
assert_eq!(ids(org), ["d"]);
remove_cached(org, "missing");
assert_eq!(ids(org), ["d"]);
remove_cached(org, "d");
assert!(list_cached(org).is_empty());
}
#[test]
fn remove_org_drops_every_row_of_that_org_only() {
let (a, b) = ("db_downtimes_org_a", "db_downtimes_org_b");
replace_org(a, vec![downtime(a, "x")]);
replace_org(b, vec![downtime(b, "y")]);
remove_org(a);
assert!(list_cached(a).is_empty());
assert_eq!(ids(b), ["y"]);
}
#[test]
fn ownership_parents_the_downtime_by_its_folder_public_id() {
let mut row = downtime("acme", "dt-1");
row.folder_id = "payments".to_string();
let owner = ownership(&row);
assert_eq!(owner.obj_id, "dt-1");
assert_eq!(owner.parent_type, "downtime_folders");
assert_eq!(owner.parent, "payments");
}
#[test]
fn the_ownership_backfill_covers_every_row_of_every_org_under_its_folder() {
let mut moved = downtime("acme", "dt-2");
moved.folder_id = "payments".to_string();
let orgs = vec![
Arc::new(vec![downtime("acme", "dt-1"), moved]),
Arc::new(vec![downtime("beta", "dt-3")]),
];
let tuples: Vec<(String, String, String)> = ownership_backfill(&orgs)
.into_iter()
.map(|(org, a)| (org, a.obj_id, a.parent))
.collect();
let expected = |org: &str, id: &str, folder: &str| {
(org.to_string(), id.to_string(), folder.to_string())
};
assert_eq!(
tuples,
vec![
expected("acme", "dt-1", "default"),
expected("acme", "dt-2", "payments"),
expected("beta", "dt-3", "default"),
]
);
assert!(ownership_backfill(&[]).is_empty());
}
#[cfg(feature = "enterprise")]
#[test]
fn a_super_cluster_replicates_downtimes_only_with_its_own_switch_on() {
let mut o2 = o2_enterprise::enterprise::common::config::O2Config::init().unwrap();
o2.super_cluster.enabled = true;
o2.downtimes.enabled = true;
o2.downtimes.replication_enabled = false;
assert!(
!super_cluster::replicates_with(&o2, false),
"a put, a delete, an un-mute and a downtime folder publish nothing"
);
o2.downtimes.replication_enabled = true;
assert!(super_cluster::replicates_with(&o2, false));
assert!(!super_cluster::replicates_with(&o2, true));
o2.downtimes.enabled = false;
assert!(!super_cluster::replicates_with(&o2, false));
o2.downtimes.enabled = true;
o2.super_cluster.enabled = false;
assert!(!super_cluster::replicates_with(&o2, false));
}
}
src/db/src/folders.rs
+31 -16
View File
@@ -80,6 +80,10 @@ pub enum FolderError {
#[error("Folder contains prompts. Please move/delete prompts from folder.")]
DeleteWithPrompts,
/// An error that occurs when trying to delete a folder that contains downtimes.
#[error("Folder contains downtimes. Please move/delete downtimes from folder.")]
DeleteWithDowntimes,
/// An error that occurs when trying to delete a folder that cannot be found.
#[error("Folder not found")]
NotFound,
@@ -128,10 +132,7 @@ pub async fn save_folder(
set_ownership(org_id, folder_type_ofga, Authz::new(&folder.folder_id)).await;
#[cfg(feature = "enterprise")]
if o2_enterprise::enterprise::common::config::get_config()
.super_cluster
.enabled
{
if publishes(folder_type) {
let _ = o2_enterprise::enterprise::super_cluster::queue::folders_create(
org_id,
_id,
@@ -177,10 +178,7 @@ pub async fn ensure_default_folder(
.await;
#[cfg(feature = "enterprise")]
if o2_enterprise::enterprise::common::config::get_config()
.super_cluster
.enabled
{
if publishes(folder_type) {
let _ = o2_enterprise::enterprise::super_cluster::queue::folders_create(
org_id,
_id,
@@ -215,10 +213,7 @@ pub async fn update_folder(
let (_, folder) = table::folders::put(org_id, None, folder, folder_type).await?;
#[cfg(feature = "enterprise")]
if o2_enterprise::enterprise::common::config::get_config()
.super_cluster
.enabled
{
if publishes(folder_type) {
let _ = o2_enterprise::enterprise::super_cluster::queue::folders_update(
org_id,
folder_id,
@@ -247,6 +242,7 @@ pub async fn list_folders(
FolderType::Reports => OFGA_MODELS.get("report_folders").unwrap().key,
FolderType::Synthetics => OFGA_MODELS.get("synthetic_folder").unwrap().key,
FolderType::Workflows => OFGA_MODELS.get("workflow_folder").unwrap().key,
FolderType::Downtimes => OFGA_MODELS.get("downtime_folders").unwrap().key,
FolderType::Prompts => OFGA_MODELS.get("prompt_folders").unwrap().key,
};
#[cfg(not(feature = "enterprise"))]
@@ -358,6 +354,14 @@ pub async fn delete_folder(
return Err(FolderError::DeleteWithWorkflows);
}
}
FolderType::Downtimes => {
if let Some(folder_pk) =
table::folders::get_pk_by_name(org_id, folder_id, folder_type).await?
&& !table::downtimes::release_folder(org_id, &folder_pk).await?
{
return Err(FolderError::DeleteWithDowntimes);
}
}
FolderType::Prompts => {
if let Some(folder_pk) =
table::folders::get_pk_by_name(org_id, folder_id, folder_type).await?
@@ -401,10 +405,7 @@ pub async fn delete_folder(
remove_ownership(org_id, folder_type_ofga, Authz::new(folder_id)).await;
#[cfg(feature = "enterprise")]
if o2_enterprise::enterprise::common::config::get_config()
.super_cluster
.enabled
{
if publishes(folder_type) {
let _ = o2_enterprise::enterprise::super_cluster::queue::folders_delete(
org_id,
folder_id,
@@ -416,6 +417,15 @@ pub async fn delete_folder(
Ok(())
}
/// A downtime folder travels with the rows filed in it, so it waits for downtime replication too.
#[cfg(feature = "enterprise")]
fn publishes(folder_type: FolderType) -> bool {
o2_enterprise::enterprise::common::config::get_config()
.super_cluster
.enabled
&& (folder_type != FolderType::Downtimes || crate::downtimes::replicates())
}
/// OpenFGA object type that owns folders of the given kind.
fn folder_type_ofga_name(folder_type: FolderType) -> &'static str {
match folder_type {
@@ -424,6 +434,7 @@ fn folder_type_ofga_name(folder_type: FolderType) -> &'static str {
FolderType::Reports => "report_folders",
FolderType::Synthetics => "synthetic_folder",
FolderType::Workflows => "workflow_folder",
FolderType::Downtimes => "downtime_folders",
FolderType::Prompts => "prompt_folders",
}
}
@@ -464,6 +475,10 @@ async fn permitted_folders(
OFGA_MODELS.get("workflow_folder").unwrap().key,
OFGA_MODELS.get("workflows").unwrap().key,
),
FolderType::Downtimes => (
OFGA_MODELS.get("downtime_folders").unwrap().key,
OFGA_MODELS.get("downtimes").unwrap().key,
),
// Prompt grants are plain `prompt:{id}` with no folder prefix, so there is no
// individual grant to lift a folder into view: only folder `GET` counts.
FolderType::Prompts => (OFGA_MODELS.get("prompt_folders").unwrap().key, ""),
src/db/src/lib.rs
+1
View File
@@ -29,6 +29,7 @@ pub mod compact;
pub mod dashboards;
pub mod db_monitoring;
pub mod distinct_values;
pub mod downtimes;
pub mod enrichment_table;
pub mod file_list;
pub mod folders;
src/infra/src/coordinator/downtimes.rs
+112
View File
@@ -0,0 +1,112 @@
// Copyright 2026 OpenObserve Inc.
//
// This program is free software: you can redistribute it and/or modify
// it under the terms of the GNU Affero General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
//
// This program is distributed in the hope that it will be useful
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU Affero General Public License for more details.
//
// You should have received a copy of the GNU Affero General Public License
// along with this program. If not, see <http://www.gnu.org/licenses/>.
use std::{future::Future, sync::Arc};
use crate::{db::Event, errors::Error};
pub const DOWNTIME_WATCHER_PREFIX: &str = "/downtimes/";
/// Tells every node that this downtime row changed; the payload is empty, nodes reload the org.
pub async fn emit_put_event(org: &str, id: &str) -> Result<(), Error> {
let cluster_coordinator = super::get_coordinator().await;
cluster_coordinator
.put(&downtime_key(org, id), bytes::Bytes::new(), true, None)
.await?;
Ok(())
}
pub async fn emit_delete_event(org: &str, id: &str) -> Result<(), Error> {
let cluster_coordinator = super::get_coordinator().await;
cluster_coordinator
.delete(&downtime_key(org, id), false, true, None)
.await
}
pub async fn watch_events<OnPut, OnPutFut, OnDelete, OnDeleteFut>(
on_put: OnPut,
on_delete: OnDelete,
) -> Result<(), anyhow::Error>
where
OnPut: Fn(String, String) -> OnPutFut,
OnPutFut: Future<Output = Result<(), anyhow::Error>>,
OnDelete: Fn(String, String) -> OnDeleteFut,
OnDeleteFut: Future<Output = Result<(), anyhow::Error>>,
{
let cluster_coordinator = super::get_coordinator().await;
let mut events = cluster_coordinator.watch(DOWNTIME_WATCHER_PREFIX).await?;
let events = Arc::get_mut(&mut events).unwrap();
log::info!("Start watching downtimes");
while let Some(ev) = events.recv().await {
let (key, is_put) = match &ev {
Event::Put(ev) => (ev.key.as_str(), true),
Event::Delete(ev) => (ev.key.as_str(), false),
Event::Empty => continue,
};
let Some((org, id)) = parse_downtime_key(key) else {
log::error!("watch_downtimes: failed to parse event key {key}");
continue;
};
let result = if is_put {
(on_put)(org, id).await
} else {
(on_delete)(org, id).await
};
if let Err(e) = result {
log::error!("watch_downtimes: handler failed: {e}");
}
}
log::error!("watch_downtimes: event channel closed");
Ok(())
}
pub fn parse_downtime_key(key: &str) -> Option<(String, String)> {
let mut parts = key.trim_start_matches('/').split('/');
if parts.next()? != "downtimes" {
return None;
}
let org = parts.next().filter(|s| !s.is_empty())?;
let id = parts.next().filter(|s| !s.is_empty())?;
Some((org.to_owned(), id.to_owned()))
}
fn downtime_key(org: &str, id: &str) -> String {
format!("{DOWNTIME_WATCHER_PREFIX}{org}/{id}")
}
#[cfg(test)]
mod tests {
use super::*;
#[test]
fn the_key_round_trips() {
let key = downtime_key("acme", "2f9KQe4b7Nq1vH3sT0mLzXpRcWd");
assert_eq!(key, "/downtimes/acme/2f9KQe4b7Nq1vH3sT0mLzXpRcWd");
assert_eq!(
parse_downtime_key(&key),
Some((
"acme".to_string(),
"2f9KQe4b7Nq1vH3sT0mLzXpRcWd".to_string()
))
);
}
#[test]
fn a_foreign_or_short_key_does_not_parse() {
assert!(parse_downtime_key("/alerts/acme/x").is_none());
assert!(parse_downtime_key("/downtimes/acme").is_none());
assert!(parse_downtime_key("/downtimes/acme/").is_none());
}
}
src/infra/src/coordinator/mod.rs
+1
View File
@@ -17,6 +17,7 @@ pub mod ai_prompts;
pub mod alerts;
pub mod dashboards;
pub mod destinations;
pub mod downtimes;
pub mod events;
pub mod ingest_rejection;
pub mod model_pricing;
src/infra/src/table/alert_incidents.rs
+416 -21
View File
@@ -18,8 +18,8 @@
//! Provides CRUD operations for incidents and incident-alert associations.
use sea_orm::{
ActiveModelTrait, ColumnTrait, EntityTrait, PaginatorTrait, QueryFilter, QueryOrder,
QuerySelect, Set, TransactionTrait,
ActiveModelTrait, ColumnTrait, ConnectionTrait, EntityTrait, PaginatorTrait, QueryFilter,
QueryOrder, QuerySelect, Set, TransactionTrait,
sea_query::{Expr, LockType},
};
use svix_ksuid::KsuidLike;
@@ -30,6 +30,14 @@ use crate::{
errors::{self, DbError, Error},
};
/// One incident [auto_resolve_stale] resolved, with the downtime that muted it until then.
#[derive(Clone, Debug, PartialEq, Eq)]
pub struct AutoResolved {
pub org_id: String,
pub incident_id: String,
pub muted_by_downtime_id: Option<String>,
}
/// The row a new incident starts as. One construction site, so the plain create
/// and the on-call promotion's transaction cannot drift as columns are added.
fn new_incident(
@@ -39,6 +47,7 @@ fn new_incident(
key_type: &str,
first_alert_at: i64,
title: Option<String>,
muted_by_downtime_id: Option<String>,
) -> alert_incidents::ActiveModel {
let now = chrono::Utc::now().timestamp_micros();
@@ -60,6 +69,7 @@ fn new_incident(
acknowledged_at: Set(None),
created_at: Set(now),
updated_at: Set(now),
muted_by_downtime_id: Set(muted_by_downtime_id),
}
}
@@ -74,6 +84,20 @@ pub async fn get(org_id: &str, id: &str) -> Result<Option<alert_incidents::Model
.map_err(|e| Error::DbError(DbError::SeaORMError(e.to_string())))
}
/// [get] on the primary, for a write that must not act on a lagging replica.
pub async fn get_rw(
org_id: &str,
id: &str,
) -> Result<Option<alert_incidents::Model>, errors::Error> {
let client = get_orm_client_rw().await;
alert_incidents::Entity::find_by_id(id)
.filter(alert_incidents::Column::OrgId.eq(org_id))
.one(client)
.await
.map_err(|e| Error::DbError(DbError::SeaORMError(e.to_string())))
}
/// Create a new incident
pub async fn create(
org_id: &str,
@@ -82,6 +106,28 @@ pub async fn create(
key_type: &str,
first_alert_at: i64,
title: Option<String>,
) -> Result<alert_incidents::Model, errors::Error> {
create_with_mute(
org_id,
severity,
group_values,
key_type,
first_alert_at,
title,
None,
)
.await
}
/// [`create`] for an incident opened by a firing that a downtime suppressed (D3).
pub async fn create_with_mute(
org_id: &str,
severity: &str,
group_values: serde_json::Value,
key_type: &str,
first_alert_at: i64,
title: Option<String>,
muted_by_downtime_id: Option<String>,
) -> Result<alert_incidents::Model, errors::Error> {
let client = get_orm_client_rw().await;
@@ -92,6 +138,7 @@ pub async fn create(
key_type,
first_alert_at,
title,
muted_by_downtime_id,
)
.insert(client)
.await
@@ -132,6 +179,7 @@ pub async fn create_and_attach_to_oncall_response(
key_type,
first_alert_at,
title,
None,
)
.insert(&txn)
.await
@@ -253,11 +301,23 @@ pub async fn update_status(
id: &str,
status: &str,
) -> Result<alert_incidents::Model, errors::Error> {
let client = get_orm_client_rw().await;
update_status_with(get_orm_client_rw().await, org_id, id, status).await
}
/// [`update_status`] on a given connection; a resolve ends the downtime mute.
pub async fn update_status_with<C: ConnectionTrait>(
conn: &C,
org_id: &str,
id: &str,
status: &str,
) -> Result<alert_incidents::Model, errors::Error> {
let now = chrono::Utc::now().timestamp_micros();
let incident = get(org_id, id)
.await?
let incident = alert_incidents::Entity::find_by_id(id)
.filter(alert_incidents::Column::OrgId.eq(org_id))
.one(conn)
.await
.map_err(|e| Error::DbError(DbError::SeaORMError(e.to_string())))?
.ok_or_else(|| Error::DbError(DbError::SeaORMError("Incident not found".to_string())))?;
let mut active: alert_incidents::ActiveModel = incident.into();
@@ -266,10 +326,12 @@ pub async fn update_status(
if status == "resolved" {
active.resolved_at = Set(Some(now));
// A manual reopen must run as a live incident, not stay muted forever.
active.muted_by_downtime_id = Set(None);
}
active
.update(client)
.update(conn)
.await
.map_err(|e| Error::DbError(DbError::SeaORMError(e.to_string())))
}
@@ -305,6 +367,67 @@ pub async fn acknowledge(
get(org_id, id).await
}
/// Clears the mute only while it names `downtime_id`, so one of two concurrent firings sees `true`.
pub async fn clear_muted_by_downtime_id(
org_id: &str,
id: &str,
downtime_id: &str,
) -> Result<bool, errors::Error> {
clear_muted_by_downtime_id_with(get_orm_client_rw().await, org_id, id, downtime_id).await
}
/// [`clear_muted_by_downtime_id`] on a given connection.
pub async fn clear_muted_by_downtime_id_with<C: ConnectionTrait>(
conn: &C,
org_id: &str,
id: &str,
downtime_id: &str,
) -> Result<bool, errors::Error> {
replace_muted_by_downtime_id_with(conn, org_id, id, downtime_id, None).await
}
/// Names the downtime that mutes the incident now, only while it still names `downtime_id`.
pub async fn retarget_muted_by_downtime_id(
org_id: &str,
id: &str,
downtime_id: &str,
current: &str,
) -> Result<bool, errors::Error> {
replace_muted_by_downtime_id_with(
get_orm_client_rw().await,
org_id,
id,
downtime_id,
Some(current),
)
.await
}
/// Compare-and-set of the mute from `downtime_id` to `next`; `true` when this call changed it.
pub async fn replace_muted_by_downtime_id_with<C: ConnectionTrait>(
conn: &C,
org_id: &str,
id: &str,
downtime_id: &str,
next: Option<&str>,
) -> Result<bool, errors::Error> {
let res = alert_incidents::Entity::update_many()
.col_expr(
alert_incidents::Column::MutedByDowntimeId,
Expr::value(next.map(str::to_string)),
)
.col_expr(
alert_incidents::Column::UpdatedAt,
Expr::value(chrono::Utc::now().timestamp_micros()),
)
.filter(alert_incidents::Column::Id.eq(id))
.filter(alert_incidents::Column::OrgId.eq(org_id))
.filter(alert_incidents::Column::MutedByDowntimeId.eq(downtime_id))
.exec(conn)
.await?;
Ok(res.rows_affected == 1)
}
/// Update incident title
pub async fn update_title(
org_id: &str,
@@ -358,8 +481,17 @@ pub async fn list(
limit: u64,
offset: u64,
) -> Result<Vec<alert_incidents::Model>, errors::Error> {
let client = get_orm_client_ro().await;
list_with(get_orm_client_ro().await, org_id, status, limit, offset).await
}
/// [`list`] on a given connection.
pub async fn list_with<C: ConnectionTrait>(
client: &C,
org_id: &str,
status: Option<&str>,
limit: u64,
offset: u64,
) -> Result<Vec<alert_incidents::Model>, errors::Error> {
let mut query = alert_incidents::Entity::find()
.select_only()
// Select all columns EXCEPT topology_context for performance
@@ -377,6 +509,7 @@ pub async fn list(
.column(alert_incidents::Column::AssignedTo)
.column(alert_incidents::Column::CreatedAt)
.column(alert_incidents::Column::UpdatedAt)
.column(alert_incidents::Column::MutedByDowntimeId)
.filter(alert_incidents::Column::OrgId.eq(org_id))
.order_by_desc(alert_incidents::Column::LastAlertAt);
@@ -498,6 +631,54 @@ pub async fn find_open_incident_containing_alert(
.map_err(|e| Error::DbError(DbError::SeaORMError(e.to_string())))
}
/// The resolved incidents among the alert's `limit` latest firings, newest first.
pub async fn resolved_incidents_for_alert(
org_id: &str,
alert_id: &str,
limit: u64,
) -> Result<Vec<String>, errors::Error> {
resolved_incidents_for_alert_with(get_orm_client_ro().await, org_id, alert_id, limit).await
}
/// [`resolved_incidents_for_alert`] against a caller-supplied connection.
pub async fn resolved_incidents_for_alert_with<C: ConnectionTrait>(
conn: &C,
org_id: &str,
alert_id: &str,
limit: u64,
) -> Result<Vec<String>, errors::Error> {
let links: Vec<String> = alert_incident_alerts::Entity::find()
.select_only()
.column(alert_incident_alerts::Column::IncidentId)
.filter(alert_incident_alerts::Column::AlertId.eq(alert_id))
.order_by_desc(alert_incident_alerts::Column::AlertFiredAt)
.limit(limit)
.into_tuple()
.all(conn)
.await
.map_err(|e| Error::DbError(DbError::SeaORMError(e.to_string())))?;
let mut ids: Vec<String> = Vec::with_capacity(links.len());
for id in links {
if !ids.contains(&id) {
ids.push(id);
}
}
if ids.is_empty() {
return Ok(ids);
}
let resolved: Vec<String> = alert_incidents::Entity::find()
.select_only()
.column(alert_incidents::Column::Id)
.filter(alert_incidents::Column::OrgId.eq(org_id))
.filter(alert_incidents::Column::Status.eq("resolved"))
.filter(alert_incidents::Column::Id.is_in(ids.clone()))
.into_tuple()
.all(conn)
.await
.map_err(|e| Error::DbError(DbError::SeaORMError(e.to_string())))?;
Ok(ids.into_iter().filter(|id| resolved.contains(id)).collect())
}
/// Get actual alert counts for multiple incidents (source of truth)
///
/// Returns a HashMap of incident_id -> actual_count from junction table.
@@ -779,18 +960,21 @@ pub async fn upgrade_incident_group_values(
Ok(())
}
/// Auto-resolve stale incidents that haven't received new alerts
///
/// Returns the number of incidents resolved
/// Returns (count, Vec<(org_id, incident_id)>) of resolved incidents
/// Resolves the incidents with no alert within the threshold; their count and the incidents.
pub async fn auto_resolve_stale(
stale_threshold_micros: i64,
) -> Result<(u64, Vec<(String, String)>), errors::Error> {
const PAGE_SIZE: u64 = 500;
let client = get_orm_client_rw().await;
) -> Result<(u64, Vec<AutoResolved>), errors::Error> {
let now = chrono::Utc::now().timestamp_micros();
let cutoff = now - stale_threshold_micros;
auto_resolve_stale_with(get_orm_client_rw().await, now - stale_threshold_micros, now).await
}
/// [auto_resolve_stale] on a given connection, for incidents with no alert since `cutoff`.
pub async fn auto_resolve_stale_with<C: ConnectionTrait>(
client: &C,
cutoff: i64,
now: i64,
) -> Result<(u64, Vec<AutoResolved>), errors::Error> {
const PAGE_SIZE: u64 = 500;
let mut resolved_ids = Vec::new();
loop {
@@ -812,17 +996,22 @@ pub async fn auto_resolve_stale(
.col_expr(alert_incidents::Column::Status, Expr::value("resolved"))
.col_expr(alert_incidents::Column::ResolvedAt, Expr::value(Some(now)))
.col_expr(alert_incidents::Column::UpdatedAt, Expr::value(now))
// A resolved incident is muted by nothing, even if recording its events fails later.
.col_expr(
alert_incidents::Column::MutedByDowntimeId,
Expr::value(Option::<String>::None),
)
.filter(alert_incidents::Column::Id.is_in(page_ids))
.filter(alert_incidents::Column::Status.ne("resolved"))
.exec(client)
.await
.map_err(|e| Error::DbError(DbError::SeaORMError(e.to_string())))?;
resolved_ids.extend(
stale_incidents
.into_iter()
.map(|incident| (incident.org_id, incident.id)),
);
resolved_ids.extend(stale_incidents.into_iter().map(|incident| AutoResolved {
org_id: incident.org_id,
incident_id: incident.id,
muted_by_downtime_id: incident.muted_by_downtime_id,
}));
if !page_full {
break;
}
@@ -844,8 +1033,189 @@ pub async fn delete_by_org(org_id: &str) -> Result<(), errors::Error> {
#[cfg(test)]
mod tests {
use sea_orm::{Database, DatabaseConnection, Schema};
use super::*;
async fn incidents_db() -> DatabaseConnection {
let db = Database::connect("sqlite::memory:").await.unwrap();
let backend = db.get_database_backend();
let stmt = Schema::new(backend).create_table_from_entity(alert_incidents::Entity);
db.execute(backend.build(&stmt)).await.unwrap();
db
}
/// An incident in `status` that alert `a1` fired into at `fired_at`.
async fn fired_into(db: &DatabaseConnection, status: &str, fired_at: i64) -> String {
let mut incident = new_incident(
"acme",
"P2",
serde_json::json!({"service": "payments"}),
"service",
fired_at,
None,
None,
);
incident.status = Set(status.to_string());
let incident = incident.insert(db).await.unwrap();
alert_incident_alerts::ActiveModel {
incident_id: Set(incident.id.clone()),
alert_id: Set("a1".to_string()),
alert_fired_at: Set(fired_at),
alert_name: Set("a1".to_string()),
alert_kind: Set("alert".to_string()),
correlation_reason: Set(None),
created_at: Set(fired_at),
resolved_at: Set(None),
}
.insert(db)
.await
.unwrap();
incident.id
}
#[tokio::test]
async fn a_firing_after_the_window_finds_the_resolved_incidents_but_not_the_open_one() {
let db = incidents_db().await;
let backend = db.get_database_backend();
let stmt = Schema::new(backend).create_table_from_entity(alert_incident_alerts::Entity);
db.execute(backend.build(&stmt)).await.unwrap();
let oldest = fired_into(&db, "resolved", 1).await;
let quiet = fired_into(&db, "resolved", 2).await;
let open = fired_into(&db, "open", 3).await;
let found = resolved_incidents_for_alert_with(&db, "acme", "a1", 10)
.await
.unwrap();
assert_eq!(found, [quiet.clone(), oldest]);
assert!(!found.contains(&open));
let latest_two = resolved_incidents_for_alert_with(&db, "acme", "a1", 2)
.await
.unwrap();
assert_eq!(latest_two, [quiet]);
assert!(
resolved_incidents_for_alert_with(&db, "other", "a1", 10)
.await
.unwrap()
.is_empty()
);
}
async fn muted_incident(db: &DatabaseConnection, downtime_id: &str) -> alert_incidents::Model {
new_incident(
"acme",
"P2",
serde_json::json!({"service": "payments"}),
"service",
1,
None,
Some(downtime_id.to_string()),
)
.insert(db)
.await
.unwrap()
}
#[tokio::test]
async fn two_concurrent_unmutes_clear_the_incident_once() {
let db = incidents_db().await;
let incident = muted_incident(&db, "dt-1").await;
let (a, b) = tokio::join!(
clear_muted_by_downtime_id_with(&db, "acme", &incident.id, "dt-1"),
clear_muted_by_downtime_id_with(&db, "acme", &incident.id, "dt-1"),
);
let winners = [a.unwrap(), b.unwrap()].iter().filter(|won| **won).count();
assert_eq!(
winners, 1,
"exactly one firing sends the opened notification"
);
let row = alert_incidents::Entity::find_by_id(incident.id.clone())
.one(&db)
.await
.unwrap()
.unwrap();
assert_eq!(row.muted_by_downtime_id, None);
}
#[tokio::test]
async fn a_second_downtime_takes_over_the_mute_of_the_first() {
let db = incidents_db().await;
let incident = muted_incident(&db, "dt-1").await;
assert!(
replace_muted_by_downtime_id_with(&db, "acme", &incident.id, "dt-1", Some("dt-2"))
.await
.unwrap()
);
assert!(
!replace_muted_by_downtime_id_with(&db, "acme", &incident.id, "dt-1", Some("dt-3"))
.await
.unwrap()
);
let row = alert_incidents::Entity::find_by_id(incident.id.clone())
.one(&db)
.await
.unwrap()
.unwrap();
assert_eq!(row.muted_by_downtime_id.as_deref(), Some("dt-2"));
}
#[tokio::test]
async fn an_unmute_for_another_downtime_or_org_changes_nothing() {
let db = incidents_db().await;
let incident = muted_incident(&db, "dt-1").await;
assert!(
!clear_muted_by_downtime_id_with(&db, "acme", &incident.id, "dt-2")
.await
.unwrap()
);
assert!(
!clear_muted_by_downtime_id_with(&db, "other", &incident.id, "dt-1")
.await
.unwrap()
);
let row = alert_incidents::Entity::find_by_id(incident.id.clone())
.one(&db)
.await
.unwrap()
.unwrap();
assert_eq!(row.muted_by_downtime_id.as_deref(), Some("dt-1"));
}
#[tokio::test]
async fn a_resolve_clears_the_mute_so_a_manual_reopen_is_live() {
let db = incidents_db().await;
let incident = muted_incident(&db, "dt-1").await;
let ack = update_status_with(&db, "acme", &incident.id, "acknowledged")
.await
.unwrap();
assert_eq!(ack.muted_by_downtime_id.as_deref(), Some("dt-1"));
let resolved = update_status_with(&db, "acme", &incident.id, "resolved")
.await
.unwrap();
assert_eq!(resolved.muted_by_downtime_id, None);
assert!(resolved.resolved_at.is_some());
let reopened = update_status_with(&db, "acme", &incident.id, "open")
.await
.unwrap();
assert_eq!(reopened.status, "open");
assert_eq!(reopened.muted_by_downtime_id, None);
assert!(
update_status_with(&db, "other", &incident.id, "resolved")
.await
.is_err()
);
}
#[tokio::test]
async fn the_list_carries_the_muting_downtime_for_the_muted_chip() {
let db = incidents_db().await;
let muted = muted_incident(&db, "dt-1").await;
let rows = list_with(&db, "acme", None, 50, 0).await.unwrap();
assert_eq!(rows.len(), 1);
assert_eq!(rows[0].id, muted.id);
assert_eq!(rows[0].muted_by_downtime_id.as_deref(), Some("dt-1"));
}
#[test]
fn test_ksuid_generation() {
let id = svix_ksuid::Ksuid::new(None, None).to_string();
@@ -976,4 +1346,29 @@ mod tests {
assert!(result.is_err(), "Should reject malformed JSON");
}
#[tokio::test]
async fn the_bulk_auto_resolve_clears_the_mute_and_reports_it() {
let db = incidents_db().await;
let incident = muted_incident(&db, "dt-1").await;
let now = incident.last_alert_at + 1_000;
let (count, resolved) = auto_resolve_stale_with(&db, now, now).await.unwrap();
assert_eq!(count, 1);
assert_eq!(
resolved,
[AutoResolved {
org_id: "acme".to_string(),
incident_id: incident.id.clone(),
muted_by_downtime_id: Some("dt-1".to_string()),
}]
);
let row = alert_incidents::Entity::find_by_id(incident.id)
.one(&db)
.await
.unwrap()
.unwrap();
assert_eq!(row.status, "resolved");
assert_eq!(row.muted_by_downtime_id, None);
}
}
src/infra/src/table/alert_states.rs
+243
View File
@@ -113,6 +113,116 @@ pub async fn get_rollups(alert_ids: &[String]) -> Result<Vec<AlertState>, errors
.collect())
}
/// Records on the rollup row which downtime suppressed the latest firing run, or clears it.
pub async fn set_last_downtime_id(
alert_id: &str,
downtime_id: Option<&str>,
) -> Result<u64, errors::Error> {
let client = get_orm_client_rw().await;
set_last_downtime_id_with(client, alert_id, downtime_id).await
}
/// [`set_last_downtime_id`] against a caller-supplied connection.
pub async fn set_last_downtime_id_with<C: sea_orm::ConnectionTrait>(
conn: &C,
alert_id: &str,
downtime_id: Option<&str>,
) -> Result<u64, errors::Error> {
let Some(downtime_id) = downtime_id else {
let res = alert_states::Entity::update_many()
.col_expr(
alert_states::Column::LastDowntimeId,
sea_orm::sea_query::Expr::value(Option::<String>::None),
)
.filter(alert_states::Column::AlertId.eq(alert_id))
.filter(alert_states::Column::GroupKey.eq(ROLLUP_GROUP_KEY))
.filter(alert_states::Column::LastDowntimeId.is_not_null())
.exec(conn)
.await?;
return Ok(res.rows_affected);
};
// A real-time alert never runs through the scheduler, so its rollup row may not exist yet.
let row = alert_states::ActiveModel {
alert_id: Set(alert_id.to_string()),
group_key: Set(ROLLUP_GROUP_KEY.to_string()),
last_downtime_id: Set(Some(downtime_id.to_string())),
..Default::default()
};
let res = alert_states::Entity::insert(row)
.on_conflict(
sea_orm::sea_query::OnConflict::columns([
alert_states::Column::AlertId,
alert_states::Column::GroupKey,
])
.update_column(alert_states::Column::LastDowntimeId)
.to_owned(),
)
.exec_without_returning(conn)
.await?;
Ok(res)
}
/// Forgets a downtime every alert recorded; returns those alerts, whose mute the caller closes.
pub async fn clear_last_downtime_id(downtime_id: &str) -> Result<Vec<String>, errors::Error> {
let client = get_orm_client_rw().await;
clear_last_downtime_id_with(client, downtime_id).await
}
/// [`clear_last_downtime_id`] against a caller-supplied connection.
pub async fn clear_last_downtime_id_with<C: sea_orm::ConnectionTrait>(
conn: &C,
downtime_id: &str,
) -> Result<Vec<String>, errors::Error> {
let mut alert_ids: Vec<String> = alert_states::Entity::find()
.select_only()
.column(alert_states::Column::AlertId)
.filter(alert_states::Column::LastDowntimeId.eq(downtime_id))
.into_tuple()
.all(conn)
.await?;
alert_ids.sort();
alert_ids.dedup();
if alert_ids.is_empty() {
return Ok(alert_ids);
}
alert_states::Entity::update_many()
.col_expr(
alert_states::Column::LastDowntimeId,
sea_orm::sea_query::Expr::value(Option::<String>::None),
)
.filter(alert_states::Column::LastDowntimeId.eq(downtime_id))
.exec(conn)
.await?;
Ok(alert_ids)
}
/// `alert_id -> last_downtime_id` for the alerts of a list page that carry one.
pub async fn last_downtime_ids(
alert_ids: &[String],
) -> Result<std::collections::HashMap<String, String>, errors::Error> {
let client = get_orm_client_ro().await;
last_downtime_ids_with(client, alert_ids).await
}
/// [`last_downtime_ids`] against a caller-supplied connection.
pub async fn last_downtime_ids_with<C: sea_orm::ConnectionTrait>(
conn: &C,
alert_ids: &[String],
) -> Result<std::collections::HashMap<String, String>, errors::Error> {
if alert_ids.is_empty() {
return Ok(Default::default());
}
Ok(alert_states::Entity::find()
.filter(alert_states::Column::AlertId.is_in(alert_ids.to_vec()))
.filter(alert_states::Column::GroupKey.eq(ROLLUP_GROUP_KEY))
.filter(alert_states::Column::LastDowntimeId.is_not_null())
.all(conn)
.await?
.into_iter()
.filter_map(|m| m.last_downtime_id.map(|id| (m.alert_id, id)))
.collect())
}
/// All per-group rows for one alert.
pub async fn list_groups(alert_id: &str) -> Result<Vec<AlertState>, errors::Error> {
let client = get_orm_client_ro().await;
@@ -557,6 +667,7 @@ where
episode_opened_at: Set(state.episode_opened_at),
episode_incident_id: Set(state.episode_incident_id.clone()),
recovering_since: Set(state.recovering_since),
last_downtime_id: sea_orm::ActiveValue::NotSet,
};
// Upsert on the composite primary key — rows are created lazily on an
@@ -879,6 +990,7 @@ mod tests {
episode_opened_at: None,
episode_incident_id: None,
recovering_since: None,
last_downtime_id: None,
}
}
@@ -1237,6 +1349,36 @@ mod tests {
assert_eq!(count_transitions(&db).await, 2);
}
#[tokio::test]
async fn last_downtime_id_is_kept_by_state_writes_and_cleared_on_demand() {
let db = db().await;
persist_with(&db, &update_for("alert-1", ROLLUP_GROUP_KEY, 1_000), None)
.await
.unwrap();
persist_with(&db, &update_for("alert-2", ROLLUP_GROUP_KEY, 1_000), None)
.await
.unwrap();
let ids = vec!["alert-1".to_string(), "alert-2".to_string()];
assert_eq!(
set_last_downtime_id_with(&db, "alert-1", Some("dt-1"))
.await
.unwrap(),
1
);
persist_with(&db, &update_for("alert-1", ROLLUP_GROUP_KEY, 2_000), None)
.await
.unwrap();
let found = last_downtime_ids_with(&db, &ids).await.unwrap();
assert_eq!(found.len(), 1);
assert_eq!(found.get("alert-1").map(String::as_str), Some("dt-1"));
set_last_downtime_id_with(&db, "alert-1", None)
.await
.unwrap();
assert!(last_downtime_ids_with(&db, &ids).await.unwrap().is_empty());
}
/// ...and on the group. One evaluation writes every group's transition at
/// the same `at`; collapsing them by `(alert_id, at)` would keep one group's
/// history and drop the rest.
@@ -1604,6 +1746,107 @@ mod tests {
assert_eq!(count_intervals(&db).await, 1);
}
async fn recorded(db: &sea_orm::DatabaseConnection, alert_id: &str, downtime_id: &str) {
use sea_orm::IntoActiveModel;
alert_states::Model {
alert_id: alert_id.to_string(),
last_downtime_id: Some(downtime_id.to_string()),
..model(Some(RunOutcome::Suppressed.to_i32()))
}
.into_active_model()
.insert(db)
.await
.unwrap();
}
#[tokio::test]
async fn an_edited_downtime_is_forgotten_by_every_alert_that_recorded_it() {
let db = db().await;
recorded(&db, "alert-1", "dt-1").await;
recorded(&db, "alert-2", "dt-1").await;
recorded(&db, "alert-3", "dt-2").await;
assert_eq!(
clear_last_downtime_id_with(&db, "dt-1").await.unwrap(),
["alert-1", "alert-2"]
);
assert!(
clear_last_downtime_id_with(&db, "dt-1")
.await
.unwrap()
.is_empty()
);
let ids: Vec<String> = ["alert-1", "alert-2", "alert-3"].map(String::from).to_vec();
let left = last_downtime_ids_with(&db, &ids).await.unwrap();
assert_eq!(left.len(), 1);
assert_eq!(left.get("alert-3").map(String::as_str), Some("dt-2"));
}
#[tokio::test]
async fn an_unsuppressed_firing_clears_the_recorded_downtime_once() {
let db = db().await;
recorded(&db, "alert-1", "dt-1").await;
assert_eq!(
set_last_downtime_id_with(&db, "alert-1", None)
.await
.unwrap(),
1
);
assert_eq!(
set_last_downtime_id_with(&db, "alert-1", None)
.await
.unwrap(),
0,
"a row with nothing recorded is not rewritten"
);
let ids = vec!["alert-1".to_string()];
assert!(last_downtime_ids_with(&db, &ids).await.unwrap().is_empty());
}
#[tokio::test]
async fn a_mute_of_an_alert_with_no_state_row_creates_its_rollup_row() {
let db = db().await;
set_last_downtime_id_with(&db, "realtime-1", Some("dt"))
.await
.unwrap();
let ids = vec!["realtime-1".to_string()];
let found = last_downtime_ids_with(&db, &ids).await.unwrap();
assert_eq!(found.get("realtime-1").map(String::as_str), Some("dt"));
let row = get_with(&db, "realtime-1", ROLLUP_GROUP_KEY)
.await
.unwrap()
.expect("the rollup row is inserted");
assert_eq!(
row.last_outcome, None,
"the other columns keep their defaults"
);
assert_eq!(row.level, None);
// A second mute updates the row it made, and a clear leaves no record.
set_last_downtime_id_with(&db, "realtime-1", Some("dt-2"))
.await
.unwrap();
let found = last_downtime_ids_with(&db, &ids).await.unwrap();
assert_eq!(found.get("realtime-1").map(String::as_str), Some("dt-2"));
assert_eq!(
set_last_downtime_id_with(&db, "realtime-1", None)
.await
.unwrap(),
1
);
assert!(last_downtime_ids_with(&db, &ids).await.unwrap().is_empty());
// A clear of an alert with no row inserts nothing.
set_last_downtime_id_with(&db, "never-ran", None)
.await
.unwrap();
assert!(
get_with(&db, "never-ran", ROLLUP_GROUP_KEY)
.await
.unwrap()
.is_none()
);
}
/// The episode axis has to survive the UPDATE half of the upsert, not only
/// the INSERT.
///
src/infra/src/table/downtime_notifications.rs
+293
View File
@@ -0,0 +1,293 @@
// Copyright 2026 OpenObserve Inc.
//
// This program is free software: you can redistribute it and/or modify
// it under the terms of the GNU Affero General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
//
// This program is distributed in the hope that it will be useful
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU Affero General Public License for more details.
//
// You should have received a copy of the GNU Affero General Public License
// along with this program. If not, see <http://www.gnu.org/licenses/>.
//! Sent downtime notifications, one row per `(downtime_id, window_start, event)`.
use sea_orm::{
ColumnTrait, ConnectionTrait, EntityTrait, QueryFilter, QueryOrder, QuerySelect, Set,
sea_query::{Expr, OnConflict},
};
use super::entity::downtime_notifications::{ActiveModel, Column, Entity, Model};
use crate::{
db::{get_orm_client_ro, get_orm_client_rw},
errors,
};
#[derive(Clone, Debug, PartialEq, Eq)]
pub struct DowntimeNotification {
pub id: String,
pub org: String,
pub downtime_id: String,
pub window_start: i64,
pub event: String,
pub sent_at: i64,
pub destinations: serde_json::Value,
pub result: Option<String>,
}
impl From<Model> for DowntimeNotification {
fn from(m: Model) -> Self {
Self {
id: m.id,
org: m.org,
downtime_id: m.downtime_id,
window_start: m.window_start,
event: m.event,
sent_at: m.sent_at,
destinations: m.destinations,
result: m.result,
}
}
}
/// False when a row for the same downtime, window and event already exists.
pub async fn insert_if_absent(record: &DowntimeNotification) -> Result<bool, errors::Error> {
let client = get_orm_client_rw().await;
insert_if_absent_with(client, record).await
}
/// Newest first by `sent_at`.
pub async fn list_for_downtime(
org: &str,
downtime_id: &str,
limit: u64,
) -> Result<Vec<DowntimeNotification>, errors::Error> {
let client = get_orm_client_ro().await;
list_for_downtime_with(client, org, downtime_id, limit).await
}
pub async fn set_result(id: &str, result: &str) -> Result<(), errors::Error> {
let client = get_orm_client_rw().await;
set_result_with(client, id, result).await
}
pub async fn delete_for_downtime(org: &str, downtime_id: &str) -> Result<u64, errors::Error> {
let client = get_orm_client_rw().await;
delete_for_downtime_with(client, org, downtime_id).await
}
pub async fn insert_if_absent_with<C: ConnectionTrait>(
conn: &C,
record: &DowntimeNotification,
) -> Result<bool, errors::Error> {
let inserted = Entity::insert(to_active_model(record))
.on_conflict(
OnConflict::columns([Column::DowntimeId, Column::WindowStart, Column::Event])
.do_nothing()
.to_owned(),
)
.exec_without_returning(conn)
.await?;
Ok(inserted > 0)
}
pub async fn set_result_with<C: ConnectionTrait>(
conn: &C,
id: &str,
result: &str,
) -> Result<(), errors::Error> {
Entity::update_many()
.col_expr(Column::Result, Expr::value(result))
.filter(Column::Id.eq(id))
.exec(conn)
.await?;
Ok(())
}
pub async fn list_for_downtime_with<C: ConnectionTrait>(
conn: &C,
org: &str,
downtime_id: &str,
limit: u64,
) -> Result<Vec<DowntimeNotification>, errors::Error> {
Ok(Entity::find()
.filter(Column::Org.eq(org))
.filter(Column::DowntimeId.eq(downtime_id))
.order_by_desc(Column::SentAt)
.order_by_desc(Column::Id)
.limit(limit)
.all(conn)
.await?
.into_iter()
.map(DowntimeNotification::from)
.collect())
}
pub async fn delete_for_downtime_with<C: ConnectionTrait>(
conn: &C,
org: &str,
downtime_id: &str,
) -> Result<u64, errors::Error> {
let res = Entity::delete_many()
.filter(Column::Org.eq(org))
.filter(Column::DowntimeId.eq(downtime_id))
.exec(conn)
.await?;
Ok(res.rows_affected)
}
fn to_active_model(r: &DowntimeNotification) -> ActiveModel {
ActiveModel {
id: Set(r.id.clone()),
org: Set(r.org.clone()),
downtime_id: Set(r.downtime_id.clone()),
window_start: Set(r.window_start),
event: Set(r.event.clone()),
sent_at: Set(r.sent_at),
destinations: Set(r.destinations.clone()),
result: Set(r.result.clone()),
}
}
#[cfg(test)]
mod tests {
use sea_orm::{Database, DatabaseConnection};
use super::*;
use crate::table::migration::create_downtime_notifications_for_test;
async fn db() -> DatabaseConnection {
let db = Database::connect("sqlite::memory:").await.unwrap();
create_downtime_notifications_for_test(&db).await.unwrap();
db
}
fn record(id: &str, downtime_id: &str, window_start: i64, event: &str) -> DowntimeNotification {
DowntimeNotification {
id: id.to_string(),
org: "acme".to_string(),
downtime_id: downtime_id.to_string(),
window_start,
event: event.to_string(),
sent_at: window_start,
destinations: serde_json::json!(["slack"]),
result: None,
}
}
fn ids(rows: &[DowntimeNotification]) -> Vec<&str> {
rows.iter().map(|r| r.id.as_str()).collect()
}
#[tokio::test]
async fn insert_if_absent_refuses_a_second_row_for_the_same_window_and_event() {
let db = db().await;
let first = record("n1", "d1", 10, "start");
assert!(insert_if_absent_with(&db, &first).await.unwrap());
assert!(
!insert_if_absent_with(&db, &record("n2", "d1", 10, "start"))
.await
.unwrap()
);
assert!(
insert_if_absent_with(&db, &record("n3", "d1", 10, "end"))
.await
.unwrap()
);
assert!(
insert_if_absent_with(&db, &record("n4", "d1", 20, "start"))
.await
.unwrap()
);
let rows = list_for_downtime_with(&db, "acme", "d1", 10).await.unwrap();
assert_eq!(ids(&rows), ["n4", "n3", "n1"]);
assert_eq!(rows[2], first);
}
#[tokio::test]
async fn set_result_records_the_outcome_on_the_claimed_row_only() {
let db = db().await;
for r in [
record("n1", "d1", 10, "start"),
record("n2", "d1", 20, "start"),
] {
insert_if_absent_with(&db, &r).await.unwrap();
}
set_result_with(&db, "n1", "slack: timed out")
.await
.unwrap();
let rows = list_for_downtime_with(&db, "acme", "d1", 10).await.unwrap();
assert_eq!(rows[0].result, None);
assert_eq!(rows[1].result.as_deref(), Some("slack: timed out"));
}
#[tokio::test]
async fn list_for_downtime_filters_by_org_and_downtime_and_honours_the_limit() {
let db = db().await;
for r in [
record("n1", "d1", 10, "start"),
record("n2", "d1", 20, "start"),
record("n3", "d2", 30, "start"),
] {
insert_if_absent_with(&db, &r).await.unwrap();
}
let mut foreign = record("n4", "d1", 40, "start");
foreign.org = "other".to_string();
insert_if_absent_with(&db, &foreign).await.unwrap();
assert_eq!(
ids(&list_for_downtime_with(&db, "acme", "d1", 10).await.unwrap()),
["n2", "n1"]
);
assert_eq!(
ids(&list_for_downtime_with(&db, "acme", "d1", 1).await.unwrap()),
["n2"]
);
assert_eq!(
ids(&list_for_downtime_with(&db, "other", "d1", 10)
.await
.unwrap()),
["n4"]
);
}
#[tokio::test]
async fn delete_for_downtime_spares_other_downtimes_and_orgs() {
let db = db().await;
let mut foreign = record("n4", "d1", 40, "start");
foreign.org = "other".to_string();
for r in [
record("n1", "d1", 10, "start"),
record("n2", "d1", 20, "start"),
record("n3", "d2", 30, "start"),
foreign,
] {
insert_if_absent_with(&db, &r).await.unwrap();
}
assert_eq!(
delete_for_downtime_with(&db, "acme", "d1").await.unwrap(),
2
);
assert!(
list_for_downtime_with(&db, "acme", "d1", 10)
.await
.unwrap()
.is_empty()
);
assert_eq!(
ids(&list_for_downtime_with(&db, "acme", "d2", 10).await.unwrap()),
["n3"]
);
assert_eq!(
ids(&list_for_downtime_with(&db, "other", "d1", 10)
.await
.unwrap()),
["n4"]
);
}
}
src/infra/src/table/downtimes.rs
+1711
View File
File diff suppressed because it is too large Load Diff
src/infra/src/table/entity/alert_incidents.rs
+5
View File
@@ -34,6 +34,9 @@ pub struct Model {
pub created_at: i64,
pub updated_at: i64,
/// Set while a downtime mutes this incident; nothing about it notifies.
pub muted_by_downtime_id: Option<String>,
}
#[derive(Copy, Clone, Debug, EnumIter, DeriveRelation)]
@@ -74,6 +77,7 @@ mod tests {
acknowledged_at: None,
created_at: 1000,
updated_at: 2000,
muted_by_downtime_id: None,
};
assert_eq!(m.id, "inc-1");
assert_eq!(m.status, "open");
@@ -105,6 +109,7 @@ mod tests {
acknowledged_at: Some(1500),
created_at: 1000,
updated_at: 1500,
muted_by_downtime_id: Some("2f9K".to_string()),
};
assert_eq!(m.acknowledged_by.as_deref(), Some("user@example.com"));
assert_eq!(m.acknowledged_at, Some(1500));
src/infra/src/table/entity/alert_states.rs
+2
View File
@@ -55,6 +55,8 @@ pub struct Model {
/// When the condition first cleared, while `keep_firing_for` holds the
/// episode open. NULL = not in the hold.
pub recovering_since: Option<i64>,
/// Rollup row only: the downtime that suppressed the latest firing run.
pub last_downtime_id: Option<String>,
}
#[derive(Copy, Clone, Debug, EnumIter, DeriveRelation)]
src/infra/src/table/entity/downtime_notifications.rs
+22
View File
@@ -0,0 +1,22 @@
//! `SeaORM` Entity for the `downtime_notifications` table.
use sea_orm::entity::prelude::*;
#[derive(Clone, Debug, PartialEq, DeriveEntityModel, Eq)]
#[sea_orm(table_name = "downtime_notifications")]
pub struct Model {
#[sea_orm(primary_key, auto_increment = false)]
pub id: String,
pub org: String,
pub downtime_id: String,
pub window_start: i64,
pub event: String,
pub sent_at: i64,
pub destinations: Json,
pub result: Option<String>,
}
#[derive(Copy, Clone, Debug, EnumIter, DeriveRelation)]
pub enum Relation {}
impl ActiveModelBehavior for ActiveModel {}
src/infra/src/table/entity/downtimes.rs
+40
View File
@@ -0,0 +1,40 @@
//! `SeaORM` Entity for the `downtimes` table.
use sea_orm::entity::prelude::*;
#[derive(Clone, Debug, PartialEq, DeriveEntityModel, Eq)]
#[sea_orm(table_name = "downtimes")]
pub struct Model {
#[sea_orm(primary_key, auto_increment = false)]
pub id: String,
pub org: String,
/// Primary key of the downtime folder, not its public id.
pub folder_id: String,
pub name: String,
pub reason: Option<String>,
pub condition: Option<Json>,
pub targets: Json,
pub show_banner: bool,
pub repeat: i16,
pub starts_at: i64,
pub ends_at: Option<i64>,
pub timezone: String,
pub start_time_local: Option<String>,
pub duration_secs: i64,
pub weekdays: Option<Json>,
pub cancelled_at: Option<i64>,
pub cancelled_by: Option<String>,
pub deleted_at: Option<i64>,
pub notifications: Option<Json>,
pub origin_region: Option<String>,
pub version: i64,
pub created_by: String,
pub created_at: i64,
pub updated_by: String,
pub updated_at: i64,
}
#[derive(Copy, Clone, Debug, EnumIter, DeriveRelation)]
pub enum Relation {}
impl ActiveModelBehavior for ActiveModel {}
src/infra/src/table/entity/mod.rs
+2
View File
@@ -19,6 +19,8 @@ pub mod compactor_manual_jobs;
pub mod dashboards;
pub mod destinations;
pub mod distinct_value_fields;
pub mod downtime_notifications;
pub mod downtimes;
pub mod enrichment_table_urls;
pub mod enrichment_tables;
pub mod external_alerts;
src/infra/src/table/entity/slo_backfill_jobs.rs
+3
View File
@@ -55,6 +55,9 @@ pub struct Model {
pub rows_written: i64,
pub error: Option<String>,
pub updated_at: i64,
/// `backfill`, or `remeasure` (D8): no status deltas, and a `reconcile` when the walk ends.
pub kind: String,
pub attempts: i32,
}
#[derive(Copy, Clone, Debug, EnumIter, DeriveRelation)]
src/infra/src/table/folders.rs
+44 -1
View File
@@ -46,6 +46,21 @@ pub(crate) fn folder_type_into_i16(folder_type: FolderType) -> i16 {
FolderType::Synthetics => 3,
FolderType::Workflows => 4,
FolderType::Prompts => 5,
FolderType::Downtimes => 6,
}
}
/// The inverse of [folder_type_into_i16].
pub fn folder_type_from_i16(value: i16) -> Option<FolderType> {
match value {
0 => Some(FolderType::Dashboards),
1 => Some(FolderType::Alerts),
2 => Some(FolderType::Reports),
3 => Some(FolderType::Synthetics),
4 => Some(FolderType::Workflows),
5 => Some(FolderType::Prompts),
6 => Some(FolderType::Downtimes),
_ => None,
}
}
@@ -157,7 +172,16 @@ pub async fn get_or_create(
folder: Folder,
folder_type: FolderType,
) -> Result<(Ksuid, Folder, bool), errors::Error> {
let client = get_orm_client_rw().await;
get_or_create_with(get_orm_client_rw().await, org_id, folder, folder_type).await
}
/// [`get_or_create`] on a given connection.
pub async fn get_or_create_with<C: ConnectionTrait>(
client: &C,
org_id: &str,
folder: Folder,
folder_type: FolderType,
) -> Result<(Ksuid, Folder, bool), errors::Error> {
let folder_id = folder.folder_id.clone();
if let Some(model) = get_model(client, org_id, &folder_id, folder_type).await? {
@@ -362,6 +386,25 @@ mod tests {
assert_eq!(folder_type_into_i16(FolderType::Dashboards), 0);
assert_eq!(folder_type_into_i16(FolderType::Alerts), 1);
assert_eq!(folder_type_into_i16(FolderType::Reports), 2);
assert_eq!(folder_type_into_i16(FolderType::Prompts), 5);
assert_eq!(folder_type_into_i16(FolderType::Downtimes), 6);
}
#[test]
fn every_folder_type_round_trips_through_its_stored_number() {
for folder_type in [
FolderType::Dashboards,
FolderType::Alerts,
FolderType::Reports,
FolderType::Synthetics,
FolderType::Workflows,
FolderType::Prompts,
FolderType::Downtimes,
] {
let n = folder_type_into_i16(folder_type);
assert_eq!(folder_type_from_i16(n), Some(folder_type));
}
assert_eq!(folder_type_from_i16(7), None);
}
#[test]
src/infra/src/table/incident_events.rs
+9 -1
View File
@@ -296,8 +296,16 @@ pub async fn append(
incident_id: &str,
event: IncidentEvent,
) -> Result<(), sea_orm::DbErr> {
let db = get_orm_client_rw().await;
append_with(get_orm_client_rw().await, org_id, incident_id, event).await
}
/// [append] on a given connection.
pub async fn append_with<C: TransactionTrait>(
db: &C,
org_id: &str,
incident_id: &str,
event: IncidentEvent,
) -> Result<(), sea_orm::DbErr> {
let txn = db.begin().await?;
let row = incident_events::Entity::find()
src/infra/src/table/migration/m20261009_000001_create_downtimes.rs
+292
View File
@@ -0,0 +1,292 @@
// Copyright 2026 OpenObserve Inc.
//
// This program is free software: you can redistribute it and/or modify
// it under the terms of the GNU Affero General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
//
// This program is distributed in the hope that it will be useful
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU Affero General Public License for more details.
//
// You should have received a copy of the GNU Affero General Public License
// along with this program. If not, see <http://www.gnu.org/licenses/>.
use sea_orm_migration::prelude::*;
#[derive(DeriveMigrationName)]
pub struct Migration;
const DOWNTIMES_ORG_IDX: &str = "downtimes_org_idx";
const DOWNTIMES_ORG_FOLDER_IDX: &str = "downtimes_org_folder_idx";
const DOWNTIMES_ORG_ENDS_AT_IDX: &str = "downtimes_org_ends_at_idx";
#[async_trait::async_trait]
impl MigrationTrait for Migration {
async fn up(&self, manager: &SchemaManager) -> Result<(), DbErr> {
manager.create_table(create_downtimes_table()).await?;
manager.create_index(create_org_idx()).await?;
manager.create_index(create_org_folder_idx()).await?;
manager.create_index(create_org_ends_at_idx()).await?;
Ok(())
}
async fn down(&self, manager: &SchemaManager) -> Result<(), DbErr> {
manager
.drop_table(Table::drop().table(Downtimes::Table).to_owned())
.await?;
Ok(())
}
}
fn create_downtimes_table() -> TableCreateStatement {
Table::create()
.table(Downtimes::Table)
.if_not_exists()
.col(
ColumnDef::new(Downtimes::Id)
.string_len(27)
.not_null()
.primary_key(),
)
.col(ColumnDef::new(Downtimes::Org).string_len(100).not_null())
.col(ColumnDef::new(Downtimes::FolderId).char_len(27).not_null())
.col(ColumnDef::new(Downtimes::Name).string_len(256).not_null())
.col(ColumnDef::new(Downtimes::Reason).text())
.col(ColumnDef::new(Downtimes::Condition).json())
.col(ColumnDef::new(Downtimes::Targets).json().not_null())
.col(
ColumnDef::new(Downtimes::ShowBanner)
.boolean()
.not_null()
.default(true),
)
.col(ColumnDef::new(Downtimes::Repeat).small_integer().not_null())
.col(ColumnDef::new(Downtimes::StartsAt).big_integer().not_null())
.col(ColumnDef::new(Downtimes::EndsAt).big_integer())
.col(
ColumnDef::new(Downtimes::Timezone)
.string_len(64)
.not_null(),
)
.col(ColumnDef::new(Downtimes::StartTimeLocal).string_len(5))
.col(
ColumnDef::new(Downtimes::DurationSecs)
.big_integer()
.not_null(),
)
.col(ColumnDef::new(Downtimes::Weekdays).json())
.col(ColumnDef::new(Downtimes::CancelledAt).big_integer())
.col(ColumnDef::new(Downtimes::CancelledBy).string_len(256))
.col(ColumnDef::new(Downtimes::DeletedAt).big_integer())
.col(ColumnDef::new(Downtimes::Notifications).json().null())
.col(
ColumnDef::new(Downtimes::OriginRegion)
.string_len(64)
.null(),
)
.col(
ColumnDef::new(Downtimes::Version)
.big_integer()
.not_null()
.default(0),
)
.col(
ColumnDef::new(Downtimes::CreatedBy)
.string_len(256)
.not_null(),
)
.col(
ColumnDef::new(Downtimes::CreatedAt)
.big_integer()
.not_null(),
)
.col(
ColumnDef::new(Downtimes::UpdatedBy)
.string_len(256)
.not_null(),
)
.col(
ColumnDef::new(Downtimes::UpdatedAt)
.big_integer()
.not_null(),
)
.foreign_key(
ForeignKey::create()
.name("downtimes_folder_fk")
.from(Downtimes::Table, Downtimes::FolderId)
.to(Folders::Table, Folders::Id),
)
.to_owned()
}
fn create_org_idx() -> IndexCreateStatement {
sea_query::Index::create()
.if_not_exists()
.name(DOWNTIMES_ORG_IDX)
.table(Downtimes::Table)
.col(Downtimes::Org)
.to_owned()
}
fn create_org_folder_idx() -> IndexCreateStatement {
sea_query::Index::create()
.if_not_exists()
.name(DOWNTIMES_ORG_FOLDER_IDX)
.table(Downtimes::Table)
.col(Downtimes::Org)
.col(Downtimes::FolderId)
.to_owned()
}
fn create_org_ends_at_idx() -> IndexCreateStatement {
sea_query::Index::create()
.if_not_exists()
.name(DOWNTIMES_ORG_ENDS_AT_IDX)
.table(Downtimes::Table)
.col(Downtimes::Org)
.col(Downtimes::EndsAt)
.to_owned()
}
#[derive(DeriveIden)]
enum Folders {
Table,
Id,
}
#[derive(DeriveIden)]
enum Downtimes {
Table,
Id,
Org,
FolderId,
Name,
Reason,
Condition,
Targets,
ShowBanner,
Repeat,
StartsAt,
EndsAt,
Timezone,
StartTimeLocal,
DurationSecs,
Weekdays,
CancelledAt,
CancelledBy,
DeletedAt,
Notifications,
OriginRegion,
Version,
CreatedBy,
CreatedAt,
UpdatedBy,
UpdatedAt,
}
#[cfg(test)]
mod tests {
use collapse::*;
use sea_orm::{ConnectionTrait, Database, Statement};
use super::*;
#[test]
fn postgres() {
collapsed_eq!(
&create_downtimes_table().to_string(PostgresQueryBuilder),
r#"
CREATE TABLE IF NOT EXISTS "downtimes" (
"id" varchar(27) NOT NULL PRIMARY KEY,
"org" varchar(100) NOT NULL,
"folder_id" char(27) NOT NULL,
"name" varchar(256) NOT NULL,
"reason" text,
"condition" json,
"targets" json NOT NULL,
"show_banner" bool NOT NULL DEFAULT TRUE,
"repeat" smallint NOT NULL,
"starts_at" bigint NOT NULL,
"ends_at" bigint,
"timezone" varchar(64) NOT NULL,
"start_time_local" varchar(5),
"duration_secs" bigint NOT NULL,
"weekdays" json,
"cancelled_at" bigint,
"cancelled_by" varchar(256),
"deleted_at" bigint,
"notifications" json NULL,
"origin_region" varchar(64) NULL,
"version" bigint NOT NULL DEFAULT 0,
"created_by" varchar(256) NOT NULL,
"created_at" bigint NOT NULL,
"updated_by" varchar(256) NOT NULL,
"updated_at" bigint NOT NULL,
CONSTRAINT "downtimes_folder_fk" FOREIGN KEY ("folder_id") REFERENCES "folders" ("id")
)"#
);
}
#[test]
fn the_indexes_cover_org_folder_and_retention() {
assert_eq!(
&create_org_idx().to_string(PostgresQueryBuilder),
r#"CREATE INDEX IF NOT EXISTS "downtimes_org_idx" ON "downtimes" ("org")"#
);
assert_eq!(
&create_org_folder_idx().to_string(PostgresQueryBuilder),
r#"CREATE INDEX IF NOT EXISTS "downtimes_org_folder_idx" ON "downtimes" ("org", "folder_id")"#
);
assert_eq!(
&create_org_ends_at_idx().to_string(PostgresQueryBuilder),
r#"CREATE INDEX IF NOT EXISTS "downtimes_org_ends_at_idx" ON "downtimes" ("org", "ends_at")"#
);
}
#[tokio::test]
async fn test_up_reruns_without_error_and_keeps_the_new_columns() {
let db = Database::connect("sqlite::memory:").await.unwrap();
db.execute_unprepared(
"CREATE TABLE folders (id char(27) PRIMARY KEY); INSERT INTO folders VALUES ('f1')",
)
.await
.unwrap();
let manager = SchemaManager::new(&db);
Migration.up(&manager).await.expect("first run");
Migration.up(&manager).await.expect("second run");
for column in ["notifications", "origin_region", "version"] {
assert!(manager.has_column("downtimes", column).await.unwrap());
}
db.execute(Statement::from_string(
sea_orm::DbBackend::Sqlite,
"INSERT INTO downtimes (id, org, folder_id, name, targets, repeat, starts_at, \
timezone, duration_secs, created_by, created_at, updated_by, updated_at) \
VALUES ('d1', 'acme', 'f1', 'n', '[]', 0, 1, 'UTC', 60, 'lin', 1, 'lin', 1)"
.to_owned(),
))
.await
.expect("a row is insertable without the new columns");
let row = db
.query_one(Statement::from_string(
sea_orm::DbBackend::Sqlite,
"SELECT version FROM downtimes WHERE id = 'd1'".to_owned(),
))
.await
.unwrap()
.unwrap();
assert_eq!(row.try_get::<i64>("", "version").unwrap(), 0);
}
#[tokio::test]
async fn test_down_drops_the_table() {
let db = Database::connect("sqlite::memory:").await.unwrap();
let manager = SchemaManager::new(&db);
Migration.up(&manager).await.unwrap();
Migration.down(&manager).await.unwrap();
assert!(!manager.has_table("downtimes").await.unwrap());
}
}
src/infra/src/table/migration/m20261009_000002_add_muted_by_downtime_id_to_alert_incidents.rs
+96
View File
@@ -0,0 +1,96 @@
// Copyright 2026 OpenObserve Inc.
//
// This program is free software: you can redistribute it and/or modify
// it under the terms of the GNU Affero General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
//
// This program is distributed in the hope that it will be useful
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU Affero General Public License for more details.
//
// You should have received a copy of the GNU Affero General Public License
// along with this program. If not, see <http://www.gnu.org/licenses/>.
use sea_orm_migration::prelude::*;
#[derive(DeriveMigrationName)]
pub struct Migration;
const TABLE: &str = "alert_incidents";
const COLUMN: &str = "muted_by_downtime_id";
#[async_trait::async_trait]
impl MigrationTrait for Migration {
async fn up(&self, manager: &SchemaManager) -> Result<(), DbErr> {
// SQLite emits a plain ADD COLUMN for add_column_if_not_exists, so guard explicitly.
if manager.has_column(TABLE, COLUMN).await? {
return Ok(());
}
manager.alter_table(add_column_statement()).await
}
async fn down(&self, manager: &SchemaManager) -> Result<(), DbErr> {
if !manager.has_column(TABLE, COLUMN).await? {
return Ok(());
}
manager
.alter_table(
Table::alter()
.table(AlertIncidents::Table)
.drop_column(AlertIncidents::MutedByDowntimeId)
.to_owned(),
)
.await
}
}
fn add_column_statement() -> TableAlterStatement {
Table::alter()
.table(AlertIncidents::Table)
.add_column(
ColumnDef::new(AlertIncidents::MutedByDowntimeId)
.string_len(27)
.null(),
)
.to_owned()
}
#[derive(DeriveIden)]
enum AlertIncidents {
Table,
MutedByDowntimeId,
}
#[cfg(test)]
mod tests {
use sea_orm::{ConnectionTrait, Database};
use super::*;
#[test]
fn postgres() {
assert_eq!(
&add_column_statement().to_string(PostgresQueryBuilder),
r#"ALTER TABLE "alert_incidents" ADD COLUMN "muted_by_downtime_id" varchar(27) NULL"#
);
}
#[tokio::test]
async fn test_up_reruns_without_adding_the_column_twice() {
let db = Database::connect("sqlite::memory:").await.unwrap();
db.execute_unprepared("CREATE TABLE alert_incidents (id varchar(27) PRIMARY KEY)")
.await
.unwrap();
let manager = SchemaManager::new(&db);
Migration.up(&manager).await.expect("first run");
Migration.up(&manager).await.expect("second run");
assert!(manager.has_column(TABLE, COLUMN).await.unwrap());
Migration.down(&manager).await.unwrap();
Migration.down(&manager).await.unwrap();
assert!(!manager.has_column(TABLE, COLUMN).await.unwrap());
}
}
src/infra/src/table/migration/m20261009_000003_add_last_downtime_id_to_alert_states.rs
+96
View File
@@ -0,0 +1,96 @@
// Copyright 2026 OpenObserve Inc.
//
// This program is free software: you can redistribute it and/or modify
// it under the terms of the GNU Affero General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
//
// This program is distributed in the hope that it will be useful
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU Affero General Public License for more details.
//
// You should have received a copy of the GNU Affero General Public License
// along with this program. If not, see <http://www.gnu.org/licenses/>.
use sea_orm_migration::prelude::*;
#[derive(DeriveMigrationName)]
pub struct Migration;
const TABLE: &str = "alert_states";
const COLUMN: &str = "last_downtime_id";
#[async_trait::async_trait]
impl MigrationTrait for Migration {
async fn up(&self, manager: &SchemaManager) -> Result<(), DbErr> {
// SQLite emits a plain ADD COLUMN for add_column_if_not_exists, so guard explicitly.
if manager.has_column(TABLE, COLUMN).await? {
return Ok(());
}
manager.alter_table(add_column_statement()).await
}
async fn down(&self, manager: &SchemaManager) -> Result<(), DbErr> {
if !manager.has_column(TABLE, COLUMN).await? {
return Ok(());
}
manager
.alter_table(
Table::alter()
.table(AlertStates::Table)
.drop_column(AlertStates::LastDowntimeId)
.to_owned(),
)
.await
}
}
fn add_column_statement() -> TableAlterStatement {
Table::alter()
.table(AlertStates::Table)
.add_column(
ColumnDef::new(AlertStates::LastDowntimeId)
.string_len(27)
.null(),
)
.to_owned()
}
#[derive(DeriveIden)]
enum AlertStates {
Table,
LastDowntimeId,
}
#[cfg(test)]
mod tests {
use sea_orm::{ConnectionTrait, Database};
use super::*;
#[test]
fn postgres() {
assert_eq!(
&add_column_statement().to_string(PostgresQueryBuilder),
r#"ALTER TABLE "alert_states" ADD COLUMN "last_downtime_id" varchar(27) NULL"#
);
}
#[tokio::test]
async fn test_up_reruns_without_adding_the_column_twice() {
let db = Database::connect("sqlite::memory:").await.unwrap();
db.execute_unprepared("CREATE TABLE alert_states (id varchar(27) PRIMARY KEY)")
.await
.unwrap();
let manager = SchemaManager::new(&db);
Migration.up(&manager).await.expect("first run");
Migration.up(&manager).await.expect("second run");
assert!(manager.has_column(TABLE, COLUMN).await.unwrap());
Migration.down(&manager).await.unwrap();
Migration.down(&manager).await.unwrap();
assert!(!manager.has_column(TABLE, COLUMN).await.unwrap());
}
}
src/infra/src/table/migration/m20261009_000004_add_kind_to_slo_backfill_jobs.rs
+142
View File
@@ -0,0 +1,142 @@
// Copyright 2026 OpenObserve Inc.
//
// This program is free software: you can redistribute it and/or modify
// it under the terms of the GNU Affero General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
//
// This program is distributed in the hope that it will be useful
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU Affero General Public License for more details.
//
// You should have received a copy of the GNU Affero General Public License
// along with this program. If not, see <http://www.gnu.org/licenses/>.
use sea_orm_migration::prelude::*;
#[derive(DeriveMigrationName)]
pub struct Migration;
const TABLE: &str = "slo_backfill_jobs";
const KIND: &str = "kind";
const ATTEMPTS: &str = "attempts";
#[async_trait::async_trait]
impl MigrationTrait for Migration {
async fn up(&self, manager: &SchemaManager) -> Result<(), DbErr> {
// SQLite emits a plain ADD COLUMN for add_column_if_not_exists, so guard explicitly.
if !manager.has_column(TABLE, KIND).await? {
manager.alter_table(add_kind_statement()).await?;
}
if !manager.has_column(TABLE, ATTEMPTS).await? {
manager.alter_table(add_attempts_statement()).await?;
}
Ok(())
}
async fn down(&self, manager: &SchemaManager) -> Result<(), DbErr> {
if manager.has_column(TABLE, ATTEMPTS).await? {
manager
.alter_table(drop_column_statement(SloBackfillJobs::Attempts))
.await?;
}
if manager.has_column(TABLE, KIND).await? {
manager
.alter_table(drop_column_statement(SloBackfillJobs::Kind))
.await?;
}
Ok(())
}
}
fn add_kind_statement() -> TableAlterStatement {
Table::alter()
.table(SloBackfillJobs::Table)
.add_column(
ColumnDef::new(SloBackfillJobs::Kind)
.string_len(16)
.not_null()
.default("backfill"),
)
.to_owned()
}
fn add_attempts_statement() -> TableAlterStatement {
Table::alter()
.table(SloBackfillJobs::Table)
.add_column(
ColumnDef::new(SloBackfillJobs::Attempts)
.integer()
.not_null()
.default(0),
)
.to_owned()
}
fn drop_column_statement(column: SloBackfillJobs) -> TableAlterStatement {
Table::alter()
.table(SloBackfillJobs::Table)
.drop_column(column)
.to_owned()
}
#[derive(DeriveIden)]
enum SloBackfillJobs {
Table,
Kind,
Attempts,
}
#[cfg(test)]
mod tests {
use sea_orm::{ConnectionTrait, Database};
use super::*;
#[test]
fn postgres() {
assert_eq!(
&add_kind_statement().to_string(PostgresQueryBuilder),
r#"ALTER TABLE "slo_backfill_jobs" ADD COLUMN "kind" varchar(16) NOT NULL DEFAULT 'backfill'"#
);
assert_eq!(
&add_attempts_statement().to_string(PostgresQueryBuilder),
r#"ALTER TABLE "slo_backfill_jobs" ADD COLUMN "attempts" integer NOT NULL DEFAULT 0"#
);
}
#[tokio::test]
async fn test_up_reruns_without_adding_either_column_twice() {
let db = Database::connect("sqlite::memory:").await.unwrap();
db.execute_unprepared("CREATE TABLE slo_backfill_jobs (slo_id varchar(27) PRIMARY KEY)")
.await
.unwrap();
let manager = SchemaManager::new(&db);
Migration.up(&manager).await.expect("first run");
Migration.up(&manager).await.expect("second run");
assert!(manager.has_column(TABLE, KIND).await.unwrap());
assert!(manager.has_column(TABLE, ATTEMPTS).await.unwrap());
Migration.down(&manager).await.unwrap();
Migration.down(&manager).await.unwrap();
assert!(!manager.has_column(TABLE, KIND).await.unwrap());
assert!(!manager.has_column(TABLE, ATTEMPTS).await.unwrap());
}
#[tokio::test]
async fn test_up_reruns_adds_attempts_to_a_table_that_already_has_kind() {
let db = Database::connect("sqlite::memory:").await.unwrap();
db.execute_unprepared(
"CREATE TABLE slo_backfill_jobs (slo_id varchar(27) PRIMARY KEY, \
kind varchar(16) NOT NULL DEFAULT 'backfill')",
)
.await
.unwrap();
let manager = SchemaManager::new(&db);
Migration.up(&manager).await.unwrap();
assert!(manager.has_column(TABLE, ATTEMPTS).await.unwrap());
}
}
src/infra/src/table/migration/m20261009_000005_create_downtime_notifications.rs
+207
View File
@@ -0,0 +1,207 @@
// Copyright 2026 OpenObserve Inc.
//
// This program is free software: you can redistribute it and/or modify
// it under the terms of the GNU Affero General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
//
// This program is distributed in the hope that it will be useful
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU Affero General Public License for more details.
//
// You should have received a copy of the GNU Affero General Public License
// along with this program. If not, see <http://www.gnu.org/licenses/>.
use sea_orm_migration::prelude::*;
const WINDOW_EVENT_IDX: &str = "downtime_notifications_window_event_idx";
const ORG_DOWNTIME_IDX: &str = "downtime_notifications_org_downtime_idx";
#[derive(DeriveMigrationName)]
pub struct Migration;
#[async_trait::async_trait]
impl MigrationTrait for Migration {
async fn up(&self, manager: &SchemaManager) -> Result<(), DbErr> {
manager.create_table(create_table_statement()).await?;
manager.create_index(create_window_event_idx()).await?;
manager.create_index(create_org_downtime_idx()).await
}
async fn down(&self, manager: &SchemaManager) -> Result<(), DbErr> {
for name in [ORG_DOWNTIME_IDX, WINDOW_EVENT_IDX] {
manager
.drop_index(
Index::drop()
.if_exists()
.name(name)
.table(DowntimeNotifications::Table)
.to_owned(),
)
.await?;
}
manager
.drop_table(
Table::drop()
.table(DowntimeNotifications::Table)
.if_exists()
.to_owned(),
)
.await
}
}
#[derive(DeriveIden)]
enum DowntimeNotifications {
Table,
Id,
Org,
DowntimeId,
WindowStart,
Event,
SentAt,
Destinations,
Result,
}
fn create_table_statement() -> TableCreateStatement {
Table::create()
.table(DowntimeNotifications::Table)
.if_not_exists()
.col(
ColumnDef::new(DowntimeNotifications::Id)
.char_len(27)
.not_null()
.primary_key(),
)
.col(
ColumnDef::new(DowntimeNotifications::Org)
.string_len(100)
.not_null(),
)
.col(
ColumnDef::new(DowntimeNotifications::DowntimeId)
.char_len(27)
.not_null(),
)
.col(
ColumnDef::new(DowntimeNotifications::WindowStart)
.big_integer()
.not_null(),
)
.col(
ColumnDef::new(DowntimeNotifications::Event)
.string_len(16)
.not_null(),
)
.col(
ColumnDef::new(DowntimeNotifications::SentAt)
.big_integer()
.not_null(),
)
.col(
ColumnDef::new(DowntimeNotifications::Destinations)
.json()
.not_null(),
)
.col(ColumnDef::new(DowntimeNotifications::Result).text().null())
.to_owned()
}
fn create_window_event_idx() -> IndexCreateStatement {
Index::create()
.if_not_exists()
.unique()
.name(WINDOW_EVENT_IDX)
.table(DowntimeNotifications::Table)
.col(DowntimeNotifications::DowntimeId)
.col(DowntimeNotifications::WindowStart)
.col(DowntimeNotifications::Event)
.to_owned()
}
fn create_org_downtime_idx() -> IndexCreateStatement {
Index::create()
.if_not_exists()
.name(ORG_DOWNTIME_IDX)
.table(DowntimeNotifications::Table)
.col(DowntimeNotifications::Org)
.col(DowntimeNotifications::DowntimeId)
.to_owned()
}
#[cfg(test)]
mod tests {
use collapse::*;
use sea_orm::{ConnectionTrait, Database};
use sea_orm_migration::MigrationName;
use super::*;
const TABLE: &str = "downtime_notifications";
#[test]
fn postgres() {
collapsed_eq!(
&create_table_statement().to_string(PostgresQueryBuilder),
r#"
CREATE TABLE IF NOT EXISTS "downtime_notifications" (
"id" char(27) NOT NULL PRIMARY KEY,
"org" varchar(100) NOT NULL,
"downtime_id" char(27) NOT NULL,
"window_start" bigint NOT NULL,
"event" varchar(16) NOT NULL,
"sent_at" bigint NOT NULL,
"destinations" json NOT NULL,
"result" text NULL
)"#
);
assert_eq!(
&create_window_event_idx().to_string(PostgresQueryBuilder),
r#"CREATE UNIQUE INDEX IF NOT EXISTS "downtime_notifications_window_event_idx" ON "downtime_notifications" ("downtime_id", "window_start", "event")"#
);
assert_eq!(
&create_org_downtime_idx().to_string(PostgresQueryBuilder),
r#"CREATE INDEX IF NOT EXISTS "downtime_notifications_org_downtime_idx" ON "downtime_notifications" ("org", "downtime_id")"#
);
}
#[tokio::test]
async fn test_up_reruns_and_the_unique_index_refuses_a_second_send() {
let db = Database::connect("sqlite::memory:").await.unwrap();
let manager = SchemaManager::new(&db);
Migration.up(&manager).await.expect("first run");
Migration.up(&manager).await.expect("second run");
assert!(manager.has_table(TABLE).await.unwrap());
let insert = |id: &str| {
format!(
"INSERT INTO downtime_notifications \
(id, org, downtime_id, window_start, event, sent_at, destinations) \
VALUES ('{id}', 'acme', 'd1', 1, 'start', 2, '[]')"
)
};
db.execute_unprepared(&insert("n1")).await.unwrap();
assert!(db.execute_unprepared(&insert("n2")).await.is_err());
}
#[tokio::test]
async fn test_down_drops_the_table() {
let db = Database::connect("sqlite::memory:").await.unwrap();
let manager = SchemaManager::new(&db);
Migration.up(&manager).await.unwrap();
Migration.down(&manager).await.unwrap();
assert!(!manager.has_table(TABLE).await.unwrap());
Migration.down(&manager).await.unwrap();
}
#[test]
fn test_migration_name() {
assert_eq!(
Migration.name(),
"m20261009_000005_create_downtime_notifications"
);
}
}
src/infra/src/table/migration/mod.rs
+49
View File
@@ -211,6 +211,11 @@ mod m20261004_000001_create_query_history;
mod m20261007_000001_scope_alert_dedup_state_by_org;
mod m20261008_000001_add_name_to_rum_pa_tombstones;
mod m20261009_000001_add_promql_rule_mode;
mod m20261009_000001_create_downtimes;
mod m20261009_000002_add_muted_by_downtime_id_to_alert_incidents;
mod m20261009_000003_add_last_downtime_id_to_alert_states;
mod m20261009_000004_add_kind_to_slo_backfill_jobs;
mod m20261009_000005_create_downtime_notifications;
/// Shared body of the two `folder_id` migrations above; not a migration itself.
mod workflow_folder_id;
@@ -314,6 +319,19 @@ pub(crate) async fn create_slo_tables_for_test(
let manager = SchemaManager::new(db);
m20260727_000001_create_slo_tables::Migration
.up(&manager)
.await?;
m20261009_000004_add_kind_to_slo_backfill_jobs::Migration
.up(&manager)
.await
}
#[cfg(test)]
pub(crate) async fn create_downtime_notifications_for_test(
db: &sea_orm::DatabaseConnection,
) -> Result<(), DbErr> {
use sea_orm_migration::MigrationTrait;
m20261009_000005_create_downtime_notifications::Migration
.up(&SchemaManager::new(db))
.await
}
@@ -529,6 +547,11 @@ impl MigratorTrait for Migrator {
Box::new(m20261007_000001_scope_alert_dedup_state_by_org::Migration),
Box::new(m20261008_000001_add_name_to_rum_pa_tombstones::Migration),
Box::new(m20261009_000001_add_promql_rule_mode::Migration),
Box::new(m20261009_000001_create_downtimes::Migration),
Box::new(m20261009_000002_add_muted_by_downtime_id_to_alert_incidents::Migration),
Box::new(m20261009_000003_add_last_downtime_id_to_alert_states::Migration),
Box::new(m20261009_000004_add_kind_to_slo_backfill_jobs::Migration),
Box::new(m20261009_000005_create_downtime_notifications::Migration),
]
}
}
@@ -588,6 +611,8 @@ mod tests {
(98, "m20261007_000001_scope_alert_dedup_state_by_org"),
(99, "m20261008_000001_add_name_to_rum_pa_tombstones"),
(100, "m20261009_000001_add_promql_rule_mode"),
(101, "m20261009_000004_add_kind_to_slo_backfill_jobs"),
(102, "m20261009_000005_create_downtime_notifications"),
];
#[test]
@@ -667,6 +692,30 @@ mod tests {
"m20260725_000001_create_alert_states_tables",
"m20260924_000001_add_recovery_episode_columns",
),
(
"m20250109_092400_recreate_tables_with_ksuids",
"m20261009_000001_create_downtimes",
),
(
"m20251204_000001_create_alert_incidents_table",
"m20261009_000002_add_muted_by_downtime_id_to_alert_incidents",
),
(
"m20260725_000001_create_alert_states_tables",
"m20261009_000003_add_last_downtime_id_to_alert_states",
),
(
"m20260727_000001_create_slo_tables",
"m20261009_000004_add_kind_to_slo_backfill_jobs",
),
(
"m20261007_000001_scope_alert_dedup_state_by_org",
"m20261009_000001_create_downtimes",
),
(
"m20261009_000001_create_downtimes",
"m20261009_000005_create_downtime_notifications",
),
] {
assert!(
position(earlier) < position(later),
src/infra/src/table/mod.rs
+2
View File
@@ -33,6 +33,8 @@ mod composite_alerts_contract_tests;
pub mod dashboards;
pub mod destinations;
pub mod distinct_values;
pub mod downtime_notifications;
pub mod downtimes;
pub mod enrichment_table_urls;
pub mod enrichment_tables;
pub mod entity;
src/infra/src/table/slo.rs
+84 -4
View File
@@ -258,8 +258,8 @@ pub async fn apply_status_in_txn<C: ConnectionTrait>(
}
/// Read one status row. `group_key = ""` is the rollup.
pub async fn load_status(
db: &DatabaseConnection,
pub async fn load_status<C: ConnectionTrait>(
db: &C,
slo_id: &str,
group_key: &str,
) -> Result<Option<slo_status::Model>, errors::Error> {
@@ -270,6 +270,21 @@ pub async fn load_status(
)
}
/// The rollup row, locked until the transaction ends; SQLite has one writer connection instead.
pub async fn load_rollup_for_update<C: ConnectionTrait>(
db: &C,
slo_id: &str,
) -> Result<Option<slo_status::Model>, errors::Error> {
use sea_orm::QuerySelect;
Ok(slo_status::Entity::find_by_id((
slo_id.to_string(),
slo_status::ROLLUP_GROUP_KEY.to_string(),
))
.lock_exclusive()
.one(db)
.await?)
}
/// Every status row for one SLO — the rollup plus each group.
pub async fn load_all_groups(
db: &DatabaseConnection,
@@ -337,8 +352,8 @@ pub async fn bump_generation(
/// This is what makes at-least-once publication safe (D64): the running
/// aggregate is a cache, and this rebuilds it from the slices that are the
/// source of truth. It is therefore **load-bearing**, not hygiene.
pub async fn reconcile_from_slices(
db: &DatabaseConnection,
pub async fn reconcile_from_slices<C: ConnectionTrait>(
db: &C,
slo_id: &str,
group_key: &str,
recomputed: (f64, f64, i32),
@@ -358,6 +373,29 @@ pub async fn reconcile_from_slices(
Ok(())
}
/// Replaces the rollup's burn cache; false if its generation or watermark moved.
pub async fn replace_burn_cache<C: ConnectionTrait>(
db: &C,
slo_id: &str,
definition_generation: i32,
watermark_end: i64,
trailing_slices: serde_json::Value,
burn_windows: serde_json::Value,
) -> Result<bool, errors::Error> {
let res = slo_status::Entity::update_many()
.col_expr(
slo_status::Column::TrailingSlices,
Expr::value(trailing_slices),
)
.col_expr(slo_status::Column::BurnWindows, Expr::value(burn_windows))
.filter(row_of(slo_id, slo_status::ROLLUP_GROUP_KEY))
.filter(slo_status::Column::DefinitionGeneration.eq(definition_generation))
.filter(slo_status::Column::WatermarkEnd.eq(watermark_end))
.exec(db)
.await?;
Ok(res.rows_affected > 0)
}
/// Delete every status row belonging to an org's SLOs — the org-teardown path.
///
/// These rows carry no org of their own, so the ids come from `slos`. This
@@ -761,6 +799,48 @@ mod tests {
);
}
#[tokio::test]
async fn a_burn_cache_replace_at_the_stored_watermark_lands() {
let db = db().await;
init_generation(&db, SLO, 1).await.unwrap();
apply_status(&db, &write_of(1)).await.unwrap();
let windows = serde_json::json!({ "3600": { "good": 12.0, "total": 12.0 } });
let trailing = serde_json::json!({"9600": [0.0, 0.0]});
assert!(
replace_burn_cache(&db, SLO, 1, 9_900, trailing.clone(), windows.clone())
.await
.unwrap()
);
let status = load_status(&db, SLO, ROLLUP).await.unwrap().unwrap();
assert_eq!(status.burn_windows, Some(windows));
assert_eq!(status.trailing_slices, Some(trailing));
}
#[tokio::test]
async fn a_burn_cache_replace_after_the_watermark_moved_writes_nothing() {
let db = db().await;
init_generation(&db, SLO, 1).await.unwrap();
apply_status(&db, &write_of(1)).await.unwrap();
let before = load_status(&db, SLO, ROLLUP).await.unwrap().unwrap();
let empty = serde_json::json!({});
assert!(
!replace_burn_cache(&db, SLO, 1, 9_600, empty.clone(), empty.clone())
.await
.unwrap()
);
assert!(
!replace_burn_cache(&db, SLO, 2, 9_900, empty.clone(), empty)
.await
.unwrap()
);
assert_eq!(
load_status(&db, SLO, ROLLUP).await.unwrap().unwrap(),
before
);
}
// ===================== concurrency ====================================
/// A genuine race. The hazard is a read-modify-write lost update — two
src/infra/src/table/slo_backfill_jobs.rs
+466 -46
View File
@@ -21,8 +21,9 @@
//! new definition's history should be.
use sea_orm::{
ActiveModelTrait, ColumnTrait, DatabaseConnection, EntityTrait, IntoActiveModel, QueryFilter,
ActiveModelTrait, ColumnTrait, ConnectionTrait, DatabaseConnection, EntityTrait, QueryFilter,
Set,
sea_query::{Expr, OnConflict},
};
use super::entity::slo_backfill_jobs;
@@ -34,8 +35,24 @@ pub const STATE_DONE: i32 = 3;
pub const STATE_FAILED: i32 = 4;
pub const STATE_CANCELLED: i32 = 5;
pub async fn get(
db: &DatabaseConnection,
pub const KIND_BACKFILL: &str = "backfill";
/// Re-measures slices a downtime corrected or un-corrected (D8).
pub const KIND_REMEASURE: &str = "remeasure";
/// Retries of [extend_range] when another writer changed the row between its read and write.
const EXTEND_ATTEMPTS: usize = 5;
/// One SLO's re-measure, queued in the transaction of the downtime write that needs it.
#[derive(Clone, Debug, PartialEq, Eq)]
pub struct Remeasure {
pub slo_id: String,
pub generation: i32,
pub range_start: i64,
pub range_end: i64,
}
pub async fn get<C: ConnectionTrait>(
db: &C,
slo_id: &str,
generation: i32,
) -> Result<Option<slo_backfill_jobs::Model>, Error> {
@@ -70,40 +87,174 @@ pub async fn queue(
rows_written: Set(0),
error: Set(None),
updated_at: Set(now),
kind: Set(KIND_BACKFILL.to_string()),
attempts: Set(0),
}
.insert(db)
.await?;
Ok(())
}
/// Record a completed chunk. `done_through` is the earliest point filled,
/// because the walk runs backwards from the present.
/// Queue a re-measure of `[range_start, range_end)`, widening the generation's job if one exists.
pub async fn queue_remeasure<C: ConnectionTrait>(
db: &C,
slo_id: &str,
generation: i32,
range_start: i64,
range_end: i64,
now: i64,
) -> Result<(), Error> {
let job = slo_backfill_jobs::ActiveModel {
slo_id: Set(slo_id.to_string()),
definition_generation: Set(generation),
state: Set(STATE_QUEUED),
range_start: Set(range_start),
range_end: Set(range_end),
done_through: Set(None),
rows_written: Set(0),
error: Set(None),
updated_at: Set(now),
kind: Set(KIND_REMEASURE.to_string()),
attempts: Set(0),
};
let inserted = slo_backfill_jobs::Entity::insert(job)
.on_conflict(
OnConflict::columns([
slo_backfill_jobs::Column::SloId,
slo_backfill_jobs::Column::DefinitionGeneration,
])
.do_nothing()
.to_owned(),
)
.exec_without_returning(db)
.await?;
if inserted == 0 {
return extend_range(db, slo_id, generation, range_start, range_end, now).await;
}
Ok(())
}
/// Queues each re-measure; inside the caller's transaction, a failure queues none of them.
pub async fn queue_remeasures<C: ConnectionTrait>(
db: &C,
remeasures: &[Remeasure],
now: i64,
) -> Result<(), Error> {
for r in remeasures {
queue_remeasure(db, &r.slo_id, r.generation, r.range_start, r.range_end, now).await?;
}
Ok(())
}
/// Makes the job a re-measure of the range and restarts its walk; an unfinished job is widened.
pub async fn extend_range<C: ConnectionTrait>(
db: &C,
slo_id: &str,
generation: i32,
range_start: i64,
range_end: i64,
now: i64,
) -> Result<(), Error> {
for _ in 0..EXTEND_ATTEMPTS {
let Some(model) = get(db, slo_id, generation).await? else {
return Ok(());
};
let (start, end) = extended_range(&model, range_start, range_end);
let res = slo_backfill_jobs::Entity::update_many()
.col_expr(slo_backfill_jobs::Column::State, Expr::value(STATE_QUEUED))
.col_expr(slo_backfill_jobs::Column::RangeStart, Expr::value(start))
.col_expr(slo_backfill_jobs::Column::RangeEnd, Expr::value(end))
.col_expr(
slo_backfill_jobs::Column::DoneThrough,
Expr::value(Option::<i64>::None),
)
.col_expr(
slo_backfill_jobs::Column::Error,
Expr::value(Option::<String>::None),
)
.col_expr(
slo_backfill_jobs::Column::UpdatedAt,
Expr::value(next_updated_at(model.updated_at, now)),
)
.col_expr(slo_backfill_jobs::Column::Kind, Expr::value(KIND_REMEASURE))
.col_expr(slo_backfill_jobs::Column::Attempts, Expr::value(0))
.filter(slo_backfill_jobs::Column::SloId.eq(slo_id))
.filter(slo_backfill_jobs::Column::DefinitionGeneration.eq(generation))
.filter(slo_backfill_jobs::Column::UpdatedAt.eq(model.updated_at))
.exec(db)
.await?;
if res.rows_affected > 0 {
return Ok(());
}
}
Err(Error::Message(format!(
"SLO backfill job {slo_id}/{generation} kept changing while its range was widened"
)))
}
/// The `updated_at` a CAS write stores: always past `expected`, so a stale writer fails.
pub fn next_updated_at(expected: i64, now: i64) -> i64 {
now.max(expected.saturating_add(1))
}
fn extended_range(model: &slo_backfill_jobs::Model, start: i64, end: i64) -> (i64, i64) {
let finished = matches!(model.state, STATE_DONE | STATE_CANCELLED | STATE_FAILED);
if finished {
(start, end)
} else {
(model.range_start.min(start), model.range_end.max(end))
}
}
/// Records a chunk (`done_through` is the earliest point filled); false if the row changed.
pub async fn record_progress(
db: &DatabaseConnection,
slo_id: &str,
generation: i32,
done_through: i64,
rows_written: i64,
) -> Result<(), Error> {
let Some(model) = get(db, slo_id, generation).await? else {
return Ok(());
};
let previous = model.rows_written;
let mut active = model.into_active_model();
active.state = Set(STATE_RUNNING);
active.done_through = Set(Some(done_through));
active.rows_written = Set(previous + rows_written);
active.update(db).await?;
Ok(())
expected_updated_at: i64,
now: i64,
) -> Result<bool, Error> {
let res = slo_backfill_jobs::Entity::update_many()
.col_expr(slo_backfill_jobs::Column::State, Expr::value(STATE_RUNNING))
.col_expr(
slo_backfill_jobs::Column::DoneThrough,
Expr::value(Some(done_through)),
)
.col_expr(
slo_backfill_jobs::Column::RowsWritten,
Expr::col(slo_backfill_jobs::Column::RowsWritten).add(rows_written),
)
.col_expr(
slo_backfill_jobs::Column::UpdatedAt,
Expr::value(next_updated_at(expected_updated_at, now)),
)
.filter(slo_backfill_jobs::Column::SloId.eq(slo_id))
.filter(slo_backfill_jobs::Column::DefinitionGeneration.eq(generation))
.filter(slo_backfill_jobs::Column::UpdatedAt.eq(expected_updated_at))
.exec(db)
.await?;
Ok(res.rows_affected > 0)
}
pub async fn mark_done(
db: &DatabaseConnection,
pub async fn mark_done<C: ConnectionTrait>(
db: &C,
slo_id: &str,
generation: i32,
) -> Result<(), Error> {
set_state(db, slo_id, generation, STATE_DONE, None).await
expected_updated_at: i64,
now: i64,
) -> Result<bool, Error> {
set_state(
db,
slo_id,
generation,
STATE_DONE,
None,
expected_updated_at,
now,
)
.await
}
pub async fn mark_failed(
@@ -111,32 +262,93 @@ pub async fn mark_failed(
slo_id: &str,
generation: i32,
error: &str,
) -> Result<(), Error> {
set_state(db, slo_id, generation, STATE_FAILED, Some(error)).await
expected_updated_at: i64,
now: i64,
) -> Result<bool, Error> {
set_state(
db,
slo_id,
generation,
STATE_FAILED,
Some(error),
expected_updated_at,
now,
)
.await
}
pub async fn cancel(db: &DatabaseConnection, slo_id: &str, generation: i32) -> Result<(), Error> {
set_state(db, slo_id, generation, STATE_CANCELLED, None).await
}
async fn set_state(
/// Counts one failed finish and keeps the job's state; false if the row changed.
pub async fn record_failed_attempt(
db: &DatabaseConnection,
slo_id: &str,
generation: i32,
error: &str,
expected_updated_at: i64,
now: i64,
) -> Result<bool, Error> {
let res = slo_backfill_jobs::Entity::update_many()
.col_expr(
slo_backfill_jobs::Column::Attempts,
Expr::col(slo_backfill_jobs::Column::Attempts).add(1),
)
.col_expr(slo_backfill_jobs::Column::Error, Expr::value(error))
.col_expr(
slo_backfill_jobs::Column::UpdatedAt,
Expr::value(next_updated_at(expected_updated_at, now)),
)
.filter(slo_backfill_jobs::Column::SloId.eq(slo_id))
.filter(slo_backfill_jobs::Column::DefinitionGeneration.eq(generation))
.filter(slo_backfill_jobs::Column::UpdatedAt.eq(expected_updated_at))
.exec(db)
.await?;
Ok(res.rows_affected > 0)
}
pub async fn cancel(
db: &DatabaseConnection,
slo_id: &str,
generation: i32,
expected_updated_at: i64,
now: i64,
) -> Result<bool, Error> {
set_state(
db,
slo_id,
generation,
STATE_CANCELLED,
None,
expected_updated_at,
now,
)
.await
}
/// False, writing nothing, if the row changed since it was read at `expected_updated_at`.
async fn set_state<C: ConnectionTrait>(
db: &C,
slo_id: &str,
generation: i32,
state: i32,
error: Option<&str>,
) -> Result<(), Error> {
let Some(model) = get(db, slo_id, generation).await? else {
return Ok(());
};
let mut active = model.into_active_model();
active.state = Set(state);
expected_updated_at: i64,
now: i64,
) -> Result<bool, Error> {
let mut update = slo_backfill_jobs::Entity::update_many()
.col_expr(slo_backfill_jobs::Column::State, Expr::value(state))
.col_expr(
slo_backfill_jobs::Column::UpdatedAt,
Expr::value(next_updated_at(expected_updated_at, now)),
);
if let Some(e) = error {
active.error = Set(Some(e.to_string()));
update = update.col_expr(slo_backfill_jobs::Column::Error, Expr::value(e));
}
active.update(db).await?;
Ok(())
let res = update
.filter(slo_backfill_jobs::Column::SloId.eq(slo_id))
.filter(slo_backfill_jobs::Column::DefinitionGeneration.eq(generation))
.filter(slo_backfill_jobs::Column::UpdatedAt.eq(expected_updated_at))
.exec(db)
.await?;
Ok(res.rows_affected > 0)
}
/// Drop every job for an SLO — used when the SLO itself is deleted.
@@ -185,6 +397,39 @@ mod tests {
db
}
async fn version(db: &DatabaseConnection, slo_id: &str, generation: i32) -> i64 {
get(db, slo_id, generation)
.await
.unwrap()
.map_or(0, |j| j.updated_at)
}
/// [record_progress] against the row as it is now, as a chunk that just loaded it.
async fn progress(
db: &DatabaseConnection,
slo_id: &str,
generation: i32,
done_through: i64,
rows_written: i64,
) -> Result<bool, Error> {
let expected = version(db, slo_id, generation).await;
record_progress(
db,
slo_id,
generation,
done_through,
rows_written,
expected,
0,
)
.await
}
async fn done(db: &DatabaseConnection, slo_id: &str, generation: i32) -> Result<bool, Error> {
let expected = version(db, slo_id, generation).await;
mark_done(db, slo_id, generation, expected, 0).await
}
#[tokio::test]
async fn a_queued_job_starts_with_no_progress() {
let db = db().await;
@@ -200,7 +445,7 @@ mod tests {
async fn requeueing_leaves_existing_progress_alone() {
let db = db().await;
queue(&db, SLO, 1, 0, 900, 100).await.unwrap();
record_progress(&db, SLO, 1, 300, 50).await.unwrap();
progress(&db, SLO, 1, 300, 50).await.unwrap();
queue(&db, SLO, 1, 0, 900, 200).await.unwrap();
let j = get(&db, SLO, 1).await.unwrap().unwrap();
@@ -212,8 +457,8 @@ mod tests {
async fn progress_accumulates_rows_but_replaces_the_resume_point() {
let db = db().await;
queue(&db, SLO, 1, 0, 900, 100).await.unwrap();
record_progress(&db, SLO, 1, 600, 10).await.unwrap();
record_progress(&db, SLO, 1, 300, 20).await.unwrap();
progress(&db, SLO, 1, 600, 10).await.unwrap();
progress(&db, SLO, 1, 300, 20).await.unwrap();
let j = get(&db, SLO, 1).await.unwrap().unwrap();
assert_eq!(j.done_through, Some(300), "the walk runs backwards");
@@ -228,7 +473,7 @@ mod tests {
async fn a_new_generation_gets_its_own_job() {
let db = db().await;
queue(&db, SLO, 1, 0, 900, 100).await.unwrap();
record_progress(&db, SLO, 1, 300, 50).await.unwrap();
progress(&db, SLO, 1, 300, 50).await.unwrap();
queue(&db, SLO, 2, 0, 900, 100).await.unwrap();
assert_eq!(get(&db, SLO, 2).await.unwrap().unwrap().done_through, None);
@@ -243,17 +488,25 @@ mod tests {
async fn a_job_can_be_completed_failed_or_cancelled() {
let db = db().await;
queue(&db, SLO, 1, 0, 900, 100).await.unwrap();
mark_done(&db, SLO, 1).await.unwrap();
assert!(done(&db, SLO, 1).await.unwrap());
assert_eq!(get(&db, SLO, 1).await.unwrap().unwrap().state, STATE_DONE);
queue(&db, SLO, 2, 0, 900, 100).await.unwrap();
mark_failed(&db, SLO, 2, "search timeout").await.unwrap();
assert!(
mark_failed(&db, SLO, 2, "search timeout", version(&db, SLO, 2).await, 0)
.await
.unwrap()
);
let j = get(&db, SLO, 2).await.unwrap().unwrap();
assert_eq!(j.state, STATE_FAILED);
assert_eq!(j.error.as_deref(), Some("search timeout"));
queue(&db, SLO, 3, 0, 900, 100).await.unwrap();
cancel(&db, SLO, 3).await.unwrap();
assert!(
cancel(&db, SLO, 3, version(&db, SLO, 3).await, 0)
.await
.unwrap()
);
assert_eq!(
get(&db, SLO, 3).await.unwrap().unwrap().state,
STATE_CANCELLED
@@ -263,7 +516,7 @@ mod tests {
#[tokio::test]
async fn progress_on_a_missing_job_is_a_no_op() {
let db = db().await;
record_progress(&db, SLO, 9, 300, 50).await.unwrap();
progress(&db, SLO, 9, 300, 50).await.unwrap();
assert!(get(&db, SLO, 9).await.unwrap().is_none());
}
@@ -332,4 +585,171 @@ mod tests {
delete_by_org(&db, ORG).await.unwrap();
assert!(get(&db, OTHER_SLO, 1).await.unwrap().is_some());
}
#[tokio::test]
async fn a_remeasure_on_a_fresh_generation_is_its_own_job() {
let db = db().await;
queue_remeasure(&db, SLO, 1, 300, 600, 100).await.unwrap();
let j = get(&db, SLO, 1).await.unwrap().unwrap();
assert_eq!(j.kind, KIND_REMEASURE);
assert_eq!((j.range_start, j.range_end), (300, 600));
assert_eq!(j.state, STATE_QUEUED);
}
#[tokio::test]
async fn a_remeasure_replaces_a_finished_backfill_and_restarts_the_walk() {
let db = db().await;
queue(&db, SLO, 1, 0, 900, 100).await.unwrap();
progress(&db, SLO, 1, 0, 50).await.unwrap();
done(&db, SLO, 1).await.unwrap();
queue_remeasure(&db, SLO, 1, 300, 600, 200).await.unwrap();
let j = get(&db, SLO, 1).await.unwrap().unwrap();
assert_eq!(j.kind, KIND_REMEASURE);
assert_eq!((j.range_start, j.range_end), (300, 600));
assert_eq!(j.done_through, None, "the walk must restart");
assert_eq!(j.state, STATE_QUEUED);
}
#[tokio::test]
async fn a_remeasure_widens_an_unfinished_job() {
let db = db().await;
queue(&db, SLO, 1, 0, 900, 100).await.unwrap();
progress(&db, SLO, 1, 600, 10).await.unwrap();
extend_range(&db, SLO, 1, 800, 1_200, 200).await.unwrap();
let j = get(&db, SLO, 1).await.unwrap().unwrap();
assert_eq!((j.range_start, j.range_end), (0, 1_200));
assert_eq!(j.done_through, None);
assert_eq!(j.kind, KIND_REMEASURE);
}
#[tokio::test]
async fn a_plain_backfill_is_queued_as_backfill() {
let db = db().await;
queue(&db, SLO, 1, 0, 900, 100).await.unwrap();
assert_eq!(get(&db, SLO, 1).await.unwrap().unwrap().kind, KIND_BACKFILL);
}
/// A chunk that loaded the job before a re-measure reset it must not write over the reset.
#[tokio::test]
async fn a_stale_chunk_records_no_progress() {
let db = db().await;
queue(&db, SLO, 1, 0, 900, 100).await.unwrap();
let loaded = version(&db, SLO, 1).await;
queue_remeasure(&db, SLO, 1, 300, 600, 100).await.unwrap();
let reset = get(&db, SLO, 1).await.unwrap().unwrap();
assert_ne!(
reset.updated_at, loaded,
"a reset in the same second still moves the version"
);
assert!(
!record_progress(&db, SLO, 1, 300, 50, loaded, 100)
.await
.unwrap()
);
assert_eq!(get(&db, SLO, 1).await.unwrap().unwrap(), reset);
}
#[tokio::test]
async fn a_stale_mark_done_leaves_the_requeued_job_queued() {
let db = db().await;
queue(&db, SLO, 1, 0, 900, 100).await.unwrap();
let loaded = version(&db, SLO, 1).await;
assert!(
record_progress(&db, SLO, 1, 0, 5, loaded, 150)
.await
.unwrap()
);
let after_chunk = next_updated_at(loaded, 150);
assert_eq!(version(&db, SLO, 1).await, after_chunk);
queue_remeasure(&db, SLO, 1, 300, 600, 150).await.unwrap();
assert!(!mark_done(&db, SLO, 1, after_chunk, 150).await.unwrap());
let j = get(&db, SLO, 1).await.unwrap().unwrap();
assert_eq!(j.state, STATE_QUEUED);
assert_eq!(j.kind, KIND_REMEASURE);
assert!(!cancel(&db, SLO, 1, after_chunk, 150).await.unwrap());
assert!(
!mark_failed(&db, SLO, 1, "x", after_chunk, 150)
.await
.unwrap()
);
assert_eq!(get(&db, SLO, 1).await.unwrap().unwrap(), j);
}
#[tokio::test]
async fn two_remeasures_of_one_job_end_in_one_row_covering_both() {
let db = db().await;
let (a, b) = tokio::join!(
queue_remeasure(&db, SLO, 1, 300, 600, 100),
queue_remeasure(&db, SLO, 1, 500, 900, 100)
);
a.unwrap();
b.unwrap();
queue_remeasure(&db, SLO, 1, 100, 200, 100).await.unwrap();
let rows = slo_backfill_jobs::Entity::find().all(&db).await.unwrap();
assert_eq!(rows.len(), 1);
assert_eq!((rows[0].range_start, rows[0].range_end), (100, 900));
assert_eq!(rows[0].state, STATE_QUEUED);
}
#[tokio::test]
async fn a_failed_attempt_counts_up_and_keeps_the_state() {
let db = db().await;
queue_remeasure(&db, SLO, 1, 300, 600, 100).await.unwrap();
let loaded = get(&db, SLO, 1).await.unwrap().unwrap();
assert!(
record_failed_attempt(&db, SLO, 1, "partial", loaded.updated_at, 100)
.await
.unwrap()
);
let j = get(&db, SLO, 1).await.unwrap().unwrap();
assert_eq!(j.attempts, 1);
assert_eq!(j.state, STATE_QUEUED);
assert_eq!(j.error.as_deref(), Some("partial"));
assert!(j.updated_at > loaded.updated_at);
}
#[tokio::test]
async fn a_stale_failed_attempt_counts_nothing() {
let db = db().await;
queue_remeasure(&db, SLO, 1, 300, 600, 100).await.unwrap();
let loaded = version(&db, SLO, 1).await;
queue_remeasure(&db, SLO, 1, 0, 900, 100).await.unwrap();
let requeued = get(&db, SLO, 1).await.unwrap().unwrap();
assert!(
!record_failed_attempt(&db, SLO, 1, "partial", loaded, 100)
.await
.unwrap()
);
assert_eq!(get(&db, SLO, 1).await.unwrap().unwrap(), requeued);
}
#[tokio::test]
async fn a_new_remeasure_resets_the_attempts() {
let db = db().await;
queue_remeasure(&db, SLO, 1, 300, 600, 100).await.unwrap();
let loaded = version(&db, SLO, 1).await;
record_failed_attempt(&db, SLO, 1, "partial", loaded, 100)
.await
.unwrap();
queue_remeasure(&db, SLO, 1, 0, 900, 100).await.unwrap();
assert_eq!(get(&db, SLO, 1).await.unwrap().unwrap().attempts, 0);
}
#[test]
fn the_next_version_always_moves_forward() {
assert_eq!(next_updated_at(100, 200), 200);
assert_eq!(next_updated_at(100, 100), 101);
assert_eq!(
next_updated_at(100, 50),
101,
"a clock behind the row still moves it"
);
}
}
src/infra/src/table/slos.rs
+22 -5
View File
@@ -109,12 +109,19 @@ pub async fn list(
if let Some(folder) = folder_id {
q = q.filter(slos::Column::FolderId.eq(folder));
}
q.order_by_asc(slos::Column::Name)
let rows = q.order_by_asc(slos::Column::Name).all(db).await?;
rows.into_iter().map(to_slo).collect()
}
/// [list] for the correction planner: a row that does not decode is logged and left out, so it
/// cannot block the org's other SLOs; every other caller still sees the error.
pub async fn list_decodable(db: &DatabaseConnection, org: &str) -> Result<Vec<Slo>, Error> {
let rows = slos::Entity::find()
.filter(slos::Column::Org.eq(org))
.order_by_asc(slos::Column::Name)
.all(db)
.await?
.into_iter()
.map(to_slo)
.collect()
.await?;
Ok(rows.into_iter().filter_map(decoded).collect())
}
/// When the current generation began.
@@ -457,6 +464,16 @@ pub async fn move_to_folder(
// conversion
// ---------------------------------------------------------------------------
/// One row that does not decode is logged and left out.
fn decoded(m: slos::Model) -> Option<Slo> {
let id = m.id.clone();
to_slo(m)
.inspect_err(|e| {
log::error!("[slo] skipping SLO {id}, its stored row does not decode: {e}")
})
.ok()
}
fn to_slo(m: slos::Model) -> Result<Slo, Error> {
let sli_config = serde_json::from_value(m.sli_config)?;
let group_by = match m.group_by {
src/infra/src/table/synthetics_locations.rs
+19 -9
View File
@@ -25,7 +25,9 @@ use std::{
time::{Duration, Instant},
};
use sea_orm::{ColumnTrait, EntityTrait, QueryFilter, QueryOrder, Set, sea_query::Expr};
use sea_orm::{
ColumnTrait, ConnectionTrait, EntityTrait, QueryFilter, QueryOrder, Set, sea_query::Expr,
};
use tokio::sync::RwLock;
use super::entity::synthetics_locations::{ActiveModel, Column, Entity, Model};
@@ -276,14 +278,7 @@ pub async fn remove(id: &str) -> Result<(), errors::Error> {
/// stale `down_notified_at` would reintroduce exactly the duplicate this fixes.
pub async fn try_claim_down_notification(id: &str, now_us: i64) -> Result<bool, errors::Error> {
let client = get_orm_client_rw().await;
let res = Entity::update_many()
.col_expr(Column::DownNotifiedAt, Expr::value(now_us))
.filter(Column::Id.eq(id))
.filter(Column::DownNotifiedAt.eq(0i64))
.exec(client)
.await
.map_err(|e| Error::DbError(DbError::SeaORMError(e.to_string())))?;
Ok(res.rows_affected > 0)
try_claim_down_notification_with(client, id, now_us).await
}
/// Clears the down flag so a future outage notifies again.
@@ -303,6 +298,21 @@ pub async fn clear_down_notification(id: &str) -> Result<(), errors::Error> {
Ok(())
}
pub async fn try_claim_down_notification_with<C: ConnectionTrait>(
conn: &C,
id: &str,
now_us: i64,
) -> Result<bool, errors::Error> {
let res = Entity::update_many()
.col_expr(Column::DownNotifiedAt, Expr::value(now_us))
.filter(Column::Id.eq(id))
.filter(Column::DownNotifiedAt.eq(0i64))
.exec(conn)
.await
.map_err(|e| Error::DbError(DbError::SeaORMError(e.to_string())))?;
Ok(res.rows_affected > 0)
}
#[cfg(test)]
mod tests {
use sea_orm::{DbBackend, QueryTrait};

Some files were not shown because too many files have changed in this diff Show More