fix(agent-awake): warn once and back off when the sleep assertion keeps failing (#33841)

* fix(agent-awake): keep sleep-assertion failures warn-once across respawns

A successful spawn reset the failure streak, so a systemd-inhibit or caffeinate that
exits right away re-warned on every 30s retry.

* fix(agent-awake): back off sleep-assertion retries after consecutive failures

* test(agent-awake): dispose the real-timer assertion

src/main/linux-lid-sleep-assertion.test.ts

@@ -344,7 +344,7 @@ describe('LinuxLidSleepAssertion', () => {
       return { assertion, failLatestChild, logger, spawn, spawnGaps }
     }
 
-    it('warns once while it keeps retrying every interval', () => {
+    it('warns once while it keeps retrying, doubling the wait up to a cap', () => {
       vi.useFakeTimers()
       try {
         const { assertion, failLatestChild, logger, spawnGaps } = createRetryingAssertion()
@@ -355,7 +355,9 @@ describe('LinuxLidSleepAssertion', () => {
           vi.advanceTimersToNextTimer()
         }
 
-        expect(spawnGaps()).toEqual(Array(8).fill(LINUX_LID_SLEEP_ASSERTION_RETRY_MS))
+        expect(spawnGaps()).toEqual(
+          [1, 2, 4, 8, 16, 16, 16, 16].map((factor) => factor * LINUX_LID_SLEEP_ASSERTION_RETRY_MS)
+        )
         expect(logger.warn).toHaveBeenCalledTimes(1)
         expect(logger.debug).toHaveBeenCalledTimes(7)
         assertion.dispose()

src/main/linux-lid-sleep-assertion.ts

@@ -1,6 +1,8 @@
 import { spawnProcess } from '@orca/process-host'
 
 export const LINUX_LID_SLEEP_ASSERTION_RETRY_MS = 30_000
+// Consecutive failures double the retry delay up to this many base delays.
+const MAX_RETRY_BACKOFF_FACTOR = 16
 
 type Logger = Pick<Console, 'debug' | 'warn'>
 
@@ -44,6 +46,7 @@ export class LinuxLidSleepAssertion {
   private readonly spawn: SystemdInhibitSpawn
   private child: SystemdInhibitProcess | null = null
   private childStartedAt: number | null = null
+  private consecutiveFailures = 0
   private retryNotBefore: number | null = null
   private retryTimer: ReturnType<typeof setTimeout> | null = null
   private systemdInhibitUnavailable = false
@@ -198,7 +201,11 @@ export class LinuxLidSleepAssertion {
       return
     }
     this.logFailure(failureKey, reason, details, failureType)
-    this.retryNotBefore = this.now() + LINUX_LID_SLEEP_ASSERTION_RETRY_MS
+    this.consecutiveFailures += 1
+    this.retryNotBefore =
+      this.now() +
+      LINUX_LID_SLEEP_ASSERTION_RETRY_MS *
+        Math.min(2 ** (this.consecutiveFailures - 1), MAX_RETRY_BACKOFF_FACTOR)
     this.scheduleRetry()
     this.onUnexpectedFailure('linux-lid-assertion-failure')
   }
@@ -231,6 +238,7 @@ export class LinuxLidSleepAssertion {
     ) {
       this.lastFailureKey = null
       this.warnedForLastFailure = false
+      this.consecutiveFailures = 0
     }
   }
 

src/main/macos-system-sleep-assertion.test.ts

@@ -220,4 +220,28 @@ describe('MacosSystemSleepAssertion', () => {
     expect(logger.debug).toHaveBeenCalledTimes(1)
     assertion.dispose()
   })
+
+  it('warns once while each respawned caffeinate exits right away', () => {
+    const logger = createLogger()
+    const children = [new FakeCaffeinateProcess(), new FakeCaffeinateProcess()]
+    const spawn = vi.fn(() => children[spawn.mock.calls.length - 1])
+    let now = 1_000
+    const assertion = new MacosSystemSleepAssertion({
+      logger,
+      now: () => now,
+      platform: 'darwin',
+      spawn
+    })
+
+    for (const child of children) {
+      assertion.start('status-change')
+      child.emit('exit', 1, null)
+      now += MACOS_SYSTEM_SLEEP_ASSERTION_RETRY_MS + 1
+    }
+
+    expect(spawn).toHaveBeenCalledTimes(2)
+    expect(logger.warn).toHaveBeenCalledTimes(1)
+    expect(logger.debug).toHaveBeenCalledTimes(1)
+    assertion.dispose()
+  })
 })

src/main/macos-system-sleep-assertion.ts

@@ -1,6 +1,8 @@
 import { spawn as nodeSpawn } from 'node:child_process'
 
 export const MACOS_SYSTEM_SLEEP_ASSERTION_RETRY_MS = 30_000
+// Consecutive failures double the retry delay up to this many base delays.
+const MAX_RETRY_BACKOFF_FACTOR = 16
 
 type Logger = Pick<Console, 'debug' | 'warn'>
 
@@ -41,9 +43,11 @@ export class MacosSystemSleepAssertion {
   private retryTimer: ReturnType<typeof setTimeout> | null = null
   private lastFailureKey: string | null = null
   private warnedForLastFailure = false
+  private consecutiveFailures = 0
   private readonly intentionalStops = new WeakSet<CaffeinateProcess>()
   private readonly reportedFailures = new WeakSet<CaffeinateProcess>()
   private readonly childCleanups = new WeakMap<CaffeinateProcess, () => void>()
+  private readonly childStartedAt = new WeakMap<CaffeinateProcess, number>()
 
   constructor(options: MacosSystemSleepAssertionOptions = {}) {
     this.logger = options.logger ?? console
@@ -92,8 +96,8 @@ export class MacosSystemSleepAssertion {
     })
     child.on('error', onError)
     child.on('exit', onExit)
+    this.childStartedAt.set(child, this.now())
     this.resetRetrySuppression()
-    this.resetFailureStreak()
     return true
   }
 
@@ -141,6 +145,13 @@ export class MacosSystemSleepAssertion {
     if (this.child === child) {
       this.child = null
     }
+    // A spawn proves nothing; only a child that outlived the retry window counts as recovered.
+    if (
+      this.now() - (this.childStartedAt.get(child) ?? this.now()) >=
+      MACOS_SYSTEM_SLEEP_ASSERTION_RETRY_MS
+    ) {
+      this.resetFailureStreak()
+    }
     this.handleFailure(failureKey, startReason, details, failureType)
   }
 
@@ -160,7 +171,11 @@ export class MacosSystemSleepAssertion {
     failureType: 'error' | 'exit' | 'spawn-error' = 'spawn-error'
   ): void {
     this.logFailure(failureKey, reason, details, failureType)
-    this.retryNotBefore = this.now() + MACOS_SYSTEM_SLEEP_ASSERTION_RETRY_MS
+    this.consecutiveFailures += 1
+    this.retryNotBefore =
+      this.now() +
+      MACOS_SYSTEM_SLEEP_ASSERTION_RETRY_MS *
+        Math.min(2 ** (this.consecutiveFailures - 1), MAX_RETRY_BACKOFF_FACTOR)
     this.scheduleRetry()
     this.onUnexpectedFailure('macos-assertion-failure')
   }
@@ -211,6 +226,7 @@ export class MacosSystemSleepAssertion {
   private resetFailureStreak(): void {
     this.lastFailureKey = null
     this.warnedForLastFailure = false
+    this.consecutiveFailures = 0
   }
 }