From ad73494fb106e93a79aa09cf99729ffaef978cea Mon Sep 17 00:00:00 2001 From: Neil <4138956+nwparker@users.noreply.github.com> Date: Mon, 26 Oct 2026 02:33:13 -0700 Subject: [PATCH] fix(agent-awake): warn once and back off when the sleep assertion keeps failing (#33841) * fix(agent-awake): keep sleep-assertion failures warn-once across respawns A successful spawn reset the failure streak, so a systemd-inhibit or caffeinate that exits right away re-warned on every 30s retry. * fix(agent-awake): back off sleep-assertion retries after consecutive failures * test(agent-awake): dispose the real-timer assertion --- src/main/linux-lid-sleep-assertion.test.ts | 6 +++-- src/main/linux-lid-sleep-assertion.ts | 10 +++++++- src/main/macos-system-sleep-assertion.test.ts | 24 +++++++++++++++++++ src/main/macos-system-sleep-assertion.ts | 20 ++++++++++++++-- 4 files changed, 55 insertions(+), 5 deletions(-) diff --git a/src/main/linux-lid-sleep-assertion.test.ts b/src/main/linux-lid-sleep-assertion.test.ts index 9c42f3630e6..98021f36dd0 100644 --- a/src/main/linux-lid-sleep-assertion.test.ts +++ b/src/main/linux-lid-sleep-assertion.test.ts @@ -344,7 +344,7 @@ describe('LinuxLidSleepAssertion', () => { return { assertion, failLatestChild, logger, spawn, spawnGaps } } - it('warns once while it keeps retrying every interval', () => { + it('warns once while it keeps retrying, doubling the wait up to a cap', () => { vi.useFakeTimers() try { const { assertion, failLatestChild, logger, spawnGaps } = createRetryingAssertion() @@ -355,7 +355,9 @@ describe('LinuxLidSleepAssertion', () => { vi.advanceTimersToNextTimer() } - expect(spawnGaps()).toEqual(Array(8).fill(LINUX_LID_SLEEP_ASSERTION_RETRY_MS)) + expect(spawnGaps()).toEqual( + [1, 2, 4, 8, 16, 16, 16, 16].map((factor) => factor * LINUX_LID_SLEEP_ASSERTION_RETRY_MS) + ) expect(logger.warn).toHaveBeenCalledTimes(1) expect(logger.debug).toHaveBeenCalledTimes(7) assertion.dispose() diff --git a/src/main/linux-lid-sleep-assertion.ts b/src/main/linux-lid-sleep-assertion.ts index 9fcd243b2a4..3f81278c0fc 100644 --- a/src/main/linux-lid-sleep-assertion.ts +++ b/src/main/linux-lid-sleep-assertion.ts @@ -1,6 +1,8 @@ import { spawnProcess } from '@orca/process-host' export const LINUX_LID_SLEEP_ASSERTION_RETRY_MS = 30_000 +// Consecutive failures double the retry delay up to this many base delays. +const MAX_RETRY_BACKOFF_FACTOR = 16 type Logger = Pick @@ -44,6 +46,7 @@ export class LinuxLidSleepAssertion { private readonly spawn: SystemdInhibitSpawn private child: SystemdInhibitProcess | null = null private childStartedAt: number | null = null + private consecutiveFailures = 0 private retryNotBefore: number | null = null private retryTimer: ReturnType | null = null private systemdInhibitUnavailable = false @@ -198,7 +201,11 @@ export class LinuxLidSleepAssertion { return } this.logFailure(failureKey, reason, details, failureType) - this.retryNotBefore = this.now() + LINUX_LID_SLEEP_ASSERTION_RETRY_MS + this.consecutiveFailures += 1 + this.retryNotBefore = + this.now() + + LINUX_LID_SLEEP_ASSERTION_RETRY_MS * + Math.min(2 ** (this.consecutiveFailures - 1), MAX_RETRY_BACKOFF_FACTOR) this.scheduleRetry() this.onUnexpectedFailure('linux-lid-assertion-failure') } @@ -231,6 +238,7 @@ export class LinuxLidSleepAssertion { ) { this.lastFailureKey = null this.warnedForLastFailure = false + this.consecutiveFailures = 0 } } diff --git a/src/main/macos-system-sleep-assertion.test.ts b/src/main/macos-system-sleep-assertion.test.ts index 8d17485a245..de6544f2dd5 100644 --- a/src/main/macos-system-sleep-assertion.test.ts +++ b/src/main/macos-system-sleep-assertion.test.ts @@ -220,4 +220,28 @@ describe('MacosSystemSleepAssertion', () => { expect(logger.debug).toHaveBeenCalledTimes(1) assertion.dispose() }) + + it('warns once while each respawned caffeinate exits right away', () => { + const logger = createLogger() + const children = [new FakeCaffeinateProcess(), new FakeCaffeinateProcess()] + const spawn = vi.fn(() => children[spawn.mock.calls.length - 1]) + let now = 1_000 + const assertion = new MacosSystemSleepAssertion({ + logger, + now: () => now, + platform: 'darwin', + spawn + }) + + for (const child of children) { + assertion.start('status-change') + child.emit('exit', 1, null) + now += MACOS_SYSTEM_SLEEP_ASSERTION_RETRY_MS + 1 + } + + expect(spawn).toHaveBeenCalledTimes(2) + expect(logger.warn).toHaveBeenCalledTimes(1) + expect(logger.debug).toHaveBeenCalledTimes(1) + assertion.dispose() + }) }) diff --git a/src/main/macos-system-sleep-assertion.ts b/src/main/macos-system-sleep-assertion.ts index 84250eb77af..1612df813e2 100644 --- a/src/main/macos-system-sleep-assertion.ts +++ b/src/main/macos-system-sleep-assertion.ts @@ -1,6 +1,8 @@ import { spawn as nodeSpawn } from 'node:child_process' export const MACOS_SYSTEM_SLEEP_ASSERTION_RETRY_MS = 30_000 +// Consecutive failures double the retry delay up to this many base delays. +const MAX_RETRY_BACKOFF_FACTOR = 16 type Logger = Pick @@ -41,9 +43,11 @@ export class MacosSystemSleepAssertion { private retryTimer: ReturnType | null = null private lastFailureKey: string | null = null private warnedForLastFailure = false + private consecutiveFailures = 0 private readonly intentionalStops = new WeakSet() private readonly reportedFailures = new WeakSet() private readonly childCleanups = new WeakMap void>() + private readonly childStartedAt = new WeakMap() constructor(options: MacosSystemSleepAssertionOptions = {}) { this.logger = options.logger ?? console @@ -92,8 +96,8 @@ export class MacosSystemSleepAssertion { }) child.on('error', onError) child.on('exit', onExit) + this.childStartedAt.set(child, this.now()) this.resetRetrySuppression() - this.resetFailureStreak() return true } @@ -141,6 +145,13 @@ export class MacosSystemSleepAssertion { if (this.child === child) { this.child = null } + // A spawn proves nothing; only a child that outlived the retry window counts as recovered. + if ( + this.now() - (this.childStartedAt.get(child) ?? this.now()) >= + MACOS_SYSTEM_SLEEP_ASSERTION_RETRY_MS + ) { + this.resetFailureStreak() + } this.handleFailure(failureKey, startReason, details, failureType) } @@ -160,7 +171,11 @@ export class MacosSystemSleepAssertion { failureType: 'error' | 'exit' | 'spawn-error' = 'spawn-error' ): void { this.logFailure(failureKey, reason, details, failureType) - this.retryNotBefore = this.now() + MACOS_SYSTEM_SLEEP_ASSERTION_RETRY_MS + this.consecutiveFailures += 1 + this.retryNotBefore = + this.now() + + MACOS_SYSTEM_SLEEP_ASSERTION_RETRY_MS * + Math.min(2 ** (this.consecutiveFailures - 1), MAX_RETRY_BACKOFF_FACTOR) this.scheduleRetry() this.onUnexpectedFailure('macos-assertion-failure') } @@ -211,6 +226,7 @@ export class MacosSystemSleepAssertion { private resetFailureStreak(): void { this.lastFailureKey = null this.warnedForLastFailure = false + this.consecutiveFailures = 0 } }