mirror of
https://github.com/stablyai/orca.git
synced 2026-09-24 16:02:41 +00:00
Tier GitHub PR lookup polling to prevent quota exhaustion (#12013)
* Tier GitHub PR lookup polling to prevent quota exhaustion The selected worktree (O(1)) checks per-minute; card list (O(N)) per-15-minutes. Introduce process-wide cache to collapse concurrent polling and gate lookups on available rate-limit budget with exponential backoff on failure. - Preserve last-known review during backoff - Invalidate cache when Orca opens a PR - Stop coordinator from double-charging * Tier GitHub PR lookup polling to prevent quota exhaustion - Return the latest reset time when both GitHub API buckets are rate-limited, preventing premature retries against still-blocked buckets. - Serve the last known review on transient lookup failures, preventing reviews from blinking out on temporary errors. - Discard in-flight lookups that predate an invalidation so stale answers cannot overwrite newly opened reviews. * fix: give rate-limit reset tests unique titles oxlint vitest/no-identical-title was failing static analysis because two cases shared the same describe title.
This commit is contained in:
@@ -170,6 +170,7 @@ import {
|
||||
getPRComments,
|
||||
getPRForBranch,
|
||||
getPRForBranchOutcome,
|
||||
getGitHubPRLookupRateLimitBlock,
|
||||
getRepoSlug,
|
||||
getRepoUpstream,
|
||||
getWorkItem,
|
||||
@@ -4609,3 +4610,70 @@ describe('GitHub GraphQL rate-limit guard', () => {
|
||||
expect(noteRateLimitSpendMock).not.toHaveBeenCalled()
|
||||
})
|
||||
})
|
||||
|
||||
describe('getGitHubPRLookupRateLimitBlock', () => {
|
||||
beforeEach(() => {
|
||||
execFileAsyncMock.mockReset()
|
||||
ghExecFileAsyncMock.mockReset()
|
||||
getOwnerRepoMock.mockReset()
|
||||
getOwnerRepoMock.mockResolvedValue({ owner: 'acme', repo: 'widgets' })
|
||||
getRemoteUrlForRepoMock.mockReset()
|
||||
gitExecFileAsyncMock.mockReset()
|
||||
getRateLimitMock.mockReset()
|
||||
getRateLimitMock.mockResolvedValue(undefined)
|
||||
rateLimitGuardMock.mockReset()
|
||||
rateLimitGuardMock.mockReturnValue({ blocked: false })
|
||||
noteRateLimitSpendMock.mockReset()
|
||||
_resetOwnerRepoCache()
|
||||
})
|
||||
|
||||
it('reports no block while every lookup bucket has budget', async () => {
|
||||
await expect(getGitHubPRLookupRateLimitBlock('/repo-root')).resolves.toBeNull()
|
||||
expect(getRateLimitMock).toHaveBeenCalled()
|
||||
})
|
||||
|
||||
it('reports a block when either lookup bucket is exhausted', async () => {
|
||||
rateLimitGuardMock.mockImplementation(((bucket: string) =>
|
||||
bucket === 'graphql'
|
||||
? { blocked: true, remaining: 4, limit: 5000, resetAt: 1_800_000_000 }
|
||||
: { blocked: false }) as () => RateLimitGuardResult)
|
||||
|
||||
await expect(getGitHubPRLookupRateLimitBlock('/repo-root')).resolves.toEqual({
|
||||
resetAt: 1_800_000_000
|
||||
})
|
||||
})
|
||||
|
||||
it('reports the latest reset when both lookup buckets are exhausted', async () => {
|
||||
rateLimitGuardMock.mockImplementation(((bucket: string) => ({
|
||||
blocked: true,
|
||||
remaining: 4,
|
||||
limit: 5000,
|
||||
// Why: core resets first, so returning it would retry into graphql's block.
|
||||
resetAt: bucket === 'core' ? 1_800_000_000 : 1_800_003_600
|
||||
})) as () => RateLimitGuardResult)
|
||||
|
||||
await expect(getGitHubPRLookupRateLimitBlock('/repo-root')).resolves.toEqual({
|
||||
resetAt: 1_800_003_600
|
||||
})
|
||||
})
|
||||
|
||||
it('reports the later reset when graphql outlasts core', async () => {
|
||||
// Retrying at the earlier reset would fail again on the bucket still blocked.
|
||||
rateLimitGuardMock.mockImplementation(((bucket: string) => ({
|
||||
blocked: true,
|
||||
remaining: 0,
|
||||
limit: 5000,
|
||||
resetAt: bucket === 'graphql' ? 1_800_000_600 : 1_800_000_000
|
||||
})) as () => RateLimitGuardResult)
|
||||
|
||||
await expect(getGitHubPRLookupRateLimitBlock('/repo-root')).resolves.toEqual({
|
||||
resetAt: 1_800_000_600
|
||||
})
|
||||
})
|
||||
|
||||
it('fails open when the exempt rate-limit probe itself fails', async () => {
|
||||
getRateLimitMock.mockRejectedValue(new Error('probe offline'))
|
||||
|
||||
await expect(getGitHubPRLookupRateLimitBlock('/repo-root')).resolves.toBeNull()
|
||||
})
|
||||
})
|
||||
|
||||
@@ -195,6 +195,52 @@ async function assertRateLimitBudget(
|
||||
}
|
||||
}
|
||||
|
||||
// Why: a branch lookup prefers REST but can fall back to `gh pr list` and
|
||||
// `gh pr view`, so both buckets are guarded and charged. Mirrors the PR refresh
|
||||
// coordinator's own estimate.
|
||||
const PR_BRANCH_LOOKUP_BUCKETS = ['core', 'graphql'] as const
|
||||
|
||||
/**
|
||||
* Rate-limit floor for GitHub PR lookups that do not run through the PR refresh
|
||||
* coordinator's queue (#11532).
|
||||
*
|
||||
* The coordinator guards and paces its own background refreshes, but
|
||||
* `hostedReview:forBranch` polls the same lookup straight from the renderer.
|
||||
* Ungated, the two paths together could spend the user's entire hourly quota —
|
||||
* which is per user and shared with their own `gh` and CLI agents.
|
||||
* Returns the reset time when the caller must not spend, else `null`.
|
||||
*/
|
||||
export async function getGitHubPRLookupRateLimitBlock(
|
||||
repoPath: string,
|
||||
connectionId?: string | null,
|
||||
localGitOptions: LocalGitExecOptions = {}
|
||||
): Promise<{ resetAt: number } | null> {
|
||||
const executionOptions = ghRepoExecOptions(
|
||||
githubRepoContext(repoPath, connectionId, localGitOptions)
|
||||
)
|
||||
// Why: identity resolution runs local git, which can fail for reasons that
|
||||
// have nothing to do with the budget; let the lookup itself classify those.
|
||||
const repository = await getOriginGitHubApiRepository(
|
||||
repoPath,
|
||||
connectionId,
|
||||
executionOptions
|
||||
).catch(() => null)
|
||||
if (repository === null) {
|
||||
return null
|
||||
}
|
||||
if (spendsSharedGitHubComQuota(repository, executionOptions)) {
|
||||
// Why: the probe only warms the snapshot and is exempt from limits, so a
|
||||
// failure must fail open rather than block the lookup (#7553).
|
||||
await getRateLimit().catch(() => undefined)
|
||||
}
|
||||
// Why: retrying at the earlier reset would fail again on the bucket that has
|
||||
// not reset yet, so the latest blocked reset is the only honest retry time.
|
||||
const resets = PR_BRANCH_LOOKUP_BUCKETS.map((bucket) =>
|
||||
repositoryRateLimitGuard(repository, bucket, executionOptions)
|
||||
).flatMap((guard) => (guard.blocked ? [guard.resetAt] : []))
|
||||
return resets.length > 0 ? { resetAt: Math.max(...resets) } : null
|
||||
}
|
||||
|
||||
function prRefreshUpstreamError(
|
||||
err: unknown
|
||||
): Extract<PRRefreshOutcome, { kind: 'upstream-error' }> {
|
||||
@@ -2956,6 +3002,13 @@ export async function getPRForBranchOutcome(
|
||||
if (connectionId && candidates.length === 0) {
|
||||
return { kind: 'no-pr', fetchedAt: Date.now() }
|
||||
}
|
||||
// Why (#11532): account every lookup, not just the coordinator's queue —
|
||||
// `hostedReview:forBranch` reaches this directly from renderer polling and
|
||||
// was spending the shared quota invisibly. headRepo is `origin`, the same
|
||||
// identity the coordinator guards on.
|
||||
for (const bucket of PR_BRANCH_LOOKUP_BUCKETS) {
|
||||
noteRepositoryRateLimitSpend(headRepo ?? candidates[0], bucket, 1, ghOptions)
|
||||
}
|
||||
let data: PullRequestLookupData | null = null
|
||||
let dataRepo: OwnerRepo | null = null
|
||||
let dataHeadRepo: OwnerRepo | null = headRepo
|
||||
|
||||
@@ -32,6 +32,7 @@ const {
|
||||
getRateLimitMock,
|
||||
rateLimitGuardMock,
|
||||
noteRateLimitSpendMock,
|
||||
noteRepositoryRateLimitSpendMock,
|
||||
ghRepoExecOptionsMock,
|
||||
githubRepoContextMock,
|
||||
getSshGitProviderMock,
|
||||
@@ -52,6 +53,7 @@ const {
|
||||
blocked: false
|
||||
})),
|
||||
noteRateLimitSpendMock: vi.fn(),
|
||||
noteRepositoryRateLimitSpendMock: vi.fn(),
|
||||
ghRepoExecOptionsMock: vi.fn((context) =>
|
||||
context.connectionId
|
||||
? {}
|
||||
@@ -116,7 +118,8 @@ vi.mock('./local-git-config-signature', () => ({
|
||||
vi.mock('./rate-limit', () => ({
|
||||
getRateLimit: getRateLimitMock,
|
||||
rateLimitGuard: rateLimitGuardMock,
|
||||
noteRateLimitSpend: noteRateLimitSpendMock
|
||||
noteRateLimitSpend: noteRateLimitSpendMock,
|
||||
noteRepositoryRateLimitSpend: noteRepositoryRateLimitSpendMock
|
||||
}))
|
||||
|
||||
import {
|
||||
|
||||
@@ -649,18 +649,9 @@ describe('pr-refresh-coordinator', () => {
|
||||
'graphql',
|
||||
executionOptions
|
||||
)
|
||||
expect(noteRepositoryRateLimitSpendMock).toHaveBeenCalledWith(
|
||||
testCase.repository,
|
||||
'core',
|
||||
1,
|
||||
executionOptions
|
||||
)
|
||||
expect(noteRepositoryRateLimitSpendMock).toHaveBeenCalledWith(
|
||||
testCase.repository,
|
||||
'graphql',
|
||||
1,
|
||||
executionOptions
|
||||
)
|
||||
// Why (#11532): the lookup itself debits the snapshot now, so every caller
|
||||
// is accounted for; the coordinator must not double-charge on top.
|
||||
expect(noteRepositoryRateLimitSpendMock).not.toHaveBeenCalled()
|
||||
expect(getPRForBranchOutcomeMock).toHaveBeenCalledTimes(1)
|
||||
})
|
||||
|
||||
|
||||
@@ -11,12 +11,11 @@ import type {
|
||||
import { getPRForBranchOutcome, type GitHubPRBranchLookupOptions } from './client'
|
||||
import { getOriginGitHubApiRepository } from './github-api-repository'
|
||||
import { ghRepoExecOptions, githubRepoContext } from './gh-utils'
|
||||
import { getRateLimit, repositoryRateLimitGuard, spendsSharedGitHubComQuota } from './rate-limit'
|
||||
import {
|
||||
getRateLimit,
|
||||
noteRepositoryRateLimitSpend,
|
||||
repositoryRateLimitGuard,
|
||||
spendsSharedGitHubComQuota
|
||||
} from './rate-limit'
|
||||
lookupBackoffDelayMs,
|
||||
NO_REVIEW_REFRESH_INTERVAL_MS
|
||||
} from '../source-control/hosted-review-refresh-pacing'
|
||||
import { recordCoalescedCrashBreadcrumb } from '../crash-reporting/crash-breadcrumb-store'
|
||||
import { sendToTrustedUIRenderer } from '../ipc/ui'
|
||||
|
||||
@@ -74,8 +73,6 @@ const BACKGROUND_BUDGET_WINDOW_MS = 5 * 60_000
|
||||
const MIN_BACKGROUND_SPACING_MS = 10_000
|
||||
const BACKGROUND_BUDGET_MAX = 20
|
||||
const POST_PUSH_DELAY_MS = 2_500
|
||||
const BACKOFF_BASE_MS = 60_000
|
||||
const BACKOFF_MAX_MS = 15 * 60_000
|
||||
const DIAGNOSTIC_BREADCRUMB_MIN_INTERVAL_MS = 30_000
|
||||
const ACTIVE_BURST_WINDOW_MS = 30_000
|
||||
const ACTIVE_BURST_MAX = 3
|
||||
@@ -424,8 +421,7 @@ function removeQueuedAliasForInvalidCandidate(key: string, alias: GitHubPRRefres
|
||||
*/
|
||||
function nextVisibleErrorRetryAt(key: string): number {
|
||||
const failures = (errorBackoff.get(key)?.failures ?? 0) + 1
|
||||
const retryAt =
|
||||
Date.now() + Math.min(BACKOFF_MAX_MS, BACKOFF_BASE_MS * 2 ** Math.min(failures - 1, 4))
|
||||
const retryAt = Date.now() + lookupBackoffDelayMs(failures)
|
||||
errorBackoff.set(key, { failures, retryAt })
|
||||
return retryAt
|
||||
}
|
||||
@@ -509,7 +505,7 @@ function refreshIntervalForCandidate(candidate: GitHubPRRefreshCandidate): numbe
|
||||
return 30 * 60_000
|
||||
}
|
||||
if (candidate.cachedHasPR === false) {
|
||||
return 15 * 60_000
|
||||
return NO_REVIEW_REFRESH_INTERVAL_MS
|
||||
}
|
||||
if (
|
||||
candidate.cachedHasPR === true &&
|
||||
@@ -776,9 +772,8 @@ async function drainQueue(): Promise<void> {
|
||||
// Why: tab/worktree churn can enqueue many distinct active refreshes that each probe local Git.
|
||||
noteActiveStart(next)
|
||||
}
|
||||
for (const bucket of buckets) {
|
||||
noteRepositoryRateLimitSpend(repository, bucket, 1, executionOptions)
|
||||
}
|
||||
// Why (#11532): the lookup itself now debits the snapshot, so every
|
||||
// caller is accounted for; debiting here too would double-count.
|
||||
}
|
||||
|
||||
const outcome = await getPRForBranchOutcome(
|
||||
|
||||
Reference in New Issue
Block a user