fix(github): enqueue a merge-queue PR when auto-merge is disabled (#24327)

On repos whose base branch requires a merge queue and that disable
auto-merge, gh pr merge --auto always calls enablePullRequestAutoMerge,
which GitHub rejects. Call the enqueuePullRequest mutation directly in
that case and report enqueue failures as merge-queue errors.

Fixes #19638

Co-authored-by: Neil <neil@stably.ai>

src/main/github/client-merge-queue-auto-merge.test.ts

@@ -387,6 +387,48 @@ describe('GitHub GraphQL rate-limit guard', () => {
     ).toBe(false)
   })
 
+  it('enqueues instead of gh pr merge --auto when a required merge queue disallows auto-merge', async () => {
+    ghExecFileAsyncMock
+      .mockResolvedValueOnce({ stdout: JSON.stringify({ stack: null }) })
+      .mockResolvedValueOnce({
+        stdout: JSON.stringify({ id: 'PR_kwDO123', headRefOid: 'head-oid', baseRefName: 'main' })
+      })
+      .mockResolvedValueOnce({
+        stdout: JSON.stringify({
+          data: { repository: { mergeQueue: { id: 'MQ_kw' }, autoMergeAllowed: false } }
+        })
+      })
+      .mockResolvedValue({ stdout: '', stderr: '' })
+
+    await expect(
+      setPRAutoMerge('/repo-root', 7, true, 'squash', undefined, {
+        owner: 'stablyai',
+        repo: 'orca',
+        host: 'github.com'
+      })
+    ).resolves.toEqual({ ok: true })
+
+    expect(ghExecFileAsyncMock).toHaveBeenCalledTimes(4)
+    expect(ghExecFileAsyncMock).toHaveBeenNthCalledWith(
+      4,
+      [
+        'api',
+        'graphql',
+        '-f',
+        expect.stringContaining('enqueuePullRequest('),
+        '-f',
+        'pullRequestId=PR_kwDO123',
+        '-f',
+        'expectedHeadOid=head-oid'
+      ],
+      expect.objectContaining({
+        cwd: '/repo-root',
+        env: expect.objectContaining({ GH_PROMPT_DISABLED: '1' }),
+        host: 'github.com'
+      })
+    )
+  })
+
   it('blocks direct merge when GitHub reports required approval', async () => {
     ghExecFileAsyncMock
       .mockResolvedValueOnce({ stdout: JSON.stringify({ stack: null }) })

src/main/github/client/merge/pr-auto-merge.ts

@@ -6,6 +6,7 @@ import { resolveGitHubRepoExecution, type GitHubApiRepository } from '../../gith
 import { githubPRStackExecutionScope, type GhExecOptions } from './../github-exec-scope'
 import { detectRepositoryMergeMetadata } from './../detect/repository-merge-metadata'
 import { getRestPRByNumber } from './../lookup/pr-number-lookup'
+import { errorMessage } from '../../../../shared/error-message'
 export const PR_AUTO_MERGE_IDENTITY_JSON_FIELDS = 'id,headRefOid,baseRefName'
 
 export const GITHUB_AUTO_MERGE_METHODS: Record<GitHubPRMergeMethod, 'MERGE' | 'SQUASH' | 'REBASE'> =
@@ -66,14 +67,14 @@ export async function runPRAutoMergeCommand(
   })
 }
 
-export async function shouldUseMergeQueueAutoMerge(
+async function resolveMergeQueueAutoMergePath(
   pr: PRAutoMergeIdentity,
   ownerRepo: GitHubApiRepository | null,
   ghOptions: GhExecOptions,
   executionScope?: string
-): Promise<boolean> {
+): Promise<'enqueue' | 'gh-auto' | null> {
   if (!ownerRepo || !pr.baseRefName) {
-    return false
+    return null
   }
   const mergeMetadata = await detectRepositoryMergeMetadata(
     ownerRepo,
@@ -81,7 +82,33 @@ export async function shouldUseMergeQueueAutoMerge(
     ghOptions,
     executionScope
   )
-  return mergeMetadata.mergeQueueRequired === true
+  if (mergeMetadata.mergeQueueRequired !== true) {
+    return null
+  }
+  // Why: gh pr merge --auto calls enablePullRequestAutoMerge, which GitHub rejects when the repo disables auto-merge.
+  return mergeMetadata.autoMergeAllowed === false ? 'enqueue' : 'gh-auto'
+}
+
+async function enqueuePullRequest(
+  pr: PRAutoMergeIdentity,
+  ghOptions: GhExecOptions
+): Promise<void> {
+  const query = `mutation($pullRequestId: ID!, $expectedHeadOid: GitObjectID) {
+    enqueuePullRequest(input: {
+      pullRequestId: $pullRequestId,
+      expectedHeadOid: $expectedHeadOid
+    }) {
+      mergeQueueEntry { id }
+    }
+  }`
+  const args = ['api', 'graphql', '-f', `query=${query}`, '-f', `pullRequestId=${pr.id}`]
+  if (pr.headRefOid) {
+    args.push('-f', `expectedHeadOid=${pr.headRefOid}`)
+  }
+  await ghExecFileAsync(args, {
+    ...ghOptions,
+    env: { ...process.env, GH_PROMPT_DISABLED: '1' }
+  })
 }
 
 export async function enablePRAutoMerge(
@@ -108,8 +135,27 @@ export async function enablePRAutoMerge(
   if (!pr?.id) {
     return { ok: false, error: 'Could not resolve GitHub pull request ID' }
   }
-  const useMergeQueue = await shouldUseMergeQueueAutoMerge(pr, ownerRepo, ghOptions, executionScope)
-  if (useMergeQueue) {
+  const mergeQueuePath = await resolveMergeQueueAutoMergePath(
+    pr,
+    ownerRepo,
+    ghOptions,
+    executionScope
+  )
+  if (mergeQueuePath === 'enqueue') {
+    try {
+      await enqueuePullRequest(pr, ghOptions)
+    } catch (err) {
+      return {
+        ok: false,
+        error: classifyPullRequestUpdateError(
+          errorMessage(err),
+          'add pull request to the merge queue'
+        ).message
+      }
+    }
+    return { ok: true }
+  }
+  if (mergeQueuePath === 'gh-auto') {
     await runPRAutoMergeCommand(prNumber, method, ownerRepo, ghOptions)
     return { ok: true }
   }