fix(ci): background steps are single run steps, so no action reads the event payload while the runner rewrites it (#30795)

* fix(ci): background steps are single run steps, so no JavaScript action reads the event payload while the runner rewrites it

* test(ci): guard the background slot cap and pin the payload-free allowlist

.github/actions/prepare-linux-package-fixture/action.yml

@@ -7,10 +7,16 @@ inputs:
   save-cache:
     description: Build and save a missing fixture on the default branch.
     default: 'false'
+  load:
+    description: Load the image here; 'false' only restores, so the caller can load it in a background run step.
+    default: 'true'
 outputs:
   image:
     description: Loaded image to use as a Docker build cache, or empty on a cache miss.
     value: ${{ steps.image.outputs.image }}
+  cache-hit:
+    description: Whether the fixture archive was restored under the exact key.
+    value: ${{ steps.cache.outputs.cache-hit }}
 runs:
   using: composite
   steps:
@@ -20,39 +26,17 @@ runs:
       continue-on-error: true
       with:
         path: ${{ runner.temp }}/orca-package-fixtures/${{ inputs.fixture }}.tar
-        key: linux-package-fixture-v1-${{ runner.os }}-${{ runner.arch }}-linux-amd64-${{ inputs.fixture }}-${{ hashFiles(format('config/docker/{0}/**', inputs.fixture), '.github/actions/prepare-linux-package-fixture/action.yml') }}
+        key: linux-package-fixture-v1-${{ runner.os }}-${{ runner.arch }}-linux-amd64-${{ inputs.fixture }}-${{ hashFiles(format('config/docker/{0}/**', inputs.fixture), '.github/actions/prepare-linux-package-fixture/action.yml', '.github/actions/prepare-linux-package-fixture/load-fixture.sh') }}
 
     - name: Load or seed fixture build cache
       id: image
+      if: inputs.load != 'false'
       shell: bash
       env:
         FIXTURE: ${{ inputs.fixture }}
         SAVE_CACHE: ${{ inputs.save-cache }}
         CACHE_HIT: ${{ steps.cache.outputs.cache-hit }}
-      run: |
-        set -euo pipefail
-        case "$FIXTURE" in
-          headless-serve-shutdown|cli-launch-contract|daemon-shutdown-descendants) ;;
-          *) echo "Unsupported package fixture: $FIXTURE" >&2; exit 1 ;;
-        esac
-        archive="$RUNNER_TEMP/orca-package-fixtures/$FIXTURE.tar"
-        image="orca-package-fixture-$FIXTURE:cache"
-        if [[ "$CACHE_HIT" == true ]] && docker load --input "$archive"; then
-          if docker image inspect "$image" > /dev/null; then
-            echo "image=$image" >> "$GITHUB_OUTPUT"
-            exit 0
-          fi
-        fi
-        if [[ "$SAVE_CACHE" != true ]]; then
-          echo 'No usable fixture cache; the package runner will build normally.'
-          exit 0
-        fi
-        docker build --platform linux/amd64 --build-arg BUILDKIT_INLINE_CACHE=1 \
-          --tag "$image" --file "config/docker/$FIXTURE/Dockerfile" "config/docker/$FIXTURE"
-        mkdir -p "$(dirname "$archive")"
-        docker save --output "$archive" "$image"
-        echo "image=$image" >> "$GITHUB_OUTPUT"
-        echo 'built=true' >> "$GITHUB_OUTPUT"
+      run: bash "$GITHUB_ACTION_PATH/load-fixture.sh"
 
     - name: Save fixture image
       if: inputs.save-cache == 'true' && steps.cache.outputs.cache-hit != 'true' && steps.image.outputs.built == 'true'

.github/actions/prepare-linux-package-fixture/load-fixture.sh

@@ -0,0 +1,25 @@
+#!/usr/bin/env bash
+# Loads a restored fixture image; only the default-branch warmer builds and exports a missing one.
+set -euo pipefail
+case "$FIXTURE" in
+  headless-serve-shutdown|cli-launch-contract|daemon-shutdown-descendants) ;;
+  *) echo "Unsupported package fixture: $FIXTURE" >&2; exit 1 ;;
+esac
+archive="$RUNNER_TEMP/orca-package-fixtures/$FIXTURE.tar"
+image="orca-package-fixture-$FIXTURE:cache"
+if [[ "${CACHE_HIT:-}" == true ]] && docker load --input "$archive"; then
+  if docker image inspect "$image" > /dev/null; then
+    echo "image=$image" >> "$GITHUB_OUTPUT"
+    exit 0
+  fi
+fi
+if [[ "${SAVE_CACHE:-}" != true ]]; then
+  echo 'No usable fixture cache; the package runner will build normally.'
+  exit 0
+fi
+docker build --platform linux/amd64 --build-arg BUILDKIT_INLINE_CACHE=1 \
+  --tag "$image" --file "config/docker/$FIXTURE/Dockerfile" "config/docker/$FIXTURE"
+mkdir -p "$(dirname "$archive")"
+docker save --output "$archive" "$image"
+echo "image=$image" >> "$GITHUB_OUTPUT"
+echo 'built=true' >> "$GITHUB_OUTPUT"

.github/actions/prepare-wayland-clipboard/action.yml

@@ -5,6 +5,14 @@ inputs:
   save-cache:
     description: Publish a validated helper from the default branch.
     default: 'false'
+  build:
+    description: Build here; 'false' only restores, so the caller can build in a background run step.
+    default: 'true'
+
+outputs:
+  cache-hit:
+    description: Whether the helper was restored under the exact key.
+    value: ${{ steps.cache.outputs.cache-hit }}
 
 runs:
   using: composite
@@ -30,22 +38,14 @@ runs:
         key: ${{ steps.identity.outputs.key }}
 
     - name: Build or validate Wayland clipboard helper
+      if: inputs.build != 'false'
       shell: bash
       env:
         CACHE_HIT: ${{ steps.cache.outputs.cache-hit }}
-      run: |
-        if node config/scripts/build-wayland-clipboard.mjs; then
-          exit 0
-        fi
-        if [ "$CACHE_HIT" != true ]; then
-          exit 1
-        fi
-        echo 'Discarding an unusable Wayland clipboard cache and rebuilding from source.'
-        rm -rf -- native/wayland-clipboard/.build
-        node config/scripts/build-wayland-clipboard.mjs
+      run: bash "$GITHUB_ACTION_PATH/build-or-validate.sh"
 
     - name: Save validated Wayland clipboard helper from main
-      if: inputs.save-cache == 'true' && steps.cache.outputs.cache-hit != 'true' && github.ref == 'refs/heads/main' && github.event_name != 'pull_request'
+      if: inputs.build != 'false' && inputs.save-cache == 'true' && steps.cache.outputs.cache-hit != 'true' && github.ref == 'refs/heads/main' && github.event_name != 'pull_request'
       uses: actions/cache/save@v5
       continue-on-error: true
       with:

.github/actions/prepare-wayland-clipboard/build-or-validate.sh

@@ -0,0 +1,12 @@
+#!/usr/bin/env bash
+# Validates a restored helper and rebuilds once from source when the restored copy is unusable.
+set -e
+if node config/scripts/build-wayland-clipboard.mjs; then
+  exit 0
+fi
+if [ "${CACHE_HIT:-}" != true ]; then
+  exit 1
+fi
+echo 'Discarding an unusable Wayland clipboard cache and rebuilding from source.'
+rm -rf -- native/wayland-clipboard/.build
+node config/scripts/build-wayland-clipboard.mjs

.github/workflows/e2e.yml

@@ -154,10 +154,8 @@ jobs:
       - uses: ./.github/actions/build-mobile-web-bundle
         if: hashFiles('.github/actions/build-mobile-web-bundle/action.yml') != ''
 
-      # Only the completed output upload overlaps the change from Node to Electron ABI.
+      # Foreground: a background JavaScript action reads the event payload while later steps rewrite it.
       - name: Upload E2E build output
-        id: e2e-build-upload
-        background: true
         uses: actions/upload-artifact@v7
         with:
           name: e2e-build-out${{ inputs.artifact_suffix }}
@@ -180,8 +178,6 @@ jobs:
         with:
           native-runtime: electron
 
-      - wait: e2e-build-upload
-
   e2e:
     name: e2e ${{ matrix.shard_name }}
     needs: [build]

.github/workflows/pr.yml

@@ -544,6 +544,7 @@ jobs:
           PREFLIGHT_PHASE_SELECTED: ${{ needs.code_paths.outputs.typecheck == 'true' }}
           PREFLIGHT_PRIOR_SUCCESS: ${{ job.status == 'success' }}
           ORCA_UNIT_SELECTION_MODE: ${{ vars.ORCA_UNIT_SELECTION_MODE || 'shadow' }}
+          ORCA_PULL_REQUEST_DRAFT: ${{ github.event.pull_request.draft }}
           # Five shards use fewer runner minutes; the repository variable can still request ten.
           ORCA_UNIT_FULL_SHARD_COUNT: ${{ vars.ORCA_UNIT_FULL_SHARD_COUNT || '5' }}
         run: |
@@ -1125,11 +1126,20 @@ jobs:
           restore-keys: |
             electron-builder-linux-
 
+      # Restore in the foreground: background steps may only be single run steps (ci-background-steps.test.mjs).
       - uses: ./.github/actions/prepare-linux-package-fixture
-        id: daemon-fixture-cache
-        background: true
+        id: daemon-fixture-cache-restore
         with:
           fixture: daemon-shutdown-descendants
+          load: 'false'
+
+      - name: Load daemon shutdown fixture image
+        id: daemon-fixture-cache
+        background: true
+        env:
+          FIXTURE: daemon-shutdown-descendants
+          CACHE_HIT: ${{ steps.daemon-fixture-cache-restore.outputs.cache-hit }}
+        run: bash .github/actions/prepare-linux-package-fixture/load-fixture.sh
 
       - uses: ./.github/actions/install-node-dependencies
         with:
@@ -1171,24 +1181,48 @@ jobs:
           src/main/browser/browser-popup-native-compatibility.electron.test.ts
           src/main/browser/browser-popup-background-compatibility.electron.test.ts
 
+      - name: Restore native components
+        id: linux-native-restore
+        uses: ./.github/actions/prepare-wayland-clipboard
+        with:
+          build: 'false'
+
       # The isolated glibc-floor build writes only the Wayland helper, so JavaScript can overlap.
       - name: Build native components
         id: linux-native
         background: true
-        uses: ./.github/actions/prepare-wayland-clipboard
+        env:
+          CACHE_HIT: ${{ steps.linux-native-restore.outputs.cache-hit }}
+        run: bash .github/actions/prepare-wayland-clipboard/build-or-validate.sh
 
-      # Restore independent fixtures during builds; keep native lifecycle probes serial.
+      # Load independent fixtures during builds; keep native lifecycle probes serial.
       - uses: ./.github/actions/prepare-linux-package-fixture
-        id: shutdown-fixture-cache
-        background: true
+        id: shutdown-fixture-cache-restore
         with:
           fixture: headless-serve-shutdown
+          load: 'false'
+
+      - name: Load headless serve shutdown fixture image
+        id: shutdown-fixture-cache
+        background: true
+        env:
+          FIXTURE: headless-serve-shutdown
+          CACHE_HIT: ${{ steps.shutdown-fixture-cache-restore.outputs.cache-hit }}
+        run: bash .github/actions/prepare-linux-package-fixture/load-fixture.sh
 
       - uses: ./.github/actions/prepare-linux-package-fixture
-        id: cli-fixture-cache
-        background: true
+        id: cli-fixture-cache-restore
         with:
           fixture: cli-launch-contract
+          load: 'false'
+
+      - name: Load CLI launch contract fixture image
+        id: cli-fixture-cache
+        background: true
+        env:
+          FIXTURE: cli-launch-contract
+          CACHE_HIT: ${{ steps.cli-fixture-cache-restore.outputs.cache-hit }}
+        run: bash .github/actions/prepare-linux-package-fixture/load-fixture.sh
 
       - name: Install Linux package tooling
         id: linux-package-tools

.github/workflows/unit-plan.yml

@@ -39,6 +39,7 @@ jobs:
         id: plan
         env:
           ORCA_UNIT_SELECTION_MODE: ${{ inputs.selection_mode }}
+          ORCA_PULL_REQUEST_DRAFT: ${{ github.event.pull_request.draft }}
         run: node config/scripts/ci-unit-plan.mjs
       - uses: actions/upload-artifact@v7
         continue-on-error: true

config/scripts/build-wayland-clipboard.test.mjs

@@ -173,6 +173,7 @@ describe('Wayland clipboard CI cache recovery', () => {
         fi
         return "$REBUILD_STATUS"
       }
+      export -f node
       ${build.run}
     `
     const result = runProcessSync({
@@ -181,6 +182,7 @@ describe('Wayland clipboard CI cache recovery', () => {
       cwd: root,
       env: {
         ...process.env,
+        GITHUB_ACTION_PATH: join(process.cwd(), '.github/actions/prepare-wayland-clipboard'),
         CACHE_HIT: String(hit),
         INITIAL_STATUS: String(initial),
         REBUILD_STATUS: String(rebuild)

config/scripts/ci-background-steps.test.mjs

@@ -0,0 +1,95 @@
+import { existsSync, readdirSync, readFileSync, statSync } from 'node:fs'
+import { join } from 'node:path'
+import { parse } from 'yaml'
+import { expect, it } from 'vitest'
+
+// Why: the runner rewrites $GITHUB_EVENT_PATH in place (truncate, then write) at every step start,
+// and background steps start on their own task. A JavaScript action (@actions/github parses the
+// payload at import) or composite (its inner steps start later) in the background therefore races
+// the foreground's step starts and reads an empty file: "Unexpected end of JSON input".
+const workflowFiles = readdirSync('.github/workflows')
+  .filter((name) => /\.ya?ml$/.test(name))
+  .map((name) => join('.github/workflows', name))
+const actionFiles = readdirSync('.github/actions')
+  .map((name) => join('.github/actions', name, 'action.yml'))
+  .filter((file) => existsSync(file))
+
+// Single-step JavaScript actions whose bundle never reads the payload (checked by searching dist/index.js).
+const payloadFreeActions = ['ruby/setup-ruby@v1']
+// The runner runs 10 at once; a queued step starts (and rewrites the payload) mid-foreground-step.
+const backgroundSlots = 10
+
+function stepLists(file) {
+  const document = parse(readFileSync(file, 'utf8'))
+  return document.jobs
+    ? Object.entries(document.jobs).map(([job, value]) => [job, value.steps ?? []])
+    : [['composite', document.runs?.steps ?? []]]
+}
+
+function backgroundSteps() {
+  const found = []
+  for (const file of [...workflowFiles, ...actionFiles]) {
+    for (const [job, steps] of stepLists(file)) {
+      for (const step of steps) {
+        if (step.background) {
+          found.push({ where: `${file} ${job} ${step.id ?? step.name}`, step })
+        }
+      }
+    }
+  }
+  return found
+}
+
+function readsEventPayload(run) {
+  const scripts = [...run.matchAll(/(?:config\/scripts|\.github\/actions)\/[\w./-]+/g)].map(
+    (match) => match[0]
+  )
+  return (
+    run.includes('GITHUB_EVENT_PATH') ||
+    scripts.some(
+      (file) =>
+        existsSync(file) &&
+        statSync(file).isFile() &&
+        readFileSync(file, 'utf8').includes('GITHUB_EVENT_PATH')
+    )
+  )
+}
+
+it('finds the background steps it guards', () => {
+  expect(backgroundSteps().length).toBeGreaterThan(10)
+})
+
+it('backgrounds only single run steps, never actions that start later steps or parse the payload', () => {
+  const actions = backgroundSteps()
+    .filter(({ step }) => typeof step.run !== 'string' && !payloadFreeActions.includes(step.uses))
+    .map(({ where }) => where)
+  expect(actions).toEqual([])
+})
+
+it('keeps the event payload out of background run steps', () => {
+  const readers = backgroundSteps()
+    .filter(({ step }) => typeof step.run === 'string' && readsEventPayload(step.run))
+    .map(({ where }) => where)
+  expect(readers).toEqual([])
+})
+
+it('never queues a background step behind a full slot pool', () => {
+  const crowded = []
+  for (const file of workflowFiles) {
+    for (const [job, steps] of stepLists(file)) {
+      const open = new Set()
+      for (const [index, step] of steps.entries()) {
+        if (step.background) {
+          open.add(step.id ?? index)
+        }
+        for (const id of [step.wait ?? []].flat()) {
+          open.delete(id)
+        }
+        if (open.size > backgroundSlots) {
+          crowded.push(`${file} ${job}`)
+        }
+      }
+    }
+  }
+  expect(crowded).toEqual([])
+})

config/scripts/ci-native-cache.test.mjs

@@ -187,7 +187,6 @@ describe('CI native cache ownership', () => {
     const cliWait = steps.find((step) => step.wait === 'e2e-cli')
     const electron = steps.find((step) => step.id === 'electron-native-runtime')
     const upload = steps.find((step) => step.name === 'Upload E2E build output')
-    const uploadWait = steps.find((step) => step.wait === upload.id)
     expect(install.with).toMatchObject({
       'native-runtime': 'node',
       'persist-native-cache': 'false',
@@ -199,11 +198,11 @@ describe('CI native cache ownership', () => {
       key: '${{ steps.deps.outputs.native-cache-key }}',
       path: '${{ steps.deps.outputs.native-cache-path }}'
     })
-    const ordered = [install, save, build, cliWait, upload, electron, uploadWait]
+    const ordered = [install, save, build, cliWait, upload, electron]
     expect(ordered.map((step) => steps.indexOf(step))).toEqual(
       ordered.map((step) => steps.indexOf(step)).sort((left, right) => left - right)
     )
-    expect(upload.background).toBe(true)
+    expect(upload.background).toBeUndefined()
     expect(upload.if).toBeUndefined()
     expect(electron.background).toBeUndefined()
     expect(electron.uses).toBe('./.github/actions/prepare-native-runtime')
@@ -211,7 +210,6 @@ describe('CI native cache ownership', () => {
       'native-runtime': 'electron',
       'node-version': '${{ steps.deps.outputs.node-version }}'
     })
-    expect(uploadWait).toEqual({ wait: upload.id })
     for (const name of ['e2e', 'changed-e2e', 'orcad-serve-mode-switch', 'orcad-ssh-docker']) {
       expect(e2e.jobs[name].needs, name).toEqual(['build'])
       expect(

config/scripts/ci-unit-plan.mjs

@@ -1,4 +1,4 @@
-import { appendFileSync, readFileSync } from 'node:fs'
+import { appendFileSync } from 'node:fs'
 import { pathToFileURL } from 'node:url'
 import { runProcessSync } from '@orca/process-host'
 import { collectUnitDependencyGraph } from './ci-unit-dependency-graph.mjs'
@@ -15,7 +15,8 @@ export function prepareUnitPlan(env = process.env) {
   const files = discoverUnitFiles()
   let plan
   try {
-    const event = JSON.parse(readFileSync(env.GITHUB_EVENT_PATH, 'utf8'))
+    // From the workflow, not the payload file: background steps can read it mid-rewrite.
+    const event = { pull_request: { draft: env.ORCA_PULL_REQUEST_DRAFT === 'true' } }
     if (env.GITHUB_EVENT_NAME !== 'pull_request') {
       throw new Error('Full reference run')
     }

config/scripts/ci-unit-plan.test.mjs

@@ -48,8 +48,6 @@ it.each([
         git(commit)
       }
       const sourceSha = git(['rev-parse', 'HEAD'])
-      const eventPath = join(root, 'event.json')
-      writeFileSync(eventPath, JSON.stringify({ pull_request: { draft: true } }))
       const result = runProcessSync({
         program: process.execPath,
         args: [fileURLToPath(new URL('./ci-unit-plan.mjs', import.meta.url))],
@@ -59,7 +57,7 @@ it.each([
           ORCA_BACKGROUND_LAUNCH: '1',
           ORCA_UNIT_SELECTION_MODE: 'selected',
           GITHUB_EVENT_NAME: 'pull_request',
-          GITHUB_EVENT_PATH: eventPath,
+          ORCA_PULL_REQUEST_DRAFT: 'true',
           GITHUB_SHA: sourceSha,
           ORCA_SHARD_SOURCE_SHA: sourceSha,
           GITHUB_OUTPUT: join(root, 'outputs'),
@@ -95,8 +93,6 @@ it.each(['5', '10', 'invalid'])(
     try {
       mkdirSync(join(root, 'src'))
       writeFileSync(join(root, 'src/retained.test.ts'), 'export const retained = true')
-      const eventPath = join(root, 'event.json')
-      writeFileSync(eventPath, '{}')
       const result = runProcessSync({
         program: process.execPath,
         args: [fileURLToPath(new URL('./ci-unit-plan.mjs', import.meta.url))],
@@ -106,7 +102,6 @@ it.each(['5', '10', 'invalid'])(
           ORCA_BACKGROUND_LAUNCH: '1',
           ORCA_UNIT_FULL_SHARD_COUNT: requestedCount,
           GITHUB_EVENT_NAME: 'workflow_dispatch',
-          GITHUB_EVENT_PATH: eventPath,
           ORCA_SHARD_SOURCE_SHA: 'full-count-fixture'
         }
       })

config/scripts/linux-package-fixture-cache.test.mjs

@@ -53,6 +53,7 @@ function exercise({
       ...process.env,
       ORCA_BACKGROUND_LAUNCH: '1',
       PATH: `${bin}${delimiter}${process.env.PATH ?? ''}`,
+      GITHUB_ACTION_PATH: join(process.cwd(), '.github/actions/prepare-linux-package-fixture'),
       RUNNER_TEMP: directory,
       GITHUB_OUTPUT: output,
       COMMAND_LOG: commandLog,
@@ -113,16 +114,30 @@ it('uses exact context/platform keys and the same restore-only action in package
   expect(cache.with.key).toContain('linux-amd64')
   expect(cache.with.key).toContain("format('config/docker/{0}/**', inputs.fixture)")
   expect(cache.with.key).toContain('.github/actions/prepare-linux-package-fixture/action.yml')
+  expect(cache.with.key).toContain('.github/actions/prepare-linux-package-fixture/load-fixture.sh')
   expect(cache.with['restore-keys']).toBeUndefined()
   expect(action.inputs['save-cache'].default).toBe('false')
   const prepared = workflow('pr').jobs.package.steps.filter((step) =>
     step.uses?.endsWith('/prepare-linux-package-fixture')
   )
   expect(prepared.map((step) => step.with)).toEqual([
-    { fixture: 'daemon-shutdown-descendants' },
-    { fixture: 'headless-serve-shutdown' },
-    { fixture: 'cli-launch-contract' }
+    { fixture: 'daemon-shutdown-descendants', load: 'false' },
+    { fixture: 'headless-serve-shutdown', load: 'false' },
+    { fixture: 'cli-launch-contract', load: 'false' }
   ])
+  const steps = workflow('pr').jobs.package.steps
+  for (const restore of prepared) {
+    // Only the single load step runs in the background; the restore action stays foreground.
+    const loader = steps.find((step) => step.env?.CACHE_HIT?.includes(`steps.${restore.id}.`))
+    expect(restore.background).toBeUndefined()
+    expect(loader).toMatchObject({
+      id: restore.id.replace(/-restore$/, ''),
+      background: true,
+      env: { FIXTURE: restore.with.fixture },
+      run: 'bash .github/actions/prepare-linux-package-fixture/load-fixture.sh'
+    })
+    expect(steps.indexOf(loader)).toBe(steps.findIndex((step) => step.id === restore.id) + 1)
+  }
   const platformSource = workflow('ci-cache-warmup-platforms').jobs['warm-platforms'].uses
   const warm = parse(readFileSync(platformSource, 'utf8')).jobs['warm-linux-package-fixtures']
   expect(warm.steps.slice(1).map((step) => step.with.fixture)).toEqual(
@@ -166,6 +181,7 @@ it.each([
 
 it.each([
   '.github/actions/prepare-linux-package-fixture/action.yml',
+  '.github/actions/prepare-linux-package-fixture/load-fixture.sh',
   'config/docker/daemon-shutdown-descendants/Dockerfile',
   'config/docker/headless-serve-shutdown/Dockerfile',
   'config/docker/cli-launch-contract/Dockerfile'