ci(relay-staging): split the staging load across two hosted runners (#27402)

One hosted runner tops out near 2,048 connections to one cell (4 shards each stalled at 512 with
control_open_timeout), short of the 2,840 the Asia flip-back test needs.

.github/workflows/cloud-prove-relay-asia-staging.yml

@@ -39,6 +39,14 @@ on:
         default: '4'
         type: choice
         options: ['1', '2', '3', '4']
+      load-runners:
+        description: >-
+          Load only, hosted runners only. Runner machines, each with load-shards generators; one
+          hosted runner holds about 2,000 connections to one cell
+        required: false
+        default: '1'
+        type: choice
+        options: ['1', '2']
       load-args:
         description: >-
           Load only. Per-shard generator arguments, e.g. --controls 150 --splices 15
@@ -367,6 +375,11 @@ jobs:
   load:
     if: ${{ vars.ORCA_CLOUD_OPERATIONS_ENABLED == 'true' && (github.ref == 'refs/heads/main') && inputs.mode == 'load' }}
     runs-on: ${{ inputs.load-region == 'asia-east2' && fromJSON('["self-hosted","linux","x64","relay-asia-east2-load"]') || 'ubuntu-latest' }}
+    strategy:
+      # One runner's failure must not drop the other half of the load mid-run.
+      fail-fast: false
+      matrix:
+        runner: ${{ fromJSON(inputs.load-runners == '2' && '[0, 1]' || '[0]') }}
     timeout-minutes: 180
     environment: staging
     env:
@@ -374,6 +387,8 @@ jobs:
       AUTH_ORIGIN: https://auth-staging.onorca.dev
       LOAD_REGION: ${{ inputs.load-region }}
       LOAD_SHARDS: ${{ inputs.load-shards }}
+      LOAD_RUNNERS: ${{ inputs.load-runners }}
+      LOAD_RUNNER_INDEX: ${{ matrix.runner }}
       LOAD_ARGS: ${{ inputs.load-args }}
     steps:
       - uses: actions/checkout@v4
@@ -387,6 +402,12 @@ jobs:
           test "${CONFIRMATION}" = LOAD_RELAY_STAGING
           [[ "${LOAD_REGION}" =~ ^(asia-east2|asia-east2-hosted|us-central1)$ ]]
           [[ "${LOAD_SHARDS}" =~ ^[1-4]$ ]]
+          [[ "${LOAD_RUNNERS}" =~ ^[12]$ ]]
+          # The Asia runner is one machine; a second matrix job would only queue behind the first.
+          if test "${LOAD_RUNNERS}" != 1 && test "${LOAD_REGION}" = asia-east2; then
+            echo "load-runners 2 needs a hosted runner" >&2
+            exit 1
+          fi
           read -r -a args <<< "${LOAD_ARGS}"
           for arg in "${args[@]}"; do
             [[ "${arg}" =~ ^[A-Za-z0-9=:/._,-]+$ ]] || { echo "unsupported characters in load-args" >&2; exit 1; }
@@ -449,8 +470,8 @@ jobs:
               --auth-origin "${AUTH_ORIGIN}" \
               --preferred-region "${LOAD_REGION%-hosted}" \
               --relay-asia-load-principals 32 \
-              --shard-count "${LOAD_SHARDS}" \
-              --shard-index "${shard}" \
+              --shard-count "$((LOAD_SHARDS * LOAD_RUNNERS))" \
+              --shard-index "$((LOAD_RUNNER_INDEX * LOAD_SHARDS + shard))" \
               "${args[@]}" \
               > "${load_dir}/shard-${shard}.jsonl" &
             pids+=("$!")
@@ -470,7 +491,7 @@ jobs:
         if: ${{ always() }}
         uses: actions/upload-artifact@v4
         with:
-          name: relay-staging-load-${{ github.run_id }}-${{ github.run_attempt }}
+          name: relay-staging-load-${{ github.run_id }}-${{ github.run_attempt }}-${{ matrix.runner }}
           path: ${{ runner.temp }}/relay-staging-load/
           if-no-files-found: warn
           retention-days: 7