From 4cc214fef70fbcc5bcbd38103c5af6285be873eb Mon Sep 17 00:00:00 2001 From: Neil <4138956+nwparker@users.noreply.github.com> Date: Sat, 5 Sep 2026 17:02:58 -0700 Subject: [PATCH] ci: avoid idle Mac allocations and cached native toolchain installs --- .../install-node-dependencies/action.yml | 24 +++-- .github/workflows/hourly-mac-build.yml | 90 +++++++++--------- config/scripts/ci-native-toolchain.test.mjs | 69 ++++++++++++++ .../hourly-preflight-workflow.test.mjs | 91 +++++++++++++++++++ docs/reference/ci-runner-efficiency.md | 38 +++++++- 5 files changed, 261 insertions(+), 51 deletions(-) create mode 100644 config/scripts/ci-native-toolchain.test.mjs create mode 100644 config/scripts/hourly-preflight-workflow.test.mjs diff --git a/.github/actions/install-node-dependencies/action.yml b/.github/actions/install-node-dependencies/action.yml index e36ec4c65d8..7695d2bec9b 100644 --- a/.github/actions/install-node-dependencies/action.yml +++ b/.github/actions/install-node-dependencies/action.yml @@ -77,14 +77,6 @@ runs: ;; esac - # pnpm's bundled gyp_main.py is not executable on fresh Linux runners. - - name: Use external node-gyp - if: runner.os == 'Linux' && inputs.native-runtime != 'none' - shell: bash - run: | - npm install -g node-gyp@11.5.0 - echo "npm_config_node_gyp=$(npm root -g)/node-gyp/bin/node-gyp.js" >> "$GITHUB_ENV" - - name: Prepare dependency install shell: bash run: | @@ -175,6 +167,22 @@ runs: node_modules/.pnpm/@vscode+windows-process-tree@*/node_modules/@vscode/windows-process-tree/build key: native-modules-${{ runner.os }}-${{ steps.native-cache-scope.outputs.scope }}-${{ runner.arch }}-${{ inputs.native-runtime }}-node${{ steps.requested-node.outputs.node-version || steps.default-node.outputs.node-version }}-${{ hashFiles('pnpm-lock.yaml', '.github/actions/install-node-dependencies/action.yml', 'config/scripts/ensure-native-runtime.mjs', 'config/scripts/rebuild-native-deps.mjs', 'config/patches/node-pty@1.1.0.patch', 'config/patches/@vscode__windows-process-tree@0.8.0.patch') }} + # pnpm's bundled gyp_main.py is not executable on fresh Linux runners. + - name: Use external node-gyp + if: runner.os == 'Linux' && inputs.native-runtime != 'none' + shell: bash + env: + NATIVE_RUNTIME: ${{ inputs.native-runtime }} + NATIVE_CACHE_HIT: ${{ steps.native-cache-restore.outputs.cache-hit || steps.native-cache-restore-only.outputs.cache-hit }} + run: | + # A cache hit can contain unusable addons; probe before skipping the rebuild toolchain. + if [ "$NATIVE_RUNTIME" = node ] && [ "$NATIVE_CACHE_HIT" = true ] && + node config/scripts/ensure-native-runtime.mjs --check-only; then + exit 0 + fi + npm install -g node-gyp@11.5.0 + echo "npm_config_node_gyp=$(npm root -g)/node-gyp/bin/node-gyp.js" >> "$GITHUB_ENV" + - name: Prepare native runtime if: inputs.native-runtime != 'none' shell: bash diff --git a/.github/workflows/hourly-mac-build.yml b/.github/workflows/hourly-mac-build.yml index ac3af92a3bc..c300b2543b8 100644 --- a/.github/workflows/hourly-mac-build.yml +++ b/.github/workflows/hourly-mac-build.yml @@ -26,7 +26,7 @@ name: Hourly macOS Dev Build # HOURLY_RELEASE_APP_ID the App's numeric id # HOURLY_RELEASE_APP_PRIVATE_KEY the App's .pem private key # -# Installation tokens live one hour, which is why this mints twice. Install and +# Installation tokens live one hour, so the build job mints twice. Install and # build need no token at all, and notarization can hold the publish step for tens # of minutes; minting again once the build is done starts the clock at the first # call that actually uses it rather than burning a third of it on `pnpm install`. @@ -60,33 +60,15 @@ env: HOURLY_RETAIN_COUNT: 72 jobs: - build-hourly-mac: + # Avoid occupying the limited Mac pool when main has not moved. + preflight: if: github.repository == 'stablyai/orca' + runs-on: ubuntu-latest + timeout-minutes: 5 outputs: - tag: ${{ steps.release.outputs.tag }} - version: ${{ steps.hourly.outputs.version }} + should_build: ${{ steps.freshness.outputs.should_build }} head_sha: ${{ steps.freshness.outputs.head_sha }} - published: ${{ steps.publish_live.outcome == 'success' && 'true' || 'false' }} - runs-on: blacksmith-6vcpu-macos-15 - # Why 150: it must exceed the worst case the retry budgets below can produce - # (install 3x10 + publish 2x45 = 120, plus ~25 for checkout/build/verify), or - # the job is killed mid-retry and no cleanup step runs at all. A typical run - # is far shorter — this is the notary queue's tail, not its median. - timeout-minutes: 150 - env: - NODE_OPTIONS: --max-old-space-size=4096 steps: - - name: Checkout - uses: actions/checkout@v6 - with: - ref: main - fetch-depth: 0 - # Why: this job only reads stablyai/orca and never pushes; every write - # goes to the hourly repo through a minted App token passed by env. - # Not persisting the checkout credential shrinks the blast radius if a - # build step is compromised (zizmor: artipacked). - persist-credentials: false - - name: Mint hourly repo token id: app_token uses: actions/create-github-app-token@v2 @@ -95,18 +77,19 @@ jobs: private-key: ${{ secrets.HOURLY_RELEASE_APP_PRIVATE_KEY }} owner: stablyai repositories: orca-hourly + permission-contents: read - # Why: main is often idle overnight. Rebuilding an unchanged commit burns a - # runner hour and adds a redundant tag to the retention window. - name: Check whether main moved since the last hourly id: freshness shell: bash env: GH_TOKEN: ${{ steps.app_token.outputs.token }} + MAIN_REPO_TOKEN: ${{ github.token }} FORCED: ${{ github.event_name == 'workflow_dispatch' && inputs.force }} run: | set -euo pipefail - head_sha="$(git rev-parse HEAD)" + head_sha="$(GH_TOKEN="$MAIN_REPO_TOKEN" gh api "repos/$GITHUB_REPOSITORY/commits/main" --jq .sha)" + [[ "$head_sha" =~ ^[0-9a-f]{40}$ ]] || { echo "::error::Could not resolve main"; exit 1; } echo "head_sha=$head_sha" >>"$GITHUB_OUTPUT" if [[ "$FORCED" == "true" ]]; then echo "should_build=true" >>"$GITHUB_OUTPUT" @@ -133,21 +116,55 @@ jobs: echo "main moved to $head_sha (last hourly built $last_sha); building." fi + build-hourly-mac: + needs: preflight + if: needs.preflight.outputs.should_build == 'true' + outputs: + tag: ${{ steps.release.outputs.tag }} + version: ${{ steps.hourly.outputs.version }} + head_sha: ${{ needs.preflight.outputs.head_sha }} + published: ${{ steps.publish_live.outcome == 'success' && 'true' || 'false' }} + runs-on: blacksmith-6vcpu-macos-15 + # Why 150: it must exceed the worst case the retry budgets below can produce + # (install 3x10 + publish 2x45 = 120, plus ~25 for checkout/build/verify), or + # the job is killed mid-retry and no cleanup step runs at all. A typical run + # is far shorter — this is the notary queue's tail, not its median. + timeout-minutes: 150 + env: + NODE_OPTIONS: --max-old-space-size=4096 + steps: + - name: Checkout + uses: actions/checkout@v6 + with: + ref: ${{ needs.preflight.outputs.head_sha }} + fetch-depth: 0 + # Why: this job only reads stablyai/orca and never pushes; every write + # goes to the hourly repo through a minted App token passed by env. + # Not persisting the checkout credential shrinks the blast radius if a + # build step is compromised (zizmor: artipacked). + persist-credentials: false + + - name: Mint hourly repo token + id: app_token + uses: actions/create-github-app-token@v2 + with: + app-id: ${{ secrets.HOURLY_RELEASE_APP_ID }} + private-key: ${{ secrets.HOURLY_RELEASE_APP_PRIVATE_KEY }} + owner: stablyai + repositories: orca-hourly + - name: Setup pnpm - if: steps.freshness.outputs.should_build == 'true' uses: pnpm/setup@v2 with: install: false - name: Setup Node.js - if: steps.freshness.outputs.should_build == 'true' uses: actions/setup-node@v6 with: node-version-file: package.json cache: pnpm - name: Cache electron-builder downloads - if: steps.freshness.outputs.should_build == 'true' uses: actions/cache@v5 with: path: | @@ -158,7 +175,6 @@ jobs: electron-builder-mac- - name: Install dependencies - if: steps.freshness.outputs.should_build == 'true' uses: nick-fields/retry@v4 with: timeout_minutes: 10 @@ -169,7 +185,6 @@ jobs: # Why: signing is what makes an hourly installable over an existing Orca, so # a missing cert must fail here rather than after a 20-minute build. - name: Verify macOS signing environment - if: steps.freshness.outputs.should_build == 'true' run: node config/scripts/verify-macos-release-env.mjs env: CSC_LINK: ${{ secrets.MAC_CERTS }} @@ -180,7 +195,6 @@ jobs: - name: Compute hourly version id: hourly - if: steps.freshness.outputs.should_build == 'true' shell: bash env: GH_TOKEN: ${{ steps.app_token.outputs.token }} @@ -211,7 +225,7 @@ jobs: node config/scripts/hourly-build-version.mjs \ >"$RUNNER_TEMP/hourly-identity.txt" grep -E '^(version|build_number)=' "$RUNNER_TEMP/hourly-identity.txt" - # Why check rather than trust: the checkout above pins `ref: main`, but a + # Why check rather than trust: the checkout above pins the resolved main commit, but a # workflow_dispatch runs this file from whatever branch was dispatched. A # branch that edits this step while main still has the old script yields # an empty name and an untitled release — silent, and only visible once @@ -223,7 +237,6 @@ jobs: cat "$RUNNER_TEMP/hourly-identity.txt" >>"$GITHUB_OUTPUT" - name: Build app - if: steps.freshness.outputs.should_build == 'true' run: pnpm build:release env: NODE_OPTIONS: --max-old-space-size=4096 @@ -239,7 +252,6 @@ jobs: # part the full budget. - name: Re-mint hourly repo token for publish id: app_token_publish - if: steps.freshness.outputs.should_build == 'true' uses: actions/create-github-app-token@v2 with: app-id: ${{ secrets.HOURLY_RELEASE_APP_ID }} @@ -249,13 +261,12 @@ jobs: - name: Create hourly release id: release - if: steps.freshness.outputs.should_build == 'true' shell: bash env: GH_TOKEN: ${{ steps.app_token_publish.outputs.token }} TAG: v${{ steps.hourly.outputs.version }} NAME: ${{ steps.hourly.outputs.name }} - SHA: ${{ steps.freshness.outputs.head_sha }} + SHA: ${{ needs.preflight.outputs.head_sha }} run: | set -euo pipefail # Kept at 12 even though the title shows 7: the freshness check above @@ -291,7 +302,6 @@ jobs: echo "tag=$TAG" >>"$GITHUB_OUTPUT" - name: Publish hourly macOS artifacts - if: steps.freshness.outputs.should_build == 'true' uses: nick-fields/retry@v4 with: # Why 45 like the release pipeline: an attempt is pack + notarize + @@ -322,7 +332,6 @@ jobs: # release missing that manifest is a tag the picker offers and the download # 404s on, so fail loudly instead of leaving a broken entry. - name: Verify update manifest published - if: steps.freshness.outputs.should_build == 'true' shell: bash env: GH_TOKEN: ${{ steps.app_token_publish.outputs.token }} @@ -352,7 +361,6 @@ jobs: # means the picker can never offer a release whose assets are incomplete. - name: Publish the verified release id: publish_live - if: steps.freshness.outputs.should_build == 'true' shell: bash env: GH_TOKEN: ${{ steps.app_token_publish.outputs.token }} diff --git a/config/scripts/ci-native-toolchain.test.mjs b/config/scripts/ci-native-toolchain.test.mjs new file mode 100644 index 00000000000..e35437da77c --- /dev/null +++ b/config/scripts/ci-native-toolchain.test.mjs @@ -0,0 +1,69 @@ +import { execFileSync } from 'node:child_process' +import { chmodSync, mkdtempSync, readFileSync, rmSync, writeFileSync } from 'node:fs' +import { tmpdir } from 'node:os' +import { join } from 'node:path' +import { parse } from 'yaml' +import { describe, expect, it } from 'vitest' + +const steps = parse(readFileSync('.github/actions/install-node-dependencies/action.yml', 'utf8')) + .runs.steps +const toolchain = steps.find((step) => step.name === 'Use external node-gyp') + +describe('CI native toolchain preparation', () => { + it('probes only after both cache restore variants and before native rebuilding', () => { + const index = steps.indexOf(toolchain) + for (const id of ['native-cache-restore', 'native-cache-restore-only']) { + expect(index).toBeGreaterThan(steps.findIndex((step) => step.id === id)) + expect(toolchain.env.NATIVE_CACHE_HIT).toContain(`steps.${id}.outputs.cache-hit`) + } + expect(index).toBeLessThan(steps.findIndex((step) => step.name === 'Prepare native runtime')) + expect(toolchain.if).toBe("runner.os == 'Linux' && inputs.native-runtime != 'none'") + }) + + // The action's toolchain workaround only runs in Linux Bash. + it.skipIf(process.platform === 'win32').each([ + ['node', 'true', '0', false], + ['node', 'true', '1', true], + ['node', 'false', '0', true], + ['node', '', '0', true], + ['electron', 'true', '0', true], + ['electron', 'false', '0', true] + ])('runtime=%s cache=%s probe=%s installs=%s', (runtime, hit, probeStatus, installs) => { + const directory = mkdtempSync(join(tmpdir(), 'orca-ci-native-toolchain-')) + const log = join(directory, 'commands') + const environment = join(directory, 'github-env') + try { + writeFileSync(log, '') + writeFileSync(environment, '') + for (const [name, source] of [ + ['node', 'echo "node $*" >> "$COMMAND_LOG"\nexit "$PROBE_STATUS"'], + ['npm', 'echo "npm $*" >> "$COMMAND_LOG"\nif [ "$1" = root ]; then echo /global; fi'] + ]) { + const path = join(directory, name) + writeFileSync(path, `#!/bin/sh\n${source}\n`) + chmodSync(path, 0o755) + } + execFileSync('bash', ['-e', '-o', 'pipefail', '-c', toolchain.run], { + env: { + ...process.env, + PATH: `${directory}:${process.env.PATH}`, + NATIVE_RUNTIME: runtime, + NATIVE_CACHE_HIT: hit, + PROBE_STATUS: probeStatus, + COMMAND_LOG: log, + GITHUB_ENV: environment + } + }) + const commands = readFileSync(log, 'utf8') + expect(commands.includes('npm install -g node-gyp@11.5.0')).toBe(installs) + expect(commands.includes('node config/scripts/ensure-native-runtime.mjs --check-only')).toBe( + runtime === 'node' && hit === 'true' + ) + expect(readFileSync(environment, 'utf8')).toBe( + installs ? 'npm_config_node_gyp=/global/node-gyp/bin/node-gyp.js\n' : '' + ) + } finally { + rmSync(directory, { recursive: true, force: true }) + } + }) +}) diff --git a/config/scripts/hourly-preflight-workflow.test.mjs b/config/scripts/hourly-preflight-workflow.test.mjs new file mode 100644 index 00000000000..2bec40b329b --- /dev/null +++ b/config/scripts/hourly-preflight-workflow.test.mjs @@ -0,0 +1,91 @@ +import { mkdtempSync, readFileSync, rmSync } from 'node:fs' +import { tmpdir } from 'node:os' +import { join } from 'node:path' +import { describe, expect, it } from 'vitest' +import { parse } from 'yaml' +import { runProcess } from '../../src/shared/child-process/run-process' + +const workflow = parse( + readFileSync(new URL('../../.github/workflows/hourly-mac-build.yml', import.meta.url), 'utf8') +) +const preflight = workflow.jobs.preflight +const freshness = preflight.steps.find((step) => step.id === 'freshness') +const head = 'abcdef0123'.repeat(4) + +async function checkFreshness(overrides = {}) { + const directory = mkdtempSync(join(tmpdir(), 'hourly-preflight-')) + const output = join(directory, 'output') + try { + const result = await runProcess({ + program: 'bash', + args: [ + '-c', + `gh() { + case "$1 $2" in + "api "*) printf '%s\\n' "$HEAD_SHA" ;; + "release list") printf '%s\\n' "$LAST_TAG" ;; + "release view") printf '%s\\n' "$LAST_SHA" ;; + *) return 1 ;; + esac + } + ${freshness.run}` + ], + env: { + ...process.env, + GITHUB_OUTPUT: output, + GITHUB_REPOSITORY: 'stablyai/orca', + MAIN_REPO_TOKEN: 'main-token', + HOURLY_REPO: 'stablyai/orca-hourly', + HEAD_SHA: head, + LAST_TAG: 'previous-hourly', + LAST_SHA: head.slice(0, 12), + FORCED: 'false', + ...overrides + } + }) + return { + exitCode: result.code, + stderr: result.stderr, + stdout: result.stdout, + output: result.code === 0 ? readFileSync(output, 'utf8') : '' + } + } finally { + rmSync(directory, { recursive: true, force: true }) + } +} + +describe('hourly build preflight', () => { + it('gates Mac allocation and pins the checkout and downstream identity', () => { + const build = workflow.jobs['build-hourly-mac'] + expect(preflight['runs-on']).toBe('ubuntu-latest') + expect(preflight.steps.some((step) => step.uses?.startsWith('actions/checkout'))).toBe(false) + expect( + preflight.steps.find((step) => step.id === 'app_token').with['permission-contents'] + ).toBe('read') + expect(build.needs).toBe('preflight') + expect(build.if).toBe("needs.preflight.outputs.should_build == 'true'") + expect(build.steps.find((step) => step.name === 'Checkout').with.ref).toBe( + build.outputs.head_sha + ) + expect(build.outputs.head_sha).toBe('${{ needs.preflight.outputs.head_sha }}') + expect(build.steps.find((step) => step.id === 'release').env.SHA).toBe(build.outputs.head_sha) + expect(workflow.concurrency).toEqual({ group: 'hourly-mac-build', 'cancel-in-progress': false }) + }) + + it.each([ + ['unchanged', {}, false], + ['changed', { LAST_SHA: '123456789012' }, true], + ['forced', { FORCED: 'true' }, true], + ['first build', { LAST_TAG: '' }, true], + ['missing prior identity', { LAST_SHA: '' }, true] + ])('%s main selects the expected build decision', async (_name, env, shouldBuild) => { + const result = await checkFreshness(env) + expect(result.exitCode, `${result.stdout} ${result.stderr}`).toBe(0) + expect(result.output).toBe(`head_sha=${head}\nshould_build=${shouldBuild}\n`) + }) + + it('fails closed when main cannot be resolved, even when forced', async () => { + const result = await checkFreshness({ HEAD_SHA: '', FORCED: 'true' }) + expect(result.exitCode).not.toBe(0) + }) +}) diff --git a/docs/reference/ci-runner-efficiency.md b/docs/reference/ci-runner-efficiency.md index e569f749102..a9f644bc435 100644 --- a/docs/reference/ci-runner-efficiency.md +++ b/docs/reference/ci-runner-efficiency.md @@ -21,8 +21,9 @@ billing minutes or queue time. This small sample is not a historical average. default Debian/RPM compression is xz. PR artifacts are inspected on the same runner, so their download size offers no benefit. Keep all AppImage, Debian, RPM, payload, launcher, and shutdown checks. Release compression is unchanged. - Compression savings need a hosted run; do not equate the full packaging step - with removable compression time. + Hosted validation in [33999422341](https://github.com/stablyai/orca/actions/runs/33999422341) + reduced the package-build step to 2m13s and the full Linux job to 6m17s, with + all existing checks passing. This is a small observational sample. - Cancel superseded Mobile Checks and Skill update round-trip PR runs. The skill matrix has 13 jobs. Preserve non-cancelling main/merge-group skill runs, with separate concurrency groups per event. @@ -36,6 +37,21 @@ caching, and changed-spec E2E routing. Increasing shards would increase setup work and simultaneous runner demand. Do not adjust the count without comparing critical-path time and aggregate job time on the same commit. +## Follow-up savings + +- Move the hourly main/release freshness lookup to a five-minute Ubuntu + preflight without a checkout. In unchanged run + [33986205749](https://github.com/stablyai/orca/actions/runs/33986205749), + Blacksmith macOS was occupied for 40 seconds, including a 30-second checkout, + before skipping. The new job-level gate avoids that Mac allocation. Actual + builds gain an Ubuntu scheduling hop; pin the Mac checkout and downstream + Windows identity to the SHA that the preflight checked. +- Avoid global `npm install -g node-gyp` for validated Linux Node-runtime cache + hits. Use the existing native-module load/provenance check before skipping; + misses, broken addons, and Electron jobs still install the rebuild toolchain. + The action file participates in cache keys, so this rollout creates fresh + native caches once. No measured warm-cache seconds are claimed yet. + ## Runner recommendations The repository is **public**, verified using the GitHub API. Standard @@ -66,6 +82,24 @@ See [GitHub Actions billing](https://docs.github.com/en/billing/concepts/product See [pricing](https://ubicloud.com/docs/about/pricing) and [setup](https://ubicloud.com/docs/github-actions-integration/quickstart). +### A bounded Ubicloud candidate + +The Linux leg of `performance-contracts.yml` took 48 seconds in +[33994756657](https://github.com/stablyai/orca/actions/runs/33994756657). +Its daily schedule and 20-minute timeout make it a small candidate: 31 ordinary +scheduled attempts permit at most 620 job-runtime minutes, before runner +startup/cleanup billing. Actual timings on Ubicloud's 2-vCPU hardware still need +measurement; the GitHub timing is only a sizing reference. + +If enabled later, route only the first attempt of the scheduled Linux job to +Ubicloud; keep PRs, manual dispatches, reruns, and macOS/Windows on GitHub. This +avoids spending the allowance on unpredictable PR volume. Check other account +usage and available credit before enabling; a workflow timeout is not an +account-wide billing cap. On September 5, the organization's GitHub App +installation list contained Blacksmith but no Ubicloud installation, so this +follow-up leaves runner selection on GitHub rather than queueing work against +an unprovisioned label. + ## Machines that also run coding agents Do not register the credentialed host directly as a public-PR runner. A PR can