ci: use ARM unit runners, overlap web builds, and reuse verifier fixtures (#23376)

* test: reuse isolated mobile bundle fixtures for verifier checks

* ci: run PR unit shards on ARM and overlap independent web builds

* docs: record controlled CI overlap and runner measurements

* test: observe WebRTC packets with the host clock

* ci: isolate Windows installer CIM probe from native test load

* docs: record native probe scheduling validation

---------

Co-authored-by: m4air <m4air@m4airs-Air.localdomain>
This commit is contained in:
OrcaWin
2026-09-27 01:06:54 -07:00
committed by GitHub
co-authored by m4air
parent 983250a12e
commit 47cebbf5d2
13 changed files with 244 additions and 69 deletions
+16
View File
@@ -70,6 +70,22 @@ jobs:
if: steps.typecheck-cache.outputs.cache-hit != 'true'
run: pnpm run typecheck
warm-linux-arm:
runs-on: ubuntu-24.04-arm
timeout-minutes: 10
env:
ORCA_BACKGROUND_LAUNCH: '1'
steps:
- uses: actions/checkout@v6
with:
persist-credentials: false
- uses: ./.github/actions/install-node-dependencies
with:
native-runtime: node
node-version: '24'
- name: Verify native cache is usable
run: node config/scripts/ensure-native-runtime.mjs --check-only
warm-windows:
strategy:
fail-fast: false
+10 -6
View File
@@ -1,8 +1,8 @@
name: Node next compatibility
name: Scheduled x86 unit compatibility
on:
schedule:
# Full future-runtime coverage is useful, but not worth doubling every PR matrix.
# Keep current and future Node coverage on x86 while PR unit shards use ARM.
- cron: '0 10 * * *'
workflow_dispatch:
@@ -14,9 +14,13 @@ permissions:
contents: read
jobs:
# A cold cache would otherwise make all eight Node 26 shards compile the same native addons.
# Prime each Node ABI before its shards restore native modules.
test_native_cache:
name: prepare test native cache node 26
name: prepare test native cache node ${{ matrix.node }}
strategy:
fail-fast: false
matrix:
node: ['24', '26']
runs-on: ubuntu-latest
steps:
@@ -28,10 +32,10 @@ jobs:
- uses: ./.github/actions/install-node-dependencies
with:
native-runtime: node
node-version: '26'
node-version: ${{ matrix.node }}
test:
needs: [test_native_cache]
uses: ./.github/workflows/unit-tests.yml
with:
node_versions: '["26"]'
node_versions: '["24", "26"]'
+11 -3
View File
@@ -606,7 +606,7 @@ jobs:
name: prepare test native cache node 24
needs: [code_paths]
if: needs.code_paths.outputs.native_cache_changed == 'true'
runs-on: ubuntu-latest
runs-on: ubuntu-24.04-arm
steps:
- name: Checkout
@@ -629,6 +629,7 @@ jobs:
uses: ./.github/workflows/unit-tests.yml
with:
node_versions: '["24"]'
runner: ubuntu-24.04-arm
# Why a separate job: the test needs a real Chrome, and the sharded `test` matrix
# would pay for it on every shard to run one file in whichever shard it landed in.
@@ -897,6 +898,8 @@ jobs:
exit "$status"
- name: Project web client from renderer build
id: web-client
background: true
run: pnpm run build:web-from-renderer
# Why here and not inside "Build package inputs": this job assembles packaging inputs step by
@@ -908,7 +911,7 @@ jobs:
- name: Build native components
run: pnpm run build:native
- wait: linux-package-tools
- wait: [linux-package-tools, web-client]
- name: Package unpacked app
env:
@@ -1006,13 +1009,18 @@ jobs:
# vitest runs here directly rather than through `pnpm test`, so the addon
# assertions only hold once install-node-dependencies has rebuilt natives.
- name: Test Windows installer process probe
# Keep cold CIM startup out of the concurrent Electron/native process workload.
run: >-
pnpm exec vitest run --config config/vitest.config.ts
config/scripts/nsis-process-check.test.mjs
- name: Test Windows-specific boundaries
run: >-
pnpm exec vitest run --config config/vitest.config.ts
config/scripts/rebuild-native-deps.test.mjs
config/scripts/rebuild-native-deps-windows-process-tree.test.mjs
config/scripts/rebuild-native-deps-node-pty.test.mjs
config/scripts/nsis-process-check.test.mjs
config/scripts/ensure-native-runtime-job-ownership.test.mjs
config/scripts/verify-packaged-node-pty-job-ownership.test.mjs
config/scripts/windows-pe-machine.test.mjs
+13 -3
View File
@@ -8,13 +8,19 @@ on:
required: true
type: string
runner:
description: Hosted runner for the unit shards; relay integration keeps its x86 host.
required: false
default: ubuntu-latest
type: string
permissions:
contents: read
jobs:
test:
name: tests node ${{ matrix.node }} ${{ matrix.shard }}/${{ matrix.shard_total }}
runs-on: ubuntu-latest
runs-on: ${{ inputs.runner }}
strategy:
fail-fast: false
matrix:
@@ -78,7 +84,11 @@ jobs:
if-no-files-found: warn
relay_integration:
name: relay integration node ${{ fromJSON(inputs.node_versions)[0] }}
name: relay integration node ${{ matrix.node }}
strategy:
fail-fast: false
matrix:
node: ${{ fromJSON(inputs.node_versions) }}
runs-on: ubuntu-latest
steps:
@@ -90,7 +100,7 @@ jobs:
- uses: ./.github/actions/install-node-dependencies
with:
native-runtime: node
node-version: ${{ fromJSON(inputs.node_versions)[0] }}
node-version: ${{ matrix.node }}
cache-electron-package: 'true'
cache-dependency-path: |
pnpm-lock.yaml
@@ -1,9 +1,13 @@
import { mkdir, mkdtemp, readFile, rm, writeFile } from 'node:fs/promises'
import { tmpdir } from 'node:os'
import { mkdir, readFile, writeFile } from 'node:fs/promises'
import { join, relative } from 'node:path'
import { fileURLToPath } from 'node:url'
import { deserialize, serialize } from 'node:v8'
import { describe, expect, it } from 'vitest'
import {
withScratch,
readAppBundle,
readWrittenBundle,
copyWrittenBundle
} from './mobile-web-app-bundle-test-fixture.mjs'
import {
MOBILE_WEB_APP_NATIVE_PARITY_STYLE,
MOBILE_WEB_APP_ROOT_RESET,
@@ -66,34 +70,6 @@ function allScriptSource({ script, chunks }) {
return [script, ...chunks.map((chunk) => chunk.bytes)].map((bytes) => bytes.toString('utf8'))
}
async function withScratch(run) {
const scratch = await mkdtemp(join(tmpdir(), 'orca-mobile-web-app-test-'))
try {
return await run(scratch)
} finally {
await rm(scratch, { recursive: true, force: true })
}
}
// Snapshots preserve Buffer methods and give each assertion its own mutable copy.
let appBundleSnapshot
let writtenBundleSnapshot
async function readAppBundle() {
appBundleSnapshot ??= bundleMobileWebApp().then(serialize)
// Deserialized Buffers alias their snapshot, so copy it before exposing them.
return deserialize(Buffer.from(await appBundleSnapshot))
}
async function readWrittenBundle() {
writtenBundleSnapshot ??= withScratch(async (scratch) => {
const { outDir, ...result } = await buildMobileWebAppBundle({ outDir: join(scratch, 'bundle') })
const html = await readFile(join(outDir, 'index.html'), 'utf8')
return serialize({ ...result, html })
})
return deserialize(await writtenBundleSnapshot)
}
describe('the CRLF pin', () => {
it('exempts the same extensions in .gitattributes as the CRLF scan skips', async () => {
const attributes = await readFile(join(projectDir, '.gitattributes'), 'utf8')
@@ -124,6 +100,19 @@ describeBundling('the app bundle', () => {
const written = await readWrittenBundle()
written.manifest.assets.length = 0
expect((await readWrittenBundle()).manifest.assets.length).toBeGreaterThan(0)
const originalByte = written.files[0].bytes[0]
written.files[0].bytes[0] ^= 255
expect((await readWrittenBundle()).files[0].bytes[0]).toBe(originalByte)
await withScratch(async (scratch) => {
const firstDir = join(scratch, 'first')
await copyWrittenBundle(firstDir)
await writeFile(join(firstDir, 'manifest.json'), 'corrupted')
const secondDir = join(scratch, 'second')
const second = await copyWrittenBundle(secondDir)
for (const { file, bytes } of second.files) {
expect((await readFile(join(secondDir, file))).equals(bytes), file).toBe(true)
}
})
}, 120_000)
it('resolves react-native to react-native-web and leaves no require.context', async () => {
@@ -642,7 +631,7 @@ describe('the verifier', () => {
async () => {
await withScratch(async (scratch) => {
const outDir = join(scratch, 'mobile-web')
await buildMobileWebAppBundle({ outDir })
await copyWrittenBundle(outDir)
await expect(verifyMobileWebAppBundle({ bundleDir: outDir })).resolves.toBeDefined()
})
},
@@ -654,7 +643,7 @@ describe('the verifier', () => {
async () => {
await withScratch(async (scratch) => {
const outDir = join(scratch, 'mobile-web')
await buildMobileWebAppBundle({ outDir })
await copyWrittenBundle(outDir)
const manifestPath = join(outDir, 'manifest.json')
const manifest = JSON.parse(await readFile(manifestPath, 'utf8'))
manifest.buildId = 'f'.repeat(64)
@@ -672,7 +661,7 @@ describe('the verifier', () => {
async () => {
await withScratch(async (scratch) => {
const outDir = join(scratch, 'mobile-web')
const { manifest } = await buildMobileWebAppBundle({ outDir })
const { manifest } = await copyWrittenBundle(outDir)
// What a stale out/ actually looks like: every digest agrees with its bytes and the
// buildId derives from the asset list, but the source has moved on. Only the two fresh
// builds the verifier runs can tell, which is the check this covers.
@@ -41,7 +41,7 @@ describe('CI background step barriers', () => {
expect(pending.delete(id), `missing background step ${id}`).toBe(true)
}
}
expect(pending.size).toBeLessThanOrEqual(3)
expect(pending.size).toBeLessThanOrEqual(job === pr.jobs.package ? 4 : 3)
}
expect([...pending]).toEqual([])
}
@@ -108,6 +108,7 @@ describe('CI background step barriers', () => {
const steps = pr.jobs.package.steps
for (const [id, consumer] of [
['linux-package-tools', 'Package unpacked app'],
['web-client', 'Package unpacked app'],
['shutdown-fixture-cache', 'Verify headless serve signal shutdown'],
['cli-fixture-cache', 'Verify Linux CLI launch contract']
]) {
@@ -8,9 +8,13 @@ const workflow = readWorkflow('ci-cache-warmup')
const steps = workflow.jobs.warm.steps
it('warms the same Linux Node runtime the PR shards restore', () => {
const install = steps.find((step) => step.uses === './.github/actions/install-node-dependencies')
const arm = workflow.jobs['warm-linux-arm']
const install = arm.steps.find(
(step) => step.uses === './.github/actions/install-node-dependencies'
)
const primer = readWorkflow('pr').jobs.test_native_cache
expect(workflow.jobs.warm['runs-on']).toBe(primer['runs-on'])
expect(arm['runs-on']).toBe(primer['runs-on'])
expect(arm.steps.at(-1).run).toBe('node config/scripts/ensure-native-runtime.mjs --check-only')
expect(install.with).toEqual(primer.steps.find((step) => step.uses === install.uses).with)
})
@@ -30,6 +34,7 @@ it('publishes incremental state under a key and prefix that new PRs restore', ()
it('bounds warming to the required platforms and validates changes without granting writes', () => {
expect(Object.keys(workflow.jobs)).toEqual([
'warm',
'warm-linux-arm',
'warm-windows',
'warm-linux-package-fixtures'
])
@@ -6,6 +6,19 @@ import { parse } from 'yaml'
const projectDir = resolve(import.meta.dirname, '../..')
describe('client-hosted browser package coverage', () => {
it('finishes the installer process probe before concurrent native boundaries', () => {
const workflow = parse(readFileSync(join(projectDir, '.github/workflows/pr.yml'), 'utf8'))
const steps = workflow.jobs.package_windows.steps
const probe = steps.findIndex((step) => step.name === 'Test Windows installer process probe')
const boundaries = steps.findIndex((step) => step.name === 'Test Windows-specific boundaries')
const file = 'config/scripts/nsis-process-check.test.mjs'
expect(probe).toBeGreaterThanOrEqual(0)
expect(probe).toBeLessThan(boundaries)
expect(steps[probe].background).toBeUndefined()
expect(steps[probe].run).toContain(file)
expect(steps[boundaries].run).not.toContain(file)
})
it('bundles the WSL browser-network relay with its version stamp', () => {
const relayBuild = readFileSync(join(projectDir, 'config/scripts/build-relay.mjs'), 'utf8')
@@ -0,0 +1,49 @@
import { mkdir, mkdtemp, readFile, rm, writeFile } from 'node:fs/promises'
import { tmpdir } from 'node:os'
import { dirname, join } from 'node:path'
import { deserialize, serialize } from 'node:v8'
import { bundleMobileWebApp, buildMobileWebAppBundle } from './build-mobile-web-app-bundle.mjs'
export async function withScratch(run) {
const scratch = await mkdtemp(join(tmpdir(), 'orca-mobile-web-app-test-'))
try {
return await run(scratch)
} finally {
await rm(scratch, { recursive: true, force: true })
}
}
// Snapshots preserve Buffer methods and give each assertion its own mutable copy.
let appBundleSnapshot
let writtenBundleSnapshot
export async function readAppBundle() {
appBundleSnapshot ??= bundleMobileWebApp().then(serialize)
// Deserialized Buffers alias their snapshot, so copy it before exposing them.
return deserialize(Buffer.from(await appBundleSnapshot))
}
export async function readWrittenBundle() {
writtenBundleSnapshot ??= withScratch(async (scratch) => {
const { outDir, ...result } = await buildMobileWebAppBundle({ outDir: join(scratch, 'bundle') })
const html = await readFile(join(outDir, 'index.html'), 'utf8')
const files = await Promise.all(
['manifest.json', ...result.manifest.assets.map((asset) => asset.path)].map(async (file) => ({
file,
bytes: await readFile(join(outDir, file))
}))
)
return serialize({ ...result, html, files })
})
return deserialize(Buffer.from(await writtenBundleSnapshot))
}
export async function copyWrittenBundle(outDir) {
const snapshot = await readWrittenBundle()
for (const { file, bytes } of snapshot.files) {
const destination = join(outDir, file)
await mkdir(dirname(destination), { recursive: true })
await writeFile(destination, bytes)
}
return { ...snapshot, outDir }
}
@@ -70,7 +70,7 @@ describe('PR workflow parallelism', () => {
expect(workflow.permissions).toEqual({ contents: 'read' })
})
it('runs Node 24 on PRs and the same eight-shard suite on Node 26 daily', () => {
it('runs all eight shards on ARM for PRs and both Node versions on x86 daily', () => {
const sharedTest = unitTestWorkflow.jobs.test
const testStep = sharedTest.steps.find((step) => step.name === 'Test shard')
const installStep = sharedTest.steps.find(
@@ -86,7 +86,22 @@ describe('PR workflow parallelism', () => {
expect(workflow.jobs.test.uses).toBe('./.github/workflows/unit-tests.yml')
expect(JSON.parse(workflow.jobs.test.with.node_versions)).toEqual(['24'])
expect(nodeNextWorkflow.jobs.test.uses).toBe('./.github/workflows/unit-tests.yml')
expect(JSON.parse(nodeNextWorkflow.jobs.test.with.node_versions)).toEqual(['26'])
expect(JSON.parse(nodeNextWorkflow.jobs.test.with.node_versions)).toEqual(['24', '26'])
expect(workflow.jobs.test.with.runner).toBe('ubuntu-24.04-arm')
expect(workflow.jobs.test_native_cache['runs-on']).toBe('ubuntu-24.04-arm')
expect(sharedTest['runs-on']).toBe('${{ inputs.runner }}')
expect(unitTestWorkflow.on.workflow_call.inputs.runner.default).toBe('ubuntu-latest')
expect(nodeNextWorkflow.jobs.test.with.runner).toBeUndefined()
expect(nodeNextWorkflow.jobs.test_native_cache['runs-on']).toBe('ubuntu-latest')
expect(nodeNextWorkflow.jobs.test_native_cache.strategy.matrix.node).toEqual(['24', '26'])
const relay = unitTestWorkflow.jobs.relay_integration
expect(relay.strategy.matrix.node).toBe('${{ fromJSON(inputs.node_versions) }}')
expect(relay['runs-on']).toBe('ubuntu-latest')
expect(
relay.steps.find((step) => step.uses === './.github/actions/install-node-dependencies').with[
'node-version'
]
).toBe('${{ matrix.node }}')
expect(nodeNextWorkflow.on.schedule).toHaveLength(1)
expect(nodeNextWorkflow.on.workflow_dispatch).toBeNull()
expect(sharedTest.strategy.matrix.node).toBe('${{ fromJSON(inputs.node_versions) }}')
@@ -104,7 +119,7 @@ describe('PR workflow parallelism', () => {
expect(primerInstall.with['node-version']).toBe('24')
expect(workflow.jobs.test.needs).toContain('test_native_cache')
expect(nodeNextPrimerInstall.with['native-runtime']).toBe('node')
expect(nodeNextPrimerInstall.with['node-version']).toBe('26')
expect(nodeNextPrimerInstall.with['node-version']).toBe('${{ matrix.node }}')
expect(nodeNextWorkflow.jobs.test.needs).toEqual(['test_native_cache'])
})
@@ -69,7 +69,10 @@ import { classifyPrJobs } from './pr-code-change-scope.mjs'
const projectDir = resolve(import.meta.dirname, '../..')
const WINDOWS_LANE_JOB = 'package_windows'
const WINDOWS_LANE_STEP = 'Test Windows-specific boundaries'
const WINDOWS_LANE_STEPS = [
'Test Windows-specific boundaries',
'Test Windows installer process probe'
]
const WINDOWS_LANE_RUNNER = 'windows-2022'
/**
@@ -288,19 +291,20 @@ function readWindowsWorkflow() {
const jobs = Object.entries(workflow.jobs ?? {})
const windowsJobs = jobs.filter(([, job]) => couldRunOnWindows(job?.['runs-on']))
const steps = workflow.jobs?.[WINDOWS_LANE_JOB]?.steps ?? []
const step = steps.find((candidate) => candidate?.name === WINDOWS_LANE_STEP)
if (!step) {
throw new Error(
`No "${WINDOWS_LANE_STEP}" step in the ${WINDOWS_LANE_JOB} job of .github/workflows/pr.yml. ` +
'If it was renamed, update WINDOWS_LANE_STEP here -- do not delete this guard.'
)
}
const run = String(step.run ?? '')
if (!run.includes('vitest run')) {
throw new Error(
`The "${WINDOWS_LANE_STEP}" step no longer invokes vitest; this guard is stale.`
)
}
const runs = WINDOWS_LANE_STEPS.map((name) => {
const step = steps.find((candidate) => candidate?.name === name)
if (!step) {
throw new Error(
`No "${name}" step in ${WINDOWS_LANE_JOB}; update WINDOWS_LANE_STEPS if renamed.`
)
}
const run = String(step.run ?? '')
if (!run.includes('vitest run')) {
throw new Error(`The "${name}" step no longer invokes vitest; this guard is stale.`)
}
return run
})
const run = runs.join(' ')
return {
windowsJobNames: windowsJobs.map(([name]) => name),
laneFiles: run.split(/\s+/).filter((token) => TEST_FILE_PATTERN.test(token))
@@ -331,7 +335,7 @@ function registrationFailure(path) {
const missing = []
if (!laneFiles.includes(path)) {
missing.push(
`add "${path}" to the "${WINDOWS_LANE_STEP}" vitest argv in .github/workflows/pr.yml ` +
`add "${path}" to the "${WINDOWS_LANE_STEPS[0]}" vitest argv in .github/workflows/pr.yml ` +
`(job ${WINDOWS_LANE_JOB})`
)
}
+57
View File
@@ -1,5 +1,62 @@
# CI efficiency and runner capacity
## September 27 follow-up
[PR #23368](https://github.com/stablyai/orca/pull/23368) overlaps shell installation
with dependency setup, starts localization extraction before the orcad smoke,
and prepares mobile route snapshots while WebKit and the bundle are being built.
Its 27 checks passed without retries; seven existing conditional checks skipped.
Same-runner comparisons in both orders measured:
| Work | Before | After | Evidence |
| --- | --- | --- | --- |
| Static block | 63.5 / 74.7s | 38.9 / 55.6s | [Full comparisons](https://github.com/stablyai/orca/actions/runs/36302208990) |
| Shell job, downloads warmed equally | 76.8 / 70.3s | 64.4 / 64.0s | [Controlled shell runs](https://github.com/stablyai/orca/actions/runs/36302612626) |
| Mobile preparation | 19.8–21.6s | 18.0–18.5s | [Eight measurements](https://github.com/stablyai/orca/actions/runs/36302877583) |
| Web projection and mobile build | 17.2–17.3s | 11.7–12.1s | [Eight measurements](https://github.com/stablyai/orca/actions/runs/36302974324) |
| Mobile verifier fixture suite | 25.47 / 25.35s | 20.04 / 19.92s | [Four full-suite runs](https://github.com/stablyai/orca/actions/runs/36302692823) |
Full mobile-job timings were dominated by first-run apt installation and browser
test variation; the controlled preparation measurement is the scheduling evidence.
All 1,248 web/mobile output files matched byte-for-byte in the build comparison.
The fixture suite kept all 47 tests, isolated mutable copies, and the verifier's
two fresh builds. No deadline, isolation, or worker-count changes were needed.
The existing unit assignment was already balanced at about 919 historical
worker-seconds per shard; fresh x86 elapsed times still ranged from 254 to 433s.
Refreshing weights alone would encode runner variation rather than resolve it.
An [identical-source architecture pilot](https://github.com/stablyai/orca/actions/runs/36302250920)
ran shards 1 and 8 on both four-CPU hosted runners. Complete jobs improved from
449 to 404s and 461 to 384s on ARM, including setup; test and skip counts matched.
A [full ARM run](https://github.com/stablyai/orca/actions/runs/36302906752) then passed
all eight shards in 329–373 test seconds (365–412 job seconds). Uploaded reports
matched the same complete x86 assignment: 9,876 files, each exactly once, no
unhandled errors. These are elapsed samples excluding queue time, not a guarantee
that every ARM allocation is faster than every x86 allocation.
PR unit shards and their cache primer now use ARM; a main-branch warmer seeds
that architecture's existing native and pnpm cache keys. Native, package, and
relay gates continue on x86. The daily workflow retains complete x86 coverage on
both Node 24 and Node 26, including relay integration. Thus PR unit architecture
changes, while x86 unit coverage remains scheduled; this is an explicit coverage
placement tradeoff rather than a claim of identical per-PR host coverage.
PR validation exposed a WebRTC probe timeout inside a hidden renderer. Isolated
and four-concurrent probes passed on both architectures; the original cause is
unproven. The probe now uses Electron main for the same three-second observation
interval. A [fault-injection comparison](https://github.com/stablyai/orca/actions/runs/36304349257)
passed with renderer timers unavailable on both architectures, while the original
probe failed the negative control. Packet assertions and deadlines are unchanged.
A subsequent Windows run timed out in the installer's real CIM process query
after verifying restricted policy. Its unchanged probe now runs before the
concurrent native suite, removing that source of contention without relaxing
the twenty-second process deadline or dropping either PowerShell architecture.
Replacing Vitest deep comparisons with Node assertions in the status-store
oracle saved only about one local second in an initial trial. The change was
not retained: that evidence did not justify changing assertion semantics.
## Four follow-up changes
- Keep the readiness event, but reuse required checks only after an Actions API
@@ -112,15 +112,19 @@ async function probe() {
iceServers: [{ urls: 'stun:\${target}:\${udpAddress.port}' }],
iceCandidatePoolSize: 1
})
globalThis.__webrtcEgressPeer = peer
peer.createDataChannel('probe')
const offer = await peer.createOffer()
await peer.setLocalDescription(offer)
await new Promise(resolve => setTimeout(resolve, 3000))
peer.close()
})()
\`
enterPhase('renderer-webrtc')
await window.webContents.executeJavaScript(script)
// Hidden renderer timers may be throttled; the packet observation clock belongs to the host.
enterPhase('observe-packets')
await new Promise((resolve) => setTimeout(resolve, 3000))
enterPhase('close-peer')
await window.webContents.executeJavaScript('globalThis.__webrtcEgressPeer.close()')
enterPhase('drain-packets')
await new Promise((resolve) => setTimeout(resolve, 500))
enterPhase('cleanup')