ci: reduce dependency, checkout, and test deadline overhead (#18968)

* ci: reduce dependency, checkout, and test deadline overhead

* ci: avoid generic E2E jobs for native-only IME changes
This commit is contained in:
Neil
2026-09-05 18:20:12 -07:00
committed by GitHub
parent d7722a698c
commit 6031c19e9f
13 changed files with 177 additions and 50 deletions
+5 -4
View File
@@ -25,9 +25,10 @@ defaults:
working-directory: cloud
jobs:
# Public-repository hosted runners preserve Blacksmith allowance for macOS.
security:
name: Secret scan
runs-on: blacksmith-2vcpu-ubuntu-2204
runs-on: ubuntu-22.04
steps:
- uses: actions/checkout@v4
with:
@@ -53,7 +54,7 @@ jobs:
# Compiles the workspace. No Postgres service: nothing here reaches a
# database, and the service container costs ~13s of startup.
build:
runs-on: blacksmith-4vcpu-ubuntu-2204
runs-on: ubuntu-22.04
steps:
- uses: actions/checkout@v4
@@ -73,7 +74,7 @@ jobs:
# package it needs through the relay pretest hook, so it does not depend on
# `pnpm build` having run.
test:
runs-on: blacksmith-4vcpu-ubuntu-2204
runs-on: ubuntu-22.04
services:
postgres:
image: postgres:16-alpine
@@ -107,7 +108,7 @@ jobs:
# Fork pull requests reach this job, so it never configures a backend, never plans, and never
# holds a credential. Only the relay root ships here; foundation and apps stay private.
terraform:
runs-on: blacksmith-2vcpu-ubuntu-2204
runs-on: ubuntu-22.04
steps:
- uses: actions/checkout@v4
+3 -2
View File
@@ -93,12 +93,13 @@ jobs:
NATIVE_IME_SOURCE_CHANGED="$(printf '%s\n' "$CHANGED" | node config/scripts/pr-e2e-source-routing.mjs --native-ime-source)"
echo "native_ime_source_changed=$NATIVE_IME_SOURCE_CHANGED" >> "$GITHUB_OUTPUT"
echo "Native IME source changed: $NATIVE_IME_SOURCE_CHANGED"
if [ "$TEST_FILES_JSON" != '[]' ]; then
SHOULD_RUN="$(printf '%s\n' "$CHANGED" | node config/scripts/pr-e2e-source-routing.mjs --reusable-workflow)"
if [ "$SHOULD_RUN" = true ]; then
echo "should_run=true" >> "$GITHUB_OUTPUT"
echo "Changed E2E specs: $TEST_FILES_JSON"
else
echo "should_run=false" >> "$GITHUB_OUTPUT"
echo "No changed E2E specs"
echo "No specs requiring the reusable E2E workflow"
fi
static_analysis:
+13 -10
View File
@@ -858,16 +858,17 @@ jobs:
if: runner.os == 'Linux'
run: sudo apt-get update && sudo apt-get install -y build-essential python3 xvfb
- name: Setup Node.js
uses: actions/setup-node@v6
with:
node-version-file: package.json
- name: Setup pnpm
uses: pnpm/setup@v2
with:
install: false
- name: Setup Node.js
uses: actions/setup-node@v6
with:
node-version-file: package.json
cache: pnpm
# Why: Linux terminal golden E2E uses the same native install path as
# release CI, which needs pnpm to bypass its non-executable gyp_main.py.
- name: Use external node-gyp to avoid pnpm's bundled copy (Linux only)
@@ -1074,16 +1075,17 @@ jobs:
if: runner.os == 'Linux'
run: sudo apt-get update && sudo apt-get install -y build-essential python3 xvfb
- name: Setup Node.js
uses: actions/setup-node@v6
with:
node-version-file: package.json
- name: Setup pnpm
uses: pnpm/setup@v2
with:
install: false
- name: Setup Node.js
uses: actions/setup-node@v6
with:
node-version-file: package.json
cache: pnpm
# Why: keep the non-blocking evidence lane on the same Linux native
# install path as the blocking golden and release build jobs.
- name: Use external node-gyp to avoid pnpm's bundled copy (Linux only)
@@ -1716,6 +1718,7 @@ jobs:
with:
name: orca-windows-unsigned-${{ needs.cut.outputs.tag }}
path: dist/orca-windows-setup.exe
compression-level: 0
if-no-files-found: error
# Why: SignPath Foundation production certificates require manual review,
@@ -45,7 +45,9 @@ jobs:
steps:
- uses: actions/checkout@v6
with:
# Historical skill snapshots need tags, but only their blobs are read.
fetch-depth: 0
filter: blob:none
persist-credentials: false
- uses: actions/setup-node@v6
with:
+2 -16
View File
@@ -38,23 +38,9 @@ jobs:
xfwm4
xvfb
- name: Setup Node.js
uses: actions/setup-node@v6
- uses: ./.github/actions/install-node-dependencies
with:
node-version-file: package.json
- name: Setup pnpm
uses: pnpm/setup@v2
with:
install: false
- name: Use external node-gyp to avoid pnpm bundled copy
run: |
npm install -g node-gyp@11.5.0
echo "npm_config_node_gyp=$(npm root -g)/node-gyp/bin/node-gyp.js" >> "$GITHUB_ENV"
- name: Install dependencies
run: pnpm install --frozen-lockfile
native-runtime: electron
- name: Build Electron app for E2E
run: pnpm exec electron-vite build --mode e2e
+6 -5
View File
@@ -67,16 +67,17 @@ jobs:
- name: Install native build tools and xvfb
run: sudo apt-get update && sudo apt-get install -y build-essential python3 xvfb zsh
- name: Setup Node.js
uses: actions/setup-node@v6
with:
node-version-file: package.json
- name: Setup pnpm
uses: pnpm/setup@v2
with:
install: false
- name: Setup Node.js
uses: actions/setup-node@v6
with:
node-version-file: package.json
cache: pnpm
# Why: this scheduled/manual workflow uses the same native install path as
# PR and E2E CI, which needs pnpm to bypass its bundled gyp_main.py.
- name: Use external node-gyp to avoid pnpm's bundled copy
@@ -215,6 +215,7 @@ jobs:
with:
name: orca-windows-installer-unsigned-${{ github.run_id }}
path: dist/orca-windows-setup.exe
compression-level: 0
if-no-files-found: error
- name: Submit Windows installer signing request
@@ -0,0 +1,33 @@
import { readFileSync } from 'node:fs'
import { describe, expect, it } from 'vitest'
import { parse } from 'yaml'
import { hasNativeImeSourceChange, shouldRunReusablePrE2e } from './pr-e2e-source-routing.mjs'
const workflow = parse(readFileSync('.github/workflows/pr.yml', 'utf8'))
const filterStep = workflow.jobs.code_paths.steps.find((step) => step.id === 'e2e_filter')
describe('native-only PR E2E routing', () => {
it('avoids generic E2E allocation for native-only changes while preserving its IME lane', () => {
for (const file of [
'tests/e2e/terminal-ibus-hangul-native.spec.ts',
'config/scripts/run-terminal-ibus-hangul-e2e.mjs'
]) {
expect(hasNativeImeSourceChange([file])).toBe(true)
expect(shouldRunReusablePrE2e([file])).toBe(false)
}
expect(shouldRunReusablePrE2e([])).toBe(false)
for (const spec of [
'tests/e2e/ssh-startup-exec-readiness.spec.ts',
'tests/e2e/paired-startup-exec-readiness.spec.ts',
'tests/e2e/terminal-ime-exact-byte.spec.ts',
'tests/e2e/future.spec.ts'
]) {
expect(shouldRunReusablePrE2e([spec])).toBe(true)
expect(shouldRunReusablePrE2e(['tests/e2e/terminal-ibus-hangul-native.spec.ts', spec])).toBe(
true
)
}
expect(filterStep.run).toContain('pr-e2e-source-routing.mjs --reusable-workflow')
expect(filterStep.run).toContain('if [ "$SHOULD_RUN" = true ]; then')
})
})
+12
View File
@@ -217,6 +217,16 @@ export function hasNativeImeSourceChange(changedPaths) {
).some((route) => changedPaths.some(route.matches))
}
export function shouldRunReusablePrE2e(changedPaths) {
// Native IME has its own workflow; SSH still runs inside the reusable workflow.
return (
hasSshSourceChange(changedPaths) ||
selectPrE2eSpecs(changedPaths).some(
(spec) => spec !== 'tests/e2e/terminal-ibus-hangul-native.spec.ts'
)
)
}
if (process.argv[1] && import.meta.url === pathToFileURL(process.argv[1]).href) {
let input = ''
process.stdin.setEncoding('utf8')
@@ -226,6 +236,8 @@ if (process.argv[1] && import.meta.url === pathToFileURL(process.argv[1]).href)
const changedPaths = input.split(/\r?\n/).filter(Boolean)
if (process.argv.includes('--ssh-source')) {
process.stdout.write(`${hasSshSourceChange(changedPaths)}\n`)
} else if (process.argv.includes('--reusable-workflow')) {
process.stdout.write(`${shouldRunReusablePrE2e(changedPaths)}\n`)
} else if (process.argv.includes('--native-ime-source')) {
process.stdout.write(`${hasNativeImeSourceChange(changedPaths)}\n`)
} else {
+66
View File
@@ -131,3 +131,69 @@ environments currently exist. An environment-gated design adds a GitHub
approval after each SignPath approval and changes the current automatic inner
signing timeout fallback; those are explicit release-policy decisions, so this
PR leaves production signing behavior unchanged.
## Second audit and hosted trials
- Cloud Verify ran 100 times in a sampled 39-hour window (84 PR and 16 push
runs). Move its four Ubuntu 22.04 jobs from Blacksmith to standard hosted
Ubuntu 22.04, preserving Postgres, secret scanning, build, tests, and Terraform
validation. Baseline [34001538145](https://github.com/stablyai/orca/actions/runs/34001538145)
used 64/72/26/19 seconds for security/test/build/Terraform respectively.
This conserves the shared provider allowance; hosted latency must be checked.
- Keep full tag history for the 13-job skill round-trip matrix, but fetch blobs
lazily. Only two historical SKILL.md files are materialized. Baseline
[33999994876](https://github.com/stablyai/orca/actions/runs/33999994876)
spent 4284 seconds per checkout, about 14 aggregate runner minutes. A hosted
trial must verify historical blob fetches on all three operating systems.
- Use the existing Electron/native dependency cache for native IME CI. Keep
both deterministic boundary and real IBus tests. Add pnpm store caching to
terminal perf and release golden/evidence lanes; retain their raw installs
because manually selected older refs may not contain the shared action.
- Disable ZIP recompression only for already-compressed NSIS installers sent
to SignPath. Installer contents, release compression, and signing stay intact.
- Advance existing placement and startup deadlines with scoped fake timers in
three renderer test files. All 34 tests pass in 62 ms of local test execution,
versus 65.182 seconds in the sampled hosted baseline. Imports and transforms
still dominate invocation time; this is not a claim of equal PR wall savings.
Eight unit shards already have balanced 260296-second sample durations.
Reducing shards or removing test isolation lacks evidence of a net gain. Real
subprocess tests intentionally cover lifecycle behavior and retain real clocks.
The 14-way E2E split retains headroom after earlier 12-way timeouts. Lowering
coverage or schedule frequency is outside this efficiency pass. Cache complexity
for a seven-second docs install is unlikely to pay back. Release build reuse
across modes risks differing telemetry identities and native platform artifacts.
Terminal Perf's baseline [33955846492](https://github.com/stablyai/orca/actions/runs/33955846492)
failed waiting 30 seconds for workspaceSessionReady in its shared-page fixture,
before measuring terminal performance. Compare hosted trials against that known
failure rather than attributing it to dependency cache changes.
Hosted trials for the second audit:
- [Cloud Verify 34002295216](https://github.com/stablyai/orca/actions/runs/34002295216)
passed all four jobs on standard hosted Ubuntu: security 57s, test 102s, build
35s, Terraform 19s. The test lane is 30s slower than the Blacksmith sample;
retain this modest latency tradeoff to conserve shared allowance.
- [Skill matrix 34002295221](https://github.com/stablyai/orca/actions/runs/34002295221)
passed all 13 legs, including historical blob materialization. Checkout took
1820s on Linux, 3945s on macOS, and 4958s on Windows, versus the earlier
4284s range across platforms. These are observational samples.
- [Native IME 34002299594](https://github.com/stablyai/orca/actions/runs/34002299594)
passed both deterministic and real IBus checks. Shared dependency setup took
29s, versus 35s for the old install/toolchain steps in the sampled baseline.
- Native-IME-only source/spec changes no longer allocate the reusable E2E
build, cache, and consumer jobs just to filter out the native spec. The
separate native workflow still runs; SSH-only and mixed spec lists still
allocate the reusable workflow. Routing contracts exercise these cases.
- [Hourly 34001816449](https://github.com/stablyai/orca/actions/runs/34001816449)
exercised the new five-second preflight and successfully published macOS.
The Windows follow-up failed in its unchanged input-vetting fetch because
remote refs differ only by case on its case-insensitive filesystem. The
requested SHA was correct; this does not validate an unchanged-main skip yet.
Moving the daily Mac freshness check has lower expected value than hourly:
only one potential idle allocation per day, and active development usually
requires that build. Defer another release-graph change until skip frequency
justifies it. The substantive remaining release occupancy opportunity is the
separately documented asynchronous signing policy decision.
@@ -199,16 +199,25 @@ async function applySnapshot(
store: TestStore,
snap: RemoteWorkspaceObservedSnapshot
): Promise<void> {
await applyDirectSshRemoteWorkspaceSnapshot({
store,
snapshot: snap,
token: token(snap.revision),
arrival: 1,
isArrivalCurrent: () => true,
isPreparationTokenCurrent: () => true,
waitForWorkspaceSessionReady: async () => true,
finalizeHydratedTerminals: () => 0
})
vi.useFakeTimers()
try {
const pending = applyDirectSshRemoteWorkspaceSnapshot({
store,
snapshot: snap,
token: token(snap.revision),
arrival: 1,
isArrivalCurrent: () => true,
isPreparationTokenCurrent: () => true,
waitForWorkspaceSessionReady: async () => true,
finalizeHydratedTerminals: () => 0
})
// Exercise the real placement deadline without spending ten wall-clock seconds per snapshot.
await vi.advanceTimersByTimeAsync(10_000)
await pending
} finally {
vi.clearAllTimers()
vi.useRealTimers()
}
}
function adoptedTabIds(store: TestStore): string[] {
@@ -680,7 +680,15 @@ describe('createRemoteWorkspaceTargetSync', () => {
]
})
await harness.sync.applyUnsolicitedSnapshot('target-a', incoming)
vi.useFakeTimers()
try {
const pending = harness.sync.applyUnsolicitedSnapshot('target-a', incoming)
await vi.advanceTimersByTimeAsync(10_000)
await pending
} finally {
harness.sync.stop()
vi.useRealTimers()
}
const merged = hydrateTabsSession.mock.calls[0][0]
expect(merged.tabsByWorktree).toEqual({
@@ -146,6 +146,7 @@ describe('restoreSshConnectionsForStartup', () => {
})
it('does not push a connected background target back into the deferred list', async () => {
vi.useFakeTimers()
installWindowApi([target('ssh-active'), target('ssh-bg')])
// The active host never answers and times out; the background host connects first.
harness.connect.mockImplementation((targetId: string) =>
@@ -154,7 +155,7 @@ describe('restoreSshConnectionsForStartup', () => {
: new Promise<SshConnectionState>(() => {})
)
await restoreSshConnectionsForStartup({
const restore = restoreSshConnectionsForStartup({
connectionIds: ['ssh-active', 'ssh-bg'],
blockingConnectionIds: ['ssh-active'],
setDeferredSshReconnectTargets: harness.setDeferredSshReconnectTargets,
@@ -162,11 +163,14 @@ describe('restoreSshConnectionsForStartup', () => {
publishSshConnectionState: harness.publishSshConnectionState
})
await vi.advanceTimersByTimeAsync(15_000)
await restore
expect(harness.removeDeferredSshReconnectTarget).toHaveBeenCalledWith('ssh-bg')
// The timed-out rewrite must not resurrect the reachable background target: a deferred
// connected target sends fresh panes down the cold-restore path instead of the normal one.
expect(harness.setDeferredSshReconnectTargets).toHaveBeenLastCalledWith(['ssh-active'])
}, 30_000)
})
it('keeps passphrase targets deferred and never dials them', async () => {
installWindowApi([target('ssh-key', true), target('ssh-bg')])