From 6031c19e9fb260e661c29ccea3a196f15e24774b Mon Sep 17 00:00:00 2001 From: Neil <4138956+nwparker@users.noreply.github.com> Date: Sat, 5 Sep 2026 18:20:12 -0700 Subject: [PATCH] ci: reduce dependency, checkout, and test deadline overhead (#18968) * ci: reduce dependency, checkout, and test deadline overhead * ci: avoid generic E2E jobs for native-only IME changes --- .github/workflows/cloud-verify.yml | 9 +-- .github/workflows/pr.yml | 5 +- .github/workflows/release-cut.yml | 23 ++++--- .github/workflows/skill-update-roundtrip.yml | 2 + .github/workflows/terminal-ime-e2e.yml | 18 +---- .github/workflows/terminal-perf.yml | 11 ++-- .../workflows/windows-signing-rehearsal.yml | 1 + .../pr-e2e-native-only-routing.test.mjs | 33 ++++++++++ config/scripts/pr-e2e-source-routing.mjs | 12 ++++ docs/reference/ci-runner-efficiency.md | 66 +++++++++++++++++++ ...ace-snapshot-unplaced-tab-adoption.test.ts | 29 +++++--- .../remote-workspace-target-sync.test.ts | 10 ++- .../startup-ssh-connection-restore.test.ts | 8 ++- 13 files changed, 177 insertions(+), 50 deletions(-) create mode 100644 config/scripts/pr-e2e-native-only-routing.test.mjs diff --git a/.github/workflows/cloud-verify.yml b/.github/workflows/cloud-verify.yml index f0cc2df2bad..e2ba9407ac4 100644 --- a/.github/workflows/cloud-verify.yml +++ b/.github/workflows/cloud-verify.yml @@ -25,9 +25,10 @@ defaults: working-directory: cloud jobs: + # Public-repository hosted runners preserve Blacksmith allowance for macOS. security: name: Secret scan - runs-on: blacksmith-2vcpu-ubuntu-2204 + runs-on: ubuntu-22.04 steps: - uses: actions/checkout@v4 with: @@ -53,7 +54,7 @@ jobs: # Compiles the workspace. No Postgres service: nothing here reaches a # database, and the service container costs ~13s of startup. build: - runs-on: blacksmith-4vcpu-ubuntu-2204 + runs-on: ubuntu-22.04 steps: - uses: actions/checkout@v4 @@ -73,7 +74,7 @@ jobs: # package it needs through the relay pretest hook, so it does not depend on # `pnpm build` having run. test: - runs-on: blacksmith-4vcpu-ubuntu-2204 + runs-on: ubuntu-22.04 services: postgres: image: postgres:16-alpine @@ -107,7 +108,7 @@ jobs: # Fork pull requests reach this job, so it never configures a backend, never plans, and never # holds a credential. Only the relay root ships here; foundation and apps stay private. terraform: - runs-on: blacksmith-2vcpu-ubuntu-2204 + runs-on: ubuntu-22.04 steps: - uses: actions/checkout@v4 diff --git a/.github/workflows/pr.yml b/.github/workflows/pr.yml index 93bc4c0afc8..0e2fa3f273c 100644 --- a/.github/workflows/pr.yml +++ b/.github/workflows/pr.yml @@ -93,12 +93,13 @@ jobs: NATIVE_IME_SOURCE_CHANGED="$(printf '%s\n' "$CHANGED" | node config/scripts/pr-e2e-source-routing.mjs --native-ime-source)" echo "native_ime_source_changed=$NATIVE_IME_SOURCE_CHANGED" >> "$GITHUB_OUTPUT" echo "Native IME source changed: $NATIVE_IME_SOURCE_CHANGED" - if [ "$TEST_FILES_JSON" != '[]' ]; then + SHOULD_RUN="$(printf '%s\n' "$CHANGED" | node config/scripts/pr-e2e-source-routing.mjs --reusable-workflow)" + if [ "$SHOULD_RUN" = true ]; then echo "should_run=true" >> "$GITHUB_OUTPUT" echo "Changed E2E specs: $TEST_FILES_JSON" else echo "should_run=false" >> "$GITHUB_OUTPUT" - echo "No changed E2E specs" + echo "No specs requiring the reusable E2E workflow" fi static_analysis: diff --git a/.github/workflows/release-cut.yml b/.github/workflows/release-cut.yml index 6f888c3a512..001eee4e03c 100644 --- a/.github/workflows/release-cut.yml +++ b/.github/workflows/release-cut.yml @@ -858,16 +858,17 @@ jobs: if: runner.os == 'Linux' run: sudo apt-get update && sudo apt-get install -y build-essential python3 xvfb - - name: Setup Node.js - uses: actions/setup-node@v6 - with: - node-version-file: package.json - - name: Setup pnpm uses: pnpm/setup@v2 with: install: false + - name: Setup Node.js + uses: actions/setup-node@v6 + with: + node-version-file: package.json + cache: pnpm + # Why: Linux terminal golden E2E uses the same native install path as # release CI, which needs pnpm to bypass its non-executable gyp_main.py. - name: Use external node-gyp to avoid pnpm's bundled copy (Linux only) @@ -1074,16 +1075,17 @@ jobs: if: runner.os == 'Linux' run: sudo apt-get update && sudo apt-get install -y build-essential python3 xvfb - - name: Setup Node.js - uses: actions/setup-node@v6 - with: - node-version-file: package.json - - name: Setup pnpm uses: pnpm/setup@v2 with: install: false + - name: Setup Node.js + uses: actions/setup-node@v6 + with: + node-version-file: package.json + cache: pnpm + # Why: keep the non-blocking evidence lane on the same Linux native # install path as the blocking golden and release build jobs. - name: Use external node-gyp to avoid pnpm's bundled copy (Linux only) @@ -1716,6 +1718,7 @@ jobs: with: name: orca-windows-unsigned-${{ needs.cut.outputs.tag }} path: dist/orca-windows-setup.exe + compression-level: 0 if-no-files-found: error # Why: SignPath Foundation production certificates require manual review, diff --git a/.github/workflows/skill-update-roundtrip.yml b/.github/workflows/skill-update-roundtrip.yml index 239f1b2f27c..96de1101275 100644 --- a/.github/workflows/skill-update-roundtrip.yml +++ b/.github/workflows/skill-update-roundtrip.yml @@ -45,7 +45,9 @@ jobs: steps: - uses: actions/checkout@v6 with: + # Historical skill snapshots need tags, but only their blobs are read. fetch-depth: 0 + filter: blob:none persist-credentials: false - uses: actions/setup-node@v6 with: diff --git a/.github/workflows/terminal-ime-e2e.yml b/.github/workflows/terminal-ime-e2e.yml index b9957b2daa9..1ab905d8783 100644 --- a/.github/workflows/terminal-ime-e2e.yml +++ b/.github/workflows/terminal-ime-e2e.yml @@ -38,23 +38,9 @@ jobs: xfwm4 xvfb - - name: Setup Node.js - uses: actions/setup-node@v6 + - uses: ./.github/actions/install-node-dependencies with: - node-version-file: package.json - - - name: Setup pnpm - uses: pnpm/setup@v2 - with: - install: false - - - name: Use external node-gyp to avoid pnpm bundled copy - run: | - npm install -g node-gyp@11.5.0 - echo "npm_config_node_gyp=$(npm root -g)/node-gyp/bin/node-gyp.js" >> "$GITHUB_ENV" - - - name: Install dependencies - run: pnpm install --frozen-lockfile + native-runtime: electron - name: Build Electron app for E2E run: pnpm exec electron-vite build --mode e2e diff --git a/.github/workflows/terminal-perf.yml b/.github/workflows/terminal-perf.yml index 38d0a25bbb7..72a0fec8992 100644 --- a/.github/workflows/terminal-perf.yml +++ b/.github/workflows/terminal-perf.yml @@ -67,16 +67,17 @@ jobs: - name: Install native build tools and xvfb run: sudo apt-get update && sudo apt-get install -y build-essential python3 xvfb zsh - - name: Setup Node.js - uses: actions/setup-node@v6 - with: - node-version-file: package.json - - name: Setup pnpm uses: pnpm/setup@v2 with: install: false + - name: Setup Node.js + uses: actions/setup-node@v6 + with: + node-version-file: package.json + cache: pnpm + # Why: this scheduled/manual workflow uses the same native install path as # PR and E2E CI, which needs pnpm to bypass its bundled gyp_main.py. - name: Use external node-gyp to avoid pnpm's bundled copy diff --git a/.github/workflows/windows-signing-rehearsal.yml b/.github/workflows/windows-signing-rehearsal.yml index 54b751908dc..6fc6fab7193 100644 --- a/.github/workflows/windows-signing-rehearsal.yml +++ b/.github/workflows/windows-signing-rehearsal.yml @@ -215,6 +215,7 @@ jobs: with: name: orca-windows-installer-unsigned-${{ github.run_id }} path: dist/orca-windows-setup.exe + compression-level: 0 if-no-files-found: error - name: Submit Windows installer signing request diff --git a/config/scripts/pr-e2e-native-only-routing.test.mjs b/config/scripts/pr-e2e-native-only-routing.test.mjs new file mode 100644 index 00000000000..b6c3662cd1d --- /dev/null +++ b/config/scripts/pr-e2e-native-only-routing.test.mjs @@ -0,0 +1,33 @@ +import { readFileSync } from 'node:fs' +import { describe, expect, it } from 'vitest' +import { parse } from 'yaml' +import { hasNativeImeSourceChange, shouldRunReusablePrE2e } from './pr-e2e-source-routing.mjs' + +const workflow = parse(readFileSync('.github/workflows/pr.yml', 'utf8')) +const filterStep = workflow.jobs.code_paths.steps.find((step) => step.id === 'e2e_filter') + +describe('native-only PR E2E routing', () => { + it('avoids generic E2E allocation for native-only changes while preserving its IME lane', () => { + for (const file of [ + 'tests/e2e/terminal-ibus-hangul-native.spec.ts', + 'config/scripts/run-terminal-ibus-hangul-e2e.mjs' + ]) { + expect(hasNativeImeSourceChange([file])).toBe(true) + expect(shouldRunReusablePrE2e([file])).toBe(false) + } + expect(shouldRunReusablePrE2e([])).toBe(false) + for (const spec of [ + 'tests/e2e/ssh-startup-exec-readiness.spec.ts', + 'tests/e2e/paired-startup-exec-readiness.spec.ts', + 'tests/e2e/terminal-ime-exact-byte.spec.ts', + 'tests/e2e/future.spec.ts' + ]) { + expect(shouldRunReusablePrE2e([spec])).toBe(true) + expect(shouldRunReusablePrE2e(['tests/e2e/terminal-ibus-hangul-native.spec.ts', spec])).toBe( + true + ) + } + expect(filterStep.run).toContain('pr-e2e-source-routing.mjs --reusable-workflow') + expect(filterStep.run).toContain('if [ "$SHOULD_RUN" = true ]; then') + }) +}) diff --git a/config/scripts/pr-e2e-source-routing.mjs b/config/scripts/pr-e2e-source-routing.mjs index 78814b663cb..5b698fb0b42 100644 --- a/config/scripts/pr-e2e-source-routing.mjs +++ b/config/scripts/pr-e2e-source-routing.mjs @@ -217,6 +217,16 @@ export function hasNativeImeSourceChange(changedPaths) { ).some((route) => changedPaths.some(route.matches)) } +export function shouldRunReusablePrE2e(changedPaths) { + // Native IME has its own workflow; SSH still runs inside the reusable workflow. + return ( + hasSshSourceChange(changedPaths) || + selectPrE2eSpecs(changedPaths).some( + (spec) => spec !== 'tests/e2e/terminal-ibus-hangul-native.spec.ts' + ) + ) +} + if (process.argv[1] && import.meta.url === pathToFileURL(process.argv[1]).href) { let input = '' process.stdin.setEncoding('utf8') @@ -226,6 +236,8 @@ if (process.argv[1] && import.meta.url === pathToFileURL(process.argv[1]).href) const changedPaths = input.split(/\r?\n/).filter(Boolean) if (process.argv.includes('--ssh-source')) { process.stdout.write(`${hasSshSourceChange(changedPaths)}\n`) + } else if (process.argv.includes('--reusable-workflow')) { + process.stdout.write(`${shouldRunReusablePrE2e(changedPaths)}\n`) } else if (process.argv.includes('--native-ime-source')) { process.stdout.write(`${hasNativeImeSourceChange(changedPaths)}\n`) } else { diff --git a/docs/reference/ci-runner-efficiency.md b/docs/reference/ci-runner-efficiency.md index a9f644bc435..6d688598097 100644 --- a/docs/reference/ci-runner-efficiency.md +++ b/docs/reference/ci-runner-efficiency.md @@ -131,3 +131,69 @@ environments currently exist. An environment-gated design adds a GitHub approval after each SignPath approval and changes the current automatic inner signing timeout fallback; those are explicit release-policy decisions, so this PR leaves production signing behavior unchanged. + +## Second audit and hosted trials + +- Cloud Verify ran 100 times in a sampled 39-hour window (84 PR and 16 push + runs). Move its four Ubuntu 22.04 jobs from Blacksmith to standard hosted + Ubuntu 22.04, preserving Postgres, secret scanning, build, tests, and Terraform + validation. Baseline [34001538145](https://github.com/stablyai/orca/actions/runs/34001538145) + used 64/72/26/19 seconds for security/test/build/Terraform respectively. + This conserves the shared provider allowance; hosted latency must be checked. +- Keep full tag history for the 13-job skill round-trip matrix, but fetch blobs + lazily. Only two historical SKILL.md files are materialized. Baseline + [33999994876](https://github.com/stablyai/orca/actions/runs/33999994876) + spent 42–84 seconds per checkout, about 14 aggregate runner minutes. A hosted + trial must verify historical blob fetches on all three operating systems. +- Use the existing Electron/native dependency cache for native IME CI. Keep + both deterministic boundary and real IBus tests. Add pnpm store caching to + terminal perf and release golden/evidence lanes; retain their raw installs + because manually selected older refs may not contain the shared action. +- Disable ZIP recompression only for already-compressed NSIS installers sent + to SignPath. Installer contents, release compression, and signing stay intact. +- Advance existing placement and startup deadlines with scoped fake timers in + three renderer test files. All 34 tests pass in 62 ms of local test execution, + versus 65.182 seconds in the sampled hosted baseline. Imports and transforms + still dominate invocation time; this is not a claim of equal PR wall savings. + +Eight unit shards already have balanced 260–296-second sample durations. +Reducing shards or removing test isolation lacks evidence of a net gain. Real +subprocess tests intentionally cover lifecycle behavior and retain real clocks. +The 14-way E2E split retains headroom after earlier 12-way timeouts. Lowering +coverage or schedule frequency is outside this efficiency pass. Cache complexity +for a seven-second docs install is unlikely to pay back. Release build reuse +across modes risks differing telemetry identities and native platform artifacts. + +Terminal Perf's baseline [33955846492](https://github.com/stablyai/orca/actions/runs/33955846492) +failed waiting 30 seconds for workspaceSessionReady in its shared-page fixture, +before measuring terminal performance. Compare hosted trials against that known +failure rather than attributing it to dependency cache changes. + +Hosted trials for the second audit: + +- [Cloud Verify 34002295216](https://github.com/stablyai/orca/actions/runs/34002295216) + passed all four jobs on standard hosted Ubuntu: security 57s, test 102s, build + 35s, Terraform 19s. The test lane is 30s slower than the Blacksmith sample; + retain this modest latency tradeoff to conserve shared allowance. +- [Skill matrix 34002295221](https://github.com/stablyai/orca/actions/runs/34002295221) + passed all 13 legs, including historical blob materialization. Checkout took + 18–20s on Linux, 39–45s on macOS, and 49–58s on Windows, versus the earlier + 42–84s range across platforms. These are observational samples. +- [Native IME 34002299594](https://github.com/stablyai/orca/actions/runs/34002299594) + passed both deterministic and real IBus checks. Shared dependency setup took + 29s, versus 35s for the old install/toolchain steps in the sampled baseline. +- Native-IME-only source/spec changes no longer allocate the reusable E2E + build, cache, and consumer jobs just to filter out the native spec. The + separate native workflow still runs; SSH-only and mixed spec lists still + allocate the reusable workflow. Routing contracts exercise these cases. +- [Hourly 34001816449](https://github.com/stablyai/orca/actions/runs/34001816449) + exercised the new five-second preflight and successfully published macOS. + The Windows follow-up failed in its unchanged input-vetting fetch because + remote refs differ only by case on its case-insensitive filesystem. The + requested SHA was correct; this does not validate an unchanged-main skip yet. + +Moving the daily Mac freshness check has lower expected value than hourly: +only one potential idle allocation per day, and active development usually +requires that build. Defer another release-graph change until skip frequency +justifies it. The substantive remaining release occupancy opportunity is the +separately documented asynchronous signing policy decision. diff --git a/src/renderer/src/hooks/remote-workspace-snapshot-unplaced-tab-adoption.test.ts b/src/renderer/src/hooks/remote-workspace-snapshot-unplaced-tab-adoption.test.ts index 13daf0c9f20..391bdf10693 100644 --- a/src/renderer/src/hooks/remote-workspace-snapshot-unplaced-tab-adoption.test.ts +++ b/src/renderer/src/hooks/remote-workspace-snapshot-unplaced-tab-adoption.test.ts @@ -199,16 +199,25 @@ async function applySnapshot( store: TestStore, snap: RemoteWorkspaceObservedSnapshot ): Promise { - await applyDirectSshRemoteWorkspaceSnapshot({ - store, - snapshot: snap, - token: token(snap.revision), - arrival: 1, - isArrivalCurrent: () => true, - isPreparationTokenCurrent: () => true, - waitForWorkspaceSessionReady: async () => true, - finalizeHydratedTerminals: () => 0 - }) + vi.useFakeTimers() + try { + const pending = applyDirectSshRemoteWorkspaceSnapshot({ + store, + snapshot: snap, + token: token(snap.revision), + arrival: 1, + isArrivalCurrent: () => true, + isPreparationTokenCurrent: () => true, + waitForWorkspaceSessionReady: async () => true, + finalizeHydratedTerminals: () => 0 + }) + // Exercise the real placement deadline without spending ten wall-clock seconds per snapshot. + await vi.advanceTimersByTimeAsync(10_000) + await pending + } finally { + vi.clearAllTimers() + vi.useRealTimers() + } } function adoptedTabIds(store: TestStore): string[] { diff --git a/src/renderer/src/hooks/remote-workspace-target-sync.test.ts b/src/renderer/src/hooks/remote-workspace-target-sync.test.ts index bc0082c03ce..7d9f662ffe0 100644 --- a/src/renderer/src/hooks/remote-workspace-target-sync.test.ts +++ b/src/renderer/src/hooks/remote-workspace-target-sync.test.ts @@ -680,7 +680,15 @@ describe('createRemoteWorkspaceTargetSync', () => { ] }) - await harness.sync.applyUnsolicitedSnapshot('target-a', incoming) + vi.useFakeTimers() + try { + const pending = harness.sync.applyUnsolicitedSnapshot('target-a', incoming) + await vi.advanceTimersByTimeAsync(10_000) + await pending + } finally { + harness.sync.stop() + vi.useRealTimers() + } const merged = hydrateTabsSession.mock.calls[0][0] expect(merged.tabsByWorktree).toEqual({ diff --git a/src/renderer/src/startup/startup-ssh-connection-restore.test.ts b/src/renderer/src/startup/startup-ssh-connection-restore.test.ts index 3464816d0d8..3f74655454a 100644 --- a/src/renderer/src/startup/startup-ssh-connection-restore.test.ts +++ b/src/renderer/src/startup/startup-ssh-connection-restore.test.ts @@ -146,6 +146,7 @@ describe('restoreSshConnectionsForStartup', () => { }) it('does not push a connected background target back into the deferred list', async () => { + vi.useFakeTimers() installWindowApi([target('ssh-active'), target('ssh-bg')]) // The active host never answers and times out; the background host connects first. harness.connect.mockImplementation((targetId: string) => @@ -154,7 +155,7 @@ describe('restoreSshConnectionsForStartup', () => { : new Promise(() => {}) ) - await restoreSshConnectionsForStartup({ + const restore = restoreSshConnectionsForStartup({ connectionIds: ['ssh-active', 'ssh-bg'], blockingConnectionIds: ['ssh-active'], setDeferredSshReconnectTargets: harness.setDeferredSshReconnectTargets, @@ -162,11 +163,14 @@ describe('restoreSshConnectionsForStartup', () => { publishSshConnectionState: harness.publishSshConnectionState }) + await vi.advanceTimersByTimeAsync(15_000) + await restore + expect(harness.removeDeferredSshReconnectTarget).toHaveBeenCalledWith('ssh-bg') // The timed-out rewrite must not resurrect the reachable background target: a deferred // connected target sends fresh panes down the cold-restore path instead of the normal one. expect(harness.setDeferredSshReconnectTargets).toHaveBeenLastCalledWith(['ssh-active']) - }, 30_000) + }) it('keeps passphrase targets deferred and never dials them', async () => { installWindowApi([target('ssh-key', true), target('ssh-bg')])