diff --git a/.github/actions/install-node-dependencies/action.yml b/.github/actions/install-node-dependencies/action.yml index dc9a2f2f439..053fcdbe0fb 100644 --- a/.github/actions/install-node-dependencies/action.yml +++ b/.github/actions/install-node-dependencies/action.yml @@ -42,10 +42,16 @@ outputs: value: ${{ steps.requested-node.outputs.node-version || steps.default-node.outputs.node-version }} native-cache-scope: description: Operating-system image scope used by the native module cache. - value: ${{ steps.native-cache-scope.outputs.scope }} + value: ${{ steps.native-runtime.outputs.cache-scope }} + native-cache-key: + description: Exact native cache key requested after installation. + value: ${{ steps.native-runtime.outputs.cache-key }} + native-cache-path: + description: Native build directories covered by the requested cache. + value: ${{ steps.native-runtime.outputs.cache-path }} native-cache-hit: description: Whether the compiled native module cache was restored. - value: ${{ steps.native-cache-restore.outputs.cache-hit || steps.native-cache-restore-only.outputs.cache-hit }} + value: ${{ steps.native-runtime.outputs.cache-hit }} runs: using: composite @@ -203,68 +209,11 @@ runs: path: ${{ steps.electron-package-cache.outputs.cache-root }} key: electron-package-${{ runner.os }}-${{ runner.arch }}-${{ steps.electron-package-cache.outputs.version }} - # Why cached: `--ignore-scripts` leaves node-pty without build/Release, so - # ensure-native-runtime node-gyp-compiles it in every job that asks for a runtime. - # The artifacts are ABI-bound, so the key carries the target runtime, the resolved - # Node version, and the patch whose contents the build has to match. - # Windows extra globs are empty on Linux. No restore-keys: a partial-match key is - # an ABI-mismatched build, and ensure-native-runtime would recompile it anyway. - # Native addons built on a newer Linux image can require glibc symbols - # missing from an older runner/container. ImageOS distinguishes hosted - # Windows/macOS images; /etc/os-release also distinguishes Linux containers. - - name: Resolve native cache scope - id: native-cache-scope - if: inputs.native-runtime != 'none' - shell: bash - run: | - scope="${ImageOS:-$RUNNER_OS}" - if [ -r /etc/os-release ]; then - . /etc/os-release - scope="${ID:-linux}-${VERSION_ID:-unknown}" - fi - echo "scope=$scope" >> "$GITHUB_OUTPUT" - - - name: Restore compiled native modules - id: native-cache-restore - if: inputs.native-runtime != 'none' && inputs.persist-native-cache != 'false' - uses: actions/cache@v5 - with: - path: | - node_modules/.pnpm/node-pty@*/node_modules/node-pty/build - native/windows-registry/build - node_modules/.pnpm/@vscode+windows-process-tre*/node_modules/@vscode/windows-process-tree/build - key: native-modules-${{ runner.os }}-${{ steps.native-cache-scope.outputs.scope }}-${{ runner.arch }}-${{ inputs.native-runtime }}-node${{ steps.requested-node.outputs.node-version || steps.default-node.outputs.node-version }}-${{ hashFiles('pnpm-lock.yaml', '.github/actions/install-node-dependencies/action.yml', 'config/scripts/ensure-native-runtime.mjs', 'config/scripts/rebuild-native-deps.mjs', 'config/patches/node-pty@1.1.0.patch', 'config/patches/@vscode__windows-process-tree@0.8.0.patch', 'native/windows-registry/src/addon.cc', 'native/windows-registry/binding.gyp', 'native/windows-registry/package.json') }} - - - name: Restore compiled native modules without saving - id: native-cache-restore-only - if: inputs.native-runtime != 'none' && inputs.persist-native-cache == 'false' - uses: actions/cache/restore@v5 - with: - path: | - node_modules/.pnpm/node-pty@*/node_modules/node-pty/build - native/windows-registry/build - node_modules/.pnpm/@vscode+windows-process-tre*/node_modules/@vscode/windows-process-tree/build - key: native-modules-${{ runner.os }}-${{ steps.native-cache-scope.outputs.scope }}-${{ runner.arch }}-${{ inputs.native-runtime }}-node${{ steps.requested-node.outputs.node-version || steps.default-node.outputs.node-version }}-${{ hashFiles('pnpm-lock.yaml', '.github/actions/install-node-dependencies/action.yml', 'config/scripts/ensure-native-runtime.mjs', 'config/scripts/rebuild-native-deps.mjs', 'config/patches/node-pty@1.1.0.patch', 'config/patches/@vscode__windows-process-tree@0.8.0.patch', 'native/windows-registry/src/addon.cc', 'native/windows-registry/binding.gyp', 'native/windows-registry/package.json') }} - - # pnpm's bundled gyp_main.py is not executable; Electron rebuild uses its own node-gyp API. - - name: Use external node-gyp - if: runner.os == 'Linux' && inputs.native-runtime == 'node' - shell: bash - env: - NATIVE_RUNTIME: ${{ inputs.native-runtime }} - NATIVE_CACHE_HIT: ${{ steps.native-cache-restore.outputs.cache-hit || steps.native-cache-restore-only.outputs.cache-hit }} - run: | - # A cache hit can contain unusable addons; probe before skipping the rebuild toolchain. - if [ "$NATIVE_RUNTIME" = node ] && [ "$NATIVE_CACHE_HIT" = true ] && - node config/scripts/ensure-native-runtime.mjs --check-only; then - exit 0 - fi - npm install -g node-gyp@11.5.0 - echo "npm_config_node_gyp=$(npm root -g)/node-gyp/bin/node-gyp.js" >> "$GITHUB_ENV" - - name: Prepare native runtime + id: native-runtime if: inputs.native-runtime != 'none' - shell: bash - env: - NATIVE_RUNTIME: ${{ inputs.native-runtime }} - run: node config/scripts/ensure-native-runtime.mjs --runtime="$NATIVE_RUNTIME" + uses: ./.github/actions/prepare-native-runtime + with: + native-runtime: ${{ inputs.native-runtime }} + node-version: ${{ steps.requested-node.outputs.node-version || steps.default-node.outputs.node-version }} + persist-native-cache: ${{ inputs.persist-native-cache }} diff --git a/.github/actions/prepare-native-runtime/action.yml b/.github/actions/prepare-native-runtime/action.yml new file mode 100644 index 00000000000..ae56feeb68e --- /dev/null +++ b/.github/actions/prepare-native-runtime/action.yml @@ -0,0 +1,120 @@ +name: Prepare native runtime +description: Restore and probe source-built native modules for the requested runtime. + +inputs: + native-runtime: + description: Native runtime to prepare (none, node, or electron). + required: false + default: none + node-version: + description: Resolved Node.js version provided by setup-node. + required: true + persist-native-cache: + description: Save native modules at job end unless a later step changes their ABI. + required: false + default: 'true' + +outputs: + cache-scope: + description: Operating-system image scope used by the native module cache. + value: ${{ steps.native-cache-scope.outputs.scope }} + cache-key: + description: Exact key requested for this runtime, independent of whether restoration succeeds. + value: ${{ steps.native-cache-scope.outputs.key }} + cache-path: + description: Native build directories covered by the requested cache. + value: ${{ steps.native-cache-scope.outputs.path }} + cache-hit: + description: Whether compiled native modules were restored under the exact requested key. + value: ${{ steps.native-cache-restore.outputs.cache-hit || steps.native-cache-restore-only.outputs.cache-hit }} + +runs: + using: composite + steps: + - name: Validate native runtime + shell: bash + env: + NATIVE_RUNTIME: ${{ inputs.native-runtime }} + NODE_VERSION: ${{ inputs.node-version }} + run: | + case "$NATIVE_RUNTIME" in + none) ;; + node|electron) test -n "$NODE_VERSION" ;; + *) + echo "::error::native-runtime must be none, node, or electron" + exit 2 + ;; + esac + + # Exact keys separate runtime, architecture and host libc; every restored addon is probed. + - name: Resolve native cache scope + id: native-cache-scope + if: inputs.native-runtime != 'none' + shell: bash + env: + NATIVE_RUNTIME: ${{ inputs.native-runtime }} + NODE_VERSION: ${{ inputs.node-version }} + NATIVE_SOURCE_HASH: ${{ hashFiles('pnpm-lock.yaml', 'pnpm-workspace.yaml', '.npmrc', '.pnpmfile.cjs', '.github/actions/prepare-native-runtime/action.yml', 'config/scripts/ensure-native-runtime.mjs', 'config/scripts/rebuild-native-deps.mjs', 'config/scripts/node-pty-job-ownership.cjs', 'config/scripts/windows-pe-machine.cjs', 'config/scripts/windows-process-tree-gyp-rebuild.mjs', 'config/scripts/windows-process-tree-creation-time.cjs', 'config/scripts/install-electron-package-binary.mjs', 'config/scripts/electron-platform-path.mjs', 'config/scripts/zip-extractor-command.mjs', 'src/shared/zip-extractor-command.ts', 'config/scripts/shared-electron-dist-cache.mjs', 'config/scripts/space-sharing-copy.mjs', 'config/patches/node-pty@1.1.0.patch', 'config/patches/@vscode__windows-process-tree@0.8.0.patch', 'native/windows-registry/src/addon.cc', 'native/windows-registry/binding.gyp', 'native/windows-registry/package.json', 'native/windows-registry/index.js') }} + run: | + scope="${ImageOS:-$RUNNER_OS}" + if [ -r /etc/os-release ]; then + . /etc/os-release + scope="${ID:-linux}-${VERSION_ID:-unknown}" + fi + pnpm_version="$(pnpm --version)" + test -n "$pnpm_version" + printf 'scope=%s\n' "$scope" >> "$GITHUB_OUTPUT" + printf 'key=native-modules-%s-%s-%s-%s-node%s-pnpm%s-%s\n' "$RUNNER_OS" "$scope" "$RUNNER_ARCH" "$NATIVE_RUNTIME" "$NODE_VERSION" "$pnpm_version" "$NATIVE_SOURCE_HASH" >> "$GITHUB_OUTPUT" + cat >> "$GITHUB_OUTPUT" <<'ORCA_NATIVE_CACHE_PATHS' + path<> "$GITHUB_ENV" + + - name: Prepare native runtime + if: inputs.native-runtime != 'none' + shell: bash + env: + NATIVE_RUNTIME: ${{ inputs.native-runtime }} + run: node config/scripts/ensure-native-runtime.mjs --runtime="$NATIVE_RUNTIME" diff --git a/.github/workflows/ci-cache-warmup.yml b/.github/workflows/ci-cache-warmup.yml index 4d69cd1c104..bdd3e185638 100644 --- a/.github/workflows/ci-cache-warmup.yml +++ b/.github/workflows/ci-cache-warmup.yml @@ -9,6 +9,7 @@ on: paths: - '.github/workflows/ci-cache-warmup.yml' - '.github/actions/install-node-dependencies/**' + - '.github/actions/prepare-native-runtime/**' - '.github/actions/prepare-git-compatibility/**' - '.github/actions/prepare-linux-package-fixture/**' - 'config/docker/headless-serve-shutdown/**' @@ -17,9 +18,21 @@ on: - 'package.json' - 'pnpm-lock.yaml' - 'pnpm-workspace.yaml' + - '.npmrc' + - '.pnpmfile.cjs' - 'config/tsconfig*.json' - 'config/scripts/ensure-native-runtime.mjs' - 'config/scripts/rebuild-native-deps.mjs' + - 'config/scripts/node-pty-job-ownership.cjs' + - 'config/scripts/windows-pe-machine.cjs' + - 'config/scripts/windows-process-tree-gyp-rebuild.mjs' + - 'config/scripts/windows-process-tree-creation-time.cjs' + - 'config/scripts/install-electron-package-binary.mjs' + - 'config/scripts/electron-platform-path.mjs' + - 'config/scripts/zip-extractor-command.mjs' + - 'src/shared/zip-extractor-command.ts' + - 'config/scripts/shared-electron-dist-cache.mjs' + - 'config/scripts/space-sharing-copy.mjs' - 'config/patches/node-pty@1.1.0.patch' - 'config/patches/@vscode__windows-process-tree@0.8.0.patch' - 'native/windows-registry/**' diff --git a/.github/workflows/node-server-tests.yml b/.github/workflows/node-server-tests.yml index ab86fcd6fe6..0a2840c60a1 100644 --- a/.github/workflows/node-server-tests.yml +++ b/.github/workflows/node-server-tests.yml @@ -16,6 +16,7 @@ on: - '.npmrc' - '.pnpmfile.cjs' - '.github/actions/install-node-dependencies/**' + - '.github/actions/prepare-native-runtime/**' - '.github/workflows/node-server-tests.yml' # The pull request qualifies one platform for an unflavoured change; this is where all six # are re-qualified, so a platform break surfaces minutes after merge instead of next cron. @@ -34,6 +35,7 @@ on: - '.npmrc' - '.pnpmfile.cjs' - '.github/actions/install-node-dependencies/**' + - '.github/actions/prepare-native-runtime/**' - '.github/workflows/node-server-tests.yml' workflow_dispatch: inputs: diff --git a/.github/workflows/pr.yml b/.github/workflows/pr.yml index 278c6ca42d5..32a167cf071 100644 --- a/.github/workflows/pr.yml +++ b/.github/workflows/pr.yml @@ -1091,11 +1091,8 @@ jobs: if: steps.deps.outputs.native-cache-hit != 'true' uses: actions/cache/save@v5 with: - path: | - node_modules/.pnpm/node-pty@*/node_modules/node-pty/build - native/windows-registry/build - node_modules/.pnpm/@vscode+windows-process-tre*/node_modules/@vscode/windows-process-tree/build - key: native-modules-${{ runner.os }}-${{ steps.deps.outputs.native-cache-scope }}-${{ runner.arch }}-node-node${{ steps.deps.outputs.node-version }}-${{ hashFiles('pnpm-lock.yaml', '.github/actions/install-node-dependencies/action.yml', 'config/scripts/ensure-native-runtime.mjs', 'config/scripts/rebuild-native-deps.mjs', 'config/patches/node-pty@1.1.0.patch', 'config/patches/@vscode__windows-process-tree@0.8.0.patch', 'native/windows-registry/src/addon.cc', 'native/windows-registry/binding.gyp', 'native/windows-registry/package.json') }} + path: ${{ steps.deps.outputs.native-cache-path }} + key: ${{ steps.deps.outputs.native-cache-key }} # vitest runs here directly rather than through `pnpm test`, so the addon # assertions only hold once install-node-dependencies has rebuilt natives. @@ -1195,17 +1192,11 @@ jobs: - name: Build package inputs run: pnpm run build:release:parallel - - name: Restore compiled Electron native modules - uses: actions/cache@v5 - with: - path: | - node_modules/.pnpm/node-pty@*/node_modules/node-pty/build - native/windows-registry/build - node_modules/.pnpm/@vscode+windows-process-tre*/node_modules/@vscode/windows-process-tree/build - key: native-modules-${{ runner.os }}-${{ steps.deps.outputs.native-cache-scope }}-${{ runner.arch }}-electron-node${{ steps.deps.outputs.node-version }}-${{ hashFiles('pnpm-lock.yaml', '.github/actions/install-node-dependencies/action.yml', 'config/scripts/ensure-native-runtime.mjs', 'config/scripts/rebuild-native-deps.mjs', 'config/patches/node-pty@1.1.0.patch', 'config/patches/@vscode__windows-process-tree@0.8.0.patch', 'native/windows-registry/src/addon.cc', 'native/windows-registry/binding.gyp', 'native/windows-registry/package.json') }} - - name: Prepare Electron native runtime - run: node config/scripts/ensure-native-runtime.mjs --runtime=electron + uses: ./.github/actions/prepare-native-runtime + with: + native-runtime: electron + node-version: ${{ steps.deps.outputs.node-version }} - name: Package unpacked app env: diff --git a/config/scripts/ci-cache-warmup-workflow.test.mjs b/config/scripts/ci-cache-warmup-workflow.test.mjs index 4b20c04b157..07c207bddce 100644 --- a/config/scripts/ci-cache-warmup-workflow.test.mjs +++ b/config/scripts/ci-cache-warmup-workflow.test.mjs @@ -8,6 +8,21 @@ const readWorkflow = (name) => const workflow = readWorkflow('ci-cache-warmup') const steps = workflow.jobs.warm.steps +it('seeds new native keys on main when any declared build or probe input changes', () => { + const native = parse(readFileSync('.github/actions/prepare-native-runtime/action.yml', 'utf8')) + const hash = native.runs.steps.find((step) => step.id === 'native-cache-scope').env + .NATIVE_SOURCE_HASH + const files = [...hash.matchAll(/'([^']+)'/g)].map((match) => match[1]) + for (const file of files) { + expect( + workflow.on.push.paths.some((pattern) => + pattern.endsWith('/**') ? file.startsWith(pattern.slice(0, -2)) : file === pattern + ), + file + ).toBe(true) + } +}) + it('warms the same Linux Node runtime the PR shards restore', () => { const arm = workflow.jobs['warm-linux-arm'] const install = arm.steps.find( @@ -56,6 +71,7 @@ it('bounds warming to the required platforms and validates changes without grant expect(workflow.jobs.warm['timeout-minutes']).toBeLessThanOrEqual(10) expect(workflow.permissions).toEqual({ contents: 'read' }) expect(workflow.on.push.branches).toEqual(['main']) + expect(workflow.on.push.paths).toContain('.github/actions/prepare-native-runtime/**') expect(workflow.on.schedule).toEqual([{ cron: '41 * * * *' }]) expect(workflow.on.pull_request.paths).toContain('.github/workflows/ci-cache-warmup.yml') expect(workflow.concurrency['cancel-in-progress']).toBe(true) diff --git a/config/scripts/ci-native-cache.test.mjs b/config/scripts/ci-native-cache.test.mjs new file mode 100644 index 00000000000..6851ed62f9b --- /dev/null +++ b/config/scripts/ci-native-cache.test.mjs @@ -0,0 +1,248 @@ +import { createHash } from 'node:crypto' +import { existsSync, mkdtempSync, readFileSync, rmSync, writeFileSync } from 'node:fs' +import { tmpdir } from 'node:os' +import { join } from 'node:path' +import { parse } from 'yaml' +import { describe, expect, it } from 'vitest' +import { classifyPrJobs } from './pr-code-change-scope.mjs' +import { runProcessSync } from './script-child-process.mjs' + +const installerPath = '.github/actions/install-node-dependencies/action.yml' +const nativePath = '.github/actions/prepare-native-runtime/action.yml' +const installer = parse(readFileSync(installerPath, 'utf8')) +const native = parse(readFileSync(nativePath, 'utf8')) +const identity = native.runs.steps.find((step) => step.id === 'native-cache-scope') +const inputs = [...identity.env.NATIVE_SOURCE_HASH.matchAll(/'([^']+)'/g)].map((match) => match[1]) +const source = new Map( + [...inputs, installerPath].map((file) => [file, existsSync(file) ? readFileSync(file) : null]) +) +const paths = [ + 'node_modules/.pnpm/node-pty@*/node_modules/node-pty/build', + 'native/windows-registry/build', + 'node_modules/.pnpm/@vscode+windows-process-tre*/node_modules/@vscode/windows-process-tree/build' +] + +function sourceHash(files = source) { + const digest = createHash('sha256') + for (const file of inputs.toSorted()) { + const contents = files.get(file) + if (contents !== null) { + digest.update(createHash('sha256').update(contents).digest()) + } + } + return digest.digest('hex') +} + +function resolveIdentity({ + runtime = 'node', + node = 'v24.21.0', + pnpm = '12.0.0', + os = 'Windows', + arch = 'X64', + image = 'win22', + libc = null, + files = source +} = {}) { + const directory = mkdtempSync(join(tmpdir(), 'orca-native-identity-')) + try { + const output = join(directory, 'output') + const osRelease = join(directory, 'os-release') + if (libc) { + writeFileSync(osRelease, `ID=${libc.id}\nVERSION_ID=${libc.version}\n`) + } + // Isolate the container file lookup while running the production identity commands. + const script = `pnpm() { printf "%s\\n" "$TEST_PNPM_VERSION"; }\n${identity.run.replaceAll('/etc/os-release', '"$TEST_OS_RELEASE"')}` + const execution = runProcessSync({ + program: 'bash', + args: ['-e', '-o', 'pipefail', '-c', script], + env: { + ...process.env, + NATIVE_RUNTIME: runtime, + NODE_VERSION: node, + RUNNER_OS: os, + RUNNER_ARCH: arch, + ImageOS: image, + NATIVE_SOURCE_HASH: sourceHash(files), + TEST_PNPM_VERSION: pnpm, + TEST_OS_RELEASE: osRelease, + GITHUB_OUTPUT: output + } + }) + expect(execution.code, execution.stderr).toBe(0) + const result = readFileSync(output, 'utf8') + return { + key: /^key=(.+)$/m.exec(result)?.[1], + scope: /^scope=(.+)$/m.exec(result)?.[1], + path: /path< { + it('hashes the complete native preparation policy and build/probe source closure', () => { + expect(inputs).toEqual([ + 'pnpm-lock.yaml', + 'pnpm-workspace.yaml', + '.npmrc', + '.pnpmfile.cjs', + nativePath, + 'config/scripts/ensure-native-runtime.mjs', + 'config/scripts/rebuild-native-deps.mjs', + 'config/scripts/node-pty-job-ownership.cjs', + 'config/scripts/windows-pe-machine.cjs', + 'config/scripts/windows-process-tree-gyp-rebuild.mjs', + 'config/scripts/windows-process-tree-creation-time.cjs', + 'config/scripts/install-electron-package-binary.mjs', + 'config/scripts/electron-platform-path.mjs', + 'config/scripts/zip-extractor-command.mjs', + 'src/shared/zip-extractor-command.ts', + 'config/scripts/shared-electron-dist-cache.mjs', + 'config/scripts/space-sharing-copy.mjs', + 'config/patches/node-pty@1.1.0.patch', + 'config/patches/@vscode__windows-process-tree@0.8.0.patch', + 'native/windows-registry/src/addon.cc', + 'native/windows-registry/binding.gyp', + 'native/windows-registry/package.json', + 'native/windows-registry/index.js' + ]) + expect(inputs).not.toContain(installerPath) + for (const file of inputs) { + expect(classifyPrJobs([file]).native_cache_changed, file).toBe(true) + } + }) + + it.skipIf(process.platform === 'win32')( + 'keeps a pnpm-only installer edit on the same native key and paths', + () => { + const changed = new Map(source) + changed.set( + installerPath, + Buffer.from( + readFileSync(installerPath, 'utf8').replace( + "inputs.cache-pnpm-verification == 'true'", + "inputs.cache-pnpm-verification == 'false'" + ) + ) + ) + expect(changed.get(installerPath)).not.toEqual(source.get(installerPath)) + expect(resolveIdentity({ files: changed })).toEqual(resolveIdentity()) + } + ) + + it.skipIf(process.platform === 'win32').each(inputs)( + 'changes the requested key when native input %s changes', + (file) => { + const changed = new Map(source) + changed.set( + file, + Buffer.concat([source.get(file) ?? Buffer.alloc(0), Buffer.from('\nchanged')]) + ) + expect(resolveIdentity({ files: changed }).key).not.toBe(resolveIdentity().key) + } + ) + + it.skipIf(process.platform === 'win32')( + 'separates runtime, Node/pnpm versions, architecture, image and libc hosts', + () => { + const baseline = resolveIdentity() + expect(baseline.path.split('\n')).toEqual(paths) + const variants = [ + { runtime: 'electron' }, + { node: 'v24.22.0' }, + { pnpm: '12.1.0' }, + { arch: 'ARM64' }, + { image: 'win11' }, + { os: 'Linux', image: 'ubuntu22', libc: { id: 'ubuntu', version: '22.04' } }, + { os: 'Linux', image: 'ubuntu22', libc: { id: 'almalinux', version: '8.10' } } + ].map(resolveIdentity) + expect(new Set([baseline, ...variants].map(({ key }) => key)).size).toBe(variants.length + 1) + expect(variants.at(-1).scope).toBe('almalinux-8.10') + expect(variants.every(({ path }) => path === baseline.path)).toBe(true) + } + ) + + it('keeps early input rejection before dependency installation and native restoration', () => { + const early = installer.runs.steps.findIndex((step) => step.name === 'Validate native runtime') + const install = installer.runs.steps.findIndex((step) => step.name === 'Install dependencies') + const prepare = installer.runs.steps.findIndex((step) => step.id === 'native-runtime') + expect(early).toBeLessThan(install) + expect(install).toBeLessThan(prepare) + expect(installer.runs.steps[early].run).toContain('none|node|electron)') + expect(installer.runs.steps[prepare].if).toBe("inputs.native-runtime != 'none'") + }) + + it.skipIf(process.platform === 'win32')( + 'rejects invalid runtime and missing resolved Node before cache restoration', + () => { + const early = installer.runs.steps.find((step) => step.name === 'Validate native runtime') + const validate = native.runs.steps.find((step) => step.name === 'Validate native runtime') + for (const run of [early.run, validate.run]) { + expect( + runProcessSync({ + program: 'bash', + args: ['-e', '-c', run], + env: { ...process.env, NATIVE_RUNTIME: 'unsupported', NODE_VERSION: 'v24.21.0' }, + stdio: 'pipe' + }).code + ).not.toBe(0) + } + expect( + runProcessSync({ + program: 'bash', + args: ['-e', '-c', validate.run], + env: { ...process.env, NATIVE_RUNTIME: 'node', NODE_VERSION: '' }, + stdio: 'pipe' + }).code + ).not.toBe(0) + } + ) + + it('uses requested identity on both restores and saves the Node ABI before Windows switches to Electron', () => { + for (const step of native.runs.steps.filter((step) => step.uses?.startsWith('actions/cache'))) { + expect(step.with.key).toBe('${{ steps.native-cache-scope.outputs.key }}') + expect(step.with.path.trim().split('\n')).toEqual(paths) + expect(step.with['restore-keys']).toBeUndefined() + } + const job = parse(readFileSync('.github/workflows/pr.yml', 'utf8')).jobs.package_windows + const save = job.steps.find((step) => step.name === 'Save compiled Node native modules') + const install = job.steps.find((step) => step.id === 'deps') + const build = job.steps.find((step) => step.name === 'Build package inputs') + const electron = job.steps.find((step) => step.name === 'Prepare Electron native runtime') + expect(install.with['persist-native-cache']).toBe('false') + expect(save.with).toEqual({ + key: '${{ steps.deps.outputs.native-cache-key }}', + path: '${{ steps.deps.outputs.native-cache-path }}' + }) + expect(job.steps.indexOf(save)).toBeLessThan(job.steps.indexOf(build)) + expect(job.steps.indexOf(build)).toBeLessThan(job.steps.indexOf(electron)) + expect(electron.uses).toBe('./.github/actions/prepare-native-runtime') + expect(electron.with).toEqual({ + 'native-runtime': 'electron', + 'node-version': '${{ steps.deps.outputs.node-version }}' + }) + expect(native.outputs['cache-key'].value).toBe('${{ steps.native-cache-scope.outputs.key }}') + }) + + it.skipIf(process.platform === 'win32')( + 'keeps cache post-job paths when the nested step output context is absent', + () => { + const postContext = { steps: {} } + const evaluateInput = (value) => + value.replace( + /\$\{\{\s*steps\.([\w-]+)\.outputs\.([\w-]+)\s*\}\}/g, + (_expression, step, output) => postContext.steps[step]?.outputs?.[output] ?? '' + ) + expect(evaluateInput('${{ steps.native-cache-scope.outputs.path }}')).toBe('') + const emitted = resolveIdentity().path + for (const step of native.runs.steps.filter((step) => + step.uses?.startsWith('actions/cache') + )) { + expect(evaluateInput(step.with.path).trim()).toBe(emitted) + } + } + ) +}) diff --git a/config/scripts/ci-native-toolchain.test.mjs b/config/scripts/ci-native-toolchain.test.mjs index b9d7be83707..f0d58b97639 100644 --- a/config/scripts/ci-native-toolchain.test.mjs +++ b/config/scripts/ci-native-toolchain.test.mjs @@ -5,8 +5,8 @@ import { join } from 'node:path' import { parse } from 'yaml' import { describe, expect, it } from 'vitest' -const steps = parse(readFileSync('.github/actions/install-node-dependencies/action.yml', 'utf8')) - .runs.steps +const steps = parse(readFileSync('.github/actions/prepare-native-runtime/action.yml', 'utf8')).runs + .steps const toolchain = steps.find((step) => step.name === 'Use external node-gyp') describe('CI native toolchain preparation', () => { diff --git a/config/scripts/mobile-release-shell-switch-workflow.test.mjs b/config/scripts/mobile-release-shell-switch-workflow.test.mjs index 40cc20a95ba..f6f05c779eb 100644 --- a/config/scripts/mobile-release-shell-switch-workflow.test.mjs +++ b/config/scripts/mobile-release-shell-switch-workflow.test.mjs @@ -171,7 +171,7 @@ describe('what the mobile jobs restore from cache', () => { expect(names).toEqual( expect.arrayContaining([ './.github/actions/install-node-dependencies: Cache Electron package archive', - './.github/actions/install-node-dependencies: Restore compiled native modules', + './.github/actions/prepare-native-runtime: Restore compiled native modules', './.github/actions/install-node-dependencies: Setup Node.js', 'ios-build: Setup Ruby and fastlane' ]) diff --git a/config/scripts/node-server-change-scope.mjs b/config/scripts/node-server-change-scope.mjs index b1a523cd120..d190855f7c0 100644 --- a/config/scripts/node-server-change-scope.mjs +++ b/config/scripts/node-server-change-scope.mjs @@ -41,6 +41,7 @@ const ALWAYS_FILES = new Set([ ]) const ALWAYS_PREFIXES = [ '.github/actions/install-node-dependencies/', + '.github/actions/prepare-native-runtime/', // These areas also contain worker paths and fixtures opened without an import. 'src/main/persistence/', 'src/main/sqlite/', diff --git a/config/scripts/node-server-change-scope.test.mjs b/config/scripts/node-server-change-scope.test.mjs index a6910fe9c76..78b511dc1a8 100644 --- a/config/scripts/node-server-change-scope.test.mjs +++ b/config/scripts/node-server-change-scope.test.mjs @@ -83,6 +83,7 @@ it.each([ 'config/patches/node-pty@1.1.0.patch', 'native/windows-registry/src/addon.cc', '.github/actions/install-node-dependencies/action.yml', + '.github/actions/prepare-native-runtime/action.yml', '.github/workflows/node-server-tests.yml', 'src/main/persistence/profile-state/new-worker.ts' ])('always selects build, native and dynamically opened inputs: %s', async (file) => { diff --git a/config/scripts/node-server-qualification.mjs b/config/scripts/node-server-qualification.mjs index 0fa5923eb4a..b1faf6eef17 100644 --- a/config/scripts/node-server-qualification.mjs +++ b/config/scripts/node-server-qualification.mjs @@ -14,6 +14,7 @@ const PLATFORM_PREFIXES = [ 'native/', 'config/patches/', '.github/actions/install-node-dependencies/', + '.github/actions/prepare-native-runtime/', 'src/main/persistence/', 'src/main/sqlite/', 'src/main/orcad/', diff --git a/config/scripts/node-server-qualification.test.mjs b/config/scripts/node-server-qualification.test.mjs index a4d367d58b2..480d7e45912 100644 --- a/config/scripts/node-server-qualification.test.mjs +++ b/config/scripts/node-server-qualification.test.mjs @@ -27,6 +27,7 @@ it.each([ 'native/windows-registry/src/addon.cc', 'config/patches/node-pty.patch', '.github/actions/install-node-dependencies/action.yml', + '.github/actions/prepare-native-runtime/action.yml', 'src/main/ssh/ssh-provider.ts', 'src/main/providers/local-pty-provider.ts', 'src/shared/child-process/run-process.ts', diff --git a/config/scripts/pr-code-change-scope.mjs b/config/scripts/pr-code-change-scope.mjs index 1b5679344c0..7f34a8304a9 100644 --- a/config/scripts/pr-code-change-scope.mjs +++ b/config/scripts/pr-code-change-scope.mjs @@ -38,6 +38,7 @@ const ALWAYS_ON_CODE_JOBS = new Set(['static_analysis', 'typecheck', 'test']) const GLOBAL_FORCE_PREFIXES = [ '.github/workflows/pr.yml', '.github/actions/install-node-dependencies/', + '.github/actions/prepare-native-runtime/', 'config/scripts/pr-code-change-scope' ] @@ -222,8 +223,26 @@ const NATIVE_CACHE_FILES = new Set([ 'package.json', 'pnpm-lock.yaml', '.github/actions/install-node-dependencies/action.yml', + '.github/actions/prepare-native-runtime/action.yml', + 'pnpm-workspace.yaml', + '.npmrc', + '.pnpmfile.cjs', 'config/scripts/ensure-native-runtime.mjs', - 'config/scripts/rebuild-native-deps.mjs' + 'config/scripts/rebuild-native-deps.mjs', + 'config/scripts/node-pty-job-ownership.cjs', + 'config/scripts/windows-pe-machine.cjs', + 'config/scripts/windows-process-tree-gyp-rebuild.mjs', + 'config/scripts/windows-process-tree-creation-time.cjs', + 'config/scripts/install-electron-package-binary.mjs', + 'config/scripts/electron-platform-path.mjs', + 'config/scripts/zip-extractor-command.mjs', + 'src/shared/zip-extractor-command.ts', + 'config/scripts/shared-electron-dist-cache.mjs', + 'config/scripts/space-sharing-copy.mjs', + 'native/windows-registry/src/addon.cc', + 'native/windows-registry/binding.gyp', + 'native/windows-registry/package.json', + 'native/windows-registry/index.js' ]) const NATIVE_CACHE_PREFIXES = [ diff --git a/config/scripts/pr-code-change-scope.test.mjs b/config/scripts/pr-code-change-scope.test.mjs index da6670845e4..297d1f8e671 100644 --- a/config/scripts/pr-code-change-scope.test.mjs +++ b/config/scripts/pr-code-change-scope.test.mjs @@ -419,6 +419,7 @@ describe('per-job path classification', () => { 'package.json', 'pnpm-lock.yaml', '.github/actions/install-node-dependencies/action.yml', + '.github/actions/prepare-native-runtime/action.yml', 'config/scripts/ensure-native-runtime.mjs', 'config/scripts/rebuild-native-deps.mjs', 'config/patches/node-pty@1.1.0.patch' diff --git a/config/scripts/pr-workflow-parallelism.test.mjs b/config/scripts/pr-workflow-parallelism.test.mjs index ac2be049bdf..9f841ffb1be 100644 --- a/config/scripts/pr-workflow-parallelism.test.mjs +++ b/config/scripts/pr-workflow-parallelism.test.mjs @@ -13,6 +13,9 @@ const nodeNextWorkflow = parse(readFileSync('.github/workflows/node-next-compat. const dependencyAction = parse( readFileSync('.github/actions/install-node-dependencies/action.yml', 'utf8') ) +const nativeAction = parse( + readFileSync('.github/actions/prepare-native-runtime/action.yml', 'utf8') +) const packageJson = JSON.parse(readFileSync('package.json', 'utf8')) const pnpmWorkspace = parse(readFileSync('pnpm-workspace.yaml', 'utf8')) const shellContractFiles = [ @@ -360,9 +363,9 @@ describe('PR workflow parallelism', () => { ).toBe("steps.deps.outputs.native-cache-hit != 'true'") expect(dependencyAction.inputs['persist-native-cache'].default).toBe('true') - expect( - dependencyAction.runs.steps.find((step) => step.name === 'Use external node-gyp').if - ).toBe("runner.os == 'Linux' && inputs.native-runtime == 'node'") + expect(nativeAction.runs.steps.find((step) => step.name === 'Use external node-gyp').if).toBe( + "runner.os == 'Linux' && inputs.native-runtime == 'node'" + ) const dependencyInstall = dependencyAction.runs.steps.find( (step) => step.name === 'Install dependencies' ) @@ -383,7 +386,13 @@ describe('PR workflow parallelism', () => { (step) => step.name === 'Prepare native runtime' ) expect(prepareRuntime.if).toBe("inputs.native-runtime != 'none'") - expect(prepareRuntime.run).toContain('ensure-native-runtime.mjs --runtime="$NATIVE_RUNTIME"') + expect(prepareRuntime.uses).toBe('./.github/actions/prepare-native-runtime') + expect(prepareRuntime.with).toEqual({ + 'native-runtime': '${{ inputs.native-runtime }}', + 'node-version': + '${{ steps.requested-node.outputs.node-version || steps.default-node.outputs.node-version }}', + 'persist-native-cache': '${{ inputs.persist-native-cache }}' + }) }) it('reuses native preparation after the dependency action gate', () => { @@ -399,14 +408,16 @@ describe('PR workflow parallelism', () => { }) it('restores compiled native modules after the install that strips them', () => { - const steps = dependencyAction.runs.steps - const installIndex = steps.findIndex((step) => step.name === 'Install dependencies') + const installerSteps = dependencyAction.runs.steps + const installIndex = installerSteps.findIndex((step) => step.name === 'Install dependencies') + const nativeIndex = installerSteps.findIndex((step) => step.id === 'native-runtime') + expect(installIndex).toBeLessThan(nativeIndex) + const steps = nativeAction.runs.steps const cacheIndex = steps.findIndex((step) => step.name === 'Restore compiled native modules') const prepareIndex = steps.findIndex((step) => step.name === 'Prepare native runtime') // `--ignore-scripts` leaves no build/, so a restore before the install would be // overwritten and one after the rebuild would never save a hit. - expect(installIndex).toBeLessThan(cacheIndex) expect(cacheIndex).toBeLessThan(prepareIndex) expect(steps[cacheIndex].if).toBe( "inputs.native-runtime != 'none' && inputs.persist-native-cache != 'false'" @@ -421,20 +432,8 @@ describe('PR workflow parallelism', () => { // Native artifacts are ABI-bound: a key missing either dimension serves a build // that cannot load, and ensure-native-runtime would recompile it anyway. for (const cacheStep of [steps[cacheIndex], restoreOnly]) { - expect(cacheStep.with.key).toContain('${{ inputs.native-runtime }}') - expect(cacheStep.with.key).toContain('${{ runner.os }}') - expect(cacheStep.with.key).toContain('${{ runner.arch }}') - expect(cacheStep.with.key).toContain( - 'steps.requested-node.outputs.node-version || steps.default-node.outputs.node-version' - ) - expect(cacheStep.with.key).toContain('steps.native-cache-scope.outputs.scope') - expect(cacheStep.with.key).toContain('config/patches/node-pty@1.1.0.patch') - expect(cacheStep.with.key).toContain( - 'config/patches/@vscode__windows-process-tree@0.8.0.patch' - ) - expect(cacheStep.with.key).toContain('.github/actions/install-node-dependencies/action.yml') - expect(cacheStep.with.key).toContain('config/scripts/ensure-native-runtime.mjs') - expect(cacheStep.with.key).toContain('config/scripts/rebuild-native-deps.mjs') + expect(cacheStep.with.key).toBe('${{ steps.native-cache-scope.outputs.key }}') + expect(cacheStep.with.path).not.toContain('${{') expect(cacheStep.with.path).toContain('node-pty@*/node_modules/node-pty/build') expect(cacheStep.with.path).toContain('native/windows-registry/build') expect(cacheStep.with.path).toContain('@vscode+windows-process-tre*') @@ -445,17 +444,24 @@ describe('PR workflow parallelism', () => { const cacheScope = steps.find((step) => step.name === 'Resolve native cache scope') expect(cacheScope.if).toBe("inputs.native-runtime != 'none'") expect(cacheScope.run).toContain('/etc/os-release') - expect(dependencyAction.outputs['native-cache-scope'].value).toBe( + expect(nativeAction.outputs['cache-scope'].value).toBe( '${{ steps.native-cache-scope.outputs.scope }}' ) - expect(dependencyAction.outputs['native-cache-hit'].value).toContain( + expect(nativeAction.outputs['cache-hit'].value).toContain( 'steps.native-cache-restore.outputs.cache-hit' ) - expect(dependencyAction.outputs['native-cache-hit'].value).toContain( + expect(nativeAction.outputs['cache-hit'].value).toContain( 'steps.native-cache-restore-only.outputs.cache-hit' ) - const electronCache = steps.find((step) => step.name === 'Cache Electron package archive') - const electronCacheResolution = steps.find( + for (const output of ['scope', 'hit', 'key', 'path']) { + expect(dependencyAction.outputs[`native-cache-${output}`].value).toBe( + `\${{ steps.native-runtime.outputs.cache-${output} }}` + ) + } + const electronCache = installerSteps.find( + (step) => step.name === 'Cache Electron package archive' + ) + const electronCacheResolution = installerSteps.find( (step) => step.name === 'Resolve Electron package cache' ) expect(electronCacheResolution.if).toBe( diff --git a/config/scripts/skills-cli-package-workflow.test.mjs b/config/scripts/skills-cli-package-workflow.test.mjs index 489437dd61f..6117ff99ba8 100644 --- a/config/scripts/skills-cli-package-workflow.test.mjs +++ b/config/scripts/skills-cli-package-workflow.test.mjs @@ -14,7 +14,11 @@ describe('packaged skills CLI PR gates', () => { expect(job['runs-on']).toBe('windows-2022') expect(buildStep.run).toBe('pnpm run build:release:parallel') - expect(prepareStep.run).toBe('node config/scripts/ensure-native-runtime.mjs --runtime=electron') + expect(prepareStep.uses).toBe('./.github/actions/prepare-native-runtime') + expect(prepareStep.with).toEqual({ + 'native-runtime': 'electron', + 'node-version': '${{ steps.deps.outputs.node-version }}' + }) expect(packageStep.run).toContain('electron-builder') expect(packageStep.run).toContain('--dir') expect(packageStep.env.ORCA_REUSE_PREPARED_NATIVE_RUNTIME).toBe('1') diff --git a/docs/reference/ci-runner-efficiency.md b/docs/reference/ci-runner-efficiency.md index 5fccd8f8b7f..f4443b05c43 100644 --- a/docs/reference/ci-runner-efficiency.md +++ b/docs/reference/ci-runner-efficiency.md @@ -1060,3 +1060,138 @@ warnings remained non-fatal. Every full-repository scan passed cleanly. Keep the existing production waves. The temporary workflow and 601-line benchmark driver were removed after recording this rejected result. + +## Native cache ownership: retain the extraction + +Native restoration, toolchain recovery, and preparation now belong to +`.github/actions/prepare-native-runtime/action.yml`. The installer forwards +its requested key and three build paths; Windows packaging saves the Node +build before calling the same action for Electron. Existing native load, +patched-build, Windows job-ownership, registry, and process-table probes remain +unchanged on restored consumers. Exact keys still separate OS/image or Linux +container libc, architecture, runtime, resolved Node version, and actual pnpm +version, without partial-key restoration. + +The source hash covers the dedicated action, `pnpm-lock.yaml`, +`pnpm-workspace.yaml`, `.npmrc`, `.pnpmfile.cjs`, both native dependency patches, +and these complete build/probe inputs: + +- `config/scripts/ensure-native-runtime.mjs`, `rebuild-native-deps.mjs`, + `node-pty-job-ownership.cjs`, `windows-pe-machine.cjs`, + `windows-process-tree-gyp-rebuild.mjs`, and + `windows-process-tree-creation-time.cjs`; +- `config/scripts/install-electron-package-binary.mjs`, + `electron-platform-path.mjs`, `zip-extractor-command.mjs`, + `shared-electron-dist-cache.mjs`, `space-sharing-copy.mjs`, and + `src/shared/zip-extractor-command.ts`; +- `native/windows-registry/src/addon.cc`, `binding.gyp`, `package.json`, and + `index.js`. + +The patches are `config/patches/node-pty@1.1.0.patch` and +`config/patches/@vscode__windows-process-tree@0.8.0.patch`. Root app version and +script metadata are excluded; installed package versions remain owned by the +full lockfile, and the external node-gyp pin belongs to the native action. +A negative control changing only the installer's +pnpm verification condition preserves the native key and paths. Every declared +native input mutation changes the key, and main warming watches those inputs. + +This policy creates one cold namespace. The bounded 50-head main sample has +49 adjacent transitions and three native-key changes under both the old and +expanded policies: the added node-pty helper export still invalidates #24448. +There is no measured historical net saving. + +The [cold warming run](https://github.com/stablyai/orca/actions/runs/36945655208/attempts/1) +published all four exact Node keys, and its +[warm rerun](https://github.com/stablyai/orca/actions/runs/36945655208/attempts/2) +restored them on fresh runners with the same frozen inputs, Node 24.21.0, and +pnpm 12.0.0. All five jobs passed in both attempts. These times cover the entire +native action: runtime validation, key resolution, restore, any toolchain +recovery, and the unchanged native preparation probes. + +| Native Node lane | Cold action | Warm action | Cold post-job save | +| ---------------- | ----------- | ----------- | ------------------ | +| Linux x64 | 18.288 s | 0.993 s | 0.387 s | +| Linux ARM64 | 11.540 s | 1.062 s | 1.113 s | +| Windows x64 | 104.482 s | 1.381 s | 2.424 s | +| Windows ARM64 | 256.662 s | 3.832 s | 1.243 s | + +Cold Windows jobs rebuilt all three native addons. Warm jobs loaded and probed +the restored builds; Linux also ran the existing check-only probe before +skipping the external node-gyp installation. Warm post-job steps recognized +their primary keys and did not save again. An earlier trial exposed unavailable +nested composite outputs during post-job saving; both cache variants now use +the same literal path inventory as the requested output, and the fixed cold +jobs published their caches without missing-path warnings. + +Both [PR package jobs](https://github.com/stablyai/orca/actions/runs/36945659474) +passed. Windows packaging consumed the same-run Node seed in 1.374 seconds +before its Node tests and Electron transition. Its Electron cache initially +missed while the modules were already healthy, so that stage does not establish +an avoided compilation. Linux's Electron cache was also published, and all 19 +bundled native binaries passed the existing glibc floor check. + +The [first six-platform headless run](https://github.com/stablyai/orca/actions/runs/36945658897) +ran every persistence lane: five passed, while Mac Intel failed waiting for a +cancel-test worker's ready file before its 500 ms timeout. That lane deliberately +uses `native-runtime: none`; its separate slot build and smoke passed. The failure +blocked the five downstream Linux glibc/musl qualifications, so this run does not +establish complete headless qualification. All six persistence lanes, Node 18 +handoffs, and Linux floor/musl gates remain; final qualification is tracked in +the [PR's latest checks](https://github.com/stablyai/orca/pull/24476/checks). + +These are single cold/warm observations, not paired medians or a measured +whole-workflow saving. They demonstrate usable exact-key reuse after publication; +future savings depend on cache availability and unchanged native inputs. The +trial seeds belong to this PR's merge ref. Other PRs require a main-branch seed +after merging this new namespace; the existing main push and hourly warming +jobs provide that seed. + +## Separate mobile install verification: retain the current policy + +Three local paired pnpm 12 mobile installs reduced the median from 17.155 to +15.871 seconds, a 1.284-second difference before cache transfer and postinstall +scripts. That narrow margin does not establish a net hosted saving, so the +separate mobile verification record was not adopted. + +## Unit shard weights: retain the current allocation + +The latest five shard wall times were 526/495/503/508/510 seconds. Reweighting +projected roughly a 4% reduction in the slowest shard without reducing total +CPU work; the evidence across runs was weak. That estimate does not justify +changing allocation, so the current weights remain. + +## Serializer oracle allocations: retain the change + +The serializer round-trip oracle now reloads one xterm cell per buffer traversal +and writes flag digits directly, avoiding fresh cell objects and flag arrays for +every comparison. Independent replay terminals, cell descriptors, transcript +fixtures, resize schedules, ConPTY modes and seeds remain unchanged. + +The [hosted comparison](https://github.com/stablyai/orca/actions/runs/36944080887) +used one Ubuntu 24.04 ARM64 runner, image 20260927.135.1, Node 24.21.0 and one +isolated fork. The baseline formatter was frozen from f69052e. Byte-parity capture +ran separately; these three alternating pairs had no payload instrumentation. +Times cover the complete Vitest invocation, including startup and shutdown. + +| Pair/order | Baseline | Candidate | Change | +| ------------------ | -------- | --------- | ------ | +| 1: baseline first | 70.631 s | 61.661 s | -12.7% | +| 2: candidate first | 70.619 s | 62.284 s | -11.8% | +| 3: baseline first | 70.750 s | 62.091 s | -12.2% | +| Median | 70.631 s | 62.091 s | -12.1% | + +Median test-body time fell from 69.519 to 60.953 seconds. All eight full-cohort +invocations preserved the same 89 passes and two existing conditional skips +across three files. Separate baseline/candidate captures produced identical +95,017,559-byte payloads for all 1,435 scenarios and 7,649 checkpoints, with zero +source crashes; both SHA256 hashes matched the local captures. + +Seven focused controls compare against the original allocating oracle, including +all 128 text-flag combinations, styled blanks, wide cells, cell reuse and immutable +snapshots. Five deliberate faults were detected: stale cell contents, a missing +bold flag, changed empty-cell policy, removed scratch reuse and a source-parser +crash. The last control also proved that crash returns enter the capture. + +This measures the three-file oracle cohort. Whole-shard timings include other +test bodies, imports and transforms, so a whole-suite saving needs separate +measurement. diff --git a/src/main/daemon/serialize-grid-cell-descriptors.test.ts b/src/main/daemon/serialize-grid-cell-descriptors.test.ts new file mode 100644 index 00000000000..490b40a1c9d --- /dev/null +++ b/src/main/daemon/serialize-grid-cell-descriptors.test.ts @@ -0,0 +1,232 @@ +import { describe, expect, it, vi } from 'vitest' +import type { Terminal } from '@xterm/headless' +import { bufferRows, cellDescriptor } from './serialize-grid-cell-descriptors' +import { createFuzzTerminal, writeTerminal } from './serialize-grid-roundtrip' + +// Frozen allocating oracle from f69052e; a reused cell must preserve every descriptor. +type BufferLine = NonNullable> +type Buffer = Terminal['buffer']['active'] + +const COLOR_MODE_P16 = 16777216 +const COLOR_MODE_P256 = 33554432 +const DEFAULT_BLANK = '▯·w1·b0:-1·000' +const CLIPPED = 'CLIPPED' + +// SerializeAddon re-emits palette 0-15 set via 38;5;N as SGR 30-37/90-97; same theme slot. +function canonicalColorMode(mode: number, colorValue: number): number { + return mode === COLOR_MODE_P256 && colorValue >= 0 && colorValue < 16 ? COLOR_MODE_P16 : mode +} + +function flags(values: boolean[]): string { + return values.map((flag) => (flag ? '1' : '0')).join('') +} + +/** Visually effective cell state, same blank-cell policy as terminal-restore-parity-fixture. */ +function allocatingCellDescriptor(line: BufferLine | undefined, x: number, cols: number): string { + if (!line || x >= line.length) { + return DEFAULT_BLANK + } + const cell = line.getCell(x) + if (!cell) { + return DEFAULT_BLANK + } + if (x === cols - 1 && line.length > cols && cell.getWidth() > 1) { + return CLIPPED + } + const chars = cell.getChars() + const fgMode = canonicalColorMode(cell.getFgColorMode(), cell.getFgColor()) + const bgMode = canonicalColorMode(cell.getBgColorMode(), cell.getBgColor()) + if (chars === '' || chars === ' ') { + const blank = chars === ' ' + const inverseFg = cell.isInverse() ? `·if${fgMode}:${cell.getFgColor()}` : '' + return `▯·w${cell.getWidth()}·b${bgMode}:${cell.getBgColor()}·${flags([ + blank && cell.isUnderline() !== 0, + blank && cell.isStrikethrough() !== 0, + blank && cell.isOverline() !== 0 + ])}${inverseFg}` + } + const cellFlags = flags([ + cell.isBold() !== 0, + cell.isDim() !== 0, + cell.isItalic() !== 0, + cell.isUnderline() !== 0, + cell.isInverse() !== 0, + cell.isInvisible() !== 0, + cell.isStrikethrough() !== 0 + ]) + return `${chars}·w${cell.getWidth()}·f${fgMode}:${cell.getFgColor()}·b${bgMode}:${cell.getBgColor()}·${cellFlags}` +} + +function rowCells(line: BufferLine | undefined, cols: number): string[] { + return Array.from({ length: cols }, (_, x) => allocatingCellDescriptor(line, x, cols)) +} + +function allocatingBufferRows( + buffer: Buffer, + start: number, + end: number, + cols: number +): string[][] { + const rows: string[][] = [] + for (let y = start; y < end; y++) { + rows.push(rowCells(buffer.getLine(y), cols)) + } + while (rows.length > 0 && rows.at(-1)!.every((c) => c === DEFAULT_BLANK)) { + rows.pop() + } + return rows +} + +describe('serialize oracle cell reuse', () => { + it.each([false, true])( + 'matches allocating cells across SGR, wide text, buffers and resize (ConPTY=%s)', + (conpty) => { + const terminal = createFuzzTerminal({ cols: 14, rows: 4, scrollback: 30, conpty }) + try { + const writes = [ + 'plain \x1b[1;2;3;4;7;8;9mstyled\x1b[0m\r\n', + '\x1b[38;5;2;48;5;10m palette \x1b[38;2;11;22;33;48;2;44;55;66m RGB \x1b[0m\r\n', + '\x1b[4:3;9;53m \x1b[0m\x1b[7m \x1b[0m界👩‍💻é\r\n', + 'scroll1\r\nscroll2\r\nscroll3\r\n', + '\x1b[?1049h\x1b[1;2;3;4;7;8;9malt界\x1b[0m', + '\x1b[?1049l\x1b[2J\x1b[Hclear' + ] + for (const data of writes) { + writeTerminal(terminal, data) + for (const buffer of [ + terminal.buffer.normal, + terminal.buffer.alternate, + terminal.buffer.active + ]) { + expect(bufferRows(buffer, -1, buffer.length + 1, terminal.cols + 2)).toEqual( + allocatingBufferRows(buffer, -1, buffer.length + 1, terminal.cols + 2) + ) + } + terminal.resize(terminal.cols === 14 ? 9 : 14, 4) + expect( + bufferRows(terminal.buffer.active, 0, terminal.buffer.active.length, terminal.cols) + ).toEqual( + allocatingBufferRows( + terminal.buffer.active, + 0, + terminal.buffer.active.length, + terminal.cols + ) + ) + } + } finally { + terminal.dispose() + } + } + ) + + it('preserves the order of every text flag combination while reloading a plain cell', () => { + const terminal = createFuzzTerminal({ cols: 4, rows: 1, scrollback: 0 }) + try { + const sgr = [1, 2, 3, 4, 7, 8, 9] + const scratch = terminal.buffer.active.getNullCell() + for (let mask = 0; mask < 128; mask++) { + const codes = sgr.filter((_code, bit) => (mask & (1 << bit)) !== 0) + writeTerminal(terminal, `\x1b[H\x1b[0m\x1b[${codes.length ? codes.join(';') : 0}mA\x1b[0mB`) + const line = terminal.buffer.active.getLine(0) + expect(cellDescriptor(line, 0, 4, scratch)).toBe(allocatingCellDescriptor(line, 0, 4)) + expect(cellDescriptor(line, 1, 4, scratch)).toBe(allocatingCellDescriptor(line, 1, 4)) + } + } finally { + terminal.dispose() + } + }) + + it('preserves styled spaces, empty cells and inverse foreground colors', () => { + const terminal = createFuzzTerminal({ cols: 4, rows: 1, scrollback: 0 }) + try { + const scratch = terminal.buffer.active.getNullCell() + for (const inverse of [false, true]) { + for (const glyph of ['', ' ']) { + for (let mask = 0; mask < 8; mask++) { + const codes = [4, 9, 53].filter((_code, bit) => (mask & (1 << bit)) !== 0) + writeTerminal( + terminal, + `\x1b[H\x1b[0m\x1b[38;2;3;4;5;48;5;2${inverse ? ';7' : ''}${codes.length ? `;${codes.join(';')}` : ''}m\x1b[2J${glyph}` + ) + const line = terminal.buffer.active.getLine(0) + expect(cellDescriptor(line, 0, 4, scratch)).toBe(allocatingCellDescriptor(line, 0, 4)) + } + } + } + } finally { + terminal.dispose() + } + }) + + it('preserves a clipped wide leading cell at the comparison grid edge', () => { + const terminal = createFuzzTerminal({ cols: 8, rows: 1, scrollback: 0 }) + try { + writeTerminal(terminal, 'abc界') + const line = terminal.buffer.active.getLine(0) + const scratch = terminal.buffer.active.getNullCell() + expect(cellDescriptor(line, 3, 4, scratch)).toBe('CLIPPED') + expect(cellDescriptor(line, 3, 4, scratch)).toBe(allocatingCellDescriptor(line, 3, 4)) + expect(bufferRows(terminal.buffer.active, 0, 1, 4)).toEqual( + allocatingBufferRows(terminal.buffer.active, 0, 1, 4) + ) + } finally { + terminal.dispose() + } + }) + + it('loads every cell into one traversal-local scratch object and retains immutable descriptors', () => { + const terminal = createFuzzTerminal({ cols: 4, rows: 2, scrollback: 0 }) + try { + writeTerminal(terminal, '\x1b[1mA\x1b[0m B界') + const buffer = terminal.buffer.active + const scratch = buffer.getNullCell() + const allocate = vi.spyOn(buffer, 'getNullCell').mockReturnValue(scratch) + const getLine = buffer.getLine.bind(buffer) + const loaded: unknown[] = [] + vi.spyOn(buffer, 'getLine').mockImplementation((y) => { + const line = getLine(y) + if (line) { + const getCell = line.getCell.bind(line) + vi.spyOn(line, 'getCell').mockImplementation((x, cell) => { + loaded.push(cell) + return getCell(x, cell) + }) + } + return line + }) + const expected = allocatingBufferRows(buffer, 0, 2, 4) + loaded.length = 0 + const actual = bufferRows(buffer, 0, 2, 4) + expect(actual).toEqual(expected) + expect(allocate).toHaveBeenCalledTimes(1) + expect(loaded).toHaveLength(8) + expect(loaded.every((cell) => cell === scratch)).toBe(true) + writeTerminal(terminal, '\x1b[2J\x1b[Hnew') + bufferRows(buffer, 0, 2, 4) + expect(actual).toEqual(expected) + } finally { + terminal.dispose() + vi.restoreAllMocks() + } + }) + + it('keeps missing lines and invalid columns blank after a styled cell occupied the scratch', () => { + const terminal = createFuzzTerminal({ cols: 4, rows: 2, scrollback: 0 }) + try { + writeTerminal(terminal, '\x1b[1;7;38;2;5;6;7mX') + const line = terminal.buffer.active.getLine(0) + const scratch = terminal.buffer.active.getNullCell() + expect(cellDescriptor(line, 0, 4, scratch)).toBe(allocatingCellDescriptor(line, 0, 4)) + for (const x of [-1, 4, 5]) { + expect(cellDescriptor(line, x, 4, scratch)).toBe(allocatingCellDescriptor(line, x, 4)) + } + expect(cellDescriptor(undefined, 0, 4, scratch)).toBe( + allocatingCellDescriptor(undefined, 0, 4) + ) + expect(cellDescriptor(line, 1, 4, scratch)).toBe(allocatingCellDescriptor(line, 1, 4)) + } finally { + terminal.dispose() + } + }) +}) diff --git a/src/main/daemon/serialize-grid-cell-descriptors.ts b/src/main/daemon/serialize-grid-cell-descriptors.ts index 391990de741..8b6c331f5fc 100644 --- a/src/main/daemon/serialize-grid-cell-descriptors.ts +++ b/src/main/daemon/serialize-grid-cell-descriptors.ts @@ -18,16 +18,17 @@ function canonicalColorMode(mode: number, colorValue: number): number { return mode === COLOR_MODE_P256 && colorValue >= 0 && colorValue < 16 ? COLOR_MODE_P16 : mode } -function flags(values: boolean[]): string { - return values.map((flag) => (flag ? '1' : '0')).join('') -} - /** Visually effective cell state, same blank-cell policy as terminal-restore-parity-fixture. */ -export function cellDescriptor(line: BufferLine | undefined, x: number, cols: number): string { +export function cellDescriptor( + line: BufferLine | undefined, + x: number, + cols: number, + reusableCell?: ReturnType +): string { if (!line || x >= line.length) { return DEFAULT_BLANK } - const cell = line.getCell(x) + const cell = line.getCell(x, reusableCell) if (!cell) { return DEFAULT_BLANK } @@ -40,26 +41,22 @@ export function cellDescriptor(line: BufferLine | undefined, x: number, cols: nu if (chars === '' || chars === ' ') { const blank = chars === ' ' const inverseFg = cell.isInverse() ? `·if${fgMode}:${cell.getFgColor()}` : '' - return `▯·w${cell.getWidth()}·b${bgMode}:${cell.getBgColor()}·${flags([ - blank && cell.isUnderline() !== 0, - blank && cell.isStrikethrough() !== 0, - blank && cell.isOverline() !== 0 - ])}${inverseFg}` + return `▯·w${cell.getWidth()}·b${bgMode}:${cell.getBgColor()}·${blank && cell.isUnderline() !== 0 ? '1' : '0'}${blank && cell.isStrikethrough() !== 0 ? '1' : '0'}${blank && cell.isOverline() !== 0 ? '1' : '0'}${inverseFg}` } - const cellFlags = flags([ - cell.isBold() !== 0, - cell.isDim() !== 0, - cell.isItalic() !== 0, - cell.isUnderline() !== 0, - cell.isInverse() !== 0, - cell.isInvisible() !== 0, - cell.isStrikethrough() !== 0 - ]) + const cellFlags = + `${cell.isBold() !== 0 ? '1' : '0'}${cell.isDim() !== 0 ? '1' : '0'}` + + `${cell.isItalic() !== 0 ? '1' : '0'}${cell.isUnderline() !== 0 ? '1' : '0'}` + + `${cell.isInverse() !== 0 ? '1' : '0'}${cell.isInvisible() !== 0 ? '1' : '0'}` + + `${cell.isStrikethrough() !== 0 ? '1' : '0'}` return `${chars}·w${cell.getWidth()}·f${fgMode}:${cell.getFgColor()}·b${bgMode}:${cell.getBgColor()}·${cellFlags}` } -function rowCells(line: BufferLine | undefined, cols: number): string[] { - return Array.from({ length: cols }, (_, x) => cellDescriptor(line, x, cols)) +function rowCells( + line: BufferLine | undefined, + cols: number, + reusableCell: ReturnType +): string[] { + return Array.from({ length: cols }, (_, x) => cellDescriptor(line, x, cols, reusableCell)) } // A wide glyph whose trailing half lies past the grid cannot be replayed; any blank is faithful. @@ -73,8 +70,9 @@ function rowsMatch(expected: string[], actual: string[]): boolean { export function bufferRows(buffer: Buffer, start: number, end: number, cols: number): string[][] { const rows: string[][] = [] + const reusableCell = buffer.getNullCell() for (let y = start; y < end; y++) { - rows.push(rowCells(buffer.getLine(y), cols)) + rows.push(rowCells(buffer.getLine(y), cols, reusableCell)) } while (rows.length > 0 && rows.at(-1)!.every((c) => c === DEFAULT_BLANK)) { rows.pop() diff --git a/src/main/persistence/profile-state/profile-state-backup-worker.test.ts b/src/main/persistence/profile-state/profile-state-backup-worker.test.ts index 26457dd4434..4f331ed711e 100644 --- a/src/main/persistence/profile-state/profile-state-backup-worker.test.ts +++ b/src/main/persistence/profile-state/profile-state-backup-worker.test.ts @@ -181,11 +181,12 @@ describe('profile state backup worker', () => { const cancellation = new AbortController() const pending = runProfileStateBackupWorker(job, { workerPath: worker, - timeoutMs: 500, + // Worker startup must not race the cancellation assertion. + timeoutMs: mode === 'cancel' ? 10_000 : 500, signal: cancellation.signal }) const failed = expect(pending).rejects.toThrow(mode === 'cancel' ? 'cancelled' : 'timed out') - await vi.waitFor(() => expect(existsSync(ready)).toBe(true)) + await vi.waitFor(() => expect(existsSync(ready)).toBe(true), { timeout: 5_000 }) expect(readdirSync(directory).filter((name) => name.startsWith('backup.db.'))).toHaveLength(4) if (mode === 'cancel') { cancellation.abort()