diff --git a/.github/workflows/ssh-relay-runtime-artifacts.yml b/.github/workflows/ssh-relay-runtime-artifacts.yml index f0d4dc9aee2..21e36216a16 100644 --- a/.github/workflows/ssh-relay-runtime-artifacts.yml +++ b/.github/workflows/ssh-relay-runtime-artifacts.yml @@ -110,6 +110,7 @@ jobs: config/scripts/ssh-relay-node-zip-inspection.test.mjs \ config/scripts/ssh-relay-runtime-artifact.test.mjs \ config/scripts/ssh-relay-runtime-pty-smoke.test.mjs \ + config/scripts/ssh-relay-runtime-reproducibility.test.mjs \ config/scripts/ssh-relay-runtime-resource-diagnostics.test.mjs \ config/scripts/ssh-relay-runtime-windows-tree.test.mjs \ config/scripts/ssh-relay-runtime-zip.test.mjs \ @@ -129,34 +130,44 @@ jobs: done printf 'NODE_INPUTS=%s\n' "$inputs" >> "$GITHUB_ENV" - - name: Build, inspect, and smoke exact runtime + - name: Build twice, inspect, smoke, and compare exact runtime shell: bash run: | set -euo pipefail - output="$RUNNER_TEMP/ssh-relay-runtime/${{ matrix.tuple }}" - mkdir -p "$(dirname "$output")" + output_root="$RUNNER_TEMP/ssh-relay-runtime/${{ matrix.tuple }}" + mkdir -p "$output_root" + first_output="$output_root/first" + second_output="$output_root/second" source_commit=$(git rev-parse HEAD) source_date_epoch=$(git show -s --format=%ct "$source_commit") - /usr/bin/time -p node config/scripts/build-ssh-relay-runtime.mjs \ + for output in "$first_output" "$second_output"; do + printf 'clean_build_output=%s\n' "$output" + /usr/bin/time -p node config/scripts/build-ssh-relay-runtime.mjs \ + --tuple '${{ matrix.tuple }}' \ + --inputs-directory "$NODE_INPUTS" \ + --output-directory "$output" \ + --source-date-epoch "$source_date_epoch" \ + --git-commit "$source_commit" + identity="$output/orca-ssh-relay-runtime-${{ matrix.tuple }}.identity.json" + archive=$(find "$output" -maxdepth 1 -name '*.tar.xz' -print -quit) + test -n "$archive" + /usr/bin/time -p node config/scripts/verify-ssh-relay-runtime.mjs \ + --runtime-directory "$output/runtime" \ + --identity "$identity" \ + --archive "$archive" + if find "$output/runtime" -type f \( -name npm -o -name npx -o -name corepack -o -name '*.map' \) -print -quit | grep -q .; then + echo 'Runtime contains a prohibited package manager or source map.' >&2 + exit 1 + fi + parcel_packages=$(find "$output/runtime/node_modules/@parcel" -mindepth 1 -maxdepth 1 -type d | wc -l | tr -d ' ') + test "$parcel_packages" = 2 + done + node config/scripts/ssh-relay-runtime-reproducibility.mjs \ --tuple '${{ matrix.tuple }}' \ - --inputs-directory "$NODE_INPUTS" \ - --output-directory "$output" \ - --source-date-epoch "$source_date_epoch" \ - --git-commit "$source_commit" - identity="$output/orca-ssh-relay-runtime-${{ matrix.tuple }}.identity.json" - archive=$(find "$output" -maxdepth 1 -name '*.tar.xz' -print -quit) - /usr/bin/time -p node config/scripts/verify-ssh-relay-runtime.mjs \ - --runtime-directory "$output/runtime" \ - --identity "$identity" \ - --archive "$archive" - if find "$output/runtime" -type f \( -name npm -o -name npx -o -name corepack -o -name '*.map' \) -print -quit | grep -q .; then - echo 'Runtime contains a prohibited package manager or source map.' >&2 - exit 1 - fi - parcel_packages=$(find "$output/runtime/node_modules/@parcel" -mindepth 1 -maxdepth 1 -type d | wc -l | tr -d ' ') - test "$parcel_packages" = 2 + --first-output-directory "$first_output" \ + --second-output-directory "$second_output" mkdir -p "$GITHUB_WORKSPACE/runtime-evidence/${{ matrix.tuple }}" - cp "$output"/*.tar.xz "$output"/*.spdx.json "$output"/*.provenance.json "$output"/*.identity.json \ + cp "$first_output"/*.tar.xz "$first_output"/*.spdx.json "$first_output"/*.provenance.json "$first_output"/*.identity.json \ "$GITHUB_WORKSPACE/runtime-evidence/${{ matrix.tuple }}/" - name: Upload unpublished artifact evidence @@ -254,6 +265,7 @@ jobs: config/scripts/ssh-relay-node-zip-inspection.test.mjs ` config/scripts/ssh-relay-runtime-artifact.test.mjs ` config/scripts/ssh-relay-runtime-pty-smoke.test.mjs ` + config/scripts/ssh-relay-runtime-reproducibility.test.mjs ` config/scripts/ssh-relay-runtime-resource-diagnostics.test.mjs ` config/scripts/ssh-relay-runtime-windows-tree.test.mjs ` config/scripts/ssh-relay-runtime-zip.test.mjs ` @@ -284,41 +296,52 @@ jobs: } "NODE_INPUTS=$inputs" | Out-File -FilePath $env:GITHUB_ENV -Encoding utf8 -Append - - name: Build, inspect, and smoke exact runtime + - name: Build twice, inspect, smoke, and compare exact runtime shell: pwsh run: | $ErrorActionPreference = 'Stop' - $output = Join-Path $env:RUNNER_TEMP 'ssh-relay-runtime/${{ matrix.tuple }}' - New-Item -ItemType Directory -Path (Split-Path -Parent $output) -Force | Out-Null + $outputRoot = Join-Path $env:RUNNER_TEMP 'ssh-relay-runtime/${{ matrix.tuple }}' + New-Item -ItemType Directory -Path $outputRoot -Force | Out-Null + $firstOutput = Join-Path $outputRoot 'first' + $secondOutput = Join-Path $outputRoot 'second' $sourceCommit = git rev-parse HEAD $sourceDateEpoch = git show -s --format=%ct $sourceCommit - node config/scripts/build-ssh-relay-runtime.mjs ` - --tuple '${{ matrix.tuple }}' ` - --inputs-directory $env:NODE_INPUTS ` - --output-directory $output ` - --source-date-epoch $sourceDateEpoch ` - --git-commit $sourceCommit - if ($LASTEXITCODE -ne 0) { throw 'runtime build failed' } - $identity = Join-Path $output 'orca-ssh-relay-runtime-${{ matrix.tuple }}.identity.json' - $archive = Get-ChildItem -LiteralPath $output -Filter '*.zip' -File | Select-Object -First 1 - if (-not $archive) { throw 'runtime ZIP is missing' } - node config/scripts/verify-ssh-relay-runtime.mjs ` - --runtime-directory (Join-Path $output 'runtime') ` - --identity $identity ` - --archive $archive.FullName - if ($LASTEXITCODE -ne 0) { throw 'runtime verification failed' } - $prohibited = Get-ChildItem -LiteralPath (Join-Path $output 'runtime') -Recurse -File | - Where-Object { $_.Name -in @('npm', 'npm.cmd', 'npx', 'npx.cmd', 'corepack', 'corepack.cmd') -or $_.Name.EndsWith('.map') } | - Select-Object -First 1 - if ($prohibited) { throw "Runtime contains a prohibited file: $($prohibited.FullName)" } - $parcelRoot = Join-Path $output 'runtime/node_modules/@parcel' - if (@(Get-ChildItem -LiteralPath $parcelRoot -Directory).Count -ne 2) { - throw 'Runtime must contain exactly the watcher wrapper and one native package' + foreach ($output in @($firstOutput, $secondOutput)) { + "clean_build_output=$output" + node config/scripts/build-ssh-relay-runtime.mjs ` + --tuple '${{ matrix.tuple }}' ` + --inputs-directory $env:NODE_INPUTS ` + --output-directory $output ` + --source-date-epoch $sourceDateEpoch ` + --git-commit $sourceCommit + if ($LASTEXITCODE -ne 0) { throw 'runtime build failed' } + $identity = Join-Path $output 'orca-ssh-relay-runtime-${{ matrix.tuple }}.identity.json' + $archive = Get-ChildItem -LiteralPath $output -Filter '*.zip' -File | Select-Object -First 1 + if (-not $archive) { throw 'runtime ZIP is missing' } + node config/scripts/verify-ssh-relay-runtime.mjs ` + --runtime-directory (Join-Path $output 'runtime') ` + --identity $identity ` + --archive $archive.FullName + if ($LASTEXITCODE -ne 0) { throw 'runtime verification failed' } + $prohibited = Get-ChildItem -LiteralPath (Join-Path $output 'runtime') -Recurse -File | + Where-Object { $_.Name -in @('npm', 'npm.cmd', 'npx', 'npx.cmd', 'corepack', 'corepack.cmd') -or $_.Name.EndsWith('.map') } | + Select-Object -First 1 + if ($prohibited) { throw "Runtime contains a prohibited file: $($prohibited.FullName)" } + $parcelRoot = Join-Path $output 'runtime/node_modules/@parcel' + if (@(Get-ChildItem -LiteralPath $parcelRoot -Directory).Count -ne 2) { + throw 'Runtime must contain exactly the watcher wrapper and one native package' + } } + node config/scripts/ssh-relay-runtime-reproducibility.mjs ` + --tuple '${{ matrix.tuple }}' ` + --first-output-directory $firstOutput ` + --second-output-directory $secondOutput + if ($LASTEXITCODE -ne 0) { throw 'runtime reproducibility verification failed' } $evidence = Join-Path $env:GITHUB_WORKSPACE 'runtime-evidence/${{ matrix.tuple }}' New-Item -ItemType Directory -Path $evidence -Force | Out-Null - Copy-Item -LiteralPath $archive.FullName -Destination $evidence - Get-ChildItem -LiteralPath $output -File | + $firstArchive = Get-ChildItem -LiteralPath $firstOutput -Filter '*.zip' -File | Select-Object -First 1 + Copy-Item -LiteralPath $firstArchive.FullName -Destination $evidence + Get-ChildItem -LiteralPath $firstOutput -File | Where-Object { $_.Name -match '\.(?:spdx|provenance|identity)\.json$' } | Copy-Item -Destination $evidence diff --git a/config/scripts/ssh-relay-runtime-reproducibility.mjs b/config/scripts/ssh-relay-runtime-reproducibility.mjs new file mode 100644 index 00000000000..febadab9db0 --- /dev/null +++ b/config/scripts/ssh-relay-runtime-reproducibility.mjs @@ -0,0 +1,268 @@ +#!/usr/bin/env node + +import { createHash } from 'node:crypto' +import { createReadStream } from 'node:fs' +import { lstat, opendir, readFile, realpath } from 'node:fs/promises' +import { join, resolve } from 'node:path' + +// The bounded output adds four release assets around a runtime tree capped at 5,000 entries. +const MAX_ENTRIES = 6_000 +const MAX_FILE_BYTES = 250 * 1024 * 1024 +const MAX_TOTAL_BYTES = 500 * 1024 * 1024 +const MAX_PATH_BYTES = 512 +const MAX_PATH_DEPTH = 40 +const MAX_IDENTITY_BYTES = 1024 * 1024 +const COMPARISON_TIMEOUT_MS = 5 * 60 * 1000 +const SUPPORTED_TUPLES = new Set([ + 'linux-x64-glibc', + 'linux-arm64-glibc', + 'darwin-x64', + 'darwin-arm64', + 'win32-x64', + 'win32-arm64' +]) + +async function sha256File(path, signal) { + const digest = createHash('sha256') + for await (const chunk of createReadStream(path, { signal })) { + signal.throwIfAborted() + digest.update(chunk) + } + return `sha256:${digest.digest('hex')}` +} + +function validateRelativePath(path) { + if ( + Buffer.byteLength(path) > MAX_PATH_BYTES || + path.split('/').length > MAX_PATH_DEPTH || + path.includes('\\') + ) { + throw new Error(`reproducibility output path exceeds limits: ${path}`) + } +} + +async function snapshotOutput(directory, label, signal) { + const entries = new Map() + let files = 0 + let bytes = 0 + + async function visit(absoluteDirectory, relativeDirectory = '') { + const children = [] + const handle = await opendir(absoluteDirectory) + for await (const child of handle) { + children.push(child.name) + } + children.sort() + for (const name of children) { + signal.throwIfAborted() + const path = relativeDirectory ? `${relativeDirectory}/${name}` : name + validateRelativePath(path) + const absolutePath = join(absoluteDirectory, name) + const metadata = await lstat(absolutePath) + if (metadata.isSymbolicLink() || (!metadata.isDirectory() && !metadata.isFile())) { + throw new Error(`${label} contains a prohibited special entry: ${path}`) + } + if (entries.size >= MAX_ENTRIES) { + throw new Error(`${label} exceeds the reproducibility entry-count limit`) + } + if (metadata.isDirectory()) { + entries.set(path, { path, type: 'directory', mode: metadata.mode & 0o777 }) + await visit(absolutePath, path) + continue + } + if (metadata.size > MAX_FILE_BYTES) { + throw new Error(`${label} file exceeds the reproducibility size limit: ${path}`) + } + files += 1 + bytes += metadata.size + if (bytes > MAX_TOTAL_BYTES) { + throw new Error(`${label} exceeds the reproducibility total-size limit`) + } + entries.set(path, { + path, + type: 'file', + mode: metadata.mode & 0o777, + size: metadata.size, + sha256: await sha256File(absolutePath, signal) + }) + } + } + + const root = await lstat(directory) + if (!root.isDirectory() || root.isSymbolicLink()) { + throw new Error(`${label} must be a real directory`) + } + await visit(directory) + return { bytes, entries, files } +} + +async function validateCompleteOutput(directory, label, tuple, snapshot) { + const identityName = `orca-ssh-relay-runtime-${tuple}.identity.json` + const required = [ + 'runtime', + 'runtime/relay.js', + 'runtime/relay-watcher.js', + 'runtime/runtime-metadata.json', + identityName, + `orca-ssh-relay-runtime-${tuple}.spdx.json`, + `orca-ssh-relay-runtime-${tuple}.provenance.json` + ] + for (const path of required) { + if (!snapshot.entries.has(path)) { + throw new Error(`${label} is missing required asset: ${path}`) + } + } + const identityEntry = snapshot.entries.get(identityName) + if (identityEntry.type !== 'file' || identityEntry.size > MAX_IDENTITY_BYTES) { + throw new Error(`${label} identity exceeds its size or type contract`) + } + let identity + try { + identity = JSON.parse(await readFile(join(directory, identityName), 'utf8')) + } catch { + throw new Error(`${label} identity is not valid JSON`) + } + if ( + identity.tupleId !== tuple || + !/^sha256:[0-9a-f]{64}$/.test(identity.contentId) || + typeof identity.archive?.name !== 'string' || + !/^sha256:[0-9a-f]{64}$/.test(identity.archive.sha256) + ) { + throw new Error(`${label} identity does not match the requested tuple or digest contract`) + } + if (!snapshot.entries.has(identity.archive.name)) { + throw new Error(`${label} is missing required asset: ${identity.archive.name}`) + } + const archiveEntry = snapshot.entries.get(identity.archive.name) + if (archiveEntry.type !== 'file' || archiveEntry.sha256 !== identity.archive.sha256) { + throw new Error(`${label} archive digest does not match its identity`) + } + const expectedTopLevel = new Set([...required.slice(4), identity.archive.name, 'runtime']) + const unexpected = [...snapshot.entries.keys()].find( + (path) => !path.includes('/') && !expectedTopLevel.has(path) + ) + if (unexpected) { + throw new Error(`${label} contains an unexpected top-level asset: ${unexpected}`) + } + return identity +} + +function compareSnapshots(first, second) { + const paths = [...new Set([...first.entries.keys(), ...second.entries.keys()])].sort() + for (const path of paths) { + const left = first.entries.get(path) + const right = second.entries.get(path) + if (!left) { + throw new Error(`first output is missing reproducibility entry: ${path}`) + } + if (!right) { + throw new Error(`second output is missing reproducibility entry: ${path}`) + } + for (const field of ['type', 'mode', 'size', 'sha256']) { + if (left[field] !== right[field]) { + throw new Error(`reproducibility mismatch for ${path}: ${field}`) + } + } + } +} + +export async function verifySshRelayRuntimeReproducibility({ + firstOutputDirectory, + secondOutputDirectory, + tuple, + signal = AbortSignal.timeout(COMPARISON_TIMEOUT_MS) +}) { + const started = process.hrtime.bigint() + if (!SUPPORTED_TUPLES.has(tuple)) { + throw new Error(`unsupported runtime reproducibility tuple: ${tuple}`) + } + const firstDirectory = resolve(firstOutputDirectory) + const secondDirectory = resolve(secondOutputDirectory) + if (firstDirectory === secondDirectory) { + throw new Error('clean-build outputs must be distinct directories') + } + signal.throwIfAborted() + const [firstRealPath, secondRealPath] = await Promise.all([ + realpath(firstDirectory), + realpath(secondDirectory) + ]) + if (firstRealPath === secondRealPath) { + throw new Error('clean-build outputs must resolve to distinct directories') + } + + // Walk sequentially so the clean-build gate cannot double file and read pressure on CI runners. + const first = await snapshotOutput(firstDirectory, 'first output', signal) + const second = await snapshotOutput(secondDirectory, 'second output', signal) + const firstIdentity = await validateCompleteOutput(firstDirectory, 'first output', tuple, first) + const secondIdentity = await validateCompleteOutput( + secondDirectory, + 'second output', + tuple, + second + ) + compareSnapshots(first, second) + if ( + firstIdentity.contentId !== secondIdentity.contentId || + firstIdentity.archive.sha256 !== secondIdentity.archive.sha256 + ) { + throw new Error('reproducibility identity or archive digest mismatch') + } + + return { + tuple, + contentId: firstIdentity.contentId, + archiveSha256: firstIdentity.archive.sha256, + entries: first.entries.size, + files: first.files, + bytes: first.bytes, + durationMs: Number(process.hrtime.bigint() - started) / 1e6 + } +} + +function valueAfter(argv, index, flag) { + const value = argv[index + 1] + if (!value || value.startsWith('--')) { + throw new Error(`${flag} requires a value`) + } + return value +} + +export function parseReproducibilityArguments(argv) { + const result = {} + for (let index = 0; index < argv.length; index += 1) { + const flag = argv[index] + const value = valueAfter(argv, index, flag) + if (flag === '--tuple') { + result.tuple = value + } else if (flag === '--first-output-directory') { + result.firstOutputDirectory = resolve(value) + } else if (flag === '--second-output-directory') { + result.secondOutputDirectory = resolve(value) + } else { + throw new Error(`Unknown argument: ${flag}`) + } + index += 1 + } + for (const field of ['tuple', 'firstOutputDirectory', 'secondOutputDirectory']) { + if (!result[field]) { + throw new Error(`Missing required reproducibility argument: ${field}`) + } + } + return result +} + +async function main() { + const result = await verifySshRelayRuntimeReproducibility( + parseReproducibilityArguments(process.argv.slice(2)) + ) + process.stdout.write(`${JSON.stringify(result, null, 2)}\n`) +} + +if (process.argv[1] && resolve(process.argv[1]) === import.meta.filename) { + main().catch((error) => { + process.stderr.write( + `SSH relay runtime reproducibility verification failed: ${error.message}\n` + ) + process.exitCode = 1 + }) +} diff --git a/config/scripts/ssh-relay-runtime-reproducibility.test.mjs b/config/scripts/ssh-relay-runtime-reproducibility.test.mjs new file mode 100644 index 00000000000..0a346a6f6c9 --- /dev/null +++ b/config/scripts/ssh-relay-runtime-reproducibility.test.mjs @@ -0,0 +1,145 @@ +import { createHash } from 'node:crypto' +import { chmod, cp, mkdir, mkdtemp, rm, symlink, writeFile } from 'node:fs/promises' +import { tmpdir } from 'node:os' +import { join } from 'node:path' + +import { afterEach, describe, expect, it } from 'vitest' + +import { verifySshRelayRuntimeReproducibility } from './ssh-relay-runtime-reproducibility.mjs' + +const temporaryDirectories = [] +const tuple = 'linux-x64-glibc' +const contentId = `sha256:${'a'.repeat(64)}` +const archiveBytes = 'archive bytes' +const archiveSha256 = `sha256:${createHash('sha256').update(archiveBytes).digest('hex')}` +const archiveName = `orca-ssh-relay-runtime-v1-${tuple}-${contentId.slice('sha256:'.length)}.tar.xz` + +afterEach(async () => { + await Promise.all(temporaryDirectories.splice(0).map((path) => rm(path, { recursive: true }))) +}) + +async function writeOutput(directory) { + await mkdir(join(directory, 'runtime'), { recursive: true }) + await Promise.all([ + writeFile(join(directory, 'runtime', 'relay.js'), 'relay bytes'), + writeFile(join(directory, 'runtime', 'relay-watcher.js'), 'watcher bytes'), + writeFile(join(directory, 'runtime', 'runtime-metadata.json'), '{"schemaVersion":1}\n'), + writeFile(join(directory, archiveName), archiveBytes), + writeFile( + join(directory, `orca-ssh-relay-runtime-${tuple}.identity.json`), + `${JSON.stringify({ tupleId: tuple, contentId, archive: { name: archiveName, sha256: archiveSha256 } })}\n` + ), + writeFile( + join(directory, `orca-ssh-relay-runtime-${tuple}.spdx.json`), + '{"spdxVersion":"SPDX-2.3"}\n' + ), + writeFile( + join(directory, `orca-ssh-relay-runtime-${tuple}.provenance.json`), + '{"predicateType":"https://slsa.dev/provenance/v1"}\n' + ) + ]) +} + +async function createOutputs() { + const directory = await mkdtemp(join(tmpdir(), 'orca-runtime-reproducibility-')) + temporaryDirectories.push(directory) + const firstOutputDirectory = join(directory, 'first') + const secondOutputDirectory = join(directory, 'second') + await writeOutput(firstOutputDirectory) + await cp(firstOutputDirectory, secondOutputDirectory, { recursive: true }) + return { directory, firstOutputDirectory, secondOutputDirectory } +} + +describe('SSH relay runtime clean-build reproducibility', () => { + it('accepts byte-and-mode-identical complete outputs', async () => { + const fixture = await createOutputs() + + await expect( + verifySshRelayRuntimeReproducibility({ + ...fixture, + tuple + }) + ).resolves.toEqual(expect.objectContaining({ tuple, contentId, archiveSha256, files: 7 })) + }) + + it('rejects runtime-tree byte drift', async () => { + const fixture = await createOutputs() + await writeFile(join(fixture.secondOutputDirectory, 'runtime', 'relay.js'), 'other bytes') + + await expect(verifySshRelayRuntimeReproducibility({ ...fixture, tuple })).rejects.toThrow( + 'reproducibility mismatch for runtime/relay.js: sha256' + ) + }) + + it('rejects SBOM byte drift', async () => { + const fixture = await createOutputs() + await writeFile( + join(fixture.secondOutputDirectory, `orca-ssh-relay-runtime-${tuple}.spdx.json`), + '{"spdxVersion":"SPDX-2.2"}\n' + ) + + await expect(verifySshRelayRuntimeReproducibility({ ...fixture, tuple })).rejects.toThrow( + `reproducibility mismatch for orca-ssh-relay-runtime-${tuple}.spdx.json: sha256` + ) + }) + + it('rejects an archive that disagrees with its identity', async () => { + const fixture = await createOutputs() + await writeFile(join(fixture.secondOutputDirectory, archiveName), 'other archive bytes') + + await expect(verifySshRelayRuntimeReproducibility({ ...fixture, tuple })).rejects.toThrow( + 'second output archive digest does not match its identity' + ) + }) + + it('rejects a missing published metadata asset', async () => { + const fixture = await createOutputs() + await rm(join(fixture.secondOutputDirectory, `orca-ssh-relay-runtime-${tuple}.spdx.json`)) + + await expect(verifySshRelayRuntimeReproducibility({ ...fixture, tuple })).rejects.toThrow( + `second output is missing required asset: orca-ssh-relay-runtime-${tuple}.spdx.json` + ) + }) + + it.skipIf(process.platform === 'win32')('rejects mode drift', async () => { + const fixture = await createOutputs() + await chmod(join(fixture.secondOutputDirectory, 'runtime', 'relay.js'), 0o755) + + await expect(verifySshRelayRuntimeReproducibility({ ...fixture, tuple })).rejects.toThrow( + 'reproducibility mismatch for runtime/relay.js: mode' + ) + }) + + it.skipIf(process.platform === 'win32')( + 'rejects links before reading their targets', + async () => { + const fixture = await createOutputs() + await symlink( + join(fixture.directory, 'outside'), + join(fixture.firstOutputDirectory, 'runtime', 'escape') + ) + + await expect(verifySshRelayRuntimeReproducibility({ ...fixture, tuple })).rejects.toThrow( + 'first output contains a prohibited special entry: runtime/escape' + ) + } + ) + + it('rejects the same directory and a pre-aborted comparison', async () => { + const fixture = await createOutputs() + await expect( + verifySshRelayRuntimeReproducibility({ + firstOutputDirectory: fixture.firstOutputDirectory, + secondOutputDirectory: fixture.firstOutputDirectory, + tuple + }) + ).rejects.toThrow('clean-build outputs must be distinct directories') + await expect( + verifySshRelayRuntimeReproducibility({ + ...fixture, + tuple, + signal: AbortSignal.abort() + }) + ).rejects.toThrow('aborted') + }) +}) diff --git a/config/scripts/ssh-relay-runtime-workflow.test.mjs b/config/scripts/ssh-relay-runtime-workflow.test.mjs index 3fc07e0ff8d..e505f7887f1 100644 --- a/config/scripts/ssh-relay-runtime-workflow.test.mjs +++ b/config/scripts/ssh-relay-runtime-workflow.test.mjs @@ -35,12 +35,13 @@ describe('SSH relay runtime artifact workflow', () => { } }) - it('uploads only archive and metadata evidence after executable verification', async () => { + it('uploads only the first output after two clean builds verify and compare', async () => { const source = await readFile(workflowUrl, 'utf8') const workflow = parse(source) const steps = workflow.jobs['build-posix-runtime'].steps + const windowsSteps = workflow.jobs['build-windows-runtime'].steps const buildIndex = steps.findIndex( - (step) => step.name === 'Build, inspect, and smoke exact runtime' + (step) => step.name === 'Build twice, inspect, smoke, and compare exact runtime' ) const uploadIndex = steps.findIndex( (step) => step.name === 'Upload unpublished artifact evidence' @@ -52,12 +53,16 @@ describe('SSH relay runtime artifact workflow', () => { expect(source).toContain('ssh-relay-runtime-workflow.test.mjs') expect(source).toContain('pnpm install --frozen-lockfile --ignore-scripts') expect(source).toContain('--connect-timeout 20 --max-time 300 --retry 2') - expect(source).toContain('mkdir -p "$(dirname "$output")"') + expect(source).toContain('mkdir -p "$output_root"') expect(source).toContain('source_commit=$(git rev-parse HEAD)') expect(source).toContain('--git-commit "$source_commit"') expect(source).not.toContain('--git-commit "$GITHUB_SHA"') - expect(source).toContain('cp "$output"/*.tar.xz') - expect(source).toContain("Get-ChildItem -LiteralPath $output -Filter '*.zip'") + expect(source).toContain('for output in "$first_output" "$second_output"') + expect(source).toContain('foreach ($output in @($firstOutput, $secondOutput))') + expect(source).toContain('ssh-relay-runtime-reproducibility.mjs') + expect(source).toContain('ssh-relay-runtime-reproducibility.test.mjs') + expect(source).toContain('cp "$first_output"/*.tar.xz') + expect(source).toContain("Get-ChildItem -LiteralPath $firstOutput -Filter '*.zip'") expect(source).toContain('ssh-relay-node-zip-inspection.test.mjs') expect(source).toContain('ssh-relay-runtime-pty-smoke.test.mjs') expect(source).toContain('ssh-relay-runtime-resource-diagnostics.test.mjs') @@ -65,6 +70,17 @@ describe('SSH relay runtime artifact workflow', () => { expect(source).toContain('node-v24.18.0-headers.tar.gz') expect(source).toContain('node_library: win-x64/node.lib') expect(source).toContain("@('gpg.exe', 'gpgv.exe')") + for (const jobSteps of [steps, windowsSteps]) { + const run = jobSteps.find( + (step) => step.name === 'Build twice, inspect, smoke, and compare exact runtime' + ).run + expect(run.indexOf('ssh-relay-runtime-reproducibility.mjs')).toBeGreaterThan( + run.indexOf('verify-ssh-relay-runtime.mjs') + ) + expect(run.indexOf('runtime-evidence/${{ matrix.tuple }}')).toBeGreaterThan( + run.indexOf('ssh-relay-runtime-reproducibility.mjs') + ) + } expect(steps[uploadIndex].with.path).toBe('runtime-evidence/${{ matrix.tuple }}/') expect(source).not.toMatch(/releases\/|gh release|contents:\s*write/i) }) diff --git a/docs/reference/plans/2026-07-14-ssh-relay-github-release-implementation-checklist.md b/docs/reference/plans/2026-07-14-ssh-relay-github-release-implementation-checklist.md index 776ae307de6..72dd4cd7ae6 100644 --- a/docs/reference/plans/2026-07-14-ssh-relay-github-release-implementation-checklist.md +++ b/docs/reference/plans/2026-07-14-ssh-relay-github-release-implementation-checklist.md @@ -2,7 +2,7 @@ Date created: 2026-07-14
Last updated: 2026-07-14
-Current phase: Milestone 3 / Work Package 2 target-native runtime assembly — add and prove a same-head, same-runner two-clean-build identity oracle now that all six glibc/macOS/Windows target-native artifact and executable cells are green; oldest-baseline, native-trust, cross-family remote, and measured-baseline gates remain open; no bundled-runtime path is enabled
+Current phase: Milestone 3 / Work Package 2 target-native runtime assembly — the bounded same-head, same-runner two-clean-build identity oracle is locally green and awaits exact-head execution on all six glibc/macOS/Windows native runners; oldest-baseline, native-trust, cross-family remote, and measured-baseline gates remain open; no bundled-runtime path is enabled
Primary design: [SSH relay GitHub Release plan](./2026-07-14-ssh-relay-github-release-plan.html)
Motivating issues: [#8450](https://github.com/stablyai/orca/issues/8450), [#1693](https://github.com/stablyai/orca/issues/1693) @@ -87,8 +87,10 @@ same change as the work it records. diff gates. The first exact-head native run exposed a macOS `/var` versus `/private/var` watcher oracle mismatch under E-M3-CI-RED-001. The corrected exact-head run passed target-native build, archive/tree verification, bundled Node, patched PTY, and watcher smoke for Linux x64/arm64 and - macOS x64/arm64 under E-M3-CI-001. Same-runner clean-rebuild identity, oldest-baseline, - native-trust, SSH, Windows, and musl cells remain open. + macOS x64/arm64 under E-M3-CI-001. A bounded comparator now rejects incomplete outputs and any + runtime-tree, archive, identity, SPDX, provenance, type, mode, size, or digest drift after two + independently verified clean builds; its focused contract is locally green, but exact-head native + execution remains pending. Oldest-baseline, native-trust, SSH, and musl cells remain open. - Windows input correction: E-M3-WINDOWS-INPUT-GAP-001 proved the official Windows ZIP lacks headers and `node.lib`. Both artifacts now require the exact signed headers archive and tuple import library as explicit inputs. The schema, signed-checksum verifier, bounded ZIP/header extraction, @@ -125,11 +127,12 @@ same change as the work it records. - Rollout control: existing per-SSH-target configuration; legacy is the default and the bundled runtime is an explicit per-target Beta opt-in under E-M1-ROLLOUT-DECISION-001. - Legacy fallback removal: not authorized. -- Next required action: add a same-head/same-runner two-clean-build identity comparison that fails - closed on any tree, archive, SBOM, provenance, or identity drift, then run it for every target- - native candidate. Keep oldest-baseline, cross-family remote infrastructure, signing/trust, and - measured legacy baseline gates open; do not introduce publication, resolver, transfer, rollout, - tuple enablement, or default behavior. +- Next required action: finish the exact implementation-commit local gates, push the bounded + two-clean-build comparator/workflow package to draft PR #8741, and require every target-native job + to compare complete runtime-tree, archive, identity, SPDX, and provenance outputs before upload. + Record every native drift instead of weakening the oracle. Keep oldest-baseline, cross-family + remote infrastructure, signing/trust, and measured legacy baseline gates open; do not introduce + publication, resolver, transfer, rollout, tuple enablement, or default behavior. ## Non-Negotiable Invariants @@ -597,8 +600,11 @@ transfer, install, fallback, rollout setting, or tuple enablement belongs in thi **Active sub-gate — 2026-07-14, Codex implementation owner:** exact-head run 29345126283 proves all six target-native artifact/executable jobs, including normal Windows x64/arm64 settlement and -unpublished upload. Add a bounded same-head/same-runner two-clean-build comparison without changing -artifact consumers, publication, the repository-wide node-pty patch, or the legacy/default path. +unpublished upload. The locally green bounded comparator now requires two independently verified +clean outputs and exact type/mode/size/SHA-256 equality for the runtime tree, archive, identity, +SPDX, and provenance before uploading only the first output. Exact-head native execution remains +open; artifact consumers, publication, the repository-wide node-pty patch, and the legacy/default +path remain unchanged. Each runtime must contain only the executable closure required by the relay. @@ -1258,6 +1264,7 @@ focused commands as their scripts/tests are introduced. - [x] `pnpm exec vitest run --config config/vitest.config.ts config/scripts/ssh-relay-runtime-pty-smoke.test.mjs` (E-M3-WINDOWS-SMOKE-SETTLEMENT-LOCAL-RED-001, E-M3-WINDOWS-SMOKE-SETTLEMENT-LOCAL-001; native evidence remains pending) - [x] `pnpm exec vitest run --config config/vitest.config.ts config/scripts/ssh-relay-runtime-resource-diagnostics.test.mjs` (E-M3-WINDOWS-RESOURCE-DIAGNOSTIC-LOCAL-001; native classification pending) - [x] `pnpm exec vitest run --config config/vitest.config.ts config/scripts/ssh-relay-node-pty-windows-settlement.test.mjs` (E-M3-WINDOWS-CONPTY-WORKER-LOCAL-RED-001, E-M3-WINDOWS-CONPTY-WORKER-LOCAL-001; native settlement pending) +- [ ] `pnpm exec vitest run --config config/vitest.config.ts config/scripts/ssh-relay-runtime-reproducibility.test.mjs config/scripts/ssh-relay-runtime-workflow.test.mjs` (locally green on the uncommitted implementation tree; rerun on the exact implementation commit before checking) - [x] `node config/scripts/verify-ssh-relay-node-release-inputs.mjs --inputs-directory --archive win32-x64` (E-M3-WINDOWS-INPUT-001; signed real Node ZIP, headers, and import library; no Windows execution) - [x] `node config/scripts/build-ssh-relay-runtime.mjs --tuple linux-arm64-glibc --inputs-directory --output-directory --source-date-epoch --git-commit ` (E-M3-RUNTIME-LOCAL-001; local native Linux arm64 only) - [x] `node config/scripts/verify-ssh-relay-runtime.mjs --runtime-directory --identity --archive ` (E-M3-RUNTIME-LOCAL-001; local native Linux arm64 only) @@ -3599,10 +3606,12 @@ The project is not complete until every applicable item below is checked with ev ## Next Required Action -Commit and push the bounded post-smoke resource diagnostic, then capture both native Windows -resource snapshots through the existing parent timeout without making another cleanup change. Use -that evidence to isolate the owning subsystem before requiring exact-head normal exit and artifact -upload. The separate clean-build identity gate, cross-family Layer B target pool, protected -manifest-signing environment, and paired legacy performance baseline remain release/default-path -blockers; no publication, desktop resolver, SSH transfer/install, per-target Beta, fallback, tuple -enablement, or default behavior may be connected by this package. +Complete the exact implementation-commit local gates, push the bounded two-clean-build +reproducibility package to draft PR #8741, and run all six target-native jobs. Each job must build, +verify, and smoke two exclusive outputs, compare every runtime-tree/archive/identity/SPDX/provenance +entry, and upload only after equality. Record exact differing paths and fields for every native red +result and correct the producer without weakening the oracle. Cross-family Layer B targets, the +protected manifest-signing environment, oldest-baseline/native-trust cells, and the paired legacy +performance baseline remain release/default-path blockers; no publication, desktop resolver, SSH +transfer/install, per-target Beta, fallback, tuple enablement, or default behavior may be connected +by this package.