build(ssh): stabilize native runtime rebuilds

Co-authored-by: Orca <help@stably.ai>

.github/workflows/ssh-relay-runtime-artifacts.yml

@@ -108,9 +108,12 @@ jobs:
           pnpm exec vitest run --config config/vitest.config.ts \
             config/scripts/ssh-relay-node-release-verification.test.mjs \
             config/scripts/ssh-relay-node-tar-inspection.test.mjs \
+            config/scripts/ssh-relay-node-pty-build.test.mjs \
+            config/scripts/ssh-relay-node-pty-windows-build-determinism.test.mjs \
             config/scripts/ssh-relay-node-pty-windows-settlement.test.mjs \
             config/scripts/ssh-relay-node-zip-inspection.test.mjs \
             config/scripts/ssh-relay-runtime-artifact.test.mjs \
+            config/scripts/ssh-relay-runtime-build.test.mjs \
             config/scripts/ssh-relay-runtime-pty-smoke.test.mjs \
             config/scripts/ssh-relay-runtime-reproducibility.test.mjs \
             config/scripts/ssh-relay-runtime-resource-diagnostics.test.mjs \
@@ -140,6 +143,7 @@ jobs:
           mkdir -p "$output_root"
           first_output="$output_root/first"
           second_output="$output_root/second"
+          work_directory="$output_root/build-work"
           source_commit=$(git rev-parse HEAD)
           source_date_epoch=$(git show -s --format=%ct "$source_commit")
           for output in "$first_output" "$second_output"; do
@@ -148,6 +152,7 @@ jobs:
               --tuple '${{ matrix.tuple }}' \
               --inputs-directory "$NODE_INPUTS" \
               --output-directory "$output" \
+              --work-directory "$work_directory" \
               --source-date-epoch "$source_date_epoch" \
               --git-commit "$source_commit"
             identity="$output/orca-ssh-relay-runtime-${{ matrix.tuple }}.identity.json"
@@ -267,9 +272,12 @@ jobs:
           pnpm exec vitest run --config config/vitest.config.ts `
             config/scripts/ssh-relay-node-release-verification.test.mjs `
             config/scripts/ssh-relay-node-tar-inspection.test.mjs `
+            config/scripts/ssh-relay-node-pty-build.test.mjs `
+            config/scripts/ssh-relay-node-pty-windows-build-determinism.test.mjs `
             config/scripts/ssh-relay-node-pty-windows-settlement.test.mjs `
             config/scripts/ssh-relay-node-zip-inspection.test.mjs `
             config/scripts/ssh-relay-runtime-artifact.test.mjs `
+            config/scripts/ssh-relay-runtime-build.test.mjs `
             config/scripts/ssh-relay-runtime-pty-smoke.test.mjs `
             config/scripts/ssh-relay-runtime-reproducibility.test.mjs `
             config/scripts/ssh-relay-runtime-resource-diagnostics.test.mjs `
@@ -310,6 +318,7 @@ jobs:
           New-Item -ItemType Directory -Path $outputRoot -Force | Out-Null
           $firstOutput = Join-Path $outputRoot 'first'
           $secondOutput = Join-Path $outputRoot 'second'
+          $workDirectory = Join-Path $outputRoot 'build-work'
           $sourceCommit = git rev-parse HEAD
           $sourceDateEpoch = git show -s --format=%ct $sourceCommit
           foreach ($output in @($firstOutput, $secondOutput)) {
@@ -318,6 +327,7 @@ jobs:
               --tuple '${{ matrix.tuple }}' `
               --inputs-directory $env:NODE_INPUTS `
               --output-directory $output `
+              --work-directory $workDirectory `
               --source-date-epoch $sourceDateEpoch `
               --git-commit $sourceCommit
             if ($LASTEXITCODE -ne 0) { throw 'runtime build failed' }

config/scripts/build-ssh-relay-runtime.mjs

@@ -1,9 +1,8 @@
 #!/usr/bin/env node
 
 import { execFile } from 'node:child_process'
-import { mkdir, mkdtemp, readFile, rm } from 'node:fs/promises'
-import { tmpdir } from 'node:os'
-import { dirname, join, resolve } from 'node:path'
+import { mkdir, readFile, rm } from 'node:fs/promises'
+import { dirname, isAbsolute, join, relative, resolve, sep } from 'node:path'
 import { promisify } from 'node:util'
 
 import { buildPatchedSshRelayNodePty } from './ssh-relay-node-pty-build.mjs'
@@ -51,6 +50,8 @@ export function parseBuildArguments(argv) {
       result.inputsDirectory = resolve(value)
     } else if (flag === '--output-directory') {
       result.outputDirectory = resolve(value)
+    } else if (flag === '--work-directory') {
+      result.workDirectory = resolve(value)
     } else if (flag === '--contract') {
       result.contractPath = resolve(value)
     } else if (flag === '--source-date-epoch') {
@@ -66,6 +67,7 @@ export function parseBuildArguments(argv) {
     'tuple',
     'inputsDirectory',
     'outputDirectory',
+    'workDirectory',
     'sourceDateEpoch',
     'gitCommit'
   ]) {
@@ -85,6 +87,20 @@ export function parseBuildArguments(argv) {
   return result
 }
 
+function containsPath(parent, candidate) {
+  const path = relative(parent, candidate)
+  return path === '' || (path !== '..' && !path.startsWith(`..${sep}`) && !isAbsolute(path))
+}
+
+export function assertDisjointRuntimeBuildDirectories(outputDirectory, workDirectory) {
+  if (
+    containsPath(outputDirectory, workDirectory) ||
+    containsPath(workDirectory, outputDirectory)
+  ) {
+    throw new Error('Runtime output and work directories must be disjoint')
+  }
+}
+
 function assertTargetNative(tuple) {
   const os = tuple.startsWith('linux-') ? 'linux' : tuple.startsWith('darwin-') ? 'darwin' : 'win32'
   const architecture = tuple.includes('arm64') ? 'arm64' : 'x64'
@@ -153,6 +169,7 @@ function relayPlatform(tuple) {
 export async function buildSshRelayRuntime(options) {
   const started = process.hrtime.bigint()
   assertTargetNative(options.tuple)
+  assertDisjointRuntimeBuildDirectories(options.outputDirectory, options.workDirectory)
   const release = validateSshRelayNodeReleaseContract(
     JSON.parse(await readFile(options.contractPath, 'utf8'))
   )
@@ -161,11 +178,17 @@ export async function buildSshRelayRuntime(options) {
     inputsDirectory: options.inputsDirectory,
     archiveTuples: [options.tuple]
   })
-  const workDirectory = await mkdtemp(join(tmpdir(), `orca-runtime-${options.tuple}-`))
-  // Why: the final directory is still created exclusively, but callers need not pre-create parents.
-  await mkdir(dirname(options.outputDirectory), { recursive: true })
+  const workDirectory = options.workDirectory
+  // Why: both final and clean-build paths are exclusive, but callers need not pre-create parents.
+  await Promise.all([
+    mkdir(dirname(options.outputDirectory), { recursive: true }),
+    mkdir(dirname(workDirectory), { recursive: true })
+  ])
   await mkdir(options.outputDirectory)
+  let workDirectoryCreated = false
   try {
+    await mkdir(workDirectory)
+    workDirectoryCreated = true
     await run(process.execPath, [resolve(scriptDirectory, 'build-relay.mjs')], { cwd: projectRoot })
     const extractedDirectory = join(workDirectory, 'node-inputs')
     const nodeArchivePath = join(options.inputsDirectory, release.archives[options.tuple].name)
@@ -244,7 +267,9 @@ export async function buildSshRelayRuntime(options) {
     await rm(options.outputDirectory, { recursive: true, force: true })
     throw error
   } finally {
-    await rm(workDirectory, { recursive: true, force: true })
+    if (workDirectoryCreated) {
+      await rm(workDirectory, { recursive: true, force: true })
+    }
   }
 }
 

config/scripts/ssh-relay-node-pty-build.mjs

@@ -5,6 +5,7 @@ import { dirname, join, resolve } from 'node:path'
 import { promisify } from 'node:util'
 
 import { applyWindowsNodePtySettlement } from './ssh-relay-node-pty-windows-settlement.mjs'
+import { applyWindowsNodePtyBuildDeterminism } from './ssh-relay-node-pty-windows-build-determinism.mjs'
 
 const require = createRequire(import.meta.url)
 const execFileAsync = promisify(execFile)
@@ -14,6 +15,21 @@ const BUILD_TIMEOUT_MS = 10 * 60 * 1000
 const COMMAND_TIMEOUT_MS = 60 * 1000
 const MAX_COMMAND_OUTPUT_BYTES = 16 * 1024 * 1024
 
+export function nodePtyNativeBuildCommands({ nodePath, nodeRoot, tuple }) {
+  const nodeGypArguments = ['--release', `--nodedir=${nodeRoot}`]
+  if (!tuple.startsWith('darwin-')) {
+    return [{ command: nodePath, args: [NODE_GYP_PATH, 'rebuild', ...nodeGypArguments] }]
+  }
+  return [
+    { command: nodePath, args: [NODE_GYP_PATH, 'configure', ...nodeGypArguments] },
+    {
+      command: 'make',
+      // Why: ld must retain a loadable UUID while ignoring incidental inputs such as build time.
+      args: ['-C', 'build', 'BUILDTYPE=Release', 'LDFLAGS.target=-Wl,-reproducible']
+    }
+  ]
+}
+
 async function runCommand(command, args, options = {}) {
   // Why: native build tools can be noisy or hang; both cases must settle within explicit bounds.
   const result = await execFileAsync(command, args, {
@@ -134,6 +150,7 @@ export async function buildPatchedSshRelayNodePty({
     dereference: true,
     filter: sourceFilter
   })
+  await applyWindowsNodePtyBuildDeterminism({ nodePtyDirectory: buildDirectory, tuple })
   await applyWindowsNodePtySettlement({
     nodePtyLibraryDirectory: join(buildDirectory, 'lib'),
     tuple
@@ -147,23 +164,26 @@ export async function buildPatchedSshRelayNodePty({
   const controller = new AbortController()
   const timeout = setTimeout(() => controller.abort(), BUILD_TIMEOUT_MS)
   try {
-    await runCommand(nodePath, [NODE_GYP_PATH, 'rebuild', '--release', `--nodedir=${nodeRoot}`], {
-      cwd: buildDirectory,
-      signal: controller.signal,
-      timeout: BUILD_TIMEOUT_MS,
-      windowsHide: true,
-      env: {
-        ...process.env,
-        PATH: `${dirname(nodePath)}${process.platform === 'win32' ? ';' : ':'}${process.env.PATH ?? ''}`,
-        npm_config_arch: tuple.includes('arm64') ? 'arm64' : 'x64',
-        npm_config_build_from_source: 'true',
-        npm_config_nodedir: nodeRoot,
-        // Why: a successful build must prove all Node inputs were staged, never fetched implicitly.
-        npm_config_offline: 'true',
-        npm_config_disturl: 'http://127.0.0.1:9',
-        npm_config_target: nodeVersion
-      }
-    })
+    const buildEnvironment = {
+      ...process.env,
+      PATH: `${dirname(nodePath)}${process.platform === 'win32' ? ';' : ':'}${process.env.PATH ?? ''}`,
+      npm_config_arch: tuple.includes('arm64') ? 'arm64' : 'x64',
+      npm_config_build_from_source: 'true',
+      npm_config_nodedir: nodeRoot,
+      // Why: a successful build must prove all Node inputs were staged, never fetched implicitly.
+      npm_config_offline: 'true',
+      npm_config_disturl: 'http://127.0.0.1:9',
+      npm_config_target: nodeVersion
+    }
+    for (const buildCommand of nodePtyNativeBuildCommands({ nodePath, nodeRoot, tuple })) {
+      await runCommand(buildCommand.command, buildCommand.args, {
+        cwd: buildDirectory,
+        signal: controller.signal,
+        timeout: BUILD_TIMEOUT_MS,
+        windowsHide: true,
+        env: buildEnvironment
+      })
+    }
   } finally {
     clearTimeout(timeout)
   }

config/scripts/ssh-relay-node-pty-build.test.mjs

@@ -0,0 +1,35 @@
+import { describe, expect, it } from 'vitest'
+
+import { nodePtyNativeBuildCommands } from './ssh-relay-node-pty-build.mjs'
+
+const nodePath = '/staged/node'
+const nodeRoot = '/staged/node-root'
+
+describe('SSH relay node-pty native build commands', () => {
+  it('uses the ordinary node-gyp rebuild on Linux and Windows', () => {
+    for (const tuple of ['linux-x64-glibc', 'linux-arm64-glibc', 'win32-x64', 'win32-arm64']) {
+      const commands = nodePtyNativeBuildCommands({ nodePath, nodeRoot, tuple })
+      expect(commands).toHaveLength(1)
+      expect(commands[0]).toMatchObject({ command: nodePath })
+      expect(commands[0].args).toEqual(
+        expect.arrayContaining(['rebuild', '--release', `--nodedir=${nodeRoot}`])
+      )
+      expect(commands[0].args).not.toContain('LDFLAGS.target=-Wl,-reproducible')
+    }
+  })
+
+  it('configures macOS before reproducibly linking a loadable Mach-O UUID', () => {
+    for (const tuple of ['darwin-x64', 'darwin-arm64']) {
+      const commands = nodePtyNativeBuildCommands({ nodePath, nodeRoot, tuple })
+      expect(commands).toHaveLength(2)
+      expect(commands[0]).toMatchObject({ command: nodePath })
+      expect(commands[0].args).toEqual(
+        expect.arrayContaining(['configure', '--release', `--nodedir=${nodeRoot}`])
+      )
+      expect(commands[1]).toEqual({
+        command: 'make',
+        args: ['-C', 'build', 'BUILDTYPE=Release', 'LDFLAGS.target=-Wl,-reproducible']
+      })
+    }
+  })
+})

config/scripts/ssh-relay-node-pty-windows-build-determinism.mjs

@@ -0,0 +1,31 @@
+import { readFile, writeFile } from 'node:fs/promises'
+import { join } from 'node:path'
+
+const LINKER_OPTIONS = `            'VCLinkerTool': {
+              'AdditionalOptions': [
+                '/DYNAMICBASE',
+                '/guard:cf'
+              ]
+            }`
+
+const REPRODUCIBLE_LINKER_OPTIONS = `            'VCLinkerTool': {
+              'AdditionalOptions': [
+                '/DYNAMICBASE',
+                '/guard:cf',
+                '/Brepro'
+              ]
+            }`
+
+export async function applyWindowsNodePtyBuildDeterminism({ nodePtyDirectory, tuple }) {
+  if (!tuple.startsWith('win32-')) {
+    return false
+  }
+  const bindingPath = join(nodePtyDirectory, 'binding.gyp')
+  const source = await readFile(bindingPath, 'utf8')
+  if (source.split(LINKER_OPTIONS).length !== 2 || source.includes("'/Brepro'")) {
+    throw new Error('node-pty Windows linker settings do not match the reviewed source')
+  }
+  // Why: MSVC otherwise embeds per-build timestamps/identities in the copied artifact binaries.
+  await writeFile(bindingPath, source.replace(LINKER_OPTIONS, REPRODUCIBLE_LINKER_OPTIONS), 'utf8')
+  return true
+}

config/scripts/ssh-relay-node-pty-windows-build-determinism.test.mjs

@@ -0,0 +1,44 @@
+import { cp, mkdtemp, readFile, rm, writeFile } from 'node:fs/promises'
+import { tmpdir } from 'node:os'
+import { join, resolve } from 'node:path'
+
+import { afterEach, describe, expect, it } from 'vitest'
+
+import { applyWindowsNodePtyBuildDeterminism } from './ssh-relay-node-pty-windows-build-determinism.mjs'
+
+const temporaryDirectories = []
+afterEach(async () => {
+  await Promise.all(temporaryDirectories.splice(0).map((path) => rm(path, { recursive: true })))
+})
+
+async function copiedSource() {
+  const directory = await mkdtemp(join(tmpdir(), 'orca-node-pty-windows-determinism-'))
+  temporaryDirectories.push(directory)
+  await cp(resolve('node_modules/node-pty/binding.gyp'), join(directory, 'binding.gyp'))
+  return directory
+}
+
+describe('SSH relay Windows node-pty build determinism', () => {
+  it('adds deterministic MSVC linking only to the copied Windows source', async () => {
+    const directory = await copiedSource()
+    await expect(
+      applyWindowsNodePtyBuildDeterminism({ nodePtyDirectory: directory, tuple: 'win32-arm64' })
+    ).resolves.toBe(true)
+    const source = await readFile(join(directory, 'binding.gyp'), 'utf8')
+    expect(source.match(/'\/Brepro'/g)).toHaveLength(1)
+  })
+
+  it('leaves POSIX source untouched and rejects an unexpected Windows source shape', async () => {
+    const directory = await copiedSource()
+    const original = await readFile(join(directory, 'binding.gyp'), 'utf8')
+    await expect(
+      applyWindowsNodePtyBuildDeterminism({ nodePtyDirectory: directory, tuple: 'darwin-arm64' })
+    ).resolves.toBe(false)
+    expect(await readFile(join(directory, 'binding.gyp'), 'utf8')).toBe(original)
+
+    await writeFile(join(directory, 'binding.gyp'), '{}', 'utf8')
+    await expect(
+      applyWindowsNodePtyBuildDeterminism({ nodePtyDirectory: directory, tuple: 'win32-x64' })
+    ).rejects.toThrow('do not match the reviewed source')
+  })
+})

config/scripts/ssh-relay-runtime-build.test.mjs

@@ -0,0 +1,51 @@
+import { resolve } from 'node:path'
+
+import { describe, expect, it } from 'vitest'
+
+import {
+  assertDisjointRuntimeBuildDirectories,
+  parseBuildArguments
+} from './build-ssh-relay-runtime.mjs'
+
+const commit = 'a'.repeat(40)
+const requiredArguments = [
+  '--tuple',
+  'darwin-arm64',
+  '--inputs-directory',
+  'inputs',
+  '--output-directory',
+  'outputs/first',
+  '--work-directory',
+  'outputs/build-work',
+  '--source-date-epoch',
+  '1752710400',
+  '--git-commit',
+  commit
+]
+
+describe('SSH relay runtime build isolation', () => {
+  it('requires and resolves a caller-owned stable clean-build directory', () => {
+    expect(parseBuildArguments(requiredArguments)).toMatchObject({
+      outputDirectory: resolve('outputs/first'),
+      workDirectory: resolve('outputs/build-work')
+    })
+    expect(() =>
+      parseBuildArguments(requiredArguments.filter((value, index) => ![6, 7].includes(index)))
+    ).toThrow('Missing required build argument: workDirectory')
+  })
+
+  it('rejects work/output overlap before either exclusive directory is created', () => {
+    expect(() => assertDisjointRuntimeBuildDirectories('/tmp/output', '/tmp/output')).toThrow(
+      'must be disjoint'
+    )
+    expect(() => assertDisjointRuntimeBuildDirectories('/tmp/output', '/tmp/output/work')).toThrow(
+      'must be disjoint'
+    )
+    expect(() => assertDisjointRuntimeBuildDirectories('/tmp/output/work', '/tmp/output')).toThrow(
+      'must be disjoint'
+    )
+    expect(() =>
+      assertDisjointRuntimeBuildDirectories('/tmp/output/first', '/tmp/output/build-work')
+    ).not.toThrow()
+  })
+})

config/scripts/ssh-relay-runtime-workflow.test.mjs

@@ -64,6 +64,10 @@ describe('SSH relay runtime artifact workflow', () => {
     expect(source).toContain(
       'node --check config/scripts/ssh-relay-runtime-reproducibility.test.mjs'
     )
+    expect(source.match(/ssh-relay-node-pty-build\.test\.mjs/g)).toHaveLength(2)
+    expect(source.match(/ssh-relay-node-pty-windows-build-determinism\.test\.mjs/g)).toHaveLength(2)
+    expect(source.match(/ssh-relay-runtime-build\.test\.mjs/g)).toHaveLength(2)
+    expect(source.match(/--work-directory/g)).toHaveLength(2)
     expect(source).toContain('cp "$first_output"/*.tar.xz')
     expect(source).toContain("Get-ChildItem -LiteralPath $firstOutput -Filter '*.zip'")
     expect(source).toContain('ssh-relay-node-zip-inspection.test.mjs')