diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index dd04a2bb42..46ceab84eb 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -80,6 +80,28 @@ jobs: - name: Run full workspace tests run: cargo nextest run --workspace --profile ci + npm-package: + name: npm package + if: github.event_name != 'pull_request' || github.event.action != 'closed' + runs-on: ubuntu-latest + timeout-minutes: 10 + steps: + - name: Checkout repository + uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 + with: + persist-credentials: false + + - name: Install Node.js + uses: actions/setup-node@49933ea5288caeca8642d1e84afbd3f7d6820020 # v4.4.0 + with: + node-version: 24.15.0 + + - name: Test npm launcher + run: npm test --prefix npm + + - name: Test release packaging scripts + run: python3 -m unittest discover --start-directory scripts/tests --verbose + protocol-smoke: name: CDP and WebDriver smoke if: github.event_name != 'pull_request' || github.event.action != 'closed' diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 9b60f340e4..f542033c5d 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -17,6 +17,11 @@ on: required: true default: false type: boolean + publish_npm: + description: Publish @lexmount/moli after creating a non-draft release + required: true + default: false + type: boolean concurrency: group: release-${{ inputs.version }} @@ -52,9 +57,16 @@ jobs: shell: bash env: INPUT_VERSION: ${{ inputs.version }} + INPUT_DRAFT: ${{ inputs.draft }} + INPUT_PUBLISH_NPM: ${{ inputs.publish_npm }} run: | set -euo pipefail + if [[ "$INPUT_DRAFT" == true && "$INPUT_PUBLISH_NPM" == true ]]; then + echo "npm publishing cannot be enabled for a draft release." >&2 + exit 1 + fi + version="${INPUT_VERSION#v}" semver_pattern='^(0|[1-9][0-9]*)\.(0|[1-9][0-9]*)\.(0|[1-9][0-9]*)(-[0-9A-Za-z-]+(\.[0-9A-Za-z-]+)*)?(\+[0-9A-Za-z-]+(\.[0-9A-Za-z-]+)*)?$' if [[ ! "$version" =~ $semver_pattern ]]; then @@ -215,6 +227,85 @@ jobs: compression-level: 0 retention-days: 7 + package-npm: + name: Package npm CLI + needs: + - validate + - build-linux + - build-macos + - build-windows + runs-on: ubuntu-latest + timeout-minutes: 15 + steps: + - name: Checkout repository + uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 + with: + persist-credentials: false + + - name: Install Node.js + uses: actions/setup-node@49933ea5288caeca8642d1e84afbd3f7d6820020 # v4.4.0 + with: + node-version: 24.15.0 + + - name: Download native release artifacts + uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1 + with: + pattern: release-* + path: dist + merge-multiple: true + + - name: Test npm launcher + run: npm test --prefix npm + + - name: Build npm packages + run: >- + python3 scripts/package_npm.py + --version "${{ needs.validate.outputs.version }}" + --input-dir dist + --output-dir dist/npm + + - name: Validate npm publish contents + run: >- + python3 scripts/publish_npm.py + dist/npm/npm-packages.json + --main-tag latest + --dry-run + + - name: Smoke-test packaged Linux CLI + shell: bash + env: + RELEASE_VERSION: ${{ needs.validate.outputs.version }} + run: | + set -euo pipefail + + smoke_dir=$(mktemp -d "${RUNNER_TEMP}/moli-npm-smoke.XXXXXX") + trap 'rm -rf "$smoke_dir"' EXIT + manifest=dist/npm/npm-packages.json + main_tarball=$(jq -r '.main.filename' "$manifest") + linux_tarball=$(jq -r \ + '.platforms[] | select(.target == "x86_64-unknown-linux-gnu") | .filename' \ + "$manifest") + main_root="$smoke_dir/node_modules/@lexmount/moli" + platform_root="$smoke_dir/node_modules/@lexmount/moli-linux-x64" + mkdir -p "$main_root" "$platform_root" + tar -xzf "dist/npm/$main_tarball" -C "$main_root" --strip-components=1 + tar -xzf "dist/npm/$linux_tarball" -C "$platform_root" --strip-components=1 + + reported_version=$(node "$main_root/bin/moli.js" --version) + if [[ "$reported_version" != "moli $RELEASE_VERSION" ]]; then + echo "Unexpected packaged CLI version: $reported_version" >&2 + exit 1 + fi + + - name: Upload npm packages + uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 + with: + name: npm-packages + path: dist/npm/ + if-no-files-found: error + compression-level: 0 + retention-days: 7 + publish: name: Create GitHub Release needs: @@ -222,6 +313,7 @@ jobs: - build-linux - build-macos - build-windows + - package-npm runs-on: ubuntu-latest timeout-minutes: 15 permissions: @@ -302,3 +394,56 @@ jobs: printf 'Created %s\n' "$release_url" printf '### Release created\n\n[%s](%s)\n' \ "$RELEASE_TAG" "$release_url" >> "$GITHUB_STEP_SUMMARY" + + publish-npm: + name: Publish npm CLI + if: ${{ inputs.publish_npm && !inputs.draft }} + needs: + - validate + - package-npm + - publish + runs-on: ubuntu-latest + timeout-minutes: 15 + environment: npm + permissions: + contents: read + id-token: write + steps: + - name: Checkout repository + uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 + with: + persist-credentials: false + + - name: Install Node.js + uses: actions/setup-node@49933ea5288caeca8642d1e84afbd3f7d6820020 # v4.4.0 + with: + node-version: 24.15.0 + registry-url: https://registry.npmjs.org + + - name: Download npm packages + uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1 + with: + name: npm-packages + path: dist/npm + + - name: Verify trusted-publishing client versions + shell: bash + run: | + node --version + npm --version + + - name: Publish platform packages, then launcher + shell: bash + env: + RELEASE_PRERELEASE: ${{ inputs.prerelease }} + run: | + set -euo pipefail + + main_tag=latest + if [[ "$RELEASE_PRERELEASE" == true ]]; then + main_tag=next + fi + python3 scripts/publish_npm.py \ + dist/npm/npm-packages.json \ + --main-tag "$main_tag" \ + --trusted-publishing diff --git a/docs/RELEASING.md b/docs/RELEASING.md index c23a843306..75148abb0f 100644 --- a/docs/RELEASING.md +++ b/docs/RELEASING.md @@ -14,6 +14,15 @@ version marker, and third-party license notices. The workflow also publishes `moli-installer.sh` and `moli-installer.ps1`. Skills are maintained separately in the repository and are not included in release assets. +The workflow also assembles an npm CLI package set from those same native +archives. The public launcher is `@lexmount/moli@`. Its optional +dependencies select one of four platform versions such as +`@lexmount/moli@-linux-x64`, so npm downloads only the binary for the +host operating system and architecture. The launcher exposes the `moli` +command and forwards arguments, standard I/O, signals, and exit status to the +native executable. Linux npm installs currently require x86-64 glibc; Linux +ARM64 and musl are not advertised as supported targets. + Stable names are intentional: the latest non-prerelease asset is always available at `https://github.com/lexmount/moli/releases/latest/download/`. @@ -56,7 +65,9 @@ treats the Windows/MSVC combination as untested. 1. Open **Actions** in GitHub and choose the **Release** workflow. 2. Select **Run workflow** and choose the Git ref containing the release. 3. Enter the version (with or without a leading `v`). -4. Choose whether the release should be a prerelease or a draft, then run it. +4. Choose whether the release should be a prerelease or a draft. +5. Enable npm publishing only after the npm trusted publisher described below + is configured, then run the workflow. The workflow validates the selected commit, builds all four native artifacts in parallel, verifies the expected archives, creates the corresponding @@ -65,3 +76,68 @@ and two installers. It stops without creating a release if any platform fails, if the requested version does not match the manifest, or if the tag already exists. A published, non-prerelease release is explicitly marked as the latest release so the stable installer URLs switch to it immediately. + +Before creating the GitHub Release, the workflow builds five npm tarballs and +smoke-tests the Linux launcher against the real packaged binary. npm publishing +is disabled by default and is never attempted for draft releases. For a stable +release the launcher receives the `latest` dist-tag; for a prerelease it +receives `next`. Native platform versions are published first under +platform-specific dist-tags, and the launcher is published last so users never +receive a package whose required binary versions are incomplete. + +## Configure npm publishing + +The unscoped `moli` package name is already owned by another publisher, so the +release uses the public scoped package `@lexmount/moli`. Confirm that the npm +organization or user controlling `@lexmount` can publish public packages before +enabling the workflow option. + +An npm Trusted Publisher can only be attached after the package exists. For the +first npm release, leave npm publishing disabled when running the Release +workflow, download its `npm-packages` artifact, sign in to npm interactively, +and publish the verified package set from the extracted artifact: + +```sh +npm login +python3 scripts/publish_npm.py /path/to/npm-packages/npm-packages.json \ + --main-tag latest +``` + +Use `--main-tag next` instead when bootstrapping from a prerelease. The script +publishes the four native versions first and the launcher last. It is safe to +retry: versions whose registry integrity matches the artifact are skipped. +After this one-time publication, configure Trusted Publishing for subsequent +releases. + +Configure an npm Trusted Publisher for the `lexmount/moli` GitHub repository +with these values: + +- Workflow filename: `release.yml` +- Environment: `npm` +- Allowed action: `npm publish` + +The publish job runs on a GitHub-hosted runner with Node 24, npm 11.5.1 or +newer, and the `id-token: write` permission. The publish script checks these +minimum versions before touching the registry. It does not use a long-lived +npm token, and npm generates provenance automatically for the public package. +Add any required reviewers or branch/tag restrictions to the GitHub `npm` +environment. After the first successful publication, installation is: + +```sh +npm install --global @lexmount/moli +moli --version +``` + +For local package validation, place all four native archives in `dist/`, then +run: + +```sh +python3 scripts/package_npm.py --version 1.0.0 +python3 scripts/publish_npm.py dist/npm/npm-packages.json --dry-run +``` + +The package manifest records the SHA-512 integrity of every tarball. A retried +publish skips an existing version only when the registry reports the same +integrity; a mismatch stops the release. npm does not allow a published +name/version pair to be replaced, so release versions must be bumped before +publishing changed contents. diff --git a/npm/bin/moli.js b/npm/bin/moli.js new file mode 100644 index 0000000000..ee4054f0ce --- /dev/null +++ b/npm/bin/moli.js @@ -0,0 +1,96 @@ +#!/usr/bin/env node + +import { spawn } from "node:child_process"; +import { existsSync, realpathSync } from "node:fs"; +import { createRequire } from "node:module"; +import path from "node:path"; +import { fileURLToPath } from "node:url"; + +import { + assertSupportedLibc, + platformDefinitionFor, +} from "../lib/platform.js"; + +const require = createRequire(import.meta.url); +const packageRoot = realpathSync( + path.join(path.dirname(fileURLToPath(import.meta.url)), ".."), +); + +function executableFor(definition) { + let vendorRoot; + try { + const packageJson = require.resolve(`${definition.package}/package.json`); + vendorRoot = path.join(path.dirname(packageJson), "vendor"); + } catch { + // This fallback makes source checkouts and assembled package smoke tests + // possible without weakening the normal optional-dependency lookup. + vendorRoot = path.join(packageRoot, "vendor"); + } + + const executable = path.join( + vendorRoot, + definition.target, + "bin", + definition.binary, + ); + if (existsSync(executable)) { + return executable; + } + + throw new Error( + `Missing optional dependency ${definition.package}. ` + + "Reinstall @lexmount/moli without omitting optional dependencies.", + ); +} + +async function run() { + const definition = platformDefinitionFor(process.platform, process.arch); + const runtimeReport = process.report?.getReport?.(); + assertSupportedLibc(definition, runtimeReport?.header?.glibcVersionRuntime); + const child = spawn(executableFor(definition), process.argv.slice(2), { + stdio: "inherit", + }); + + const forwardedSignals = + process.platform === "win32" + ? ["SIGINT", "SIGTERM"] + : ["SIGINT", "SIGTERM", "SIGHUP"]; + const signalHandlers = new Map(); + for (const signal of forwardedSignals) { + const handler = () => { + if (!child.killed) { + child.kill(signal); + } + }; + signalHandlers.set(signal, handler); + process.on(signal, handler); + } + + let result; + try { + result = await new Promise((resolve, reject) => { + child.once("error", reject); + child.once("exit", (exitCode, signal) => { + resolve(signal ? { signal } : { exitCode: exitCode ?? 1 }); + }); + }); + } finally { + for (const [signal, handler] of signalHandlers) { + process.off(signal, handler); + } + } + + if ("signal" in result) { + process.kill(process.pid, result.signal); + return; + } + process.exitCode = result.exitCode; +} + +try { + await run(); +} catch (error) { + const message = error instanceof Error ? error.message : String(error); + console.error(`moli: ${message}`); + process.exitCode = 1; +} diff --git a/npm/lib/platform.js b/npm/lib/platform.js new file mode 100644 index 0000000000..edd7943744 --- /dev/null +++ b/npm/lib/platform.js @@ -0,0 +1,32 @@ +import { readFileSync } from "node:fs"; + +const definitions = JSON.parse( + readFileSync(new URL("../platforms.json", import.meta.url), "utf8"), +); + +export const PLATFORM_DEFINITIONS = Object.freeze( + definitions.map((definition) => Object.freeze(definition)), +); + +export function platformDefinitionFor(platform, arch) { + const definition = PLATFORM_DEFINITIONS.find( + (candidate) => + candidate.platform === platform && candidate.arch === arch, + ); + if (!definition) { + throw new Error(`Unsupported platform: ${platform} (${arch})`); + } + return definition; +} + +export function assertSupportedLibc(definition, glibcVersionRuntime) { + if ( + definition.libc?.includes("glibc") && + (typeof glibcVersionRuntime !== "string" || glibcVersionRuntime.length === 0) + ) { + throw new Error( + `Unsupported libc for ${definition.platform} (${definition.arch}): ` + + "this Moli package requires glibc", + ); + } +} diff --git a/npm/package.json b/npm/package.json new file mode 100644 index 0000000000..05b94706b1 --- /dev/null +++ b/npm/package.json @@ -0,0 +1,17 @@ +{ + "name": "@lexmount/moli", + "version": "0.0.0-development", + "private": true, + "description": "npm launcher sources for the Moli CLI", + "license": "MIT OR Apache-2.0", + "type": "module", + "bin": { + "moli": "bin/moli.js" + }, + "engines": { + "node": ">=18" + }, + "scripts": { + "test": "node --test" + } +} diff --git a/npm/platforms.json b/npm/platforms.json new file mode 100644 index 0000000000..5aee44404c --- /dev/null +++ b/npm/platforms.json @@ -0,0 +1,41 @@ +[ + { + "id": "linux-x64", + "platform": "linux", + "arch": "x64", + "target": "x86_64-unknown-linux-gnu", + "package": "@lexmount/moli-linux-x64", + "archive": "moli-x86_64-unknown-linux-gnu.tar.gz", + "binary": "moli", + "libc": [ + "glibc" + ] + }, + { + "id": "darwin-x64", + "platform": "darwin", + "arch": "x64", + "target": "x86_64-apple-darwin", + "package": "@lexmount/moli-darwin-x64", + "archive": "moli-x86_64-apple-darwin.tar.gz", + "binary": "moli" + }, + { + "id": "darwin-arm64", + "platform": "darwin", + "arch": "arm64", + "target": "aarch64-apple-darwin", + "package": "@lexmount/moli-darwin-arm64", + "archive": "moli-aarch64-apple-darwin.tar.gz", + "binary": "moli" + }, + { + "id": "win32-x64", + "platform": "win32", + "arch": "x64", + "target": "x86_64-pc-windows-msvc", + "package": "@lexmount/moli-win32-x64", + "archive": "moli-x86_64-pc-windows-msvc.zip", + "binary": "moli.exe" + } +] diff --git a/npm/tests/launcher.test.mjs b/npm/tests/launcher.test.mjs new file mode 100644 index 0000000000..475f6b9b13 --- /dev/null +++ b/npm/tests/launcher.test.mjs @@ -0,0 +1,131 @@ +import assert from "node:assert/strict"; +import { cp, chmod, mkdir, mkdtemp, rm, writeFile } from "node:fs/promises"; +import os from "node:os"; +import path from "node:path"; +import { spawn, spawnSync } from "node:child_process"; +import test from "node:test"; +import { fileURLToPath } from "node:url"; + +import { platformDefinitionFor } from "../lib/platform.js"; + +const npmSourceRoot = fileURLToPath(new URL("..", import.meta.url)); + +async function createLauncherFixture(context, binarySource) { + const fixtureRoot = await mkdtemp(path.join(os.tmpdir(), "moli-npm-launcher-")); + context.after(() => rm(fixtureRoot, { recursive: true, force: true })); + + await cp(path.join(npmSourceRoot, "bin"), path.join(fixtureRoot, "bin"), { + recursive: true, + }); + await cp(path.join(npmSourceRoot, "lib"), path.join(fixtureRoot, "lib"), { + recursive: true, + }); + await cp( + path.join(npmSourceRoot, "platforms.json"), + path.join(fixtureRoot, "platforms.json"), + ); + + const definition = platformDefinitionFor(process.platform, process.arch); + const fakeBinary = path.join( + fixtureRoot, + "vendor", + definition.target, + "bin", + definition.binary, + ); + await mkdir(path.dirname(fakeBinary), { recursive: true }); + await writeFile(fakeBinary, binarySource, "utf8"); + await chmod(fakeBinary, 0o755); + return { + fixtureRoot, + launcher: path.join(fixtureRoot, "bin", "moli.js"), + }; +} + +test( + "launcher forwards arguments, stdio, and exit status", + { skip: process.platform === "win32" }, + async (context) => { + const { launcher } = await createLauncherFixture( + context, + [ + "#!/usr/bin/env node", + "const result = {", + " args: process.argv.slice(2),", + "};", + "process.stdout.write(JSON.stringify(result));", + "process.stderr.write('native stderr');", + "process.exit(Number(process.env.MOLI_TEST_EXIT_CODE));", + "", + ].join("\n"), + ); + + const result = spawnSync( + process.execPath, + [launcher, "argument with spaces", "--flag"], + { + encoding: "utf8", + env: { ...process.env, MOLI_TEST_EXIT_CODE: "23" }, + }, + ); + + assert.equal(result.status, 23); + assert.equal(result.stderr, "native stderr"); + assert.deepEqual(JSON.parse(result.stdout), { + args: ["argument with spaces", "--flag"], + }); + }, +); + +test( + "launcher forwards termination signals and exits with the same signal", + { skip: process.platform === "win32" }, + async (context) => { + const { launcher } = await createLauncherFixture( + context, + [ + "#!/usr/bin/env node", + "process.stdout.write('ready\\n');", + "setInterval(() => {}, 1000);", + "", + ].join("\n"), + ); + const child = spawn(process.execPath, [launcher], { + stdio: ["ignore", "pipe", "pipe"], + }); + context.after(() => { + if (child.exitCode === null && child.signalCode === null) { + child.kill("SIGKILL"); + } + }); + + await new Promise((resolve, reject) => { + child.once("error", reject); + child.stdout.once("data", (chunk) => { + assert.equal(chunk.toString(), "ready\n"); + resolve(); + }); + }); + const exited = new Promise((resolve) => { + child.once("exit", (...result) => resolve(result)); + }); + child.kill("SIGTERM"); + const [exitCode, signal] = await exited; + + assert.equal(exitCode, null); + assert.equal(signal, "SIGTERM"); + }, +); + +test("launcher reports a missing optional dependency clearly", () => { + const result = spawnSync( + process.execPath, + [path.join(npmSourceRoot, "bin", "moli.js"), "--version"], + { encoding: "utf8" }, + ); + + assert.equal(result.status, 1); + const definition = platformDefinitionFor(process.platform, process.arch); + assert.match(result.stderr, new RegExp(`Missing optional dependency ${definition.package}`)); + assert.match(result.stderr, /without omitting optional dependencies/); +}); diff --git a/npm/tests/platform.test.mjs b/npm/tests/platform.test.mjs new file mode 100644 index 0000000000..36eb88def3 --- /dev/null +++ b/npm/tests/platform.test.mjs @@ -0,0 +1,53 @@ +import assert from "node:assert/strict"; +import test from "node:test"; + +import { + PLATFORM_DEFINITIONS, + assertSupportedLibc, + platformDefinitionFor, +} from "../lib/platform.js"; + +test("maps every supported Node platform to one native target", () => { + const expected = new Map([ + ["linux:x64", "x86_64-unknown-linux-gnu"], + ["darwin:x64", "x86_64-apple-darwin"], + ["darwin:arm64", "aarch64-apple-darwin"], + ["win32:x64", "x86_64-pc-windows-msvc"], + ]); + + assert.equal(PLATFORM_DEFINITIONS.length, expected.size); + for (const [key, target] of expected) { + const [platform, arch] = key.split(":"); + assert.equal(platformDefinitionFor(platform, arch).target, target); + } +}); + +test("keeps package aliases and release assets unique", () => { + for (const field of ["id", "target", "package", "archive"]) { + const values = PLATFORM_DEFINITIONS.map((definition) => definition[field]); + assert.equal(new Set(values).size, values.length, `${field} must be unique`); + } +}); + +test("rejects unsupported platforms without falling back to a wrong binary", () => { + assert.throws( + () => platformDefinitionFor("linux", "arm64"), + /Unsupported platform: linux \(arm64\)/, + ); + assert.throws( + () => platformDefinitionFor("freebsd", "x64"), + /Unsupported platform: freebsd \(x64\)/, + ); +}); + +test("rejects musl before attempting to run the glibc Linux package", () => { + const linux = platformDefinitionFor("linux", "x64"); + assert.doesNotThrow(() => assertSupportedLibc(linux, "2.36")); + assert.throws( + () => assertSupportedLibc(linux, undefined), + /this Moli package requires glibc/, + ); + + const darwin = platformDefinitionFor("darwin", "arm64"); + assert.doesNotThrow(() => assertSupportedLibc(darwin, undefined)); +}); diff --git a/scripts/package_npm.py b/scripts/package_npm.py new file mode 100644 index 0000000000..0b186a56b3 --- /dev/null +++ b/scripts/package_npm.py @@ -0,0 +1,568 @@ +#!/usr/bin/env python3 +"""Build npm launcher and native platform packages from Moli release archives.""" + +from __future__ import annotations + +import argparse +import base64 +import hashlib +import json +import re +import shutil +import subprocess +import sys +import tarfile +import tempfile +import tomllib +import zipfile +from dataclasses import dataclass +from pathlib import Path +from typing import Any + + +REPO_ROOT = Path(__file__).resolve().parent.parent +MANIFEST_PATH = REPO_ROOT / "moli" / "Cargo.toml" +NPM_SOURCE_DIR = REPO_ROOT / "npm" +PLATFORMS_PATH = NPM_SOURCE_DIR / "platforms.json" +PACKAGE_NAME = "@lexmount/moli" +SEMVER_PATTERN = re.compile( + r"^(0|[1-9][0-9]*)\." + r"(0|[1-9][0-9]*)\." + r"(0|[1-9][0-9]*)" + r"(?:-[0-9A-Za-z-]+(?:\.[0-9A-Za-z-]+)*)?" + r"(?:\+[0-9A-Za-z-]+(?:\.[0-9A-Za-z-]+)*)?$" +) + + +class NpmPackageError(RuntimeError): + """An npm package input or build step was invalid.""" + + +@dataclass(frozen=True) +class PlatformDefinition: + id: str + platform: str + arch: str + target: str + package: str + archive: str + binary: str + libc: tuple[str, ...] = () + + +def normalize_version(raw_version: str) -> str: + version = raw_version.removeprefix("v") + if not SEMVER_PATTERN.fullmatch(version): + raise NpmPackageError(f"invalid semantic version: {raw_version}") + return version + + +def manifest_version() -> str: + with MANIFEST_PATH.open("rb") as manifest_file: + manifest = tomllib.load(manifest_file) + try: + version = manifest["package"]["version"] + except (KeyError, TypeError) as error: + raise NpmPackageError( + f"package.version is missing from {MANIFEST_PATH}" + ) from error + if not isinstance(version, str): + raise NpmPackageError( + f"package.version in {MANIFEST_PATH} is not a string" + ) + return version + + +def resolve_repo_path(raw_path: str) -> Path: + path = Path(raw_path).expanduser() + if not path.is_absolute(): + path = REPO_ROOT / path + return path.resolve() + + +def load_platforms() -> list[PlatformDefinition]: + try: + raw_platforms = json.loads(PLATFORMS_PATH.read_text(encoding="utf-8")) + except (OSError, json.JSONDecodeError) as error: + raise NpmPackageError(f"could not read {PLATFORMS_PATH}: {error}") from error + if not isinstance(raw_platforms, list) or not raw_platforms: + raise NpmPackageError(f"{PLATFORMS_PATH} must contain a non-empty array") + + platforms: list[PlatformDefinition] = [] + for index, raw in enumerate(raw_platforms): + if not isinstance(raw, dict): + raise NpmPackageError(f"platform entry {index} must be an object") + raw_libc = raw.get("libc", []) + if not isinstance(raw_libc, list): + raise NpmPackageError(f"platform entry {index} libc must be an array") + try: + definition = PlatformDefinition( + id=raw["id"], + platform=raw["platform"], + arch=raw["arch"], + target=raw["target"], + package=raw["package"], + archive=raw["archive"], + binary=raw["binary"], + libc=tuple(raw_libc), + ) + except (KeyError, TypeError) as error: + raise NpmPackageError(f"invalid platform entry {index}: {error}") from error + values = ( + definition.id, + definition.platform, + definition.arch, + definition.target, + definition.package, + definition.archive, + definition.binary, + *definition.libc, + ) + if not all(isinstance(value, str) and value for value in values): + raise NpmPackageError( + f"platform entry {index} contains an empty or non-string value" + ) + expected_package = f"{PACKAGE_NAME}-{definition.id}" + if definition.package != expected_package: + raise NpmPackageError( + f"platform entry {index} package must be {expected_package}" + ) + platforms.append(definition) + + for field in ("id", "target", "package", "archive"): + values = [getattr(platform, field) for platform in platforms] + if len(values) != len(set(values)): + raise NpmPackageError(f"platform definitions contain duplicate {field} values") + node_platforms = [(platform.platform, platform.arch) for platform in platforms] + if len(node_platforms) != len(set(node_platforms)): + raise NpmPackageError( + "platform definitions contain duplicate platform/architecture pairs" + ) + return platforms + + +def platform_version(version: str, platform_id: str) -> str: + core, separator, build = version.partition("+") + variant = f"{core}-{platform_id}" + if separator: + variant = f"{variant}+{build}" + if not SEMVER_PATTERN.fullmatch(variant): + raise NpmPackageError(f"invalid npm platform version: {variant}") + return variant + + +def repository_metadata() -> dict[str, str]: + return { + "type": "git", + "url": "git+https://github.com/lexmount/moli.git", + } + + +def copy_project_file(destination: Path, relative_path: str) -> None: + source = REPO_ROOT / relative_path + if not source.is_file(): + raise NpmPackageError(f"required npm package file is missing: {source}") + shutil.copy2(source, destination / source.name) + + +def copy_package_metadata(destination: Path, *, third_party: bool) -> None: + for relative_path in ("README.md", "LICENSE", "LICENSE-APACHE", "LICENSE-MIT"): + copy_project_file(destination, relative_path) + if not third_party: + return + + notices_root = destination / "third_party" + notices_root.mkdir() + for name in ("licenses", "notices"): + source = REPO_ROOT / "third_party" / name + if not source.is_dir(): + raise NpmPackageError( + f"required third-party license directory is missing: {source}" + ) + shutil.copytree(source, notices_root / name) + + +def write_json(path: Path, value: Any) -> None: + path.write_text( + json.dumps(value, indent=2, ensure_ascii=False) + "\n", encoding="utf-8" + ) + + +def main_package_manifest( + version: str, platforms: list[PlatformDefinition] +) -> dict[str, Any]: + optional_dependencies = { + platform.package: f"npm:{PACKAGE_NAME}@{platform_version(version, platform.id)}" + for platform in platforms + } + return { + "name": PACKAGE_NAME, + "version": version, + "description": ( + "A structured-first headless browser engine for AI agents" + ), + "license": "MIT OR Apache-2.0", + "type": "module", + "bin": {"moli": "bin/moli.js"}, + "engines": {"node": ">=18"}, + "files": [ + "bin", + "lib", + "platforms.json", + "README.md", + "LICENSE", + "LICENSE-APACHE", + "LICENSE-MIT", + ], + "repository": repository_metadata(), + "homepage": "https://github.com/lexmount/moli", + "bugs": {"url": "https://github.com/lexmount/moli/issues"}, + "publishConfig": {"access": "public"}, + "optionalDependencies": optional_dependencies, + } + + +def platform_package_manifest( + version: str, definition: PlatformDefinition +) -> dict[str, Any]: + manifest: dict[str, Any] = { + "name": PACKAGE_NAME, + "version": platform_version(version, definition.id), + "description": f"Moli native binary for {definition.id}", + "license": "MIT OR Apache-2.0", + "engines": {"node": ">=18"}, + "os": [definition.platform], + "cpu": [definition.arch], + "files": [ + "vendor", + "README.md", + "LICENSE", + "LICENSE-APACHE", + "LICENSE-MIT", + "third_party", + ], + "repository": repository_metadata(), + "homepage": "https://github.com/lexmount/moli", + "bugs": {"url": "https://github.com/lexmount/moli/issues"}, + "publishConfig": {"access": "public"}, + } + if definition.libc: + manifest["libc"] = list(definition.libc) + return manifest + + +def extract_platform_binary( + archive_path: Path, + version: str, + definition: PlatformDefinition, + destination: Path, +) -> None: + package_root = f"moli-v{version}-{definition.target}" + binary_member = f"{package_root}/{definition.binary}" + version_member = f"{package_root}/VERSION" + try: + if archive_path.suffix == ".zip": + with zipfile.ZipFile(archive_path, mode="r") as archive: + archived_version = archive.read(version_member) + with archive.open(binary_member, mode="r") as source, destination.open( + "wb" + ) as output: + shutil.copyfileobj(source, output) + else: + with tarfile.open(archive_path, mode="r:gz") as archive: + version_entry = archive.getmember(version_member) + binary_entry = archive.getmember(binary_member) + if not version_entry.isfile() or not binary_entry.isfile(): + raise NpmPackageError( + f"release archive contains a non-file package member: {archive_path}" + ) + version_source = archive.extractfile(version_entry) + binary_source = archive.extractfile(binary_entry) + if version_source is None or binary_source is None: + raise NpmPackageError( + f"could not read package members from {archive_path}" + ) + with version_source: + archived_version = version_source.read() + with binary_source, destination.open("wb") as output: + shutil.copyfileobj(binary_source, output) + except (KeyError, OSError, tarfile.TarError, zipfile.BadZipFile) as error: + raise NpmPackageError(f"could not unpack {archive_path}: {error}") from error + + try: + decoded_version = archived_version.decode("utf-8").strip() + except UnicodeDecodeError as error: + raise NpmPackageError( + f"release archive VERSION is not UTF-8: {archive_path}" + ) from error + if decoded_version != version: + raise NpmPackageError( + f"release archive version mismatch for {archive_path.name}: " + f"expected {version}, got {decoded_version or ''}" + ) + destination.chmod(0o644 if definition.platform == "win32" else 0o755) + + +def package_integrity(path: Path) -> str: + digest = hashlib.sha512() + with path.open("rb") as package_file: + for chunk in iter(lambda: package_file.read(1024 * 1024), b""): + digest.update(chunk) + return "sha512-" + base64.b64encode(digest.digest()).decode("ascii") + + +def npm_pack(package_dir: Path, output_dir: Path) -> dict[str, Any]: + command = [ + "npm", + "pack", + "--json", + "--pack-destination", + str(output_dir), + ] + try: + result = subprocess.run( + command, + cwd=package_dir, + check=True, + text=True, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + ) + except FileNotFoundError as error: + raise NpmPackageError("npm is required to build npm packages") from error + except subprocess.CalledProcessError as error: + detail = error.stderr.strip() or error.stdout.strip() + raise NpmPackageError(f"npm pack failed: {detail}") from error + + try: + payload = json.loads(result.stdout) + except json.JSONDecodeError as error: + raise NpmPackageError(f"npm pack returned invalid JSON: {error}") from error + if not isinstance(payload, list) or len(payload) != 1 or not isinstance(payload[0], dict): + raise NpmPackageError("npm pack did not describe exactly one package") + + packed = payload[0] + required_fields = ( + "name", + "version", + "filename", + "integrity", + "shasum", + "size", + "unpackedSize", + "files", + ) + if any(field not in packed for field in required_fields): + raise NpmPackageError("npm pack output is missing required metadata") + for field in ("name", "version", "filename", "integrity", "shasum"): + if not isinstance(packed[field], str) or not packed[field]: + raise NpmPackageError(f"npm pack output contains invalid {field}") + if Path(packed["filename"]).name != packed["filename"]: + raise NpmPackageError("npm pack output filename must not contain a path") + tarball = output_dir / packed["filename"] + if not tarball.is_file(): + raise NpmPackageError(f"npm pack did not create {tarball}") + actual_integrity = package_integrity(tarball) + if packed["integrity"] != actual_integrity: + raise NpmPackageError( + f"npm package integrity mismatch for {tarball.name}: " + f"expected {packed['integrity']}, got {actual_integrity}" + ) + return packed + + +def packed_file_paths(packed: dict[str, Any]) -> set[str]: + files = packed["files"] + if not isinstance(files, list): + raise NpmPackageError("npm pack file metadata must be an array") + paths: set[str] = set() + for file in files: + if not isinstance(file, dict) or not isinstance(file.get("path"), str): + raise NpmPackageError("npm pack returned invalid file metadata") + paths.add(file["path"]) + return paths + + +def compact_pack_metadata(packed: dict[str, Any]) -> dict[str, Any]: + return { + field: packed[field] + for field in ( + "name", + "version", + "filename", + "integrity", + "shasum", + "size", + "unpackedSize", + ) + } + + +def build_main_package( + stage: Path, + output_dir: Path, + version: str, + platforms: list[PlatformDefinition], +) -> dict[str, Any]: + shutil.copytree(NPM_SOURCE_DIR / "bin", stage / "bin") + shutil.copytree(NPM_SOURCE_DIR / "lib", stage / "lib") + shutil.copy2(PLATFORMS_PATH, stage / "platforms.json") + (stage / "bin" / "moli.js").chmod(0o755) + copy_package_metadata(stage, third_party=False) + write_json(stage / "package.json", main_package_manifest(version, platforms)) + + packed = npm_pack(stage, output_dir) + if packed["name"] != PACKAGE_NAME or packed["version"] != version: + raise NpmPackageError("npm packed an unexpected main package identity") + expected_files = { + "bin/moli.js", + "lib/platform.js", + "platforms.json", + "package.json", + } + missing = expected_files - packed_file_paths(packed) + if missing: + raise NpmPackageError( + f"main npm package is missing files: {', '.join(sorted(missing))}" + ) + return compact_pack_metadata(packed) + + +def build_platform_package( + stage: Path, + output_dir: Path, + input_dir: Path, + version: str, + definition: PlatformDefinition, +) -> dict[str, Any]: + archive_path = input_dir / definition.archive + if not archive_path.is_file(): + raise NpmPackageError(f"required release archive is missing: {archive_path}") + + binary = stage / "vendor" / definition.target / "bin" / definition.binary + binary.parent.mkdir(parents=True) + extract_platform_binary(archive_path, version, definition, binary) + copy_package_metadata(stage, third_party=True) + write_json(stage / "package.json", platform_package_manifest(version, definition)) + + packed = npm_pack(stage, output_dir) + expected_version = platform_version(version, definition.id) + if packed["name"] != PACKAGE_NAME or packed["version"] != expected_version: + raise NpmPackageError( + f"npm packed an unexpected package identity for {definition.id}" + ) + expected_binary = f"vendor/{definition.target}/bin/{definition.binary}" + if expected_binary not in packed_file_paths(packed): + raise NpmPackageError( + f"platform npm package is missing binary: {expected_binary}" + ) + metadata = compact_pack_metadata(packed) + metadata.update( + { + "alias": definition.package, + "target": definition.target, + "distTag": definition.id, + } + ) + return metadata + + +def build_packages( + *, version: str, input_dir: Path, output_dir: Path +) -> dict[str, Any]: + declared_version = manifest_version() + if version != declared_version: + raise NpmPackageError( + f"requested version {version} does not match " + f"moli/Cargo.toml ({declared_version})" + ) + if not input_dir.is_dir(): + raise NpmPackageError(f"release input directory does not exist: {input_dir}") + if output_dir.exists(): + raise NpmPackageError(f"npm output path already exists: {output_dir}") + output_dir.parent.mkdir(parents=True, exist_ok=True) + + platforms = load_platforms() + for definition in platforms: + archive_path = input_dir / definition.archive + if not archive_path.is_file(): + raise NpmPackageError(f"required release archive is missing: {archive_path}") + + with tempfile.TemporaryDirectory( + prefix=".moli-npm-", dir=output_dir.parent + ) as temporary: + work_dir = Path(temporary) + packed_dir = work_dir / "packed" + packed_dir.mkdir() + + platform_packages: list[dict[str, Any]] = [] + for definition in platforms: + with tempfile.TemporaryDirectory( + prefix=f"{definition.id}-", dir=work_dir + ) as platform_stage: + platform_packages.append( + build_platform_package( + Path(platform_stage), + packed_dir, + input_dir, + version, + definition, + ) + ) + + with tempfile.TemporaryDirectory(prefix="main-", dir=work_dir) as main_stage: + main_package = build_main_package( + Path(main_stage), packed_dir, version, platforms + ) + + release_manifest = { + "schemaVersion": 1, + "package": PACKAGE_NAME, + "version": version, + "main": main_package, + "platforms": platform_packages, + } + write_json(packed_dir / "npm-packages.json", release_manifest) + packed_dir.rename(output_dir) + return release_manifest + + +def parse_args() -> argparse.Namespace: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--version", required=True, help="release version, with optional v") + parser.add_argument( + "--input-dir", + default="dist", + help="directory containing native release archives (default: dist)", + ) + parser.add_argument( + "--output-dir", + default="dist/npm", + help="new directory for npm tarballs (default: dist/npm)", + ) + return parser.parse_args() + + +def main() -> int: + args = parse_args() + try: + version = normalize_version(args.version) + output_dir = resolve_repo_path(args.output_dir) + manifest = build_packages( + version=version, + input_dir=resolve_repo_path(args.input_dir), + output_dir=output_dir, + ) + print(f"Created npm package set for {manifest['package']}@{version}") + for platform in manifest["platforms"]: + print(f"Created: {output_dir / platform['filename']}") + print(f"Created: {output_dir / manifest['main']['filename']}") + print(f"Created: {output_dir / 'npm-packages.json'}") + return 0 + except NpmPackageError as error: + print(f"npm package error: {error}", file=sys.stderr) + return 1 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/scripts/publish_npm.py b/scripts/publish_npm.py new file mode 100644 index 0000000000..e869525af9 --- /dev/null +++ b/scripts/publish_npm.py @@ -0,0 +1,348 @@ +#!/usr/bin/env python3 +"""Publish a verified Moli npm package set, with the launcher published last.""" + +from __future__ import annotations + +import argparse +import base64 +import hashlib +import json +import re +import shlex +import subprocess +import sys +import tarfile +from pathlib import Path +from typing import Any + + +DIST_TAG_PATTERN = re.compile(r"^[A-Za-z][A-Za-z0-9._-]*$") +TOOL_VERSION_PATTERN = re.compile( + r"^v?(0|[1-9][0-9]*)\." + r"(0|[1-9][0-9]*)\." + r"(0|[1-9][0-9]*)" + r"(?:-[0-9A-Za-z.-]+)?$" +) +MINIMUM_TRUSTED_NPM_VERSION = (11, 5, 1) +MINIMUM_TRUSTED_NODE_VERSION = (22, 14, 0) +MAXIMUM_PACKAGE_JSON_SIZE = 1024 * 1024 + + +class NpmPublishError(RuntimeError): + """An npm package set was invalid or could not be published safely.""" + + +def package_integrity(path: Path) -> str: + digest = hashlib.sha512() + with path.open("rb") as package_file: + for chunk in iter(lambda: package_file.read(1024 * 1024), b""): + digest.update(chunk) + return "sha512-" + base64.b64encode(digest.digest()).decode("ascii") + + +def load_manifest(path: Path) -> dict[str, Any]: + try: + manifest = json.loads(path.read_text(encoding="utf-8")) + except (OSError, json.JSONDecodeError) as error: + raise NpmPublishError( + f"could not read npm package manifest {path}: {error}" + ) from error + if not isinstance(manifest, dict) or manifest.get("schemaVersion") != 1: + raise NpmPublishError("unsupported npm package manifest schema") + if not isinstance(manifest.get("package"), str) or not isinstance( + manifest.get("version"), str + ): + raise NpmPublishError("npm package manifest is missing package identity") + if not isinstance(manifest.get("main"), dict) or not isinstance( + manifest.get("platforms"), list + ): + raise NpmPublishError("npm package manifest is missing package entries") + if not manifest["platforms"]: + raise NpmPublishError("npm package manifest contains no platform packages") + return manifest + + +def validate_dist_tag(tag: str) -> str: + if not DIST_TAG_PATTERN.fullmatch(tag): + raise NpmPublishError(f"invalid npm dist-tag: {tag}") + return tag + + +def tool_version(command: str) -> tuple[int, int, int]: + try: + result = subprocess.run( + [command, "--version"], + check=True, + text=True, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + ) + except FileNotFoundError as error: + raise NpmPublishError( + f"{command} is required to publish npm packages" + ) from error + except subprocess.CalledProcessError as error: + detail = error.stderr.strip() or error.stdout.strip() + raise NpmPublishError( + f"could not determine {command} version: {detail}" + ) from error + + raw_version = result.stdout.strip() + match = TOOL_VERSION_PATTERN.fullmatch(raw_version) + if match is None: + raise NpmPublishError(f"could not parse {command} version: {raw_version}") + return tuple(int(component) for component in match.groups()) + + +def require_trusted_publishing_toolchain() -> None: + requirements = ( + ("node", MINIMUM_TRUSTED_NODE_VERSION), + ("npm", MINIMUM_TRUSTED_NPM_VERSION), + ) + for command, minimum in requirements: + actual = tool_version(command) + if actual < minimum: + minimum_display = ".".join(str(component) for component in minimum) + actual_display = ".".join(str(component) for component in actual) + raise NpmPublishError( + f"npm trusted publishing requires {command} >= {minimum_display}; " + f"found {actual_display}" + ) + + +def validate_package_entry( + entry: dict[str, Any], package_dir: Path +) -> tuple[Path, str, str]: + for field in ("name", "version", "filename", "integrity"): + if not isinstance(entry.get(field), str) or not entry[field]: + raise NpmPublishError(f"npm package entry has invalid {field}") + filename = entry["filename"] + if Path(filename).name != filename: + raise NpmPublishError( + f"npm package filename must not contain a path: {filename}" + ) + tarball = package_dir / filename + if not tarball.is_file(): + raise NpmPublishError(f"npm package tarball is missing: {tarball}") + actual_integrity = package_integrity(tarball) + if actual_integrity != entry["integrity"]: + raise NpmPublishError( + f"npm package integrity mismatch for {filename}: " + f"expected {entry['integrity']}, got {actual_integrity}" + ) + + try: + with tarfile.open(tarball, mode="r:gz") as archive: + package_json_entry = archive.getmember("package/package.json") + if ( + not package_json_entry.isfile() + or package_json_entry.size > MAXIMUM_PACKAGE_JSON_SIZE + ): + raise NpmPublishError( + f"npm package has an invalid package/package.json: {filename}" + ) + package_json = archive.extractfile(package_json_entry) + if package_json is None: + raise NpmPublishError( + f"npm package is missing package/package.json: {filename}" + ) + with package_json: + identity = json.load(package_json) + except ( + KeyError, + OSError, + UnicodeDecodeError, + tarfile.TarError, + json.JSONDecodeError, + ) as error: + raise NpmPublishError( + f"could not read npm package identity from {filename}: {error}" + ) from error + if not isinstance(identity, dict): + raise NpmPublishError(f"npm package identity is invalid in {filename}") + if ( + identity.get("name") != entry["name"] + or identity.get("version") != entry["version"] + ): + raise NpmPublishError( + f"npm package identity mismatch for {filename}: expected " + f"{entry['name']}@{entry['version']}" + ) + return tarball, entry["name"], entry["version"] + + +def npm_view_integrity(name: str, version: str) -> str | None: + try: + result = subprocess.run( + ["npm", "view", f"{name}@{version}", "dist.integrity", "--json"], + check=False, + text=True, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + ) + except FileNotFoundError as error: + raise NpmPublishError("npm is required to publish npm packages") from error + if result.returncode != 0: + if "E404" in result.stderr or "E404" in result.stdout: + return None + detail = result.stderr.strip() or result.stdout.strip() + raise NpmPublishError(f"npm view failed for {name}@{version}: {detail}") + try: + integrity = json.loads(result.stdout) + except json.JSONDecodeError as error: + raise NpmPublishError( + f"npm view returned invalid JSON for {name}@{version}: {error}" + ) from error + if not isinstance(integrity, str) or not integrity: + raise NpmPublishError(f"npm view returned no integrity for {name}@{version}") + return integrity + + +def run_checked(command: list[str], description: str) -> None: + print("+ " + shlex.join(command), flush=True) + try: + subprocess.run(command, check=True) + except FileNotFoundError as error: + raise NpmPublishError("npm is required to publish npm packages") from error + except subprocess.CalledProcessError as error: + raise NpmPublishError( + f"{description} failed with exit code {error.returncode}" + ) from error + + +def publish_package( + entry: dict[str, Any], + package_dir: Path, + *, + dist_tag: str, + dry_run: bool, +) -> None: + tarball, name, version = validate_package_entry(entry, package_dir) + dist_tag = validate_dist_tag(dist_tag) + + if dry_run: + command = [ + "npm", + "publish", + str(tarball), + "--access", + "public", + "--tag", + dist_tag, + "--dry-run", + ] + run_checked(command, f"npm publish dry run for {name}@{version}") + return + + published_integrity = npm_view_integrity(name, version) + if published_integrity is not None: + if published_integrity != entry["integrity"]: + raise NpmPublishError( + f"refusing to reuse {name}@{version}: registry integrity differs" + ) + print(f"Already published with matching integrity: {name}@{version}") + else: + command = [ + "npm", + "publish", + str(tarball), + "--access", + "public", + "--tag", + dist_tag, + ] + run_checked(command, f"npm publish for {name}@{version}") + + +def publish_package_set( + manifest_path: Path, + *, + main_tag: str, + trusted_publishing: bool, + dry_run: bool, +) -> None: + if trusted_publishing and not dry_run: + require_trusted_publishing_toolchain() + + manifest = load_manifest(manifest_path) + package_dir = manifest_path.parent + package_name = manifest["package"] + version = manifest["version"] + + platform_entries: list[tuple[dict[str, Any], str]] = [] + for entry in manifest["platforms"]: + if not isinstance(entry, dict): + raise NpmPublishError("npm platform package entry must be an object") + if entry.get("name") != package_name: + raise NpmPublishError("npm platform package name does not match package set") + dist_tag = entry.get("distTag") + if not isinstance(dist_tag, str): + raise NpmPublishError("npm platform package is missing its dist-tag") + validate_dist_tag(dist_tag) + validate_package_entry(entry, package_dir) + platform_entries.append((entry, dist_tag)) + + platform_versions = [entry["version"] for entry, _ in platform_entries] + if len(platform_versions) != len(set(platform_versions)): + raise NpmPublishError("npm platform package versions must be unique") + + main = manifest["main"] + if main.get("name") != package_name or main.get("version") != version: + raise NpmPublishError("main npm package identity does not match package set") + validate_dist_tag(main_tag) + validate_package_entry(main, package_dir) + + for entry, dist_tag in platform_entries: + publish_package( + entry, + package_dir, + dist_tag=dist_tag, + dry_run=dry_run, + ) + + publish_package( + main, + package_dir, + dist_tag=main_tag, + dry_run=dry_run, + ) + + +def parse_args() -> argparse.Namespace: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("manifest", help="path to npm-packages.json") + parser.add_argument( + "--main-tag", + default="latest", + help="dist-tag for the launcher package (default: latest)", + ) + parser.add_argument( + "--trusted-publishing", + action="store_true", + help="require a supported Node/npm OIDC toolchain before publishing", + ) + parser.add_argument( + "--dry-run", + action="store_true", + help="validate every tarball with npm publish --dry-run", + ) + return parser.parse_args() + + +def main() -> int: + args = parse_args() + try: + publish_package_set( + Path(args.manifest).resolve(), + main_tag=args.main_tag, + trusted_publishing=args.trusted_publishing, + dry_run=args.dry_run, + ) + return 0 + except NpmPublishError as error: + print(f"npm publish error: {error}", file=sys.stderr) + return 1 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/scripts/release.py b/scripts/release.py index 615b349573..b229ac0a58 100755 --- a/scripts/release.py +++ b/scripts/release.py @@ -153,7 +153,14 @@ def strip_and_sign(binary: Path, target: str) -> tuple[int, int]: def binary_reported_version(binary: Path) -> str: - return run_checked([str(binary), "version"], capture_output=True).stdout.strip() + output = run_checked([str(binary), "--version"], capture_output=True).stdout.strip() + match = re.fullmatch(r"moli\s+(\S+)", output) + if match is None: + raise ReleaseError( + "packaged binary returned an unexpected `--version` response: " + f"{output or ''}" + ) + return match.group(1) def copy_release_materials(package_dir: Path) -> None: diff --git a/scripts/tests/__init__.py b/scripts/tests/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/scripts/tests/test_package_npm.py b/scripts/tests/test_package_npm.py new file mode 100644 index 0000000000..aa4947c979 --- /dev/null +++ b/scripts/tests/test_package_npm.py @@ -0,0 +1,233 @@ +from __future__ import annotations + +import io +import json +import subprocess +import sys +import tarfile +import tempfile +import tomllib +import unittest +import zipfile +from pathlib import Path + + +REPO_ROOT = Path(__file__).resolve().parents[2] +PACKAGE_SCRIPT = REPO_ROOT / "scripts" / "package_npm.py" +PUBLISH_SCRIPT = REPO_ROOT / "scripts" / "publish_npm.py" +PLATFORMS = json.loads( + (REPO_ROOT / "npm" / "platforms.json").read_text(encoding="utf-8") +) + + +def current_version() -> str: + with (REPO_ROOT / "moli" / "Cargo.toml").open("rb") as manifest_file: + return tomllib.load(manifest_file)["package"]["version"] + + +def add_tar_bytes(archive: tarfile.TarFile, name: str, value: bytes, mode: int) -> None: + member = tarfile.TarInfo(name) + member.size = len(value) + member.mode = mode + archive.addfile(member, io.BytesIO(value)) + + +def write_release_archive(directory: Path, platform: dict[str, object], version: str) -> None: + target = str(platform["target"]) + binary = str(platform["binary"]) + archive_path = directory / str(platform["archive"]) + package_root = f"moli-v{version}-{target}" + binary_contents = f"fixture binary for {target}\n".encode() + if archive_path.suffix == ".zip": + with zipfile.ZipFile(archive_path, mode="w") as archive: + archive.writestr(f"{package_root}/VERSION", f"{version}\n") + archive.writestr(f"{package_root}/{binary}", binary_contents) + return + + with tarfile.open(archive_path, mode="w:gz") as archive: + add_tar_bytes( + archive, + f"{package_root}/VERSION", + f"{version}\n".encode(), + 0o644, + ) + add_tar_bytes( + archive, + f"{package_root}/{binary}", + binary_contents, + 0o755, + ) + + +def read_tgz_json(path: Path, member: str) -> dict[str, object]: + with tarfile.open(path, mode="r:gz") as archive: + source = archive.extractfile(member) + if source is None: + raise AssertionError(f"missing {member} in {path}") + return json.load(source) + + +class PackageNpmTests(unittest.TestCase): + def test_builds_launcher_and_one_native_package_per_supported_platform(self) -> None: + version = current_version() + with tempfile.TemporaryDirectory(prefix="moli-npm-package-test-") as raw: + root = Path(raw) + input_dir = root / "release" + output_dir = root / "npm" + input_dir.mkdir() + for platform in PLATFORMS: + write_release_archive(input_dir, platform, version) + + result = subprocess.run( + [ + sys.executable, + str(PACKAGE_SCRIPT), + "--version", + version, + "--input-dir", + str(input_dir), + "--output-dir", + str(output_dir), + ], + cwd=REPO_ROOT, + check=False, + text=True, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + ) + self.assertEqual(result.returncode, 0, result.stderr) + + package_set = json.loads( + (output_dir / "npm-packages.json").read_text(encoding="utf-8") + ) + self.assertEqual(package_set["package"], "@lexmount/moli") + self.assertEqual(package_set["version"], version) + self.assertEqual(len(package_set["platforms"]), len(PLATFORMS)) + + main_tarball = output_dir / package_set["main"]["filename"] + main_manifest = read_tgz_json(main_tarball, "package/package.json") + self.assertEqual(main_manifest["version"], version) + self.assertEqual(main_manifest["bin"], {"moli": "bin/moli.js"}) + expected_aliases = { + package["alias"]: ( + f"npm:@lexmount/moli@{package['version']}" + ) + for package in package_set["platforms"] + } + self.assertEqual( + main_manifest["optionalDependencies"], expected_aliases + ) + with tarfile.open(main_tarball, mode="r:gz") as archive: + launcher = archive.getmember("package/bin/moli.js") + self.assertNotEqual(launcher.mode & 0o111, 0) + + platform_by_target = { + str(platform["target"]): platform for platform in PLATFORMS + } + for package in package_set["platforms"]: + platform = platform_by_target[package["target"]] + tarball = output_dir / package["filename"] + manifest = read_tgz_json(tarball, "package/package.json") + self.assertEqual(manifest["name"], "@lexmount/moli") + self.assertEqual(manifest["os"], [platform["platform"]]) + self.assertEqual(manifest["cpu"], [platform["arch"]]) + if "libc" in platform: + self.assertEqual(manifest["libc"], platform["libc"]) + + binary_member = ( + f"package/vendor/{platform['target']}/bin/{platform['binary']}" + ) + with tarfile.open(tarball, mode="r:gz") as archive: + member = archive.getmember(binary_member) + source = archive.extractfile(member) + self.assertIsNotNone(source) + assert source is not None + self.assertEqual( + source.read(), + f"fixture binary for {platform['target']}\n".encode(), + ) + if platform["platform"] != "win32": + self.assertNotEqual(member.mode & 0o111, 0) + + publish_dry_run = subprocess.run( + [ + sys.executable, + str(PUBLISH_SCRIPT), + str(output_dir / "npm-packages.json"), + "--main-tag", + "latest", + "--dry-run", + ], + cwd=REPO_ROOT, + check=False, + text=True, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + ) + self.assertEqual(publish_dry_run.returncode, 0, publish_dry_run.stderr) + + def test_rejects_an_incomplete_release_before_creating_output(self) -> None: + version = current_version() + with tempfile.TemporaryDirectory(prefix="moli-npm-package-test-") as raw: + root = Path(raw) + input_dir = root / "release" + output_dir = root / "npm" + input_dir.mkdir() + for platform in PLATFORMS[:-1]: + write_release_archive(input_dir, platform, version) + + result = subprocess.run( + [ + sys.executable, + str(PACKAGE_SCRIPT), + "--version", + version, + "--input-dir", + str(input_dir), + "--output-dir", + str(output_dir), + ], + cwd=REPO_ROOT, + check=False, + text=True, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + ) + self.assertNotEqual(result.returncode, 0) + self.assertIn("required release archive is missing", result.stderr) + self.assertFalse(output_dir.exists()) + + def test_refuses_to_overwrite_an_existing_output_path(self) -> None: + version = current_version() + with tempfile.TemporaryDirectory(prefix="moli-npm-package-test-") as raw: + root = Path(raw) + input_dir = root / "release" + output_dir = root / "npm" + input_dir.mkdir() + output_dir.mkdir() + for platform in PLATFORMS: + write_release_archive(input_dir, platform, version) + + result = subprocess.run( + [ + sys.executable, + str(PACKAGE_SCRIPT), + "--version", + version, + "--input-dir", + str(input_dir), + "--output-dir", + str(output_dir), + ], + cwd=REPO_ROOT, + check=False, + text=True, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + ) + self.assertNotEqual(result.returncode, 0) + self.assertIn("npm output path already exists", result.stderr) + + +if __name__ == "__main__": + unittest.main() diff --git a/scripts/tests/test_publish_npm.py b/scripts/tests/test_publish_npm.py new file mode 100644 index 0000000000..4fc1c78f0f --- /dev/null +++ b/scripts/tests/test_publish_npm.py @@ -0,0 +1,178 @@ +from __future__ import annotations + +import base64 +import hashlib +import json +import subprocess +import sys +import tarfile +import tempfile +import unittest +from io import BytesIO +from pathlib import Path +from unittest.mock import patch + + +SCRIPTS_DIR = Path(__file__).resolve().parents[1] +sys.path.insert(0, str(SCRIPTS_DIR)) + +import publish_npm # noqa: E402 + + +def integrity(value: bytes) -> str: + digest = hashlib.sha512(value).digest() + return "sha512-" + base64.b64encode(digest).decode("ascii") + + +def write_package(path: Path, name: str, version: str) -> None: + package_json = json.dumps({"name": name, "version": version}).encode() + member = tarfile.TarInfo("package/package.json") + member.size = len(package_json) + with tarfile.open(path, mode="w:gz") as archive: + archive.addfile(member, BytesIO(package_json)) + + +class PublishNpmTests(unittest.TestCase): + def test_publishes_all_platform_packages_before_the_launcher(self) -> None: + manifest = { + "schemaVersion": 1, + "package": "@lexmount/moli", + "version": "1.0.0", + "platforms": [ + { + "name": "@lexmount/moli", + "version": "1.0.0-linux-x64", + "filename": "linux.tgz", + "integrity": "unused", + "distTag": "linux-x64", + }, + { + "name": "@lexmount/moli", + "version": "1.0.0-darwin-arm64", + "filename": "darwin.tgz", + "integrity": "unused", + "distTag": "darwin-arm64", + }, + ], + "main": { + "name": "@lexmount/moli", + "version": "1.0.0", + "filename": "main.tgz", + "integrity": "unused", + }, + } + with tempfile.TemporaryDirectory(prefix="moli-npm-publish-test-") as raw: + package_dir = Path(raw) + for entry in [*manifest["platforms"], manifest["main"]]: + tarball = package_dir / entry["filename"] + write_package(tarball, entry["name"], entry["version"]) + entry["integrity"] = publish_npm.package_integrity(tarball) + manifest_path = package_dir / "npm-packages.json" + manifest_path.write_text(json.dumps(manifest), encoding="utf-8") + calls: list[tuple[str, str]] = [] + + def record(entry, _package_dir, *, dist_tag, dry_run): + self.assertFalse(dry_run) + calls.append((entry["version"], dist_tag)) + + with ( + patch.object( + publish_npm, "require_trusted_publishing_toolchain" + ) as require_toolchain, + patch.object(publish_npm, "publish_package", side_effect=record), + ): + publish_npm.publish_package_set( + manifest_path, + main_tag="latest", + trusted_publishing=True, + dry_run=False, + ) + require_toolchain.assert_called_once_with() + + self.assertEqual( + calls, + [ + ("1.0.0-linux-x64", "linux-x64"), + ("1.0.0-darwin-arm64", "darwin-arm64"), + ("1.0.0", "latest"), + ], + ) + + def test_rejects_changed_tarball_contents(self) -> None: + original = b"original package" + with tempfile.TemporaryDirectory(prefix="moli-npm-publish-test-") as raw: + package_dir = Path(raw) + tarball = package_dir / "package.tgz" + tarball.write_bytes(b"changed package") + entry = { + "name": "@lexmount/moli", + "version": "1.0.0", + "filename": tarball.name, + "integrity": integrity(original), + } + with self.assertRaisesRegex( + publish_npm.NpmPublishError, "integrity mismatch" + ): + publish_npm.validate_package_entry(entry, package_dir) + + def test_rejects_tarball_paths_outside_the_package_directory(self) -> None: + with tempfile.TemporaryDirectory(prefix="moli-npm-publish-test-") as raw: + entry = { + "name": "@lexmount/moli", + "version": "1.0.0", + "filename": "../package.tgz", + "integrity": "unused", + } + with self.assertRaisesRegex( + publish_npm.NpmPublishError, "must not contain a path" + ): + publish_npm.validate_package_entry(entry, Path(raw)) + + def test_rejects_package_identity_mismatch(self) -> None: + with tempfile.TemporaryDirectory(prefix="moli-npm-publish-test-") as raw: + package_dir = Path(raw) + tarball = package_dir / "package.tgz" + write_package(tarball, "@lexmount/not-moli", "1.0.0") + entry = { + "name": "@lexmount/moli", + "version": "1.0.0", + "filename": tarball.name, + "integrity": publish_npm.package_integrity(tarball), + } + with self.assertRaisesRegex( + publish_npm.NpmPublishError, "identity mismatch" + ): + publish_npm.validate_package_entry(entry, package_dir) + + def test_trusted_publishing_rejects_old_npm(self) -> None: + versions = {"node": (24, 15, 0), "npm": (11, 5, 0)} + with ( + patch.object(publish_npm, "tool_version", side_effect=versions.__getitem__), + self.assertRaisesRegex( + publish_npm.NpmPublishError, + r"npm trusted publishing requires npm >= 11\.5\.1; found 11\.5\.0", + ), + ): + publish_npm.require_trusted_publishing_toolchain() + + def test_trusted_publishing_rejects_old_node(self) -> None: + versions = {"node": (22, 13, 9), "npm": (11, 18, 0)} + with ( + patch.object(publish_npm, "tool_version", side_effect=versions.__getitem__), + self.assertRaisesRegex( + publish_npm.NpmPublishError, + r"npm trusted publishing requires node >= 22\.14\.0; found 22\.13\.9", + ), + ): + publish_npm.require_trusted_publishing_toolchain() + + def test_tool_version_accepts_node_prefix_and_prerelease(self) -> None: + completed = subprocess.CompletedProcess( + ["node", "--version"], 0, stdout="v24.15.0-rc.1\n", stderr="" + ) + with patch.object(publish_npm.subprocess, "run", return_value=completed): + self.assertEqual(publish_npm.tool_version("node"), (24, 15, 0)) + + +if __name__ == "__main__": + unittest.main() diff --git a/scripts/tests/test_release.py b/scripts/tests/test_release.py new file mode 100644 index 0000000000..83c620a990 --- /dev/null +++ b/scripts/tests/test_release.py @@ -0,0 +1,39 @@ +from __future__ import annotations + +import subprocess +import sys +import unittest +from pathlib import Path +from unittest.mock import patch + + +SCRIPTS_DIR = Path(__file__).resolve().parents[1] +sys.path.insert(0, str(SCRIPTS_DIR)) + +import release # noqa: E402 + + +class ReleaseVersionTests(unittest.TestCase): + def test_reads_standard_cli_version_flag(self) -> None: + completed = subprocess.CompletedProcess( + args=[], returncode=0, stdout="moli 1.2.3\n", stderr="" + ) + with patch.object(release, "run_checked", return_value=completed) as run: + version = release.binary_reported_version(Path("/tmp/moli")) + + self.assertEqual(version, "1.2.3") + run.assert_called_once_with(["/tmp/moli", "--version"], capture_output=True) + + def test_rejects_an_unexpected_version_response(self) -> None: + completed = subprocess.CompletedProcess( + args=[], returncode=0, stdout="1.2.3\n", stderr="" + ) + with patch.object(release, "run_checked", return_value=completed): + with self.assertRaisesRegex( + release.ReleaseError, "unexpected `--version` response" + ): + release.binary_reported_version(Path("/tmp/moli")) + + +if __name__ == "__main__": + unittest.main()