diff --git a/.github/workflows/build-codex-claude-workstation.yml b/.github/workflows/build-codex-claude-workstation.yml index 0bb6e70..de025b4 100644 --- a/.github/workflows/build-codex-claude-workstation.yml +++ b/.github/workflows/build-codex-claude-workstation.yml @@ -27,6 +27,8 @@ on: - 'scripts/evaluate-trivy-policy.py' - 'scripts/test-evaluate-trivy-policy.py' - 'scripts/publish-tested-image.py' + - 'scripts/registry_image.py' + - 'scripts/test-registry-image.py' - 'scripts/test-publish-tested-image.py' permissions: diff --git a/.github/workflows/build-gemini-skill-browser-linuxserver.yml b/.github/workflows/build-gemini-skill-browser-linuxserver.yml index 6be5bb5..ff40839 100644 --- a/.github/workflows/build-gemini-skill-browser-linuxserver.yml +++ b/.github/workflows/build-gemini-skill-browser-linuxserver.yml @@ -30,10 +30,13 @@ on: - '.github/workflows/build-gemini-skill-browser-linuxserver.yml' - '.github/workflows/release-gemini-browser.yml' - 'scripts/smoke-gemini-browser.py' + - 'scripts/test-smoke-gemini-browser.py' - 'scripts/trivy-image-gate.sh' - 'scripts/evaluate-trivy-policy.py' - 'scripts/test-evaluate-trivy-policy.py' - 'scripts/publish-tested-image.py' + - 'scripts/registry_image.py' + - 'scripts/test-registry-image.py' - 'scripts/test-publish-tested-image.py' permissions: diff --git a/.github/workflows/build-gemini-skill-browser.yml b/.github/workflows/build-gemini-skill-browser.yml index 23f139a..f247810 100644 --- a/.github/workflows/build-gemini-skill-browser.yml +++ b/.github/workflows/build-gemini-skill-browser.yml @@ -30,10 +30,13 @@ on: - '.github/workflows/build-gemini-skill-browser.yml' - '.github/workflows/release-gemini-browser.yml' - 'scripts/smoke-gemini-browser.py' + - 'scripts/test-smoke-gemini-browser.py' - 'scripts/trivy-image-gate.sh' - 'scripts/evaluate-trivy-policy.py' - 'scripts/test-evaluate-trivy-policy.py' - 'scripts/publish-tested-image.py' + - 'scripts/registry_image.py' + - 'scripts/test-registry-image.py' - 'scripts/test-publish-tested-image.py' permissions: diff --git a/.github/workflows/build-opencode-workstation.yml b/.github/workflows/build-opencode-workstation.yml index a0d1f34..b0ced36 100644 --- a/.github/workflows/build-opencode-workstation.yml +++ b/.github/workflows/build-opencode-workstation.yml @@ -27,6 +27,8 @@ on: - 'scripts/evaluate-trivy-policy.py' - 'scripts/test-evaluate-trivy-policy.py' - 'scripts/publish-tested-image.py' + - 'scripts/registry_image.py' + - 'scripts/test-registry-image.py' - 'scripts/test-publish-tested-image.py' permissions: diff --git a/.github/workflows/openclaw-upstream-docker.yml b/.github/workflows/openclaw-upstream-docker.yml index 8bccabe..afd21ae 100644 --- a/.github/workflows/openclaw-upstream-docker.yml +++ b/.github/workflows/openclaw-upstream-docker.yml @@ -17,6 +17,8 @@ on: - 'scripts/evaluate-trivy-policy.py' - 'scripts/test-evaluate-trivy-policy.py' - 'scripts/publish-tested-image.py' + - 'scripts/registry_image.py' + - 'scripts/test-registry-image.py' - 'scripts/test-publish-tested-image.py' - '.github/workflows/openclaw-upstream-docker.yml' @@ -51,7 +53,9 @@ jobs: run: | python3 scripts/test-resolve-openclaw-inputs.py bash scripts/test-apply-openclaw-runtime-hardening.sh + node --test scripts/test-openclaw-npm-bundle.mjs python3 scripts/test-evaluate-trivy-policy.py + python3 scripts/test-registry-image.py python3 scripts/test-publish-tested-image.py - name: Resolve immutable sources and security refresh id: inputs @@ -168,6 +172,10 @@ jobs: REPOSITORY: ${{ needs.prepare.outputs.repository }} DOCKERHUB_NAMESPACE: ${{ vars.DOCKERHUB_USERNAME || secrets.DOCKERHUB_USERNAME }} IMAGE_TAG: ${{ needs.prepare.outputs.image_tag }} + GHCR_USERNAME: ${{ github.actor }} + GHCR_TOKEN: ${{ github.token }} + DOCKERHUB_USERNAME: ${{ vars.DOCKERHUB_USERNAME || secrets.DOCKERHUB_USERNAME }} + DOCKERHUB_TOKEN: ${{ secrets.DOCKERHUB_TOKEN }} run: | set -euo pipefail repositories=("$REPOSITORY" "docker.io/$DOCKERHUB_NAMESPACE/openclaw-sandbox") diff --git a/.github/workflows/release-deepseek-harness.yml b/.github/workflows/release-deepseek-harness.yml index eb62347..f86bff7 100644 --- a/.github/workflows/release-deepseek-harness.yml +++ b/.github/workflows/release-deepseek-harness.yml @@ -60,6 +60,7 @@ jobs: - name: Check component pins and publication safety regressions run: | bash deepseek-harness-builder/scripts/check-component-pins.sh + python3 scripts/test-registry-image.py python3 deepseek-harness-builder/scripts/test-tested-image-artifact.py - name: Set up Node.js @@ -226,6 +227,10 @@ jobs: IMAGE_TAG: ${{ needs.prepare.outputs.image_tag }} LATEST_TAG: ${{ needs.prepare.outputs.latest_tag }} PUSH_LATEST: ${{ inputs.push_latest }} + GHCR_USERNAME: ${{ github.actor }} + GHCR_TOKEN: ${{ github.token }} + DOCKERHUB_USERNAME: ${{ env.DOCKERHUB_NAMESPACE }} + DOCKERHUB_TOKEN: ${{ secrets.DOCKERHUB_TOKEN }} run: | set -euo pipefail floating_args=() diff --git a/.github/workflows/release-gemini-browser.yml b/.github/workflows/release-gemini-browser.yml index 9c0b81e..90a59e6 100644 --- a/.github/workflows/release-gemini-browser.yml +++ b/.github/workflows/release-gemini-browser.yml @@ -69,6 +69,8 @@ jobs: - name: Validate input and publication contracts run: | python3 gemini-skill-browser-builder/scripts/test-resolve-browser-inputs.py + python3 scripts/test-smoke-gemini-browser.py + python3 scripts/test-registry-image.py python3 scripts/test-publish-tested-image.py python3 scripts/test-evaluate-trivy-policy.py - uses: actions/setup-node@v7 @@ -133,6 +135,8 @@ jobs: REPOSITORY: ${{ steps.params.outputs.repository }} IMAGE_TAG: ${{ steps.params.outputs.image_tag }} LATEST_TAG: ${{ steps.params.outputs.release_latest }} + GHCR_USERNAME: ${{ github.actor }} + GHCR_TOKEN: ${{ github.token }} run: | python3 scripts/publish-tested-image.py stage --image "$IMAGE" --image-id "$IMAGE" \ --variant "$VARIANT" --platform linux/amd64 --repository "$REPOSITORY" \ diff --git a/.github/workflows/release-workstations.yml b/.github/workflows/release-workstations.yml index b1dafe6..a44947f 100644 --- a/.github/workflows/release-workstations.yml +++ b/.github/workflows/release-workstations.yml @@ -83,6 +83,7 @@ jobs: env: VARIANT: ${{ inputs.variant }} run: | + python3 scripts/test-registry-image.py python3 scripts/test-publish-tested-image.py python3 scripts/test-evaluate-trivy-policy.py if [[ "$VARIANT" == codex ]]; then @@ -180,6 +181,8 @@ jobs: VARIANT: ${{ inputs.variant }} REPOSITORY: ${{ needs.prepare.outputs.repository }} ARCH: ${{ matrix.arch }} + GHCR_USERNAME: ${{ github.actor }} + GHCR_TOKEN: ${{ github.token }} run: | python3 scripts/publish-tested-image.py stage --image "$IMAGE" --image-id "$IMAGE" \ --variant "$VARIANT" --platform "linux/$ARCH" --repository "$REPOSITORY" \ diff --git a/.github/workflows/validate-deepseek-harness-component-pins.yml b/.github/workflows/validate-deepseek-harness-component-pins.yml index 17c3326..ea82378 100644 --- a/.github/workflows/validate-deepseek-harness-component-pins.yml +++ b/.github/workflows/validate-deepseek-harness-component-pins.yml @@ -12,6 +12,8 @@ on: - ".github/actions/verify-deepseek-harness/**" - "deepseek-harness-builder/**" - "scripts/trivy-image-gate.sh" + - "scripts/registry_image.py" + - "scripts/test-registry-image.py" - "scripts/evaluate-trivy-policy.py" - "scripts/test-evaluate-trivy-policy.py" - "SECURITY_SCAN.md" @@ -54,7 +56,9 @@ jobs: run: python3 scripts/test-evaluate-trivy-policy.py - name: Run tested image publication regression tests - run: python3 deepseek-harness-builder/scripts/test-tested-image-artifact.py + run: | + python3 scripts/test-registry-image.py + python3 deepseek-harness-builder/scripts/test-tested-image-artifact.py - name: Run DeepSeek Harness version resolver regression tests run: | diff --git a/.github/workflows/verify-deepseek-harness-pr.yml b/.github/workflows/verify-deepseek-harness-pr.yml index 0e999bd..f895c63 100644 --- a/.github/workflows/verify-deepseek-harness-pr.yml +++ b/.github/workflows/verify-deepseek-harness-pr.yml @@ -11,6 +11,8 @@ on: - ".github/actions/verify-deepseek-harness/**" - "deepseek-harness-builder/**" - "scripts/trivy-image-gate.sh" + - "scripts/registry_image.py" + - "scripts/test-registry-image.py" - "scripts/evaluate-trivy-policy.py" - "scripts/test-evaluate-trivy-policy.py" - "SECURITY_SCAN.md" diff --git a/codex-claude-workstation-builder/image/Dockerfile b/codex-claude-workstation-builder/image/Dockerfile index f1b57ab..b50ef65 100644 --- a/codex-claude-workstation-builder/image/Dockerfile +++ b/codex-claude-workstation-builder/image/Dockerfile @@ -341,6 +341,20 @@ RUN [[ "${CODE_SERVER_TAR_VERSION}" =~ ^[0-9]+\.[0-9]+\.[0-9]+$ ]] \ "${code_server_tar_dir}/package.json" "${CODE_SERVER_TAR_VERSION}" \ && find /tmp -maxdepth 1 -name "${tarball}" -delete +# Patch the HTTP client bundled by VS Code until code-server includes the fix. +ARG CODE_SERVER_UNDICI_VERSION=7.29.1 +RUN [[ "${CODE_SERVER_UNDICI_VERSION}" =~ ^[0-9]+[.][0-9]+[.][0-9]+$ ]] \ + && undici_dir="/usr/lib/code-server/lib/vscode/node_modules/undici" \ + && test "$(realpath "${undici_dir}")" = "${undici_dir}" \ + && test "$(node -p "require('${undici_dir}/package.json').name")" = undici \ + && tarball="$(npm pack --ignore-scripts --silent --pack-destination /tmp "undici@${CODE_SERVER_UNDICI_VERSION}")" \ + && [[ "${tarball}" == "undici-${CODE_SERVER_UNDICI_VERSION}.tgz" ]] \ + && find "${undici_dir}" -mindepth 1 -delete \ + && tar -xzf "/tmp/${tarball}" --strip-components=1 -C "${undici_dir}" \ + && node -e 'const [directory, expected] = process.argv.slice(1); if (require(`${directory}/package.json`).version !== expected) throw new Error("unexpected code-server undici version"); require(directory)' \ + "${undici_dir}" "${CODE_SERVER_UNDICI_VERSION}" \ + && rm -f "/tmp/${tarball}" + # ── npm globals and Corepack package managers ── # Claude's audited postinstall links its platform-native optional dependency. # Keep lifecycle scripts from all other global packages blocked by npm 12. diff --git a/codex-claude-workstation-builder/image/paseo-runtime/package-lock.json b/codex-claude-workstation-builder/image/paseo-runtime/package-lock.json index 7cd687b..a14aeaa 100644 --- a/codex-claude-workstation-builder/image/paseo-runtime/package-lock.json +++ b/codex-claude-workstation-builder/image/paseo-runtime/package-lock.json @@ -2056,9 +2056,9 @@ } }, "node_modules/fast-uri": { - "version": "3.1.6", - "resolved": "https://registry.npmjs.org/fast-uri/-/fast-uri-3.1.6.tgz", - "integrity": "sha512-7Ical1vFEMr0onbVzEDIreM22I4khW+fzyQPwvAFWBp1iwdshSZRsL4jjRvPG9JP1uiqMHRto+YU6R2/CzDz5Q==", + "version": "3.1.8", + "resolved": "https://registry.npmjs.org/fast-uri/-/fast-uri-3.1.8.tgz", + "integrity": "sha512-GZMtZUTNRpOVIECoXwLNZS5xUGE+mVNbTB8h/7Rwh2TFWcBQiPzTgyZi05BF9UMZKkLJv8XBRJTlU7zg8+ZfMg==", "funding": [ { "type": "github", diff --git a/codex-claude-workstation-builder/image/paseo-runtime/package.json b/codex-claude-workstation-builder/image/paseo-runtime/package.json index ef286a4..7c81739 100644 --- a/codex-claude-workstation-builder/image/paseo-runtime/package.json +++ b/codex-claude-workstation-builder/image/paseo-runtime/package.json @@ -8,6 +8,6 @@ "@getpaseo/cli": "0.3.1" }, "overrides": { - "fast-uri": "3.1.6" + "fast-uri": "3.1.8" } } diff --git a/codex-claude-workstation-builder/scripts/test-paseo-contract.sh b/codex-claude-workstation-builder/scripts/test-paseo-contract.sh index d00d725..4d14f70 100755 --- a/codex-claude-workstation-builder/scripts/test-paseo-contract.sh +++ b/codex-claude-workstation-builder/scripts/test-paseo-contract.sh @@ -112,11 +112,11 @@ if (root?.dependencies?.["@getpaseo/cli"] !== "0.3.1") { if (lock.packages?.["node_modules/@getpaseo/cli"]?.version !== "0.3.1") { throw new Error("package-lock does not resolve @getpaseo/cli 0.3.1"); } -if (packageJson.overrides?.["fast-uri"] !== "3.1.6") { - throw new Error("package.json must pin the fast-uri security override to 3.1.6"); +if (packageJson.overrides?.["fast-uri"] !== "3.1.8") { + throw new Error("package.json must pin the fast-uri security override to 3.1.8"); } -if (lock.packages?.["node_modules/fast-uri"]?.version !== "3.1.6") { - throw new Error("package-lock does not resolve fast-uri 3.1.6"); +if (lock.packages?.["node_modules/fast-uri"]?.version !== "3.1.8") { + throw new Error("package-lock does not resolve fast-uri 3.1.8"); } const installScripts = []; diff --git a/deepseek-harness-builder/README.md b/deepseek-harness-builder/README.md index 7f9f3ba..b2f42e2 100644 --- a/deepseek-harness-builder/README.md +++ b/deepseek-harness-builder/README.md @@ -11,11 +11,11 @@ One Dockerfile produces two independently tested variants under one image reposi | `latest` | `` | `runtime` | Lightweight 1Panel service with essential shell and repository tools. This remains the default final target. | | `workstation` | `-workstation` | `workstation` | Full interactive development environment with compiler and language toolchains. | -Both workflows publish the same tags to `ghcr.io/okxlin/deepseek-harness` and to `docker.io/$DOCKERHUB_USERNAME/deepseek-harness`. Configure `DOCKERHUB_USERNAME` as a GitHub Actions repository variable or secret and configure `DOCKERHUB_TOKEN` as a repository secret. The Docker Hub token is used only by the registry login action and is never passed to the image build. +Both workflows publish the same tags to `ghcr.io/okxlin/deepseek-harness` and to `docker.io/$DOCKERHUB_USERNAME/deepseek-harness`. Configure `DOCKERHUB_USERNAME` as a GitHub Actions repository variable or secret and configure `DOCKERHUB_TOKEN` as a repository secret. The Docker Hub token is used only for registry authentication and verified-image publication; it is never passed to the image build. Scheduled runs and manual runs with an empty version select the highest non-draft `dsh-v*` source release from DeepSeek Harness GitHub Releases, verify its tag, commit, and source-archive SHA-256, then publish matching `` and `-workstation` tags. This supports upstream releases that are available from GitHub before they are published to npm. An explicit `dsh-v*` selects that source release; an npm version or dist-tag still selects a published package release. `image/dsh-source.json` remains the reproducible local/PR baseline, while future source releases no longer require a Dockerfile edit. Manual workflow runs can also override the published image tag. Use a floating tag for an AppStore `latest` channel and the matching version tag for a numbered AppStore version. -The committed source baseline is [0.1.7-rc.2](https://github.com/deepseek-ai/deepseek-harness/releases/tag/dsh-v0.1.7-rc.2). This baseline includes the PTC package name `ptc-runtime` and workflow executor `workflow-ptc`; custom profiles using the old names need updating. DeepSeek now defaults to the Messages protocol. A manually configured old official API root should be removed or changed to `https://api.deepseek.com/anthropic`; custom provider URLs remain unchanged. +The committed source baseline is [0.2.0-rc.2](https://github.com/deepseek-ai/deepseek-harness/releases/tag/dsh-v0.2.0-rc.2). This baseline includes the PTC package name `ptc-runtime` and workflow executor `workflow-ptc`; custom profiles using the old names need updating. DeepSeek now defaults to the Messages protocol. A manually configured old official API root should be removed or changed to `https://api.deepseek.com/anthropic`; custom provider URLs remain unchanged. Build versions, base-image digests, and source/tool checksums live in [`image/components.lock.json`](image/components.lock.json). Python packages have a hash-locked [`image/python-requirements.lock`](image/python-requirements.lock); the legacy npm path retains its own [package manifest](image/package.json) and [pnpm lock](image/pnpm-lock.yaml). [`image/dsh-source.json`](image/dsh-source.json) pins the source release to a commit archive and checksum. Local builds and CI use `scripts/component-inputs.py` to resolve the same build arguments; ordinary component updates do not require editing Dockerfile instructions. Source builds currently install locally packed runtime tarballs with npm, and the dependency audit checks that installed tree. diff --git a/deepseek-harness-builder/README.zh-CN.md b/deepseek-harness-builder/README.zh-CN.md index 73c80ff..d83857d 100644 --- a/deepseek-harness-builder/README.zh-CN.md +++ b/deepseek-harness-builder/README.zh-CN.md @@ -11,11 +11,11 @@ | `latest` | `` | `runtime` | 轻量级 1Panel 服务镜像，包含必要 Shell 和仓库工具。它仍是默认最终 target。 | | `workstation` | `-workstation` | `workstation` | 完整交互式开发环境，包含编译器和语言工具链。 | -两个工作流都会把相同标签发布到 `ghcr.io/okxlin/deepseek-harness` 和 `docker.io/$DOCKERHUB_USERNAME/deepseek-harness`。请把 `DOCKERHUB_USERNAME` 配置为 GitHub Actions 仓库变量或 Secret，并把 `DOCKERHUB_TOKEN` 配置为仓库 Secret。Docker Hub token 只用于 Registry 登录，不会传入镜像构建上下文。 +两个工作流都会把相同标签发布到 `ghcr.io/okxlin/deepseek-harness` 和 `docker.io/$DOCKERHUB_USERNAME/deepseek-harness`。请把 `DOCKERHUB_USERNAME` 配置为 GitHub Actions 仓库变量或 Secret，并把 `DOCKERHUB_TOKEN` 配置为仓库 Secret。Docker Hub token 只用于 Registry 认证和已验证镜像的发布，不会传入镜像构建上下文。 定时任务以及留空版本的手动运行会从 DeepSeek Harness GitHub Releases 选择最高的非 draft `dsh-v*` 源码版本，校验对应 tag、commit 和源码归档 SHA-256 后构建，并发布匹配的 `` 和 `-workstation` 标签；因此上游源码版本可以在发布到 npm 之前进入镜像。手动显式传入 `dsh-v*` 可以选择指定的源码 release，传入已发布的 npm 版本或 dist-tag 时仍按请求的 npm selector 解析。`image/dsh-source.json` 保留可复现的本地/PR 基线，未来源码 release 不需要修改 Dockerfile。手动工作流也可覆盖发布标签。AppStore `latest` 通道使用浮动标签，编号 AppStore 版本使用匹配的版本标签。 -当前提交的源码基线是 [0.1.7-rc.2](https://github.com/deepseek-ai/deepseek-harness/releases/tag/dsh-v0.1.7-rc.2)。该基线包含 PTC 包名 `ptc-runtime` 和 workflow 执行器 `workflow-ptc`；使用旧名称的自定义 profile 需要更新。DeepSeek 现在默认使用 Messages 协议；手动配置的旧官方 API 根地址应删除或改为 `https://api.deepseek.com/anthropic`，自定义 provider URL 不受影响。 +当前提交的源码基线是 [0.2.0-rc.2](https://github.com/deepseek-ai/deepseek-harness/releases/tag/dsh-v0.2.0-rc.2)。该基线包含 PTC 包名 `ptc-runtime` 和 workflow 执行器 `workflow-ptc`；使用旧名称的自定义 profile 需要更新。DeepSeek 现在默认使用 Messages 协议；手动配置的旧官方 API 根地址应删除或改为 `https://api.deepseek.com/anthropic`，自定义 provider URL 不受影响。 构建版本、基础镜像 digest、源码和工具校验和集中保存在 [`image/components.lock.json`](image/components.lock.json)。Python 依赖使用独立的 [`image/python-requirements.lock`](image/python-requirements.lock)，旧版 npm 路径保留自己的 [package.json](image/package.json) 和 [pnpm-lock.yaml](image/pnpm-lock.yaml)。[`image/dsh-source.json`](image/dsh-source.json) 把源码归档绑定到确定的 commit 和 SHA-256。本地构建与 CI 都通过 `scripts/component-inputs.py` 解析参数，常规组件更新无需手改 Dockerfile 指令。源码构建当前仍由 npm 安装本地打包的 runtime tarball，依赖审计检查这棵实际安装树。 diff --git a/deepseek-harness-builder/image/components.lock.json b/deepseek-harness-builder/image/components.lock.json index 1432a29..34bc400 100644 --- a/deepseek-harness-builder/image/components.lock.json +++ b/deepseek-harness-builder/image/components.lock.json @@ -8,7 +8,7 @@ "CADDY_RATELIMIT_LICENSE_SHA256": "094bedda295c0702807d1027a00e1134f4cb71d523dc49f6d4b3caf589053068", "CADDY_RATELIMIT_REF": "5625512f24f6f59d6f64fb3aafe5eecff0b286db", "CADDY_SECURITY_LICENSE_SHA256": "c71d239df91726fc519c6eb72d318ec65820627232b2f796219e87dcf35d0ab4", - "CADDY_SECURITY_VERSION": "1.2.2", + "CADDY_SECURITY_VERSION": "1.3.0", "CADDY_SOURCE_ARCHIVE_SHA256": "2c3d02078286a6282cdb4d1d8744077788d556659dac0b64d8ed5886a7e5aeb9", "CADDY_VERSION": "2.11.4", "CEL_GO_LICENSE_SHA256": "4cdb9af102dfbb0ca03d87d6f650a505df098646a4080f4665b389ad9c6caa02", @@ -23,8 +23,8 @@ "DOCKER_COMPOSE_VERSION": "5.5.1", "DOCKER_VERSION": "29.8.1", "GO_AUTHCRUNCH_LICENSE_SHA256": "c71d239df91726fc519c6eb72d318ec65820627232b2f796219e87dcf35d0ab4", - "GO_AUTHCRUNCH_SOURCE_SHA256": "166d9743a7fa0979747341929472b680fd1d2ee88879b33f3fc775315579b699", - "GO_AUTHCRUNCH_VERSION": "1.3.6", + "GO_AUTHCRUNCH_SOURCE_SHA256": "b0c0ff46975fdbfff1eee591dfda20339a9170e133e41e269fa2b82b913ad84b", + "GO_AUTHCRUNCH_VERSION": "1.3.10", "GO_IMAGE": "golang:1.27.1-trixie@sha256:9baa6b4187bbb98d240372a8a235ac0bb6b5ddd52bba1431dc2f7c0705862728", "GO_VERSION": "1.27.1", "GRPC_VERSION": "1.84.0", @@ -34,8 +34,8 @@ "NODE_VERSION": "24.21.0", "NPM_ARCHIVE_SHA256": "d1a92f40e6c407b84c3a00c3cf978a10b24fd42f153c527e2016cef7bb34a483", "NPM_VERSION": "11.20.0", - "PNPM_ARCHIVE_SHA256": "05b7b921fbb31564505c967eabf825895a1cc18f50935c00be98815272cc9d56", - "PNPM_VERSION": "12.6.0", + "PNPM_ARCHIVE_SHA256": "90762b8105e833164186b011efe5cde13841693ac8eaf784d2974a6e1c01fba3", + "PNPM_VERSION": "12.8.1", "PYTEST_VERSION": "9.1.1", "PYTHON_IMAGE": "python:3.12.14-slim-trixie@sha256:2c941e860699f878900b0edc2403613c234d4b32eda3cc9fa7036991a2a63c4a", "PYTHON_VERSION": "3.12.14", @@ -45,15 +45,15 @@ "RUFF_VERSION": "0.16.9", "UV_LICENSE_APACHE_SHA256": "c71d239df91726fc519c6eb72d318ec65820627232b2f796219e87dcf35d0ab4", "UV_LICENSE_MIT_SHA256": "860e3d7a86b84e6a7012c7a635fc64df475cebc6cce34dfeb73a5982ec58176c", - "UV_SHA256_AMD64": "23bf5552d220e0842b65c862097b2ebaeba0064b74eda5e565e77fd25969d8c8", - "UV_SHA256_ARM64": "0804e9b164c64b6914182d5920c08551958a095986f10a3731056df701126436", - "UV_VERSION": "0.12.19", + "UV_SHA256_AMD64": "23f02075b652bb1df64178cfae41b5caf160822e720e2663568f3f5d63bc52c0", + "UV_SHA256_ARM64": "030b69227b40af8c1981b7301793dc66e71ed3c796ea8688209dd268bd91ec51", + "UV_VERSION": "0.12.21", "X_CRYPTO_VERSION": "0.57.0", "X_MOD_VERSION": "0.41.0", "X_TEXT_VERSION": "0.42.0", "YQ_LICENSE_SHA256": "697db34dabb21562fe84487a2ccd031fbd45382b89c2cbdec8ef31682c486040", - "YQ_SHA256_AMD64": "c5f056448f973ae7d39b5401949648a78f2dc1947d6a8eb65be60d5c504b9385", - "YQ_SHA256_ARM64": "88a1016bc1d657375a35864e4f44b6f333df8ff97b559f51bba0adcb2169df09", - "YQ_VERSION": "4.53.6" + "YQ_SHA256_AMD64": "8e34fc298390875de416e6a4afcb8cabeceb25d9aa8506c1a2f9353cf702ea5f", + "YQ_SHA256_ARM64": "189088da0c6429ec5178dfaab1a114805f6cab0b61b165ab236efedf1d57a71b", + "YQ_VERSION": "4.54.1" } } diff --git a/deepseek-harness-builder/image/dsh-source.json b/deepseek-harness-builder/image/dsh-source.json index 53b98cf..1b4c724 100644 --- a/deepseek-harness-builder/image/dsh-source.json +++ b/deepseek-harness-builder/image/dsh-source.json @@ -1,8 +1,8 @@ { - "version": "0.1.7-rc.2", + "version": "0.2.0-rc.2", "repository": "deepseek-ai/deepseek-harness", - "ref": "dsh-v0.1.7-rc.2", - "commit": "477b4f420553e8a52c2fbccc464d7561b239c443", - "archiveSha256": "a6b78577dec0653bf336f6aa093c55dd93cb3350f220f91efde951b18c7eda83", - "archiveUrl": "https://codeload.github.com/deepseek-ai/deepseek-harness/tar.gz/477b4f420553e8a52c2fbccc464d7561b239c443" + "ref": "dsh-v0.2.0-rc.2", + "commit": "639ed015397290b3745d163aafe02ffee4aa3f84", + "archiveSha256": "bbc09e888e1df3aa37be049abdb7720951e76442d21e5432365d87a465d628f1", + "archiveUrl": "https://codeload.github.com/deepseek-ai/deepseek-harness/tar.gz/639ed015397290b3745d163aafe02ffee4aa3f84" } diff --git a/deepseek-harness-builder/image/package.json b/deepseek-harness-builder/image/package.json index 268daac..9394c64 100644 --- a/deepseek-harness-builder/image/package.json +++ b/deepseek-harness-builder/image/package.json @@ -4,7 +4,7 @@ "private": true, "description": "Pinned runtime dependency closure for the DeepSeek Harness container image.", "license": "MIT", - "packageManager": "pnpm@12.6.0", + "packageManager": "pnpm@12.8.1", "engines": { "node": "24.21.0" }, diff --git a/deepseek-harness-builder/image/scripts/patch-dsh-telemetry.mjs b/deepseek-harness-builder/image/scripts/patch-dsh-telemetry.mjs index bb2364c..159dfa9 100644 --- a/deepseek-harness-builder/image/scripts/patch-dsh-telemetry.mjs +++ b/deepseek-harness-builder/image/scripts/patch-dsh-telemetry.mjs @@ -14,15 +14,13 @@ const baseRequire = createRequire(basePath) // These 0.1.6 request contributors are independent of the CLI's OTel switch. // Preserve the image's opt-out for every profile, including explicit overlays. -// https://github.com/deepseek-ai/deepseek-harness/tree/dsh-v0.1.7-rc.2/packages/session/session-log-deepseek +// https://github.com/deepseek-ai/deepseek-harness/tree/dsh-v0.2.0-rc.2/packages/session/session-log-deepseek const guards = [ - ['@deepseek-ai/dsh-session-log-deepseek', 'config.enabled !== true'], - ['@deepseek-ai/dsh-plugin-package-inventory-deepseek', 'config.enabled === false'], + ['@deepseek-ai/dsh-session-log-deepseek', ['config.enabled !== true', '!config.enabled.get()']], + ['@deepseek-ai/dsh-plugin-package-inventory-deepseek', ['config.enabled === false']], ] -for (const [name, guard] of guards) { +for (const [name, supportedGuards] of guards) { if (!Object.hasOwn(base.dependencies ?? {}, name)) continue // Older npm releases have neither contributor. - const original = `if (${guard})` - const replacement = `if (${guard} || process.env.DSH_TELEMETRY_DISABLED)` const target = baseRequire.resolve(name) const stat = await lstat(target) const canonical = await realpath(target) @@ -33,10 +31,24 @@ for (const [name, guard] of guards) { } const source = await readFile(target, 'utf8') const count = needle => source.split(needle).length - 1 - if (count(original) === 0 && count(replacement) === 1) continue - if (count(original) !== 1 || count(replacement) !== 0) { + const matches = supportedGuards.map(guard => [ + `if (${guard})`, `if (${guard} || process.env.DSH_TELEMETRY_DISABLED)`, + ]).filter(([original, replacement]) => count(original) + count(replacement) > 0) + if (matches.length !== 1) throw new Error(`unexpected telemetry activation guard in ${name}`) + const [original, replacement] = matches[0] + if (count(original) + count(replacement) !== 1) { throw new Error(`unexpected telemetry activation guard in ${name}`) } - await writeFile(target, source.replace(original, replacement), 'utf8') + let patched = source.replace(original, replacement) + // 0.2 registers first and reads the volatile switch inside prepare(). Keep the + // image opt-out at plugin activation as well as at request preparation. + if (original.includes('config.enabled.get()')) { + const entry = 'function apply(ctx, config) {' + const activation = `${entry}\n if (process.env.DSH_TELEMETRY_DISABLED) return;` + if (count(entry) !== 1) throw new Error(`unexpected telemetry plugin entry in ${name}`) + if (!patched.includes(activation)) patched = patched.replace(entry, activation) + } + if (patched === source) continue + await writeFile(target, patched, 'utf8') process.stdout.write(`[dsh-patch] ${name} honors DSH_TELEMETRY_DISABLED\n`) } diff --git a/deepseek-harness-builder/scripts/test-patch-dsh-telemetry.mjs b/deepseek-harness-builder/scripts/test-patch-dsh-telemetry.mjs index 89ed0fc..0ef9462 100644 --- a/deepseek-harness-builder/scripts/test-patch-dsh-telemetry.mjs +++ b/deepseek-harness-builder/scripts/test-patch-dsh-telemetry.mjs @@ -60,6 +60,40 @@ test('legacy releases without the request contributors remain supported', async assert.equal(run(root).status, 0) }) +test('0.2 volatile session-log config retains the telemetry opt-out', async t => { + const root = await fixture(t) + const target = join(root, 'node_modules', names[0], 'index.js') + await writeFile(target, `function apply(ctx, config) { + ctx.register({ prepare() { + if (!config.enabled.get()) return undefined; + return 'session log'; + } }); + } + export { apply };\n`) + assert.equal(run(root).status, 0) + assert.equal(run(root).status, 0, 'patch is idempotent') + const { apply } = await import(pathToFileURL(target)) + const previous = process.env.DSH_TELEMETRY_DISABLED + t.after(() => { + if (previous === undefined) delete process.env.DSH_TELEMETRY_DISABLED + else process.env.DSH_TELEMETRY_DISABLED = previous + }) + for (const disabled of ['', '1']) { + process.env.DSH_TELEMETRY_DISABLED = disabled + for (const enabled of [false, true]) { + const registrations = [] + apply({ register(value) { registrations.push(value) } }, { enabled: { get: () => enabled } }) + assert.equal(registrations.length, disabled ? 0 : 1, 'opt-out prevents registration') + if (registrations.length) { + assert.equal(registrations[0].prepare(), enabled ? 'session log' : undefined) + process.env.DSH_TELEMETRY_DISABLED = '1' + assert.equal(registrations[0].prepare(), undefined, 'request-time opt-out still applies') + process.env.DSH_TELEMETRY_DISABLED = disabled + } + } + } +}) + test('unknown activation code and missing declared packages fail closed', async t => { const root = await fixture(t) const target = join(root, 'node_modules', names[0], 'index.js') diff --git a/deepseek-harness-builder/scripts/test-tested-image-artifact.py b/deepseek-harness-builder/scripts/test-tested-image-artifact.py index 3565154..678f012 100644 --- a/deepseek-harness-builder/scripts/test-tested-image-artifact.py +++ b/deepseek-harness-builder/scripts/test-tested-image-artifact.py @@ -1,20 +1,19 @@ #!/usr/bin/env python3 """Publication must fail closed when a verified image or release input changes.""" -from argparse import Namespace import contextlib import hashlib import importlib.util import io import json -from pathlib import Path import subprocess import tarfile import tempfile import unittest +from argparse import Namespace +from pathlib import Path from unittest.mock import patch - SPEC = importlib.util.spec_from_file_location("artifacts", Path(__file__).with_name("tested-image-artifact.py")) artifacts = importlib.util.module_from_spec(SPEC) SPEC.loader.exec_module(artifacts) @@ -24,6 +23,25 @@ def encode(value): return json.dumps(value, separators=(",", ":")).encode() +class FakeRegistryClient: + def __init__(self, case, repository): + self.case = case + self.repository = repository + + def publish_platform_manifest(self, prepared): + if self.case.failed_repository == self.repository: + raise subprocess.CalledProcessError(1, "registry upload") + raw = prepared["path"].read_bytes() + manifest = json.loads(raw) + if self.case.corrupt_platform_repository == self.repository: + manifest["config"]["digest"] = "sha256:" + "e" * 64 + raw = encode(manifest) + digest = "sha256:" + hashlib.sha256(raw).hexdigest() + self.case.registry_calls.append((self.repository, digest, manifest)) + self.case.remote[f"{self.repository}@{digest}"] = raw + return digest + + class TestedImageArtifactTests(unittest.TestCase): def setUp(self): self.temporary = tempfile.TemporaryDirectory() @@ -38,15 +56,21 @@ class TestedImageArtifactTests(unittest.TestCase): self.configs = {} self.calls = [] self.remote = {} - self.tagged = {} + self.registry_calls = [] + self.failed_repository = None + self.corrupt_platform_repository = None self.write_artifact("amd64") self.write_artifact("arm64") def directory(self, arch): return self.root / f"deepseek-harness-{self.expected['variant']}-image-{arch}" - def write_artifact(self, arch, *, config_changes=None, docker_layout=False): + def write_artifact(self, arch, *, config_changes=None, docker_layout=False, with_layer=False): + layer_content = b"verified layer bytes" config = {"os": "linux", "architecture": arch, + "rootfs": {"type": "layers", "diff_ids": [ + "sha256:" + hashlib.sha256(layer_content).hexdigest() + ] if with_layer else []}, "config": {"Env": [f"DSH_IMAGE_VARIANT={self.expected['variant']}"], "Labels": { "org.opencontainers.image.revision": self.expected["revision"], "org.opencontainers.image.version": self.expected["dsh_version"]}}} @@ -57,7 +81,9 @@ class TestedImageArtifactTests(unittest.TestCase): image_id = "sha256:" + digest self.configs[image_id] = config config_name = f"{digest}.json" if docker_layout else f"blobs/sha256/{digest}" - manifest = [{"Config": config_name, "RepoTags": None, "Layers": []}] + layer_name = "layer.tar" + manifest = [{"Config": config_name, "RepoTags": None, + "Layers": [layer_name] if with_layer else []}] directory = self.directory(arch) directory.mkdir(exist_ok=True) with tarfile.open(directory / "image.tar.gz", "w:gz") as saved: @@ -65,6 +91,11 @@ class TestedImageArtifactTests(unittest.TestCase): member = tarfile.TarInfo(name) member.size = len(content) saved.addfile(member, io.BytesIO(content)) + if with_layer: + content = layer_content + member = tarfile.TarInfo(layer_name) + member.size = len(content) + saved.addfile(member, io.BytesIO(content)) receipt = {"schema_version": 1, **self.expected, "platform": f"linux/{arch}", "image_id": image_id, "archive_sha256": artifacts.sha256(directory / "image.tar.gz")} (directory / "receipt.json").write_text(json.dumps(receipt), encoding="utf-8") @@ -85,15 +116,6 @@ class TestedImageArtifactTests(unittest.TestCase): config = self.configs[image_id] return encode([{"Id": image_id, "Os": config["os"], "Architecture": config["architecture"], "Config": config["config"]}]) - if args[:2] == ("image", "tag"): - self.tagged[args[3]] = args[2] - return None - if args[:2] == ("image", "push"): - reference = args[2] - manifest = {"schemaVersion": 2, "mediaType": "application/vnd.oci.image.manifest.v1+json", - "config": {"digest": self.tagged[reference]}} - self.remote[reference] = encode(manifest) - return None if args[:4] == ("buildx", "imagetools", "inspect", "--raw"): return self.remote[args[4]] if args[:3] == ("buildx", "imagetools", "create"): @@ -113,31 +135,68 @@ class TestedImageArtifactTests(unittest.TestCase): self.fail(f"unexpected Docker operation (especially a rebuild): {args}") def publish(self, docker=None): - with patch.object(artifacts, "docker", docker or self.docker), contextlib.redirect_stdout(io.StringIO()): + with patch.object(artifacts, "docker", docker or self.docker), \ + patch.object(artifacts, "create_registry_client", + lambda repository: FakeRegistryClient(self, repository)), \ + contextlib.redirect_stdout(io.StringIO()): artifacts.publish_images(self.options, self.expected) def assert_rejected_without_registry_writes(self): with self.assertRaises((ValueError, tarfile.TarError)): self.publish() - self.assertFalse(any(call[:2] in (("image", "push"), ("image", "tag")) - or call[:3] == ("buildx", "imagetools", "create") for call in self.calls)) + self.assertFalse(self.registry_calls) + self.assertFalse(any(call[:3] == ("buildx", "imagetools", "create") for call in self.calls)) - def test_publishes_tested_config_ids_to_both_registries_before_floating_tags(self): + def test_publishes_tested_manifests_by_digest_before_normal_tags(self): self.publish() - pushes = [call for call in self.calls if call[:2] == ("image", "push")] - self.assertEqual(len(pushes), 4) + self.assertEqual(len(self.registry_calls), 4) first_manifest = next(i for i, call in enumerate(self.calls) if call[:3] == ("buildx", "imagetools", "create")) - self.assertEqual(sum(call[:2] == ("image", "push") for call in self.calls[:first_manifest]), 4) + self.assertTrue(all("ci-" not in str(call) for call in self.calls[:first_manifest])) creates = [call for call in self.calls if call[:3] == ("buildx", "imagetools", "create")] self.assertEqual([call[5].rsplit(":", 1)[1] for call in creates], ["0.1.5-rc.1", "0.1.5-rc.1", "latest", "latest"]) - self.assertEqual(set(self.tagged.values()), set(self.configs)) self.assertTrue(all("@sha256:" in source for call in creates for source in call[6:])) + self.assertTrue(all("ci-" not in source for call in creates for source in call[6:])) + self.assertEqual({call[2]["config"]["digest"] for call in self.registry_calls}, set(self.configs)) def test_supports_both_docker_archive_config_layouts(self): self.write_artifact("amd64", docker_layout=True) self.publish() + def test_prepares_uncompressed_layers_as_gzip_blobs(self): + receipt = self.write_artifact("amd64", with_layer=True) + with tempfile.TemporaryDirectory() as staging: + prepared = artifacts.prepare_platform_manifest( + self.directory("amd64"), receipt, Path(staging)) + self.assertEqual(len(prepared["manifest"]["layers"]), 1) + layer = prepared["manifest"]["layers"][0] + self.assertEqual(layer["mediaType"], "application/vnd.docker.image.rootfs.diff.tar.gzip") + self.assertEqual(prepared["blobs"][layer["digest"]].read_bytes()[:2], b"\x1f\x8b") + + def test_registry_client_uploads_blobs_and_manifest_by_digest_without_a_tag(self): + receipt = self.write_artifact("amd64", with_layer=True) + with tempfile.TemporaryDirectory() as staging: + prepared = artifacts.prepare_platform_manifest( + self.directory("amd64"), receipt, Path(staging)) + client = artifacts.RegistryClient("ghcr.io/okxlin/deepseek-harness", "user", "token") + client.token = "access-token" + requests = [] + + def send(method, url, headers, *, body=None, body_path=None): + requests.append((method, url, body, body_path)) + if method == "HEAD": + return 404, {} + if method == "POST": + return 202, {"location": "https://ghcr.io/v2/okxlin/deepseek-harness/blobs/uploads/1"} + return 201, {} + + with patch.object(client, "_send", side_effect=send): + self.assertEqual(client.publish_platform_manifest(prepared), prepared["digest"]) + self.assertEqual(sum(method == "PUT" for method, *_ in requests), 3) + manifest_put = next(url for method, url, *_ in requests if method == "PUT" and "/manifests/" in url) + self.assertIn("/manifests/sha256:", manifest_put) + self.assertNotIn(":ci-", manifest_put) + def test_workstation_arm_only_publishes_only_the_requested_platform(self): self.root = self.root / "arm-only" self.root.mkdir() @@ -148,8 +207,8 @@ class TestedImageArtifactTests(unittest.TestCase): self.options.latest_tag = "workstation" receipt = self.write_artifact("arm64") self.publish() - self.assertEqual(set(self.tagged.values()), {receipt["image_id"]}) - self.assertEqual(len(self.tagged), 2) + self.assertEqual({call[2]["config"]["digest"] for call in self.registry_calls}, {receipt["image_id"]}) + self.assertEqual(len(self.registry_calls), 2) def test_dry_run_checks_artifacts_without_docker_or_registry_access(self): self.options.dry_run = True @@ -212,24 +271,15 @@ class TestedImageArtifactTests(unittest.TestCase): self.assertFalse(any(call[:2] == ("image", "push") for call in self.calls)) def test_registry_push_failure_never_updates_release_tags(self): - def failed_push(*args, **kwargs): - if args[:2] == ("image", "push") and args[2].startswith("docker.io/"): - raise subprocess.CalledProcessError(1, "docker push") - return self.docker(*args, **kwargs) + self.failed_repository = "docker.io/okxlin/deepseek-harness" with self.assertRaises(subprocess.CalledProcessError): - self.publish(failed_push) + self.publish() self.assertFalse(any(call[:3] == ("buildx", "imagetools", "create") for call in self.calls)) def test_registry_config_mismatch_never_updates_release_tags(self): - def wrong_config(*args, **kwargs): - result = self.docker(*args, **kwargs) - if args[:4] == ("buildx", "imagetools", "inspect", "--raw"): - manifest = json.loads(result) - manifest["config"]["digest"] = "sha256:" + "e" * 64 - return encode(manifest) - return result - with self.assertRaisesRegex(ValueError, "pushed image does not match"): - self.publish(wrong_config) + self.corrupt_platform_repository = "ghcr.io/okxlin/deepseek-harness" + with self.assertRaisesRegex(ValueError, "unexpected platform digest|published image digest changed"): + self.publish() self.assertFalse(any(call[:3] == ("buildx", "imagetools", "create") for call in self.calls)) def test_incomplete_version_manifest_never_updates_floating_tags(self): diff --git a/deepseek-harness-builder/scripts/tested-image-artifact.py b/deepseek-harness-builder/scripts/tested-image-artifact.py index 81520d8..82736a6 100644 --- a/deepseek-harness-builder/scripts/tested-image-artifact.py +++ b/deepseek-harness-builder/scripts/tested-image-artifact.py @@ -2,27 +2,33 @@ """Export verified images and publish those exact images without rebuilding.""" import argparse -import gzip import hashlib import json -from pathlib import Path import re import shutil import subprocess import sys import tarfile +import tempfile +from pathlib import Path +LIBRARY = Path(__file__).resolve().parents[2] / "scripts" +if str(LIBRARY) not in sys.path: + sys.path.insert(0, str(LIBRARY)) +from registry_image import ( + DIGEST, + IMAGE_TYPES, + INDEX_TYPES, + RegistryClient, # noqa: F401 - re-exported for the focused Registry V2 test + archive_metadata, + create_registry_client, + save_image, +) +from registry_image import ( + prepare_platform_manifest as prepare_image_manifest, +) -DIGEST = re.compile(r"sha256:[a-f0-9]{64}") PLATFORMS = ("linux/amd64", "linux/arm64") -IMAGE_TYPES = { - "application/vnd.oci.image.manifest.v1+json", - "application/vnd.docker.distribution.manifest.v2+json", -} -INDEX_TYPES = { - "application/vnd.oci.image.index.v1+json", - "application/vnd.docker.distribution.manifest.list.v2+json", -} def require(condition, message): @@ -80,23 +86,7 @@ def check_archive(directory, expected): require(sha256(archive) == receipt.get("archive_sha256"), f"archive checksum mismatch: {archive}") # Read metadata without extracting any archive paths into the host filesystem. - digest = receipt["image_id"].removeprefix("sha256:") - config_names = {f"{digest}.json", f"blobs/sha256/{digest}"} - metadata = {} - with tarfile.open(archive, "r|gz") as saved: - for member in saved: - if member.name not in config_names | {"manifest.json"}: - continue - require(member.isfile() and member.size <= 4 * 1024 * 1024, - "invalid image archive metadata member") - require(member.name not in metadata, "duplicate image archive metadata") - metadata[member.name] = saved.extractfile(member).read() - manifest = json.loads(metadata.get("manifest.json", b"null")) - require(isinstance(manifest, list) and len(manifest) == 1, "archive must contain exactly one image") - config_name = manifest[0].get("Config") - require(config_name in config_names and config_name in metadata, "archive config does not match tested image ID") - config_bytes = metadata[config_name] - require(hashlib.sha256(config_bytes).hexdigest() == digest, "archive image config digest mismatch") + config_bytes, _ = archive_metadata(archive, receipt["image_id"]) check_config(json.loads(config_bytes), receipt) return receipt @@ -116,11 +106,7 @@ def export_image(options, expected): flush=True) archive = directory / "image.tar.gz" # Saving by config ID binds the archive to the tested image, even if a tag moves. - with archive.open("xb") as target: - with subprocess.Popen(["docker", "image", "save", options.image_id], stdout=subprocess.PIPE) as saved: - with gzip.GzipFile(filename="", mode="wb", fileobj=target, compresslevel=1, mtime=0) as zipped: - shutil.copyfileobj(saved.stdout, zipped) - require(saved.wait() == 0, "docker image save failed") + save_image(options.image_id, archive) receipt = {"schema_version": 1, **expected, "archive_sha256": sha256(archive)} (directory / "receipt.json").write_text(json.dumps(receipt, indent=2) + "\n", encoding="utf-8") print(json.dumps(receipt, indent=2)) @@ -132,6 +118,11 @@ def remote_manifest(reference): return json.loads(raw), "sha256:" + hashlib.sha256(raw).hexdigest() +def prepare_platform_manifest(directory, receipt, destination): + """Convert one verified Docker-save archive into a digest-addressed manifest.""" + return prepare_image_manifest(directory / "image.tar.gz", receipt["image_id"], destination) + + def check_index(reference, expected_digests): manifest, digest = remote_manifest(reference) require(manifest.get("schemaVersion") == 2 and manifest.get("mediaType") in INDEX_TYPES, @@ -172,34 +163,41 @@ def publish_images(options, expected): return # Validate every archive and loaded image before making the first registry write. - for directory, receipt in artifacts: - docker("image", "load", "--input", directory / "image.tar.gz") - inspect_image(receipt["image_id"], receipt) + with tempfile.TemporaryDirectory(prefix="deepseek-harness-manifests-") as staging: + prepared = [] + for directory, receipt in artifacts: + prepared.append((receipt, prepare_platform_manifest( + directory, receipt, Path(staging) / receipt["platform"].split("/")[1]))) + for directory, receipt in artifacts: + docker("image", "load", "--input", directory / "image.tar.gz") + inspect_image(receipt["image_id"], receipt) - staged = {} - for repository in options.repository: - staged[repository] = {} - for _, receipt in artifacts: - arch = receipt["platform"].split("/")[1] - reference = f"{repository}:ci-{expected['run_id']}-{options.run_attempt}-{expected['variant']}-{arch}" - docker("image", "tag", receipt["image_id"], reference) - docker("image", "push", reference) - manifest, digest = remote_manifest(reference) - require(manifest.get("schemaVersion") == 2 and manifest.get("mediaType") in IMAGE_TYPES - and manifest.get("config", {}).get("digest") == receipt["image_id"], - f"pushed image does not match tested image ID: {reference}") - staged[repository][receipt["platform"]] = digest + clients = {repository: create_registry_client(repository) for repository in options.repository} + staged = {} + for repository, client in clients.items(): + staged[repository] = {} + for receipt, platform_manifest in prepared: + digest = client.publish_platform_manifest(platform_manifest) + require(digest == platform_manifest["digest"], + f"registry returned an unexpected platform digest: {repository}") + reference = f"{repository}@{digest}" + manifest, remote_digest = remote_manifest(reference) + require(manifest.get("schemaVersion") == 2 and manifest.get("mediaType") in IMAGE_TYPES + and manifest.get("config", {}).get("digest") == receipt["image_id"], + f"published image does not match tested image ID: {reference}") + require(remote_digest == digest, f"published image digest changed: {reference}") + staged[repository][receipt["platform"]] = digest - # Both registries must have all images before updating release tags. Floating - # tags follow only after the version tag has been verified in both registries. - published = [] - for tag in tags: - for repository, digests in staged.items(): - reference = f"{repository}:{tag}" - docker("buildx", "imagetools", "create", "--prefer-index=true", "--tag", reference, - *(f"{repository}@{digest}" for digest in digests.values())) - digest = check_index(reference, digests) - published.append(f"{reference} -> {digest}") + # Both registries must have all images before updating release tags. Floating + # tags follow only after the version tag has been verified in both registries. + published = [] + for tag in tags: + for repository, digests in staged.items(): + reference = f"{repository}:{tag}" + docker("buildx", "imagetools", "create", "--prefer-index=true", "--tag", reference, + *(f"{repository}@{digest}" for digest in digests.values())) + digest = check_index(reference, digests) + published.append(f"{reference} -> {digest}") for item in published: print(item) if options.summary: @@ -239,7 +237,8 @@ def main(): else: publish_images(options, expected) return 0 - except (OSError, ValueError, KeyError, TypeError, AttributeError, tarfile.TarError, subprocess.SubprocessError) as exc: + except (OSError, ValueError, KeyError, TypeError, AttributeError, RuntimeError, + tarfile.TarError, subprocess.SubprocessError) as exc: print(f"ERROR: {exc}", file=sys.stderr) return 1 diff --git a/gemini-skill-browser-builder/image/runtime/package-lock.json b/gemini-skill-browser-builder/image/runtime/package-lock.json index f30c240..95e9623 100644 --- a/gemini-skill-browser-builder/image/runtime/package-lock.json +++ b/gemini-skill-browser-builder/image/runtime/package-lock.json @@ -792,9 +792,9 @@ } }, "node_modules/brace-expansion": { - "version": "1.1.18", - "resolved": "https://registry.npmjs.org/brace-expansion/-/brace-expansion-1.1.18.tgz", - "integrity": "sha512-Edep/X9fGqVNmzKBVsDYIOtD+z1tuezV70LBjdCst9Tqu76lsnvRiZ6oTic1n+/BIwX6QDGAO94PN4N2SADvtw==", + "version": "1.1.21", + "resolved": "https://registry.npmjs.org/brace-expansion/-/brace-expansion-1.1.21.tgz", + "integrity": "sha512-9zeA+KLZNNzglF2TPKRQEDyx6Yby7daAkuy8MiPzpXPsYDWi/DRM8jmwUDxokQjYqBpv5DgPiwD4h4ZZSy1Ujw==", "license": "MIT", "dependencies": { "balanced-match": "^1.0.0", @@ -1217,9 +1217,9 @@ "license": "MIT" }, "node_modules/fast-uri": { - "version": "3.1.7", - "resolved": "https://registry.npmjs.org/fast-uri/-/fast-uri-3.1.7.tgz", - "integrity": "sha512-dOvZVzjdZdz7phd9v6jCbwxrBW3fK6n8Rc0CtdmM4bumzMnxywBYhuph6J819RRw/ku+rLbelwfMunktuzVVHg==", + "version": "3.1.8", + "resolved": "https://registry.npmjs.org/fast-uri/-/fast-uri-3.1.8.tgz", + "integrity": "sha512-GZMtZUTNRpOVIECoXwLNZS5xUGE+mVNbTB8h/7Rwh2TFWcBQiPzTgyZi05BF9UMZKkLJv8XBRJTlU7zg8+ZfMg==", "funding": [ { "type": "github", @@ -1505,9 +1505,9 @@ "license": "ISC" }, "node_modules/ip-address": { - "version": "10.7.0", - "resolved": "https://registry.npmjs.org/ip-address/-/ip-address-10.7.0.tgz", - "integrity": "sha512-BGFsyJd5mpXp3rK6jIdADLNgpJUK1jnjzvYF8lK+VyDab9JAmqN0YOKDdP17HlgKb2+ehPgDc8EtnRLbGCAMhA==", + "version": "10.7.2", + "resolved": "https://registry.npmjs.org/ip-address/-/ip-address-10.7.2.tgz", + "integrity": "sha512-7H/2gFSIitxc0hG3nOI1glS8QLo/EHBFFLk8vEUjXY/xu0AdL8jZ9U1IzO2PUm0d2D/ofQcAifb0g6OBkt8U7w==", "license": "MIT", "engines": { "node": ">= 12" diff --git a/openclaw-builder/configs/components.json b/openclaw-builder/configs/components.json index fadaa69..db89466 100644 --- a/openclaw-builder/configs/components.json +++ b/openclaw-builder/configs/components.json @@ -18,5 +18,19 @@ "sha512": "5e57abc8c85bba4b1d2ada1f9d667d4a4fb80944db9ace0c6ff114df6499c13a00c99e51815a2cfe48bc9fecabd0b5873862ae6a46d74eeb983472d08e175d82" }, "test_daemon_image": "docker.io/library/docker:29-dind@sha256:5efed980cba3fc126cf54e21a5a6ff8849d05b6e0623d6e7612f48e9cd6cd17e", - "test_sandbox_image": "docker.io/library/busybox:1.37@sha256:9db7b59979c38555a39def84a31fb98b5296952f9e3afd4f6f11f05b07adfab0" + "test_sandbox_image": "docker.io/library/busybox:1.37@sha256:9db7b59979c38555a39def84a31fb98b5296952f9e3afd4f6f11f05b07adfab0", + "npm_bundled": { + "brace-expansion": { + "from": "5.0.9", + "version": "5.0.12", + "url": "https://registry.npmjs.org/brace-expansion/-/brace-expansion-5.0.12.tgz", + "sha512": "628bd0debce168b308ac38cd0cc90d4b4d6d79af77aa11211b9c72f1febe47497e770dca8bee6c0a822823de368ae2d94c75a881692d830543854d3d88d12299" + }, + "undici": { + "from": "6.28.0", + "version": "6.28.1", + "url": "https://registry.npmjs.org/undici/-/undici-6.28.1.tgz", + "sha512": "cd6a5d4d50f9e07e3c088c9b2f4ad6437ba4a5bf5ddb7c0cede1f946d7dd99e3fbd1c587363b5fa3b3f8bd95fee42a0370ee772783eee6e5e9ee535f8dce8344" + } + } } diff --git a/openclaw-builder/image/patch-vendored-deps.mjs b/openclaw-builder/image/patch-vendored-deps.mjs index fae7d02..56504a5 100644 --- a/openclaw-builder/image/patch-vendored-deps.mjs +++ b/openclaw-builder/image/patch-vendored-deps.mjs @@ -34,8 +34,68 @@ export function verifyArchiveReader(directory) { assert.equal(oversized, false, 'archive reader trusted the attacker-declared size'); } +export async function patchNpmBundles(root, pins) { + root = fs.realpathSync(root); + const store = path.join(root, '.pnpm'); + for (const [name, pin] of Object.entries(pins)) { + assert.ok(['brace-expansion', 'undici'].includes(name)); + assert.match(pin.version, /^\d+\.\d+\.\d+$/); + assert.equal(pin.url, `https://registry.npmjs.org/${name}/-/${name}-${pin.version}.tgz`); + assert.match(pin.sha512, /^[a-f0-9]{128}$/); + const outdated = []; + for (const entry of fs.readdirSync(store)) { + if (!/^npm@\d/.test(entry)) continue; + const directory = path.join(store, entry, 'node_modules/npm/node_modules', name); + if (!fs.existsSync(directory)) continue; + assert.equal(fs.realpathSync(directory), directory, 'unexpected bundled npm dependency path'); + const pkg = JSON.parse(fs.readFileSync(path.join(directory, 'package.json'))); + assert.equal(pkg.name, name); + if (pkg.version === pin.from) outdated.push(directory); + else { + assert.match(pkg.version, /^\d+\.\d+\.\d+$/); + const fixed = pin.version.split('.').map(Number); + assert.ok(pkg.version.split('.').map(Number).reduce((order, value, index) => + order || value - fixed[index], 0) >= 0, `unexpected vulnerable ${name} version`); + } + } + if (!outdated.length) continue; + const temporary = fs.mkdtempSync('/tmp/openclaw-npm-bundle-'); + try { + const response = await fetch(pin.url, {redirect: 'error', signal: AbortSignal.timeout(60000)}); + assert.equal(response.status, 200); + const chunks = []; + let size = 0; + for await (const chunk of response.body) { + size += chunk.length; + assert.ok(size <= 4 * 1024 * 1024, 'npm dependency archive exceeds expected size'); + chunks.push(chunk); + } + const bytes = Buffer.concat(chunks); + assert.equal(createHash('sha512').update(bytes).digest('hex'), pin.sha512); + const archive = path.join(temporary, 'dependency.tgz'); + fs.writeFileSync(archive, bytes); + execFileSync('tar', ['-xzf', archive, '--no-same-owner', '--no-same-permissions', '-C', temporary]); + const replacement = path.join(temporary, 'package'); + const pkg = JSON.parse(fs.readFileSync(path.join(replacement, 'package.json'))); + assert.equal(pkg.name, name); + assert.equal(pkg.version, pin.version); + for (const directory of outdated) { + const {uid, gid} = fs.statSync(directory); + fs.rmSync(directory, {recursive: true}); + fs.cpSync(replacement, directory, {recursive: true}); + execFileSync('chown', ['-R', `${uid}:${gid}`, directory]); + console.log(`Replace verified bundled npm dependency ${name} ${pin.from} -> ${pin.version}: ${directory}`); + } + } finally { + fs.rmSync(temporary, {recursive: true}); + } + } +} + async function main() { - const pin = JSON.parse(fs.readFileSync(new URL('./components.json', import.meta.url))).adm_zip; + const components = JSON.parse(fs.readFileSync(new URL('./components.json', import.meta.url))); + await patchNpmBundles('/app/node_modules', components.npm_bundled); + const pin = components.adm_zip; assert.match(pin.version, /^\d+\.\d+\.\d+$/); assert.equal(pin.url, `https://registry.npmjs.org/adm-zip/-/adm-zip-${pin.version}.tgz`); assert.match(pin.sha512, /^[a-f0-9]{128}$/); diff --git a/scripts/publish-tested-image.py b/scripts/publish-tested-image.py index e599176..ae3dd88 100644 --- a/scripts/publish-tested-image.py +++ b/scripts/publish-tested-image.py @@ -3,10 +3,20 @@ import argparse import hashlib import json -from pathlib import Path import re import subprocess import sys +import tempfile +from pathlib import Path + +LIBRARY = Path(__file__).resolve().parent +if str(LIBRARY) not in sys.path: + sys.path.insert(0, str(LIBRARY)) +from registry_image import ( + create_registry_client, + prepare_platform_manifest, + save_image, +) REPOSITORIES = { "codex": "codex-claude-workstation", "opencode": "opencode-workstation", @@ -59,12 +69,15 @@ def stage(args, expected): receipt_path = args.receipt require(not receipt_path.exists() and not receipt_path.is_symlink(), "receipt must be a new file") receipt_path.parent.mkdir(parents=True, exist_ok=True) - reference = f"{args.repository}:ci-{args.run_id}-{args.run_attempt}-{args.variant}-{args.platform.split('/')[1]}" - # Tag the recorded ID, never a mutable local tag. - docker("image", "tag", args.image_id, reference) - docker("image", "push", reference) - manifest, digest = remote_manifest(reference) - check_manifest(manifest, args.image_id) + with tempfile.TemporaryDirectory(prefix="tested-image-") as staging: + archive = Path(staging) / "image.tar.gz" + save_image(args.image_id, archive) + prepared = prepare_platform_manifest(archive, args.image_id, Path(staging) / "manifest") + digest = create_registry_client(args.repository).publish_platform_manifest(prepared) + reference = f"{args.repository}@{digest}" + manifest, remote_digest = remote_manifest(reference) + require(remote_digest == digest, "registry manifest digest mismatch after publication") + check_manifest(manifest, args.image_id) receipt = {"schema_version": 1, **expected, "platform": args.platform, "image_id": args.image_id, "manifest_digest": digest} with receipt_path.open("x", encoding="utf-8") as output: @@ -163,6 +176,7 @@ def main(): if __name__ == "__main__": try: main() - except (ValueError, OSError, subprocess.CalledProcessError) as error: + except (ValueError, OSError, KeyError, TypeError, AttributeError, RuntimeError, + subprocess.CalledProcessError) as error: print(f"ERROR: {error}", file=sys.stderr) sys.exit(1) diff --git a/scripts/registry_image.py b/scripts/registry_image.py new file mode 100644 index 0000000..ed5e7d9 --- /dev/null +++ b/scripts/registry_image.py @@ -0,0 +1,318 @@ +"""Publish verified Docker-save images without creating staging tags.""" + +import base64 +import gzip +import hashlib +import http.client +import json +import os +import re +import shutil +import subprocess +import tarfile +import urllib.error +import urllib.parse +import urllib.request + +DIGEST = re.compile(r"sha256:[a-f0-9]{64}") +IMAGE_TYPES = { + "application/vnd.oci.image.manifest.v1+json", + "application/vnd.docker.distribution.manifest.v2+json", +} +INDEX_TYPES = { + "application/vnd.oci.image.index.v1+json", + "application/vnd.docker.distribution.manifest.list.v2+json", +} +REGISTRY_CONFIG = { + "ghcr.io": { + "registry_host": "ghcr.io", + "token_url": "https://ghcr.io/token", + "service": "ghcr.io", + "upload_hosts": {"ghcr.io"}, + }, + "docker.io": { + "registry_host": "registry-1.docker.io", + "token_url": "https://auth.docker.io/token", + "service": "registry.docker.io", + "upload_hosts": {"registry-1.docker.io", "docker.io"}, + }, +} +BLOB_CHUNK_SIZE = 1024 * 1024 + + +def require(condition, message): + if not condition: + raise ValueError(message) + + +def sha256(path): + with path.open("rb") as source: + return hashlib.file_digest(source, "sha256").hexdigest() + + +def save_image(image_id, archive): + """Save one local image as the gzip-wrapped Docker archive format.""" + with archive.open("xb") as target, subprocess.Popen( + ["docker", "image", "save", image_id], stdout=subprocess.PIPE + ) as saved, gzip.GzipFile(filename="", mode="wb", fileobj=target, + compresslevel=1, mtime=0) as zipped: + shutil.copyfileobj(saved.stdout, zipped, BLOB_CHUNK_SIZE) + require(saved.wait() == 0, "docker image save failed") + + +def archive_metadata(archive, image_id): + digest = image_id.removeprefix("sha256:") + config_names = {f"{digest}.json", f"blobs/sha256/{digest}"} + metadata = {} + with tarfile.open(archive, "r|gz") as saved: + for member in saved: + if member.name not in config_names | {"manifest.json"}: + continue + require(member.isfile() and member.size <= 4 * 1024 * 1024, + "invalid image archive metadata member") + require(member.name not in metadata, "duplicate image archive metadata") + source = saved.extractfile(member) + require(source is not None, f"cannot read image archive metadata: {member.name}") + with source: + metadata[member.name] = source.read() + manifest = json.loads(metadata.get("manifest.json", b"null")) + require(isinstance(manifest, list) and len(manifest) == 1 + and isinstance(manifest[0], dict), "archive must contain exactly one image") + config_name = manifest[0].get("Config") + require(isinstance(config_name, str) and config_name in config_names and config_name in metadata, + "archive config does not match tested image ID") + config_bytes = metadata[config_name] + require(hashlib.sha256(config_bytes).hexdigest() == digest, "archive image config digest mismatch") + return config_bytes, manifest[0] + + +def prepare_platform_manifest(archive, image_id, destination): + """Convert one Docker-save archive into a digest-addressed platform manifest.""" + require(DIGEST.fullmatch(image_id), "invalid image ID") + config_bytes, saved_manifest = archive_metadata(archive, image_id) + layer_names = saved_manifest.get("Layers") + require(isinstance(layer_names, list) and all(isinstance(name, str) and name for name in layer_names), + "archive image layers are invalid") + config = json.loads(config_bytes) + require(isinstance(config, dict), "image config is invalid") + rootfs = config.get("rootfs") + require(isinstance(rootfs, dict) and rootfs.get("type") == "layers", "image rootfs is invalid") + diff_ids = rootfs.get("diff_ids") + require(isinstance(diff_ids, list) and len(diff_ids) == len(layer_names) + and all(isinstance(value, str) and DIGEST.fullmatch(value) for value in diff_ids), + "image rootfs layer digests are invalid") + + destination.mkdir(parents=True, exist_ok=True) + config_path = destination / "config.json" + config_path.write_bytes(config_bytes) + blobs = {image_id: config_path} + layers = {} + uncompressed_digests = {} + found_layers = set() + with tarfile.open(archive, "r|gz") as saved: + for member in saved: + if member.name not in layer_names: + continue + require(member.isfile(), f"image archive layer is not a regular file: {member.name}") + require(member.name not in found_layers, f"duplicate image archive layer member: {member.name}") + found_layers.add(member.name) + source = saved.extractfile(member) + require(source is not None, f"cannot read image archive layer: {member.name}") + require(member.size >= 0 and member.size <= shutil.disk_usage(destination).free, + f"image archive layer exceeds available staging space: {member.name}") + raw_layer = destination / f"layer-{len(layers)}.tar" + with source, raw_layer.open("wb") as target: + shutil.copyfileobj(source, target, BLOB_CHUNK_SIZE) + require(raw_layer.stat().st_size == member.size, + f"image archive layer size changed while reading: {member.name}") + compressed_layer = destination / f"layer-{len(layers)}.tar.gz" + with raw_layer.open("rb") as source: + magic = source.read(4) + require(magic != b"\x28\xb5\x2f\xfd", "zstd image archive layers are not supported") + if magic.startswith(b"\x1f\x8b"): + with gzip.open(raw_layer, "rb") as source: + uncompressed_digests[member.name] = "sha256:" + hashlib.file_digest(source, "sha256").hexdigest() + shutil.copyfile(raw_layer, compressed_layer) + else: + uncompressed_digests[member.name] = "sha256:" + sha256(raw_layer) + with raw_layer.open("rb") as source, compressed_layer.open("wb") as target, \ + gzip.GzipFile(filename="", mode="wb", fileobj=target, + compresslevel=1, mtime=0) as zipped: + shutil.copyfileobj(source, zipped, BLOB_CHUNK_SIZE) + raw_layer.unlink() + digest = "sha256:" + sha256(compressed_layer) + blobs[digest] = compressed_layer + layers[member.name] = { + "mediaType": "application/vnd.docker.image.rootfs.diff.tar.gzip", + "size": compressed_layer.stat().st_size, + "digest": digest, + } + require(found_layers == set(layer_names), "image archive is missing a declared layer") + require([uncompressed_digests[name] for name in layer_names] == diff_ids, + "image archive layer content does not match tested rootfs") + + manifest = { + "schemaVersion": 2, + "mediaType": "application/vnd.docker.distribution.manifest.v2+json", + "config": { + "mediaType": "application/vnd.docker.container.image.v1+json", + "size": len(config_bytes), + "digest": image_id, + }, + # Docker save may write tar members in digest order, not filesystem order. + "layers": [layers[name] for name in layer_names], + } + raw_manifest = json.dumps(manifest, separators=(",", ":")).encode() + manifest_path = destination / "manifest.json" + manifest_path.write_bytes(raw_manifest) + return { + "path": manifest_path, + "digest": "sha256:" + hashlib.sha256(raw_manifest).hexdigest(), + "manifest": manifest, + "blobs": blobs, + } + + +class _NoRedirect(urllib.request.HTTPRedirectHandler): + def redirect_request(self, request, fp, code, msg, headers, new_url): + return None + + +class RegistryClient: + """Minimal Registry V2 client for digest-only platform publication.""" + + def __init__(self, repository, username, password): + registry, name = repository.split("/", 1) + settings = REGISTRY_CONFIG[registry] + self.repository = name + self.registry_host = settings["registry_host"] + self.token_url = settings["token_url"] + self.service = settings["service"] + self.upload_hosts = settings["upload_hosts"] + self.username = username + self.password = password + self.token = None + + def _get_token(self): + query = urllib.parse.urlencode({ + "service": self.service, + "scope": f"repository:{self.repository}:pull,push", + }) + credentials = base64.b64encode(f"{self.username}:{self.password}".encode()).decode() + request = urllib.request.Request( + f"{self.token_url}?{query}", + headers={"Accept": "application/json", "Authorization": f"Basic {credentials}"}, + ) + opener = urllib.request.build_opener(urllib.request.ProxyHandler({}), _NoRedirect()) + try: + with opener.open(request, timeout=60) as response: + payload = json.loads(response.read(4 * 1024 * 1024)) + except (urllib.error.HTTPError, urllib.error.URLError, TimeoutError, json.JSONDecodeError) as exc: + raise RuntimeError(f"registry token request failed for {self.repository}: {type(exc).__name__}") from None + require(isinstance(payload, dict), "registry token response is invalid") + token = payload.get("token") or payload.get("access_token") + require(isinstance(token, str) and token and not any(char.isspace() for char in token), + "registry token response is invalid") + self.token = token + return token + + def _request(self, method, url, expected, *, body=None, body_path=None, content_type=None): + require(body is None or body_path is None, "registry request body is ambiguous") + for attempt in range(2): + headers = {"Authorization": f"Bearer {self.token or self._get_token()}"} + if body is not None: + headers["Content-Length"] = str(len(body)) + if body_path is not None: + headers["Content-Length"] = str(body_path.stat().st_size) + if content_type: + headers["Content-Type"] = content_type + status, response_headers = self._send(method, url, headers, body=body, body_path=body_path) + if status != 401: + require(status in expected, + f"registry {method} failed for {self.repository}: HTTP {status}") + return status, response_headers + self.token = None + if attempt == 1: + raise RuntimeError(f"registry authorization failed for {self.repository}") + raise RuntimeError("registry request retry failed") + + @staticmethod + def _send(method, url, headers, *, body=None, body_path=None): + parsed = urllib.parse.urlsplit(url) + require(parsed.scheme == "https" and parsed.hostname, "registry URL must use HTTPS") + target = urllib.parse.urlunsplit(("", "", parsed.path or "/", parsed.query, "")) + connection = http.client.HTTPSConnection(parsed.hostname, parsed.port, timeout=120) + try: + connection.putrequest(method, target) + for key, value in headers.items(): + connection.putheader(key, value) + connection.endheaders() + if body is not None: + connection.send(body) + elif body_path is not None: + with body_path.open("rb") as source: + while chunk := source.read(BLOB_CHUNK_SIZE): + connection.send(chunk) + response = connection.getresponse() + response_headers = {key.lower(): value for key, value in response.getheaders()} + response.read(4 * 1024 * 1024 + 1) + return response.status, response_headers + finally: + connection.close() + + def _upload_url(self, location): + upload_url = urllib.parse.urljoin(f"https://{self.registry_host}", location) + parsed = urllib.parse.urlsplit(upload_url) + require(parsed.scheme == "https" and parsed.hostname in self.upload_hosts + and not parsed.username and not parsed.password, + "registry upload location is not an allowed registry host") + return upload_url + + def _path(self, suffix): + return f"https://{self.registry_host}/v2/{self.repository}/{suffix}" + + def _upload_blob(self, digest, path): + blob_url = self._path(f"blobs/{digest}") + status, _ = self._request("HEAD", blob_url, (200, 404)) + if status == 200: + return + _, headers = self._request("POST", self._path("blobs/uploads/"), (202,)) + location = headers.get("location") + require(location, "registry did not return a blob upload location") + upload_url = self._upload_url(location) + parsed = urllib.parse.urlsplit(upload_url) + # Upload locations can contain opaque signed state; preserve its encoding. + # https://github.com/opencontainers/distribution-spec/blob/main/spec.md#post-then-put + query = parsed.query + ("&" if parsed.query else "") + urllib.parse.urlencode({"digest": digest}) + upload_url = urllib.parse.urlunsplit((parsed.scheme, parsed.netloc, parsed.path, + query, "")) + self._request("PUT", upload_url, (201,), body_path=path, + content_type="application/octet-stream") + + def publish_platform_manifest(self, prepared): + raw_manifest = prepared["path"].read_bytes() + digest = prepared["digest"] + require("sha256:" + hashlib.sha256(raw_manifest).hexdigest() == digest, + "prepared platform manifest digest changed") + manifest = prepared["manifest"] + descriptors = [manifest["config"], *manifest.get("layers", [])] + for descriptor in descriptors: + blob_digest = descriptor["digest"] + blob_path = prepared["blobs"].get(blob_digest) + require(blob_path is not None and blob_path.is_file(), + f"missing prepared image blob: {blob_digest}") + self._upload_blob(blob_digest, blob_path) + self._request("PUT", self._path(f"manifests/{digest}"), (201,), body=raw_manifest, + content_type=manifest["mediaType"]) + return digest + + +def create_registry_client(repository): + registry = repository.split("/", 1)[0] + credentials = { + "ghcr.io": (os.environ.get("GHCR_USERNAME"), os.environ.get("GHCR_TOKEN")), + "docker.io": (os.environ.get("DOCKERHUB_USERNAME"), os.environ.get("DOCKERHUB_TOKEN")), + }[registry] + require(all(isinstance(value, str) and value for value in credentials), + f"credentials are not configured for {registry}") + return RegistryClient(repository, *credentials) diff --git a/scripts/smoke-gemini-browser.py b/scripts/smoke-gemini-browser.py index 3bec510..d83aab7 100644 --- a/scripts/smoke-gemini-browser.py +++ b/scripts/smoke-gemini-browser.py @@ -2,6 +2,7 @@ """Exercise the desktop, daemon, Puppeteer/stealth, sharp and persisted cookies.""" import argparse import json +import re import subprocess import sys import time @@ -100,11 +101,20 @@ def main(image, variant): else: raise RuntimeError("desktop/daemon did not become ready") if variant == "linuxserver": - docker("exec", name, "nginx", "-t") + nginx = docker("exec", name, "nginx", "-T") + # Selkies 2 moved file downloads behind the /api proxy. + # https://github.com/linuxserver/docker-baseimage-selkies/commit/c28cd610744dae7c278f2a49ca28748dbf387c1a + config = nginx.stdout + nginx.stderr + files_path = "/api/files/" if re.search(r"\blocation\s+/api\s*\{", config) else "/files/" + files_url = "https://127.0.0.1:3001" + files_path docker("exec", name, "sh", "-c", "printf '%s' 'release-factory files smoke' > /config/Desktop/rf-smoke.txt") - files = docker("exec", name, "curl", "-kfsS", "--max-time", "5", "-u", - "smoke:ci-gemini-password", "https://127.0.0.1:3001/files/") + # Nginx can serve the desktop before the Selkies API is ready. + curl = ("exec", name, "curl", "-kfsS", "--max-time", "5", "--retry", "10", + "--retry-delay", "1", "--retry-max-time", "30", "-u", "smoke:ci-gemini-password") + files = docker(*curl, files_url) assert "rf-smoke.txt" in files.stdout, "file browser did not list the test file" + downloaded = docker(*curl, files_url + "rf-smoke.txt") + assert downloaded.stdout == "release-factory files smoke", "file download content mismatch" result = docker("exec", "-w", "/opt/gemini-skill", "-e", "SMOKE_PHASE=" + phase, name, "node", "--input-type=module", "-e", PROBE) print(result.stdout, end="", flush=True) diff --git a/scripts/test-openclaw-npm-bundle.mjs b/scripts/test-openclaw-npm-bundle.mjs new file mode 100644 index 0000000..578cf71 --- /dev/null +++ b/scripts/test-openclaw-npm-bundle.mjs @@ -0,0 +1,53 @@ +import assert from 'node:assert/strict'; +import fs from 'node:fs'; +import os from 'node:os'; +import path from 'node:path'; +import test from 'node:test'; +import { patchNpmBundles } from '../openclaw-builder/image/patch-vendored-deps.mjs'; + +const pins = JSON.parse(fs.readFileSync(new URL('../openclaw-builder/configs/components.json', import.meta.url))).npm_bundled; + +test('repairs only the two vulnerable dependencies inside bundled npm', async () => { + const temporary = fs.mkdtempSync(path.join(os.tmpdir(), 'openclaw-npm-test-')); + try { + const root = path.join(temporary, 'node_modules'); + const npm = path.join(root, '.pnpm/npm@11.20.0/node_modules/npm'); + fs.mkdirSync(npm, {recursive: true}); + fs.writeFileSync(path.join(npm, 'package.json'), '{"name":"npm","version":"11.20.0"}\n'); + for (const [name, pin] of Object.entries(pins)) { + const directory = path.join(npm, 'node_modules', name); + fs.mkdirSync(directory, {recursive: true}); + fs.writeFileSync(path.join(directory, 'package.json'), JSON.stringify({name, version: pin.from})); + } + const unrelated = path.join(root, 'unrelated'); + fs.mkdirSync(unrelated); + fs.writeFileSync(path.join(unrelated, 'package.json'), '{"name":"undici","version":"6.28.0"}'); + await patchNpmBundles(root, pins); + for (const [name, pin] of Object.entries(pins)) { + const pkg = JSON.parse(fs.readFileSync(path.join(npm, 'node_modules', name, 'package.json'))); + assert.equal(pkg.version, pin.version); + assert.equal(pkg.name, name); + } + assert.equal(JSON.parse(fs.readFileSync(path.join(unrelated, 'package.json'))).version, '6.28.0'); + assert.equal(fs.readFileSync(path.join(npm, 'package.json'), 'utf8'), '{"name":"npm","version":"11.20.0"}\n'); + await patchNpmBundles(root, pins); // Idempotent and does not download the archives again. + } finally { + fs.rmSync(temporary, {recursive: true}); + } +}); + +test('rejects a bundle dependency that escapes its npm root', async () => { + const temporary = fs.mkdtempSync(path.join(os.tmpdir(), 'openclaw-npm-escape-')); + try { + const root = path.join(temporary, 'node_modules'); + const npm = path.join(root, '.pnpm/npm@11.20.0/node_modules/npm/node_modules'); + const outside = path.join(temporary, 'outside'); + fs.mkdirSync(npm, {recursive: true}); + fs.mkdirSync(outside); + fs.writeFileSync(path.join(outside, 'package.json'), '{"name":"undici","version":"6.28.0"}'); + fs.symlinkSync(outside, path.join(npm, 'undici')); + await assert.rejects(patchNpmBundles(root, pins), /unexpected bundled npm dependency path/); + } finally { + fs.rmSync(temporary, {recursive: true}); + } +}); diff --git a/scripts/test-publish-tested-image.py b/scripts/test-publish-tested-image.py index 9aac251..ea31ea2 100644 --- a/scripts/test-publish-tested-image.py +++ b/scripts/test-publish-tested-image.py @@ -3,13 +3,13 @@ import importlib.util import json import os -from pathlib import Path import subprocess import sys import tempfile import textwrap -from types import SimpleNamespace import unittest +from pathlib import Path +from types import SimpleNamespace from unittest.mock import patch SPEC = importlib.util.spec_from_file_location("publish_tested", Path(__file__).with_name("publish-tested-image.py")) @@ -49,7 +49,7 @@ class PublicationTests(unittest.TestCase): directory.mkdir() (directory / "receipt.json").write_text(json.dumps(receipt)) - def test_stage_tags_the_recorded_id_and_checks_remote_config(self): + def test_stage_publishes_the_recorded_id_by_digest_without_a_ci_tag(self): with tempfile.TemporaryDirectory() as tmp: args = SimpleNamespace(**self.expected, image="mutable:tag", image_id=self.image_id, platform="linux/amd64", receipt=Path(tmp) / "receipt.json") @@ -59,9 +59,21 @@ class PublicationTests(unittest.TestCase): def docker(*argv, **kwargs): calls.append(argv) return json.dumps([image]).encode() if kwargs.get("capture") else None - with patch.object(MODULE, "docker", docker), patch.object(MODULE, "remote_manifest", return_value=(self.manifest, self.digest)): + class Client: + def __init__(self, digest): + self.digest = digest + + def publish_platform_manifest(self, prepared): + return self.digest + + with patch.object(MODULE, "docker", docker), \ + patch.object(MODULE, "save_image"), \ + patch.object(MODULE, "prepare_platform_manifest", return_value=object()), \ + patch.object(MODULE, "create_registry_client", return_value=Client(self.digest)), \ + patch.object(MODULE, "remote_manifest", return_value=(self.manifest, self.digest)): MODULE.stage(args, self.expected) - self.assertEqual(calls[1][0:3], ("image", "tag", self.image_id)) + self.assertEqual(calls, [("image", "inspect", "mutable:tag")]) + self.assertNotIn(":ci-", str(json.loads(args.receipt.read_text()))) self.assertEqual(json.loads(args.receipt.read_text())["image_id"], self.image_id) def test_changed_image_or_platform_never_reaches_registry_write(self): diff --git a/scripts/test-registry-image.py b/scripts/test-registry-image.py new file mode 100644 index 0000000..0983b91 --- /dev/null +++ b/scripts/test-registry-image.py @@ -0,0 +1,130 @@ +#!/usr/bin/env python3 +"""Verify image-layer identity and the Registry V2 digest-only upload contract.""" +import gzip +import hashlib +import io +import json +import tarfile +import tempfile +import unittest +from pathlib import Path +from unittest.mock import patch + +import registry_image as registry + + +def digest(data): + return "sha256:" + hashlib.sha256(data).hexdigest() + + +def layer(content): + stream = io.BytesIO() + with tarfile.open(fileobj=stream, mode="w") as tar: + member = tarfile.TarInfo("value.txt") + member.size = len(content) + tar.addfile(member, io.BytesIO(content)) + return stream.getvalue() + + +class LayerTests(unittest.TestCase): + def prepare(self, *, compressed=False, corrupt=None): + tmp = tempfile.TemporaryDirectory() + self.addCleanup(tmp.cleanup) + root = Path(tmp.name) + data = [layer(b"base"), layer(b"upper")] + names = ["base/layer.tar", "upper/layer.tar"] + config = {"rootfs": {"type": "layers", "diff_ids": [digest(b) for b in data]}} + if corrupt == "count": + config["rootfs"]["diff_ids"].pop() + if corrupt == "digest": + config["rootfs"]["diff_ids"][0] = "invalid" + if corrupt == "type": + config["rootfs"]["type"] = "unknown" + config_bytes = json.dumps(config).encode() + image_id = digest(config_bytes) + config_name = image_id[7:] + ".json" + manifest = [{"Config": config_name, "Layers": names}] + members = [(config_name, config_bytes), ("manifest.json", json.dumps(manifest).encode())] + for name, raw in reversed(list(zip(names, data))): + if corrupt == "missing" and name == names[0]: + continue + if corrupt == "bytes" and name == names[0]: + raw = layer(b"wrong") + stored = gzip.compress(raw, mtime=0) if compressed else raw + if corrupt == "gzip" and name == names[0]: + stored = gzip.compress(raw)[:-8] + if corrupt == "zstd" and name == names[0]: + stored = b"\x28\xb5\x2f\xfd" + raw + members.append((name, stored)) + archive = root / "image.tar.gz" + with tarfile.open(archive, "w:gz") as tar: + for name, body in members: + member = tarfile.TarInfo(name) + member.size = len(body) + tar.addfile(member, io.BytesIO(body)) + prepared = registry.prepare_platform_manifest(archive, image_id, root / "prepared") + return prepared, config + + def test_tar_member_order_does_not_change_filesystem_layer_order(self): + for compressed in (False, True): + with self.subTest(compressed=compressed): + prepared, config = self.prepare(compressed=compressed) + actual = [digest(gzip.decompress(prepared["blobs"][d["digest"]].read_bytes())) + for d in prepared["manifest"]["layers"]] + self.assertEqual(actual, config["rootfs"]["diff_ids"]) + + def test_invalid_layers_fail_before_publication(self): + for corrupt in ("count", "digest", "type", "missing", "bytes", "gzip", "zstd"): + with self.subTest(corrupt=corrupt), self.assertRaises((ValueError, EOFError, OSError)): + self.prepare(corrupt=corrupt) + + +class RegistryTests(unittest.TestCase): + def client(self, host="ghcr.io"): + client = registry.RegistryClient(host + "/owner/image", "user", "test-password") + client.token = "test-token" + return client + + def test_upload_preserves_opaque_location_query(self): + for host in ("ghcr.io", "docker.io"): + for absolute in (False, True): + with self.subTest(host=host, absolute=absolute), tempfile.TemporaryDirectory() as tmp: + client = self.client(host) + path = Path(tmp) / "blob" + path.write_bytes(b"blob") + location = "/v2/owner/image/blobs/uploads/1?token=a%2fb&space=%20&empty&x=1&x=2" + if absolute: + location = "https://" + client.registry_host + location + with patch.object(client, "_send", side_effect=[ + (404, {}), (202, {"location": location}), (201, {}), + ]) as send: + client._upload_blob(digest(b"blob"), path) + expected = (location if absolute else "https://" + client.registry_host + location) + self.assertEqual(send.call_args.args[1], expected + "&digest=" + digest(b"blob").replace(":", "%3A")) + + def test_incomplete_upload_is_rejected(self): + client = self.client() + with tempfile.TemporaryDirectory() as tmp: + path = Path(tmp) / "blob" + path.write_bytes(b"blob") + with patch.object(client, "_send", side_effect=[ + (404, {}), (202, {"location": "/v2/owner/image/blobs/uploads/1"}), (202, {}), + ]), self.assertRaisesRegex(ValueError, "HTTP 202"): + client._upload_blob(digest(b"blob"), path) + + def test_existing_blob_does_not_start_an_upload(self): + client = self.client() + with patch.object(client, "_send", return_value=(200, {})) as send: + client._upload_blob(digest(b"blob"), Path("unused")) + self.assertEqual(send.call_count, 1) + self.assertEqual(send.call_args.args[0], "HEAD") + + def test_untrusted_upload_locations_are_rejected(self): + client = self.client() + for url in ("https://evil.example/upload", "http://ghcr.io/upload", "https://user@ghcr.io/upload"): + with self.subTest(url=url), self.assertRaises(ValueError): + client._upload_url(url) + + +if __name__ == "__main__": + unittest.main() diff --git a/scripts/test-smoke-gemini-browser.py b/scripts/test-smoke-gemini-browser.py new file mode 100644 index 0000000..09b5783 --- /dev/null +++ b/scripts/test-smoke-gemini-browser.py @@ -0,0 +1,61 @@ +#!/usr/bin/env python3 +"""Keep file browsing covered across LinuxServer's Selkies route migration.""" +import importlib.util +import subprocess +import unittest +from pathlib import Path +from unittest.mock import patch + +SPEC = importlib.util.spec_from_file_location("smoke", Path(__file__).with_name("smoke-gemini-browser.py")) +smoke = importlib.util.module_from_spec(SPEC) +SPEC.loader.exec_module(smoke) + + +class FileBrowserTests(unittest.TestCase): + def run_smoke(self, route, *, listing="rf-smoke.txt", content="release-factory files smoke"): + requested = [] + + def docker(*args, **kwargs): + output = "" + if args[0] == "exec" and "curl" in args: + url = args[-1] + if url.endswith("/health"): + output = '{"service":"browser-daemon"}' + elif url == "https://127.0.0.1:3001/": + output = "desktop" + else: + requested.append(url) + if not url.startswith("https://127.0.0.1:3001" + route): + raise RuntimeError("curl: (22) The requested URL returned error: 404") + output = content if url.endswith("rf-smoke.txt") else listing + elif args[0] == "exec" and "nginx" in args: + output = "server { location " + ("/api" if route == "/api/files/" else "/files") + " { } }" + return subprocess.CompletedProcess(args, 0, output, "") + + with patch.object(smoke, "docker", side_effect=docker): + smoke.main("test-image", "linuxserver") + return requested + + def test_selkies_v2_lists_and_downloads_through_api(self): + self.assertEqual(self.run_smoke("/api/files/"), [ + "https://127.0.0.1:3001/api/files/", + "https://127.0.0.1:3001/api/files/rf-smoke.txt", + ] * 2) + + def test_older_base_images_keep_the_legacy_file_route(self): + self.assertEqual(self.run_smoke("/files/"), [ + "https://127.0.0.1:3001/files/", + "https://127.0.0.1:3001/files/rf-smoke.txt", + ] * 2) + + def test_missing_file_still_fails(self): + with self.assertRaisesRegex(AssertionError, "did not list"): + self.run_smoke("/api/files/", listing="empty directory") + + def test_wrong_download_still_fails(self): + with self.assertRaisesRegex(AssertionError, "content"): + self.run_smoke("/api/files/", content="wrong file") + + +if __name__ == "__main__": + unittest.main()