fix(release): repair image builds, update DSH, and stop ci tag publication (#90)

Repair Selkies file smoke checks and DSH 0.2 telemetry compatibility; update pinned components and vulnerable bundled dependencies. Publish tested images by verified digest before advancing release tags, without ci tags.

Supersedes #88. All final-head native build, runtime and security checks passed.
This commit is contained in:
okxlin
2026-09-30 17:24:37 +08:00
committed by GitHub
parent 7e0ce1b4ce
commit 2ffcf17dfd
33 changed files with 982 additions and 165 deletions
@@ -27,6 +27,8 @@ on:
- 'scripts/evaluate-trivy-policy.py'
- 'scripts/test-evaluate-trivy-policy.py'
- 'scripts/publish-tested-image.py'
- 'scripts/registry_image.py'
- 'scripts/test-registry-image.py'
- 'scripts/test-publish-tested-image.py'
permissions:
@@ -30,10 +30,13 @@ on:
- '.github/workflows/build-gemini-skill-browser-linuxserver.yml'
- '.github/workflows/release-gemini-browser.yml'
- 'scripts/smoke-gemini-browser.py'
- 'scripts/test-smoke-gemini-browser.py'
- 'scripts/trivy-image-gate.sh'
- 'scripts/evaluate-trivy-policy.py'
- 'scripts/test-evaluate-trivy-policy.py'
- 'scripts/publish-tested-image.py'
- 'scripts/registry_image.py'
- 'scripts/test-registry-image.py'
- 'scripts/test-publish-tested-image.py'
permissions:
@@ -30,10 +30,13 @@ on:
- '.github/workflows/build-gemini-skill-browser.yml'
- '.github/workflows/release-gemini-browser.yml'
- 'scripts/smoke-gemini-browser.py'
- 'scripts/test-smoke-gemini-browser.py'
- 'scripts/trivy-image-gate.sh'
- 'scripts/evaluate-trivy-policy.py'
- 'scripts/test-evaluate-trivy-policy.py'
- 'scripts/publish-tested-image.py'
- 'scripts/registry_image.py'
- 'scripts/test-registry-image.py'
- 'scripts/test-publish-tested-image.py'
permissions:
@@ -27,6 +27,8 @@ on:
- 'scripts/evaluate-trivy-policy.py'
- 'scripts/test-evaluate-trivy-policy.py'
- 'scripts/publish-tested-image.py'
- 'scripts/registry_image.py'
- 'scripts/test-registry-image.py'
- 'scripts/test-publish-tested-image.py'
permissions:
@@ -17,6 +17,8 @@ on:
- 'scripts/evaluate-trivy-policy.py'
- 'scripts/test-evaluate-trivy-policy.py'
- 'scripts/publish-tested-image.py'
- 'scripts/registry_image.py'
- 'scripts/test-registry-image.py'
- 'scripts/test-publish-tested-image.py'
- '.github/workflows/openclaw-upstream-docker.yml'
@@ -51,7 +53,9 @@ jobs:
run: |
python3 scripts/test-resolve-openclaw-inputs.py
bash scripts/test-apply-openclaw-runtime-hardening.sh
node --test scripts/test-openclaw-npm-bundle.mjs
python3 scripts/test-evaluate-trivy-policy.py
python3 scripts/test-registry-image.py
python3 scripts/test-publish-tested-image.py
- name: Resolve immutable sources and security refresh
id: inputs
@@ -168,6 +172,10 @@ jobs:
REPOSITORY: ${{ needs.prepare.outputs.repository }}
DOCKERHUB_NAMESPACE: ${{ vars.DOCKERHUB_USERNAME || secrets.DOCKERHUB_USERNAME }}
IMAGE_TAG: ${{ needs.prepare.outputs.image_tag }}
GHCR_USERNAME: ${{ github.actor }}
GHCR_TOKEN: ${{ github.token }}
DOCKERHUB_USERNAME: ${{ vars.DOCKERHUB_USERNAME || secrets.DOCKERHUB_USERNAME }}
DOCKERHUB_TOKEN: ${{ secrets.DOCKERHUB_TOKEN }}
run: |
set -euo pipefail
repositories=("$REPOSITORY" "docker.io/$DOCKERHUB_NAMESPACE/openclaw-sandbox")
@@ -60,6 +60,7 @@ jobs:
- name: Check component pins and publication safety regressions
run: |
bash deepseek-harness-builder/scripts/check-component-pins.sh
python3 scripts/test-registry-image.py
python3 deepseek-harness-builder/scripts/test-tested-image-artifact.py
- name: Set up Node.js
@@ -226,6 +227,10 @@ jobs:
IMAGE_TAG: ${{ needs.prepare.outputs.image_tag }}
LATEST_TAG: ${{ needs.prepare.outputs.latest_tag }}
PUSH_LATEST: ${{ inputs.push_latest }}
GHCR_USERNAME: ${{ github.actor }}
GHCR_TOKEN: ${{ github.token }}
DOCKERHUB_USERNAME: ${{ env.DOCKERHUB_NAMESPACE }}
DOCKERHUB_TOKEN: ${{ secrets.DOCKERHUB_TOKEN }}
run: |
set -euo pipefail
floating_args=()
@@ -69,6 +69,8 @@ jobs:
- name: Validate input and publication contracts
run: |
python3 gemini-skill-browser-builder/scripts/test-resolve-browser-inputs.py
python3 scripts/test-smoke-gemini-browser.py
python3 scripts/test-registry-image.py
python3 scripts/test-publish-tested-image.py
python3 scripts/test-evaluate-trivy-policy.py
- uses: actions/setup-node@v7
@@ -133,6 +135,8 @@ jobs:
REPOSITORY: ${{ steps.params.outputs.repository }}
IMAGE_TAG: ${{ steps.params.outputs.image_tag }}
LATEST_TAG: ${{ steps.params.outputs.release_latest }}
GHCR_USERNAME: ${{ github.actor }}
GHCR_TOKEN: ${{ github.token }}
run: |
python3 scripts/publish-tested-image.py stage --image "$IMAGE" --image-id "$IMAGE" \
--variant "$VARIANT" --platform linux/amd64 --repository "$REPOSITORY" \
@@ -83,6 +83,7 @@ jobs:
env:
VARIANT: ${{ inputs.variant }}
run: |
python3 scripts/test-registry-image.py
python3 scripts/test-publish-tested-image.py
python3 scripts/test-evaluate-trivy-policy.py
if [[ "$VARIANT" == codex ]]; then
@@ -180,6 +181,8 @@ jobs:
VARIANT: ${{ inputs.variant }}
REPOSITORY: ${{ needs.prepare.outputs.repository }}
ARCH: ${{ matrix.arch }}
GHCR_USERNAME: ${{ github.actor }}
GHCR_TOKEN: ${{ github.token }}
run: |
python3 scripts/publish-tested-image.py stage --image "$IMAGE" --image-id "$IMAGE" \
--variant "$VARIANT" --platform "linux/$ARCH" --repository "$REPOSITORY" \
@@ -12,6 +12,8 @@ on:
- ".github/actions/verify-deepseek-harness/**"
- "deepseek-harness-builder/**"
- "scripts/trivy-image-gate.sh"
- "scripts/registry_image.py"
- "scripts/test-registry-image.py"
- "scripts/evaluate-trivy-policy.py"
- "scripts/test-evaluate-trivy-policy.py"
- "SECURITY_SCAN.md"
@@ -54,7 +56,9 @@ jobs:
run: python3 scripts/test-evaluate-trivy-policy.py
- name: Run tested image publication regression tests
run: python3 deepseek-harness-builder/scripts/test-tested-image-artifact.py
run: |
python3 scripts/test-registry-image.py
python3 deepseek-harness-builder/scripts/test-tested-image-artifact.py
- name: Run DeepSeek Harness version resolver regression tests
run: |
@@ -11,6 +11,8 @@ on:
- ".github/actions/verify-deepseek-harness/**"
- "deepseek-harness-builder/**"
- "scripts/trivy-image-gate.sh"
- "scripts/registry_image.py"
- "scripts/test-registry-image.py"
- "scripts/evaluate-trivy-policy.py"
- "scripts/test-evaluate-trivy-policy.py"
- "SECURITY_SCAN.md"
@@ -341,6 +341,20 @@ RUN [[ "${CODE_SERVER_TAR_VERSION}" =~ ^[0-9]+\.[0-9]+\.[0-9]+$ ]] \
"${code_server_tar_dir}/package.json" "${CODE_SERVER_TAR_VERSION}" \
&& find /tmp -maxdepth 1 -name "${tarball}" -delete
# Patch the HTTP client bundled by VS Code until code-server includes the fix.
ARG CODE_SERVER_UNDICI_VERSION=7.29.1
RUN [[ "${CODE_SERVER_UNDICI_VERSION}" =~ ^[0-9]+[.][0-9]+[.][0-9]+$ ]] \
&& undici_dir="/usr/lib/code-server/lib/vscode/node_modules/undici" \
&& test "$(realpath "${undici_dir}")" = "${undici_dir}" \
&& test "$(node -p "require('${undici_dir}/package.json').name")" = undici \
&& tarball="$(npm pack --ignore-scripts --silent --pack-destination /tmp "undici@${CODE_SERVER_UNDICI_VERSION}")" \
&& [[ "${tarball}" == "undici-${CODE_SERVER_UNDICI_VERSION}.tgz" ]] \
&& find "${undici_dir}" -mindepth 1 -delete \
&& tar -xzf "/tmp/${tarball}" --strip-components=1 -C "${undici_dir}" \
&& node -e 'const [directory, expected] = process.argv.slice(1); if (require(`${directory}/package.json`).version !== expected) throw new Error("unexpected code-server undici version"); require(directory)' \
"${undici_dir}" "${CODE_SERVER_UNDICI_VERSION}" \
&& rm -f "/tmp/${tarball}"
# ── npm globals and Corepack package managers ──
# Claude's audited postinstall links its platform-native optional dependency.
# Keep lifecycle scripts from all other global packages blocked by npm 12.
@@ -2056,9 +2056,9 @@
}
},
"node_modules/fast-uri": {
"version": "3.1.6",
"resolved": "https://registry.npmjs.org/fast-uri/-/fast-uri-3.1.6.tgz",
"integrity": "sha512-7Ical1vFEMr0onbVzEDIreM22I4khW+fzyQPwvAFWBp1iwdshSZRsL4jjRvPG9JP1uiqMHRto+YU6R2/CzDz5Q==",
"version": "3.1.8",
"resolved": "https://registry.npmjs.org/fast-uri/-/fast-uri-3.1.8.tgz",
"integrity": "sha512-GZMtZUTNRpOVIECoXwLNZS5xUGE+mVNbTB8h/7Rwh2TFWcBQiPzTgyZi05BF9UMZKkLJv8XBRJTlU7zg8+ZfMg==",
"funding": [
{
"type": "github",
@@ -8,6 +8,6 @@
"@getpaseo/cli": "0.3.1"
},
"overrides": {
"fast-uri": "3.1.6"
"fast-uri": "3.1.8"
}
}
@@ -112,11 +112,11 @@ if (root?.dependencies?.["@getpaseo/cli"] !== "0.3.1") {
if (lock.packages?.["node_modules/@getpaseo/cli"]?.version !== "0.3.1") {
throw new Error("package-lock does not resolve @getpaseo/cli 0.3.1");
}
if (packageJson.overrides?.["fast-uri"] !== "3.1.6") {
throw new Error("package.json must pin the fast-uri security override to 3.1.6");
if (packageJson.overrides?.["fast-uri"] !== "3.1.8") {
throw new Error("package.json must pin the fast-uri security override to 3.1.8");
}
if (lock.packages?.["node_modules/fast-uri"]?.version !== "3.1.6") {
throw new Error("package-lock does not resolve fast-uri 3.1.6");
if (lock.packages?.["node_modules/fast-uri"]?.version !== "3.1.8") {
throw new Error("package-lock does not resolve fast-uri 3.1.8");
}
const installScripts = [];
+2 -2
View File
@@ -11,11 +11,11 @@ One Dockerfile produces two independently tested variants under one image reposi
| `latest` | `<DSH_VERSION>` | `runtime` | Lightweight 1Panel service with essential shell and repository tools. This remains the default final target. |
| `workstation` | `<DSH_VERSION>-workstation` | `workstation` | Full interactive development environment with compiler and language toolchains. |
Both workflows publish the same tags to `ghcr.io/okxlin/deepseek-harness` and to `docker.io/$DOCKERHUB_USERNAME/deepseek-harness`. Configure `DOCKERHUB_USERNAME` as a GitHub Actions repository variable or secret and configure `DOCKERHUB_TOKEN` as a repository secret. The Docker Hub token is used only by the registry login action and is never passed to the image build.
Both workflows publish the same tags to `ghcr.io/okxlin/deepseek-harness` and to `docker.io/$DOCKERHUB_USERNAME/deepseek-harness`. Configure `DOCKERHUB_USERNAME` as a GitHub Actions repository variable or secret and configure `DOCKERHUB_TOKEN` as a repository secret. The Docker Hub token is used only for registry authentication and verified-image publication; it is never passed to the image build.
Scheduled runs and manual runs with an empty version select the highest non-draft `dsh-v*` source release from DeepSeek Harness GitHub Releases, verify its tag, commit, and source-archive SHA-256, then publish matching `<DSH_VERSION>` and `<DSH_VERSION>-workstation` tags. This supports upstream releases that are available from GitHub before they are published to npm. An explicit `dsh-v*` selects that source release; an npm version or dist-tag still selects a published package release. `image/dsh-source.json` remains the reproducible local/PR baseline, while future source releases no longer require a Dockerfile edit. Manual workflow runs can also override the published image tag. Use a floating tag for an AppStore `latest` channel and the matching version tag for a numbered AppStore version.
The committed source baseline is [0.1.7-rc.2](https://github.com/deepseek-ai/deepseek-harness/releases/tag/dsh-v0.1.7-rc.2). This baseline includes the PTC package name `ptc-runtime` and workflow executor `workflow-ptc`; custom profiles using the old names need updating. DeepSeek now defaults to the Messages protocol. A manually configured old official API root should be removed or changed to `https://api.deepseek.com/anthropic`; custom provider URLs remain unchanged.
The committed source baseline is [0.2.0-rc.2](https://github.com/deepseek-ai/deepseek-harness/releases/tag/dsh-v0.2.0-rc.2). This baseline includes the PTC package name `ptc-runtime` and workflow executor `workflow-ptc`; custom profiles using the old names need updating. DeepSeek now defaults to the Messages protocol. A manually configured old official API root should be removed or changed to `https://api.deepseek.com/anthropic`; custom provider URLs remain unchanged.
Build versions, base-image digests, and source/tool checksums live in [`image/components.lock.json`](image/components.lock.json). Python packages have a hash-locked [`image/python-requirements.lock`](image/python-requirements.lock); the legacy npm path retains its own [package manifest](image/package.json) and [pnpm lock](image/pnpm-lock.yaml). [`image/dsh-source.json`](image/dsh-source.json) pins the source release to a commit archive and checksum. Local builds and CI use `scripts/component-inputs.py` to resolve the same build arguments; ordinary component updates do not require editing Dockerfile instructions. Source builds currently install locally packed runtime tarballs with npm, and the dependency audit checks that installed tree.
+2 -2
View File
@@ -11,11 +11,11 @@
| `latest` | `<DSH_VERSION>` | `runtime` | 轻量级 1Panel 服务镜像,包含必要 Shell 和仓库工具。它仍是默认最终 target。 |
| `workstation` | `<DSH_VERSION>-workstation` | `workstation` | 完整交互式开发环境,包含编译器和语言工具链。 |
两个工作流都会把相同标签发布到 `ghcr.io/okxlin/deepseek-harness` 和 `docker.io/$DOCKERHUB_USERNAME/deepseek-harness`。请把 `DOCKERHUB_USERNAME` 配置为 GitHub Actions 仓库变量或 Secret,并把 `DOCKERHUB_TOKEN` 配置为仓库 Secret。Docker Hub token 只用于 Registry 登录,不会传入镜像构建上下文。
两个工作流都会把相同标签发布到 `ghcr.io/okxlin/deepseek-harness` 和 `docker.io/$DOCKERHUB_USERNAME/deepseek-harness`。请把 `DOCKERHUB_USERNAME` 配置为 GitHub Actions 仓库变量或 Secret,并把 `DOCKERHUB_TOKEN` 配置为仓库 Secret。Docker Hub token 只用于 Registry 认证和已验证镜像的发布,不会传入镜像构建上下文。
定时任务以及留空版本的手动运行会从 DeepSeek Harness GitHub Releases 选择最高的非 draft `dsh-v*` 源码版本,校验对应 tag、commit 和源码归档 SHA-256 后构建,并发布匹配的 `<DSH_VERSION>` 和 `<DSH_VERSION>-workstation` 标签;因此上游源码版本可以在发布到 npm 之前进入镜像。手动显式传入 `dsh-v*` 可以选择指定的源码 release,传入已发布的 npm 版本或 dist-tag 时仍按请求的 npm selector 解析。`image/dsh-source.json` 保留可复现的本地/PR 基线,未来源码 release 不需要修改 Dockerfile。手动工作流也可覆盖发布标签。AppStore `latest` 通道使用浮动标签,编号 AppStore 版本使用匹配的版本标签。
当前提交的源码基线是 [0.1.7-rc.2](https://github.com/deepseek-ai/deepseek-harness/releases/tag/dsh-v0.1.7-rc.2)。该基线包含 PTC 包名 `ptc-runtime` 和 workflow 执行器 `workflow-ptc`;使用旧名称的自定义 profile 需要更新。DeepSeek 现在默认使用 Messages 协议;手动配置的旧官方 API 根地址应删除或改为 `https://api.deepseek.com/anthropic`,自定义 provider URL 不受影响。
当前提交的源码基线是 [0.2.0-rc.2](https://github.com/deepseek-ai/deepseek-harness/releases/tag/dsh-v0.2.0-rc.2)。该基线包含 PTC 包名 `ptc-runtime` 和 workflow 执行器 `workflow-ptc`;使用旧名称的自定义 profile 需要更新。DeepSeek 现在默认使用 Messages 协议;手动配置的旧官方 API 根地址应删除或改为 `https://api.deepseek.com/anthropic`,自定义 provider URL 不受影响。
构建版本、基础镜像 digest、源码和工具校验和集中保存在 [`image/components.lock.json`](image/components.lock.json)。Python 依赖使用独立的 [`image/python-requirements.lock`](image/python-requirements.lock),旧版 npm 路径保留自己的 [package.json](image/package.json) 和 [pnpm-lock.yaml](image/pnpm-lock.yaml)。[`image/dsh-source.json`](image/dsh-source.json) 把源码归档绑定到确定的 commit 和 SHA-256。本地构建与 CI 都通过 `scripts/component-inputs.py` 解析参数,常规组件更新无需手改 Dockerfile 指令。源码构建当前仍由 npm 安装本地打包的 runtime tarball,依赖审计检查这棵实际安装树。
@@ -8,7 +8,7 @@
"CADDY_RATELIMIT_LICENSE_SHA256": "094bedda295c0702807d1027a00e1134f4cb71d523dc49f6d4b3caf589053068",
"CADDY_RATELIMIT_REF": "5625512f24f6f59d6f64fb3aafe5eecff0b286db",
"CADDY_SECURITY_LICENSE_SHA256": "c71d239df91726fc519c6eb72d318ec65820627232b2f796219e87dcf35d0ab4",
"CADDY_SECURITY_VERSION": "1.2.2",
"CADDY_SECURITY_VERSION": "1.3.0",
"CADDY_SOURCE_ARCHIVE_SHA256": "2c3d02078286a6282cdb4d1d8744077788d556659dac0b64d8ed5886a7e5aeb9",
"CADDY_VERSION": "2.11.4",
"CEL_GO_LICENSE_SHA256": "4cdb9af102dfbb0ca03d87d6f650a505df098646a4080f4665b389ad9c6caa02",
@@ -23,8 +23,8 @@
"DOCKER_COMPOSE_VERSION": "5.5.1",
"DOCKER_VERSION": "29.8.1",
"GO_AUTHCRUNCH_LICENSE_SHA256": "c71d239df91726fc519c6eb72d318ec65820627232b2f796219e87dcf35d0ab4",
"GO_AUTHCRUNCH_SOURCE_SHA256": "166d9743a7fa0979747341929472b680fd1d2ee88879b33f3fc775315579b699",
"GO_AUTHCRUNCH_VERSION": "1.3.6",
"GO_AUTHCRUNCH_SOURCE_SHA256": "b0c0ff46975fdbfff1eee591dfda20339a9170e133e41e269fa2b82b913ad84b",
"GO_AUTHCRUNCH_VERSION": "1.3.10",
"GO_IMAGE": "golang:1.27.1-trixie@sha256:9baa6b4187bbb98d240372a8a235ac0bb6b5ddd52bba1431dc2f7c0705862728",
"GO_VERSION": "1.27.1",
"GRPC_VERSION": "1.84.0",
@@ -34,8 +34,8 @@
"NODE_VERSION": "24.21.0",
"NPM_ARCHIVE_SHA256": "d1a92f40e6c407b84c3a00c3cf978a10b24fd42f153c527e2016cef7bb34a483",
"NPM_VERSION": "11.20.0",
"PNPM_ARCHIVE_SHA256": "05b7b921fbb31564505c967eabf825895a1cc18f50935c00be98815272cc9d56",
"PNPM_VERSION": "12.6.0",
"PNPM_ARCHIVE_SHA256": "90762b8105e833164186b011efe5cde13841693ac8eaf784d2974a6e1c01fba3",
"PNPM_VERSION": "12.8.1",
"PYTEST_VERSION": "9.1.1",
"PYTHON_IMAGE": "python:3.12.14-slim-trixie@sha256:2c941e860699f878900b0edc2403613c234d4b32eda3cc9fa7036991a2a63c4a",
"PYTHON_VERSION": "3.12.14",
@@ -45,15 +45,15 @@
"RUFF_VERSION": "0.16.9",
"UV_LICENSE_APACHE_SHA256": "c71d239df91726fc519c6eb72d318ec65820627232b2f796219e87dcf35d0ab4",
"UV_LICENSE_MIT_SHA256": "860e3d7a86b84e6a7012c7a635fc64df475cebc6cce34dfeb73a5982ec58176c",
"UV_SHA256_AMD64": "23bf5552d220e0842b65c862097b2ebaeba0064b74eda5e565e77fd25969d8c8",
"UV_SHA256_ARM64": "0804e9b164c64b6914182d5920c08551958a095986f10a3731056df701126436",
"UV_VERSION": "0.12.19",
"UV_SHA256_AMD64": "23f02075b652bb1df64178cfae41b5caf160822e720e2663568f3f5d63bc52c0",
"UV_SHA256_ARM64": "030b69227b40af8c1981b7301793dc66e71ed3c796ea8688209dd268bd91ec51",
"UV_VERSION": "0.12.21",
"X_CRYPTO_VERSION": "0.57.0",
"X_MOD_VERSION": "0.41.0",
"X_TEXT_VERSION": "0.42.0",
"YQ_LICENSE_SHA256": "697db34dabb21562fe84487a2ccd031fbd45382b89c2cbdec8ef31682c486040",
"YQ_SHA256_AMD64": "c5f056448f973ae7d39b5401949648a78f2dc1947d6a8eb65be60d5c504b9385",
"YQ_SHA256_ARM64": "88a1016bc1d657375a35864e4f44b6f333df8ff97b559f51bba0adcb2169df09",
"YQ_VERSION": "4.53.6"
"YQ_SHA256_AMD64": "8e34fc298390875de416e6a4afcb8cabeceb25d9aa8506c1a2f9353cf702ea5f",
"YQ_SHA256_ARM64": "189088da0c6429ec5178dfaab1a114805f6cab0b61b165ab236efedf1d57a71b",
"YQ_VERSION": "4.54.1"
}
}
@@ -1,8 +1,8 @@
{
"version": "0.1.7-rc.2",
"version": "0.2.0-rc.2",
"repository": "deepseek-ai/deepseek-harness",
"ref": "dsh-v0.1.7-rc.2",
"commit": "477b4f420553e8a52c2fbccc464d7561b239c443",
"archiveSha256": "a6b78577dec0653bf336f6aa093c55dd93cb3350f220f91efde951b18c7eda83",
"archiveUrl": "https://codeload.github.com/deepseek-ai/deepseek-harness/tar.gz/477b4f420553e8a52c2fbccc464d7561b239c443"
"ref": "dsh-v0.2.0-rc.2",
"commit": "639ed015397290b3745d163aafe02ffee4aa3f84",
"archiveSha256": "bbc09e888e1df3aa37be049abdb7720951e76442d21e5432365d87a465d628f1",
"archiveUrl": "https://codeload.github.com/deepseek-ai/deepseek-harness/tar.gz/639ed015397290b3745d163aafe02ffee4aa3f84"
}
+1 -1
View File
@@ -4,7 +4,7 @@
"private": true,
"description": "Pinned runtime dependency closure for the DeepSeek Harness container image.",
"license": "MIT",
"packageManager": "pnpm@12.6.0",
"packageManager": "pnpm@12.8.1",
"engines": {
"node": "24.21.0"
},
@@ -14,15 +14,13 @@ const baseRequire = createRequire(basePath)
// These 0.1.6 request contributors are independent of the CLI's OTel switch.
// Preserve the image's opt-out for every profile, including explicit overlays.
// https://github.com/deepseek-ai/deepseek-harness/tree/dsh-v0.1.7-rc.2/packages/session/session-log-deepseek
// https://github.com/deepseek-ai/deepseek-harness/tree/dsh-v0.2.0-rc.2/packages/session/session-log-deepseek
const guards = [
['@deepseek-ai/dsh-session-log-deepseek', 'config.enabled !== true'],
['@deepseek-ai/dsh-plugin-package-inventory-deepseek', 'config.enabled === false'],
['@deepseek-ai/dsh-session-log-deepseek', ['config.enabled !== true', '!config.enabled.get()']],
['@deepseek-ai/dsh-plugin-package-inventory-deepseek', ['config.enabled === false']],
]
for (const [name, guard] of guards) {
for (const [name, supportedGuards] of guards) {
if (!Object.hasOwn(base.dependencies ?? {}, name)) continue // Older npm releases have neither contributor.
const original = `if (${guard})`
const replacement = `if (${guard} || process.env.DSH_TELEMETRY_DISABLED)`
const target = baseRequire.resolve(name)
const stat = await lstat(target)
const canonical = await realpath(target)
@@ -33,10 +31,24 @@ for (const [name, guard] of guards) {
}
const source = await readFile(target, 'utf8')
const count = needle => source.split(needle).length - 1
if (count(original) === 0 && count(replacement) === 1) continue
if (count(original) !== 1 || count(replacement) !== 0) {
const matches = supportedGuards.map(guard => [
`if (${guard})`, `if (${guard} || process.env.DSH_TELEMETRY_DISABLED)`,
]).filter(([original, replacement]) => count(original) + count(replacement) > 0)
if (matches.length !== 1) throw new Error(`unexpected telemetry activation guard in ${name}`)
const [original, replacement] = matches[0]
if (count(original) + count(replacement) !== 1) {
throw new Error(`unexpected telemetry activation guard in ${name}`)
}
await writeFile(target, source.replace(original, replacement), 'utf8')
let patched = source.replace(original, replacement)
// 0.2 registers first and reads the volatile switch inside prepare(). Keep the
// image opt-out at plugin activation as well as at request preparation.
if (original.includes('config.enabled.get()')) {
const entry = 'function apply(ctx, config) {'
const activation = `${entry}\n if (process.env.DSH_TELEMETRY_DISABLED) return;`
if (count(entry) !== 1) throw new Error(`unexpected telemetry plugin entry in ${name}`)
if (!patched.includes(activation)) patched = patched.replace(entry, activation)
}
if (patched === source) continue
await writeFile(target, patched, 'utf8')
process.stdout.write(`[dsh-patch] ${name} honors DSH_TELEMETRY_DISABLED\n`)
}
@@ -60,6 +60,40 @@ test('legacy releases without the request contributors remain supported', async
assert.equal(run(root).status, 0)
})
test('0.2 volatile session-log config retains the telemetry opt-out', async t => {
const root = await fixture(t)
const target = join(root, 'node_modules', names[0], 'index.js')
await writeFile(target, `function apply(ctx, config) {
ctx.register({ prepare() {
if (!config.enabled.get()) return undefined;
return 'session log';
} });
}
export { apply };\n`)
assert.equal(run(root).status, 0)
assert.equal(run(root).status, 0, 'patch is idempotent')
const { apply } = await import(pathToFileURL(target))
const previous = process.env.DSH_TELEMETRY_DISABLED
t.after(() => {
if (previous === undefined) delete process.env.DSH_TELEMETRY_DISABLED
else process.env.DSH_TELEMETRY_DISABLED = previous
})
for (const disabled of ['', '1']) {
process.env.DSH_TELEMETRY_DISABLED = disabled
for (const enabled of [false, true]) {
const registrations = []
apply({ register(value) { registrations.push(value) } }, { enabled: { get: () => enabled } })
assert.equal(registrations.length, disabled ? 0 : 1, 'opt-out prevents registration')
if (registrations.length) {
assert.equal(registrations[0].prepare(), enabled ? 'session log' : undefined)
process.env.DSH_TELEMETRY_DISABLED = '1'
assert.equal(registrations[0].prepare(), undefined, 'request-time opt-out still applies')
process.env.DSH_TELEMETRY_DISABLED = disabled
}
}
}
})
test('unknown activation code and missing declared packages fail closed', async t => {
const root = await fixture(t)
const target = join(root, 'node_modules', names[0], 'index.js')
@@ -1,20 +1,19 @@
#!/usr/bin/env python3
"""Publication must fail closed when a verified image or release input changes."""
from argparse import Namespace
import contextlib
import hashlib
import importlib.util
import io
import json
from pathlib import Path
import subprocess
import tarfile
import tempfile
import unittest
from argparse import Namespace
from pathlib import Path
from unittest.mock import patch
SPEC = importlib.util.spec_from_file_location("artifacts", Path(__file__).with_name("tested-image-artifact.py"))
artifacts = importlib.util.module_from_spec(SPEC)
SPEC.loader.exec_module(artifacts)
@@ -24,6 +23,25 @@ def encode(value):
return json.dumps(value, separators=(",", ":")).encode()
class FakeRegistryClient:
def __init__(self, case, repository):
self.case = case
self.repository = repository
def publish_platform_manifest(self, prepared):
if self.case.failed_repository == self.repository:
raise subprocess.CalledProcessError(1, "registry upload")
raw = prepared["path"].read_bytes()
manifest = json.loads(raw)
if self.case.corrupt_platform_repository == self.repository:
manifest["config"]["digest"] = "sha256:" + "e" * 64
raw = encode(manifest)
digest = "sha256:" + hashlib.sha256(raw).hexdigest()
self.case.registry_calls.append((self.repository, digest, manifest))
self.case.remote[f"{self.repository}@{digest}"] = raw
return digest
class TestedImageArtifactTests(unittest.TestCase):
def setUp(self):
self.temporary = tempfile.TemporaryDirectory()
@@ -38,15 +56,21 @@ class TestedImageArtifactTests(unittest.TestCase):
self.configs = {}
self.calls = []
self.remote = {}
self.tagged = {}
self.registry_calls = []
self.failed_repository = None
self.corrupt_platform_repository = None
self.write_artifact("amd64")
self.write_artifact("arm64")
def directory(self, arch):
return self.root / f"deepseek-harness-{self.expected['variant']}-image-{arch}"
def write_artifact(self, arch, *, config_changes=None, docker_layout=False):
def write_artifact(self, arch, *, config_changes=None, docker_layout=False, with_layer=False):
layer_content = b"verified layer bytes"
config = {"os": "linux", "architecture": arch,
"rootfs": {"type": "layers", "diff_ids": [
"sha256:" + hashlib.sha256(layer_content).hexdigest()
] if with_layer else []},
"config": {"Env": [f"DSH_IMAGE_VARIANT={self.expected['variant']}"], "Labels": {
"org.opencontainers.image.revision": self.expected["revision"],
"org.opencontainers.image.version": self.expected["dsh_version"]}}}
@@ -57,7 +81,9 @@ class TestedImageArtifactTests(unittest.TestCase):
image_id = "sha256:" + digest
self.configs[image_id] = config
config_name = f"{digest}.json" if docker_layout else f"blobs/sha256/{digest}"
manifest = [{"Config": config_name, "RepoTags": None, "Layers": []}]
layer_name = "layer.tar"
manifest = [{"Config": config_name, "RepoTags": None,
"Layers": [layer_name] if with_layer else []}]
directory = self.directory(arch)
directory.mkdir(exist_ok=True)
with tarfile.open(directory / "image.tar.gz", "w:gz") as saved:
@@ -65,6 +91,11 @@ class TestedImageArtifactTests(unittest.TestCase):
member = tarfile.TarInfo(name)
member.size = len(content)
saved.addfile(member, io.BytesIO(content))
if with_layer:
content = layer_content
member = tarfile.TarInfo(layer_name)
member.size = len(content)
saved.addfile(member, io.BytesIO(content))
receipt = {"schema_version": 1, **self.expected, "platform": f"linux/{arch}",
"image_id": image_id, "archive_sha256": artifacts.sha256(directory / "image.tar.gz")}
(directory / "receipt.json").write_text(json.dumps(receipt), encoding="utf-8")
@@ -85,15 +116,6 @@ class TestedImageArtifactTests(unittest.TestCase):
config = self.configs[image_id]
return encode([{"Id": image_id, "Os": config["os"], "Architecture": config["architecture"],
"Config": config["config"]}])
if args[:2] == ("image", "tag"):
self.tagged[args[3]] = args[2]
return None
if args[:2] == ("image", "push"):
reference = args[2]
manifest = {"schemaVersion": 2, "mediaType": "application/vnd.oci.image.manifest.v1+json",
"config": {"digest": self.tagged[reference]}}
self.remote[reference] = encode(manifest)
return None
if args[:4] == ("buildx", "imagetools", "inspect", "--raw"):
return self.remote[args[4]]
if args[:3] == ("buildx", "imagetools", "create"):
@@ -113,31 +135,68 @@ class TestedImageArtifactTests(unittest.TestCase):
self.fail(f"unexpected Docker operation (especially a rebuild): {args}")
def publish(self, docker=None):
with patch.object(artifacts, "docker", docker or self.docker), contextlib.redirect_stdout(io.StringIO()):
with patch.object(artifacts, "docker", docker or self.docker), \
patch.object(artifacts, "create_registry_client",
lambda repository: FakeRegistryClient(self, repository)), \
contextlib.redirect_stdout(io.StringIO()):
artifacts.publish_images(self.options, self.expected)
def assert_rejected_without_registry_writes(self):
with self.assertRaises((ValueError, tarfile.TarError)):
self.publish()
self.assertFalse(any(call[:2] in (("image", "push"), ("image", "tag"))
or call[:3] == ("buildx", "imagetools", "create") for call in self.calls))
self.assertFalse(self.registry_calls)
self.assertFalse(any(call[:3] == ("buildx", "imagetools", "create") for call in self.calls))
def test_publishes_tested_config_ids_to_both_registries_before_floating_tags(self):
def test_publishes_tested_manifests_by_digest_before_normal_tags(self):
self.publish()
pushes = [call for call in self.calls if call[:2] == ("image", "push")]
self.assertEqual(len(pushes), 4)
self.assertEqual(len(self.registry_calls), 4)
first_manifest = next(i for i, call in enumerate(self.calls) if call[:3] == ("buildx", "imagetools", "create"))
self.assertEqual(sum(call[:2] == ("image", "push") for call in self.calls[:first_manifest]), 4)
self.assertTrue(all("ci-" not in str(call) for call in self.calls[:first_manifest]))
creates = [call for call in self.calls if call[:3] == ("buildx", "imagetools", "create")]
self.assertEqual([call[5].rsplit(":", 1)[1] for call in creates],
["0.1.5-rc.1", "0.1.5-rc.1", "latest", "latest"])
self.assertEqual(set(self.tagged.values()), set(self.configs))
self.assertTrue(all("@sha256:" in source for call in creates for source in call[6:]))
self.assertTrue(all("ci-" not in source for call in creates for source in call[6:]))
self.assertEqual({call[2]["config"]["digest"] for call in self.registry_calls}, set(self.configs))
def test_supports_both_docker_archive_config_layouts(self):
self.write_artifact("amd64", docker_layout=True)
self.publish()
def test_prepares_uncompressed_layers_as_gzip_blobs(self):
receipt = self.write_artifact("amd64", with_layer=True)
with tempfile.TemporaryDirectory() as staging:
prepared = artifacts.prepare_platform_manifest(
self.directory("amd64"), receipt, Path(staging))
self.assertEqual(len(prepared["manifest"]["layers"]), 1)
layer = prepared["manifest"]["layers"][0]
self.assertEqual(layer["mediaType"], "application/vnd.docker.image.rootfs.diff.tar.gzip")
self.assertEqual(prepared["blobs"][layer["digest"]].read_bytes()[:2], b"\x1f\x8b")
def test_registry_client_uploads_blobs_and_manifest_by_digest_without_a_tag(self):
receipt = self.write_artifact("amd64", with_layer=True)
with tempfile.TemporaryDirectory() as staging:
prepared = artifacts.prepare_platform_manifest(
self.directory("amd64"), receipt, Path(staging))
client = artifacts.RegistryClient("ghcr.io/okxlin/deepseek-harness", "user", "token")
client.token = "access-token"
requests = []
def send(method, url, headers, *, body=None, body_path=None):
requests.append((method, url, body, body_path))
if method == "HEAD":
return 404, {}
if method == "POST":
return 202, {"location": "https://ghcr.io/v2/okxlin/deepseek-harness/blobs/uploads/1"}
return 201, {}
with patch.object(client, "_send", side_effect=send):
self.assertEqual(client.publish_platform_manifest(prepared), prepared["digest"])
self.assertEqual(sum(method == "PUT" for method, *_ in requests), 3)
manifest_put = next(url for method, url, *_ in requests if method == "PUT" and "/manifests/" in url)
self.assertIn("/manifests/sha256:", manifest_put)
self.assertNotIn(":ci-", manifest_put)
def test_workstation_arm_only_publishes_only_the_requested_platform(self):
self.root = self.root / "arm-only"
self.root.mkdir()
@@ -148,8 +207,8 @@ class TestedImageArtifactTests(unittest.TestCase):
self.options.latest_tag = "workstation"
receipt = self.write_artifact("arm64")
self.publish()
self.assertEqual(set(self.tagged.values()), {receipt["image_id"]})
self.assertEqual(len(self.tagged), 2)
self.assertEqual({call[2]["config"]["digest"] for call in self.registry_calls}, {receipt["image_id"]})
self.assertEqual(len(self.registry_calls), 2)
def test_dry_run_checks_artifacts_without_docker_or_registry_access(self):
self.options.dry_run = True
@@ -212,24 +271,15 @@ class TestedImageArtifactTests(unittest.TestCase):
self.assertFalse(any(call[:2] == ("image", "push") for call in self.calls))
def test_registry_push_failure_never_updates_release_tags(self):
def failed_push(*args, **kwargs):
if args[:2] == ("image", "push") and args[2].startswith("docker.io/"):
raise subprocess.CalledProcessError(1, "docker push")
return self.docker(*args, **kwargs)
self.failed_repository = "docker.io/okxlin/deepseek-harness"
with self.assertRaises(subprocess.CalledProcessError):
self.publish(failed_push)
self.publish()
self.assertFalse(any(call[:3] == ("buildx", "imagetools", "create") for call in self.calls))
def test_registry_config_mismatch_never_updates_release_tags(self):
def wrong_config(*args, **kwargs):
result = self.docker(*args, **kwargs)
if args[:4] == ("buildx", "imagetools", "inspect", "--raw"):
manifest = json.loads(result)
manifest["config"]["digest"] = "sha256:" + "e" * 64
return encode(manifest)
return result
with self.assertRaisesRegex(ValueError, "pushed image does not match"):
self.publish(wrong_config)
self.corrupt_platform_repository = "ghcr.io/okxlin/deepseek-harness"
with self.assertRaisesRegex(ValueError, "unexpected platform digest|published image digest changed"):
self.publish()
self.assertFalse(any(call[:3] == ("buildx", "imagetools", "create") for call in self.calls))
def test_incomplete_version_manifest_never_updates_floating_tags(self):
@@ -2,27 +2,33 @@
"""Export verified images and publish those exact images without rebuilding."""
import argparse
import gzip
import hashlib
import json
from pathlib import Path
import re
import shutil
import subprocess
import sys
import tarfile
import tempfile
from pathlib import Path
LIBRARY = Path(__file__).resolve().parents[2] / "scripts"
if str(LIBRARY) not in sys.path:
sys.path.insert(0, str(LIBRARY))
from registry_image import (
DIGEST,
IMAGE_TYPES,
INDEX_TYPES,
RegistryClient, # noqa: F401 - re-exported for the focused Registry V2 test
archive_metadata,
create_registry_client,
save_image,
)
from registry_image import (
prepare_platform_manifest as prepare_image_manifest,
)
DIGEST = re.compile(r"sha256:[a-f0-9]{64}")
PLATFORMS = ("linux/amd64", "linux/arm64")
IMAGE_TYPES = {
"application/vnd.oci.image.manifest.v1+json",
"application/vnd.docker.distribution.manifest.v2+json",
}
INDEX_TYPES = {
"application/vnd.oci.image.index.v1+json",
"application/vnd.docker.distribution.manifest.list.v2+json",
}
def require(condition, message):
@@ -80,23 +86,7 @@ def check_archive(directory, expected):
require(sha256(archive) == receipt.get("archive_sha256"), f"archive checksum mismatch: {archive}")
# Read metadata without extracting any archive paths into the host filesystem.
digest = receipt["image_id"].removeprefix("sha256:")
config_names = {f"{digest}.json", f"blobs/sha256/{digest}"}
metadata = {}
with tarfile.open(archive, "r|gz") as saved:
for member in saved:
if member.name not in config_names | {"manifest.json"}:
continue
require(member.isfile() and member.size <= 4 * 1024 * 1024,
"invalid image archive metadata member")
require(member.name not in metadata, "duplicate image archive metadata")
metadata[member.name] = saved.extractfile(member).read()
manifest = json.loads(metadata.get("manifest.json", b"null"))
require(isinstance(manifest, list) and len(manifest) == 1, "archive must contain exactly one image")
config_name = manifest[0].get("Config")
require(config_name in config_names and config_name in metadata, "archive config does not match tested image ID")
config_bytes = metadata[config_name]
require(hashlib.sha256(config_bytes).hexdigest() == digest, "archive image config digest mismatch")
config_bytes, _ = archive_metadata(archive, receipt["image_id"])
check_config(json.loads(config_bytes), receipt)
return receipt
@@ -116,11 +106,7 @@ def export_image(options, expected):
flush=True)
archive = directory / "image.tar.gz"
# Saving by config ID binds the archive to the tested image, even if a tag moves.
with archive.open("xb") as target:
with subprocess.Popen(["docker", "image", "save", options.image_id], stdout=subprocess.PIPE) as saved:
with gzip.GzipFile(filename="", mode="wb", fileobj=target, compresslevel=1, mtime=0) as zipped:
shutil.copyfileobj(saved.stdout, zipped)
require(saved.wait() == 0, "docker image save failed")
save_image(options.image_id, archive)
receipt = {"schema_version": 1, **expected, "archive_sha256": sha256(archive)}
(directory / "receipt.json").write_text(json.dumps(receipt, indent=2) + "\n", encoding="utf-8")
print(json.dumps(receipt, indent=2))
@@ -132,6 +118,11 @@ def remote_manifest(reference):
return json.loads(raw), "sha256:" + hashlib.sha256(raw).hexdigest()
def prepare_platform_manifest(directory, receipt, destination):
"""Convert one verified Docker-save archive into a digest-addressed manifest."""
return prepare_image_manifest(directory / "image.tar.gz", receipt["image_id"], destination)
def check_index(reference, expected_digests):
manifest, digest = remote_manifest(reference)
require(manifest.get("schemaVersion") == 2 and manifest.get("mediaType") in INDEX_TYPES,
@@ -172,34 +163,41 @@ def publish_images(options, expected):
return
# Validate every archive and loaded image before making the first registry write.
for directory, receipt in artifacts:
docker("image", "load", "--input", directory / "image.tar.gz")
inspect_image(receipt["image_id"], receipt)
with tempfile.TemporaryDirectory(prefix="deepseek-harness-manifests-") as staging:
prepared = []
for directory, receipt in artifacts:
prepared.append((receipt, prepare_platform_manifest(
directory, receipt, Path(staging) / receipt["platform"].split("/")[1])))
for directory, receipt in artifacts:
docker("image", "load", "--input", directory / "image.tar.gz")
inspect_image(receipt["image_id"], receipt)
staged = {}
for repository in options.repository:
staged[repository] = {}
for _, receipt in artifacts:
arch = receipt["platform"].split("/")[1]
reference = f"{repository}:ci-{expected['run_id']}-{options.run_attempt}-{expected['variant']}-{arch}"
docker("image", "tag", receipt["image_id"], reference)
docker("image", "push", reference)
manifest, digest = remote_manifest(reference)
require(manifest.get("schemaVersion") == 2 and manifest.get("mediaType") in IMAGE_TYPES
and manifest.get("config", {}).get("digest") == receipt["image_id"],
f"pushed image does not match tested image ID: {reference}")
staged[repository][receipt["platform"]] = digest
clients = {repository: create_registry_client(repository) for repository in options.repository}
staged = {}
for repository, client in clients.items():
staged[repository] = {}
for receipt, platform_manifest in prepared:
digest = client.publish_platform_manifest(platform_manifest)
require(digest == platform_manifest["digest"],
f"registry returned an unexpected platform digest: {repository}")
reference = f"{repository}@{digest}"
manifest, remote_digest = remote_manifest(reference)
require(manifest.get("schemaVersion") == 2 and manifest.get("mediaType") in IMAGE_TYPES
and manifest.get("config", {}).get("digest") == receipt["image_id"],
f"published image does not match tested image ID: {reference}")
require(remote_digest == digest, f"published image digest changed: {reference}")
staged[repository][receipt["platform"]] = digest
# Both registries must have all images before updating release tags. Floating
# tags follow only after the version tag has been verified in both registries.
published = []
for tag in tags:
for repository, digests in staged.items():
reference = f"{repository}:{tag}"
docker("buildx", "imagetools", "create", "--prefer-index=true", "--tag", reference,
*(f"{repository}@{digest}" for digest in digests.values()))
digest = check_index(reference, digests)
published.append(f"{reference} -> {digest}")
# Both registries must have all images before updating release tags. Floating
# tags follow only after the version tag has been verified in both registries.
published = []
for tag in tags:
for repository, digests in staged.items():
reference = f"{repository}:{tag}"
docker("buildx", "imagetools", "create", "--prefer-index=true", "--tag", reference,
*(f"{repository}@{digest}" for digest in digests.values()))
digest = check_index(reference, digests)
published.append(f"{reference} -> {digest}")
for item in published:
print(item)
if options.summary:
@@ -239,7 +237,8 @@ def main():
else:
publish_images(options, expected)
return 0
except (OSError, ValueError, KeyError, TypeError, AttributeError, tarfile.TarError, subprocess.SubprocessError) as exc:
except (OSError, ValueError, KeyError, TypeError, AttributeError, RuntimeError,
tarfile.TarError, subprocess.SubprocessError) as exc:
print(f"ERROR: {exc}", file=sys.stderr)
return 1
@@ -792,9 +792,9 @@
}
},
"node_modules/brace-expansion": {
"version": "1.1.18",
"resolved": "https://registry.npmjs.org/brace-expansion/-/brace-expansion-1.1.18.tgz",
"integrity": "sha512-Edep/X9fGqVNmzKBVsDYIOtD+z1tuezV70LBjdCst9Tqu76lsnvRiZ6oTic1n+/BIwX6QDGAO94PN4N2SADvtw==",
"version": "1.1.21",
"resolved": "https://registry.npmjs.org/brace-expansion/-/brace-expansion-1.1.21.tgz",
"integrity": "sha512-9zeA+KLZNNzglF2TPKRQEDyx6Yby7daAkuy8MiPzpXPsYDWi/DRM8jmwUDxokQjYqBpv5DgPiwD4h4ZZSy1Ujw==",
"license": "MIT",
"dependencies": {
"balanced-match": "^1.0.0",
@@ -1217,9 +1217,9 @@
"license": "MIT"
},
"node_modules/fast-uri": {
"version": "3.1.7",
"resolved": "https://registry.npmjs.org/fast-uri/-/fast-uri-3.1.7.tgz",
"integrity": "sha512-dOvZVzjdZdz7phd9v6jCbwxrBW3fK6n8Rc0CtdmM4bumzMnxywBYhuph6J819RRw/ku+rLbelwfMunktuzVVHg==",
"version": "3.1.8",
"resolved": "https://registry.npmjs.org/fast-uri/-/fast-uri-3.1.8.tgz",
"integrity": "sha512-GZMtZUTNRpOVIECoXwLNZS5xUGE+mVNbTB8h/7Rwh2TFWcBQiPzTgyZi05BF9UMZKkLJv8XBRJTlU7zg8+ZfMg==",
"funding": [
{
"type": "github",
@@ -1505,9 +1505,9 @@
"license": "ISC"
},
"node_modules/ip-address": {
"version": "10.7.0",
"resolved": "https://registry.npmjs.org/ip-address/-/ip-address-10.7.0.tgz",
"integrity": "sha512-BGFsyJd5mpXp3rK6jIdADLNgpJUK1jnjzvYF8lK+VyDab9JAmqN0YOKDdP17HlgKb2+ehPgDc8EtnRLbGCAMhA==",
"version": "10.7.2",
"resolved": "https://registry.npmjs.org/ip-address/-/ip-address-10.7.2.tgz",
"integrity": "sha512-7H/2gFSIitxc0hG3nOI1glS8QLo/EHBFFLk8vEUjXY/xu0AdL8jZ9U1IzO2PUm0d2D/ofQcAifb0g6OBkt8U7w==",
"license": "MIT",
"engines": {
"node": ">= 12"
+15 -1
View File
@@ -18,5 +18,19 @@
"sha512": "5e57abc8c85bba4b1d2ada1f9d667d4a4fb80944db9ace0c6ff114df6499c13a00c99e51815a2cfe48bc9fecabd0b5873862ae6a46d74eeb983472d08e175d82"
},
"test_daemon_image": "docker.io/library/docker:29-dind@sha256:5efed980cba3fc126cf54e21a5a6ff8849d05b6e0623d6e7612f48e9cd6cd17e",
"test_sandbox_image": "docker.io/library/busybox:1.37@sha256:9db7b59979c38555a39def84a31fb98b5296952f9e3afd4f6f11f05b07adfab0"
"test_sandbox_image": "docker.io/library/busybox:1.37@sha256:9db7b59979c38555a39def84a31fb98b5296952f9e3afd4f6f11f05b07adfab0",
"npm_bundled": {
"brace-expansion": {
"from": "5.0.9",
"version": "5.0.12",
"url": "https://registry.npmjs.org/brace-expansion/-/brace-expansion-5.0.12.tgz",
"sha512": "628bd0debce168b308ac38cd0cc90d4b4d6d79af77aa11211b9c72f1febe47497e770dca8bee6c0a822823de368ae2d94c75a881692d830543854d3d88d12299"
},
"undici": {
"from": "6.28.0",
"version": "6.28.1",
"url": "https://registry.npmjs.org/undici/-/undici-6.28.1.tgz",
"sha512": "cd6a5d4d50f9e07e3c088c9b2f4ad6437ba4a5bf5ddb7c0cede1f946d7dd99e3fbd1c587363b5fa3b3f8bd95fee42a0370ee772783eee6e5e9ee535f8dce8344"
}
}
}
+61 -1
View File
@@ -34,8 +34,68 @@ export function verifyArchiveReader(directory) {
assert.equal(oversized, false, 'archive reader trusted the attacker-declared size');
}
export async function patchNpmBundles(root, pins) {
root = fs.realpathSync(root);
const store = path.join(root, '.pnpm');
for (const [name, pin] of Object.entries(pins)) {
assert.ok(['brace-expansion', 'undici'].includes(name));
assert.match(pin.version, /^\d+\.\d+\.\d+$/);
assert.equal(pin.url, `https://registry.npmjs.org/${name}/-/${name}-${pin.version}.tgz`);
assert.match(pin.sha512, /^[a-f0-9]{128}$/);
const outdated = [];
for (const entry of fs.readdirSync(store)) {
if (!/^npm@\d/.test(entry)) continue;
const directory = path.join(store, entry, 'node_modules/npm/node_modules', name);
if (!fs.existsSync(directory)) continue;
assert.equal(fs.realpathSync(directory), directory, 'unexpected bundled npm dependency path');
const pkg = JSON.parse(fs.readFileSync(path.join(directory, 'package.json')));
assert.equal(pkg.name, name);
if (pkg.version === pin.from) outdated.push(directory);
else {
assert.match(pkg.version, /^\d+\.\d+\.\d+$/);
const fixed = pin.version.split('.').map(Number);
assert.ok(pkg.version.split('.').map(Number).reduce((order, value, index) =>
order || value - fixed[index], 0) >= 0, `unexpected vulnerable ${name} version`);
}
}
if (!outdated.length) continue;
const temporary = fs.mkdtempSync('/tmp/openclaw-npm-bundle-');
try {
const response = await fetch(pin.url, {redirect: 'error', signal: AbortSignal.timeout(60000)});
assert.equal(response.status, 200);
const chunks = [];
let size = 0;
for await (const chunk of response.body) {
size += chunk.length;
assert.ok(size <= 4 * 1024 * 1024, 'npm dependency archive exceeds expected size');
chunks.push(chunk);
}
const bytes = Buffer.concat(chunks);
assert.equal(createHash('sha512').update(bytes).digest('hex'), pin.sha512);
const archive = path.join(temporary, 'dependency.tgz');
fs.writeFileSync(archive, bytes);
execFileSync('tar', ['-xzf', archive, '--no-same-owner', '--no-same-permissions', '-C', temporary]);
const replacement = path.join(temporary, 'package');
const pkg = JSON.parse(fs.readFileSync(path.join(replacement, 'package.json')));
assert.equal(pkg.name, name);
assert.equal(pkg.version, pin.version);
for (const directory of outdated) {
const {uid, gid} = fs.statSync(directory);
fs.rmSync(directory, {recursive: true});
fs.cpSync(replacement, directory, {recursive: true});
execFileSync('chown', ['-R', `${uid}:${gid}`, directory]);
console.log(`Replace verified bundled npm dependency ${name} ${pin.from} -> ${pin.version}: ${directory}`);
}
} finally {
fs.rmSync(temporary, {recursive: true});
}
}
}
async function main() {
const pin = JSON.parse(fs.readFileSync(new URL('./components.json', import.meta.url))).adm_zip;
const components = JSON.parse(fs.readFileSync(new URL('./components.json', import.meta.url)));
await patchNpmBundles('/app/node_modules', components.npm_bundled);
const pin = components.adm_zip;
assert.match(pin.version, /^\d+\.\d+\.\d+$/);
assert.equal(pin.url, `https://registry.npmjs.org/adm-zip/-/adm-zip-${pin.version}.tgz`);
assert.match(pin.sha512, /^[a-f0-9]{128}$/);
+22 -8
View File
@@ -3,10 +3,20 @@
import argparse
import hashlib
import json
from pathlib import Path
import re
import subprocess
import sys
import tempfile
from pathlib import Path
LIBRARY = Path(__file__).resolve().parent
if str(LIBRARY) not in sys.path:
sys.path.insert(0, str(LIBRARY))
from registry_image import (
create_registry_client,
prepare_platform_manifest,
save_image,
)
REPOSITORIES = {
"codex": "codex-claude-workstation", "opencode": "opencode-workstation",
@@ -59,12 +69,15 @@ def stage(args, expected):
receipt_path = args.receipt
require(not receipt_path.exists() and not receipt_path.is_symlink(), "receipt must be a new file")
receipt_path.parent.mkdir(parents=True, exist_ok=True)
reference = f"{args.repository}:ci-{args.run_id}-{args.run_attempt}-{args.variant}-{args.platform.split('/')[1]}"
# Tag the recorded ID, never a mutable local tag.
docker("image", "tag", args.image_id, reference)
docker("image", "push", reference)
manifest, digest = remote_manifest(reference)
check_manifest(manifest, args.image_id)
with tempfile.TemporaryDirectory(prefix="tested-image-") as staging:
archive = Path(staging) / "image.tar.gz"
save_image(args.image_id, archive)
prepared = prepare_platform_manifest(archive, args.image_id, Path(staging) / "manifest")
digest = create_registry_client(args.repository).publish_platform_manifest(prepared)
reference = f"{args.repository}@{digest}"
manifest, remote_digest = remote_manifest(reference)
require(remote_digest == digest, "registry manifest digest mismatch after publication")
check_manifest(manifest, args.image_id)
receipt = {"schema_version": 1, **expected, "platform": args.platform,
"image_id": args.image_id, "manifest_digest": digest}
with receipt_path.open("x", encoding="utf-8") as output:
@@ -163,6 +176,7 @@ def main():
if __name__ == "__main__":
try:
main()
except (ValueError, OSError, subprocess.CalledProcessError) as error:
except (ValueError, OSError, KeyError, TypeError, AttributeError, RuntimeError,
subprocess.CalledProcessError) as error:
print(f"ERROR: {error}", file=sys.stderr)
sys.exit(1)
+318
View File
@@ -0,0 +1,318 @@
"""Publish verified Docker-save images without creating staging tags."""
import base64
import gzip
import hashlib
import http.client
import json
import os
import re
import shutil
import subprocess
import tarfile
import urllib.error
import urllib.parse
import urllib.request
DIGEST = re.compile(r"sha256:[a-f0-9]{64}")
IMAGE_TYPES = {
"application/vnd.oci.image.manifest.v1+json",
"application/vnd.docker.distribution.manifest.v2+json",
}
INDEX_TYPES = {
"application/vnd.oci.image.index.v1+json",
"application/vnd.docker.distribution.manifest.list.v2+json",
}
REGISTRY_CONFIG = {
"ghcr.io": {
"registry_host": "ghcr.io",
"token_url": "https://ghcr.io/token",
"service": "ghcr.io",
"upload_hosts": {"ghcr.io"},
},
"docker.io": {
"registry_host": "registry-1.docker.io",
"token_url": "https://auth.docker.io/token",
"service": "registry.docker.io",
"upload_hosts": {"registry-1.docker.io", "docker.io"},
},
}
BLOB_CHUNK_SIZE = 1024 * 1024
def require(condition, message):
if not condition:
raise ValueError(message)
def sha256(path):
with path.open("rb") as source:
return hashlib.file_digest(source, "sha256").hexdigest()
def save_image(image_id, archive):
"""Save one local image as the gzip-wrapped Docker archive format."""
with archive.open("xb") as target, subprocess.Popen(
["docker", "image", "save", image_id], stdout=subprocess.PIPE
) as saved, gzip.GzipFile(filename="", mode="wb", fileobj=target,
compresslevel=1, mtime=0) as zipped:
shutil.copyfileobj(saved.stdout, zipped, BLOB_CHUNK_SIZE)
require(saved.wait() == 0, "docker image save failed")
def archive_metadata(archive, image_id):
digest = image_id.removeprefix("sha256:")
config_names = {f"{digest}.json", f"blobs/sha256/{digest}"}
metadata = {}
with tarfile.open(archive, "r|gz") as saved:
for member in saved:
if member.name not in config_names | {"manifest.json"}:
continue
require(member.isfile() and member.size <= 4 * 1024 * 1024,
"invalid image archive metadata member")
require(member.name not in metadata, "duplicate image archive metadata")
source = saved.extractfile(member)
require(source is not None, f"cannot read image archive metadata: {member.name}")
with source:
metadata[member.name] = source.read()
manifest = json.loads(metadata.get("manifest.json", b"null"))
require(isinstance(manifest, list) and len(manifest) == 1
and isinstance(manifest[0], dict), "archive must contain exactly one image")
config_name = manifest[0].get("Config")
require(isinstance(config_name, str) and config_name in config_names and config_name in metadata,
"archive config does not match tested image ID")
config_bytes = metadata[config_name]
require(hashlib.sha256(config_bytes).hexdigest() == digest, "archive image config digest mismatch")
return config_bytes, manifest[0]
def prepare_platform_manifest(archive, image_id, destination):
"""Convert one Docker-save archive into a digest-addressed platform manifest."""
require(DIGEST.fullmatch(image_id), "invalid image ID")
config_bytes, saved_manifest = archive_metadata(archive, image_id)
layer_names = saved_manifest.get("Layers")
require(isinstance(layer_names, list) and all(isinstance(name, str) and name for name in layer_names),
"archive image layers are invalid")
config = json.loads(config_bytes)
require(isinstance(config, dict), "image config is invalid")
rootfs = config.get("rootfs")
require(isinstance(rootfs, dict) and rootfs.get("type") == "layers", "image rootfs is invalid")
diff_ids = rootfs.get("diff_ids")
require(isinstance(diff_ids, list) and len(diff_ids) == len(layer_names)
and all(isinstance(value, str) and DIGEST.fullmatch(value) for value in diff_ids),
"image rootfs layer digests are invalid")
destination.mkdir(parents=True, exist_ok=True)
config_path = destination / "config.json"
config_path.write_bytes(config_bytes)
blobs = {image_id: config_path}
layers = {}
uncompressed_digests = {}
found_layers = set()
with tarfile.open(archive, "r|gz") as saved:
for member in saved:
if member.name not in layer_names:
continue
require(member.isfile(), f"image archive layer is not a regular file: {member.name}")
require(member.name not in found_layers, f"duplicate image archive layer member: {member.name}")
found_layers.add(member.name)
source = saved.extractfile(member)
require(source is not None, f"cannot read image archive layer: {member.name}")
require(member.size >= 0 and member.size <= shutil.disk_usage(destination).free,
f"image archive layer exceeds available staging space: {member.name}")
raw_layer = destination / f"layer-{len(layers)}.tar"
with source, raw_layer.open("wb") as target:
shutil.copyfileobj(source, target, BLOB_CHUNK_SIZE)
require(raw_layer.stat().st_size == member.size,
f"image archive layer size changed while reading: {member.name}")
compressed_layer = destination / f"layer-{len(layers)}.tar.gz"
with raw_layer.open("rb") as source:
magic = source.read(4)
require(magic != b"\x28\xb5\x2f\xfd", "zstd image archive layers are not supported")
if magic.startswith(b"\x1f\x8b"):
with gzip.open(raw_layer, "rb") as source:
uncompressed_digests[member.name] = "sha256:" + hashlib.file_digest(source, "sha256").hexdigest()
shutil.copyfile(raw_layer, compressed_layer)
else:
uncompressed_digests[member.name] = "sha256:" + sha256(raw_layer)
with raw_layer.open("rb") as source, compressed_layer.open("wb") as target, \
gzip.GzipFile(filename="", mode="wb", fileobj=target,
compresslevel=1, mtime=0) as zipped:
shutil.copyfileobj(source, zipped, BLOB_CHUNK_SIZE)
raw_layer.unlink()
digest = "sha256:" + sha256(compressed_layer)
blobs[digest] = compressed_layer
layers[member.name] = {
"mediaType": "application/vnd.docker.image.rootfs.diff.tar.gzip",
"size": compressed_layer.stat().st_size,
"digest": digest,
}
require(found_layers == set(layer_names), "image archive is missing a declared layer")
require([uncompressed_digests[name] for name in layer_names] == diff_ids,
"image archive layer content does not match tested rootfs")
manifest = {
"schemaVersion": 2,
"mediaType": "application/vnd.docker.distribution.manifest.v2+json",
"config": {
"mediaType": "application/vnd.docker.container.image.v1+json",
"size": len(config_bytes),
"digest": image_id,
},
# Docker save may write tar members in digest order, not filesystem order.
"layers": [layers[name] for name in layer_names],
}
raw_manifest = json.dumps(manifest, separators=(",", ":")).encode()
manifest_path = destination / "manifest.json"
manifest_path.write_bytes(raw_manifest)
return {
"path": manifest_path,
"digest": "sha256:" + hashlib.sha256(raw_manifest).hexdigest(),
"manifest": manifest,
"blobs": blobs,
}
class _NoRedirect(urllib.request.HTTPRedirectHandler):
def redirect_request(self, request, fp, code, msg, headers, new_url):
return None
class RegistryClient:
"""Minimal Registry V2 client for digest-only platform publication."""
def __init__(self, repository, username, password):
registry, name = repository.split("/", 1)
settings = REGISTRY_CONFIG[registry]
self.repository = name
self.registry_host = settings["registry_host"]
self.token_url = settings["token_url"]
self.service = settings["service"]
self.upload_hosts = settings["upload_hosts"]
self.username = username
self.password = password
self.token = None
def _get_token(self):
query = urllib.parse.urlencode({
"service": self.service,
"scope": f"repository:{self.repository}:pull,push",
})
credentials = base64.b64encode(f"{self.username}:{self.password}".encode()).decode()
request = urllib.request.Request(
f"{self.token_url}?{query}",
headers={"Accept": "application/json", "Authorization": f"Basic {credentials}"},
)
opener = urllib.request.build_opener(urllib.request.ProxyHandler({}), _NoRedirect())
try:
with opener.open(request, timeout=60) as response:
payload = json.loads(response.read(4 * 1024 * 1024))
except (urllib.error.HTTPError, urllib.error.URLError, TimeoutError, json.JSONDecodeError) as exc:
raise RuntimeError(f"registry token request failed for {self.repository}: {type(exc).__name__}") from None
require(isinstance(payload, dict), "registry token response is invalid")
token = payload.get("token") or payload.get("access_token")
require(isinstance(token, str) and token and not any(char.isspace() for char in token),
"registry token response is invalid")
self.token = token
return token
def _request(self, method, url, expected, *, body=None, body_path=None, content_type=None):
require(body is None or body_path is None, "registry request body is ambiguous")
for attempt in range(2):
headers = {"Authorization": f"Bearer {self.token or self._get_token()}"}
if body is not None:
headers["Content-Length"] = str(len(body))
if body_path is not None:
headers["Content-Length"] = str(body_path.stat().st_size)
if content_type:
headers["Content-Type"] = content_type
status, response_headers = self._send(method, url, headers, body=body, body_path=body_path)
if status != 401:
require(status in expected,
f"registry {method} failed for {self.repository}: HTTP {status}")
return status, response_headers
self.token = None
if attempt == 1:
raise RuntimeError(f"registry authorization failed for {self.repository}")
raise RuntimeError("registry request retry failed")
@staticmethod
def _send(method, url, headers, *, body=None, body_path=None):
parsed = urllib.parse.urlsplit(url)
require(parsed.scheme == "https" and parsed.hostname, "registry URL must use HTTPS")
target = urllib.parse.urlunsplit(("", "", parsed.path or "/", parsed.query, ""))
connection = http.client.HTTPSConnection(parsed.hostname, parsed.port, timeout=120)
try:
connection.putrequest(method, target)
for key, value in headers.items():
connection.putheader(key, value)
connection.endheaders()
if body is not None:
connection.send(body)
elif body_path is not None:
with body_path.open("rb") as source:
while chunk := source.read(BLOB_CHUNK_SIZE):
connection.send(chunk)
response = connection.getresponse()
response_headers = {key.lower(): value for key, value in response.getheaders()}
response.read(4 * 1024 * 1024 + 1)
return response.status, response_headers
finally:
connection.close()
def _upload_url(self, location):
upload_url = urllib.parse.urljoin(f"https://{self.registry_host}", location)
parsed = urllib.parse.urlsplit(upload_url)
require(parsed.scheme == "https" and parsed.hostname in self.upload_hosts
and not parsed.username and not parsed.password,
"registry upload location is not an allowed registry host")
return upload_url
def _path(self, suffix):
return f"https://{self.registry_host}/v2/{self.repository}/{suffix}"
def _upload_blob(self, digest, path):
blob_url = self._path(f"blobs/{digest}")
status, _ = self._request("HEAD", blob_url, (200, 404))
if status == 200:
return
_, headers = self._request("POST", self._path("blobs/uploads/"), (202,))
location = headers.get("location")
require(location, "registry did not return a blob upload location")
upload_url = self._upload_url(location)
parsed = urllib.parse.urlsplit(upload_url)
# Upload locations can contain opaque signed state; preserve its encoding.
# https://github.com/opencontainers/distribution-spec/blob/main/spec.md#post-then-put
query = parsed.query + ("&" if parsed.query else "") + urllib.parse.urlencode({"digest": digest})
upload_url = urllib.parse.urlunsplit((parsed.scheme, parsed.netloc, parsed.path,
query, ""))
self._request("PUT", upload_url, (201,), body_path=path,
content_type="application/octet-stream")
def publish_platform_manifest(self, prepared):
raw_manifest = prepared["path"].read_bytes()
digest = prepared["digest"]
require("sha256:" + hashlib.sha256(raw_manifest).hexdigest() == digest,
"prepared platform manifest digest changed")
manifest = prepared["manifest"]
descriptors = [manifest["config"], *manifest.get("layers", [])]
for descriptor in descriptors:
blob_digest = descriptor["digest"]
blob_path = prepared["blobs"].get(blob_digest)
require(blob_path is not None and blob_path.is_file(),
f"missing prepared image blob: {blob_digest}")
self._upload_blob(blob_digest, blob_path)
self._request("PUT", self._path(f"manifests/{digest}"), (201,), body=raw_manifest,
content_type=manifest["mediaType"])
return digest
def create_registry_client(repository):
registry = repository.split("/", 1)[0]
credentials = {
"ghcr.io": (os.environ.get("GHCR_USERNAME"), os.environ.get("GHCR_TOKEN")),
"docker.io": (os.environ.get("DOCKERHUB_USERNAME"), os.environ.get("DOCKERHUB_TOKEN")),
}[registry]
require(all(isinstance(value, str) and value for value in credentials),
f"credentials are not configured for {registry}")
return RegistryClient(repository, *credentials)
+13 -3
View File
@@ -2,6 +2,7 @@
"""Exercise the desktop, daemon, Puppeteer/stealth, sharp and persisted cookies."""
import argparse
import json
import re
import subprocess
import sys
import time
@@ -100,11 +101,20 @@ def main(image, variant):
else:
raise RuntimeError("desktop/daemon did not become ready")
if variant == "linuxserver":
docker("exec", name, "nginx", "-t")
nginx = docker("exec", name, "nginx", "-T")
# Selkies 2 moved file downloads behind the /api proxy.
# https://github.com/linuxserver/docker-baseimage-selkies/commit/c28cd610744dae7c278f2a49ca28748dbf387c1a
config = nginx.stdout + nginx.stderr
files_path = "/api/files/" if re.search(r"\blocation\s+/api\s*\{", config) else "/files/"
files_url = "https://127.0.0.1:3001" + files_path
docker("exec", name, "sh", "-c", "printf '%s' 'release-factory files smoke' > /config/Desktop/rf-smoke.txt")
files = docker("exec", name, "curl", "-kfsS", "--max-time", "5", "-u",
"smoke:ci-gemini-password", "https://127.0.0.1:3001/files/")
# Nginx can serve the desktop before the Selkies API is ready.
curl = ("exec", name, "curl", "-kfsS", "--max-time", "5", "--retry", "10",
"--retry-delay", "1", "--retry-max-time", "30", "-u", "smoke:ci-gemini-password")
files = docker(*curl, files_url)
assert "rf-smoke.txt" in files.stdout, "file browser did not list the test file"
downloaded = docker(*curl, files_url + "rf-smoke.txt")
assert downloaded.stdout == "release-factory files smoke", "file download content mismatch"
result = docker("exec", "-w", "/opt/gemini-skill", "-e", "SMOKE_PHASE=" + phase,
name, "node", "--input-type=module", "-e", PROBE)
print(result.stdout, end="", flush=True)
+53
View File
@@ -0,0 +1,53 @@
import assert from 'node:assert/strict';
import fs from 'node:fs';
import os from 'node:os';
import path from 'node:path';
import test from 'node:test';
import { patchNpmBundles } from '../openclaw-builder/image/patch-vendored-deps.mjs';
const pins = JSON.parse(fs.readFileSync(new URL('../openclaw-builder/configs/components.json', import.meta.url))).npm_bundled;
test('repairs only the two vulnerable dependencies inside bundled npm', async () => {
const temporary = fs.mkdtempSync(path.join(os.tmpdir(), 'openclaw-npm-test-'));
try {
const root = path.join(temporary, 'node_modules');
const npm = path.join(root, '.pnpm/npm@11.20.0/node_modules/npm');
fs.mkdirSync(npm, {recursive: true});
fs.writeFileSync(path.join(npm, 'package.json'), '{"name":"npm","version":"11.20.0"}\n');
for (const [name, pin] of Object.entries(pins)) {
const directory = path.join(npm, 'node_modules', name);
fs.mkdirSync(directory, {recursive: true});
fs.writeFileSync(path.join(directory, 'package.json'), JSON.stringify({name, version: pin.from}));
}
const unrelated = path.join(root, 'unrelated');
fs.mkdirSync(unrelated);
fs.writeFileSync(path.join(unrelated, 'package.json'), '{"name":"undici","version":"6.28.0"}');
await patchNpmBundles(root, pins);
for (const [name, pin] of Object.entries(pins)) {
const pkg = JSON.parse(fs.readFileSync(path.join(npm, 'node_modules', name, 'package.json')));
assert.equal(pkg.version, pin.version);
assert.equal(pkg.name, name);
}
assert.equal(JSON.parse(fs.readFileSync(path.join(unrelated, 'package.json'))).version, '6.28.0');
assert.equal(fs.readFileSync(path.join(npm, 'package.json'), 'utf8'), '{"name":"npm","version":"11.20.0"}\n');
await patchNpmBundles(root, pins); // Idempotent and does not download the archives again.
} finally {
fs.rmSync(temporary, {recursive: true});
}
});
test('rejects a bundle dependency that escapes its npm root', async () => {
const temporary = fs.mkdtempSync(path.join(os.tmpdir(), 'openclaw-npm-escape-'));
try {
const root = path.join(temporary, 'node_modules');
const npm = path.join(root, '.pnpm/npm@11.20.0/node_modules/npm/node_modules');
const outside = path.join(temporary, 'outside');
fs.mkdirSync(npm, {recursive: true});
fs.mkdirSync(outside);
fs.writeFileSync(path.join(outside, 'package.json'), '{"name":"undici","version":"6.28.0"}');
fs.symlinkSync(outside, path.join(npm, 'undici'));
await assert.rejects(patchNpmBundles(root, pins), /unexpected bundled npm dependency path/);
} finally {
fs.rmSync(temporary, {recursive: true});
}
});
+17 -5
View File
@@ -3,13 +3,13 @@
import importlib.util
import json
import os
from pathlib import Path
import subprocess
import sys
import tempfile
import textwrap
from types import SimpleNamespace
import unittest
from pathlib import Path
from types import SimpleNamespace
from unittest.mock import patch
SPEC = importlib.util.spec_from_file_location("publish_tested", Path(__file__).with_name("publish-tested-image.py"))
@@ -49,7 +49,7 @@ class PublicationTests(unittest.TestCase):
directory.mkdir()
(directory / "receipt.json").write_text(json.dumps(receipt))
def test_stage_tags_the_recorded_id_and_checks_remote_config(self):
def test_stage_publishes_the_recorded_id_by_digest_without_a_ci_tag(self):
with tempfile.TemporaryDirectory() as tmp:
args = SimpleNamespace(**self.expected, image="mutable:tag", image_id=self.image_id,
platform="linux/amd64", receipt=Path(tmp) / "receipt.json")
@@ -59,9 +59,21 @@ class PublicationTests(unittest.TestCase):
def docker(*argv, **kwargs):
calls.append(argv)
return json.dumps([image]).encode() if kwargs.get("capture") else None
with patch.object(MODULE, "docker", docker), patch.object(MODULE, "remote_manifest", return_value=(self.manifest, self.digest)):
class Client:
def __init__(self, digest):
self.digest = digest
def publish_platform_manifest(self, prepared):
return self.digest
with patch.object(MODULE, "docker", docker), \
patch.object(MODULE, "save_image"), \
patch.object(MODULE, "prepare_platform_manifest", return_value=object()), \
patch.object(MODULE, "create_registry_client", return_value=Client(self.digest)), \
patch.object(MODULE, "remote_manifest", return_value=(self.manifest, self.digest)):
MODULE.stage(args, self.expected)
self.assertEqual(calls[1][0:3], ("image", "tag", self.image_id))
self.assertEqual(calls, [("image", "inspect", "mutable:tag")])
self.assertNotIn(":ci-", str(json.loads(args.receipt.read_text())))
self.assertEqual(json.loads(args.receipt.read_text())["image_id"], self.image_id)
def test_changed_image_or_platform_never_reaches_registry_write(self):
+130
View File
@@ -0,0 +1,130 @@
#!/usr/bin/env python3
"""Verify image-layer identity and the Registry V2 digest-only upload contract."""
import gzip
import hashlib
import io
import json
import tarfile
import tempfile
import unittest
from pathlib import Path
from unittest.mock import patch
import registry_image as registry
def digest(data):
return "sha256:" + hashlib.sha256(data).hexdigest()
def layer(content):
stream = io.BytesIO()
with tarfile.open(fileobj=stream, mode="w") as tar:
member = tarfile.TarInfo("value.txt")
member.size = len(content)
tar.addfile(member, io.BytesIO(content))
return stream.getvalue()
class LayerTests(unittest.TestCase):
def prepare(self, *, compressed=False, corrupt=None):
tmp = tempfile.TemporaryDirectory()
self.addCleanup(tmp.cleanup)
root = Path(tmp.name)
data = [layer(b"base"), layer(b"upper")]
names = ["base/layer.tar", "upper/layer.tar"]
config = {"rootfs": {"type": "layers", "diff_ids": [digest(b) for b in data]}}
if corrupt == "count":
config["rootfs"]["diff_ids"].pop()
if corrupt == "digest":
config["rootfs"]["diff_ids"][0] = "invalid"
if corrupt == "type":
config["rootfs"]["type"] = "unknown"
config_bytes = json.dumps(config).encode()
image_id = digest(config_bytes)
config_name = image_id[7:] + ".json"
manifest = [{"Config": config_name, "Layers": names}]
members = [(config_name, config_bytes), ("manifest.json", json.dumps(manifest).encode())]
for name, raw in reversed(list(zip(names, data))):
if corrupt == "missing" and name == names[0]:
continue
if corrupt == "bytes" and name == names[0]:
raw = layer(b"wrong")
stored = gzip.compress(raw, mtime=0) if compressed else raw
if corrupt == "gzip" and name == names[0]:
stored = gzip.compress(raw)[:-8]
if corrupt == "zstd" and name == names[0]:
stored = b"\x28\xb5\x2f\xfd" + raw
members.append((name, stored))
archive = root / "image.tar.gz"
with tarfile.open(archive, "w:gz") as tar:
for name, body in members:
member = tarfile.TarInfo(name)
member.size = len(body)
tar.addfile(member, io.BytesIO(body))
prepared = registry.prepare_platform_manifest(archive, image_id, root / "prepared")
return prepared, config
def test_tar_member_order_does_not_change_filesystem_layer_order(self):
for compressed in (False, True):
with self.subTest(compressed=compressed):
prepared, config = self.prepare(compressed=compressed)
actual = [digest(gzip.decompress(prepared["blobs"][d["digest"]].read_bytes()))
for d in prepared["manifest"]["layers"]]
self.assertEqual(actual, config["rootfs"]["diff_ids"])
def test_invalid_layers_fail_before_publication(self):
for corrupt in ("count", "digest", "type", "missing", "bytes", "gzip", "zstd"):
with self.subTest(corrupt=corrupt), self.assertRaises((ValueError, EOFError, OSError)):
self.prepare(corrupt=corrupt)
class RegistryTests(unittest.TestCase):
def client(self, host="ghcr.io"):
client = registry.RegistryClient(host + "/owner/image", "user", "test-password")
client.token = "test-token"
return client
def test_upload_preserves_opaque_location_query(self):
for host in ("ghcr.io", "docker.io"):
for absolute in (False, True):
with self.subTest(host=host, absolute=absolute), tempfile.TemporaryDirectory() as tmp:
client = self.client(host)
path = Path(tmp) / "blob"
path.write_bytes(b"blob")
location = "/v2/owner/image/blobs/uploads/1?token=a%2fb&space=%20&empty&x=1&x=2"
if absolute:
location = "https://" + client.registry_host + location
with patch.object(client, "_send", side_effect=[
(404, {}), (202, {"location": location}), (201, {}),
]) as send:
client._upload_blob(digest(b"blob"), path)
expected = (location if absolute else "https://" + client.registry_host + location)
self.assertEqual(send.call_args.args[1], expected + "&digest=" + digest(b"blob").replace(":", "%3A"))
def test_incomplete_upload_is_rejected(self):
client = self.client()
with tempfile.TemporaryDirectory() as tmp:
path = Path(tmp) / "blob"
path.write_bytes(b"blob")
with patch.object(client, "_send", side_effect=[
(404, {}), (202, {"location": "/v2/owner/image/blobs/uploads/1"}), (202, {}),
]), self.assertRaisesRegex(ValueError, "HTTP 202"):
client._upload_blob(digest(b"blob"), path)
def test_existing_blob_does_not_start_an_upload(self):
client = self.client()
with patch.object(client, "_send", return_value=(200, {})) as send:
client._upload_blob(digest(b"blob"), Path("unused"))
self.assertEqual(send.call_count, 1)
self.assertEqual(send.call_args.args[0], "HEAD")
def test_untrusted_upload_locations_are_rejected(self):
client = self.client()
for url in ("https://evil.example/upload", "http://ghcr.io/upload", "https://user@ghcr.io/upload"):
with self.subTest(url=url), self.assertRaises(ValueError):
client._upload_url(url)
if __name__ == "__main__":
unittest.main()
+61
View File
@@ -0,0 +1,61 @@
#!/usr/bin/env python3
"""Keep file browsing covered across LinuxServer's Selkies route migration."""
import importlib.util
import subprocess
import unittest
from pathlib import Path
from unittest.mock import patch
SPEC = importlib.util.spec_from_file_location("smoke", Path(__file__).with_name("smoke-gemini-browser.py"))
smoke = importlib.util.module_from_spec(SPEC)
SPEC.loader.exec_module(smoke)
class FileBrowserTests(unittest.TestCase):
def run_smoke(self, route, *, listing="rf-smoke.txt", content="release-factory files smoke"):
requested = []
def docker(*args, **kwargs):
output = ""
if args[0] == "exec" and "curl" in args:
url = args[-1]
if url.endswith("/health"):
output = '{"service":"browser-daemon"}'
elif url == "https://127.0.0.1:3001/":
output = "desktop"
else:
requested.append(url)
if not url.startswith("https://127.0.0.1:3001" + route):
raise RuntimeError("curl: (22) The requested URL returned error: 404")
output = content if url.endswith("rf-smoke.txt") else listing
elif args[0] == "exec" and "nginx" in args:
output = "server { location " + ("/api" if route == "/api/files/" else "/files") + " { } }"
return subprocess.CompletedProcess(args, 0, output, "")
with patch.object(smoke, "docker", side_effect=docker):
smoke.main("test-image", "linuxserver")
return requested
def test_selkies_v2_lists_and_downloads_through_api(self):
self.assertEqual(self.run_smoke("/api/files/"), [
"https://127.0.0.1:3001/api/files/",
"https://127.0.0.1:3001/api/files/rf-smoke.txt",
] * 2)
def test_older_base_images_keep_the_legacy_file_route(self):
self.assertEqual(self.run_smoke("/files/"), [
"https://127.0.0.1:3001/files/",
"https://127.0.0.1:3001/files/rf-smoke.txt",
] * 2)
def test_missing_file_still_fails(self):
with self.assertRaisesRegex(AssertionError, "did not list"):
self.run_smoke("/api/files/", listing="empty directory")
def test_wrong_download_still_fails(self):
with self.assertRaisesRegex(AssertionError, "content"):
self.run_smoke("/api/files/", content="wrong file")
if __name__ == "__main__":
unittest.main()