mirror of
https://github.com/okxlin/release-factory.git
synced 2026-10-01 08:01:39 +00:00
fix(release): repair image builds, update DSH, and stop ci tag publication (#90)
Repair Selkies file smoke checks and DSH 0.2 telemetry compatibility; update pinned components and vulnerable bundled dependencies. Publish tested images by verified digest before advancing release tags, without ci tags. Supersedes #88. All final-head native build, runtime and security checks passed.
This commit is contained in:
@@ -27,6 +27,8 @@ on:
|
||||
- 'scripts/evaluate-trivy-policy.py'
|
||||
- 'scripts/test-evaluate-trivy-policy.py'
|
||||
- 'scripts/publish-tested-image.py'
|
||||
- 'scripts/registry_image.py'
|
||||
- 'scripts/test-registry-image.py'
|
||||
- 'scripts/test-publish-tested-image.py'
|
||||
|
||||
permissions:
|
||||
|
||||
@@ -30,10 +30,13 @@ on:
|
||||
- '.github/workflows/build-gemini-skill-browser-linuxserver.yml'
|
||||
- '.github/workflows/release-gemini-browser.yml'
|
||||
- 'scripts/smoke-gemini-browser.py'
|
||||
- 'scripts/test-smoke-gemini-browser.py'
|
||||
- 'scripts/trivy-image-gate.sh'
|
||||
- 'scripts/evaluate-trivy-policy.py'
|
||||
- 'scripts/test-evaluate-trivy-policy.py'
|
||||
- 'scripts/publish-tested-image.py'
|
||||
- 'scripts/registry_image.py'
|
||||
- 'scripts/test-registry-image.py'
|
||||
- 'scripts/test-publish-tested-image.py'
|
||||
|
||||
permissions:
|
||||
|
||||
@@ -30,10 +30,13 @@ on:
|
||||
- '.github/workflows/build-gemini-skill-browser.yml'
|
||||
- '.github/workflows/release-gemini-browser.yml'
|
||||
- 'scripts/smoke-gemini-browser.py'
|
||||
- 'scripts/test-smoke-gemini-browser.py'
|
||||
- 'scripts/trivy-image-gate.sh'
|
||||
- 'scripts/evaluate-trivy-policy.py'
|
||||
- 'scripts/test-evaluate-trivy-policy.py'
|
||||
- 'scripts/publish-tested-image.py'
|
||||
- 'scripts/registry_image.py'
|
||||
- 'scripts/test-registry-image.py'
|
||||
- 'scripts/test-publish-tested-image.py'
|
||||
|
||||
permissions:
|
||||
|
||||
@@ -27,6 +27,8 @@ on:
|
||||
- 'scripts/evaluate-trivy-policy.py'
|
||||
- 'scripts/test-evaluate-trivy-policy.py'
|
||||
- 'scripts/publish-tested-image.py'
|
||||
- 'scripts/registry_image.py'
|
||||
- 'scripts/test-registry-image.py'
|
||||
- 'scripts/test-publish-tested-image.py'
|
||||
|
||||
permissions:
|
||||
|
||||
@@ -17,6 +17,8 @@ on:
|
||||
- 'scripts/evaluate-trivy-policy.py'
|
||||
- 'scripts/test-evaluate-trivy-policy.py'
|
||||
- 'scripts/publish-tested-image.py'
|
||||
- 'scripts/registry_image.py'
|
||||
- 'scripts/test-registry-image.py'
|
||||
- 'scripts/test-publish-tested-image.py'
|
||||
- '.github/workflows/openclaw-upstream-docker.yml'
|
||||
|
||||
@@ -51,7 +53,9 @@ jobs:
|
||||
run: |
|
||||
python3 scripts/test-resolve-openclaw-inputs.py
|
||||
bash scripts/test-apply-openclaw-runtime-hardening.sh
|
||||
node --test scripts/test-openclaw-npm-bundle.mjs
|
||||
python3 scripts/test-evaluate-trivy-policy.py
|
||||
python3 scripts/test-registry-image.py
|
||||
python3 scripts/test-publish-tested-image.py
|
||||
- name: Resolve immutable sources and security refresh
|
||||
id: inputs
|
||||
@@ -168,6 +172,10 @@ jobs:
|
||||
REPOSITORY: ${{ needs.prepare.outputs.repository }}
|
||||
DOCKERHUB_NAMESPACE: ${{ vars.DOCKERHUB_USERNAME || secrets.DOCKERHUB_USERNAME }}
|
||||
IMAGE_TAG: ${{ needs.prepare.outputs.image_tag }}
|
||||
GHCR_USERNAME: ${{ github.actor }}
|
||||
GHCR_TOKEN: ${{ github.token }}
|
||||
DOCKERHUB_USERNAME: ${{ vars.DOCKERHUB_USERNAME || secrets.DOCKERHUB_USERNAME }}
|
||||
DOCKERHUB_TOKEN: ${{ secrets.DOCKERHUB_TOKEN }}
|
||||
run: |
|
||||
set -euo pipefail
|
||||
repositories=("$REPOSITORY" "docker.io/$DOCKERHUB_NAMESPACE/openclaw-sandbox")
|
||||
|
||||
@@ -60,6 +60,7 @@ jobs:
|
||||
- name: Check component pins and publication safety regressions
|
||||
run: |
|
||||
bash deepseek-harness-builder/scripts/check-component-pins.sh
|
||||
python3 scripts/test-registry-image.py
|
||||
python3 deepseek-harness-builder/scripts/test-tested-image-artifact.py
|
||||
|
||||
- name: Set up Node.js
|
||||
@@ -226,6 +227,10 @@ jobs:
|
||||
IMAGE_TAG: ${{ needs.prepare.outputs.image_tag }}
|
||||
LATEST_TAG: ${{ needs.prepare.outputs.latest_tag }}
|
||||
PUSH_LATEST: ${{ inputs.push_latest }}
|
||||
GHCR_USERNAME: ${{ github.actor }}
|
||||
GHCR_TOKEN: ${{ github.token }}
|
||||
DOCKERHUB_USERNAME: ${{ env.DOCKERHUB_NAMESPACE }}
|
||||
DOCKERHUB_TOKEN: ${{ secrets.DOCKERHUB_TOKEN }}
|
||||
run: |
|
||||
set -euo pipefail
|
||||
floating_args=()
|
||||
|
||||
@@ -69,6 +69,8 @@ jobs:
|
||||
- name: Validate input and publication contracts
|
||||
run: |
|
||||
python3 gemini-skill-browser-builder/scripts/test-resolve-browser-inputs.py
|
||||
python3 scripts/test-smoke-gemini-browser.py
|
||||
python3 scripts/test-registry-image.py
|
||||
python3 scripts/test-publish-tested-image.py
|
||||
python3 scripts/test-evaluate-trivy-policy.py
|
||||
- uses: actions/setup-node@v7
|
||||
@@ -133,6 +135,8 @@ jobs:
|
||||
REPOSITORY: ${{ steps.params.outputs.repository }}
|
||||
IMAGE_TAG: ${{ steps.params.outputs.image_tag }}
|
||||
LATEST_TAG: ${{ steps.params.outputs.release_latest }}
|
||||
GHCR_USERNAME: ${{ github.actor }}
|
||||
GHCR_TOKEN: ${{ github.token }}
|
||||
run: |
|
||||
python3 scripts/publish-tested-image.py stage --image "$IMAGE" --image-id "$IMAGE" \
|
||||
--variant "$VARIANT" --platform linux/amd64 --repository "$REPOSITORY" \
|
||||
|
||||
@@ -83,6 +83,7 @@ jobs:
|
||||
env:
|
||||
VARIANT: ${{ inputs.variant }}
|
||||
run: |
|
||||
python3 scripts/test-registry-image.py
|
||||
python3 scripts/test-publish-tested-image.py
|
||||
python3 scripts/test-evaluate-trivy-policy.py
|
||||
if [[ "$VARIANT" == codex ]]; then
|
||||
@@ -180,6 +181,8 @@ jobs:
|
||||
VARIANT: ${{ inputs.variant }}
|
||||
REPOSITORY: ${{ needs.prepare.outputs.repository }}
|
||||
ARCH: ${{ matrix.arch }}
|
||||
GHCR_USERNAME: ${{ github.actor }}
|
||||
GHCR_TOKEN: ${{ github.token }}
|
||||
run: |
|
||||
python3 scripts/publish-tested-image.py stage --image "$IMAGE" --image-id "$IMAGE" \
|
||||
--variant "$VARIANT" --platform "linux/$ARCH" --repository "$REPOSITORY" \
|
||||
|
||||
@@ -12,6 +12,8 @@ on:
|
||||
- ".github/actions/verify-deepseek-harness/**"
|
||||
- "deepseek-harness-builder/**"
|
||||
- "scripts/trivy-image-gate.sh"
|
||||
- "scripts/registry_image.py"
|
||||
- "scripts/test-registry-image.py"
|
||||
- "scripts/evaluate-trivy-policy.py"
|
||||
- "scripts/test-evaluate-trivy-policy.py"
|
||||
- "SECURITY_SCAN.md"
|
||||
@@ -54,7 +56,9 @@ jobs:
|
||||
run: python3 scripts/test-evaluate-trivy-policy.py
|
||||
|
||||
- name: Run tested image publication regression tests
|
||||
run: python3 deepseek-harness-builder/scripts/test-tested-image-artifact.py
|
||||
run: |
|
||||
python3 scripts/test-registry-image.py
|
||||
python3 deepseek-harness-builder/scripts/test-tested-image-artifact.py
|
||||
|
||||
- name: Run DeepSeek Harness version resolver regression tests
|
||||
run: |
|
||||
|
||||
@@ -11,6 +11,8 @@ on:
|
||||
- ".github/actions/verify-deepseek-harness/**"
|
||||
- "deepseek-harness-builder/**"
|
||||
- "scripts/trivy-image-gate.sh"
|
||||
- "scripts/registry_image.py"
|
||||
- "scripts/test-registry-image.py"
|
||||
- "scripts/evaluate-trivy-policy.py"
|
||||
- "scripts/test-evaluate-trivy-policy.py"
|
||||
- "SECURITY_SCAN.md"
|
||||
|
||||
@@ -341,6 +341,20 @@ RUN [[ "${CODE_SERVER_TAR_VERSION}" =~ ^[0-9]+\.[0-9]+\.[0-9]+$ ]] \
|
||||
"${code_server_tar_dir}/package.json" "${CODE_SERVER_TAR_VERSION}" \
|
||||
&& find /tmp -maxdepth 1 -name "${tarball}" -delete
|
||||
|
||||
# Patch the HTTP client bundled by VS Code until code-server includes the fix.
|
||||
ARG CODE_SERVER_UNDICI_VERSION=7.29.1
|
||||
RUN [[ "${CODE_SERVER_UNDICI_VERSION}" =~ ^[0-9]+[.][0-9]+[.][0-9]+$ ]] \
|
||||
&& undici_dir="/usr/lib/code-server/lib/vscode/node_modules/undici" \
|
||||
&& test "$(realpath "${undici_dir}")" = "${undici_dir}" \
|
||||
&& test "$(node -p "require('${undici_dir}/package.json').name")" = undici \
|
||||
&& tarball="$(npm pack --ignore-scripts --silent --pack-destination /tmp "undici@${CODE_SERVER_UNDICI_VERSION}")" \
|
||||
&& [[ "${tarball}" == "undici-${CODE_SERVER_UNDICI_VERSION}.tgz" ]] \
|
||||
&& find "${undici_dir}" -mindepth 1 -delete \
|
||||
&& tar -xzf "/tmp/${tarball}" --strip-components=1 -C "${undici_dir}" \
|
||||
&& node -e 'const [directory, expected] = process.argv.slice(1); if (require(`${directory}/package.json`).version !== expected) throw new Error("unexpected code-server undici version"); require(directory)' \
|
||||
"${undici_dir}" "${CODE_SERVER_UNDICI_VERSION}" \
|
||||
&& rm -f "/tmp/${tarball}"
|
||||
|
||||
# ── npm globals and Corepack package managers ──
|
||||
# Claude's audited postinstall links its platform-native optional dependency.
|
||||
# Keep lifecycle scripts from all other global packages blocked by npm 12.
|
||||
|
||||
@@ -2056,9 +2056,9 @@
|
||||
}
|
||||
},
|
||||
"node_modules/fast-uri": {
|
||||
"version": "3.1.6",
|
||||
"resolved": "https://registry.npmjs.org/fast-uri/-/fast-uri-3.1.6.tgz",
|
||||
"integrity": "sha512-7Ical1vFEMr0onbVzEDIreM22I4khW+fzyQPwvAFWBp1iwdshSZRsL4jjRvPG9JP1uiqMHRto+YU6R2/CzDz5Q==",
|
||||
"version": "3.1.8",
|
||||
"resolved": "https://registry.npmjs.org/fast-uri/-/fast-uri-3.1.8.tgz",
|
||||
"integrity": "sha512-GZMtZUTNRpOVIECoXwLNZS5xUGE+mVNbTB8h/7Rwh2TFWcBQiPzTgyZi05BF9UMZKkLJv8XBRJTlU7zg8+ZfMg==",
|
||||
"funding": [
|
||||
{
|
||||
"type": "github",
|
||||
|
||||
@@ -8,6 +8,6 @@
|
||||
"@getpaseo/cli": "0.3.1"
|
||||
},
|
||||
"overrides": {
|
||||
"fast-uri": "3.1.6"
|
||||
"fast-uri": "3.1.8"
|
||||
}
|
||||
}
|
||||
|
||||
@@ -112,11 +112,11 @@ if (root?.dependencies?.["@getpaseo/cli"] !== "0.3.1") {
|
||||
if (lock.packages?.["node_modules/@getpaseo/cli"]?.version !== "0.3.1") {
|
||||
throw new Error("package-lock does not resolve @getpaseo/cli 0.3.1");
|
||||
}
|
||||
if (packageJson.overrides?.["fast-uri"] !== "3.1.6") {
|
||||
throw new Error("package.json must pin the fast-uri security override to 3.1.6");
|
||||
if (packageJson.overrides?.["fast-uri"] !== "3.1.8") {
|
||||
throw new Error("package.json must pin the fast-uri security override to 3.1.8");
|
||||
}
|
||||
if (lock.packages?.["node_modules/fast-uri"]?.version !== "3.1.6") {
|
||||
throw new Error("package-lock does not resolve fast-uri 3.1.6");
|
||||
if (lock.packages?.["node_modules/fast-uri"]?.version !== "3.1.8") {
|
||||
throw new Error("package-lock does not resolve fast-uri 3.1.8");
|
||||
}
|
||||
|
||||
const installScripts = [];
|
||||
|
||||
@@ -11,11 +11,11 @@ One Dockerfile produces two independently tested variants under one image reposi
|
||||
| `latest` | `<DSH_VERSION>` | `runtime` | Lightweight 1Panel service with essential shell and repository tools. This remains the default final target. |
|
||||
| `workstation` | `<DSH_VERSION>-workstation` | `workstation` | Full interactive development environment with compiler and language toolchains. |
|
||||
|
||||
Both workflows publish the same tags to `ghcr.io/okxlin/deepseek-harness` and to `docker.io/$DOCKERHUB_USERNAME/deepseek-harness`. Configure `DOCKERHUB_USERNAME` as a GitHub Actions repository variable or secret and configure `DOCKERHUB_TOKEN` as a repository secret. The Docker Hub token is used only by the registry login action and is never passed to the image build.
|
||||
Both workflows publish the same tags to `ghcr.io/okxlin/deepseek-harness` and to `docker.io/$DOCKERHUB_USERNAME/deepseek-harness`. Configure `DOCKERHUB_USERNAME` as a GitHub Actions repository variable or secret and configure `DOCKERHUB_TOKEN` as a repository secret. The Docker Hub token is used only for registry authentication and verified-image publication; it is never passed to the image build.
|
||||
|
||||
Scheduled runs and manual runs with an empty version select the highest non-draft `dsh-v*` source release from DeepSeek Harness GitHub Releases, verify its tag, commit, and source-archive SHA-256, then publish matching `<DSH_VERSION>` and `<DSH_VERSION>-workstation` tags. This supports upstream releases that are available from GitHub before they are published to npm. An explicit `dsh-v*` selects that source release; an npm version or dist-tag still selects a published package release. `image/dsh-source.json` remains the reproducible local/PR baseline, while future source releases no longer require a Dockerfile edit. Manual workflow runs can also override the published image tag. Use a floating tag for an AppStore `latest` channel and the matching version tag for a numbered AppStore version.
|
||||
|
||||
The committed source baseline is [0.1.7-rc.2](https://github.com/deepseek-ai/deepseek-harness/releases/tag/dsh-v0.1.7-rc.2). This baseline includes the PTC package name `ptc-runtime` and workflow executor `workflow-ptc`; custom profiles using the old names need updating. DeepSeek now defaults to the Messages protocol. A manually configured old official API root should be removed or changed to `https://api.deepseek.com/anthropic`; custom provider URLs remain unchanged.
|
||||
The committed source baseline is [0.2.0-rc.2](https://github.com/deepseek-ai/deepseek-harness/releases/tag/dsh-v0.2.0-rc.2). This baseline includes the PTC package name `ptc-runtime` and workflow executor `workflow-ptc`; custom profiles using the old names need updating. DeepSeek now defaults to the Messages protocol. A manually configured old official API root should be removed or changed to `https://api.deepseek.com/anthropic`; custom provider URLs remain unchanged.
|
||||
|
||||
Build versions, base-image digests, and source/tool checksums live in [`image/components.lock.json`](image/components.lock.json). Python packages have a hash-locked [`image/python-requirements.lock`](image/python-requirements.lock); the legacy npm path retains its own [package manifest](image/package.json) and [pnpm lock](image/pnpm-lock.yaml). [`image/dsh-source.json`](image/dsh-source.json) pins the source release to a commit archive and checksum. Local builds and CI use `scripts/component-inputs.py` to resolve the same build arguments; ordinary component updates do not require editing Dockerfile instructions. Source builds currently install locally packed runtime tarballs with npm, and the dependency audit checks that installed tree.
|
||||
|
||||
|
||||
@@ -11,11 +11,11 @@
|
||||
| `latest` | `<DSH_VERSION>` | `runtime` | 轻量级 1Panel 服务镜像,包含必要 Shell 和仓库工具。它仍是默认最终 target。 |
|
||||
| `workstation` | `<DSH_VERSION>-workstation` | `workstation` | 完整交互式开发环境,包含编译器和语言工具链。 |
|
||||
|
||||
两个工作流都会把相同标签发布到 `ghcr.io/okxlin/deepseek-harness` 和 `docker.io/$DOCKERHUB_USERNAME/deepseek-harness`。请把 `DOCKERHUB_USERNAME` 配置为 GitHub Actions 仓库变量或 Secret,并把 `DOCKERHUB_TOKEN` 配置为仓库 Secret。Docker Hub token 只用于 Registry 登录,不会传入镜像构建上下文。
|
||||
两个工作流都会把相同标签发布到 `ghcr.io/okxlin/deepseek-harness` 和 `docker.io/$DOCKERHUB_USERNAME/deepseek-harness`。请把 `DOCKERHUB_USERNAME` 配置为 GitHub Actions 仓库变量或 Secret,并把 `DOCKERHUB_TOKEN` 配置为仓库 Secret。Docker Hub token 只用于 Registry 认证和已验证镜像的发布,不会传入镜像构建上下文。
|
||||
|
||||
定时任务以及留空版本的手动运行会从 DeepSeek Harness GitHub Releases 选择最高的非 draft `dsh-v*` 源码版本,校验对应 tag、commit 和源码归档 SHA-256 后构建,并发布匹配的 `<DSH_VERSION>` 和 `<DSH_VERSION>-workstation` 标签;因此上游源码版本可以在发布到 npm 之前进入镜像。手动显式传入 `dsh-v*` 可以选择指定的源码 release,传入已发布的 npm 版本或 dist-tag 时仍按请求的 npm selector 解析。`image/dsh-source.json` 保留可复现的本地/PR 基线,未来源码 release 不需要修改 Dockerfile。手动工作流也可覆盖发布标签。AppStore `latest` 通道使用浮动标签,编号 AppStore 版本使用匹配的版本标签。
|
||||
|
||||
当前提交的源码基线是 [0.1.7-rc.2](https://github.com/deepseek-ai/deepseek-harness/releases/tag/dsh-v0.1.7-rc.2)。该基线包含 PTC 包名 `ptc-runtime` 和 workflow 执行器 `workflow-ptc`;使用旧名称的自定义 profile 需要更新。DeepSeek 现在默认使用 Messages 协议;手动配置的旧官方 API 根地址应删除或改为 `https://api.deepseek.com/anthropic`,自定义 provider URL 不受影响。
|
||||
当前提交的源码基线是 [0.2.0-rc.2](https://github.com/deepseek-ai/deepseek-harness/releases/tag/dsh-v0.2.0-rc.2)。该基线包含 PTC 包名 `ptc-runtime` 和 workflow 执行器 `workflow-ptc`;使用旧名称的自定义 profile 需要更新。DeepSeek 现在默认使用 Messages 协议;手动配置的旧官方 API 根地址应删除或改为 `https://api.deepseek.com/anthropic`,自定义 provider URL 不受影响。
|
||||
|
||||
构建版本、基础镜像 digest、源码和工具校验和集中保存在 [`image/components.lock.json`](image/components.lock.json)。Python 依赖使用独立的 [`image/python-requirements.lock`](image/python-requirements.lock),旧版 npm 路径保留自己的 [package.json](image/package.json) 和 [pnpm-lock.yaml](image/pnpm-lock.yaml)。[`image/dsh-source.json`](image/dsh-source.json) 把源码归档绑定到确定的 commit 和 SHA-256。本地构建与 CI 都通过 `scripts/component-inputs.py` 解析参数,常规组件更新无需手改 Dockerfile 指令。源码构建当前仍由 npm 安装本地打包的 runtime tarball,依赖审计检查这棵实际安装树。
|
||||
|
||||
|
||||
@@ -8,7 +8,7 @@
|
||||
"CADDY_RATELIMIT_LICENSE_SHA256": "094bedda295c0702807d1027a00e1134f4cb71d523dc49f6d4b3caf589053068",
|
||||
"CADDY_RATELIMIT_REF": "5625512f24f6f59d6f64fb3aafe5eecff0b286db",
|
||||
"CADDY_SECURITY_LICENSE_SHA256": "c71d239df91726fc519c6eb72d318ec65820627232b2f796219e87dcf35d0ab4",
|
||||
"CADDY_SECURITY_VERSION": "1.2.2",
|
||||
"CADDY_SECURITY_VERSION": "1.3.0",
|
||||
"CADDY_SOURCE_ARCHIVE_SHA256": "2c3d02078286a6282cdb4d1d8744077788d556659dac0b64d8ed5886a7e5aeb9",
|
||||
"CADDY_VERSION": "2.11.4",
|
||||
"CEL_GO_LICENSE_SHA256": "4cdb9af102dfbb0ca03d87d6f650a505df098646a4080f4665b389ad9c6caa02",
|
||||
@@ -23,8 +23,8 @@
|
||||
"DOCKER_COMPOSE_VERSION": "5.5.1",
|
||||
"DOCKER_VERSION": "29.8.1",
|
||||
"GO_AUTHCRUNCH_LICENSE_SHA256": "c71d239df91726fc519c6eb72d318ec65820627232b2f796219e87dcf35d0ab4",
|
||||
"GO_AUTHCRUNCH_SOURCE_SHA256": "166d9743a7fa0979747341929472b680fd1d2ee88879b33f3fc775315579b699",
|
||||
"GO_AUTHCRUNCH_VERSION": "1.3.6",
|
||||
"GO_AUTHCRUNCH_SOURCE_SHA256": "b0c0ff46975fdbfff1eee591dfda20339a9170e133e41e269fa2b82b913ad84b",
|
||||
"GO_AUTHCRUNCH_VERSION": "1.3.10",
|
||||
"GO_IMAGE": "golang:1.27.1-trixie@sha256:9baa6b4187bbb98d240372a8a235ac0bb6b5ddd52bba1431dc2f7c0705862728",
|
||||
"GO_VERSION": "1.27.1",
|
||||
"GRPC_VERSION": "1.84.0",
|
||||
@@ -34,8 +34,8 @@
|
||||
"NODE_VERSION": "24.21.0",
|
||||
"NPM_ARCHIVE_SHA256": "d1a92f40e6c407b84c3a00c3cf978a10b24fd42f153c527e2016cef7bb34a483",
|
||||
"NPM_VERSION": "11.20.0",
|
||||
"PNPM_ARCHIVE_SHA256": "05b7b921fbb31564505c967eabf825895a1cc18f50935c00be98815272cc9d56",
|
||||
"PNPM_VERSION": "12.6.0",
|
||||
"PNPM_ARCHIVE_SHA256": "90762b8105e833164186b011efe5cde13841693ac8eaf784d2974a6e1c01fba3",
|
||||
"PNPM_VERSION": "12.8.1",
|
||||
"PYTEST_VERSION": "9.1.1",
|
||||
"PYTHON_IMAGE": "python:3.12.14-slim-trixie@sha256:2c941e860699f878900b0edc2403613c234d4b32eda3cc9fa7036991a2a63c4a",
|
||||
"PYTHON_VERSION": "3.12.14",
|
||||
@@ -45,15 +45,15 @@
|
||||
"RUFF_VERSION": "0.16.9",
|
||||
"UV_LICENSE_APACHE_SHA256": "c71d239df91726fc519c6eb72d318ec65820627232b2f796219e87dcf35d0ab4",
|
||||
"UV_LICENSE_MIT_SHA256": "860e3d7a86b84e6a7012c7a635fc64df475cebc6cce34dfeb73a5982ec58176c",
|
||||
"UV_SHA256_AMD64": "23bf5552d220e0842b65c862097b2ebaeba0064b74eda5e565e77fd25969d8c8",
|
||||
"UV_SHA256_ARM64": "0804e9b164c64b6914182d5920c08551958a095986f10a3731056df701126436",
|
||||
"UV_VERSION": "0.12.19",
|
||||
"UV_SHA256_AMD64": "23f02075b652bb1df64178cfae41b5caf160822e720e2663568f3f5d63bc52c0",
|
||||
"UV_SHA256_ARM64": "030b69227b40af8c1981b7301793dc66e71ed3c796ea8688209dd268bd91ec51",
|
||||
"UV_VERSION": "0.12.21",
|
||||
"X_CRYPTO_VERSION": "0.57.0",
|
||||
"X_MOD_VERSION": "0.41.0",
|
||||
"X_TEXT_VERSION": "0.42.0",
|
||||
"YQ_LICENSE_SHA256": "697db34dabb21562fe84487a2ccd031fbd45382b89c2cbdec8ef31682c486040",
|
||||
"YQ_SHA256_AMD64": "c5f056448f973ae7d39b5401949648a78f2dc1947d6a8eb65be60d5c504b9385",
|
||||
"YQ_SHA256_ARM64": "88a1016bc1d657375a35864e4f44b6f333df8ff97b559f51bba0adcb2169df09",
|
||||
"YQ_VERSION": "4.53.6"
|
||||
"YQ_SHA256_AMD64": "8e34fc298390875de416e6a4afcb8cabeceb25d9aa8506c1a2f9353cf702ea5f",
|
||||
"YQ_SHA256_ARM64": "189088da0c6429ec5178dfaab1a114805f6cab0b61b165ab236efedf1d57a71b",
|
||||
"YQ_VERSION": "4.54.1"
|
||||
}
|
||||
}
|
||||
|
||||
@@ -1,8 +1,8 @@
|
||||
{
|
||||
"version": "0.1.7-rc.2",
|
||||
"version": "0.2.0-rc.2",
|
||||
"repository": "deepseek-ai/deepseek-harness",
|
||||
"ref": "dsh-v0.1.7-rc.2",
|
||||
"commit": "477b4f420553e8a52c2fbccc464d7561b239c443",
|
||||
"archiveSha256": "a6b78577dec0653bf336f6aa093c55dd93cb3350f220f91efde951b18c7eda83",
|
||||
"archiveUrl": "https://codeload.github.com/deepseek-ai/deepseek-harness/tar.gz/477b4f420553e8a52c2fbccc464d7561b239c443"
|
||||
"ref": "dsh-v0.2.0-rc.2",
|
||||
"commit": "639ed015397290b3745d163aafe02ffee4aa3f84",
|
||||
"archiveSha256": "bbc09e888e1df3aa37be049abdb7720951e76442d21e5432365d87a465d628f1",
|
||||
"archiveUrl": "https://codeload.github.com/deepseek-ai/deepseek-harness/tar.gz/639ed015397290b3745d163aafe02ffee4aa3f84"
|
||||
}
|
||||
|
||||
@@ -4,7 +4,7 @@
|
||||
"private": true,
|
||||
"description": "Pinned runtime dependency closure for the DeepSeek Harness container image.",
|
||||
"license": "MIT",
|
||||
"packageManager": "pnpm@12.6.0",
|
||||
"packageManager": "pnpm@12.8.1",
|
||||
"engines": {
|
||||
"node": "24.21.0"
|
||||
},
|
||||
|
||||
@@ -14,15 +14,13 @@ const baseRequire = createRequire(basePath)
|
||||
|
||||
// These 0.1.6 request contributors are independent of the CLI's OTel switch.
|
||||
// Preserve the image's opt-out for every profile, including explicit overlays.
|
||||
// https://github.com/deepseek-ai/deepseek-harness/tree/dsh-v0.1.7-rc.2/packages/session/session-log-deepseek
|
||||
// https://github.com/deepseek-ai/deepseek-harness/tree/dsh-v0.2.0-rc.2/packages/session/session-log-deepseek
|
||||
const guards = [
|
||||
['@deepseek-ai/dsh-session-log-deepseek', 'config.enabled !== true'],
|
||||
['@deepseek-ai/dsh-plugin-package-inventory-deepseek', 'config.enabled === false'],
|
||||
['@deepseek-ai/dsh-session-log-deepseek', ['config.enabled !== true', '!config.enabled.get()']],
|
||||
['@deepseek-ai/dsh-plugin-package-inventory-deepseek', ['config.enabled === false']],
|
||||
]
|
||||
for (const [name, guard] of guards) {
|
||||
for (const [name, supportedGuards] of guards) {
|
||||
if (!Object.hasOwn(base.dependencies ?? {}, name)) continue // Older npm releases have neither contributor.
|
||||
const original = `if (${guard})`
|
||||
const replacement = `if (${guard} || process.env.DSH_TELEMETRY_DISABLED)`
|
||||
const target = baseRequire.resolve(name)
|
||||
const stat = await lstat(target)
|
||||
const canonical = await realpath(target)
|
||||
@@ -33,10 +31,24 @@ for (const [name, guard] of guards) {
|
||||
}
|
||||
const source = await readFile(target, 'utf8')
|
||||
const count = needle => source.split(needle).length - 1
|
||||
if (count(original) === 0 && count(replacement) === 1) continue
|
||||
if (count(original) !== 1 || count(replacement) !== 0) {
|
||||
const matches = supportedGuards.map(guard => [
|
||||
`if (${guard})`, `if (${guard} || process.env.DSH_TELEMETRY_DISABLED)`,
|
||||
]).filter(([original, replacement]) => count(original) + count(replacement) > 0)
|
||||
if (matches.length !== 1) throw new Error(`unexpected telemetry activation guard in ${name}`)
|
||||
const [original, replacement] = matches[0]
|
||||
if (count(original) + count(replacement) !== 1) {
|
||||
throw new Error(`unexpected telemetry activation guard in ${name}`)
|
||||
}
|
||||
await writeFile(target, source.replace(original, replacement), 'utf8')
|
||||
let patched = source.replace(original, replacement)
|
||||
// 0.2 registers first and reads the volatile switch inside prepare(). Keep the
|
||||
// image opt-out at plugin activation as well as at request preparation.
|
||||
if (original.includes('config.enabled.get()')) {
|
||||
const entry = 'function apply(ctx, config) {'
|
||||
const activation = `${entry}\n if (process.env.DSH_TELEMETRY_DISABLED) return;`
|
||||
if (count(entry) !== 1) throw new Error(`unexpected telemetry plugin entry in ${name}`)
|
||||
if (!patched.includes(activation)) patched = patched.replace(entry, activation)
|
||||
}
|
||||
if (patched === source) continue
|
||||
await writeFile(target, patched, 'utf8')
|
||||
process.stdout.write(`[dsh-patch] ${name} honors DSH_TELEMETRY_DISABLED\n`)
|
||||
}
|
||||
|
||||
@@ -60,6 +60,40 @@ test('legacy releases without the request contributors remain supported', async
|
||||
assert.equal(run(root).status, 0)
|
||||
})
|
||||
|
||||
test('0.2 volatile session-log config retains the telemetry opt-out', async t => {
|
||||
const root = await fixture(t)
|
||||
const target = join(root, 'node_modules', names[0], 'index.js')
|
||||
await writeFile(target, `function apply(ctx, config) {
|
||||
ctx.register({ prepare() {
|
||||
if (!config.enabled.get()) return undefined;
|
||||
return 'session log';
|
||||
} });
|
||||
}
|
||||
export { apply };\n`)
|
||||
assert.equal(run(root).status, 0)
|
||||
assert.equal(run(root).status, 0, 'patch is idempotent')
|
||||
const { apply } = await import(pathToFileURL(target))
|
||||
const previous = process.env.DSH_TELEMETRY_DISABLED
|
||||
t.after(() => {
|
||||
if (previous === undefined) delete process.env.DSH_TELEMETRY_DISABLED
|
||||
else process.env.DSH_TELEMETRY_DISABLED = previous
|
||||
})
|
||||
for (const disabled of ['', '1']) {
|
||||
process.env.DSH_TELEMETRY_DISABLED = disabled
|
||||
for (const enabled of [false, true]) {
|
||||
const registrations = []
|
||||
apply({ register(value) { registrations.push(value) } }, { enabled: { get: () => enabled } })
|
||||
assert.equal(registrations.length, disabled ? 0 : 1, 'opt-out prevents registration')
|
||||
if (registrations.length) {
|
||||
assert.equal(registrations[0].prepare(), enabled ? 'session log' : undefined)
|
||||
process.env.DSH_TELEMETRY_DISABLED = '1'
|
||||
assert.equal(registrations[0].prepare(), undefined, 'request-time opt-out still applies')
|
||||
process.env.DSH_TELEMETRY_DISABLED = disabled
|
||||
}
|
||||
}
|
||||
}
|
||||
})
|
||||
|
||||
test('unknown activation code and missing declared packages fail closed', async t => {
|
||||
const root = await fixture(t)
|
||||
const target = join(root, 'node_modules', names[0], 'index.js')
|
||||
|
||||
@@ -1,20 +1,19 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Publication must fail closed when a verified image or release input changes."""
|
||||
|
||||
from argparse import Namespace
|
||||
import contextlib
|
||||
import hashlib
|
||||
import importlib.util
|
||||
import io
|
||||
import json
|
||||
from pathlib import Path
|
||||
import subprocess
|
||||
import tarfile
|
||||
import tempfile
|
||||
import unittest
|
||||
from argparse import Namespace
|
||||
from pathlib import Path
|
||||
from unittest.mock import patch
|
||||
|
||||
|
||||
SPEC = importlib.util.spec_from_file_location("artifacts", Path(__file__).with_name("tested-image-artifact.py"))
|
||||
artifacts = importlib.util.module_from_spec(SPEC)
|
||||
SPEC.loader.exec_module(artifacts)
|
||||
@@ -24,6 +23,25 @@ def encode(value):
|
||||
return json.dumps(value, separators=(",", ":")).encode()
|
||||
|
||||
|
||||
class FakeRegistryClient:
|
||||
def __init__(self, case, repository):
|
||||
self.case = case
|
||||
self.repository = repository
|
||||
|
||||
def publish_platform_manifest(self, prepared):
|
||||
if self.case.failed_repository == self.repository:
|
||||
raise subprocess.CalledProcessError(1, "registry upload")
|
||||
raw = prepared["path"].read_bytes()
|
||||
manifest = json.loads(raw)
|
||||
if self.case.corrupt_platform_repository == self.repository:
|
||||
manifest["config"]["digest"] = "sha256:" + "e" * 64
|
||||
raw = encode(manifest)
|
||||
digest = "sha256:" + hashlib.sha256(raw).hexdigest()
|
||||
self.case.registry_calls.append((self.repository, digest, manifest))
|
||||
self.case.remote[f"{self.repository}@{digest}"] = raw
|
||||
return digest
|
||||
|
||||
|
||||
class TestedImageArtifactTests(unittest.TestCase):
|
||||
def setUp(self):
|
||||
self.temporary = tempfile.TemporaryDirectory()
|
||||
@@ -38,15 +56,21 @@ class TestedImageArtifactTests(unittest.TestCase):
|
||||
self.configs = {}
|
||||
self.calls = []
|
||||
self.remote = {}
|
||||
self.tagged = {}
|
||||
self.registry_calls = []
|
||||
self.failed_repository = None
|
||||
self.corrupt_platform_repository = None
|
||||
self.write_artifact("amd64")
|
||||
self.write_artifact("arm64")
|
||||
|
||||
def directory(self, arch):
|
||||
return self.root / f"deepseek-harness-{self.expected['variant']}-image-{arch}"
|
||||
|
||||
def write_artifact(self, arch, *, config_changes=None, docker_layout=False):
|
||||
def write_artifact(self, arch, *, config_changes=None, docker_layout=False, with_layer=False):
|
||||
layer_content = b"verified layer bytes"
|
||||
config = {"os": "linux", "architecture": arch,
|
||||
"rootfs": {"type": "layers", "diff_ids": [
|
||||
"sha256:" + hashlib.sha256(layer_content).hexdigest()
|
||||
] if with_layer else []},
|
||||
"config": {"Env": [f"DSH_IMAGE_VARIANT={self.expected['variant']}"], "Labels": {
|
||||
"org.opencontainers.image.revision": self.expected["revision"],
|
||||
"org.opencontainers.image.version": self.expected["dsh_version"]}}}
|
||||
@@ -57,7 +81,9 @@ class TestedImageArtifactTests(unittest.TestCase):
|
||||
image_id = "sha256:" + digest
|
||||
self.configs[image_id] = config
|
||||
config_name = f"{digest}.json" if docker_layout else f"blobs/sha256/{digest}"
|
||||
manifest = [{"Config": config_name, "RepoTags": None, "Layers": []}]
|
||||
layer_name = "layer.tar"
|
||||
manifest = [{"Config": config_name, "RepoTags": None,
|
||||
"Layers": [layer_name] if with_layer else []}]
|
||||
directory = self.directory(arch)
|
||||
directory.mkdir(exist_ok=True)
|
||||
with tarfile.open(directory / "image.tar.gz", "w:gz") as saved:
|
||||
@@ -65,6 +91,11 @@ class TestedImageArtifactTests(unittest.TestCase):
|
||||
member = tarfile.TarInfo(name)
|
||||
member.size = len(content)
|
||||
saved.addfile(member, io.BytesIO(content))
|
||||
if with_layer:
|
||||
content = layer_content
|
||||
member = tarfile.TarInfo(layer_name)
|
||||
member.size = len(content)
|
||||
saved.addfile(member, io.BytesIO(content))
|
||||
receipt = {"schema_version": 1, **self.expected, "platform": f"linux/{arch}",
|
||||
"image_id": image_id, "archive_sha256": artifacts.sha256(directory / "image.tar.gz")}
|
||||
(directory / "receipt.json").write_text(json.dumps(receipt), encoding="utf-8")
|
||||
@@ -85,15 +116,6 @@ class TestedImageArtifactTests(unittest.TestCase):
|
||||
config = self.configs[image_id]
|
||||
return encode([{"Id": image_id, "Os": config["os"], "Architecture": config["architecture"],
|
||||
"Config": config["config"]}])
|
||||
if args[:2] == ("image", "tag"):
|
||||
self.tagged[args[3]] = args[2]
|
||||
return None
|
||||
if args[:2] == ("image", "push"):
|
||||
reference = args[2]
|
||||
manifest = {"schemaVersion": 2, "mediaType": "application/vnd.oci.image.manifest.v1+json",
|
||||
"config": {"digest": self.tagged[reference]}}
|
||||
self.remote[reference] = encode(manifest)
|
||||
return None
|
||||
if args[:4] == ("buildx", "imagetools", "inspect", "--raw"):
|
||||
return self.remote[args[4]]
|
||||
if args[:3] == ("buildx", "imagetools", "create"):
|
||||
@@ -113,31 +135,68 @@ class TestedImageArtifactTests(unittest.TestCase):
|
||||
self.fail(f"unexpected Docker operation (especially a rebuild): {args}")
|
||||
|
||||
def publish(self, docker=None):
|
||||
with patch.object(artifacts, "docker", docker or self.docker), contextlib.redirect_stdout(io.StringIO()):
|
||||
with patch.object(artifacts, "docker", docker or self.docker), \
|
||||
patch.object(artifacts, "create_registry_client",
|
||||
lambda repository: FakeRegistryClient(self, repository)), \
|
||||
contextlib.redirect_stdout(io.StringIO()):
|
||||
artifacts.publish_images(self.options, self.expected)
|
||||
|
||||
def assert_rejected_without_registry_writes(self):
|
||||
with self.assertRaises((ValueError, tarfile.TarError)):
|
||||
self.publish()
|
||||
self.assertFalse(any(call[:2] in (("image", "push"), ("image", "tag"))
|
||||
or call[:3] == ("buildx", "imagetools", "create") for call in self.calls))
|
||||
self.assertFalse(self.registry_calls)
|
||||
self.assertFalse(any(call[:3] == ("buildx", "imagetools", "create") for call in self.calls))
|
||||
|
||||
def test_publishes_tested_config_ids_to_both_registries_before_floating_tags(self):
|
||||
def test_publishes_tested_manifests_by_digest_before_normal_tags(self):
|
||||
self.publish()
|
||||
pushes = [call for call in self.calls if call[:2] == ("image", "push")]
|
||||
self.assertEqual(len(pushes), 4)
|
||||
self.assertEqual(len(self.registry_calls), 4)
|
||||
first_manifest = next(i for i, call in enumerate(self.calls) if call[:3] == ("buildx", "imagetools", "create"))
|
||||
self.assertEqual(sum(call[:2] == ("image", "push") for call in self.calls[:first_manifest]), 4)
|
||||
self.assertTrue(all("ci-" not in str(call) for call in self.calls[:first_manifest]))
|
||||
creates = [call for call in self.calls if call[:3] == ("buildx", "imagetools", "create")]
|
||||
self.assertEqual([call[5].rsplit(":", 1)[1] for call in creates],
|
||||
["0.1.5-rc.1", "0.1.5-rc.1", "latest", "latest"])
|
||||
self.assertEqual(set(self.tagged.values()), set(self.configs))
|
||||
self.assertTrue(all("@sha256:" in source for call in creates for source in call[6:]))
|
||||
self.assertTrue(all("ci-" not in source for call in creates for source in call[6:]))
|
||||
self.assertEqual({call[2]["config"]["digest"] for call in self.registry_calls}, set(self.configs))
|
||||
|
||||
def test_supports_both_docker_archive_config_layouts(self):
|
||||
self.write_artifact("amd64", docker_layout=True)
|
||||
self.publish()
|
||||
|
||||
def test_prepares_uncompressed_layers_as_gzip_blobs(self):
|
||||
receipt = self.write_artifact("amd64", with_layer=True)
|
||||
with tempfile.TemporaryDirectory() as staging:
|
||||
prepared = artifacts.prepare_platform_manifest(
|
||||
self.directory("amd64"), receipt, Path(staging))
|
||||
self.assertEqual(len(prepared["manifest"]["layers"]), 1)
|
||||
layer = prepared["manifest"]["layers"][0]
|
||||
self.assertEqual(layer["mediaType"], "application/vnd.docker.image.rootfs.diff.tar.gzip")
|
||||
self.assertEqual(prepared["blobs"][layer["digest"]].read_bytes()[:2], b"\x1f\x8b")
|
||||
|
||||
def test_registry_client_uploads_blobs_and_manifest_by_digest_without_a_tag(self):
|
||||
receipt = self.write_artifact("amd64", with_layer=True)
|
||||
with tempfile.TemporaryDirectory() as staging:
|
||||
prepared = artifacts.prepare_platform_manifest(
|
||||
self.directory("amd64"), receipt, Path(staging))
|
||||
client = artifacts.RegistryClient("ghcr.io/okxlin/deepseek-harness", "user", "token")
|
||||
client.token = "access-token"
|
||||
requests = []
|
||||
|
||||
def send(method, url, headers, *, body=None, body_path=None):
|
||||
requests.append((method, url, body, body_path))
|
||||
if method == "HEAD":
|
||||
return 404, {}
|
||||
if method == "POST":
|
||||
return 202, {"location": "https://ghcr.io/v2/okxlin/deepseek-harness/blobs/uploads/1"}
|
||||
return 201, {}
|
||||
|
||||
with patch.object(client, "_send", side_effect=send):
|
||||
self.assertEqual(client.publish_platform_manifest(prepared), prepared["digest"])
|
||||
self.assertEqual(sum(method == "PUT" for method, *_ in requests), 3)
|
||||
manifest_put = next(url for method, url, *_ in requests if method == "PUT" and "/manifests/" in url)
|
||||
self.assertIn("/manifests/sha256:", manifest_put)
|
||||
self.assertNotIn(":ci-", manifest_put)
|
||||
|
||||
def test_workstation_arm_only_publishes_only_the_requested_platform(self):
|
||||
self.root = self.root / "arm-only"
|
||||
self.root.mkdir()
|
||||
@@ -148,8 +207,8 @@ class TestedImageArtifactTests(unittest.TestCase):
|
||||
self.options.latest_tag = "workstation"
|
||||
receipt = self.write_artifact("arm64")
|
||||
self.publish()
|
||||
self.assertEqual(set(self.tagged.values()), {receipt["image_id"]})
|
||||
self.assertEqual(len(self.tagged), 2)
|
||||
self.assertEqual({call[2]["config"]["digest"] for call in self.registry_calls}, {receipt["image_id"]})
|
||||
self.assertEqual(len(self.registry_calls), 2)
|
||||
|
||||
def test_dry_run_checks_artifacts_without_docker_or_registry_access(self):
|
||||
self.options.dry_run = True
|
||||
@@ -212,24 +271,15 @@ class TestedImageArtifactTests(unittest.TestCase):
|
||||
self.assertFalse(any(call[:2] == ("image", "push") for call in self.calls))
|
||||
|
||||
def test_registry_push_failure_never_updates_release_tags(self):
|
||||
def failed_push(*args, **kwargs):
|
||||
if args[:2] == ("image", "push") and args[2].startswith("docker.io/"):
|
||||
raise subprocess.CalledProcessError(1, "docker push")
|
||||
return self.docker(*args, **kwargs)
|
||||
self.failed_repository = "docker.io/okxlin/deepseek-harness"
|
||||
with self.assertRaises(subprocess.CalledProcessError):
|
||||
self.publish(failed_push)
|
||||
self.publish()
|
||||
self.assertFalse(any(call[:3] == ("buildx", "imagetools", "create") for call in self.calls))
|
||||
|
||||
def test_registry_config_mismatch_never_updates_release_tags(self):
|
||||
def wrong_config(*args, **kwargs):
|
||||
result = self.docker(*args, **kwargs)
|
||||
if args[:4] == ("buildx", "imagetools", "inspect", "--raw"):
|
||||
manifest = json.loads(result)
|
||||
manifest["config"]["digest"] = "sha256:" + "e" * 64
|
||||
return encode(manifest)
|
||||
return result
|
||||
with self.assertRaisesRegex(ValueError, "pushed image does not match"):
|
||||
self.publish(wrong_config)
|
||||
self.corrupt_platform_repository = "ghcr.io/okxlin/deepseek-harness"
|
||||
with self.assertRaisesRegex(ValueError, "unexpected platform digest|published image digest changed"):
|
||||
self.publish()
|
||||
self.assertFalse(any(call[:3] == ("buildx", "imagetools", "create") for call in self.calls))
|
||||
|
||||
def test_incomplete_version_manifest_never_updates_floating_tags(self):
|
||||
|
||||
@@ -2,27 +2,33 @@
|
||||
"""Export verified images and publish those exact images without rebuilding."""
|
||||
|
||||
import argparse
|
||||
import gzip
|
||||
import hashlib
|
||||
import json
|
||||
from pathlib import Path
|
||||
import re
|
||||
import shutil
|
||||
import subprocess
|
||||
import sys
|
||||
import tarfile
|
||||
import tempfile
|
||||
from pathlib import Path
|
||||
|
||||
LIBRARY = Path(__file__).resolve().parents[2] / "scripts"
|
||||
if str(LIBRARY) not in sys.path:
|
||||
sys.path.insert(0, str(LIBRARY))
|
||||
from registry_image import (
|
||||
DIGEST,
|
||||
IMAGE_TYPES,
|
||||
INDEX_TYPES,
|
||||
RegistryClient, # noqa: F401 - re-exported for the focused Registry V2 test
|
||||
archive_metadata,
|
||||
create_registry_client,
|
||||
save_image,
|
||||
)
|
||||
from registry_image import (
|
||||
prepare_platform_manifest as prepare_image_manifest,
|
||||
)
|
||||
|
||||
DIGEST = re.compile(r"sha256:[a-f0-9]{64}")
|
||||
PLATFORMS = ("linux/amd64", "linux/arm64")
|
||||
IMAGE_TYPES = {
|
||||
"application/vnd.oci.image.manifest.v1+json",
|
||||
"application/vnd.docker.distribution.manifest.v2+json",
|
||||
}
|
||||
INDEX_TYPES = {
|
||||
"application/vnd.oci.image.index.v1+json",
|
||||
"application/vnd.docker.distribution.manifest.list.v2+json",
|
||||
}
|
||||
|
||||
|
||||
def require(condition, message):
|
||||
@@ -80,23 +86,7 @@ def check_archive(directory, expected):
|
||||
require(sha256(archive) == receipt.get("archive_sha256"), f"archive checksum mismatch: {archive}")
|
||||
|
||||
# Read metadata without extracting any archive paths into the host filesystem.
|
||||
digest = receipt["image_id"].removeprefix("sha256:")
|
||||
config_names = {f"{digest}.json", f"blobs/sha256/{digest}"}
|
||||
metadata = {}
|
||||
with tarfile.open(archive, "r|gz") as saved:
|
||||
for member in saved:
|
||||
if member.name not in config_names | {"manifest.json"}:
|
||||
continue
|
||||
require(member.isfile() and member.size <= 4 * 1024 * 1024,
|
||||
"invalid image archive metadata member")
|
||||
require(member.name not in metadata, "duplicate image archive metadata")
|
||||
metadata[member.name] = saved.extractfile(member).read()
|
||||
manifest = json.loads(metadata.get("manifest.json", b"null"))
|
||||
require(isinstance(manifest, list) and len(manifest) == 1, "archive must contain exactly one image")
|
||||
config_name = manifest[0].get("Config")
|
||||
require(config_name in config_names and config_name in metadata, "archive config does not match tested image ID")
|
||||
config_bytes = metadata[config_name]
|
||||
require(hashlib.sha256(config_bytes).hexdigest() == digest, "archive image config digest mismatch")
|
||||
config_bytes, _ = archive_metadata(archive, receipt["image_id"])
|
||||
check_config(json.loads(config_bytes), receipt)
|
||||
return receipt
|
||||
|
||||
@@ -116,11 +106,7 @@ def export_image(options, expected):
|
||||
flush=True)
|
||||
archive = directory / "image.tar.gz"
|
||||
# Saving by config ID binds the archive to the tested image, even if a tag moves.
|
||||
with archive.open("xb") as target:
|
||||
with subprocess.Popen(["docker", "image", "save", options.image_id], stdout=subprocess.PIPE) as saved:
|
||||
with gzip.GzipFile(filename="", mode="wb", fileobj=target, compresslevel=1, mtime=0) as zipped:
|
||||
shutil.copyfileobj(saved.stdout, zipped)
|
||||
require(saved.wait() == 0, "docker image save failed")
|
||||
save_image(options.image_id, archive)
|
||||
receipt = {"schema_version": 1, **expected, "archive_sha256": sha256(archive)}
|
||||
(directory / "receipt.json").write_text(json.dumps(receipt, indent=2) + "\n", encoding="utf-8")
|
||||
print(json.dumps(receipt, indent=2))
|
||||
@@ -132,6 +118,11 @@ def remote_manifest(reference):
|
||||
return json.loads(raw), "sha256:" + hashlib.sha256(raw).hexdigest()
|
||||
|
||||
|
||||
def prepare_platform_manifest(directory, receipt, destination):
|
||||
"""Convert one verified Docker-save archive into a digest-addressed manifest."""
|
||||
return prepare_image_manifest(directory / "image.tar.gz", receipt["image_id"], destination)
|
||||
|
||||
|
||||
def check_index(reference, expected_digests):
|
||||
manifest, digest = remote_manifest(reference)
|
||||
require(manifest.get("schemaVersion") == 2 and manifest.get("mediaType") in INDEX_TYPES,
|
||||
@@ -172,34 +163,41 @@ def publish_images(options, expected):
|
||||
return
|
||||
|
||||
# Validate every archive and loaded image before making the first registry write.
|
||||
for directory, receipt in artifacts:
|
||||
docker("image", "load", "--input", directory / "image.tar.gz")
|
||||
inspect_image(receipt["image_id"], receipt)
|
||||
with tempfile.TemporaryDirectory(prefix="deepseek-harness-manifests-") as staging:
|
||||
prepared = []
|
||||
for directory, receipt in artifacts:
|
||||
prepared.append((receipt, prepare_platform_manifest(
|
||||
directory, receipt, Path(staging) / receipt["platform"].split("/")[1])))
|
||||
for directory, receipt in artifacts:
|
||||
docker("image", "load", "--input", directory / "image.tar.gz")
|
||||
inspect_image(receipt["image_id"], receipt)
|
||||
|
||||
staged = {}
|
||||
for repository in options.repository:
|
||||
staged[repository] = {}
|
||||
for _, receipt in artifacts:
|
||||
arch = receipt["platform"].split("/")[1]
|
||||
reference = f"{repository}:ci-{expected['run_id']}-{options.run_attempt}-{expected['variant']}-{arch}"
|
||||
docker("image", "tag", receipt["image_id"], reference)
|
||||
docker("image", "push", reference)
|
||||
manifest, digest = remote_manifest(reference)
|
||||
require(manifest.get("schemaVersion") == 2 and manifest.get("mediaType") in IMAGE_TYPES
|
||||
and manifest.get("config", {}).get("digest") == receipt["image_id"],
|
||||
f"pushed image does not match tested image ID: {reference}")
|
||||
staged[repository][receipt["platform"]] = digest
|
||||
clients = {repository: create_registry_client(repository) for repository in options.repository}
|
||||
staged = {}
|
||||
for repository, client in clients.items():
|
||||
staged[repository] = {}
|
||||
for receipt, platform_manifest in prepared:
|
||||
digest = client.publish_platform_manifest(platform_manifest)
|
||||
require(digest == platform_manifest["digest"],
|
||||
f"registry returned an unexpected platform digest: {repository}")
|
||||
reference = f"{repository}@{digest}"
|
||||
manifest, remote_digest = remote_manifest(reference)
|
||||
require(manifest.get("schemaVersion") == 2 and manifest.get("mediaType") in IMAGE_TYPES
|
||||
and manifest.get("config", {}).get("digest") == receipt["image_id"],
|
||||
f"published image does not match tested image ID: {reference}")
|
||||
require(remote_digest == digest, f"published image digest changed: {reference}")
|
||||
staged[repository][receipt["platform"]] = digest
|
||||
|
||||
# Both registries must have all images before updating release tags. Floating
|
||||
# tags follow only after the version tag has been verified in both registries.
|
||||
published = []
|
||||
for tag in tags:
|
||||
for repository, digests in staged.items():
|
||||
reference = f"{repository}:{tag}"
|
||||
docker("buildx", "imagetools", "create", "--prefer-index=true", "--tag", reference,
|
||||
*(f"{repository}@{digest}" for digest in digests.values()))
|
||||
digest = check_index(reference, digests)
|
||||
published.append(f"{reference} -> {digest}")
|
||||
# Both registries must have all images before updating release tags. Floating
|
||||
# tags follow only after the version tag has been verified in both registries.
|
||||
published = []
|
||||
for tag in tags:
|
||||
for repository, digests in staged.items():
|
||||
reference = f"{repository}:{tag}"
|
||||
docker("buildx", "imagetools", "create", "--prefer-index=true", "--tag", reference,
|
||||
*(f"{repository}@{digest}" for digest in digests.values()))
|
||||
digest = check_index(reference, digests)
|
||||
published.append(f"{reference} -> {digest}")
|
||||
for item in published:
|
||||
print(item)
|
||||
if options.summary:
|
||||
@@ -239,7 +237,8 @@ def main():
|
||||
else:
|
||||
publish_images(options, expected)
|
||||
return 0
|
||||
except (OSError, ValueError, KeyError, TypeError, AttributeError, tarfile.TarError, subprocess.SubprocessError) as exc:
|
||||
except (OSError, ValueError, KeyError, TypeError, AttributeError, RuntimeError,
|
||||
tarfile.TarError, subprocess.SubprocessError) as exc:
|
||||
print(f"ERROR: {exc}", file=sys.stderr)
|
||||
return 1
|
||||
|
||||
|
||||
@@ -792,9 +792,9 @@
|
||||
}
|
||||
},
|
||||
"node_modules/brace-expansion": {
|
||||
"version": "1.1.18",
|
||||
"resolved": "https://registry.npmjs.org/brace-expansion/-/brace-expansion-1.1.18.tgz",
|
||||
"integrity": "sha512-Edep/X9fGqVNmzKBVsDYIOtD+z1tuezV70LBjdCst9Tqu76lsnvRiZ6oTic1n+/BIwX6QDGAO94PN4N2SADvtw==",
|
||||
"version": "1.1.21",
|
||||
"resolved": "https://registry.npmjs.org/brace-expansion/-/brace-expansion-1.1.21.tgz",
|
||||
"integrity": "sha512-9zeA+KLZNNzglF2TPKRQEDyx6Yby7daAkuy8MiPzpXPsYDWi/DRM8jmwUDxokQjYqBpv5DgPiwD4h4ZZSy1Ujw==",
|
||||
"license": "MIT",
|
||||
"dependencies": {
|
||||
"balanced-match": "^1.0.0",
|
||||
@@ -1217,9 +1217,9 @@
|
||||
"license": "MIT"
|
||||
},
|
||||
"node_modules/fast-uri": {
|
||||
"version": "3.1.7",
|
||||
"resolved": "https://registry.npmjs.org/fast-uri/-/fast-uri-3.1.7.tgz",
|
||||
"integrity": "sha512-dOvZVzjdZdz7phd9v6jCbwxrBW3fK6n8Rc0CtdmM4bumzMnxywBYhuph6J819RRw/ku+rLbelwfMunktuzVVHg==",
|
||||
"version": "3.1.8",
|
||||
"resolved": "https://registry.npmjs.org/fast-uri/-/fast-uri-3.1.8.tgz",
|
||||
"integrity": "sha512-GZMtZUTNRpOVIECoXwLNZS5xUGE+mVNbTB8h/7Rwh2TFWcBQiPzTgyZi05BF9UMZKkLJv8XBRJTlU7zg8+ZfMg==",
|
||||
"funding": [
|
||||
{
|
||||
"type": "github",
|
||||
@@ -1505,9 +1505,9 @@
|
||||
"license": "ISC"
|
||||
},
|
||||
"node_modules/ip-address": {
|
||||
"version": "10.7.0",
|
||||
"resolved": "https://registry.npmjs.org/ip-address/-/ip-address-10.7.0.tgz",
|
||||
"integrity": "sha512-BGFsyJd5mpXp3rK6jIdADLNgpJUK1jnjzvYF8lK+VyDab9JAmqN0YOKDdP17HlgKb2+ehPgDc8EtnRLbGCAMhA==",
|
||||
"version": "10.7.2",
|
||||
"resolved": "https://registry.npmjs.org/ip-address/-/ip-address-10.7.2.tgz",
|
||||
"integrity": "sha512-7H/2gFSIitxc0hG3nOI1glS8QLo/EHBFFLk8vEUjXY/xu0AdL8jZ9U1IzO2PUm0d2D/ofQcAifb0g6OBkt8U7w==",
|
||||
"license": "MIT",
|
||||
"engines": {
|
||||
"node": ">= 12"
|
||||
|
||||
@@ -18,5 +18,19 @@
|
||||
"sha512": "5e57abc8c85bba4b1d2ada1f9d667d4a4fb80944db9ace0c6ff114df6499c13a00c99e51815a2cfe48bc9fecabd0b5873862ae6a46d74eeb983472d08e175d82"
|
||||
},
|
||||
"test_daemon_image": "docker.io/library/docker:29-dind@sha256:5efed980cba3fc126cf54e21a5a6ff8849d05b6e0623d6e7612f48e9cd6cd17e",
|
||||
"test_sandbox_image": "docker.io/library/busybox:1.37@sha256:9db7b59979c38555a39def84a31fb98b5296952f9e3afd4f6f11f05b07adfab0"
|
||||
"test_sandbox_image": "docker.io/library/busybox:1.37@sha256:9db7b59979c38555a39def84a31fb98b5296952f9e3afd4f6f11f05b07adfab0",
|
||||
"npm_bundled": {
|
||||
"brace-expansion": {
|
||||
"from": "5.0.9",
|
||||
"version": "5.0.12",
|
||||
"url": "https://registry.npmjs.org/brace-expansion/-/brace-expansion-5.0.12.tgz",
|
||||
"sha512": "628bd0debce168b308ac38cd0cc90d4b4d6d79af77aa11211b9c72f1febe47497e770dca8bee6c0a822823de368ae2d94c75a881692d830543854d3d88d12299"
|
||||
},
|
||||
"undici": {
|
||||
"from": "6.28.0",
|
||||
"version": "6.28.1",
|
||||
"url": "https://registry.npmjs.org/undici/-/undici-6.28.1.tgz",
|
||||
"sha512": "cd6a5d4d50f9e07e3c088c9b2f4ad6437ba4a5bf5ddb7c0cede1f946d7dd99e3fbd1c587363b5fa3b3f8bd95fee42a0370ee772783eee6e5e9ee535f8dce8344"
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -34,8 +34,68 @@ export function verifyArchiveReader(directory) {
|
||||
assert.equal(oversized, false, 'archive reader trusted the attacker-declared size');
|
||||
}
|
||||
|
||||
export async function patchNpmBundles(root, pins) {
|
||||
root = fs.realpathSync(root);
|
||||
const store = path.join(root, '.pnpm');
|
||||
for (const [name, pin] of Object.entries(pins)) {
|
||||
assert.ok(['brace-expansion', 'undici'].includes(name));
|
||||
assert.match(pin.version, /^\d+\.\d+\.\d+$/);
|
||||
assert.equal(pin.url, `https://registry.npmjs.org/${name}/-/${name}-${pin.version}.tgz`);
|
||||
assert.match(pin.sha512, /^[a-f0-9]{128}$/);
|
||||
const outdated = [];
|
||||
for (const entry of fs.readdirSync(store)) {
|
||||
if (!/^npm@\d/.test(entry)) continue;
|
||||
const directory = path.join(store, entry, 'node_modules/npm/node_modules', name);
|
||||
if (!fs.existsSync(directory)) continue;
|
||||
assert.equal(fs.realpathSync(directory), directory, 'unexpected bundled npm dependency path');
|
||||
const pkg = JSON.parse(fs.readFileSync(path.join(directory, 'package.json')));
|
||||
assert.equal(pkg.name, name);
|
||||
if (pkg.version === pin.from) outdated.push(directory);
|
||||
else {
|
||||
assert.match(pkg.version, /^\d+\.\d+\.\d+$/);
|
||||
const fixed = pin.version.split('.').map(Number);
|
||||
assert.ok(pkg.version.split('.').map(Number).reduce((order, value, index) =>
|
||||
order || value - fixed[index], 0) >= 0, `unexpected vulnerable ${name} version`);
|
||||
}
|
||||
}
|
||||
if (!outdated.length) continue;
|
||||
const temporary = fs.mkdtempSync('/tmp/openclaw-npm-bundle-');
|
||||
try {
|
||||
const response = await fetch(pin.url, {redirect: 'error', signal: AbortSignal.timeout(60000)});
|
||||
assert.equal(response.status, 200);
|
||||
const chunks = [];
|
||||
let size = 0;
|
||||
for await (const chunk of response.body) {
|
||||
size += chunk.length;
|
||||
assert.ok(size <= 4 * 1024 * 1024, 'npm dependency archive exceeds expected size');
|
||||
chunks.push(chunk);
|
||||
}
|
||||
const bytes = Buffer.concat(chunks);
|
||||
assert.equal(createHash('sha512').update(bytes).digest('hex'), pin.sha512);
|
||||
const archive = path.join(temporary, 'dependency.tgz');
|
||||
fs.writeFileSync(archive, bytes);
|
||||
execFileSync('tar', ['-xzf', archive, '--no-same-owner', '--no-same-permissions', '-C', temporary]);
|
||||
const replacement = path.join(temporary, 'package');
|
||||
const pkg = JSON.parse(fs.readFileSync(path.join(replacement, 'package.json')));
|
||||
assert.equal(pkg.name, name);
|
||||
assert.equal(pkg.version, pin.version);
|
||||
for (const directory of outdated) {
|
||||
const {uid, gid} = fs.statSync(directory);
|
||||
fs.rmSync(directory, {recursive: true});
|
||||
fs.cpSync(replacement, directory, {recursive: true});
|
||||
execFileSync('chown', ['-R', `${uid}:${gid}`, directory]);
|
||||
console.log(`Replace verified bundled npm dependency ${name} ${pin.from} -> ${pin.version}: ${directory}`);
|
||||
}
|
||||
} finally {
|
||||
fs.rmSync(temporary, {recursive: true});
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
async function main() {
|
||||
const pin = JSON.parse(fs.readFileSync(new URL('./components.json', import.meta.url))).adm_zip;
|
||||
const components = JSON.parse(fs.readFileSync(new URL('./components.json', import.meta.url)));
|
||||
await patchNpmBundles('/app/node_modules', components.npm_bundled);
|
||||
const pin = components.adm_zip;
|
||||
assert.match(pin.version, /^\d+\.\d+\.\d+$/);
|
||||
assert.equal(pin.url, `https://registry.npmjs.org/adm-zip/-/adm-zip-${pin.version}.tgz`);
|
||||
assert.match(pin.sha512, /^[a-f0-9]{128}$/);
|
||||
|
||||
@@ -3,10 +3,20 @@
|
||||
import argparse
|
||||
import hashlib
|
||||
import json
|
||||
from pathlib import Path
|
||||
import re
|
||||
import subprocess
|
||||
import sys
|
||||
import tempfile
|
||||
from pathlib import Path
|
||||
|
||||
LIBRARY = Path(__file__).resolve().parent
|
||||
if str(LIBRARY) not in sys.path:
|
||||
sys.path.insert(0, str(LIBRARY))
|
||||
from registry_image import (
|
||||
create_registry_client,
|
||||
prepare_platform_manifest,
|
||||
save_image,
|
||||
)
|
||||
|
||||
REPOSITORIES = {
|
||||
"codex": "codex-claude-workstation", "opencode": "opencode-workstation",
|
||||
@@ -59,12 +69,15 @@ def stage(args, expected):
|
||||
receipt_path = args.receipt
|
||||
require(not receipt_path.exists() and not receipt_path.is_symlink(), "receipt must be a new file")
|
||||
receipt_path.parent.mkdir(parents=True, exist_ok=True)
|
||||
reference = f"{args.repository}:ci-{args.run_id}-{args.run_attempt}-{args.variant}-{args.platform.split('/')[1]}"
|
||||
# Tag the recorded ID, never a mutable local tag.
|
||||
docker("image", "tag", args.image_id, reference)
|
||||
docker("image", "push", reference)
|
||||
manifest, digest = remote_manifest(reference)
|
||||
check_manifest(manifest, args.image_id)
|
||||
with tempfile.TemporaryDirectory(prefix="tested-image-") as staging:
|
||||
archive = Path(staging) / "image.tar.gz"
|
||||
save_image(args.image_id, archive)
|
||||
prepared = prepare_platform_manifest(archive, args.image_id, Path(staging) / "manifest")
|
||||
digest = create_registry_client(args.repository).publish_platform_manifest(prepared)
|
||||
reference = f"{args.repository}@{digest}"
|
||||
manifest, remote_digest = remote_manifest(reference)
|
||||
require(remote_digest == digest, "registry manifest digest mismatch after publication")
|
||||
check_manifest(manifest, args.image_id)
|
||||
receipt = {"schema_version": 1, **expected, "platform": args.platform,
|
||||
"image_id": args.image_id, "manifest_digest": digest}
|
||||
with receipt_path.open("x", encoding="utf-8") as output:
|
||||
@@ -163,6 +176,7 @@ def main():
|
||||
if __name__ == "__main__":
|
||||
try:
|
||||
main()
|
||||
except (ValueError, OSError, subprocess.CalledProcessError) as error:
|
||||
except (ValueError, OSError, KeyError, TypeError, AttributeError, RuntimeError,
|
||||
subprocess.CalledProcessError) as error:
|
||||
print(f"ERROR: {error}", file=sys.stderr)
|
||||
sys.exit(1)
|
||||
|
||||
@@ -0,0 +1,318 @@
|
||||
"""Publish verified Docker-save images without creating staging tags."""
|
||||
|
||||
import base64
|
||||
import gzip
|
||||
import hashlib
|
||||
import http.client
|
||||
import json
|
||||
import os
|
||||
import re
|
||||
import shutil
|
||||
import subprocess
|
||||
import tarfile
|
||||
import urllib.error
|
||||
import urllib.parse
|
||||
import urllib.request
|
||||
|
||||
DIGEST = re.compile(r"sha256:[a-f0-9]{64}")
|
||||
IMAGE_TYPES = {
|
||||
"application/vnd.oci.image.manifest.v1+json",
|
||||
"application/vnd.docker.distribution.manifest.v2+json",
|
||||
}
|
||||
INDEX_TYPES = {
|
||||
"application/vnd.oci.image.index.v1+json",
|
||||
"application/vnd.docker.distribution.manifest.list.v2+json",
|
||||
}
|
||||
REGISTRY_CONFIG = {
|
||||
"ghcr.io": {
|
||||
"registry_host": "ghcr.io",
|
||||
"token_url": "https://ghcr.io/token",
|
||||
"service": "ghcr.io",
|
||||
"upload_hosts": {"ghcr.io"},
|
||||
},
|
||||
"docker.io": {
|
||||
"registry_host": "registry-1.docker.io",
|
||||
"token_url": "https://auth.docker.io/token",
|
||||
"service": "registry.docker.io",
|
||||
"upload_hosts": {"registry-1.docker.io", "docker.io"},
|
||||
},
|
||||
}
|
||||
BLOB_CHUNK_SIZE = 1024 * 1024
|
||||
|
||||
|
||||
def require(condition, message):
|
||||
if not condition:
|
||||
raise ValueError(message)
|
||||
|
||||
|
||||
def sha256(path):
|
||||
with path.open("rb") as source:
|
||||
return hashlib.file_digest(source, "sha256").hexdigest()
|
||||
|
||||
|
||||
def save_image(image_id, archive):
|
||||
"""Save one local image as the gzip-wrapped Docker archive format."""
|
||||
with archive.open("xb") as target, subprocess.Popen(
|
||||
["docker", "image", "save", image_id], stdout=subprocess.PIPE
|
||||
) as saved, gzip.GzipFile(filename="", mode="wb", fileobj=target,
|
||||
compresslevel=1, mtime=0) as zipped:
|
||||
shutil.copyfileobj(saved.stdout, zipped, BLOB_CHUNK_SIZE)
|
||||
require(saved.wait() == 0, "docker image save failed")
|
||||
|
||||
|
||||
def archive_metadata(archive, image_id):
|
||||
digest = image_id.removeprefix("sha256:")
|
||||
config_names = {f"{digest}.json", f"blobs/sha256/{digest}"}
|
||||
metadata = {}
|
||||
with tarfile.open(archive, "r|gz") as saved:
|
||||
for member in saved:
|
||||
if member.name not in config_names | {"manifest.json"}:
|
||||
continue
|
||||
require(member.isfile() and member.size <= 4 * 1024 * 1024,
|
||||
"invalid image archive metadata member")
|
||||
require(member.name not in metadata, "duplicate image archive metadata")
|
||||
source = saved.extractfile(member)
|
||||
require(source is not None, f"cannot read image archive metadata: {member.name}")
|
||||
with source:
|
||||
metadata[member.name] = source.read()
|
||||
manifest = json.loads(metadata.get("manifest.json", b"null"))
|
||||
require(isinstance(manifest, list) and len(manifest) == 1
|
||||
and isinstance(manifest[0], dict), "archive must contain exactly one image")
|
||||
config_name = manifest[0].get("Config")
|
||||
require(isinstance(config_name, str) and config_name in config_names and config_name in metadata,
|
||||
"archive config does not match tested image ID")
|
||||
config_bytes = metadata[config_name]
|
||||
require(hashlib.sha256(config_bytes).hexdigest() == digest, "archive image config digest mismatch")
|
||||
return config_bytes, manifest[0]
|
||||
|
||||
|
||||
def prepare_platform_manifest(archive, image_id, destination):
|
||||
"""Convert one Docker-save archive into a digest-addressed platform manifest."""
|
||||
require(DIGEST.fullmatch(image_id), "invalid image ID")
|
||||
config_bytes, saved_manifest = archive_metadata(archive, image_id)
|
||||
layer_names = saved_manifest.get("Layers")
|
||||
require(isinstance(layer_names, list) and all(isinstance(name, str) and name for name in layer_names),
|
||||
"archive image layers are invalid")
|
||||
config = json.loads(config_bytes)
|
||||
require(isinstance(config, dict), "image config is invalid")
|
||||
rootfs = config.get("rootfs")
|
||||
require(isinstance(rootfs, dict) and rootfs.get("type") == "layers", "image rootfs is invalid")
|
||||
diff_ids = rootfs.get("diff_ids")
|
||||
require(isinstance(diff_ids, list) and len(diff_ids) == len(layer_names)
|
||||
and all(isinstance(value, str) and DIGEST.fullmatch(value) for value in diff_ids),
|
||||
"image rootfs layer digests are invalid")
|
||||
|
||||
destination.mkdir(parents=True, exist_ok=True)
|
||||
config_path = destination / "config.json"
|
||||
config_path.write_bytes(config_bytes)
|
||||
blobs = {image_id: config_path}
|
||||
layers = {}
|
||||
uncompressed_digests = {}
|
||||
found_layers = set()
|
||||
with tarfile.open(archive, "r|gz") as saved:
|
||||
for member in saved:
|
||||
if member.name not in layer_names:
|
||||
continue
|
||||
require(member.isfile(), f"image archive layer is not a regular file: {member.name}")
|
||||
require(member.name not in found_layers, f"duplicate image archive layer member: {member.name}")
|
||||
found_layers.add(member.name)
|
||||
source = saved.extractfile(member)
|
||||
require(source is not None, f"cannot read image archive layer: {member.name}")
|
||||
require(member.size >= 0 and member.size <= shutil.disk_usage(destination).free,
|
||||
f"image archive layer exceeds available staging space: {member.name}")
|
||||
raw_layer = destination / f"layer-{len(layers)}.tar"
|
||||
with source, raw_layer.open("wb") as target:
|
||||
shutil.copyfileobj(source, target, BLOB_CHUNK_SIZE)
|
||||
require(raw_layer.stat().st_size == member.size,
|
||||
f"image archive layer size changed while reading: {member.name}")
|
||||
compressed_layer = destination / f"layer-{len(layers)}.tar.gz"
|
||||
with raw_layer.open("rb") as source:
|
||||
magic = source.read(4)
|
||||
require(magic != b"\x28\xb5\x2f\xfd", "zstd image archive layers are not supported")
|
||||
if magic.startswith(b"\x1f\x8b"):
|
||||
with gzip.open(raw_layer, "rb") as source:
|
||||
uncompressed_digests[member.name] = "sha256:" + hashlib.file_digest(source, "sha256").hexdigest()
|
||||
shutil.copyfile(raw_layer, compressed_layer)
|
||||
else:
|
||||
uncompressed_digests[member.name] = "sha256:" + sha256(raw_layer)
|
||||
with raw_layer.open("rb") as source, compressed_layer.open("wb") as target, \
|
||||
gzip.GzipFile(filename="", mode="wb", fileobj=target,
|
||||
compresslevel=1, mtime=0) as zipped:
|
||||
shutil.copyfileobj(source, zipped, BLOB_CHUNK_SIZE)
|
||||
raw_layer.unlink()
|
||||
digest = "sha256:" + sha256(compressed_layer)
|
||||
blobs[digest] = compressed_layer
|
||||
layers[member.name] = {
|
||||
"mediaType": "application/vnd.docker.image.rootfs.diff.tar.gzip",
|
||||
"size": compressed_layer.stat().st_size,
|
||||
"digest": digest,
|
||||
}
|
||||
require(found_layers == set(layer_names), "image archive is missing a declared layer")
|
||||
require([uncompressed_digests[name] for name in layer_names] == diff_ids,
|
||||
"image archive layer content does not match tested rootfs")
|
||||
|
||||
manifest = {
|
||||
"schemaVersion": 2,
|
||||
"mediaType": "application/vnd.docker.distribution.manifest.v2+json",
|
||||
"config": {
|
||||
"mediaType": "application/vnd.docker.container.image.v1+json",
|
||||
"size": len(config_bytes),
|
||||
"digest": image_id,
|
||||
},
|
||||
# Docker save may write tar members in digest order, not filesystem order.
|
||||
"layers": [layers[name] for name in layer_names],
|
||||
}
|
||||
raw_manifest = json.dumps(manifest, separators=(",", ":")).encode()
|
||||
manifest_path = destination / "manifest.json"
|
||||
manifest_path.write_bytes(raw_manifest)
|
||||
return {
|
||||
"path": manifest_path,
|
||||
"digest": "sha256:" + hashlib.sha256(raw_manifest).hexdigest(),
|
||||
"manifest": manifest,
|
||||
"blobs": blobs,
|
||||
}
|
||||
|
||||
|
||||
class _NoRedirect(urllib.request.HTTPRedirectHandler):
|
||||
def redirect_request(self, request, fp, code, msg, headers, new_url):
|
||||
return None
|
||||
|
||||
|
||||
class RegistryClient:
|
||||
"""Minimal Registry V2 client for digest-only platform publication."""
|
||||
|
||||
def __init__(self, repository, username, password):
|
||||
registry, name = repository.split("/", 1)
|
||||
settings = REGISTRY_CONFIG[registry]
|
||||
self.repository = name
|
||||
self.registry_host = settings["registry_host"]
|
||||
self.token_url = settings["token_url"]
|
||||
self.service = settings["service"]
|
||||
self.upload_hosts = settings["upload_hosts"]
|
||||
self.username = username
|
||||
self.password = password
|
||||
self.token = None
|
||||
|
||||
def _get_token(self):
|
||||
query = urllib.parse.urlencode({
|
||||
"service": self.service,
|
||||
"scope": f"repository:{self.repository}:pull,push",
|
||||
})
|
||||
credentials = base64.b64encode(f"{self.username}:{self.password}".encode()).decode()
|
||||
request = urllib.request.Request(
|
||||
f"{self.token_url}?{query}",
|
||||
headers={"Accept": "application/json", "Authorization": f"Basic {credentials}"},
|
||||
)
|
||||
opener = urllib.request.build_opener(urllib.request.ProxyHandler({}), _NoRedirect())
|
||||
try:
|
||||
with opener.open(request, timeout=60) as response:
|
||||
payload = json.loads(response.read(4 * 1024 * 1024))
|
||||
except (urllib.error.HTTPError, urllib.error.URLError, TimeoutError, json.JSONDecodeError) as exc:
|
||||
raise RuntimeError(f"registry token request failed for {self.repository}: {type(exc).__name__}") from None
|
||||
require(isinstance(payload, dict), "registry token response is invalid")
|
||||
token = payload.get("token") or payload.get("access_token")
|
||||
require(isinstance(token, str) and token and not any(char.isspace() for char in token),
|
||||
"registry token response is invalid")
|
||||
self.token = token
|
||||
return token
|
||||
|
||||
def _request(self, method, url, expected, *, body=None, body_path=None, content_type=None):
|
||||
require(body is None or body_path is None, "registry request body is ambiguous")
|
||||
for attempt in range(2):
|
||||
headers = {"Authorization": f"Bearer {self.token or self._get_token()}"}
|
||||
if body is not None:
|
||||
headers["Content-Length"] = str(len(body))
|
||||
if body_path is not None:
|
||||
headers["Content-Length"] = str(body_path.stat().st_size)
|
||||
if content_type:
|
||||
headers["Content-Type"] = content_type
|
||||
status, response_headers = self._send(method, url, headers, body=body, body_path=body_path)
|
||||
if status != 401:
|
||||
require(status in expected,
|
||||
f"registry {method} failed for {self.repository}: HTTP {status}")
|
||||
return status, response_headers
|
||||
self.token = None
|
||||
if attempt == 1:
|
||||
raise RuntimeError(f"registry authorization failed for {self.repository}")
|
||||
raise RuntimeError("registry request retry failed")
|
||||
|
||||
@staticmethod
|
||||
def _send(method, url, headers, *, body=None, body_path=None):
|
||||
parsed = urllib.parse.urlsplit(url)
|
||||
require(parsed.scheme == "https" and parsed.hostname, "registry URL must use HTTPS")
|
||||
target = urllib.parse.urlunsplit(("", "", parsed.path or "/", parsed.query, ""))
|
||||
connection = http.client.HTTPSConnection(parsed.hostname, parsed.port, timeout=120)
|
||||
try:
|
||||
connection.putrequest(method, target)
|
||||
for key, value in headers.items():
|
||||
connection.putheader(key, value)
|
||||
connection.endheaders()
|
||||
if body is not None:
|
||||
connection.send(body)
|
||||
elif body_path is not None:
|
||||
with body_path.open("rb") as source:
|
||||
while chunk := source.read(BLOB_CHUNK_SIZE):
|
||||
connection.send(chunk)
|
||||
response = connection.getresponse()
|
||||
response_headers = {key.lower(): value for key, value in response.getheaders()}
|
||||
response.read(4 * 1024 * 1024 + 1)
|
||||
return response.status, response_headers
|
||||
finally:
|
||||
connection.close()
|
||||
|
||||
def _upload_url(self, location):
|
||||
upload_url = urllib.parse.urljoin(f"https://{self.registry_host}", location)
|
||||
parsed = urllib.parse.urlsplit(upload_url)
|
||||
require(parsed.scheme == "https" and parsed.hostname in self.upload_hosts
|
||||
and not parsed.username and not parsed.password,
|
||||
"registry upload location is not an allowed registry host")
|
||||
return upload_url
|
||||
|
||||
def _path(self, suffix):
|
||||
return f"https://{self.registry_host}/v2/{self.repository}/{suffix}"
|
||||
|
||||
def _upload_blob(self, digest, path):
|
||||
blob_url = self._path(f"blobs/{digest}")
|
||||
status, _ = self._request("HEAD", blob_url, (200, 404))
|
||||
if status == 200:
|
||||
return
|
||||
_, headers = self._request("POST", self._path("blobs/uploads/"), (202,))
|
||||
location = headers.get("location")
|
||||
require(location, "registry did not return a blob upload location")
|
||||
upload_url = self._upload_url(location)
|
||||
parsed = urllib.parse.urlsplit(upload_url)
|
||||
# Upload locations can contain opaque signed state; preserve its encoding.
|
||||
# https://github.com/opencontainers/distribution-spec/blob/main/spec.md#post-then-put
|
||||
query = parsed.query + ("&" if parsed.query else "") + urllib.parse.urlencode({"digest": digest})
|
||||
upload_url = urllib.parse.urlunsplit((parsed.scheme, parsed.netloc, parsed.path,
|
||||
query, ""))
|
||||
self._request("PUT", upload_url, (201,), body_path=path,
|
||||
content_type="application/octet-stream")
|
||||
|
||||
def publish_platform_manifest(self, prepared):
|
||||
raw_manifest = prepared["path"].read_bytes()
|
||||
digest = prepared["digest"]
|
||||
require("sha256:" + hashlib.sha256(raw_manifest).hexdigest() == digest,
|
||||
"prepared platform manifest digest changed")
|
||||
manifest = prepared["manifest"]
|
||||
descriptors = [manifest["config"], *manifest.get("layers", [])]
|
||||
for descriptor in descriptors:
|
||||
blob_digest = descriptor["digest"]
|
||||
blob_path = prepared["blobs"].get(blob_digest)
|
||||
require(blob_path is not None and blob_path.is_file(),
|
||||
f"missing prepared image blob: {blob_digest}")
|
||||
self._upload_blob(blob_digest, blob_path)
|
||||
self._request("PUT", self._path(f"manifests/{digest}"), (201,), body=raw_manifest,
|
||||
content_type=manifest["mediaType"])
|
||||
return digest
|
||||
|
||||
|
||||
def create_registry_client(repository):
|
||||
registry = repository.split("/", 1)[0]
|
||||
credentials = {
|
||||
"ghcr.io": (os.environ.get("GHCR_USERNAME"), os.environ.get("GHCR_TOKEN")),
|
||||
"docker.io": (os.environ.get("DOCKERHUB_USERNAME"), os.environ.get("DOCKERHUB_TOKEN")),
|
||||
}[registry]
|
||||
require(all(isinstance(value, str) and value for value in credentials),
|
||||
f"credentials are not configured for {registry}")
|
||||
return RegistryClient(repository, *credentials)
|
||||
@@ -2,6 +2,7 @@
|
||||
"""Exercise the desktop, daemon, Puppeteer/stealth, sharp and persisted cookies."""
|
||||
import argparse
|
||||
import json
|
||||
import re
|
||||
import subprocess
|
||||
import sys
|
||||
import time
|
||||
@@ -100,11 +101,20 @@ def main(image, variant):
|
||||
else:
|
||||
raise RuntimeError("desktop/daemon did not become ready")
|
||||
if variant == "linuxserver":
|
||||
docker("exec", name, "nginx", "-t")
|
||||
nginx = docker("exec", name, "nginx", "-T")
|
||||
# Selkies 2 moved file downloads behind the /api proxy.
|
||||
# https://github.com/linuxserver/docker-baseimage-selkies/commit/c28cd610744dae7c278f2a49ca28748dbf387c1a
|
||||
config = nginx.stdout + nginx.stderr
|
||||
files_path = "/api/files/" if re.search(r"\blocation\s+/api\s*\{", config) else "/files/"
|
||||
files_url = "https://127.0.0.1:3001" + files_path
|
||||
docker("exec", name, "sh", "-c", "printf '%s' 'release-factory files smoke' > /config/Desktop/rf-smoke.txt")
|
||||
files = docker("exec", name, "curl", "-kfsS", "--max-time", "5", "-u",
|
||||
"smoke:ci-gemini-password", "https://127.0.0.1:3001/files/")
|
||||
# Nginx can serve the desktop before the Selkies API is ready.
|
||||
curl = ("exec", name, "curl", "-kfsS", "--max-time", "5", "--retry", "10",
|
||||
"--retry-delay", "1", "--retry-max-time", "30", "-u", "smoke:ci-gemini-password")
|
||||
files = docker(*curl, files_url)
|
||||
assert "rf-smoke.txt" in files.stdout, "file browser did not list the test file"
|
||||
downloaded = docker(*curl, files_url + "rf-smoke.txt")
|
||||
assert downloaded.stdout == "release-factory files smoke", "file download content mismatch"
|
||||
result = docker("exec", "-w", "/opt/gemini-skill", "-e", "SMOKE_PHASE=" + phase,
|
||||
name, "node", "--input-type=module", "-e", PROBE)
|
||||
print(result.stdout, end="", flush=True)
|
||||
|
||||
@@ -0,0 +1,53 @@
|
||||
import assert from 'node:assert/strict';
|
||||
import fs from 'node:fs';
|
||||
import os from 'node:os';
|
||||
import path from 'node:path';
|
||||
import test from 'node:test';
|
||||
import { patchNpmBundles } from '../openclaw-builder/image/patch-vendored-deps.mjs';
|
||||
|
||||
const pins = JSON.parse(fs.readFileSync(new URL('../openclaw-builder/configs/components.json', import.meta.url))).npm_bundled;
|
||||
|
||||
test('repairs only the two vulnerable dependencies inside bundled npm', async () => {
|
||||
const temporary = fs.mkdtempSync(path.join(os.tmpdir(), 'openclaw-npm-test-'));
|
||||
try {
|
||||
const root = path.join(temporary, 'node_modules');
|
||||
const npm = path.join(root, '.pnpm/npm@11.20.0/node_modules/npm');
|
||||
fs.mkdirSync(npm, {recursive: true});
|
||||
fs.writeFileSync(path.join(npm, 'package.json'), '{"name":"npm","version":"11.20.0"}\n');
|
||||
for (const [name, pin] of Object.entries(pins)) {
|
||||
const directory = path.join(npm, 'node_modules', name);
|
||||
fs.mkdirSync(directory, {recursive: true});
|
||||
fs.writeFileSync(path.join(directory, 'package.json'), JSON.stringify({name, version: pin.from}));
|
||||
}
|
||||
const unrelated = path.join(root, 'unrelated');
|
||||
fs.mkdirSync(unrelated);
|
||||
fs.writeFileSync(path.join(unrelated, 'package.json'), '{"name":"undici","version":"6.28.0"}');
|
||||
await patchNpmBundles(root, pins);
|
||||
for (const [name, pin] of Object.entries(pins)) {
|
||||
const pkg = JSON.parse(fs.readFileSync(path.join(npm, 'node_modules', name, 'package.json')));
|
||||
assert.equal(pkg.version, pin.version);
|
||||
assert.equal(pkg.name, name);
|
||||
}
|
||||
assert.equal(JSON.parse(fs.readFileSync(path.join(unrelated, 'package.json'))).version, '6.28.0');
|
||||
assert.equal(fs.readFileSync(path.join(npm, 'package.json'), 'utf8'), '{"name":"npm","version":"11.20.0"}\n');
|
||||
await patchNpmBundles(root, pins); // Idempotent and does not download the archives again.
|
||||
} finally {
|
||||
fs.rmSync(temporary, {recursive: true});
|
||||
}
|
||||
});
|
||||
|
||||
test('rejects a bundle dependency that escapes its npm root', async () => {
|
||||
const temporary = fs.mkdtempSync(path.join(os.tmpdir(), 'openclaw-npm-escape-'));
|
||||
try {
|
||||
const root = path.join(temporary, 'node_modules');
|
||||
const npm = path.join(root, '.pnpm/npm@11.20.0/node_modules/npm/node_modules');
|
||||
const outside = path.join(temporary, 'outside');
|
||||
fs.mkdirSync(npm, {recursive: true});
|
||||
fs.mkdirSync(outside);
|
||||
fs.writeFileSync(path.join(outside, 'package.json'), '{"name":"undici","version":"6.28.0"}');
|
||||
fs.symlinkSync(outside, path.join(npm, 'undici'));
|
||||
await assert.rejects(patchNpmBundles(root, pins), /unexpected bundled npm dependency path/);
|
||||
} finally {
|
||||
fs.rmSync(temporary, {recursive: true});
|
||||
}
|
||||
});
|
||||
@@ -3,13 +3,13 @@
|
||||
import importlib.util
|
||||
import json
|
||||
import os
|
||||
from pathlib import Path
|
||||
import subprocess
|
||||
import sys
|
||||
import tempfile
|
||||
import textwrap
|
||||
from types import SimpleNamespace
|
||||
import unittest
|
||||
from pathlib import Path
|
||||
from types import SimpleNamespace
|
||||
from unittest.mock import patch
|
||||
|
||||
SPEC = importlib.util.spec_from_file_location("publish_tested", Path(__file__).with_name("publish-tested-image.py"))
|
||||
@@ -49,7 +49,7 @@ class PublicationTests(unittest.TestCase):
|
||||
directory.mkdir()
|
||||
(directory / "receipt.json").write_text(json.dumps(receipt))
|
||||
|
||||
def test_stage_tags_the_recorded_id_and_checks_remote_config(self):
|
||||
def test_stage_publishes_the_recorded_id_by_digest_without_a_ci_tag(self):
|
||||
with tempfile.TemporaryDirectory() as tmp:
|
||||
args = SimpleNamespace(**self.expected, image="mutable:tag", image_id=self.image_id,
|
||||
platform="linux/amd64", receipt=Path(tmp) / "receipt.json")
|
||||
@@ -59,9 +59,21 @@ class PublicationTests(unittest.TestCase):
|
||||
def docker(*argv, **kwargs):
|
||||
calls.append(argv)
|
||||
return json.dumps([image]).encode() if kwargs.get("capture") else None
|
||||
with patch.object(MODULE, "docker", docker), patch.object(MODULE, "remote_manifest", return_value=(self.manifest, self.digest)):
|
||||
class Client:
|
||||
def __init__(self, digest):
|
||||
self.digest = digest
|
||||
|
||||
def publish_platform_manifest(self, prepared):
|
||||
return self.digest
|
||||
|
||||
with patch.object(MODULE, "docker", docker), \
|
||||
patch.object(MODULE, "save_image"), \
|
||||
patch.object(MODULE, "prepare_platform_manifest", return_value=object()), \
|
||||
patch.object(MODULE, "create_registry_client", return_value=Client(self.digest)), \
|
||||
patch.object(MODULE, "remote_manifest", return_value=(self.manifest, self.digest)):
|
||||
MODULE.stage(args, self.expected)
|
||||
self.assertEqual(calls[1][0:3], ("image", "tag", self.image_id))
|
||||
self.assertEqual(calls, [("image", "inspect", "mutable:tag")])
|
||||
self.assertNotIn(":ci-", str(json.loads(args.receipt.read_text())))
|
||||
self.assertEqual(json.loads(args.receipt.read_text())["image_id"], self.image_id)
|
||||
|
||||
def test_changed_image_or_platform_never_reaches_registry_write(self):
|
||||
|
||||
@@ -0,0 +1,130 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Verify image-layer identity and the Registry V2 digest-only upload contract."""
|
||||
import gzip
|
||||
import hashlib
|
||||
import io
|
||||
import json
|
||||
import tarfile
|
||||
import tempfile
|
||||
import unittest
|
||||
from pathlib import Path
|
||||
from unittest.mock import patch
|
||||
|
||||
import registry_image as registry
|
||||
|
||||
|
||||
def digest(data):
|
||||
return "sha256:" + hashlib.sha256(data).hexdigest()
|
||||
|
||||
|
||||
def layer(content):
|
||||
stream = io.BytesIO()
|
||||
with tarfile.open(fileobj=stream, mode="w") as tar:
|
||||
member = tarfile.TarInfo("value.txt")
|
||||
member.size = len(content)
|
||||
tar.addfile(member, io.BytesIO(content))
|
||||
return stream.getvalue()
|
||||
|
||||
|
||||
class LayerTests(unittest.TestCase):
|
||||
def prepare(self, *, compressed=False, corrupt=None):
|
||||
tmp = tempfile.TemporaryDirectory()
|
||||
self.addCleanup(tmp.cleanup)
|
||||
root = Path(tmp.name)
|
||||
data = [layer(b"base"), layer(b"upper")]
|
||||
names = ["base/layer.tar", "upper/layer.tar"]
|
||||
config = {"rootfs": {"type": "layers", "diff_ids": [digest(b) for b in data]}}
|
||||
if corrupt == "count":
|
||||
config["rootfs"]["diff_ids"].pop()
|
||||
if corrupt == "digest":
|
||||
config["rootfs"]["diff_ids"][0] = "invalid"
|
||||
if corrupt == "type":
|
||||
config["rootfs"]["type"] = "unknown"
|
||||
config_bytes = json.dumps(config).encode()
|
||||
image_id = digest(config_bytes)
|
||||
config_name = image_id[7:] + ".json"
|
||||
manifest = [{"Config": config_name, "Layers": names}]
|
||||
members = [(config_name, config_bytes), ("manifest.json", json.dumps(manifest).encode())]
|
||||
for name, raw in reversed(list(zip(names, data))):
|
||||
if corrupt == "missing" and name == names[0]:
|
||||
continue
|
||||
if corrupt == "bytes" and name == names[0]:
|
||||
raw = layer(b"wrong")
|
||||
stored = gzip.compress(raw, mtime=0) if compressed else raw
|
||||
if corrupt == "gzip" and name == names[0]:
|
||||
stored = gzip.compress(raw)[:-8]
|
||||
if corrupt == "zstd" and name == names[0]:
|
||||
stored = b"\x28\xb5\x2f\xfd" + raw
|
||||
members.append((name, stored))
|
||||
archive = root / "image.tar.gz"
|
||||
with tarfile.open(archive, "w:gz") as tar:
|
||||
for name, body in members:
|
||||
member = tarfile.TarInfo(name)
|
||||
member.size = len(body)
|
||||
tar.addfile(member, io.BytesIO(body))
|
||||
prepared = registry.prepare_platform_manifest(archive, image_id, root / "prepared")
|
||||
return prepared, config
|
||||
|
||||
def test_tar_member_order_does_not_change_filesystem_layer_order(self):
|
||||
for compressed in (False, True):
|
||||
with self.subTest(compressed=compressed):
|
||||
prepared, config = self.prepare(compressed=compressed)
|
||||
actual = [digest(gzip.decompress(prepared["blobs"][d["digest"]].read_bytes()))
|
||||
for d in prepared["manifest"]["layers"]]
|
||||
self.assertEqual(actual, config["rootfs"]["diff_ids"])
|
||||
|
||||
def test_invalid_layers_fail_before_publication(self):
|
||||
for corrupt in ("count", "digest", "type", "missing", "bytes", "gzip", "zstd"):
|
||||
with self.subTest(corrupt=corrupt), self.assertRaises((ValueError, EOFError, OSError)):
|
||||
self.prepare(corrupt=corrupt)
|
||||
|
||||
|
||||
class RegistryTests(unittest.TestCase):
|
||||
def client(self, host="ghcr.io"):
|
||||
client = registry.RegistryClient(host + "/owner/image", "user", "test-password")
|
||||
client.token = "test-token"
|
||||
return client
|
||||
|
||||
def test_upload_preserves_opaque_location_query(self):
|
||||
for host in ("ghcr.io", "docker.io"):
|
||||
for absolute in (False, True):
|
||||
with self.subTest(host=host, absolute=absolute), tempfile.TemporaryDirectory() as tmp:
|
||||
client = self.client(host)
|
||||
path = Path(tmp) / "blob"
|
||||
path.write_bytes(b"blob")
|
||||
location = "/v2/owner/image/blobs/uploads/1?token=a%2fb&space=%20&empty&x=1&x=2"
|
||||
if absolute:
|
||||
location = "https://" + client.registry_host + location
|
||||
with patch.object(client, "_send", side_effect=[
|
||||
(404, {}), (202, {"location": location}), (201, {}),
|
||||
]) as send:
|
||||
client._upload_blob(digest(b"blob"), path)
|
||||
expected = (location if absolute else "https://" + client.registry_host + location)
|
||||
self.assertEqual(send.call_args.args[1], expected + "&digest=" + digest(b"blob").replace(":", "%3A"))
|
||||
|
||||
def test_incomplete_upload_is_rejected(self):
|
||||
client = self.client()
|
||||
with tempfile.TemporaryDirectory() as tmp:
|
||||
path = Path(tmp) / "blob"
|
||||
path.write_bytes(b"blob")
|
||||
with patch.object(client, "_send", side_effect=[
|
||||
(404, {}), (202, {"location": "/v2/owner/image/blobs/uploads/1"}), (202, {}),
|
||||
]), self.assertRaisesRegex(ValueError, "HTTP 202"):
|
||||
client._upload_blob(digest(b"blob"), path)
|
||||
|
||||
def test_existing_blob_does_not_start_an_upload(self):
|
||||
client = self.client()
|
||||
with patch.object(client, "_send", return_value=(200, {})) as send:
|
||||
client._upload_blob(digest(b"blob"), Path("unused"))
|
||||
self.assertEqual(send.call_count, 1)
|
||||
self.assertEqual(send.call_args.args[0], "HEAD")
|
||||
|
||||
def test_untrusted_upload_locations_are_rejected(self):
|
||||
client = self.client()
|
||||
for url in ("https://evil.example/upload", "http://ghcr.io/upload", "https://user@ghcr.io/upload"):
|
||||
with self.subTest(url=url), self.assertRaises(ValueError):
|
||||
client._upload_url(url)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
@@ -0,0 +1,61 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Keep file browsing covered across LinuxServer's Selkies route migration."""
|
||||
import importlib.util
|
||||
import subprocess
|
||||
import unittest
|
||||
from pathlib import Path
|
||||
from unittest.mock import patch
|
||||
|
||||
SPEC = importlib.util.spec_from_file_location("smoke", Path(__file__).with_name("smoke-gemini-browser.py"))
|
||||
smoke = importlib.util.module_from_spec(SPEC)
|
||||
SPEC.loader.exec_module(smoke)
|
||||
|
||||
|
||||
class FileBrowserTests(unittest.TestCase):
|
||||
def run_smoke(self, route, *, listing="rf-smoke.txt", content="release-factory files smoke"):
|
||||
requested = []
|
||||
|
||||
def docker(*args, **kwargs):
|
||||
output = ""
|
||||
if args[0] == "exec" and "curl" in args:
|
||||
url = args[-1]
|
||||
if url.endswith("/health"):
|
||||
output = '{"service":"browser-daemon"}'
|
||||
elif url == "https://127.0.0.1:3001/":
|
||||
output = "desktop"
|
||||
else:
|
||||
requested.append(url)
|
||||
if not url.startswith("https://127.0.0.1:3001" + route):
|
||||
raise RuntimeError("curl: (22) The requested URL returned error: 404")
|
||||
output = content if url.endswith("rf-smoke.txt") else listing
|
||||
elif args[0] == "exec" and "nginx" in args:
|
||||
output = "server { location " + ("/api" if route == "/api/files/" else "/files") + " { } }"
|
||||
return subprocess.CompletedProcess(args, 0, output, "")
|
||||
|
||||
with patch.object(smoke, "docker", side_effect=docker):
|
||||
smoke.main("test-image", "linuxserver")
|
||||
return requested
|
||||
|
||||
def test_selkies_v2_lists_and_downloads_through_api(self):
|
||||
self.assertEqual(self.run_smoke("/api/files/"), [
|
||||
"https://127.0.0.1:3001/api/files/",
|
||||
"https://127.0.0.1:3001/api/files/rf-smoke.txt",
|
||||
] * 2)
|
||||
|
||||
def test_older_base_images_keep_the_legacy_file_route(self):
|
||||
self.assertEqual(self.run_smoke("/files/"), [
|
||||
"https://127.0.0.1:3001/files/",
|
||||
"https://127.0.0.1:3001/files/rf-smoke.txt",
|
||||
] * 2)
|
||||
|
||||
def test_missing_file_still_fails(self):
|
||||
with self.assertRaisesRegex(AssertionError, "did not list"):
|
||||
self.run_smoke("/api/files/", listing="empty directory")
|
||||
|
||||
def test_wrong_download_still_fails(self):
|
||||
with self.assertRaisesRegex(AssertionError, "content"):
|
||||
self.run_smoke("/api/files/", content="wrong file")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
Reference in New Issue
Block a user