diff --git a/.github/actions/cargo-cache/Dockerfile b/.github/actions/cargo-cache/Dockerfile new file mode 100644 index 000000000..7306ad16e --- /dev/null +++ b/.github/actions/cargo-cache/Dockerfile @@ -0,0 +1,27 @@ +# syntax=docker/dockerfile:1.7@sha256:a57df69d0ea827fb7266491f2813635de6f17269be881f696fbfdf2d83dda33e + +FROM busybox:1.37.0-musl@sha256:5cec3fc171c87218698e85a52af7087de727372aae264a787b8112901a5b0092 AS restore +ARG CACHE_STAMP +RUN --mount=type=bind,target=/cache \ + --mount=type=cache,target=/usr/local/cargo/registry \ + --mount=type=cache,target=/usr/local/cargo/git \ + --mount=type=cache,target=/app/backend/target \ + test -n "$CACHE_STAMP" \ + && cp -a /cache/registry/. /usr/local/cargo/registry/ \ + && cp -a /cache/git/. /usr/local/cargo/git/ \ + && cp -a /cache/target/. /app/backend/target/ + +FROM busybox:1.37.0-musl@sha256:5cec3fc171c87218698e85a52af7087de727372aae264a787b8112901a5b0092 AS collect +ARG CACHE_STAMP +# 一次导出三个 mount,避免逐目录创建镜像、载入 Docker 再复制文件。 +RUN --mount=type=cache,target=/usr/local/cargo/registry \ + --mount=type=cache,target=/usr/local/cargo/git \ + --mount=type=cache,target=/app/backend/target \ + test -n "$CACHE_STAMP" \ + && mkdir -p /cache/registry /cache/git /cache/target \ + && cp -a /usr/local/cargo/registry/. /cache/registry/ \ + && cp -a /usr/local/cargo/git/. /cache/git/ \ + && cp -a /app/backend/target/. /cache/target/ + +FROM scratch AS snapshot +COPY --from=collect /cache/ / diff --git a/.github/actions/cargo-cache/action.yml b/.github/actions/cargo-cache/action.yml new file mode 100644 index 000000000..a7e1bad06 --- /dev/null +++ b/.github/actions/cargo-cache/action.yml @@ -0,0 +1,31 @@ +name: Cargo cache mounts +description: Restore or export a validated Cargo snapshot for a BuildKit builder +inputs: + operation: + description: Cache operation, restore or snapshot + required: true + builder: + description: Buildx builder name + required: true + path: + description: Cargo snapshot directory on the runner + required: true + architecture: + description: Cargo target directory architecture, amd64 or arm64 + required: true +outputs: + usable: + description: Whether the restored or exported snapshot contains Cargo dependencies + value: ${{ steps.cache.outputs.usable }} +runs: + using: composite + steps: + - id: cache + shell: bash + env: + ACTION_PATH: ${{ github.action_path }} + OPERATION: ${{ inputs.operation }} + BUILDER: ${{ inputs.builder }} + CACHE_PATH: ${{ inputs.path }} + ARCHITECTURE: ${{ inputs.architecture }} + run: bash "$ACTION_PATH/cargo-cache.sh" "$OPERATION" "$BUILDER" "$CACHE_PATH" "$ARCHITECTURE" diff --git a/.github/actions/cargo-cache/cargo-cache.sh b/.github/actions/cargo-cache/cargo-cache.sh new file mode 100644 index 000000000..ec4d178a4 --- /dev/null +++ b/.github/actions/cargo-cache/cargo-cache.sh @@ -0,0 +1,56 @@ +#!/usr/bin/env bash +set -euo pipefail + +if [ "$#" -ne 4 ]; then + echo "usage: $0 restore|snapshot BUILDER DIRECTORY amd64|arm64" >&2 + exit 2 +fi +operation="$1" +builder="$2" +directory="$(realpath -m "$3")" +architecture="$4" +case "$operation" in restore|snapshot) ;; *) exit 2 ;; esac +case "$architecture" in amd64|arm64) ;; *) exit 2 ;; esac +dockerfile="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)/Dockerfile" + +usable() { + local root="$1" path crates=false libraries=false + # 目录或 buildstamp 存在不代表编译发生过,必须同时有下载包与目标依赖。 + for path in "$root"/registry/cache/*/*.crate; do + if [ -f "$path" ] && [ -s "$path" ]; then crates=true; break; fi + done + for path in "$root"/target/"$architecture"/release/deps/*.rlib; do + if [ -f "$path" ] && [ -s "$path" ]; then libraries=true; break; fi + done + [ "$crates" = true ] && [ "$libraries" = true ] +} + +command=(docker buildx build --builder "$builder" --file "$dockerfile" --progress plain) +# mount 的内容不参与层指纹,用时间戳重跑复制;--no-cache 会重置 cache mount。 +command+=(--build-arg "CACHE_STAMP=$(date +%s%N)" --target "$operation") +valid=false +if [ "$operation" = restore ]; then + if usable "$directory"; then + mkdir -p "$directory"/{registry,git,target} + "${command[@]}" --output type=cacheonly "$directory" + valid=true + else + echo "No usable Cargo snapshot; cached image layers may still be available" + fi +else + # 导出只发送空上下文,新快照独立校验后替换旧目录,避免残留文件造成误判。 + mkdir -p "$(dirname "$directory")" + temporary="$(mktemp -d "${directory}.XXXXXX")" + trap 'rm -rf "$temporary"' EXIT + mkdir "$temporary/context" + "${command[@]}" --output "type=local,dest=$temporary/snapshot" "$temporary/context" + if usable "$temporary/snapshot"; then + rm -rf "$directory" + mv "$temporary/snapshot" "$directory" + valid=true + fi +fi + +echo "Cargo snapshot for $architecture: usable=$valid" +if [ "$valid" = true ]; then du -sh "$directory"; fi +if [ -n "${GITHUB_OUTPUT:-}" ]; then printf 'usable=%s\n' "$valid" >> "$GITHUB_OUTPUT"; fi diff --git a/.github/dependabot.yml b/.github/dependabot.yml index 82e184a78..a6002ea85 100644 --- a/.github/dependabot.yml +++ b/.github/dependabot.yml @@ -68,7 +68,9 @@ updates: - patch - package-ecosystem: docker - directory: /deploy + directories: + - /deploy + - /.github/actions/cargo-cache labels: [] schedule: interval: weekly diff --git a/.github/workflows/_container.yml b/.github/workflows/_container.yml index 7de06319b..0b906c2e0 100644 --- a/.github/workflows/_container.yml +++ b/.github/workflows/_container.yml @@ -30,39 +30,25 @@ jobs: id: buildx uses: docker/setup-buildx-action@f87e5991a6d7451dcb8d9637bfbc97413f497069 # v4.4.1 - - name: Cache Cargo mounts - if: github.event_name == 'push' && github.ref == 'refs/heads/main' + - name: Restore Cargo snapshot id: cargo-cache - uses: actions/cache@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6.1.0 - with: - path: ${{ runner.temp }}/container-cargo - # 同一平台和依赖复用快照,源码变化由 Cargo 指纹判定是否重编译 - key: container-cargo-v2-${{ runner.os }}-${{ runner.arch }}-${{ hashFiles('backend/Cargo.lock', 'backend/rust-toolchain.toml', 'backend/**/Cargo.toml', 'backend/.cargo/config.toml', 'deploy/Dockerfile') }} - restore-keys: | - container-cargo-v2-${{ runner.os }}-${{ runner.arch }}- - - - name: Restore Cargo mounts for pull requests - if: github.event_name != 'push' || github.ref != 'refs/heads/main' uses: actions/cache/restore@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6.1.0 with: path: ${{ runner.temp }}/container-cargo - key: container-cargo-v2-${{ runner.os }}-${{ runner.arch }}-${{ hashFiles('backend/Cargo.lock', 'backend/rust-toolchain.toml', 'backend/**/Cargo.toml', 'backend/.cargo/config.toml', 'deploy/Dockerfile') }} + # 同一平台和依赖复用快照,源码变化由 Cargo 指纹判定是否重编译 + key: container-cargo-v3-${{ runner.os }}-${{ runner.arch }}-${{ hashFiles('backend/Cargo.lock', 'backend/rust-toolchain.toml', 'backend/**/Cargo.toml', 'backend/.cargo/config.toml', 'deploy/Dockerfile', '.github/actions/cargo-cache/Dockerfile') }} restore-keys: | + container-cargo-v3-${{ runner.os }}-${{ runner.arch }}- container-cargo-v2-${{ runner.os }}-${{ runner.arch }}- - # GHA 层缓存不含 cache mount,只有主分支缺少当前快照时才回收并保存 - name: Restore Cargo mounts to BuildKit - uses: reproducible-containers/buildkit-cache-dance@5422eac04292c961a382e0f584ea0f03ad9da723 # v3.4.0 + id: cargo-restore + uses: ./.github/actions/cargo-cache with: + operation: restore builder: ${{ steps.buildx.outputs.name }} - scratch-dir: ${{ runner.temp }}/container-cache-dance - cache-map: | - { - "${{ runner.temp }}/container-cargo/registry": "/usr/local/cargo/registry", - "${{ runner.temp }}/container-cargo/git": "/usr/local/cargo/git", - "${{ runner.temp }}/container-cargo/target": "/app/backend/target" - } - skip-extraction: ${{ github.event_name != 'push' || github.ref != 'refs/heads/main' || steps.cargo-cache.outputs.cache-hit == 'true' }} + path: ${{ runner.temp }}/container-cargo + architecture: amd64 - name: Build runtime image uses: docker/build-push-action@c3c9e263c25d99ce0380d002d59b67737d91b0dc # v7.4.0 @@ -84,3 +70,23 @@ jobs: image: ${{ env.CPR_IMAGE }} platform: linux/amd64 artifact-name: container-diagnostics-${{ github.run_id }}-${{ github.run_attempt }}-amd64 + + # 构建层命中时 mount 仍可能为空,只有实际检查过内容的快照才允许保存。 + - name: Export Cargo snapshot + if: >- + github.event_name == 'push' && github.ref == 'refs/heads/main' && + (steps.cargo-cache.outputs.cache-hit != 'true' || steps.cargo-restore.outputs.usable != 'true') + id: cargo-snapshot + uses: ./.github/actions/cargo-cache + with: + operation: snapshot + builder: ${{ steps.buildx.outputs.name }} + path: ${{ runner.temp }}/container-cargo + architecture: amd64 + + - name: Save Cargo snapshot + if: steps.cargo-snapshot.outputs.usable == 'true' && steps.cargo-cache.outputs.cache-hit != 'true' + uses: actions/cache/save@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6.1.0 + with: + path: ${{ runner.temp }}/container-cargo + key: ${{ steps.cargo-cache.outputs.cache-primary-key }} diff --git a/.github/workflows/_quality.yml b/.github/workflows/_quality.yml index fe923ddfa..1290de1e9 100644 --- a/.github/workflows/_quality.yml +++ b/.github/workflows/_quality.yml @@ -211,5 +211,8 @@ jobs: -color .github/workflows/*.yml - - name: Check release script syntax - run: bash -n release/publish + - name: Check workflow script syntax + run: | + bash -n release/publish + shopt -s globstar nullglob + for script in .github/**/*.sh; do bash -n "$script"; done diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 966e0eb87..acc4b3ca3 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -44,12 +44,14 @@ jobs: backend: - 'backend/**' - 'release/version.yaml' - - '.github/workflows/**' + - '.github/workflows/ci.yml' + - '.github/workflows/_quality.yml' - '!**/*.md' frontend: - 'frontend/**' - '.github/actions/build-frontend/**' - - '.github/workflows/**' + - '.github/workflows/ci.yml' + - '.github/workflows/_quality.yml' - '!**/*.md' source: - '.gitmodules' @@ -65,7 +67,8 @@ jobs: - 'deploy/**' - '.dockerignore' - 'release/version.yaml' - - '.github/actions/**' + - '.github/actions/verify-container/**' + - '.github/actions/cargo-cache/**' - '.github/workflows/_container.yml' - '.github/workflows/ci.yml' - '!**/*.md' diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 618622fe2..fa10e9db0 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -165,6 +165,39 @@ jobs: puts "build_type=#{version.split("+", 2).first.include?("-exp.") ? "experimental" : "release"}" RUBY + - name: Seal plugin release manifest + env: + VERSION: ${{ steps.version.outputs.version }} + GIT_SHA: ${{ steps.version.outputs.git_sha }} + run: | + ruby <<'RUBY' + require "fileutils" + require "json" + + host_path = "backend/crates/gateway-plugin/runtime/plugin-host-compatibility.json" + host = JSON.parse(File.read(host_path)) + abort "Invalid plugin host compatibility declaration: #{host_path}" unless host["schema_version"] == 1 + # 清单只依赖版本与兼容声明,平台归档和镜像消费同一份封口结果。 + manifest = { + "schema_version" => 1, + "sealed" => true, + "gateway_version" => ENV.fetch("VERSION"), + "gateway_git_sha" => ENV.fetch("GIT_SHA"), + "plugin_host" => host, + "plugins" => [], + } + FileUtils.mkdir_p("dist/plugin-release-assets") + File.write("dist/plugin-release-assets/plugin-release-manifest.json", JSON.pretty_generate(manifest) + "\n") + RUBY + + - name: Upload sealed plugin assets + uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 + with: + name: plugin-release-assets + path: dist/plugin-release-assets + if-no-files-found: error + retention-days: 1 + quality: needs: [release-info] uses: ./.github/workflows/_quality.yml @@ -329,12 +362,17 @@ jobs: pattern: codex-proxy-rs-* path: dist/binaries + - name: Download sealed plugin assets + uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1 + with: + name: plugin-release-assets + path: dist/plugin-release-assets + - name: Package release assets shell: bash env: VERSION: ${{ needs.release-info.outputs.version }} DOCKER_VERSION: ${{ needs.release-info.outputs.docker_version }} - GIT_SHA: ${{ needs.release-info.outputs.git_sha }} run: | set -euo pipefail ruby <<'RUBY' > release-platforms.json @@ -409,21 +447,8 @@ jobs: raise SystemExit(f"Unsupported archive format: {platform['archive']}") release_packages.append((platform, package_root, archive_path)) - plugin_host_path = pathlib.Path("backend/crates/gateway-plugin/runtime/plugin-host-compatibility.json") - plugin_host = json.loads(plugin_host_path.read_text()) - if plugin_host.get("schema_version") != 1: - raise SystemExit(f"Invalid plugin host compatibility declaration: {plugin_host_path}") - plugin_manifest_path = release_dir / "plugin-release-manifest.json" - plugin_manifest_path.write_text(json.dumps({ - "schema_version": 1, - "sealed": True, - "gateway_version": version, - "gateway_git_sha": os.environ["GIT_SHA"], - "plugin_host": plugin_host, - # 示例独立发版;当前不内置插件,仍保留更新与回滚所需的封口清单。 - "plugins": [], - }, indent=2, sort_keys=True) + "\n") + shutil.copy2("dist/plugin-release-assets/plugin-release-manifest.json", plugin_manifest_path) assets.append(plugin_manifest_path) # 保留运行时清单目录合同,宿主包不再携带教学示例。 @@ -456,10 +481,12 @@ jobs: path: dist/release if-no-files-found: error retention-days: 1 + # tar.gz 已压缩,Actions 只负责传递原始发行字节。 + compression-level: 0 build-docker-image: name: build-docker-image (${{ matrix.dockerPlatform }}) - needs: [release-info, build-docker-binary, package-assets] + needs: [release-info, frontend, build-docker-binary] strategy: fail-fast: false matrix: ${{ fromJson(needs.release-info.outputs.docker_matrix) }} @@ -492,11 +519,11 @@ jobs: name: codex-proxy-rs-${{ matrix.os }}-${{ matrix.arch }} path: dist/docker-artifacts/codex-proxy-rs-${{ matrix.os }}-${{ matrix.arch }} - - name: Download sealed release assets + - name: Download sealed plugin assets uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1 with: - name: release-assets - path: dist/release-assets + name: plugin-release-assets + path: dist/plugin-release-assets - name: Prepare Docker prebuilt assets shell: bash @@ -524,7 +551,7 @@ jobs: shutil.copy2(binary_src, binary_dest) binary_dest.chmod(0o755) - release_dir = pathlib.Path("dist/release-assets") + release_dir = pathlib.Path("dist/plugin-release-assets") manifest_path = release_dir / "plugin-release-manifest.json" manifest = json.loads(manifest_path.read_text()) if manifest.get("schema_version") != 1 or manifest.get("sealed") is not True: diff --git a/.github/workflows/security.yml b/.github/workflows/security.yml index f750d5af9..905037359 100644 --- a/.github/workflows/security.yml +++ b/.github/workflows/security.yml @@ -27,20 +27,15 @@ jobs: with: ref: ${{ github.event.pull_request.head.sha || github.sha }} - - name: Setup Rust - uses: dtolnay/rust-toolchain@1a3a6d54512beeaffd394f8d516ca16f2c506a20 # 1.97 - - - name: Cache Rust - uses: Swatinem/rust-cache@6323deb102c322ba6fcbdcafc7e3dddab59af2b6 # v2.9.2 - with: - workspaces: backend -> target - - name: Install cargo-audit - run: cargo install cargo-audit --version 0.22.2 --locked + uses: taiki-e/install-action@4cef1412cce204788f482e778a0b9187f9626a29 # v2.87.21 + with: + tool: cargo-audit@0.22.2 + fallback: none - name: Run Cargo audits run: | - cargo audit --file backend/Cargo.lock + cargo-audit audit --file backend/Cargo.lock frontend-security: runs-on: ubuntu-latest