mirror of
https://github.com/zyycn/codex-proxy-rs.git
synced 2026-11-06 08:06:19 +00:00
fix(ci): reduce Cargo cache snapshot disk usage
Keep Cargo snapshots compressed during transfer, extract directly into BuildKit cache mounts, and release restored runner snapshots. Export archives without an extra copy layer and use a new cache format key. Co-authored-by: Codex <noreply@openai.com>
This commit is contained in:
@@ -7,21 +7,18 @@ RUN --mount=type=bind,target=/cache \
|
||||
--mount=type=cache,target=/usr/local/cargo/git \
|
||||
--mount=type=cache,target=/app/backend/target \
|
||||
test -n "$CACHE_STAMP" \
|
||||
&& cp -a /cache/registry/. /usr/local/cargo/registry/ \
|
||||
&& cp -a /cache/git/. /usr/local/cargo/git/ \
|
||||
&& cp -a /cache/target/. /app/backend/target/
|
||||
&& tar -xzf /cache/registry.tar.gz -C /usr/local/cargo/registry \
|
||||
&& tar -xzf /cache/git.tar.gz -C /usr/local/cargo/git \
|
||||
&& tar -xzf /cache/target.tar.gz -C /app/backend/target
|
||||
|
||||
FROM busybox:1.37.0-musl@sha256:5cec3fc171c87218698e85a52af7087de727372aae264a787b8112901a5b0092 AS collect
|
||||
FROM busybox:1.37.0-musl@sha256:5cec3fc171c87218698e85a52af7087de727372aae264a787b8112901a5b0092 AS snapshot
|
||||
ARG CACHE_STAMP
|
||||
# 一次导出三个 mount,避免逐目录创建镜像、载入 Docker 再复制文件。
|
||||
# 直接压缩 mount 内容,构建层和 runner 只保存归档,避免多份展开的编译产物占满磁盘
|
||||
RUN --mount=type=cache,target=/usr/local/cargo/registry \
|
||||
--mount=type=cache,target=/usr/local/cargo/git \
|
||||
--mount=type=cache,target=/app/backend/target \
|
||||
test -n "$CACHE_STAMP" \
|
||||
&& mkdir -p /cache/registry /cache/git /cache/target \
|
||||
&& cp -a /usr/local/cargo/registry/. /cache/registry/ \
|
||||
&& cp -a /usr/local/cargo/git/. /cache/git/ \
|
||||
&& cp -a /app/backend/target/. /cache/target/
|
||||
|
||||
FROM scratch AS snapshot
|
||||
COPY --from=collect /cache/ /
|
||||
&& mkdir -p /cache \
|
||||
&& tar -czf /cache/registry.tar.gz -C /usr/local/cargo/registry . \
|
||||
&& tar -czf /cache/git.tar.gz -C /usr/local/cargo/git . \
|
||||
&& tar -czf /cache/target.tar.gz -C /app/backend/target .
|
||||
|
||||
@@ -14,24 +14,42 @@ case "$architecture" in amd64|arm64) ;; *) exit 2 ;; esac
|
||||
dockerfile="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)/Dockerfile"
|
||||
|
||||
usable() {
|
||||
local root="$1" path crates=false libraries=false
|
||||
# 目录或 buildstamp 存在不代表编译发生过,必须同时有下载包与目标依赖。
|
||||
for path in "$root"/registry/cache/*/*.crate; do
|
||||
if [ -f "$path" ] && [ -s "$path" ]; then crates=true; break; fi
|
||||
done
|
||||
for path in "$root"/target/"$architecture"/release/deps/*.rlib; do
|
||||
if [ -f "$path" ] && [ -s "$path" ]; then libraries=true; break; fi
|
||||
done
|
||||
[ "$crates" = true ] && [ "$libraries" = true ]
|
||||
# 流式检查归档中的非空依赖文件,校验时不在 runner 展开编译产物
|
||||
python3 - "$1" "$architecture" <<'PY'
|
||||
import pathlib
|
||||
import re
|
||||
import sys
|
||||
import tarfile
|
||||
|
||||
root = pathlib.Path(sys.argv[1])
|
||||
architecture = sys.argv[2]
|
||||
|
||||
def contains(filename, pattern):
|
||||
with tarfile.open(root / filename, mode="r|gz") as archive:
|
||||
return any(
|
||||
member.isfile() and member.size > 0
|
||||
and re.fullmatch(pattern, member.name.removeprefix("./"))
|
||||
for member in archive
|
||||
)
|
||||
|
||||
try:
|
||||
valid = (
|
||||
(root / "git.tar.gz").stat().st_size > 0
|
||||
and contains("registry.tar.gz", r"cache/[^/]+/[^/]+\.crate")
|
||||
and contains("target.tar.gz", rf"{architecture}/release/deps/[^/]+\.rlib")
|
||||
)
|
||||
except (OSError, tarfile.TarError):
|
||||
valid = False
|
||||
sys.exit(0 if valid else 1)
|
||||
PY
|
||||
}
|
||||
|
||||
command=(docker buildx build --builder "$builder" --file "$dockerfile" --progress plain)
|
||||
# mount 的内容不参与层指纹,用时间戳重跑复制;--no-cache 会重置 cache mount。
|
||||
# mount 的内容不参与层指纹,用时间戳重跑归档操作;--no-cache 会重置 cache mount
|
||||
command+=(--build-arg "CACHE_STAMP=$(date +%s%N)" --target "$operation")
|
||||
valid=false
|
||||
if [ "$operation" = restore ]; then
|
||||
if usable "$directory"; then
|
||||
mkdir -p "$directory"/{registry,git,target}
|
||||
"${command[@]}" --output type=cacheonly "$directory"
|
||||
valid=true
|
||||
else
|
||||
@@ -43,10 +61,11 @@ else
|
||||
temporary="$(mktemp -d "${directory}.XXXXXX")"
|
||||
trap 'rm -rf "$temporary"' EXIT
|
||||
mkdir "$temporary"/{context,snapshot}
|
||||
# tar 流避开 local exporter 的逐文件同步阻塞,直接解包以免额外落盘整份归档。
|
||||
# tar 流避开 local exporter 的逐文件同步阻塞,只接收 /cache 下的压缩归档
|
||||
# 直接导出打包阶段,避免再创建一层归档副本
|
||||
# https://github.com/moby/buildkit/issues/2950
|
||||
"${command[@]}" --output type=tar,dest=- "$temporary/context" \
|
||||
| tar -xf - -C "$temporary/snapshot"
|
||||
| tar -xf - -C "$temporary/snapshot" --strip-components=1 cache/
|
||||
if usable "$temporary/snapshot"; then
|
||||
rm -rf "$directory"
|
||||
mv "$temporary/snapshot" "$directory"
|
||||
|
||||
@@ -36,10 +36,9 @@ jobs:
|
||||
with:
|
||||
path: ${{ runner.temp }}/container-cargo
|
||||
# 同一平台和依赖复用快照,源码变化由 Cargo 指纹判定是否重编译
|
||||
key: container-cargo-v3-${{ runner.os }}-${{ runner.arch }}-${{ hashFiles('backend/Cargo.lock', 'backend/rust-toolchain.toml', 'backend/**/Cargo.toml', 'backend/.cargo/config.toml', 'deploy/Dockerfile', '.github/actions/cargo-cache/Dockerfile') }}
|
||||
key: container-cargo-v4-${{ runner.os }}-${{ runner.arch }}-${{ hashFiles('backend/Cargo.lock', 'backend/rust-toolchain.toml', 'backend/**/Cargo.toml', 'backend/.cargo/config.toml', 'deploy/Dockerfile', '.github/actions/cargo-cache/Dockerfile') }}
|
||||
restore-keys: |
|
||||
container-cargo-v3-${{ runner.os }}-${{ runner.arch }}-
|
||||
container-cargo-v2-${{ runner.os }}-${{ runner.arch }}-
|
||||
container-cargo-v4-${{ runner.os }}-${{ runner.arch }}-
|
||||
|
||||
- name: Restore Cargo mounts to BuildKit
|
||||
id: cargo-restore
|
||||
@@ -50,6 +49,9 @@ jobs:
|
||||
path: ${{ runner.temp }}/container-cargo
|
||||
architecture: amd64
|
||||
|
||||
- name: Release restored Cargo snapshot
|
||||
run: rm -rf "$RUNNER_TEMP/container-cargo"
|
||||
|
||||
- name: Build runtime image
|
||||
uses: docker/build-push-action@c3c9e263c25d99ce0380d002d59b67737d91b0dc # v7.4.0
|
||||
with:
|
||||
|
||||
Reference in New Issue
Block a user