From 41b520259ec8f5c3f176b95ccdc172b6a8be310c Mon Sep 17 00:00:00 2001 From: Neil <4138956+nwparker@users.noreply.github.com> Date: Fri, 4 Sep 2026 22:24:18 -0700 Subject: [PATCH] ci(package): retry apt fetches and docker builds behind the Ubuntu mirror (#18797) The package job builds three Docker images whose apt-get update/install hit archive.ubuntu.com with no retry, timeout, or mirror fallback. When the mirror is mid-sync every build dies in one of three ways: - per-package fetch stalls (~64 s each, `Ign:` lines) until the runner's 10-minute docker build timeout fires: https://github.com/stablyai/orca/actions/runs/33935104546/job/101221447425 - `apt-get update` exit 100 with `Hash Sum mismatch` on noble-updates/restricted/Packages.gz: https://github.com/stablyai/orca/actions/runs/33935104546/job/101226099525 - `apt-get update` exit 100 with `File has unexpected size ... Mirror sync in progress?`: https://github.com/stablyai/orca/actions/runs/33935244026/job/101231083497 Each Dockerfile now retries `apt-get update` up to five times with Acquire::Retries and a 30 s HTTP timeout, clearing /var/lib/apt/lists between attempts so a half-synced index is never reused, and passes the same acquire options to `apt-get install`. Each runner script retries the whole `docker build` once when the first attempt fails or times out. --- config/docker/cli-launch-contract/Dockerfile | 9 +++-- config/docker/headless-pairing/Dockerfile | 9 +++-- .../docker/headless-serve-shutdown/Dockerfile | 9 +++-- .../run-headless-linux-pairing-docker.mjs | 13 +++++-- .../run-headless-serve-shutdown-docker.mjs | 12 +++++-- .../run-linux-cli-launch-contract-docker.mjs | 34 +++++++++++-------- 6 files changed, 62 insertions(+), 24 deletions(-) diff --git a/config/docker/cli-launch-contract/Dockerfile b/config/docker/cli-launch-contract/Dockerfile index f6a618a8ece..c90cbcd979c 100644 --- a/config/docker/cli-launch-contract/Dockerfile +++ b/config/docker/cli-launch-contract/Dockerfile @@ -6,8 +6,13 @@ ARG LIBASOUND_PACKAGE=libasound2t64 ENV DEBIAN_FRONTEND=noninteractive # Install Electron's link-time libraries without adding a display server or FUSE. -RUN apt-get update \ - && apt-get install -y --no-install-recommends \ +# Why: archive.ubuntu.com mid-sync returns Hash Sum mismatch / wrong-size indexes and stalls per-package fetches; retry with bounded timeouts and drop half-synced lists between attempts. +RUN for attempt in 1 2 3 4 5; do \ + apt-get -o Acquire::Retries=5 -o Acquire::http::Timeout=30 update && break; \ + if [ "$attempt" = 5 ]; then exit 100; fi; \ + rm -rf /var/lib/apt/lists/*; sleep 20; \ + done \ + && apt-get -o Acquire::Retries=5 -o Acquire::http::Timeout=30 install -y --no-install-recommends \ bash \ ca-certificates \ coreutils \ diff --git a/config/docker/headless-pairing/Dockerfile b/config/docker/headless-pairing/Dockerfile index 03664f68b0d..e4b4cafeefc 100644 --- a/config/docker/headless-pairing/Dockerfile +++ b/config/docker/headless-pairing/Dockerfile @@ -5,8 +5,13 @@ ARG LIBASOUND_PACKAGE=libasound2t64 ENV DEBIAN_FRONTEND=noninteractive -RUN apt-get update \ - && apt-get install -y --no-install-recommends \ +# Why: archive.ubuntu.com mid-sync returns Hash Sum mismatch / wrong-size indexes and stalls per-package fetches; retry with bounded timeouts and drop half-synced lists between attempts. +RUN for attempt in 1 2 3 4 5; do \ + apt-get -o Acquire::Retries=5 -o Acquire::http::Timeout=30 update && break; \ + if [ "$attempt" = 5 ]; then exit 100; fi; \ + rm -rf /var/lib/apt/lists/*; sleep 20; \ + done \ + && apt-get -o Acquire::Retries=5 -o Acquire::http::Timeout=30 install -y --no-install-recommends \ bash \ ca-certificates \ dbus-x11 \ diff --git a/config/docker/headless-serve-shutdown/Dockerfile b/config/docker/headless-serve-shutdown/Dockerfile index 13b1ed2b69f..8ee7b942499 100644 --- a/config/docker/headless-serve-shutdown/Dockerfile +++ b/config/docker/headless-serve-shutdown/Dockerfile @@ -2,8 +2,13 @@ FROM ubuntu@sha256:678c6550cc43645e08669028bc177f50be4e7c5b8cca677067b1914d4afc7 ENV DEBIAN_FRONTEND=noninteractive -RUN apt-get update \ - && apt-get install -y --no-install-recommends \ +# Why: archive.ubuntu.com mid-sync returns Hash Sum mismatch / wrong-size indexes and stalls per-package fetches; retry with bounded timeouts and drop half-synced lists between attempts. +RUN for attempt in 1 2 3 4 5; do \ + apt-get -o Acquire::Retries=5 -o Acquire::http::Timeout=30 update && break; \ + if [ "$attempt" = 5 ]; then exit 100; fi; \ + rm -rf /var/lib/apt/lists/*; sleep 20; \ + done \ + && apt-get -o Acquire::Retries=5 -o Acquire::http::Timeout=30 install -y --no-install-recommends \ bash \ ca-certificates \ dbus-x11 \ diff --git a/config/scripts/run-headless-linux-pairing-docker.mjs b/config/scripts/run-headless-linux-pairing-docker.mjs index 635c66348cc..799b8d73ab1 100644 --- a/config/scripts/run-headless-linux-pairing-docker.mjs +++ b/config/scripts/run-headless-linux-pairing-docker.mjs @@ -70,7 +70,7 @@ function valueAfter(flag) { function buildImage(image) { console.log(`Building ${image.name} fixture...`) - docker([ + const buildArgs = [ 'build', '--build-arg', `BASE_IMAGE=${image.base}`, @@ -81,7 +81,16 @@ function buildImage(image) { '-t', image.tag, '.' - ]) + ] + // Why: apt fetches from archive.ubuntu.com stall or fail mid-sync; a second build usually lands on a healthy index. + try { + docker(buildArgs) + } catch (error) { + console.error( + `${error instanceof Error ? error.message : String(error)}\nRetrying docker build once...` + ) + docker(buildArgs) + } } function extractAppImage(image) { diff --git a/config/scripts/run-headless-serve-shutdown-docker.mjs b/config/scripts/run-headless-serve-shutdown-docker.mjs index 184713c41a0..d8dcdd345ad 100755 --- a/config/scripts/run-headless-serve-shutdown-docker.mjs +++ b/config/scripts/run-headless-serve-shutdown-docker.mjs @@ -43,7 +43,7 @@ const artifactVolume = `orca-headless-serve-shutdown-${suffix}` const sha256 = createHash('sha256').update(readFileSync(appImage)).digest('hex') try { - docker([ + const buildArgs = [ 'build', '--platform', platform, @@ -52,7 +52,15 @@ try { '-t', image, shutdownDockerDirectory - ]) + ] + // Why: apt fetches from archive.ubuntu.com stall or fail mid-sync; a second build usually lands on a healthy index. + const firstBuild = docker(buildArgs, { allowFailure: true }) + if (firstBuild.status !== 0) { + process.stderr.write( + `${firstBuild.stdout}${firstBuild.stderr}\ndocker build failed with status ${firstBuild.status}; retrying once...\n` + ) + docker(buildArgs) + } docker(['volume', 'create', artifactVolume]) runDesktopStartupOracle({ image, appImage, platform }) docker([ diff --git a/config/scripts/run-linux-cli-launch-contract-docker.mjs b/config/scripts/run-linux-cli-launch-contract-docker.mjs index 901e0877e85..bd414947026 100755 --- a/config/scripts/run-linux-cli-launch-contract-docker.mjs +++ b/config/scripts/run-linux-cli-launch-contract-docker.mjs @@ -173,20 +173,26 @@ function runCase(caseName) { function buildImage() { console.log(`Building ${tag}…`) - docker( - [ - 'build', - ...dockerPlatformArgs, - '--build-arg', - `BASE_IMAGE=${base}`, - '-f', - 'config/docker/cli-launch-contract/Dockerfile', - '-t', - tag, - 'config/docker/cli-launch-contract' - ], - { timeoutMs: BUILD_TIMEOUT_MS } - ) + const buildArgs = [ + 'build', + ...dockerPlatformArgs, + '--build-arg', + `BASE_IMAGE=${base}`, + '-f', + 'config/docker/cli-launch-contract/Dockerfile', + '-t', + tag, + 'config/docker/cli-launch-contract' + ] + // Why: apt fetches from archive.ubuntu.com stall or fail mid-sync; a second build usually lands on a healthy index. + try { + docker(buildArgs, { timeoutMs: BUILD_TIMEOUT_MS }) + } catch (error) { + console.error( + `${error instanceof Error ? error.message : String(error)}\nRetrying docker build once…` + ) + docker(buildArgs, { timeoutMs: BUILD_TIMEOUT_MS }) + } } // Extract unprivileged so chrome-sandbox is not root-owned setuid.