ci(package): retry apt fetches and docker builds behind the Ubuntu mirror (#18797)

The package job builds three Docker images whose apt-get update/install
hit archive.ubuntu.com with no retry, timeout, or mirror fallback. When
the mirror is mid-sync every build dies in one of three ways:

- per-package fetch stalls (~64 s each, `Ign:` lines) until the runner's
  10-minute docker build timeout fires:
  https://github.com/stablyai/orca/actions/runs/33935104546/job/101221447425
- `apt-get update` exit 100 with `Hash Sum mismatch` on
  noble-updates/restricted/Packages.gz:
  https://github.com/stablyai/orca/actions/runs/33935104546/job/101226099525
- `apt-get update` exit 100 with `File has unexpected size ... Mirror
  sync in progress?`:
  https://github.com/stablyai/orca/actions/runs/33935244026/job/101231083497

Each Dockerfile now retries `apt-get update` up to five times with
Acquire::Retries and a 30 s HTTP timeout, clearing /var/lib/apt/lists
between attempts so a half-synced index is never reused, and passes the
same acquire options to `apt-get install`. Each runner script retries the
whole `docker build` once when the first attempt fails or times out.

config/docker/cli-launch-contract/Dockerfile

@@ -6,8 +6,13 @@ ARG LIBASOUND_PACKAGE=libasound2t64
 ENV DEBIAN_FRONTEND=noninteractive
 
 # Install Electron's link-time libraries without adding a display server or FUSE.
-RUN apt-get update \
-  && apt-get install -y --no-install-recommends \
+# Why: archive.ubuntu.com mid-sync returns Hash Sum mismatch / wrong-size indexes and stalls per-package fetches; retry with bounded timeouts and drop half-synced lists between attempts.
+RUN for attempt in 1 2 3 4 5; do \
+      apt-get -o Acquire::Retries=5 -o Acquire::http::Timeout=30 update && break; \
+      if [ "$attempt" = 5 ]; then exit 100; fi; \
+      rm -rf /var/lib/apt/lists/*; sleep 20; \
+    done \
+  && apt-get -o Acquire::Retries=5 -o Acquire::http::Timeout=30 install -y --no-install-recommends \
     bash \
     ca-certificates \
     coreutils \

config/docker/headless-pairing/Dockerfile

@@ -5,8 +5,13 @@ ARG LIBASOUND_PACKAGE=libasound2t64
 
 ENV DEBIAN_FRONTEND=noninteractive
 
-RUN apt-get update \
-  && apt-get install -y --no-install-recommends \
+# Why: archive.ubuntu.com mid-sync returns Hash Sum mismatch / wrong-size indexes and stalls per-package fetches; retry with bounded timeouts and drop half-synced lists between attempts.
+RUN for attempt in 1 2 3 4 5; do \
+      apt-get -o Acquire::Retries=5 -o Acquire::http::Timeout=30 update && break; \
+      if [ "$attempt" = 5 ]; then exit 100; fi; \
+      rm -rf /var/lib/apt/lists/*; sleep 20; \
+    done \
+  && apt-get -o Acquire::Retries=5 -o Acquire::http::Timeout=30 install -y --no-install-recommends \
     bash \
     ca-certificates \
     dbus-x11 \

config/docker/headless-serve-shutdown/Dockerfile

@@ -2,8 +2,13 @@ FROM ubuntu@sha256:678c6550cc43645e08669028bc177f50be4e7c5b8cca677067b1914d4afc7
 
 ENV DEBIAN_FRONTEND=noninteractive
 
-RUN apt-get update \
-  && apt-get install -y --no-install-recommends \
+# Why: archive.ubuntu.com mid-sync returns Hash Sum mismatch / wrong-size indexes and stalls per-package fetches; retry with bounded timeouts and drop half-synced lists between attempts.
+RUN for attempt in 1 2 3 4 5; do \
+      apt-get -o Acquire::Retries=5 -o Acquire::http::Timeout=30 update && break; \
+      if [ "$attempt" = 5 ]; then exit 100; fi; \
+      rm -rf /var/lib/apt/lists/*; sleep 20; \
+    done \
+  && apt-get -o Acquire::Retries=5 -o Acquire::http::Timeout=30 install -y --no-install-recommends \
     bash \
     ca-certificates \
     dbus-x11 \

config/scripts/run-headless-linux-pairing-docker.mjs

@@ -70,7 +70,7 @@ function valueAfter(flag) {
 
 function buildImage(image) {
   console.log(`Building ${image.name} fixture...`)
-  docker([
+  const buildArgs = [
     'build',
     '--build-arg',
     `BASE_IMAGE=${image.base}`,
@@ -81,7 +81,16 @@ function buildImage(image) {
     '-t',
     image.tag,
     '.'
-  ])
+  ]
+  // Why: apt fetches from archive.ubuntu.com stall or fail mid-sync; a second build usually lands on a healthy index.
+  try {
+    docker(buildArgs)
+  } catch (error) {
+    console.error(
+      `${error instanceof Error ? error.message : String(error)}\nRetrying docker build once...`
+    )
+    docker(buildArgs)
+  }
 }
 
 function extractAppImage(image) {

config/scripts/run-headless-serve-shutdown-docker.mjs

@@ -43,7 +43,7 @@ const artifactVolume = `orca-headless-serve-shutdown-${suffix}`
 const sha256 = createHash('sha256').update(readFileSync(appImage)).digest('hex')
 
 try {
-  docker([
+  const buildArgs = [
     'build',
     '--platform',
     platform,
@@ -52,7 +52,15 @@ try {
     '-t',
     image,
     shutdownDockerDirectory
-  ])
+  ]
+  // Why: apt fetches from archive.ubuntu.com stall or fail mid-sync; a second build usually lands on a healthy index.
+  const firstBuild = docker(buildArgs, { allowFailure: true })
+  if (firstBuild.status !== 0) {
+    process.stderr.write(
+      `${firstBuild.stdout}${firstBuild.stderr}\ndocker build failed with status ${firstBuild.status}; retrying once...\n`
+    )
+    docker(buildArgs)
+  }
   docker(['volume', 'create', artifactVolume])
   runDesktopStartupOracle({ image, appImage, platform })
   docker([

config/scripts/run-linux-cli-launch-contract-docker.mjs

@@ -173,20 +173,26 @@ function runCase(caseName) {
 
 function buildImage() {
   console.log(`Building ${tag}…`)
-  docker(
-    [
-      'build',
-      ...dockerPlatformArgs,
-      '--build-arg',
-      `BASE_IMAGE=${base}`,
-      '-f',
-      'config/docker/cli-launch-contract/Dockerfile',
-      '-t',
-      tag,
-      'config/docker/cli-launch-contract'
-    ],
-    { timeoutMs: BUILD_TIMEOUT_MS }
-  )
+  const buildArgs = [
+    'build',
+    ...dockerPlatformArgs,
+    '--build-arg',
+    `BASE_IMAGE=${base}`,
+    '-f',
+    'config/docker/cli-launch-contract/Dockerfile',
+    '-t',
+    tag,
+    'config/docker/cli-launch-contract'
+  ]
+  // Why: apt fetches from archive.ubuntu.com stall or fail mid-sync; a second build usually lands on a healthy index.
+  try {
+    docker(buildArgs, { timeoutMs: BUILD_TIMEOUT_MS })
+  } catch (error) {
+    console.error(
+      `${error instanceof Error ? error.message : String(error)}\nRetrying docker build once…`
+    )
+    docker(buildArgs, { timeoutMs: BUILD_TIMEOUT_MS })
+  }
 }
 
 // Extract unprivileged so chrome-sandbox is not root-owned setuid.