Merge pull request #373 from warmbly/fix/issue-371-public-images

fix: publish images a stranger can actually pull, and fail the release when they cannot

.github/workflows/build-push.yml

@@ -255,3 +255,29 @@ jobs:
             -t ${{ env.IMAGE_PREFIX }}/${{ matrix.service }}:${{ github.sha }} \
             -t ${{ env.IMAGE_PREFIX }}/${{ matrix.service }}:dev \
             $(printf '${{ env.IMAGE_PREFIX }}/${{ matrix.service }}@sha256:%s ' *)
+
+  # A package on GHCR is created private and does not inherit the repository's
+  # visibility, and no API can change that: an owner has to flip it by hand.
+  # So a new service image is born unpullable, and the first sign of it would
+  # otherwise be a stranger's failed install (#371).
+  #
+  # This is the early warning: main is where a new image first appears, and
+  # finding out here costs nothing, while finding out at tag time blocks a
+  # release. It only warns, because the fix is not in this repo and nobody can
+  # land a PR that makes main green again.
+  verify-public:
+    name: Warn if images are not publicly pullable
+    needs: [build-go, merge-native]
+    if: always() && github.event_name == 'push'
+    runs-on: ubuntu-latest
+    permissions:
+      contents: read
+    steps:
+      - uses: actions/checkout@v4
+      # Only what this workflow publishes. web, admin and cli are built at
+      # tag time and have no :dev tag, so checking them here would warn about
+      # something no push to main can fix.
+      - name: Pull-test every image anonymously
+        run: |
+          ./scripts/check-images-public.sh --prefix "$IMAGE_PREFIX" --tag dev --warn \
+            backend consumer worker forms updater tracking realtime

.github/workflows/release.yml

@@ -264,15 +264,39 @@ jobs:
           path: dist/
           retention-days: 1
 
+  # GHCR creates a package private and does not inherit the repository's
+  # visibility, and there is no API to change it, so a green build proves
+  # nothing about whether a stranger can pull what it published. Releases
+  # v0.1.0 to v0.4.0 all shipped an installer that could not pull one byte
+  # (#371). This is the only step here that talks to the registry the way an
+  # operator does: no login, no credentials, no `packages` permission.
+  #
+  # It gates the release rather than trailing it, because install.sh resolves
+  # its default version from the newest GitHub Release. Failing before the
+  # release exists is what keeps an unpullable build from becoming the version
+  # every `curl | sh` picks up. Fix the visibility, then re-run this job; the
+  # images and their digests are untouched by it.
+  verify-public:
+    name: Verify images are publicly pullable
+    needs: [validate-tag, build-go, build-frontend, merge-native]
+    runs-on: ubuntu-latest
+    # contents: read is what checkout needs. The absence of `packages` is the
+    # point: this job must not be able to see a private image.
+    permissions:
+      contents: read
+    steps:
+      - uses: actions/checkout@v4
+      - name: Pull-test every image anonymously
+        env:
+          TAG: ${{ github.ref_name }}
+        run: ./scripts/check-images-public.sh --prefix "$IMAGE_PREFIX" --tag "$TAG"
+
   create-release:
     name: Create GitHub Release
-    needs: [validate-tag, build-go, build-frontend, merge-native, build-cli]
+    needs: [validate-tag, build-go, build-frontend, merge-native, build-cli, verify-public]
     runs-on: ubuntu-latest
     permissions:
       contents: write
-      # The manifest step logs in to GHCR and inspects each published image;
-      # without read access that fails on a private package.
-      packages: read
     steps:
       - uses: actions/checkout@v4
         with:
@@ -287,34 +311,36 @@ jobs:
       # The installer verifies what it pulled against this file, so it is what
       # makes "curl | sh" checkable after the fact rather than only before it.
       # One line per service, because the thing that reads it is a POSIX shell.
-      - name: Log in to GHCR
-        uses: docker/login-action@v3
-        with:
-          registry: ${{ env.REGISTRY }}
-          username: ${{ github.actor }}
-          password: ${{ secrets.GITHUB_TOKEN }}
-
+      # Digests are read anonymously, from the same view the operator who
+      # checks them has. Reading them with a token that can see private
+      # packages is what let four releases publish a manifest for images
+      # nobody else could fetch.
       - name: Publish the image manifest
         env:
           TAG: ${{ github.ref_name }}
         run: |
           set -euo pipefail
+          ./scripts/check-images-public.sh --prefix "$IMAGE_PREFIX" --tag "$TAG" \
+            backend consumer worker forms updater web admin tracking realtime \
+            | grep -E '^[a-z]+[[:space:]]+sha256:' > /tmp/digests.tsv
           {
             printf '{\n'
             printf '  "tag": "%s",\n' "$TAG"
             printf '  "registry": "%s",\n' "$IMAGE_PREFIX"
             printf '  "images": {\n'
             first=1
-            for s in backend consumer worker forms updater web admin tracking realtime; do
-              digest=$(docker buildx imagetools inspect "$IMAGE_PREFIX/$s:$TAG" \
-                --format '{{json .Manifest.Digest}}' | tr -d '"')
+            while IFS=$'\t' read -r svc digest; do
               [ "$first" = 1 ] || printf ',\n'
               first=0
-              printf '    "%s": "%s"' "$s" "$digest"
-            done
+              printf '    "%s": "%s"' "$svc" "$digest"
+            done < /tmp/digests.tsv
             printf '\n  }\n}\n'
           } > /tmp/images.json
           cat /tmp/images.json
+          # Nine services in, nine digests out, or the manifest is incomplete
+          # and the installer would silently skip whatever is missing.
+          [ "$(wc -l < /tmp/digests.tsv)" -eq 9 ] || {
+            echo "::error::expected 9 image digests, got $(wc -l < /tmp/digests.tsv)"; exit 1; }
 
       - name: Generate release notes
         env:

Makefile

@@ -42,7 +42,7 @@ PROTO_GEN_FILES := $(PROTO_DIR)/tasks.pb.go
         restart restart-go restart-all infra infra-down app app-down app-logs \
         backend forms forms-web consumer worker run dev tracking realtime web \
         admin site docs grant-admin revoke-admin gen-key installer-sha installer-check installer-demo \
-        db-reset db-wipe migrate warmbly warmbly-dist cli-sha cli-check
+        db-reset db-wipe migrate warmbly warmbly-dist cli-sha cli-check images-check
 
 setup-tools:
 	@echo "Installing required Go tools into $(GO_BIN)"
@@ -704,6 +704,15 @@ installer-sha:
 installer-check:
 	@./scripts/check-installer.sh
 
+# Every published image has to be pullable by a stranger, and nothing else we
+# run proves it: a package on GHCR is created private, does not inherit the
+# repository's visibility, and no API can change that, so every check that
+# talks to the registry with a token passes while the world sees nothing
+# (#371). This one carries no credentials on purpose. No TAG means the newest
+# release, which is what a fresh install resolves to.
+images-check:
+	@./scripts/check-images-public.sh $(if $(TAG),--tag $(TAG),)
+
 # Walk the installer's wizard without installing anything: the real questions,
 # the real review, and a played pull and start. Writes no file, pulls no image,
 # needs no Docker. This is the "what does it look like" target.

docs/content/docs/development/deployment-guide.mdx

@@ -752,6 +752,8 @@ CI publishes multi-arch images to `ghcr.io/<owner>/warmbly/`, which works on any
 | Push to `main` | backend, consumer, worker, forms, updater, tracking, realtime | `:<sha>`, `:dev` |
 | Tag `vX.Y.Z` | all of the above plus web and admin | `:vX.Y.Z`, `:vX.Y`, `:vX`, `:prod` |
 
+One thing to know if you are running a fork: GHCR creates each package private and does not inherit your repository's visibility, so images published by a green workflow stay unreadable to everyone else until you make each package public by hand. On a personal fork that is the package's own settings page, reached from the Packages tab on your profile. On an organization fork it is the same page reached from the org's Packages, and public packages have to be permitted in the org's package settings first, or the control is greyed out. There is no API for either. `make images-check` tells you where a given tag stands, with no credentials, and it is the same check the release runs before it publishes.
+
 Every app service in `docker-compose.yml` carries both an `image:` and a `build:` key, so the same file serves both paths. `docker compose pull && docker compose up -d` runs the published images and compiles nothing; `docker compose up --build` still builds this checkout. Pin a release with `WARMBLY_TAG=v1.4.2` in `.env`, or point `WARMBLY_IMAGE_PREFIX` at your own registry.
 
 To move a worker fleet onto a new release, update `WORKER_IMAGE` and use "Pull latest and restart" on each worker. Control-plane migrations are forward-only, so prefer rolling forward over rolling back.

docs/content/docs/development/install.mdx

@@ -22,7 +22,7 @@ curl -fsSL https://warmbly.com/install.sh | sh -s -- --wizard
 | Where | `/opt/warmbly`, or wherever you point `--dir` |
 | What | `docker-compose.yml`, a `.env` at 0600, `keys-backup.txt`, and the data root |
 | Version | The newest release, resolved once and pinned in `.env`. Never `latest` |
-| Images | `ghcr.io/warmbly/warmbly/*`, multi-arch for amd64 and arm64 |
+| Images | `ghcr.io/warmbly/warmbly/*`, multi-arch for amd64 and arm64. Public, so no registry login and no GitHub account |
 | Services | Backend, consumer, worker, dashboard, admin, Postgres, Redis, NATS, and (unless you say core only) tracking, realtime and forms |
 
 Nothing is built on your machine. The clone-and-build path still exists and is still supported; see [self-hosting](/development/deployment-guide/) for when you want it.
@@ -305,6 +305,7 @@ Stops and removes the containers and leaves every byte of data where it is. Addi
 | `Docker is installed but not answering` | The daemon is not running, or your user is not in the `docker` group yet. `sudo systemctl start docker`, then log out and back in |
 | `Something already listens on 3000` | Another self-hosted tool. Change the published port in `docker-compose.yml` before starting |
 | `Could not pull the release images` | The tag does not exist, or this host cannot reach `ghcr.io`. Check [releases](https://github.com/warmbly/warmbly/releases) |
+| `The registry refused to serve the release images` | The images are not readable without an account, which is never how a release is meant to ship. Please [report it](https://github.com/warmbly/warmbly/issues). If you passed `--registry`, check it for a typo first: a namespace that does not exist is refused the same way. To get past it, point `--registry` at a mirror you can read, or `docker login ghcr.io` as someone who already has access to the packages. Logging in does not by itself grant it |
 | The API never answers | The first boot applies every migration. `docker compose -p warmbly logs -f backend` |
 | No claim link printed | The database already has accounts, so there is nothing to claim. See [first run](/development/first-run/) |
 

scripts/check-images-public.sh

@@ -0,0 +1,166 @@
+#!/usr/bin/env bash
+#
+# Checks that every published image can be pulled by a stranger.
+#
+# GHCR creates a package private and does NOT inherit the repository's
+# visibility, so an image published from a public repo by a green workflow is
+# still unreadable to everyone outside the org until an owner flips it by hand
+# in the UI. There is no API for that flip, which means nothing in CI can fix
+# it and everything in CI must at least notice it: releases v0.1.0 through
+# v0.4.0 all shipped a `curl | sh` installer that could not pull a single byte,
+# and every check we had passed, because each one ran authenticated (#371).
+#
+# So this speaks to the registry the way an anonymous `docker pull` does: an
+# unauthenticated token, then the manifest. No docker, no login, no
+# credentials to accidentally inherit from the runner.
+#
+# It prints "<service><TAB><digest>" per image on stdout, so the release can
+# build images.json out of the same public view it just verified rather than
+# out of a privileged one.
+#
+#   scripts/check-images-public.sh [--prefix P] [--tag T] [--warn] [service...]
+set -euo pipefail
+
+PREFIX="ghcr.io/warmbly/warmbly"
+TAG=""
+WARN_ONLY=0
+SERVICES=()
+
+while [[ $# -gt 0 ]]; do
+  case $1 in
+    --prefix) PREFIX=$2; shift 2 ;;
+    --tag) TAG=$2; shift 2 ;;
+    --warn) WARN_ONLY=1; shift ;;
+    -h|--help)
+      sed -n '2,20p' "$0" | sed 's|^# \{0,1\}||'
+      exit 0 ;;
+    -*) echo "unknown flag: $1" >&2; exit 2 ;;
+    *) SERVICES+=("$1"); shift ;;
+  esac
+done
+
+# Every image the release advertises. The nine compose services, plus the cli
+# image, which the release notes table tells people to pull.
+if [[ ${#SERVICES[@]} -eq 0 ]]; then
+  SERVICES=(backend consumer worker forms updater web admin tracking realtime cli)
+fi
+
+fail() { printf '\033[31m✗\033[0m %s\n' "$*" >&2; }
+pass() { printf '\033[32m✓\033[0m %s\n' "$*" >&2; }
+info() { printf '%s\n' "$*" >&2; }
+
+HOST=${PREFIX%%/*}
+NAMESPACE=${PREFIX#*/}
+if [[ $HOST != "ghcr.io" ]]; then
+  info "· $PREFIX is not on ghcr.io; the anonymous check only speaks GHCR's token flow. Skipped."
+  exit 0
+fi
+
+# No tag given means "whatever a fresh install would resolve to", which is the
+# newest GitHub release, because that is what install.sh pins.
+if [[ -z $TAG ]]; then
+  TAG=$(curl -fsSL -H 'Accept: application/vnd.github+json' \
+    https://api.github.com/repos/warmbly/warmbly/releases/latest 2>/dev/null |
+    sed -n 's/.*"tag_name": *"\([^"]*\)".*/\1/p' | head -1)
+  [[ -n $TAG ]] || { fail "could not resolve the newest release; pass --tag"; exit 1; }
+fi
+
+info "Checking $PREFIX/*:$TAG is pullable with no credentials"
+info ""
+
+# A manifest list is what a multi-arch pull resolves, and its digest is what
+# `docker image inspect` reports as the RepoDigest, so this Accept set is what
+# makes the printed digest comparable to what lands on an operator's machine.
+ACCEPT='application/vnd.oci.image.index.v1+json,application/vnd.docker.distribution.manifest.list.v2+json,application/vnd.oci.image.manifest.v1+json,application/vnd.docker.distribution.manifest.v2+json'
+
+# Sets CODE and DIGEST rather than printing them: the response headers carry
+# both, and a command substitution would run this in a subshell and lose them.
+CODE=""
+DIGEST=""
+probe() {
+  local svc=$1 token headers
+  CODE="000"; DIGEST=""
+  token=$(curl -fsSL --max-time 20 \
+    "https://${HOST}/token?scope=repository:${NAMESPACE}/${svc}:pull&service=${HOST}" 2>/dev/null |
+    sed -n 's/.*"token":"\([^"]*\)".*/\1/p') || true
+  # No token at all is a network problem, not a visibility one; the caller
+  # retries before believing it.
+  [[ -n $token ]] || return 0
+  headers=$(curl -sS -I --max-time 20 -H "Authorization: Bearer $token" -H "Accept: $ACCEPT" \
+    "https://${HOST}/v2/${NAMESPACE}/${svc}/manifests/${TAG}" 2>/dev/null) || true
+  CODE=$(printf '%s' "$headers" | sed -n 's|^HTTP/[0-9.]* \([0-9][0-9][0-9]\).*|\1|p' | tail -1)
+  CODE=${CODE:-000}
+  DIGEST=$(printf '%s' "$headers" |
+    sed -n 's/[Dd]ocker-[Cc]ontent-[Dd]igest: *//p' | tr -d '\r' | head -1)
+}
+
+private=()
+missing=()
+for svc in "${SERVICES[@]}"; do
+  probe "$svc"
+  # 5xx and a dead token are transient often enough that one retry is worth
+  # more than a flaky release gate.
+  if [[ $CODE == "000" || $CODE == 5* ]]; then
+    sleep 2
+    probe "$svc"
+  fi
+  case $CODE in
+    200)
+      pass "$svc:$TAG is public${DIGEST:+ ($DIGEST)}"
+      if [[ -n $DIGEST ]]; then printf '%s\t%s\n' "$svc" "$DIGEST"; fi
+      ;;
+    403|401)
+      # GHCR answers 403 for private and for does-not-exist alike, so this
+      # cannot tell them apart and does not pretend to.
+      fail "$svc:$TAG is NOT publicly pullable (HTTP $CODE)"
+      private+=("$svc")
+      ;;
+    404)
+      fail "$svc:$TAG does not exist (HTTP 404)"
+      missing+=("$svc")
+      ;;
+    *)
+      fail "$svc:$TAG could not be checked (HTTP $CODE)"
+      private+=("$svc")
+      ;;
+  esac
+done
+
+if [[ ${#private[@]} -eq 0 && ${#missing[@]} -eq 0 ]]; then
+  info ""
+  printf '\033[32mAll %d images are pullable with no credentials.\033[0m\n' "${#SERVICES[@]}" >&2
+  exit 0
+fi
+
+broken=()
+if [[ ${#private[@]} -gt 0 ]]; then broken+=("${private[@]}"); fi
+if [[ ${#missing[@]} -gt 0 ]]; then broken+=("${missing[@]}"); fi
+summary="not publicly pullable: ${broken[*]}"
+
+info ""
+info "  $summary"
+info ""
+info "  A package on GHCR is created private and does not inherit the"
+info "  repository's visibility. There is no API for the fix; an org owner has"
+info "  to do it in the UI, once per package:"
+info ""
+info "    1. https://github.com/organizations/warmbly/settings/packages"
+info "       Package Creation must allow Public, or the control below is greyed out."
+info "    2. https://github.com/orgs/warmbly/packages"
+info "       each package > Package settings > Danger Zone > Change visibility"
+info ""
+info "  It applies to every tag at once and cannot be undone."
+info ""
+
+if [[ -n ${GITHUB_ACTIONS:-} ]]; then
+  if [[ $WARN_ONLY == 1 ]]; then
+    echo "::warning title=Images are not public::$summary. The next release will fail its publicity gate. See scripts/check-images-public.sh."
+  else
+    echo "::error title=Images are not public::$summary. A self-host install of this release cannot pull them."
+  fi
+fi
+
+if [[ $WARN_ONLY == 1 ]]; then
+  exit 0
+fi
+exit 1

scripts/check-installer.sh

@@ -177,6 +177,44 @@ else
   echo "· python3 not installed; the width check was skipped"
 fi
 
+# A registry that will not serve an image answers "unauthorized", and
+# reporting that as a missing tag is what sent the first person who hit it
+# looking in entirely the wrong place (#371). Both branches of the diagnosis
+# are checked, against the literal text docker produces.
+diag() {
+  local log=$1
+  {
+    sed -n '/^pull_failed()/,/^}/p' "$SCRIPT"
+    cat <<'STUB'
+show_log() { :; }
+fail_with() { printf '%s\n' "$@"; exit 1; }
+REGISTRY=ghcr.io/warmbly/warmbly
+RESOLVED_TAG=v0.0.0-test
+REPO=warmbly/warmbly
+pull_failed
+STUB
+  } | LOGFILE="$log" sh || true
+}
+
+printf 'Error response from daemon: Head "https://ghcr.io/v2/warmbly/warmbly/forms/manifests/v0.4.0": unauthorized\n' >"$work/log.unauth"
+printf 'Error response from daemon: manifest unknown\n' >"$work/log.missing"
+printf 'Error response from daemon: denied\n' >"$work/log.denied"
+
+diag "$work/log.unauth" | grep -q 'refused to serve' ||
+  fail "an unauthorized pull is not diagnosed as a registry refusal"
+diag "$work/log.unauth" | grep -q 'probably fine' ||
+  fail "an unauthorized pull still blames the tag"
+diag "$work/log.unauth" | grep -q 'check it for a typo' ||
+  fail "an unauthorized pull does not mention a mistyped --registry, which"
+diag "$work/log.missing" | grep -q 'may not exist' ||
+  fail "an ordinary pull failure lost its generic message"
+if diag "$work/log.missing" | grep -q 'refused to serve'; then
+  fail "an ordinary pull failure is misreported as a registry refusal"
+fi
+diag "$work/log.denied" | grep -q 'check it for a typo' ||
+  fail "a bare denied does not offer the mistyped-registry reading"
+pass "diagnoses an unauthorized pull separately from a missing tag"
+
 # The checksum is the whole answer to "why would I pipe this into a shell", so
 # a stale one is a failure, not a warning.
 if [[ ! -f $SUMFILE ]]; then

site/public/install.sh

@@ -2273,17 +2273,45 @@ SECRET_KEY_BASE=$SECRET_KEY_BASE
 KEYSEOF
 }
 
+# A registry that refuses to serve an image answers "unauthorized", which the
+# generic message below would report as a missing tag and send the operator
+# looking in entirely the wrong place. Every Warmbly image is meant to be
+# pullable with no account at all, so this is a bug in the release rather than
+# anything wrong with their machine (#371).
+pull_failed() {
+    show_log
+    if [ -s "$LOGFILE" ] &&
+       grep -Eqi 'unauthorized|denied|authentication required' "$LOGFILE"; then
+        _host=${REGISTRY%%/*}
+        fail_with "The registry refused to serve the release images." \
+                  "It answered unauthorized rather than 'no such tag', so" \
+                  "${RESOLVED_TAG} is probably fine and the images are simply" \
+                  "not readable without an account." \
+                  "" \
+                  "Every Warmbly image is meant to be public. If you did not" \
+                  "pass --registry, that makes this our bug, not yours:" \
+                  "please tell us at https://github.com/$REPO/issues" \
+                  "" \
+                  "If you did pass it, check it for a typo. A namespace that" \
+                  "does not exist is refused in exactly the same way as a" \
+                  "private one." \
+                  "" \
+                  "To get past it now: docker login $_host as someone who can" \
+                  "read these images, or point --registry at a mirror you can."
+    fi
+    fail_with "Could not pull the release images." \
+              "The tag ${RESOLVED_TAG} may not exist, or this host cannot" \
+              "reach ${REGISTRY}." \
+              "Releases: https://github.com/$REPO/releases"
+}
+
 # pull_images shows one row per image, flipping to a tick the moment that image
 # lands. The pull itself runs in the background, so this is compose's own
 # parallel pull with a readable face on it rather than a serial one.
 pull_images() {
     _refs=$(cd "$DIR" && composec config --images 2>/dev/null | sort -u)
     if [ -z "$_refs" ]; then
-        spin "Pulling images" sh -c "cd '$DIR' && $COMPOSE pull" || {
-            show_log
-            fail_with "Could not pull the release images." \
-                      "Check the tag exists: https://github.com/$REPO/pkgs/container/warmbly%2Fbackend"
-        }
+        spin "Pulling images" sh -c "cd '$DIR' && $COMPOSE pull" || pull_failed
         return 0
     fi
     _total=$(printf '%s\n' "$_refs" | wc -l | tr -d ' ')
@@ -2296,7 +2324,7 @@ pull_images() {
         say "  ... pulling $_total images"
         wait "$_pid" 2>/dev/null && _rc=0 || _rc=$?
         [ "$_rc" = 0 ] && say "  ok  pulled $_total images"
-        [ "$_rc" = 0 ] || { show_log; fail_with "Could not pull the release images."; }
+        [ "$_rc" = 0 ] || pull_failed
         return 0
     fi
 
@@ -2336,10 +2364,7 @@ pull_images() {
     printf '%b' "$SHOW"
     wait "$_pid" 2>/dev/null && _rc=0 || _rc=$?
     if [ "$_rc" != 0 ]; then
-        show_log
-        fail_with "Could not pull the release images." \
-                  "The tag ${RESOLVED_TAG} may not exist, or this host cannot reach ${REGISTRY}." \
-                  "Releases: https://github.com/$REPO/releases"
+        pull_failed
     fi
 }
 

site/public/install.sh.sha256

@@ -1 +1 @@
-e029f3a9c9de2f4f3bfb648018435a2da20eee4d0dabd4e1659d80afc4eb44d1  install.sh
+76dea57a84aa5ee84126dc657d2df87725fba90998eb4224ed861ccb71b0b08e  install.sh