diff --git a/.github/actions/sign-attest-image/action.yml b/.github/actions/sign-attest-image/action.yml new file mode 100644 index 0000000000..de0133dac0 --- /dev/null +++ b/.github/actions/sign-attest-image/action.yml @@ -0,0 +1,56 @@ +name: Sign image and attach provenance +description: > + Keyless-signs a pushed image digest with cosign (index and per-arch + manifests) and records SLSA provenance as a GitHub artifact attestation + pushed to the registry. SBOMs are not generated here: the build step embeds + them as BuildKit attestation manifests (depot `sbom: true`), which the index + signature then covers. The calling job must already be logged in to the + registry and must have id-token: write, attestations: write and + packages: write permissions (write-all covers all three). +inputs: + image: + description: "Fully-qualified image name without tag, e.g. ghcr.io/windmill-labs/windmill" + required: true + digest: + description: "Pushed manifest digest (sha256:...) from build-push-action" + required: true +runs: + using: composite + steps: + - name: Preflight + shell: bash + env: + DIGEST: ${{ inputs.digest }} + run: | + if [ -z "${ACTIONS_ID_TOKEN_REQUEST_URL:-}" ]; then + echo "::error::No OIDC token available; the calling job needs id-token: write" + exit 1 + fi + case "$DIGEST" in + sha256:*) ;; + *) + echo "::error::digest '$DIGEST' is not a sha256: digest" + exit 1 + ;; + esac + + # cosign v2 writes the classic sha256-.sig tag format that the + # installed base of cosign clients can verify; v3's bundle format cannot be + # verified by v2 clients yet, so stay on v2 until v3 verification is common. + - uses: sigstore/cosign-installer@v4.1.2 + with: + cosign-release: "v2.6.5" + + - name: Cosign keyless sign (index + per-arch manifests) + shell: bash + env: + IMAGE: ${{ inputs.image }} + DIGEST: ${{ inputs.digest }} + run: cosign sign --yes --recursive "${IMAGE}@${DIGEST}" + + - name: SLSA provenance (GitHub artifact attestation) + uses: actions/attest-build-provenance@v4 + with: + subject-name: ${{ inputs.image }} + subject-digest: ${{ inputs.digest }} + push-to-registry: true diff --git a/.github/workflows/build_cli_image.yml b/.github/workflows/build_cli_image.yml index 5b71a786aa..de64a76583 100644 --- a/.github/workflows/build_cli_image.yml +++ b/.github/workflows/build_cli_image.yml @@ -13,9 +13,13 @@ permissions: contents: read id-token: write packages: write + attestations: write jobs: publish_cli: + # a tag-targeted dispatch would republish the release tags unsigned, + # un-verifying the release; to republish a release, re-push its tag + if: github.event_name == 'push' || !startsWith(github.ref, 'refs/tags/') runs-on: ubicloud steps: - uses: actions/checkout@v4 @@ -42,14 +46,23 @@ jobs: password: ${{ secrets.GITHUB_TOKEN }} - name: Build and push publicly + id: docker_build uses: depot/build-push-action@v1 with: file: "./docker/DockerfileCli" platforms: linux/amd64,linux/arm64 push: true + sbom: ${{ startsWith(github.ref, 'refs/tags/v') && github.event_name == 'push' }} tags: | ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}:latest ${{ steps.meta.outputs.tags }} labels: | ${{ steps.meta.outputs.labels }} org.opencontainers.image.licenses=AGPLv3 + + - name: Sign and attest release image + if: startsWith(github.ref, 'refs/tags/v') && github.event_name == 'push' + uses: ./.github/actions/sign-attest-image + with: + image: ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }} + digest: ${{ steps.docker_build.outputs.digest }} diff --git a/.github/workflows/docker-image.yml b/.github/workflows/docker-image.yml index 433f3a86b3..a8b20623bc 100644 --- a/.github/workflows/docker-image.yml +++ b/.github/workflows/docker-image.yml @@ -86,11 +86,13 @@ jobs: type=semver,pattern={{major}}.{{minor}} - name: Build and push publicly + id: docker_build uses: depot/build-push-action@v1 with: context: . platforms: linux/amd64,linux/arm64 push: true + sbom: ${{ startsWith(github.ref, 'refs/tags/v') && github.event_name == 'push' }} build-args: | features=ce WM_BUILD_VERSION=${{ github.sha }} @@ -100,6 +102,13 @@ jobs: labels: | ${{ steps.meta-public.outputs.labels }} + - name: Sign and attest release image + if: startsWith(github.ref, 'refs/tags/v') && github.event_name == 'push' + uses: ./.github/actions/sign-attest-image + with: + image: ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }} + digest: ${{ steps.docker_build.outputs.digest }} + build_ee: runs-on: ubicloud if: (github.event_name != 'workflow_dispatch') || github.event.inputs.ee @@ -149,11 +158,13 @@ jobs: ./backend/substitute_ee_code.sh --copy --dir ./windmill-ee-private - name: Build and push publicly ee + id: docker_build uses: depot/build-push-action@v1 with: context: . platforms: linux/amd64,linux/arm64 push: true + sbom: ${{ startsWith(github.ref, 'refs/tags/v') && github.event_name == 'push' }} build-args: | features=ee WM_BUILD_VERSION=${{ github.sha }} @@ -164,6 +175,13 @@ jobs: ${{ steps.meta-ee-public.outputs.labels }} org.opencontainers.image.licenses=Windmill-Enterprise-License + - name: Sign and attest release image + if: startsWith(github.ref, 'refs/tags/v') && github.event_name == 'push' + uses: ./.github/actions/sign-attest-image + with: + image: ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}-ee + digest: ${{ steps.docker_build.outputs.digest }} + attach_amd64_binary_to_release: needs: [build, build_ee] runs-on: ubicloud @@ -358,6 +376,21 @@ jobs: docker buildx imagetools create ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}:${{ env.DEV_SHA }} --tag ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}:latest docker buildx imagetools create ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}:${{ env.DEV_SHA }} --tag ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}:main + - uses: sigstore/cosign-installer@v4.1.2 + if: startsWith(github.ref, 'refs/tags/v') + with: + cosign-release: "v2.6.5" + # end-to-end release guard: the version tag pushed by this run must + # verify against this exact run's identity (the mutable :latest/:dev + # tags race with concurrent main builds, so they are not asserted here) + - name: Verify release image is signed + if: startsWith(github.ref, 'refs/tags/v') + run: | + cosign verify \ + --certificate-oidc-issuer https://token.actions.githubusercontent.com \ + --certificate-identity "https://github.com/windmill-labs/windmill/.github/workflows/docker-image.yml@${GITHUB_REF}" \ + "${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}:${GITHUB_REF_NAME#v}" + tag_latest_ee: runs-on: ubicloud needs: [run_integration_test, build_ee] @@ -379,6 +412,21 @@ jobs: docker buildx imagetools create ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}-ee:${{ env.DEV_SHA }} --tag ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}-ee:latest docker buildx imagetools create ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}-ee:${{ env.DEV_SHA }} --tag ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}-ee:main + - uses: sigstore/cosign-installer@v4.1.2 + if: startsWith(github.ref, 'refs/tags/v') + with: + cosign-release: "v2.6.5" + # end-to-end release guard: the version tag pushed by this run must + # verify against this exact run's identity (the mutable :latest/:dev + # tags race with concurrent main builds, so they are not asserted here) + - name: Verify release ee image is signed + if: startsWith(github.ref, 'refs/tags/v') + run: | + cosign verify \ + --certificate-oidc-issuer https://token.actions.githubusercontent.com \ + --certificate-identity "https://github.com/windmill-labs/windmill/.github/workflows/docker-image.yml@${GITHUB_REF}" \ + "${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}-ee:${GITHUB_REF_NAME#v}" + verify_ee_image_vulnerabilities: runs-on: ubicloud needs: [tag_latest_ee] @@ -493,11 +541,13 @@ jobs: password: ${{ secrets.GITHUB_TOKEN }} - name: Build and push publicly ee + id: docker_build uses: depot/build-push-action@v1 with: context: . platforms: linux/amd64 push: true + sbom: ${{ startsWith(github.ref, 'refs/tags/v') && github.event_name == 'push' }} file: "./docker/DockerfileCuda" tags: | ${{ steps.meta-ee-public.outputs.tags }} @@ -505,6 +555,13 @@ jobs: ${{ steps.meta-ee-public.outputs.labels }} org.opencontainers.image.licenses=Windmill-Enterprise-License + - name: Sign and attest release image + if: startsWith(github.ref, 'refs/tags/v') && github.event_name == 'push' + uses: ./.github/actions/sign-attest-image + with: + image: ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}-ee-cuda + digest: ${{ steps.docker_build.outputs.digest }} + build_slim: if: ${{ startsWith(github.ref, 'refs/tags/v') }} needs: [build] @@ -537,17 +594,26 @@ jobs: password: ${{ secrets.GITHUB_TOKEN }} - name: Build and push publicly ee + id: docker_build uses: depot/build-push-action@v1 with: context: . platforms: linux/amd64 push: true + sbom: ${{ startsWith(github.ref, 'refs/tags/v') && github.event_name == 'push' }} file: "./docker/DockerfileSlim" tags: | ${{ steps.meta-ee-public.outputs.tags }} labels: | ${{ steps.meta-ee-public.outputs.labels }} + - name: Sign and attest release image + if: startsWith(github.ref, 'refs/tags/v') && github.event_name == 'push' + uses: ./.github/actions/sign-attest-image + with: + image: ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}-slim + digest: ${{ steps.docker_build.outputs.digest }} + build_ee_slim: needs: [build_ee] runs-on: ubicloud @@ -582,11 +648,13 @@ jobs: password: ${{ secrets.GITHUB_TOKEN }} - name: Build and push publicly ee + id: docker_build uses: depot/build-push-action@v1 with: context: . platforms: linux/amd64,linux/arm64 push: true + sbom: ${{ startsWith(github.ref, 'refs/tags/v') && github.event_name == 'push' }} file: "./docker/DockerfileSlimEe" tags: | ${{ steps.meta-ee-public.outputs.tags }} @@ -594,6 +662,13 @@ jobs: ${{ steps.meta-ee-public.outputs.labels }} org.opencontainers.image.licenses=Windmill-Enterprise-License + - name: Sign and attest release image + if: startsWith(github.ref, 'refs/tags/v') && github.event_name == 'push' + uses: ./.github/actions/sign-attest-image + with: + image: ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}-ee-slim + digest: ${{ steps.docker_build.outputs.digest }} + build_full: if: ${{ startsWith(github.ref, 'refs/tags/v') }} needs: [build] @@ -626,17 +701,26 @@ jobs: password: ${{ secrets.GITHUB_TOKEN }} - name: Build and push publicly + id: docker_build uses: depot/build-push-action@v1 with: context: . platforms: linux/amd64,linux/arm64 push: true + sbom: ${{ startsWith(github.ref, 'refs/tags/v') && github.event_name == 'push' }} file: "./docker/DockerfileFull" tags: | ${{ steps.meta-public.outputs.tags }} labels: | ${{ steps.meta-public.outputs.labels }} + - name: Sign and attest release image + if: startsWith(github.ref, 'refs/tags/v') && github.event_name == 'push' + uses: ./.github/actions/sign-attest-image + with: + image: ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}-full + digest: ${{ steps.docker_build.outputs.digest }} + build_ee_full: if: ${{ startsWith(github.ref, 'refs/tags/v') }} needs: [build_ee] @@ -669,14 +753,23 @@ jobs: password: ${{ secrets.GITHUB_TOKEN }} - name: Build and push publicly ee + id: docker_build uses: depot/build-push-action@v1 with: context: . platforms: linux/amd64,linux/arm64 push: true + sbom: ${{ startsWith(github.ref, 'refs/tags/v') && github.event_name == 'push' }} file: "./docker/DockerfileFullEe" tags: | ${{ steps.meta-ee-public.outputs.tags }} labels: | ${{ steps.meta-ee-public.outputs.labels }} org.opencontainers.image.licenses=Windmill-Enterprise-License + + - name: Sign and attest release image + if: startsWith(github.ref, 'refs/tags/v') && github.event_name == 'push' + uses: ./.github/actions/sign-attest-image + with: + image: ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}-ee-full + digest: ${{ steps.docker_build.outputs.digest }} diff --git a/.github/workflows/publish_extra.yml b/.github/workflows/publish_extra.yml index 49964c95d9..ca62933292 100644 --- a/.github/workflows/publish_extra.yml +++ b/.github/workflows/publish_extra.yml @@ -84,6 +84,9 @@ jobs: publish_extra: needs: [sleep, test_extra] + # a tag-targeted dispatch would republish the release tags unsigned, + # un-verifying the release; to republish a release, re-push its tag + if: github.event_name == 'push' || !startsWith(github.ref, 'refs/tags/') runs-on: ubicloud-standard-8 steps: - uses: actions/checkout@v4 @@ -112,15 +115,24 @@ jobs: password: ${{ secrets.GITHUB_TOKEN }} - name: Build and push publicly + id: docker_build uses: depot/build-push-action@v1 with: context: . file: ./docker/DockerfileExtra platforms: linux/amd64,linux/arm64 push: true + sbom: ${{ startsWith(github.ref, 'refs/tags/v') && github.event_name == 'push' }} tags: | ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}:latest ${{ steps.meta.outputs.tags }} labels: | ${{ steps.meta.outputs.labels }} org.opencontainers.image.licenses=AGPLv3 + + - name: Sign and attest release image + if: startsWith(github.ref, 'refs/tags/v') && github.event_name == 'push' + uses: ./.github/actions/sign-attest-image + with: + image: ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }} + digest: ${{ steps.docker_build.outputs.digest }} diff --git a/backend/THREAT_MODEL.md b/backend/THREAT_MODEL.md index f5725d65a3..af33877078 100644 --- a/backend/THREAT_MODEL.md +++ b/backend/THREAT_MODEL.md @@ -104,7 +104,7 @@ published advisory history (73 GHSA advisories, several rated 9.9 critical). | T6 | Disclosure of secrets, resource credentials, and workspace encryption keys across the authorization boundary (AI proxy, MCP, caches, export); database read additionally yields plaintext instance-level `global_settings` secrets | remote_auth | EP6, EP14, EP13 | Secret variables, encryption keys, resource creds, global settings | critical | likely | partially_mitigated | RLS on `$var:`, cache scoping by caller, admin checks on export; per-workspace secret *variables* encrypted at rest, but `global_settings` is plaintext under the default DB secret backend | GHSA-jwg4-v3cj-rvfm, GHSA-8m2p-2crh-9h3w, GHSA-6635-6fch-v8px, GHSA-437f-725p-7w84, GHSA-f27g-j463-q85w (CVE-2026-26964), GHSA-j679-v6vj-jfxc, GHSA-6vrr-fq33-qpfp, 0ba128afe7, 7836a4e733, ff8e39c69b | | T7 | Full instance compromise from insecure deployment defaults (dind control, default admin/`changeme`, exposed Postgres, publicly readable SUPERADMIN_SECRET) | remote_unauth | EP15 | All assets | critical | likely | partially_mitigated | first-time-setup warning on default admin; docs recommend hardening | GHSA-3vpp-vf62-wqp6, GHSA-24fr-44f8-fqwg (CVE-2026-29059), GHSA-6q36-5p3h-766j | | T8 | Unauthenticated RCE via the Debugger WebSocket: `/ws_debug/*` exposed by the gateway/ingress with the debugger service as the auth boundary; signature gate was bypassable via `program`-mode launches (read+exec an arbitrary server-side file path, never signed) even with signing on, and the WS handshake had no Origin check (CSWSH) | remote_unauth | EP15 | Worker host, all assets | critical | possible | partially_mitigated | `program`-mode launches now rejected when `REQUIRE_SIGNED_DEBUG_REQUESTS` is on (signing covers every launch, not just inline `code`); shipped `docker-compose` now defaults `REQUIRE_SIGNED_DEBUG_REQUESTS=true`; opt-in `DEBUG_ALLOWED_ORIGINS` allowlist rejects cross-origin handshakes. Residual: code default is secure but operators can still set `=false`; origin allowlist is opt-in | GHSA-725h-99vx-9xr4 | -| T9 | Supply-chain compromise via cached hub scripts, GitHub workflow command injection, or vulnerable base-image deps | supply_chain | EP16 | Worker host, build integrity | critical | possible | partially_mitigated | hub-script re-pin to patched versions; HUB_BASE_URL override | GHSA-w2m9-q5f7-3gpq, edf340c4d4, GHSA-8rq7-w7g6-8wvr, GHSA-vch9-39v5-4wg7 (CVE-2024-37371) | +| T9 | Supply-chain compromise via cached hub scripts, GitHub workflow command injection, or vulnerable base-image deps | supply_chain | EP16 | Worker host, build integrity | critical | possible | partially_mitigated | hub-script re-pin to patched versions; HUB_BASE_URL override; release images (`v*` tags) keyless-signed with cosign, with per-platform SPDX SBOMs embedded at build time (covered by the signed index digest) + SLSA provenance (GitHub artifact attestations) | GHSA-w2m9-q5f7-3gpq, edf340c4d4, GHSA-8rq7-w7g6-8wvr, GHSA-vch9-39v5-4wg7 (CVE-2024-37371) | | T10 | Unauthenticated disclosure of job results, args, logs, and admin config via missing-authz public endpoints | remote_unauth | EP2, EP13 | Job results/args/logs, global settings, scripts | high | likely | partially_mitigated | anonymous-job checks, log-endpoint authz hardening | GHSA-qfg7-x243-5hg4, GHSA-v448-fmm4-52fp, 108a88a180, bb90f4ce83 | | T11 | Stored XSS leading to admin/account takeover via app HTML component, markdown, S3 download content-type, or a script-chosen `text/html` content type on `run_wait_result` / sync HTTP-route responses (GET-reachable with the `SameSite=Lax` session cookie) | remote_auth | EP12 | Admin session, accounts | high | likely | partially_mitigated | DOMPurify markdown sanitization, `X-Content-Type-Options: nosniff` + CSP sandbox on downloads and on every `result_to_response` composite result (inserted after `wm_headers`; hop-by-hop names such as `Connection` rejected so a proxy cannot strip them) | GHSA-9c5c-hh3c-r9mc, GHSA-qxj7-hpx3-r892, GHSA-cf2x-rg8c-v63v, bb78b1c06d, 625b67dff0, WIN-2471 | | T12 | Webhook authentication bypass / signature replay forges trigger invocations and approvals | remote_unauth | EP3 | Job execution integrity, approvals | high | likely | partially_mitigated | HMAC verification on some triggers; signing-oracle fix | GHSA-jw8c-h45c-xpjw, GHSA-hh9x-rcf8-xjr2, GHSA-q9g3-q6fj-hc2x, GHSA-8jc4-wj2p-2vmp, ab2a15b2a8 | @@ -169,4 +169,4 @@ check. | Canonicalize + confine all file-path inputs to a base dir and never follow symlinks in log/file readers | T13 | yes | S | | Mask secrets at the log sink and keep secrets out of worker process env (`/proc`) — pass via files/pipes scrubbed after use | T15 | partial | M | | Add global rate limiting and per-tenant resource/queue quotas at the edge | T16, T18 | partial | M | -| Pin and integrity-verify hub scripts and CI actions; SBOM + automated base-image CVE scanning in release | T9 | partial | M | +| Pin and integrity-verify hub scripts and CI actions; SBOM + automated base-image CVE scanning in release — release images now cosign-signed with SBOM + SLSA provenance attestations; remaining: CI action SHA-pinning, hub-script integrity, rhel/rpi images | T9 | partial | M | diff --git a/docs/docker-security.md b/docs/docker-security.md index f0a9f23073..5b5196f8b4 100644 --- a/docs/docker-security.md +++ b/docs/docker-security.md @@ -63,3 +63,65 @@ gap, rebuild and republish the `latest` / patch tags: Scan the published images (e.g. Trivy / Defender) after rebuilds to confirm the base-OS finding count stays low. + +# Verifying image signatures, SBOMs and provenance + +Release images are signed and attested at publish time: + +- **cosign keyless signature** on the pushed manifest digest (index and + per-arch manifests), via GitHub OIDC — no long-lived signing key exists + (`.github/actions/sign-attest-image`). +- **SBOMs** are generated at build time (`sbom: true` on the depot build + step) and embedded in the image index as BuildKit attestation manifests — + one SPDX document per platform. They are part of the signed index digest, + so the cosign signature covers them. They are not sent to a transparency + log: SPDX documents for these images run tens of MB, beyond what Rekor or + GitHub attestations accept as payloads. +- **SLSA build provenance** recorded as a GitHub artifact attestation and + pushed to the registry (`actions/attest-build-provenance`). + +## What is covered + +Only images published from a release tag (`v*`) are signed: `windmill`, +`windmill-ee`, `windmill-ee-cuda`, `windmill-slim`, `windmill-ee-slim`, +`windmill-full`, `windmill-ee-full` (`.github/workflows/docker-image.yml`), +`windmill-cli` (`build_cli_image.yml`) and `windmill-extra` +(`publish_extra.yml`). The `:latest` and `:main` tags are repointed on +every `main` push as well as on releases, so they resolve to a signed +digest only until the next `main` build lands — verify a version tag or a +digest, not `:latest`. Development images (`:dev`, branch builds, +`windmill-test`), the dispatch-only RHEL/rpi images and the `caddy-l4` +image are not signed. + +## How to verify + +Signatures are keyless: trust is anchored in the Fulcio certificate identity, +which for these images is the *calling workflow file at a `v*` tag ref* in +this repository. Verify a signature with cosign (v2.x): + +```bash +cosign verify \ + --certificate-oidc-issuer https://token.actions.githubusercontent.com \ + --certificate-identity-regexp '^https://github.com/windmill-labs/windmill/\.github/workflows/(docker-image|publish_extra|build_cli_image)\.yml@refs/tags/v' \ + ghcr.io/windmill-labs/windmill: +``` + +Extract the embedded SBOM (per platform; verify the signature first — it +covers the index these documents live in): + +```bash +docker buildx imagetools inspect ghcr.io/windmill-labs/windmill: \ + --format '{{ json .SBOM }}' +``` + +Verify SLSA provenance through GitHub's attestation API: + +```bash +gh attestation verify oci://ghcr.io/windmill-labs/windmill: \ + -R windmill-labs/windmill +``` + +Note for registry housekeeping: cosign stores signatures as extra +`sha256-.sig` tags in the same ghcr package, and the pushed +provenance attestations live there as referrer artifacts — any +tag-retention automation must not prune them.