From 280a3e64ac302afbc57b1e6f8db1d45fbbdc6c8f Mon Sep 17 00:00:00 2001 From: Matthew Meszaros Date: Fri, 11 Sep 2026 22:31:41 -0700 Subject: [PATCH] feat: publish Kafka-linked image variants (#448) * feat: publish Kafka-linked images for backend, consumer, worker and tracking as -kafka tag variants built per-arch on native runners, because the cgo librdkafka link cannot cross-compile on the build-go path * feat: carry BUILT_AT inside the Go Kafka targets' build-args instead of the shared build-native block, so tracking and realtime stop warning about an unconsumed arg, and emit the matrix with printf because echo expands the separating backslash-n in some shells --- .github/workflows/build-push.yml | 110 ++++++++++++------ .github/workflows/release.yml | 69 +++++++---- deploy/README.md | 4 +- deploy/config/env.example | 5 +- .../docs/development/configuration.mdx | 4 +- .../docs/development/deployment-guide.mdx | 13 ++- docs/content/docs/development/events.mdx | 4 +- 7 files changed, 141 insertions(+), 68 deletions(-) diff --git a/.github/workflows/build-push.yml b/.github/workflows/build-push.yml index 619d1f64..bce97557 100644 --- a/.github/workflows/build-push.yml +++ b/.github/workflows/build-push.yml @@ -23,10 +23,12 @@ jobs: name: Detect Changes runs-on: ubuntu-latest outputs: - # JSON lists consumed as matrix inputs. `go` services cross-compile both - # arches on one runner; `native` services (Rust, Elixir) have no - # cross-compiler, so each arch builds on a native runner and the digests - # are merged into one manifest. + # JSON consumed as matrix inputs. `go` is a list of service names that + # cross-compile both arches on one runner. `native` is a list of + # {service, suffix, args} objects that cannot cross-compile, so each arch + # builds on a native runner and the digests are merged into one manifest: + # Rust and Elixir have no cross-compiler, and the Kafka variants of the Go + # services link librdkafka through cgo, which needs a native toolchain. go: ${{ steps.matrix.outputs.go }} native: ${{ steps.matrix.outputs.native }} steps: @@ -87,28 +89,53 @@ jobs: UPDATER: ${{ steps.filter.outputs.updater }} TRACKING: ${{ steps.filter.outputs.tracking }} REALTIME: ${{ steps.filter.outputs.realtime }} + BUILT_AT: ${{ github.event.head_commit.timestamp }} run: | go="" native="" + # A service that talks to the event bus gets a second, Kafka-linked + # image published as :-kafka. backend, consumer, worker and + # tracking are the four that do; forms and updater never touch it, and + # realtime is fed by the others rather than reading the bus itself. + add() { + case "$1" in + backend|consumer|worker) + go="$go\"$1\"," + native="$native{\"service\":\"$1\",\"suffix\":\"-kafka\",\"args\":\"GO_TAGS=kafka\\nBUILT_AT=$BUILT_AT\"}," + ;; + forms|updater) + go="$go\"$1\"," + ;; + tracking) + native="$native{\"service\":\"tracking\",\"suffix\":\"\",\"args\":\"\"}," + native="$native{\"service\":\"tracking\",\"suffix\":\"-kafka\",\"args\":\"CARGO_FEATURES=kafka\"}," + ;; + realtime) + native="$native{\"service\":\"realtime\",\"suffix\":\"\",\"args\":\"\"}," + ;; + esac + } if [ "$EVENT" = "workflow_dispatch" ]; then sel="${SELECTED:-all}" - for s in backend consumer worker forms updater; do - if [ "$sel" = "all" ] || [ "$sel" = "$s" ]; then go="$go\"$s\","; fi - done - for s in tracking realtime; do - if [ "$sel" = "all" ] || [ "$sel" = "$s" ]; then native="$native\"$s\","; fi + for s in backend consumer worker forms updater tracking realtime; do + if [ "$sel" = "all" ] || [ "$sel" = "$s" ]; then add "$s"; fi done else - if [ "$BACKEND" = "true" ]; then go="$go\"backend\","; fi - if [ "$CONSUMER" = "true" ]; then go="$go\"consumer\","; fi - if [ "$WORKER" = "true" ]; then go="$go\"worker\","; fi - if [ "$FORMS" = "true" ]; then go="$go\"forms\","; fi - if [ "$UPDATER" = "true" ]; then go="$go\"updater\","; fi - if [ "$TRACKING" = "true" ]; then native="$native\"tracking\","; fi - if [ "$REALTIME" = "true" ]; then native="$native\"realtime\","; fi + [ "$BACKEND" = "true" ] && add backend + [ "$CONSUMER" = "true" ] && add consumer + [ "$WORKER" = "true" ] && add worker + [ "$FORMS" = "true" ] && add forms + [ "$UPDATER" = "true" ] && add updater + [ "$TRACKING" = "true" ] && add tracking + [ "$REALTIME" = "true" ] && add realtime + true fi - echo "go=[${go%,}]" >> "$GITHUB_OUTPUT" - echo "native=[${native%,}]" >> "$GITHUB_OUTPUT" + # printf, not echo: the Go Kafka args carry a literal \n separating two + # build-args, and echo expands backslash escapes in some shells, which + # would split the value across two lines and break both the JSON and + # the single-line $GITHUB_OUTPUT format. + printf 'go=[%s]\n' "${go%,}" >> "$GITHUB_OUTPUT" + printf 'native=[%s]\n' "${native%,}" >> "$GITHUB_OUTPUT" # Go services: the Dockerfiles build on $BUILDPLATFORM and cross-compile to # each target arch, so one amd64 runner produces both platforms without QEMU. @@ -154,18 +181,19 @@ jobs: cache-from: type=gha,scope=${{ matrix.service }} cache-to: type=gha,mode=max,scope=${{ matrix.service }} - # Rust (tracking) and Elixir (realtime) have no cross-compiler; an emulated - # arm64 build under QEMU runs for an hour or more. Build each arch on a - # native runner and merge the digests into one manifest (the + # Everything that cannot cross-compile: Rust (tracking), Elixir (realtime), + # and the Kafka variants of the Go services, which link librdkafka through + # cgo. An emulated arm64 build under QEMU runs for an hour or more, so build + # each arch on a native runner and merge the digests into one manifest (the # Docker-documented multi-runner pattern). build-native: - name: Build ${{ matrix.service }} (${{ matrix.platform }}) + name: Build ${{ matrix.target.service }}${{ matrix.target.suffix }} (${{ matrix.platform }}) needs: changes if: needs.changes.outputs.native != '[]' strategy: fail-fast: false matrix: - service: ${{ fromJSON(needs.changes.outputs.native) }} + target: ${{ fromJSON(needs.changes.outputs.native) }} platform: [linux/amd64, linux/arm64] runs-on: ${{ matrix.platform == 'linux/arm64' && 'ubuntu-24.04-arm' || 'ubuntu-latest' }} permissions: @@ -174,11 +202,18 @@ jobs: steps: - uses: actions/checkout@v4 - - name: Prepare platform pair + # `key` is the per-variant identifier: the image name plus the tag suffix, + # so a Kafka build never shares a cache scope or a digest artifact with + # the plain build of the same service. + - name: Prepare identifiers id: prep - run: echo "pair=${PLATFORM//\//-}" >> "$GITHUB_OUTPUT" + run: | + echo "pair=${PLATFORM//\//-}" >> "$GITHUB_OUTPUT" + echo "key=${SERVICE}${SUFFIX}" >> "$GITHUB_OUTPUT" env: PLATFORM: ${{ matrix.platform }} + SERVICE: ${{ matrix.target.service }} + SUFFIX: ${{ matrix.target.suffix }} - name: Set up Docker Buildx uses: docker/setup-buildx-action@v3 @@ -194,15 +229,16 @@ jobs: id: build uses: docker/build-push-action@v6 with: - context: ${{ matrix.service == 'tracking' && './tracking' || '.' }} - file: ${{ matrix.service == 'tracking' && './tracking/Dockerfile' || format('deploy/docker/{0}.Dockerfile', matrix.service) }} + context: ${{ matrix.target.service == 'tracking' && './tracking' || '.' }} + file: ${{ matrix.target.service == 'tracking' && './tracking/Dockerfile' || format('deploy/docker/{0}.Dockerfile', matrix.target.service) }} platforms: ${{ matrix.platform }} build-args: | VERSION=dev-${{ github.sha }} COMMIT=${{ github.sha }} - cache-from: type=gha,scope=${{ matrix.service }}-${{ steps.prep.outputs.pair }} - cache-to: type=gha,mode=max,scope=${{ matrix.service }}-${{ steps.prep.outputs.pair }} - outputs: type=image,name=${{ env.IMAGE_PREFIX }}/${{ matrix.service }},push-by-digest=true,name-canonical=true,push=true + ${{ matrix.target.args }} + cache-from: type=gha,scope=${{ steps.prep.outputs.key }}-${{ steps.prep.outputs.pair }} + cache-to: type=gha,mode=max,scope=${{ steps.prep.outputs.key }}-${{ steps.prep.outputs.pair }} + outputs: type=image,name=${{ env.IMAGE_PREFIX }}/${{ matrix.target.service }},push-by-digest=true,name-canonical=true,push=true - name: Export digest run: | @@ -213,19 +249,19 @@ jobs: - name: Upload digest uses: actions/upload-artifact@v4 with: - name: digests-${{ matrix.service }}-${{ steps.prep.outputs.pair }} + name: digests-${{ steps.prep.outputs.key }}-${{ steps.prep.outputs.pair }} path: /tmp/digests/* if-no-files-found: error retention-days: 1 merge-native: - name: Merge ${{ matrix.service }} manifest + name: Merge ${{ matrix.target.service }}${{ matrix.target.suffix }} manifest needs: [changes, build-native] if: needs.changes.outputs.native != '[]' strategy: fail-fast: false matrix: - service: ${{ fromJSON(needs.changes.outputs.native) }} + target: ${{ fromJSON(needs.changes.outputs.native) }} runs-on: ubuntu-latest permissions: contents: read @@ -235,7 +271,7 @@ jobs: uses: actions/download-artifact@v4 with: path: /tmp/digests - pattern: digests-${{ matrix.service }}-* + pattern: digests-${{ matrix.target.service }}${{ matrix.target.suffix }}-linux-* merge-multiple: true - name: Log in to GHCR @@ -252,9 +288,9 @@ jobs: working-directory: /tmp/digests run: | docker buildx imagetools create \ - -t ${{ env.IMAGE_PREFIX }}/${{ matrix.service }}:${{ github.sha }} \ - -t ${{ env.IMAGE_PREFIX }}/${{ matrix.service }}:dev \ - $(printf '${{ env.IMAGE_PREFIX }}/${{ matrix.service }}@sha256:%s ' *) + -t ${{ env.IMAGE_PREFIX }}/${{ matrix.target.service }}:${{ github.sha }}${{ matrix.target.suffix }} \ + -t ${{ env.IMAGE_PREFIX }}/${{ matrix.target.service }}:dev${{ matrix.target.suffix }} \ + $(printf '${{ env.IMAGE_PREFIX }}/${{ matrix.target.service }}@sha256:%s ' *) # A package on GHCR is created private and does not inherit the repository's # visibility, and no API can change that: an owner has to flip it by hand. diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 9f4eb0d6..84839be3 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -129,17 +129,28 @@ jobs: cache-from: type=gha,scope=${{ matrix.service }} cache-to: type=gha,mode=max,scope=${{ matrix.service }} - # Rust (tracking) and Elixir (realtime) have no cross-compiler; an emulated - # arm64 build under QEMU runs for an hour or more. Build each arch on a - # native runner and merge the digests into one manifest (the - # Docker-documented multi-runner pattern). + # Everything that cannot cross-compile: Rust (tracking), Elixir (realtime), + # and the Kafka variants of the bus-facing Go services, which link librdkafka + # through cgo. An emulated arm64 build under QEMU runs for an hour or more, so + # build each arch on a native runner and merge the digests into one manifest + # (the Docker-documented multi-runner pattern). + # + # The Kafka variants publish under the same image names with a -kafka tag + # suffix, so an operator on Kafka pins ...:prod-kafka and gets the same + # release built with EVENTBUS_PROVIDER=kafka available. build-native: - name: Build ${{ matrix.service }} (${{ matrix.platform }}) + name: Build ${{ matrix.target.service }}${{ matrix.target.suffix }} (${{ matrix.platform }}) needs: validate-tag strategy: fail-fast: false matrix: - service: [tracking, realtime] + target: + - { service: tracking, suffix: "", args: "" } + - { service: tracking, suffix: "-kafka", args: "CARGO_FEATURES=kafka" } + - { service: realtime, suffix: "", args: "" } + - { service: backend, suffix: "-kafka", args: "GO_TAGS=kafka" } + - { service: consumer, suffix: "-kafka", args: "GO_TAGS=kafka" } + - { service: worker, suffix: "-kafka", args: "GO_TAGS=kafka" } platform: [linux/amd64, linux/arm64] runs-on: ${{ matrix.platform == 'linux/arm64' && 'ubuntu-24.04-arm' || 'ubuntu-latest' }} permissions: @@ -148,11 +159,18 @@ jobs: steps: - uses: actions/checkout@v4 - - name: Prepare platform pair + # `key` is the per-variant identifier: the image name plus the tag suffix, + # so a Kafka build never shares a cache scope or a digest artifact with + # the plain build of the same service. + - name: Prepare identifiers id: prep - run: echo "pair=${PLATFORM//\//-}" >> "$GITHUB_OUTPUT" + run: | + echo "pair=${PLATFORM//\//-}" >> "$GITHUB_OUTPUT" + echo "key=${SERVICE}${SUFFIX}" >> "$GITHUB_OUTPUT" env: PLATFORM: ${{ matrix.platform }} + SERVICE: ${{ matrix.target.service }} + SUFFIX: ${{ matrix.target.suffix }} - name: Set up Docker Buildx uses: docker/setup-buildx-action@v3 @@ -168,15 +186,16 @@ jobs: id: build uses: docker/build-push-action@v6 with: - context: ${{ matrix.service == 'tracking' && './tracking' || '.' }} - file: ${{ matrix.service == 'tracking' && './tracking/Dockerfile' || format('deploy/docker/{0}.Dockerfile', matrix.service) }} + context: ${{ matrix.target.service == 'tracking' && './tracking' || '.' }} + file: ${{ matrix.target.service == 'tracking' && './tracking/Dockerfile' || format('deploy/docker/{0}.Dockerfile', matrix.target.service) }} platforms: ${{ matrix.platform }} build-args: | VERSION=${{ github.ref_name }} COMMIT=${{ github.sha }} - cache-from: type=gha,scope=${{ matrix.service }}-${{ steps.prep.outputs.pair }} - cache-to: type=gha,mode=max,scope=${{ matrix.service }}-${{ steps.prep.outputs.pair }} - outputs: type=image,name=${{ env.IMAGE_PREFIX }}/${{ matrix.service }},push-by-digest=true,name-canonical=true,push=true + ${{ matrix.target.args }} + cache-from: type=gha,scope=${{ steps.prep.outputs.key }}-${{ steps.prep.outputs.pair }} + cache-to: type=gha,mode=max,scope=${{ steps.prep.outputs.key }}-${{ steps.prep.outputs.pair }} + outputs: type=image,name=${{ env.IMAGE_PREFIX }}/${{ matrix.target.service }},push-by-digest=true,name-canonical=true,push=true - name: Export digest run: | @@ -187,18 +206,24 @@ jobs: - name: Upload digest uses: actions/upload-artifact@v4 with: - name: digests-${{ matrix.service }}-${{ steps.prep.outputs.pair }} + name: digests-${{ steps.prep.outputs.key }}-${{ steps.prep.outputs.pair }} path: /tmp/digests/* if-no-files-found: error retention-days: 1 merge-native: - name: Merge ${{ matrix.service }} manifest + name: Merge ${{ matrix.target.service }}${{ matrix.target.suffix }} manifest needs: [validate-tag, build-native] strategy: fail-fast: false matrix: - service: [tracking, realtime] + target: + - { service: tracking, suffix: "" } + - { service: tracking, suffix: "-kafka" } + - { service: realtime, suffix: "" } + - { service: backend, suffix: "-kafka" } + - { service: consumer, suffix: "-kafka" } + - { service: worker, suffix: "-kafka" } runs-on: ubuntu-latest permissions: contents: read @@ -208,7 +233,7 @@ jobs: uses: actions/download-artifact@v4 with: path: /tmp/digests - pattern: digests-${{ matrix.service }}-* + pattern: digests-${{ matrix.target.service }}${{ matrix.target.suffix }}-linux-* merge-multiple: true - name: Log in to GHCR @@ -225,11 +250,11 @@ jobs: working-directory: /tmp/digests run: | docker buildx imagetools create \ - -t ${{ env.IMAGE_PREFIX }}/${{ matrix.service }}:${{ github.ref_name }} \ - -t ${{ env.IMAGE_PREFIX }}/${{ matrix.service }}:v${{ needs.validate-tag.outputs.minor }} \ - -t ${{ env.IMAGE_PREFIX }}/${{ matrix.service }}:v${{ needs.validate-tag.outputs.major }} \ - -t ${{ env.IMAGE_PREFIX }}/${{ matrix.service }}:prod \ - $(printf '${{ env.IMAGE_PREFIX }}/${{ matrix.service }}@sha256:%s ' *) + -t ${{ env.IMAGE_PREFIX }}/${{ matrix.target.service }}:${{ github.ref_name }}${{ matrix.target.suffix }} \ + -t ${{ env.IMAGE_PREFIX }}/${{ matrix.target.service }}:v${{ needs.validate-tag.outputs.minor }}${{ matrix.target.suffix }} \ + -t ${{ env.IMAGE_PREFIX }}/${{ matrix.target.service }}:v${{ needs.validate-tag.outputs.major }}${{ matrix.target.suffix }} \ + -t ${{ env.IMAGE_PREFIX }}/${{ matrix.target.service }}:prod${{ matrix.target.suffix }} \ + $(printf '${{ env.IMAGE_PREFIX }}/${{ matrix.target.service }}@sha256:%s ' *) # The `warmbly` CLI is a plain static binary, so it cross-compiles for every # platform on one runner. diff --git a/deploy/README.md b/deploy/README.md index 351388e3..d4e76242 100644 --- a/deploy/README.md +++ b/deploy/README.md @@ -42,7 +42,9 @@ docker build -f web/Dockerfile -t warmbly/web web/ docker build -f admin/Dockerfile -t warmbly/admin admin/ ``` -The default builds have no Kafka/Avro support; add `--build-arg GO_TAGS=kafka` (Go images) or `--build-arg CARGO_FEATURES=kafka` (tracking) to opt in. +The default builds have no Kafka/Avro support; add `--build-arg GO_TAGS=kafka` (Go images) or `--build-arg CARGO_FEATURES=kafka` (tracking) to opt in. Those builds link librdkafka through cgo, which cannot cross-compile, so each architecture has to be built on a machine of that architecture. + +CI already publishes them, so you rarely need to: `backend`, `consumer`, `worker`, and `tracking` each get a second tag with a `-kafka` suffix (`ghcr.io/warmbly/warmbly/backend:prod-kafka`). GitHub Actions publishes these to GHCR automatically. See [the self-hosting guide](https://docs.warmbly.com/development/deployment-guide/). diff --git a/deploy/config/env.example b/deploy/config/env.example index 7204a407..78554a7e 100644 --- a/deploy/config/env.example +++ b/deploy/config/env.example @@ -60,8 +60,9 @@ REDIS=redis://localhost:6379 # === Provider switches (no-cloud defaults) === AWS_CONFIG_ENABLED=false # true => read secrets from AWS SSM/Secrets Manager -# Event bus. nats (default): one small JetStream binary. kafka: build the images -# with GO_TAGS=kafka / CARGO_FEATURES=kafka and set KAFKA_* below. +# Event bus. nats (default): one small JetStream binary. kafka: run the -kafka +# images (ghcr.io/warmbly/warmbly/backend:prod-kafka and the same for consumer, +# worker, tracking) and set KAFKA_* below. EVENTBUS_PROVIDER=nats NATS_URL=nats://localhost:4222 # NATS_STREAM_NAME=warmbly diff --git a/docs/content/docs/development/configuration.mdx b/docs/content/docs/development/configuration.mdx index b90c7191..51cec201 100644 --- a/docs/content/docs/development/configuration.mdx +++ b/docs/content/docs/development/configuration.mdx @@ -289,7 +289,7 @@ On `filesystem`, a remote worker writes blobs to its own disk rather than a volu | Variable | What it does | Default | Restart needed | |---|---|---|---| -| `EVENTBUS_PROVIDER` | `nats` or `kafka`. Kafka needs images built with `GO_TAGS=kafka` | `nats` under compose, `kafka` for a bare binary | yes | +| `EVENTBUS_PROVIDER` | `nats` or `kafka`. Kafka needs the `-kafka` images, or a build with `GO_TAGS=kafka` | `nats` under compose, `kafka` for a bare binary | yes | | `NATS_URL` | JetStream address. Credentials in the URL are honored by every service, including the Rust tracking publisher: `nats://user:pass@host:4222` for a user, `nats://token@host:4222` for a token, `tls://` for TLS | `nats://nats:4222` | yes | | `NATS_CREDS` | Path to a NATS credentials file (user JWT plus nkey seed), for a bus that authenticates with JWT rather than a token. Synadia Cloud and any nsc-managed account issue one | unset | yes | | `NATS_CREDS_B64` | The same file, base64 encoded, as a single line. This is the form the fleet uses: a node receives environment variables rather than files, and the env file docker reads cannot express a multi-line value. Takes precedence over `NATS_CREDS` | unset | yes | @@ -566,7 +566,7 @@ The Rust open and click service. It reads its own environment, so these have to | `TRACKING_SCANNER_NETWORKS` | Extra scanner sources, comma separated, each a CIDR or `asn:` with an optional label. For sources that never carry a person's own request, so their pixel fetches and their clicks are both treated as automated | empty | | `TRACKING_SCANNER_CLICK_NETWORKS` | The same, for sources that also proxy a mail client's own image fetches. Only their click tickets are treated as automated, because their pixel fetches are genuine opens | empty | | `TRACKING_SCANNER_ASN_HEADER` | Header a trusted proxy sets with the source ASN, which is what makes `asn:` entries match. Read only from a proxy in `TRACKING_TRUSTED_PROXIES`. On Cloudflare, a transform rule writing `ip.src.asnum`. Empty disables ASN matching | empty | -| `EVENTBUS_PROVIDER` | `nats` or `kafka`. Kafka needs an image built with `CARGO_FEATURES=kafka` | `nats` | +| `EVENTBUS_PROVIDER` | `nats` or `kafka`. Kafka needs the `tracking:*-kafka` image, or a build with `CARGO_FEATURES=kafka` | `nats` | | `NATS_URL`, `NATS_SUBJECT_PREFIX` | JetStream address and subject prefix. The publish subject is `.` | `nats://localhost:4222`, `warmbly` | | `KAFKA_TRACKING_TOPIC` | Event topic, read by the Rust publisher **and** the Go subscriber | `tracking-events` | | `KAFKA_BOOTSTRAP_SERVERS`, `KAFKA_SASL_USERNAME`, `KAFKA_SASL_PASSWORD` | Broker transport when `EVENTBUS_PROVIDER=kafka` | unset | diff --git a/docs/content/docs/development/deployment-guide.mdx b/docs/content/docs/development/deployment-guide.mdx index b3d333e3..5d7c6a57 100644 --- a/docs/content/docs/development/deployment-guide.mdx +++ b/docs/content/docs/development/deployment-guide.mdx @@ -329,7 +329,7 @@ DEPLOYMENT_MODE=self_hosted # AWS_SECRET_ACCESS_KEY= # KMS_PROVIDER=aws # KMS_AWS_KEY_ID=alias/warmbly -# EVENTBUS_PROVIDER=kafka # needs images built with GO_TAGS=kafka +# EVENTBUS_PROVIDER=kafka # needs the -kafka images (see below) # KAFKA_BOOTSTRAP_SERVERS=broker:9092 # PRIMARY_DB=postgres://user:pass@host:5432/warmbly?sslmode=require # REDIS=redis://host:6379 @@ -633,7 +633,16 @@ Pair it with `DISABLE_PASSWORD_LOGIN=true` for an SSO-only deployment. Three things to know: - **Bare binaries default to the cloud values** (`kafka`, `avro`, `s3`, `aws`). Compose, the Makefile, and both env templates set the local values for you. A hand-rolled environment must set them or the process exits at boot. -- Kafka and Avro are also build-time opt-ins: `--build-arg GO_TAGS=kafka` for Go, `--build-arg CARGO_FEATURES=kafka` for tracking. +- Kafka and Avro are build-time opt-ins, so `EVENTBUS_PROVIDER=kafka` needs a build that has them. Each release publishes those builds as a second tag on the same image, suffixed `-kafka`, for the four services that touch the bus: + + ``` + ghcr.io/warmbly/warmbly/backend:prod-kafka + ghcr.io/warmbly/warmbly/consumer:prod-kafka + ghcr.io/warmbly/warmbly/worker:prod-kafka + ghcr.io/warmbly/warmbly/tracking:prod-kafka + ``` + + `forms`, `updater`, `realtime`, `web`, and `admin` never read the bus and have one image each. To build your own instead, pass `--build-arg GO_TAGS=kafka` (Go) or `--build-arg CARGO_FEATURES=kafka` (tracking); cgo cannot cross-compile, so each architecture has to be built on a machine of that architecture. - `PUBSUB_ENABLED` must match across backend, consumer, and realtime. ## Optional subsystems diff --git a/docs/content/docs/development/events.mdx b/docs/content/docs/development/events.mdx index 770b9150..99a8fc50 100644 --- a/docs/content/docs/development/events.mdx +++ b/docs/content/docs/development/events.mdx @@ -10,14 +10,14 @@ Warmbly's services communicate asynchronously through a pluggable event bus. The The bus is an abstraction (`internal/infrastructure/eventbus/`) with two providers, selected by `EVENTBUS_PROVIDER`: - `nats`: NATS JetStream. One durable stream (default name `warmbly`) holds every topic as a subject under a prefix: topic `jobs.worker-events` becomes subject `warmbly.jobs.worker-events`. Messages are retained for 7 days, consumer groups map to JetStream durable consumers, and failed handlers are redelivered up to 10 times. This is what `docker-compose.yml` runs. -- `kafka`: Apache Kafka behind the `kafka` Go build tag. The default binaries do not include it (no librdkafka, no CGO); build with `GO_TAGS=kafka` to enable it, otherwise `EVENTBUS_PROVIDER=kafka` fails at boot with a clear error. +- `kafka`: Apache Kafka behind the `kafka` Go build tag. The default binaries do not include it (no librdkafka, no CGO), so `EVENTBUS_PROVIDER=kafka` on a default build fails at boot with a clear error. Every service that reads the bus publishes a second image for it, the same tag with a `-kafka` suffix: `ghcr.io/warmbly/warmbly/backend:prod-kafka`, and the same for `consumer`, `worker`, and `tracking`. Building your own needs `GO_TAGS=kafka` (Go) or `CARGO_FEATURES=kafka` (tracking). Message encoding is orthogonal to transport, selected by `CODEC_PROVIDER`: - `json`: plain JSON, no external dependencies. Required for the self-host stack (the worker command and result envelopes carry untyped bodies Avro cannot serialize). - `avro`: Avro with Confluent Schema Registry (`SCHEMA_REGISTRY_URL` plus optional key/secret), also behind the `kafka` build tag. The topic name is used as the Schema Registry subject. -The Rust tracking service follows the same switch: it publishes JSON to JetStream by default, and only speaks Kafka plus Avro when compiled with its `kafka` cargo feature. +The Rust tracking service follows the same switch: it publishes JSON to JetStream by default, and only speaks Kafka plus Avro when compiled with its `kafka` cargo feature, which is what the `tracking:*-kafka` image is. ## Topics