mirror of
https://github.com/warmbly/warmbly.git
synced 2026-09-12 16:04:25 +00:00
feat: publish Kafka-linked image variants (#448)
* feat: publish Kafka-linked images for backend, consumer, worker and tracking as -kafka tag variants built per-arch on native runners, because the cgo librdkafka link cannot cross-compile on the build-go path * feat: carry BUILT_AT inside the Go Kafka targets' build-args instead of the shared build-native block, so tracking and realtime stop warning about an unconsumed arg, and emit the matrix with printf because echo expands the separating backslash-n in some shells
This commit is contained in:
Matthew Meszaros
@@ -23,10 +23,12 @@ jobs:
|
||||
name: Detect Changes
|
||||
runs-on: ubuntu-latest
|
||||
outputs:
|
||||
# JSON lists consumed as matrix inputs. `go` services cross-compile both
|
||||
# arches on one runner; `native` services (Rust, Elixir) have no
|
||||
# cross-compiler, so each arch builds on a native runner and the digests
|
||||
# are merged into one manifest.
|
||||
# JSON consumed as matrix inputs. `go` is a list of service names that
|
||||
# cross-compile both arches on one runner. `native` is a list of
|
||||
# {service, suffix, args} objects that cannot cross-compile, so each arch
|
||||
# builds on a native runner and the digests are merged into one manifest:
|
||||
# Rust and Elixir have no cross-compiler, and the Kafka variants of the Go
|
||||
# services link librdkafka through cgo, which needs a native toolchain.
|
||||
go: ${{ steps.matrix.outputs.go }}
|
||||
native: ${{ steps.matrix.outputs.native }}
|
||||
steps:
|
||||
@@ -87,28 +89,53 @@ jobs:
|
||||
UPDATER: ${{ steps.filter.outputs.updater }}
|
||||
TRACKING: ${{ steps.filter.outputs.tracking }}
|
||||
REALTIME: ${{ steps.filter.outputs.realtime }}
|
||||
BUILT_AT: ${{ github.event.head_commit.timestamp }}
|
||||
run: |
|
||||
go=""
|
||||
native=""
|
||||
# A service that talks to the event bus gets a second, Kafka-linked
|
||||
# image published as :<tag>-kafka. backend, consumer, worker and
|
||||
# tracking are the four that do; forms and updater never touch it, and
|
||||
# realtime is fed by the others rather than reading the bus itself.
|
||||
add() {
|
||||
case "$1" in
|
||||
backend|consumer|worker)
|
||||
go="$go\"$1\","
|
||||
native="$native{\"service\":\"$1\",\"suffix\":\"-kafka\",\"args\":\"GO_TAGS=kafka\\nBUILT_AT=$BUILT_AT\"},"
|
||||
;;
|
||||
forms|updater)
|
||||
go="$go\"$1\","
|
||||
;;
|
||||
tracking)
|
||||
native="$native{\"service\":\"tracking\",\"suffix\":\"\",\"args\":\"\"},"
|
||||
native="$native{\"service\":\"tracking\",\"suffix\":\"-kafka\",\"args\":\"CARGO_FEATURES=kafka\"},"
|
||||
;;
|
||||
realtime)
|
||||
native="$native{\"service\":\"realtime\",\"suffix\":\"\",\"args\":\"\"},"
|
||||
;;
|
||||
esac
|
||||
}
|
||||
if [ "$EVENT" = "workflow_dispatch" ]; then
|
||||
sel="${SELECTED:-all}"
|
||||
for s in backend consumer worker forms updater; do
|
||||
if [ "$sel" = "all" ] || [ "$sel" = "$s" ]; then go="$go\"$s\","; fi
|
||||
done
|
||||
for s in tracking realtime; do
|
||||
if [ "$sel" = "all" ] || [ "$sel" = "$s" ]; then native="$native\"$s\","; fi
|
||||
for s in backend consumer worker forms updater tracking realtime; do
|
||||
if [ "$sel" = "all" ] || [ "$sel" = "$s" ]; then add "$s"; fi
|
||||
done
|
||||
else
|
||||
if [ "$BACKEND" = "true" ]; then go="$go\"backend\","; fi
|
||||
if [ "$CONSUMER" = "true" ]; then go="$go\"consumer\","; fi
|
||||
if [ "$WORKER" = "true" ]; then go="$go\"worker\","; fi
|
||||
if [ "$FORMS" = "true" ]; then go="$go\"forms\","; fi
|
||||
if [ "$UPDATER" = "true" ]; then go="$go\"updater\","; fi
|
||||
if [ "$TRACKING" = "true" ]; then native="$native\"tracking\","; fi
|
||||
if [ "$REALTIME" = "true" ]; then native="$native\"realtime\","; fi
|
||||
[ "$BACKEND" = "true" ] && add backend
|
||||
[ "$CONSUMER" = "true" ] && add consumer
|
||||
[ "$WORKER" = "true" ] && add worker
|
||||
[ "$FORMS" = "true" ] && add forms
|
||||
[ "$UPDATER" = "true" ] && add updater
|
||||
[ "$TRACKING" = "true" ] && add tracking
|
||||
[ "$REALTIME" = "true" ] && add realtime
|
||||
true
|
||||
fi
|
||||
echo "go=[${go%,}]" >> "$GITHUB_OUTPUT"
|
||||
echo "native=[${native%,}]" >> "$GITHUB_OUTPUT"
|
||||
# printf, not echo: the Go Kafka args carry a literal \n separating two
|
||||
# build-args, and echo expands backslash escapes in some shells, which
|
||||
# would split the value across two lines and break both the JSON and
|
||||
# the single-line $GITHUB_OUTPUT format.
|
||||
printf 'go=[%s]\n' "${go%,}" >> "$GITHUB_OUTPUT"
|
||||
printf 'native=[%s]\n' "${native%,}" >> "$GITHUB_OUTPUT"
|
||||
|
||||
# Go services: the Dockerfiles build on $BUILDPLATFORM and cross-compile to
|
||||
# each target arch, so one amd64 runner produces both platforms without QEMU.
|
||||
@@ -154,18 +181,19 @@ jobs:
|
||||
cache-from: type=gha,scope=${{ matrix.service }}
|
||||
cache-to: type=gha,mode=max,scope=${{ matrix.service }}
|
||||
|
||||
# Rust (tracking) and Elixir (realtime) have no cross-compiler; an emulated
|
||||
# arm64 build under QEMU runs for an hour or more. Build each arch on a
|
||||
# native runner and merge the digests into one manifest (the
|
||||
# Everything that cannot cross-compile: Rust (tracking), Elixir (realtime),
|
||||
# and the Kafka variants of the Go services, which link librdkafka through
|
||||
# cgo. An emulated arm64 build under QEMU runs for an hour or more, so build
|
||||
# each arch on a native runner and merge the digests into one manifest (the
|
||||
# Docker-documented multi-runner pattern).
|
||||
build-native:
|
||||
name: Build ${{ matrix.service }} (${{ matrix.platform }})
|
||||
name: Build ${{ matrix.target.service }}${{ matrix.target.suffix }} (${{ matrix.platform }})
|
||||
needs: changes
|
||||
if: needs.changes.outputs.native != '[]'
|
||||
strategy:
|
||||
fail-fast: false
|
||||
matrix:
|
||||
service: ${{ fromJSON(needs.changes.outputs.native) }}
|
||||
target: ${{ fromJSON(needs.changes.outputs.native) }}
|
||||
platform: [linux/amd64, linux/arm64]
|
||||
runs-on: ${{ matrix.platform == 'linux/arm64' && 'ubuntu-24.04-arm' || 'ubuntu-latest' }}
|
||||
permissions:
|
||||
@@ -174,11 +202,18 @@ jobs:
|
||||
steps:
|
||||
- uses: actions/checkout@v4
|
||||
|
||||
- name: Prepare platform pair
|
||||
# `key` is the per-variant identifier: the image name plus the tag suffix,
|
||||
# so a Kafka build never shares a cache scope or a digest artifact with
|
||||
# the plain build of the same service.
|
||||
- name: Prepare identifiers
|
||||
id: prep
|
||||
run: echo "pair=${PLATFORM//\//-}" >> "$GITHUB_OUTPUT"
|
||||
run: |
|
||||
echo "pair=${PLATFORM//\//-}" >> "$GITHUB_OUTPUT"
|
||||
echo "key=${SERVICE}${SUFFIX}" >> "$GITHUB_OUTPUT"
|
||||
env:
|
||||
PLATFORM: ${{ matrix.platform }}
|
||||
SERVICE: ${{ matrix.target.service }}
|
||||
SUFFIX: ${{ matrix.target.suffix }}
|
||||
|
||||
- name: Set up Docker Buildx
|
||||
uses: docker/setup-buildx-action@v3
|
||||
@@ -194,15 +229,16 @@ jobs:
|
||||
id: build
|
||||
uses: docker/build-push-action@v6
|
||||
with:
|
||||
context: ${{ matrix.service == 'tracking' && './tracking' || '.' }}
|
||||
file: ${{ matrix.service == 'tracking' && './tracking/Dockerfile' || format('deploy/docker/{0}.Dockerfile', matrix.service) }}
|
||||
context: ${{ matrix.target.service == 'tracking' && './tracking' || '.' }}
|
||||
file: ${{ matrix.target.service == 'tracking' && './tracking/Dockerfile' || format('deploy/docker/{0}.Dockerfile', matrix.target.service) }}
|
||||
platforms: ${{ matrix.platform }}
|
||||
build-args: |
|
||||
VERSION=dev-${{ github.sha }}
|
||||
COMMIT=${{ github.sha }}
|
||||
cache-from: type=gha,scope=${{ matrix.service }}-${{ steps.prep.outputs.pair }}
|
||||
cache-to: type=gha,mode=max,scope=${{ matrix.service }}-${{ steps.prep.outputs.pair }}
|
||||
outputs: type=image,name=${{ env.IMAGE_PREFIX }}/${{ matrix.service }},push-by-digest=true,name-canonical=true,push=true
|
||||
${{ matrix.target.args }}
|
||||
cache-from: type=gha,scope=${{ steps.prep.outputs.key }}-${{ steps.prep.outputs.pair }}
|
||||
cache-to: type=gha,mode=max,scope=${{ steps.prep.outputs.key }}-${{ steps.prep.outputs.pair }}
|
||||
outputs: type=image,name=${{ env.IMAGE_PREFIX }}/${{ matrix.target.service }},push-by-digest=true,name-canonical=true,push=true
|
||||
|
||||
- name: Export digest
|
||||
run: |
|
||||
@@ -213,19 +249,19 @@ jobs:
|
||||
- name: Upload digest
|
||||
uses: actions/upload-artifact@v4
|
||||
with:
|
||||
name: digests-${{ matrix.service }}-${{ steps.prep.outputs.pair }}
|
||||
name: digests-${{ steps.prep.outputs.key }}-${{ steps.prep.outputs.pair }}
|
||||
path: /tmp/digests/*
|
||||
if-no-files-found: error
|
||||
retention-days: 1
|
||||
|
||||
merge-native:
|
||||
name: Merge ${{ matrix.service }} manifest
|
||||
name: Merge ${{ matrix.target.service }}${{ matrix.target.suffix }} manifest
|
||||
needs: [changes, build-native]
|
||||
if: needs.changes.outputs.native != '[]'
|
||||
strategy:
|
||||
fail-fast: false
|
||||
matrix:
|
||||
service: ${{ fromJSON(needs.changes.outputs.native) }}
|
||||
target: ${{ fromJSON(needs.changes.outputs.native) }}
|
||||
runs-on: ubuntu-latest
|
||||
permissions:
|
||||
contents: read
|
||||
@@ -235,7 +271,7 @@ jobs:
|
||||
uses: actions/download-artifact@v4
|
||||
with:
|
||||
path: /tmp/digests
|
||||
pattern: digests-${{ matrix.service }}-*
|
||||
pattern: digests-${{ matrix.target.service }}${{ matrix.target.suffix }}-linux-*
|
||||
merge-multiple: true
|
||||
|
||||
- name: Log in to GHCR
|
||||
@@ -252,9 +288,9 @@ jobs:
|
||||
working-directory: /tmp/digests
|
||||
run: |
|
||||
docker buildx imagetools create \
|
||||
-t ${{ env.IMAGE_PREFIX }}/${{ matrix.service }}:${{ github.sha }} \
|
||||
-t ${{ env.IMAGE_PREFIX }}/${{ matrix.service }}:dev \
|
||||
$(printf '${{ env.IMAGE_PREFIX }}/${{ matrix.service }}@sha256:%s ' *)
|
||||
-t ${{ env.IMAGE_PREFIX }}/${{ matrix.target.service }}:${{ github.sha }}${{ matrix.target.suffix }} \
|
||||
-t ${{ env.IMAGE_PREFIX }}/${{ matrix.target.service }}:dev${{ matrix.target.suffix }} \
|
||||
$(printf '${{ env.IMAGE_PREFIX }}/${{ matrix.target.service }}@sha256:%s ' *)
|
||||
|
||||
# A package on GHCR is created private and does not inherit the repository's
|
||||
# visibility, and no API can change that: an owner has to flip it by hand.
|
||||
|
||||
@@ -129,17 +129,28 @@ jobs:
|
||||
cache-from: type=gha,scope=${{ matrix.service }}
|
||||
cache-to: type=gha,mode=max,scope=${{ matrix.service }}
|
||||
|
||||
# Rust (tracking) and Elixir (realtime) have no cross-compiler; an emulated
|
||||
# arm64 build under QEMU runs for an hour or more. Build each arch on a
|
||||
# native runner and merge the digests into one manifest (the
|
||||
# Docker-documented multi-runner pattern).
|
||||
# Everything that cannot cross-compile: Rust (tracking), Elixir (realtime),
|
||||
# and the Kafka variants of the bus-facing Go services, which link librdkafka
|
||||
# through cgo. An emulated arm64 build under QEMU runs for an hour or more, so
|
||||
# build each arch on a native runner and merge the digests into one manifest
|
||||
# (the Docker-documented multi-runner pattern).
|
||||
#
|
||||
# The Kafka variants publish under the same image names with a -kafka tag
|
||||
# suffix, so an operator on Kafka pins ...:prod-kafka and gets the same
|
||||
# release built with EVENTBUS_PROVIDER=kafka available.
|
||||
build-native:
|
||||
name: Build ${{ matrix.service }} (${{ matrix.platform }})
|
||||
name: Build ${{ matrix.target.service }}${{ matrix.target.suffix }} (${{ matrix.platform }})
|
||||
needs: validate-tag
|
||||
strategy:
|
||||
fail-fast: false
|
||||
matrix:
|
||||
service: [tracking, realtime]
|
||||
target:
|
||||
- { service: tracking, suffix: "", args: "" }
|
||||
- { service: tracking, suffix: "-kafka", args: "CARGO_FEATURES=kafka" }
|
||||
- { service: realtime, suffix: "", args: "" }
|
||||
- { service: backend, suffix: "-kafka", args: "GO_TAGS=kafka" }
|
||||
- { service: consumer, suffix: "-kafka", args: "GO_TAGS=kafka" }
|
||||
- { service: worker, suffix: "-kafka", args: "GO_TAGS=kafka" }
|
||||
platform: [linux/amd64, linux/arm64]
|
||||
runs-on: ${{ matrix.platform == 'linux/arm64' && 'ubuntu-24.04-arm' || 'ubuntu-latest' }}
|
||||
permissions:
|
||||
@@ -148,11 +159,18 @@ jobs:
|
||||
steps:
|
||||
- uses: actions/checkout@v4
|
||||
|
||||
- name: Prepare platform pair
|
||||
# `key` is the per-variant identifier: the image name plus the tag suffix,
|
||||
# so a Kafka build never shares a cache scope or a digest artifact with
|
||||
# the plain build of the same service.
|
||||
- name: Prepare identifiers
|
||||
id: prep
|
||||
run: echo "pair=${PLATFORM//\//-}" >> "$GITHUB_OUTPUT"
|
||||
run: |
|
||||
echo "pair=${PLATFORM//\//-}" >> "$GITHUB_OUTPUT"
|
||||
echo "key=${SERVICE}${SUFFIX}" >> "$GITHUB_OUTPUT"
|
||||
env:
|
||||
PLATFORM: ${{ matrix.platform }}
|
||||
SERVICE: ${{ matrix.target.service }}
|
||||
SUFFIX: ${{ matrix.target.suffix }}
|
||||
|
||||
- name: Set up Docker Buildx
|
||||
uses: docker/setup-buildx-action@v3
|
||||
@@ -168,15 +186,16 @@ jobs:
|
||||
id: build
|
||||
uses: docker/build-push-action@v6
|
||||
with:
|
||||
context: ${{ matrix.service == 'tracking' && './tracking' || '.' }}
|
||||
file: ${{ matrix.service == 'tracking' && './tracking/Dockerfile' || format('deploy/docker/{0}.Dockerfile', matrix.service) }}
|
||||
context: ${{ matrix.target.service == 'tracking' && './tracking' || '.' }}
|
||||
file: ${{ matrix.target.service == 'tracking' && './tracking/Dockerfile' || format('deploy/docker/{0}.Dockerfile', matrix.target.service) }}
|
||||
platforms: ${{ matrix.platform }}
|
||||
build-args: |
|
||||
VERSION=${{ github.ref_name }}
|
||||
COMMIT=${{ github.sha }}
|
||||
cache-from: type=gha,scope=${{ matrix.service }}-${{ steps.prep.outputs.pair }}
|
||||
cache-to: type=gha,mode=max,scope=${{ matrix.service }}-${{ steps.prep.outputs.pair }}
|
||||
outputs: type=image,name=${{ env.IMAGE_PREFIX }}/${{ matrix.service }},push-by-digest=true,name-canonical=true,push=true
|
||||
${{ matrix.target.args }}
|
||||
cache-from: type=gha,scope=${{ steps.prep.outputs.key }}-${{ steps.prep.outputs.pair }}
|
||||
cache-to: type=gha,mode=max,scope=${{ steps.prep.outputs.key }}-${{ steps.prep.outputs.pair }}
|
||||
outputs: type=image,name=${{ env.IMAGE_PREFIX }}/${{ matrix.target.service }},push-by-digest=true,name-canonical=true,push=true
|
||||
|
||||
- name: Export digest
|
||||
run: |
|
||||
@@ -187,18 +206,24 @@ jobs:
|
||||
- name: Upload digest
|
||||
uses: actions/upload-artifact@v4
|
||||
with:
|
||||
name: digests-${{ matrix.service }}-${{ steps.prep.outputs.pair }}
|
||||
name: digests-${{ steps.prep.outputs.key }}-${{ steps.prep.outputs.pair }}
|
||||
path: /tmp/digests/*
|
||||
if-no-files-found: error
|
||||
retention-days: 1
|
||||
|
||||
merge-native:
|
||||
name: Merge ${{ matrix.service }} manifest
|
||||
name: Merge ${{ matrix.target.service }}${{ matrix.target.suffix }} manifest
|
||||
needs: [validate-tag, build-native]
|
||||
strategy:
|
||||
fail-fast: false
|
||||
matrix:
|
||||
service: [tracking, realtime]
|
||||
target:
|
||||
- { service: tracking, suffix: "" }
|
||||
- { service: tracking, suffix: "-kafka" }
|
||||
- { service: realtime, suffix: "" }
|
||||
- { service: backend, suffix: "-kafka" }
|
||||
- { service: consumer, suffix: "-kafka" }
|
||||
- { service: worker, suffix: "-kafka" }
|
||||
runs-on: ubuntu-latest
|
||||
permissions:
|
||||
contents: read
|
||||
@@ -208,7 +233,7 @@ jobs:
|
||||
uses: actions/download-artifact@v4
|
||||
with:
|
||||
path: /tmp/digests
|
||||
pattern: digests-${{ matrix.service }}-*
|
||||
pattern: digests-${{ matrix.target.service }}${{ matrix.target.suffix }}-linux-*
|
||||
merge-multiple: true
|
||||
|
||||
- name: Log in to GHCR
|
||||
@@ -225,11 +250,11 @@ jobs:
|
||||
working-directory: /tmp/digests
|
||||
run: |
|
||||
docker buildx imagetools create \
|
||||
-t ${{ env.IMAGE_PREFIX }}/${{ matrix.service }}:${{ github.ref_name }} \
|
||||
-t ${{ env.IMAGE_PREFIX }}/${{ matrix.service }}:v${{ needs.validate-tag.outputs.minor }} \
|
||||
-t ${{ env.IMAGE_PREFIX }}/${{ matrix.service }}:v${{ needs.validate-tag.outputs.major }} \
|
||||
-t ${{ env.IMAGE_PREFIX }}/${{ matrix.service }}:prod \
|
||||
$(printf '${{ env.IMAGE_PREFIX }}/${{ matrix.service }}@sha256:%s ' *)
|
||||
-t ${{ env.IMAGE_PREFIX }}/${{ matrix.target.service }}:${{ github.ref_name }}${{ matrix.target.suffix }} \
|
||||
-t ${{ env.IMAGE_PREFIX }}/${{ matrix.target.service }}:v${{ needs.validate-tag.outputs.minor }}${{ matrix.target.suffix }} \
|
||||
-t ${{ env.IMAGE_PREFIX }}/${{ matrix.target.service }}:v${{ needs.validate-tag.outputs.major }}${{ matrix.target.suffix }} \
|
||||
-t ${{ env.IMAGE_PREFIX }}/${{ matrix.target.service }}:prod${{ matrix.target.suffix }} \
|
||||
$(printf '${{ env.IMAGE_PREFIX }}/${{ matrix.target.service }}@sha256:%s ' *)
|
||||
|
||||
# The `warmbly` CLI is a plain static binary, so it cross-compiles for every
|
||||
# platform on one runner.
|
||||
|
||||
+3
-1
@@ -42,7 +42,9 @@ docker build -f web/Dockerfile -t warmbly/web web/
|
||||
docker build -f admin/Dockerfile -t warmbly/admin admin/
|
||||
```
|
||||
|
||||
The default builds have no Kafka/Avro support; add `--build-arg GO_TAGS=kafka` (Go images) or `--build-arg CARGO_FEATURES=kafka` (tracking) to opt in.
|
||||
The default builds have no Kafka/Avro support; add `--build-arg GO_TAGS=kafka` (Go images) or `--build-arg CARGO_FEATURES=kafka` (tracking) to opt in. Those builds link librdkafka through cgo, which cannot cross-compile, so each architecture has to be built on a machine of that architecture.
|
||||
|
||||
CI already publishes them, so you rarely need to: `backend`, `consumer`, `worker`, and `tracking` each get a second tag with a `-kafka` suffix (`ghcr.io/warmbly/warmbly/backend:prod-kafka`).
|
||||
|
||||
GitHub Actions publishes these to GHCR automatically. See [the self-hosting guide](https://docs.warmbly.com/development/deployment-guide/).
|
||||
|
||||
|
||||
@@ -60,8 +60,9 @@ REDIS=redis://localhost:6379
|
||||
# === Provider switches (no-cloud defaults) ===
|
||||
AWS_CONFIG_ENABLED=false # true => read secrets from AWS SSM/Secrets Manager
|
||||
|
||||
# Event bus. nats (default): one small JetStream binary. kafka: build the images
|
||||
# with GO_TAGS=kafka / CARGO_FEATURES=kafka and set KAFKA_* below.
|
||||
# Event bus. nats (default): one small JetStream binary. kafka: run the -kafka
|
||||
# images (ghcr.io/warmbly/warmbly/backend:prod-kafka and the same for consumer,
|
||||
# worker, tracking) and set KAFKA_* below.
|
||||
EVENTBUS_PROVIDER=nats
|
||||
NATS_URL=nats://localhost:4222
|
||||
# NATS_STREAM_NAME=warmbly
|
||||
|
||||
@@ -289,7 +289,7 @@ On `filesystem`, a remote worker writes blobs to its own disk rather than a volu
|
||||
|
||||
| Variable | What it does | Default | Restart needed |
|
||||
|---|---|---|---|
|
||||
| `EVENTBUS_PROVIDER` | `nats` or `kafka`. Kafka needs images built with `GO_TAGS=kafka` | `nats` under compose, `kafka` for a bare binary | yes |
|
||||
| `EVENTBUS_PROVIDER` | `nats` or `kafka`. Kafka needs the `-kafka` images, or a build with `GO_TAGS=kafka` | `nats` under compose, `kafka` for a bare binary | yes |
|
||||
| `NATS_URL` | JetStream address. Credentials in the URL are honored by every service, including the Rust tracking publisher: `nats://user:pass@host:4222` for a user, `nats://token@host:4222` for a token, `tls://` for TLS | `nats://nats:4222` | yes |
|
||||
| `NATS_CREDS` | Path to a NATS credentials file (user JWT plus nkey seed), for a bus that authenticates with JWT rather than a token. Synadia Cloud and any nsc-managed account issue one | unset | yes |
|
||||
| `NATS_CREDS_B64` | The same file, base64 encoded, as a single line. This is the form the fleet uses: a node receives environment variables rather than files, and the env file docker reads cannot express a multi-line value. Takes precedence over `NATS_CREDS` | unset | yes |
|
||||
@@ -566,7 +566,7 @@ The Rust open and click service. It reads its own environment, so these have to
|
||||
| `TRACKING_SCANNER_NETWORKS` | Extra scanner sources, comma separated, each a CIDR or `asn:<number>` with an optional label. For sources that never carry a person's own request, so their pixel fetches and their clicks are both treated as automated | empty |
|
||||
| `TRACKING_SCANNER_CLICK_NETWORKS` | The same, for sources that also proxy a mail client's own image fetches. Only their click tickets are treated as automated, because their pixel fetches are genuine opens | empty |
|
||||
| `TRACKING_SCANNER_ASN_HEADER` | Header a trusted proxy sets with the source ASN, which is what makes `asn:` entries match. Read only from a proxy in `TRACKING_TRUSTED_PROXIES`. On Cloudflare, a transform rule writing `ip.src.asnum`. Empty disables ASN matching | empty |
|
||||
| `EVENTBUS_PROVIDER` | `nats` or `kafka`. Kafka needs an image built with `CARGO_FEATURES=kafka` | `nats` |
|
||||
| `EVENTBUS_PROVIDER` | `nats` or `kafka`. Kafka needs the `tracking:*-kafka` image, or a build with `CARGO_FEATURES=kafka` | `nats` |
|
||||
| `NATS_URL`, `NATS_SUBJECT_PREFIX` | JetStream address and subject prefix. The publish subject is `<prefix>.<topic>` | `nats://localhost:4222`, `warmbly` |
|
||||
| `KAFKA_TRACKING_TOPIC` | Event topic, read by the Rust publisher **and** the Go subscriber | `tracking-events` |
|
||||
| `KAFKA_BOOTSTRAP_SERVERS`, `KAFKA_SASL_USERNAME`, `KAFKA_SASL_PASSWORD` | Broker transport when `EVENTBUS_PROVIDER=kafka` | unset |
|
||||
|
||||
@@ -329,7 +329,7 @@ DEPLOYMENT_MODE=self_hosted
|
||||
# AWS_SECRET_ACCESS_KEY=
|
||||
# KMS_PROVIDER=aws
|
||||
# KMS_AWS_KEY_ID=alias/warmbly
|
||||
# EVENTBUS_PROVIDER=kafka # needs images built with GO_TAGS=kafka
|
||||
# EVENTBUS_PROVIDER=kafka # needs the -kafka images (see below)
|
||||
# KAFKA_BOOTSTRAP_SERVERS=broker:9092
|
||||
# PRIMARY_DB=postgres://user:pass@host:5432/warmbly?sslmode=require
|
||||
# REDIS=redis://host:6379
|
||||
@@ -633,7 +633,16 @@ Pair it with `DISABLE_PASSWORD_LOGIN=true` for an SSO-only deployment.
|
||||
Three things to know:
|
||||
|
||||
- **Bare binaries default to the cloud values** (`kafka`, `avro`, `s3`, `aws`). Compose, the Makefile, and both env templates set the local values for you. A hand-rolled environment must set them or the process exits at boot.
|
||||
- Kafka and Avro are also build-time opt-ins: `--build-arg GO_TAGS=kafka` for Go, `--build-arg CARGO_FEATURES=kafka` for tracking.
|
||||
- Kafka and Avro are build-time opt-ins, so `EVENTBUS_PROVIDER=kafka` needs a build that has them. Each release publishes those builds as a second tag on the same image, suffixed `-kafka`, for the four services that touch the bus:
|
||||
|
||||
```
|
||||
ghcr.io/warmbly/warmbly/backend:prod-kafka
|
||||
ghcr.io/warmbly/warmbly/consumer:prod-kafka
|
||||
ghcr.io/warmbly/warmbly/worker:prod-kafka
|
||||
ghcr.io/warmbly/warmbly/tracking:prod-kafka
|
||||
```
|
||||
|
||||
`forms`, `updater`, `realtime`, `web`, and `admin` never read the bus and have one image each. To build your own instead, pass `--build-arg GO_TAGS=kafka` (Go) or `--build-arg CARGO_FEATURES=kafka` (tracking); cgo cannot cross-compile, so each architecture has to be built on a machine of that architecture.
|
||||
- `PUBSUB_ENABLED` must match across backend, consumer, and realtime.
|
||||
|
||||
## Optional subsystems
|
||||
|
||||
@@ -10,14 +10,14 @@ Warmbly's services communicate asynchronously through a pluggable event bus. The
|
||||
The bus is an abstraction (`internal/infrastructure/eventbus/`) with two providers, selected by `EVENTBUS_PROVIDER`:
|
||||
|
||||
- `nats`: NATS JetStream. One durable stream (default name `warmbly`) holds every topic as a subject under a prefix: topic `jobs.worker-events` becomes subject `warmbly.jobs.worker-events`. Messages are retained for 7 days, consumer groups map to JetStream durable consumers, and failed handlers are redelivered up to 10 times. This is what `docker-compose.yml` runs.
|
||||
- `kafka`: Apache Kafka behind the `kafka` Go build tag. The default binaries do not include it (no librdkafka, no CGO); build with `GO_TAGS=kafka` to enable it, otherwise `EVENTBUS_PROVIDER=kafka` fails at boot with a clear error.
|
||||
- `kafka`: Apache Kafka behind the `kafka` Go build tag. The default binaries do not include it (no librdkafka, no CGO), so `EVENTBUS_PROVIDER=kafka` on a default build fails at boot with a clear error. Every service that reads the bus publishes a second image for it, the same tag with a `-kafka` suffix: `ghcr.io/warmbly/warmbly/backend:prod-kafka`, and the same for `consumer`, `worker`, and `tracking`. Building your own needs `GO_TAGS=kafka` (Go) or `CARGO_FEATURES=kafka` (tracking).
|
||||
|
||||
Message encoding is orthogonal to transport, selected by `CODEC_PROVIDER`:
|
||||
|
||||
- `json`: plain JSON, no external dependencies. Required for the self-host stack (the worker command and result envelopes carry untyped bodies Avro cannot serialize).
|
||||
- `avro`: Avro with Confluent Schema Registry (`SCHEMA_REGISTRY_URL` plus optional key/secret), also behind the `kafka` build tag. The topic name is used as the Schema Registry subject.
|
||||
|
||||
The Rust tracking service follows the same switch: it publishes JSON to JetStream by default, and only speaks Kafka plus Avro when compiled with its `kafka` cargo feature.
|
||||
The Rust tracking service follows the same switch: it publishes JSON to JetStream by default, and only speaks Kafka plus Avro when compiled with its `kafka` cargo feature, which is what the `tracking:*-kafka` image is.
|
||||
|
||||
## Topics
|
||||
|
||||
|
||||
Reference in New Issue
Block a user