mirror of
https://github.com/stablyai/orca.git
synced 2026-09-21 16:02:20 +00:00
feat(cloud): native push gateway and dedicated infrastructure (1/3) (#19912)
* refactor(cloud): share PostgreSQL schema startup between services * feat(cloud): add durable native push notification gateway * infra(push): define dedicated gateway resources and operational checks * fix(push): bound cross-host admission and simplify gateway configuration * fix(push): validate deploy configuration and preserve topic-error registrations
This commit is contained in:
@@ -106,7 +106,7 @@ jobs:
|
||||
set -euo pipefail
|
||||
docker run --rm --network none --entrypoint node "${IMAGE}" --input-type=module -e '
|
||||
import { loadPushConfig } from "./apps/push/dist/config.js";
|
||||
const env = { ORCA_PUSH_PUBLIC_URL: "https://push.onorca.dev", ORCA_PUSH_MODE: "validation" };
|
||||
const env = { ORCA_PUSH_PUBLIC_URL: "https://push.onorca.dev", ORCA_PUSH_MODE: "validation", ORCA_PUSH_FCM_PROJECT_ID: "onorca-cloud" };
|
||||
if (loadPushConfig(env).mode !== "validation") throw new Error("validation_mode_unsupported");
|
||||
let rejected = false;
|
||||
try { loadPushConfig({ ...env, ORCA_PUSH_MODE: "invalid" }); } catch { rejected = true; }
|
||||
|
||||
@@ -90,6 +90,7 @@ jobs:
|
||||
--health-timeout 5s
|
||||
--health-retries 10
|
||||
env:
|
||||
ORCA_PUSH_TEST_DATABASE_URL: postgres://relay_test:relay_test@127.0.0.1:5432/orca_relay_test
|
||||
ORCA_RELAY_TEST_POSTGRES_URL: postgres://relay_test:relay_test@127.0.0.1:5432/orca_relay_test
|
||||
steps:
|
||||
- uses: actions/checkout@v4
|
||||
|
||||
+43
-7
@@ -24,6 +24,40 @@ the repository's root [MIT license](../LICENSE).
|
||||
- `apps/relay-ops`: the relay operations console and the incident monitor
|
||||
behind `pnpm ops:relay`, `pnpm incident:relay`, and
|
||||
`pnpm incident:relay-preflight`.
|
||||
- `apps/push` and `packages/push-contract`: the mobile push gateway that holds
|
||||
the APNs key and sends to phones through APNs and FCM, and its wire contract.
|
||||
It is deployed and operated from here but is not part of the relay data path;
|
||||
see [docs/push-gateway.md](docs/push-gateway.md).
|
||||
|
||||
## Mobile push gateway
|
||||
|
||||
`apps/push` is a separate Cloud Run service from the relay. Phones never hold an
|
||||
Orca credential for it: the desktop host authenticates with the same X25519
|
||||
key it uses for the relay, answering an encrypted challenge to mint a 24 hour
|
||||
session, then registers each paired phone's native push token and asks the
|
||||
gateway to push. The gateway queues each event as its own notification,
|
||||
enforces per-host quotas and request limits, and retires a
|
||||
registration as soon as Apple or Google reports the token unregistered.
|
||||
Provider push is the only ordinary mobile OS-banner path. The notification
|
||||
socket is retained only for live dismissal and reconnect tray reconciliation;
|
||||
it never creates or recovers banners. Desktop notification categories remain
|
||||
authoritative.
|
||||
Each delivery is persisted as one notification event. Before deploying an
|
||||
incompatible queue format, stop all older push gateway revisions and clear only
|
||||
unpublished push delivery fixtures; no queue preservation or migration is required.
|
||||
FCM notification messages are inherently collapsible while offline and have a
|
||||
small concurrent collapse-key budget, so every pending alert is not guaranteed.
|
||||
|
||||
Storage follows the relay pattern: PostgreSQL in production, SQLite for tests
|
||||
and local development. Configure it with `ORCA_PUSH_PUBLIC_URL`, `ORCA_PUSH_FCM_PROJECT_ID`,
|
||||
`ORCA_PUSH_DATABASE_URL`, the three APNs variables (`ORCA_PUSH_APNS_KEY`,
|
||||
`ORCA_PUSH_APNS_KEY_ID`, `ORCA_PUSH_APPLE_TEAM_ID`, all three or none), and
|
||||
optionally `ORCA_PUSH_APNS_TOPIC`. The FCM credential comes from
|
||||
the runtime service account, so no key material is configured for Android. See
|
||||
[push gateway operations](docs/push-gateway.md) for deployment and recovery.
|
||||
|
||||
Logging is aggregate counters only. Tokens, notification titles, notification
|
||||
bodies, and full host fingerprints never reach a log line.
|
||||
|
||||
## Infrastructure and operations
|
||||
|
||||
@@ -38,16 +72,18 @@ the repository's root [MIT license](../LICENSE).
|
||||
- `dev/contracts` and `dev/fixtures`: the checked-in data those contract tests
|
||||
read, including the Terraform root partition.
|
||||
- `docs/`: the relay runbooks, capacity-testing guide, incident-monitor
|
||||
reference, and the workflow variable reference in `docs/relay-workflows.md`.
|
||||
reference, the workflow variable reference in `docs/relay-workflows.md`, and
|
||||
the push gateway runbook in `docs/push-gateway.md`.
|
||||
|
||||
## Workflows
|
||||
|
||||
The 24 `.github/workflows/cloud-*.yml` workflows are the relay's deploy and
|
||||
operate surface: publish and deploy the director, roll GCE cell capacity,
|
||||
operate Asia admission and regional rehoming, prove staging capacity, monitor
|
||||
production, and power staging up and down. `.github/actions/cloud-sql-rollout-lease`
|
||||
is the compare-and-swap lease that serializes every rollout against the shared
|
||||
Cloud SQL instance.
|
||||
The 25 `.github/workflows/cloud-*.yml` workflows are the deploy and operate
|
||||
surface: publish and deploy the director, roll GCE cell capacity, operate Asia
|
||||
admission and regional rehoming, prove staging capacity, monitor production,
|
||||
power staging up and down, and deploy the mobile push gateway.
|
||||
`.github/actions/cloud-sql-rollout-lease` is the compare-and-swap lease that
|
||||
serializes rollouts against the shared Cloud SQL instance. Push reuses that
|
||||
action with its own lease object and deployment concurrency group.
|
||||
|
||||
Every one of them is inert. Each top-level job is gated on
|
||||
`vars.ORCA_CLOUD_OPERATIONS_ENABLED == 'true'`, a repository variable that is
|
||||
|
||||
@@ -0,0 +1,29 @@
|
||||
FROM node:24-alpine AS build
|
||||
WORKDIR /app
|
||||
RUN corepack enable
|
||||
COPY package.json pnpm-lock.yaml pnpm-workspace.yaml tsconfig.base.json ./
|
||||
COPY packages/push-contract/package.json packages/push-contract/package.json
|
||||
COPY packages/postgres-schema/package.json packages/postgres-schema/package.json
|
||||
COPY apps/push/package.json apps/push/package.json
|
||||
RUN pnpm install --frozen-lockfile
|
||||
COPY packages/push-contract packages/push-contract
|
||||
COPY apps/push apps/push
|
||||
COPY packages/postgres-schema packages/postgres-schema
|
||||
RUN pnpm --filter @orca-cloud/postgres-schema build && pnpm --filter @orca-cloud/push-contract build && pnpm --filter @orca-cloud/push build
|
||||
|
||||
FROM node:24-alpine AS runtime
|
||||
ENV NODE_ENV=production
|
||||
ENV PORT=8080
|
||||
WORKDIR /app
|
||||
RUN corepack enable
|
||||
COPY package.json pnpm-lock.yaml pnpm-workspace.yaml ./
|
||||
COPY packages/push-contract/package.json packages/push-contract/package.json
|
||||
COPY packages/postgres-schema/package.json packages/postgres-schema/package.json
|
||||
COPY apps/push/package.json apps/push/package.json
|
||||
COPY --from=build /app/packages/push-contract/dist packages/push-contract/dist
|
||||
COPY --from=build /app/packages/postgres-schema/dist packages/postgres-schema/dist
|
||||
COPY --from=build /app/apps/push/dist apps/push/dist
|
||||
RUN pnpm install --prod --frozen-lockfile --filter @orca-cloud/push...
|
||||
USER node
|
||||
EXPOSE 8080
|
||||
CMD ["node", "apps/push/dist/index.js"]
|
||||
@@ -0,0 +1,35 @@
|
||||
{
|
||||
"name": "@orca-cloud/push",
|
||||
"private": true,
|
||||
"version": "0.0.0",
|
||||
"type": "module",
|
||||
"main": "dist/index.js",
|
||||
"scripts": {
|
||||
"build": "pnpm clean && tsc -p tsconfig.build.json",
|
||||
"clean": "node -e \"require('fs').rmSync('dist', { recursive: true, force: true })\"",
|
||||
"dev": "tsx watch src/index.ts",
|
||||
"lint": "tsc -p tsconfig.json --noEmit",
|
||||
"pretest": "pnpm --filter @orca-cloud/postgres-schema build && pnpm --filter @orca-cloud/push-contract build",
|
||||
"start": "node dist/index.js",
|
||||
"test": "vitest run",
|
||||
"typecheck": "tsc -p tsconfig.json --noEmit"
|
||||
},
|
||||
"dependencies": {
|
||||
"@hono/node-server": "^1.19.17",
|
||||
"@orca-cloud/postgres-schema": "workspace:*",
|
||||
"@orca-cloud/push-contract": "workspace:*",
|
||||
"google-auth-library": "^10.5.0",
|
||||
"hono": "^4.13.7",
|
||||
"pg": "^8.22.0",
|
||||
"pg-connection-string": "2.14.0",
|
||||
"tweetnacl": "^1.0.3",
|
||||
"zod": "^3.25.76"
|
||||
},
|
||||
"devDependencies": {
|
||||
"@types/node": "^24.10.0",
|
||||
"@types/pg": "^8.20.0",
|
||||
"tsx": "^4.21.0",
|
||||
"typescript": "^5.9.3",
|
||||
"vitest": "^4.0.8"
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,42 @@
|
||||
import { createPrivateKey, type KeyObject, sign } from 'node:crypto'
|
||||
import type { ApnsCredentials } from './config.js'
|
||||
|
||||
// Apple rejects a provider token older than an hour and throttles reissue
|
||||
// under about 20 minutes, so 50 minutes is the safe rotation point.
|
||||
export const APNS_TOKEN_ROTATION_MS = 50 * 60 * 1000
|
||||
|
||||
function base64UrlJson(value: Record<string, unknown>): string {
|
||||
return Buffer.from(JSON.stringify(value), 'utf8').toString('base64url')
|
||||
}
|
||||
|
||||
export class ApnsAuthenticationToken {
|
||||
private readonly privateKey: KeyObject
|
||||
private cached: { token: string; issuedAtMs: number } | null = null
|
||||
|
||||
constructor(
|
||||
private readonly credentials: ApnsCredentials,
|
||||
private readonly now: () => number = Date.now,
|
||||
private readonly rotationMs: number = APNS_TOKEN_ROTATION_MS
|
||||
) {
|
||||
this.privateKey = createPrivateKey(credentials.keyPem)
|
||||
}
|
||||
|
||||
value(): string {
|
||||
const nowMs = this.now()
|
||||
if (this.cached && nowMs - this.cached.issuedAtMs < this.rotationMs) return this.cached.token
|
||||
const header = base64UrlJson({ alg: 'ES256', kid: this.credentials.keyId })
|
||||
const payload = base64UrlJson({
|
||||
iss: this.credentials.teamId,
|
||||
iat: Math.floor(nowMs / 1000)
|
||||
})
|
||||
const signingInput = `${header}.${payload}`
|
||||
// ES256 requires the raw r||s pair; Node emits DER unless asked otherwise.
|
||||
const signature = sign('sha256', Buffer.from(signingInput, 'utf8'), {
|
||||
key: this.privateKey,
|
||||
dsaEncoding: 'ieee-p1363'
|
||||
}).toString('base64url')
|
||||
const token = `${signingInput}.${signature}`
|
||||
this.cached = { token, issuedAtMs: nowMs }
|
||||
return token
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,220 @@
|
||||
import { generateKeyPairSync } from 'node:crypto'
|
||||
import { describe, expect, it } from 'vitest'
|
||||
import { ApnsAuthenticationToken, APNS_TOKEN_ROTATION_MS } from './apns-authentication-token.js'
|
||||
import { ApnsClient } from './apns-client.js'
|
||||
import type { ApnsRequest, ApnsResponse } from './apns-http2-transport.js'
|
||||
import type { ApnsCredentials } from './config.js'
|
||||
import { buildPushDelivery } from './push-delivery-message.js'
|
||||
|
||||
const HOST = 'abcdefghijklmnop'
|
||||
|
||||
function credentials(): ApnsCredentials {
|
||||
const { privateKey } = generateKeyPairSync('ec', {
|
||||
namedCurve: 'P-256',
|
||||
privateKeyEncoding: { type: 'pkcs8', format: 'pem' },
|
||||
publicKeyEncoding: { type: 'spki', format: 'pem' }
|
||||
})
|
||||
return { keyPem: privateKey, keyId: 'ABCDE12345', teamId: 'TEAM123456' }
|
||||
}
|
||||
|
||||
function delivery(now = Date.now()) {
|
||||
return buildPushDelivery({
|
||||
expiresAt: now + 300_000,
|
||||
registrationId: 'reg-1',
|
||||
hostFingerprint: HOST,
|
||||
notification: {
|
||||
notificationId: 'note-1',
|
||||
notificationSeq: 7,
|
||||
notificationEpoch: 'epoch-1',
|
||||
source: 'agent-task-complete',
|
||||
agentState: 'needs-input',
|
||||
title: 'Agent needs input',
|
||||
body: 'Waiting on your answer',
|
||||
worktreeId: 'wt-1'
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
function fakeTransport(response: ApnsResponse) {
|
||||
const requests: ApnsRequest[] = []
|
||||
return {
|
||||
requests,
|
||||
transport: async (request: ApnsRequest): Promise<ApnsResponse> => {
|
||||
requests.push(request)
|
||||
return response
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
describe('apns authentication token', () => {
|
||||
it('signs an ES256 provider token and caches it until the rotation point', () => {
|
||||
let clock = 1_700_000_000_000
|
||||
const authentication = new ApnsAuthenticationToken(credentials(), () => clock)
|
||||
const first = authentication.value()
|
||||
const [header, payload, signature] = first.split('.')
|
||||
expect(JSON.parse(Buffer.from(header!, 'base64url').toString('utf8'))).toEqual({
|
||||
alg: 'ES256',
|
||||
kid: 'ABCDE12345'
|
||||
})
|
||||
expect(JSON.parse(Buffer.from(payload!, 'base64url').toString('utf8'))).toEqual({
|
||||
iss: 'TEAM123456',
|
||||
iat: Math.floor(clock / 1000)
|
||||
})
|
||||
expect(Buffer.from(signature!, 'base64url').byteLength).toBe(64)
|
||||
|
||||
clock += APNS_TOKEN_ROTATION_MS - 1
|
||||
expect(authentication.value()).toBe(first)
|
||||
clock += 1
|
||||
expect(authentication.value()).not.toBe(first)
|
||||
})
|
||||
})
|
||||
|
||||
describe('apns client', () => {
|
||||
it('sends the specified headers, path, and alert body', async () => {
|
||||
const clock = 1_700_000_000_000
|
||||
const fake = fakeTransport({ status: 200, body: '' })
|
||||
const client = new ApnsClient({
|
||||
topic: 'com.stably.orca.mobile',
|
||||
credentials: credentials(),
|
||||
transport: fake.transport,
|
||||
now: () => clock
|
||||
})
|
||||
await expect(
|
||||
client.send(delivery(clock), { token: 'a'.repeat(64), apnsEnvironment: 'production' })
|
||||
).resolves.toEqual({ status: 'sent' })
|
||||
const request = fake.requests[0]!
|
||||
expect(request.host).toBe('api.push.apple.com')
|
||||
expect(request.path).toBe(`/3/device/${'a'.repeat(64)}`)
|
||||
expect(request.headers).toMatchObject({
|
||||
'apns-topic': 'com.stably.orca.mobile',
|
||||
'apns-push-type': 'alert',
|
||||
'apns-priority': '10',
|
||||
'apns-expiration': String(Math.floor(clock / 1000) + 5 * 60),
|
||||
'apns-collapse-id': expect.stringMatching(/^[a-f0-9]{64}$/)
|
||||
})
|
||||
expect(request.headers.authorization).toMatch(/^bearer /)
|
||||
expect(JSON.parse(request.body)).toEqual({
|
||||
aps: {
|
||||
alert: { title: 'Agent needs input', body: 'Waiting on your answer' },
|
||||
sound: 'default',
|
||||
'thread-id': HOST
|
||||
},
|
||||
orca: {
|
||||
hostFingerprint: HOST,
|
||||
worktreeId: 'wt-1',
|
||||
notificationId: 'note-1',
|
||||
notificationSeq: 7,
|
||||
notificationEpoch: 'epoch-1',
|
||||
source: 'agent-task-complete',
|
||||
agentState: 'needs-input'
|
||||
}
|
||||
})
|
||||
})
|
||||
|
||||
it('targets the sandbox host and keeps the individual collapse id', async () => {
|
||||
const fake = fakeTransport({ status: 200, body: '' })
|
||||
const client = new ApnsClient({
|
||||
topic: 'com.stably.orca.mobile',
|
||||
credentials: credentials(),
|
||||
transport: fake.transport
|
||||
})
|
||||
await client.send(delivery(), { token: 'b'.repeat(64), apnsEnvironment: 'sandbox' })
|
||||
expect(fake.requests[0]?.host).toBe('api.sandbox.push.apple.com')
|
||||
expect(fake.requests[0]?.headers['apns-collapse-id']).toMatch(/^[a-f0-9]{64}$/)
|
||||
})
|
||||
|
||||
it.each([
|
||||
[410, 'Unregistered'],
|
||||
[400, 'BadDeviceToken'],
|
||||
[400, 'Unregistered']
|
||||
])('classifies %i %s as a dead token', async (status, reason) => {
|
||||
const fake = fakeTransport({ status, body: JSON.stringify({ reason }) })
|
||||
const client = new ApnsClient({
|
||||
topic: 'com.stably.orca.mobile',
|
||||
credentials: credentials(),
|
||||
transport: fake.transport
|
||||
})
|
||||
await expect(
|
||||
client.send(delivery(), { token: 'a'.repeat(64), apnsEnvironment: 'production' })
|
||||
).resolves.toEqual({ status: 'dead', reason })
|
||||
})
|
||||
|
||||
it.each([
|
||||
[400, 'PayloadTooLarge'],
|
||||
[400, 'DeviceTokenNotForTopic'],
|
||||
[429, 'TooManyRequests'],
|
||||
[500, 'InternalServerError']
|
||||
])('treats %i %s with the appropriate retry policy', async (status, reason) => {
|
||||
const fake = fakeTransport({ status, body: JSON.stringify({ reason }) })
|
||||
const client = new ApnsClient({
|
||||
topic: 'com.stably.orca.mobile',
|
||||
credentials: credentials(),
|
||||
transport: fake.transport
|
||||
})
|
||||
await expect(
|
||||
client.send(delivery(), { token: 'a'.repeat(64), apnsEnvironment: 'production' })
|
||||
).resolves.toEqual({ status: 'error', reason, retryable: status === 429 || status >= 500 })
|
||||
})
|
||||
|
||||
it('reports a transport failure as an error rather than throwing', async () => {
|
||||
const client = new ApnsClient({
|
||||
topic: 'com.stably.orca.mobile',
|
||||
credentials: credentials(),
|
||||
transport: async () => {
|
||||
throw new Error('socket hang up')
|
||||
}
|
||||
})
|
||||
await expect(
|
||||
client.send(delivery(), { token: 'a'.repeat(64), apnsEnvironment: 'production' })
|
||||
).resolves.toEqual({ status: 'error', reason: 'Error', retryable: true })
|
||||
})
|
||||
})
|
||||
|
||||
it('does not collapse background dismissals with visible alerts', async () => {
|
||||
const fake = fakeTransport({ status: 200, body: '' })
|
||||
const apns = new ApnsClient({
|
||||
topic: 'test',
|
||||
credentials: credentials(),
|
||||
transport: fake.transport
|
||||
})
|
||||
const alert = delivery()
|
||||
await apns.send(
|
||||
{ ...alert, orca: { ...alert.orca, kind: 'dismiss' } },
|
||||
{
|
||||
token: 'test',
|
||||
apnsEnvironment: 'sandbox'
|
||||
}
|
||||
)
|
||||
expect(fake.requests[0]?.headers).not.toHaveProperty('apns-collapse-id')
|
||||
expect(fake.requests[0]?.headers).toMatchObject({
|
||||
'apns-push-type': 'background',
|
||||
'apns-priority': '5'
|
||||
})
|
||||
expect(JSON.parse(fake.requests[0]!.body).aps).toEqual({ 'content-available': 1 })
|
||||
})
|
||||
|
||||
it('keeps the absolute deadline across retries and refuses expired delivery', async () => {
|
||||
let now = 1_700_000_000_000
|
||||
const fake = fakeTransport({ status: 503, body: '{}' })
|
||||
const client = new ApnsClient({
|
||||
topic: 'test',
|
||||
credentials: credentials(),
|
||||
now: () => now,
|
||||
transport: fake.transport
|
||||
})
|
||||
const pending = delivery(now)
|
||||
const device = { token: 'test', apnsEnvironment: 'sandbox' as const }
|
||||
await client.send(pending, device)
|
||||
now += 60_000
|
||||
await client.send(pending, device)
|
||||
expect(fake.requests.map((request) => request.headers['apns-expiration'])).toEqual([
|
||||
String(pending.expiresAt / 1000),
|
||||
String(pending.expiresAt / 1000)
|
||||
])
|
||||
now = pending.expiresAt
|
||||
await expect(client.send(pending, device)).resolves.toEqual({
|
||||
status: 'error',
|
||||
reason: 'expired'
|
||||
})
|
||||
expect(fake.requests).toHaveLength(2)
|
||||
})
|
||||
@@ -0,0 +1,95 @@
|
||||
import type { ApnsEnvironment } from '@orca-cloud/push-contract'
|
||||
import { ApnsAuthenticationToken } from './apns-authentication-token.js'
|
||||
import type { ApnsTransport } from './apns-http2-transport.js'
|
||||
import type { ApnsCredentials } from './config.js'
|
||||
import type { PushDelivery } from './push-delivery-message.js'
|
||||
import type { PushProviderOutcome } from './push-provider-outcome.js'
|
||||
|
||||
const APNS_HOSTS: Record<ApnsEnvironment, string> = {
|
||||
production: 'api.push.apple.com',
|
||||
sandbox: 'api.sandbox.push.apple.com'
|
||||
}
|
||||
|
||||
const DEAD_TOKEN_REASONS = new Set(['BadDeviceToken', 'Unregistered'])
|
||||
|
||||
export type ApnsClientOptions = {
|
||||
topic: string
|
||||
credentials: ApnsCredentials
|
||||
transport: ApnsTransport
|
||||
now?: () => number
|
||||
}
|
||||
|
||||
function readReason(body: string): string {
|
||||
try {
|
||||
const parsed = JSON.parse(body) as { reason?: unknown }
|
||||
return typeof parsed.reason === 'string' ? parsed.reason : 'unknown'
|
||||
} catch {
|
||||
return 'unparseable'
|
||||
}
|
||||
}
|
||||
|
||||
export function apnsBody(delivery: PushDelivery): string {
|
||||
return JSON.stringify({
|
||||
aps:
|
||||
delivery.orca.kind === 'dismiss'
|
||||
? { 'content-available': 1 }
|
||||
: {
|
||||
alert: { title: delivery.title, body: delivery.body },
|
||||
...(delivery.sound === false ? {} : { sound: 'default' }),
|
||||
'thread-id': delivery.hostFingerprint
|
||||
},
|
||||
orca: delivery.orca
|
||||
})
|
||||
}
|
||||
|
||||
export class ApnsClient {
|
||||
private readonly authentication: ApnsAuthenticationToken
|
||||
private readonly now: () => number
|
||||
|
||||
constructor(private readonly options: ApnsClientOptions) {
|
||||
this.now = options.now ?? Date.now
|
||||
this.authentication = new ApnsAuthenticationToken(options.credentials, this.now)
|
||||
}
|
||||
|
||||
async send(
|
||||
delivery: PushDelivery,
|
||||
device: { token: string; apnsEnvironment: ApnsEnvironment }
|
||||
): Promise<PushProviderOutcome> {
|
||||
const expiration = Math.floor(delivery.expiresAt / 1000)
|
||||
if (expiration * 1000 <= this.now()) return { status: 'error', reason: 'expired' }
|
||||
let response
|
||||
try {
|
||||
response = await this.options.transport({
|
||||
host: APNS_HOSTS[device.apnsEnvironment],
|
||||
path: `/3/device/${device.token}`,
|
||||
headers: {
|
||||
authorization: `bearer ${this.authentication.value()}`,
|
||||
'apns-topic': this.options.topic,
|
||||
'apns-push-type': delivery.orca.kind === 'dismiss' ? 'background' : 'alert',
|
||||
'apns-priority': delivery.orca.kind === 'dismiss' ? '5' : '10',
|
||||
'apns-expiration': String(expiration),
|
||||
...(delivery.orca.kind === 'dismiss' ? {} : { 'apns-collapse-id': delivery.collapseId })
|
||||
},
|
||||
body: apnsBody(delivery)
|
||||
})
|
||||
} catch (error) {
|
||||
return {
|
||||
status: 'error',
|
||||
reason: error instanceof Error ? error.name : 'transport_failed',
|
||||
retryable: true
|
||||
}
|
||||
}
|
||||
if (response.status === 200) return { status: 'sent' }
|
||||
const reason = readReason(response.body)
|
||||
if (response.status === 410) return { status: 'dead', reason }
|
||||
if (response.status === 400 && DEAD_TOKEN_REASONS.has(reason)) {
|
||||
return { status: 'dead', reason }
|
||||
}
|
||||
return {
|
||||
status: 'error',
|
||||
reason,
|
||||
retryable: response.status === 429 || response.status >= 500,
|
||||
...(response.retryAfterMs === undefined ? {} : { retryAfterMs: response.retryAfterMs })
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,50 @@
|
||||
import { connect, constants, type ClientHttp2Session } from 'node:http2'
|
||||
import { readApnsStreamResponse, type ApnsResponse } from './apns-stream-response.js'
|
||||
|
||||
export type ApnsRequest = {
|
||||
host: string
|
||||
path: string
|
||||
headers: Record<string, string>
|
||||
body: string
|
||||
}
|
||||
|
||||
export type { ApnsResponse }
|
||||
export type ApnsTransport = (request: ApnsRequest) => Promise<ApnsResponse>
|
||||
|
||||
// APNs requires HTTP/2 and rewards a long-lived session per host, so sessions
|
||||
// are cached and only dropped when the socket itself goes away.
|
||||
export function createApnsHttp2Transport(): ApnsTransport & { close(): void } {
|
||||
const sessions = new Map<string, ClientHttp2Session>()
|
||||
|
||||
const sessionFor = (host: string): ClientHttp2Session => {
|
||||
const existing = sessions.get(host)
|
||||
if (existing && !existing.closed && !existing.destroyed) return existing
|
||||
const session = connect(`https://${host}`)
|
||||
const forget = (): void => {
|
||||
if (sessions.get(host) === session) sessions.delete(host)
|
||||
}
|
||||
session.on('error', forget)
|
||||
session.on('close', forget)
|
||||
sessions.set(host, session)
|
||||
return session
|
||||
}
|
||||
|
||||
const transport = async (request: ApnsRequest): Promise<ApnsResponse> => {
|
||||
const stream = sessionFor(request.host).request({
|
||||
...request.headers,
|
||||
[constants.HTTP2_HEADER_METHOD]: 'POST',
|
||||
[constants.HTTP2_HEADER_PATH]: request.path,
|
||||
[constants.HTTP2_HEADER_AUTHORITY]: request.host,
|
||||
'content-type': 'application/json',
|
||||
'content-length': String(Buffer.byteLength(request.body))
|
||||
})
|
||||
return await readApnsStreamResponse(stream, request.body)
|
||||
}
|
||||
|
||||
return Object.assign(transport, {
|
||||
close(): void {
|
||||
for (const session of sessions.values()) session.close()
|
||||
sessions.clear()
|
||||
}
|
||||
})
|
||||
}
|
||||
@@ -0,0 +1,45 @@
|
||||
import { EventEmitter } from 'node:events'
|
||||
import { expect, it, vi } from 'vitest'
|
||||
const mocks = vi.hoisted(() => ({
|
||||
connect: vi.fn(),
|
||||
read: vi.fn(async () => ({ status: 200, body: '' }))
|
||||
}))
|
||||
vi.mock('node:http2', async (original) => ({
|
||||
...(await original<typeof import('node:http2')>()),
|
||||
connect: mocks.connect
|
||||
}))
|
||||
vi.mock('./apns-stream-response.js', () => ({ readApnsStreamResponse: mocks.read }))
|
||||
import { createApnsHttp2Transport } from './apns-http2-transport.js'
|
||||
|
||||
it('keeps the replacement cached when the draining session closes later', async () => {
|
||||
const sessions: Array<
|
||||
EventEmitter & {
|
||||
closed: boolean
|
||||
destroyed: boolean
|
||||
request: ReturnType<typeof vi.fn>
|
||||
close: ReturnType<typeof vi.fn>
|
||||
}
|
||||
> = []
|
||||
mocks.connect.mockImplementation(() => {
|
||||
const session = Object.assign(new EventEmitter(), {
|
||||
closed: false,
|
||||
destroyed: false,
|
||||
request: vi.fn(() => ({})),
|
||||
close: vi.fn()
|
||||
})
|
||||
sessions.push(session)
|
||||
return session
|
||||
})
|
||||
const transport = createApnsHttp2Transport()
|
||||
const request = { host: 'api.push.apple.com', path: '/synthetic', headers: {}, body: '{}' }
|
||||
await transport(request)
|
||||
sessions[0]!.closed = true
|
||||
await transport(request)
|
||||
sessions[0]!.emit('close')
|
||||
sessions[0]!.emit('error', new Error('old-session'))
|
||||
await transport(request)
|
||||
expect(sessions).toHaveLength(2)
|
||||
expect(sessions[1]!.request).toHaveBeenCalledTimes(2)
|
||||
transport.close()
|
||||
expect(sessions[1]!.close).toHaveBeenCalledOnce()
|
||||
})
|
||||
@@ -0,0 +1,82 @@
|
||||
import { EventEmitter } from 'node:events'
|
||||
import { describe, expect, it } from 'vitest'
|
||||
import { readApnsStreamResponse, type ApnsResponseStream } from './apns-stream-response.js'
|
||||
|
||||
type FakeStream = ApnsResponseStream & {
|
||||
sentBody: string | null
|
||||
destroyedWith: Error | null
|
||||
fireTimeout(): void
|
||||
}
|
||||
|
||||
function fakeApnsStream(): FakeStream {
|
||||
const emitter = new EventEmitter() as FakeStream
|
||||
emitter.sentBody = null
|
||||
emitter.destroyedWith = null
|
||||
let onTimeout: (() => void) | null = null
|
||||
emitter.setTimeout = (_ms, callback) => {
|
||||
onTimeout = callback
|
||||
}
|
||||
emitter.destroy = (error?: Error) => {
|
||||
emitter.destroyedWith = error ?? null
|
||||
if (error) emitter.emit('error', error)
|
||||
}
|
||||
emitter.end = (body: string) => {
|
||||
emitter.sentBody = body
|
||||
}
|
||||
emitter.fireTimeout = () => onTimeout?.()
|
||||
return emitter
|
||||
}
|
||||
|
||||
describe('apns stream response', () => {
|
||||
it('resolves with the status and the concatenated body', async () => {
|
||||
const stream = fakeApnsStream()
|
||||
const pending = readApnsStreamResponse(stream, '{"aps":{}}')
|
||||
expect(stream.sentBody).toBe('{"aps":{}}')
|
||||
stream.emit('response', { ':status': '200' })
|
||||
stream.emit('data', Buffer.from('{"re'))
|
||||
stream.emit('data', Buffer.from('ason":"ok"}'))
|
||||
stream.emit('end')
|
||||
await expect(pending).resolves.toEqual({ status: 200, body: '{"reason":"ok"}' })
|
||||
})
|
||||
|
||||
it('rejects when the peer resets the stream without an end or an error', async () => {
|
||||
const stream = fakeApnsStream()
|
||||
const pending = readApnsStreamResponse(stream, 'body')
|
||||
stream.emit('response', { ':status': '200' })
|
||||
// NGHTTP2_NO_ERROR: node emits only 'close', so nothing else would settle.
|
||||
stream.emit('close')
|
||||
await expect(pending).rejects.toThrow('apns_stream_closed')
|
||||
})
|
||||
|
||||
it('keeps the resolved response when close follows a completed end', async () => {
|
||||
const stream = fakeApnsStream()
|
||||
const pending = readApnsStreamResponse(stream, 'body')
|
||||
stream.emit('response', { ':status': '410' })
|
||||
stream.emit('end')
|
||||
stream.emit('close')
|
||||
await expect(pending).resolves.toEqual({ status: 410, body: '' })
|
||||
})
|
||||
|
||||
it('keeps the original error when close follows a stream error', async () => {
|
||||
const stream = fakeApnsStream()
|
||||
const pending = readApnsStreamResponse(stream, 'body')
|
||||
stream.emit('error', new Error('socket_hang_up'))
|
||||
stream.emit('close')
|
||||
await expect(pending).rejects.toThrow('socket_hang_up')
|
||||
})
|
||||
|
||||
it('destroys the stream on timeout and surfaces the timeout error', async () => {
|
||||
const stream = fakeApnsStream()
|
||||
const pending = readApnsStreamResponse(stream, 'body', 10)
|
||||
stream.fireTimeout()
|
||||
await expect(pending).rejects.toThrow('apns_timeout')
|
||||
expect(stream.destroyedWith?.message).toBe('apns_timeout')
|
||||
})
|
||||
|
||||
it('reports a missing status header as zero rather than NaN', async () => {
|
||||
const stream = fakeApnsStream()
|
||||
const pending = readApnsStreamResponse(stream, 'body')
|
||||
stream.emit('end')
|
||||
await expect(pending).resolves.toEqual({ status: 0, body: '' })
|
||||
})
|
||||
})
|
||||
@@ -0,0 +1,53 @@
|
||||
import type { EventEmitter } from 'node:events'
|
||||
import { providerRetryAfter } from './provider-retry-delay.js'
|
||||
import { constants } from 'node:http2'
|
||||
|
||||
export type ApnsResponse = { status: number; body: string; retryAfterMs?: number }
|
||||
|
||||
// The subset of ClientHttp2Stream this module drives, so a fake emitter can
|
||||
// stand in for a real APNs stream in tests.
|
||||
export type ApnsResponseStream = EventEmitter & {
|
||||
setTimeout(ms: number, callback: () => void): void
|
||||
destroy(error?: Error): void
|
||||
end(body: string): void
|
||||
}
|
||||
|
||||
export const APNS_REQUEST_TIMEOUT_MS = 10_000
|
||||
|
||||
export function readApnsStreamResponse(
|
||||
stream: ApnsResponseStream,
|
||||
body: string,
|
||||
timeoutMs = APNS_REQUEST_TIMEOUT_MS
|
||||
): Promise<ApnsResponse> {
|
||||
return new Promise<ApnsResponse>((resolve, reject) => {
|
||||
let settled = false
|
||||
const settle = (run: () => void): void => {
|
||||
if (settled) return
|
||||
settled = true
|
||||
run()
|
||||
}
|
||||
let status = 0
|
||||
let retryAfterMs: number | undefined
|
||||
const chunks: Buffer[] = []
|
||||
stream.setTimeout(timeoutMs, () => stream.destroy(new Error('apns_timeout')))
|
||||
stream.on('response', (headers: Record<string, unknown>) => {
|
||||
status = Number(headers[constants.HTTP2_HEADER_STATUS] ?? 0)
|
||||
retryAfterMs = providerRetryAfter(String(headers['retry-after'] ?? ''))
|
||||
})
|
||||
stream.on('data', (chunk: Buffer) => chunks.push(chunk))
|
||||
stream.on('error', (error: Error) => settle(() => reject(error)))
|
||||
stream.on('end', () =>
|
||||
settle(() =>
|
||||
resolve({
|
||||
status,
|
||||
body: Buffer.concat(chunks).toString('utf8'),
|
||||
...(retryAfterMs === undefined ? {} : { retryAfterMs })
|
||||
})
|
||||
)
|
||||
)
|
||||
// A peer reset with NGHTTP2_NO_ERROR emits neither 'end' nor 'error', which
|
||||
// would leave the worker's delivery pending for the life of the process.
|
||||
stream.on('close', () => settle(() => reject(new Error('apns_stream_closed'))))
|
||||
stream.end(body)
|
||||
})
|
||||
}
|
||||
@@ -0,0 +1,48 @@
|
||||
import { expect, it } from 'vitest'
|
||||
import { createPushHostKeypair } from './host-challenge-answering.test-fixture.js'
|
||||
import {
|
||||
APNS_TOKEN,
|
||||
createPushServerHarness,
|
||||
notification
|
||||
} from './push-server-harness.test-fixture.js'
|
||||
|
||||
it('delivers again after a topic error without re-registering the phone', async () => {
|
||||
const h = await createPushServerHarness()
|
||||
try {
|
||||
const token = await h.signIn(createPushHostKeypair(71))
|
||||
const registered = await h.post(
|
||||
'/v1/devices',
|
||||
{
|
||||
v: 1,
|
||||
deviceId: 'phone',
|
||||
platform: 'ios',
|
||||
token: APNS_TOKEN,
|
||||
apnsEnvironment: 'sandbox'
|
||||
},
|
||||
token
|
||||
)
|
||||
const { registrationId } = (await registered.json()) as { registrationId: string }
|
||||
h.setApnsResponse({ status: 400, body: JSON.stringify({ reason: 'DeviceTokenNotForTopic' }) })
|
||||
for (const seq of [1, 2]) {
|
||||
const sent = await h.post(
|
||||
'/v1/send',
|
||||
{
|
||||
v: 1,
|
||||
registrationIds: [registrationId],
|
||||
notification: notification({ notificationId: `topic-${seq}`, notificationSeq: seq })
|
||||
},
|
||||
token
|
||||
)
|
||||
expect(await sent.json()).toEqual({ results: [{ registrationId, status: 'queued' }] })
|
||||
await h.flushDeliveries()
|
||||
if (seq === 1) {
|
||||
expect(h.server.observability.consume().delivery_error).toBe(1)
|
||||
h.setApnsResponse({ status: 200, body: '' })
|
||||
}
|
||||
}
|
||||
expect(h.apnsRequests).toHaveLength(2)
|
||||
expect(h.server.observability.consume().delivery_sent).toBe(1)
|
||||
} finally {
|
||||
await h.close()
|
||||
}
|
||||
})
|
||||
@@ -0,0 +1,9 @@
|
||||
// Rejects the many base64 spellings of the same bytes: a non-canonical
|
||||
// encoding would change the transcript the host signs without changing the key.
|
||||
export function decodeCanonicalBase64(value: string, expectedBytes: number): Buffer | null {
|
||||
if (!/^(?:[A-Za-z0-9+/]{4})*(?:[A-Za-z0-9+/]{2}==|[A-Za-z0-9+/]{3}=)?$/.test(value)) return null
|
||||
const decoded = Buffer.from(value, 'base64')
|
||||
return decoded.byteLength === expectedBytes && decoded.toString('base64') === value
|
||||
? decoded
|
||||
: null
|
||||
}
|
||||
@@ -0,0 +1,169 @@
|
||||
import { PUSH_LIMITS } from '@orca-cloud/push-contract'
|
||||
import { Hono } from 'hono'
|
||||
import { describe, expect, it, vi } from 'vitest'
|
||||
import { ClientIpRateLimiter, clientIpRateLimit } from './client-ip-rate-limit.js'
|
||||
|
||||
const CAPACITY = PUSH_LIMITS.unauthenticatedRequestsPerMinutePerIp
|
||||
|
||||
function limiterApp(limiter: ClientIpRateLimiter, trustedProxyHops = 0): Hono {
|
||||
const app = new Hono()
|
||||
app.post('/probe', clientIpRateLimit(limiter, { trustedProxyHops }), (context) =>
|
||||
context.json({ ok: true })
|
||||
)
|
||||
return app
|
||||
}
|
||||
|
||||
describe('client ip rate limiter', () => {
|
||||
it('admits exactly the per-minute allowance and refuses the next request', () => {
|
||||
const limiter = new ClientIpRateLimiter({ now: () => 1_000 })
|
||||
for (let index = 0; index < CAPACITY; index++) {
|
||||
expect(limiter.allow('203.0.113.7')).toBe(true)
|
||||
}
|
||||
expect(limiter.allow('203.0.113.7')).toBe(false)
|
||||
})
|
||||
|
||||
it('keeps one client ip from spending another one budget', () => {
|
||||
const limiter = new ClientIpRateLimiter({ now: () => 1_000 })
|
||||
for (let index = 0; index < CAPACITY; index++) limiter.allow('203.0.113.7')
|
||||
expect(limiter.allow('203.0.113.7')).toBe(false)
|
||||
expect(limiter.allow('198.51.100.9')).toBe(true)
|
||||
})
|
||||
|
||||
it('refills over the window rather than resetting on a boundary', () => {
|
||||
let clock = 1_000
|
||||
const limiter = new ClientIpRateLimiter({ now: () => clock })
|
||||
for (let index = 0; index < CAPACITY; index++) limiter.allow('203.0.113.7')
|
||||
expect(limiter.allow('203.0.113.7')).toBe(false)
|
||||
|
||||
// Half a window buys back half the allowance, no more.
|
||||
clock += 30_000
|
||||
for (let index = 0; index < CAPACITY / 2; index++) {
|
||||
expect(limiter.allow('203.0.113.7')).toBe(true)
|
||||
}
|
||||
expect(limiter.allow('203.0.113.7')).toBe(false)
|
||||
})
|
||||
|
||||
it('bounds what it remembers when a flood of distinct ips arrives', () => {
|
||||
let clock = 1_000
|
||||
const limiter = new ClientIpRateLimiter({ now: () => clock, maxTrackedIps: 8 })
|
||||
for (let index = 0; index < 200; index++) {
|
||||
clock += 1
|
||||
limiter.allow(`198.51.100.${index}`)
|
||||
}
|
||||
expect(limiter.trackedIpCount()).toBeLessThanOrEqual(8)
|
||||
})
|
||||
|
||||
it('evicts the least recently used bucket without scanning the map', () => {
|
||||
const limiter = new ClientIpRateLimiter({ capacity: 1, maxTrackedIps: 2, now: () => 1_000 })
|
||||
limiter.allow('old')
|
||||
limiter.allow('recent')
|
||||
expect(limiter.allow('old')).toBe(false)
|
||||
const entries = vi.spyOn(Map.prototype, 'entries')
|
||||
const iterator = vi.spyOn(Map.prototype, Symbol.iterator)
|
||||
try {
|
||||
limiter.allow('new')
|
||||
expect(entries).not.toHaveBeenCalled()
|
||||
expect(iterator).not.toHaveBeenCalled()
|
||||
} finally {
|
||||
entries.mockRestore()
|
||||
iterator.mockRestore()
|
||||
}
|
||||
expect(limiter.available('old')).toBe(false)
|
||||
expect(limiter.available('recent')).toBe(true)
|
||||
expect(limiter.trackedIpCount()).toBe(2)
|
||||
})
|
||||
|
||||
it('answers 429 with a rate_limited body once the bucket is empty', async () => {
|
||||
const app = limiterApp(new ClientIpRateLimiter({ now: () => 1_000 }))
|
||||
const headers = { 'x-forwarded-for': '10.0.0.1, 10.0.0.2, 203.0.113.7' }
|
||||
for (let index = 0; index < CAPACITY; index++) {
|
||||
expect((await app.request('/probe', { method: 'POST', headers })).status).toBe(200)
|
||||
}
|
||||
const limited = await app.request('/probe', { method: 'POST', headers })
|
||||
expect(limited.status).toBe(429)
|
||||
expect(await limited.json()).toEqual({ error: 'rate_limited' })
|
||||
})
|
||||
|
||||
it('buckets on the last forwarded hop, the only one the platform appended', async () => {
|
||||
const app = limiterApp(new ClientIpRateLimiter({ now: () => 1_000 }))
|
||||
for (let index = 0; index < CAPACITY; index++) {
|
||||
await app.request('/probe', {
|
||||
method: 'POST',
|
||||
headers: { 'x-forwarded-for': `10.0.0.${index}, 203.0.113.7` }
|
||||
})
|
||||
}
|
||||
const sameClient = await app.request('/probe', {
|
||||
method: 'POST',
|
||||
headers: { 'x-forwarded-for': '10.9.9.9, 203.0.113.7' }
|
||||
})
|
||||
expect(sameClient.status).toBe(429)
|
||||
const otherClient = await app.request('/probe', {
|
||||
method: 'POST',
|
||||
headers: { 'x-forwarded-for': '10.0.0.1, 198.51.100.9' }
|
||||
})
|
||||
expect(otherClient.status).toBe(200)
|
||||
})
|
||||
|
||||
it('gives a spoofed left-most hop no escape from the caller own bucket', async () => {
|
||||
const app = limiterApp(new ClientIpRateLimiter({ now: () => 1_000 }))
|
||||
// A caller that rewrites its own x-forwarded-for on every request still ends
|
||||
// up behind the one value Cloud Run appended.
|
||||
for (let index = 0; index < CAPACITY; index++) {
|
||||
const allowed = await app.request('/probe', {
|
||||
method: 'POST',
|
||||
headers: { 'x-forwarded-for': `198.51.100.${index}, 203.0.113.7` }
|
||||
})
|
||||
expect(allowed.status).toBe(200)
|
||||
}
|
||||
const spoofed = await app.request('/probe', {
|
||||
method: 'POST',
|
||||
headers: { 'x-forwarded-for': '198.51.100.250, 10.1.1.1, 203.0.113.7' }
|
||||
})
|
||||
expect(spoofed.status).toBe(429)
|
||||
})
|
||||
|
||||
it('skips the configured trusted proxies when counting from the right', async () => {
|
||||
const app = limiterApp(new ClientIpRateLimiter({ now: () => 1_000, capacity: 1 }), 1)
|
||||
// <client>, <cloud run>, <load balancer>: one trusted hop after the client.
|
||||
const headers = { 'x-forwarded-for': '203.0.113.7, 10.0.0.1' }
|
||||
expect((await app.request('/probe', { method: 'POST', headers })).status).toBe(200)
|
||||
expect((await app.request('/probe', { method: 'POST', headers })).status).toBe(429)
|
||||
expect(
|
||||
(
|
||||
await app.request('/probe', {
|
||||
method: 'POST',
|
||||
headers: { 'x-forwarded-for': '198.51.100.9, 10.0.0.1' }
|
||||
})
|
||||
).status
|
||||
).toBe(200)
|
||||
})
|
||||
|
||||
it('trusts nothing when the header is shorter than the configured depth', async () => {
|
||||
const app = limiterApp(new ClientIpRateLimiter({ now: () => 1_000, capacity: 1 }), 1)
|
||||
// Only one hop, so the client value the depth points at does not exist.
|
||||
const headers = { 'x-forwarded-for': '203.0.113.7' }
|
||||
expect((await app.request('/probe', { method: 'POST', headers })).status).toBe(200)
|
||||
expect(
|
||||
(
|
||||
await app.request('/probe', {
|
||||
method: 'POST',
|
||||
headers: { 'x-forwarded-for': '198.51.100.9' }
|
||||
})
|
||||
).status
|
||||
).toBe(429)
|
||||
})
|
||||
|
||||
it('ignores spoofable x-real-ip and uses a single shared bucket', async () => {
|
||||
const app = limiterApp(new ClientIpRateLimiter({ now: () => 1_000, capacity: 1 }))
|
||||
expect(
|
||||
(await app.request('/probe', { method: 'POST', headers: { 'x-real-ip': '198.51.100.9' } }))
|
||||
.status
|
||||
).toBe(200)
|
||||
expect(
|
||||
(await app.request('/probe', { method: 'POST', headers: { 'x-real-ip': '203.0.113.7' } }))
|
||||
.status
|
||||
).toBe(429)
|
||||
expect((await app.request('/probe', { method: 'POST' })).status).toBe(429)
|
||||
expect((await app.request('/probe', { method: 'POST' })).status).toBe(429)
|
||||
})
|
||||
})
|
||||
@@ -0,0 +1,95 @@
|
||||
import { PUSH_LIMITS } from '@orca-cloud/push-contract'
|
||||
import type { Context, MiddlewareHandler } from 'hono'
|
||||
|
||||
const REFILL_WINDOW_MS = 60_000
|
||||
const MAX_TRACKED_IPS = 10_000
|
||||
const UNKNOWN_CLIENT_IP = 'unknown'
|
||||
|
||||
export type ClientIpRateLimiterOptions = {
|
||||
capacity?: number
|
||||
windowMs?: number
|
||||
maxTrackedIps?: number
|
||||
now?: () => number
|
||||
}
|
||||
|
||||
type Bucket = { tokens: number; updatedAt: number }
|
||||
|
||||
// Read x-forwarded-for from the right. Cloud Run appends the connecting peer,
|
||||
// so the last value is the only one it wrote; everything to its left is
|
||||
// whatever the caller sent and can be a fresh forgery on every request.
|
||||
// trustedProxyHops is how many appenders sit between Cloud Run and the client
|
||||
// (0 today, 1 once a load balancer fronts it). A header too short for that
|
||||
// depth is not trusted at all and falls through to the shared bucket, which
|
||||
// throttles rather than opens.
|
||||
export function readClientIp(context: Context, trustedProxyHops = 0): string {
|
||||
const hops =
|
||||
context.req
|
||||
.header('x-forwarded-for')
|
||||
?.split(',')
|
||||
.map((hop) => hop.trim())
|
||||
.filter((hop) => hop.length > 0) ?? []
|
||||
const client = hops[hops.length - 1 - trustedProxyHops]
|
||||
return client ?? UNKNOWN_CLIENT_IP
|
||||
}
|
||||
|
||||
// Per-instance admission avoids a database round trip; capacity scales with instance count.
|
||||
export class ClientIpRateLimiter {
|
||||
private readonly buckets = new Map<string, Bucket>()
|
||||
private readonly capacity: number
|
||||
private readonly windowMs: number
|
||||
private readonly maxTrackedIps: number
|
||||
private readonly now: () => number
|
||||
|
||||
constructor(options: ClientIpRateLimiterOptions = {}) {
|
||||
this.capacity = options.capacity ?? PUSH_LIMITS.unauthenticatedRequestsPerMinutePerIp
|
||||
this.windowMs = options.windowMs ?? REFILL_WINDOW_MS
|
||||
this.maxTrackedIps = options.maxTrackedIps ?? MAX_TRACKED_IPS
|
||||
this.now = options.now ?? Date.now
|
||||
}
|
||||
|
||||
available(clientIp: string): boolean {
|
||||
return this.tokensAt(this.buckets.get(clientIp), this.now()) >= 1
|
||||
}
|
||||
|
||||
allow(clientIp: string): boolean {
|
||||
const now = this.now()
|
||||
const tokens = this.tokensAt(this.buckets.get(clientIp), now)
|
||||
this.buckets.delete(clientIp)
|
||||
this.buckets.set(clientIp, { tokens: tokens < 1 ? tokens : tokens - 1, updatedAt: now })
|
||||
if (this.buckets.size > this.maxTrackedIps) {
|
||||
const oldest = this.buckets.keys().next().value
|
||||
if (oldest !== undefined) this.buckets.delete(oldest)
|
||||
}
|
||||
return tokens >= 1
|
||||
}
|
||||
|
||||
trackedIpCount(): number {
|
||||
return this.buckets.size
|
||||
}
|
||||
|
||||
private tokensAt(bucket: Bucket | undefined, now: number): number {
|
||||
if (!bucket) return this.capacity
|
||||
const refilled = ((now - bucket.updatedAt) * this.capacity) / this.windowMs
|
||||
return Math.min(this.capacity, bucket.tokens + Math.max(0, refilled))
|
||||
}
|
||||
}
|
||||
|
||||
export type ClientIpRateLimitOptions = {
|
||||
trustedProxyHops?: number
|
||||
onLimited?: () => void
|
||||
}
|
||||
|
||||
export function clientIpRateLimit(
|
||||
limiter: ClientIpRateLimiter,
|
||||
options: ClientIpRateLimitOptions = {}
|
||||
): MiddlewareHandler {
|
||||
const trustedProxyHops = options.trustedProxyHops ?? 0
|
||||
return async (context, next) => {
|
||||
if (!limiter.allow(readClientIp(context, trustedProxyHops))) {
|
||||
options.onLimited?.()
|
||||
return context.json({ error: 'rate_limited' }, 429)
|
||||
}
|
||||
await next()
|
||||
return
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,109 @@
|
||||
import { generateKeyPairSync } from 'node:crypto'
|
||||
import { PUSH_DEFAULTS } from '@orca-cloud/push-contract'
|
||||
import { describe, expect, it } from 'vitest'
|
||||
import { loadPushConfig, PUSH_DATABASE_POOL_MAX } from './config.js'
|
||||
|
||||
function apnsKeyPem(): string {
|
||||
return generateKeyPairSync('ec', {
|
||||
namedCurve: 'P-256',
|
||||
privateKeyEncoding: { type: 'pkcs8', format: 'pem' },
|
||||
publicKeyEncoding: { type: 'spki', format: 'pem' }
|
||||
}).privateKey
|
||||
}
|
||||
|
||||
const MINIMAL = {
|
||||
ORCA_PUSH_PUBLIC_URL: 'https://push.onorca.dev',
|
||||
ORCA_PUSH_FCM_PROJECT_ID: 'onorca-cloud'
|
||||
}
|
||||
|
||||
describe('push gateway config', () => {
|
||||
it('applies the documented defaults', () => {
|
||||
expect(loadPushConfig(MINIMAL)).toEqual({
|
||||
mode: 'active',
|
||||
port: 8080,
|
||||
publicUrl: 'https://push.onorca.dev',
|
||||
databaseUrl: undefined,
|
||||
dataDir: './data/push',
|
||||
databasePoolMax: PUSH_DATABASE_POOL_MAX,
|
||||
apns: undefined,
|
||||
apnsTopic: PUSH_DEFAULTS.apnsTopic,
|
||||
fcmProjectId: 'onorca-cloud',
|
||||
trustedProxyHops: 0
|
||||
})
|
||||
})
|
||||
|
||||
it('reads a full APNs credential and the overridable knobs', () => {
|
||||
const keyPem = apnsKeyPem()
|
||||
const config = loadPushConfig({
|
||||
...MINIMAL,
|
||||
PORT: '9090',
|
||||
ORCA_PUSH_DATABASE_URL: 'postgres://localhost/orca_push',
|
||||
ORCA_PUSH_DATA_DIR: '/var/lib/push',
|
||||
ORCA_PUSH_APNS_KEY: keyPem,
|
||||
ORCA_PUSH_APNS_KEY_ID: 'ABCDE12345',
|
||||
ORCA_PUSH_APPLE_TEAM_ID: 'TEAM123456',
|
||||
ORCA_PUSH_APNS_TOPIC: 'com.stably.orca.mobile.dev',
|
||||
ORCA_PUSH_FCM_PROJECT_ID: 'onorca-staging',
|
||||
ORCA_PUSH_TRUSTED_PROXY_HOPS: '1'
|
||||
})
|
||||
expect(config).toMatchObject({
|
||||
port: 9090,
|
||||
databaseUrl: 'postgres://localhost/orca_push',
|
||||
dataDir: '/var/lib/push',
|
||||
apns: { keyPem, keyId: 'ABCDE12345', teamId: 'TEAM123456' },
|
||||
apnsTopic: 'com.stably.orca.mobile.dev',
|
||||
trustedProxyHops: 1,
|
||||
fcmProjectId: 'onorca-staging'
|
||||
})
|
||||
})
|
||||
|
||||
it('requires an explicit FCM project instead of silently targeting production', () => {
|
||||
expect(() => loadPushConfig({ ...MINIMAL, ORCA_PUSH_FCM_PROJECT_ID: undefined })).toThrow()
|
||||
expect(() => loadPushConfig({ ...MINIMAL, ORCA_PUSH_FCM_PROJECT_ID: ' ' })).toThrow()
|
||||
})
|
||||
|
||||
it('refuses a partial APNs credential', () => {
|
||||
expect(() => loadPushConfig({ ...MINIMAL, ORCA_PUSH_APNS_KEY: apnsKeyPem() })).toThrow(
|
||||
'configured together'
|
||||
)
|
||||
expect(() =>
|
||||
loadPushConfig({
|
||||
...MINIMAL,
|
||||
ORCA_PUSH_APNS_KEY: 'not-a-pem',
|
||||
ORCA_PUSH_APNS_KEY_ID: 'ABCDE12345',
|
||||
ORCA_PUSH_APPLE_TEAM_ID: 'TEAM123456'
|
||||
})
|
||||
).toThrow('PEM text')
|
||||
})
|
||||
|
||||
it('requires a canonical HTTPS origin outside loopback', () => {
|
||||
expect(() =>
|
||||
loadPushConfig({ ...MINIMAL, ORCA_PUSH_PUBLIC_URL: 'https://push.onorca.dev/v1' })
|
||||
).toThrow('must be an origin')
|
||||
expect(() =>
|
||||
loadPushConfig({ ...MINIMAL, ORCA_PUSH_PUBLIC_URL: 'http://push.onorca.dev' })
|
||||
).toThrow('must use HTTPS')
|
||||
expect(
|
||||
loadPushConfig({ ...MINIMAL, ORCA_PUSH_PUBLIC_URL: 'http://localhost:8080' }).publicUrl
|
||||
).toBe('http://localhost:8080')
|
||||
})
|
||||
|
||||
it('treats an empty optional variable as unset', () => {
|
||||
expect(
|
||||
loadPushConfig({ ...MINIMAL, ORCA_PUSH_DATABASE_URL: '', ORCA_PUSH_APNS_KEY_ID: '' })
|
||||
).toMatchObject({ databaseUrl: undefined, apns: undefined })
|
||||
})
|
||||
})
|
||||
|
||||
it('treats blank defaulted environment settings as absent', () => {
|
||||
const blanks = Object.fromEntries(
|
||||
[
|
||||
'PORT',
|
||||
'ORCA_PUSH_DATA_DIR',
|
||||
'ORCA_PUSH_APNS_TOPIC',
|
||||
'ORCA_PUSH_DATABASE_POOL_MAX',
|
||||
'ORCA_PUSH_TRUSTED_PROXY_HOPS'
|
||||
].map((key) => [key, ' '])
|
||||
)
|
||||
expect(loadPushConfig({ ...MINIMAL, ...blanks })).toEqual(loadPushConfig(MINIMAL))
|
||||
})
|
||||
@@ -0,0 +1,108 @@
|
||||
import { PUSH_DEFAULTS } from '@orca-cloud/push-contract'
|
||||
import { z } from 'zod'
|
||||
|
||||
export const PUSH_DATABASE_POOL_MAX = 10
|
||||
|
||||
const OptionalTextSchema = z.preprocess(
|
||||
(value) => (value === '' ? undefined : value),
|
||||
z.string().min(1).optional()
|
||||
)
|
||||
|
||||
const EnvSchema = z.object({
|
||||
ORCA_PUSH_MODE: z.enum(['active', 'validation']).default('active'),
|
||||
PORT: z.coerce.number().int().positive().default(8080),
|
||||
ORCA_PUSH_PUBLIC_URL: z.string().url(),
|
||||
ORCA_PUSH_DATABASE_URL: OptionalTextSchema,
|
||||
ORCA_PUSH_DATA_DIR: z.string().min(1).default('./data/push'),
|
||||
ORCA_PUSH_DATABASE_POOL_MAX: z.coerce.number().int().positive().max(100).optional(),
|
||||
ORCA_PUSH_APNS_KEY: OptionalTextSchema,
|
||||
ORCA_PUSH_APNS_KEY_ID: z.preprocess(
|
||||
(value) => (value === '' ? undefined : value),
|
||||
z
|
||||
.string()
|
||||
.regex(/^[A-Z0-9]{10}$/)
|
||||
.optional()
|
||||
),
|
||||
ORCA_PUSH_APPLE_TEAM_ID: z.preprocess(
|
||||
(value) => (value === '' ? undefined : value),
|
||||
z
|
||||
.string()
|
||||
.regex(/^[A-Z0-9]{10}$/)
|
||||
.optional()
|
||||
),
|
||||
ORCA_PUSH_APNS_TOPIC: z.string().min(1).max(255).default(PUSH_DEFAULTS.apnsTopic),
|
||||
ORCA_PUSH_FCM_PROJECT_ID: z.string().regex(/^[a-z0-9-]{4,64}$/),
|
||||
// How many proxies append to x-forwarded-for after the client. 0 is Cloud Run
|
||||
// alone; raise it to 1 when a load balancer fronts the service.
|
||||
ORCA_PUSH_TRUSTED_PROXY_HOPS: z.coerce.number().int().nonnegative().max(8).default(0)
|
||||
})
|
||||
|
||||
export type ApnsCredentials = { keyPem: string; keyId: string; teamId: string }
|
||||
|
||||
export type PushConfig = {
|
||||
mode: 'active' | 'validation'
|
||||
port: number
|
||||
publicUrl: string
|
||||
databaseUrl?: string
|
||||
dataDir: string
|
||||
databasePoolMax: number
|
||||
apns?: ApnsCredentials
|
||||
apnsTopic: string
|
||||
fcmProjectId: string
|
||||
trustedProxyHops: number
|
||||
}
|
||||
|
||||
function canonicalOrigin(value: string, name: string): string {
|
||||
const url = new URL(value)
|
||||
if (url.origin !== value || url.pathname !== '/') throw new Error(`${name} must be an origin`)
|
||||
const loopback = ['127.0.0.1', 'localhost', '::1', '[::1]'].includes(url.hostname)
|
||||
if (url.protocol !== 'https:' && !(loopback && url.protocol === 'http:')) {
|
||||
throw new Error(`${name} must use HTTPS outside loopback development`)
|
||||
}
|
||||
return value
|
||||
}
|
||||
|
||||
// The APNs key, key id, and team id are one credential; a partial set would
|
||||
// pass startup and then fail every iOS send at runtime.
|
||||
function readApnsCredentials(parsed: z.infer<typeof EnvSchema>): ApnsCredentials | undefined {
|
||||
const parts = [
|
||||
parsed.ORCA_PUSH_APNS_KEY,
|
||||
parsed.ORCA_PUSH_APNS_KEY_ID,
|
||||
parsed.ORCA_PUSH_APPLE_TEAM_ID
|
||||
]
|
||||
const present = parts.filter((value) => value !== undefined).length
|
||||
if (present === 0) return undefined
|
||||
if (present !== parts.length) {
|
||||
throw new Error('APNs key, key id, and team id must be configured together')
|
||||
}
|
||||
const keyPem = parsed.ORCA_PUSH_APNS_KEY!
|
||||
if (!keyPem.includes('-----BEGIN')) throw new Error('ORCA_PUSH_APNS_KEY must be PEM text')
|
||||
return {
|
||||
keyPem,
|
||||
keyId: parsed.ORCA_PUSH_APNS_KEY_ID!,
|
||||
teamId: parsed.ORCA_PUSH_APPLE_TEAM_ID!
|
||||
}
|
||||
}
|
||||
|
||||
export function loadPushConfig(env: NodeJS.ProcessEnv = process.env): PushConfig {
|
||||
const parsed = EnvSchema.parse(
|
||||
Object.fromEntries(
|
||||
Object.entries(env).map(([key, value]) => [
|
||||
key,
|
||||
key !== 'ORCA_PUSH_MODE' && value?.trim() === '' ? undefined : value
|
||||
])
|
||||
)
|
||||
)
|
||||
return {
|
||||
mode: parsed.ORCA_PUSH_MODE,
|
||||
port: parsed.PORT,
|
||||
publicUrl: canonicalOrigin(parsed.ORCA_PUSH_PUBLIC_URL, 'ORCA_PUSH_PUBLIC_URL'),
|
||||
databaseUrl: parsed.ORCA_PUSH_DATABASE_URL,
|
||||
dataDir: parsed.ORCA_PUSH_DATA_DIR,
|
||||
databasePoolMax: parsed.ORCA_PUSH_DATABASE_POOL_MAX ?? PUSH_DATABASE_POOL_MAX,
|
||||
apns: readApnsCredentials(parsed),
|
||||
apnsTopic: parsed.ORCA_PUSH_APNS_TOPIC,
|
||||
fcmProjectId: parsed.ORCA_PUSH_FCM_PROJECT_ID,
|
||||
trustedProxyHops: parsed.ORCA_PUSH_TRUSTED_PROXY_HOPS
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,47 @@
|
||||
import { describe, expect, it } from 'vitest'
|
||||
import { createHmac } from 'node:crypto'
|
||||
import vector from '../../../packages/push-contract/src/push-host-proof-vector.json' with { type: 'json' }
|
||||
import { answerPushHostChallenge, createPushHostKeypair } from './host-challenge-answering.test-fixture.js'
|
||||
import { PushHostChallengeStore } from './host-challenge-store.js'
|
||||
import { deriveHostFingerprint } from './host-fingerprint.js'
|
||||
import { openInMemoryPushDatabase } from './push-database.js'
|
||||
|
||||
// Why: the desktop answers challenges in a workspace this one cannot import.
|
||||
// Both sides replay the same checked-in vector, so a transcript drift on
|
||||
// either side fails in that side's own suite.
|
||||
describe('desktop host proof interop', () => {
|
||||
it('the checked-in vector answers to the same proof the fixture host computes', () => {
|
||||
const secretKey = new Uint8Array(Buffer.from(vector.hostSecretKeyB64, 'base64'))
|
||||
const keypair = { publicKey: new Uint8Array(Buffer.from(vector.hostPublicKeyB64, 'base64')), secretKey }
|
||||
expect(deriveHostFingerprint(keypair.publicKey)).toBe(vector.hostFingerprint)
|
||||
const proof = answerPushHostChallenge(vector.challenge, {
|
||||
gatewayOrigin: vector.gatewayOrigin,
|
||||
keypair,
|
||||
now: () => vector.issuedAt + 1_000
|
||||
})
|
||||
const expected = createHmac('sha256', Buffer.from(vector.challengeSecretB64, 'base64'))
|
||||
.update(Buffer.from('orca-push-host-proof/v1\0ack\0'))
|
||||
.update(Buffer.from(vector.transcriptB64, 'base64'))
|
||||
.digest('base64')
|
||||
expect(proof).toBe(expected)
|
||||
})
|
||||
|
||||
it('a live challenge from the store round-trips through the fixture host once', async () => {
|
||||
const database = await openInMemoryPushDatabase()
|
||||
const store = new PushHostChallengeStore(database, vector.gatewayOrigin)
|
||||
const keypair = createPushHostKeypair(11)
|
||||
const challenge = await store.issue(Buffer.from(keypair.publicKey).toString('base64'))
|
||||
expect(challenge).not.toBeNull()
|
||||
const proof = answerPushHostChallenge(challenge!, { gatewayOrigin: vector.gatewayOrigin, keypair })
|
||||
expect(proof).not.toBeNull()
|
||||
expect(await store.verify(challenge!.challengeId, proof!)).toEqual({
|
||||
ok: true,
|
||||
hostFingerprint: deriveHostFingerprint(keypair.publicKey)
|
||||
})
|
||||
expect(await store.verify(challenge!.challengeId, proof!)).toEqual({
|
||||
ok: false,
|
||||
reason: 'already_consumed'
|
||||
})
|
||||
await database.close()
|
||||
})
|
||||
})
|
||||
@@ -0,0 +1,88 @@
|
||||
import { expect, it, vi } from 'vitest'
|
||||
import { openPushDatabase, type PushDatabase } from './push-database.js'
|
||||
import { PushDeviceRegistryStore } from './device-registry-store.js'
|
||||
|
||||
const databaseUrl = process.env.ORCA_PUSH_TEST_DATABASE_URL
|
||||
it.skipIf(!databaseUrl)(
|
||||
'serializes deletion with a registration that has already read its row',
|
||||
async () => {
|
||||
if (!process.env.CI && new URL(databaseUrl!).port !== '55440')
|
||||
throw new Error('isolated_postgres_port_required')
|
||||
const database = await openPushDatabase({
|
||||
databaseUrl,
|
||||
dataDir: '',
|
||||
poolMax: 4,
|
||||
applicationName: 'push-delete-race'
|
||||
})
|
||||
let release!: () => void
|
||||
const paused = new Promise<void>((resolve) => {
|
||||
release = resolve
|
||||
})
|
||||
let read = false
|
||||
let pause = false
|
||||
const wrapped: PushDatabase = {
|
||||
dialect: database.dialect,
|
||||
query: database.query.bind(database),
|
||||
close: database.close.bind(database),
|
||||
lockQuotaScope: database.lockQuotaScope.bind(database),
|
||||
transaction: (run) =>
|
||||
database.transaction((tx) =>
|
||||
run({
|
||||
dialect: tx.dialect,
|
||||
close: tx.close.bind(tx),
|
||||
transaction: tx.transaction.bind(tx),
|
||||
lockQuotaScope: tx.lockQuotaScope.bind(tx),
|
||||
query: async (sql, params) => {
|
||||
const rows = await tx.query(sql, params)
|
||||
if (pause && sql.startsWith('SELECT registration_id FROM push_devices')) {
|
||||
read = true
|
||||
await paused
|
||||
}
|
||||
return rows
|
||||
}
|
||||
})
|
||||
)
|
||||
}
|
||||
const devices = new PushDeviceRegistryStore(wrapped)
|
||||
const input = {
|
||||
hostFingerprint: 'delete-race-host',
|
||||
deviceId: 'phone',
|
||||
platform: 'android' as const,
|
||||
token: 'synthetic'
|
||||
}
|
||||
let registration: Promise<unknown> | undefined
|
||||
let deletion: Promise<boolean> | undefined
|
||||
try {
|
||||
await database.query('DELETE FROM push_devices WHERE host_fingerprint = ?', [
|
||||
input.hostFingerprint
|
||||
])
|
||||
const first = await devices.upsert(input)
|
||||
if (!first.ok) throw new Error('registration refused')
|
||||
pause = true
|
||||
registration = devices.upsert(input)
|
||||
await vi.waitFor(() => expect(read).toBe(true))
|
||||
let deleted = false
|
||||
deletion = devices.deleteOwned(input.hostFingerprint, first.registrationId).then((value) => {
|
||||
deleted = true
|
||||
return value
|
||||
})
|
||||
await vi.waitFor(async () => {
|
||||
const rows = await database.query(
|
||||
"SELECT 1 FROM pg_stat_activity WHERE application_name = 'push-delete-race' AND wait_event_type = 'Lock'"
|
||||
)
|
||||
expect(deleted || rows.length > 0).toBe(true)
|
||||
})
|
||||
expect(deleted).toBe(false)
|
||||
release()
|
||||
expect(await registration).toEqual(first)
|
||||
expect(await deletion).toBe(true)
|
||||
} finally {
|
||||
release()
|
||||
await Promise.allSettled([registration, deletion])
|
||||
await database.query('DELETE FROM push_devices WHERE host_fingerprint = ?', [
|
||||
input.hostFingerprint
|
||||
])
|
||||
await database.close()
|
||||
}
|
||||
}
|
||||
)
|
||||
@@ -0,0 +1,191 @@
|
||||
import { PUSH_LIMITS } from '@orca-cloud/push-contract'
|
||||
import { afterEach, beforeEach, describe, expect, it } from 'vitest'
|
||||
import { PushDeviceRegistryStore, type PushDeviceUpsert } from './device-registry-store.js'
|
||||
import { openInMemoryPushDatabase, type PushDatabase } from './push-database.js'
|
||||
|
||||
const OWNER = 'abcdefghijklmnop'
|
||||
const OTHER = 'ponmlkjihgfedcba'
|
||||
|
||||
describe('push device registry store', () => {
|
||||
let database: PushDatabase
|
||||
let clock = 1_700_000_000_000
|
||||
let devices: PushDeviceRegistryStore
|
||||
|
||||
beforeEach(async () => {
|
||||
database = await openInMemoryPushDatabase()
|
||||
clock = 1_700_000_000_000
|
||||
devices = new PushDeviceRegistryStore(database, () => clock)
|
||||
})
|
||||
|
||||
afterEach(async () => {
|
||||
await database.close()
|
||||
})
|
||||
|
||||
async function upsertOk(input: PushDeviceUpsert): Promise<string> {
|
||||
const result = await devices.upsert(input)
|
||||
if (!result.ok) throw new Error(`unexpected upsert refusal: ${result.reason}`)
|
||||
return result.registrationId
|
||||
}
|
||||
|
||||
function androidDevice(deviceId: string): PushDeviceUpsert {
|
||||
return {
|
||||
hostFingerprint: OWNER,
|
||||
deviceId,
|
||||
platform: 'android',
|
||||
token: `token-${deviceId}`
|
||||
}
|
||||
}
|
||||
|
||||
it('keeps one registration per host and device while replacing the token', async () => {
|
||||
const first = await upsertOk({
|
||||
hostFingerprint: OWNER,
|
||||
deviceId: 'device-1',
|
||||
platform: 'ios',
|
||||
token: 'a'.repeat(64),
|
||||
apnsEnvironment: 'sandbox'
|
||||
})
|
||||
clock += 1_000
|
||||
const second = await upsertOk({
|
||||
hostFingerprint: OWNER,
|
||||
deviceId: 'device-1',
|
||||
platform: 'ios',
|
||||
token: 'b'.repeat(64),
|
||||
apnsEnvironment: 'production'
|
||||
})
|
||||
expect(second).toBe(first)
|
||||
const registration = await devices.findById(first)
|
||||
expect(registration).toMatchObject({
|
||||
token: 'b'.repeat(64),
|
||||
apnsEnvironment: 'production',
|
||||
dead: false
|
||||
})
|
||||
expect(await devices.list(OWNER)).toHaveLength(1)
|
||||
})
|
||||
|
||||
it('revives a registration that a re-registered token replaces', async () => {
|
||||
const registrationId = await upsertOk({
|
||||
hostFingerprint: OWNER,
|
||||
deviceId: 'device-1',
|
||||
platform: 'android',
|
||||
token: 'token-one'
|
||||
})
|
||||
await devices.markDead((await devices.findById(registrationId))!)
|
||||
expect((await devices.findById(registrationId))?.dead).toBe(true)
|
||||
await upsertOk({
|
||||
hostFingerprint: OWNER,
|
||||
deviceId: 'device-1',
|
||||
platform: 'android',
|
||||
token: 'token-two'
|
||||
})
|
||||
expect(await devices.findById(registrationId)).toMatchObject({
|
||||
token: 'token-two',
|
||||
dead: false
|
||||
})
|
||||
})
|
||||
|
||||
it('lets only the owning host delete a registration', async () => {
|
||||
const registrationId = await upsertOk({
|
||||
hostFingerprint: OWNER,
|
||||
deviceId: 'device-1',
|
||||
platform: 'android',
|
||||
token: 'token-one'
|
||||
})
|
||||
expect(await devices.deleteOwned(OTHER, registrationId)).toBe(false)
|
||||
expect(await devices.findById(registrationId)).not.toBeNull()
|
||||
expect(await devices.deleteOwned(OWNER, registrationId)).toBe(true)
|
||||
expect(await devices.findById(registrationId)).toBeNull()
|
||||
})
|
||||
|
||||
it('scopes lookups and listings to the owning host', async () => {
|
||||
const owned = await upsertOk({
|
||||
hostFingerprint: OWNER,
|
||||
deviceId: 'device-1',
|
||||
platform: 'android',
|
||||
token: 'token-one'
|
||||
})
|
||||
const foreign = await upsertOk({
|
||||
hostFingerprint: OTHER,
|
||||
deviceId: 'device-2',
|
||||
platform: 'android',
|
||||
token: 'token-two'
|
||||
})
|
||||
const found = await devices.findOwned(OWNER, [owned, foreign])
|
||||
expect([...found.keys()]).toEqual([owned])
|
||||
expect(await devices.list(OTHER)).toEqual([
|
||||
{ registrationId: foreign, deviceId: 'device-2', platform: 'android', dead: false }
|
||||
])
|
||||
expect(await devices.findOwned(OWNER, [])).toEqual(new Map())
|
||||
})
|
||||
|
||||
it('refuses a new device once the host reaches its registration cap', async () => {
|
||||
for (let index = 0; index < PUSH_LIMITS.maxDevicesPerHost; index++) {
|
||||
await upsertOk(androidDevice(`device-${index}`))
|
||||
}
|
||||
expect(await devices.upsert(androidDevice('one-too-many'))).toEqual({
|
||||
ok: false,
|
||||
reason: 'too_many_devices'
|
||||
})
|
||||
expect(await devices.list(OWNER)).toHaveLength(PUSH_LIMITS.maxDevicesPerHost)
|
||||
})
|
||||
|
||||
it('still lets a capped host re-register a device it already owns', async () => {
|
||||
for (let index = 0; index < PUSH_LIMITS.maxDevicesPerHost; index++) {
|
||||
await upsertOk(androidDevice(`device-${index}`))
|
||||
}
|
||||
const rotated = await devices.upsert({ ...androidDevice('device-0'), token: 'rotated-token' })
|
||||
expect(rotated.ok).toBe(true)
|
||||
expect(await devices.list(OWNER)).toHaveLength(PUSH_LIMITS.maxDevicesPerHost)
|
||||
})
|
||||
|
||||
it('frees a slot when a registration is deleted', async () => {
|
||||
const first = await upsertOk(androidDevice('device-0'))
|
||||
for (let index = 1; index < PUSH_LIMITS.maxDevicesPerHost; index++) {
|
||||
await upsertOk(androidDevice(`device-${index}`))
|
||||
}
|
||||
expect((await devices.upsert(androidDevice('extra'))).ok).toBe(false)
|
||||
expect(await devices.deleteOwned(OWNER, first)).toBe(true)
|
||||
expect((await devices.upsert(androidDevice('extra'))).ok).toBe(true)
|
||||
})
|
||||
|
||||
it('counts the cap per host, not across the whole table', async () => {
|
||||
for (let index = 0; index < PUSH_LIMITS.maxDevicesPerHost; index++) {
|
||||
await upsertOk(androidDevice(`device-${index}`))
|
||||
}
|
||||
expect((await devices.upsert(androidDevice('extra'))).ok).toBe(false)
|
||||
expect(
|
||||
(await devices.upsert({ ...androidDevice('device-0'), hostFingerprint: OTHER })).ok
|
||||
).toBe(true)
|
||||
})
|
||||
|
||||
it('bounds list reads to the host device allowance', async () => {
|
||||
// Straight past the per-host cap, so only the query LIMIT can bound this.
|
||||
const rows = PUSH_LIMITS.maxDevicesPerHost + 5
|
||||
for (let index = 0; index < rows; index++) {
|
||||
await database.query(
|
||||
`INSERT INTO push_devices (registration_id, host_fingerprint, device_id, platform, token,
|
||||
created_at, updated_at)
|
||||
VALUES (?, ?, ?, ?, ?, ?, ?)`,
|
||||
[`reg-${index}`, OWNER, `device-${index}`, 'android', 'token', clock + index, clock]
|
||||
)
|
||||
}
|
||||
expect(await devices.list(OWNER)).toHaveLength(PUSH_LIMITS.maxDevicesPerHost)
|
||||
})
|
||||
|
||||
it('separates the same device id registered against two hosts', async () => {
|
||||
const first = await upsertOk({
|
||||
hostFingerprint: OWNER,
|
||||
deviceId: 'shared-device',
|
||||
platform: 'ios',
|
||||
token: 'a'.repeat(64),
|
||||
apnsEnvironment: 'sandbox'
|
||||
})
|
||||
const second = await upsertOk({
|
||||
hostFingerprint: OTHER,
|
||||
deviceId: 'shared-device',
|
||||
platform: 'ios',
|
||||
token: 'c'.repeat(64),
|
||||
apnsEnvironment: 'sandbox'
|
||||
})
|
||||
expect(first).not.toBe(second)
|
||||
})
|
||||
})
|
||||
@@ -0,0 +1,177 @@
|
||||
import { randomUUID } from 'node:crypto'
|
||||
import {
|
||||
PUSH_LIMITS,
|
||||
type ApnsEnvironment,
|
||||
type PushDeviceSummary,
|
||||
type PushPlatform
|
||||
} from '@orca-cloud/push-contract'
|
||||
import type { PushDatabase, SqlRow } from './push-database.js'
|
||||
|
||||
const DEVICE_CAP_LOCK_PREFIX = 'orca-push-device-cap:'
|
||||
|
||||
export type PushDeviceRegistration = {
|
||||
registrationId: string
|
||||
hostFingerprint: string
|
||||
deviceId: string
|
||||
platform: PushPlatform
|
||||
token: string
|
||||
apnsEnvironment?: ApnsEnvironment
|
||||
dead: boolean
|
||||
}
|
||||
|
||||
export type PushDeviceUpsertResult =
|
||||
| { ok: true; registrationId: string }
|
||||
| { ok: false; reason: 'too_many_devices' }
|
||||
|
||||
export type PushDeviceUpsert = {
|
||||
hostFingerprint: string
|
||||
deviceId: string
|
||||
platform: PushPlatform
|
||||
token: string
|
||||
apnsEnvironment?: ApnsEnvironment
|
||||
}
|
||||
|
||||
function toRegistration(row: SqlRow): PushDeviceRegistration {
|
||||
const apnsEnvironment = row.apns_environment
|
||||
return {
|
||||
registrationId: String(row.registration_id),
|
||||
hostFingerprint: String(row.host_fingerprint),
|
||||
deviceId: String(row.device_id),
|
||||
platform: String(row.platform) as PushPlatform,
|
||||
token: String(row.token),
|
||||
...(apnsEnvironment === null || apnsEnvironment === undefined
|
||||
? {}
|
||||
: { apnsEnvironment: String(apnsEnvironment) as ApnsEnvironment }),
|
||||
dead: row.dead_at !== null && row.dead_at !== undefined
|
||||
}
|
||||
}
|
||||
|
||||
export class PushDeviceRegistryStore {
|
||||
constructor(
|
||||
private readonly database: PushDatabase,
|
||||
private readonly now: () => number = Date.now
|
||||
) {}
|
||||
|
||||
// The registration id is stable for a (host, device) pair so a re-registered
|
||||
// phone keeps the id the desktop already persisted; only the token rotates.
|
||||
async upsert(input: PushDeviceUpsert): Promise<PushDeviceUpsertResult> {
|
||||
const now = this.now()
|
||||
return await this.database.transaction<PushDeviceUpsertResult>(async (transaction) => {
|
||||
// deviceId is caller-chosen, so counting and inserting must not interleave
|
||||
// or a burst of new ids would walk straight past the cap.
|
||||
await transaction.lockQuotaScope(`${DEVICE_CAP_LOCK_PREFIX}${input.hostFingerprint}`)
|
||||
const [existing] = await transaction.query(
|
||||
'SELECT registration_id FROM push_devices WHERE host_fingerprint = ? AND device_id = ?',
|
||||
[input.hostFingerprint, input.deviceId]
|
||||
)
|
||||
if (existing) {
|
||||
const registrationId = String(existing.registration_id)
|
||||
await transaction.query(
|
||||
`UPDATE push_devices
|
||||
SET platform = ?, token = ?, apns_environment = ?,
|
||||
dead_at = NULL, updated_at = ?
|
||||
WHERE registration_id = ?`,
|
||||
[input.platform, input.token, input.apnsEnvironment ?? null, now, registrationId]
|
||||
)
|
||||
return { ok: true, registrationId }
|
||||
}
|
||||
const [countRow] = await transaction.query(
|
||||
'SELECT COUNT(*) AS devices FROM push_devices WHERE host_fingerprint = ?',
|
||||
[input.hostFingerprint]
|
||||
)
|
||||
if (Number(countRow?.devices ?? 0) >= PUSH_LIMITS.maxDevicesPerHost) {
|
||||
return { ok: false, reason: 'too_many_devices' }
|
||||
}
|
||||
const registrationId = randomUUID()
|
||||
await transaction.query(
|
||||
`INSERT INTO push_devices
|
||||
(registration_id, host_fingerprint, device_id, platform, token, apns_environment,
|
||||
dead_at, created_at, updated_at)
|
||||
VALUES (?, ?, ?, ?, ?, ?, NULL, ?, ?)`,
|
||||
[
|
||||
registrationId,
|
||||
input.hostFingerprint,
|
||||
input.deviceId,
|
||||
input.platform,
|
||||
input.token,
|
||||
input.apnsEnvironment ?? null,
|
||||
now,
|
||||
now
|
||||
]
|
||||
)
|
||||
return { ok: true, registrationId }
|
||||
})
|
||||
}
|
||||
|
||||
async deleteOwned(hostFingerprint: string, registrationId: string): Promise<boolean> {
|
||||
return this.database.transaction(async (transaction) => {
|
||||
await transaction.lockQuotaScope(`${DEVICE_CAP_LOCK_PREFIX}${hostFingerprint}`)
|
||||
const [result] = await transaction.query(
|
||||
'DELETE FROM push_devices WHERE registration_id = ? AND host_fingerprint = ?',
|
||||
[registrationId, hostFingerprint]
|
||||
)
|
||||
return Number(result?.changes ?? 0) > 0
|
||||
})
|
||||
}
|
||||
|
||||
async list(hostFingerprint: string): Promise<PushDeviceSummary[]> {
|
||||
const rows = await this.database.query(
|
||||
// Bounded by the device-list response limit, so an
|
||||
// oversized table degrades to a truncated list instead of a 500.
|
||||
`SELECT registration_id, device_id, platform, dead_at
|
||||
FROM push_devices WHERE host_fingerprint = ? ORDER BY created_at ASC LIMIT ?`,
|
||||
[hostFingerprint, PUSH_LIMITS.maxDevicesPerHost]
|
||||
)
|
||||
return rows.map((row) => ({
|
||||
registrationId: String(row.registration_id),
|
||||
deviceId: String(row.device_id),
|
||||
platform: String(row.platform) as PushPlatform,
|
||||
dead: row.dead_at !== null && row.dead_at !== undefined
|
||||
}))
|
||||
}
|
||||
|
||||
async findOwned(
|
||||
hostFingerprint: string,
|
||||
registrationIds: readonly string[]
|
||||
): Promise<Map<string, PushDeviceRegistration>> {
|
||||
if (registrationIds.length === 0) return new Map()
|
||||
const placeholders = registrationIds.map(() => '?').join(', ')
|
||||
const rows = await this.database.query(
|
||||
`SELECT registration_id, host_fingerprint, device_id, platform, token, apns_environment,
|
||||
dead_at
|
||||
FROM push_devices
|
||||
WHERE host_fingerprint = ? AND registration_id IN (${placeholders})`,
|
||||
[hostFingerprint, ...registrationIds]
|
||||
)
|
||||
return new Map(
|
||||
rows.map((row) => {
|
||||
const registration = toRegistration(row)
|
||||
return [registration.registrationId, registration]
|
||||
})
|
||||
)
|
||||
}
|
||||
|
||||
async findById(registrationId: string): Promise<PushDeviceRegistration | null> {
|
||||
const [row] = await this.database.query(
|
||||
`SELECT registration_id, host_fingerprint, device_id, platform, token, apns_environment,
|
||||
dead_at
|
||||
FROM push_devices WHERE registration_id = ?`,
|
||||
[registrationId]
|
||||
)
|
||||
return row ? toRegistration(row) : null
|
||||
}
|
||||
|
||||
async markDead(observed: PushDeviceRegistration): Promise<void> {
|
||||
await this.database.query(
|
||||
`UPDATE push_devices SET dead_at = ?, updated_at = ? WHERE registration_id = ? AND token = ? AND platform = ? AND COALESCE(apns_environment, '') = ?`,
|
||||
[
|
||||
this.now(),
|
||||
this.now(),
|
||||
observed.registrationId,
|
||||
observed.token,
|
||||
observed.platform,
|
||||
observed.apnsEnvironment ?? ''
|
||||
]
|
||||
)
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,45 @@
|
||||
export const DURABLE_PUSH_SCHEMA = `
|
||||
CREATE TABLE IF NOT EXISTS push_dismissed_events (
|
||||
host_fingerprint TEXT NOT NULL,
|
||||
notification_epoch TEXT NOT NULL,
|
||||
notification_id TEXT NOT NULL,
|
||||
notification_seq BIGINT NOT NULL,
|
||||
created_at BIGINT NOT NULL,
|
||||
PRIMARY KEY(host_fingerprint, notification_epoch, notification_id)
|
||||
);
|
||||
CREATE INDEX IF NOT EXISTS push_dismissed_retention ON push_dismissed_events(created_at);
|
||||
CREATE TABLE IF NOT EXISTS push_events (
|
||||
event_id TEXT PRIMARY KEY,
|
||||
host_fingerprint TEXT NOT NULL,
|
||||
kind TEXT NOT NULL,
|
||||
fingerprint TEXT NOT NULL,
|
||||
created_at BIGINT NOT NULL,
|
||||
expires_at BIGINT NOT NULL
|
||||
);
|
||||
CREATE INDEX IF NOT EXISTS push_events_quota ON push_events(host_fingerprint, kind, created_at);
|
||||
CREATE TABLE IF NOT EXISTS push_event_recipients (
|
||||
event_id TEXT NOT NULL,
|
||||
registration_id TEXT NOT NULL,
|
||||
created_at BIGINT NOT NULL,
|
||||
PRIMARY KEY(event_id, registration_id)
|
||||
);
|
||||
CREATE TABLE IF NOT EXISTS push_delivery_batches (
|
||||
batch_id TEXT PRIMARY KEY,
|
||||
host_fingerprint TEXT NOT NULL,
|
||||
registration_id TEXT NOT NULL,
|
||||
kind TEXT NOT NULL,
|
||||
payload_json TEXT NOT NULL,
|
||||
state TEXT NOT NULL,
|
||||
due_at BIGINT NOT NULL,
|
||||
expires_at BIGINT NOT NULL,
|
||||
lease_token TEXT,
|
||||
lease_until BIGINT NOT NULL,
|
||||
attempts BIGINT NOT NULL,
|
||||
created_at BIGINT NOT NULL
|
||||
);
|
||||
CREATE INDEX IF NOT EXISTS push_events_retention ON push_events(created_at);
|
||||
CREATE INDEX IF NOT EXISTS push_recipients_retention ON push_event_recipients(created_at);
|
||||
CREATE INDEX IF NOT EXISTS push_batches_expiry ON push_delivery_batches(expires_at);
|
||||
CREATE INDEX IF NOT EXISTS push_batches_due ON push_delivery_batches(state, due_at);
|
||||
CREATE INDEX IF NOT EXISTS push_batches_registration ON push_delivery_batches(registration_id, state);
|
||||
`
|
||||
@@ -0,0 +1,289 @@
|
||||
import { randomUUID } from 'node:crypto'
|
||||
import pg from 'pg'
|
||||
import { afterEach, describe, expect, it } from 'vitest'
|
||||
import { openInMemoryPushDatabase, openPushDatabase, type PushDatabase } from './push-database.js'
|
||||
import { DurablePushStore, DELIVERY_LEASE_MS } from './durable-push-store.js'
|
||||
import type { PushNotification } from '@orca-cloud/push-contract'
|
||||
|
||||
const cleanups: (() => Promise<void>)[] = []
|
||||
afterEach(async () => {
|
||||
await Promise.all(cleanups.splice(0).map((cleanup) => cleanup()))
|
||||
})
|
||||
const notification = (seq: number, kind: 'alert' | 'dismiss' = 'alert'): PushNotification => ({
|
||||
notificationId: `notification-${seq}`,
|
||||
notificationEpoch: 'epoch',
|
||||
notificationSeq: seq,
|
||||
source: 'agent-task-complete',
|
||||
agentState: 'finished',
|
||||
title: 'Done',
|
||||
body: '',
|
||||
kind
|
||||
})
|
||||
async function fixture() {
|
||||
const databaseUrl =
|
||||
process.env.ORCA_PUSH_DURABLE_TEST_POSTGRES_URL ?? process.env.ORCA_PUSH_TEST_DATABASE_URL
|
||||
if (databaseUrl && !process.env.CI && new URL(databaseUrl).port !== '55440')
|
||||
throw new Error('isolated_postgres_port_required')
|
||||
let db: PushDatabase
|
||||
if (databaseUrl) {
|
||||
const admin = new pg.Client({ connectionString: databaseUrl })
|
||||
await admin.connect()
|
||||
const schema = `durable_${randomUUID().replaceAll('-', '')}`
|
||||
let scoped: PushDatabase | undefined
|
||||
cleanups.push(async () => {
|
||||
try {
|
||||
await scoped?.close()
|
||||
} finally {
|
||||
try {
|
||||
await admin.query(`DROP SCHEMA IF EXISTS ${schema} CASCADE`)
|
||||
} finally {
|
||||
await admin.end()
|
||||
}
|
||||
}
|
||||
})
|
||||
await admin.query(`CREATE SCHEMA ${schema}`)
|
||||
const url = new URL(databaseUrl)
|
||||
url.searchParams.set('options', `-c search_path=${schema}`)
|
||||
db = scoped = await openPushDatabase({ databaseUrl: url.toString(), dataDir: '', poolMax: 4 })
|
||||
} else {
|
||||
db = await openInMemoryPushDatabase()
|
||||
cleanups.push(() => db.close())
|
||||
}
|
||||
let now = 1_000_000
|
||||
const clock = () => now
|
||||
return {
|
||||
db,
|
||||
store: new DurablePushStore(db, clock),
|
||||
clock,
|
||||
advance: (ms: number) => {
|
||||
now += ms
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
describe('durable push acceptance', () => {
|
||||
it('counts a logical event once across phones and separates the 300/15min dismissal budget', async () => {
|
||||
const { store, advance } = await fixture()
|
||||
for (let i = 0; i < 300; i++) {
|
||||
expect(await store.accept('host', 'phone1', notification(i))).toBe('queued')
|
||||
expect(await store.accept('host', 'phone2', notification(i))).toBe('queued')
|
||||
expect(await store.accept('host', 'phone1', notification(i, 'dismiss'))).toBe('queued')
|
||||
}
|
||||
expect(await store.accept('host', 'phone1', notification(300))).toBe('rate_limited')
|
||||
expect(await store.accept('host', 'phone1', notification(300, 'dismiss'))).toBe('rate_limited')
|
||||
expect(await store.accept('another-host', 'phone3', notification(300))).toBe('queued')
|
||||
advance(15 * 60_000)
|
||||
expect(await store.accept('host', 'phone1', notification(301))).toBe('queued')
|
||||
})
|
||||
|
||||
it('queues one delivery per event and recovers work across service instances', async () => {
|
||||
const { db, store, clock, advance } = await fixture()
|
||||
await store.accept('host', 'phone', notification(1))
|
||||
const restarted = new DurablePushStore(db, clock)
|
||||
await restarted.accept('host', 'phone', notification(1))
|
||||
advance(1)
|
||||
await restarted.accept('host', 'phone', notification(2))
|
||||
const rows = await db.query(
|
||||
"SELECT payload_json, due_at, created_at FROM push_delivery_batches WHERE registration_id = ? AND state = 'pending' ORDER BY created_at, batch_id",
|
||||
['phone']
|
||||
)
|
||||
expect(rows).toHaveLength(2)
|
||||
expect(rows.map((row) => JSON.parse(String(row.payload_json)))).toEqual([
|
||||
notification(1),
|
||||
notification(2)
|
||||
])
|
||||
expect(rows.every((row) => Number(row.due_at) >= Number(row.created_at))).toBe(true)
|
||||
const delivery = await restarted.claim()
|
||||
expect(delivery?.notification.notificationSeq).toBe(1)
|
||||
expect(await store.claim()).toBeNull()
|
||||
advance(DELIVERY_LEASE_MS)
|
||||
const reclaimed = await store.claim()
|
||||
expect(reclaimed?.id).toBe(delivery?.id)
|
||||
expect(reclaimed?.lease).not.toBe(delivery?.lease)
|
||||
await restarted.finish(delivery!)
|
||||
expect(await store.claim()).toBeNull()
|
||||
await store.finish(reclaimed!)
|
||||
const second = await restarted.claim()
|
||||
expect(second?.notification.notificationSeq).toBe(2)
|
||||
await restarted.finish(second!)
|
||||
expect(await restarted.claim()).toBeNull()
|
||||
})
|
||||
|
||||
it('never extends expiry and refuses conflicting duplicate content', async () => {
|
||||
const { store, advance } = await fixture()
|
||||
await store.accept('host', 'phone', notification(1))
|
||||
expect(await store.accept('host', 'phone', { ...notification(1), body: 'changed' })).toBe(
|
||||
'error'
|
||||
)
|
||||
const delivery = (await store.claim())!
|
||||
await store.finish(delivery, 10 * 60_000)
|
||||
advance(60_000)
|
||||
expect(await store.claim()).toBeNull()
|
||||
advance(5 * 60_000)
|
||||
expect(await store.accept('host', 'phone', notification(1))).toBe('error')
|
||||
})
|
||||
|
||||
it('orders a due retry before a fresh first attempt without delaying the retry', async () => {
|
||||
const { store, advance } = await fixture()
|
||||
await store.accept('host', 'phone', notification(1))
|
||||
const first = (await store.claim())!
|
||||
await store.finish(first, 1000)
|
||||
expect(await store.claim()).toBeNull()
|
||||
|
||||
advance(1000)
|
||||
await store.accept('host', 'phone', notification(2))
|
||||
const retry = (await store.claim())!
|
||||
expect(retry.notification.notificationSeq).toBe(1)
|
||||
await store.finish(retry)
|
||||
const fresh = (await store.claim())!
|
||||
expect(fresh?.notification.notificationSeq).toBe(2)
|
||||
await store.finish(fresh!)
|
||||
})
|
||||
|
||||
it('orders an expired first-attempt lease by creation time after a retry becomes due', async () => {
|
||||
const { db, store, clock, advance } = await fixture()
|
||||
await store.accept('host', 'phone', notification(1))
|
||||
const retry = (await store.claim())!
|
||||
await store.finish(retry, 1000)
|
||||
|
||||
advance(2000)
|
||||
await db.query(
|
||||
`INSERT INTO push_delivery_batches(batch_id, host_fingerprint, registration_id, kind, payload_json, state, due_at, expires_at, lease_until, attempts, created_at)
|
||||
VALUES ('crashed-singleton', 'host', 'phone', 'alert', ?, 'pending', ?, ?, 0, 1, ?)`,
|
||||
[JSON.stringify(notification(2)), clock() - 1, clock() + 300_000, clock()]
|
||||
)
|
||||
const reclaimedRetry = (await store.claim())!
|
||||
expect(reclaimedRetry.notification.notificationSeq).toBe(1)
|
||||
await store.finish(reclaimedRetry)
|
||||
const reclaimedCrash = (await store.claim())!
|
||||
expect(reclaimedCrash.notification.notificationSeq).toBe(2)
|
||||
await store.finish(reclaimedCrash)
|
||||
})
|
||||
|
||||
it('rolls quota and payload back together if persistence fails', async () => {
|
||||
const { db, store } = await fixture()
|
||||
await db.query('ALTER TABLE push_delivery_batches RENAME TO push_delivery_batches_unavailable')
|
||||
try {
|
||||
const databaseUrl =
|
||||
process.env.ORCA_PUSH_DURABLE_TEST_POSTGRES_URL ?? process.env.ORCA_PUSH_TEST_DATABASE_URL
|
||||
if (databaseUrl) {
|
||||
const concurrent = await openPushDatabase({ databaseUrl, dataDir: '' })
|
||||
try {
|
||||
await expect(
|
||||
concurrent.query('SELECT COUNT(*) FROM push_delivery_batches')
|
||||
).resolves.toHaveLength(1)
|
||||
} finally {
|
||||
await concurrent.close()
|
||||
}
|
||||
}
|
||||
await expect(store.accept('host', 'phone', notification(1))).rejects.toThrow()
|
||||
expect(await db.query('SELECT * FROM push_events')).toEqual([])
|
||||
expect(await db.query('SELECT * FROM push_event_recipients')).toEqual([])
|
||||
} finally {
|
||||
await db.query(
|
||||
'ALTER TABLE push_delivery_batches_unavailable RENAME TO push_delivery_batches'
|
||||
)
|
||||
}
|
||||
})
|
||||
})
|
||||
|
||||
it('serializes concurrent instances at the quota boundary', async () => {
|
||||
const { db, store, clock } = await fixture()
|
||||
for (let seq = 0; seq < 299; seq++) await store.accept('host', 'phone', notification(seq))
|
||||
const second = new DurablePushStore(db, clock)
|
||||
const results = await Promise.all(
|
||||
Array.from({ length: 6 }, (_, index) =>
|
||||
(index % 2 ? store : second).accept('host', 'phone', notification(400 + index))
|
||||
)
|
||||
)
|
||||
expect(results.filter((result) => result === 'queued')).toHaveLength(1)
|
||||
expect(results.filter((result) => result === 'rate_limited')).toHaveLength(5)
|
||||
})
|
||||
|
||||
it('cancels unsent alerts and prevents an older replay after dismissal', async () => {
|
||||
const { store } = await fixture()
|
||||
const alert = notification(1)
|
||||
await store.accept('host', 'phone', alert)
|
||||
await store.accept('host', 'phone', {
|
||||
...notification(2, 'dismiss'),
|
||||
notificationId: alert.notificationId
|
||||
})
|
||||
const delivery = (await store.claim())!
|
||||
expect(delivery.notification.kind).toBe('dismiss')
|
||||
await store.finish(delivery)
|
||||
expect(await store.claim()).toBeNull()
|
||||
await store.accept('host', 'another-phone', alert)
|
||||
expect(await store.claim()).toBeNull()
|
||||
})
|
||||
|
||||
it('does not resurrect an in-flight alert after a dismissal and transient provider failure', async () => {
|
||||
const { store, advance } = await fixture()
|
||||
await store.accept('host', 'phone', notification(1))
|
||||
const inFlight = (await store.claim())!
|
||||
await store.accept('host', 'phone', {
|
||||
...notification(2, 'dismiss'),
|
||||
notificationId: notification(1).notificationId
|
||||
})
|
||||
await store.finish(inFlight, 1000)
|
||||
const dismissal = (await store.claim())!
|
||||
expect(dismissal.notification.kind).toBe('dismiss')
|
||||
await store.finish(dismissal)
|
||||
advance(1000)
|
||||
expect(await store.claim()).toBeNull()
|
||||
expect(await store.pendingCount('phone')).toBe(0)
|
||||
})
|
||||
|
||||
it.each([false, true])(
|
||||
'normalizes default alert kind (explicit first: %s)',
|
||||
async (explicitFirst) => {
|
||||
const { db, store } = await fixture()
|
||||
const { kind: _kind, ...implicit } = notification(1)
|
||||
const explicit = { kind: 'alert' as const, ...implicit }
|
||||
for (const event of explicitFirst ? [explicit, implicit] : [implicit, explicit]) {
|
||||
expect(await store.accept('host', 'phone', event)).toBe('queued')
|
||||
}
|
||||
expect(await store.pendingCount('phone')).toBe(1)
|
||||
expect(await db.query('SELECT event_id FROM push_events')).toHaveLength(1)
|
||||
expect(await store.accept('host', 'phone', { ...explicit, body: 'changed' })).toBe('error')
|
||||
expect(await store.accept('host', 'phone', { ...implicit, kind: 'dismiss' })).toBe('queued')
|
||||
expect(await db.query('SELECT event_id FROM push_events')).toHaveLength(2)
|
||||
}
|
||||
)
|
||||
|
||||
it('fences late renew and finish after an expired claim is dismissed', async () => {
|
||||
const { db, store, advance } = await fixture()
|
||||
const alert = notification(1)
|
||||
await store.accept('host', 'phone', alert)
|
||||
const stale = (await store.claim())!
|
||||
advance(DELIVERY_LEASE_MS)
|
||||
await store.accept('host', 'phone', {
|
||||
...notification(2, 'dismiss'),
|
||||
notificationId: alert.notificationId
|
||||
})
|
||||
const read = async () =>
|
||||
(
|
||||
await db.query(
|
||||
'SELECT state, payload_json, lease_until FROM push_delivery_batches WHERE batch_id = ?',
|
||||
[stale.id]
|
||||
)
|
||||
)[0]
|
||||
const cancelled = await read()
|
||||
expect(cancelled).toMatchObject({ state: 'dismissed', payload_json: '{}' })
|
||||
await store.renew(stale)
|
||||
expect(await read()).toEqual(cancelled)
|
||||
await store.finish(stale, 1000)
|
||||
expect(await read()).toEqual(cancelled)
|
||||
await store.finish(stale)
|
||||
expect(await read()).toEqual(cancelled)
|
||||
const dismissal = (await store.claim())!
|
||||
expect(dismissal.notification.kind).toBe('dismiss')
|
||||
await store.finish(dismissal)
|
||||
await store.accept('host', 'phone', notification(3))
|
||||
const fresh = (await store.claim())!
|
||||
await store.finish(fresh, 1000)
|
||||
advance(1000)
|
||||
const retry = (await store.claim())!
|
||||
expect(retry.id).toBe(fresh.id)
|
||||
await store.finish(retry)
|
||||
expect(await store.claim()).toBeNull()
|
||||
})
|
||||
@@ -0,0 +1,194 @@
|
||||
import { isDismissedAlert, reconcileQueuedDismissal } from './push-queued-dismissal.js'
|
||||
import { parsePushDeliveryPayload } from './push-delivery-payload.js'
|
||||
import { createHash, randomUUID } from 'node:crypto'
|
||||
import { PUSH_LIMITS, type PushNotification } from '@orca-cloud/push-contract'
|
||||
import type { PushDatabase, SqlRow } from './push-database.js'
|
||||
|
||||
const RETENTION_MS = 24 * 60 * 60_000
|
||||
export const DELIVERY_LEASE_MS = 30_000
|
||||
export type QueuedPushDelivery = {
|
||||
id: string
|
||||
registrationId: string
|
||||
hostFingerprint: string
|
||||
notification: PushNotification
|
||||
expiresAt: number
|
||||
lease: string
|
||||
attempts: number
|
||||
}
|
||||
|
||||
export class DurablePushStore {
|
||||
constructor(
|
||||
private readonly database: PushDatabase,
|
||||
private readonly now = Date.now
|
||||
) {}
|
||||
|
||||
async accept(
|
||||
host: string,
|
||||
registrationId: string,
|
||||
notification: PushNotification
|
||||
): Promise<'queued' | 'rate_limited' | 'error'> {
|
||||
const now = this.now()
|
||||
const kind = notification.kind ?? 'alert'
|
||||
const eventId = createHash('sha256')
|
||||
.update(
|
||||
JSON.stringify([host, kind, notification.notificationEpoch, notification.notificationSeq])
|
||||
)
|
||||
.digest('hex')
|
||||
const { sound: _sound, kind: _kind, ...content } = notification
|
||||
const fingerprint = createHash('sha256')
|
||||
.update(JSON.stringify({ kind, ...content }))
|
||||
.digest('hex')
|
||||
return this.database.transaction(async (tx) => {
|
||||
await tx.lockQuotaScope(`push-events:${host}`)
|
||||
const [existing] = await tx.query('SELECT * FROM push_events WHERE event_id = ?', [eventId])
|
||||
if (existing && existing.fingerprint !== fingerprint) return 'error'
|
||||
const expiresAt = existing
|
||||
? Number(existing.expires_at)
|
||||
: Math.min(
|
||||
notification.expiresAt ?? Infinity,
|
||||
now + PUSH_LIMITS.notificationTtlSeconds * 1000
|
||||
)
|
||||
if (expiresAt <= now) return 'error'
|
||||
if (!existing) {
|
||||
const [count] = await tx.query(
|
||||
'SELECT COUNT(*) AS total FROM push_events WHERE host_fingerprint = ? AND kind = ? AND created_at > ?',
|
||||
[host, kind, now - PUSH_LIMITS.eventQuotaWindowMs]
|
||||
)
|
||||
if (Number(count?.total ?? 0) >= PUSH_LIMITS.hostEventsPerWindow) return 'rate_limited'
|
||||
await tx.query(
|
||||
'INSERT INTO push_events(event_id, host_fingerprint, kind, fingerprint, created_at, expires_at) VALUES (?, ?, ?, ?, ?, ?)',
|
||||
[eventId, host, kind, fingerprint, now, expiresAt]
|
||||
)
|
||||
}
|
||||
const [recipient] = await tx.query(
|
||||
'SELECT event_id FROM push_event_recipients WHERE event_id = ? AND registration_id = ?',
|
||||
[eventId, registrationId]
|
||||
)
|
||||
if (recipient) return 'queued'
|
||||
if (await reconcileQueuedDismissal(tx, host, registrationId, notification, now))
|
||||
return 'queued'
|
||||
await tx.query(
|
||||
`INSERT INTO push_delivery_batches(batch_id, host_fingerprint, registration_id, kind, payload_json, state, due_at, expires_at, lease_until, attempts, created_at)
|
||||
VALUES (?, ?, ?, ?, ?, 'pending', ?, ?, 0, 0, ?)`,
|
||||
[
|
||||
randomUUID(),
|
||||
host,
|
||||
registrationId,
|
||||
kind,
|
||||
JSON.stringify(notification),
|
||||
now,
|
||||
expiresAt,
|
||||
now
|
||||
]
|
||||
)
|
||||
await tx.query(
|
||||
'INSERT INTO push_event_recipients(event_id, registration_id, created_at) VALUES (?, ?, ?)',
|
||||
[eventId, registrationId, now]
|
||||
)
|
||||
return 'queued'
|
||||
})
|
||||
}
|
||||
|
||||
async claim(): Promise<QueuedPushDelivery | null> {
|
||||
return this.database.transaction(async (tx) => {
|
||||
await tx.lockQuotaScope('push-worker-claim')
|
||||
const now = this.now()
|
||||
const params = [now, now, now, now]
|
||||
const predicate =
|
||||
"state = 'pending' AND lease_until <= ? AND expires_at > ? AND due_at <= ? AND NOT EXISTS (SELECT 1 FROM push_delivery_batches busy WHERE busy.registration_id = push_delivery_batches.registration_id AND busy.lease_until > ?)"
|
||||
let [row] = await tx.query(
|
||||
`SELECT * FROM push_delivery_batches WHERE ${predicate} ORDER BY due_at, created_at, batch_id LIMIT 1`,
|
||||
params
|
||||
)
|
||||
if (!row) return null
|
||||
await tx.lockQuotaScope(`push-events:${String(row.host_fingerprint)}`)
|
||||
;[row] = await tx.query('SELECT * FROM push_delivery_batches WHERE batch_id = ?', [
|
||||
row.batch_id
|
||||
])
|
||||
if (!row || row.state !== 'pending' || Number(row.expires_at) <= now) return null
|
||||
const notification = parsePushDeliveryPayload(String(row.payload_json))
|
||||
if (await isDismissedAlert(tx, String(row.host_fingerprint), notification)) {
|
||||
await tx.query(
|
||||
"UPDATE push_delivery_batches SET state = 'dismissed', payload_json = '{}' WHERE batch_id = ?",
|
||||
[row.batch_id]
|
||||
)
|
||||
return null
|
||||
}
|
||||
const lease = randomUUID()
|
||||
await tx.query(
|
||||
'UPDATE push_delivery_batches SET lease_token = ?, lease_until = ?, attempts = attempts + 1 WHERE batch_id = ?',
|
||||
[lease, now + DELIVERY_LEASE_MS, row.batch_id]
|
||||
)
|
||||
return this.delivery(row, lease)
|
||||
})
|
||||
}
|
||||
|
||||
private delivery(row: SqlRow, lease: string): QueuedPushDelivery {
|
||||
return {
|
||||
id: String(row.batch_id),
|
||||
registrationId: String(row.registration_id),
|
||||
hostFingerprint: String(row.host_fingerprint),
|
||||
notification: parsePushDeliveryPayload(String(row.payload_json)),
|
||||
expiresAt: Number(row.expires_at),
|
||||
lease,
|
||||
attempts: Number(row.attempts) + 1
|
||||
}
|
||||
}
|
||||
|
||||
async renew(delivery: QueuedPushDelivery): Promise<void> {
|
||||
await this.database.query(
|
||||
"UPDATE push_delivery_batches SET lease_until = ? WHERE batch_id = ? AND lease_token = ? AND state = 'pending'",
|
||||
[this.now() + DELIVERY_LEASE_MS, delivery.id, delivery.lease]
|
||||
)
|
||||
}
|
||||
|
||||
async finish(
|
||||
delivery: QueuedPushDelivery,
|
||||
retryAfterMs?: number,
|
||||
outcome = 'done'
|
||||
): Promise<void> {
|
||||
const now = this.now()
|
||||
const retryAt = retryAfterMs === undefined ? Infinity : now + Math.max(1000, retryAfterMs)
|
||||
const retry = retryAt < delivery.expiresAt
|
||||
await this.database.query(
|
||||
`UPDATE push_delivery_batches SET state = ?, payload_json = ?, due_at = ?, lease_until = 0, lease_token = NULL
|
||||
WHERE batch_id = ? AND lease_token = ? AND state = 'pending'`,
|
||||
[
|
||||
retry ? 'pending' : retryAfterMs !== undefined ? 'expired' : outcome,
|
||||
retry ? JSON.stringify(delivery.notification) : '{}',
|
||||
retry ? retryAt : now,
|
||||
delivery.id,
|
||||
delivery.lease
|
||||
]
|
||||
)
|
||||
}
|
||||
|
||||
async pendingCount(registrationId: string): Promise<number> {
|
||||
const [row] = await this.database.query(
|
||||
"SELECT COUNT(*) AS total FROM push_delivery_batches WHERE registration_id = ? AND state = 'pending'",
|
||||
[registrationId]
|
||||
)
|
||||
return Number(row?.total ?? 0)
|
||||
}
|
||||
|
||||
async prune(): Promise<number> {
|
||||
const now = this.now()
|
||||
await this.database.query(
|
||||
"UPDATE push_delivery_batches SET state = 'expired', payload_json = '{}' WHERE expires_at <= ? AND state = 'pending'",
|
||||
[now]
|
||||
)
|
||||
await this.database.query('DELETE FROM push_dismissed_events WHERE created_at < ?', [
|
||||
now - RETENTION_MS
|
||||
])
|
||||
await this.database.query('DELETE FROM push_delivery_batches WHERE expires_at < ?', [
|
||||
now - RETENTION_MS
|
||||
])
|
||||
await this.database.query('DELETE FROM push_event_recipients WHERE created_at < ?', [
|
||||
now - RETENTION_MS
|
||||
])
|
||||
const [result] = await this.database.query('DELETE FROM push_events WHERE created_at < ?', [
|
||||
now - RETENTION_MS
|
||||
])
|
||||
return Number(result?.changes ?? 0)
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,221 @@
|
||||
import { afterEach, expect, it, vi } from 'vitest'
|
||||
import type { PushNotification } from '@orca-cloud/push-contract'
|
||||
import { DurablePushStore } from './durable-push-store.js'
|
||||
import { DurablePushWorker } from './durable-push-worker.js'
|
||||
import { PushDispatcher } from './push-dispatcher.js'
|
||||
import { PushDeviceRegistryStore } from './device-registry-store.js'
|
||||
import { openInMemoryPushDatabase } from './push-database.js'
|
||||
import type { PushDelivery } from './push-delivery-message.js'
|
||||
import type { PushProviderOutcome } from './push-provider-outcome.js'
|
||||
|
||||
const cleanups: (() => Promise<void>)[] = []
|
||||
afterEach(async () => {
|
||||
for (const cleanup of cleanups.splice(0)) await cleanup()
|
||||
vi.useRealTimers()
|
||||
vi.restoreAllMocks()
|
||||
})
|
||||
const note = (seq: number, overrides: Partial<PushNotification> = {}): PushNotification => ({
|
||||
notificationId: `note-${seq}`,
|
||||
notificationSeq: seq,
|
||||
notificationEpoch: 'epoch',
|
||||
source: 'agent-task-complete',
|
||||
agentState: 'finished',
|
||||
title: 'Done',
|
||||
body: 'Finished task',
|
||||
...overrides
|
||||
})
|
||||
async function fixture() {
|
||||
const db = await openInMemoryPushDatabase()
|
||||
let time = 1_000_000
|
||||
const now = () => time
|
||||
const store = new DurablePushStore(db, now)
|
||||
const devices = new PushDeviceRegistryStore(db, now)
|
||||
const device = await devices.upsert({
|
||||
hostFingerprint: 'host',
|
||||
deviceId: 'phone',
|
||||
platform: 'android',
|
||||
token: 'test-token'
|
||||
})
|
||||
if (!device.ok) throw new Error('registration failed')
|
||||
const send = vi.fn(async (_delivery: PushDelivery): Promise<PushProviderOutcome> => ({
|
||||
status: 'sent'
|
||||
}))
|
||||
const onRetry = vi.fn()
|
||||
const dispatcher = new PushDispatcher({ devices, fcm: { send } as never })
|
||||
const worker = new DurablePushWorker(store, dispatcher, { now, onRetry })
|
||||
cleanups.push(async () => {
|
||||
await worker.stop()
|
||||
await db.close()
|
||||
})
|
||||
vi.spyOn(console, 'warn').mockImplementation(() => {})
|
||||
return {
|
||||
db,
|
||||
store,
|
||||
devices,
|
||||
worker,
|
||||
dispatcher,
|
||||
send,
|
||||
onRetry,
|
||||
now,
|
||||
registrationId: device.registrationId,
|
||||
accept: (notification: PushNotification) =>
|
||||
store.accept('host', device.registrationId, notification),
|
||||
advance: (ms: number) => {
|
||||
time += ms
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
it('sends every burst event immediately with its original content and identity', async () => {
|
||||
const h = await fixture()
|
||||
await h.accept(note(1))
|
||||
await h.accept(
|
||||
note(2, { agentState: 'needs-input', title: 'Answer needed', body: 'Please respond' })
|
||||
)
|
||||
await h.worker.runDue()
|
||||
expect(h.send).toHaveBeenCalledTimes(2)
|
||||
const first = h.send.mock.calls.find(([delivery]) => delivery.orca.notificationSeq === 1)![0]
|
||||
const second = h.send.mock.calls.find(([delivery]) => delivery.orca.notificationSeq === 2)![0]
|
||||
expect(first).toMatchObject({
|
||||
title: 'Done',
|
||||
body: 'Finished task',
|
||||
orca: {
|
||||
notificationId: 'note-1',
|
||||
notificationSeq: 1
|
||||
}
|
||||
})
|
||||
expect(second).toMatchObject({
|
||||
title: 'Answer needed',
|
||||
body: 'Please respond',
|
||||
orca: { notificationId: 'note-2', notificationSeq: 2 }
|
||||
})
|
||||
expect(first.collapseId).not.toBe(second.collapseId)
|
||||
expect(h.send.mock.calls.every(([delivery]) => !('coalescedCount' in delivery.orca))).toBe(true)
|
||||
expect(h.send.mock.calls.every(([delivery]) => !('summaryMembers' in delivery.orca))).toBe(true)
|
||||
expect(h.onRetry).not.toHaveBeenCalled()
|
||||
})
|
||||
|
||||
it('keeps untrackable bells and per-phone deliveries individually replaceable', async () => {
|
||||
const h = await fixture()
|
||||
const other = await h.devices.upsert({
|
||||
hostFingerprint: 'host',
|
||||
deviceId: 'phone2',
|
||||
platform: 'android',
|
||||
token: 'other-token'
|
||||
})
|
||||
if (!other.ok) throw new Error('registration failed')
|
||||
await h.accept(note(1, { notificationId: undefined, source: 'terminal-bell', agentState: null }))
|
||||
await h.accept(note(2))
|
||||
await h.accept(note(3))
|
||||
await h.store.accept('host', other.registrationId, note(2))
|
||||
await h.worker.runDue()
|
||||
expect(h.send).toHaveBeenCalledTimes(4)
|
||||
const deliveries = h.send.mock.calls.map(([delivery]) => delivery)
|
||||
const primary = deliveries.filter((delivery) => delivery.registrationId === h.registrationId)
|
||||
expect(primary).toHaveLength(3)
|
||||
expect(new Set(primary.map((delivery) => delivery.collapseId)).size).toBe(3)
|
||||
expect(
|
||||
deliveries.find((delivery) => delivery.registrationId === other.registrationId)
|
||||
).toMatchObject({ orca: { notificationId: 'note-2', notificationSeq: 2 } })
|
||||
})
|
||||
|
||||
it('persists provider retry delay and resumes it through a new worker', async () => {
|
||||
const h = await fixture()
|
||||
h.send.mockResolvedValueOnce({
|
||||
status: 'error',
|
||||
reason: 'busy',
|
||||
retryable: true,
|
||||
retryAfterMs: 10000
|
||||
})
|
||||
await h.accept(note(1))
|
||||
await h.worker.runDue()
|
||||
expect(h.send).toHaveBeenCalledOnce()
|
||||
await h.worker.stop()
|
||||
const restarted = new DurablePushWorker(h.store, h.dispatcher, { now: h.now, onRetry: h.onRetry })
|
||||
h.advance(9999)
|
||||
await restarted.runDue()
|
||||
expect(h.send).toHaveBeenCalledOnce()
|
||||
h.advance(1)
|
||||
await restarted.runDue()
|
||||
expect(h.send).toHaveBeenCalledTimes(2)
|
||||
expect(h.send.mock.calls.map(([delivery]) => delivery.expiresAt)).toEqual([1_300_000, 1_300_000])
|
||||
expect(h.onRetry).toHaveBeenCalledOnce()
|
||||
expect(await h.store.pendingCount(h.registrationId)).toBe(0)
|
||||
await restarted.stop()
|
||||
})
|
||||
|
||||
it('expires instead of shortening a provider delay beyond the delivery lifetime', async () => {
|
||||
const h = await fixture()
|
||||
h.send.mockResolvedValue({
|
||||
status: 'error',
|
||||
reason: 'busy',
|
||||
retryable: true,
|
||||
retryAfterMs: 600000
|
||||
})
|
||||
await h.accept(note(1))
|
||||
await h.worker.runDue()
|
||||
h.advance(600000)
|
||||
await h.worker.runDue()
|
||||
expect(h.send).toHaveBeenCalledOnce()
|
||||
expect(await h.store.pendingCount(h.registrationId)).toBe(0)
|
||||
expect(h.onRetry).not.toHaveBeenCalled()
|
||||
})
|
||||
|
||||
it('rechecks the device before a persisted retry and does not send after unregistration', async () => {
|
||||
const h = await fixture()
|
||||
h.send.mockResolvedValue({ status: 'error', reason: 'timeout', retryable: true })
|
||||
await h.accept(note(1))
|
||||
await h.worker.runDue()
|
||||
await h.devices.deleteOwned('host', h.registrationId)
|
||||
h.advance(3000)
|
||||
await h.worker.runDue()
|
||||
expect(h.send).toHaveBeenCalledOnce()
|
||||
expect(await h.store.pendingCount(h.registrationId)).toBe(0)
|
||||
})
|
||||
|
||||
it('joins active work on shutdown and leaves unclaimed work for the next instance', async () => {
|
||||
const h = await fixture()
|
||||
let finish!: (outcome: PushProviderOutcome) => void
|
||||
let started!: () => void
|
||||
const entered = new Promise<void>((resolve) => {
|
||||
started = resolve
|
||||
})
|
||||
h.send.mockImplementationOnce(() => {
|
||||
started()
|
||||
return new Promise((resolve) => {
|
||||
finish = resolve
|
||||
})
|
||||
})
|
||||
await h.accept(note(1))
|
||||
const pending = h.worker.runDue()
|
||||
await entered
|
||||
await h.accept(note(2))
|
||||
let stopped = false
|
||||
const stopping = h.worker.stop().then(() => {
|
||||
stopped = true
|
||||
})
|
||||
await Promise.resolve()
|
||||
expect(stopped).toBe(false)
|
||||
finish({ status: 'sent' })
|
||||
await Promise.all([pending, stopping])
|
||||
expect(stopped).toBe(true)
|
||||
expect(h.send).toHaveBeenCalledOnce()
|
||||
const resumed = new DurablePushWorker(h.store, h.dispatcher, { now: h.now })
|
||||
await resumed.runDue()
|
||||
expect(h.send).toHaveBeenCalledTimes(2)
|
||||
await resumed.stop()
|
||||
})
|
||||
|
||||
it('runs due work on its timer and releases the timer on stop', async () => {
|
||||
const h = await fixture()
|
||||
vi.useFakeTimers()
|
||||
await h.accept(note(1))
|
||||
h.worker.start()
|
||||
h.worker.start()
|
||||
expect(vi.getTimerCount()).toBe(1)
|
||||
await vi.advanceTimersByTimeAsync(1000)
|
||||
await h.worker.runDue()
|
||||
expect(h.send).toHaveBeenCalledOnce()
|
||||
await h.worker.stop()
|
||||
expect(vi.getTimerCount()).toBe(0)
|
||||
})
|
||||
@@ -0,0 +1,89 @@
|
||||
import { buildPushDelivery } from './push-delivery-message.js'
|
||||
import type { PushDispatcher } from './push-dispatcher.js'
|
||||
import type { DurablePushStore } from './durable-push-store.js'
|
||||
|
||||
export class DurablePushWorker {
|
||||
private timer?: NodeJS.Timeout
|
||||
private running: Promise<void> | null = null
|
||||
private stopped = false
|
||||
constructor(
|
||||
private readonly store: DurablePushStore,
|
||||
private readonly dispatcher: PushDispatcher,
|
||||
private readonly options: { now?: () => number; onRetry?: () => void } = {}
|
||||
) {}
|
||||
|
||||
start(): void {
|
||||
if (this.timer) return
|
||||
this.stopped = false
|
||||
this.timer = setInterval(() => {
|
||||
void this.runDue().catch(() => {
|
||||
console.warn(JSON.stringify({ event: 'orca_push_worker_failed' }))
|
||||
})
|
||||
}, 1000)
|
||||
this.timer.unref()
|
||||
}
|
||||
|
||||
async runDue(): Promise<void> {
|
||||
if (this.running) {
|
||||
await this.running
|
||||
return
|
||||
}
|
||||
if (this.stopped) return
|
||||
const pending = Promise.allSettled(Array.from({ length: 4 }, () => this.drain())).then(
|
||||
(results) => {
|
||||
const failure = results.find((result) => result.status === 'rejected')
|
||||
if (failure?.status === 'rejected') throw failure.reason
|
||||
}
|
||||
)
|
||||
this.running = pending
|
||||
try {
|
||||
await pending
|
||||
} finally {
|
||||
this.running = null
|
||||
}
|
||||
}
|
||||
|
||||
private async drain(): Promise<void> {
|
||||
for (let count = 0; count < 25 && !this.stopped; count++) {
|
||||
const queued = await this.store.claim()
|
||||
if (!queued) return
|
||||
const delivery = buildPushDelivery({
|
||||
expiresAt: queued.expiresAt,
|
||||
registrationId: queued.registrationId,
|
||||
hostFingerprint: queued.hostFingerprint,
|
||||
notification: queued.notification
|
||||
})
|
||||
if ((this.options.now ?? Date.now)() >= queued.expiresAt) {
|
||||
await this.store.finish(queued)
|
||||
continue
|
||||
}
|
||||
const heartbeat = setInterval(() => {
|
||||
void this.store.renew(queued).catch(() => {})
|
||||
}, 10_000)
|
||||
heartbeat.unref()
|
||||
try {
|
||||
if (queued.attempts > 1) this.options.onRetry?.()
|
||||
const outcome = await this.dispatcher.sendOnce(delivery)
|
||||
const retryAfterMs =
|
||||
outcome.status === 'error' && outcome.retryable
|
||||
? Math.max(
|
||||
outcome.retryAfterMs ?? 0,
|
||||
Math.min(30_000, 1000 * 2 ** Math.min(queued.attempts, 5))
|
||||
)
|
||||
: undefined
|
||||
await this.store.finish(queued, retryAfterMs, outcome.status)
|
||||
} catch {
|
||||
await this.store.finish(queued, 5000)
|
||||
} finally {
|
||||
clearInterval(heartbeat)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
async stop(): Promise<void> {
|
||||
this.stopped = true
|
||||
if (this.timer) clearInterval(this.timer)
|
||||
this.timer = undefined
|
||||
await this.running
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,15 @@
|
||||
import { GoogleAuth } from 'google-auth-library'
|
||||
import { FCM_SCOPE } from './fcm-client.js'
|
||||
|
||||
// Resolves the runtime service account credential from the GCE metadata server
|
||||
// in Cloud Run and from GOOGLE_APPLICATION_CREDENTIALS locally; the library
|
||||
// caches and refreshes the token itself.
|
||||
export function createFcmAccessTokenProvider(): () => Promise<string> {
|
||||
const auth = new GoogleAuth({ scopes: [FCM_SCOPE] })
|
||||
return async () => {
|
||||
const client = await auth.getClient()
|
||||
const token = await client.getAccessToken()
|
||||
if (!token.token) throw new Error('fcm_access_token_unavailable')
|
||||
return token.token
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,240 @@
|
||||
import { createHash } from 'node:crypto'
|
||||
import { describe, expect, it } from 'vitest'
|
||||
import { fcmCollapseKey, FcmClient, type FcmRequest, type FcmResponse } from './fcm-client.js'
|
||||
import { buildPushDelivery } from './push-delivery-message.js'
|
||||
|
||||
const NOW = 1_700_000_000_000
|
||||
const HOST = 'abcdefghijklmnop'
|
||||
const TOKEN = 'cQ1abcDEF_gh:APA91bZZ-zz0123456789abcdefghijklmnopqrstuvwxyz'
|
||||
|
||||
function delivery(agentState: 'needs-input' | null = 'needs-input') {
|
||||
return buildPushDelivery({
|
||||
expiresAt: NOW + 300_000,
|
||||
registrationId: 'reg-1',
|
||||
hostFingerprint: HOST,
|
||||
notification: {
|
||||
notificationId: 'note-1',
|
||||
notificationSeq: 7,
|
||||
notificationEpoch: 'epoch-1',
|
||||
source: 'agent-task-complete',
|
||||
agentState,
|
||||
title: 'Agent needs input',
|
||||
body: 'Waiting on your answer',
|
||||
worktreeId: 'wt-1'
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
function fakeTransport(response: FcmResponse) {
|
||||
const requests: FcmRequest[] = []
|
||||
return {
|
||||
requests,
|
||||
transport: async (request: FcmRequest): Promise<FcmResponse> => {
|
||||
requests.push(request)
|
||||
return response
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
function client(response: FcmResponse) {
|
||||
const fake = fakeTransport(response)
|
||||
return {
|
||||
fake,
|
||||
client: new FcmClient({
|
||||
projectId: 'onorca-cloud',
|
||||
now: () => NOW,
|
||||
accessToken: async () => 'access-token',
|
||||
transport: fake.transport
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
describe('fcm client', () => {
|
||||
it('posts the v1 send payload for the configured project', async () => {
|
||||
const { fake, client: fcm } = client({ status: 200, body: '{"name":"projects/x/messages/1"}' })
|
||||
await expect(fcm.send(delivery(), { token: TOKEN })).resolves.toEqual({ status: 'sent' })
|
||||
const request = fake.requests[0]!
|
||||
expect(request.url).toBe('https://fcm.googleapis.com/v1/projects/onorca-cloud/messages:send')
|
||||
expect(request.accessToken).toBe('access-token')
|
||||
expect(JSON.parse(request.body)).toEqual({
|
||||
message: {
|
||||
token: TOKEN,
|
||||
notification: { title: 'Agent needs input', body: 'Waiting on your answer' },
|
||||
android: {
|
||||
priority: 'HIGH',
|
||||
ttl: '300s',
|
||||
collapse_key: createHash('sha256')
|
||||
.update(
|
||||
createHash('sha256')
|
||||
.update(JSON.stringify([HOST, 'note-1']))
|
||||
.digest('hex')
|
||||
)
|
||||
.digest('hex')
|
||||
.slice(0, 32),
|
||||
notification: {
|
||||
channel_id: 'orca-desktop',
|
||||
tag: createHash('sha256')
|
||||
.update(JSON.stringify([HOST, 'note-1']))
|
||||
.digest('hex')
|
||||
}
|
||||
},
|
||||
data: {
|
||||
hostFingerprint: HOST,
|
||||
worktreeId: 'wt-1',
|
||||
notificationId: 'note-1',
|
||||
notificationSeq: '7',
|
||||
notificationEpoch: 'epoch-1',
|
||||
source: 'agent-task-complete',
|
||||
agentState: 'needs-input'
|
||||
}
|
||||
}
|
||||
})
|
||||
})
|
||||
|
||||
it('carries every data value as a string and omits a null agent state', async () => {
|
||||
const { fake, client: fcm } = client({ status: 200, body: '{}' })
|
||||
await fcm.send(delivery(null), { token: TOKEN })
|
||||
const message = JSON.parse(fake.requests[0]!.body) as {
|
||||
message: {
|
||||
android: { collapse_key: string; notification: { tag: string } }
|
||||
data: Record<string, string>
|
||||
}
|
||||
}
|
||||
expect(Object.values(message.message.data).every((value) => typeof value === 'string')).toBe(
|
||||
true
|
||||
)
|
||||
expect(message.message.data.agentState).toBeUndefined()
|
||||
const tag = createHash('sha256')
|
||||
.update(JSON.stringify([HOST, 'note-1']))
|
||||
.digest('hex')
|
||||
expect(message.message.data.coalescedCount).toBeUndefined()
|
||||
expect(message.message.android.notification.tag).toBe(tag)
|
||||
expect(message.message.android.collapse_key).toBe(fcmCollapseKey(tag))
|
||||
expect(message.message.android.collapse_key).toHaveLength(32)
|
||||
})
|
||||
|
||||
it('marks an unregistered token dead from the status or the error detail', async () => {
|
||||
const byStatus = client({
|
||||
status: 404,
|
||||
body: JSON.stringify({ error: { status: 'UNREGISTERED', message: 'not registered' } })
|
||||
})
|
||||
await expect(byStatus.client.send(delivery(), { token: TOKEN })).resolves.toEqual({
|
||||
status: 'dead',
|
||||
reason: 'UNREGISTERED'
|
||||
})
|
||||
const byDetail = client({
|
||||
status: 404,
|
||||
body: JSON.stringify({
|
||||
error: {
|
||||
status: 'NOT_FOUND',
|
||||
message: 'Requested entity was not found.',
|
||||
details: [{ errorCode: 'UNREGISTERED' }]
|
||||
}
|
||||
})
|
||||
})
|
||||
await expect(byDetail.client.send(delivery(), { token: TOKEN })).resolves.toEqual({
|
||||
status: 'dead',
|
||||
reason: 'UNREGISTERED'
|
||||
})
|
||||
})
|
||||
|
||||
it('marks an invalid-argument that names the token dead, and others an error', async () => {
|
||||
const named = client({
|
||||
status: 400,
|
||||
body: JSON.stringify({
|
||||
error: { status: 'INVALID_ARGUMENT', message: 'The registration token is not valid.' }
|
||||
})
|
||||
})
|
||||
await expect(named.client.send(delivery(), { token: TOKEN })).resolves.toEqual({
|
||||
status: 'dead',
|
||||
reason: 'INVALID_ARGUMENT'
|
||||
})
|
||||
const unnamed = client({
|
||||
status: 400,
|
||||
body: JSON.stringify({
|
||||
error: { status: 'INVALID_ARGUMENT', message: 'Invalid value at message.android.ttl' }
|
||||
})
|
||||
})
|
||||
await expect(unnamed.client.send(delivery(), { token: TOKEN })).resolves.toEqual({
|
||||
status: 'error',
|
||||
reason: 'INVALID_ARGUMENT',
|
||||
retryable: false,
|
||||
retryAfterMs: 10000
|
||||
})
|
||||
})
|
||||
|
||||
it('treats a server fault and a transport failure as errors', async () => {
|
||||
const faulted = client({
|
||||
status: 503,
|
||||
body: JSON.stringify({ error: { status: 'UNAVAILABLE', message: 'backend busy' } })
|
||||
})
|
||||
await expect(faulted.client.send(delivery(), { token: TOKEN })).resolves.toEqual({
|
||||
status: 'error',
|
||||
reason: 'UNAVAILABLE',
|
||||
retryable: true,
|
||||
retryAfterMs: 10000
|
||||
})
|
||||
const broken = new FcmClient({
|
||||
projectId: 'onorca-cloud',
|
||||
now: () => NOW,
|
||||
accessToken: async () => 'access-token',
|
||||
transport: async () => {
|
||||
throw new Error('ECONNRESET')
|
||||
}
|
||||
})
|
||||
await expect(broken.send(delivery(), { token: TOKEN })).resolves.toEqual({
|
||||
status: 'error',
|
||||
reason: 'Error',
|
||||
retryable: true
|
||||
})
|
||||
})
|
||||
})
|
||||
|
||||
it('does not send when credential refresh crosses the absolute expiry', async () => {
|
||||
let now = 1000
|
||||
const fake = fakeTransport({ status: 200, body: '{}' })
|
||||
const fcm = new FcmClient({
|
||||
projectId: 'test',
|
||||
now: () => now,
|
||||
accessToken: async () => {
|
||||
now = 3000
|
||||
return 'test-token'
|
||||
},
|
||||
transport: fake.transport
|
||||
})
|
||||
await expect(fcm.send({ ...delivery(), expiresAt: 2000 }, { token: TOKEN })).resolves.toEqual({
|
||||
status: 'error',
|
||||
reason: 'expired'
|
||||
})
|
||||
expect(fake.requests).toHaveLength(0)
|
||||
})
|
||||
|
||||
it('decreases retry TTL and refuses expired delivery before refreshing credentials', async () => {
|
||||
let now = NOW
|
||||
let refreshes = 0
|
||||
const fake = fakeTransport({ status: 503, body: '{}' })
|
||||
const fcm = new FcmClient({
|
||||
projectId: 'test',
|
||||
now: () => now,
|
||||
accessToken: async () => {
|
||||
refreshes++
|
||||
return 'test-token'
|
||||
},
|
||||
transport: fake.transport
|
||||
})
|
||||
const pending = delivery()
|
||||
await fcm.send(pending, { token: TOKEN })
|
||||
now += 60_000
|
||||
await fcm.send(pending, { token: TOKEN })
|
||||
expect(fake.requests.map((request) => JSON.parse(request.body).message.android.ttl)).toEqual([
|
||||
'300s',
|
||||
'240s'
|
||||
])
|
||||
now = pending.expiresAt
|
||||
await expect(fcm.send(pending, { token: TOKEN })).resolves.toEqual({
|
||||
status: 'error',
|
||||
reason: 'expired'
|
||||
})
|
||||
expect(fake.requests).toHaveLength(2)
|
||||
expect(refreshes).toBe(2)
|
||||
})
|
||||
@@ -0,0 +1,147 @@
|
||||
import { providerRetryAfter } from './provider-retry-delay.js'
|
||||
import { createHash } from 'node:crypto'
|
||||
import { PUSH_DEFAULTS } from '@orca-cloud/push-contract'
|
||||
import { orcaDataStrings, type PushDelivery } from './push-delivery-message.js'
|
||||
import type { PushProviderOutcome } from './push-provider-outcome.js'
|
||||
|
||||
export const FCM_SCOPE = 'https://www.googleapis.com/auth/firebase.messaging'
|
||||
|
||||
export type FcmRequest = { url: string; accessToken: string; body: string }
|
||||
export type FcmResponse = { status: number; body: string; retryAfterMs?: number }
|
||||
export type FcmTransport = (request: FcmRequest) => Promise<FcmResponse>
|
||||
|
||||
export type FcmClientOptions = {
|
||||
projectId: string
|
||||
accessToken: () => Promise<string>
|
||||
transport: FcmTransport
|
||||
channelId?: string
|
||||
now?: () => number
|
||||
}
|
||||
|
||||
type FcmErrorBody = {
|
||||
error?: { status?: unknown; message?: unknown; details?: { errorCode?: unknown }[] }
|
||||
}
|
||||
|
||||
// FCM collapse_key is a short opaque string, so the collapse id is hashed
|
||||
// rather than truncated: truncation would merge unrelated notifications.
|
||||
export function fcmCollapseKey(collapseId: string): string {
|
||||
return createHash('sha256').update(collapseId).digest('hex').slice(0, 32)
|
||||
}
|
||||
|
||||
export function fcmMessageBody(input: {
|
||||
delivery: PushDelivery
|
||||
token: string
|
||||
channelId: string
|
||||
now?: number
|
||||
}): string {
|
||||
const { delivery } = input
|
||||
const now = input.now ?? Date.now()
|
||||
return JSON.stringify({
|
||||
message: {
|
||||
token: input.token,
|
||||
...(delivery.orca.kind === 'dismiss'
|
||||
? {}
|
||||
: { notification: { title: delivery.title, body: delivery.body } }),
|
||||
android: {
|
||||
priority: 'HIGH',
|
||||
ttl: `${Math.max(0, Math.ceil((delivery.expiresAt - now) / 1000))}s`,
|
||||
collapse_key: fcmCollapseKey(delivery.collapseId),
|
||||
...(delivery.orca.kind === 'dismiss'
|
||||
? {}
|
||||
: {
|
||||
notification: {
|
||||
channel_id:
|
||||
delivery.sound === false ? `${input.channelId}-silent` : input.channelId,
|
||||
tag: delivery.collapseId
|
||||
}
|
||||
})
|
||||
},
|
||||
data: orcaDataStrings(delivery.orca)
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
function readFcmError(body: string): { status: string; message: string; errorCodes: string[] } {
|
||||
try {
|
||||
const parsed = JSON.parse(body) as FcmErrorBody
|
||||
return {
|
||||
status: typeof parsed.error?.status === 'string' ? parsed.error.status : 'unknown',
|
||||
message: typeof parsed.error?.message === 'string' ? parsed.error.message : '',
|
||||
errorCodes: (parsed.error?.details ?? [])
|
||||
.map((detail) => detail.errorCode)
|
||||
.filter((code): code is string => typeof code === 'string')
|
||||
}
|
||||
} catch {
|
||||
return { status: 'unparseable', message: '', errorCodes: [] }
|
||||
}
|
||||
}
|
||||
|
||||
export class FcmClient {
|
||||
private readonly channelId: string
|
||||
|
||||
constructor(private readonly options: FcmClientOptions) {
|
||||
this.channelId = options.channelId ?? PUSH_DEFAULTS.androidChannelId
|
||||
}
|
||||
|
||||
async send(delivery: PushDelivery, device: { token: string }): Promise<PushProviderOutcome> {
|
||||
if (delivery.expiresAt <= (this.options.now ?? Date.now)())
|
||||
return { status: 'error', reason: 'expired' }
|
||||
let response: FcmResponse
|
||||
try {
|
||||
const accessToken = await this.options.accessToken()
|
||||
const now = (this.options.now ?? Date.now)()
|
||||
if (delivery.expiresAt <= now) return { status: 'error', reason: 'expired' }
|
||||
response = await this.options.transport({
|
||||
url: `https://fcm.googleapis.com/v1/projects/${this.options.projectId}/messages:send`,
|
||||
accessToken,
|
||||
body: fcmMessageBody({
|
||||
delivery,
|
||||
token: device.token,
|
||||
channelId: this.channelId,
|
||||
now
|
||||
})
|
||||
})
|
||||
} catch (error) {
|
||||
return {
|
||||
status: 'error',
|
||||
reason: error instanceof Error ? error.name : 'transport_failed',
|
||||
retryable: true
|
||||
}
|
||||
}
|
||||
if (response.status >= 200 && response.status < 300) return { status: 'sent' }
|
||||
const failure = readFcmError(response.body)
|
||||
if (failure.status === 'UNREGISTERED' || failure.errorCodes.includes('UNREGISTERED')) {
|
||||
return { status: 'dead', reason: 'UNREGISTERED' }
|
||||
}
|
||||
// A revoked token also surfaces as INVALID_ARGUMENT naming the token field.
|
||||
if (failure.status === 'INVALID_ARGUMENT' && /\btoken\b/i.test(failure.message)) {
|
||||
return { status: 'dead', reason: 'INVALID_ARGUMENT' }
|
||||
}
|
||||
return {
|
||||
status: 'error',
|
||||
reason: failure.status,
|
||||
retryable: response.status === 429 || response.status >= 500,
|
||||
retryAfterMs: Math.max(response.status === 429 ? 60_000 : 10_000, response.retryAfterMs ?? 0)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
export function createFcmFetchTransport(fetchImpl: typeof fetch = fetch): FcmTransport {
|
||||
return async (request) => {
|
||||
const response = await fetchImpl(request.url, {
|
||||
method: 'POST',
|
||||
headers: {
|
||||
authorization: `Bearer ${request.accessToken}`,
|
||||
'content-type': 'application/json'
|
||||
},
|
||||
body: request.body,
|
||||
redirect: 'error',
|
||||
signal: AbortSignal.timeout(10_000)
|
||||
})
|
||||
return {
|
||||
status: response.status,
|
||||
body: await response.text(),
|
||||
retryAfterMs: providerRetryAfter(response.headers.get('retry-after') ?? undefined)
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,163 @@
|
||||
import { createHmac, timingSafeEqual } from 'node:crypto'
|
||||
import {
|
||||
PUSH_HOST_CHALLENGE_PLAINTEXT_DOMAIN,
|
||||
PUSH_HOST_PROOF_TRANSCRIPT_DOMAIN,
|
||||
PUSH_HOST_PROOF_TRANSCRIPT_FIELD_COUNT,
|
||||
PUSH_LIMITS
|
||||
} from '@orca-cloud/push-contract'
|
||||
import nacl from 'tweetnacl'
|
||||
import { decodeCanonicalBase64 } from './canonical-base64.js'
|
||||
import { deriveHostFingerprint } from './host-fingerprint.js'
|
||||
|
||||
// The desktop side of the push challenge, written the way the shipped host
|
||||
// will answer it, so the gateway is exercised against a real box-opening peer.
|
||||
const textEncoder = new TextEncoder()
|
||||
const textDecoder = new TextDecoder()
|
||||
|
||||
export type PushHostKeypair = { publicKey: Uint8Array; secretKey: Uint8Array }
|
||||
|
||||
export type PushChallengeWire = {
|
||||
challengeId: string
|
||||
gatewayEphemeralPublicKeyB64: string
|
||||
nonceB64: string
|
||||
ciphertextB64: string
|
||||
expiresAt: number
|
||||
}
|
||||
|
||||
export function createPushHostKeypair(seed?: number): PushHostKeypair {
|
||||
const pair =
|
||||
seed === undefined
|
||||
? nacl.box.keyPair()
|
||||
: nacl.box.keyPair.fromSecretKey(new Uint8Array(32).fill(seed))
|
||||
return { publicKey: pair.publicKey, secretKey: pair.secretKey }
|
||||
}
|
||||
|
||||
export function hostPublicKeyB64(keypair: PushHostKeypair): string {
|
||||
return Buffer.from(keypair.publicKey).toString('base64')
|
||||
}
|
||||
|
||||
function equal(left: Uint8Array | undefined, right: Uint8Array): boolean {
|
||||
return Boolean(left && left.byteLength === right.byteLength && timingSafeEqual(left, right))
|
||||
}
|
||||
|
||||
function uint64(value: number): Uint8Array {
|
||||
const bytes = new Uint8Array(8)
|
||||
new DataView(bytes.buffer).setBigUint64(0, BigInt(value), false)
|
||||
return bytes
|
||||
}
|
||||
|
||||
function parseTranscript(transcript: Uint8Array): Map<string, Uint8Array> | null {
|
||||
const fields = new Map<string, Uint8Array>()
|
||||
const view = new DataView(transcript.buffer, transcript.byteOffset, transcript.byteLength)
|
||||
let offset = 0
|
||||
try {
|
||||
while (offset < transcript.byteLength) {
|
||||
const nameLength = view.getUint32(offset, false)
|
||||
offset += 4
|
||||
const name = textDecoder.decode(transcript.slice(offset, offset + nameLength))
|
||||
offset += nameLength
|
||||
const valueLength = view.getUint32(offset, false)
|
||||
offset += 4
|
||||
if (fields.has(name) || offset + valueLength > transcript.byteLength) return null
|
||||
fields.set(name, transcript.slice(offset, offset + valueLength))
|
||||
offset += valueLength
|
||||
}
|
||||
} catch {
|
||||
return null
|
||||
}
|
||||
return offset === transcript.byteLength ? fields : null
|
||||
}
|
||||
|
||||
function readUint64(value: Uint8Array | undefined): number | null {
|
||||
if (!value || value.byteLength !== 8) return null
|
||||
const parsed = new DataView(value.buffer, value.byteOffset, value.byteLength).getBigUint64(0, false)
|
||||
return parsed <= BigInt(Number.MAX_SAFE_INTEGER) ? Number(parsed) : null
|
||||
}
|
||||
|
||||
export type PushHostProofContext = {
|
||||
gatewayOrigin: string
|
||||
keypair: PushHostKeypair
|
||||
now?: () => number
|
||||
onInvalid?: (reason: string) => void
|
||||
}
|
||||
|
||||
function validateTranscript(
|
||||
transcript: Uint8Array,
|
||||
challenge: PushChallengeWire,
|
||||
context: PushHostProofContext,
|
||||
gatewayKey: Uint8Array,
|
||||
nonce: Uint8Array
|
||||
): boolean {
|
||||
const fields = parseTranscript(transcript)
|
||||
if (!fields || fields.size !== PUSH_HOST_PROOF_TRANSCRIPT_FIELD_COUNT) {
|
||||
context.onInvalid?.('transcript-structure')
|
||||
return false
|
||||
}
|
||||
const now = (context.now ?? Date.now)()
|
||||
const issuedAt = readUint64(fields.get('issuedAt'))
|
||||
const expiresAt = readUint64(fields.get('expiresAt'))
|
||||
const fingerprint = deriveHostFingerprint(context.keypair.publicKey)
|
||||
const checks: [string, boolean][] = [
|
||||
['issuedAt-readable', issuedAt !== null],
|
||||
[
|
||||
'issuedAt-not-future',
|
||||
issuedAt === null || issuedAt - PUSH_LIMITS.clockSkewToleranceMs <= now
|
||||
],
|
||||
['not-expired', now - PUSH_LIMITS.clockSkewToleranceMs <= challenge.expiresAt],
|
||||
['issuedAt-before-expiry', issuedAt === null || issuedAt <= challenge.expiresAt],
|
||||
[
|
||||
'window',
|
||||
issuedAt === null || challenge.expiresAt - issuedAt <= PUSH_LIMITS.challengeTtlMs
|
||||
],
|
||||
['expiry-consistent', expiresAt === challenge.expiresAt],
|
||||
['protocol', equal(fields.get('protocol'), textEncoder.encode(PUSH_HOST_PROOF_TRANSCRIPT_DOMAIN))],
|
||||
['version', equal(fields.get('version'), new Uint8Array([1]))],
|
||||
['gatewayOrigin', equal(fields.get('gatewayOrigin'), textEncoder.encode(context.gatewayOrigin))],
|
||||
['gatewayEphemeralPublicKey', equal(fields.get('gatewayEphemeralPublicKey'), gatewayKey)],
|
||||
['challengeNonce', equal(fields.get('challengeNonce'), nonce)],
|
||||
['challengeId', equal(fields.get('challengeId'), textEncoder.encode(challenge.challengeId))],
|
||||
['hostFingerprint', equal(fields.get('hostFingerprint'), textEncoder.encode(fingerprint))],
|
||||
['hostPublicKey', equal(fields.get('hostPublicKey'), context.keypair.publicKey)],
|
||||
['issuedAt-value', issuedAt === null || uint64(issuedAt).byteLength === 8]
|
||||
]
|
||||
const failed = checks.filter(([, ok]) => !ok).map(([name]) => name)
|
||||
if (failed.length === 0) return true
|
||||
context.onInvalid?.(`transcript:${failed.join('+')}`)
|
||||
return false
|
||||
}
|
||||
|
||||
export function answerPushHostChallenge(
|
||||
challenge: PushChallengeWire,
|
||||
context: PushHostProofContext
|
||||
): string | null {
|
||||
const gatewayKey = decodeCanonicalBase64(challenge.gatewayEphemeralPublicKeyB64, 32)
|
||||
const nonce = decodeCanonicalBase64(challenge.nonceB64, 24)
|
||||
const ciphertext = Buffer.from(challenge.ciphertextB64, 'base64')
|
||||
if (!gatewayKey || !nonce || ciphertext.toString('base64') !== challenge.ciphertextB64) return null
|
||||
const plaintext = nacl.box.open(ciphertext, nonce, gatewayKey, context.keypair.secretKey)
|
||||
if (!plaintext) {
|
||||
context.onInvalid?.('challenge-box-open')
|
||||
return null
|
||||
}
|
||||
const domain = textEncoder.encode(`${PUSH_HOST_CHALLENGE_PLAINTEXT_DOMAIN}\0`)
|
||||
if (
|
||||
!equal(plaintext.slice(0, domain.byteLength), domain) ||
|
||||
plaintext.byteLength < domain.byteLength + 36
|
||||
) {
|
||||
return null
|
||||
}
|
||||
const transcriptLength = new DataView(
|
||||
plaintext.buffer,
|
||||
plaintext.byteOffset + domain.byteLength,
|
||||
4
|
||||
).getUint32(0, false)
|
||||
const transcriptStart = domain.byteLength + 4
|
||||
const secretStart = transcriptStart + transcriptLength
|
||||
if (secretStart + 32 !== plaintext.byteLength) return null
|
||||
const transcript = plaintext.slice(transcriptStart, secretStart)
|
||||
if (!validateTranscript(transcript, challenge, context, gatewayKey, nonce)) return null
|
||||
return createHmac('sha256', plaintext.slice(secretStart))
|
||||
.update(textEncoder.encode(`${PUSH_HOST_PROOF_TRANSCRIPT_DOMAIN}\0ack\0`))
|
||||
.update(transcript)
|
||||
.digest('base64')
|
||||
}
|
||||
@@ -0,0 +1,191 @@
|
||||
import { PUSH_LIMITS } from '@orca-cloud/push-contract'
|
||||
import { afterEach, beforeEach, describe, expect, it } from 'vitest'
|
||||
import {
|
||||
answerPushHostChallenge,
|
||||
createPushHostKeypair,
|
||||
hostPublicKeyB64
|
||||
} from './host-challenge-answering.test-fixture.js'
|
||||
import { PushHostChallengeStore } from './host-challenge-store.js'
|
||||
import { deriveHostFingerprint } from './host-fingerprint.js'
|
||||
import { openInMemoryPushDatabase, type PushDatabase } from './push-database.js'
|
||||
|
||||
const GATEWAY_ORIGIN = 'https://push.onorca.dev'
|
||||
|
||||
describe('push host challenge store', () => {
|
||||
let database: PushDatabase
|
||||
let clock = 1_700_000_000_000
|
||||
let store: PushHostChallengeStore
|
||||
|
||||
beforeEach(async () => {
|
||||
database = await openInMemoryPushDatabase()
|
||||
clock = 1_700_000_000_000
|
||||
store = new PushHostChallengeStore(database, GATEWAY_ORIGIN, () => clock)
|
||||
})
|
||||
|
||||
afterEach(async () => {
|
||||
await database.close()
|
||||
})
|
||||
|
||||
it('completes a challenge, proof, and consume round trip', async () => {
|
||||
const host = createPushHostKeypair(1)
|
||||
const challenge = await store.issue(hostPublicKeyB64(host))
|
||||
expect(challenge).not.toBeNull()
|
||||
expect(challenge!.expiresAt).toBe(clock + PUSH_LIMITS.challengeTtlMs)
|
||||
expect(challenge!.hostFingerprint).toBe(deriveHostFingerprint(host.publicKey))
|
||||
|
||||
const proof = answerPushHostChallenge(challenge!, {
|
||||
gatewayOrigin: GATEWAY_ORIGIN,
|
||||
keypair: host,
|
||||
now: () => clock
|
||||
})
|
||||
expect(proof).not.toBeNull()
|
||||
await expect(store.verify(challenge!.challengeId, proof!)).resolves.toEqual({
|
||||
ok: true,
|
||||
hostFingerprint: deriveHostFingerprint(host.publicKey)
|
||||
})
|
||||
})
|
||||
|
||||
it('never stores material that reproduces the proof', async () => {
|
||||
const host = createPushHostKeypair(2)
|
||||
const challenge = await store.issue(hostPublicKeyB64(host))
|
||||
const proof = answerPushHostChallenge(challenge!, {
|
||||
gatewayOrigin: GATEWAY_ORIGIN,
|
||||
keypair: host,
|
||||
now: () => clock
|
||||
})
|
||||
const [row] = await database.query('SELECT secret_hash FROM push_challenges')
|
||||
expect(String(row?.secret_hash)).not.toBe(proof)
|
||||
expect(Buffer.from(String(row?.secret_hash), 'base64url').byteLength).toBe(32)
|
||||
})
|
||||
|
||||
it('rejects a replayed challenge', async () => {
|
||||
const host = createPushHostKeypair(3)
|
||||
const challenge = await store.issue(hostPublicKeyB64(host))
|
||||
const proof = answerPushHostChallenge(challenge!, {
|
||||
gatewayOrigin: GATEWAY_ORIGIN,
|
||||
keypair: host,
|
||||
now: () => clock
|
||||
})!
|
||||
await expect(store.verify(challenge!.challengeId, proof)).resolves.toMatchObject({ ok: true })
|
||||
await expect(store.verify(challenge!.challengeId, proof)).resolves.toEqual({
|
||||
ok: false,
|
||||
reason: 'already_consumed'
|
||||
})
|
||||
})
|
||||
|
||||
it('rejects a challenge the moment its own ttl elapses', async () => {
|
||||
const host = createPushHostKeypair(4)
|
||||
const challenge = await store.issue(hostPublicKeyB64(host))
|
||||
const proof = answerPushHostChallenge(challenge!, {
|
||||
gatewayOrigin: GATEWAY_ORIGIN,
|
||||
keypair: host,
|
||||
now: () => clock
|
||||
})!
|
||||
clock += PUSH_LIMITS.challengeTtlMs + 1
|
||||
await expect(store.verify(challenge!.challengeId, proof)).resolves.toEqual({
|
||||
ok: false,
|
||||
reason: 'expired'
|
||||
})
|
||||
})
|
||||
|
||||
it('spends no skew tolerance on its own expiry, so the ttl is the whole window', async () => {
|
||||
const host = createPushHostKeypair(5)
|
||||
const challenge = await store.issue(hostPublicKeyB64(host))
|
||||
const proof = answerPushHostChallenge(challenge!, {
|
||||
gatewayOrigin: GATEWAY_ORIGIN,
|
||||
keypair: host,
|
||||
now: () => clock
|
||||
})!
|
||||
// A proof that the host would still consider in-window is refused here: the
|
||||
// gateway issued expires_at against this clock and needs no allowance.
|
||||
clock += PUSH_LIMITS.challengeTtlMs + PUSH_LIMITS.clockSkewToleranceMs - 1
|
||||
await expect(store.verify(challenge!.challengeId, proof)).resolves.toEqual({
|
||||
ok: false,
|
||||
reason: 'expired'
|
||||
})
|
||||
})
|
||||
|
||||
it('accepts a proof that lands just inside the ttl', async () => {
|
||||
const host = createPushHostKeypair(26)
|
||||
const challenge = await store.issue(hostPublicKeyB64(host))
|
||||
const proof = answerPushHostChallenge(challenge!, {
|
||||
gatewayOrigin: GATEWAY_ORIGIN,
|
||||
keypair: host,
|
||||
now: () => clock
|
||||
})!
|
||||
clock += PUSH_LIMITS.challengeTtlMs
|
||||
await expect(store.verify(challenge!.challengeId, proof)).resolves.toMatchObject({ ok: true })
|
||||
})
|
||||
|
||||
it('keeps an expired row long enough to answer expired rather than unknown', async () => {
|
||||
const host = createPushHostKeypair(27)
|
||||
const challenge = await store.issue(hostPublicKeyB64(host))
|
||||
const proof = answerPushHostChallenge(challenge!, {
|
||||
gatewayOrigin: GATEWAY_ORIGIN,
|
||||
keypair: host,
|
||||
now: () => clock
|
||||
})!
|
||||
clock += PUSH_LIMITS.challengeTtlMs + 1
|
||||
expect(await store.pruneExpired()).toBe(0)
|
||||
await expect(store.verify(challenge!.challengeId, proof)).resolves.toEqual({
|
||||
ok: false,
|
||||
reason: 'expired'
|
||||
})
|
||||
})
|
||||
|
||||
it('refuses a wrong host: the box will not open and a foreign proof will not match', async () => {
|
||||
const owner = createPushHostKeypair(6)
|
||||
const intruder = createPushHostKeypair(7)
|
||||
const ownerChallenge = await store.issue(hostPublicKeyB64(owner))
|
||||
expect(
|
||||
answerPushHostChallenge(ownerChallenge!, {
|
||||
gatewayOrigin: GATEWAY_ORIGIN,
|
||||
keypair: intruder,
|
||||
now: () => clock
|
||||
})
|
||||
).toBeNull()
|
||||
|
||||
const intruderChallenge = await store.issue(hostPublicKeyB64(intruder))
|
||||
const intruderProof = answerPushHostChallenge(intruderChallenge!, {
|
||||
gatewayOrigin: GATEWAY_ORIGIN,
|
||||
keypair: intruder,
|
||||
now: () => clock
|
||||
})!
|
||||
await expect(store.verify(ownerChallenge!.challengeId, intruderProof)).resolves.toEqual({
|
||||
ok: false,
|
||||
reason: 'proof_mismatch'
|
||||
})
|
||||
})
|
||||
|
||||
it('rejects a proof bound to a different gateway origin', async () => {
|
||||
const host = createPushHostKeypair(8)
|
||||
const challenge = await store.issue(hostPublicKeyB64(host))
|
||||
const reasons: string[] = []
|
||||
expect(
|
||||
answerPushHostChallenge(challenge!, {
|
||||
gatewayOrigin: 'https://push.example.test',
|
||||
keypair: host,
|
||||
now: () => clock,
|
||||
onInvalid: (reason) => reasons.push(reason)
|
||||
})
|
||||
).toBeNull()
|
||||
expect(reasons.join()).toContain('gatewayOrigin')
|
||||
})
|
||||
|
||||
it('rejects an unknown challenge id and a malformed public key', async () => {
|
||||
await expect(store.verify('missing', Buffer.alloc(32, 9).toString('base64'))).resolves.toEqual({
|
||||
ok: false,
|
||||
reason: 'unknown_challenge'
|
||||
})
|
||||
await expect(store.issue('not-base64!!')).resolves.toBeNull()
|
||||
await expect(store.issue(Buffer.alloc(31, 1).toString('base64'))).resolves.toBeNull()
|
||||
})
|
||||
|
||||
it('prunes challenges that fell out of the skew window', async () => {
|
||||
const host = createPushHostKeypair(9)
|
||||
await store.issue(hostPublicKeyB64(host))
|
||||
expect(await store.pruneExpired()).toBe(0)
|
||||
clock += PUSH_LIMITS.challengeTtlMs + PUSH_LIMITS.clockSkewToleranceMs + 1
|
||||
expect(await store.pruneExpired()).toBe(1)
|
||||
})
|
||||
})
|
||||
@@ -0,0 +1,135 @@
|
||||
import { createHash, createHmac, randomBytes, randomUUID, timingSafeEqual } from 'node:crypto'
|
||||
import {
|
||||
buildPushHostChallengePlaintext,
|
||||
buildPushHostProofMacInput,
|
||||
buildPushHostProofTranscript,
|
||||
PUSH_LIMITS
|
||||
} from '@orca-cloud/push-contract'
|
||||
import nacl from 'tweetnacl'
|
||||
import { decodeCanonicalBase64 } from './canonical-base64.js'
|
||||
import { deriveHostFingerprint } from './host-fingerprint.js'
|
||||
import type { PushDatabase } from './push-database.js'
|
||||
|
||||
export type IssuedPushChallenge = {
|
||||
challengeId: string
|
||||
gatewayEphemeralPublicKeyB64: string
|
||||
nonceB64: string
|
||||
ciphertextB64: string
|
||||
expiresAt: number
|
||||
hostFingerprint: string
|
||||
}
|
||||
|
||||
export type PushProofVerification =
|
||||
| { ok: true; hostFingerprint: string }
|
||||
| { ok: false; reason: 'unknown_challenge' | 'already_consumed' | 'expired' | 'proof_mismatch' }
|
||||
|
||||
function sha256(value: Uint8Array): string {
|
||||
return createHash('sha256').update(value).digest('base64url')
|
||||
}
|
||||
|
||||
function equalDigest(left: string, right: string): boolean {
|
||||
const leftBytes = Buffer.from(left)
|
||||
const rightBytes = Buffer.from(right)
|
||||
return leftBytes.length === rightBytes.length && timingSafeEqual(leftBytes, rightBytes)
|
||||
}
|
||||
|
||||
export class PushHostChallengeStore {
|
||||
constructor(
|
||||
private readonly database: PushDatabase,
|
||||
private readonly gatewayOrigin: string,
|
||||
private readonly now: () => number = Date.now
|
||||
) {}
|
||||
|
||||
async issue(hostPublicKeyB64: string): Promise<IssuedPushChallenge | null> {
|
||||
const hostPublicKey = decodeCanonicalBase64(hostPublicKeyB64, 32)
|
||||
if (!hostPublicKey) return null
|
||||
const hostFingerprint = deriveHostFingerprint(hostPublicKey)
|
||||
const ephemeral = nacl.box.keyPair()
|
||||
const challengeNonce = randomBytes(nacl.box.nonceLength)
|
||||
const challengeSecret = randomBytes(32)
|
||||
const challengeId = randomUUID()
|
||||
const issuedAt = this.now()
|
||||
const expiresAt = issuedAt + PUSH_LIMITS.challengeTtlMs
|
||||
const transcript = buildPushHostProofTranscript({
|
||||
gatewayOrigin: this.gatewayOrigin,
|
||||
gatewayEphemeralPublicKey: ephemeral.publicKey,
|
||||
challengeNonce,
|
||||
challengeId,
|
||||
issuedAt,
|
||||
expiresAt,
|
||||
hostFingerprint,
|
||||
hostPublicKey
|
||||
})
|
||||
const ciphertext = nacl.box(
|
||||
buildPushHostChallengePlaintext(transcript, challengeSecret),
|
||||
challengeNonce,
|
||||
hostPublicKey,
|
||||
ephemeral.secretKey
|
||||
)
|
||||
const expectedProof = createHmac('sha256', challengeSecret)
|
||||
.update(buildPushHostProofMacInput(transcript))
|
||||
.digest()
|
||||
await this.database.query(
|
||||
`INSERT INTO push_challenges
|
||||
(challenge_id, host_fingerprint, secret_hash, expires_at,
|
||||
consumed_at)
|
||||
VALUES (?, ?, ?, ?, NULL)`,
|
||||
[
|
||||
challengeId,
|
||||
hostFingerprint,
|
||||
// The stored digest is of the ack the secret produces, never of the
|
||||
// secret itself: a database reader must not be able to forge a proof.
|
||||
sha256(expectedProof),
|
||||
expiresAt
|
||||
]
|
||||
)
|
||||
return {
|
||||
challengeId,
|
||||
gatewayEphemeralPublicKeyB64: Buffer.from(ephemeral.publicKey).toString('base64'),
|
||||
nonceB64: Buffer.from(challengeNonce).toString('base64'),
|
||||
ciphertextB64: Buffer.from(ciphertext).toString('base64'),
|
||||
expiresAt,
|
||||
hostFingerprint
|
||||
}
|
||||
}
|
||||
|
||||
async verify(challengeId: string, proofB64: string): Promise<PushProofVerification> {
|
||||
const proof = decodeCanonicalBase64(proofB64, 32)
|
||||
return await this.database.transaction<PushProofVerification>(async (transaction) => {
|
||||
const [row] = await transaction.query(
|
||||
`SELECT host_fingerprint, secret_hash, expires_at, consumed_at
|
||||
FROM push_challenges WHERE challenge_id = ?`,
|
||||
[challengeId]
|
||||
)
|
||||
if (!row) return { ok: false, reason: 'unknown_challenge' }
|
||||
if (row.consumed_at !== null && row.consumed_at !== undefined) {
|
||||
return { ok: false, reason: 'already_consumed' }
|
||||
}
|
||||
const now = this.now()
|
||||
// No skew allowance here: the gateway set expires_at from this same clock.
|
||||
// The tolerance belongs to the host, which validates a foreign timestamp.
|
||||
if (now > Number(row.expires_at)) return { ok: false, reason: 'expired' }
|
||||
if (!proof || !equalDigest(sha256(proof), String(row.secret_hash))) {
|
||||
return { ok: false, reason: 'proof_mismatch' }
|
||||
}
|
||||
// Consume under the same predicate the read used, so two concurrent
|
||||
// proofs for one challenge cannot both mint a session.
|
||||
const [consumed] = await transaction.query(
|
||||
'UPDATE push_challenges SET consumed_at = ? WHERE challenge_id = ? AND consumed_at IS NULL',
|
||||
[now, challengeId]
|
||||
)
|
||||
if (Number(consumed?.changes ?? 0) !== 1) return { ok: false, reason: 'already_consumed' }
|
||||
return { ok: true, hostFingerprint: String(row.host_fingerprint) }
|
||||
})
|
||||
}
|
||||
|
||||
// Rows outlive the expiry check by the skew tolerance so a late proof reads
|
||||
// as 'expired' rather than as an unknown challenge.
|
||||
async pruneExpired(): Promise<number> {
|
||||
const cutoff = this.now() - PUSH_LIMITS.clockSkewToleranceMs
|
||||
const [result] = await this.database.query('DELETE FROM push_challenges WHERE expires_at < ?', [
|
||||
cutoff
|
||||
])
|
||||
return Number(result?.changes ?? 0)
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,16 @@
|
||||
import { createHash } from 'node:crypto'
|
||||
import { PUSH_HOST_FINGERPRINT_LENGTH } from '@orca-cloud/push-contract'
|
||||
|
||||
// Identical derivation to deriveRelayHostId on the desktop, so a host and a
|
||||
// phone reach the same fingerprint from the same X25519 public key.
|
||||
export function deriveHostFingerprint(hostPublicKey: Uint8Array): string {
|
||||
return createHash('sha256')
|
||||
.update(hostPublicKey)
|
||||
.digest('base64url')
|
||||
.slice(0, PUSH_HOST_FINGERPRINT_LENGTH)
|
||||
}
|
||||
|
||||
// Logs may carry at most this much of a fingerprint.
|
||||
export function fingerprintLogPrefix(hostFingerprint: string): string {
|
||||
return hostFingerprint.slice(0, 4)
|
||||
}
|
||||
@@ -0,0 +1,47 @@
|
||||
import { expect, it } from 'vitest'
|
||||
import { openInMemoryPushDatabase, openPushDatabase } from './push-database.js'
|
||||
import { PushHostChallengeStore } from './host-challenge-store.js'
|
||||
import {
|
||||
answerPushHostChallenge,
|
||||
createPushHostKeypair,
|
||||
hostPublicKeyB64
|
||||
} from './host-challenge-answering.test-fixture.js'
|
||||
|
||||
it('accepts independent proofs but consumes each challenge only once under concurrency', async () => {
|
||||
const databaseUrl = process.env.ORCA_PUSH_TEST_DATABASE_URL
|
||||
if (databaseUrl && !process.env.CI && new URL(databaseUrl).port !== '55440') {
|
||||
throw new Error('isolated_postgres_port_required')
|
||||
}
|
||||
const db = databaseUrl
|
||||
? await openPushDatabase({ databaseUrl, dataDir: '', poolMax: 4 })
|
||||
: await openInMemoryPushDatabase()
|
||||
const host = createPushHostKeypair()
|
||||
const origin = 'https://push.onorca.dev'
|
||||
const store = new PushHostChallengeStore(db, origin)
|
||||
const challenges = await Promise.all([
|
||||
store.issue(hostPublicKeyB64(host)),
|
||||
store.issue(hostPublicKeyB64(host))
|
||||
])
|
||||
try {
|
||||
const proofs = challenges.map((challenge) =>
|
||||
answerPushHostChallenge(challenge!, { gatewayOrigin: origin, keypair: host })!
|
||||
)
|
||||
const results = await Promise.all(
|
||||
challenges.flatMap((challenge, index) =>
|
||||
Array.from({ length: 5 }, () => store.verify(challenge!.challengeId, proofs[index]!))
|
||||
)
|
||||
)
|
||||
expect(results.filter((result) => result.ok)).toEqual([
|
||||
{ ok: true, hostFingerprint: challenges[0]!.hostFingerprint },
|
||||
{ ok: true, hostFingerprint: challenges[0]!.hostFingerprint }
|
||||
])
|
||||
expect(results.filter((result) => !result.ok)).toEqual(
|
||||
Array.from({ length: 8 }, () => ({ ok: false, reason: 'already_consumed' }))
|
||||
)
|
||||
} finally {
|
||||
for (const challenge of challenges) {
|
||||
await db.query('DELETE FROM push_challenges WHERE challenge_id = ?', [challenge!.challengeId])
|
||||
}
|
||||
await db.close()
|
||||
}
|
||||
})
|
||||
@@ -0,0 +1,70 @@
|
||||
import { PUSH_LIMITS } from '@orca-cloud/push-contract'
|
||||
import { afterEach, beforeEach, describe, expect, it } from 'vitest'
|
||||
import { PushHostSessionStore } from './host-session-store.js'
|
||||
import { openInMemoryPushDatabase, type PushDatabase } from './push-database.js'
|
||||
|
||||
const HOST = 'abcdefghijklmnop'
|
||||
|
||||
describe('push host session store', () => {
|
||||
let database: PushDatabase
|
||||
let clock = 1_700_000_000_000
|
||||
let sessions: PushHostSessionStore
|
||||
|
||||
beforeEach(async () => {
|
||||
database = await openInMemoryPushDatabase()
|
||||
clock = 1_700_000_000_000
|
||||
sessions = new PushHostSessionStore(database, () => clock)
|
||||
})
|
||||
|
||||
afterEach(async () => {
|
||||
await database.close()
|
||||
})
|
||||
|
||||
it('mints a 24 hour session and stores only its hash', async () => {
|
||||
const session = await sessions.create(HOST)
|
||||
expect(session.expiresAt).toBe(clock + PUSH_LIMITS.sessionTtlMs)
|
||||
expect(Buffer.from(session.sessionToken, 'base64url').byteLength).toBe(32)
|
||||
const [row] = await database.query('SELECT token_hash FROM push_sessions')
|
||||
expect(String(row?.token_hash)).not.toBe(session.sessionToken)
|
||||
await expect(sessions.resolve(session.sessionToken)).resolves.toMatchObject({
|
||||
ok: true,
|
||||
hostFingerprint: HOST
|
||||
})
|
||||
})
|
||||
|
||||
it('reports expiry separately from an unknown token', async () => {
|
||||
const session = await sessions.create(HOST)
|
||||
clock += PUSH_LIMITS.sessionTtlMs + 1
|
||||
await expect(sessions.resolve(session.sessionToken)).resolves.toEqual({
|
||||
ok: false,
|
||||
reason: 'session_expired'
|
||||
})
|
||||
await expect(sessions.resolve('not-a-session')).resolves.toEqual({
|
||||
ok: false,
|
||||
reason: 'unknown_session'
|
||||
})
|
||||
})
|
||||
|
||||
it('accepts a session on its final millisecond', async () => {
|
||||
const session = await sessions.create(HOST)
|
||||
clock += PUSH_LIMITS.sessionTtlMs
|
||||
await expect(sessions.resolve(session.sessionToken)).resolves.toMatchObject({ ok: true })
|
||||
})
|
||||
|
||||
it('keeps one live session per host and prunes it once expired', async () => {
|
||||
const first = await sessions.create(HOST)
|
||||
const second = await sessions.create(HOST)
|
||||
// The earlier session is gone the moment its host proves again, so a flood
|
||||
// of proofs leaves one row per host rather than one per proof.
|
||||
await expect(sessions.resolve(first.sessionToken)).resolves.toEqual({
|
||||
ok: false,
|
||||
reason: 'unknown_session'
|
||||
})
|
||||
await expect(sessions.resolve(second.sessionToken)).resolves.toMatchObject({ ok: true })
|
||||
const other = await sessions.create('ponmlkjihgfedcba')
|
||||
await expect(sessions.resolve(second.sessionToken)).resolves.toMatchObject({ ok: true })
|
||||
clock += PUSH_LIMITS.sessionTtlMs + 1
|
||||
expect(await sessions.pruneExpired()).toBe(2)
|
||||
await expect(sessions.resolve(other.sessionToken)).resolves.toMatchObject({ ok: false })
|
||||
})
|
||||
})
|
||||
@@ -0,0 +1,65 @@
|
||||
import { createHash, randomBytes } from 'node:crypto'
|
||||
import { PUSH_LIMITS } from '@orca-cloud/push-contract'
|
||||
import type { PushDatabase } from './push-database.js'
|
||||
|
||||
export type IssuedPushSession = {
|
||||
sessionToken: string
|
||||
expiresAt: number
|
||||
hostFingerprint: string
|
||||
}
|
||||
|
||||
export type PushSessionLookup =
|
||||
| { ok: true; hostFingerprint: string; expiresAt: number }
|
||||
| { ok: false; reason: 'unknown_session' | 'session_expired' }
|
||||
|
||||
function hashSessionToken(sessionToken: string): string {
|
||||
return createHash('sha256').update(sessionToken).digest('base64url')
|
||||
}
|
||||
|
||||
export class PushHostSessionStore {
|
||||
constructor(
|
||||
private readonly database: PushDatabase,
|
||||
private readonly now: () => number = Date.now
|
||||
) {}
|
||||
|
||||
async create(hostFingerprint: string): Promise<IssuedPushSession> {
|
||||
const sessionToken = randomBytes(32).toString('base64url')
|
||||
const createdAt = this.now()
|
||||
const expiresAt = createdAt + PUSH_LIMITS.sessionTtlMs
|
||||
await this.database.transaction(async (transaction) => {
|
||||
// Why: a desktop holds one session at a time and only re-proves once it is
|
||||
// gone, so an earlier row is dead weight. It also bounds the table to one
|
||||
// row per host however many proofs a self-minted identity answers.
|
||||
await transaction.lockQuotaScope(`orca-push-session:${hostFingerprint}`)
|
||||
await transaction.query('DELETE FROM push_sessions WHERE host_fingerprint = ?', [
|
||||
hostFingerprint
|
||||
])
|
||||
await transaction.query(
|
||||
`INSERT INTO push_sessions (token_hash, host_fingerprint, expires_at, created_at)
|
||||
VALUES (?, ?, ?, ?)`,
|
||||
[hashSessionToken(sessionToken), hostFingerprint, expiresAt, createdAt]
|
||||
)
|
||||
})
|
||||
return { sessionToken, expiresAt, hostFingerprint }
|
||||
}
|
||||
|
||||
async resolve(sessionToken: string): Promise<PushSessionLookup> {
|
||||
const [row] = await this.database.query(
|
||||
'SELECT host_fingerprint, expires_at FROM push_sessions WHERE token_hash = ?',
|
||||
[hashSessionToken(sessionToken)]
|
||||
)
|
||||
if (!row) return { ok: false, reason: 'unknown_session' }
|
||||
const expiresAt = Number(row.expires_at)
|
||||
// No skew grace here: a 24h session that just expired should be re-minted
|
||||
// through the challenge, which is cheap and already handled by the host.
|
||||
if (this.now() > expiresAt) return { ok: false, reason: 'session_expired' }
|
||||
return { ok: true, hostFingerprint: String(row.host_fingerprint), expiresAt }
|
||||
}
|
||||
|
||||
async pruneExpired(): Promise<number> {
|
||||
const [result] = await this.database.query('DELETE FROM push_sessions WHERE expires_at < ?', [
|
||||
this.now()
|
||||
])
|
||||
return Number(result?.changes ?? 0)
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,55 @@
|
||||
import { startPushBackground } from './push-background.js'
|
||||
import { loadPushConfig } from './config.js'
|
||||
import { openPushDatabase } from './push-database.js'
|
||||
import { createPushServer } from './push-server.js'
|
||||
|
||||
const config = loadPushConfig()
|
||||
const database = await openPushDatabase({
|
||||
...(config.databaseUrl === undefined ? {} : { databaseUrl: config.databaseUrl }),
|
||||
dataDir: config.dataDir,
|
||||
poolMax: config.databasePoolMax,
|
||||
applicationName: 'orca-push',
|
||||
readOnly: config.mode === 'validation'
|
||||
})
|
||||
const {
|
||||
server,
|
||||
challenges,
|
||||
sessions,
|
||||
deliveryStore,
|
||||
worker,
|
||||
observability,
|
||||
closeTransports,
|
||||
requestDrain
|
||||
} = createPushServer(config, database)
|
||||
|
||||
const stopBackground = startPushBackground(config, { challenges, sessions, deliveryStore, worker })
|
||||
observability.start()
|
||||
|
||||
server.listen(config.port, () => {
|
||||
console.log(`[orca-push] listening on ${config.publicUrl} (port ${config.port})`)
|
||||
})
|
||||
|
||||
let stopping = false
|
||||
const shutdown = (): void => {
|
||||
if (stopping) return
|
||||
stopping = true
|
||||
// Cloud Run sends SIGKILL after ten seconds; leave time for explicit cleanup.
|
||||
const deadline = setTimeout(() => process.exit(1), 9_000)
|
||||
deadline.unref()
|
||||
const requests = requestDrain.begin()
|
||||
const deliveries = stopBackground()
|
||||
const connections = new Promise<void>((resolve) => server.close(() => resolve()))
|
||||
void Promise.all([requests, connections, deliveries])
|
||||
.then(async () => {
|
||||
closeTransports()
|
||||
await database.close()
|
||||
observability.stop()
|
||||
clearTimeout(deadline)
|
||||
})
|
||||
.catch(() => {
|
||||
console.warn(JSON.stringify({ event: 'orca_push_shutdown_failed' }))
|
||||
process.exitCode = 1
|
||||
})
|
||||
}
|
||||
process.once('SIGTERM', shutdown)
|
||||
process.once('SIGINT', shutdown)
|
||||
@@ -0,0 +1,9 @@
|
||||
export function providerRetryAfter(
|
||||
value: string | undefined,
|
||||
now = Date.now()
|
||||
): number | undefined {
|
||||
if (!value) return undefined
|
||||
const seconds = Number(value)
|
||||
const delay = Number.isFinite(seconds) ? seconds * 1000 : Date.parse(value) - now
|
||||
return Number.isFinite(delay) ? Math.max(0, delay) : undefined
|
||||
}
|
||||
@@ -0,0 +1,21 @@
|
||||
// Bound unauthenticated database lookups independently of Cloud Run HTTP concurrency.
|
||||
export class PushAuthAdmission {
|
||||
private active = 0
|
||||
private readonly waiting: (() => void)[] = []
|
||||
|
||||
async run<T>(operation: () => Promise<T>): Promise<T | null> {
|
||||
if (this.active >= 4) {
|
||||
if (this.waiting.length >= 32) return null
|
||||
await new Promise<void>((resolve) => this.waiting.push(resolve))
|
||||
} else {
|
||||
this.active++
|
||||
}
|
||||
try {
|
||||
return await operation()
|
||||
} finally {
|
||||
const next = this.waiting.shift()
|
||||
if (next) next()
|
||||
else this.active--
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,50 @@
|
||||
import { PUSH_LIMITS } from '@orca-cloud/push-contract'
|
||||
import { expect, it } from 'vitest'
|
||||
import { createPushHostKeypair } from './host-challenge-answering.test-fixture.js'
|
||||
import { createPushServerHarness, FCM_TOKEN } from './push-server-harness.test-fixture.js'
|
||||
|
||||
it('bounds valid hosts together across routes without letting key rotation reset the IP budget', async () => {
|
||||
const harness = await createPushServerHarness()
|
||||
const headers = { 'x-forwarded-for': '203.0.113.7' }
|
||||
try {
|
||||
const hostCount =
|
||||
PUSH_LIMITS.authenticatedRequestsPerMinutePerIp /
|
||||
PUSH_LIMITS.authenticatedRequestsPerMinutePerHost
|
||||
for (let host = 0; host < hostCount; host++) {
|
||||
const token = await harness.signIn(createPushHostKeypair(host + 1))
|
||||
for (
|
||||
let request = 0;
|
||||
request < PUSH_LIMITS.authenticatedRequestsPerMinutePerHost;
|
||||
request++
|
||||
) {
|
||||
expect((await harness.authorized('/v1/devices', { headers }, token)).status).toBe(200)
|
||||
}
|
||||
}
|
||||
const token = await harness.signIn(createPushHostKeypair(99))
|
||||
const registration = {
|
||||
method: 'POST',
|
||||
headers: { ...headers, 'content-type': 'application/json' },
|
||||
body: JSON.stringify({ v: 1, deviceId: 'phone', platform: 'android', token: FCM_TOKEN })
|
||||
}
|
||||
expect((await harness.authorized('/v1/devices', registration, token)).status).toBe(429)
|
||||
expect((await harness.authorized('/v1/send', { method: 'POST', headers }, token)).status).toBe(
|
||||
429
|
||||
)
|
||||
expect(
|
||||
(
|
||||
await harness.authorized(
|
||||
'/v1/devices',
|
||||
{
|
||||
...registration,
|
||||
headers: { ...registration.headers, 'x-forwarded-for': '198.51.100.9' }
|
||||
},
|
||||
token
|
||||
)
|
||||
).status
|
||||
).toBe(200)
|
||||
harness.advanceClock(60_000)
|
||||
expect((await harness.authorized('/v1/devices', registration, token)).status).toBe(200)
|
||||
} finally {
|
||||
await harness.close()
|
||||
}
|
||||
}, 30_000)
|
||||
@@ -0,0 +1,43 @@
|
||||
import type { PushConfig } from './config.js'
|
||||
import type { createPushServer } from './push-server.js'
|
||||
|
||||
const CHALLENGE_PRUNE_INTERVAL_MS = 60_000
|
||||
const SESSION_PRUNE_INTERVAL_MS = 10 * 60_000
|
||||
const DELIVERY_PRUNE_INTERVAL_MS = 60_000
|
||||
|
||||
function prune(label: string, run: () => Promise<number>, intervalMs: number): NodeJS.Timeout {
|
||||
const timer = setInterval(() => {
|
||||
void run().catch((error: unknown) => {
|
||||
console.warn(
|
||||
JSON.stringify({
|
||||
event: 'orca_push_prune_failed',
|
||||
target: label,
|
||||
error: error instanceof Error ? error.name : 'unknown'
|
||||
})
|
||||
)
|
||||
})
|
||||
}, intervalMs)
|
||||
timer.unref()
|
||||
return timer
|
||||
}
|
||||
|
||||
export function startPushBackground(
|
||||
config: Pick<PushConfig, 'mode'>,
|
||||
runtime: Pick<
|
||||
ReturnType<typeof createPushServer>,
|
||||
'challenges' | 'sessions' | 'deliveryStore' | 'worker'
|
||||
>
|
||||
): () => Promise<void> {
|
||||
if (config.mode === 'validation') return async () => {}
|
||||
const { challenges, sessions, deliveryStore, worker } = runtime
|
||||
const timers = [
|
||||
prune('challenges', () => challenges.pruneExpired(), CHALLENGE_PRUNE_INTERVAL_MS),
|
||||
prune('sessions', () => sessions.pruneExpired(), SESSION_PRUNE_INTERVAL_MS),
|
||||
prune('deliveries', () => deliveryStore.prune(), DELIVERY_PRUNE_INTERVAL_MS)
|
||||
]
|
||||
worker.start()
|
||||
return async () => {
|
||||
for (const timer of timers) clearInterval(timer)
|
||||
await worker.stop()
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,121 @@
|
||||
import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest'
|
||||
import { randomUUID } from 'node:crypto'
|
||||
|
||||
const fakes = vi.hoisted(() => ({
|
||||
configs: [] as Array<Record<string, unknown>>,
|
||||
lifecycle: [] as string[],
|
||||
query: vi.fn(async (_sql: string) => ({ rows: [], rowCount: 0 })),
|
||||
release: vi.fn()
|
||||
}))
|
||||
|
||||
vi.mock('pg', () => ({
|
||||
default: {
|
||||
Pool: class {
|
||||
on = vi.fn()
|
||||
connect = vi.fn(async () => ({ query: fakes.query, release: fakes.release }))
|
||||
private readonly label: string
|
||||
|
||||
constructor(config: Record<string, unknown>) {
|
||||
fakes.configs.push(config)
|
||||
this.label = `max=${String(config.max)} statement_timeout=${String(config.statement_timeout)}`
|
||||
fakes.lifecycle.push(`open ${this.label}`)
|
||||
}
|
||||
|
||||
async end(): Promise<void> {
|
||||
fakes.lifecycle.push(`end ${this.label}`)
|
||||
}
|
||||
}
|
||||
}
|
||||
}))
|
||||
|
||||
import { openPushDatabase } from './push-database.js'
|
||||
import { pushSchemaStatements } from './push-schema.js'
|
||||
|
||||
describe('PostgreSQL push gateway startup', () => {
|
||||
beforeEach(() => {
|
||||
fakes.configs.length = 0
|
||||
fakes.lifecycle.length = 0
|
||||
fakes.query.mockClear()
|
||||
})
|
||||
|
||||
afterEach(() => {
|
||||
vi.restoreAllMocks()
|
||||
})
|
||||
|
||||
const socketPassword = `${randomUUID()}@/`
|
||||
const socketUrl = `postgresql://push:${encodeURIComponent(socketPassword)}@/orca_push?host=/cloudsql/test:region:instance`
|
||||
|
||||
it('passes the Terraform socket URL unchanged to both active pools', async () => {
|
||||
const database = await openPushDatabase({ databaseUrl: socketUrl, dataDir: '/unused' })
|
||||
expect(fakes.configs.map((config) => config.connectionString)).toEqual([socketUrl, socketUrl])
|
||||
await database.close()
|
||||
})
|
||||
|
||||
it('parses socket credentials and query options before enforcing validation read-only', async () => {
|
||||
const options = '-c search_path=validation -c default_transaction_read_only=off'
|
||||
const database = await openPushDatabase({
|
||||
databaseUrl: `${socketUrl}&port=5433&sslmode=disable&options=${encodeURIComponent(options)}`,
|
||||
dataDir: '/unused',
|
||||
readOnly: true
|
||||
})
|
||||
expect(fakes.configs).toHaveLength(1)
|
||||
expect(fakes.configs[0]).toMatchObject({
|
||||
host: '/cloudsql/test:region:instance',
|
||||
user: 'push',
|
||||
password: socketPassword,
|
||||
database: 'orca_push',
|
||||
port: 5433,
|
||||
ssl: false,
|
||||
options: `${options} -c default_transaction_read_only=on`
|
||||
})
|
||||
expect(fakes.configs[0]).not.toHaveProperty('connectionString')
|
||||
expect(fakes.query).not.toHaveBeenCalled()
|
||||
await database.close()
|
||||
})
|
||||
|
||||
// Why: a CREATE INDEX on a grown table can outlive the 5s request deadline,
|
||||
// and a schema that inherits it fails every startup at the same statement.
|
||||
it('applies the schema on an untimed pool that is gone before the serving pool opens', async () => {
|
||||
const database = await openPushDatabase({
|
||||
databaseUrl: 'postgresql://push@localhost:55440/orca_push',
|
||||
dataDir: '/unused',
|
||||
poolMax: 2,
|
||||
applicationName: 'orca-push'
|
||||
})
|
||||
expect(fakes.lifecycle).toEqual([
|
||||
'open max=1 statement_timeout=0',
|
||||
'end max=1 statement_timeout=0',
|
||||
'open max=2 statement_timeout=5000'
|
||||
])
|
||||
expect(fakes.configs[0]).toMatchObject({
|
||||
application_name: 'orca-push/schema',
|
||||
lock_timeout: 1_000,
|
||||
idle_in_transaction_session_timeout: 5_000
|
||||
})
|
||||
expect(
|
||||
fakes.query.mock.calls.map(([sql]) => sql).slice(0, pushSchemaStatements().length)
|
||||
).toEqual(pushSchemaStatements())
|
||||
await database.close()
|
||||
})
|
||||
|
||||
it('retries a transaction the pool statement_timeout aborted', async () => {
|
||||
const database = await openPushDatabase({
|
||||
databaseUrl: 'postgresql://push@localhost:55440/orca_push',
|
||||
dataDir: '/unused'
|
||||
})
|
||||
const warn = vi.spyOn(console, 'warn').mockImplementation(() => undefined)
|
||||
let attempts = 0
|
||||
const result = await database.transaction(async () => {
|
||||
attempts += 1
|
||||
if (attempts === 1) throw Object.assign(new Error('canceling statement'), { code: '57014' })
|
||||
return 'done'
|
||||
})
|
||||
expect(result).toBe('done')
|
||||
expect(attempts).toBe(2)
|
||||
expect(warn.mock.calls.map(([line]) => String(line))).toEqual([
|
||||
expect.stringContaining('"code":"57014"')
|
||||
])
|
||||
warn.mockRestore()
|
||||
await database.close()
|
||||
})
|
||||
})
|
||||
@@ -0,0 +1,283 @@
|
||||
import { mkdirSync } from 'node:fs'
|
||||
import { join } from 'node:path'
|
||||
import { DatabaseSync } from 'node:sqlite'
|
||||
import pg from 'pg'
|
||||
import { parseIntoClientConfig } from 'pg-connection-string'
|
||||
import { applyPostgresSchema } from '@orca-cloud/postgres-schema'
|
||||
import { pushSchemaStatements } from './push-schema.js'
|
||||
|
||||
const POSTGRES_LOCK_TIMEOUT_MS = 1_000
|
||||
const POSTGRES_CONNECTION_TIMEOUT_MS = 2_000
|
||||
const POSTGRES_STATEMENT_TIMEOUT_MS = 5_000
|
||||
const POSTGRES_IDLE_TRANSACTION_TIMEOUT_MS = 5_000
|
||||
const POSTGRES_TRANSACTION_ATTEMPTS = 3
|
||||
const POSTGRES_RETRY_MAX_DELAY_MS = 25
|
||||
|
||||
export type SqlRow = Record<string, unknown>
|
||||
|
||||
export interface PushDatabase {
|
||||
readonly dialect: 'sqlite' | 'postgres'
|
||||
query(sql: string, params?: unknown[]): Promise<SqlRow[]>
|
||||
transaction<T>(operation: (transaction: PushDatabase) => Promise<T>): Promise<T>
|
||||
// Serializes every transaction that reads then writes the same identity's
|
||||
// quota rows. Must be called inside a transaction; it releases at commit.
|
||||
lockQuotaScope(key: string): Promise<void>
|
||||
close(): Promise<void>
|
||||
}
|
||||
|
||||
function postgresSql(sql: string): string {
|
||||
let index = 0
|
||||
return sql.replace(/\?/g, () => `$${++index}`)
|
||||
}
|
||||
|
||||
function returnsRows(sql: string): boolean {
|
||||
return /^\s*(select|with)/i.test(sql) || /returning/i.test(sql)
|
||||
}
|
||||
|
||||
class SqliteTransaction implements PushDatabase {
|
||||
readonly dialect = 'sqlite' as const
|
||||
|
||||
constructor(protected readonly database: DatabaseSync) {}
|
||||
|
||||
async query(sql: string, params: unknown[] = []): Promise<SqlRow[]> {
|
||||
const statement = this.database.prepare(sql)
|
||||
const bound = params.map((value) => (value === undefined ? null : value)) as never[]
|
||||
if (returnsRows(sql)) return statement.all(...bound) as SqlRow[]
|
||||
const result = statement.run(...bound)
|
||||
return [{ changes: Number(result.changes) }]
|
||||
}
|
||||
|
||||
async transaction<T>(operation: (transaction: PushDatabase) => Promise<T>): Promise<T> {
|
||||
return await operation(this)
|
||||
}
|
||||
|
||||
// BEGIN IMMEDIATE already holds the single writer lock for the whole
|
||||
// transaction, so there is nothing narrower left to take.
|
||||
async lockQuotaScope(): Promise<void> {}
|
||||
|
||||
async close(): Promise<void> {}
|
||||
}
|
||||
|
||||
class SqliteDatabase extends SqliteTransaction {
|
||||
// node:sqlite is synchronous and has no nested transactions, so overlapping
|
||||
// callers are serialized behind one tail promise instead of racing BEGIN.
|
||||
private tail: Promise<void> = Promise.resolve()
|
||||
|
||||
override async query(sql: string, params: unknown[] = []): Promise<SqlRow[]> {
|
||||
await this.tail
|
||||
return await super.query(sql, params)
|
||||
}
|
||||
|
||||
override async transaction<T>(operation: (transaction: PushDatabase) => Promise<T>): Promise<T> {
|
||||
const previous = this.tail
|
||||
let release!: () => void
|
||||
this.tail = new Promise((resolve) => (release = resolve))
|
||||
await previous
|
||||
this.database.exec('BEGIN IMMEDIATE')
|
||||
const transaction = new SqliteTransaction(this.database)
|
||||
try {
|
||||
const result = await operation(transaction)
|
||||
this.database.exec('COMMIT')
|
||||
return result
|
||||
} catch (error) {
|
||||
this.database.exec('ROLLBACK')
|
||||
throw error
|
||||
} finally {
|
||||
release()
|
||||
}
|
||||
}
|
||||
|
||||
override async close(): Promise<void> {
|
||||
await this.tail
|
||||
this.database.close()
|
||||
}
|
||||
}
|
||||
|
||||
class PostgresTransaction implements PushDatabase {
|
||||
readonly dialect = 'postgres' as const
|
||||
|
||||
constructor(private readonly client: pg.PoolClient) {}
|
||||
|
||||
async query(sql: string, params: unknown[] = []): Promise<SqlRow[]> {
|
||||
const result = await this.client.query(postgresSql(sql), params)
|
||||
return returnsRows(sql) ? (result.rows as SqlRow[]) : [{ changes: result.rowCount ?? 0 }]
|
||||
}
|
||||
|
||||
async transaction<T>(operation: (transaction: PushDatabase) => Promise<T>): Promise<T> {
|
||||
return await operation(this)
|
||||
}
|
||||
|
||||
// READ COMMITTED lets a concurrent count-then-insert read the same
|
||||
// under-quota total, so the identity is serialized for the whole transaction.
|
||||
async lockQuotaScope(key: string): Promise<void> {
|
||||
await this.query('SELECT pg_advisory_xact_lock(hashtext(?::text))', [key])
|
||||
}
|
||||
|
||||
async close(): Promise<void> {}
|
||||
}
|
||||
|
||||
function retryablePostgresTransactionError(error: unknown): boolean {
|
||||
const code = String((error as { code?: unknown }).code)
|
||||
// 57014 is the pool statement_timeout firing. It aborts the transaction the
|
||||
// same way a lock timeout does, so it takes the bounded retry path too.
|
||||
return code === '40P01' || code === '40001' || code === '55P03' || code === '57014'
|
||||
}
|
||||
|
||||
async function waitForPostgresRetry(): Promise<void> {
|
||||
const delayMs = Math.floor(Math.random() * (POSTGRES_RETRY_MAX_DELAY_MS + 1))
|
||||
await new Promise((resolve) => setTimeout(resolve, delayMs))
|
||||
}
|
||||
|
||||
class PostgresDatabase implements PushDatabase {
|
||||
readonly dialect = 'postgres' as const
|
||||
|
||||
constructor(private readonly pool: pg.Pool) {}
|
||||
|
||||
async query(sql: string, params: unknown[] = []): Promise<SqlRow[]> {
|
||||
const client = await this.pool.connect()
|
||||
try {
|
||||
const result = await client.query(postgresSql(sql), params)
|
||||
return returnsRows(sql) ? (result.rows as SqlRow[]) : [{ changes: result.rowCount ?? 0 }]
|
||||
} finally {
|
||||
client.release()
|
||||
}
|
||||
}
|
||||
|
||||
async transaction<T>(operation: (transaction: PushDatabase) => Promise<T>): Promise<T> {
|
||||
for (let attempt = 1; attempt <= POSTGRES_TRANSACTION_ATTEMPTS; attempt++) {
|
||||
const client = await this.pool.connect()
|
||||
try {
|
||||
await client.query('BEGIN')
|
||||
const result = await operation(new PostgresTransaction(client))
|
||||
await client.query('COMMIT')
|
||||
return result
|
||||
} catch (error) {
|
||||
await client.query('ROLLBACK').catch(() => undefined)
|
||||
if (
|
||||
!retryablePostgresTransactionError(error) ||
|
||||
attempt === POSTGRES_TRANSACTION_ATTEMPTS
|
||||
) {
|
||||
throw error
|
||||
}
|
||||
console.warn(
|
||||
JSON.stringify({
|
||||
event: 'orca_push_postgres_transaction_retry',
|
||||
code: String((error as { code?: unknown }).code),
|
||||
attempt
|
||||
})
|
||||
)
|
||||
} finally {
|
||||
client.release()
|
||||
}
|
||||
// A PostgreSQL transaction is unusable after an abort, so retry all work
|
||||
// on a fresh pooled client with a small full-jitter delay.
|
||||
await waitForPostgresRetry()
|
||||
}
|
||||
throw new Error('postgres_transaction_retry_exhausted')
|
||||
}
|
||||
|
||||
// An advisory transaction lock taken outside a transaction is released by the
|
||||
// implicit commit before the caller reads anything, which protects nothing.
|
||||
async lockQuotaScope(): Promise<void> {
|
||||
throw new Error('lock_quota_scope_requires_transaction')
|
||||
}
|
||||
|
||||
async close(): Promise<void> {
|
||||
await this.pool.end()
|
||||
}
|
||||
}
|
||||
|
||||
async function applySchema(database: PushDatabase): Promise<void> {
|
||||
for (const statement of pushSchemaStatements()) await database.query(statement)
|
||||
}
|
||||
|
||||
// Why: DDL is not a request. A CREATE INDEX on a grown table can legitimately
|
||||
// outlive the request statement_timeout, and inheriting it would fail every
|
||||
// startup at the same statement instead of finishing once. One connection of
|
||||
// its own, closed before the serving pool opens, keeps the untimed session off
|
||||
// the request path entirely.
|
||||
async function applySchemaOnUntimedPool(
|
||||
databaseUrl: string,
|
||||
applicationName: string | undefined
|
||||
): Promise<void> {
|
||||
const pool = new pg.Pool({
|
||||
connectionString: databaseUrl,
|
||||
max: 1,
|
||||
application_name: applicationName ? `${applicationName}/schema` : undefined,
|
||||
connectionTimeoutMillis: POSTGRES_CONNECTION_TIMEOUT_MS,
|
||||
statement_timeout: 0,
|
||||
lock_timeout: POSTGRES_LOCK_TIMEOUT_MS,
|
||||
idle_in_transaction_session_timeout: POSTGRES_IDLE_TRANSACTION_TIMEOUT_MS
|
||||
})
|
||||
absorbPostgresIdleClientErrors(pool)
|
||||
const database = new PostgresDatabase(pool)
|
||||
try {
|
||||
await applyPostgresSchema(pushSchemaStatements(), (statement) => database.query(statement), {
|
||||
eventPrefix: 'orca_push_postgres_schema'
|
||||
})
|
||||
} finally {
|
||||
await database.close().catch(() => undefined)
|
||||
}
|
||||
}
|
||||
|
||||
export function absorbPostgresIdleClientErrors(pool: Pick<pg.Pool, 'on'>): void {
|
||||
pool.on('error', () => {
|
||||
// node-postgres removes failed idle clients itself; an unhandled 'error'
|
||||
// would crash the service and turn a SQL blip into a restart loop.
|
||||
console.warn('[orca-push] idle PostgreSQL client failed')
|
||||
})
|
||||
}
|
||||
|
||||
export async function openPushDatabase(input: {
|
||||
databaseUrl?: string
|
||||
dataDir: string
|
||||
poolMax?: number
|
||||
applicationName?: string
|
||||
readOnly?: boolean
|
||||
}): Promise<PushDatabase> {
|
||||
let database: PushDatabase
|
||||
if (input.databaseUrl) {
|
||||
if (!input.readOnly) await applySchemaOnUntimedPool(input.databaseUrl, input.applicationName)
|
||||
let connection: pg.ClientConfig = { connectionString: input.databaseUrl }
|
||||
if (input.readOnly) {
|
||||
connection = parseIntoClientConfig(input.databaseUrl)
|
||||
// A URL parameter must not trigger a second parse that overrides read-only options.
|
||||
delete connection.connectionString
|
||||
connection.options = `${connection.options ?? ''} -c default_transaction_read_only=on`.trim()
|
||||
}
|
||||
const pool = new pg.Pool({
|
||||
...connection,
|
||||
max: input.poolMax ?? 10,
|
||||
application_name: input.applicationName,
|
||||
connectionTimeoutMillis: POSTGRES_CONNECTION_TIMEOUT_MS,
|
||||
statement_timeout: POSTGRES_STATEMENT_TIMEOUT_MS,
|
||||
lock_timeout: POSTGRES_LOCK_TIMEOUT_MS,
|
||||
idle_in_transaction_session_timeout: POSTGRES_IDLE_TRANSACTION_TIMEOUT_MS
|
||||
})
|
||||
absorbPostgresIdleClientErrors(pool)
|
||||
database = new PostgresDatabase(pool)
|
||||
} else {
|
||||
mkdirSync(input.dataDir, { recursive: true })
|
||||
const sqlite = new DatabaseSync(join(input.dataDir, 'orca-push.sqlite'), {
|
||||
readOnly: input.readOnly ?? false
|
||||
})
|
||||
if (!input.readOnly) sqlite.exec('PRAGMA journal_mode = WAL; PRAGMA foreign_keys = ON;')
|
||||
database = new SqliteDatabase(sqlite)
|
||||
}
|
||||
if (database.dialect === 'postgres' || input.readOnly) return database
|
||||
try {
|
||||
await applySchema(database)
|
||||
return database
|
||||
} catch (error) {
|
||||
await database.close().catch(() => undefined)
|
||||
throw error
|
||||
}
|
||||
}
|
||||
|
||||
export async function openInMemoryPushDatabase(): Promise<PushDatabase> {
|
||||
const sqlite = new DatabaseSync(':memory:')
|
||||
sqlite.exec('PRAGMA foreign_keys = ON;')
|
||||
const database = new SqliteDatabase(sqlite)
|
||||
await applySchema(database)
|
||||
return database
|
||||
}
|
||||
@@ -0,0 +1,85 @@
|
||||
import { afterEach, expect, it, vi } from 'vitest'
|
||||
import { Hono } from 'hono'
|
||||
import { PushRequestDrain } from './push-request-drain.js'
|
||||
import { PushDispatcher } from './push-dispatcher.js'
|
||||
import { PushDeviceRegistryStore } from './device-registry-store.js'
|
||||
import { openInMemoryPushDatabase, type PushDatabase } from './push-database.js'
|
||||
import { buildPushDelivery } from './push-delivery-message.js'
|
||||
import { PushNotificationSchema } from '@orca-cloud/push-contract'
|
||||
import { notification } from './push-server-harness.test-fixture.js'
|
||||
|
||||
const databases: PushDatabase[] = []
|
||||
afterEach(async () => {
|
||||
await Promise.all(databases.splice(0).map((db) => db.close()))
|
||||
vi.restoreAllMocks()
|
||||
})
|
||||
const note = PushNotificationSchema.parse(notification())
|
||||
const tick = () => new Promise((resolve) => setImmediate(resolve))
|
||||
function deferred() {
|
||||
let resolve!: () => void
|
||||
const promise = new Promise<void>((done) => {
|
||||
resolve = done
|
||||
})
|
||||
return { promise, resolve }
|
||||
}
|
||||
async function registered() {
|
||||
const db = await openInMemoryPushDatabase()
|
||||
databases.push(db)
|
||||
const devices = new PushDeviceRegistryStore(db)
|
||||
const input = {
|
||||
hostFingerprint: 'abcdefghijklmnop',
|
||||
deviceId: 'device',
|
||||
platform: 'android' as const,
|
||||
token: 'old-token'
|
||||
}
|
||||
const row = await devices.upsert(input)
|
||||
if (!row.ok) throw new Error('registration failed')
|
||||
const delivery = buildPushDelivery({
|
||||
expiresAt: Date.now() + 300_000,
|
||||
registrationId: row.registrationId,
|
||||
hostFingerprint: input.hostFingerprint,
|
||||
notification: note
|
||||
})
|
||||
return { db, devices, input, delivery }
|
||||
}
|
||||
|
||||
it('does not retire a refreshed token after the old token fails', async () => {
|
||||
const h = await registered()
|
||||
const gate = deferred()
|
||||
const send = vi.fn(async () => {
|
||||
await gate.promise
|
||||
return { status: 'dead', reason: 'UNREGISTERED' }
|
||||
})
|
||||
vi.spyOn(console, 'warn').mockImplementation(() => {})
|
||||
const dispatcher = new PushDispatcher({ devices: h.devices, fcm: { send } as never })
|
||||
const pending = dispatcher.sendOnce(h.delivery)
|
||||
await tick()
|
||||
await h.devices.upsert({ ...h.input, token: 'replacement-token' })
|
||||
gate.resolve()
|
||||
await pending
|
||||
expect(await h.devices.findById(h.delivery.registrationId)).toMatchObject({
|
||||
token: 'replacement-token',
|
||||
dead: false
|
||||
})
|
||||
})
|
||||
|
||||
it('rejects new requests during drain and waits for an admitted handler', async () => {
|
||||
const gate = deferred()
|
||||
const requests = new PushRequestDrain()
|
||||
const app = new Hono().use('*', requests.middleware).post('/send', async (c) => {
|
||||
await gate.promise
|
||||
return c.json({ queued: true })
|
||||
})
|
||||
const pending = app.request('/send', { method: 'POST' })
|
||||
await tick()
|
||||
let drained = false
|
||||
const drain = requests.begin().then(() => {
|
||||
drained = true
|
||||
})
|
||||
expect((await app.request('/send', { method: 'POST' })).status).toBe(503)
|
||||
expect(drained).toBe(false)
|
||||
gate.resolve()
|
||||
expect((await pending).status).toBe(200)
|
||||
await drain
|
||||
expect(drained).toBe(true)
|
||||
})
|
||||
@@ -0,0 +1,75 @@
|
||||
import { createHash } from 'node:crypto'
|
||||
import { type PushNotification } from '@orca-cloud/push-contract'
|
||||
|
||||
export type PushOrcaData = {
|
||||
kind?: 'alert' | 'dismiss'
|
||||
hostFingerprint: string
|
||||
worktreeId?: string
|
||||
notificationId?: string
|
||||
notificationSeq: number
|
||||
notificationEpoch: string
|
||||
source: string
|
||||
agentState: string | null
|
||||
}
|
||||
|
||||
export type PushDelivery = {
|
||||
expiresAt: number
|
||||
sound?: boolean
|
||||
registrationId: string
|
||||
hostFingerprint: string
|
||||
title: string
|
||||
body: string
|
||||
collapseId: string
|
||||
orca: PushOrcaData
|
||||
}
|
||||
|
||||
export function collapseIdFor(notification: PushNotification, hostFingerprint: string): string {
|
||||
const identity =
|
||||
notification.notificationId === undefined
|
||||
? [notification.notificationEpoch, notification.notificationSeq]
|
||||
: notification.notificationId
|
||||
return createHash('sha256')
|
||||
.update(JSON.stringify([hostFingerprint, identity]))
|
||||
.digest('hex')
|
||||
}
|
||||
|
||||
export function buildPushDelivery(input: {
|
||||
expiresAt: number
|
||||
registrationId: string
|
||||
hostFingerprint: string
|
||||
notification: PushNotification
|
||||
}): PushDelivery {
|
||||
const { notification, hostFingerprint } = input
|
||||
return {
|
||||
...(notification.sound === false ? { sound: false } : {}),
|
||||
expiresAt: input.expiresAt,
|
||||
registrationId: input.registrationId,
|
||||
hostFingerprint,
|
||||
title: notification.title,
|
||||
body: notification.body,
|
||||
collapseId: collapseIdFor(notification, hostFingerprint),
|
||||
orca: {
|
||||
...(notification.kind ? { kind: notification.kind } : {}),
|
||||
hostFingerprint,
|
||||
...(notification.worktreeId === undefined ? {} : { worktreeId: notification.worktreeId }),
|
||||
...(notification.notificationId === undefined
|
||||
? {}
|
||||
: { notificationId: notification.notificationId }),
|
||||
notificationSeq: notification.notificationSeq,
|
||||
notificationEpoch: notification.notificationEpoch,
|
||||
source: notification.source,
|
||||
agentState: notification.agentState
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
export function orcaDataStrings(orca: PushOrcaData): Record<string, string> {
|
||||
return Object.fromEntries(
|
||||
Object.entries(orca)
|
||||
.filter(([, value]) => value !== undefined && value !== null)
|
||||
.map(([key, value]) => [
|
||||
key,
|
||||
typeof value === 'object' ? JSON.stringify(value) : String(value)
|
||||
])
|
||||
)
|
||||
}
|
||||
@@ -0,0 +1,8 @@
|
||||
import type { PushNotification } from '@orca-cloud/push-contract'
|
||||
|
||||
export function parsePushDeliveryPayload(payload: string): PushNotification {
|
||||
const value: unknown = JSON.parse(payload)
|
||||
if (!value || typeof value !== 'object' || Array.isArray(value))
|
||||
throw new Error('invalid_push_delivery_payload')
|
||||
return value as PushNotification
|
||||
}
|
||||
@@ -0,0 +1,17 @@
|
||||
import { readFileSync } from 'node:fs'
|
||||
import { expect, it } from 'vitest'
|
||||
import { loadPushConfig } from './config.js'
|
||||
|
||||
it('runs the deployment image preflight against the actual config loader', () => {
|
||||
const workflow = readFileSync(
|
||||
new URL('../../../../.github/workflows/cloud-push-deploy.yml', import.meta.url),
|
||||
'utf8'
|
||||
)
|
||||
const step = workflow.split('- name: Require image support for inert validation')[1]!
|
||||
const script = step.match(/--input-type=module -e '([\s\S]*?)'/)?.[1]
|
||||
expect(script).toBeDefined()
|
||||
const run = new Function('loadPushConfig', script!.replace(/import .*?;/, ''))
|
||||
expect(() => run(loadPushConfig)).not.toThrow()
|
||||
expect(() => run(() => ({ mode: 'active' }))).toThrow('validation_mode_unsupported')
|
||||
expect(() => run(() => ({ mode: 'validation' }))).toThrow('validation_mode_not_fail_closed')
|
||||
})
|
||||
@@ -0,0 +1,27 @@
|
||||
import { expect, it } from 'vitest'
|
||||
import { apnsBody } from './apns-client.js'
|
||||
import { fcmMessageBody } from './fcm-client.js'
|
||||
import { buildPushDelivery } from './push-delivery-message.js'
|
||||
|
||||
it('dismissal provider payloads cannot display a new alert or play a sound', () => {
|
||||
const delivery = buildPushDelivery({
|
||||
expiresAt: Date.now() + 300_000,
|
||||
registrationId: 'reg',
|
||||
hostFingerprint: 'host',
|
||||
notification: {
|
||||
kind: 'dismiss',
|
||||
notificationId: 'note',
|
||||
notificationSeq: 2,
|
||||
notificationEpoch: 'epoch',
|
||||
source: 'agent-task-complete',
|
||||
agentState: null,
|
||||
title: 'Orca',
|
||||
body: ''
|
||||
}
|
||||
})
|
||||
expect(JSON.parse(apnsBody(delivery)).aps).toEqual({ 'content-available': 1 })
|
||||
const android = JSON.parse(fcmMessageBody({ delivery, token: 'test', channelId: 'test' })).message
|
||||
expect(android).not.toHaveProperty('notification')
|
||||
expect(android.android).not.toHaveProperty('notification')
|
||||
expect(android.data.kind).toBe('dismiss')
|
||||
})
|
||||
@@ -0,0 +1,52 @@
|
||||
import type { ApnsClient } from './apns-client.js'
|
||||
import type { PushDeviceRegistryStore } from './device-registry-store.js'
|
||||
import type { FcmClient } from './fcm-client.js'
|
||||
import { fingerprintLogPrefix } from './host-fingerprint.js'
|
||||
import type { PushDelivery } from './push-delivery-message.js'
|
||||
import type { PushProviderOutcome } from './push-provider-outcome.js'
|
||||
|
||||
export type PushDispatcherOptions = {
|
||||
devices: PushDeviceRegistryStore
|
||||
apns?: ApnsClient
|
||||
fcm?: FcmClient
|
||||
onOutcome?: (outcome: PushProviderOutcome['status']) => void
|
||||
}
|
||||
|
||||
// Retires the registration when the provider says the token is gone.
|
||||
export class PushDispatcher {
|
||||
constructor(private readonly options: PushDispatcherOptions) {}
|
||||
|
||||
async sendOnce(delivery: PushDelivery): Promise<PushProviderOutcome> {
|
||||
const device = await this.options.devices.findById(delivery.registrationId)
|
||||
if (!device || device.dead) return { status: 'dead', reason: 'registration_unavailable' }
|
||||
let outcome: PushProviderOutcome
|
||||
if (device.platform === 'ios') {
|
||||
outcome = this.options.apns
|
||||
? await this.options.apns.send(delivery, {
|
||||
token: device.token,
|
||||
apnsEnvironment: device.apnsEnvironment ?? 'production'
|
||||
})
|
||||
: { status: 'error', reason: 'apns_not_configured' }
|
||||
} else {
|
||||
outcome = this.options.fcm
|
||||
? await this.options.fcm.send(delivery, { token: device.token })
|
||||
: { status: 'error', reason: 'fcm_not_configured' }
|
||||
}
|
||||
this.options.onOutcome?.(outcome.status)
|
||||
if (outcome.status === 'dead') {
|
||||
await this.options.devices.markDead(device)
|
||||
}
|
||||
if (outcome.status !== 'sent') {
|
||||
console.warn(
|
||||
JSON.stringify({
|
||||
event: 'orca_push_delivery_failed',
|
||||
platform: device.platform,
|
||||
status: outcome.status,
|
||||
reason: outcome.reason,
|
||||
host: fingerprintLogPrefix(delivery.hostFingerprint)
|
||||
})
|
||||
)
|
||||
}
|
||||
return outcome
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,29 @@
|
||||
import { expect, it } from 'vitest'
|
||||
import { apnsBody } from './apns-client.js'
|
||||
import { fcmMessageBody } from './fcm-client.js'
|
||||
import { buildPushDelivery } from './push-delivery-message.js'
|
||||
import { PushNotificationSchema } from '@orca-cloud/push-contract'
|
||||
|
||||
it('carries a silent preference through validation to APNs and Android payloads', () => {
|
||||
const notification = PushNotificationSchema.parse({
|
||||
notificationSeq: 1,
|
||||
notificationEpoch: 'epoch',
|
||||
source: 'terminal-bell',
|
||||
agentState: null,
|
||||
title: 'Bell',
|
||||
body: '',
|
||||
sound: false
|
||||
})
|
||||
const delivery = buildPushDelivery({
|
||||
expiresAt: Date.now() + 300_000,
|
||||
registrationId: 'reg',
|
||||
hostFingerprint: 'host',
|
||||
notification
|
||||
})
|
||||
expect(JSON.parse(apnsBody(delivery)).aps).not.toHaveProperty('sound')
|
||||
expect(
|
||||
JSON.parse(fcmMessageBody({ delivery, token: 'test-token', channelId: 'orca-desktop' })).message
|
||||
.android.notification.channel_id
|
||||
).toBe('orca-desktop-silent')
|
||||
expect(JSON.parse(apnsBody({ ...delivery, sound: undefined })).aps.sound).toBe('default')
|
||||
})
|
||||
@@ -0,0 +1,56 @@
|
||||
const COUNTER_NAMES = [
|
||||
'ip_rate_limited',
|
||||
'request_error',
|
||||
'challenge_issued',
|
||||
'challenge_rejected',
|
||||
'session_issued',
|
||||
'session_rejected',
|
||||
'device_registered',
|
||||
'device_rejected',
|
||||
'device_deleted',
|
||||
'send_queued',
|
||||
'send_dead',
|
||||
'send_rate_limited',
|
||||
'send_error',
|
||||
'delivery_sent',
|
||||
'delivery_dead',
|
||||
'delivery_error',
|
||||
'delivery_retry'
|
||||
] as const
|
||||
|
||||
type PushCounterName = (typeof COUNTER_NAMES)[number]
|
||||
|
||||
// Aggregate counters only. Nothing here may accept a token, a title, a body,
|
||||
// or more than the first four characters of a host fingerprint.
|
||||
export class PushObservability {
|
||||
private counters = new Map<PushCounterName, number>()
|
||||
private timer: NodeJS.Timeout | null = null
|
||||
|
||||
record(name: PushCounterName, delta = 1): void {
|
||||
this.counters.set(name, (this.counters.get(name) ?? 0) + delta)
|
||||
}
|
||||
|
||||
consume(): Record<PushCounterName, number> {
|
||||
const snapshot = Object.fromEntries(
|
||||
COUNTER_NAMES.map((name) => [name, this.counters.get(name) ?? 0])
|
||||
) as Record<PushCounterName, number>
|
||||
this.counters = new Map()
|
||||
return snapshot
|
||||
}
|
||||
|
||||
start(intervalMs = 60_000): void {
|
||||
if (this.timer) return
|
||||
this.timer = setInterval(() => {
|
||||
const counters = this.consume()
|
||||
if (Object.values(counters).every((value) => value === 0)) return
|
||||
console.warn(JSON.stringify({ event: 'orca_push_counters', ...counters }))
|
||||
}, intervalMs)
|
||||
this.timer.unref()
|
||||
}
|
||||
|
||||
stop(): void {
|
||||
if (!this.timer) return
|
||||
clearInterval(this.timer)
|
||||
this.timer = null
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,6 @@
|
||||
// What a provider send resolved to, before the send route maps it onto the
|
||||
// contract's queued / dead / rate_limited / error statuses.
|
||||
export type PushProviderOutcome =
|
||||
| { status: 'sent' }
|
||||
| { status: 'dead'; reason: string }
|
||||
| { status: 'error'; reason: string; retryable?: boolean; retryAfterMs?: number }
|
||||
@@ -0,0 +1,57 @@
|
||||
import type { PushNotification } from '@orca-cloud/push-contract'
|
||||
import type { PushDatabase } from './push-database.js'
|
||||
import { parsePushDeliveryPayload } from './push-delivery-payload.js'
|
||||
|
||||
export async function reconcileQueuedDismissal(
|
||||
tx: PushDatabase,
|
||||
host: string,
|
||||
registrationId: string,
|
||||
notification: PushNotification,
|
||||
now: number
|
||||
): Promise<boolean> {
|
||||
if (!notification.notificationId) return false
|
||||
const key = [host, notification.notificationEpoch, notification.notificationId]
|
||||
const [dismissed] = await tx.query(
|
||||
'SELECT notification_seq FROM push_dismissed_events WHERE host_fingerprint = ? AND notification_epoch = ? AND notification_id = ?',
|
||||
key
|
||||
)
|
||||
if (notification.kind !== 'dismiss')
|
||||
return Number(dismissed?.notification_seq ?? -1) >= notification.notificationSeq
|
||||
await tx.query(
|
||||
`INSERT INTO push_dismissed_events(host_fingerprint, notification_epoch, notification_id, notification_seq, created_at)
|
||||
VALUES (?, ?, ?, ?, ?) ON CONFLICT(host_fingerprint, notification_epoch, notification_id)
|
||||
DO UPDATE SET notification_seq = CASE WHEN push_dismissed_events.notification_seq > excluded.notification_seq THEN push_dismissed_events.notification_seq ELSE excluded.notification_seq END, created_at = excluded.created_at`,
|
||||
[...key, notification.notificationSeq, now]
|
||||
)
|
||||
const deliveries = await tx.query(
|
||||
"SELECT batch_id, payload_json FROM push_delivery_batches WHERE host_fingerprint = ? AND registration_id = ? AND kind = 'alert' AND state = 'pending' AND lease_until <= ?",
|
||||
[host, registrationId, now]
|
||||
)
|
||||
for (const delivery of deliveries) {
|
||||
const queued = parsePushDeliveryPayload(String(delivery.payload_json))
|
||||
if (
|
||||
queued.notificationEpoch !== notification.notificationEpoch ||
|
||||
queued.notificationId !== notification.notificationId ||
|
||||
queued.notificationSeq > notification.notificationSeq
|
||||
)
|
||||
continue
|
||||
await tx.query(
|
||||
'UPDATE push_delivery_batches SET payload_json = ?, state = ? WHERE batch_id = ?',
|
||||
['{}', 'dismissed', delivery.batch_id]
|
||||
)
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
export async function isDismissedAlert(
|
||||
tx: PushDatabase,
|
||||
host: string,
|
||||
notification: PushNotification
|
||||
): Promise<boolean> {
|
||||
if (notification.kind === 'dismiss' || !notification.notificationId) return false
|
||||
const rows = await tx.query(
|
||||
'SELECT notification_seq FROM push_dismissed_events WHERE host_fingerprint = ? AND notification_epoch = ? AND notification_id = ?',
|
||||
[host, notification.notificationEpoch, notification.notificationId]
|
||||
)
|
||||
return Number(rows[0]?.notification_seq ?? -1) >= notification.notificationSeq
|
||||
}
|
||||
@@ -0,0 +1,26 @@
|
||||
import { expect, it, vi } from 'vitest'
|
||||
import { createPushReadiness } from './push-readiness.js'
|
||||
import type { PushDatabase } from './push-database.js'
|
||||
|
||||
it('shares a slow check and caches failures before retrying', async () => {
|
||||
let clock = 0
|
||||
let reject!: (error: Error) => void
|
||||
const query = vi.fn(
|
||||
() =>
|
||||
new Promise<never>((_, fail) => {
|
||||
reject = fail
|
||||
})
|
||||
)
|
||||
const ready = createPushReadiness({ query } as unknown as PushDatabase, { now: () => clock })
|
||||
const checks = Array.from({ length: 100 }, () => ready())
|
||||
expect(query).toHaveBeenCalledTimes(1)
|
||||
reject(new Error('offline'))
|
||||
expect(await Promise.all(checks)).toEqual(Array(100).fill(false))
|
||||
expect(await ready()).toBe(false)
|
||||
expect(query).toHaveBeenCalledTimes(1)
|
||||
clock = 10_000
|
||||
const retry = ready()
|
||||
expect(query).toHaveBeenCalledTimes(2)
|
||||
reject(new Error('offline'))
|
||||
expect(await retry).toBe(false)
|
||||
})
|
||||
@@ -0,0 +1,39 @@
|
||||
import type { PushDatabase } from './push-database.js'
|
||||
|
||||
export type PushReadinessOptions = {
|
||||
cacheMs?: number
|
||||
now?: () => number
|
||||
}
|
||||
|
||||
// The gateway holds no JWKS dependency, so readiness is exactly "can we reach
|
||||
// the database": /health stays unconditional for the container probe.
|
||||
export function createPushReadiness(
|
||||
database: PushDatabase,
|
||||
options: PushReadinessOptions = {}
|
||||
): () => Promise<boolean> {
|
||||
const cacheMs = options.cacheMs ?? 10_000
|
||||
const now = options.now ?? Date.now
|
||||
let cachedAt = Number.NEGATIVE_INFINITY
|
||||
let cached = false
|
||||
|
||||
let pending: Promise<boolean> | null = null
|
||||
|
||||
async function check(): Promise<boolean> {
|
||||
try {
|
||||
await database.query('SELECT 1 AS ready')
|
||||
cached = true
|
||||
} catch {
|
||||
cached = false
|
||||
}
|
||||
cachedAt = now()
|
||||
return cached
|
||||
}
|
||||
|
||||
return async () => {
|
||||
if (now() - cachedAt < cacheMs) return cached
|
||||
pending ??= check().finally(() => {
|
||||
pending = null
|
||||
})
|
||||
return pending
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,28 @@
|
||||
import type { MiddlewareHandler } from 'hono'
|
||||
|
||||
export class PushRequestDrain {
|
||||
private draining = false
|
||||
private active = 0
|
||||
private readonly waiters = new Set<() => void>()
|
||||
|
||||
readonly middleware: MiddlewareHandler = async (context, next) => {
|
||||
if (this.draining) return context.json({ error: 'shutting_down' }, 503)
|
||||
this.active++
|
||||
try {
|
||||
await next()
|
||||
} finally {
|
||||
this.active--
|
||||
if (this.active === 0) {
|
||||
for (const resolve of this.waiters) resolve()
|
||||
this.waiters.clear()
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
begin(): Promise<void> {
|
||||
this.draining = true
|
||||
return this.active === 0
|
||||
? Promise.resolve()
|
||||
: new Promise((resolve) => this.waiters.add(resolve))
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,46 @@
|
||||
import { DURABLE_PUSH_SCHEMA } from './durable-push-schema.js'
|
||||
// Applied at startup for both dialects, including additive queue tables,
|
||||
// so every column type has to read the same in SQLite and PostgreSQL.
|
||||
const PUSH_SCHEMA = `
|
||||
CREATE TABLE IF NOT EXISTS push_challenges (
|
||||
challenge_id TEXT PRIMARY KEY,
|
||||
host_fingerprint TEXT NOT NULL,
|
||||
secret_hash TEXT NOT NULL,
|
||||
expires_at BIGINT NOT NULL,
|
||||
consumed_at BIGINT
|
||||
);
|
||||
CREATE INDEX IF NOT EXISTS push_challenges_expires_at ON push_challenges(expires_at);
|
||||
|
||||
CREATE TABLE IF NOT EXISTS push_sessions (
|
||||
token_hash TEXT PRIMARY KEY,
|
||||
host_fingerprint TEXT NOT NULL,
|
||||
expires_at BIGINT NOT NULL,
|
||||
created_at BIGINT NOT NULL
|
||||
);
|
||||
CREATE UNIQUE INDEX IF NOT EXISTS push_sessions_host ON push_sessions(host_fingerprint);
|
||||
CREATE INDEX IF NOT EXISTS push_sessions_expires_at ON push_sessions(expires_at);
|
||||
|
||||
CREATE TABLE IF NOT EXISTS push_devices (
|
||||
registration_id TEXT PRIMARY KEY,
|
||||
host_fingerprint TEXT NOT NULL,
|
||||
device_id TEXT NOT NULL,
|
||||
platform TEXT NOT NULL,
|
||||
token TEXT NOT NULL,
|
||||
apns_environment TEXT,
|
||||
dead_at BIGINT,
|
||||
created_at BIGINT NOT NULL,
|
||||
updated_at BIGINT NOT NULL
|
||||
);
|
||||
CREATE UNIQUE INDEX IF NOT EXISTS push_devices_host_device
|
||||
ON push_devices(host_fingerprint, device_id);
|
||||
`
|
||||
|
||||
export function pushSchemaStatements(): string[] {
|
||||
// Comments are stripped before the split so a ';' inside one cannot cut a
|
||||
// statement in half and hand SQLite an "incomplete input" fragment.
|
||||
return (PUSH_SCHEMA + DURABLE_PUSH_SCHEMA)
|
||||
.replace(/--[^\n]*/g, '')
|
||||
.split(';')
|
||||
.map((statement) => statement.trim())
|
||||
.filter((statement) => statement.length > 0)
|
||||
}
|
||||
@@ -0,0 +1,64 @@
|
||||
import { afterEach, expect, it } from 'vitest'
|
||||
import { createPushServerHarness, notification } from './push-server-harness.test-fixture.js'
|
||||
import { createPushHostKeypair } from './host-challenge-answering.test-fixture.js'
|
||||
const harnesses: Awaited<ReturnType<typeof createPushServerHarness>>[] = []
|
||||
afterEach(async () => {
|
||||
await Promise.all(harnesses.splice(0).map((h) => h.close()))
|
||||
})
|
||||
|
||||
it('returns queued for concurrent retries without double quota or delivery', async () => {
|
||||
const h = await createPushServerHarness()
|
||||
harnesses.push(h)
|
||||
const token = await h.signIn(createPushHostKeypair(2))
|
||||
const registrationId = await h.registerAndroid(token)
|
||||
const body = { v: 1, registrationIds: [registrationId], notification: notification() }
|
||||
const responses = await Promise.all(
|
||||
Array.from({ length: 10 }, () => h.post('/v1/send', body, token))
|
||||
)
|
||||
for (const response of responses)
|
||||
expect(await response.json()).toEqual({ results: [{ registrationId, status: 'queued' }] })
|
||||
expect(await h.server.deliveryStore.pendingCount(registrationId)).toBe(1)
|
||||
await h.flushDeliveries()
|
||||
await h.post('/v1/send', body, token)
|
||||
await h.flushDeliveries()
|
||||
expect(h.fcmRequests).toHaveLength(1)
|
||||
expect(JSON.parse(h.fcmRequests[0]!.body).message.data.coalescedCount).toBeUndefined()
|
||||
expect(
|
||||
Number((await h.database.query('SELECT COUNT(*) AS count FROM push_events'))[0]?.count)
|
||||
).toBe(1)
|
||||
await h.post(
|
||||
'/v1/send',
|
||||
{ ...body, notification: notification({ notificationEpoch: 'new-epoch' }) },
|
||||
token
|
||||
)
|
||||
await h.flushDeliveries()
|
||||
expect(h.fcmRequests).toHaveLength(2)
|
||||
})
|
||||
|
||||
it.each([false, true])(
|
||||
'accepts default alert kind equivalently through the API (explicit first: %s)',
|
||||
async (explicitFirst) => {
|
||||
const h = await createPushServerHarness()
|
||||
harnesses.push(h)
|
||||
const token = await h.signIn(createPushHostKeypair(3))
|
||||
const registrationId = await h.registerAndroid(token)
|
||||
const implicit = notification()
|
||||
const explicit = { kind: 'alert', ...implicit }
|
||||
for (const event of explicitFirst ? [explicit, implicit] : [implicit, explicit]) {
|
||||
const response = await h.post(
|
||||
'/v1/send',
|
||||
{ v: 1, registrationIds: [registrationId], notification: event },
|
||||
token
|
||||
)
|
||||
expect(await response.json()).toEqual({ results: [{ registrationId, status: 'queued' }] })
|
||||
}
|
||||
await h.flushDeliveries()
|
||||
expect(h.fcmRequests).toHaveLength(1)
|
||||
const changed = await h.post(
|
||||
'/v1/send',
|
||||
{ v: 1, registrationIds: [registrationId], notification: { ...explicit, body: 'changed' } },
|
||||
token
|
||||
)
|
||||
expect(await changed.json()).toEqual({ results: [{ registrationId, status: 'error' }] })
|
||||
}
|
||||
)
|
||||
@@ -0,0 +1,162 @@
|
||||
import { PUSH_LIMITS } from '@orca-cloud/push-contract'
|
||||
import { afterEach, beforeEach, describe, expect, it } from 'vitest'
|
||||
import { createPushHostKeypair } from './host-challenge-answering.test-fixture.js'
|
||||
import type { PushDatabase } from './push-database.js'
|
||||
import { createPushServer } from './push-server.js'
|
||||
import {
|
||||
createPushServerHarness,
|
||||
testPushConfig
|
||||
} from './push-server-harness.test-fixture.js'
|
||||
|
||||
describe('push gateway authentication and device routes', () => {
|
||||
let harness: Awaited<ReturnType<typeof createPushServerHarness>>
|
||||
|
||||
beforeEach(async () => {
|
||||
harness = await createPushServerHarness()
|
||||
})
|
||||
|
||||
afterEach(async () => {
|
||||
await harness.close()
|
||||
})
|
||||
|
||||
it('answers health unconditionally and ready from the database', async () => {
|
||||
expect((await harness.server.app.request('/health')).status).toBe(200)
|
||||
expect((await harness.server.app.request('/ready')).status).toBe(200)
|
||||
})
|
||||
|
||||
it('reports not ready when the database is unreachable', async () => {
|
||||
const unreachable: PushDatabase = {
|
||||
dialect: 'sqlite',
|
||||
query: async () => {
|
||||
throw new Error('no connection')
|
||||
},
|
||||
transaction: async (operation) => await operation(unreachable),
|
||||
lockQuotaScope: async () => undefined,
|
||||
close: async () => undefined
|
||||
}
|
||||
const broken = createPushServer(testPushConfig(), unreachable, {
|
||||
fcmAccessToken: async () => 'token',
|
||||
fcmTransport: async () => ({ status: 200, body: '{}' })
|
||||
})
|
||||
expect((await broken.app.request('/health')).status).toBe(200)
|
||||
expect((await broken.app.request('/ready')).status).toBe(503)
|
||||
await broken.worker.stop()
|
||||
})
|
||||
|
||||
it('completes challenge, session, register, list, delete', async () => {
|
||||
const sessionToken = await harness.signIn(createPushHostKeypair(11))
|
||||
const registrationId = await harness.registerAndroid(sessionToken)
|
||||
|
||||
const list = await harness.authorized('/v1/devices', {}, sessionToken)
|
||||
expect(await list.json()).toEqual({
|
||||
devices: [{ registrationId, deviceId: 'device-1', platform: 'android', dead: false }]
|
||||
})
|
||||
|
||||
const deleted = await harness.authorized(
|
||||
`/v1/devices/${registrationId}`,
|
||||
{ method: 'DELETE' },
|
||||
sessionToken
|
||||
)
|
||||
expect(deleted.status).toBe(204)
|
||||
expect(await harness.server.devices.findById(registrationId)).toBeNull()
|
||||
})
|
||||
|
||||
it('refuses a request with no bearer, a bogus bearer, and an expired session', async () => {
|
||||
const sessionToken = await harness.signIn(createPushHostKeypair(12))
|
||||
expect((await harness.server.app.request('/v1/devices')).status).toBe(401)
|
||||
const bogus = await harness.authorized('/v1/devices', {}, 'nonsense')
|
||||
expect(bogus.status).toBe(401)
|
||||
expect(await bogus.json()).toEqual({ error: 'invalid_token' })
|
||||
|
||||
harness.advanceClock(PUSH_LIMITS.sessionTtlMs + 1)
|
||||
const expired = await harness.authorized('/v1/devices', {}, sessionToken)
|
||||
expect(expired.status).toBe(401)
|
||||
expect(await expired.json()).toEqual({ error: 'session_expired' })
|
||||
})
|
||||
|
||||
it('refuses a replayed proof and an unknown challenge', async () => {
|
||||
const host = createPushHostKeypair(13)
|
||||
const challenge = await harness.issueChallenge(host)
|
||||
const proof = harness.answer(challenge, host)
|
||||
expect(
|
||||
(
|
||||
await harness.post('/v1/host/session', {
|
||||
v: 1,
|
||||
challengeId: challenge.challengeId,
|
||||
proofB64: proof
|
||||
})
|
||||
).status
|
||||
).toBe(200)
|
||||
|
||||
const replay = await harness.post('/v1/host/session', {
|
||||
v: 1,
|
||||
challengeId: challenge.challengeId,
|
||||
proofB64: proof
|
||||
})
|
||||
expect(replay.status).toBe(401)
|
||||
expect(await replay.json()).toEqual({ error: 'invalid_proof' })
|
||||
|
||||
const unknown = await harness.post('/v1/host/session', {
|
||||
v: 1,
|
||||
challengeId: 'no-such-challenge',
|
||||
proofB64: proof
|
||||
})
|
||||
expect(await unknown.json()).toEqual({ error: 'invalid_challenge' })
|
||||
})
|
||||
|
||||
it('never returns the host fingerprint on the challenge itself', async () => {
|
||||
const challenge = await harness.issueChallenge(createPushHostKeypair(22))
|
||||
expect(Object.keys(challenge).sort()).toEqual([
|
||||
'challengeId',
|
||||
'ciphertextB64',
|
||||
'expiresAt',
|
||||
'gatewayEphemeralPublicKeyB64',
|
||||
'nonceB64'
|
||||
])
|
||||
})
|
||||
|
||||
it('lets only the owning host delete a registration', async () => {
|
||||
const ownerToken = await harness.signIn(createPushHostKeypair(14))
|
||||
const intruderToken = await harness.signIn(createPushHostKeypair(15))
|
||||
const registrationId = await harness.registerAndroid(ownerToken)
|
||||
|
||||
const forbidden = await harness.authorized(
|
||||
`/v1/devices/${registrationId}`,
|
||||
{ method: 'DELETE' },
|
||||
intruderToken
|
||||
)
|
||||
expect(forbidden.status).toBe(404)
|
||||
expect(await forbidden.json()).toEqual({ error: 'not_found' })
|
||||
expect(await harness.server.devices.findById(registrationId)).not.toBeNull()
|
||||
})
|
||||
|
||||
it('replaces the token on a re-registration and keeps one registration id', async () => {
|
||||
const sessionToken = await harness.signIn(createPushHostKeypair(23))
|
||||
const first = await harness.registerAndroid(sessionToken)
|
||||
const again = await harness.post(
|
||||
'/v1/devices',
|
||||
{
|
||||
v: 1,
|
||||
deviceId: 'device-1',
|
||||
platform: 'android',
|
||||
token: 'rotated_token:APA91b-newnewnewnewnewnewnewnewnewnew'
|
||||
},
|
||||
sessionToken
|
||||
)
|
||||
expect(await again.json()).toEqual({ registrationId: first })
|
||||
expect(await harness.server.devices.findById(first)).toMatchObject({
|
||||
token: 'rotated_token:APA91b-newnewnewnewnewnewnewnewnewnew'
|
||||
})
|
||||
})
|
||||
|
||||
it('rejects a malformed registration body', async () => {
|
||||
const sessionToken = await harness.signIn(createPushHostKeypair(16))
|
||||
const bad = await harness.post(
|
||||
'/v1/devices',
|
||||
{ v: 1, deviceId: 'device-1', platform: 'ios', token: 'not-hex' },
|
||||
sessionToken
|
||||
)
|
||||
expect(bad.status).toBe(400)
|
||||
expect(await bad.json()).toEqual({ error: 'invalid_request' })
|
||||
})
|
||||
})
|
||||
@@ -0,0 +1,162 @@
|
||||
import { generateKeyPairSync } from 'node:crypto'
|
||||
import { expect } from 'vitest'
|
||||
import type { ApnsRequest, ApnsResponse } from './apns-http2-transport.js'
|
||||
import type { PushConfig } from './config.js'
|
||||
import type { FcmRequest, FcmResponse } from './fcm-client.js'
|
||||
import {
|
||||
answerPushHostChallenge,
|
||||
hostPublicKeyB64,
|
||||
type PushHostKeypair
|
||||
} from './host-challenge-answering.test-fixture.js'
|
||||
import { openInMemoryPushDatabase, type PushDatabase } from './push-database.js'
|
||||
import { createPushServer } from './push-server.js'
|
||||
|
||||
export const GATEWAY_ORIGIN = 'https://push.onorca.dev'
|
||||
export const APNS_TOKEN = 'a'.repeat(64)
|
||||
export const FCM_TOKEN = 'cQ1abcDEF_gh:APA91bZZ-zz0123456789abcdefghijklmnopqrstuvwxyz'
|
||||
|
||||
export function notification(overrides: Record<string, unknown> = {}): Record<string, unknown> {
|
||||
return {
|
||||
notificationId: 'note-1',
|
||||
notificationSeq: 1,
|
||||
notificationEpoch: 'epoch-1',
|
||||
source: 'agent-task-complete',
|
||||
agentState: 'needs-input',
|
||||
title: 'Agent needs input',
|
||||
body: 'Waiting on your answer',
|
||||
worktreeId: 'wt-1',
|
||||
...overrides
|
||||
}
|
||||
}
|
||||
|
||||
export function testPushConfig(): PushConfig {
|
||||
const { privateKey } = generateKeyPairSync('ec', {
|
||||
namedCurve: 'P-256',
|
||||
privateKeyEncoding: { type: 'pkcs8', format: 'pem' },
|
||||
publicKeyEncoding: { type: 'spki', format: 'pem' }
|
||||
})
|
||||
return {
|
||||
mode: 'active',
|
||||
port: 0,
|
||||
publicUrl: GATEWAY_ORIGIN,
|
||||
dataDir: './data/push-test',
|
||||
databasePoolMax: 10,
|
||||
apns: { keyPem: privateKey, keyId: 'ABCDE12345', teamId: 'TEAM123456' },
|
||||
apnsTopic: 'com.stably.orca.mobile',
|
||||
fcmProjectId: 'onorca-cloud',
|
||||
trustedProxyHops: 0
|
||||
}
|
||||
}
|
||||
|
||||
type ChallengeWire = {
|
||||
challengeId: string
|
||||
gatewayEphemeralPublicKeyB64: string
|
||||
nonceB64: string
|
||||
ciphertextB64: string
|
||||
expiresAt: number
|
||||
}
|
||||
|
||||
export async function createPushServerHarness() {
|
||||
const database: PushDatabase = await openInMemoryPushDatabase()
|
||||
let clock = 1_700_000_000_000
|
||||
const apnsRequests: ApnsRequest[] = []
|
||||
const fcmRequests: FcmRequest[] = []
|
||||
let apnsResponse: ApnsResponse = { status: 200, body: '' }
|
||||
let fcmResponse: FcmResponse = { status: 200, body: '{}' }
|
||||
const server = createPushServer(testPushConfig(), database, {
|
||||
now: () => clock,
|
||||
apnsTransport: async (request) => {
|
||||
apnsRequests.push(request)
|
||||
return apnsResponse
|
||||
},
|
||||
fcmTransport: async (request) => {
|
||||
fcmRequests.push(request)
|
||||
return fcmResponse
|
||||
},
|
||||
fcmAccessToken: async () => 'access-token'
|
||||
})
|
||||
|
||||
const post = async (path: string, body: unknown, token?: string): Promise<Response> =>
|
||||
await server.app.request(path, {
|
||||
method: 'POST',
|
||||
headers: {
|
||||
'content-type': 'application/json',
|
||||
...(token ? { authorization: `Bearer ${token}` } : {})
|
||||
},
|
||||
body: JSON.stringify(body)
|
||||
})
|
||||
|
||||
const issueChallenge = async (keypair: PushHostKeypair): Promise<ChallengeWire> => {
|
||||
const response = await post('/v1/host/challenge', {
|
||||
v: 1,
|
||||
hostPublicKeyB64: hostPublicKeyB64(keypair)
|
||||
})
|
||||
expect(response.status).toBe(200)
|
||||
return (await response.json()) as ChallengeWire
|
||||
}
|
||||
|
||||
const answer = (challenge: ChallengeWire, keypair: PushHostKeypair): string => {
|
||||
const proof = answerPushHostChallenge(challenge, {
|
||||
gatewayOrigin: GATEWAY_ORIGIN,
|
||||
keypair,
|
||||
now: () => clock
|
||||
})
|
||||
expect(proof).not.toBeNull()
|
||||
return proof!
|
||||
}
|
||||
|
||||
return {
|
||||
server,
|
||||
database,
|
||||
apnsRequests,
|
||||
fcmRequests,
|
||||
post,
|
||||
issueChallenge,
|
||||
answer,
|
||||
now: () => clock,
|
||||
flushDeliveries: async (): Promise<void> => {
|
||||
await server.worker.runDue()
|
||||
},
|
||||
advanceClock: (deltaMs: number): void => {
|
||||
clock += deltaMs
|
||||
},
|
||||
setApnsResponse: (response: ApnsResponse): void => {
|
||||
apnsResponse = response
|
||||
},
|
||||
setFcmResponse: (response: FcmResponse): void => {
|
||||
fcmResponse = response
|
||||
},
|
||||
authorized: async (path: string, init: RequestInit = {}, token?: string): Promise<Response> =>
|
||||
await server.app.request(path, {
|
||||
...init,
|
||||
headers: {
|
||||
...(init.headers as Record<string, string> | undefined),
|
||||
...(token ? { authorization: `Bearer ${token}` } : {})
|
||||
}
|
||||
}),
|
||||
signIn: async (keypair: PushHostKeypair): Promise<string> => {
|
||||
const challenge = await issueChallenge(keypair)
|
||||
const response = await post('/v1/host/session', {
|
||||
v: 1,
|
||||
challengeId: challenge.challengeId,
|
||||
proofB64: answer(challenge, keypair)
|
||||
})
|
||||
expect(response.status).toBe(200)
|
||||
return ((await response.json()) as { sessionToken: string }).sessionToken
|
||||
},
|
||||
registerAndroid: async (token: string, deviceId = 'device-1'): Promise<string> => {
|
||||
const response = await post(
|
||||
'/v1/devices',
|
||||
{ v: 1, deviceId, platform: 'android', token: FCM_TOKEN },
|
||||
token
|
||||
)
|
||||
expect(response.status).toBe(200)
|
||||
return ((await response.json()) as { registrationId: string }).registrationId
|
||||
},
|
||||
close: async (): Promise<void> => {
|
||||
await server.worker.stop()
|
||||
// A test may close the database itself to provoke a route failure.
|
||||
await database.close().catch(() => undefined)
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,273 @@
|
||||
import { PUSH_LIMITS } from '@orca-cloud/push-contract'
|
||||
import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest'
|
||||
import { createPushHostKeypair, hostPublicKeyB64 } from './host-challenge-answering.test-fixture.js'
|
||||
import {
|
||||
createPushServerHarness,
|
||||
FCM_TOKEN,
|
||||
notification
|
||||
} from './push-server-harness.test-fixture.js'
|
||||
|
||||
const CLIENT_IP = '203.0.113.7'
|
||||
const OTHER_CLIENT_IP = '198.51.100.9'
|
||||
|
||||
function oversizedChallengeBody(): string {
|
||||
return JSON.stringify({ v: 1, filler: 'x'.repeat(PUSH_LIMITS.maxHttpBodyBytes) })
|
||||
}
|
||||
|
||||
function chunkedRequest(path: string, body: string): Request {
|
||||
const stream = new ReadableStream<Uint8Array>({
|
||||
start(controller) {
|
||||
controller.enqueue(new TextEncoder().encode(body))
|
||||
controller.close()
|
||||
}
|
||||
})
|
||||
return new Request(`http://push.test${path}`, {
|
||||
method: 'POST',
|
||||
headers: { 'content-type': 'application/json' },
|
||||
body: stream,
|
||||
duplex: 'half'
|
||||
} as RequestInit)
|
||||
}
|
||||
|
||||
describe('push gateway request limits', () => {
|
||||
let harness: Awaited<ReturnType<typeof createPushServerHarness>>
|
||||
|
||||
beforeEach(async () => {
|
||||
harness = await createPushServerHarness()
|
||||
})
|
||||
|
||||
afterEach(async () => {
|
||||
await harness.close()
|
||||
})
|
||||
|
||||
it('refuses an oversized chunked body that declares no content length', async () => {
|
||||
const request = chunkedRequest('/v1/host/challenge', oversizedChallengeBody())
|
||||
expect(request.headers.get('content-length')).toBeNull()
|
||||
|
||||
const response = await harness.server.app.request(request)
|
||||
expect(response.status).toBe(413)
|
||||
expect(await response.json()).toEqual({ error: 'request_too_large' })
|
||||
})
|
||||
|
||||
it('still refuses an oversized body that declares a content length', async () => {
|
||||
const body = oversizedChallengeBody()
|
||||
const response = await harness.server.app.request('/v1/host/challenge', {
|
||||
method: 'POST',
|
||||
headers: {
|
||||
'content-type': 'application/json',
|
||||
'content-length': String(Buffer.byteLength(body))
|
||||
},
|
||||
body
|
||||
})
|
||||
expect(response.status).toBe(413)
|
||||
expect(await response.json()).toEqual({ error: 'request_too_large' })
|
||||
})
|
||||
|
||||
it('lets a chunked body under the cap through to schema validation', async () => {
|
||||
const response = await harness.server.app.request(
|
||||
chunkedRequest(
|
||||
'/v1/host/challenge',
|
||||
JSON.stringify({ v: 1, hostPublicKeyB64: hostPublicKeyB64(createPushHostKeypair(60)) })
|
||||
)
|
||||
)
|
||||
expect(response.status).toBe(200)
|
||||
})
|
||||
|
||||
it('caps an authenticated oversized send as well', async () => {
|
||||
const sessionToken = await harness.signIn(createPushHostKeypair(61))
|
||||
const response = await harness.server.app.request(
|
||||
new Request('http://push.test/v1/send', {
|
||||
method: 'POST',
|
||||
headers: {
|
||||
'content-type': 'application/json',
|
||||
authorization: `Bearer ${sessionToken}`
|
||||
},
|
||||
body: new ReadableStream<Uint8Array>({
|
||||
start(controller) {
|
||||
controller.enqueue(new TextEncoder().encode(oversizedChallengeBody()))
|
||||
controller.close()
|
||||
}
|
||||
}),
|
||||
duplex: 'half'
|
||||
} as RequestInit)
|
||||
)
|
||||
expect(response.status).toBe(413)
|
||||
expect(await response.json()).toEqual({ error: 'request_too_large' })
|
||||
})
|
||||
|
||||
it('rate limits one client ip across both unauthenticated routes', async () => {
|
||||
const body = JSON.stringify({
|
||||
v: 1,
|
||||
hostPublicKeyB64: hostPublicKeyB64(createPushHostKeypair(62))
|
||||
})
|
||||
// Cloud Run appends the peer, so the caller's own IP is the last value.
|
||||
const headers = {
|
||||
'content-type': 'application/json',
|
||||
'x-forwarded-for': `10.0.0.1, ${CLIENT_IP}`
|
||||
}
|
||||
for (let index = 0; index < PUSH_LIMITS.unauthenticatedRequestsPerMinutePerIp; index++) {
|
||||
const allowed = await harness.server.app.request('/v1/host/challenge', {
|
||||
method: 'POST',
|
||||
headers,
|
||||
body
|
||||
})
|
||||
expect(allowed.status).toBe(200)
|
||||
}
|
||||
|
||||
const limited = await harness.server.app.request('/v1/host/challenge', {
|
||||
method: 'POST',
|
||||
headers,
|
||||
body
|
||||
})
|
||||
expect(limited.status).toBe(429)
|
||||
expect(await limited.json()).toEqual({ error: 'rate_limited' })
|
||||
|
||||
// The session route draws on the same bucket, so a flood cannot simply move.
|
||||
const session = await harness.server.app.request('/v1/host/session', {
|
||||
method: 'POST',
|
||||
headers,
|
||||
body: JSON.stringify({ v: 1, challengeId: 'anything', proofB64: 'x'.repeat(44) })
|
||||
})
|
||||
expect(session.status).toBe(429)
|
||||
|
||||
const other = await harness.server.app.request('/v1/host/challenge', {
|
||||
method: 'POST',
|
||||
headers: { ...headers, 'x-forwarded-for': `10.0.0.1, ${OTHER_CLIENT_IP}` },
|
||||
body
|
||||
})
|
||||
expect(other.status).toBe(200)
|
||||
|
||||
// A caller rewriting the left of the chain lands in its own bucket anyway.
|
||||
const spoofed = await harness.server.app.request('/v1/host/challenge', {
|
||||
method: 'POST',
|
||||
headers: { ...headers, 'x-forwarded-for': `198.51.100.250, ${CLIENT_IP}` },
|
||||
body
|
||||
})
|
||||
expect(spoofed.status).toBe(429)
|
||||
})
|
||||
|
||||
it('lets a throttled client back in once the window refills', async () => {
|
||||
const body = JSON.stringify({
|
||||
v: 1,
|
||||
hostPublicKeyB64: hostPublicKeyB64(createPushHostKeypair(63))
|
||||
})
|
||||
const headers = { 'content-type': 'application/json', 'x-forwarded-for': CLIENT_IP }
|
||||
for (let index = 0; index < PUSH_LIMITS.unauthenticatedRequestsPerMinutePerIp; index++) {
|
||||
await harness.server.app.request('/v1/host/challenge', { method: 'POST', headers, body })
|
||||
}
|
||||
expect(
|
||||
(await harness.server.app.request('/v1/host/challenge', { method: 'POST', headers, body }))
|
||||
.status
|
||||
).toBe(429)
|
||||
|
||||
harness.advanceClock(60_000)
|
||||
expect(
|
||||
(await harness.server.app.request('/v1/host/challenge', { method: 'POST', headers, body }))
|
||||
.status
|
||||
).toBe(200)
|
||||
})
|
||||
|
||||
it('limits authenticated hosts independently behind the same IP', async () => {
|
||||
const sessionToken = await harness.signIn(createPushHostKeypair(64))
|
||||
const headers = { 'x-forwarded-for': CLIENT_IP }
|
||||
for (let index = 0; index < 600; index++) {
|
||||
const listed = await harness.authorized('/v1/devices', { headers }, sessionToken)
|
||||
expect(listed.status).toBe(200)
|
||||
}
|
||||
const limited = await harness.authorized('/v1/devices', { headers }, sessionToken)
|
||||
expect(limited.status).toBe(429)
|
||||
const otherToken = await harness.signIn(createPushHostKeypair(68))
|
||||
expect((await harness.authorized('/v1/devices', { headers }, otherToken)).status).toBe(200)
|
||||
// The handshake bucket is untouched by any of that.
|
||||
const challenge = await harness.server.app.request('/v1/host/challenge', {
|
||||
method: 'POST',
|
||||
headers: { ...headers, 'content-type': 'application/json' },
|
||||
body: JSON.stringify({ v: 1, hostPublicKeyB64: hostPublicKeyB64(createPushHostKeypair(67)) })
|
||||
})
|
||||
expect(challenge.status).toBe(200)
|
||||
})
|
||||
|
||||
it('stops repeated forged bearers after the invalid-auth budget is exhausted', async () => {
|
||||
const headers = { 'x-forwarded-for': CLIENT_IP }
|
||||
const [before] = await harness.database.query('SELECT COUNT(*) AS sessions FROM push_sessions')
|
||||
for (let index = 0; index < PUSH_LIMITS.unauthenticatedRequestsPerMinutePerIp; index++) {
|
||||
const refused = await harness.authorized('/v1/send', { method: 'POST', headers }, 'forged')
|
||||
expect(refused.status).toBe(401)
|
||||
}
|
||||
const limited = await harness.authorized('/v1/send', { method: 'POST', headers }, 'forged')
|
||||
expect(limited.status).toBe(429)
|
||||
expect(await limited.json()).toEqual({ error: 'rate_limited' })
|
||||
expect(harness.server.unauthenticatedIps.trackedIpCount()).toBe(0)
|
||||
const [after] = await harness.database.query('SELECT COUNT(*) AS sessions FROM push_sessions')
|
||||
expect(Number(after?.sessions)).toBe(Number(before?.sessions))
|
||||
})
|
||||
|
||||
it('answers 409 once a host has registered its device allowance', async () => {
|
||||
const sessionToken = await harness.signIn(createPushHostKeypair(66))
|
||||
for (let index = 0; index < PUSH_LIMITS.maxDevicesPerHost; index++) {
|
||||
const accepted = await harness.post(
|
||||
'/v1/devices',
|
||||
{
|
||||
v: 1,
|
||||
deviceId: `device-${index}`,
|
||||
platform: 'android',
|
||||
token: FCM_TOKEN
|
||||
},
|
||||
sessionToken
|
||||
)
|
||||
expect(accepted.status).toBe(200)
|
||||
}
|
||||
|
||||
const refused = await harness.post(
|
||||
'/v1/devices',
|
||||
{ v: 1, deviceId: 'one-too-many', platform: 'android', token: FCM_TOKEN },
|
||||
sessionToken
|
||||
)
|
||||
expect(refused.status).toBe(409)
|
||||
expect(await refused.json()).toEqual({ error: 'too_many_devices' })
|
||||
|
||||
const listed = await harness.authorized('/v1/devices', {}, sessionToken)
|
||||
expect(((await listed.json()) as { devices: unknown[] }).devices).toHaveLength(
|
||||
PUSH_LIMITS.maxDevicesPerHost
|
||||
)
|
||||
})
|
||||
|
||||
// Why: a database error carries the failing row in its message. The response
|
||||
// and the log must both stop at the error's name.
|
||||
it('answers an unexpected route failure with a bare 500 and logs only the name', async () => {
|
||||
const sessionToken = await harness.signIn(createPushHostKeypair(66))
|
||||
const warn = vi.spyOn(console, 'warn').mockImplementation(() => undefined)
|
||||
try {
|
||||
await harness.database.close()
|
||||
const response = await harness.authorized('/v1/devices', {}, sessionToken)
|
||||
expect(response.status).toBe(500)
|
||||
expect(await response.json()).toEqual({ error: 'internal' })
|
||||
const logged = warn.mock.calls.map((call) => String(call[0])).join('\n')
|
||||
expect(logged).toContain('"event":"orca_push_request_failed"')
|
||||
expect(logged).not.toContain('SELECT')
|
||||
expect(logged).not.toContain('push_devices')
|
||||
expect(harness.server.observability.consume().request_error).toBe(1)
|
||||
} finally {
|
||||
warn.mockRestore()
|
||||
}
|
||||
})
|
||||
|
||||
it('charges a repeated registration id once and returns one result', async () => {
|
||||
const sessionToken = await harness.signIn(createPushHostKeypair(65))
|
||||
const registrationId = await harness.registerAndroid(sessionToken)
|
||||
|
||||
const response = await harness.post(
|
||||
'/v1/send',
|
||||
{
|
||||
v: 1,
|
||||
registrationIds: [registrationId, registrationId, registrationId],
|
||||
notification: notification()
|
||||
},
|
||||
sessionToken
|
||||
)
|
||||
expect(await response.json()).toEqual({ results: [{ registrationId, status: 'queued' }] })
|
||||
expect(await harness.server.deliveryStore.pendingCount(registrationId)).toBe(1)
|
||||
const [row] = await harness.database.query('SELECT COUNT(*) AS sends FROM push_events')
|
||||
expect(Number(row?.sends)).toBe(1)
|
||||
})
|
||||
})
|
||||
@@ -0,0 +1,229 @@
|
||||
import { PushNotificationSchema } from '@orca-cloud/push-contract'
|
||||
import { PUSH_LIMITS } from '@orca-cloud/push-contract'
|
||||
import { afterEach, beforeEach, describe, expect, it } from 'vitest'
|
||||
import { createPushHostKeypair } from './host-challenge-answering.test-fixture.js'
|
||||
import {
|
||||
APNS_TOKEN,
|
||||
createPushServerHarness,
|
||||
FCM_TOKEN,
|
||||
notification
|
||||
} from './push-server-harness.test-fixture.js'
|
||||
|
||||
describe('push gateway send route', () => {
|
||||
let harness: Awaited<ReturnType<typeof createPushServerHarness>>
|
||||
|
||||
beforeEach(async () => {
|
||||
harness = await createPushServerHarness()
|
||||
})
|
||||
|
||||
afterEach(async () => {
|
||||
await harness.close()
|
||||
})
|
||||
|
||||
it('rejects a batch over the registration cap', async () => {
|
||||
const sessionToken = await harness.signIn(createPushHostKeypair(16))
|
||||
const oversized = await harness.post(
|
||||
'/v1/send',
|
||||
{
|
||||
v: 1,
|
||||
registrationIds: Array.from(
|
||||
{ length: PUSH_LIMITS.maxRegistrationIdsPerSend + 1 },
|
||||
(_, index) => `reg-${index}`
|
||||
),
|
||||
notification: notification()
|
||||
},
|
||||
sessionToken
|
||||
)
|
||||
expect(oversized.status).toBe(400)
|
||||
expect(await oversized.json()).toEqual({ error: 'invalid_request' })
|
||||
})
|
||||
|
||||
it('queues a send, delivers it to fcm, and reports a dead token on the next send', async () => {
|
||||
const sessionToken = await harness.signIn(createPushHostKeypair(17))
|
||||
const registrationId = await harness.registerAndroid(sessionToken)
|
||||
|
||||
const queued = await harness.post(
|
||||
'/v1/send',
|
||||
{ v: 1, registrationIds: [registrationId], notification: notification() },
|
||||
sessionToken
|
||||
)
|
||||
expect(await queued.json()).toEqual({ results: [{ registrationId, status: 'queued' }] })
|
||||
|
||||
harness.setFcmResponse({
|
||||
status: 404,
|
||||
body: JSON.stringify({ error: { status: 'UNREGISTERED', message: 'gone' } })
|
||||
})
|
||||
await harness.flushDeliveries()
|
||||
expect(harness.fcmRequests).toHaveLength(1)
|
||||
expect(JSON.parse(harness.fcmRequests[0]!.body)).toMatchObject({
|
||||
message: { token: FCM_TOKEN, notification: { title: 'Agent needs input' } }
|
||||
})
|
||||
|
||||
const afterDeath = await harness.post(
|
||||
'/v1/send',
|
||||
{ v: 1, registrationIds: [registrationId], notification: notification() },
|
||||
sessionToken
|
||||
)
|
||||
expect(await afterDeath.json()).toEqual({ results: [{ registrationId, status: 'dead' }] })
|
||||
|
||||
const listed = await harness.authorized('/v1/devices', {}, sessionToken)
|
||||
expect(await listed.json()).toEqual({
|
||||
devices: [{ registrationId, deviceId: 'device-1', platform: 'android', dead: true }]
|
||||
})
|
||||
})
|
||||
|
||||
it('leaves a live registration alone when the provider reports a transient failure', async () => {
|
||||
const sessionToken = await harness.signIn(createPushHostKeypair(24))
|
||||
const registrationId = await harness.registerAndroid(sessionToken)
|
||||
await harness.post(
|
||||
'/v1/send',
|
||||
{ v: 1, registrationIds: [registrationId], notification: notification() },
|
||||
sessionToken
|
||||
)
|
||||
harness.setFcmResponse({
|
||||
status: 503,
|
||||
body: JSON.stringify({ error: { status: 'UNAVAILABLE', message: 'backend busy' } })
|
||||
})
|
||||
await harness.flushDeliveries()
|
||||
expect(await harness.server.devices.findById(registrationId)).toMatchObject({ dead: false })
|
||||
})
|
||||
|
||||
it('reports retries from the durable worker after the provider delay', async () => {
|
||||
const token = await harness.signIn(createPushHostKeypair(26))
|
||||
const registrationId = await harness.registerAndroid(token)
|
||||
await harness.post(
|
||||
'/v1/send',
|
||||
{
|
||||
v: 1,
|
||||
registrationIds: [registrationId],
|
||||
notification: notification()
|
||||
},
|
||||
token
|
||||
)
|
||||
harness.setFcmResponse({ status: 503, body: '{}' })
|
||||
await harness.flushDeliveries()
|
||||
expect(harness.server.observability.consume()).toMatchObject({
|
||||
delivery_error: 1,
|
||||
delivery_retry: 0
|
||||
})
|
||||
harness.advanceClock(10_000)
|
||||
harness.setFcmResponse({ status: 200, body: '{}' })
|
||||
await harness.server.worker.runDue()
|
||||
expect(harness.server.observability.consume()).toMatchObject({
|
||||
delivery_sent: 1,
|
||||
delivery_retry: 1
|
||||
})
|
||||
})
|
||||
|
||||
it('sends a burst as individual APNs alerts grouped by the host thread', async () => {
|
||||
const sessionToken = await harness.signIn(createPushHostKeypair(18))
|
||||
const registration = await harness.post(
|
||||
'/v1/devices',
|
||||
{
|
||||
v: 1,
|
||||
deviceId: 'iphone-1',
|
||||
platform: 'ios',
|
||||
token: APNS_TOKEN,
|
||||
apnsEnvironment: 'sandbox'
|
||||
},
|
||||
sessionToken
|
||||
)
|
||||
const { registrationId } = (await registration.json()) as { registrationId: string }
|
||||
for (const seq of [1, 2, 3]) {
|
||||
await harness.post(
|
||||
'/v1/send',
|
||||
{
|
||||
v: 1,
|
||||
registrationIds: [registrationId],
|
||||
notification: notification({ notificationId: `note-${seq}`, notificationSeq: seq })
|
||||
},
|
||||
sessionToken
|
||||
)
|
||||
}
|
||||
await harness.flushDeliveries()
|
||||
expect(harness.apnsRequests).toHaveLength(3)
|
||||
const bodies = harness.apnsRequests.map(
|
||||
(request) =>
|
||||
JSON.parse(request.body) as {
|
||||
aps: { alert: { title: string; body: string }; 'thread-id': string }
|
||||
orca: Record<string, unknown> & { notificationSeq: number }
|
||||
}
|
||||
)
|
||||
expect(
|
||||
harness.apnsRequests.every((request) => request.host === 'api.sandbox.push.apple.com')
|
||||
).toBe(true)
|
||||
expect(bodies.map((body) => body.aps.alert)).toEqual(
|
||||
Array.from({ length: 3 }, () => ({
|
||||
title: 'Agent needs input',
|
||||
body: 'Waiting on your answer'
|
||||
}))
|
||||
)
|
||||
expect(new Set(bodies.map((body) => body.aps['thread-id'])).size).toBe(1)
|
||||
expect(bodies.map((body) => body.orca.notificationSeq).sort((a, b) => a - b)).toEqual([1, 2, 3])
|
||||
expect(bodies.every((body) => !('coalescedCount' in body.orca))).toBe(true)
|
||||
expect(bodies.every((body) => !('summaryMembers' in body.orca))).toBe(true)
|
||||
expect(
|
||||
new Set(harness.apnsRequests.map((request) => request.headers['apns-collapse-id'])).size
|
||||
).toBe(3)
|
||||
})
|
||||
|
||||
it('sends a lone event through unchanged with its own collapse id', async () => {
|
||||
const sessionToken = await harness.signIn(createPushHostKeypair(25))
|
||||
const registrationId = await harness.registerAndroid(sessionToken)
|
||||
await harness.post(
|
||||
'/v1/send',
|
||||
{ v: 1, registrationIds: [registrationId], notification: notification() },
|
||||
sessionToken
|
||||
)
|
||||
await harness.flushDeliveries()
|
||||
const message = JSON.parse(harness.fcmRequests[0]!.body) as {
|
||||
message: { android: { notification: { tag: string } }; data: Record<string, string> }
|
||||
}
|
||||
expect(message.message.android.notification.tag).toMatch(/^[a-f0-9]{64}$/)
|
||||
expect(message.message.data.coalescedCount).toBeUndefined()
|
||||
})
|
||||
|
||||
it('reports an error for a registration the host does not own', async () => {
|
||||
const ownerToken = await harness.signIn(createPushHostKeypair(19))
|
||||
const intruderToken = await harness.signIn(createPushHostKeypair(20))
|
||||
const registrationId = await harness.registerAndroid(ownerToken)
|
||||
|
||||
const foreign = await harness.post(
|
||||
'/v1/send',
|
||||
{ v: 1, registrationIds: [registrationId, 'made-up'], notification: notification() },
|
||||
intruderToken
|
||||
)
|
||||
expect(await foreign.json()).toEqual({
|
||||
results: [
|
||||
{ registrationId, status: 'error' },
|
||||
{ registrationId: 'made-up', status: 'error' }
|
||||
]
|
||||
})
|
||||
expect(await harness.server.deliveryStore.pendingCount(registrationId)).toBe(0)
|
||||
})
|
||||
|
||||
it('rate limits a host that exhausted its 15-minute allowance', async () => {
|
||||
const sessionToken = await harness.signIn(createPushHostKeypair(21))
|
||||
const registrationId = await harness.registerAndroid(sessionToken)
|
||||
const hostFingerprint = (await harness.server.devices.findById(registrationId))!.hostFingerprint
|
||||
for (let index = 0; index < PUSH_LIMITS.hostEventsPerWindow; index++) {
|
||||
expect(
|
||||
await harness.server.deliveryStore.accept(
|
||||
hostFingerprint,
|
||||
registrationId,
|
||||
PushNotificationSchema.parse(
|
||||
notification({ notificationId: `note-${index + 1000}`, notificationSeq: index + 1000 })
|
||||
)
|
||||
)
|
||||
).toBe('queued')
|
||||
}
|
||||
const limited = await harness.post(
|
||||
'/v1/send',
|
||||
{ v: 1, registrationIds: [registrationId], notification: notification() },
|
||||
sessionToken
|
||||
)
|
||||
expect(limited.status).toBe(200)
|
||||
expect(await limited.json()).toEqual({ results: [{ registrationId, status: 'rate_limited' }] })
|
||||
expect(await harness.server.deliveryStore.pendingCount(registrationId)).toBe(300)
|
||||
})
|
||||
})
|
||||
@@ -0,0 +1,298 @@
|
||||
import { PushAuthAdmission } from './push-auth-admission.js'
|
||||
import { createAdaptorServer } from '@hono/node-server'
|
||||
import {
|
||||
PUSH_LIMITS,
|
||||
PushDeviceRegistrationRequestSchema,
|
||||
PushHostChallengeRequestSchema,
|
||||
PushHostSessionRequestSchema,
|
||||
PushSendRequestSchema,
|
||||
type PushSendResult
|
||||
} from '@orca-cloud/push-contract'
|
||||
import { Hono, type MiddlewareHandler } from 'hono'
|
||||
import { bodyLimit } from 'hono/body-limit'
|
||||
import { ApnsClient } from './apns-client.js'
|
||||
import { createApnsHttp2Transport, type ApnsTransport } from './apns-http2-transport.js'
|
||||
import { clientIpRateLimit, ClientIpRateLimiter, readClientIp } from './client-ip-rate-limit.js'
|
||||
import { DurablePushStore } from './durable-push-store.js'
|
||||
import { DurablePushWorker } from './durable-push-worker.js'
|
||||
import type { PushConfig } from './config.js'
|
||||
import { PushDeviceRegistryStore } from './device-registry-store.js'
|
||||
import { createFcmAccessTokenProvider } from './fcm-access-token.js'
|
||||
import { createFcmFetchTransport, FcmClient, type FcmTransport } from './fcm-client.js'
|
||||
import { PushHostChallengeStore } from './host-challenge-store.js'
|
||||
import { PushHostSessionStore } from './host-session-store.js'
|
||||
import type { PushDatabase } from './push-database.js'
|
||||
import { PushDispatcher } from './push-dispatcher.js'
|
||||
import { PushObservability } from './push-observability.js'
|
||||
import { createPushReadiness } from './push-readiness.js'
|
||||
import { PushRequestDrain } from './push-request-drain.js'
|
||||
|
||||
export type PushServerOptions = {
|
||||
now?: () => number
|
||||
apnsTransport?: ApnsTransport
|
||||
fcmTransport?: FcmTransport
|
||||
fcmAccessToken?: () => Promise<string>
|
||||
}
|
||||
|
||||
type PushVariables = { hostFingerprint: string }
|
||||
|
||||
export function readBearer(header: string | undefined): string | null {
|
||||
if (!header) return null
|
||||
const [scheme, ...rest] = header.split(' ')
|
||||
const token = rest.join(' ').trim()
|
||||
return scheme?.toLowerCase() === 'bearer' && token.length > 0 ? token : null
|
||||
}
|
||||
|
||||
// Hono's body limit, not a Content-Length check: a chunked body declares no
|
||||
// length, and req.json() would buffer all of it before any handler ran.
|
||||
const limitBody = bodyLimit({
|
||||
maxSize: PUSH_LIMITS.maxHttpBodyBytes,
|
||||
onError: (context) => context.json({ error: 'request_too_large' }, 413)
|
||||
})
|
||||
|
||||
export function createPushServer(
|
||||
config: PushConfig,
|
||||
database: PushDatabase,
|
||||
options: PushServerOptions = {}
|
||||
) {
|
||||
const now = options.now ?? Date.now
|
||||
const observability = new PushObservability()
|
||||
const challenges = new PushHostChallengeStore(database, config.publicUrl, now)
|
||||
const sessions = new PushHostSessionStore(database, now)
|
||||
const devices = new PushDeviceRegistryStore(database, now)
|
||||
const deliveryStore = new DurablePushStore(database, now)
|
||||
const apnsTransport = options.apnsTransport ?? (config.apns ? createApnsHttp2Transport() : null)
|
||||
const dispatcher = new PushDispatcher({
|
||||
devices,
|
||||
...(config.apns && apnsTransport
|
||||
? {
|
||||
apns: new ApnsClient({
|
||||
topic: config.apnsTopic,
|
||||
credentials: config.apns,
|
||||
transport: apnsTransport,
|
||||
now
|
||||
})
|
||||
}
|
||||
: {}),
|
||||
fcm: new FcmClient({
|
||||
now,
|
||||
projectId: config.fcmProjectId,
|
||||
accessToken: options.fcmAccessToken ?? createFcmAccessTokenProvider(),
|
||||
transport: options.fcmTransport ?? createFcmFetchTransport()
|
||||
}),
|
||||
onOutcome: (status) =>
|
||||
observability.record(
|
||||
status === 'sent' ? 'delivery_sent' : status === 'dead' ? 'delivery_dead' : 'delivery_error'
|
||||
)
|
||||
})
|
||||
const worker = new DurablePushWorker(deliveryStore, dispatcher, {
|
||||
now,
|
||||
onRetry: () => observability.record('delivery_retry')
|
||||
})
|
||||
const ready = createPushReadiness(database, { now })
|
||||
const unauthenticatedIps = new ClientIpRateLimiter({ now })
|
||||
const limitUnauthenticatedIp = clientIpRateLimit(unauthenticatedIps, {
|
||||
trustedProxyHops: config.trustedProxyHops,
|
||||
onLimited: () => observability.record('ip_rate_limited')
|
||||
})
|
||||
const limitAuthenticatedIp = clientIpRateLimit(
|
||||
new ClientIpRateLimiter({
|
||||
now,
|
||||
capacity: PUSH_LIMITS.authenticatedRequestsPerMinutePerIp
|
||||
}),
|
||||
{
|
||||
trustedProxyHops: config.trustedProxyHops,
|
||||
onLimited: () => observability.record('ip_rate_limited')
|
||||
}
|
||||
)
|
||||
const authAdmission = new PushAuthAdmission()
|
||||
const invalidBearerIps = new ClientIpRateLimiter({ now })
|
||||
const authenticatedHosts = new ClientIpRateLimiter({
|
||||
now,
|
||||
capacity: PUSH_LIMITS.authenticatedRequestsPerMinutePerHost
|
||||
})
|
||||
const app = new Hono<{ Variables: PushVariables }>()
|
||||
const requestDrain = new PushRequestDrain()
|
||||
app.use('*', requestDrain.middleware)
|
||||
// Hono's default handler prints the whole error, and a pg error carries the
|
||||
// offending row in `detail`. Only the error's name may reach the logs.
|
||||
app.onError((error, context) => {
|
||||
observability.record('request_error')
|
||||
console.warn(
|
||||
JSON.stringify({
|
||||
event: 'orca_push_request_failed',
|
||||
error: error instanceof Error ? error.name : 'unknown'
|
||||
})
|
||||
)
|
||||
return context.json({ error: 'internal' }, 500)
|
||||
})
|
||||
|
||||
app.get('/health', (context) =>
|
||||
context.json({ ok: true, pushProtocol: 1, deliveryProtocol: 2, mode: config.mode })
|
||||
)
|
||||
app.get('/ready', limitUnauthenticatedIp, async (context) =>
|
||||
(await ready())
|
||||
? context.json({ ok: true })
|
||||
: context.json({ error: 'dependency_unavailable' }, 503)
|
||||
)
|
||||
|
||||
if (config.mode === 'validation') {
|
||||
app.use('*', async (context) => context.json({ error: 'validation_only' }, 503))
|
||||
}
|
||||
|
||||
const bearerSession: MiddlewareHandler<{ Variables: PushVariables }> = async (context, next) => {
|
||||
const ip = readClientIp(context, config.trustedProxyHops)
|
||||
if (!invalidBearerIps.available(ip)) return context.json({ error: 'rate_limited' }, 429)
|
||||
const bearer = readBearer(context.req.header('authorization'))
|
||||
if (!bearer) {
|
||||
invalidBearerIps.allow(ip)
|
||||
return context.json({ error: 'invalid_token' }, 401)
|
||||
}
|
||||
const session = await authAdmission.run(async () => {
|
||||
if (!invalidBearerIps.available(ip)) return null
|
||||
const result = await sessions.resolve(bearer)
|
||||
if (!result.ok) invalidBearerIps.allow(ip)
|
||||
return result
|
||||
})
|
||||
if (!session) {
|
||||
context.header('Retry-After', '1')
|
||||
return context.json({ error: 'busy' }, 503)
|
||||
}
|
||||
if (!session.ok) {
|
||||
return context.json(
|
||||
{ error: session.reason === 'session_expired' ? 'session_expired' : 'invalid_token' },
|
||||
401
|
||||
)
|
||||
}
|
||||
if (!authenticatedHosts.allow(session.hostFingerprint))
|
||||
return context.json({ error: 'rate_limited' }, 429)
|
||||
context.set('hostFingerprint', session.hostFingerprint)
|
||||
await next()
|
||||
return
|
||||
}
|
||||
// `/v1/devices/*` matches `/v1/devices` itself; a second registration for the
|
||||
// bare path would run both middlewares twice on it.
|
||||
app.use('/v1/devices/*', limitAuthenticatedIp, bearerSession)
|
||||
app.use('/v1/send', limitAuthenticatedIp, bearerSession)
|
||||
|
||||
app.post('/v1/host/challenge', limitUnauthenticatedIp, limitBody, async (context) => {
|
||||
const body = PushHostChallengeRequestSchema.safeParse(
|
||||
await context.req.json().catch(() => null)
|
||||
)
|
||||
if (!body.success) return context.json({ error: 'invalid_request' }, 400)
|
||||
const issued = await challenges.issue(body.data.hostPublicKeyB64)
|
||||
if (!issued) {
|
||||
observability.record('challenge_rejected')
|
||||
return context.json({ error: 'invalid_request' }, 400)
|
||||
}
|
||||
observability.record('challenge_issued')
|
||||
const { hostFingerprint: _bound, ...response } = issued
|
||||
return context.json(response)
|
||||
})
|
||||
|
||||
app.post('/v1/host/session', limitUnauthenticatedIp, limitBody, async (context) => {
|
||||
const body = PushHostSessionRequestSchema.safeParse(await context.req.json().catch(() => null))
|
||||
if (!body.success) return context.json({ error: 'invalid_request' }, 400)
|
||||
const verification = await challenges.verify(body.data.challengeId, body.data.proofB64)
|
||||
if (!verification.ok) {
|
||||
observability.record('session_rejected')
|
||||
return context.json(
|
||||
{
|
||||
error: verification.reason === 'unknown_challenge' ? 'invalid_challenge' : 'invalid_proof'
|
||||
},
|
||||
401
|
||||
)
|
||||
}
|
||||
observability.record('session_issued')
|
||||
return context.json(await sessions.create(verification.hostFingerprint))
|
||||
})
|
||||
|
||||
app.post('/v1/devices', limitBody, async (context) => {
|
||||
const body = PushDeviceRegistrationRequestSchema.safeParse(
|
||||
await context.req.json().catch(() => null)
|
||||
)
|
||||
if (!body.success) return context.json({ error: 'invalid_request' }, 400)
|
||||
const registered = await devices.upsert({
|
||||
hostFingerprint: context.get('hostFingerprint'),
|
||||
deviceId: body.data.deviceId,
|
||||
platform: body.data.platform,
|
||||
token: body.data.token,
|
||||
...(body.data.apnsEnvironment === undefined
|
||||
? {}
|
||||
: { apnsEnvironment: body.data.apnsEnvironment })
|
||||
})
|
||||
if (!registered.ok) {
|
||||
observability.record('device_rejected')
|
||||
return context.json({ error: 'too_many_devices' }, 409)
|
||||
}
|
||||
observability.record('device_registered')
|
||||
return context.json({ registrationId: registered.registrationId })
|
||||
})
|
||||
|
||||
app.delete('/v1/devices/:registrationId', async (context) => {
|
||||
const deleted = await devices.deleteOwned(
|
||||
context.get('hostFingerprint'),
|
||||
context.req.param('registrationId')
|
||||
)
|
||||
if (!deleted) return context.json({ error: 'not_found' }, 404)
|
||||
observability.record('device_deleted')
|
||||
return context.body(null, 204)
|
||||
})
|
||||
|
||||
app.get('/v1/devices', async (context) =>
|
||||
context.json({ devices: await devices.list(context.get('hostFingerprint')) })
|
||||
)
|
||||
|
||||
app.post('/v1/send', limitBody, async (context) => {
|
||||
const body = PushSendRequestSchema.safeParse(await context.req.json().catch(() => null))
|
||||
if (!body.success) return context.json({ error: 'invalid_request' }, 400)
|
||||
const hostFingerprint = context.get('hostFingerprint')
|
||||
const owned = await devices.findOwned(hostFingerprint, body.data.registrationIds)
|
||||
const results: PushSendResult[] = []
|
||||
for (const registrationId of body.data.registrationIds) {
|
||||
const device = owned.get(registrationId)
|
||||
if (!device) {
|
||||
observability.record('send_error')
|
||||
results.push({ registrationId, status: 'error' })
|
||||
continue
|
||||
}
|
||||
if (device.dead) {
|
||||
observability.record('send_dead')
|
||||
results.push({ registrationId, status: 'dead' })
|
||||
continue
|
||||
}
|
||||
const reservation = await deliveryStore.accept(
|
||||
hostFingerprint,
|
||||
registrationId,
|
||||
body.data.notification
|
||||
)
|
||||
if (reservation !== 'queued') {
|
||||
observability.record(reservation === 'rate_limited' ? 'send_rate_limited' : 'send_error')
|
||||
results.push({ registrationId, status: reservation })
|
||||
continue
|
||||
}
|
||||
observability.record('send_queued')
|
||||
results.push({ registrationId, status: 'queued' })
|
||||
}
|
||||
return context.json({ results })
|
||||
})
|
||||
|
||||
return {
|
||||
app,
|
||||
requestDrain,
|
||||
server: createAdaptorServer(app),
|
||||
challenges,
|
||||
sessions,
|
||||
devices,
|
||||
deliveryStore,
|
||||
unauthenticatedIps,
|
||||
worker,
|
||||
observability,
|
||||
ready,
|
||||
closeTransports: (): void => {
|
||||
if (apnsTransport && 'close' in apnsTransport) {
|
||||
;(apnsTransport as { close: () => void }).close()
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,65 @@
|
||||
import { randomUUID } from 'node:crypto'
|
||||
import { tmpdir } from 'node:os'
|
||||
import { afterEach, describe, expect, it } from 'vitest'
|
||||
import { openInMemoryPushDatabase, openPushDatabase, type PushDatabase } from './push-database.js'
|
||||
import { PushHostSessionStore } from './host-session-store.js'
|
||||
const databases: PushDatabase[] = []
|
||||
afterEach(async () => {
|
||||
await Promise.all(databases.splice(0).map((db) => db.close()))
|
||||
})
|
||||
|
||||
async function concurrentSessions(db: PushDatabase) {
|
||||
databases.push(db)
|
||||
const host = randomUUID()
|
||||
const store = new PushHostSessionStore(db)
|
||||
try {
|
||||
const sessions = await Promise.all(Array.from({ length: 20 }, () => store.create(host)))
|
||||
const decisions = await Promise.all(
|
||||
sessions.map((session) => store.resolve(session.sessionToken))
|
||||
)
|
||||
expect(decisions.filter((decision) => decision.ok)).toHaveLength(1)
|
||||
const [row] = await db.query(
|
||||
'SELECT COUNT(*) AS count FROM push_sessions WHERE host_fingerprint = ?',
|
||||
[host]
|
||||
)
|
||||
expect(Number(row?.count)).toBe(1)
|
||||
} finally {
|
||||
await db.query('DELETE FROM push_sessions WHERE host_fingerprint = ?', [host])
|
||||
}
|
||||
}
|
||||
it('serializes sessions on SQLite', async () => {
|
||||
await concurrentSessions(await openInMemoryPushDatabase())
|
||||
})
|
||||
|
||||
it('enforces one session per host directly in the schema', async () => {
|
||||
const db = await openInMemoryPushDatabase()
|
||||
databases.push(db)
|
||||
await db.query('INSERT INTO push_sessions VALUES (?, ?, ?, ?)', ['first', 'host', 100, 1])
|
||||
await expect(
|
||||
db.query('INSERT INTO push_sessions VALUES (?, ?, ?, ?)', ['second', 'host', 100, 2])
|
||||
).rejects.toThrow()
|
||||
expect(await db.query('SELECT token_hash FROM push_sessions')).toEqual([{ token_hash: 'first' }])
|
||||
})
|
||||
|
||||
describe.skipIf(!process.env.ORCA_PUSH_TEST_DATABASE_URL)('PostgreSQL push sessions', () => {
|
||||
it('leaves exactly one live token after concurrent creates', async () => {
|
||||
await concurrentSessions(
|
||||
await openPushDatabase({
|
||||
databaseUrl: process.env.ORCA_PUSH_TEST_DATABASE_URL!,
|
||||
dataDir: tmpdir()
|
||||
})
|
||||
)
|
||||
})
|
||||
it('allows concurrent schema startup', async () => {
|
||||
const opened = await Promise.all(
|
||||
Array.from({ length: 4 }, () =>
|
||||
openPushDatabase({
|
||||
databaseUrl: process.env.ORCA_PUSH_TEST_DATABASE_URL!,
|
||||
dataDir: tmpdir()
|
||||
})
|
||||
)
|
||||
)
|
||||
databases.push(...opened)
|
||||
for (const db of opened) expect(await db.query('SELECT 1 AS ok')).toEqual([{ ok: 1 }])
|
||||
})
|
||||
})
|
||||
@@ -0,0 +1,90 @@
|
||||
import { expect, it, vi } from 'vitest'
|
||||
import { loadPushConfig } from './config.js'
|
||||
import { openPushDatabase } from './push-database.js'
|
||||
import { createPushServer } from './push-server.js'
|
||||
import { startPushBackground } from './push-background.js'
|
||||
|
||||
it('fails closed on an invalid validation mode', () => {
|
||||
for (const mode of ['typo', '', ' ']) {
|
||||
expect(() =>
|
||||
loadPushConfig({
|
||||
ORCA_PUSH_FCM_PROJECT_ID: 'onorca-cloud',
|
||||
ORCA_PUSH_PUBLIC_URL: 'https://push.onorca.dev',
|
||||
ORCA_PUSH_MODE: mode
|
||||
})
|
||||
).toThrow()
|
||||
}
|
||||
})
|
||||
|
||||
const databaseUrl = process.env.ORCA_PUSH_TEST_DATABASE_URL
|
||||
it.skipIf(!databaseUrl)(
|
||||
'validation cannot write PostgreSQL and starts no consumers or pruners',
|
||||
async () => {
|
||||
if (!process.env.CI && new URL(databaseUrl!).port !== '55440')
|
||||
throw new Error('isolated_postgres_port_required')
|
||||
const active = await openPushDatabase({ databaseUrl, dataDir: '' })
|
||||
const schema = `validation_${Date.now()}`
|
||||
await active.query(`CREATE SCHEMA ${schema}`)
|
||||
const isolatedUrl = new URL(databaseUrl!)
|
||||
isolatedUrl.searchParams.set(
|
||||
'options',
|
||||
`-c search_path=${schema} -c default_transaction_read_only=off`
|
||||
)
|
||||
isolatedUrl.searchParams.set('host', isolatedUrl.hostname)
|
||||
isolatedUrl.searchParams.set('port', isolatedUrl.port)
|
||||
const hostlessUrl = `postgresql://${isolatedUrl.username}:${isolatedUrl.password}@${isolatedUrl.pathname}?${isolatedUrl.searchParams}`
|
||||
const database = await openPushDatabase({
|
||||
databaseUrl: hostlessUrl,
|
||||
dataDir: '',
|
||||
readOnly: true
|
||||
})
|
||||
const config = loadPushConfig({
|
||||
ORCA_PUSH_FCM_PROJECT_ID: 'onorca-cloud',
|
||||
ORCA_PUSH_PUBLIC_URL: 'https://push.onorca.dev',
|
||||
ORCA_PUSH_MODE: 'validation'
|
||||
})
|
||||
const runtime = createPushServer(config, database)
|
||||
let stop: (() => Promise<void>) | undefined
|
||||
try {
|
||||
const [setting] = await database.query(
|
||||
"SELECT current_setting('default_transaction_read_only') AS default_transaction_read_only"
|
||||
)
|
||||
expect(setting!.default_transaction_read_only).toBe('on')
|
||||
await expect(
|
||||
database.query(`CREATE TABLE ${schema}.forbidden (id integer)`)
|
||||
).rejects.toMatchObject({ code: '25006' })
|
||||
// An empty schema stays empty: validation must not run startup DDL.
|
||||
expect(
|
||||
await active.query('SELECT tablename FROM pg_tables WHERE schemaname = ?', [schema])
|
||||
).toEqual([])
|
||||
const calls = vi.spyOn(database, 'query')
|
||||
const claim = vi.spyOn(runtime.deliveryStore, 'claim')
|
||||
const send = vi.spyOn(runtime.worker, 'start')
|
||||
vi.useFakeTimers()
|
||||
stop = startPushBackground(config, runtime)
|
||||
await vi.advanceTimersByTimeAsync(31 * 60_000)
|
||||
expect(calls).not.toHaveBeenCalled()
|
||||
expect(claim).not.toHaveBeenCalled()
|
||||
expect(send).not.toHaveBeenCalled()
|
||||
vi.useRealTimers()
|
||||
expect(await (await runtime.app.request('/health')).json()).toMatchObject({
|
||||
mode: 'validation'
|
||||
})
|
||||
expect((await runtime.app.request('/ready')).status).toBe(200)
|
||||
expect((await runtime.app.request('/v1/host/challenge', { method: 'POST' })).status).toBe(503)
|
||||
expect((await runtime.app.request('/v1/send', { method: 'POST' })).status).toBe(503)
|
||||
expect(calls.mock.calls.map(([sql]) => sql)).toEqual(['SELECT 1 AS ready'])
|
||||
await expect(
|
||||
database.query('DELETE FROM public.push_challenges WHERE false')
|
||||
).rejects.toMatchObject({ code: '25006' })
|
||||
} finally {
|
||||
vi.useRealTimers()
|
||||
await stop?.()
|
||||
runtime.closeTransports()
|
||||
await database.close()
|
||||
await active.query(`DROP SCHEMA ${schema} CASCADE`)
|
||||
await active.close()
|
||||
vi.restoreAllMocks()
|
||||
}
|
||||
}
|
||||
)
|
||||
@@ -0,0 +1,10 @@
|
||||
{
|
||||
"extends": "./tsconfig.json",
|
||||
"compilerOptions": {
|
||||
"declaration": true,
|
||||
"noEmit": false,
|
||||
"outDir": "dist",
|
||||
"rootDir": "src"
|
||||
},
|
||||
"exclude": ["src/**/*.test.ts", "src/**/*.test-fixture.ts"]
|
||||
}
|
||||
@@ -0,0 +1,5 @@
|
||||
{
|
||||
"extends": "../../tsconfig.base.json",
|
||||
"compilerOptions": { "noEmit": true },
|
||||
"include": ["src/**/*.ts"]
|
||||
}
|
||||
@@ -0,0 +1,5 @@
|
||||
import { defineConfig } from 'vitest/config'
|
||||
|
||||
export default defineConfig({
|
||||
test: { name: 'push', include: ['src/**/*.test.ts'], testTimeout: 15_000, hookTimeout: 15_000 }
|
||||
})
|
||||
@@ -3,11 +3,13 @@ WORKDIR /app
|
||||
RUN corepack enable
|
||||
COPY package.json pnpm-lock.yaml pnpm-workspace.yaml tsconfig.base.json ./
|
||||
COPY packages/relay-contract/package.json packages/relay-contract/package.json
|
||||
COPY packages/postgres-schema/package.json packages/postgres-schema/package.json
|
||||
COPY apps/relay/package.json apps/relay/package.json
|
||||
RUN pnpm install --frozen-lockfile
|
||||
COPY packages/relay-contract packages/relay-contract
|
||||
COPY apps/relay apps/relay
|
||||
RUN pnpm --filter @orca-cloud/relay-contract build && pnpm --filter @orca-cloud/relay build
|
||||
COPY packages/postgres-schema packages/postgres-schema
|
||||
RUN pnpm --filter @orca-cloud/postgres-schema build && pnpm --filter @orca-cloud/relay-contract build && pnpm --filter @orca-cloud/relay build
|
||||
|
||||
FROM node:24-alpine AS runtime
|
||||
ENV NODE_ENV=production
|
||||
@@ -16,8 +18,10 @@ WORKDIR /app
|
||||
RUN corepack enable
|
||||
COPY package.json pnpm-lock.yaml pnpm-workspace.yaml ./
|
||||
COPY packages/relay-contract/package.json packages/relay-contract/package.json
|
||||
COPY packages/postgres-schema/package.json packages/postgres-schema/package.json
|
||||
COPY apps/relay/package.json apps/relay/package.json
|
||||
COPY --from=build /app/packages/relay-contract/dist packages/relay-contract/dist
|
||||
COPY --from=build /app/packages/postgres-schema/dist packages/postgres-schema/dist
|
||||
COPY --from=build /app/apps/relay/dist apps/relay/dist
|
||||
RUN pnpm install --prod --frozen-lockfile --filter @orca-cloud/relay...
|
||||
USER node
|
||||
|
||||
@@ -9,13 +9,14 @@
|
||||
"clean": "node -e \"require('fs').rmSync('dist', { recursive: true, force: true })\"",
|
||||
"dev": "tsx watch src/index.ts",
|
||||
"lint": "tsc -p tsconfig.json --noEmit",
|
||||
"pretest": "pnpm --filter @orca-cloud/relay-contract build",
|
||||
"pretest": "pnpm --filter @orca-cloud/postgres-schema build && pnpm --filter @orca-cloud/relay-contract build",
|
||||
"start": "node dist/index.js",
|
||||
"test": "vitest run",
|
||||
"typecheck": "tsc -p tsconfig.json --noEmit"
|
||||
},
|
||||
"dependencies": {
|
||||
"@hono/node-server": "^1.19.17",
|
||||
"@orca-cloud/postgres-schema": "workspace:*",
|
||||
"@orca-cloud/relay-contract": "workspace:*",
|
||||
"hono": "^4.13.7",
|
||||
"jose": "^6.1.3",
|
||||
|
||||
@@ -1,120 +1 @@
|
||||
const RETRYABLE_SCHEMA_CODES = new Set(['55P03', '57014'])
|
||||
const DEFAULT_RETRY_DEADLINE_MS = 30_000
|
||||
const RETRY_BASE_DELAY_MS = 250
|
||||
const RETRY_MAX_DELAY_MS = 2_000
|
||||
|
||||
type SchemaStartupOptions = {
|
||||
now?: () => number
|
||||
random?: () => number
|
||||
retryDeadlineMs?: number
|
||||
wait?: (delayMs: number) => Promise<void>
|
||||
}
|
||||
|
||||
function retryDelayMs(attempt: number, random: () => number): number {
|
||||
const ceiling = Math.min(
|
||||
RETRY_BASE_DELAY_MS * 2 ** (attempt - 1),
|
||||
RETRY_MAX_DELAY_MS
|
||||
)
|
||||
return Math.ceil(ceiling * (0.5 + random() * 0.5))
|
||||
}
|
||||
|
||||
function wait(delayMs: number): Promise<void> {
|
||||
return new Promise((resolve) => setTimeout(resolve, delayMs))
|
||||
}
|
||||
|
||||
const CREATE_TABLE_IF_NOT_EXISTS = /^\s*CREATE\s+TABLE\s+IF\s+NOT\s+EXISTS\b/i
|
||||
const CREATE_INDEX_IF_NOT_EXISTS = /^\s*CREATE\s+(?:UNIQUE\s+)?INDEX\s+IF\s+NOT\s+EXISTS\b/i
|
||||
|
||||
// `IF NOT EXISTS` only checks the name before the catalog inserts, so the loser of a concurrent
|
||||
// CREATE can fail on the catalog unique index (23505) or, when the winner has already committed by
|
||||
// the time the loser reaches TypeCreate/heap_create_with_catalog, on the name check those routines
|
||||
// repeat (42710 duplicate type, 42P07 duplicate relation). Each is a no-op on the next attempt.
|
||||
function concurrentCreateCollision(
|
||||
value: { code?: unknown; constraint?: unknown },
|
||||
statement: string
|
||||
): boolean {
|
||||
if (CREATE_TABLE_IF_NOT_EXISTS.test(statement)) {
|
||||
return (
|
||||
(value.code === '23505' && value.constraint === 'pg_type_typname_nsp_index') ||
|
||||
value.code === '42710' ||
|
||||
value.code === '42P07'
|
||||
)
|
||||
}
|
||||
if (CREATE_INDEX_IF_NOT_EXISTS.test(statement)) {
|
||||
return (
|
||||
(value.code === '23505' && value.constraint === 'pg_class_relname_nsp_index') ||
|
||||
value.code === '42P07'
|
||||
)
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
const ALTER_TABLE_ADD_CONSTRAINT =
|
||||
/^\s*ALTER\s+TABLE\s+\S+\s+ADD\s+CONSTRAINT\b/i
|
||||
|
||||
// Postgres has no `ADD CONSTRAINT IF NOT EXISTS`, so a re-run and a concurrent
|
||||
// startup both land on 42710 once the constraint exists. Unlike a CREATE race
|
||||
// this is terminal, not transient: retrying only repeats it, so the statement
|
||||
// counts as applied.
|
||||
function constraintAlreadyApplied(error: unknown, statement: string): boolean {
|
||||
return (
|
||||
ALTER_TABLE_ADD_CONSTRAINT.test(statement) &&
|
||||
(error as { code?: unknown }).code === '42710'
|
||||
)
|
||||
}
|
||||
|
||||
function retryableSchemaError(error: unknown, statement: string): boolean {
|
||||
const value = error as { code?: unknown; constraint?: unknown }
|
||||
return (
|
||||
RETRYABLE_SCHEMA_CODES.has(String(value.code)) || concurrentCreateCollision(value, statement)
|
||||
)
|
||||
}
|
||||
|
||||
export async function applyPostgresSchema(
|
||||
statements: string[],
|
||||
query: (statement: string) => Promise<unknown>,
|
||||
options: SchemaStartupOptions = {}
|
||||
): Promise<void> {
|
||||
const now = options.now ?? Date.now
|
||||
const random = options.random ?? Math.random
|
||||
const pause = options.wait ?? wait
|
||||
const deadlineAt = now() + (options.retryDeadlineMs ?? DEFAULT_RETRY_DEADLINE_MS)
|
||||
|
||||
for (const statement of statements) {
|
||||
let attempt = 1
|
||||
while (true) {
|
||||
try {
|
||||
await query(statement)
|
||||
break
|
||||
} catch (error) {
|
||||
if (constraintAlreadyApplied(error, statement)) break
|
||||
const code = String((error as { code?: unknown }).code)
|
||||
const remainingMs = deadlineAt - now()
|
||||
const retryable = retryableSchemaError(error, statement)
|
||||
if (!retryable || remainingMs <= 0) {
|
||||
if (retryable) {
|
||||
console.warn(
|
||||
JSON.stringify({
|
||||
event: 'orca_relay_postgres_schema_retry_exhausted',
|
||||
code,
|
||||
attempts: attempt
|
||||
})
|
||||
)
|
||||
}
|
||||
throw error
|
||||
}
|
||||
const delayMs = Math.min(remainingMs, retryDelayMs(attempt, random))
|
||||
console.warn(
|
||||
JSON.stringify({
|
||||
event: 'orca_relay_postgres_schema_retry',
|
||||
code,
|
||||
attempt,
|
||||
delayMs
|
||||
})
|
||||
)
|
||||
await pause(delayMs)
|
||||
attempt += 1
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
export { applyPostgresSchema } from '@orca-cloud/postgres-schema'
|
||||
|
||||
@@ -86,17 +86,21 @@
|
||||
"relay": [
|
||||
"google_artifact_registry_repository_iam_member.github_production_relay_staging_mirror_writer",
|
||||
"google_artifact_registry_repository_iam_member.github_production_relay_writer",
|
||||
"google_artifact_registry_repository_iam_member.github_push_artifact_writer",
|
||||
"google_artifact_registry_repository_iam_member.github_relay_asia_topology_artifact_reader",
|
||||
"google_artifact_registry_repository_iam_member.github_staging_relay_deploy_artifact_reader",
|
||||
"google_certificate_manager_certificate.relay_gce",
|
||||
"google_certificate_manager_certificate_map.relay_gce",
|
||||
"google_certificate_manager_certificate_map_entry.relay_gce",
|
||||
"google_certificate_manager_dns_authorization.relay_gce",
|
||||
"google_cloud_run_domain_mapping.push",
|
||||
"google_cloud_run_domain_mapping.relay",
|
||||
"google_cloud_run_domain_mapping.relay_cell",
|
||||
"google_cloud_run_v2_service.push",
|
||||
"google_cloud_run_v2_service.relay",
|
||||
"google_cloud_run_v2_service.relay_cell",
|
||||
"google_cloud_run_v2_service.relay_fence_broker",
|
||||
"google_cloud_run_v2_service_iam_member.github_production_push_developer",
|
||||
"google_cloud_run_v2_service_iam_member.github_production_relay_director_developer",
|
||||
"google_cloud_run_v2_service_iam_member.github_production_relay_fence_broker_developer",
|
||||
"google_cloud_run_v2_service_iam_member.github_staging_relay_capacity_developer",
|
||||
@@ -125,6 +129,7 @@
|
||||
"google_iam_workload_identity_pool_provider.github_fence",
|
||||
"google_iam_workload_identity_pool_provider.github_monitor",
|
||||
"google_iam_workload_identity_pool_provider.github_production_relay_capacity",
|
||||
"google_iam_workload_identity_pool_provider.github_push",
|
||||
"google_iam_workload_identity_pool_provider.github_relay_asia_proof",
|
||||
"google_iam_workload_identity_pool_provider.github_relay_asia_topology",
|
||||
"google_iam_workload_identity_pool_provider.github_staging_relay_capacity",
|
||||
@@ -172,6 +177,9 @@
|
||||
"google_project_iam_member.github_staging_relay_capacity_viewer",
|
||||
"google_project_iam_member.github_staging_relay_deploy_compute_viewer",
|
||||
"google_project_iam_member.github_staging_relay_power",
|
||||
"google_project_iam_member.push_runtime_cloudsql_client",
|
||||
"google_project_iam_member.push_runtime_fcm_admin",
|
||||
"google_project_iam_member.push_runtime_service_usage_consumer",
|
||||
"google_project_iam_member.relay_director_runtime_cloudsql_client",
|
||||
"google_project_iam_member.relay_fence_broker_artifact_reader",
|
||||
"google_project_iam_member.relay_fence_broker_compute_viewer",
|
||||
@@ -180,9 +188,13 @@
|
||||
"google_project_iam_member.relay_runtime_artifact_reader",
|
||||
"google_project_iam_member.relay_runtime_cloudsql_client",
|
||||
"google_project_iam_member.relay_runtime_log_writer",
|
||||
"google_secret_manager_secret.push_dedicated_database_url",
|
||||
"google_secret_manager_secret.push_provider",
|
||||
"google_secret_manager_secret.relay_assignment_signing_key",
|
||||
"google_secret_manager_secret.relay_database_url",
|
||||
"google_secret_manager_secret.relay_regional_placement_enabled",
|
||||
"google_secret_manager_secret_iam_member.push_dedicated_database_url_accessor",
|
||||
"google_secret_manager_secret_iam_member.push_provider_runtime_accessor",
|
||||
"google_secret_manager_secret_iam_member.relay_assignment_signing_key_accessor",
|
||||
"google_secret_manager_secret_iam_member.relay_assignment_signing_key_director_accessor",
|
||||
"google_secret_manager_secret_iam_member.relay_database_url_accessor",
|
||||
@@ -192,24 +204,30 @@
|
||||
"google_secret_manager_secret_iam_member.relay_regional_placement_deploy_viewer",
|
||||
"google_secret_manager_secret_iam_member.relay_regional_placement_director_accessor",
|
||||
"google_secret_manager_secret_iam_member.relay_regional_placement_runtime_accessor",
|
||||
"google_secret_manager_secret_version.push_dedicated_database_url",
|
||||
"google_secret_manager_secret_version.relay_assignment_signing_key",
|
||||
"google_secret_manager_secret_version.relay_database_url",
|
||||
"google_secret_manager_secret_version.relay_regional_placement_enabled",
|
||||
"google_service_account.github_fence",
|
||||
"google_service_account.github_monitor",
|
||||
"google_service_account.github_production_relay_capacity",
|
||||
"google_service_account.github_push_deploy",
|
||||
"google_service_account.github_relay_asia_proof",
|
||||
"google_service_account.github_relay_asia_topology",
|
||||
"google_service_account.github_staging_relay_capacity",
|
||||
"google_service_account.github_staging_relay_deploy",
|
||||
"google_service_account.push_runtime",
|
||||
"google_service_account.relay_director_runtime",
|
||||
"google_service_account.relay_fence_broker",
|
||||
"google_service_account.relay_runtime",
|
||||
"google_service_account_iam_member.github_accepted_repository_workload_identity_user",
|
||||
"google_service_account_iam_member.github_fence_workload_identity_user",
|
||||
"google_service_account_iam_member.github_monitor_workload_identity_user",
|
||||
"google_service_account_iam_member.github_production_push_runtime_token_creator",
|
||||
"google_service_account_iam_member.github_production_push_runtime_user",
|
||||
"google_service_account_iam_member.github_production_relay_capacity_runtime_user",
|
||||
"google_service_account_iam_member.github_production_relay_capacity_workload_identity_user",
|
||||
"google_service_account_iam_member.github_push_workload_identity_user",
|
||||
"google_service_account_iam_member.github_relay_asia_proof_workload_identity_user",
|
||||
"google_service_account_iam_member.github_relay_asia_topology_runtime_user",
|
||||
"google_service_account_iam_member.github_relay_asia_topology_workload_identity_user",
|
||||
@@ -221,9 +239,13 @@
|
||||
"google_service_account_iam_member.github_staging_relay_deploy_auth_runtime_user",
|
||||
"google_service_account_iam_member.github_staging_relay_deploy_workload_identity_user",
|
||||
"google_service_account_iam_member.relay_fence_broker_requester_token_creator",
|
||||
"google_sql_database.push_dedicated",
|
||||
"google_sql_database.relay",
|
||||
"google_sql_database_instance.push_dedicated",
|
||||
"google_sql_user.push_dedicated",
|
||||
"google_sql_user.relay",
|
||||
"google_storage_bucket_iam_member.github_production_relay_capacity_state",
|
||||
"google_storage_bucket_iam_member.github_push_rollout_lease",
|
||||
"google_storage_bucket_iam_member.github_relay_asia_topology_state",
|
||||
"google_storage_bucket_iam_member.github_relay_asia_topology_state_list",
|
||||
"google_storage_bucket_iam_member.github_staging_relay_capacity_state",
|
||||
@@ -231,6 +253,7 @@
|
||||
"google_storage_bucket_iam_member.github_staging_relay_deploy_state_list",
|
||||
"google_storage_bucket_iam_member.relay_fence_broker_bucket_reader",
|
||||
"google_storage_bucket_iam_member.relay_fence_broker_state_objects",
|
||||
"random_password.push_dedicated_database",
|
||||
"random_password.relay_assignment_signing_key",
|
||||
"random_password.relay_database"
|
||||
],
|
||||
|
||||
@@ -0,0 +1,140 @@
|
||||
import { readFileSync, writeFileSync } from 'node:fs'
|
||||
|
||||
// Executable fake gcloud: revision deletion obeys the platform's latest/traffic constraints.
|
||||
const path = process.env.MODEL_STATE
|
||||
const state = JSON.parse(readFileSync(path, 'utf8'))
|
||||
const args = process.argv.slice(2)
|
||||
const option = (name) => args[args.indexOf(name) + 1]
|
||||
const has = (name) => args.includes(name)
|
||||
const fail = (message) => {
|
||||
throw new Error(message)
|
||||
}
|
||||
const persist = () => writeFileSync(path, JSON.stringify(state))
|
||||
const output = (value) => console.log(typeof value === 'string' ? value : JSON.stringify(value))
|
||||
const revision = (name) => state.revisions[name] ?? fail(`missing revision ${name}`)
|
||||
const traffic = () => [
|
||||
{ revisionName: state.serving, percent: 100 },
|
||||
...Object.entries(state.tags).map(([tag, name]) => ({
|
||||
tag,
|
||||
revisionName: name,
|
||||
url: `https://${tag}.test`
|
||||
}))
|
||||
]
|
||||
state.trace.push(args.join(' '))
|
||||
try {
|
||||
if (args[0] === 'curl') {
|
||||
if (state.failure === 'public' && args.some((arg) => arg.includes('https://public.test'))) {
|
||||
fail('public check failed')
|
||||
}
|
||||
const url = args.find((arg) => arg.startsWith('https://'))
|
||||
const tag = new URL(url).hostname.split('.')[0]
|
||||
const name = state.tags[tag] ?? state.serving
|
||||
if (has('-w')) {
|
||||
output('200')
|
||||
} else {
|
||||
output({
|
||||
ok: true,
|
||||
deliveryProtocol: 2,
|
||||
mode: revision(name).spec.containers[0].env.some((entry) => entry.value === 'validation')
|
||||
? 'validation'
|
||||
: 'active'
|
||||
})
|
||||
}
|
||||
} else if (args.slice(0, 2).join(' ') === 'run deploy') {
|
||||
const name = `${option('deploy')}-${option('--revision-suffix')}`
|
||||
if (state.failure === 'deploy-before') {
|
||||
fail('deploy failed before create')
|
||||
}
|
||||
const item = structuredClone(revision(state.latest))
|
||||
item.metadata.name = name
|
||||
item.spec.containers[0].image = option('--image')
|
||||
item.status.imageDigest = option('--image')
|
||||
item.spec.containers[0].env = item.spec.containers[0].env.filter(
|
||||
(entry) => entry.name !== 'ORCA_PUSH_MODE'
|
||||
)
|
||||
if (has('--update-env-vars')) {
|
||||
item.spec.containers[0].env.push({ name: 'ORCA_PUSH_MODE', value: 'validation' })
|
||||
}
|
||||
state.revisions[name] = item
|
||||
state.latest = name
|
||||
if (has('--tag')) {
|
||||
state.tags[option('--tag')] = name
|
||||
}
|
||||
state.peak = Math.max(state.peak, Object.keys(state.revisions).length)
|
||||
if (state.peak > 3) {
|
||||
fail('three-revision budget exceeded')
|
||||
}
|
||||
if (state.failure === 'deploy-after') {
|
||||
fail('deploy failed after create')
|
||||
}
|
||||
} else if (args.slice(0, 3).join(' ') === 'run services describe') {
|
||||
if (state.failure === 'describe') {
|
||||
fail('describe failed')
|
||||
}
|
||||
if (args.some((arg) => arg.includes('value(status.latestCreatedRevisionName)'))) {
|
||||
output(state.latest)
|
||||
} else {
|
||||
const template = structuredClone(revision(state.latest))
|
||||
delete template.spec.containers[0].name
|
||||
output({
|
||||
spec: { template },
|
||||
status: { latestCreatedRevisionName: state.latest, traffic: traffic() }
|
||||
})
|
||||
}
|
||||
} else if (args.slice(0, 3).join(' ') === 'run services update-traffic') {
|
||||
if (has('--to-revisions')) {
|
||||
const name = option('--to-revisions').split('=')[0]
|
||||
revision(name)
|
||||
state.serving = name
|
||||
}
|
||||
if (has('--remove-tags')) {
|
||||
for (const tag of option('--remove-tags').split(',')) {
|
||||
delete state.tags[tag]
|
||||
}
|
||||
}
|
||||
if (has('--clear-tags')) {
|
||||
state.tags = {}
|
||||
}
|
||||
if (state.failure === 'traffic-after') {
|
||||
fail('traffic changed but response failed')
|
||||
}
|
||||
} else if (args.slice(0, 3).join(' ') === 'run revisions list') {
|
||||
const names = Object.keys(state.revisions)
|
||||
output(
|
||||
(has('--filter')
|
||||
? names.filter((name) => name === option('--filter').split('=')[1])
|
||||
: names
|
||||
).join('\n')
|
||||
)
|
||||
} else if (args.slice(0, 3).join(' ') === 'run revisions describe') {
|
||||
const item = revision(args[3])
|
||||
const format = args.find((arg) => arg.startsWith('--format=')) ?? option('--format')
|
||||
if (format.includes('minScale')) {
|
||||
output(item.metadata.annotations['autoscaling.knative.dev/minScale'])
|
||||
} else if (format.includes('maxScale')) {
|
||||
output(item.metadata.annotations['autoscaling.knative.dev/maxScale'])
|
||||
} else {
|
||||
output(item)
|
||||
}
|
||||
} else if (args.slice(0, 3).join(' ') === 'run revisions delete') {
|
||||
const name = args[3]
|
||||
if (name === state.latest) {
|
||||
fail('FAILED_PRECONDITION: latest created Revision cannot be directly deleted')
|
||||
}
|
||||
if (name === state.serving || Object.values(state.tags).includes(name)) {
|
||||
fail('revision has traffic or tags')
|
||||
}
|
||||
if (state.failure === 'delete') {
|
||||
fail('delete failed')
|
||||
}
|
||||
revision(name)
|
||||
delete state.revisions[name]
|
||||
} else {
|
||||
fail(`unmodeled gcloud call ${args.join(' ')}`)
|
||||
}
|
||||
} catch (error) {
|
||||
console.error(error.message)
|
||||
process.exitCode = 1
|
||||
} finally {
|
||||
persist()
|
||||
}
|
||||
@@ -0,0 +1,373 @@
|
||||
import assert from 'node:assert/strict'
|
||||
import { mkdirSync, mkdtempSync, readFileSync, rmSync, writeFileSync } from 'node:fs'
|
||||
import { tmpdir } from 'node:os'
|
||||
import { join } from 'node:path'
|
||||
import { fileURLToPath } from 'node:url'
|
||||
import { spawnSync } from 'node:child_process'
|
||||
import test from 'node:test'
|
||||
import { readRelayWorkflow } from './relay-repository.mjs'
|
||||
|
||||
const workflow = readRelayWorkflow('push-deploy.yml')
|
||||
function step(name) {
|
||||
const start = workflow.indexOf(` - name: ${name}\n`)
|
||||
assert.notEqual(start, -1)
|
||||
const end = workflow.indexOf('\n - ', start + 1)
|
||||
const block = workflow.slice(start, end === -1 ? undefined : end)
|
||||
return block
|
||||
.slice(block.indexOf(' run: |\n') + ' run: |\n'.length)
|
||||
.split('\n')
|
||||
.filter((line) => line.startsWith(' '))
|
||||
.map((line) => line.slice(10))
|
||||
.join('\n')
|
||||
}
|
||||
const names = {
|
||||
preflight: 'Record the serving revision and require its Terraform-owned scaling',
|
||||
candidate: 'Deploy the candidate revision with no traffic',
|
||||
activate: 'Retire inert validation and activate the verified image',
|
||||
shift: 'Shift all traffic to the verified candidate',
|
||||
public: 'Verify the public origin after the shift',
|
||||
rollback: 'Roll traffic back to the previous revision',
|
||||
restore: 'Restore the known-good service template',
|
||||
promoteRecovery: 'Promote and verify the known-good recovery revision',
|
||||
cleanup: 'Delete the rejected candidate revision',
|
||||
retire: 'Retire previous consumers after public checks'
|
||||
}
|
||||
const image = `registry/push@sha256:${'a'.repeat(64)}`
|
||||
const spec = {
|
||||
serviceAccountName: 'runtime@test',
|
||||
containerConcurrency: 40,
|
||||
containers: [
|
||||
{
|
||||
image,
|
||||
name: 'push-test-1',
|
||||
env: [
|
||||
{
|
||||
name: 'ORCA_PUSH_DATABASE_URL',
|
||||
valueFrom: { secretKeyRef: { name: 'database', key: '7' } }
|
||||
},
|
||||
{ name: 'ORCA_PUSH_DATABASE_POOL_MAX', value: '2' }
|
||||
]
|
||||
}
|
||||
]
|
||||
}
|
||||
const prior = {
|
||||
metadata: {
|
||||
name: 'push-test-old',
|
||||
annotations: {
|
||||
'autoscaling.knative.dev/minScale': '1',
|
||||
'autoscaling.knative.dev/maxScale': '2'
|
||||
}
|
||||
},
|
||||
spec,
|
||||
status: { imageDigest: image }
|
||||
}
|
||||
const model = fileURLToPath(new URL('./push-cloud-run-model.mjs', import.meta.url))
|
||||
const options = { skip: process.platform === 'win32' }
|
||||
function exercise(callback) {
|
||||
const dir = mkdtempSync(join(tmpdir(), 'push-workflow-'))
|
||||
const statePath = join(dir, 'state.json')
|
||||
writeFileSync(
|
||||
statePath,
|
||||
JSON.stringify({
|
||||
revisions: { 'push-test-old': prior },
|
||||
latest: 'push-test-old',
|
||||
serving: 'push-test-old',
|
||||
tags: {},
|
||||
peak: 1,
|
||||
trace: []
|
||||
})
|
||||
)
|
||||
writeFileSync(join(dir, 'env'), '')
|
||||
const env = {
|
||||
...process.env,
|
||||
SERVICE_NAME: 'push-test',
|
||||
GCP_PROJECT_ID: 'test',
|
||||
GCP_REGION: 'test',
|
||||
GITHUB_RUN_ID: '123',
|
||||
GITHUB_RUN_ATTEMPT: '1',
|
||||
IMAGE: `registry/push@sha256:${'b'.repeat(64)}`,
|
||||
PUSH_MIN_INSTANCES: '1',
|
||||
PUSH_MAX_INSTANCES: '2',
|
||||
PUSH_RUNTIME_SERVICE_ACCOUNT: 'runtime@test',
|
||||
PUSH_ORIGIN: 'https://public.test',
|
||||
MODEL_STATE: statePath,
|
||||
MODEL_SCRIPT: model,
|
||||
RUNNER_TEMP: dir,
|
||||
GITHUB_ENV: join(dir, 'env'),
|
||||
GITHUB_STEP_SUMMARY: join(dir, 'summary')
|
||||
}
|
||||
const state = () => JSON.parse(readFileSync(statePath, 'utf8'))
|
||||
const change = (edit) => {
|
||||
const value = state()
|
||||
edit(value)
|
||||
writeFileSync(statePath, JSON.stringify(value))
|
||||
}
|
||||
const run = (key, ok = true, extra = '') => {
|
||||
const result = spawnSync(
|
||||
'bash',
|
||||
[
|
||||
'-c',
|
||||
`
|
||||
set -a
|
||||
source "$GITHUB_ENV"
|
||||
gcloud() { node "$MODEL_SCRIPT" "$@"; }
|
||||
curl() { node "$MODEL_SCRIPT" curl "$@"; }
|
||||
sleep() { :; }
|
||||
${extra}
|
||||
${names[key] ? step(names[key]) : key}
|
||||
`
|
||||
],
|
||||
{ cwd: dir, env, encoding: 'utf8', timeout: 30000 }
|
||||
)
|
||||
assert.equal(result.status === 0, ok, `${key}: ${result.stderr}\n${result.stdout}`)
|
||||
return result
|
||||
}
|
||||
try {
|
||||
callback({ run, state, change, dir })
|
||||
} finally {
|
||||
rmSync(dir, { recursive: true, force: true })
|
||||
}
|
||||
}
|
||||
function recover(h) {
|
||||
h.change((state) => {
|
||||
delete state.failure
|
||||
})
|
||||
h.run('restore')
|
||||
h.run('promoteRecovery')
|
||||
h.run('cleanup')
|
||||
h.run('retire')
|
||||
const state = h.state()
|
||||
assert.equal(state.serving, 'push-test-r123-1')
|
||||
assert.equal(state.latest, state.serving)
|
||||
assert.deepEqual(Object.keys(state.revisions), [state.serving])
|
||||
assert.equal(state.revisions[state.serving].spec.containers[0].image, image)
|
||||
assert.ok(state.peak <= 3)
|
||||
}
|
||||
|
||||
test(
|
||||
'the executable Cloud Run model rejects deleting latest even without tags or traffic',
|
||||
options,
|
||||
() =>
|
||||
exercise((h) => {
|
||||
h.run('preflight')
|
||||
h.run('candidate')
|
||||
h.run('gcloud run services update-traffic "$SERVICE_NAME" --clear-tags')
|
||||
const result = h.run('gcloud run revisions delete "$CANDIDATE_REVISION"', false)
|
||||
assert.match(result.stderr, /FAILED_PRECONDITION: latest created Revision/)
|
||||
})
|
||||
)
|
||||
|
||||
test(
|
||||
'success creates successor before retirement and repeated rollouts retain one consumer',
|
||||
options,
|
||||
() =>
|
||||
exercise((h) => {
|
||||
for (const attempt of ['1', '2']) {
|
||||
if (attempt === '2') {
|
||||
writeFileSync(join(h.dir, 'env'), 'GITHUB_RUN_ATTEMPT=2\n')
|
||||
}
|
||||
for (const key of ['preflight', 'candidate', 'activate', 'shift', 'public', 'retire']) {
|
||||
h.run(key)
|
||||
}
|
||||
const state = h.state()
|
||||
assert.equal(state.serving, `push-test-a123-${attempt}`)
|
||||
assert.deepEqual(Object.keys(state.revisions), [state.serving])
|
||||
assert.equal(state.peak, 3)
|
||||
}
|
||||
})
|
||||
)
|
||||
|
||||
for (const failure of ['deploy-before', 'deploy-after', 'describe']) {
|
||||
test(`validation ${failure} recovers without deleting latest`, options, () =>
|
||||
exercise((h) => {
|
||||
h.run('preflight')
|
||||
h.change((state) => {
|
||||
state.failure = failure
|
||||
})
|
||||
h.run('candidate', false)
|
||||
recover(h)
|
||||
})
|
||||
)
|
||||
}
|
||||
for (const failure of ['deploy-before', 'deploy-after', 'delete', 'describe']) {
|
||||
test(
|
||||
`activation ${failure} frees validation slot before recovery and stays within three`,
|
||||
options,
|
||||
() =>
|
||||
exercise((h) => {
|
||||
h.run('preflight')
|
||||
h.run('candidate')
|
||||
h.change((state) => {
|
||||
state.failure = failure
|
||||
})
|
||||
h.run('activate', false)
|
||||
recover(h)
|
||||
})
|
||||
)
|
||||
}
|
||||
|
||||
test(
|
||||
'ambiguous traffic shift records intent before mutation, rolls back and recovers',
|
||||
options,
|
||||
() =>
|
||||
exercise((h) => {
|
||||
h.run('preflight')
|
||||
h.run('candidate')
|
||||
h.run('activate')
|
||||
h.change((state) => {
|
||||
state.failure = 'traffic-after'
|
||||
})
|
||||
h.run('shift', false)
|
||||
assert.match(readFileSync(join(h.dir, 'env'), 'utf8'), /TRAFFIC_SHIFT_ATTEMPTED=true/)
|
||||
h.change((state) => {
|
||||
delete state.failure
|
||||
})
|
||||
h.run('rollback')
|
||||
recover(h)
|
||||
})
|
||||
)
|
||||
|
||||
test('failed public check rolls back and recovers', options, () =>
|
||||
exercise((h) => {
|
||||
for (const key of ['preflight', 'candidate', 'activate', 'shift']) {
|
||||
h.run(key)
|
||||
}
|
||||
h.change((state) => {
|
||||
state.failure = 'public'
|
||||
})
|
||||
h.run('public', false)
|
||||
h.change((state) => {
|
||||
delete state.failure
|
||||
})
|
||||
h.run('rollback')
|
||||
recover(h)
|
||||
})
|
||||
)
|
||||
|
||||
for (const defect of [
|
||||
'runtime',
|
||||
'secret',
|
||||
'mode',
|
||||
'image',
|
||||
'traffic',
|
||||
'scaling',
|
||||
'deploy-before',
|
||||
'deploy-after'
|
||||
]) {
|
||||
test(`recovery rejects ${defect} and preserves partial-create state`, options, () =>
|
||||
exercise((h) => {
|
||||
h.run('preflight')
|
||||
h.run('candidate')
|
||||
h.change((state) => {
|
||||
const revision = state.revisions[state.latest]
|
||||
if (defect === 'runtime') {
|
||||
revision.spec.serviceAccountName = 'wrong@test'
|
||||
}
|
||||
if (defect === 'secret') {
|
||||
revision.spec.containers[0].env[0].valueFrom.secretKeyRef.key = '8'
|
||||
}
|
||||
if (defect === 'scaling') {
|
||||
revision.metadata.annotations['autoscaling.knative.dev/maxScale'] = '3'
|
||||
}
|
||||
if (defect === 'traffic') {
|
||||
state.serving = state.latest
|
||||
}
|
||||
if (defect.startsWith('deploy-')) {
|
||||
state.failure = defect
|
||||
}
|
||||
})
|
||||
// Corrupt the recovery response after the modeled deploy while keeping real jq assertions.
|
||||
const extra = ['mode', 'image'].includes(defect)
|
||||
? `
|
||||
gcloud() {
|
||||
node "$MODEL_SCRIPT" "$@" > "$RUNNER_TEMP/out" || return $?
|
||||
if [[ "$*" == 'run services describe '* && "$*" == *'--format=json'* ]]; then
|
||||
jq '${defect === 'mode' ? '.spec.template.spec.containers[0].env += [{name:"ORCA_PUSH_MODE",value:"validation"}]' : '.spec.template.spec.containers[0].image = "wrong"'}' "$RUNNER_TEMP/out"
|
||||
else cat "$RUNNER_TEMP/out"; fi
|
||||
}`
|
||||
: ''
|
||||
h.run('restore', false, extra)
|
||||
const recorded = readFileSync(join(h.dir, 'env'), 'utf8')
|
||||
assert.match(recorded, /TEMPLATE_RECOVERY_REVISION=push-test-r123-1/)
|
||||
assert.doesNotMatch(recorded, /TEMPLATE_RESTORED=true/)
|
||||
assert.ok(h.state().peak <= 3)
|
||||
})
|
||||
)
|
||||
}
|
||||
|
||||
test('failed validation retirement blocks a fourth revision during recovery', options, () =>
|
||||
exercise((h) => {
|
||||
h.run('preflight')
|
||||
h.run('candidate')
|
||||
h.change((state) => {
|
||||
state.failure = 'delete'
|
||||
})
|
||||
h.run('activate', false)
|
||||
h.run('restore', false)
|
||||
assert.equal(h.state().peak, 3)
|
||||
assert.equal(h.state().revisions['push-test-r123-1'], undefined)
|
||||
})
|
||||
)
|
||||
|
||||
test(
|
||||
'failed retirement after public checks leaves verified serving and blocks the next run',
|
||||
options,
|
||||
() =>
|
||||
exercise((h) => {
|
||||
for (const key of ['preflight', 'candidate', 'activate', 'shift', 'public']) {
|
||||
h.run(key)
|
||||
}
|
||||
h.change((state) => {
|
||||
state.failure = 'delete'
|
||||
})
|
||||
h.run('retire', false)
|
||||
assert.equal(h.state().serving, 'push-test-a123-1')
|
||||
h.run('preflight', false)
|
||||
assert.match(workflow, /env.ROLLOUT_VERIFIED != 'true'/)
|
||||
})
|
||||
)
|
||||
|
||||
test(
|
||||
'recovery promotion failure keeps consumers for operator diagnosis and blocks new rollout',
|
||||
options,
|
||||
() =>
|
||||
exercise((h) => {
|
||||
h.run('preflight')
|
||||
h.run('candidate')
|
||||
h.run('restore')
|
||||
h.change((state) => {
|
||||
state.failure = 'public'
|
||||
})
|
||||
h.run('promoteRecovery', false)
|
||||
assert.doesNotMatch(readFileSync(join(h.dir, 'env'), 'utf8'), /RECOVERY_VERIFIED=true/)
|
||||
h.run('preflight', false)
|
||||
})
|
||||
)
|
||||
|
||||
const capability = step('Require image support for inert validation')
|
||||
for (const [label, source, ok] of [
|
||||
['old image', 'export function loadPushConfig() { return {}; }', false],
|
||||
[
|
||||
'invalid mode accepted',
|
||||
'export function loadPushConfig(env) { return { mode: env.ORCA_PUSH_MODE }; }',
|
||||
false
|
||||
],
|
||||
[
|
||||
'validation supported',
|
||||
`export function loadPushConfig(env) {
|
||||
if (env.ORCA_PUSH_MODE !== 'validation') throw new Error('invalid mode');
|
||||
return { mode: 'validation' };
|
||||
}`,
|
||||
true
|
||||
]
|
||||
]) {
|
||||
test(`pre-production image smoke: ${label}`, options, () =>
|
||||
exercise((h) => {
|
||||
const dist = join(h.dir, 'apps', 'push', 'dist')
|
||||
mkdirSync(dist, { recursive: true })
|
||||
writeFileSync(join(h.dir, 'package.json'), '{"type":"module"}')
|
||||
writeFileSync(join(dist, 'config.js'), source)
|
||||
h.run(capability, ok, 'docker() { node "${@: -3}"; }')
|
||||
})
|
||||
)
|
||||
}
|
||||
@@ -0,0 +1,334 @@
|
||||
import assert from 'node:assert/strict'
|
||||
import { readFileSync } from 'node:fs'
|
||||
import test from 'node:test'
|
||||
import {
|
||||
concurrencyBlocks,
|
||||
jobIf,
|
||||
jobs,
|
||||
LEASE_ACTION,
|
||||
leaseSteps
|
||||
} from './cloud-sql-rollout-lock-census.mjs'
|
||||
import { readRelayWorkflow, relayWorkflowFile } from './relay-repository.mjs'
|
||||
|
||||
// Why: the push gateway holds the APNs key and is the only thing standing between a paired
|
||||
// phone and a silent notification pipeline. Its deploy is a blue/green rollout against the
|
||||
// dedicated Cloud SQL instance, and each of the guarantees below is one careless edit from gone.
|
||||
const WORKFLOW = 'push-deploy.yml'
|
||||
const workflow = readRelayWorkflow(WORKFLOW)
|
||||
const deploy = () => {
|
||||
const job = jobs(workflow).find((entry) => entry.id === 'deploy')
|
||||
assert.ok(job, 'the workflow no longer declares a deploy job')
|
||||
return job
|
||||
}
|
||||
|
||||
function terraform(file) {
|
||||
return readFileSync(new URL(`../../infra/terraform/${file}`, import.meta.url), 'utf8')
|
||||
}
|
||||
|
||||
// The ordered step names; every assertion below reads positions out of this list rather than
|
||||
// restating them, so a reordering that breaks the no-traffic guarantee fails here.
|
||||
const stepNames = () => [...workflow.matchAll(/^ {6}- name: (.+)$/gm)].map((match) => match[1])
|
||||
|
||||
const indexOfStep = (name) => {
|
||||
const index = stepNames().indexOf(name)
|
||||
assert.notEqual(index, -1, `the workflow no longer has a "${name}" step`)
|
||||
return index
|
||||
}
|
||||
|
||||
test('the whole surface stays inert until the owner enables cloud operations', () => {
|
||||
const guard = jobIf(deploy().text)
|
||||
assert.ok(guard.includes("vars.ORCA_CLOUD_OPERATIONS_ENABLED == 'true'"), guard)
|
||||
assert.ok(guard.includes("github.ref == 'refs/heads/main'"), guard)
|
||||
assert.equal(jobs(workflow).length, 1, 'a second job would need its own gate')
|
||||
})
|
||||
|
||||
test('it authenticates through Workload Identity and holds no repository secret', () => {
|
||||
assert.match(workflow, /uses: google-github-actions\/auth@v2/)
|
||||
assert.match(workflow, /workload_identity_provider: \$\{\{ vars\.PRODUCTION_GCP_PUSH_DEPLOY_WORKLOAD_IDENTITY_PROVIDER \}\}/)
|
||||
assert.match(workflow, /service_account: \$\{\{ vars\.PRODUCTION_GCP_PUSH_DEPLOY_SERVICE_ACCOUNT \}\}/)
|
||||
assert.match(workflow, /environment: production/)
|
||||
for (const [, name] of workflow.matchAll(/secrets\.([A-Za-z_][A-Za-z0-9_]*)/g)) {
|
||||
assert.equal(name, 'GITHUB_TOKEN', `the workflow reads secrets.${name}`)
|
||||
}
|
||||
})
|
||||
|
||||
// Why: Terraform trusts exact workflow filenames, not a prefix. A rename here without the
|
||||
// matching tfvars-independent list entry would fail authentication at dispatch time only.
|
||||
test('Terraform trusts this exact workflow file on the production deploy provider', () => {
|
||||
assert.match(terraform('push-deploy-identity.tf'), /push-deploy\.yml@refs\/heads\/main/)
|
||||
assert.doesNotMatch(terraform('relay-github-actions.tf'), /push-deploy\.yml/)
|
||||
assert.equal(relayWorkflowFile(WORKFLOW), 'cloud-push-deploy.yml')
|
||||
})
|
||||
|
||||
test('the rollout is serialized and leases its dedicated push rollout lock', () => {
|
||||
const blocks = concurrencyBlocks(workflow)
|
||||
assert.equal(blocks.length, 1)
|
||||
assert.equal(blocks[0].group, 'production-push-rollout')
|
||||
assert.equal(blocks[0].cancelInProgress, 'false')
|
||||
const steps = leaseSteps(workflow)
|
||||
assert.equal(steps.length, 1, 'exactly one lease step, held for the whole run')
|
||||
assert.equal(steps[0].bucket, 'onorca-cloud-terraform-state')
|
||||
assert.equal(steps[0].object, 'terraform/state/push-rollout/production.lock')
|
||||
assert.equal(steps[0].release, undefined, 'release stays at its default for a single-job run')
|
||||
})
|
||||
|
||||
// Why: the ops guardrail is that a piped command only fails the step when pipefail is set, and
|
||||
// pipefail only applies under an explicit bash shell. Every multi-line body here opts in.
|
||||
test('every multi-line command runs under bash with pipefail', () => {
|
||||
const bodies = [...workflow.matchAll(/^ {8}(shell: bash\n {8})?run: \|\n((?: {10}.*\n|\n)+)/gm)]
|
||||
assert.ok(bodies.length >= 8, `only ${bodies.length} multi-line commands were found`)
|
||||
for (const match of bodies) {
|
||||
assert.ok(match[1], `a multi-line command does not declare shell: bash:\n${match[2].slice(0, 120)}`)
|
||||
assert.match(match[2], /^ {10}set -euo pipefail$/m)
|
||||
}
|
||||
})
|
||||
|
||||
test('the candidate revision takes no traffic and is addressed by its own tag', () => {
|
||||
assert.match(workflow, /gcloud run deploy "\$\{SERVICE_NAME\}"/)
|
||||
assert.match(workflow, /^ {12}--no-traffic \\$/m)
|
||||
assert.match(workflow, /--tag "\$\{tag\}"/)
|
||||
assert.match(workflow, /test "\$\{CANDIDATE_REVISION\}" != "\$\{ROLLBACK_REVISION\}"/)
|
||||
assert.ok(
|
||||
indexOfStep('Record the serving revision and require its Terraform-owned scaling') <
|
||||
indexOfStep('Deploy the candidate revision with no traffic'),
|
||||
'the rollback target must be captured before the candidate exists'
|
||||
)
|
||||
})
|
||||
|
||||
// Why: scaling is a Terraform-owned field that `lifecycle.ignore_changes` does not cover, so a
|
||||
// deploy that passed --max-instances would revert a later push_max_instances raise on every run.
|
||||
// The workflow asserts the shape instead of writing it, on the serving revision before the
|
||||
// candidate exists and on the candidate that inherits it.
|
||||
test('the deploy asserts the Terraform-owned scaling instead of mutating it', () => {
|
||||
assert.doesNotMatch(workflow, /--max-instances/, 'the deploy must not write a scaling field')
|
||||
assert.doesNotMatch(workflow, /--min-instances "/, 'the deploy must not write a scaling field')
|
||||
// The floor is the variables.tf default; production.tfvars overrides only the ceiling, down to
|
||||
// the two instances the Cloud SQL connection budget leaves room for.
|
||||
assert.match(workflow, /PUSH_MIN_INSTANCES: 1$/m)
|
||||
assert.match(workflow, /PUSH_MAX_INSTANCES: 2$/m)
|
||||
assert.match(terraform('variables.tf'), /variable "push_min_instances"[\s\S]*?default {5}= 1/)
|
||||
assert.match(terraform('environments/production.tfvars'), /^push_max_instances {9}= 2$/m)
|
||||
const gate = indexOfStep('Record the serving revision and require its Terraform-owned scaling')
|
||||
assert.ok(gate < indexOfStep('Deploy the candidate revision with no traffic'))
|
||||
assert.match(workflow, /autoscaling\.knative\.dev\/minScale/)
|
||||
assert.match(workflow, /\[\[ "\$\{floor:-0\}" -lt "\$\{PUSH_MIN_INSTANCES\}" \]\]/)
|
||||
assert.match(workflow, /test "\$\{ceiling\}" = "\$\{PUSH_MAX_INSTANCES\}"/)
|
||||
assert.match(workflow, /test "\$\{candidate_ceiling\}" = "\$\{PUSH_MAX_INSTANCES\}"/)
|
||||
})
|
||||
|
||||
// Why: the image build is not a Cloud SQL operation, and the lease is a global serialization
|
||||
// point. A build inside it blocks every relay deploy and rehome for its duration.
|
||||
test('the image is built before the rollout lease is taken', () => {
|
||||
const lease = workflow.indexOf(`- uses: ${LEASE_ACTION}`)
|
||||
assert.notEqual(lease, -1)
|
||||
const build = workflow.indexOf('- name: Build and publish the immutable gateway image')
|
||||
const deployCandidate = workflow.indexOf('- name: Deploy the candidate revision with no traffic')
|
||||
assert.ok(build < lease, 'the build must finish before the run takes the lease')
|
||||
assert.ok(lease < deployCandidate, 'the lease must still cover the deploy, probe, and shift')
|
||||
})
|
||||
|
||||
// Why: the gateway's Cloud SQL draw is instances x pool, and the root that takes the rollout
|
||||
// lease can only account for a pool it declares. Leaving it at the application default hid it.
|
||||
test('the database pool size is Terraform-owned and bounded at plan time', () => {
|
||||
const source = terraform('push-gateway.tf')
|
||||
assert.match(source, /name {2}= "ORCA_PUSH_DATABASE_POOL_MAX"/)
|
||||
assert.match(source, /value = tostring\(var\.push_database_pool_max\)/)
|
||||
assert.match(terraform('variables.tf'), /variable "push_database_pool_max"[\s\S]*?default {5}= 2/)
|
||||
const block = /resource "google_cloud_run_v2_service" "push"[\s\S]*?\n lifecycle \{([\s\S]*?)\n \}/.exec(source)
|
||||
assert.ok(block, 'the push service no longer declares a lifecycle block')
|
||||
assert.match(
|
||||
block[1],
|
||||
/var\.push_max_instances \* var\.push_database_pool_max \* 3 <= 64/,
|
||||
'instances x pool must be bounded at plan time'
|
||||
)
|
||||
assert.match(
|
||||
readFileSync(new URL('../../apps/push/src/config.ts', import.meta.url), 'utf8'),
|
||||
/ORCA_PUSH_DATABASE_POOL_MAX/,
|
||||
'the gateway must read the variable Terraform sets'
|
||||
)
|
||||
})
|
||||
|
||||
test('the candidate is probed on its own URL before any traffic moves', () => {
|
||||
const probe = indexOfStep('Probe the candidate readiness endpoint')
|
||||
assert.ok(probe > indexOfStep('Deploy the candidate revision with no traffic'))
|
||||
assert.ok(probe < indexOfStep('Shift all traffic to the verified candidate'))
|
||||
assert.match(workflow, /"\$\{CANDIDATE_URL\}\/ready"/)
|
||||
assert.match(workflow, /test "\$\{code\}" = 200/)
|
||||
assert.ok(workflow.indexOf('${CANDIDATE_URL}/ready') < workflow.indexOf('${CANDIDATE_URL}/health'))
|
||||
assert.match(workflow, /\.deliveryProtocol == 2/, 'verify the durable gateway after readiness')
|
||||
})
|
||||
|
||||
// Why: a gateway that answers /ready can still hold no usable FCM credential. The probe must be
|
||||
// validate-only, must use a token that cannot exist, and must treat a denied credential as the
|
||||
// failure. Accepting PERMISSION_DENIED would make the whole step decorative.
|
||||
test('the FCM probe is validate-only and separates a bad token from a bad credential', () => {
|
||||
const fcm = indexOfStep('Prove the runtime identity can reach FCM')
|
||||
assert.ok(fcm > indexOfStep('Probe the candidate readiness endpoint'))
|
||||
assert.ok(fcm < indexOfStep('Shift all traffic to the verified candidate'))
|
||||
assert.match(workflow, /"validate_only":true/)
|
||||
assert.match(workflow, /https:\/\/fcm\.googleapis\.com\/v1\/projects\/\$\{GCP_PROJECT_ID\}\/messages:send/)
|
||||
assert.match(workflow, /GCP_PROJECT_ID: onorca-cloud$/m)
|
||||
assert.match(workflow, /orca-push-deploy-probe-invalid-token/)
|
||||
assert.match(workflow, /test "\$\{status\}" = INVALID_ARGUMENT/)
|
||||
assert.match(workflow, /test "\$\{status\}" = PERMISSION_DENIED/)
|
||||
// Only those four answers are conclusive; a 429 or a 5xx says nothing about the credential, so
|
||||
// it is retried rather than read as either verdict. A denied credential still fails at once.
|
||||
assert.match(workflow, /for attempt in \$\(seq 1 5\); do/)
|
||||
const probe = workflow.slice(
|
||||
workflow.indexOf('- name: Prove the runtime identity can reach FCM'),
|
||||
workflow.indexOf('- name: Shift all traffic to the verified candidate')
|
||||
)
|
||||
assert.match(probe, /for attempt in \$\(seq 1 5\); do/)
|
||||
assert.match(probe, /test "\$\{code\}" = 401 \|\| test "\$\{code\}" = 403; then\n {14}break/)
|
||||
assert.match(
|
||||
workflow,
|
||||
/--impersonate-service-account "\$\{PUSH_RUNTIME_SERVICE_ACCOUNT\}"/,
|
||||
'the probe must exercise the runtime credential, not the deploy identity'
|
||||
)
|
||||
// Why: that token reads the Apple signing key. Masking it means a later `set -x` or a
|
||||
// debug re-run cannot print it into a public log.
|
||||
assert.match(
|
||||
probe,
|
||||
/test -n "\$\{token\}"\n {10}echo "::add-mask::\$\{token\}"/,
|
||||
'the impersonated token must be masked before anything else runs'
|
||||
)
|
||||
assert.match(workflow, /PUSH_RUNTIME_SERVICE_ACCOUNT: orca-cloud-push@onorca-cloud\.iam\.gserviceaccount\.com/)
|
||||
})
|
||||
|
||||
// Why: a deploy ends with traffic pinned to an exact revision, and a rollback pins it to the
|
||||
// previous one. Terraform reverting the service to 100% LATEST would undo either silently.
|
||||
test('Terraform does not own the image or the traffic split', () => {
|
||||
const source = terraform('push-gateway.tf')
|
||||
const block = /resource "google_cloud_run_v2_service" "push"[\s\S]*?\n lifecycle \{([\s\S]*?)\n \}/.exec(source)
|
||||
assert.ok(block, 'the push service no longer declares a lifecycle block')
|
||||
assert.match(block[1], /template\[0\]\.containers\[0\]\.image/)
|
||||
assert.match(block[1], /^\s*traffic$/m)
|
||||
})
|
||||
|
||||
test('impersonating the runtime identity is a Terraform-declared grant', () => {
|
||||
const source = terraform('push-gateway.tf')
|
||||
assert.match(source, /resource "google_service_account_iam_member" "github_production_push_runtime_token_creator"/)
|
||||
assert.match(source, /role\s+= "roles\/iam\.serviceAccountTokenCreator"/)
|
||||
assert.match(source, /resource "google_cloud_run_v2_service_iam_member" "github_production_push_developer"/)
|
||||
})
|
||||
|
||||
test('the traffic shift is all-or-nothing and is verified after the fact', () => {
|
||||
const shift = indexOfStep('Shift all traffic to the verified candidate')
|
||||
assert.match(workflow, /gcloud run services update-traffic "\$\{SERVICE_NAME\}"/)
|
||||
assert.match(workflow, /--to-revisions "\$\{CANDIDATE_REVISION\}=100"/)
|
||||
assert.match(workflow, /test "\$\{serving\}" = "\$\{CANDIDATE_REVISION\}"/)
|
||||
assert.ok(shift < indexOfStep('Verify the public origin after the shift'))
|
||||
assert.match(workflow, /PUSH_ORIGIN: https:\/\/push\.onorca\.dev/)
|
||||
assert.match(workflow, /"\$\{PUSH_ORIGIN\}\/ready"/)
|
||||
})
|
||||
|
||||
// Why: the origin can lag the traffic move by seconds, and a single unlucky curl would otherwise
|
||||
// roll a healthy deploy back. It retries on the same schedule as the candidate probe.
|
||||
test('the post-shift origin check retries like the candidate probe', () => {
|
||||
const check = workflow.slice(
|
||||
workflow.indexOf('- name: Verify the public origin after the shift'),
|
||||
workflow.indexOf('- name: Roll traffic back to the previous revision')
|
||||
)
|
||||
assert.match(check, /for attempt in \$\(seq 1 30\); do/)
|
||||
assert.match(check, /sleep 5/)
|
||||
assert.match(check, /test "\$\{code\}" = 200/)
|
||||
})
|
||||
|
||||
// Why: the summary carries the rollback target. Writing it after the origin check meant the one
|
||||
// run that needed it, the run whose check failed, was the one run that never got it.
|
||||
test('the summary is written before anything that can fail after the shift', () => {
|
||||
const summary = indexOfStep('Publish the rollout summary')
|
||||
assert.ok(summary > indexOfStep('Shift all traffic to the verified candidate'))
|
||||
assert.ok(summary < indexOfStep('Verify the public origin after the shift'))
|
||||
assert.match(workflow, /Known-good image:/)
|
||||
assert.match(workflow, /GITHUB_STEP_SUMMARY/)
|
||||
})
|
||||
|
||||
// Why: everything after the shift runs with production on the candidate, so a failure there is a
|
||||
// live gateway that has to go back. The marker is what separates that case from a failure before
|
||||
// the shift, where production never moved and the candidate is the thing to clean up.
|
||||
test('a failure after the shift rolls production back automatically', () => {
|
||||
const rollback = indexOfStep('Roll traffic back to the previous revision')
|
||||
assert.ok(rollback > indexOfStep('Verify the public origin after the shift'))
|
||||
assert.match(workflow, /echo "TRAFFIC_SHIFTED=true" >> "\$\{GITHUB_ENV\}"/)
|
||||
const shift = workflow.indexOf('- name: Shift all traffic to the verified candidate')
|
||||
assert.ok(
|
||||
workflow.indexOf('echo "TRAFFIC_SHIFTED=true"') > shift,
|
||||
'the success marker follows the shift step'
|
||||
)
|
||||
const body = workflow.slice(
|
||||
workflow.indexOf('- name: Roll traffic back to the previous revision'),
|
||||
workflow.indexOf('- name: Delete the rejected candidate revision')
|
||||
)
|
||||
assert.match(
|
||||
body,
|
||||
/if: \$\{\{ \(failure\(\) \|\| cancelled\(\)\) && env\.TRAFFIC_SHIFT_ATTEMPTED == 'true' && env\.ROLLOUT_VERIFIED != 'true' \}\}/,
|
||||
'the rollback must be conditioned on both failure and the shift marker'
|
||||
)
|
||||
assert.match(body, /test -n "\$\{ROLLBACK_REVISION:-\}"/)
|
||||
assert.match(body, /--to-revisions "\$\{ROLLBACK_REVISION\}=100"/)
|
||||
assert.match(body, /test "\$\{serving\}" = "\$\{ROLLBACK_REVISION\}"/)
|
||||
assert.match(body, /GITHUB_STEP_SUMMARY/, 'the rollback must be reported in the summary')
|
||||
})
|
||||
|
||||
// Why: a candidate that never took traffic still holds a warm instance and a Cloud SQL pool. Its
|
||||
// tag comes off first, because Cloud Run refuses to delete a revision a traffic target names.
|
||||
test('verified recovery authorizes rejected candidate deletion', () => {
|
||||
const body = workflow.slice(
|
||||
workflow.indexOf('- name: Delete the rejected candidate revision'),
|
||||
workflow.indexOf('- name: Drop the candidate traffic tag')
|
||||
)
|
||||
assert.match(
|
||||
body,
|
||||
/env\.RECOVERY_VERIFIED == 'true'/,
|
||||
'cleanup must wait for verified recovery traffic and public checks'
|
||||
)
|
||||
assert.match(body, /if test -z "\$\{CANDIDATE_REVISION:-\}"; then/)
|
||||
assert.ok(
|
||||
body.indexOf('--remove-tags') < body.indexOf('gcloud run revisions delete'),
|
||||
'the tag must come off before the revision is deleted'
|
||||
)
|
||||
assert.match(body, /echo "CANDIDATE_TAG=" >> "\$\{GITHUB_ENV\}"/)
|
||||
})
|
||||
|
||||
test('the run always drops its traffic tag', () => {
|
||||
const cleanup = indexOfStep('Drop the candidate traffic tag')
|
||||
assert.equal(cleanup, stepNames().length - 1, 'tag cleanup must be the last step')
|
||||
assert.match(workflow, /--remove-tags "\$\{CANDIDATE_TAG\}"/)
|
||||
const body = workflow.slice(workflow.indexOf('- name: Drop the candidate traffic tag'))
|
||||
assert.match(body, /if: always\(\)/)
|
||||
assert.match(body, /test -n "\$\{CANDIDATE_TAG:-\}" \|\| exit 0/)
|
||||
})
|
||||
|
||||
test('push credentials cannot assume the shared Relay deploy identity', () => {
|
||||
const source = terraform('push-deploy-identity.tf')
|
||||
assert.match(source, /"attribute.push_deploy"\s*=\s*"'production'"/)
|
||||
assert.doesNotMatch(source, /"attribute.repository"\s*=/)
|
||||
assert.match(source, /attribute\.push_deploy\/production/)
|
||||
assert.doesNotMatch(workflow, /PRODUCTION_GCP_RELAY_DEPLOY_/)
|
||||
assert.doesNotMatch(terraform('push-gateway.tf'), /member\s*=\s*local\.relay_github_deploy_service_account_member/)
|
||||
})
|
||||
|
||||
// A latest revision needs a successor even when validation is inert.
|
||||
test('dedicated database admits three simultaneous revision pools', () => {
|
||||
assert.match(terraform('push-gateway.tf'), /var\.push_max_instances \* var\.push_database_pool_max \* 3 <= 64/)
|
||||
})
|
||||
|
||||
test('push has only a dedicated database attachment and a narrowly scoped deployment lease', () => {
|
||||
const service = terraform('push-gateway.tf')
|
||||
const database = terraform('push-dedicated-database.tf')
|
||||
assert.match(service, /instances = \[google_sql_database_instance\.push_dedicated\[0\]\.connection_name\]/)
|
||||
assert.match(service, /secret\s*= google_secret_manager_secret\.push_dedicated_database_url\[0\]\.secret_id/)
|
||||
assert.match(service, /version = google_secret_manager_secret_version\.push_dedicated_database_url\[0\]\.version/)
|
||||
assert.doesNotMatch(service + database, /push_dedicated_database_(?:active|enabled)|local\.relay_database_connection_name|resource "google_sql_database" "push"/)
|
||||
assert.match(database, /tier\s*= "db-custom-2-7680"/)
|
||||
assert.match(database, /availability_type = "REGIONAL"/)
|
||||
assert.match(database, /deletion_protection\s*= true/)
|
||||
assert.match(database, /deletion_protection_enabled = true/)
|
||||
const identity = terraform('push-deploy-identity.tf')
|
||||
const lease = identity.match(/resource "google_storage_bucket_iam_member" "github_push_rollout_lease" \{([\s\S]*?)\n\}/)?.[1]
|
||||
assert.ok(lease)
|
||||
assert.match(lease, /member = local\.push_deploy_member/)
|
||||
assert.match(lease, /role\s*= "roles\/storage.objectAdmin"/)
|
||||
assert.match(lease, /resource.name == 'projects\/_\/buckets\/\$\{var.project_id\}-terraform-state\/objects\/terraform\/state\/push-rollout\/production.lock'/)
|
||||
})
|
||||
@@ -0,0 +1,48 @@
|
||||
import assert from 'node:assert/strict'
|
||||
import { readFileSync } from 'node:fs'
|
||||
import test from 'node:test'
|
||||
import { readRelayWorkflow } from './relay-repository.mjs'
|
||||
|
||||
const workflow = readRelayWorkflow('push-deploy.yml')
|
||||
const position = (name) => {
|
||||
const index = workflow.indexOf(`- name: ${name}`)
|
||||
assert.notEqual(index, -1)
|
||||
return index
|
||||
}
|
||||
const capability = position('Require image support for inert validation')
|
||||
const deploy = position('Deploy the candidate revision with no traffic')
|
||||
const activation = position('Retire inert validation and activate the verified image')
|
||||
const shift = position('Shift all traffic to the verified candidate')
|
||||
|
||||
test('the exact build digest must support validation before production boot', () => {
|
||||
assert.match(workflow, /docker buildx build --push --platform linux\/amd64 --provenance=false --metadata-file/)
|
||||
assert.match(workflow, /containerimage\.digest/)
|
||||
assert.doesNotMatch(workflow, /gcloud artifacts docker images describe/)
|
||||
assert.ok(capability < deploy)
|
||||
const preflight = workflow.slice(capability, deploy)
|
||||
assert.match(preflight, /docker run --rm --network none --entrypoint node "\$\{IMAGE\}"/)
|
||||
assert.match(preflight, /loadPushConfig\(env\)\.mode !== "validation"/)
|
||||
assert.match(preflight, /validation_mode_not_fail_closed/)
|
||||
})
|
||||
|
||||
test('inert validation and credential checks precede deliberate activation of the same digest', () => {
|
||||
assert.match(workflow.slice(deploy, activation), /--update-env-vars ORCA_PUSH_MODE=validation/)
|
||||
assert.match(workflow.slice(deploy, activation), /\.mode == "validation"/)
|
||||
assert.ok(position('Prove the runtime identity can reach FCM') < activation)
|
||||
const active = workflow.slice(activation, shift)
|
||||
assert.ok(active.indexOf('gcloud run deploy') < active.indexOf('gcloud run revisions delete'))
|
||||
assert.match(active, /--image "\$\{IMAGE\}"/)
|
||||
assert.match(active, /--remove-env-vars ORCA_PUSH_MODE/)
|
||||
assert.match(active, /\.spec\.containers\[0\]\.image == \$image/)
|
||||
assert.match(active, /\.spec\.serviceAccountName == \$account/)
|
||||
assert.match(active, /\.mode == "active"/)
|
||||
assert.ok(active.indexOf('ACTIVATION_ATTEMPTED=true') < active.indexOf('gcloud run deploy'))
|
||||
assert.match(workflow, /deletion below must stop its workers/)
|
||||
})
|
||||
|
||||
test('production startup connects read-only and gates all background work in validation', () => {
|
||||
const entry = readFileSync(new URL('../../apps/push/src/index.ts', import.meta.url), 'utf8')
|
||||
assert.match(entry, /readOnly: config\.mode === 'validation'/)
|
||||
assert.match(entry, /startPushBackground\(config,/)
|
||||
assert.doesNotMatch(entry, /worker\.start\(/)
|
||||
})
|
||||
@@ -6,7 +6,7 @@ import {
|
||||
readRelayCloudSqlConnectionBudget
|
||||
} from './relay-cloud-sql-connection-budget.mjs'
|
||||
|
||||
test('production plus three Asia pools preserves allowance and reserve below the ceiling', () => {
|
||||
test('production shared consumers keep allowance and reserve below the ceiling', () => {
|
||||
const report = readRelayCloudSqlConnectionBudget()
|
||||
|
||||
assert.deepEqual(report.consumers, { cells: 230, directors: 15, auth: 20, api: 50 })
|
||||
@@ -63,7 +63,11 @@ test('excludes fenced cell pools and reads per-cell pool overrides', () => {
|
||||
}
|
||||
}
|
||||
`,
|
||||
terraformVariables: 'variable "relay_director_database_pool_max" { default = 3 }',
|
||||
terraformVariables: [
|
||||
'variable "relay_director_database_pool_max" { default = 3 }',
|
||||
'variable "push_max_instances" { default = 1 }',
|
||||
'variable "push_database_pool_max" { default = 2 }'
|
||||
].join('\n'),
|
||||
relayConfig: 'export const RELAY_DATABASE_POOL_MAX = 10'
|
||||
},
|
||||
maxConnections: 100,
|
||||
@@ -76,6 +80,37 @@ test('excludes fenced cell pools and reads per-cell pool overrides', () => {
|
||||
assert.equal(report.budgetedTotal, 47)
|
||||
})
|
||||
|
||||
test('dedicated push scaling does not consume shared capacity', () => {
|
||||
const report = readRelayCloudSqlConnectionBudget({
|
||||
proposedAsiaCellCount: 1,
|
||||
appConsumers: { authInstances: 1, authPoolMax: 10, apiInstances: 1, apiPoolMax: 5, maxConnections: 100 },
|
||||
sources: {
|
||||
productionTfvars: `
|
||||
relay_max_instances = 1
|
||||
push_max_instances = 3
|
||||
relay_gce_fenced_cells = []
|
||||
relay_gce_cells = {
|
||||
"production-gce-c2" = { database_pool_max = 4
|
||||
}
|
||||
}
|
||||
`,
|
||||
terraformVariables: [
|
||||
'variable "relay_director_database_pool_max" { default = 3 }',
|
||||
'variable "push_max_instances" { default = 1 }',
|
||||
'variable "push_database_pool_max" { default = 2 }'
|
||||
].join('\n'),
|
||||
relayConfig: 'export const RELAY_DATABASE_POOL_MAX = 10'
|
||||
},
|
||||
maxConnections: 100,
|
||||
maintenanceAdminAllowance: 1,
|
||||
explicitReserve: 1
|
||||
})
|
||||
|
||||
assert.equal(report.consumers.push, undefined)
|
||||
assert.equal(report.rolloutOverlap.pushCandidate, undefined)
|
||||
assert.equal(report.operatingMaximum, 46)
|
||||
})
|
||||
|
||||
test('requires strict headroom below the physical ceiling', () => {
|
||||
const report = calculateRelayCloudSqlConnectionBudget({
|
||||
cellPoolTotal: 20,
|
||||
|
||||
@@ -18,6 +18,7 @@ const TERRAFORM_ROOTS = {
|
||||
'infra/terraform/relay-shared.tf',
|
||||
'infra/terraform/relay-github-workflow-trust.tf',
|
||||
'infra/terraform/relay-github-actions.tf',
|
||||
'infra/terraform/push-deploy-identity.tf',
|
||||
'infra/terraform/relay-staging-deploy-iam.tf',
|
||||
'infra/terraform/relay-asia-topology-iam.tf',
|
||||
'infra/terraform/relay-asia-proof-iam.tf'
|
||||
@@ -475,7 +476,7 @@ function collectTfvars(source, variables) {
|
||||
offset += line.length + 1
|
||||
continue
|
||||
}
|
||||
const structured = /^([A-Za-z_][A-Za-z0-9_]*)\s*=\s*(?=[[{])/.exec(line)
|
||||
const structured = /^([A-Za-z_][A-Za-z0-9_]*)\s*=\s*/.exec(line)
|
||||
if (structured) {
|
||||
try {
|
||||
variables[structured[1]] = parseValueAt(source, offset + structured[0].length)
|
||||
|
||||
@@ -30,6 +30,9 @@ const EXPECTED_CONDITIONS = {
|
||||
},
|
||||
production: {
|
||||
relay: {
|
||||
github_push:
|
||||
"assertion.repository == 'stablyai/orca' && assertion.repository_id == '1183888342' && assertion.repository_owner_id == '127256420' && assertion.ref == 'refs/heads/main' && assertion.environment == 'production' && assertion.event_name == 'workflow_dispatch' && assertion.workflow_ref == 'stablyai/orca/.github/workflows/cloud-push-deploy.yml@refs/heads/main' && assertion.job_workflow_ref == 'stablyai/orca/.github/workflows/cloud-push-deploy.yml@refs/heads/main'",
|
||||
|
||||
github:
|
||||
"assertion.repository == 'stablyai/orca' && assertion.repository_id == '1183888342' && assertion.repository_owner_id == '127256420' && assertion.ref == 'refs/heads/main' && assertion.environment == 'production' && ((assertion.workflow_ref == 'stablyai/orca/.github/workflows/cloud-deploy-relay-fence-broker.yml@refs/heads/main' || assertion.workflow_ref == 'stablyai/orca/.github/workflows/cloud-deploy-relay-production-capacity.yml@refs/heads/main' || assertion.workflow_ref == 'stablyai/orca/.github/workflows/cloud-deploy-relay-production-director.yml@refs/heads/main' || assertion.workflow_ref == 'stablyai/orca/.github/workflows/cloud-deploy-relay-production-multi-target.yml@refs/heads/main' || assertion.workflow_ref == 'stablyai/orca/.github/workflows/cloud-deploy-relay-production.yml@refs/heads/main' || assertion.workflow_ref == 'stablyai/orca/.github/workflows/cloud-operate-relay-asia-admission.yml@refs/heads/main' || assertion.workflow_ref == 'stablyai/orca/.github/workflows/cloud-publish-relay-production.yml@refs/heads/main') || (assertion.workflow_ref == 'stablyai/orca/.github/workflows/cloud-operate-relay-production-rehome.yml@refs/heads/main' && assertion.job_workflow_ref == 'stablyai/orca/.github/workflows/cloud-operate-relay-production-rehome-job.yml@refs/heads/main') || (assertion.workflow_ref == 'stablyai/orca/.github/workflows/cloud-deploy-relay-production-same-cap.yml@refs/heads/main' && (assertion.job_workflow_ref == 'stablyai/orca/.github/workflows/cloud-deploy-relay-production-same-cap-job.yml@refs/heads/main' || assertion.job_workflow_ref == 'stablyai/orca/.github/workflows/cloud-deploy-relay-production-same-cap.yml@refs/heads/main')))",
|
||||
github_monitor:
|
||||
|
||||
@@ -0,0 +1,112 @@
|
||||
# Dedicated push database operations
|
||||
|
||||
Push attaches only to its dedicated PostgreSQL 17 instance: regional HA, 2 vCPU,
|
||||
7.5 GiB RAM, 50 GiB SSD with automatic growth, seven retained backups and seven-day
|
||||
point-in-time recovery. Cloud SQL and Terraform deletion protections remain enabled.
|
||||
The Cloud SQL connector uses the dedicated URL secret pinned to its managed version.
|
||||
There is no shared-storage fallback or provision/activate switch.
|
||||
|
||||
## Existing-resource cleanup: operator prerequisite
|
||||
|
||||
This is a plan/runbook, not authorization to apply or delete resources. Preserve the
|
||||
shared Orca instance, dedicated push instance, all dedicated data and identities, and
|
||||
unrelated resources. The dedicated resource addresses remain unchanged:
|
||||
|
||||
- `google_sql_database_instance.push_dedicated[0]`
|
||||
- `google_sql_database.push_dedicated[0]`
|
||||
- `random_password.push_dedicated_database[0]`
|
||||
- `google_sql_user.push_dedicated[0]`
|
||||
- `google_secret_manager_secret.push_dedicated_database_url[0]`
|
||||
- `google_secret_manager_secret_version.push_dedicated_database_url[0]`
|
||||
- `google_secret_manager_secret_iam_member.push_dedicated_database_url_accessor[0]`
|
||||
|
||||
The relay state may still own these six obsolete shared-store resources, whose
|
||||
configuration is removed. An untargeted plan would propose deleting them; do not apply it:
|
||||
|
||||
- `google_sql_database.push[0]`
|
||||
- `google_sql_user.push[0]`
|
||||
- `random_password.push_database[0]`
|
||||
- `google_secret_manager_secret.push_database_url[0]`
|
||||
- `google_secret_manager_secret_version.push_database_url[0]`
|
||||
- `google_secret_manager_secret_iam_member.push_database_url_runtime_accessor[0]`
|
||||
|
||||
1. Use the production backend in `infra/terraform/README.md`. Inspect state addresses and
|
||||
the live service attachment, pinned secret reference and revision resources without
|
||||
printing credentials. Require the dedicated attachment and no old shared-store consumers;
|
||||
source connection drain needs an authorized operator's read-only observation.
|
||||
2. Have the shared database owner adopt the six legacy resources in an explicitly owned
|
||||
archival configuration before retiring their relay-state ownership. Retain the former
|
||||
database's `prevent_destroy` protection and secret versions; do not disable protection,
|
||||
drop databases, rotate passwords or introduce a second runtime attachment. A reviewed
|
||||
exact-address state transfer must preserve remote IDs and secret material in approved
|
||||
Terraform storage, with no credential exports to local files or terminal output.
|
||||
3. Require the owner's import/ownership plan to preserve existing resources and then an
|
||||
empty plan for those addresses. Only after adoption is proven may the operator remove
|
||||
precisely the six former addresses from relay state under backend locking. Do not
|
||||
automate this via `removed` blocks, broad `state rm`, force, or an untargeted apply.
|
||||
4. Review a fresh relay plan. Reject every delete or replace affecting either SQL instance,
|
||||
dedicated databases/users/secrets, or unrelated resources. Target only the intended push
|
||||
service and lease IAM grant for rollout; review their dependency closure too. Existing
|
||||
unrelated drift must be handled by its owner, outside this cleanup.
|
||||
|
||||
No data transfer, dedicated database reset, or phone re-registration is part of this cleanup.
|
||||
|
||||
## Schema prerequisite for existing internal test databases
|
||||
|
||||
New schemas omit `push_hosts` and the unused `host_public_key` and `transcript` columns
|
||||
on `push_challenges`. Authentication still verifies the encrypted transcript and consumes
|
||||
its challenge digest once; sessions and device ownership are unchanged. No compatibility
|
||||
migration for unpublished builds runs at application startup.
|
||||
|
||||
Before deploying onto an older internal schema, an operator must arrange a separately
|
||||
reviewed schema-preparation job through the approved database execution path. Its entire
|
||||
scope is dropping `push_hosts` (including its index) and those two unused challenge columns;
|
||||
preserve challenge digest/expiry/consumption fields and every session, device and delivery
|
||||
table. Verify that the old NOT NULL columns are absent before admitting the new image.
|
||||
Do not hand-edit production SQL or reset the dedicated database to satisfy this prerequisite.
|
||||
Until that job is reviewed and executed, the new image is not ready for an existing schema.
|
||||
|
||||
## Deployment serialization transition
|
||||
|
||||
Finish all old push workflow runs before changing the workflow's lock namespace. An old
|
||||
shared-lock push run and a new push-lock run do not exclude each other. Hold off new push
|
||||
dispatches while preparing the following exact changes:
|
||||
|
||||
1. Review the relay-root plan for
|
||||
`google_storage_bucket_iam_member.github_push_rollout_lease[0]`. It grants only
|
||||
`roles/storage.objectAdmin` on
|
||||
`projects/_/buckets/onorca-cloud-terraform-state/objects/terraform/state/push-rollout/production.lock`
|
||||
to the dedicated push deploy account. The lease action uses object GET/upload/delete,
|
||||
so no bucket-wide listing or Terraform-state access is needed.
|
||||
2. After approval, apply only the reviewed IAM/dependency plan. Verify the exact condition
|
||||
and principal independently. If foundation still grants push membership in the old
|
||||
`cloud_sql_rollout_lease_members`, its owner removes only that push member; keep Relay's
|
||||
existing members and permissions. Do not mutate foundation through the relay root.
|
||||
3. Publish the reviewed workflow on main with `production-push-rollout`, cancellation
|
||||
disabled, and the existing lease action pointed at the dedicated object. The durable
|
||||
lease covers admission, candidate validation, activation, traffic changes and recovery.
|
||||
A stale/conflicting lease stops the run; it is never stolen or force-deleted.
|
||||
4. Deploy the reviewed image through `cloud-push-deploy.yml`. Preserve candidate readiness,
|
||||
runtime-provider validation, exact digest/configuration checks, and explicit activation.
|
||||
Verify the public origin and real notification delivery/dismissal afterward.
|
||||
|
||||
## Recovery and capacity
|
||||
|
||||
Activation starts schema writes and workers before HTTP promotion. Traffic rollback cannot
|
||||
undo queue or schema changes. Cloud Run cannot delete its latest revision, so failed
|
||||
activation creates a known-good successor, verifies it, promotes it, then retires rejected
|
||||
and previous revisions. When partial activation leaves three resources, retire non-latest
|
||||
inert validation before creating recovery. Failed retirement stops automation. Admission
|
||||
requires one serving revision resource; retire historical leftovers under the push lease.
|
||||
Keep the dedicated attachment for application rollback and retain an immutable compatible
|
||||
image. An image requiring the removed challenge columns needs separate schema review.
|
||||
|
||||
The two-instance ceiling and two-connection pool draw four configured connections, twelve
|
||||
across three simultaneous revision resources. Terraform caps instances × pool × 3 at 64
|
||||
for serving, validation/rejected and active/recovery pools. Push does not draw from Relay's
|
||||
shared connection budget. Source connections must have drained before treating that old
|
||||
allocation as free. Increase capacity only after measuring deployed contention.
|
||||
|
||||
Cloud SQL resizing can interrupt connections despite HA. Durable accepted events remain in
|
||||
SQL; workers retry within each event's original five-minute deadline. Schedule resizes and
|
||||
verify reconnection, queue recovery, readiness and real delivery afterward.
|
||||
@@ -0,0 +1,392 @@
|
||||
# Orca mobile push gateway
|
||||
|
||||
`orca-cloud-push` is a public Cloud Run service in `onorca-cloud` that turns a desktop
|
||||
notification into an APNs or FCM push for a paired phone. The desktop registers each phone's
|
||||
native token with it and calls `POST /v1/send` after the socket fan-out it already does; the
|
||||
phone treats APNs/FCM as the sole ordinary OS-banner path. The notification socket is retained only
|
||||
for live dismissal and reconnect tray reconciliation; it does not create or recover banners. Desktop
|
||||
notification categories remain authoritative. The service is the only place the Apple
|
||||
`.p8` signing key is readable, which is the reason it exists as a service at all.
|
||||
|
||||
The request schemas live in `packages/push-contract/src/`. This document covers Terraform
|
||||
ownership, deployment, credential rotation, and recovery.
|
||||
|
||||
**There is no staging push gateway.** That is a decision, not an omission. `push_gateway_enabled`
|
||||
is false in `environments/staging.tfvars` and true in `environments/production.tfvars`, and every
|
||||
resource in `infra/terraform/push-gateway.tf` is behind it. A staging gateway would be a tfvars
|
||||
edit plus a second set of Apple credentials.
|
||||
|
||||
## Shape
|
||||
|
||||
| Setting | Value | Where |
|
||||
| ----------------- | ------------------------------------------------------ | -------------------------------------------- |
|
||||
| Cloud Run service | `orca-cloud-push` | `push_cloud_run_service_name` |
|
||||
| Region | `us-central1` | `region` |
|
||||
| Instances | min 1, max 2 | `push_min_instances`, `push_max_instances` |
|
||||
| Database pool | 2 per instance | `push_database_pool_max` |
|
||||
| Concurrency | 80 | `push_concurrency` |
|
||||
| Ingress | all | `INGRESS_TRAFFIC_ALL` |
|
||||
| Invoker | IAM disabled | `invoker_iam_disabled = true` on the service |
|
||||
| Runtime identity | `orca-cloud-push@onorca-cloud.iam.gserviceaccount.com` | `google_service_account.push_runtime` |
|
||||
| Database | `orca_push` on dedicated HA PostgreSQL 17 | `google_sql_database.push_dedicated` |
|
||||
| Hostname | `push.onorca.dev` | `push_base_url` |
|
||||
|
||||
The minimum of one instance is deliberate and did not move when the ceiling came down to two. A
|
||||
cold start delays a notification past the point where it is worth showing, so the floor is what
|
||||
keeps a notification prompt. The
|
||||
ceiling is a different question, answered below.
|
||||
|
||||
Push uses its approved dedicated two-vCPU HA database. Two instances with a two-connection
|
||||
pool draw four connections; three simultaneous revision resources draw twelve. Tagged
|
||||
candidates can run outside the service-wide cap, so Terraform bounds instances × pool × 3
|
||||
at 64 connections, leaving dedicated capacity for maintenance and operators. Increase pool
|
||||
sizes only after measuring contention. The shared Relay budget excludes push entirely.
|
||||
|
||||
Authentication is the host proof in `POST /v1/host/challenge`, not Cloud Run IAM, so the service
|
||||
opts out of invoker IAM with `invoker_iam_disabled = true`, exactly as the relay director does.
|
||||
The project's domain-restricted-sharing policy refuses an `allUsers` invoker binding, so that is
|
||||
the only way to reach an open service here.
|
||||
|
||||
## Environment
|
||||
|
||||
Set on the container by Terraform:
|
||||
|
||||
| Variable | Source |
|
||||
| ----------------------------- | -------------------------------------------------------- |
|
||||
| `PORT` | Cloud Run, container port 8080 |
|
||||
| `ORCA_PUSH_PUBLIC_URL` | `push_base_url` |
|
||||
| `ORCA_PUSH_FCM_PROJECT_ID` | `project_id` (required for standalone runtime) |
|
||||
| `ORCA_PUSH_DATABASE_URL` | Secret `orca-cloud-push-dedicated-database-url`, pinned version |
|
||||
| `ORCA_PUSH_DATABASE_POOL_MAX` | `push_database_pool_max`, 2 per instance |
|
||||
| `ORCA_PUSH_APNS_KEY` | Secret `orca-cloud-push-apns-key`, version `latest` |
|
||||
| `ORCA_PUSH_APNS_KEY_ID` | Secret `orca-cloud-push-apns-key-id`, version `latest` |
|
||||
| `ORCA_PUSH_APPLE_TEAM_ID` | Secret `orca-cloud-push-apple-team-id`, version `latest` |
|
||||
|
||||
`ORCA_PUSH_APNS_TOPIC` is left to its application default (`com.stably.orca.mobile`). Add it here
|
||||
only when it has to differ from the code default, so that a code-side change stays visible rather
|
||||
than silently overridden.
|
||||
|
||||
Terraform owns the three Apple secret **names, labels, and replication, and never a version.**
|
||||
The `.p8` is issued by the Apple developer portal, so a Terraform-managed version would put the
|
||||
private key in state and would fight the rotation below. The database URL secret is different:
|
||||
Terraform generates that password, so it owns that version, exactly as `relay-database.tf` does.
|
||||
That puts the generated password and the full database URL in the state bucket, which the shared
|
||||
deploy identity can read; the Apple key never appears there. The three Apple secrets and the
|
||||
`orca_push` database carry `prevent_destroy`, so disabling the gateway fails the plan instead
|
||||
of deleting the only copy of the signing key or every live device token.
|
||||
|
||||
## Importing what already exists
|
||||
|
||||
The runtime account, the three Apple secrets, and their accessor bindings were created out of
|
||||
band alongside the Apple credentials. They are declared so a plan is clean, and imported once.
|
||||
Run these from `cloud/` after `pnpm infra:init --env production`, review the resulting plan, and
|
||||
expect the imported resources to show no changes.
|
||||
|
||||
```sh
|
||||
terraform -chdir=infra/terraform import -var-file=environments/production.tfvars \
|
||||
'google_service_account.push_runtime[0]' \
|
||||
projects/onorca-cloud/serviceAccounts/orca-cloud-push@onorca-cloud.iam.gserviceaccount.com
|
||||
|
||||
terraform -chdir=infra/terraform import -var-file=environments/production.tfvars \
|
||||
'google_project_iam_member.push_runtime_fcm_admin[0]' \
|
||||
'onorca-cloud roles/firebasecloudmessaging.admin serviceAccount:orca-cloud-push@onorca-cloud.iam.gserviceaccount.com'
|
||||
|
||||
terraform -chdir=infra/terraform import -var-file=environments/production.tfvars \
|
||||
'google_project_iam_member.push_runtime_service_usage_consumer[0]' \
|
||||
'onorca-cloud roles/serviceusage.serviceUsageConsumer serviceAccount:orca-cloud-push@onorca-cloud.iam.gserviceaccount.com'
|
||||
|
||||
terraform -chdir=infra/terraform import -var-file=environments/production.tfvars \
|
||||
'google_secret_manager_secret.push_provider["orca-cloud-push-apns-key"]' \
|
||||
projects/onorca-cloud/secrets/orca-cloud-push-apns-key
|
||||
|
||||
terraform -chdir=infra/terraform import -var-file=environments/production.tfvars \
|
||||
'google_secret_manager_secret.push_provider["orca-cloud-push-apns-key-id"]' \
|
||||
projects/onorca-cloud/secrets/orca-cloud-push-apns-key-id
|
||||
|
||||
terraform -chdir=infra/terraform import -var-file=environments/production.tfvars \
|
||||
'google_secret_manager_secret.push_provider["orca-cloud-push-apple-team-id"]' \
|
||||
projects/onorca-cloud/secrets/orca-cloud-push-apple-team-id
|
||||
|
||||
terraform -chdir=infra/terraform import -var-file=environments/production.tfvars \
|
||||
'google_secret_manager_secret_iam_member.push_provider_runtime_accessor["orca-cloud-push-apns-key"]' \
|
||||
'projects/onorca-cloud/secrets/orca-cloud-push-apns-key roles/secretmanager.secretAccessor serviceAccount:orca-cloud-push@onorca-cloud.iam.gserviceaccount.com'
|
||||
|
||||
terraform -chdir=infra/terraform import -var-file=environments/production.tfvars \
|
||||
'google_secret_manager_secret_iam_member.push_provider_runtime_accessor["orca-cloud-push-apns-key-id"]' \
|
||||
'projects/onorca-cloud/secrets/orca-cloud-push-apns-key-id roles/secretmanager.secretAccessor serviceAccount:orca-cloud-push@onorca-cloud.iam.gserviceaccount.com'
|
||||
|
||||
terraform -chdir=infra/terraform import -var-file=environments/production.tfvars \
|
||||
'google_secret_manager_secret_iam_member.push_provider_runtime_accessor["orca-cloud-push-apple-team-id"]' \
|
||||
'projects/onorca-cloud/secrets/orca-cloud-push-apple-team-id roles/secretmanager.secretAccessor serviceAccount:orca-cloud-push@onorca-cloud.iam.gserviceaccount.com'
|
||||
```
|
||||
|
||||
The push resources already exist in production. Preserve their addresses, dedicated database
|
||||
and identities; review the [database cleanup runbook](./push-database-cutover.md) before applying
|
||||
changes. This root has unrelated standing drift, so an untargeted apply is never automatic.
|
||||
|
||||
Two things this root does **not** declare, because the carve assigns them elsewhere. Neither
|
||||
affects whether this root's plan is clean, since an undeclared resource is invisible to it.
|
||||
|
||||
- `firebase.googleapis.com` and `fcm.googleapis.com` are project service enablement, which is
|
||||
`google_project_service.required` in the foundation root. They are already enabled; add them
|
||||
to the foundation root's list so a foundation plan stays clean.
|
||||
- The Firebase attachment on `onorca-cloud` is project-level and belongs with foundation for the
|
||||
same reason. It exists already.
|
||||
|
||||
## Deploying
|
||||
|
||||
`Deploy Push Gateway Production` (`.github/workflows/cloud-push-deploy.yml`) is the only
|
||||
supported path. Like every `cloud-*` workflow it does nothing until `ORCA_CLOUD_OPERATIONS_ENABLED`
|
||||
is `true`, it runs only on `main`, and it needs the confirmation string `DEPLOY_PUSH_GATEWAY`.
|
||||
|
||||
It authenticates as the dedicated `orca-cloud-gha-push` identity through
|
||||
`PRODUCTION_GCP_PUSH_DEPLOY_WORKLOAD_IDENTITY_PROVIDER` and
|
||||
`PRODUCTION_GCP_PUSH_DEPLOY_SERVICE_ACCOUNT`. `push-deploy-identity.tf` restricts Workload Identity
|
||||
to this exact dispatch workflow on main in the production environment. Its distinct principal
|
||||
attribute cannot assume the shared Relay deploy identity.
|
||||
|
||||
The account can write images to Artifact Registry, deploy the push service, impersonate only
|
||||
the push runtime account, and manage exactly `terraform/state/push-rollout/production.lock`
|
||||
in the production state bucket. The relay root owns that conditional lease grant. It grants
|
||||
no Terraform-state object access. Publish `github_push_workload_identity_provider` and
|
||||
`github_push_deploy_service_account` as the production-environment variables above.
|
||||
|
||||
The workflow uses the `production-push-rollout` concurrency group with cancellation disabled
|
||||
and the existing durable lease action on the push-specific object. Push and Relay deploy
|
||||
independently; two push deploys cannot race traffic changes. Finish every old shared-lock push
|
||||
run before enabling the new workflow and lease grant. See the cleanup runbook for the bounded
|
||||
IAM transition and removal of any obsolete foundation-owned push membership.
|
||||
|
||||
The run builds the reviewed `source_sha` while the workflow stays on `main`. Buildx returns
|
||||
its own pushed digest (no mutable-tag lookup); every subsequent check and deployment uses that
|
||||
same digest. Before any production boot, a network-isolated container checks that the image
|
||||
recognizes `ORCA_PUSH_MODE=validation` and rejects invalid modes. Older images that lack this
|
||||
capability are refused before they can connect to production.
|
||||
|
||||
Under the production push rollout lease, it records the serving rollback revision and
|
||||
asserts Terraform-owned scaling. It deploys a tagged, zero-traffic validation revision:
|
||||
|
||||
- Validation opens PostgreSQL with `default_transaction_read_only=on` and skips schema setup.
|
||||
- No delivery worker or challenge, session, or delivery pruner starts.
|
||||
- Only `/health` and `/ready` are available; all application routes return 503.
|
||||
- `/health` attests `mode: validation`; `/ready` checks database connectivity only. It does not
|
||||
prove schema compatibility, provider delivery, or active-worker readiness. Container probes
|
||||
can still use `/health` without treating an inert process as unhealthy.
|
||||
|
||||
The build explicitly targets `linux/amd64` with provenance disabled so build metadata records a
|
||||
single manifest digest, rather than an OCI index that Cloud Run resolves to a different digest.
|
||||
The workflow verifies the exact image and scaling, probes readiness and mode, and checks the
|
||||
runtime identity with a validate-only FCM request. Cloud Run rejects deletion of the latest
|
||||
created revision even when it has no tag or traffic. Activation therefore creates a successor
|
||||
before removing the validation tag and deleting validation. The dedicated 64-connection budget
|
||||
reserves three simultaneous revision pools: serving, validation/rejected,
|
||||
and active/recovery successor (12 configured pool connections at the current two-by-two shape).
|
||||
Revision deletion is not proof of physical SQL session drain; verify termination and SQL sessions
|
||||
in controlled rollout acceptance. There is no shutdown sleep used as a drain gate.
|
||||
|
||||
**Activation deliberately starts production effects.** The distinct active revision uses the exact
|
||||
validated digest with the validation override removed. Schema setup runs on its existing
|
||||
one-connection untimed pool, followed by workers and pruners, before HTTP promotion. The workflow
|
||||
checks digest, full runtime spec and secret-reference shape, scaling, readiness and active mode,
|
||||
then moves HTTP traffic and checks the public origin. Those checks commit the new serving revision;
|
||||
subsequent retirement failures do not trigger rollback to a possibly deleted previous revision.
|
||||
The previous consumer is retired and all tags are cleared. Retain the previous immutable image
|
||||
from the summary: later recovery redeploys that digest, because the previous revision is deleted.
|
||||
|
||||
Before any candidate creation, the workflow requires exactly one revision resource, the sole HTTP
|
||||
serving revision. Existing historical revisions or leftovers from interrupted runs require explicit
|
||||
operator review and cleanup under the lease first; the workflow does not blindly delete them.
|
||||
This gate and retirement after every successful rollout prevent repeated runs accumulating workers.
|
||||
Terraform still owns configuration and scaling; removing validation mode adds no ignored field.
|
||||
|
||||
On failure before public checks pass, any attempted traffic shift is first rolled back and verified.
|
||||
If partial activation created a successor, recovery retires non-latest validation first; deletion
|
||||
failure stops recovery before a fourth resource can be created. Recovery then deploys the captured
|
||||
known-good digest as a tagged, zero-traffic successor with normal mode. It verifies template shape,
|
||||
secret references and scaling, probes tagged readiness and active mode, promotes the recovery
|
||||
revision, verifies traffic and public health, and only then deletes rejected and previous revisions.
|
||||
The latest recovery revision remains serving. Known-good recovery schema and workers can execute
|
||||
before promotion; neither recovery nor traffic rollback undoes schema changes or sent notifications.
|
||||
|
||||
Partial creates record deterministic names before mutation. Failed recovery or deletion requires
|
||||
operator cleanup under the lease; the next automated run refuses leftover resources. A canceled
|
||||
runner can require the same intervention. Traffic restoration alone does not stop queue consumers.
|
||||
|
||||
Manual recovery must preserve the three-resource bound and keep the successor serving:
|
||||
|
||||
```sh
|
||||
# Hold the rollout lease; inspect latest, traffic, tags and existing revisions first.
|
||||
# If three resources remain after partial activation, retire non-latest inert validation first.
|
||||
# Restore previous traffic if its revision still exists and a failed candidate took traffic.
|
||||
gcloud run deploy orca-cloud-push \
|
||||
--project onorca-cloud --region us-central1 --image <known-good-image-at-digest> \
|
||||
--remove-env-vars ORCA_PUSH_MODE --no-traffic \
|
||||
--tag <unique-recovery-tag> --revision-suffix <unique-recovery-suffix>
|
||||
# Verify exact digest, template spec/secret references/scaling, tagged /ready and active /health.
|
||||
gcloud run services update-traffic orca-cloud-push \
|
||||
--project onorca-cloud --region us-central1 --to-revisions <recovery-revision>=100
|
||||
# Verify traffic and public /ready and /health before retiring old consumers.
|
||||
gcloud run services update-traffic orca-cloud-push \
|
||||
--project onorca-cloud --region us-central1 --clear-tags
|
||||
gcloud run revisions delete <rejected-or-previous-revision> \
|
||||
--project onorca-cloud --region us-central1
|
||||
# Repeat only for reviewed obsolete revisions; retain the latest serving recovery revision.
|
||||
```
|
||||
|
||||
Never merely remove validation mode while the template still holds a rejected image. Terraform
|
||||
owns environment configuration but ignores the image, so that would activate rejected code.
|
||||
Remove a tag only if it remains present. Verify the recovery revision is serving, the template is safe,
|
||||
and obsolete revision deletion and connection drain completed;
|
||||
already accepted provider sends cannot be undone. Activation-time schema changes must be additive
|
||||
and compatible with the rollback image: rollback does not reverse migrations or queue mutations.
|
||||
The inert phase intentionally cannot validate a new schema by applying it to production. Review
|
||||
migrations and validate them against isolated PostgreSQL before dispatch. No actual Cloud Run
|
||||
rollout, provider delivery or physical-device acceptance is implied by local contract tests.
|
||||
|
||||
### Incompatible queue rollout prerequisite
|
||||
|
||||
The queue stores one notification object per delivery. Before deploying a revision that changes this
|
||||
format, stop every older push gateway revision and clear only unpublished push delivery fixtures from
|
||||
the push database. This is an unpublished feature, so do not preserve or migrate queued fixtures; no
|
||||
production mutation is implied by this prerequisite.
|
||||
|
||||
### Why the FCM probe impersonates the runtime account
|
||||
|
||||
A gateway that boots and answers `/ready` can still be unable to send: the FCM grant lives on
|
||||
the runtime service account, not on anything the readiness check touches. The probe therefore
|
||||
mints an access token for `orca-cloud-push@onorca-cloud.iam.gserviceaccount.com` and posts
|
||||
`validate_only: true` with a token that cannot exist. `validate_only` stops Google before any
|
||||
delivery, and a healthy credential answers `INVALID_ARGUMENT` because the device token is
|
||||
garbage. `PERMISSION_DENIED`, `401`, and `403` are the failures the step exists to catch, and
|
||||
they fail the run immediately, before traffic moves. Those four answers are the only conclusive
|
||||
ones: a `429`, a `5xx`, or a transport failure says nothing about the credential, so the send is
|
||||
retried up to five times at five-second intervals rather than read as either verdict. Probing as the deploy identity instead would prove
|
||||
something true about the wrong account.
|
||||
|
||||
## Rotating the APNs key
|
||||
|
||||
Apple keys do not expire, so this is for a suspected compromise or a routine rotation. Order
|
||||
matters: the new key must be serving before the old one is revoked, or every iOS push fails in
|
||||
the window between.
|
||||
|
||||
1. In the Apple developer portal, create a **new** APNs authentication key. Download the `.p8`
|
||||
once; Apple will not show it again. Note the new key ID. A team may hold two APNs keys at a
|
||||
time, which is what makes this overlap possible.
|
||||
2. Add a version to each changed secret, without printing the value:
|
||||
|
||||
```sh
|
||||
gcloud secrets versions add orca-cloud-push-apns-key \
|
||||
--project onorca-cloud --data-file /path/to/AuthKey_NEW.p8
|
||||
printf '%s' '<new key id>' | gcloud secrets versions add orca-cloud-push-apns-key-id \
|
||||
--project onorca-cloud --data-file=-
|
||||
```
|
||||
|
||||
The team ID does not change, so `orca-cloud-push-apple-team-id` is untouched.
|
||||
|
||||
3. Dispatch `Deploy Push Gateway Production`. The container reads `latest` at start, so only a
|
||||
new revision picks the key up; there is no in-place reload.
|
||||
4. Verify from a real device that an iOS notification still arrives. The workflow's FCM probe
|
||||
covers Android only, and APNs has no validate-only equivalent.
|
||||
5. Only then revoke the old key in the Apple portal, and disable the superseded secret versions:
|
||||
|
||||
```sh
|
||||
gcloud secrets versions disable <old-version> \
|
||||
--project onorca-cloud --secret orca-cloud-push-apns-key
|
||||
```
|
||||
|
||||
Disable rather than destroy, so a rollback to the previous revision still works. Destroy
|
||||
after the next clean deploy.
|
||||
|
||||
Delete the downloaded `.p8` from disk when you are done. It is the whole credential.
|
||||
|
||||
## Dead tokens
|
||||
|
||||
A push token stops working when the app is uninstalled, when the user restores to a new device,
|
||||
or when iOS reissues it. Both providers report this, and the shapes differ:
|
||||
|
||||
- APNs: HTTP 410, or 400 with `BadDeviceToken` or `Unregistered`.
|
||||
`DeviceTokenNotForTopic` is a provider configuration error and leaves the registration live.
|
||||
Check the APNs topic and environment; future notifications can resume after correction without
|
||||
phone re-registration. The failed notification is not retried for this non-transient error.
|
||||
- FCM: `UNREGISTERED`, or `INVALID_ARGUMENT` whose message names the token.
|
||||
|
||||
The gateway marks the registration `dead_at` and returns `status: "dead"` for it, and the
|
||||
desktop drops the registration when it sees that. Nothing here retries a dead token. A phone
|
||||
that comes back re-registers the same host/device pair, retaining its `registrationId` and
|
||||
clearing `dead_at`. The per-minute `delivery_dead` counter measures delivery outcomes, not
|
||||
currently dead registrations. A spike across many hosts warrants checking credentials and topics.
|
||||
|
||||
## Quotas
|
||||
|
||||
Two independent limits, both enforced in the gateway and both returning HTTP 200 with
|
||||
`status: "rate_limited"` per result rather than failing the request:
|
||||
|
||||
| Limit | Scope |
|
||||
| --------------------------------------------- | --------------------------------------- |
|
||||
| 300 logical alerts per rolling 15 minutes | per `hostFingerprint` |
|
||||
| 300 logical dismissals per rolling 15 minutes | per `hostFingerprint`, separate budget |
|
||||
| 20 `registrationIds` | per request, hard cap, HTTP 400 over it |
|
||||
|
||||
Fanout to several phones counts one logical event; there is no per-phone daily allowance.
|
||||
Unauthenticated handshakes and invalid bearer attempts have separate 30/minute IP buckets.
|
||||
Authenticated routes use a 600/minute host bucket and a shared 6,000/minute client-IP bucket
|
||||
per instance. The IP budget cannot be reset by generating another host key. It is shared by
|
||||
clients behind one NAT and is an abuse safeguard, not a global provider-spending cap. Auth database lookup concurrency
|
||||
and waiting work are bounded independently of HTTP concurrency.
|
||||
|
||||
`push_events` backs quota accounting. `push_event_recipients` deduplicates fanout and
|
||||
`push_delivery_batches` retains its historical name and persists individual deliveries, worker
|
||||
leases, retries and outcomes. Identity metadata
|
||||
is retained for 24 hours. Payloads expire within five minutes and are cleared on completion or by
|
||||
minute-level expiry cleanup. FCM project-level provider quotas remain independent of host limits.
|
||||
|
||||
Logging is aggregate counters only. Never log a token, a title, a body, or a full fingerprint;
|
||||
the first four characters of a fingerprint are the most that may appear.
|
||||
|
||||
## DNS: one hand-managed record
|
||||
|
||||
The Cloud Run domain mapping is created here, and Google issues and renews the certificate. The
|
||||
`onorca.dev` zone is not in this root: it is a Cloudflare zone whose Terraform-managed records
|
||||
live in the apps root in `stablyai/orca-cloud`, and whose relay and auth records are managed by
|
||||
hand. The push record follows the relay's precedent and was created by hand on 2026-09-04:
|
||||
|
||||
```text
|
||||
push.onorca.dev. CNAME ghs.googlehosted.com. (DNS only, not proxied)
|
||||
```
|
||||
|
||||
`terraform -chdir=infra/terraform output push_dns_record` prints the same three fields. If the
|
||||
record is ever lost, recreate it exactly like that; Cloudflare proxying blocks certificate
|
||||
issuance and breaks Cloud Run host routing.
|
||||
|
||||
### Recovery and delivery guarantees
|
||||
|
||||
Candidate tags and deterministic revision names are recorded before deployment. Promotion intent is
|
||||
recorded before changing traffic, so a failed verification or ambiguous mutation result still triggers
|
||||
rollback. A known-good successor must exist before the rejected latest revision can be deleted.
|
||||
After verified recovery promotion and public checks, rejected and previous consumers are retired;
|
||||
the recovery revision remains serving. Failed cleanup blocks subsequent rollout admission.
|
||||
The summary runs even if candidate discovery or traffic verification fails.
|
||||
|
||||
Push uses the relay's schema-startup retry implementation through `@orca-cloud/postgres-schema`.
|
||||
Session replacement is serialized per host and a unique host index upgrades older databases by
|
||||
retaining their newest session. Cloud Verify runs push concurrency tests against PostgreSQL.
|
||||
|
||||
Accepted sends commit quota and pending work together before returning `queued`. Workers resume
|
||||
unfinished deliveries after restarts without relying on desktop retries. The durable queue and
|
||||
expiring leases coordinate replicas. All provider attempts retain the original five-minute deadline
|
||||
and respect provider backoff; no retry extends alert life. Silent dismissal messages have their own
|
||||
quota and cancel matching unsent alerts. Mobile OS delivery/execution is not guaranteed.
|
||||
|
||||
Shutdown stops admission and new claims; unfinished leases remain recoverable. Provider acceptance
|
||||
and SQL completion cannot be atomic, so repeated transport delivery remains possible after a crash.
|
||||
Stable per-event replacement identities reduce duplicates without promising exactly-once visible
|
||||
delivery. FCM notification messages are inherently collapsible while offline and support only a
|
||||
small number of concurrent collapse keys per device, so excess pending messages may be discarded and
|
||||
every offline alert is not guaranteed to appear. Socket reconnect reconciles dismissals against the
|
||||
current native tray; it has no stored replay watermark and never recovers a missed OS banner.
|
||||
|
||||
### Dedicated database operations
|
||||
|
||||
Push has one dedicated database attachment, with stable Terraform addresses and deletion
|
||||
protection. There is no switch to shared storage. Follow the [database operations runbook](./push-database-cutover.md)
|
||||
for deployment prerequisites, legacy resource ownership, capacity and recovery.
|
||||
@@ -400,3 +400,30 @@ after checkout and authentication, before package installation, revision checks,
|
||||
Their typed confirmations are `PAUSE_REGIONAL_REHOMING` and `DISABLE_REGIONAL_REHOMING`. Keep the
|
||||
default 3,600,000 ms drain grace so existing splices can finish. The job summary contains only fresh
|
||||
aggregate active, receipt, registration, completion, and abort counts.
|
||||
|
||||
## Mobile push gateway
|
||||
|
||||
`Deploy Push Gateway Production` (`.github/workflows/cloud-push-deploy.yml`) is the deploy path
|
||||
for `orca-cloud-push`, the mobile push gateway. It is the one `cloud-*` workflow that is not a
|
||||
relay operation, and it is here because it shares the Artifact Registry repository and rollout
|
||||
lease. Push uses a dedicated Cloud SQL instance.
|
||||
|
||||
It authenticates through `PRODUCTION_GCP_PUSH_DEPLOY_WORKLOAD_IDENTITY_PROVIDER` and
|
||||
`PRODUCTION_GCP_PUSH_DEPLOY_SERVICE_ACCOUNT`. The dedicated identity has Artifact Registry writer,
|
||||
Cloud Run developer on the push service, and impersonation of only the push runtime account.
|
||||
Its provider pins the repository, production environment, main branch, and exact dispatch workflow;
|
||||
its distinct principal attribute cannot assume the shared Relay deploy account.
|
||||
|
||||
Foundation grants the dedicated account access to the rollout-lock prefix and bucket metadata.
|
||||
Apply that companion grant and publish the identity outputs before running the workflow. See
|
||||
[push gateway deployment setup](./push-gateway.md#deploying) for the activation steps.
|
||||
|
||||
The run builds the exact reviewed image digest before taking the lease and rejects images
|
||||
without validation-mode support using a network-isolated container. Under the lease it boots
|
||||
an inert, read-only validation revision, checks readiness, mode, scaling and FCM credentials,
|
||||
then deletes it before deliberately activating the same digest in a new revision. Activation
|
||||
starts schema writes, pruners and queue consumers before HTTP promotion. Rollback requires
|
||||
restoring traffic, deleting the rejected active revision, and restoring the service template to
|
||||
the known-good image in normal mode. The untagged template-recovery revision can run known-good
|
||||
workers and is retired before lease release; see the
|
||||
[deployment and rollback contract](./push-gateway.md#deploying).
|
||||
|
||||
@@ -412,3 +412,12 @@ relay_region_rehome_source_cell_ids = [
|
||||
# Slack #orca-relay-alerts, created out of band on 2026-08-05. Declared here because an apply
|
||||
# was otherwise going to strip it from every policy, leaving the alerts firing at nobody.
|
||||
relay_alert_notification_channels = ["projects/onorca-cloud/notificationChannels/4879431412695417284"]
|
||||
|
||||
# Mobile push gateway. Production is the only environment that runs one; the runtime account,
|
||||
# the three Apple secrets, and their accessor bindings already exist and are imported once
|
||||
# (see docs/push-gateway.md).
|
||||
push_gateway_enabled = true
|
||||
push_base_url = "https://push.onorca.dev"
|
||||
# Dedicated push pools allow three revision resources during validation and recovery.
|
||||
push_max_instances = 2
|
||||
manage_push_domain_mapping = true
|
||||
|
||||
@@ -81,3 +81,6 @@ relay_gce_cells = {
|
||||
}
|
||||
|
||||
relay_region_rehome_source_cell_ids = ["staging-gce-c2", "staging-gce-c3"]
|
||||
|
||||
# Push is currently provisioned only in production.
|
||||
push_gateway_enabled = false
|
||||
|
||||
@@ -189,3 +189,27 @@ output "relay_gce_cell_deployments" {
|
||||
error_message = "relay_gce_fenced_cells may contain only configured relay_gce_cells keys."
|
||||
}
|
||||
}
|
||||
|
||||
output "push_cloud_run_service_uri" {
|
||||
value = try(google_cloud_run_v2_service.push[0].uri, null)
|
||||
description = "Default push gateway service URI for pre-domain smoke tests."
|
||||
}
|
||||
|
||||
output "push_runtime_service_account" {
|
||||
value = try(google_service_account.push_runtime[0].email, null)
|
||||
description = "Runtime identity that holds the APNs key and sends through FCM."
|
||||
}
|
||||
|
||||
output "push_database_name" {
|
||||
value = try(google_sql_database.push_dedicated[0].name, null)
|
||||
description = "Database isolated for durable push gateway state."
|
||||
}
|
||||
|
||||
output "push_dns_record" {
|
||||
value = var.push_gateway_enabled ? {
|
||||
name = local.push_fqdn
|
||||
type = "CNAME"
|
||||
data = "ghs.googlehosted.com."
|
||||
} : null
|
||||
description = "Record the stablyai/orca-cloud apps root must publish in the onorca.dev zone."
|
||||
}
|
||||
|
||||
@@ -0,0 +1,111 @@
|
||||
resource "google_sql_database_instance" "push_dedicated" {
|
||||
count = local.push_gateway_count
|
||||
|
||||
project = var.project_id
|
||||
name = "${var.name_prefix}-push-db"
|
||||
region = var.region
|
||||
database_version = "POSTGRES_17"
|
||||
deletion_protection = true
|
||||
|
||||
settings {
|
||||
tier = "db-custom-2-7680"
|
||||
availability_type = "REGIONAL"
|
||||
edition = "ENTERPRISE"
|
||||
disk_type = "PD_SSD"
|
||||
disk_size = 50
|
||||
disk_autoresize = true
|
||||
user_labels = local.relay_shared_labels
|
||||
|
||||
backup_configuration {
|
||||
enabled = true
|
||||
point_in_time_recovery_enabled = true
|
||||
transaction_log_retention_days = 7
|
||||
start_time = "05:00"
|
||||
backup_retention_settings {
|
||||
retained_backups = 7
|
||||
}
|
||||
}
|
||||
|
||||
ip_configuration {
|
||||
ipv4_enabled = true
|
||||
ssl_mode = "ENCRYPTED_ONLY"
|
||||
}
|
||||
|
||||
maintenance_window {
|
||||
day = 7
|
||||
hour = 6
|
||||
update_track = "stable"
|
||||
}
|
||||
|
||||
deletion_protection_enabled = true
|
||||
}
|
||||
|
||||
lifecycle {
|
||||
prevent_destroy = true
|
||||
}
|
||||
}
|
||||
|
||||
resource "google_sql_database" "push_dedicated" {
|
||||
count = local.push_gateway_count
|
||||
|
||||
project = var.project_id
|
||||
name = "orca_push"
|
||||
instance = google_sql_database_instance.push_dedicated[0].name
|
||||
|
||||
lifecycle {
|
||||
prevent_destroy = true
|
||||
}
|
||||
}
|
||||
|
||||
resource "random_password" "push_dedicated_database" {
|
||||
count = local.push_gateway_count
|
||||
length = 32
|
||||
special = false
|
||||
}
|
||||
|
||||
resource "google_sql_user" "push_dedicated" {
|
||||
count = local.push_gateway_count
|
||||
|
||||
project = var.project_id
|
||||
name = "orca_push"
|
||||
instance = google_sql_database_instance.push_dedicated[0].name
|
||||
password = random_password.push_dedicated_database[0].result
|
||||
}
|
||||
|
||||
resource "google_secret_manager_secret" "push_dedicated_database_url" {
|
||||
count = local.push_gateway_count
|
||||
|
||||
project = var.project_id
|
||||
secret_id = "${var.name_prefix}-push-dedicated-database-url"
|
||||
labels = local.relay_shared_labels
|
||||
|
||||
replication {
|
||||
auto {}
|
||||
}
|
||||
|
||||
lifecycle {
|
||||
prevent_destroy = true
|
||||
}
|
||||
}
|
||||
|
||||
resource "google_secret_manager_secret_version" "push_dedicated_database_url" {
|
||||
count = local.push_gateway_count
|
||||
|
||||
secret = google_secret_manager_secret.push_dedicated_database_url[0].id
|
||||
secret_data = format(
|
||||
"postgresql://%s:%s@/%s?host=/cloudsql/%s",
|
||||
google_sql_user.push_dedicated[0].name,
|
||||
random_password.push_dedicated_database[0].result,
|
||||
google_sql_database.push_dedicated[0].name,
|
||||
google_sql_database_instance.push_dedicated[0].connection_name
|
||||
)
|
||||
}
|
||||
|
||||
resource "google_secret_manager_secret_iam_member" "push_dedicated_database_url_accessor" {
|
||||
count = local.push_gateway_count
|
||||
|
||||
project = var.project_id
|
||||
secret_id = google_secret_manager_secret.push_dedicated_database_url[0].secret_id
|
||||
role = "roles/secretmanager.secretAccessor"
|
||||
member = google_service_account.push_runtime[0].member
|
||||
}
|
||||
@@ -0,0 +1,82 @@
|
||||
locals {
|
||||
# Deployment uses its own production-only identity.
|
||||
push_gateway_deploy_count = (
|
||||
var.push_gateway_enabled && local.relay_create_production_ops_identity ? 1 : 0
|
||||
)
|
||||
github_push_workflow_clauses = [
|
||||
for prefix in local.relay_github_workflow_ref_prefixes :
|
||||
"assertion.workflow_ref == '${prefix}push-deploy.yml@refs/heads/main' && assertion.job_workflow_ref == '${prefix}push-deploy.yml@refs/heads/main'"
|
||||
]
|
||||
push_deploy_member = one(google_service_account.github_push_deploy[*].member)
|
||||
}
|
||||
|
||||
resource "google_service_account" "github_push_deploy" {
|
||||
count = local.push_gateway_deploy_count
|
||||
account_id = "${var.name_prefix}-gha-push"
|
||||
display_name = "Orca push production deploy"
|
||||
}
|
||||
|
||||
resource "google_iam_workload_identity_pool_provider" "github_push" {
|
||||
count = local.push_gateway_deploy_count
|
||||
|
||||
project = var.project_id
|
||||
workload_identity_pool_id = local.relay_workload_identity_pool_id
|
||||
workload_identity_pool_provider_id = "github-push-deploy"
|
||||
display_name = "GitHub push production deploy"
|
||||
# Do not map attribute.repository: that principal set can assume the shared deploy account.
|
||||
attribute_mapping = {
|
||||
"google.subject" = "assertion.sub"
|
||||
"attribute.push_deploy" = "'production'"
|
||||
}
|
||||
attribute_condition = join(" && ", concat(local.relay_github_leading_repository_claims, [
|
||||
"assertion.ref == 'refs/heads/main'",
|
||||
"assertion.environment == 'production'",
|
||||
"assertion.event_name == 'workflow_dispatch'",
|
||||
local.relay_github_workflow_conditions["github_push"]
|
||||
]))
|
||||
|
||||
oidc {
|
||||
issuer_uri = "https://token.actions.githubusercontent.com"
|
||||
}
|
||||
}
|
||||
|
||||
resource "google_service_account_iam_member" "github_push_workload_identity_user" {
|
||||
count = local.push_gateway_deploy_count
|
||||
|
||||
service_account_id = google_service_account.github_push_deploy[0].name
|
||||
role = "roles/iam.workloadIdentityUser"
|
||||
member = "principalSet://iam.googleapis.com/${local.relay_workload_identity_pool_name}/attribute.push_deploy/production"
|
||||
}
|
||||
|
||||
resource "google_artifact_registry_repository_iam_member" "github_push_artifact_writer" {
|
||||
count = local.push_gateway_deploy_count
|
||||
|
||||
project = var.project_id
|
||||
location = var.region
|
||||
repository = var.artifact_repository_id
|
||||
role = "roles/artifactregistry.writer"
|
||||
member = local.push_deploy_member
|
||||
}
|
||||
|
||||
output "github_push_workload_identity_provider" {
|
||||
value = try(google_iam_workload_identity_pool_provider.github_push[0].name, null)
|
||||
}
|
||||
|
||||
output "github_push_deploy_service_account" {
|
||||
value = try(google_service_account.github_push_deploy[0].email, null)
|
||||
}
|
||||
|
||||
|
||||
resource "google_storage_bucket_iam_member" "github_push_rollout_lease" {
|
||||
count = local.push_gateway_deploy_count
|
||||
|
||||
bucket = "${var.project_id}-terraform-state"
|
||||
role = "roles/storage.objectAdmin"
|
||||
member = local.push_deploy_member
|
||||
|
||||
condition {
|
||||
title = "push_rollout_lease"
|
||||
description = "Limits push deployment coordination to its own lease object."
|
||||
expression = "resource.name == 'projects/_/buckets/${var.project_id}-terraform-state/objects/terraform/state/push-rollout/production.lock'"
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,305 @@
|
||||
# Orca mobile push gateway (`cloud/apps/push`).
|
||||
#
|
||||
# One public Cloud Run service that holds the APNs key and sends through APNs and FCM V1 on
|
||||
# behalf of paired phones. Operations: `docs/push-gateway.md`.
|
||||
#
|
||||
# There is no staging push gateway by decision, so every resource here is behind
|
||||
# `var.push_gateway_enabled`, which only `environments/production.tfvars` sets true. The file
|
||||
# still reads every environment-shaped value from a variable, like the rest of this root, so a
|
||||
# future staging gateway is a tfvars edit rather than a rewrite.
|
||||
#
|
||||
# Several resources below already exist in `onorca-cloud`; they are declared so a plan is clean
|
||||
# and imported once. `docs/push-gateway.md` carries the exact `terraform import` commands.
|
||||
|
||||
locals {
|
||||
push_gateway_count = var.push_gateway_enabled ? 1 : 0
|
||||
|
||||
# The runtime account, the three provider secrets, and their accessor bindings already exist in
|
||||
# production and were created out of band with the Apple credentials.
|
||||
push_runtime_service_account_id = "${var.name_prefix}-push"
|
||||
|
||||
# Secret Manager holds the Apple credentials. Terraform owns the secret names, labels, and
|
||||
# replication; it never owns a version. The `.p8` is issued by the Apple developer portal and
|
||||
# rotated by `docs/push-gateway.md`, so a Terraform-managed version would either put the key in
|
||||
# state or fight the rotation. `ignore_changes` on the whole resource is not available, so the
|
||||
# versions are simply not declared and every consumer reads `latest`.
|
||||
push_provider_secret_ids = var.push_gateway_enabled ? toset([
|
||||
"${var.name_prefix}-push-apns-key",
|
||||
"${var.name_prefix}-push-apns-key-id",
|
||||
"${var.name_prefix}-push-apple-team-id"
|
||||
]) : toset([])
|
||||
|
||||
push_provider_secret_env = {
|
||||
"${var.name_prefix}-push-apns-key" = "ORCA_PUSH_APNS_KEY"
|
||||
"${var.name_prefix}-push-apns-key-id" = "ORCA_PUSH_APNS_KEY_ID"
|
||||
"${var.name_prefix}-push-apple-team-id" = "ORCA_PUSH_APPLE_TEAM_ID"
|
||||
}
|
||||
|
||||
push_fqdn = replace(replace(var.push_base_url, "https://", ""), "http://", "")
|
||||
|
||||
|
||||
}
|
||||
|
||||
# --- Runtime identity ---------------------------------------------------------------------
|
||||
|
||||
resource "google_service_account" "push_runtime" {
|
||||
count = local.push_gateway_count
|
||||
|
||||
project = var.project_id
|
||||
account_id = local.push_runtime_service_account_id
|
||||
display_name = "Orca mobile push gateway"
|
||||
description = "Runtime identity for the Orca mobile push gateway; sends through FCM V1."
|
||||
}
|
||||
|
||||
# FCM V1 sends are authorized by the runtime account's own metadata-server token.
|
||||
resource "google_project_iam_member" "push_runtime_fcm_admin" {
|
||||
count = local.push_gateway_count
|
||||
|
||||
project = var.project_id
|
||||
role = "roles/firebasecloudmessaging.admin"
|
||||
member = google_service_account.push_runtime[0].member
|
||||
}
|
||||
|
||||
# The FCM V1 endpoint bills against the caller's project quota, which the caller must consume.
|
||||
resource "google_project_iam_member" "push_runtime_service_usage_consumer" {
|
||||
count = local.push_gateway_count
|
||||
|
||||
project = var.project_id
|
||||
role = "roles/serviceusage.serviceUsageConsumer"
|
||||
member = google_service_account.push_runtime[0].member
|
||||
}
|
||||
|
||||
resource "google_project_iam_member" "push_runtime_cloudsql_client" {
|
||||
count = local.push_gateway_count
|
||||
|
||||
project = var.project_id
|
||||
role = "roles/cloudsql.client"
|
||||
member = google_service_account.push_runtime[0].member
|
||||
}
|
||||
|
||||
# --- Apple credentials ----------------------------------------------------------------------
|
||||
|
||||
resource "google_secret_manager_secret" "push_provider" {
|
||||
for_each = local.push_provider_secret_ids
|
||||
|
||||
project = var.project_id
|
||||
secret_id = each.value
|
||||
labels = local.relay_shared_labels
|
||||
|
||||
replication {
|
||||
auto {}
|
||||
}
|
||||
|
||||
# Why: Apple issues a `.p8` once and Secret Manager has no undelete. Turning the gateway off
|
||||
# must fail the plan rather than destroy the only copy of the signing key.
|
||||
lifecycle {
|
||||
prevent_destroy = true
|
||||
}
|
||||
}
|
||||
|
||||
resource "google_secret_manager_secret_iam_member" "push_provider_runtime_accessor" {
|
||||
for_each = local.push_provider_secret_ids
|
||||
|
||||
project = var.project_id
|
||||
secret_id = google_secret_manager_secret.push_provider[each.value].secret_id
|
||||
role = "roles/secretmanager.secretAccessor"
|
||||
member = google_service_account.push_runtime[0].member
|
||||
}
|
||||
|
||||
# --- Service --------------------------------------------------------------------------------
|
||||
|
||||
resource "google_cloud_run_v2_service" "push" {
|
||||
count = local.push_gateway_count
|
||||
|
||||
project = var.project_id
|
||||
name = var.push_cloud_run_service_name
|
||||
location = var.region
|
||||
ingress = "INGRESS_TRAFFIC_ALL"
|
||||
# Why: the host proof in `POST /v1/host/challenge` is the authentication, not Cloud Run IAM.
|
||||
# The project's domain-restricted-sharing policy refuses an `allUsers` invoker binding, so the
|
||||
# service opts out of invoker IAM exactly as the relay director does.
|
||||
invoker_iam_disabled = true
|
||||
deletion_protection = var.environment == "production"
|
||||
labels = local.relay_shared_labels
|
||||
|
||||
template {
|
||||
service_account = google_service_account.push_runtime[0].email
|
||||
timeout = "${var.push_request_timeout_seconds}s"
|
||||
max_instance_request_concurrency = var.push_concurrency
|
||||
|
||||
scaling {
|
||||
min_instance_count = var.push_min_instances
|
||||
max_instance_count = var.push_max_instances
|
||||
}
|
||||
|
||||
volumes {
|
||||
name = "cloudsql"
|
||||
|
||||
cloud_sql_instance {
|
||||
instances = [google_sql_database_instance.push_dedicated[0].connection_name]
|
||||
}
|
||||
}
|
||||
|
||||
containers {
|
||||
image = var.push_cloud_run_image
|
||||
|
||||
ports {
|
||||
container_port = 8080
|
||||
}
|
||||
|
||||
volume_mounts {
|
||||
name = "cloudsql"
|
||||
mount_path = "/cloudsql"
|
||||
}
|
||||
|
||||
env {
|
||||
name = "ORCA_PUSH_PUBLIC_URL"
|
||||
value = var.push_base_url
|
||||
}
|
||||
|
||||
env {
|
||||
name = "ORCA_PUSH_FCM_PROJECT_ID"
|
||||
value = var.project_id
|
||||
}
|
||||
|
||||
# Bound the declared pool against the dedicated database rollout budget.
|
||||
env {
|
||||
name = "ORCA_PUSH_DATABASE_POOL_MAX"
|
||||
value = tostring(var.push_database_pool_max)
|
||||
}
|
||||
|
||||
env {
|
||||
name = "ORCA_PUSH_DATABASE_URL"
|
||||
|
||||
value_source {
|
||||
secret_key_ref {
|
||||
secret = google_secret_manager_secret.push_dedicated_database_url[0].secret_id
|
||||
version = google_secret_manager_secret_version.push_dedicated_database_url[0].version
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
# Rotation adds a new version and redeploys; `latest` is what the redeploy picks up.
|
||||
dynamic "env" {
|
||||
for_each = local.push_provider_secret_env
|
||||
|
||||
content {
|
||||
name = env.value
|
||||
|
||||
value_source {
|
||||
secret_key_ref {
|
||||
secret = google_secret_manager_secret.push_provider[env.key].secret_id
|
||||
version = "latest"
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
resources {
|
||||
limits = {
|
||||
cpu = var.push_cloud_run_cpu
|
||||
memory = var.push_cloud_run_memory
|
||||
}
|
||||
|
||||
cpu_idle = false
|
||||
}
|
||||
|
||||
startup_probe {
|
||||
failure_threshold = 12
|
||||
initial_delay_seconds = 0
|
||||
period_seconds = 5
|
||||
timeout_seconds = 2
|
||||
|
||||
http_get {
|
||||
path = "/health"
|
||||
port = 8080
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
# Deploys update the immutable image and shift traffic; Terraform owns the shape and IAM.
|
||||
#
|
||||
# `traffic` is ignored as well as the image. A deploy ends with traffic pinned to an exact
|
||||
# revision and a rollback pins it to the previous one; an apply that reset the service to
|
||||
# 100% LATEST would silently undo either, and this root carries unrelated standing drift, so
|
||||
# that apply need not be a push change at all.
|
||||
lifecycle {
|
||||
precondition {
|
||||
condition = var.push_max_instances * var.push_database_pool_max * 3 <= 64
|
||||
error_message = "Dedicated push serving, validation/rejected and successor pools must fit the 64-connection rollout budget."
|
||||
}
|
||||
|
||||
ignore_changes = [
|
||||
client,
|
||||
client_version,
|
||||
template[0].containers[0].image,
|
||||
traffic
|
||||
]
|
||||
}
|
||||
|
||||
depends_on = [
|
||||
data.google_artifact_registry_repository.relay_images,
|
||||
google_project_iam_member.push_runtime_cloudsql_client,
|
||||
google_secret_manager_secret_iam_member.push_provider_runtime_accessor,
|
||||
google_secret_manager_secret_version.push_dedicated_database_url,
|
||||
google_secret_manager_secret_iam_member.push_dedicated_database_url_accessor
|
||||
]
|
||||
}
|
||||
|
||||
# Google issues and renews the certificate for the mapping. The DNS record itself is a
|
||||
# hand-managed Cloudflare CNAME to ghs.googlehosted.com, like relay.onorca.dev; this root has no
|
||||
# Cloudflare surface by design. `terraform output push_dns_record` prints the record.
|
||||
resource "google_cloud_run_domain_mapping" "push" {
|
||||
count = var.push_gateway_enabled && var.manage_push_domain_mapping ? 1 : 0
|
||||
|
||||
location = var.region
|
||||
name = local.push_fqdn
|
||||
|
||||
metadata {
|
||||
namespace = var.project_id
|
||||
}
|
||||
|
||||
spec {
|
||||
route_name = google_cloud_run_v2_service.push[0].name
|
||||
}
|
||||
|
||||
# Same reason as relay-dns.tf: a gcloud-created mapping reports an empty legacy
|
||||
# certificate_mode, and replacing it would reset issuance for no behavioral change.
|
||||
lifecycle {
|
||||
ignore_changes = [spec[0].certificate_mode]
|
||||
}
|
||||
}
|
||||
|
||||
# --- Deploy identity grants -------------------------------------------------------------------
|
||||
# Push deploy authority is isolated from Relay; foundation grants its rollout-lock access.
|
||||
|
||||
resource "google_cloud_run_v2_service_iam_member" "github_production_push_developer" {
|
||||
count = local.push_gateway_deploy_count
|
||||
|
||||
project = var.project_id
|
||||
location = var.region
|
||||
name = google_cloud_run_v2_service.push[0].name
|
||||
role = "roles/run.developer"
|
||||
member = local.push_deploy_member
|
||||
}
|
||||
|
||||
resource "google_service_account_iam_member" "github_production_push_runtime_user" {
|
||||
count = local.push_gateway_deploy_count
|
||||
|
||||
service_account_id = google_service_account.push_runtime[0].name
|
||||
role = "roles/iam.serviceAccountUser"
|
||||
member = local.push_deploy_member
|
||||
}
|
||||
|
||||
# Why: the deploy workflow's validate-only FCM send has to exercise the credential the gateway
|
||||
# will actually use. Impersonating the runtime account proves its firebasecloudmessaging grant;
|
||||
# granting the deploy account FCM admin outright would prove nothing about the runtime account
|
||||
# and would widen a project-level role on the shared identity.
|
||||
resource "google_service_account_iam_member" "github_production_push_runtime_token_creator" {
|
||||
count = local.push_gateway_deploy_count
|
||||
|
||||
service_account_id = google_service_account.push_runtime[0].name
|
||||
role = "roles/iam.serviceAccountTokenCreator"
|
||||
member = local.push_deploy_member
|
||||
}
|
||||
@@ -20,6 +20,7 @@ locals {
|
||||
"deploy-relay-production.yml",
|
||||
"operate-relay-asia-admission.yml",
|
||||
"publish-relay-production.yml"
|
||||
|
||||
]
|
||||
github_production_relay_capacity_workflow_file = "deploy-relay-production-capacity.yml"
|
||||
github_production_relay_capacity_job_workflow_file = "deploy-relay-production-capacity-job.yml"
|
||||
|
||||
@@ -10,6 +10,7 @@
|
||||
locals {
|
||||
relay_github_workflow_clauses = {
|
||||
github = local.github_production_relay_workflow_clauses
|
||||
github_push = local.github_push_workflow_clauses
|
||||
github_monitor = local.github_monitor_workflow_clauses
|
||||
github_fence = local.github_fence_workflow_clauses
|
||||
github_production_relay_capacity = local.github_production_relay_capacity_workflow_clauses
|
||||
|
||||
@@ -484,3 +484,95 @@ variable "relay_gce_cloud_sql_proxy_image" {
|
||||
error_message = "relay_gce_cloud_sql_proxy_image must be pinned by sha256 digest."
|
||||
}
|
||||
}
|
||||
|
||||
# --- Mobile push gateway ---------------------------------------------------------------------
|
||||
# There is no staging push gateway by decision, so this defaults false and only
|
||||
# environments/production.tfvars turns it on. Everything in push-gateway.tf is behind it.
|
||||
variable "push_gateway_enabled" {
|
||||
type = bool
|
||||
description = "Create the Orca mobile push gateway, its database, secrets, and identity."
|
||||
default = false
|
||||
}
|
||||
|
||||
variable "push_base_url" {
|
||||
type = string
|
||||
description = "Public TLS origin of the mobile push gateway."
|
||||
default = "https://push.onorca.dev"
|
||||
|
||||
validation {
|
||||
condition = can(regex("^https://[^/]+$", var.push_base_url))
|
||||
error_message = "push_base_url must be an HTTPS origin with no path."
|
||||
}
|
||||
}
|
||||
|
||||
variable "push_cloud_run_service_name" {
|
||||
type = string
|
||||
description = "Cloud Run service name for the mobile push gateway."
|
||||
default = "orca-cloud-push"
|
||||
}
|
||||
|
||||
variable "push_cloud_run_image" {
|
||||
type = string
|
||||
description = "Initial image for the Terraform-created push gateway service; deploys own it after."
|
||||
default = "us-docker.pkg.dev/cloudrun/container/hello"
|
||||
}
|
||||
|
||||
variable "push_cloud_run_cpu" {
|
||||
type = string
|
||||
description = "CPU limit for the push gateway container."
|
||||
default = "1"
|
||||
}
|
||||
|
||||
variable "push_cloud_run_memory" {
|
||||
type = string
|
||||
description = "Memory limit for the push gateway container."
|
||||
default = "512Mi"
|
||||
}
|
||||
|
||||
# Keep a warm instance to run durable delivery retries without incoming requests.
|
||||
variable "push_min_instances" {
|
||||
type = number
|
||||
description = "Minimum instances for the push gateway."
|
||||
default = 1
|
||||
}
|
||||
|
||||
variable "push_max_instances" {
|
||||
type = number
|
||||
description = "Maximum instances for the push gateway."
|
||||
default = 4
|
||||
|
||||
validation {
|
||||
condition = var.push_max_instances >= 1
|
||||
error_message = "The push gateway needs at least one instance."
|
||||
}
|
||||
}
|
||||
|
||||
# The dedicated database budget counts pools across all three rollout revision resources.
|
||||
variable "push_database_pool_max" {
|
||||
type = number
|
||||
description = "Push gateway database pool size per instance; instances x pool is its Cloud SQL draw."
|
||||
default = 2
|
||||
|
||||
validation {
|
||||
condition = var.push_database_pool_max >= 1 && var.push_database_pool_max <= 100
|
||||
error_message = "The push gateway pool must hold at least one connection and stay under the per-service bound."
|
||||
}
|
||||
}
|
||||
|
||||
variable "push_concurrency" {
|
||||
type = number
|
||||
description = "Cloud Run concurrency for short-lived push gateway HTTP requests."
|
||||
default = 80
|
||||
}
|
||||
|
||||
variable "push_request_timeout_seconds" {
|
||||
type = number
|
||||
description = "Cloud Run timeout for push gateway requests; every route is short-lived."
|
||||
default = 30
|
||||
}
|
||||
|
||||
variable "manage_push_domain_mapping" {
|
||||
type = bool
|
||||
description = "Manage the push gateway Cloud Run domain mapping; the DNS record stays in the apps root."
|
||||
default = false
|
||||
}
|
||||
|
||||
+1
-1
@@ -21,7 +21,7 @@
|
||||
"load:relay:recovery-gate": "node dev/scripts/run-relay-recovery-wave-gate.mjs",
|
||||
"ops:relay": "pnpm --filter @orca-cloud/relay-ops dev",
|
||||
"pretest": "node --test dev/scripts/capture-terraform-plan-baseline.test.mjs dev/scripts/operate-relay-asia-admission.test.mjs dev/scripts/prepare-relay-asia-director-cells.test.mjs dev/scripts/prepare-relay-asia-topology-input.test.mjs dev/scripts/production-cloud-sql-rollout-lock.test.mjs dev/scripts/read-relay-serving-regional-placement-version.test.mjs dev/scripts/relay-asia-admission-workflow.test.mjs dev/scripts/relay-asia-rollout-evidence.test.mjs dev/scripts/relay-asia-topology-workflow.test.mjs dev/scripts/relay-cloud-sql-connection-budget.test.mjs dev/scripts/relay-load-reader-evidence.test.mjs dev/scripts/relay-region-hint-metrics.test.mjs dev/scripts/relay-staging-deploy-identity.test.mjs dev/scripts/sanitize-relay-asia-admission-result.test.mjs dev/scripts/terraform-root-partition.test.mjs dev/scripts/validate-relay-asia-topology-plan.test.mjs ../.github/actions/cloud-sql-rollout-lease/action-contract.test.mjs ../.github/actions/cloud-sql-rollout-lease/storage-lease.test.mjs",
|
||||
"test": "pnpm -r test && node --test dev/scripts/classify-relay-production-capacity-director.test.mjs dev/scripts/classify-relay-staging-bootstrap.test.mjs dev/scripts/deploy-relay-blue-green.test.mjs dev/scripts/deploy-relay-gce-candidate.test.mjs dev/scripts/deploy-relay-gce-multi-target.test.mjs dev/scripts/github-smoke-token.test.mjs dev/scripts/infra.test.mjs dev/scripts/operate-relay-regional-rehome.test.mjs dev/scripts/power-staging-relay.test.mjs dev/scripts/prepare-relay-capacity-canary.test.mjs dev/scripts/prepare-relay-production-capacity-canary.test.mjs dev/scripts/probe-relay-legacy-admission.test.mjs dev/scripts/probe-relay-rehome-trust.test.mjs dev/scripts/production-cell-image-digest-consistency.test.mjs dev/scripts/read-relay-production-capacity-identity.test.mjs dev/scripts/relay-admin-endpoint-retry-workflow.test.mjs dev/scripts/relay-admin-transient-retry.test.mjs dev/scripts/relay-admission-selector.test.mjs dev/scripts/relay-gce-terraform-fence.test.mjs dev/scripts/relay-load-connection-failure.test.mjs dev/scripts/relay-load-control-peer.test.mjs dev/scripts/relay-load-director-capacity-gate.test.mjs dev/scripts/relay-load-model.test.mjs dev/scripts/relay-load-phase-barrier.test.mjs dev/scripts/relay-load-placement-boundary.test.mjs dev/scripts/relay-load-profile.test.mjs dev/scripts/relay-load-rebind-boundary.test.mjs dev/scripts/relay-load-region-behavior.test.mjs dev/scripts/relay-load-request-unit-boundary.test.mjs dev/scripts/relay-load-run-lifecycle.test.mjs dev/scripts/relay-monitor-evidence.test.mjs dev/scripts/relay-production-capacity-wave.test.mjs dev/scripts/relay-production-capacity-workflow.test.mjs dev/scripts/relay-production-identity-boundaries.test.mjs dev/scripts/relay-production-same-cap-wave.test.mjs dev/scripts/relay-public-workflow-contract.test.mjs dev/scripts/relay-recovery-wave-gate.test.mjs dev/scripts/relay-region-observation-evidence.test.mjs dev/scripts/relay-regional-rehome-workflow.test.mjs dev/scripts/relay-rehome-aggregate-evidence.test.mjs dev/scripts/relay-repository.test.mjs dev/scripts/relay-same-cap-script-census.test.mjs dev/scripts/relay-staging-c4-refresh-workflow.test.mjs dev/scripts/relay-staging-capacity-identity.test.mjs dev/scripts/staging-relay-apply-guard.test.mjs dev/scripts/validate-relay-capacity-plan.test.mjs dev/scripts/verify-relay-capacity-transition.test.mjs dev/scripts/verify-relay-legacy-bootstrap.test.mjs dev/scripts/workload-identity-attribute-conditions.test.mjs",
|
||||
"test": "pnpm -r test && node --test dev/scripts/classify-relay-production-capacity-director.test.mjs dev/scripts/classify-relay-staging-bootstrap.test.mjs dev/scripts/deploy-relay-blue-green.test.mjs dev/scripts/deploy-relay-gce-candidate.test.mjs dev/scripts/deploy-relay-gce-multi-target.test.mjs dev/scripts/github-smoke-token.test.mjs dev/scripts/infra.test.mjs dev/scripts/operate-relay-regional-rehome.test.mjs dev/scripts/power-staging-relay.test.mjs dev/scripts/prepare-relay-capacity-canary.test.mjs dev/scripts/prepare-relay-production-capacity-canary.test.mjs dev/scripts/probe-relay-legacy-admission.test.mjs dev/scripts/probe-relay-rehome-trust.test.mjs dev/scripts/production-cell-image-digest-consistency.test.mjs dev/scripts/push-gateway-workflow.test.mjs dev/scripts/push-gateway-recovery.test.mjs dev/scripts/push-validation-workflow.test.mjs dev/scripts/read-relay-production-capacity-identity.test.mjs dev/scripts/relay-admin-endpoint-retry-workflow.test.mjs dev/scripts/relay-admin-transient-retry.test.mjs dev/scripts/relay-admission-selector.test.mjs dev/scripts/relay-gce-terraform-fence.test.mjs dev/scripts/relay-load-connection-failure.test.mjs dev/scripts/relay-load-control-peer.test.mjs dev/scripts/relay-load-director-capacity-gate.test.mjs dev/scripts/relay-load-model.test.mjs dev/scripts/relay-load-phase-barrier.test.mjs dev/scripts/relay-load-placement-boundary.test.mjs dev/scripts/relay-load-profile.test.mjs dev/scripts/relay-load-rebind-boundary.test.mjs dev/scripts/relay-load-region-behavior.test.mjs dev/scripts/relay-load-request-unit-boundary.test.mjs dev/scripts/relay-load-run-lifecycle.test.mjs dev/scripts/relay-monitor-evidence.test.mjs dev/scripts/relay-production-capacity-wave.test.mjs dev/scripts/relay-production-capacity-workflow.test.mjs dev/scripts/relay-production-identity-boundaries.test.mjs dev/scripts/relay-production-same-cap-wave.test.mjs dev/scripts/relay-public-workflow-contract.test.mjs dev/scripts/relay-recovery-wave-gate.test.mjs dev/scripts/relay-region-observation-evidence.test.mjs dev/scripts/relay-regional-rehome-workflow.test.mjs dev/scripts/relay-rehome-aggregate-evidence.test.mjs dev/scripts/relay-repository.test.mjs dev/scripts/relay-same-cap-script-census.test.mjs dev/scripts/relay-staging-c4-refresh-workflow.test.mjs dev/scripts/relay-staging-capacity-identity.test.mjs dev/scripts/staging-relay-apply-guard.test.mjs dev/scripts/validate-relay-capacity-plan.test.mjs dev/scripts/verify-relay-capacity-transition.test.mjs dev/scripts/verify-relay-legacy-bootstrap.test.mjs dev/scripts/workload-identity-attribute-conditions.test.mjs",
|
||||
"typecheck": "pnpm -r typecheck"
|
||||
},
|
||||
"devDependencies": {
|
||||
|
||||
@@ -0,0 +1,20 @@
|
||||
{
|
||||
"name": "@orca-cloud/postgres-schema",
|
||||
"version": "0.0.0",
|
||||
"private": true,
|
||||
"type": "module",
|
||||
"main": "dist/index.js",
|
||||
"types": "dist/index.d.ts",
|
||||
"scripts": {
|
||||
"build": "tsc -p tsconfig.build.json",
|
||||
"clean": "node -e \"require('fs').rmSync('dist', { recursive: true, force: true })\"",
|
||||
"lint": "tsc -p tsconfig.json --noEmit",
|
||||
"test": "pnpm build",
|
||||
"typecheck": "tsc -p tsconfig.json --noEmit"
|
||||
},
|
||||
"devDependencies": {
|
||||
"@types/node": "^24.10.0",
|
||||
"typescript": "^5.9.3",
|
||||
"vitest": "^4.0.8"
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,113 @@
|
||||
const RETRYABLE_SCHEMA_CODES = new Set(['55P03', '57014'])
|
||||
const DEFAULT_RETRY_DEADLINE_MS = 30_000
|
||||
const RETRY_BASE_DELAY_MS = 250
|
||||
const RETRY_MAX_DELAY_MS = 2_000
|
||||
|
||||
type SchemaStartupOptions = {
|
||||
eventPrefix?: string
|
||||
now?: () => number
|
||||
random?: () => number
|
||||
retryDeadlineMs?: number
|
||||
wait?: (delayMs: number) => Promise<void>
|
||||
}
|
||||
|
||||
function retryDelayMs(attempt: number, random: () => number): number {
|
||||
const ceiling = Math.min(RETRY_BASE_DELAY_MS * 2 ** (attempt - 1), RETRY_MAX_DELAY_MS)
|
||||
return Math.ceil(ceiling * (0.5 + random() * 0.5))
|
||||
}
|
||||
|
||||
function wait(delayMs: number): Promise<void> {
|
||||
return new Promise((resolve) => setTimeout(resolve, delayMs))
|
||||
}
|
||||
|
||||
const CREATE_TABLE_IF_NOT_EXISTS = /^\s*CREATE\s+TABLE\s+IF\s+NOT\s+EXISTS\b/i
|
||||
const CREATE_INDEX_IF_NOT_EXISTS = /^\s*CREATE\s+(?:UNIQUE\s+)?INDEX\s+IF\s+NOT\s+EXISTS\b/i
|
||||
|
||||
// `IF NOT EXISTS` only checks the name before the catalog inserts, so the loser of a concurrent
|
||||
// CREATE can fail on the catalog unique index (23505) or, when the winner has already committed by
|
||||
// the time the loser reaches TypeCreate/heap_create_with_catalog, on the name check those routines
|
||||
// repeat (42710 duplicate type, 42P07 duplicate relation). Each is a no-op on the next attempt.
|
||||
function concurrentCreateCollision(
|
||||
value: { code?: unknown; constraint?: unknown },
|
||||
statement: string
|
||||
): boolean {
|
||||
if (CREATE_TABLE_IF_NOT_EXISTS.test(statement)) {
|
||||
return (
|
||||
(value.code === '23505' && value.constraint === 'pg_type_typname_nsp_index') ||
|
||||
value.code === '42710' ||
|
||||
value.code === '42P07'
|
||||
)
|
||||
}
|
||||
if (CREATE_INDEX_IF_NOT_EXISTS.test(statement)) {
|
||||
return (
|
||||
(value.code === '23505' && value.constraint === 'pg_class_relname_nsp_index') ||
|
||||
value.code === '42P07'
|
||||
)
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
const ALTER_TABLE_ADD_CONSTRAINT = /^\s*ALTER\s+TABLE\s+\S+\s+ADD\s+CONSTRAINT\b/i
|
||||
|
||||
function constraintAlreadyApplied(error: unknown, statement: string): boolean {
|
||||
return (
|
||||
ALTER_TABLE_ADD_CONSTRAINT.test(statement) &&
|
||||
(error as { code?: unknown }).code === '42710'
|
||||
)
|
||||
}
|
||||
|
||||
function retryableSchemaError(error: unknown, statement: string): boolean {
|
||||
const value = error as { code?: unknown; constraint?: unknown }
|
||||
return (
|
||||
RETRYABLE_SCHEMA_CODES.has(String(value.code)) || concurrentCreateCollision(value, statement)
|
||||
)
|
||||
}
|
||||
|
||||
export async function applyPostgresSchema(
|
||||
statements: string[],
|
||||
query: (statement: string) => Promise<unknown>,
|
||||
options: SchemaStartupOptions = {}
|
||||
): Promise<void> {
|
||||
const now = options.now ?? Date.now
|
||||
const random = options.random ?? Math.random
|
||||
const pause = options.wait ?? wait
|
||||
const deadlineAt = now() + (options.retryDeadlineMs ?? DEFAULT_RETRY_DEADLINE_MS)
|
||||
|
||||
for (const statement of statements) {
|
||||
let attempt = 1
|
||||
while (true) {
|
||||
try {
|
||||
await query(statement)
|
||||
break
|
||||
} catch (error) {
|
||||
if (constraintAlreadyApplied(error, statement)) break
|
||||
const code = String((error as { code?: unknown }).code)
|
||||
const remainingMs = deadlineAt - now()
|
||||
const retryable = retryableSchemaError(error, statement)
|
||||
if (!retryable || remainingMs <= 0) {
|
||||
if (retryable) {
|
||||
console.warn(
|
||||
JSON.stringify({
|
||||
event: `${options.eventPrefix ?? 'orca_relay_postgres_schema'}_retry_exhausted`,
|
||||
code,
|
||||
attempts: attempt
|
||||
})
|
||||
)
|
||||
}
|
||||
throw error
|
||||
}
|
||||
const delayMs = Math.min(remainingMs, retryDelayMs(attempt, random))
|
||||
console.warn(
|
||||
JSON.stringify({
|
||||
event: `${options.eventPrefix ?? 'orca_relay_postgres_schema'}_retry`,
|
||||
code,
|
||||
attempt,
|
||||
delayMs
|
||||
})
|
||||
)
|
||||
await pause(delayMs)
|
||||
attempt += 1
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,11 @@
|
||||
{
|
||||
"extends": "./tsconfig.json",
|
||||
"compilerOptions": {
|
||||
"declaration": true,
|
||||
"emitDeclarationOnly": false,
|
||||
"noEmit": false,
|
||||
"outDir": "dist",
|
||||
"rootDir": "src"
|
||||
},
|
||||
"exclude": ["src/**/*.test.ts"]
|
||||
}
|
||||
@@ -0,0 +1,5 @@
|
||||
{
|
||||
"extends": "../../tsconfig.base.json",
|
||||
"compilerOptions": { "noEmit": true },
|
||||
"include": ["src/**/*.ts"]
|
||||
}
|
||||
@@ -0,0 +1,23 @@
|
||||
{
|
||||
"name": "@orca-cloud/push-contract",
|
||||
"private": true,
|
||||
"version": "0.0.0",
|
||||
"type": "module",
|
||||
"main": "dist/index.js",
|
||||
"types": "dist/index.d.ts",
|
||||
"scripts": {
|
||||
"build": "pnpm clean && tsc -p tsconfig.build.json",
|
||||
"clean": "node -e \"require('fs').rmSync('dist', { recursive: true, force: true })\"",
|
||||
"lint": "tsc -p tsconfig.json --noEmit",
|
||||
"test": "vitest run",
|
||||
"typecheck": "tsc -p tsconfig.json --noEmit"
|
||||
},
|
||||
"dependencies": {
|
||||
"zod": "^3.25.76"
|
||||
},
|
||||
"devDependencies": {
|
||||
"@types/node": "^24.10.0",
|
||||
"typescript": "^5.9.3",
|
||||
"vitest": "^4.0.8"
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,26 @@
|
||||
import { expect, it } from 'vitest'
|
||||
import { PushDeviceRegistrationRequestSchema } from './device-registration-messages.js'
|
||||
|
||||
const registration = (token: string) => ({
|
||||
v: 1,
|
||||
deviceId: 'qa-device',
|
||||
platform: 'ios',
|
||||
token,
|
||||
apnsEnvironment: 'sandbox'
|
||||
})
|
||||
|
||||
it.each([32, 64, 160, 256])(
|
||||
'accepts variable-length APNs device tokens (%i hex characters)',
|
||||
(length) => {
|
||||
expect(
|
||||
PushDeviceRegistrationRequestSchema.safeParse(registration('aB'.repeat(length / 2))).success
|
||||
).toBe(true)
|
||||
}
|
||||
)
|
||||
|
||||
it.each(['', 'abc', 'not-hex', 'ab cd', 'ab'.repeat(2049)])(
|
||||
'rejects malformed or oversized APNs tokens',
|
||||
(token) => {
|
||||
expect(PushDeviceRegistrationRequestSchema.safeParse(registration(token)).success).toBe(false)
|
||||
}
|
||||
)
|
||||
@@ -0,0 +1,162 @@
|
||||
import { describe, expect, it } from 'vitest'
|
||||
import {
|
||||
ApnsEnvironmentSchema,
|
||||
PushDeviceRegistrationRequestSchema
|
||||
} from './device-registration-messages.js'
|
||||
import {
|
||||
PushHostChallengeRequestSchema,
|
||||
PushHostChallengeResponseSchema,
|
||||
PushHostSessionRequestSchema,
|
||||
PushHostSessionResponseSchema
|
||||
} from './host-auth-messages.js'
|
||||
import { PUSH_DEFAULTS, PUSH_LIMITS } from './push-limits.js'
|
||||
|
||||
const KEY_B64 = Buffer.alloc(32, 1).toString('base64')
|
||||
const NONCE_B64 = Buffer.alloc(24, 2).toString('base64')
|
||||
const SESSION_TOKEN = Buffer.alloc(32, 3).toString('base64url')
|
||||
const FINGERPRINT = 'abcdefghijklmnop'
|
||||
const APNS_TOKEN = 'a'.repeat(64)
|
||||
const FCM_TOKEN = 'cQ1abcDEF_gh:APA91bZZ-zz0123456789abcdefghijklmnopqrstuvwxyz'
|
||||
|
||||
function notification(): Record<string, unknown> {
|
||||
return {
|
||||
notificationId: 'note-1',
|
||||
notificationSeq: 4,
|
||||
notificationEpoch: '5c9e9a1e-0000-4000-8000-000000000000',
|
||||
source: 'agent-task-complete',
|
||||
agentState: 'needs-input',
|
||||
title: 'Agent needs input',
|
||||
body: 'Waiting on your answer',
|
||||
worktreeId: 'wt-1'
|
||||
}
|
||||
}
|
||||
|
||||
describe('push contract limits', () => {
|
||||
it('locks the normative limits the desktop and gateway both assume', () => {
|
||||
expect(PUSH_LIMITS).toMatchObject({
|
||||
titleMaxChars: 80,
|
||||
bodyMaxChars: 180,
|
||||
maxRegistrationIdsPerSend: 20,
|
||||
maxDevicesPerHost: 64,
|
||||
hostEventsPerWindow: 300,
|
||||
eventQuotaWindowMs: 900_000,
|
||||
challengeTtlMs: 10_000,
|
||||
clockSkewToleranceMs: 30_000,
|
||||
sessionTtlMs: 86_400_000,
|
||||
notificationTtlSeconds: 300,
|
||||
unauthenticatedRequestsPerMinutePerIp: 30,
|
||||
authenticatedRequestsPerMinutePerIp: 6_000,
|
||||
authenticatedRequestsPerMinutePerHost: 600
|
||||
})
|
||||
expect(PUSH_DEFAULTS.apnsTopic).toBe('com.stably.orca.mobile')
|
||||
expect(PUSH_DEFAULTS.androidChannelId).toBe('orca-desktop')
|
||||
})
|
||||
})
|
||||
|
||||
describe('host authentication schemas', () => {
|
||||
it('accepts a well formed challenge round trip', () => {
|
||||
expect(
|
||||
PushHostChallengeRequestSchema.safeParse({ v: 1, hostPublicKeyB64: KEY_B64 }).success
|
||||
).toBe(true)
|
||||
expect(
|
||||
PushHostChallengeResponseSchema.safeParse({
|
||||
challengeId: 'challenge-1',
|
||||
gatewayEphemeralPublicKeyB64: KEY_B64,
|
||||
nonceB64: NONCE_B64,
|
||||
ciphertextB64: Buffer.alloc(96, 5).toString('base64'),
|
||||
expiresAt: 1_700_000_010_000
|
||||
}).success
|
||||
).toBe(true)
|
||||
expect(
|
||||
PushHostSessionRequestSchema.safeParse({
|
||||
v: 1,
|
||||
challengeId: 'challenge-1',
|
||||
proofB64: KEY_B64
|
||||
}).success
|
||||
).toBe(true)
|
||||
expect(
|
||||
PushHostSessionResponseSchema.safeParse({
|
||||
sessionToken: SESSION_TOKEN,
|
||||
expiresAt: 1_700_086_400_000,
|
||||
hostFingerprint: FINGERPRINT
|
||||
}).success
|
||||
).toBe(true)
|
||||
})
|
||||
|
||||
it('rejects unknown keys, wrong versions, and mis-sized keys', () => {
|
||||
expect(
|
||||
PushHostChallengeRequestSchema.safeParse({
|
||||
v: 1,
|
||||
hostPublicKeyB64: KEY_B64,
|
||||
extra: true
|
||||
}).success
|
||||
).toBe(false)
|
||||
expect(
|
||||
PushHostChallengeRequestSchema.safeParse({ v: 2, hostPublicKeyB64: KEY_B64 }).success
|
||||
).toBe(false)
|
||||
expect(
|
||||
PushHostChallengeRequestSchema.safeParse({
|
||||
v: 1,
|
||||
hostPublicKeyB64: Buffer.alloc(31, 1).toString('base64')
|
||||
}).success
|
||||
).toBe(false)
|
||||
expect(
|
||||
PushHostSessionResponseSchema.safeParse({
|
||||
sessionToken: SESSION_TOKEN,
|
||||
expiresAt: 1_700_086_400_000,
|
||||
hostFingerprint: 'short'
|
||||
}).success
|
||||
).toBe(false)
|
||||
})
|
||||
})
|
||||
|
||||
describe('device registration schemas', () => {
|
||||
it('requires an apns environment and a hex token for ios', () => {
|
||||
expect(
|
||||
PushDeviceRegistrationRequestSchema.safeParse({
|
||||
v: 1,
|
||||
deviceId: 'device-1',
|
||||
platform: 'ios',
|
||||
token: APNS_TOKEN,
|
||||
apnsEnvironment: 'sandbox'
|
||||
}).success
|
||||
).toBe(true)
|
||||
expect(
|
||||
PushDeviceRegistrationRequestSchema.safeParse({
|
||||
v: 1,
|
||||
deviceId: 'device-1',
|
||||
platform: 'ios',
|
||||
token: APNS_TOKEN
|
||||
}).success
|
||||
).toBe(false)
|
||||
expect(
|
||||
PushDeviceRegistrationRequestSchema.safeParse({
|
||||
v: 1,
|
||||
deviceId: 'device-1',
|
||||
platform: 'ios',
|
||||
token: 'not-hex',
|
||||
apnsEnvironment: 'production'
|
||||
}).success
|
||||
).toBe(false)
|
||||
})
|
||||
|
||||
it('rejects an apns environment on android and accepts an fcm token', () => {
|
||||
expect(
|
||||
PushDeviceRegistrationRequestSchema.safeParse({
|
||||
v: 1,
|
||||
deviceId: 'device-2',
|
||||
platform: 'android',
|
||||
token: FCM_TOKEN
|
||||
}).success
|
||||
).toBe(true)
|
||||
expect(
|
||||
PushDeviceRegistrationRequestSchema.safeParse({
|
||||
v: 1,
|
||||
deviceId: 'device-2',
|
||||
platform: 'android',
|
||||
token: FCM_TOKEN,
|
||||
apnsEnvironment: 'sandbox'
|
||||
}).success
|
||||
).toBe(false)
|
||||
})
|
||||
})
|
||||
Some files were not shown because too many files have changed in this diff Show More
Reference in New Issue
Block a user