diff --git a/.github/workflows/cloud-push-deploy.yml b/.github/workflows/cloud-push-deploy.yml index 6014681372d..1b909d42f63 100644 --- a/.github/workflows/cloud-push-deploy.yml +++ b/.github/workflows/cloud-push-deploy.yml @@ -106,7 +106,7 @@ jobs: set -euo pipefail docker run --rm --network none --entrypoint node "${IMAGE}" --input-type=module -e ' import { loadPushConfig } from "./apps/push/dist/config.js"; - const env = { ORCA_PUSH_PUBLIC_URL: "https://push.onorca.dev", ORCA_PUSH_MODE: "validation" }; + const env = { ORCA_PUSH_PUBLIC_URL: "https://push.onorca.dev", ORCA_PUSH_MODE: "validation", ORCA_PUSH_FCM_PROJECT_ID: "onorca-cloud" }; if (loadPushConfig(env).mode !== "validation") throw new Error("validation_mode_unsupported"); let rejected = false; try { loadPushConfig({ ...env, ORCA_PUSH_MODE: "invalid" }); } catch { rejected = true; } diff --git a/.github/workflows/cloud-verify.yml b/.github/workflows/cloud-verify.yml index e2ba9407ac4..5e24cae76cc 100644 --- a/.github/workflows/cloud-verify.yml +++ b/.github/workflows/cloud-verify.yml @@ -90,6 +90,7 @@ jobs: --health-timeout 5s --health-retries 10 env: + ORCA_PUSH_TEST_DATABASE_URL: postgres://relay_test:relay_test@127.0.0.1:5432/orca_relay_test ORCA_RELAY_TEST_POSTGRES_URL: postgres://relay_test:relay_test@127.0.0.1:5432/orca_relay_test steps: - uses: actions/checkout@v4 diff --git a/cloud/README.md b/cloud/README.md index 8ffcd9fa6b3..6ca96f4cc99 100644 --- a/cloud/README.md +++ b/cloud/README.md @@ -24,6 +24,40 @@ the repository's root [MIT license](../LICENSE). - `apps/relay-ops`: the relay operations console and the incident monitor behind `pnpm ops:relay`, `pnpm incident:relay`, and `pnpm incident:relay-preflight`. +- `apps/push` and `packages/push-contract`: the mobile push gateway that holds + the APNs key and sends to phones through APNs and FCM, and its wire contract. + It is deployed and operated from here but is not part of the relay data path; + see [docs/push-gateway.md](docs/push-gateway.md). + +## Mobile push gateway + +`apps/push` is a separate Cloud Run service from the relay. Phones never hold an +Orca credential for it: the desktop host authenticates with the same X25519 +key it uses for the relay, answering an encrypted challenge to mint a 24 hour +session, then registers each paired phone's native push token and asks the +gateway to push. The gateway queues each event as its own notification, +enforces per-host quotas and request limits, and retires a +registration as soon as Apple or Google reports the token unregistered. +Provider push is the only ordinary mobile OS-banner path. The notification +socket is retained only for live dismissal and reconnect tray reconciliation; +it never creates or recovers banners. Desktop notification categories remain +authoritative. +Each delivery is persisted as one notification event. Before deploying an +incompatible queue format, stop all older push gateway revisions and clear only +unpublished push delivery fixtures; no queue preservation or migration is required. +FCM notification messages are inherently collapsible while offline and have a +small concurrent collapse-key budget, so every pending alert is not guaranteed. + +Storage follows the relay pattern: PostgreSQL in production, SQLite for tests +and local development. Configure it with `ORCA_PUSH_PUBLIC_URL`, `ORCA_PUSH_FCM_PROJECT_ID`, +`ORCA_PUSH_DATABASE_URL`, the three APNs variables (`ORCA_PUSH_APNS_KEY`, +`ORCA_PUSH_APNS_KEY_ID`, `ORCA_PUSH_APPLE_TEAM_ID`, all three or none), and +optionally `ORCA_PUSH_APNS_TOPIC`. The FCM credential comes from +the runtime service account, so no key material is configured for Android. See +[push gateway operations](docs/push-gateway.md) for deployment and recovery. + +Logging is aggregate counters only. Tokens, notification titles, notification +bodies, and full host fingerprints never reach a log line. ## Infrastructure and operations @@ -38,16 +72,18 @@ the repository's root [MIT license](../LICENSE). - `dev/contracts` and `dev/fixtures`: the checked-in data those contract tests read, including the Terraform root partition. - `docs/`: the relay runbooks, capacity-testing guide, incident-monitor - reference, and the workflow variable reference in `docs/relay-workflows.md`. + reference, the workflow variable reference in `docs/relay-workflows.md`, and + the push gateway runbook in `docs/push-gateway.md`. ## Workflows -The 24 `.github/workflows/cloud-*.yml` workflows are the relay's deploy and -operate surface: publish and deploy the director, roll GCE cell capacity, -operate Asia admission and regional rehoming, prove staging capacity, monitor -production, and power staging up and down. `.github/actions/cloud-sql-rollout-lease` -is the compare-and-swap lease that serializes every rollout against the shared -Cloud SQL instance. +The 25 `.github/workflows/cloud-*.yml` workflows are the deploy and operate +surface: publish and deploy the director, roll GCE cell capacity, operate Asia +admission and regional rehoming, prove staging capacity, monitor production, +power staging up and down, and deploy the mobile push gateway. +`.github/actions/cloud-sql-rollout-lease` is the compare-and-swap lease that +serializes rollouts against the shared Cloud SQL instance. Push reuses that +action with its own lease object and deployment concurrency group. Every one of them is inert. Each top-level job is gated on `vars.ORCA_CLOUD_OPERATIONS_ENABLED == 'true'`, a repository variable that is diff --git a/cloud/apps/push/Dockerfile b/cloud/apps/push/Dockerfile new file mode 100644 index 00000000000..efdc85fc404 --- /dev/null +++ b/cloud/apps/push/Dockerfile @@ -0,0 +1,29 @@ +FROM node:24-alpine AS build +WORKDIR /app +RUN corepack enable +COPY package.json pnpm-lock.yaml pnpm-workspace.yaml tsconfig.base.json ./ +COPY packages/push-contract/package.json packages/push-contract/package.json +COPY packages/postgres-schema/package.json packages/postgres-schema/package.json +COPY apps/push/package.json apps/push/package.json +RUN pnpm install --frozen-lockfile +COPY packages/push-contract packages/push-contract +COPY apps/push apps/push +COPY packages/postgres-schema packages/postgres-schema +RUN pnpm --filter @orca-cloud/postgres-schema build && pnpm --filter @orca-cloud/push-contract build && pnpm --filter @orca-cloud/push build + +FROM node:24-alpine AS runtime +ENV NODE_ENV=production +ENV PORT=8080 +WORKDIR /app +RUN corepack enable +COPY package.json pnpm-lock.yaml pnpm-workspace.yaml ./ +COPY packages/push-contract/package.json packages/push-contract/package.json +COPY packages/postgres-schema/package.json packages/postgres-schema/package.json +COPY apps/push/package.json apps/push/package.json +COPY --from=build /app/packages/push-contract/dist packages/push-contract/dist +COPY --from=build /app/packages/postgres-schema/dist packages/postgres-schema/dist +COPY --from=build /app/apps/push/dist apps/push/dist +RUN pnpm install --prod --frozen-lockfile --filter @orca-cloud/push... +USER node +EXPOSE 8080 +CMD ["node", "apps/push/dist/index.js"] diff --git a/cloud/apps/push/package.json b/cloud/apps/push/package.json new file mode 100644 index 00000000000..1f27749024c --- /dev/null +++ b/cloud/apps/push/package.json @@ -0,0 +1,35 @@ +{ + "name": "@orca-cloud/push", + "private": true, + "version": "0.0.0", + "type": "module", + "main": "dist/index.js", + "scripts": { + "build": "pnpm clean && tsc -p tsconfig.build.json", + "clean": "node -e \"require('fs').rmSync('dist', { recursive: true, force: true })\"", + "dev": "tsx watch src/index.ts", + "lint": "tsc -p tsconfig.json --noEmit", + "pretest": "pnpm --filter @orca-cloud/postgres-schema build && pnpm --filter @orca-cloud/push-contract build", + "start": "node dist/index.js", + "test": "vitest run", + "typecheck": "tsc -p tsconfig.json --noEmit" + }, + "dependencies": { + "@hono/node-server": "^1.19.17", + "@orca-cloud/postgres-schema": "workspace:*", + "@orca-cloud/push-contract": "workspace:*", + "google-auth-library": "^10.5.0", + "hono": "^4.13.7", + "pg": "^8.22.0", + "pg-connection-string": "2.14.0", + "tweetnacl": "^1.0.3", + "zod": "^3.25.76" + }, + "devDependencies": { + "@types/node": "^24.10.0", + "@types/pg": "^8.20.0", + "tsx": "^4.21.0", + "typescript": "^5.9.3", + "vitest": "^4.0.8" + } +} diff --git a/cloud/apps/push/src/apns-authentication-token.ts b/cloud/apps/push/src/apns-authentication-token.ts new file mode 100644 index 00000000000..34def16e86e --- /dev/null +++ b/cloud/apps/push/src/apns-authentication-token.ts @@ -0,0 +1,42 @@ +import { createPrivateKey, type KeyObject, sign } from 'node:crypto' +import type { ApnsCredentials } from './config.js' + +// Apple rejects a provider token older than an hour and throttles reissue +// under about 20 minutes, so 50 minutes is the safe rotation point. +export const APNS_TOKEN_ROTATION_MS = 50 * 60 * 1000 + +function base64UrlJson(value: Record): string { + return Buffer.from(JSON.stringify(value), 'utf8').toString('base64url') +} + +export class ApnsAuthenticationToken { + private readonly privateKey: KeyObject + private cached: { token: string; issuedAtMs: number } | null = null + + constructor( + private readonly credentials: ApnsCredentials, + private readonly now: () => number = Date.now, + private readonly rotationMs: number = APNS_TOKEN_ROTATION_MS + ) { + this.privateKey = createPrivateKey(credentials.keyPem) + } + + value(): string { + const nowMs = this.now() + if (this.cached && nowMs - this.cached.issuedAtMs < this.rotationMs) return this.cached.token + const header = base64UrlJson({ alg: 'ES256', kid: this.credentials.keyId }) + const payload = base64UrlJson({ + iss: this.credentials.teamId, + iat: Math.floor(nowMs / 1000) + }) + const signingInput = `${header}.${payload}` + // ES256 requires the raw r||s pair; Node emits DER unless asked otherwise. + const signature = sign('sha256', Buffer.from(signingInput, 'utf8'), { + key: this.privateKey, + dsaEncoding: 'ieee-p1363' + }).toString('base64url') + const token = `${signingInput}.${signature}` + this.cached = { token, issuedAtMs: nowMs } + return token + } +} diff --git a/cloud/apps/push/src/apns-client.test.ts b/cloud/apps/push/src/apns-client.test.ts new file mode 100644 index 00000000000..e4af2bb1ecb --- /dev/null +++ b/cloud/apps/push/src/apns-client.test.ts @@ -0,0 +1,220 @@ +import { generateKeyPairSync } from 'node:crypto' +import { describe, expect, it } from 'vitest' +import { ApnsAuthenticationToken, APNS_TOKEN_ROTATION_MS } from './apns-authentication-token.js' +import { ApnsClient } from './apns-client.js' +import type { ApnsRequest, ApnsResponse } from './apns-http2-transport.js' +import type { ApnsCredentials } from './config.js' +import { buildPushDelivery } from './push-delivery-message.js' + +const HOST = 'abcdefghijklmnop' + +function credentials(): ApnsCredentials { + const { privateKey } = generateKeyPairSync('ec', { + namedCurve: 'P-256', + privateKeyEncoding: { type: 'pkcs8', format: 'pem' }, + publicKeyEncoding: { type: 'spki', format: 'pem' } + }) + return { keyPem: privateKey, keyId: 'ABCDE12345', teamId: 'TEAM123456' } +} + +function delivery(now = Date.now()) { + return buildPushDelivery({ + expiresAt: now + 300_000, + registrationId: 'reg-1', + hostFingerprint: HOST, + notification: { + notificationId: 'note-1', + notificationSeq: 7, + notificationEpoch: 'epoch-1', + source: 'agent-task-complete', + agentState: 'needs-input', + title: 'Agent needs input', + body: 'Waiting on your answer', + worktreeId: 'wt-1' + } + }) +} + +function fakeTransport(response: ApnsResponse) { + const requests: ApnsRequest[] = [] + return { + requests, + transport: async (request: ApnsRequest): Promise => { + requests.push(request) + return response + } + } +} + +describe('apns authentication token', () => { + it('signs an ES256 provider token and caches it until the rotation point', () => { + let clock = 1_700_000_000_000 + const authentication = new ApnsAuthenticationToken(credentials(), () => clock) + const first = authentication.value() + const [header, payload, signature] = first.split('.') + expect(JSON.parse(Buffer.from(header!, 'base64url').toString('utf8'))).toEqual({ + alg: 'ES256', + kid: 'ABCDE12345' + }) + expect(JSON.parse(Buffer.from(payload!, 'base64url').toString('utf8'))).toEqual({ + iss: 'TEAM123456', + iat: Math.floor(clock / 1000) + }) + expect(Buffer.from(signature!, 'base64url').byteLength).toBe(64) + + clock += APNS_TOKEN_ROTATION_MS - 1 + expect(authentication.value()).toBe(first) + clock += 1 + expect(authentication.value()).not.toBe(first) + }) +}) + +describe('apns client', () => { + it('sends the specified headers, path, and alert body', async () => { + const clock = 1_700_000_000_000 + const fake = fakeTransport({ status: 200, body: '' }) + const client = new ApnsClient({ + topic: 'com.stably.orca.mobile', + credentials: credentials(), + transport: fake.transport, + now: () => clock + }) + await expect( + client.send(delivery(clock), { token: 'a'.repeat(64), apnsEnvironment: 'production' }) + ).resolves.toEqual({ status: 'sent' }) + const request = fake.requests[0]! + expect(request.host).toBe('api.push.apple.com') + expect(request.path).toBe(`/3/device/${'a'.repeat(64)}`) + expect(request.headers).toMatchObject({ + 'apns-topic': 'com.stably.orca.mobile', + 'apns-push-type': 'alert', + 'apns-priority': '10', + 'apns-expiration': String(Math.floor(clock / 1000) + 5 * 60), + 'apns-collapse-id': expect.stringMatching(/^[a-f0-9]{64}$/) + }) + expect(request.headers.authorization).toMatch(/^bearer /) + expect(JSON.parse(request.body)).toEqual({ + aps: { + alert: { title: 'Agent needs input', body: 'Waiting on your answer' }, + sound: 'default', + 'thread-id': HOST + }, + orca: { + hostFingerprint: HOST, + worktreeId: 'wt-1', + notificationId: 'note-1', + notificationSeq: 7, + notificationEpoch: 'epoch-1', + source: 'agent-task-complete', + agentState: 'needs-input' + } + }) + }) + + it('targets the sandbox host and keeps the individual collapse id', async () => { + const fake = fakeTransport({ status: 200, body: '' }) + const client = new ApnsClient({ + topic: 'com.stably.orca.mobile', + credentials: credentials(), + transport: fake.transport + }) + await client.send(delivery(), { token: 'b'.repeat(64), apnsEnvironment: 'sandbox' }) + expect(fake.requests[0]?.host).toBe('api.sandbox.push.apple.com') + expect(fake.requests[0]?.headers['apns-collapse-id']).toMatch(/^[a-f0-9]{64}$/) + }) + + it.each([ + [410, 'Unregistered'], + [400, 'BadDeviceToken'], + [400, 'Unregistered'] + ])('classifies %i %s as a dead token', async (status, reason) => { + const fake = fakeTransport({ status, body: JSON.stringify({ reason }) }) + const client = new ApnsClient({ + topic: 'com.stably.orca.mobile', + credentials: credentials(), + transport: fake.transport + }) + await expect( + client.send(delivery(), { token: 'a'.repeat(64), apnsEnvironment: 'production' }) + ).resolves.toEqual({ status: 'dead', reason }) + }) + + it.each([ + [400, 'PayloadTooLarge'], + [400, 'DeviceTokenNotForTopic'], + [429, 'TooManyRequests'], + [500, 'InternalServerError'] + ])('treats %i %s with the appropriate retry policy', async (status, reason) => { + const fake = fakeTransport({ status, body: JSON.stringify({ reason }) }) + const client = new ApnsClient({ + topic: 'com.stably.orca.mobile', + credentials: credentials(), + transport: fake.transport + }) + await expect( + client.send(delivery(), { token: 'a'.repeat(64), apnsEnvironment: 'production' }) + ).resolves.toEqual({ status: 'error', reason, retryable: status === 429 || status >= 500 }) + }) + + it('reports a transport failure as an error rather than throwing', async () => { + const client = new ApnsClient({ + topic: 'com.stably.orca.mobile', + credentials: credentials(), + transport: async () => { + throw new Error('socket hang up') + } + }) + await expect( + client.send(delivery(), { token: 'a'.repeat(64), apnsEnvironment: 'production' }) + ).resolves.toEqual({ status: 'error', reason: 'Error', retryable: true }) + }) +}) + +it('does not collapse background dismissals with visible alerts', async () => { + const fake = fakeTransport({ status: 200, body: '' }) + const apns = new ApnsClient({ + topic: 'test', + credentials: credentials(), + transport: fake.transport + }) + const alert = delivery() + await apns.send( + { ...alert, orca: { ...alert.orca, kind: 'dismiss' } }, + { + token: 'test', + apnsEnvironment: 'sandbox' + } + ) + expect(fake.requests[0]?.headers).not.toHaveProperty('apns-collapse-id') + expect(fake.requests[0]?.headers).toMatchObject({ + 'apns-push-type': 'background', + 'apns-priority': '5' + }) + expect(JSON.parse(fake.requests[0]!.body).aps).toEqual({ 'content-available': 1 }) +}) + +it('keeps the absolute deadline across retries and refuses expired delivery', async () => { + let now = 1_700_000_000_000 + const fake = fakeTransport({ status: 503, body: '{}' }) + const client = new ApnsClient({ + topic: 'test', + credentials: credentials(), + now: () => now, + transport: fake.transport + }) + const pending = delivery(now) + const device = { token: 'test', apnsEnvironment: 'sandbox' as const } + await client.send(pending, device) + now += 60_000 + await client.send(pending, device) + expect(fake.requests.map((request) => request.headers['apns-expiration'])).toEqual([ + String(pending.expiresAt / 1000), + String(pending.expiresAt / 1000) + ]) + now = pending.expiresAt + await expect(client.send(pending, device)).resolves.toEqual({ + status: 'error', + reason: 'expired' + }) + expect(fake.requests).toHaveLength(2) +}) diff --git a/cloud/apps/push/src/apns-client.ts b/cloud/apps/push/src/apns-client.ts new file mode 100644 index 00000000000..827efdfea79 --- /dev/null +++ b/cloud/apps/push/src/apns-client.ts @@ -0,0 +1,95 @@ +import type { ApnsEnvironment } from '@orca-cloud/push-contract' +import { ApnsAuthenticationToken } from './apns-authentication-token.js' +import type { ApnsTransport } from './apns-http2-transport.js' +import type { ApnsCredentials } from './config.js' +import type { PushDelivery } from './push-delivery-message.js' +import type { PushProviderOutcome } from './push-provider-outcome.js' + +const APNS_HOSTS: Record = { + production: 'api.push.apple.com', + sandbox: 'api.sandbox.push.apple.com' +} + +const DEAD_TOKEN_REASONS = new Set(['BadDeviceToken', 'Unregistered']) + +export type ApnsClientOptions = { + topic: string + credentials: ApnsCredentials + transport: ApnsTransport + now?: () => number +} + +function readReason(body: string): string { + try { + const parsed = JSON.parse(body) as { reason?: unknown } + return typeof parsed.reason === 'string' ? parsed.reason : 'unknown' + } catch { + return 'unparseable' + } +} + +export function apnsBody(delivery: PushDelivery): string { + return JSON.stringify({ + aps: + delivery.orca.kind === 'dismiss' + ? { 'content-available': 1 } + : { + alert: { title: delivery.title, body: delivery.body }, + ...(delivery.sound === false ? {} : { sound: 'default' }), + 'thread-id': delivery.hostFingerprint + }, + orca: delivery.orca + }) +} + +export class ApnsClient { + private readonly authentication: ApnsAuthenticationToken + private readonly now: () => number + + constructor(private readonly options: ApnsClientOptions) { + this.now = options.now ?? Date.now + this.authentication = new ApnsAuthenticationToken(options.credentials, this.now) + } + + async send( + delivery: PushDelivery, + device: { token: string; apnsEnvironment: ApnsEnvironment } + ): Promise { + const expiration = Math.floor(delivery.expiresAt / 1000) + if (expiration * 1000 <= this.now()) return { status: 'error', reason: 'expired' } + let response + try { + response = await this.options.transport({ + host: APNS_HOSTS[device.apnsEnvironment], + path: `/3/device/${device.token}`, + headers: { + authorization: `bearer ${this.authentication.value()}`, + 'apns-topic': this.options.topic, + 'apns-push-type': delivery.orca.kind === 'dismiss' ? 'background' : 'alert', + 'apns-priority': delivery.orca.kind === 'dismiss' ? '5' : '10', + 'apns-expiration': String(expiration), + ...(delivery.orca.kind === 'dismiss' ? {} : { 'apns-collapse-id': delivery.collapseId }) + }, + body: apnsBody(delivery) + }) + } catch (error) { + return { + status: 'error', + reason: error instanceof Error ? error.name : 'transport_failed', + retryable: true + } + } + if (response.status === 200) return { status: 'sent' } + const reason = readReason(response.body) + if (response.status === 410) return { status: 'dead', reason } + if (response.status === 400 && DEAD_TOKEN_REASONS.has(reason)) { + return { status: 'dead', reason } + } + return { + status: 'error', + reason, + retryable: response.status === 429 || response.status >= 500, + ...(response.retryAfterMs === undefined ? {} : { retryAfterMs: response.retryAfterMs }) + } + } +} diff --git a/cloud/apps/push/src/apns-http2-transport.ts b/cloud/apps/push/src/apns-http2-transport.ts new file mode 100644 index 00000000000..167b4d14e38 --- /dev/null +++ b/cloud/apps/push/src/apns-http2-transport.ts @@ -0,0 +1,50 @@ +import { connect, constants, type ClientHttp2Session } from 'node:http2' +import { readApnsStreamResponse, type ApnsResponse } from './apns-stream-response.js' + +export type ApnsRequest = { + host: string + path: string + headers: Record + body: string +} + +export type { ApnsResponse } +export type ApnsTransport = (request: ApnsRequest) => Promise + +// APNs requires HTTP/2 and rewards a long-lived session per host, so sessions +// are cached and only dropped when the socket itself goes away. +export function createApnsHttp2Transport(): ApnsTransport & { close(): void } { + const sessions = new Map() + + const sessionFor = (host: string): ClientHttp2Session => { + const existing = sessions.get(host) + if (existing && !existing.closed && !existing.destroyed) return existing + const session = connect(`https://${host}`) + const forget = (): void => { + if (sessions.get(host) === session) sessions.delete(host) + } + session.on('error', forget) + session.on('close', forget) + sessions.set(host, session) + return session + } + + const transport = async (request: ApnsRequest): Promise => { + const stream = sessionFor(request.host).request({ + ...request.headers, + [constants.HTTP2_HEADER_METHOD]: 'POST', + [constants.HTTP2_HEADER_PATH]: request.path, + [constants.HTTP2_HEADER_AUTHORITY]: request.host, + 'content-type': 'application/json', + 'content-length': String(Buffer.byteLength(request.body)) + }) + return await readApnsStreamResponse(stream, request.body) + } + + return Object.assign(transport, { + close(): void { + for (const session of sessions.values()) session.close() + sessions.clear() + } + }) +} diff --git a/cloud/apps/push/src/apns-session-replacement.test.ts b/cloud/apps/push/src/apns-session-replacement.test.ts new file mode 100644 index 00000000000..2678732ca94 --- /dev/null +++ b/cloud/apps/push/src/apns-session-replacement.test.ts @@ -0,0 +1,45 @@ +import { EventEmitter } from 'node:events' +import { expect, it, vi } from 'vitest' +const mocks = vi.hoisted(() => ({ + connect: vi.fn(), + read: vi.fn(async () => ({ status: 200, body: '' })) +})) +vi.mock('node:http2', async (original) => ({ + ...(await original()), + connect: mocks.connect +})) +vi.mock('./apns-stream-response.js', () => ({ readApnsStreamResponse: mocks.read })) +import { createApnsHttp2Transport } from './apns-http2-transport.js' + +it('keeps the replacement cached when the draining session closes later', async () => { + const sessions: Array< + EventEmitter & { + closed: boolean + destroyed: boolean + request: ReturnType + close: ReturnType + } + > = [] + mocks.connect.mockImplementation(() => { + const session = Object.assign(new EventEmitter(), { + closed: false, + destroyed: false, + request: vi.fn(() => ({})), + close: vi.fn() + }) + sessions.push(session) + return session + }) + const transport = createApnsHttp2Transport() + const request = { host: 'api.push.apple.com', path: '/synthetic', headers: {}, body: '{}' } + await transport(request) + sessions[0]!.closed = true + await transport(request) + sessions[0]!.emit('close') + sessions[0]!.emit('error', new Error('old-session')) + await transport(request) + expect(sessions).toHaveLength(2) + expect(sessions[1]!.request).toHaveBeenCalledTimes(2) + transport.close() + expect(sessions[1]!.close).toHaveBeenCalledOnce() +}) diff --git a/cloud/apps/push/src/apns-stream-response.test.ts b/cloud/apps/push/src/apns-stream-response.test.ts new file mode 100644 index 00000000000..c87b9031ca1 --- /dev/null +++ b/cloud/apps/push/src/apns-stream-response.test.ts @@ -0,0 +1,82 @@ +import { EventEmitter } from 'node:events' +import { describe, expect, it } from 'vitest' +import { readApnsStreamResponse, type ApnsResponseStream } from './apns-stream-response.js' + +type FakeStream = ApnsResponseStream & { + sentBody: string | null + destroyedWith: Error | null + fireTimeout(): void +} + +function fakeApnsStream(): FakeStream { + const emitter = new EventEmitter() as FakeStream + emitter.sentBody = null + emitter.destroyedWith = null + let onTimeout: (() => void) | null = null + emitter.setTimeout = (_ms, callback) => { + onTimeout = callback + } + emitter.destroy = (error?: Error) => { + emitter.destroyedWith = error ?? null + if (error) emitter.emit('error', error) + } + emitter.end = (body: string) => { + emitter.sentBody = body + } + emitter.fireTimeout = () => onTimeout?.() + return emitter +} + +describe('apns stream response', () => { + it('resolves with the status and the concatenated body', async () => { + const stream = fakeApnsStream() + const pending = readApnsStreamResponse(stream, '{"aps":{}}') + expect(stream.sentBody).toBe('{"aps":{}}') + stream.emit('response', { ':status': '200' }) + stream.emit('data', Buffer.from('{"re')) + stream.emit('data', Buffer.from('ason":"ok"}')) + stream.emit('end') + await expect(pending).resolves.toEqual({ status: 200, body: '{"reason":"ok"}' }) + }) + + it('rejects when the peer resets the stream without an end or an error', async () => { + const stream = fakeApnsStream() + const pending = readApnsStreamResponse(stream, 'body') + stream.emit('response', { ':status': '200' }) + // NGHTTP2_NO_ERROR: node emits only 'close', so nothing else would settle. + stream.emit('close') + await expect(pending).rejects.toThrow('apns_stream_closed') + }) + + it('keeps the resolved response when close follows a completed end', async () => { + const stream = fakeApnsStream() + const pending = readApnsStreamResponse(stream, 'body') + stream.emit('response', { ':status': '410' }) + stream.emit('end') + stream.emit('close') + await expect(pending).resolves.toEqual({ status: 410, body: '' }) + }) + + it('keeps the original error when close follows a stream error', async () => { + const stream = fakeApnsStream() + const pending = readApnsStreamResponse(stream, 'body') + stream.emit('error', new Error('socket_hang_up')) + stream.emit('close') + await expect(pending).rejects.toThrow('socket_hang_up') + }) + + it('destroys the stream on timeout and surfaces the timeout error', async () => { + const stream = fakeApnsStream() + const pending = readApnsStreamResponse(stream, 'body', 10) + stream.fireTimeout() + await expect(pending).rejects.toThrow('apns_timeout') + expect(stream.destroyedWith?.message).toBe('apns_timeout') + }) + + it('reports a missing status header as zero rather than NaN', async () => { + const stream = fakeApnsStream() + const pending = readApnsStreamResponse(stream, 'body') + stream.emit('end') + await expect(pending).resolves.toEqual({ status: 0, body: '' }) + }) +}) diff --git a/cloud/apps/push/src/apns-stream-response.ts b/cloud/apps/push/src/apns-stream-response.ts new file mode 100644 index 00000000000..9fa189304e8 --- /dev/null +++ b/cloud/apps/push/src/apns-stream-response.ts @@ -0,0 +1,53 @@ +import type { EventEmitter } from 'node:events' +import { providerRetryAfter } from './provider-retry-delay.js' +import { constants } from 'node:http2' + +export type ApnsResponse = { status: number; body: string; retryAfterMs?: number } + +// The subset of ClientHttp2Stream this module drives, so a fake emitter can +// stand in for a real APNs stream in tests. +export type ApnsResponseStream = EventEmitter & { + setTimeout(ms: number, callback: () => void): void + destroy(error?: Error): void + end(body: string): void +} + +export const APNS_REQUEST_TIMEOUT_MS = 10_000 + +export function readApnsStreamResponse( + stream: ApnsResponseStream, + body: string, + timeoutMs = APNS_REQUEST_TIMEOUT_MS +): Promise { + return new Promise((resolve, reject) => { + let settled = false + const settle = (run: () => void): void => { + if (settled) return + settled = true + run() + } + let status = 0 + let retryAfterMs: number | undefined + const chunks: Buffer[] = [] + stream.setTimeout(timeoutMs, () => stream.destroy(new Error('apns_timeout'))) + stream.on('response', (headers: Record) => { + status = Number(headers[constants.HTTP2_HEADER_STATUS] ?? 0) + retryAfterMs = providerRetryAfter(String(headers['retry-after'] ?? '')) + }) + stream.on('data', (chunk: Buffer) => chunks.push(chunk)) + stream.on('error', (error: Error) => settle(() => reject(error))) + stream.on('end', () => + settle(() => + resolve({ + status, + body: Buffer.concat(chunks).toString('utf8'), + ...(retryAfterMs === undefined ? {} : { retryAfterMs }) + }) + ) + ) + // A peer reset with NGHTTP2_NO_ERROR emits neither 'end' nor 'error', which + // would leave the worker's delivery pending for the life of the process. + stream.on('close', () => settle(() => reject(new Error('apns_stream_closed')))) + stream.end(body) + }) +} diff --git a/cloud/apps/push/src/apns-topic-recovery.test.ts b/cloud/apps/push/src/apns-topic-recovery.test.ts new file mode 100644 index 00000000000..583d4536b5a --- /dev/null +++ b/cloud/apps/push/src/apns-topic-recovery.test.ts @@ -0,0 +1,48 @@ +import { expect, it } from 'vitest' +import { createPushHostKeypair } from './host-challenge-answering.test-fixture.js' +import { + APNS_TOKEN, + createPushServerHarness, + notification +} from './push-server-harness.test-fixture.js' + +it('delivers again after a topic error without re-registering the phone', async () => { + const h = await createPushServerHarness() + try { + const token = await h.signIn(createPushHostKeypair(71)) + const registered = await h.post( + '/v1/devices', + { + v: 1, + deviceId: 'phone', + platform: 'ios', + token: APNS_TOKEN, + apnsEnvironment: 'sandbox' + }, + token + ) + const { registrationId } = (await registered.json()) as { registrationId: string } + h.setApnsResponse({ status: 400, body: JSON.stringify({ reason: 'DeviceTokenNotForTopic' }) }) + for (const seq of [1, 2]) { + const sent = await h.post( + '/v1/send', + { + v: 1, + registrationIds: [registrationId], + notification: notification({ notificationId: `topic-${seq}`, notificationSeq: seq }) + }, + token + ) + expect(await sent.json()).toEqual({ results: [{ registrationId, status: 'queued' }] }) + await h.flushDeliveries() + if (seq === 1) { + expect(h.server.observability.consume().delivery_error).toBe(1) + h.setApnsResponse({ status: 200, body: '' }) + } + } + expect(h.apnsRequests).toHaveLength(2) + expect(h.server.observability.consume().delivery_sent).toBe(1) + } finally { + await h.close() + } +}) diff --git a/cloud/apps/push/src/canonical-base64.ts b/cloud/apps/push/src/canonical-base64.ts new file mode 100644 index 00000000000..e13ea982cb6 --- /dev/null +++ b/cloud/apps/push/src/canonical-base64.ts @@ -0,0 +1,9 @@ +// Rejects the many base64 spellings of the same bytes: a non-canonical +// encoding would change the transcript the host signs without changing the key. +export function decodeCanonicalBase64(value: string, expectedBytes: number): Buffer | null { + if (!/^(?:[A-Za-z0-9+/]{4})*(?:[A-Za-z0-9+/]{2}==|[A-Za-z0-9+/]{3}=)?$/.test(value)) return null + const decoded = Buffer.from(value, 'base64') + return decoded.byteLength === expectedBytes && decoded.toString('base64') === value + ? decoded + : null +} diff --git a/cloud/apps/push/src/client-ip-rate-limit.test.ts b/cloud/apps/push/src/client-ip-rate-limit.test.ts new file mode 100644 index 00000000000..85a88f7d1cd --- /dev/null +++ b/cloud/apps/push/src/client-ip-rate-limit.test.ts @@ -0,0 +1,169 @@ +import { PUSH_LIMITS } from '@orca-cloud/push-contract' +import { Hono } from 'hono' +import { describe, expect, it, vi } from 'vitest' +import { ClientIpRateLimiter, clientIpRateLimit } from './client-ip-rate-limit.js' + +const CAPACITY = PUSH_LIMITS.unauthenticatedRequestsPerMinutePerIp + +function limiterApp(limiter: ClientIpRateLimiter, trustedProxyHops = 0): Hono { + const app = new Hono() + app.post('/probe', clientIpRateLimit(limiter, { trustedProxyHops }), (context) => + context.json({ ok: true }) + ) + return app +} + +describe('client ip rate limiter', () => { + it('admits exactly the per-minute allowance and refuses the next request', () => { + const limiter = new ClientIpRateLimiter({ now: () => 1_000 }) + for (let index = 0; index < CAPACITY; index++) { + expect(limiter.allow('203.0.113.7')).toBe(true) + } + expect(limiter.allow('203.0.113.7')).toBe(false) + }) + + it('keeps one client ip from spending another one budget', () => { + const limiter = new ClientIpRateLimiter({ now: () => 1_000 }) + for (let index = 0; index < CAPACITY; index++) limiter.allow('203.0.113.7') + expect(limiter.allow('203.0.113.7')).toBe(false) + expect(limiter.allow('198.51.100.9')).toBe(true) + }) + + it('refills over the window rather than resetting on a boundary', () => { + let clock = 1_000 + const limiter = new ClientIpRateLimiter({ now: () => clock }) + for (let index = 0; index < CAPACITY; index++) limiter.allow('203.0.113.7') + expect(limiter.allow('203.0.113.7')).toBe(false) + + // Half a window buys back half the allowance, no more. + clock += 30_000 + for (let index = 0; index < CAPACITY / 2; index++) { + expect(limiter.allow('203.0.113.7')).toBe(true) + } + expect(limiter.allow('203.0.113.7')).toBe(false) + }) + + it('bounds what it remembers when a flood of distinct ips arrives', () => { + let clock = 1_000 + const limiter = new ClientIpRateLimiter({ now: () => clock, maxTrackedIps: 8 }) + for (let index = 0; index < 200; index++) { + clock += 1 + limiter.allow(`198.51.100.${index}`) + } + expect(limiter.trackedIpCount()).toBeLessThanOrEqual(8) + }) + + it('evicts the least recently used bucket without scanning the map', () => { + const limiter = new ClientIpRateLimiter({ capacity: 1, maxTrackedIps: 2, now: () => 1_000 }) + limiter.allow('old') + limiter.allow('recent') + expect(limiter.allow('old')).toBe(false) + const entries = vi.spyOn(Map.prototype, 'entries') + const iterator = vi.spyOn(Map.prototype, Symbol.iterator) + try { + limiter.allow('new') + expect(entries).not.toHaveBeenCalled() + expect(iterator).not.toHaveBeenCalled() + } finally { + entries.mockRestore() + iterator.mockRestore() + } + expect(limiter.available('old')).toBe(false) + expect(limiter.available('recent')).toBe(true) + expect(limiter.trackedIpCount()).toBe(2) + }) + + it('answers 429 with a rate_limited body once the bucket is empty', async () => { + const app = limiterApp(new ClientIpRateLimiter({ now: () => 1_000 })) + const headers = { 'x-forwarded-for': '10.0.0.1, 10.0.0.2, 203.0.113.7' } + for (let index = 0; index < CAPACITY; index++) { + expect((await app.request('/probe', { method: 'POST', headers })).status).toBe(200) + } + const limited = await app.request('/probe', { method: 'POST', headers }) + expect(limited.status).toBe(429) + expect(await limited.json()).toEqual({ error: 'rate_limited' }) + }) + + it('buckets on the last forwarded hop, the only one the platform appended', async () => { + const app = limiterApp(new ClientIpRateLimiter({ now: () => 1_000 })) + for (let index = 0; index < CAPACITY; index++) { + await app.request('/probe', { + method: 'POST', + headers: { 'x-forwarded-for': `10.0.0.${index}, 203.0.113.7` } + }) + } + const sameClient = await app.request('/probe', { + method: 'POST', + headers: { 'x-forwarded-for': '10.9.9.9, 203.0.113.7' } + }) + expect(sameClient.status).toBe(429) + const otherClient = await app.request('/probe', { + method: 'POST', + headers: { 'x-forwarded-for': '10.0.0.1, 198.51.100.9' } + }) + expect(otherClient.status).toBe(200) + }) + + it('gives a spoofed left-most hop no escape from the caller own bucket', async () => { + const app = limiterApp(new ClientIpRateLimiter({ now: () => 1_000 })) + // A caller that rewrites its own x-forwarded-for on every request still ends + // up behind the one value Cloud Run appended. + for (let index = 0; index < CAPACITY; index++) { + const allowed = await app.request('/probe', { + method: 'POST', + headers: { 'x-forwarded-for': `198.51.100.${index}, 203.0.113.7` } + }) + expect(allowed.status).toBe(200) + } + const spoofed = await app.request('/probe', { + method: 'POST', + headers: { 'x-forwarded-for': '198.51.100.250, 10.1.1.1, 203.0.113.7' } + }) + expect(spoofed.status).toBe(429) + }) + + it('skips the configured trusted proxies when counting from the right', async () => { + const app = limiterApp(new ClientIpRateLimiter({ now: () => 1_000, capacity: 1 }), 1) + // , , : one trusted hop after the client. + const headers = { 'x-forwarded-for': '203.0.113.7, 10.0.0.1' } + expect((await app.request('/probe', { method: 'POST', headers })).status).toBe(200) + expect((await app.request('/probe', { method: 'POST', headers })).status).toBe(429) + expect( + ( + await app.request('/probe', { + method: 'POST', + headers: { 'x-forwarded-for': '198.51.100.9, 10.0.0.1' } + }) + ).status + ).toBe(200) + }) + + it('trusts nothing when the header is shorter than the configured depth', async () => { + const app = limiterApp(new ClientIpRateLimiter({ now: () => 1_000, capacity: 1 }), 1) + // Only one hop, so the client value the depth points at does not exist. + const headers = { 'x-forwarded-for': '203.0.113.7' } + expect((await app.request('/probe', { method: 'POST', headers })).status).toBe(200) + expect( + ( + await app.request('/probe', { + method: 'POST', + headers: { 'x-forwarded-for': '198.51.100.9' } + }) + ).status + ).toBe(429) + }) + + it('ignores spoofable x-real-ip and uses a single shared bucket', async () => { + const app = limiterApp(new ClientIpRateLimiter({ now: () => 1_000, capacity: 1 })) + expect( + (await app.request('/probe', { method: 'POST', headers: { 'x-real-ip': '198.51.100.9' } })) + .status + ).toBe(200) + expect( + (await app.request('/probe', { method: 'POST', headers: { 'x-real-ip': '203.0.113.7' } })) + .status + ).toBe(429) + expect((await app.request('/probe', { method: 'POST' })).status).toBe(429) + expect((await app.request('/probe', { method: 'POST' })).status).toBe(429) + }) +}) diff --git a/cloud/apps/push/src/client-ip-rate-limit.ts b/cloud/apps/push/src/client-ip-rate-limit.ts new file mode 100644 index 00000000000..18518a2d316 --- /dev/null +++ b/cloud/apps/push/src/client-ip-rate-limit.ts @@ -0,0 +1,95 @@ +import { PUSH_LIMITS } from '@orca-cloud/push-contract' +import type { Context, MiddlewareHandler } from 'hono' + +const REFILL_WINDOW_MS = 60_000 +const MAX_TRACKED_IPS = 10_000 +const UNKNOWN_CLIENT_IP = 'unknown' + +export type ClientIpRateLimiterOptions = { + capacity?: number + windowMs?: number + maxTrackedIps?: number + now?: () => number +} + +type Bucket = { tokens: number; updatedAt: number } + +// Read x-forwarded-for from the right. Cloud Run appends the connecting peer, +// so the last value is the only one it wrote; everything to its left is +// whatever the caller sent and can be a fresh forgery on every request. +// trustedProxyHops is how many appenders sit between Cloud Run and the client +// (0 today, 1 once a load balancer fronts it). A header too short for that +// depth is not trusted at all and falls through to the shared bucket, which +// throttles rather than opens. +export function readClientIp(context: Context, trustedProxyHops = 0): string { + const hops = + context.req + .header('x-forwarded-for') + ?.split(',') + .map((hop) => hop.trim()) + .filter((hop) => hop.length > 0) ?? [] + const client = hops[hops.length - 1 - trustedProxyHops] + return client ?? UNKNOWN_CLIENT_IP +} + +// Per-instance admission avoids a database round trip; capacity scales with instance count. +export class ClientIpRateLimiter { + private readonly buckets = new Map() + private readonly capacity: number + private readonly windowMs: number + private readonly maxTrackedIps: number + private readonly now: () => number + + constructor(options: ClientIpRateLimiterOptions = {}) { + this.capacity = options.capacity ?? PUSH_LIMITS.unauthenticatedRequestsPerMinutePerIp + this.windowMs = options.windowMs ?? REFILL_WINDOW_MS + this.maxTrackedIps = options.maxTrackedIps ?? MAX_TRACKED_IPS + this.now = options.now ?? Date.now + } + + available(clientIp: string): boolean { + return this.tokensAt(this.buckets.get(clientIp), this.now()) >= 1 + } + + allow(clientIp: string): boolean { + const now = this.now() + const tokens = this.tokensAt(this.buckets.get(clientIp), now) + this.buckets.delete(clientIp) + this.buckets.set(clientIp, { tokens: tokens < 1 ? tokens : tokens - 1, updatedAt: now }) + if (this.buckets.size > this.maxTrackedIps) { + const oldest = this.buckets.keys().next().value + if (oldest !== undefined) this.buckets.delete(oldest) + } + return tokens >= 1 + } + + trackedIpCount(): number { + return this.buckets.size + } + + private tokensAt(bucket: Bucket | undefined, now: number): number { + if (!bucket) return this.capacity + const refilled = ((now - bucket.updatedAt) * this.capacity) / this.windowMs + return Math.min(this.capacity, bucket.tokens + Math.max(0, refilled)) + } +} + +export type ClientIpRateLimitOptions = { + trustedProxyHops?: number + onLimited?: () => void +} + +export function clientIpRateLimit( + limiter: ClientIpRateLimiter, + options: ClientIpRateLimitOptions = {} +): MiddlewareHandler { + const trustedProxyHops = options.trustedProxyHops ?? 0 + return async (context, next) => { + if (!limiter.allow(readClientIp(context, trustedProxyHops))) { + options.onLimited?.() + return context.json({ error: 'rate_limited' }, 429) + } + await next() + return + } +} diff --git a/cloud/apps/push/src/config.test.ts b/cloud/apps/push/src/config.test.ts new file mode 100644 index 00000000000..84efe7db7d7 --- /dev/null +++ b/cloud/apps/push/src/config.test.ts @@ -0,0 +1,109 @@ +import { generateKeyPairSync } from 'node:crypto' +import { PUSH_DEFAULTS } from '@orca-cloud/push-contract' +import { describe, expect, it } from 'vitest' +import { loadPushConfig, PUSH_DATABASE_POOL_MAX } from './config.js' + +function apnsKeyPem(): string { + return generateKeyPairSync('ec', { + namedCurve: 'P-256', + privateKeyEncoding: { type: 'pkcs8', format: 'pem' }, + publicKeyEncoding: { type: 'spki', format: 'pem' } + }).privateKey +} + +const MINIMAL = { + ORCA_PUSH_PUBLIC_URL: 'https://push.onorca.dev', + ORCA_PUSH_FCM_PROJECT_ID: 'onorca-cloud' +} + +describe('push gateway config', () => { + it('applies the documented defaults', () => { + expect(loadPushConfig(MINIMAL)).toEqual({ + mode: 'active', + port: 8080, + publicUrl: 'https://push.onorca.dev', + databaseUrl: undefined, + dataDir: './data/push', + databasePoolMax: PUSH_DATABASE_POOL_MAX, + apns: undefined, + apnsTopic: PUSH_DEFAULTS.apnsTopic, + fcmProjectId: 'onorca-cloud', + trustedProxyHops: 0 + }) + }) + + it('reads a full APNs credential and the overridable knobs', () => { + const keyPem = apnsKeyPem() + const config = loadPushConfig({ + ...MINIMAL, + PORT: '9090', + ORCA_PUSH_DATABASE_URL: 'postgres://localhost/orca_push', + ORCA_PUSH_DATA_DIR: '/var/lib/push', + ORCA_PUSH_APNS_KEY: keyPem, + ORCA_PUSH_APNS_KEY_ID: 'ABCDE12345', + ORCA_PUSH_APPLE_TEAM_ID: 'TEAM123456', + ORCA_PUSH_APNS_TOPIC: 'com.stably.orca.mobile.dev', + ORCA_PUSH_FCM_PROJECT_ID: 'onorca-staging', + ORCA_PUSH_TRUSTED_PROXY_HOPS: '1' + }) + expect(config).toMatchObject({ + port: 9090, + databaseUrl: 'postgres://localhost/orca_push', + dataDir: '/var/lib/push', + apns: { keyPem, keyId: 'ABCDE12345', teamId: 'TEAM123456' }, + apnsTopic: 'com.stably.orca.mobile.dev', + trustedProxyHops: 1, + fcmProjectId: 'onorca-staging' + }) + }) + + it('requires an explicit FCM project instead of silently targeting production', () => { + expect(() => loadPushConfig({ ...MINIMAL, ORCA_PUSH_FCM_PROJECT_ID: undefined })).toThrow() + expect(() => loadPushConfig({ ...MINIMAL, ORCA_PUSH_FCM_PROJECT_ID: ' ' })).toThrow() + }) + + it('refuses a partial APNs credential', () => { + expect(() => loadPushConfig({ ...MINIMAL, ORCA_PUSH_APNS_KEY: apnsKeyPem() })).toThrow( + 'configured together' + ) + expect(() => + loadPushConfig({ + ...MINIMAL, + ORCA_PUSH_APNS_KEY: 'not-a-pem', + ORCA_PUSH_APNS_KEY_ID: 'ABCDE12345', + ORCA_PUSH_APPLE_TEAM_ID: 'TEAM123456' + }) + ).toThrow('PEM text') + }) + + it('requires a canonical HTTPS origin outside loopback', () => { + expect(() => + loadPushConfig({ ...MINIMAL, ORCA_PUSH_PUBLIC_URL: 'https://push.onorca.dev/v1' }) + ).toThrow('must be an origin') + expect(() => + loadPushConfig({ ...MINIMAL, ORCA_PUSH_PUBLIC_URL: 'http://push.onorca.dev' }) + ).toThrow('must use HTTPS') + expect( + loadPushConfig({ ...MINIMAL, ORCA_PUSH_PUBLIC_URL: 'http://localhost:8080' }).publicUrl + ).toBe('http://localhost:8080') + }) + + it('treats an empty optional variable as unset', () => { + expect( + loadPushConfig({ ...MINIMAL, ORCA_PUSH_DATABASE_URL: '', ORCA_PUSH_APNS_KEY_ID: '' }) + ).toMatchObject({ databaseUrl: undefined, apns: undefined }) + }) +}) + +it('treats blank defaulted environment settings as absent', () => { + const blanks = Object.fromEntries( + [ + 'PORT', + 'ORCA_PUSH_DATA_DIR', + 'ORCA_PUSH_APNS_TOPIC', + 'ORCA_PUSH_DATABASE_POOL_MAX', + 'ORCA_PUSH_TRUSTED_PROXY_HOPS' + ].map((key) => [key, ' ']) + ) + expect(loadPushConfig({ ...MINIMAL, ...blanks })).toEqual(loadPushConfig(MINIMAL)) +}) diff --git a/cloud/apps/push/src/config.ts b/cloud/apps/push/src/config.ts new file mode 100644 index 00000000000..2ed0d2111f1 --- /dev/null +++ b/cloud/apps/push/src/config.ts @@ -0,0 +1,108 @@ +import { PUSH_DEFAULTS } from '@orca-cloud/push-contract' +import { z } from 'zod' + +export const PUSH_DATABASE_POOL_MAX = 10 + +const OptionalTextSchema = z.preprocess( + (value) => (value === '' ? undefined : value), + z.string().min(1).optional() +) + +const EnvSchema = z.object({ + ORCA_PUSH_MODE: z.enum(['active', 'validation']).default('active'), + PORT: z.coerce.number().int().positive().default(8080), + ORCA_PUSH_PUBLIC_URL: z.string().url(), + ORCA_PUSH_DATABASE_URL: OptionalTextSchema, + ORCA_PUSH_DATA_DIR: z.string().min(1).default('./data/push'), + ORCA_PUSH_DATABASE_POOL_MAX: z.coerce.number().int().positive().max(100).optional(), + ORCA_PUSH_APNS_KEY: OptionalTextSchema, + ORCA_PUSH_APNS_KEY_ID: z.preprocess( + (value) => (value === '' ? undefined : value), + z + .string() + .regex(/^[A-Z0-9]{10}$/) + .optional() + ), + ORCA_PUSH_APPLE_TEAM_ID: z.preprocess( + (value) => (value === '' ? undefined : value), + z + .string() + .regex(/^[A-Z0-9]{10}$/) + .optional() + ), + ORCA_PUSH_APNS_TOPIC: z.string().min(1).max(255).default(PUSH_DEFAULTS.apnsTopic), + ORCA_PUSH_FCM_PROJECT_ID: z.string().regex(/^[a-z0-9-]{4,64}$/), + // How many proxies append to x-forwarded-for after the client. 0 is Cloud Run + // alone; raise it to 1 when a load balancer fronts the service. + ORCA_PUSH_TRUSTED_PROXY_HOPS: z.coerce.number().int().nonnegative().max(8).default(0) +}) + +export type ApnsCredentials = { keyPem: string; keyId: string; teamId: string } + +export type PushConfig = { + mode: 'active' | 'validation' + port: number + publicUrl: string + databaseUrl?: string + dataDir: string + databasePoolMax: number + apns?: ApnsCredentials + apnsTopic: string + fcmProjectId: string + trustedProxyHops: number +} + +function canonicalOrigin(value: string, name: string): string { + const url = new URL(value) + if (url.origin !== value || url.pathname !== '/') throw new Error(`${name} must be an origin`) + const loopback = ['127.0.0.1', 'localhost', '::1', '[::1]'].includes(url.hostname) + if (url.protocol !== 'https:' && !(loopback && url.protocol === 'http:')) { + throw new Error(`${name} must use HTTPS outside loopback development`) + } + return value +} + +// The APNs key, key id, and team id are one credential; a partial set would +// pass startup and then fail every iOS send at runtime. +function readApnsCredentials(parsed: z.infer): ApnsCredentials | undefined { + const parts = [ + parsed.ORCA_PUSH_APNS_KEY, + parsed.ORCA_PUSH_APNS_KEY_ID, + parsed.ORCA_PUSH_APPLE_TEAM_ID + ] + const present = parts.filter((value) => value !== undefined).length + if (present === 0) return undefined + if (present !== parts.length) { + throw new Error('APNs key, key id, and team id must be configured together') + } + const keyPem = parsed.ORCA_PUSH_APNS_KEY! + if (!keyPem.includes('-----BEGIN')) throw new Error('ORCA_PUSH_APNS_KEY must be PEM text') + return { + keyPem, + keyId: parsed.ORCA_PUSH_APNS_KEY_ID!, + teamId: parsed.ORCA_PUSH_APPLE_TEAM_ID! + } +} + +export function loadPushConfig(env: NodeJS.ProcessEnv = process.env): PushConfig { + const parsed = EnvSchema.parse( + Object.fromEntries( + Object.entries(env).map(([key, value]) => [ + key, + key !== 'ORCA_PUSH_MODE' && value?.trim() === '' ? undefined : value + ]) + ) + ) + return { + mode: parsed.ORCA_PUSH_MODE, + port: parsed.PORT, + publicUrl: canonicalOrigin(parsed.ORCA_PUSH_PUBLIC_URL, 'ORCA_PUSH_PUBLIC_URL'), + databaseUrl: parsed.ORCA_PUSH_DATABASE_URL, + dataDir: parsed.ORCA_PUSH_DATA_DIR, + databasePoolMax: parsed.ORCA_PUSH_DATABASE_POOL_MAX ?? PUSH_DATABASE_POOL_MAX, + apns: readApnsCredentials(parsed), + apnsTopic: parsed.ORCA_PUSH_APNS_TOPIC, + fcmProjectId: parsed.ORCA_PUSH_FCM_PROJECT_ID, + trustedProxyHops: parsed.ORCA_PUSH_TRUSTED_PROXY_HOPS + } +} diff --git a/cloud/apps/push/src/desktop-host-proof-interop.test.ts b/cloud/apps/push/src/desktop-host-proof-interop.test.ts new file mode 100644 index 00000000000..654423b0de8 --- /dev/null +++ b/cloud/apps/push/src/desktop-host-proof-interop.test.ts @@ -0,0 +1,47 @@ +import { describe, expect, it } from 'vitest' +import { createHmac } from 'node:crypto' +import vector from '../../../packages/push-contract/src/push-host-proof-vector.json' with { type: 'json' } +import { answerPushHostChallenge, createPushHostKeypair } from './host-challenge-answering.test-fixture.js' +import { PushHostChallengeStore } from './host-challenge-store.js' +import { deriveHostFingerprint } from './host-fingerprint.js' +import { openInMemoryPushDatabase } from './push-database.js' + +// Why: the desktop answers challenges in a workspace this one cannot import. +// Both sides replay the same checked-in vector, so a transcript drift on +// either side fails in that side's own suite. +describe('desktop host proof interop', () => { + it('the checked-in vector answers to the same proof the fixture host computes', () => { + const secretKey = new Uint8Array(Buffer.from(vector.hostSecretKeyB64, 'base64')) + const keypair = { publicKey: new Uint8Array(Buffer.from(vector.hostPublicKeyB64, 'base64')), secretKey } + expect(deriveHostFingerprint(keypair.publicKey)).toBe(vector.hostFingerprint) + const proof = answerPushHostChallenge(vector.challenge, { + gatewayOrigin: vector.gatewayOrigin, + keypair, + now: () => vector.issuedAt + 1_000 + }) + const expected = createHmac('sha256', Buffer.from(vector.challengeSecretB64, 'base64')) + .update(Buffer.from('orca-push-host-proof/v1\0ack\0')) + .update(Buffer.from(vector.transcriptB64, 'base64')) + .digest('base64') + expect(proof).toBe(expected) + }) + + it('a live challenge from the store round-trips through the fixture host once', async () => { + const database = await openInMemoryPushDatabase() + const store = new PushHostChallengeStore(database, vector.gatewayOrigin) + const keypair = createPushHostKeypair(11) + const challenge = await store.issue(Buffer.from(keypair.publicKey).toString('base64')) + expect(challenge).not.toBeNull() + const proof = answerPushHostChallenge(challenge!, { gatewayOrigin: vector.gatewayOrigin, keypair }) + expect(proof).not.toBeNull() + expect(await store.verify(challenge!.challengeId, proof!)).toEqual({ + ok: true, + hostFingerprint: deriveHostFingerprint(keypair.publicKey) + }) + expect(await store.verify(challenge!.challengeId, proof!)).toEqual({ + ok: false, + reason: 'already_consumed' + }) + await database.close() + }) +}) diff --git a/cloud/apps/push/src/device-registration-delete-race.test.ts b/cloud/apps/push/src/device-registration-delete-race.test.ts new file mode 100644 index 00000000000..2fe51e30516 --- /dev/null +++ b/cloud/apps/push/src/device-registration-delete-race.test.ts @@ -0,0 +1,88 @@ +import { expect, it, vi } from 'vitest' +import { openPushDatabase, type PushDatabase } from './push-database.js' +import { PushDeviceRegistryStore } from './device-registry-store.js' + +const databaseUrl = process.env.ORCA_PUSH_TEST_DATABASE_URL +it.skipIf(!databaseUrl)( + 'serializes deletion with a registration that has already read its row', + async () => { + if (!process.env.CI && new URL(databaseUrl!).port !== '55440') + throw new Error('isolated_postgres_port_required') + const database = await openPushDatabase({ + databaseUrl, + dataDir: '', + poolMax: 4, + applicationName: 'push-delete-race' + }) + let release!: () => void + const paused = new Promise((resolve) => { + release = resolve + }) + let read = false + let pause = false + const wrapped: PushDatabase = { + dialect: database.dialect, + query: database.query.bind(database), + close: database.close.bind(database), + lockQuotaScope: database.lockQuotaScope.bind(database), + transaction: (run) => + database.transaction((tx) => + run({ + dialect: tx.dialect, + close: tx.close.bind(tx), + transaction: tx.transaction.bind(tx), + lockQuotaScope: tx.lockQuotaScope.bind(tx), + query: async (sql, params) => { + const rows = await tx.query(sql, params) + if (pause && sql.startsWith('SELECT registration_id FROM push_devices')) { + read = true + await paused + } + return rows + } + }) + ) + } + const devices = new PushDeviceRegistryStore(wrapped) + const input = { + hostFingerprint: 'delete-race-host', + deviceId: 'phone', + platform: 'android' as const, + token: 'synthetic' + } + let registration: Promise | undefined + let deletion: Promise | undefined + try { + await database.query('DELETE FROM push_devices WHERE host_fingerprint = ?', [ + input.hostFingerprint + ]) + const first = await devices.upsert(input) + if (!first.ok) throw new Error('registration refused') + pause = true + registration = devices.upsert(input) + await vi.waitFor(() => expect(read).toBe(true)) + let deleted = false + deletion = devices.deleteOwned(input.hostFingerprint, first.registrationId).then((value) => { + deleted = true + return value + }) + await vi.waitFor(async () => { + const rows = await database.query( + "SELECT 1 FROM pg_stat_activity WHERE application_name = 'push-delete-race' AND wait_event_type = 'Lock'" + ) + expect(deleted || rows.length > 0).toBe(true) + }) + expect(deleted).toBe(false) + release() + expect(await registration).toEqual(first) + expect(await deletion).toBe(true) + } finally { + release() + await Promise.allSettled([registration, deletion]) + await database.query('DELETE FROM push_devices WHERE host_fingerprint = ?', [ + input.hostFingerprint + ]) + await database.close() + } + } +) diff --git a/cloud/apps/push/src/device-registry-store.test.ts b/cloud/apps/push/src/device-registry-store.test.ts new file mode 100644 index 00000000000..f325d7611b1 --- /dev/null +++ b/cloud/apps/push/src/device-registry-store.test.ts @@ -0,0 +1,191 @@ +import { PUSH_LIMITS } from '@orca-cloud/push-contract' +import { afterEach, beforeEach, describe, expect, it } from 'vitest' +import { PushDeviceRegistryStore, type PushDeviceUpsert } from './device-registry-store.js' +import { openInMemoryPushDatabase, type PushDatabase } from './push-database.js' + +const OWNER = 'abcdefghijklmnop' +const OTHER = 'ponmlkjihgfedcba' + +describe('push device registry store', () => { + let database: PushDatabase + let clock = 1_700_000_000_000 + let devices: PushDeviceRegistryStore + + beforeEach(async () => { + database = await openInMemoryPushDatabase() + clock = 1_700_000_000_000 + devices = new PushDeviceRegistryStore(database, () => clock) + }) + + afterEach(async () => { + await database.close() + }) + + async function upsertOk(input: PushDeviceUpsert): Promise { + const result = await devices.upsert(input) + if (!result.ok) throw new Error(`unexpected upsert refusal: ${result.reason}`) + return result.registrationId + } + + function androidDevice(deviceId: string): PushDeviceUpsert { + return { + hostFingerprint: OWNER, + deviceId, + platform: 'android', + token: `token-${deviceId}` + } + } + + it('keeps one registration per host and device while replacing the token', async () => { + const first = await upsertOk({ + hostFingerprint: OWNER, + deviceId: 'device-1', + platform: 'ios', + token: 'a'.repeat(64), + apnsEnvironment: 'sandbox' + }) + clock += 1_000 + const second = await upsertOk({ + hostFingerprint: OWNER, + deviceId: 'device-1', + platform: 'ios', + token: 'b'.repeat(64), + apnsEnvironment: 'production' + }) + expect(second).toBe(first) + const registration = await devices.findById(first) + expect(registration).toMatchObject({ + token: 'b'.repeat(64), + apnsEnvironment: 'production', + dead: false + }) + expect(await devices.list(OWNER)).toHaveLength(1) + }) + + it('revives a registration that a re-registered token replaces', async () => { + const registrationId = await upsertOk({ + hostFingerprint: OWNER, + deviceId: 'device-1', + platform: 'android', + token: 'token-one' + }) + await devices.markDead((await devices.findById(registrationId))!) + expect((await devices.findById(registrationId))?.dead).toBe(true) + await upsertOk({ + hostFingerprint: OWNER, + deviceId: 'device-1', + platform: 'android', + token: 'token-two' + }) + expect(await devices.findById(registrationId)).toMatchObject({ + token: 'token-two', + dead: false + }) + }) + + it('lets only the owning host delete a registration', async () => { + const registrationId = await upsertOk({ + hostFingerprint: OWNER, + deviceId: 'device-1', + platform: 'android', + token: 'token-one' + }) + expect(await devices.deleteOwned(OTHER, registrationId)).toBe(false) + expect(await devices.findById(registrationId)).not.toBeNull() + expect(await devices.deleteOwned(OWNER, registrationId)).toBe(true) + expect(await devices.findById(registrationId)).toBeNull() + }) + + it('scopes lookups and listings to the owning host', async () => { + const owned = await upsertOk({ + hostFingerprint: OWNER, + deviceId: 'device-1', + platform: 'android', + token: 'token-one' + }) + const foreign = await upsertOk({ + hostFingerprint: OTHER, + deviceId: 'device-2', + platform: 'android', + token: 'token-two' + }) + const found = await devices.findOwned(OWNER, [owned, foreign]) + expect([...found.keys()]).toEqual([owned]) + expect(await devices.list(OTHER)).toEqual([ + { registrationId: foreign, deviceId: 'device-2', platform: 'android', dead: false } + ]) + expect(await devices.findOwned(OWNER, [])).toEqual(new Map()) + }) + + it('refuses a new device once the host reaches its registration cap', async () => { + for (let index = 0; index < PUSH_LIMITS.maxDevicesPerHost; index++) { + await upsertOk(androidDevice(`device-${index}`)) + } + expect(await devices.upsert(androidDevice('one-too-many'))).toEqual({ + ok: false, + reason: 'too_many_devices' + }) + expect(await devices.list(OWNER)).toHaveLength(PUSH_LIMITS.maxDevicesPerHost) + }) + + it('still lets a capped host re-register a device it already owns', async () => { + for (let index = 0; index < PUSH_LIMITS.maxDevicesPerHost; index++) { + await upsertOk(androidDevice(`device-${index}`)) + } + const rotated = await devices.upsert({ ...androidDevice('device-0'), token: 'rotated-token' }) + expect(rotated.ok).toBe(true) + expect(await devices.list(OWNER)).toHaveLength(PUSH_LIMITS.maxDevicesPerHost) + }) + + it('frees a slot when a registration is deleted', async () => { + const first = await upsertOk(androidDevice('device-0')) + for (let index = 1; index < PUSH_LIMITS.maxDevicesPerHost; index++) { + await upsertOk(androidDevice(`device-${index}`)) + } + expect((await devices.upsert(androidDevice('extra'))).ok).toBe(false) + expect(await devices.deleteOwned(OWNER, first)).toBe(true) + expect((await devices.upsert(androidDevice('extra'))).ok).toBe(true) + }) + + it('counts the cap per host, not across the whole table', async () => { + for (let index = 0; index < PUSH_LIMITS.maxDevicesPerHost; index++) { + await upsertOk(androidDevice(`device-${index}`)) + } + expect((await devices.upsert(androidDevice('extra'))).ok).toBe(false) + expect( + (await devices.upsert({ ...androidDevice('device-0'), hostFingerprint: OTHER })).ok + ).toBe(true) + }) + + it('bounds list reads to the host device allowance', async () => { + // Straight past the per-host cap, so only the query LIMIT can bound this. + const rows = PUSH_LIMITS.maxDevicesPerHost + 5 + for (let index = 0; index < rows; index++) { + await database.query( + `INSERT INTO push_devices (registration_id, host_fingerprint, device_id, platform, token, + created_at, updated_at) + VALUES (?, ?, ?, ?, ?, ?, ?)`, + [`reg-${index}`, OWNER, `device-${index}`, 'android', 'token', clock + index, clock] + ) + } + expect(await devices.list(OWNER)).toHaveLength(PUSH_LIMITS.maxDevicesPerHost) + }) + + it('separates the same device id registered against two hosts', async () => { + const first = await upsertOk({ + hostFingerprint: OWNER, + deviceId: 'shared-device', + platform: 'ios', + token: 'a'.repeat(64), + apnsEnvironment: 'sandbox' + }) + const second = await upsertOk({ + hostFingerprint: OTHER, + deviceId: 'shared-device', + platform: 'ios', + token: 'c'.repeat(64), + apnsEnvironment: 'sandbox' + }) + expect(first).not.toBe(second) + }) +}) diff --git a/cloud/apps/push/src/device-registry-store.ts b/cloud/apps/push/src/device-registry-store.ts new file mode 100644 index 00000000000..1729f7d1b82 --- /dev/null +++ b/cloud/apps/push/src/device-registry-store.ts @@ -0,0 +1,177 @@ +import { randomUUID } from 'node:crypto' +import { + PUSH_LIMITS, + type ApnsEnvironment, + type PushDeviceSummary, + type PushPlatform +} from '@orca-cloud/push-contract' +import type { PushDatabase, SqlRow } from './push-database.js' + +const DEVICE_CAP_LOCK_PREFIX = 'orca-push-device-cap:' + +export type PushDeviceRegistration = { + registrationId: string + hostFingerprint: string + deviceId: string + platform: PushPlatform + token: string + apnsEnvironment?: ApnsEnvironment + dead: boolean +} + +export type PushDeviceUpsertResult = + | { ok: true; registrationId: string } + | { ok: false; reason: 'too_many_devices' } + +export type PushDeviceUpsert = { + hostFingerprint: string + deviceId: string + platform: PushPlatform + token: string + apnsEnvironment?: ApnsEnvironment +} + +function toRegistration(row: SqlRow): PushDeviceRegistration { + const apnsEnvironment = row.apns_environment + return { + registrationId: String(row.registration_id), + hostFingerprint: String(row.host_fingerprint), + deviceId: String(row.device_id), + platform: String(row.platform) as PushPlatform, + token: String(row.token), + ...(apnsEnvironment === null || apnsEnvironment === undefined + ? {} + : { apnsEnvironment: String(apnsEnvironment) as ApnsEnvironment }), + dead: row.dead_at !== null && row.dead_at !== undefined + } +} + +export class PushDeviceRegistryStore { + constructor( + private readonly database: PushDatabase, + private readonly now: () => number = Date.now + ) {} + + // The registration id is stable for a (host, device) pair so a re-registered + // phone keeps the id the desktop already persisted; only the token rotates. + async upsert(input: PushDeviceUpsert): Promise { + const now = this.now() + return await this.database.transaction(async (transaction) => { + // deviceId is caller-chosen, so counting and inserting must not interleave + // or a burst of new ids would walk straight past the cap. + await transaction.lockQuotaScope(`${DEVICE_CAP_LOCK_PREFIX}${input.hostFingerprint}`) + const [existing] = await transaction.query( + 'SELECT registration_id FROM push_devices WHERE host_fingerprint = ? AND device_id = ?', + [input.hostFingerprint, input.deviceId] + ) + if (existing) { + const registrationId = String(existing.registration_id) + await transaction.query( + `UPDATE push_devices + SET platform = ?, token = ?, apns_environment = ?, + dead_at = NULL, updated_at = ? + WHERE registration_id = ?`, + [input.platform, input.token, input.apnsEnvironment ?? null, now, registrationId] + ) + return { ok: true, registrationId } + } + const [countRow] = await transaction.query( + 'SELECT COUNT(*) AS devices FROM push_devices WHERE host_fingerprint = ?', + [input.hostFingerprint] + ) + if (Number(countRow?.devices ?? 0) >= PUSH_LIMITS.maxDevicesPerHost) { + return { ok: false, reason: 'too_many_devices' } + } + const registrationId = randomUUID() + await transaction.query( + `INSERT INTO push_devices + (registration_id, host_fingerprint, device_id, platform, token, apns_environment, + dead_at, created_at, updated_at) + VALUES (?, ?, ?, ?, ?, ?, NULL, ?, ?)`, + [ + registrationId, + input.hostFingerprint, + input.deviceId, + input.platform, + input.token, + input.apnsEnvironment ?? null, + now, + now + ] + ) + return { ok: true, registrationId } + }) + } + + async deleteOwned(hostFingerprint: string, registrationId: string): Promise { + return this.database.transaction(async (transaction) => { + await transaction.lockQuotaScope(`${DEVICE_CAP_LOCK_PREFIX}${hostFingerprint}`) + const [result] = await transaction.query( + 'DELETE FROM push_devices WHERE registration_id = ? AND host_fingerprint = ?', + [registrationId, hostFingerprint] + ) + return Number(result?.changes ?? 0) > 0 + }) + } + + async list(hostFingerprint: string): Promise { + const rows = await this.database.query( + // Bounded by the device-list response limit, so an + // oversized table degrades to a truncated list instead of a 500. + `SELECT registration_id, device_id, platform, dead_at + FROM push_devices WHERE host_fingerprint = ? ORDER BY created_at ASC LIMIT ?`, + [hostFingerprint, PUSH_LIMITS.maxDevicesPerHost] + ) + return rows.map((row) => ({ + registrationId: String(row.registration_id), + deviceId: String(row.device_id), + platform: String(row.platform) as PushPlatform, + dead: row.dead_at !== null && row.dead_at !== undefined + })) + } + + async findOwned( + hostFingerprint: string, + registrationIds: readonly string[] + ): Promise> { + if (registrationIds.length === 0) return new Map() + const placeholders = registrationIds.map(() => '?').join(', ') + const rows = await this.database.query( + `SELECT registration_id, host_fingerprint, device_id, platform, token, apns_environment, + dead_at + FROM push_devices + WHERE host_fingerprint = ? AND registration_id IN (${placeholders})`, + [hostFingerprint, ...registrationIds] + ) + return new Map( + rows.map((row) => { + const registration = toRegistration(row) + return [registration.registrationId, registration] + }) + ) + } + + async findById(registrationId: string): Promise { + const [row] = await this.database.query( + `SELECT registration_id, host_fingerprint, device_id, platform, token, apns_environment, + dead_at + FROM push_devices WHERE registration_id = ?`, + [registrationId] + ) + return row ? toRegistration(row) : null + } + + async markDead(observed: PushDeviceRegistration): Promise { + await this.database.query( + `UPDATE push_devices SET dead_at = ?, updated_at = ? WHERE registration_id = ? AND token = ? AND platform = ? AND COALESCE(apns_environment, '') = ?`, + [ + this.now(), + this.now(), + observed.registrationId, + observed.token, + observed.platform, + observed.apnsEnvironment ?? '' + ] + ) + } +} diff --git a/cloud/apps/push/src/durable-push-schema.ts b/cloud/apps/push/src/durable-push-schema.ts new file mode 100644 index 00000000000..770c2a1fbce --- /dev/null +++ b/cloud/apps/push/src/durable-push-schema.ts @@ -0,0 +1,45 @@ +export const DURABLE_PUSH_SCHEMA = ` +CREATE TABLE IF NOT EXISTS push_dismissed_events ( + host_fingerprint TEXT NOT NULL, + notification_epoch TEXT NOT NULL, + notification_id TEXT NOT NULL, + notification_seq BIGINT NOT NULL, + created_at BIGINT NOT NULL, + PRIMARY KEY(host_fingerprint, notification_epoch, notification_id) +); +CREATE INDEX IF NOT EXISTS push_dismissed_retention ON push_dismissed_events(created_at); +CREATE TABLE IF NOT EXISTS push_events ( + event_id TEXT PRIMARY KEY, + host_fingerprint TEXT NOT NULL, + kind TEXT NOT NULL, + fingerprint TEXT NOT NULL, + created_at BIGINT NOT NULL, + expires_at BIGINT NOT NULL +); +CREATE INDEX IF NOT EXISTS push_events_quota ON push_events(host_fingerprint, kind, created_at); +CREATE TABLE IF NOT EXISTS push_event_recipients ( + event_id TEXT NOT NULL, + registration_id TEXT NOT NULL, + created_at BIGINT NOT NULL, + PRIMARY KEY(event_id, registration_id) +); +CREATE TABLE IF NOT EXISTS push_delivery_batches ( + batch_id TEXT PRIMARY KEY, + host_fingerprint TEXT NOT NULL, + registration_id TEXT NOT NULL, + kind TEXT NOT NULL, + payload_json TEXT NOT NULL, + state TEXT NOT NULL, + due_at BIGINT NOT NULL, + expires_at BIGINT NOT NULL, + lease_token TEXT, + lease_until BIGINT NOT NULL, + attempts BIGINT NOT NULL, + created_at BIGINT NOT NULL +); +CREATE INDEX IF NOT EXISTS push_events_retention ON push_events(created_at); +CREATE INDEX IF NOT EXISTS push_recipients_retention ON push_event_recipients(created_at); +CREATE INDEX IF NOT EXISTS push_batches_expiry ON push_delivery_batches(expires_at); +CREATE INDEX IF NOT EXISTS push_batches_due ON push_delivery_batches(state, due_at); +CREATE INDEX IF NOT EXISTS push_batches_registration ON push_delivery_batches(registration_id, state); +` diff --git a/cloud/apps/push/src/durable-push-store.test.ts b/cloud/apps/push/src/durable-push-store.test.ts new file mode 100644 index 00000000000..136c0832211 --- /dev/null +++ b/cloud/apps/push/src/durable-push-store.test.ts @@ -0,0 +1,289 @@ +import { randomUUID } from 'node:crypto' +import pg from 'pg' +import { afterEach, describe, expect, it } from 'vitest' +import { openInMemoryPushDatabase, openPushDatabase, type PushDatabase } from './push-database.js' +import { DurablePushStore, DELIVERY_LEASE_MS } from './durable-push-store.js' +import type { PushNotification } from '@orca-cloud/push-contract' + +const cleanups: (() => Promise)[] = [] +afterEach(async () => { + await Promise.all(cleanups.splice(0).map((cleanup) => cleanup())) +}) +const notification = (seq: number, kind: 'alert' | 'dismiss' = 'alert'): PushNotification => ({ + notificationId: `notification-${seq}`, + notificationEpoch: 'epoch', + notificationSeq: seq, + source: 'agent-task-complete', + agentState: 'finished', + title: 'Done', + body: '', + kind +}) +async function fixture() { + const databaseUrl = + process.env.ORCA_PUSH_DURABLE_TEST_POSTGRES_URL ?? process.env.ORCA_PUSH_TEST_DATABASE_URL + if (databaseUrl && !process.env.CI && new URL(databaseUrl).port !== '55440') + throw new Error('isolated_postgres_port_required') + let db: PushDatabase + if (databaseUrl) { + const admin = new pg.Client({ connectionString: databaseUrl }) + await admin.connect() + const schema = `durable_${randomUUID().replaceAll('-', '')}` + let scoped: PushDatabase | undefined + cleanups.push(async () => { + try { + await scoped?.close() + } finally { + try { + await admin.query(`DROP SCHEMA IF EXISTS ${schema} CASCADE`) + } finally { + await admin.end() + } + } + }) + await admin.query(`CREATE SCHEMA ${schema}`) + const url = new URL(databaseUrl) + url.searchParams.set('options', `-c search_path=${schema}`) + db = scoped = await openPushDatabase({ databaseUrl: url.toString(), dataDir: '', poolMax: 4 }) + } else { + db = await openInMemoryPushDatabase() + cleanups.push(() => db.close()) + } + let now = 1_000_000 + const clock = () => now + return { + db, + store: new DurablePushStore(db, clock), + clock, + advance: (ms: number) => { + now += ms + } + } +} + +describe('durable push acceptance', () => { + it('counts a logical event once across phones and separates the 300/15min dismissal budget', async () => { + const { store, advance } = await fixture() + for (let i = 0; i < 300; i++) { + expect(await store.accept('host', 'phone1', notification(i))).toBe('queued') + expect(await store.accept('host', 'phone2', notification(i))).toBe('queued') + expect(await store.accept('host', 'phone1', notification(i, 'dismiss'))).toBe('queued') + } + expect(await store.accept('host', 'phone1', notification(300))).toBe('rate_limited') + expect(await store.accept('host', 'phone1', notification(300, 'dismiss'))).toBe('rate_limited') + expect(await store.accept('another-host', 'phone3', notification(300))).toBe('queued') + advance(15 * 60_000) + expect(await store.accept('host', 'phone1', notification(301))).toBe('queued') + }) + + it('queues one delivery per event and recovers work across service instances', async () => { + const { db, store, clock, advance } = await fixture() + await store.accept('host', 'phone', notification(1)) + const restarted = new DurablePushStore(db, clock) + await restarted.accept('host', 'phone', notification(1)) + advance(1) + await restarted.accept('host', 'phone', notification(2)) + const rows = await db.query( + "SELECT payload_json, due_at, created_at FROM push_delivery_batches WHERE registration_id = ? AND state = 'pending' ORDER BY created_at, batch_id", + ['phone'] + ) + expect(rows).toHaveLength(2) + expect(rows.map((row) => JSON.parse(String(row.payload_json)))).toEqual([ + notification(1), + notification(2) + ]) + expect(rows.every((row) => Number(row.due_at) >= Number(row.created_at))).toBe(true) + const delivery = await restarted.claim() + expect(delivery?.notification.notificationSeq).toBe(1) + expect(await store.claim()).toBeNull() + advance(DELIVERY_LEASE_MS) + const reclaimed = await store.claim() + expect(reclaimed?.id).toBe(delivery?.id) + expect(reclaimed?.lease).not.toBe(delivery?.lease) + await restarted.finish(delivery!) + expect(await store.claim()).toBeNull() + await store.finish(reclaimed!) + const second = await restarted.claim() + expect(second?.notification.notificationSeq).toBe(2) + await restarted.finish(second!) + expect(await restarted.claim()).toBeNull() + }) + + it('never extends expiry and refuses conflicting duplicate content', async () => { + const { store, advance } = await fixture() + await store.accept('host', 'phone', notification(1)) + expect(await store.accept('host', 'phone', { ...notification(1), body: 'changed' })).toBe( + 'error' + ) + const delivery = (await store.claim())! + await store.finish(delivery, 10 * 60_000) + advance(60_000) + expect(await store.claim()).toBeNull() + advance(5 * 60_000) + expect(await store.accept('host', 'phone', notification(1))).toBe('error') + }) + + it('orders a due retry before a fresh first attempt without delaying the retry', async () => { + const { store, advance } = await fixture() + await store.accept('host', 'phone', notification(1)) + const first = (await store.claim())! + await store.finish(first, 1000) + expect(await store.claim()).toBeNull() + + advance(1000) + await store.accept('host', 'phone', notification(2)) + const retry = (await store.claim())! + expect(retry.notification.notificationSeq).toBe(1) + await store.finish(retry) + const fresh = (await store.claim())! + expect(fresh?.notification.notificationSeq).toBe(2) + await store.finish(fresh!) + }) + + it('orders an expired first-attempt lease by creation time after a retry becomes due', async () => { + const { db, store, clock, advance } = await fixture() + await store.accept('host', 'phone', notification(1)) + const retry = (await store.claim())! + await store.finish(retry, 1000) + + advance(2000) + await db.query( + `INSERT INTO push_delivery_batches(batch_id, host_fingerprint, registration_id, kind, payload_json, state, due_at, expires_at, lease_until, attempts, created_at) + VALUES ('crashed-singleton', 'host', 'phone', 'alert', ?, 'pending', ?, ?, 0, 1, ?)`, + [JSON.stringify(notification(2)), clock() - 1, clock() + 300_000, clock()] + ) + const reclaimedRetry = (await store.claim())! + expect(reclaimedRetry.notification.notificationSeq).toBe(1) + await store.finish(reclaimedRetry) + const reclaimedCrash = (await store.claim())! + expect(reclaimedCrash.notification.notificationSeq).toBe(2) + await store.finish(reclaimedCrash) + }) + + it('rolls quota and payload back together if persistence fails', async () => { + const { db, store } = await fixture() + await db.query('ALTER TABLE push_delivery_batches RENAME TO push_delivery_batches_unavailable') + try { + const databaseUrl = + process.env.ORCA_PUSH_DURABLE_TEST_POSTGRES_URL ?? process.env.ORCA_PUSH_TEST_DATABASE_URL + if (databaseUrl) { + const concurrent = await openPushDatabase({ databaseUrl, dataDir: '' }) + try { + await expect( + concurrent.query('SELECT COUNT(*) FROM push_delivery_batches') + ).resolves.toHaveLength(1) + } finally { + await concurrent.close() + } + } + await expect(store.accept('host', 'phone', notification(1))).rejects.toThrow() + expect(await db.query('SELECT * FROM push_events')).toEqual([]) + expect(await db.query('SELECT * FROM push_event_recipients')).toEqual([]) + } finally { + await db.query( + 'ALTER TABLE push_delivery_batches_unavailable RENAME TO push_delivery_batches' + ) + } + }) +}) + +it('serializes concurrent instances at the quota boundary', async () => { + const { db, store, clock } = await fixture() + for (let seq = 0; seq < 299; seq++) await store.accept('host', 'phone', notification(seq)) + const second = new DurablePushStore(db, clock) + const results = await Promise.all( + Array.from({ length: 6 }, (_, index) => + (index % 2 ? store : second).accept('host', 'phone', notification(400 + index)) + ) + ) + expect(results.filter((result) => result === 'queued')).toHaveLength(1) + expect(results.filter((result) => result === 'rate_limited')).toHaveLength(5) +}) + +it('cancels unsent alerts and prevents an older replay after dismissal', async () => { + const { store } = await fixture() + const alert = notification(1) + await store.accept('host', 'phone', alert) + await store.accept('host', 'phone', { + ...notification(2, 'dismiss'), + notificationId: alert.notificationId + }) + const delivery = (await store.claim())! + expect(delivery.notification.kind).toBe('dismiss') + await store.finish(delivery) + expect(await store.claim()).toBeNull() + await store.accept('host', 'another-phone', alert) + expect(await store.claim()).toBeNull() +}) + +it('does not resurrect an in-flight alert after a dismissal and transient provider failure', async () => { + const { store, advance } = await fixture() + await store.accept('host', 'phone', notification(1)) + const inFlight = (await store.claim())! + await store.accept('host', 'phone', { + ...notification(2, 'dismiss'), + notificationId: notification(1).notificationId + }) + await store.finish(inFlight, 1000) + const dismissal = (await store.claim())! + expect(dismissal.notification.kind).toBe('dismiss') + await store.finish(dismissal) + advance(1000) + expect(await store.claim()).toBeNull() + expect(await store.pendingCount('phone')).toBe(0) +}) + +it.each([false, true])( + 'normalizes default alert kind (explicit first: %s)', + async (explicitFirst) => { + const { db, store } = await fixture() + const { kind: _kind, ...implicit } = notification(1) + const explicit = { kind: 'alert' as const, ...implicit } + for (const event of explicitFirst ? [explicit, implicit] : [implicit, explicit]) { + expect(await store.accept('host', 'phone', event)).toBe('queued') + } + expect(await store.pendingCount('phone')).toBe(1) + expect(await db.query('SELECT event_id FROM push_events')).toHaveLength(1) + expect(await store.accept('host', 'phone', { ...explicit, body: 'changed' })).toBe('error') + expect(await store.accept('host', 'phone', { ...implicit, kind: 'dismiss' })).toBe('queued') + expect(await db.query('SELECT event_id FROM push_events')).toHaveLength(2) + } +) + +it('fences late renew and finish after an expired claim is dismissed', async () => { + const { db, store, advance } = await fixture() + const alert = notification(1) + await store.accept('host', 'phone', alert) + const stale = (await store.claim())! + advance(DELIVERY_LEASE_MS) + await store.accept('host', 'phone', { + ...notification(2, 'dismiss'), + notificationId: alert.notificationId + }) + const read = async () => + ( + await db.query( + 'SELECT state, payload_json, lease_until FROM push_delivery_batches WHERE batch_id = ?', + [stale.id] + ) + )[0] + const cancelled = await read() + expect(cancelled).toMatchObject({ state: 'dismissed', payload_json: '{}' }) + await store.renew(stale) + expect(await read()).toEqual(cancelled) + await store.finish(stale, 1000) + expect(await read()).toEqual(cancelled) + await store.finish(stale) + expect(await read()).toEqual(cancelled) + const dismissal = (await store.claim())! + expect(dismissal.notification.kind).toBe('dismiss') + await store.finish(dismissal) + await store.accept('host', 'phone', notification(3)) + const fresh = (await store.claim())! + await store.finish(fresh, 1000) + advance(1000) + const retry = (await store.claim())! + expect(retry.id).toBe(fresh.id) + await store.finish(retry) + expect(await store.claim()).toBeNull() +}) diff --git a/cloud/apps/push/src/durable-push-store.ts b/cloud/apps/push/src/durable-push-store.ts new file mode 100644 index 00000000000..3003cc6d384 --- /dev/null +++ b/cloud/apps/push/src/durable-push-store.ts @@ -0,0 +1,194 @@ +import { isDismissedAlert, reconcileQueuedDismissal } from './push-queued-dismissal.js' +import { parsePushDeliveryPayload } from './push-delivery-payload.js' +import { createHash, randomUUID } from 'node:crypto' +import { PUSH_LIMITS, type PushNotification } from '@orca-cloud/push-contract' +import type { PushDatabase, SqlRow } from './push-database.js' + +const RETENTION_MS = 24 * 60 * 60_000 +export const DELIVERY_LEASE_MS = 30_000 +export type QueuedPushDelivery = { + id: string + registrationId: string + hostFingerprint: string + notification: PushNotification + expiresAt: number + lease: string + attempts: number +} + +export class DurablePushStore { + constructor( + private readonly database: PushDatabase, + private readonly now = Date.now + ) {} + + async accept( + host: string, + registrationId: string, + notification: PushNotification + ): Promise<'queued' | 'rate_limited' | 'error'> { + const now = this.now() + const kind = notification.kind ?? 'alert' + const eventId = createHash('sha256') + .update( + JSON.stringify([host, kind, notification.notificationEpoch, notification.notificationSeq]) + ) + .digest('hex') + const { sound: _sound, kind: _kind, ...content } = notification + const fingerprint = createHash('sha256') + .update(JSON.stringify({ kind, ...content })) + .digest('hex') + return this.database.transaction(async (tx) => { + await tx.lockQuotaScope(`push-events:${host}`) + const [existing] = await tx.query('SELECT * FROM push_events WHERE event_id = ?', [eventId]) + if (existing && existing.fingerprint !== fingerprint) return 'error' + const expiresAt = existing + ? Number(existing.expires_at) + : Math.min( + notification.expiresAt ?? Infinity, + now + PUSH_LIMITS.notificationTtlSeconds * 1000 + ) + if (expiresAt <= now) return 'error' + if (!existing) { + const [count] = await tx.query( + 'SELECT COUNT(*) AS total FROM push_events WHERE host_fingerprint = ? AND kind = ? AND created_at > ?', + [host, kind, now - PUSH_LIMITS.eventQuotaWindowMs] + ) + if (Number(count?.total ?? 0) >= PUSH_LIMITS.hostEventsPerWindow) return 'rate_limited' + await tx.query( + 'INSERT INTO push_events(event_id, host_fingerprint, kind, fingerprint, created_at, expires_at) VALUES (?, ?, ?, ?, ?, ?)', + [eventId, host, kind, fingerprint, now, expiresAt] + ) + } + const [recipient] = await tx.query( + 'SELECT event_id FROM push_event_recipients WHERE event_id = ? AND registration_id = ?', + [eventId, registrationId] + ) + if (recipient) return 'queued' + if (await reconcileQueuedDismissal(tx, host, registrationId, notification, now)) + return 'queued' + await tx.query( + `INSERT INTO push_delivery_batches(batch_id, host_fingerprint, registration_id, kind, payload_json, state, due_at, expires_at, lease_until, attempts, created_at) + VALUES (?, ?, ?, ?, ?, 'pending', ?, ?, 0, 0, ?)`, + [ + randomUUID(), + host, + registrationId, + kind, + JSON.stringify(notification), + now, + expiresAt, + now + ] + ) + await tx.query( + 'INSERT INTO push_event_recipients(event_id, registration_id, created_at) VALUES (?, ?, ?)', + [eventId, registrationId, now] + ) + return 'queued' + }) + } + + async claim(): Promise { + return this.database.transaction(async (tx) => { + await tx.lockQuotaScope('push-worker-claim') + const now = this.now() + const params = [now, now, now, now] + const predicate = + "state = 'pending' AND lease_until <= ? AND expires_at > ? AND due_at <= ? AND NOT EXISTS (SELECT 1 FROM push_delivery_batches busy WHERE busy.registration_id = push_delivery_batches.registration_id AND busy.lease_until > ?)" + let [row] = await tx.query( + `SELECT * FROM push_delivery_batches WHERE ${predicate} ORDER BY due_at, created_at, batch_id LIMIT 1`, + params + ) + if (!row) return null + await tx.lockQuotaScope(`push-events:${String(row.host_fingerprint)}`) + ;[row] = await tx.query('SELECT * FROM push_delivery_batches WHERE batch_id = ?', [ + row.batch_id + ]) + if (!row || row.state !== 'pending' || Number(row.expires_at) <= now) return null + const notification = parsePushDeliveryPayload(String(row.payload_json)) + if (await isDismissedAlert(tx, String(row.host_fingerprint), notification)) { + await tx.query( + "UPDATE push_delivery_batches SET state = 'dismissed', payload_json = '{}' WHERE batch_id = ?", + [row.batch_id] + ) + return null + } + const lease = randomUUID() + await tx.query( + 'UPDATE push_delivery_batches SET lease_token = ?, lease_until = ?, attempts = attempts + 1 WHERE batch_id = ?', + [lease, now + DELIVERY_LEASE_MS, row.batch_id] + ) + return this.delivery(row, lease) + }) + } + + private delivery(row: SqlRow, lease: string): QueuedPushDelivery { + return { + id: String(row.batch_id), + registrationId: String(row.registration_id), + hostFingerprint: String(row.host_fingerprint), + notification: parsePushDeliveryPayload(String(row.payload_json)), + expiresAt: Number(row.expires_at), + lease, + attempts: Number(row.attempts) + 1 + } + } + + async renew(delivery: QueuedPushDelivery): Promise { + await this.database.query( + "UPDATE push_delivery_batches SET lease_until = ? WHERE batch_id = ? AND lease_token = ? AND state = 'pending'", + [this.now() + DELIVERY_LEASE_MS, delivery.id, delivery.lease] + ) + } + + async finish( + delivery: QueuedPushDelivery, + retryAfterMs?: number, + outcome = 'done' + ): Promise { + const now = this.now() + const retryAt = retryAfterMs === undefined ? Infinity : now + Math.max(1000, retryAfterMs) + const retry = retryAt < delivery.expiresAt + await this.database.query( + `UPDATE push_delivery_batches SET state = ?, payload_json = ?, due_at = ?, lease_until = 0, lease_token = NULL + WHERE batch_id = ? AND lease_token = ? AND state = 'pending'`, + [ + retry ? 'pending' : retryAfterMs !== undefined ? 'expired' : outcome, + retry ? JSON.stringify(delivery.notification) : '{}', + retry ? retryAt : now, + delivery.id, + delivery.lease + ] + ) + } + + async pendingCount(registrationId: string): Promise { + const [row] = await this.database.query( + "SELECT COUNT(*) AS total FROM push_delivery_batches WHERE registration_id = ? AND state = 'pending'", + [registrationId] + ) + return Number(row?.total ?? 0) + } + + async prune(): Promise { + const now = this.now() + await this.database.query( + "UPDATE push_delivery_batches SET state = 'expired', payload_json = '{}' WHERE expires_at <= ? AND state = 'pending'", + [now] + ) + await this.database.query('DELETE FROM push_dismissed_events WHERE created_at < ?', [ + now - RETENTION_MS + ]) + await this.database.query('DELETE FROM push_delivery_batches WHERE expires_at < ?', [ + now - RETENTION_MS + ]) + await this.database.query('DELETE FROM push_event_recipients WHERE created_at < ?', [ + now - RETENTION_MS + ]) + const [result] = await this.database.query('DELETE FROM push_events WHERE created_at < ?', [ + now - RETENTION_MS + ]) + return Number(result?.changes ?? 0) + } +} diff --git a/cloud/apps/push/src/durable-push-worker.test.ts b/cloud/apps/push/src/durable-push-worker.test.ts new file mode 100644 index 00000000000..7cbd6074b36 --- /dev/null +++ b/cloud/apps/push/src/durable-push-worker.test.ts @@ -0,0 +1,221 @@ +import { afterEach, expect, it, vi } from 'vitest' +import type { PushNotification } from '@orca-cloud/push-contract' +import { DurablePushStore } from './durable-push-store.js' +import { DurablePushWorker } from './durable-push-worker.js' +import { PushDispatcher } from './push-dispatcher.js' +import { PushDeviceRegistryStore } from './device-registry-store.js' +import { openInMemoryPushDatabase } from './push-database.js' +import type { PushDelivery } from './push-delivery-message.js' +import type { PushProviderOutcome } from './push-provider-outcome.js' + +const cleanups: (() => Promise)[] = [] +afterEach(async () => { + for (const cleanup of cleanups.splice(0)) await cleanup() + vi.useRealTimers() + vi.restoreAllMocks() +}) +const note = (seq: number, overrides: Partial = {}): PushNotification => ({ + notificationId: `note-${seq}`, + notificationSeq: seq, + notificationEpoch: 'epoch', + source: 'agent-task-complete', + agentState: 'finished', + title: 'Done', + body: 'Finished task', + ...overrides +}) +async function fixture() { + const db = await openInMemoryPushDatabase() + let time = 1_000_000 + const now = () => time + const store = new DurablePushStore(db, now) + const devices = new PushDeviceRegistryStore(db, now) + const device = await devices.upsert({ + hostFingerprint: 'host', + deviceId: 'phone', + platform: 'android', + token: 'test-token' + }) + if (!device.ok) throw new Error('registration failed') + const send = vi.fn(async (_delivery: PushDelivery): Promise => ({ + status: 'sent' + })) + const onRetry = vi.fn() + const dispatcher = new PushDispatcher({ devices, fcm: { send } as never }) + const worker = new DurablePushWorker(store, dispatcher, { now, onRetry }) + cleanups.push(async () => { + await worker.stop() + await db.close() + }) + vi.spyOn(console, 'warn').mockImplementation(() => {}) + return { + db, + store, + devices, + worker, + dispatcher, + send, + onRetry, + now, + registrationId: device.registrationId, + accept: (notification: PushNotification) => + store.accept('host', device.registrationId, notification), + advance: (ms: number) => { + time += ms + } + } +} + +it('sends every burst event immediately with its original content and identity', async () => { + const h = await fixture() + await h.accept(note(1)) + await h.accept( + note(2, { agentState: 'needs-input', title: 'Answer needed', body: 'Please respond' }) + ) + await h.worker.runDue() + expect(h.send).toHaveBeenCalledTimes(2) + const first = h.send.mock.calls.find(([delivery]) => delivery.orca.notificationSeq === 1)![0] + const second = h.send.mock.calls.find(([delivery]) => delivery.orca.notificationSeq === 2)![0] + expect(first).toMatchObject({ + title: 'Done', + body: 'Finished task', + orca: { + notificationId: 'note-1', + notificationSeq: 1 + } + }) + expect(second).toMatchObject({ + title: 'Answer needed', + body: 'Please respond', + orca: { notificationId: 'note-2', notificationSeq: 2 } + }) + expect(first.collapseId).not.toBe(second.collapseId) + expect(h.send.mock.calls.every(([delivery]) => !('coalescedCount' in delivery.orca))).toBe(true) + expect(h.send.mock.calls.every(([delivery]) => !('summaryMembers' in delivery.orca))).toBe(true) + expect(h.onRetry).not.toHaveBeenCalled() +}) + +it('keeps untrackable bells and per-phone deliveries individually replaceable', async () => { + const h = await fixture() + const other = await h.devices.upsert({ + hostFingerprint: 'host', + deviceId: 'phone2', + platform: 'android', + token: 'other-token' + }) + if (!other.ok) throw new Error('registration failed') + await h.accept(note(1, { notificationId: undefined, source: 'terminal-bell', agentState: null })) + await h.accept(note(2)) + await h.accept(note(3)) + await h.store.accept('host', other.registrationId, note(2)) + await h.worker.runDue() + expect(h.send).toHaveBeenCalledTimes(4) + const deliveries = h.send.mock.calls.map(([delivery]) => delivery) + const primary = deliveries.filter((delivery) => delivery.registrationId === h.registrationId) + expect(primary).toHaveLength(3) + expect(new Set(primary.map((delivery) => delivery.collapseId)).size).toBe(3) + expect( + deliveries.find((delivery) => delivery.registrationId === other.registrationId) + ).toMatchObject({ orca: { notificationId: 'note-2', notificationSeq: 2 } }) +}) + +it('persists provider retry delay and resumes it through a new worker', async () => { + const h = await fixture() + h.send.mockResolvedValueOnce({ + status: 'error', + reason: 'busy', + retryable: true, + retryAfterMs: 10000 + }) + await h.accept(note(1)) + await h.worker.runDue() + expect(h.send).toHaveBeenCalledOnce() + await h.worker.stop() + const restarted = new DurablePushWorker(h.store, h.dispatcher, { now: h.now, onRetry: h.onRetry }) + h.advance(9999) + await restarted.runDue() + expect(h.send).toHaveBeenCalledOnce() + h.advance(1) + await restarted.runDue() + expect(h.send).toHaveBeenCalledTimes(2) + expect(h.send.mock.calls.map(([delivery]) => delivery.expiresAt)).toEqual([1_300_000, 1_300_000]) + expect(h.onRetry).toHaveBeenCalledOnce() + expect(await h.store.pendingCount(h.registrationId)).toBe(0) + await restarted.stop() +}) + +it('expires instead of shortening a provider delay beyond the delivery lifetime', async () => { + const h = await fixture() + h.send.mockResolvedValue({ + status: 'error', + reason: 'busy', + retryable: true, + retryAfterMs: 600000 + }) + await h.accept(note(1)) + await h.worker.runDue() + h.advance(600000) + await h.worker.runDue() + expect(h.send).toHaveBeenCalledOnce() + expect(await h.store.pendingCount(h.registrationId)).toBe(0) + expect(h.onRetry).not.toHaveBeenCalled() +}) + +it('rechecks the device before a persisted retry and does not send after unregistration', async () => { + const h = await fixture() + h.send.mockResolvedValue({ status: 'error', reason: 'timeout', retryable: true }) + await h.accept(note(1)) + await h.worker.runDue() + await h.devices.deleteOwned('host', h.registrationId) + h.advance(3000) + await h.worker.runDue() + expect(h.send).toHaveBeenCalledOnce() + expect(await h.store.pendingCount(h.registrationId)).toBe(0) +}) + +it('joins active work on shutdown and leaves unclaimed work for the next instance', async () => { + const h = await fixture() + let finish!: (outcome: PushProviderOutcome) => void + let started!: () => void + const entered = new Promise((resolve) => { + started = resolve + }) + h.send.mockImplementationOnce(() => { + started() + return new Promise((resolve) => { + finish = resolve + }) + }) + await h.accept(note(1)) + const pending = h.worker.runDue() + await entered + await h.accept(note(2)) + let stopped = false + const stopping = h.worker.stop().then(() => { + stopped = true + }) + await Promise.resolve() + expect(stopped).toBe(false) + finish({ status: 'sent' }) + await Promise.all([pending, stopping]) + expect(stopped).toBe(true) + expect(h.send).toHaveBeenCalledOnce() + const resumed = new DurablePushWorker(h.store, h.dispatcher, { now: h.now }) + await resumed.runDue() + expect(h.send).toHaveBeenCalledTimes(2) + await resumed.stop() +}) + +it('runs due work on its timer and releases the timer on stop', async () => { + const h = await fixture() + vi.useFakeTimers() + await h.accept(note(1)) + h.worker.start() + h.worker.start() + expect(vi.getTimerCount()).toBe(1) + await vi.advanceTimersByTimeAsync(1000) + await h.worker.runDue() + expect(h.send).toHaveBeenCalledOnce() + await h.worker.stop() + expect(vi.getTimerCount()).toBe(0) +}) diff --git a/cloud/apps/push/src/durable-push-worker.ts b/cloud/apps/push/src/durable-push-worker.ts new file mode 100644 index 00000000000..4efe329bfda --- /dev/null +++ b/cloud/apps/push/src/durable-push-worker.ts @@ -0,0 +1,89 @@ +import { buildPushDelivery } from './push-delivery-message.js' +import type { PushDispatcher } from './push-dispatcher.js' +import type { DurablePushStore } from './durable-push-store.js' + +export class DurablePushWorker { + private timer?: NodeJS.Timeout + private running: Promise | null = null + private stopped = false + constructor( + private readonly store: DurablePushStore, + private readonly dispatcher: PushDispatcher, + private readonly options: { now?: () => number; onRetry?: () => void } = {} + ) {} + + start(): void { + if (this.timer) return + this.stopped = false + this.timer = setInterval(() => { + void this.runDue().catch(() => { + console.warn(JSON.stringify({ event: 'orca_push_worker_failed' })) + }) + }, 1000) + this.timer.unref() + } + + async runDue(): Promise { + if (this.running) { + await this.running + return + } + if (this.stopped) return + const pending = Promise.allSettled(Array.from({ length: 4 }, () => this.drain())).then( + (results) => { + const failure = results.find((result) => result.status === 'rejected') + if (failure?.status === 'rejected') throw failure.reason + } + ) + this.running = pending + try { + await pending + } finally { + this.running = null + } + } + + private async drain(): Promise { + for (let count = 0; count < 25 && !this.stopped; count++) { + const queued = await this.store.claim() + if (!queued) return + const delivery = buildPushDelivery({ + expiresAt: queued.expiresAt, + registrationId: queued.registrationId, + hostFingerprint: queued.hostFingerprint, + notification: queued.notification + }) + if ((this.options.now ?? Date.now)() >= queued.expiresAt) { + await this.store.finish(queued) + continue + } + const heartbeat = setInterval(() => { + void this.store.renew(queued).catch(() => {}) + }, 10_000) + heartbeat.unref() + try { + if (queued.attempts > 1) this.options.onRetry?.() + const outcome = await this.dispatcher.sendOnce(delivery) + const retryAfterMs = + outcome.status === 'error' && outcome.retryable + ? Math.max( + outcome.retryAfterMs ?? 0, + Math.min(30_000, 1000 * 2 ** Math.min(queued.attempts, 5)) + ) + : undefined + await this.store.finish(queued, retryAfterMs, outcome.status) + } catch { + await this.store.finish(queued, 5000) + } finally { + clearInterval(heartbeat) + } + } + } + + async stop(): Promise { + this.stopped = true + if (this.timer) clearInterval(this.timer) + this.timer = undefined + await this.running + } +} diff --git a/cloud/apps/push/src/fcm-access-token.ts b/cloud/apps/push/src/fcm-access-token.ts new file mode 100644 index 00000000000..542e0e8d0ed --- /dev/null +++ b/cloud/apps/push/src/fcm-access-token.ts @@ -0,0 +1,15 @@ +import { GoogleAuth } from 'google-auth-library' +import { FCM_SCOPE } from './fcm-client.js' + +// Resolves the runtime service account credential from the GCE metadata server +// in Cloud Run and from GOOGLE_APPLICATION_CREDENTIALS locally; the library +// caches and refreshes the token itself. +export function createFcmAccessTokenProvider(): () => Promise { + const auth = new GoogleAuth({ scopes: [FCM_SCOPE] }) + return async () => { + const client = await auth.getClient() + const token = await client.getAccessToken() + if (!token.token) throw new Error('fcm_access_token_unavailable') + return token.token + } +} diff --git a/cloud/apps/push/src/fcm-client.test.ts b/cloud/apps/push/src/fcm-client.test.ts new file mode 100644 index 00000000000..4a8d1fb41f7 --- /dev/null +++ b/cloud/apps/push/src/fcm-client.test.ts @@ -0,0 +1,240 @@ +import { createHash } from 'node:crypto' +import { describe, expect, it } from 'vitest' +import { fcmCollapseKey, FcmClient, type FcmRequest, type FcmResponse } from './fcm-client.js' +import { buildPushDelivery } from './push-delivery-message.js' + +const NOW = 1_700_000_000_000 +const HOST = 'abcdefghijklmnop' +const TOKEN = 'cQ1abcDEF_gh:APA91bZZ-zz0123456789abcdefghijklmnopqrstuvwxyz' + +function delivery(agentState: 'needs-input' | null = 'needs-input') { + return buildPushDelivery({ + expiresAt: NOW + 300_000, + registrationId: 'reg-1', + hostFingerprint: HOST, + notification: { + notificationId: 'note-1', + notificationSeq: 7, + notificationEpoch: 'epoch-1', + source: 'agent-task-complete', + agentState, + title: 'Agent needs input', + body: 'Waiting on your answer', + worktreeId: 'wt-1' + } + }) +} + +function fakeTransport(response: FcmResponse) { + const requests: FcmRequest[] = [] + return { + requests, + transport: async (request: FcmRequest): Promise => { + requests.push(request) + return response + } + } +} + +function client(response: FcmResponse) { + const fake = fakeTransport(response) + return { + fake, + client: new FcmClient({ + projectId: 'onorca-cloud', + now: () => NOW, + accessToken: async () => 'access-token', + transport: fake.transport + }) + } +} + +describe('fcm client', () => { + it('posts the v1 send payload for the configured project', async () => { + const { fake, client: fcm } = client({ status: 200, body: '{"name":"projects/x/messages/1"}' }) + await expect(fcm.send(delivery(), { token: TOKEN })).resolves.toEqual({ status: 'sent' }) + const request = fake.requests[0]! + expect(request.url).toBe('https://fcm.googleapis.com/v1/projects/onorca-cloud/messages:send') + expect(request.accessToken).toBe('access-token') + expect(JSON.parse(request.body)).toEqual({ + message: { + token: TOKEN, + notification: { title: 'Agent needs input', body: 'Waiting on your answer' }, + android: { + priority: 'HIGH', + ttl: '300s', + collapse_key: createHash('sha256') + .update( + createHash('sha256') + .update(JSON.stringify([HOST, 'note-1'])) + .digest('hex') + ) + .digest('hex') + .slice(0, 32), + notification: { + channel_id: 'orca-desktop', + tag: createHash('sha256') + .update(JSON.stringify([HOST, 'note-1'])) + .digest('hex') + } + }, + data: { + hostFingerprint: HOST, + worktreeId: 'wt-1', + notificationId: 'note-1', + notificationSeq: '7', + notificationEpoch: 'epoch-1', + source: 'agent-task-complete', + agentState: 'needs-input' + } + } + }) + }) + + it('carries every data value as a string and omits a null agent state', async () => { + const { fake, client: fcm } = client({ status: 200, body: '{}' }) + await fcm.send(delivery(null), { token: TOKEN }) + const message = JSON.parse(fake.requests[0]!.body) as { + message: { + android: { collapse_key: string; notification: { tag: string } } + data: Record + } + } + expect(Object.values(message.message.data).every((value) => typeof value === 'string')).toBe( + true + ) + expect(message.message.data.agentState).toBeUndefined() + const tag = createHash('sha256') + .update(JSON.stringify([HOST, 'note-1'])) + .digest('hex') + expect(message.message.data.coalescedCount).toBeUndefined() + expect(message.message.android.notification.tag).toBe(tag) + expect(message.message.android.collapse_key).toBe(fcmCollapseKey(tag)) + expect(message.message.android.collapse_key).toHaveLength(32) + }) + + it('marks an unregistered token dead from the status or the error detail', async () => { + const byStatus = client({ + status: 404, + body: JSON.stringify({ error: { status: 'UNREGISTERED', message: 'not registered' } }) + }) + await expect(byStatus.client.send(delivery(), { token: TOKEN })).resolves.toEqual({ + status: 'dead', + reason: 'UNREGISTERED' + }) + const byDetail = client({ + status: 404, + body: JSON.stringify({ + error: { + status: 'NOT_FOUND', + message: 'Requested entity was not found.', + details: [{ errorCode: 'UNREGISTERED' }] + } + }) + }) + await expect(byDetail.client.send(delivery(), { token: TOKEN })).resolves.toEqual({ + status: 'dead', + reason: 'UNREGISTERED' + }) + }) + + it('marks an invalid-argument that names the token dead, and others an error', async () => { + const named = client({ + status: 400, + body: JSON.stringify({ + error: { status: 'INVALID_ARGUMENT', message: 'The registration token is not valid.' } + }) + }) + await expect(named.client.send(delivery(), { token: TOKEN })).resolves.toEqual({ + status: 'dead', + reason: 'INVALID_ARGUMENT' + }) + const unnamed = client({ + status: 400, + body: JSON.stringify({ + error: { status: 'INVALID_ARGUMENT', message: 'Invalid value at message.android.ttl' } + }) + }) + await expect(unnamed.client.send(delivery(), { token: TOKEN })).resolves.toEqual({ + status: 'error', + reason: 'INVALID_ARGUMENT', + retryable: false, + retryAfterMs: 10000 + }) + }) + + it('treats a server fault and a transport failure as errors', async () => { + const faulted = client({ + status: 503, + body: JSON.stringify({ error: { status: 'UNAVAILABLE', message: 'backend busy' } }) + }) + await expect(faulted.client.send(delivery(), { token: TOKEN })).resolves.toEqual({ + status: 'error', + reason: 'UNAVAILABLE', + retryable: true, + retryAfterMs: 10000 + }) + const broken = new FcmClient({ + projectId: 'onorca-cloud', + now: () => NOW, + accessToken: async () => 'access-token', + transport: async () => { + throw new Error('ECONNRESET') + } + }) + await expect(broken.send(delivery(), { token: TOKEN })).resolves.toEqual({ + status: 'error', + reason: 'Error', + retryable: true + }) + }) +}) + +it('does not send when credential refresh crosses the absolute expiry', async () => { + let now = 1000 + const fake = fakeTransport({ status: 200, body: '{}' }) + const fcm = new FcmClient({ + projectId: 'test', + now: () => now, + accessToken: async () => { + now = 3000 + return 'test-token' + }, + transport: fake.transport + }) + await expect(fcm.send({ ...delivery(), expiresAt: 2000 }, { token: TOKEN })).resolves.toEqual({ + status: 'error', + reason: 'expired' + }) + expect(fake.requests).toHaveLength(0) +}) + +it('decreases retry TTL and refuses expired delivery before refreshing credentials', async () => { + let now = NOW + let refreshes = 0 + const fake = fakeTransport({ status: 503, body: '{}' }) + const fcm = new FcmClient({ + projectId: 'test', + now: () => now, + accessToken: async () => { + refreshes++ + return 'test-token' + }, + transport: fake.transport + }) + const pending = delivery() + await fcm.send(pending, { token: TOKEN }) + now += 60_000 + await fcm.send(pending, { token: TOKEN }) + expect(fake.requests.map((request) => JSON.parse(request.body).message.android.ttl)).toEqual([ + '300s', + '240s' + ]) + now = pending.expiresAt + await expect(fcm.send(pending, { token: TOKEN })).resolves.toEqual({ + status: 'error', + reason: 'expired' + }) + expect(fake.requests).toHaveLength(2) + expect(refreshes).toBe(2) +}) diff --git a/cloud/apps/push/src/fcm-client.ts b/cloud/apps/push/src/fcm-client.ts new file mode 100644 index 00000000000..c22bd3309cd --- /dev/null +++ b/cloud/apps/push/src/fcm-client.ts @@ -0,0 +1,147 @@ +import { providerRetryAfter } from './provider-retry-delay.js' +import { createHash } from 'node:crypto' +import { PUSH_DEFAULTS } from '@orca-cloud/push-contract' +import { orcaDataStrings, type PushDelivery } from './push-delivery-message.js' +import type { PushProviderOutcome } from './push-provider-outcome.js' + +export const FCM_SCOPE = 'https://www.googleapis.com/auth/firebase.messaging' + +export type FcmRequest = { url: string; accessToken: string; body: string } +export type FcmResponse = { status: number; body: string; retryAfterMs?: number } +export type FcmTransport = (request: FcmRequest) => Promise + +export type FcmClientOptions = { + projectId: string + accessToken: () => Promise + transport: FcmTransport + channelId?: string + now?: () => number +} + +type FcmErrorBody = { + error?: { status?: unknown; message?: unknown; details?: { errorCode?: unknown }[] } +} + +// FCM collapse_key is a short opaque string, so the collapse id is hashed +// rather than truncated: truncation would merge unrelated notifications. +export function fcmCollapseKey(collapseId: string): string { + return createHash('sha256').update(collapseId).digest('hex').slice(0, 32) +} + +export function fcmMessageBody(input: { + delivery: PushDelivery + token: string + channelId: string + now?: number +}): string { + const { delivery } = input + const now = input.now ?? Date.now() + return JSON.stringify({ + message: { + token: input.token, + ...(delivery.orca.kind === 'dismiss' + ? {} + : { notification: { title: delivery.title, body: delivery.body } }), + android: { + priority: 'HIGH', + ttl: `${Math.max(0, Math.ceil((delivery.expiresAt - now) / 1000))}s`, + collapse_key: fcmCollapseKey(delivery.collapseId), + ...(delivery.orca.kind === 'dismiss' + ? {} + : { + notification: { + channel_id: + delivery.sound === false ? `${input.channelId}-silent` : input.channelId, + tag: delivery.collapseId + } + }) + }, + data: orcaDataStrings(delivery.orca) + } + }) +} + +function readFcmError(body: string): { status: string; message: string; errorCodes: string[] } { + try { + const parsed = JSON.parse(body) as FcmErrorBody + return { + status: typeof parsed.error?.status === 'string' ? parsed.error.status : 'unknown', + message: typeof parsed.error?.message === 'string' ? parsed.error.message : '', + errorCodes: (parsed.error?.details ?? []) + .map((detail) => detail.errorCode) + .filter((code): code is string => typeof code === 'string') + } + } catch { + return { status: 'unparseable', message: '', errorCodes: [] } + } +} + +export class FcmClient { + private readonly channelId: string + + constructor(private readonly options: FcmClientOptions) { + this.channelId = options.channelId ?? PUSH_DEFAULTS.androidChannelId + } + + async send(delivery: PushDelivery, device: { token: string }): Promise { + if (delivery.expiresAt <= (this.options.now ?? Date.now)()) + return { status: 'error', reason: 'expired' } + let response: FcmResponse + try { + const accessToken = await this.options.accessToken() + const now = (this.options.now ?? Date.now)() + if (delivery.expiresAt <= now) return { status: 'error', reason: 'expired' } + response = await this.options.transport({ + url: `https://fcm.googleapis.com/v1/projects/${this.options.projectId}/messages:send`, + accessToken, + body: fcmMessageBody({ + delivery, + token: device.token, + channelId: this.channelId, + now + }) + }) + } catch (error) { + return { + status: 'error', + reason: error instanceof Error ? error.name : 'transport_failed', + retryable: true + } + } + if (response.status >= 200 && response.status < 300) return { status: 'sent' } + const failure = readFcmError(response.body) + if (failure.status === 'UNREGISTERED' || failure.errorCodes.includes('UNREGISTERED')) { + return { status: 'dead', reason: 'UNREGISTERED' } + } + // A revoked token also surfaces as INVALID_ARGUMENT naming the token field. + if (failure.status === 'INVALID_ARGUMENT' && /\btoken\b/i.test(failure.message)) { + return { status: 'dead', reason: 'INVALID_ARGUMENT' } + } + return { + status: 'error', + reason: failure.status, + retryable: response.status === 429 || response.status >= 500, + retryAfterMs: Math.max(response.status === 429 ? 60_000 : 10_000, response.retryAfterMs ?? 0) + } + } +} + +export function createFcmFetchTransport(fetchImpl: typeof fetch = fetch): FcmTransport { + return async (request) => { + const response = await fetchImpl(request.url, { + method: 'POST', + headers: { + authorization: `Bearer ${request.accessToken}`, + 'content-type': 'application/json' + }, + body: request.body, + redirect: 'error', + signal: AbortSignal.timeout(10_000) + }) + return { + status: response.status, + body: await response.text(), + retryAfterMs: providerRetryAfter(response.headers.get('retry-after') ?? undefined) + } + } +} diff --git a/cloud/apps/push/src/host-challenge-answering.test-fixture.ts b/cloud/apps/push/src/host-challenge-answering.test-fixture.ts new file mode 100644 index 00000000000..4dec1e48c5b --- /dev/null +++ b/cloud/apps/push/src/host-challenge-answering.test-fixture.ts @@ -0,0 +1,163 @@ +import { createHmac, timingSafeEqual } from 'node:crypto' +import { + PUSH_HOST_CHALLENGE_PLAINTEXT_DOMAIN, + PUSH_HOST_PROOF_TRANSCRIPT_DOMAIN, + PUSH_HOST_PROOF_TRANSCRIPT_FIELD_COUNT, + PUSH_LIMITS +} from '@orca-cloud/push-contract' +import nacl from 'tweetnacl' +import { decodeCanonicalBase64 } from './canonical-base64.js' +import { deriveHostFingerprint } from './host-fingerprint.js' + +// The desktop side of the push challenge, written the way the shipped host +// will answer it, so the gateway is exercised against a real box-opening peer. +const textEncoder = new TextEncoder() +const textDecoder = new TextDecoder() + +export type PushHostKeypair = { publicKey: Uint8Array; secretKey: Uint8Array } + +export type PushChallengeWire = { + challengeId: string + gatewayEphemeralPublicKeyB64: string + nonceB64: string + ciphertextB64: string + expiresAt: number +} + +export function createPushHostKeypair(seed?: number): PushHostKeypair { + const pair = + seed === undefined + ? nacl.box.keyPair() + : nacl.box.keyPair.fromSecretKey(new Uint8Array(32).fill(seed)) + return { publicKey: pair.publicKey, secretKey: pair.secretKey } +} + +export function hostPublicKeyB64(keypair: PushHostKeypair): string { + return Buffer.from(keypair.publicKey).toString('base64') +} + +function equal(left: Uint8Array | undefined, right: Uint8Array): boolean { + return Boolean(left && left.byteLength === right.byteLength && timingSafeEqual(left, right)) +} + +function uint64(value: number): Uint8Array { + const bytes = new Uint8Array(8) + new DataView(bytes.buffer).setBigUint64(0, BigInt(value), false) + return bytes +} + +function parseTranscript(transcript: Uint8Array): Map | null { + const fields = new Map() + const view = new DataView(transcript.buffer, transcript.byteOffset, transcript.byteLength) + let offset = 0 + try { + while (offset < transcript.byteLength) { + const nameLength = view.getUint32(offset, false) + offset += 4 + const name = textDecoder.decode(transcript.slice(offset, offset + nameLength)) + offset += nameLength + const valueLength = view.getUint32(offset, false) + offset += 4 + if (fields.has(name) || offset + valueLength > transcript.byteLength) return null + fields.set(name, transcript.slice(offset, offset + valueLength)) + offset += valueLength + } + } catch { + return null + } + return offset === transcript.byteLength ? fields : null +} + +function readUint64(value: Uint8Array | undefined): number | null { + if (!value || value.byteLength !== 8) return null + const parsed = new DataView(value.buffer, value.byteOffset, value.byteLength).getBigUint64(0, false) + return parsed <= BigInt(Number.MAX_SAFE_INTEGER) ? Number(parsed) : null +} + +export type PushHostProofContext = { + gatewayOrigin: string + keypair: PushHostKeypair + now?: () => number + onInvalid?: (reason: string) => void +} + +function validateTranscript( + transcript: Uint8Array, + challenge: PushChallengeWire, + context: PushHostProofContext, + gatewayKey: Uint8Array, + nonce: Uint8Array +): boolean { + const fields = parseTranscript(transcript) + if (!fields || fields.size !== PUSH_HOST_PROOF_TRANSCRIPT_FIELD_COUNT) { + context.onInvalid?.('transcript-structure') + return false + } + const now = (context.now ?? Date.now)() + const issuedAt = readUint64(fields.get('issuedAt')) + const expiresAt = readUint64(fields.get('expiresAt')) + const fingerprint = deriveHostFingerprint(context.keypair.publicKey) + const checks: [string, boolean][] = [ + ['issuedAt-readable', issuedAt !== null], + [ + 'issuedAt-not-future', + issuedAt === null || issuedAt - PUSH_LIMITS.clockSkewToleranceMs <= now + ], + ['not-expired', now - PUSH_LIMITS.clockSkewToleranceMs <= challenge.expiresAt], + ['issuedAt-before-expiry', issuedAt === null || issuedAt <= challenge.expiresAt], + [ + 'window', + issuedAt === null || challenge.expiresAt - issuedAt <= PUSH_LIMITS.challengeTtlMs + ], + ['expiry-consistent', expiresAt === challenge.expiresAt], + ['protocol', equal(fields.get('protocol'), textEncoder.encode(PUSH_HOST_PROOF_TRANSCRIPT_DOMAIN))], + ['version', equal(fields.get('version'), new Uint8Array([1]))], + ['gatewayOrigin', equal(fields.get('gatewayOrigin'), textEncoder.encode(context.gatewayOrigin))], + ['gatewayEphemeralPublicKey', equal(fields.get('gatewayEphemeralPublicKey'), gatewayKey)], + ['challengeNonce', equal(fields.get('challengeNonce'), nonce)], + ['challengeId', equal(fields.get('challengeId'), textEncoder.encode(challenge.challengeId))], + ['hostFingerprint', equal(fields.get('hostFingerprint'), textEncoder.encode(fingerprint))], + ['hostPublicKey', equal(fields.get('hostPublicKey'), context.keypair.publicKey)], + ['issuedAt-value', issuedAt === null || uint64(issuedAt).byteLength === 8] + ] + const failed = checks.filter(([, ok]) => !ok).map(([name]) => name) + if (failed.length === 0) return true + context.onInvalid?.(`transcript:${failed.join('+')}`) + return false +} + +export function answerPushHostChallenge( + challenge: PushChallengeWire, + context: PushHostProofContext +): string | null { + const gatewayKey = decodeCanonicalBase64(challenge.gatewayEphemeralPublicKeyB64, 32) + const nonce = decodeCanonicalBase64(challenge.nonceB64, 24) + const ciphertext = Buffer.from(challenge.ciphertextB64, 'base64') + if (!gatewayKey || !nonce || ciphertext.toString('base64') !== challenge.ciphertextB64) return null + const plaintext = nacl.box.open(ciphertext, nonce, gatewayKey, context.keypair.secretKey) + if (!plaintext) { + context.onInvalid?.('challenge-box-open') + return null + } + const domain = textEncoder.encode(`${PUSH_HOST_CHALLENGE_PLAINTEXT_DOMAIN}\0`) + if ( + !equal(plaintext.slice(0, domain.byteLength), domain) || + plaintext.byteLength < domain.byteLength + 36 + ) { + return null + } + const transcriptLength = new DataView( + plaintext.buffer, + plaintext.byteOffset + domain.byteLength, + 4 + ).getUint32(0, false) + const transcriptStart = domain.byteLength + 4 + const secretStart = transcriptStart + transcriptLength + if (secretStart + 32 !== plaintext.byteLength) return null + const transcript = plaintext.slice(transcriptStart, secretStart) + if (!validateTranscript(transcript, challenge, context, gatewayKey, nonce)) return null + return createHmac('sha256', plaintext.slice(secretStart)) + .update(textEncoder.encode(`${PUSH_HOST_PROOF_TRANSCRIPT_DOMAIN}\0ack\0`)) + .update(transcript) + .digest('base64') +} diff --git a/cloud/apps/push/src/host-challenge-store.test.ts b/cloud/apps/push/src/host-challenge-store.test.ts new file mode 100644 index 00000000000..c5aa7928ba9 --- /dev/null +++ b/cloud/apps/push/src/host-challenge-store.test.ts @@ -0,0 +1,191 @@ +import { PUSH_LIMITS } from '@orca-cloud/push-contract' +import { afterEach, beforeEach, describe, expect, it } from 'vitest' +import { + answerPushHostChallenge, + createPushHostKeypair, + hostPublicKeyB64 +} from './host-challenge-answering.test-fixture.js' +import { PushHostChallengeStore } from './host-challenge-store.js' +import { deriveHostFingerprint } from './host-fingerprint.js' +import { openInMemoryPushDatabase, type PushDatabase } from './push-database.js' + +const GATEWAY_ORIGIN = 'https://push.onorca.dev' + +describe('push host challenge store', () => { + let database: PushDatabase + let clock = 1_700_000_000_000 + let store: PushHostChallengeStore + + beforeEach(async () => { + database = await openInMemoryPushDatabase() + clock = 1_700_000_000_000 + store = new PushHostChallengeStore(database, GATEWAY_ORIGIN, () => clock) + }) + + afterEach(async () => { + await database.close() + }) + + it('completes a challenge, proof, and consume round trip', async () => { + const host = createPushHostKeypair(1) + const challenge = await store.issue(hostPublicKeyB64(host)) + expect(challenge).not.toBeNull() + expect(challenge!.expiresAt).toBe(clock + PUSH_LIMITS.challengeTtlMs) + expect(challenge!.hostFingerprint).toBe(deriveHostFingerprint(host.publicKey)) + + const proof = answerPushHostChallenge(challenge!, { + gatewayOrigin: GATEWAY_ORIGIN, + keypair: host, + now: () => clock + }) + expect(proof).not.toBeNull() + await expect(store.verify(challenge!.challengeId, proof!)).resolves.toEqual({ + ok: true, + hostFingerprint: deriveHostFingerprint(host.publicKey) + }) + }) + + it('never stores material that reproduces the proof', async () => { + const host = createPushHostKeypair(2) + const challenge = await store.issue(hostPublicKeyB64(host)) + const proof = answerPushHostChallenge(challenge!, { + gatewayOrigin: GATEWAY_ORIGIN, + keypair: host, + now: () => clock + }) + const [row] = await database.query('SELECT secret_hash FROM push_challenges') + expect(String(row?.secret_hash)).not.toBe(proof) + expect(Buffer.from(String(row?.secret_hash), 'base64url').byteLength).toBe(32) + }) + + it('rejects a replayed challenge', async () => { + const host = createPushHostKeypair(3) + const challenge = await store.issue(hostPublicKeyB64(host)) + const proof = answerPushHostChallenge(challenge!, { + gatewayOrigin: GATEWAY_ORIGIN, + keypair: host, + now: () => clock + })! + await expect(store.verify(challenge!.challengeId, proof)).resolves.toMatchObject({ ok: true }) + await expect(store.verify(challenge!.challengeId, proof)).resolves.toEqual({ + ok: false, + reason: 'already_consumed' + }) + }) + + it('rejects a challenge the moment its own ttl elapses', async () => { + const host = createPushHostKeypair(4) + const challenge = await store.issue(hostPublicKeyB64(host)) + const proof = answerPushHostChallenge(challenge!, { + gatewayOrigin: GATEWAY_ORIGIN, + keypair: host, + now: () => clock + })! + clock += PUSH_LIMITS.challengeTtlMs + 1 + await expect(store.verify(challenge!.challengeId, proof)).resolves.toEqual({ + ok: false, + reason: 'expired' + }) + }) + + it('spends no skew tolerance on its own expiry, so the ttl is the whole window', async () => { + const host = createPushHostKeypair(5) + const challenge = await store.issue(hostPublicKeyB64(host)) + const proof = answerPushHostChallenge(challenge!, { + gatewayOrigin: GATEWAY_ORIGIN, + keypair: host, + now: () => clock + })! + // A proof that the host would still consider in-window is refused here: the + // gateway issued expires_at against this clock and needs no allowance. + clock += PUSH_LIMITS.challengeTtlMs + PUSH_LIMITS.clockSkewToleranceMs - 1 + await expect(store.verify(challenge!.challengeId, proof)).resolves.toEqual({ + ok: false, + reason: 'expired' + }) + }) + + it('accepts a proof that lands just inside the ttl', async () => { + const host = createPushHostKeypair(26) + const challenge = await store.issue(hostPublicKeyB64(host)) + const proof = answerPushHostChallenge(challenge!, { + gatewayOrigin: GATEWAY_ORIGIN, + keypair: host, + now: () => clock + })! + clock += PUSH_LIMITS.challengeTtlMs + await expect(store.verify(challenge!.challengeId, proof)).resolves.toMatchObject({ ok: true }) + }) + + it('keeps an expired row long enough to answer expired rather than unknown', async () => { + const host = createPushHostKeypair(27) + const challenge = await store.issue(hostPublicKeyB64(host)) + const proof = answerPushHostChallenge(challenge!, { + gatewayOrigin: GATEWAY_ORIGIN, + keypair: host, + now: () => clock + })! + clock += PUSH_LIMITS.challengeTtlMs + 1 + expect(await store.pruneExpired()).toBe(0) + await expect(store.verify(challenge!.challengeId, proof)).resolves.toEqual({ + ok: false, + reason: 'expired' + }) + }) + + it('refuses a wrong host: the box will not open and a foreign proof will not match', async () => { + const owner = createPushHostKeypair(6) + const intruder = createPushHostKeypair(7) + const ownerChallenge = await store.issue(hostPublicKeyB64(owner)) + expect( + answerPushHostChallenge(ownerChallenge!, { + gatewayOrigin: GATEWAY_ORIGIN, + keypair: intruder, + now: () => clock + }) + ).toBeNull() + + const intruderChallenge = await store.issue(hostPublicKeyB64(intruder)) + const intruderProof = answerPushHostChallenge(intruderChallenge!, { + gatewayOrigin: GATEWAY_ORIGIN, + keypair: intruder, + now: () => clock + })! + await expect(store.verify(ownerChallenge!.challengeId, intruderProof)).resolves.toEqual({ + ok: false, + reason: 'proof_mismatch' + }) + }) + + it('rejects a proof bound to a different gateway origin', async () => { + const host = createPushHostKeypair(8) + const challenge = await store.issue(hostPublicKeyB64(host)) + const reasons: string[] = [] + expect( + answerPushHostChallenge(challenge!, { + gatewayOrigin: 'https://push.example.test', + keypair: host, + now: () => clock, + onInvalid: (reason) => reasons.push(reason) + }) + ).toBeNull() + expect(reasons.join()).toContain('gatewayOrigin') + }) + + it('rejects an unknown challenge id and a malformed public key', async () => { + await expect(store.verify('missing', Buffer.alloc(32, 9).toString('base64'))).resolves.toEqual({ + ok: false, + reason: 'unknown_challenge' + }) + await expect(store.issue('not-base64!!')).resolves.toBeNull() + await expect(store.issue(Buffer.alloc(31, 1).toString('base64'))).resolves.toBeNull() + }) + + it('prunes challenges that fell out of the skew window', async () => { + const host = createPushHostKeypair(9) + await store.issue(hostPublicKeyB64(host)) + expect(await store.pruneExpired()).toBe(0) + clock += PUSH_LIMITS.challengeTtlMs + PUSH_LIMITS.clockSkewToleranceMs + 1 + expect(await store.pruneExpired()).toBe(1) + }) +}) diff --git a/cloud/apps/push/src/host-challenge-store.ts b/cloud/apps/push/src/host-challenge-store.ts new file mode 100644 index 00000000000..fd12af85983 --- /dev/null +++ b/cloud/apps/push/src/host-challenge-store.ts @@ -0,0 +1,135 @@ +import { createHash, createHmac, randomBytes, randomUUID, timingSafeEqual } from 'node:crypto' +import { + buildPushHostChallengePlaintext, + buildPushHostProofMacInput, + buildPushHostProofTranscript, + PUSH_LIMITS +} from '@orca-cloud/push-contract' +import nacl from 'tweetnacl' +import { decodeCanonicalBase64 } from './canonical-base64.js' +import { deriveHostFingerprint } from './host-fingerprint.js' +import type { PushDatabase } from './push-database.js' + +export type IssuedPushChallenge = { + challengeId: string + gatewayEphemeralPublicKeyB64: string + nonceB64: string + ciphertextB64: string + expiresAt: number + hostFingerprint: string +} + +export type PushProofVerification = + | { ok: true; hostFingerprint: string } + | { ok: false; reason: 'unknown_challenge' | 'already_consumed' | 'expired' | 'proof_mismatch' } + +function sha256(value: Uint8Array): string { + return createHash('sha256').update(value).digest('base64url') +} + +function equalDigest(left: string, right: string): boolean { + const leftBytes = Buffer.from(left) + const rightBytes = Buffer.from(right) + return leftBytes.length === rightBytes.length && timingSafeEqual(leftBytes, rightBytes) +} + +export class PushHostChallengeStore { + constructor( + private readonly database: PushDatabase, + private readonly gatewayOrigin: string, + private readonly now: () => number = Date.now + ) {} + + async issue(hostPublicKeyB64: string): Promise { + const hostPublicKey = decodeCanonicalBase64(hostPublicKeyB64, 32) + if (!hostPublicKey) return null + const hostFingerprint = deriveHostFingerprint(hostPublicKey) + const ephemeral = nacl.box.keyPair() + const challengeNonce = randomBytes(nacl.box.nonceLength) + const challengeSecret = randomBytes(32) + const challengeId = randomUUID() + const issuedAt = this.now() + const expiresAt = issuedAt + PUSH_LIMITS.challengeTtlMs + const transcript = buildPushHostProofTranscript({ + gatewayOrigin: this.gatewayOrigin, + gatewayEphemeralPublicKey: ephemeral.publicKey, + challengeNonce, + challengeId, + issuedAt, + expiresAt, + hostFingerprint, + hostPublicKey + }) + const ciphertext = nacl.box( + buildPushHostChallengePlaintext(transcript, challengeSecret), + challengeNonce, + hostPublicKey, + ephemeral.secretKey + ) + const expectedProof = createHmac('sha256', challengeSecret) + .update(buildPushHostProofMacInput(transcript)) + .digest() + await this.database.query( + `INSERT INTO push_challenges + (challenge_id, host_fingerprint, secret_hash, expires_at, + consumed_at) + VALUES (?, ?, ?, ?, NULL)`, + [ + challengeId, + hostFingerprint, + // The stored digest is of the ack the secret produces, never of the + // secret itself: a database reader must not be able to forge a proof. + sha256(expectedProof), + expiresAt + ] + ) + return { + challengeId, + gatewayEphemeralPublicKeyB64: Buffer.from(ephemeral.publicKey).toString('base64'), + nonceB64: Buffer.from(challengeNonce).toString('base64'), + ciphertextB64: Buffer.from(ciphertext).toString('base64'), + expiresAt, + hostFingerprint + } + } + + async verify(challengeId: string, proofB64: string): Promise { + const proof = decodeCanonicalBase64(proofB64, 32) + return await this.database.transaction(async (transaction) => { + const [row] = await transaction.query( + `SELECT host_fingerprint, secret_hash, expires_at, consumed_at + FROM push_challenges WHERE challenge_id = ?`, + [challengeId] + ) + if (!row) return { ok: false, reason: 'unknown_challenge' } + if (row.consumed_at !== null && row.consumed_at !== undefined) { + return { ok: false, reason: 'already_consumed' } + } + const now = this.now() + // No skew allowance here: the gateway set expires_at from this same clock. + // The tolerance belongs to the host, which validates a foreign timestamp. + if (now > Number(row.expires_at)) return { ok: false, reason: 'expired' } + if (!proof || !equalDigest(sha256(proof), String(row.secret_hash))) { + return { ok: false, reason: 'proof_mismatch' } + } + // Consume under the same predicate the read used, so two concurrent + // proofs for one challenge cannot both mint a session. + const [consumed] = await transaction.query( + 'UPDATE push_challenges SET consumed_at = ? WHERE challenge_id = ? AND consumed_at IS NULL', + [now, challengeId] + ) + if (Number(consumed?.changes ?? 0) !== 1) return { ok: false, reason: 'already_consumed' } + return { ok: true, hostFingerprint: String(row.host_fingerprint) } + }) + } + + // Rows outlive the expiry check by the skew tolerance so a late proof reads + // as 'expired' rather than as an unknown challenge. + async pruneExpired(): Promise { + const cutoff = this.now() - PUSH_LIMITS.clockSkewToleranceMs + const [result] = await this.database.query('DELETE FROM push_challenges WHERE expires_at < ?', [ + cutoff + ]) + return Number(result?.changes ?? 0) + } +} diff --git a/cloud/apps/push/src/host-fingerprint.ts b/cloud/apps/push/src/host-fingerprint.ts new file mode 100644 index 00000000000..955b1ac8ecb --- /dev/null +++ b/cloud/apps/push/src/host-fingerprint.ts @@ -0,0 +1,16 @@ +import { createHash } from 'node:crypto' +import { PUSH_HOST_FINGERPRINT_LENGTH } from '@orca-cloud/push-contract' + +// Identical derivation to deriveRelayHostId on the desktop, so a host and a +// phone reach the same fingerprint from the same X25519 public key. +export function deriveHostFingerprint(hostPublicKey: Uint8Array): string { + return createHash('sha256') + .update(hostPublicKey) + .digest('base64url') + .slice(0, PUSH_HOST_FINGERPRINT_LENGTH) +} + +// Logs may carry at most this much of a fingerprint. +export function fingerprintLogPrefix(hostFingerprint: string): string { + return hostFingerprint.slice(0, 4) +} diff --git a/cloud/apps/push/src/host-proof-concurrency.test.ts b/cloud/apps/push/src/host-proof-concurrency.test.ts new file mode 100644 index 00000000000..031ca61c7fd --- /dev/null +++ b/cloud/apps/push/src/host-proof-concurrency.test.ts @@ -0,0 +1,47 @@ +import { expect, it } from 'vitest' +import { openInMemoryPushDatabase, openPushDatabase } from './push-database.js' +import { PushHostChallengeStore } from './host-challenge-store.js' +import { + answerPushHostChallenge, + createPushHostKeypair, + hostPublicKeyB64 +} from './host-challenge-answering.test-fixture.js' + +it('accepts independent proofs but consumes each challenge only once under concurrency', async () => { + const databaseUrl = process.env.ORCA_PUSH_TEST_DATABASE_URL + if (databaseUrl && !process.env.CI && new URL(databaseUrl).port !== '55440') { + throw new Error('isolated_postgres_port_required') + } + const db = databaseUrl + ? await openPushDatabase({ databaseUrl, dataDir: '', poolMax: 4 }) + : await openInMemoryPushDatabase() + const host = createPushHostKeypair() + const origin = 'https://push.onorca.dev' + const store = new PushHostChallengeStore(db, origin) + const challenges = await Promise.all([ + store.issue(hostPublicKeyB64(host)), + store.issue(hostPublicKeyB64(host)) + ]) + try { + const proofs = challenges.map((challenge) => + answerPushHostChallenge(challenge!, { gatewayOrigin: origin, keypair: host })! + ) + const results = await Promise.all( + challenges.flatMap((challenge, index) => + Array.from({ length: 5 }, () => store.verify(challenge!.challengeId, proofs[index]!)) + ) + ) + expect(results.filter((result) => result.ok)).toEqual([ + { ok: true, hostFingerprint: challenges[0]!.hostFingerprint }, + { ok: true, hostFingerprint: challenges[0]!.hostFingerprint } + ]) + expect(results.filter((result) => !result.ok)).toEqual( + Array.from({ length: 8 }, () => ({ ok: false, reason: 'already_consumed' })) + ) + } finally { + for (const challenge of challenges) { + await db.query('DELETE FROM push_challenges WHERE challenge_id = ?', [challenge!.challengeId]) + } + await db.close() + } +}) diff --git a/cloud/apps/push/src/host-session-store.test.ts b/cloud/apps/push/src/host-session-store.test.ts new file mode 100644 index 00000000000..129dba2134c --- /dev/null +++ b/cloud/apps/push/src/host-session-store.test.ts @@ -0,0 +1,70 @@ +import { PUSH_LIMITS } from '@orca-cloud/push-contract' +import { afterEach, beforeEach, describe, expect, it } from 'vitest' +import { PushHostSessionStore } from './host-session-store.js' +import { openInMemoryPushDatabase, type PushDatabase } from './push-database.js' + +const HOST = 'abcdefghijklmnop' + +describe('push host session store', () => { + let database: PushDatabase + let clock = 1_700_000_000_000 + let sessions: PushHostSessionStore + + beforeEach(async () => { + database = await openInMemoryPushDatabase() + clock = 1_700_000_000_000 + sessions = new PushHostSessionStore(database, () => clock) + }) + + afterEach(async () => { + await database.close() + }) + + it('mints a 24 hour session and stores only its hash', async () => { + const session = await sessions.create(HOST) + expect(session.expiresAt).toBe(clock + PUSH_LIMITS.sessionTtlMs) + expect(Buffer.from(session.sessionToken, 'base64url').byteLength).toBe(32) + const [row] = await database.query('SELECT token_hash FROM push_sessions') + expect(String(row?.token_hash)).not.toBe(session.sessionToken) + await expect(sessions.resolve(session.sessionToken)).resolves.toMatchObject({ + ok: true, + hostFingerprint: HOST + }) + }) + + it('reports expiry separately from an unknown token', async () => { + const session = await sessions.create(HOST) + clock += PUSH_LIMITS.sessionTtlMs + 1 + await expect(sessions.resolve(session.sessionToken)).resolves.toEqual({ + ok: false, + reason: 'session_expired' + }) + await expect(sessions.resolve('not-a-session')).resolves.toEqual({ + ok: false, + reason: 'unknown_session' + }) + }) + + it('accepts a session on its final millisecond', async () => { + const session = await sessions.create(HOST) + clock += PUSH_LIMITS.sessionTtlMs + await expect(sessions.resolve(session.sessionToken)).resolves.toMatchObject({ ok: true }) + }) + + it('keeps one live session per host and prunes it once expired', async () => { + const first = await sessions.create(HOST) + const second = await sessions.create(HOST) + // The earlier session is gone the moment its host proves again, so a flood + // of proofs leaves one row per host rather than one per proof. + await expect(sessions.resolve(first.sessionToken)).resolves.toEqual({ + ok: false, + reason: 'unknown_session' + }) + await expect(sessions.resolve(second.sessionToken)).resolves.toMatchObject({ ok: true }) + const other = await sessions.create('ponmlkjihgfedcba') + await expect(sessions.resolve(second.sessionToken)).resolves.toMatchObject({ ok: true }) + clock += PUSH_LIMITS.sessionTtlMs + 1 + expect(await sessions.pruneExpired()).toBe(2) + await expect(sessions.resolve(other.sessionToken)).resolves.toMatchObject({ ok: false }) + }) +}) diff --git a/cloud/apps/push/src/host-session-store.ts b/cloud/apps/push/src/host-session-store.ts new file mode 100644 index 00000000000..899bacabcc8 --- /dev/null +++ b/cloud/apps/push/src/host-session-store.ts @@ -0,0 +1,65 @@ +import { createHash, randomBytes } from 'node:crypto' +import { PUSH_LIMITS } from '@orca-cloud/push-contract' +import type { PushDatabase } from './push-database.js' + +export type IssuedPushSession = { + sessionToken: string + expiresAt: number + hostFingerprint: string +} + +export type PushSessionLookup = + | { ok: true; hostFingerprint: string; expiresAt: number } + | { ok: false; reason: 'unknown_session' | 'session_expired' } + +function hashSessionToken(sessionToken: string): string { + return createHash('sha256').update(sessionToken).digest('base64url') +} + +export class PushHostSessionStore { + constructor( + private readonly database: PushDatabase, + private readonly now: () => number = Date.now + ) {} + + async create(hostFingerprint: string): Promise { + const sessionToken = randomBytes(32).toString('base64url') + const createdAt = this.now() + const expiresAt = createdAt + PUSH_LIMITS.sessionTtlMs + await this.database.transaction(async (transaction) => { + // Why: a desktop holds one session at a time and only re-proves once it is + // gone, so an earlier row is dead weight. It also bounds the table to one + // row per host however many proofs a self-minted identity answers. + await transaction.lockQuotaScope(`orca-push-session:${hostFingerprint}`) + await transaction.query('DELETE FROM push_sessions WHERE host_fingerprint = ?', [ + hostFingerprint + ]) + await transaction.query( + `INSERT INTO push_sessions (token_hash, host_fingerprint, expires_at, created_at) + VALUES (?, ?, ?, ?)`, + [hashSessionToken(sessionToken), hostFingerprint, expiresAt, createdAt] + ) + }) + return { sessionToken, expiresAt, hostFingerprint } + } + + async resolve(sessionToken: string): Promise { + const [row] = await this.database.query( + 'SELECT host_fingerprint, expires_at FROM push_sessions WHERE token_hash = ?', + [hashSessionToken(sessionToken)] + ) + if (!row) return { ok: false, reason: 'unknown_session' } + const expiresAt = Number(row.expires_at) + // No skew grace here: a 24h session that just expired should be re-minted + // through the challenge, which is cheap and already handled by the host. + if (this.now() > expiresAt) return { ok: false, reason: 'session_expired' } + return { ok: true, hostFingerprint: String(row.host_fingerprint), expiresAt } + } + + async pruneExpired(): Promise { + const [result] = await this.database.query('DELETE FROM push_sessions WHERE expires_at < ?', [ + this.now() + ]) + return Number(result?.changes ?? 0) + } +} diff --git a/cloud/apps/push/src/index.ts b/cloud/apps/push/src/index.ts new file mode 100644 index 00000000000..d44148874c7 --- /dev/null +++ b/cloud/apps/push/src/index.ts @@ -0,0 +1,55 @@ +import { startPushBackground } from './push-background.js' +import { loadPushConfig } from './config.js' +import { openPushDatabase } from './push-database.js' +import { createPushServer } from './push-server.js' + +const config = loadPushConfig() +const database = await openPushDatabase({ + ...(config.databaseUrl === undefined ? {} : { databaseUrl: config.databaseUrl }), + dataDir: config.dataDir, + poolMax: config.databasePoolMax, + applicationName: 'orca-push', + readOnly: config.mode === 'validation' +}) +const { + server, + challenges, + sessions, + deliveryStore, + worker, + observability, + closeTransports, + requestDrain +} = createPushServer(config, database) + +const stopBackground = startPushBackground(config, { challenges, sessions, deliveryStore, worker }) +observability.start() + +server.listen(config.port, () => { + console.log(`[orca-push] listening on ${config.publicUrl} (port ${config.port})`) +}) + +let stopping = false +const shutdown = (): void => { + if (stopping) return + stopping = true + // Cloud Run sends SIGKILL after ten seconds; leave time for explicit cleanup. + const deadline = setTimeout(() => process.exit(1), 9_000) + deadline.unref() + const requests = requestDrain.begin() + const deliveries = stopBackground() + const connections = new Promise((resolve) => server.close(() => resolve())) + void Promise.all([requests, connections, deliveries]) + .then(async () => { + closeTransports() + await database.close() + observability.stop() + clearTimeout(deadline) + }) + .catch(() => { + console.warn(JSON.stringify({ event: 'orca_push_shutdown_failed' })) + process.exitCode = 1 + }) +} +process.once('SIGTERM', shutdown) +process.once('SIGINT', shutdown) diff --git a/cloud/apps/push/src/provider-retry-delay.ts b/cloud/apps/push/src/provider-retry-delay.ts new file mode 100644 index 00000000000..4c77b3c6dc7 --- /dev/null +++ b/cloud/apps/push/src/provider-retry-delay.ts @@ -0,0 +1,9 @@ +export function providerRetryAfter( + value: string | undefined, + now = Date.now() +): number | undefined { + if (!value) return undefined + const seconds = Number(value) + const delay = Number.isFinite(seconds) ? seconds * 1000 : Date.parse(value) - now + return Number.isFinite(delay) ? Math.max(0, delay) : undefined +} diff --git a/cloud/apps/push/src/push-auth-admission.ts b/cloud/apps/push/src/push-auth-admission.ts new file mode 100644 index 00000000000..cbc951abe25 --- /dev/null +++ b/cloud/apps/push/src/push-auth-admission.ts @@ -0,0 +1,21 @@ +// Bound unauthenticated database lookups independently of Cloud Run HTTP concurrency. +export class PushAuthAdmission { + private active = 0 + private readonly waiting: (() => void)[] = [] + + async run(operation: () => Promise): Promise { + if (this.active >= 4) { + if (this.waiting.length >= 32) return null + await new Promise((resolve) => this.waiting.push(resolve)) + } else { + this.active++ + } + try { + return await operation() + } finally { + const next = this.waiting.shift() + if (next) next() + else this.active-- + } + } +} diff --git a/cloud/apps/push/src/push-authenticated-ip-limit.test.ts b/cloud/apps/push/src/push-authenticated-ip-limit.test.ts new file mode 100644 index 00000000000..a626b80060a --- /dev/null +++ b/cloud/apps/push/src/push-authenticated-ip-limit.test.ts @@ -0,0 +1,50 @@ +import { PUSH_LIMITS } from '@orca-cloud/push-contract' +import { expect, it } from 'vitest' +import { createPushHostKeypair } from './host-challenge-answering.test-fixture.js' +import { createPushServerHarness, FCM_TOKEN } from './push-server-harness.test-fixture.js' + +it('bounds valid hosts together across routes without letting key rotation reset the IP budget', async () => { + const harness = await createPushServerHarness() + const headers = { 'x-forwarded-for': '203.0.113.7' } + try { + const hostCount = + PUSH_LIMITS.authenticatedRequestsPerMinutePerIp / + PUSH_LIMITS.authenticatedRequestsPerMinutePerHost + for (let host = 0; host < hostCount; host++) { + const token = await harness.signIn(createPushHostKeypair(host + 1)) + for ( + let request = 0; + request < PUSH_LIMITS.authenticatedRequestsPerMinutePerHost; + request++ + ) { + expect((await harness.authorized('/v1/devices', { headers }, token)).status).toBe(200) + } + } + const token = await harness.signIn(createPushHostKeypair(99)) + const registration = { + method: 'POST', + headers: { ...headers, 'content-type': 'application/json' }, + body: JSON.stringify({ v: 1, deviceId: 'phone', platform: 'android', token: FCM_TOKEN }) + } + expect((await harness.authorized('/v1/devices', registration, token)).status).toBe(429) + expect((await harness.authorized('/v1/send', { method: 'POST', headers }, token)).status).toBe( + 429 + ) + expect( + ( + await harness.authorized( + '/v1/devices', + { + ...registration, + headers: { ...registration.headers, 'x-forwarded-for': '198.51.100.9' } + }, + token + ) + ).status + ).toBe(200) + harness.advanceClock(60_000) + expect((await harness.authorized('/v1/devices', registration, token)).status).toBe(200) + } finally { + await harness.close() + } +}, 30_000) diff --git a/cloud/apps/push/src/push-background.ts b/cloud/apps/push/src/push-background.ts new file mode 100644 index 00000000000..8925fd41fed --- /dev/null +++ b/cloud/apps/push/src/push-background.ts @@ -0,0 +1,43 @@ +import type { PushConfig } from './config.js' +import type { createPushServer } from './push-server.js' + +const CHALLENGE_PRUNE_INTERVAL_MS = 60_000 +const SESSION_PRUNE_INTERVAL_MS = 10 * 60_000 +const DELIVERY_PRUNE_INTERVAL_MS = 60_000 + +function prune(label: string, run: () => Promise, intervalMs: number): NodeJS.Timeout { + const timer = setInterval(() => { + void run().catch((error: unknown) => { + console.warn( + JSON.stringify({ + event: 'orca_push_prune_failed', + target: label, + error: error instanceof Error ? error.name : 'unknown' + }) + ) + }) + }, intervalMs) + timer.unref() + return timer +} + +export function startPushBackground( + config: Pick, + runtime: Pick< + ReturnType, + 'challenges' | 'sessions' | 'deliveryStore' | 'worker' + > +): () => Promise { + if (config.mode === 'validation') return async () => {} + const { challenges, sessions, deliveryStore, worker } = runtime + const timers = [ + prune('challenges', () => challenges.pruneExpired(), CHALLENGE_PRUNE_INTERVAL_MS), + prune('sessions', () => sessions.pruneExpired(), SESSION_PRUNE_INTERVAL_MS), + prune('deliveries', () => deliveryStore.prune(), DELIVERY_PRUNE_INTERVAL_MS) + ] + worker.start() + return async () => { + for (const timer of timers) clearInterval(timer) + await worker.stop() + } +} diff --git a/cloud/apps/push/src/push-database-postgres-startup.test.ts b/cloud/apps/push/src/push-database-postgres-startup.test.ts new file mode 100644 index 00000000000..b6fd23071fc --- /dev/null +++ b/cloud/apps/push/src/push-database-postgres-startup.test.ts @@ -0,0 +1,121 @@ +import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest' +import { randomUUID } from 'node:crypto' + +const fakes = vi.hoisted(() => ({ + configs: [] as Array>, + lifecycle: [] as string[], + query: vi.fn(async (_sql: string) => ({ rows: [], rowCount: 0 })), + release: vi.fn() +})) + +vi.mock('pg', () => ({ + default: { + Pool: class { + on = vi.fn() + connect = vi.fn(async () => ({ query: fakes.query, release: fakes.release })) + private readonly label: string + + constructor(config: Record) { + fakes.configs.push(config) + this.label = `max=${String(config.max)} statement_timeout=${String(config.statement_timeout)}` + fakes.lifecycle.push(`open ${this.label}`) + } + + async end(): Promise { + fakes.lifecycle.push(`end ${this.label}`) + } + } + } +})) + +import { openPushDatabase } from './push-database.js' +import { pushSchemaStatements } from './push-schema.js' + +describe('PostgreSQL push gateway startup', () => { + beforeEach(() => { + fakes.configs.length = 0 + fakes.lifecycle.length = 0 + fakes.query.mockClear() + }) + + afterEach(() => { + vi.restoreAllMocks() + }) + + const socketPassword = `${randomUUID()}@/` + const socketUrl = `postgresql://push:${encodeURIComponent(socketPassword)}@/orca_push?host=/cloudsql/test:region:instance` + + it('passes the Terraform socket URL unchanged to both active pools', async () => { + const database = await openPushDatabase({ databaseUrl: socketUrl, dataDir: '/unused' }) + expect(fakes.configs.map((config) => config.connectionString)).toEqual([socketUrl, socketUrl]) + await database.close() + }) + + it('parses socket credentials and query options before enforcing validation read-only', async () => { + const options = '-c search_path=validation -c default_transaction_read_only=off' + const database = await openPushDatabase({ + databaseUrl: `${socketUrl}&port=5433&sslmode=disable&options=${encodeURIComponent(options)}`, + dataDir: '/unused', + readOnly: true + }) + expect(fakes.configs).toHaveLength(1) + expect(fakes.configs[0]).toMatchObject({ + host: '/cloudsql/test:region:instance', + user: 'push', + password: socketPassword, + database: 'orca_push', + port: 5433, + ssl: false, + options: `${options} -c default_transaction_read_only=on` + }) + expect(fakes.configs[0]).not.toHaveProperty('connectionString') + expect(fakes.query).not.toHaveBeenCalled() + await database.close() + }) + + // Why: a CREATE INDEX on a grown table can outlive the 5s request deadline, + // and a schema that inherits it fails every startup at the same statement. + it('applies the schema on an untimed pool that is gone before the serving pool opens', async () => { + const database = await openPushDatabase({ + databaseUrl: 'postgresql://push@localhost:55440/orca_push', + dataDir: '/unused', + poolMax: 2, + applicationName: 'orca-push' + }) + expect(fakes.lifecycle).toEqual([ + 'open max=1 statement_timeout=0', + 'end max=1 statement_timeout=0', + 'open max=2 statement_timeout=5000' + ]) + expect(fakes.configs[0]).toMatchObject({ + application_name: 'orca-push/schema', + lock_timeout: 1_000, + idle_in_transaction_session_timeout: 5_000 + }) + expect( + fakes.query.mock.calls.map(([sql]) => sql).slice(0, pushSchemaStatements().length) + ).toEqual(pushSchemaStatements()) + await database.close() + }) + + it('retries a transaction the pool statement_timeout aborted', async () => { + const database = await openPushDatabase({ + databaseUrl: 'postgresql://push@localhost:55440/orca_push', + dataDir: '/unused' + }) + const warn = vi.spyOn(console, 'warn').mockImplementation(() => undefined) + let attempts = 0 + const result = await database.transaction(async () => { + attempts += 1 + if (attempts === 1) throw Object.assign(new Error('canceling statement'), { code: '57014' }) + return 'done' + }) + expect(result).toBe('done') + expect(attempts).toBe(2) + expect(warn.mock.calls.map(([line]) => String(line))).toEqual([ + expect.stringContaining('"code":"57014"') + ]) + warn.mockRestore() + await database.close() + }) +}) diff --git a/cloud/apps/push/src/push-database.ts b/cloud/apps/push/src/push-database.ts new file mode 100644 index 00000000000..a2ed2bce9d2 --- /dev/null +++ b/cloud/apps/push/src/push-database.ts @@ -0,0 +1,283 @@ +import { mkdirSync } from 'node:fs' +import { join } from 'node:path' +import { DatabaseSync } from 'node:sqlite' +import pg from 'pg' +import { parseIntoClientConfig } from 'pg-connection-string' +import { applyPostgresSchema } from '@orca-cloud/postgres-schema' +import { pushSchemaStatements } from './push-schema.js' + +const POSTGRES_LOCK_TIMEOUT_MS = 1_000 +const POSTGRES_CONNECTION_TIMEOUT_MS = 2_000 +const POSTGRES_STATEMENT_TIMEOUT_MS = 5_000 +const POSTGRES_IDLE_TRANSACTION_TIMEOUT_MS = 5_000 +const POSTGRES_TRANSACTION_ATTEMPTS = 3 +const POSTGRES_RETRY_MAX_DELAY_MS = 25 + +export type SqlRow = Record + +export interface PushDatabase { + readonly dialect: 'sqlite' | 'postgres' + query(sql: string, params?: unknown[]): Promise + transaction(operation: (transaction: PushDatabase) => Promise): Promise + // Serializes every transaction that reads then writes the same identity's + // quota rows. Must be called inside a transaction; it releases at commit. + lockQuotaScope(key: string): Promise + close(): Promise +} + +function postgresSql(sql: string): string { + let index = 0 + return sql.replace(/\?/g, () => `$${++index}`) +} + +function returnsRows(sql: string): boolean { + return /^\s*(select|with)/i.test(sql) || /returning/i.test(sql) +} + +class SqliteTransaction implements PushDatabase { + readonly dialect = 'sqlite' as const + + constructor(protected readonly database: DatabaseSync) {} + + async query(sql: string, params: unknown[] = []): Promise { + const statement = this.database.prepare(sql) + const bound = params.map((value) => (value === undefined ? null : value)) as never[] + if (returnsRows(sql)) return statement.all(...bound) as SqlRow[] + const result = statement.run(...bound) + return [{ changes: Number(result.changes) }] + } + + async transaction(operation: (transaction: PushDatabase) => Promise): Promise { + return await operation(this) + } + + // BEGIN IMMEDIATE already holds the single writer lock for the whole + // transaction, so there is nothing narrower left to take. + async lockQuotaScope(): Promise {} + + async close(): Promise {} +} + +class SqliteDatabase extends SqliteTransaction { + // node:sqlite is synchronous and has no nested transactions, so overlapping + // callers are serialized behind one tail promise instead of racing BEGIN. + private tail: Promise = Promise.resolve() + + override async query(sql: string, params: unknown[] = []): Promise { + await this.tail + return await super.query(sql, params) + } + + override async transaction(operation: (transaction: PushDatabase) => Promise): Promise { + const previous = this.tail + let release!: () => void + this.tail = new Promise((resolve) => (release = resolve)) + await previous + this.database.exec('BEGIN IMMEDIATE') + const transaction = new SqliteTransaction(this.database) + try { + const result = await operation(transaction) + this.database.exec('COMMIT') + return result + } catch (error) { + this.database.exec('ROLLBACK') + throw error + } finally { + release() + } + } + + override async close(): Promise { + await this.tail + this.database.close() + } +} + +class PostgresTransaction implements PushDatabase { + readonly dialect = 'postgres' as const + + constructor(private readonly client: pg.PoolClient) {} + + async query(sql: string, params: unknown[] = []): Promise { + const result = await this.client.query(postgresSql(sql), params) + return returnsRows(sql) ? (result.rows as SqlRow[]) : [{ changes: result.rowCount ?? 0 }] + } + + async transaction(operation: (transaction: PushDatabase) => Promise): Promise { + return await operation(this) + } + + // READ COMMITTED lets a concurrent count-then-insert read the same + // under-quota total, so the identity is serialized for the whole transaction. + async lockQuotaScope(key: string): Promise { + await this.query('SELECT pg_advisory_xact_lock(hashtext(?::text))', [key]) + } + + async close(): Promise {} +} + +function retryablePostgresTransactionError(error: unknown): boolean { + const code = String((error as { code?: unknown }).code) + // 57014 is the pool statement_timeout firing. It aborts the transaction the + // same way a lock timeout does, so it takes the bounded retry path too. + return code === '40P01' || code === '40001' || code === '55P03' || code === '57014' +} + +async function waitForPostgresRetry(): Promise { + const delayMs = Math.floor(Math.random() * (POSTGRES_RETRY_MAX_DELAY_MS + 1)) + await new Promise((resolve) => setTimeout(resolve, delayMs)) +} + +class PostgresDatabase implements PushDatabase { + readonly dialect = 'postgres' as const + + constructor(private readonly pool: pg.Pool) {} + + async query(sql: string, params: unknown[] = []): Promise { + const client = await this.pool.connect() + try { + const result = await client.query(postgresSql(sql), params) + return returnsRows(sql) ? (result.rows as SqlRow[]) : [{ changes: result.rowCount ?? 0 }] + } finally { + client.release() + } + } + + async transaction(operation: (transaction: PushDatabase) => Promise): Promise { + for (let attempt = 1; attempt <= POSTGRES_TRANSACTION_ATTEMPTS; attempt++) { + const client = await this.pool.connect() + try { + await client.query('BEGIN') + const result = await operation(new PostgresTransaction(client)) + await client.query('COMMIT') + return result + } catch (error) { + await client.query('ROLLBACK').catch(() => undefined) + if ( + !retryablePostgresTransactionError(error) || + attempt === POSTGRES_TRANSACTION_ATTEMPTS + ) { + throw error + } + console.warn( + JSON.stringify({ + event: 'orca_push_postgres_transaction_retry', + code: String((error as { code?: unknown }).code), + attempt + }) + ) + } finally { + client.release() + } + // A PostgreSQL transaction is unusable after an abort, so retry all work + // on a fresh pooled client with a small full-jitter delay. + await waitForPostgresRetry() + } + throw new Error('postgres_transaction_retry_exhausted') + } + + // An advisory transaction lock taken outside a transaction is released by the + // implicit commit before the caller reads anything, which protects nothing. + async lockQuotaScope(): Promise { + throw new Error('lock_quota_scope_requires_transaction') + } + + async close(): Promise { + await this.pool.end() + } +} + +async function applySchema(database: PushDatabase): Promise { + for (const statement of pushSchemaStatements()) await database.query(statement) +} + +// Why: DDL is not a request. A CREATE INDEX on a grown table can legitimately +// outlive the request statement_timeout, and inheriting it would fail every +// startup at the same statement instead of finishing once. One connection of +// its own, closed before the serving pool opens, keeps the untimed session off +// the request path entirely. +async function applySchemaOnUntimedPool( + databaseUrl: string, + applicationName: string | undefined +): Promise { + const pool = new pg.Pool({ + connectionString: databaseUrl, + max: 1, + application_name: applicationName ? `${applicationName}/schema` : undefined, + connectionTimeoutMillis: POSTGRES_CONNECTION_TIMEOUT_MS, + statement_timeout: 0, + lock_timeout: POSTGRES_LOCK_TIMEOUT_MS, + idle_in_transaction_session_timeout: POSTGRES_IDLE_TRANSACTION_TIMEOUT_MS + }) + absorbPostgresIdleClientErrors(pool) + const database = new PostgresDatabase(pool) + try { + await applyPostgresSchema(pushSchemaStatements(), (statement) => database.query(statement), { + eventPrefix: 'orca_push_postgres_schema' + }) + } finally { + await database.close().catch(() => undefined) + } +} + +export function absorbPostgresIdleClientErrors(pool: Pick): void { + pool.on('error', () => { + // node-postgres removes failed idle clients itself; an unhandled 'error' + // would crash the service and turn a SQL blip into a restart loop. + console.warn('[orca-push] idle PostgreSQL client failed') + }) +} + +export async function openPushDatabase(input: { + databaseUrl?: string + dataDir: string + poolMax?: number + applicationName?: string + readOnly?: boolean +}): Promise { + let database: PushDatabase + if (input.databaseUrl) { + if (!input.readOnly) await applySchemaOnUntimedPool(input.databaseUrl, input.applicationName) + let connection: pg.ClientConfig = { connectionString: input.databaseUrl } + if (input.readOnly) { + connection = parseIntoClientConfig(input.databaseUrl) + // A URL parameter must not trigger a second parse that overrides read-only options. + delete connection.connectionString + connection.options = `${connection.options ?? ''} -c default_transaction_read_only=on`.trim() + } + const pool = new pg.Pool({ + ...connection, + max: input.poolMax ?? 10, + application_name: input.applicationName, + connectionTimeoutMillis: POSTGRES_CONNECTION_TIMEOUT_MS, + statement_timeout: POSTGRES_STATEMENT_TIMEOUT_MS, + lock_timeout: POSTGRES_LOCK_TIMEOUT_MS, + idle_in_transaction_session_timeout: POSTGRES_IDLE_TRANSACTION_TIMEOUT_MS + }) + absorbPostgresIdleClientErrors(pool) + database = new PostgresDatabase(pool) + } else { + mkdirSync(input.dataDir, { recursive: true }) + const sqlite = new DatabaseSync(join(input.dataDir, 'orca-push.sqlite'), { + readOnly: input.readOnly ?? false + }) + if (!input.readOnly) sqlite.exec('PRAGMA journal_mode = WAL; PRAGMA foreign_keys = ON;') + database = new SqliteDatabase(sqlite) + } + if (database.dialect === 'postgres' || input.readOnly) return database + try { + await applySchema(database) + return database + } catch (error) { + await database.close().catch(() => undefined) + throw error + } +} + +export async function openInMemoryPushDatabase(): Promise { + const sqlite = new DatabaseSync(':memory:') + sqlite.exec('PRAGMA foreign_keys = ON;') + const database = new SqliteDatabase(sqlite) + await applySchema(database) + return database +} diff --git a/cloud/apps/push/src/push-delivery-lifecycle.test.ts b/cloud/apps/push/src/push-delivery-lifecycle.test.ts new file mode 100644 index 00000000000..c94108e459c --- /dev/null +++ b/cloud/apps/push/src/push-delivery-lifecycle.test.ts @@ -0,0 +1,85 @@ +import { afterEach, expect, it, vi } from 'vitest' +import { Hono } from 'hono' +import { PushRequestDrain } from './push-request-drain.js' +import { PushDispatcher } from './push-dispatcher.js' +import { PushDeviceRegistryStore } from './device-registry-store.js' +import { openInMemoryPushDatabase, type PushDatabase } from './push-database.js' +import { buildPushDelivery } from './push-delivery-message.js' +import { PushNotificationSchema } from '@orca-cloud/push-contract' +import { notification } from './push-server-harness.test-fixture.js' + +const databases: PushDatabase[] = [] +afterEach(async () => { + await Promise.all(databases.splice(0).map((db) => db.close())) + vi.restoreAllMocks() +}) +const note = PushNotificationSchema.parse(notification()) +const tick = () => new Promise((resolve) => setImmediate(resolve)) +function deferred() { + let resolve!: () => void + const promise = new Promise((done) => { + resolve = done + }) + return { promise, resolve } +} +async function registered() { + const db = await openInMemoryPushDatabase() + databases.push(db) + const devices = new PushDeviceRegistryStore(db) + const input = { + hostFingerprint: 'abcdefghijklmnop', + deviceId: 'device', + platform: 'android' as const, + token: 'old-token' + } + const row = await devices.upsert(input) + if (!row.ok) throw new Error('registration failed') + const delivery = buildPushDelivery({ + expiresAt: Date.now() + 300_000, + registrationId: row.registrationId, + hostFingerprint: input.hostFingerprint, + notification: note + }) + return { db, devices, input, delivery } +} + +it('does not retire a refreshed token after the old token fails', async () => { + const h = await registered() + const gate = deferred() + const send = vi.fn(async () => { + await gate.promise + return { status: 'dead', reason: 'UNREGISTERED' } + }) + vi.spyOn(console, 'warn').mockImplementation(() => {}) + const dispatcher = new PushDispatcher({ devices: h.devices, fcm: { send } as never }) + const pending = dispatcher.sendOnce(h.delivery) + await tick() + await h.devices.upsert({ ...h.input, token: 'replacement-token' }) + gate.resolve() + await pending + expect(await h.devices.findById(h.delivery.registrationId)).toMatchObject({ + token: 'replacement-token', + dead: false + }) +}) + +it('rejects new requests during drain and waits for an admitted handler', async () => { + const gate = deferred() + const requests = new PushRequestDrain() + const app = new Hono().use('*', requests.middleware).post('/send', async (c) => { + await gate.promise + return c.json({ queued: true }) + }) + const pending = app.request('/send', { method: 'POST' }) + await tick() + let drained = false + const drain = requests.begin().then(() => { + drained = true + }) + expect((await app.request('/send', { method: 'POST' })).status).toBe(503) + expect(drained).toBe(false) + gate.resolve() + expect((await pending).status).toBe(200) + await drain + expect(drained).toBe(true) +}) diff --git a/cloud/apps/push/src/push-delivery-message.ts b/cloud/apps/push/src/push-delivery-message.ts new file mode 100644 index 00000000000..bc2c1a5d9b2 --- /dev/null +++ b/cloud/apps/push/src/push-delivery-message.ts @@ -0,0 +1,75 @@ +import { createHash } from 'node:crypto' +import { type PushNotification } from '@orca-cloud/push-contract' + +export type PushOrcaData = { + kind?: 'alert' | 'dismiss' + hostFingerprint: string + worktreeId?: string + notificationId?: string + notificationSeq: number + notificationEpoch: string + source: string + agentState: string | null +} + +export type PushDelivery = { + expiresAt: number + sound?: boolean + registrationId: string + hostFingerprint: string + title: string + body: string + collapseId: string + orca: PushOrcaData +} + +export function collapseIdFor(notification: PushNotification, hostFingerprint: string): string { + const identity = + notification.notificationId === undefined + ? [notification.notificationEpoch, notification.notificationSeq] + : notification.notificationId + return createHash('sha256') + .update(JSON.stringify([hostFingerprint, identity])) + .digest('hex') +} + +export function buildPushDelivery(input: { + expiresAt: number + registrationId: string + hostFingerprint: string + notification: PushNotification +}): PushDelivery { + const { notification, hostFingerprint } = input + return { + ...(notification.sound === false ? { sound: false } : {}), + expiresAt: input.expiresAt, + registrationId: input.registrationId, + hostFingerprint, + title: notification.title, + body: notification.body, + collapseId: collapseIdFor(notification, hostFingerprint), + orca: { + ...(notification.kind ? { kind: notification.kind } : {}), + hostFingerprint, + ...(notification.worktreeId === undefined ? {} : { worktreeId: notification.worktreeId }), + ...(notification.notificationId === undefined + ? {} + : { notificationId: notification.notificationId }), + notificationSeq: notification.notificationSeq, + notificationEpoch: notification.notificationEpoch, + source: notification.source, + agentState: notification.agentState + } + } +} + +export function orcaDataStrings(orca: PushOrcaData): Record { + return Object.fromEntries( + Object.entries(orca) + .filter(([, value]) => value !== undefined && value !== null) + .map(([key, value]) => [ + key, + typeof value === 'object' ? JSON.stringify(value) : String(value) + ]) + ) +} diff --git a/cloud/apps/push/src/push-delivery-payload.ts b/cloud/apps/push/src/push-delivery-payload.ts new file mode 100644 index 00000000000..825286125b7 --- /dev/null +++ b/cloud/apps/push/src/push-delivery-payload.ts @@ -0,0 +1,8 @@ +import type { PushNotification } from '@orca-cloud/push-contract' + +export function parsePushDeliveryPayload(payload: string): PushNotification { + const value: unknown = JSON.parse(payload) + if (!value || typeof value !== 'object' || Array.isArray(value)) + throw new Error('invalid_push_delivery_payload') + return value as PushNotification +} diff --git a/cloud/apps/push/src/push-deploy-preflight.test.ts b/cloud/apps/push/src/push-deploy-preflight.test.ts new file mode 100644 index 00000000000..4652ef8edaa --- /dev/null +++ b/cloud/apps/push/src/push-deploy-preflight.test.ts @@ -0,0 +1,17 @@ +import { readFileSync } from 'node:fs' +import { expect, it } from 'vitest' +import { loadPushConfig } from './config.js' + +it('runs the deployment image preflight against the actual config loader', () => { + const workflow = readFileSync( + new URL('../../../../.github/workflows/cloud-push-deploy.yml', import.meta.url), + 'utf8' + ) + const step = workflow.split('- name: Require image support for inert validation')[1]! + const script = step.match(/--input-type=module -e '([\s\S]*?)'/)?.[1] + expect(script).toBeDefined() + const run = new Function('loadPushConfig', script!.replace(/import .*?;/, '')) + expect(() => run(loadPushConfig)).not.toThrow() + expect(() => run(() => ({ mode: 'active' }))).toThrow('validation_mode_unsupported') + expect(() => run(() => ({ mode: 'validation' }))).toThrow('validation_mode_not_fail_closed') +}) diff --git a/cloud/apps/push/src/push-dismissal-provider.test.ts b/cloud/apps/push/src/push-dismissal-provider.test.ts new file mode 100644 index 00000000000..15362105777 --- /dev/null +++ b/cloud/apps/push/src/push-dismissal-provider.test.ts @@ -0,0 +1,27 @@ +import { expect, it } from 'vitest' +import { apnsBody } from './apns-client.js' +import { fcmMessageBody } from './fcm-client.js' +import { buildPushDelivery } from './push-delivery-message.js' + +it('dismissal provider payloads cannot display a new alert or play a sound', () => { + const delivery = buildPushDelivery({ + expiresAt: Date.now() + 300_000, + registrationId: 'reg', + hostFingerprint: 'host', + notification: { + kind: 'dismiss', + notificationId: 'note', + notificationSeq: 2, + notificationEpoch: 'epoch', + source: 'agent-task-complete', + agentState: null, + title: 'Orca', + body: '' + } + }) + expect(JSON.parse(apnsBody(delivery)).aps).toEqual({ 'content-available': 1 }) + const android = JSON.parse(fcmMessageBody({ delivery, token: 'test', channelId: 'test' })).message + expect(android).not.toHaveProperty('notification') + expect(android.android).not.toHaveProperty('notification') + expect(android.data.kind).toBe('dismiss') +}) diff --git a/cloud/apps/push/src/push-dispatcher.ts b/cloud/apps/push/src/push-dispatcher.ts new file mode 100644 index 00000000000..ca9006bc435 --- /dev/null +++ b/cloud/apps/push/src/push-dispatcher.ts @@ -0,0 +1,52 @@ +import type { ApnsClient } from './apns-client.js' +import type { PushDeviceRegistryStore } from './device-registry-store.js' +import type { FcmClient } from './fcm-client.js' +import { fingerprintLogPrefix } from './host-fingerprint.js' +import type { PushDelivery } from './push-delivery-message.js' +import type { PushProviderOutcome } from './push-provider-outcome.js' + +export type PushDispatcherOptions = { + devices: PushDeviceRegistryStore + apns?: ApnsClient + fcm?: FcmClient + onOutcome?: (outcome: PushProviderOutcome['status']) => void +} + +// Retires the registration when the provider says the token is gone. +export class PushDispatcher { + constructor(private readonly options: PushDispatcherOptions) {} + + async sendOnce(delivery: PushDelivery): Promise { + const device = await this.options.devices.findById(delivery.registrationId) + if (!device || device.dead) return { status: 'dead', reason: 'registration_unavailable' } + let outcome: PushProviderOutcome + if (device.platform === 'ios') { + outcome = this.options.apns + ? await this.options.apns.send(delivery, { + token: device.token, + apnsEnvironment: device.apnsEnvironment ?? 'production' + }) + : { status: 'error', reason: 'apns_not_configured' } + } else { + outcome = this.options.fcm + ? await this.options.fcm.send(delivery, { token: device.token }) + : { status: 'error', reason: 'fcm_not_configured' } + } + this.options.onOutcome?.(outcome.status) + if (outcome.status === 'dead') { + await this.options.devices.markDead(device) + } + if (outcome.status !== 'sent') { + console.warn( + JSON.stringify({ + event: 'orca_push_delivery_failed', + platform: device.platform, + status: outcome.status, + reason: outcome.reason, + host: fingerprintLogPrefix(delivery.hostFingerprint) + }) + ) + } + return outcome + } +} diff --git a/cloud/apps/push/src/push-notification-sound.test.ts b/cloud/apps/push/src/push-notification-sound.test.ts new file mode 100644 index 00000000000..4dd1b85504f --- /dev/null +++ b/cloud/apps/push/src/push-notification-sound.test.ts @@ -0,0 +1,29 @@ +import { expect, it } from 'vitest' +import { apnsBody } from './apns-client.js' +import { fcmMessageBody } from './fcm-client.js' +import { buildPushDelivery } from './push-delivery-message.js' +import { PushNotificationSchema } from '@orca-cloud/push-contract' + +it('carries a silent preference through validation to APNs and Android payloads', () => { + const notification = PushNotificationSchema.parse({ + notificationSeq: 1, + notificationEpoch: 'epoch', + source: 'terminal-bell', + agentState: null, + title: 'Bell', + body: '', + sound: false + }) + const delivery = buildPushDelivery({ + expiresAt: Date.now() + 300_000, + registrationId: 'reg', + hostFingerprint: 'host', + notification + }) + expect(JSON.parse(apnsBody(delivery)).aps).not.toHaveProperty('sound') + expect( + JSON.parse(fcmMessageBody({ delivery, token: 'test-token', channelId: 'orca-desktop' })).message + .android.notification.channel_id + ).toBe('orca-desktop-silent') + expect(JSON.parse(apnsBody({ ...delivery, sound: undefined })).aps.sound).toBe('default') +}) diff --git a/cloud/apps/push/src/push-observability.ts b/cloud/apps/push/src/push-observability.ts new file mode 100644 index 00000000000..6f0dccb8a09 --- /dev/null +++ b/cloud/apps/push/src/push-observability.ts @@ -0,0 +1,56 @@ +const COUNTER_NAMES = [ + 'ip_rate_limited', + 'request_error', + 'challenge_issued', + 'challenge_rejected', + 'session_issued', + 'session_rejected', + 'device_registered', + 'device_rejected', + 'device_deleted', + 'send_queued', + 'send_dead', + 'send_rate_limited', + 'send_error', + 'delivery_sent', + 'delivery_dead', + 'delivery_error', + 'delivery_retry' +] as const + +type PushCounterName = (typeof COUNTER_NAMES)[number] + +// Aggregate counters only. Nothing here may accept a token, a title, a body, +// or more than the first four characters of a host fingerprint. +export class PushObservability { + private counters = new Map() + private timer: NodeJS.Timeout | null = null + + record(name: PushCounterName, delta = 1): void { + this.counters.set(name, (this.counters.get(name) ?? 0) + delta) + } + + consume(): Record { + const snapshot = Object.fromEntries( + COUNTER_NAMES.map((name) => [name, this.counters.get(name) ?? 0]) + ) as Record + this.counters = new Map() + return snapshot + } + + start(intervalMs = 60_000): void { + if (this.timer) return + this.timer = setInterval(() => { + const counters = this.consume() + if (Object.values(counters).every((value) => value === 0)) return + console.warn(JSON.stringify({ event: 'orca_push_counters', ...counters })) + }, intervalMs) + this.timer.unref() + } + + stop(): void { + if (!this.timer) return + clearInterval(this.timer) + this.timer = null + } +} diff --git a/cloud/apps/push/src/push-provider-outcome.ts b/cloud/apps/push/src/push-provider-outcome.ts new file mode 100644 index 00000000000..bc65d10c175 --- /dev/null +++ b/cloud/apps/push/src/push-provider-outcome.ts @@ -0,0 +1,6 @@ +// What a provider send resolved to, before the send route maps it onto the +// contract's queued / dead / rate_limited / error statuses. +export type PushProviderOutcome = + | { status: 'sent' } + | { status: 'dead'; reason: string } + | { status: 'error'; reason: string; retryable?: boolean; retryAfterMs?: number } diff --git a/cloud/apps/push/src/push-queued-dismissal.ts b/cloud/apps/push/src/push-queued-dismissal.ts new file mode 100644 index 00000000000..3ca5e8c7e66 --- /dev/null +++ b/cloud/apps/push/src/push-queued-dismissal.ts @@ -0,0 +1,57 @@ +import type { PushNotification } from '@orca-cloud/push-contract' +import type { PushDatabase } from './push-database.js' +import { parsePushDeliveryPayload } from './push-delivery-payload.js' + +export async function reconcileQueuedDismissal( + tx: PushDatabase, + host: string, + registrationId: string, + notification: PushNotification, + now: number +): Promise { + if (!notification.notificationId) return false + const key = [host, notification.notificationEpoch, notification.notificationId] + const [dismissed] = await tx.query( + 'SELECT notification_seq FROM push_dismissed_events WHERE host_fingerprint = ? AND notification_epoch = ? AND notification_id = ?', + key + ) + if (notification.kind !== 'dismiss') + return Number(dismissed?.notification_seq ?? -1) >= notification.notificationSeq + await tx.query( + `INSERT INTO push_dismissed_events(host_fingerprint, notification_epoch, notification_id, notification_seq, created_at) + VALUES (?, ?, ?, ?, ?) ON CONFLICT(host_fingerprint, notification_epoch, notification_id) + DO UPDATE SET notification_seq = CASE WHEN push_dismissed_events.notification_seq > excluded.notification_seq THEN push_dismissed_events.notification_seq ELSE excluded.notification_seq END, created_at = excluded.created_at`, + [...key, notification.notificationSeq, now] + ) + const deliveries = await tx.query( + "SELECT batch_id, payload_json FROM push_delivery_batches WHERE host_fingerprint = ? AND registration_id = ? AND kind = 'alert' AND state = 'pending' AND lease_until <= ?", + [host, registrationId, now] + ) + for (const delivery of deliveries) { + const queued = parsePushDeliveryPayload(String(delivery.payload_json)) + if ( + queued.notificationEpoch !== notification.notificationEpoch || + queued.notificationId !== notification.notificationId || + queued.notificationSeq > notification.notificationSeq + ) + continue + await tx.query( + 'UPDATE push_delivery_batches SET payload_json = ?, state = ? WHERE batch_id = ?', + ['{}', 'dismissed', delivery.batch_id] + ) + } + return false +} + +export async function isDismissedAlert( + tx: PushDatabase, + host: string, + notification: PushNotification +): Promise { + if (notification.kind === 'dismiss' || !notification.notificationId) return false + const rows = await tx.query( + 'SELECT notification_seq FROM push_dismissed_events WHERE host_fingerprint = ? AND notification_epoch = ? AND notification_id = ?', + [host, notification.notificationEpoch, notification.notificationId] + ) + return Number(rows[0]?.notification_seq ?? -1) >= notification.notificationSeq +} diff --git a/cloud/apps/push/src/push-readiness.test.ts b/cloud/apps/push/src/push-readiness.test.ts new file mode 100644 index 00000000000..45a57da4330 --- /dev/null +++ b/cloud/apps/push/src/push-readiness.test.ts @@ -0,0 +1,26 @@ +import { expect, it, vi } from 'vitest' +import { createPushReadiness } from './push-readiness.js' +import type { PushDatabase } from './push-database.js' + +it('shares a slow check and caches failures before retrying', async () => { + let clock = 0 + let reject!: (error: Error) => void + const query = vi.fn( + () => + new Promise((_, fail) => { + reject = fail + }) + ) + const ready = createPushReadiness({ query } as unknown as PushDatabase, { now: () => clock }) + const checks = Array.from({ length: 100 }, () => ready()) + expect(query).toHaveBeenCalledTimes(1) + reject(new Error('offline')) + expect(await Promise.all(checks)).toEqual(Array(100).fill(false)) + expect(await ready()).toBe(false) + expect(query).toHaveBeenCalledTimes(1) + clock = 10_000 + const retry = ready() + expect(query).toHaveBeenCalledTimes(2) + reject(new Error('offline')) + expect(await retry).toBe(false) +}) diff --git a/cloud/apps/push/src/push-readiness.ts b/cloud/apps/push/src/push-readiness.ts new file mode 100644 index 00000000000..c071a0750e4 --- /dev/null +++ b/cloud/apps/push/src/push-readiness.ts @@ -0,0 +1,39 @@ +import type { PushDatabase } from './push-database.js' + +export type PushReadinessOptions = { + cacheMs?: number + now?: () => number +} + +// The gateway holds no JWKS dependency, so readiness is exactly "can we reach +// the database": /health stays unconditional for the container probe. +export function createPushReadiness( + database: PushDatabase, + options: PushReadinessOptions = {} +): () => Promise { + const cacheMs = options.cacheMs ?? 10_000 + const now = options.now ?? Date.now + let cachedAt = Number.NEGATIVE_INFINITY + let cached = false + + let pending: Promise | null = null + + async function check(): Promise { + try { + await database.query('SELECT 1 AS ready') + cached = true + } catch { + cached = false + } + cachedAt = now() + return cached + } + + return async () => { + if (now() - cachedAt < cacheMs) return cached + pending ??= check().finally(() => { + pending = null + }) + return pending + } +} diff --git a/cloud/apps/push/src/push-request-drain.ts b/cloud/apps/push/src/push-request-drain.ts new file mode 100644 index 00000000000..4acaf09ca67 --- /dev/null +++ b/cloud/apps/push/src/push-request-drain.ts @@ -0,0 +1,28 @@ +import type { MiddlewareHandler } from 'hono' + +export class PushRequestDrain { + private draining = false + private active = 0 + private readonly waiters = new Set<() => void>() + + readonly middleware: MiddlewareHandler = async (context, next) => { + if (this.draining) return context.json({ error: 'shutting_down' }, 503) + this.active++ + try { + await next() + } finally { + this.active-- + if (this.active === 0) { + for (const resolve of this.waiters) resolve() + this.waiters.clear() + } + } + } + + begin(): Promise { + this.draining = true + return this.active === 0 + ? Promise.resolve() + : new Promise((resolve) => this.waiters.add(resolve)) + } +} diff --git a/cloud/apps/push/src/push-schema.ts b/cloud/apps/push/src/push-schema.ts new file mode 100644 index 00000000000..7375d475703 --- /dev/null +++ b/cloud/apps/push/src/push-schema.ts @@ -0,0 +1,46 @@ +import { DURABLE_PUSH_SCHEMA } from './durable-push-schema.js' +// Applied at startup for both dialects, including additive queue tables, +// so every column type has to read the same in SQLite and PostgreSQL. +const PUSH_SCHEMA = ` +CREATE TABLE IF NOT EXISTS push_challenges ( + challenge_id TEXT PRIMARY KEY, + host_fingerprint TEXT NOT NULL, + secret_hash TEXT NOT NULL, + expires_at BIGINT NOT NULL, + consumed_at BIGINT +); +CREATE INDEX IF NOT EXISTS push_challenges_expires_at ON push_challenges(expires_at); + +CREATE TABLE IF NOT EXISTS push_sessions ( + token_hash TEXT PRIMARY KEY, + host_fingerprint TEXT NOT NULL, + expires_at BIGINT NOT NULL, + created_at BIGINT NOT NULL +); +CREATE UNIQUE INDEX IF NOT EXISTS push_sessions_host ON push_sessions(host_fingerprint); +CREATE INDEX IF NOT EXISTS push_sessions_expires_at ON push_sessions(expires_at); + +CREATE TABLE IF NOT EXISTS push_devices ( + registration_id TEXT PRIMARY KEY, + host_fingerprint TEXT NOT NULL, + device_id TEXT NOT NULL, + platform TEXT NOT NULL, + token TEXT NOT NULL, + apns_environment TEXT, + dead_at BIGINT, + created_at BIGINT NOT NULL, + updated_at BIGINT NOT NULL +); +CREATE UNIQUE INDEX IF NOT EXISTS push_devices_host_device + ON push_devices(host_fingerprint, device_id); +` + +export function pushSchemaStatements(): string[] { + // Comments are stripped before the split so a ';' inside one cannot cut a + // statement in half and hand SQLite an "incomplete input" fragment. + return (PUSH_SCHEMA + DURABLE_PUSH_SCHEMA) + .replace(/--[^\n]*/g, '') + .split(';') + .map((statement) => statement.trim()) + .filter((statement) => statement.length > 0) +} diff --git a/cloud/apps/push/src/push-send-idempotency.test.ts b/cloud/apps/push/src/push-send-idempotency.test.ts new file mode 100644 index 00000000000..1a687a35449 --- /dev/null +++ b/cloud/apps/push/src/push-send-idempotency.test.ts @@ -0,0 +1,64 @@ +import { afterEach, expect, it } from 'vitest' +import { createPushServerHarness, notification } from './push-server-harness.test-fixture.js' +import { createPushHostKeypair } from './host-challenge-answering.test-fixture.js' +const harnesses: Awaited>[] = [] +afterEach(async () => { + await Promise.all(harnesses.splice(0).map((h) => h.close())) +}) + +it('returns queued for concurrent retries without double quota or delivery', async () => { + const h = await createPushServerHarness() + harnesses.push(h) + const token = await h.signIn(createPushHostKeypair(2)) + const registrationId = await h.registerAndroid(token) + const body = { v: 1, registrationIds: [registrationId], notification: notification() } + const responses = await Promise.all( + Array.from({ length: 10 }, () => h.post('/v1/send', body, token)) + ) + for (const response of responses) + expect(await response.json()).toEqual({ results: [{ registrationId, status: 'queued' }] }) + expect(await h.server.deliveryStore.pendingCount(registrationId)).toBe(1) + await h.flushDeliveries() + await h.post('/v1/send', body, token) + await h.flushDeliveries() + expect(h.fcmRequests).toHaveLength(1) + expect(JSON.parse(h.fcmRequests[0]!.body).message.data.coalescedCount).toBeUndefined() + expect( + Number((await h.database.query('SELECT COUNT(*) AS count FROM push_events'))[0]?.count) + ).toBe(1) + await h.post( + '/v1/send', + { ...body, notification: notification({ notificationEpoch: 'new-epoch' }) }, + token + ) + await h.flushDeliveries() + expect(h.fcmRequests).toHaveLength(2) +}) + +it.each([false, true])( + 'accepts default alert kind equivalently through the API (explicit first: %s)', + async (explicitFirst) => { + const h = await createPushServerHarness() + harnesses.push(h) + const token = await h.signIn(createPushHostKeypair(3)) + const registrationId = await h.registerAndroid(token) + const implicit = notification() + const explicit = { kind: 'alert', ...implicit } + for (const event of explicitFirst ? [explicit, implicit] : [implicit, explicit]) { + const response = await h.post( + '/v1/send', + { v: 1, registrationIds: [registrationId], notification: event }, + token + ) + expect(await response.json()).toEqual({ results: [{ registrationId, status: 'queued' }] }) + } + await h.flushDeliveries() + expect(h.fcmRequests).toHaveLength(1) + const changed = await h.post( + '/v1/send', + { v: 1, registrationIds: [registrationId], notification: { ...explicit, body: 'changed' } }, + token + ) + expect(await changed.json()).toEqual({ results: [{ registrationId, status: 'error' }] }) + } +) diff --git a/cloud/apps/push/src/push-server-auth.test.ts b/cloud/apps/push/src/push-server-auth.test.ts new file mode 100644 index 00000000000..a66a0c3bbe6 --- /dev/null +++ b/cloud/apps/push/src/push-server-auth.test.ts @@ -0,0 +1,162 @@ +import { PUSH_LIMITS } from '@orca-cloud/push-contract' +import { afterEach, beforeEach, describe, expect, it } from 'vitest' +import { createPushHostKeypair } from './host-challenge-answering.test-fixture.js' +import type { PushDatabase } from './push-database.js' +import { createPushServer } from './push-server.js' +import { + createPushServerHarness, + testPushConfig +} from './push-server-harness.test-fixture.js' + +describe('push gateway authentication and device routes', () => { + let harness: Awaited> + + beforeEach(async () => { + harness = await createPushServerHarness() + }) + + afterEach(async () => { + await harness.close() + }) + + it('answers health unconditionally and ready from the database', async () => { + expect((await harness.server.app.request('/health')).status).toBe(200) + expect((await harness.server.app.request('/ready')).status).toBe(200) + }) + + it('reports not ready when the database is unreachable', async () => { + const unreachable: PushDatabase = { + dialect: 'sqlite', + query: async () => { + throw new Error('no connection') + }, + transaction: async (operation) => await operation(unreachable), + lockQuotaScope: async () => undefined, + close: async () => undefined + } + const broken = createPushServer(testPushConfig(), unreachable, { + fcmAccessToken: async () => 'token', + fcmTransport: async () => ({ status: 200, body: '{}' }) + }) + expect((await broken.app.request('/health')).status).toBe(200) + expect((await broken.app.request('/ready')).status).toBe(503) + await broken.worker.stop() + }) + + it('completes challenge, session, register, list, delete', async () => { + const sessionToken = await harness.signIn(createPushHostKeypair(11)) + const registrationId = await harness.registerAndroid(sessionToken) + + const list = await harness.authorized('/v1/devices', {}, sessionToken) + expect(await list.json()).toEqual({ + devices: [{ registrationId, deviceId: 'device-1', platform: 'android', dead: false }] + }) + + const deleted = await harness.authorized( + `/v1/devices/${registrationId}`, + { method: 'DELETE' }, + sessionToken + ) + expect(deleted.status).toBe(204) + expect(await harness.server.devices.findById(registrationId)).toBeNull() + }) + + it('refuses a request with no bearer, a bogus bearer, and an expired session', async () => { + const sessionToken = await harness.signIn(createPushHostKeypair(12)) + expect((await harness.server.app.request('/v1/devices')).status).toBe(401) + const bogus = await harness.authorized('/v1/devices', {}, 'nonsense') + expect(bogus.status).toBe(401) + expect(await bogus.json()).toEqual({ error: 'invalid_token' }) + + harness.advanceClock(PUSH_LIMITS.sessionTtlMs + 1) + const expired = await harness.authorized('/v1/devices', {}, sessionToken) + expect(expired.status).toBe(401) + expect(await expired.json()).toEqual({ error: 'session_expired' }) + }) + + it('refuses a replayed proof and an unknown challenge', async () => { + const host = createPushHostKeypair(13) + const challenge = await harness.issueChallenge(host) + const proof = harness.answer(challenge, host) + expect( + ( + await harness.post('/v1/host/session', { + v: 1, + challengeId: challenge.challengeId, + proofB64: proof + }) + ).status + ).toBe(200) + + const replay = await harness.post('/v1/host/session', { + v: 1, + challengeId: challenge.challengeId, + proofB64: proof + }) + expect(replay.status).toBe(401) + expect(await replay.json()).toEqual({ error: 'invalid_proof' }) + + const unknown = await harness.post('/v1/host/session', { + v: 1, + challengeId: 'no-such-challenge', + proofB64: proof + }) + expect(await unknown.json()).toEqual({ error: 'invalid_challenge' }) + }) + + it('never returns the host fingerprint on the challenge itself', async () => { + const challenge = await harness.issueChallenge(createPushHostKeypair(22)) + expect(Object.keys(challenge).sort()).toEqual([ + 'challengeId', + 'ciphertextB64', + 'expiresAt', + 'gatewayEphemeralPublicKeyB64', + 'nonceB64' + ]) + }) + + it('lets only the owning host delete a registration', async () => { + const ownerToken = await harness.signIn(createPushHostKeypair(14)) + const intruderToken = await harness.signIn(createPushHostKeypair(15)) + const registrationId = await harness.registerAndroid(ownerToken) + + const forbidden = await harness.authorized( + `/v1/devices/${registrationId}`, + { method: 'DELETE' }, + intruderToken + ) + expect(forbidden.status).toBe(404) + expect(await forbidden.json()).toEqual({ error: 'not_found' }) + expect(await harness.server.devices.findById(registrationId)).not.toBeNull() + }) + + it('replaces the token on a re-registration and keeps one registration id', async () => { + const sessionToken = await harness.signIn(createPushHostKeypair(23)) + const first = await harness.registerAndroid(sessionToken) + const again = await harness.post( + '/v1/devices', + { + v: 1, + deviceId: 'device-1', + platform: 'android', + token: 'rotated_token:APA91b-newnewnewnewnewnewnewnewnewnew' + }, + sessionToken + ) + expect(await again.json()).toEqual({ registrationId: first }) + expect(await harness.server.devices.findById(first)).toMatchObject({ + token: 'rotated_token:APA91b-newnewnewnewnewnewnewnewnewnew' + }) + }) + + it('rejects a malformed registration body', async () => { + const sessionToken = await harness.signIn(createPushHostKeypair(16)) + const bad = await harness.post( + '/v1/devices', + { v: 1, deviceId: 'device-1', platform: 'ios', token: 'not-hex' }, + sessionToken + ) + expect(bad.status).toBe(400) + expect(await bad.json()).toEqual({ error: 'invalid_request' }) + }) +}) diff --git a/cloud/apps/push/src/push-server-harness.test-fixture.ts b/cloud/apps/push/src/push-server-harness.test-fixture.ts new file mode 100644 index 00000000000..f1567dd9ab7 --- /dev/null +++ b/cloud/apps/push/src/push-server-harness.test-fixture.ts @@ -0,0 +1,162 @@ +import { generateKeyPairSync } from 'node:crypto' +import { expect } from 'vitest' +import type { ApnsRequest, ApnsResponse } from './apns-http2-transport.js' +import type { PushConfig } from './config.js' +import type { FcmRequest, FcmResponse } from './fcm-client.js' +import { + answerPushHostChallenge, + hostPublicKeyB64, + type PushHostKeypair +} from './host-challenge-answering.test-fixture.js' +import { openInMemoryPushDatabase, type PushDatabase } from './push-database.js' +import { createPushServer } from './push-server.js' + +export const GATEWAY_ORIGIN = 'https://push.onorca.dev' +export const APNS_TOKEN = 'a'.repeat(64) +export const FCM_TOKEN = 'cQ1abcDEF_gh:APA91bZZ-zz0123456789abcdefghijklmnopqrstuvwxyz' + +export function notification(overrides: Record = {}): Record { + return { + notificationId: 'note-1', + notificationSeq: 1, + notificationEpoch: 'epoch-1', + source: 'agent-task-complete', + agentState: 'needs-input', + title: 'Agent needs input', + body: 'Waiting on your answer', + worktreeId: 'wt-1', + ...overrides + } +} + +export function testPushConfig(): PushConfig { + const { privateKey } = generateKeyPairSync('ec', { + namedCurve: 'P-256', + privateKeyEncoding: { type: 'pkcs8', format: 'pem' }, + publicKeyEncoding: { type: 'spki', format: 'pem' } + }) + return { + mode: 'active', + port: 0, + publicUrl: GATEWAY_ORIGIN, + dataDir: './data/push-test', + databasePoolMax: 10, + apns: { keyPem: privateKey, keyId: 'ABCDE12345', teamId: 'TEAM123456' }, + apnsTopic: 'com.stably.orca.mobile', + fcmProjectId: 'onorca-cloud', + trustedProxyHops: 0 + } +} + +type ChallengeWire = { + challengeId: string + gatewayEphemeralPublicKeyB64: string + nonceB64: string + ciphertextB64: string + expiresAt: number +} + +export async function createPushServerHarness() { + const database: PushDatabase = await openInMemoryPushDatabase() + let clock = 1_700_000_000_000 + const apnsRequests: ApnsRequest[] = [] + const fcmRequests: FcmRequest[] = [] + let apnsResponse: ApnsResponse = { status: 200, body: '' } + let fcmResponse: FcmResponse = { status: 200, body: '{}' } + const server = createPushServer(testPushConfig(), database, { + now: () => clock, + apnsTransport: async (request) => { + apnsRequests.push(request) + return apnsResponse + }, + fcmTransport: async (request) => { + fcmRequests.push(request) + return fcmResponse + }, + fcmAccessToken: async () => 'access-token' + }) + + const post = async (path: string, body: unknown, token?: string): Promise => + await server.app.request(path, { + method: 'POST', + headers: { + 'content-type': 'application/json', + ...(token ? { authorization: `Bearer ${token}` } : {}) + }, + body: JSON.stringify(body) + }) + + const issueChallenge = async (keypair: PushHostKeypair): Promise => { + const response = await post('/v1/host/challenge', { + v: 1, + hostPublicKeyB64: hostPublicKeyB64(keypair) + }) + expect(response.status).toBe(200) + return (await response.json()) as ChallengeWire + } + + const answer = (challenge: ChallengeWire, keypair: PushHostKeypair): string => { + const proof = answerPushHostChallenge(challenge, { + gatewayOrigin: GATEWAY_ORIGIN, + keypair, + now: () => clock + }) + expect(proof).not.toBeNull() + return proof! + } + + return { + server, + database, + apnsRequests, + fcmRequests, + post, + issueChallenge, + answer, + now: () => clock, + flushDeliveries: async (): Promise => { + await server.worker.runDue() + }, + advanceClock: (deltaMs: number): void => { + clock += deltaMs + }, + setApnsResponse: (response: ApnsResponse): void => { + apnsResponse = response + }, + setFcmResponse: (response: FcmResponse): void => { + fcmResponse = response + }, + authorized: async (path: string, init: RequestInit = {}, token?: string): Promise => + await server.app.request(path, { + ...init, + headers: { + ...(init.headers as Record | undefined), + ...(token ? { authorization: `Bearer ${token}` } : {}) + } + }), + signIn: async (keypair: PushHostKeypair): Promise => { + const challenge = await issueChallenge(keypair) + const response = await post('/v1/host/session', { + v: 1, + challengeId: challenge.challengeId, + proofB64: answer(challenge, keypair) + }) + expect(response.status).toBe(200) + return ((await response.json()) as { sessionToken: string }).sessionToken + }, + registerAndroid: async (token: string, deviceId = 'device-1'): Promise => { + const response = await post( + '/v1/devices', + { v: 1, deviceId, platform: 'android', token: FCM_TOKEN }, + token + ) + expect(response.status).toBe(200) + return ((await response.json()) as { registrationId: string }).registrationId + }, + close: async (): Promise => { + await server.worker.stop() + // A test may close the database itself to provoke a route failure. + await database.close().catch(() => undefined) + } + } +} diff --git a/cloud/apps/push/src/push-server-limits.test.ts b/cloud/apps/push/src/push-server-limits.test.ts new file mode 100644 index 00000000000..d5931d3ed35 --- /dev/null +++ b/cloud/apps/push/src/push-server-limits.test.ts @@ -0,0 +1,273 @@ +import { PUSH_LIMITS } from '@orca-cloud/push-contract' +import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest' +import { createPushHostKeypair, hostPublicKeyB64 } from './host-challenge-answering.test-fixture.js' +import { + createPushServerHarness, + FCM_TOKEN, + notification +} from './push-server-harness.test-fixture.js' + +const CLIENT_IP = '203.0.113.7' +const OTHER_CLIENT_IP = '198.51.100.9' + +function oversizedChallengeBody(): string { + return JSON.stringify({ v: 1, filler: 'x'.repeat(PUSH_LIMITS.maxHttpBodyBytes) }) +} + +function chunkedRequest(path: string, body: string): Request { + const stream = new ReadableStream({ + start(controller) { + controller.enqueue(new TextEncoder().encode(body)) + controller.close() + } + }) + return new Request(`http://push.test${path}`, { + method: 'POST', + headers: { 'content-type': 'application/json' }, + body: stream, + duplex: 'half' + } as RequestInit) +} + +describe('push gateway request limits', () => { + let harness: Awaited> + + beforeEach(async () => { + harness = await createPushServerHarness() + }) + + afterEach(async () => { + await harness.close() + }) + + it('refuses an oversized chunked body that declares no content length', async () => { + const request = chunkedRequest('/v1/host/challenge', oversizedChallengeBody()) + expect(request.headers.get('content-length')).toBeNull() + + const response = await harness.server.app.request(request) + expect(response.status).toBe(413) + expect(await response.json()).toEqual({ error: 'request_too_large' }) + }) + + it('still refuses an oversized body that declares a content length', async () => { + const body = oversizedChallengeBody() + const response = await harness.server.app.request('/v1/host/challenge', { + method: 'POST', + headers: { + 'content-type': 'application/json', + 'content-length': String(Buffer.byteLength(body)) + }, + body + }) + expect(response.status).toBe(413) + expect(await response.json()).toEqual({ error: 'request_too_large' }) + }) + + it('lets a chunked body under the cap through to schema validation', async () => { + const response = await harness.server.app.request( + chunkedRequest( + '/v1/host/challenge', + JSON.stringify({ v: 1, hostPublicKeyB64: hostPublicKeyB64(createPushHostKeypair(60)) }) + ) + ) + expect(response.status).toBe(200) + }) + + it('caps an authenticated oversized send as well', async () => { + const sessionToken = await harness.signIn(createPushHostKeypair(61)) + const response = await harness.server.app.request( + new Request('http://push.test/v1/send', { + method: 'POST', + headers: { + 'content-type': 'application/json', + authorization: `Bearer ${sessionToken}` + }, + body: new ReadableStream({ + start(controller) { + controller.enqueue(new TextEncoder().encode(oversizedChallengeBody())) + controller.close() + } + }), + duplex: 'half' + } as RequestInit) + ) + expect(response.status).toBe(413) + expect(await response.json()).toEqual({ error: 'request_too_large' }) + }) + + it('rate limits one client ip across both unauthenticated routes', async () => { + const body = JSON.stringify({ + v: 1, + hostPublicKeyB64: hostPublicKeyB64(createPushHostKeypair(62)) + }) + // Cloud Run appends the peer, so the caller's own IP is the last value. + const headers = { + 'content-type': 'application/json', + 'x-forwarded-for': `10.0.0.1, ${CLIENT_IP}` + } + for (let index = 0; index < PUSH_LIMITS.unauthenticatedRequestsPerMinutePerIp; index++) { + const allowed = await harness.server.app.request('/v1/host/challenge', { + method: 'POST', + headers, + body + }) + expect(allowed.status).toBe(200) + } + + const limited = await harness.server.app.request('/v1/host/challenge', { + method: 'POST', + headers, + body + }) + expect(limited.status).toBe(429) + expect(await limited.json()).toEqual({ error: 'rate_limited' }) + + // The session route draws on the same bucket, so a flood cannot simply move. + const session = await harness.server.app.request('/v1/host/session', { + method: 'POST', + headers, + body: JSON.stringify({ v: 1, challengeId: 'anything', proofB64: 'x'.repeat(44) }) + }) + expect(session.status).toBe(429) + + const other = await harness.server.app.request('/v1/host/challenge', { + method: 'POST', + headers: { ...headers, 'x-forwarded-for': `10.0.0.1, ${OTHER_CLIENT_IP}` }, + body + }) + expect(other.status).toBe(200) + + // A caller rewriting the left of the chain lands in its own bucket anyway. + const spoofed = await harness.server.app.request('/v1/host/challenge', { + method: 'POST', + headers: { ...headers, 'x-forwarded-for': `198.51.100.250, ${CLIENT_IP}` }, + body + }) + expect(spoofed.status).toBe(429) + }) + + it('lets a throttled client back in once the window refills', async () => { + const body = JSON.stringify({ + v: 1, + hostPublicKeyB64: hostPublicKeyB64(createPushHostKeypair(63)) + }) + const headers = { 'content-type': 'application/json', 'x-forwarded-for': CLIENT_IP } + for (let index = 0; index < PUSH_LIMITS.unauthenticatedRequestsPerMinutePerIp; index++) { + await harness.server.app.request('/v1/host/challenge', { method: 'POST', headers, body }) + } + expect( + (await harness.server.app.request('/v1/host/challenge', { method: 'POST', headers, body })) + .status + ).toBe(429) + + harness.advanceClock(60_000) + expect( + (await harness.server.app.request('/v1/host/challenge', { method: 'POST', headers, body })) + .status + ).toBe(200) + }) + + it('limits authenticated hosts independently behind the same IP', async () => { + const sessionToken = await harness.signIn(createPushHostKeypair(64)) + const headers = { 'x-forwarded-for': CLIENT_IP } + for (let index = 0; index < 600; index++) { + const listed = await harness.authorized('/v1/devices', { headers }, sessionToken) + expect(listed.status).toBe(200) + } + const limited = await harness.authorized('/v1/devices', { headers }, sessionToken) + expect(limited.status).toBe(429) + const otherToken = await harness.signIn(createPushHostKeypair(68)) + expect((await harness.authorized('/v1/devices', { headers }, otherToken)).status).toBe(200) + // The handshake bucket is untouched by any of that. + const challenge = await harness.server.app.request('/v1/host/challenge', { + method: 'POST', + headers: { ...headers, 'content-type': 'application/json' }, + body: JSON.stringify({ v: 1, hostPublicKeyB64: hostPublicKeyB64(createPushHostKeypair(67)) }) + }) + expect(challenge.status).toBe(200) + }) + + it('stops repeated forged bearers after the invalid-auth budget is exhausted', async () => { + const headers = { 'x-forwarded-for': CLIENT_IP } + const [before] = await harness.database.query('SELECT COUNT(*) AS sessions FROM push_sessions') + for (let index = 0; index < PUSH_LIMITS.unauthenticatedRequestsPerMinutePerIp; index++) { + const refused = await harness.authorized('/v1/send', { method: 'POST', headers }, 'forged') + expect(refused.status).toBe(401) + } + const limited = await harness.authorized('/v1/send', { method: 'POST', headers }, 'forged') + expect(limited.status).toBe(429) + expect(await limited.json()).toEqual({ error: 'rate_limited' }) + expect(harness.server.unauthenticatedIps.trackedIpCount()).toBe(0) + const [after] = await harness.database.query('SELECT COUNT(*) AS sessions FROM push_sessions') + expect(Number(after?.sessions)).toBe(Number(before?.sessions)) + }) + + it('answers 409 once a host has registered its device allowance', async () => { + const sessionToken = await harness.signIn(createPushHostKeypair(66)) + for (let index = 0; index < PUSH_LIMITS.maxDevicesPerHost; index++) { + const accepted = await harness.post( + '/v1/devices', + { + v: 1, + deviceId: `device-${index}`, + platform: 'android', + token: FCM_TOKEN + }, + sessionToken + ) + expect(accepted.status).toBe(200) + } + + const refused = await harness.post( + '/v1/devices', + { v: 1, deviceId: 'one-too-many', platform: 'android', token: FCM_TOKEN }, + sessionToken + ) + expect(refused.status).toBe(409) + expect(await refused.json()).toEqual({ error: 'too_many_devices' }) + + const listed = await harness.authorized('/v1/devices', {}, sessionToken) + expect(((await listed.json()) as { devices: unknown[] }).devices).toHaveLength( + PUSH_LIMITS.maxDevicesPerHost + ) + }) + + // Why: a database error carries the failing row in its message. The response + // and the log must both stop at the error's name. + it('answers an unexpected route failure with a bare 500 and logs only the name', async () => { + const sessionToken = await harness.signIn(createPushHostKeypair(66)) + const warn = vi.spyOn(console, 'warn').mockImplementation(() => undefined) + try { + await harness.database.close() + const response = await harness.authorized('/v1/devices', {}, sessionToken) + expect(response.status).toBe(500) + expect(await response.json()).toEqual({ error: 'internal' }) + const logged = warn.mock.calls.map((call) => String(call[0])).join('\n') + expect(logged).toContain('"event":"orca_push_request_failed"') + expect(logged).not.toContain('SELECT') + expect(logged).not.toContain('push_devices') + expect(harness.server.observability.consume().request_error).toBe(1) + } finally { + warn.mockRestore() + } + }) + + it('charges a repeated registration id once and returns one result', async () => { + const sessionToken = await harness.signIn(createPushHostKeypair(65)) + const registrationId = await harness.registerAndroid(sessionToken) + + const response = await harness.post( + '/v1/send', + { + v: 1, + registrationIds: [registrationId, registrationId, registrationId], + notification: notification() + }, + sessionToken + ) + expect(await response.json()).toEqual({ results: [{ registrationId, status: 'queued' }] }) + expect(await harness.server.deliveryStore.pendingCount(registrationId)).toBe(1) + const [row] = await harness.database.query('SELECT COUNT(*) AS sends FROM push_events') + expect(Number(row?.sends)).toBe(1) + }) +}) diff --git a/cloud/apps/push/src/push-server-send.test.ts b/cloud/apps/push/src/push-server-send.test.ts new file mode 100644 index 00000000000..2a93020ed1f --- /dev/null +++ b/cloud/apps/push/src/push-server-send.test.ts @@ -0,0 +1,229 @@ +import { PushNotificationSchema } from '@orca-cloud/push-contract' +import { PUSH_LIMITS } from '@orca-cloud/push-contract' +import { afterEach, beforeEach, describe, expect, it } from 'vitest' +import { createPushHostKeypair } from './host-challenge-answering.test-fixture.js' +import { + APNS_TOKEN, + createPushServerHarness, + FCM_TOKEN, + notification +} from './push-server-harness.test-fixture.js' + +describe('push gateway send route', () => { + let harness: Awaited> + + beforeEach(async () => { + harness = await createPushServerHarness() + }) + + afterEach(async () => { + await harness.close() + }) + + it('rejects a batch over the registration cap', async () => { + const sessionToken = await harness.signIn(createPushHostKeypair(16)) + const oversized = await harness.post( + '/v1/send', + { + v: 1, + registrationIds: Array.from( + { length: PUSH_LIMITS.maxRegistrationIdsPerSend + 1 }, + (_, index) => `reg-${index}` + ), + notification: notification() + }, + sessionToken + ) + expect(oversized.status).toBe(400) + expect(await oversized.json()).toEqual({ error: 'invalid_request' }) + }) + + it('queues a send, delivers it to fcm, and reports a dead token on the next send', async () => { + const sessionToken = await harness.signIn(createPushHostKeypair(17)) + const registrationId = await harness.registerAndroid(sessionToken) + + const queued = await harness.post( + '/v1/send', + { v: 1, registrationIds: [registrationId], notification: notification() }, + sessionToken + ) + expect(await queued.json()).toEqual({ results: [{ registrationId, status: 'queued' }] }) + + harness.setFcmResponse({ + status: 404, + body: JSON.stringify({ error: { status: 'UNREGISTERED', message: 'gone' } }) + }) + await harness.flushDeliveries() + expect(harness.fcmRequests).toHaveLength(1) + expect(JSON.parse(harness.fcmRequests[0]!.body)).toMatchObject({ + message: { token: FCM_TOKEN, notification: { title: 'Agent needs input' } } + }) + + const afterDeath = await harness.post( + '/v1/send', + { v: 1, registrationIds: [registrationId], notification: notification() }, + sessionToken + ) + expect(await afterDeath.json()).toEqual({ results: [{ registrationId, status: 'dead' }] }) + + const listed = await harness.authorized('/v1/devices', {}, sessionToken) + expect(await listed.json()).toEqual({ + devices: [{ registrationId, deviceId: 'device-1', platform: 'android', dead: true }] + }) + }) + + it('leaves a live registration alone when the provider reports a transient failure', async () => { + const sessionToken = await harness.signIn(createPushHostKeypair(24)) + const registrationId = await harness.registerAndroid(sessionToken) + await harness.post( + '/v1/send', + { v: 1, registrationIds: [registrationId], notification: notification() }, + sessionToken + ) + harness.setFcmResponse({ + status: 503, + body: JSON.stringify({ error: { status: 'UNAVAILABLE', message: 'backend busy' } }) + }) + await harness.flushDeliveries() + expect(await harness.server.devices.findById(registrationId)).toMatchObject({ dead: false }) + }) + + it('reports retries from the durable worker after the provider delay', async () => { + const token = await harness.signIn(createPushHostKeypair(26)) + const registrationId = await harness.registerAndroid(token) + await harness.post( + '/v1/send', + { + v: 1, + registrationIds: [registrationId], + notification: notification() + }, + token + ) + harness.setFcmResponse({ status: 503, body: '{}' }) + await harness.flushDeliveries() + expect(harness.server.observability.consume()).toMatchObject({ + delivery_error: 1, + delivery_retry: 0 + }) + harness.advanceClock(10_000) + harness.setFcmResponse({ status: 200, body: '{}' }) + await harness.server.worker.runDue() + expect(harness.server.observability.consume()).toMatchObject({ + delivery_sent: 1, + delivery_retry: 1 + }) + }) + + it('sends a burst as individual APNs alerts grouped by the host thread', async () => { + const sessionToken = await harness.signIn(createPushHostKeypair(18)) + const registration = await harness.post( + '/v1/devices', + { + v: 1, + deviceId: 'iphone-1', + platform: 'ios', + token: APNS_TOKEN, + apnsEnvironment: 'sandbox' + }, + sessionToken + ) + const { registrationId } = (await registration.json()) as { registrationId: string } + for (const seq of [1, 2, 3]) { + await harness.post( + '/v1/send', + { + v: 1, + registrationIds: [registrationId], + notification: notification({ notificationId: `note-${seq}`, notificationSeq: seq }) + }, + sessionToken + ) + } + await harness.flushDeliveries() + expect(harness.apnsRequests).toHaveLength(3) + const bodies = harness.apnsRequests.map( + (request) => + JSON.parse(request.body) as { + aps: { alert: { title: string; body: string }; 'thread-id': string } + orca: Record & { notificationSeq: number } + } + ) + expect( + harness.apnsRequests.every((request) => request.host === 'api.sandbox.push.apple.com') + ).toBe(true) + expect(bodies.map((body) => body.aps.alert)).toEqual( + Array.from({ length: 3 }, () => ({ + title: 'Agent needs input', + body: 'Waiting on your answer' + })) + ) + expect(new Set(bodies.map((body) => body.aps['thread-id'])).size).toBe(1) + expect(bodies.map((body) => body.orca.notificationSeq).sort((a, b) => a - b)).toEqual([1, 2, 3]) + expect(bodies.every((body) => !('coalescedCount' in body.orca))).toBe(true) + expect(bodies.every((body) => !('summaryMembers' in body.orca))).toBe(true) + expect( + new Set(harness.apnsRequests.map((request) => request.headers['apns-collapse-id'])).size + ).toBe(3) + }) + + it('sends a lone event through unchanged with its own collapse id', async () => { + const sessionToken = await harness.signIn(createPushHostKeypair(25)) + const registrationId = await harness.registerAndroid(sessionToken) + await harness.post( + '/v1/send', + { v: 1, registrationIds: [registrationId], notification: notification() }, + sessionToken + ) + await harness.flushDeliveries() + const message = JSON.parse(harness.fcmRequests[0]!.body) as { + message: { android: { notification: { tag: string } }; data: Record } + } + expect(message.message.android.notification.tag).toMatch(/^[a-f0-9]{64}$/) + expect(message.message.data.coalescedCount).toBeUndefined() + }) + + it('reports an error for a registration the host does not own', async () => { + const ownerToken = await harness.signIn(createPushHostKeypair(19)) + const intruderToken = await harness.signIn(createPushHostKeypair(20)) + const registrationId = await harness.registerAndroid(ownerToken) + + const foreign = await harness.post( + '/v1/send', + { v: 1, registrationIds: [registrationId, 'made-up'], notification: notification() }, + intruderToken + ) + expect(await foreign.json()).toEqual({ + results: [ + { registrationId, status: 'error' }, + { registrationId: 'made-up', status: 'error' } + ] + }) + expect(await harness.server.deliveryStore.pendingCount(registrationId)).toBe(0) + }) + + it('rate limits a host that exhausted its 15-minute allowance', async () => { + const sessionToken = await harness.signIn(createPushHostKeypair(21)) + const registrationId = await harness.registerAndroid(sessionToken) + const hostFingerprint = (await harness.server.devices.findById(registrationId))!.hostFingerprint + for (let index = 0; index < PUSH_LIMITS.hostEventsPerWindow; index++) { + expect( + await harness.server.deliveryStore.accept( + hostFingerprint, + registrationId, + PushNotificationSchema.parse( + notification({ notificationId: `note-${index + 1000}`, notificationSeq: index + 1000 }) + ) + ) + ).toBe('queued') + } + const limited = await harness.post( + '/v1/send', + { v: 1, registrationIds: [registrationId], notification: notification() }, + sessionToken + ) + expect(limited.status).toBe(200) + expect(await limited.json()).toEqual({ results: [{ registrationId, status: 'rate_limited' }] }) + expect(await harness.server.deliveryStore.pendingCount(registrationId)).toBe(300) + }) +}) diff --git a/cloud/apps/push/src/push-server.ts b/cloud/apps/push/src/push-server.ts new file mode 100644 index 00000000000..911f7890155 --- /dev/null +++ b/cloud/apps/push/src/push-server.ts @@ -0,0 +1,298 @@ +import { PushAuthAdmission } from './push-auth-admission.js' +import { createAdaptorServer } from '@hono/node-server' +import { + PUSH_LIMITS, + PushDeviceRegistrationRequestSchema, + PushHostChallengeRequestSchema, + PushHostSessionRequestSchema, + PushSendRequestSchema, + type PushSendResult +} from '@orca-cloud/push-contract' +import { Hono, type MiddlewareHandler } from 'hono' +import { bodyLimit } from 'hono/body-limit' +import { ApnsClient } from './apns-client.js' +import { createApnsHttp2Transport, type ApnsTransport } from './apns-http2-transport.js' +import { clientIpRateLimit, ClientIpRateLimiter, readClientIp } from './client-ip-rate-limit.js' +import { DurablePushStore } from './durable-push-store.js' +import { DurablePushWorker } from './durable-push-worker.js' +import type { PushConfig } from './config.js' +import { PushDeviceRegistryStore } from './device-registry-store.js' +import { createFcmAccessTokenProvider } from './fcm-access-token.js' +import { createFcmFetchTransport, FcmClient, type FcmTransport } from './fcm-client.js' +import { PushHostChallengeStore } from './host-challenge-store.js' +import { PushHostSessionStore } from './host-session-store.js' +import type { PushDatabase } from './push-database.js' +import { PushDispatcher } from './push-dispatcher.js' +import { PushObservability } from './push-observability.js' +import { createPushReadiness } from './push-readiness.js' +import { PushRequestDrain } from './push-request-drain.js' + +export type PushServerOptions = { + now?: () => number + apnsTransport?: ApnsTransport + fcmTransport?: FcmTransport + fcmAccessToken?: () => Promise +} + +type PushVariables = { hostFingerprint: string } + +export function readBearer(header: string | undefined): string | null { + if (!header) return null + const [scheme, ...rest] = header.split(' ') + const token = rest.join(' ').trim() + return scheme?.toLowerCase() === 'bearer' && token.length > 0 ? token : null +} + +// Hono's body limit, not a Content-Length check: a chunked body declares no +// length, and req.json() would buffer all of it before any handler ran. +const limitBody = bodyLimit({ + maxSize: PUSH_LIMITS.maxHttpBodyBytes, + onError: (context) => context.json({ error: 'request_too_large' }, 413) +}) + +export function createPushServer( + config: PushConfig, + database: PushDatabase, + options: PushServerOptions = {} +) { + const now = options.now ?? Date.now + const observability = new PushObservability() + const challenges = new PushHostChallengeStore(database, config.publicUrl, now) + const sessions = new PushHostSessionStore(database, now) + const devices = new PushDeviceRegistryStore(database, now) + const deliveryStore = new DurablePushStore(database, now) + const apnsTransport = options.apnsTransport ?? (config.apns ? createApnsHttp2Transport() : null) + const dispatcher = new PushDispatcher({ + devices, + ...(config.apns && apnsTransport + ? { + apns: new ApnsClient({ + topic: config.apnsTopic, + credentials: config.apns, + transport: apnsTransport, + now + }) + } + : {}), + fcm: new FcmClient({ + now, + projectId: config.fcmProjectId, + accessToken: options.fcmAccessToken ?? createFcmAccessTokenProvider(), + transport: options.fcmTransport ?? createFcmFetchTransport() + }), + onOutcome: (status) => + observability.record( + status === 'sent' ? 'delivery_sent' : status === 'dead' ? 'delivery_dead' : 'delivery_error' + ) + }) + const worker = new DurablePushWorker(deliveryStore, dispatcher, { + now, + onRetry: () => observability.record('delivery_retry') + }) + const ready = createPushReadiness(database, { now }) + const unauthenticatedIps = new ClientIpRateLimiter({ now }) + const limitUnauthenticatedIp = clientIpRateLimit(unauthenticatedIps, { + trustedProxyHops: config.trustedProxyHops, + onLimited: () => observability.record('ip_rate_limited') + }) + const limitAuthenticatedIp = clientIpRateLimit( + new ClientIpRateLimiter({ + now, + capacity: PUSH_LIMITS.authenticatedRequestsPerMinutePerIp + }), + { + trustedProxyHops: config.trustedProxyHops, + onLimited: () => observability.record('ip_rate_limited') + } + ) + const authAdmission = new PushAuthAdmission() + const invalidBearerIps = new ClientIpRateLimiter({ now }) + const authenticatedHosts = new ClientIpRateLimiter({ + now, + capacity: PUSH_LIMITS.authenticatedRequestsPerMinutePerHost + }) + const app = new Hono<{ Variables: PushVariables }>() + const requestDrain = new PushRequestDrain() + app.use('*', requestDrain.middleware) + // Hono's default handler prints the whole error, and a pg error carries the + // offending row in `detail`. Only the error's name may reach the logs. + app.onError((error, context) => { + observability.record('request_error') + console.warn( + JSON.stringify({ + event: 'orca_push_request_failed', + error: error instanceof Error ? error.name : 'unknown' + }) + ) + return context.json({ error: 'internal' }, 500) + }) + + app.get('/health', (context) => + context.json({ ok: true, pushProtocol: 1, deliveryProtocol: 2, mode: config.mode }) + ) + app.get('/ready', limitUnauthenticatedIp, async (context) => + (await ready()) + ? context.json({ ok: true }) + : context.json({ error: 'dependency_unavailable' }, 503) + ) + + if (config.mode === 'validation') { + app.use('*', async (context) => context.json({ error: 'validation_only' }, 503)) + } + + const bearerSession: MiddlewareHandler<{ Variables: PushVariables }> = async (context, next) => { + const ip = readClientIp(context, config.trustedProxyHops) + if (!invalidBearerIps.available(ip)) return context.json({ error: 'rate_limited' }, 429) + const bearer = readBearer(context.req.header('authorization')) + if (!bearer) { + invalidBearerIps.allow(ip) + return context.json({ error: 'invalid_token' }, 401) + } + const session = await authAdmission.run(async () => { + if (!invalidBearerIps.available(ip)) return null + const result = await sessions.resolve(bearer) + if (!result.ok) invalidBearerIps.allow(ip) + return result + }) + if (!session) { + context.header('Retry-After', '1') + return context.json({ error: 'busy' }, 503) + } + if (!session.ok) { + return context.json( + { error: session.reason === 'session_expired' ? 'session_expired' : 'invalid_token' }, + 401 + ) + } + if (!authenticatedHosts.allow(session.hostFingerprint)) + return context.json({ error: 'rate_limited' }, 429) + context.set('hostFingerprint', session.hostFingerprint) + await next() + return + } + // `/v1/devices/*` matches `/v1/devices` itself; a second registration for the + // bare path would run both middlewares twice on it. + app.use('/v1/devices/*', limitAuthenticatedIp, bearerSession) + app.use('/v1/send', limitAuthenticatedIp, bearerSession) + + app.post('/v1/host/challenge', limitUnauthenticatedIp, limitBody, async (context) => { + const body = PushHostChallengeRequestSchema.safeParse( + await context.req.json().catch(() => null) + ) + if (!body.success) return context.json({ error: 'invalid_request' }, 400) + const issued = await challenges.issue(body.data.hostPublicKeyB64) + if (!issued) { + observability.record('challenge_rejected') + return context.json({ error: 'invalid_request' }, 400) + } + observability.record('challenge_issued') + const { hostFingerprint: _bound, ...response } = issued + return context.json(response) + }) + + app.post('/v1/host/session', limitUnauthenticatedIp, limitBody, async (context) => { + const body = PushHostSessionRequestSchema.safeParse(await context.req.json().catch(() => null)) + if (!body.success) return context.json({ error: 'invalid_request' }, 400) + const verification = await challenges.verify(body.data.challengeId, body.data.proofB64) + if (!verification.ok) { + observability.record('session_rejected') + return context.json( + { + error: verification.reason === 'unknown_challenge' ? 'invalid_challenge' : 'invalid_proof' + }, + 401 + ) + } + observability.record('session_issued') + return context.json(await sessions.create(verification.hostFingerprint)) + }) + + app.post('/v1/devices', limitBody, async (context) => { + const body = PushDeviceRegistrationRequestSchema.safeParse( + await context.req.json().catch(() => null) + ) + if (!body.success) return context.json({ error: 'invalid_request' }, 400) + const registered = await devices.upsert({ + hostFingerprint: context.get('hostFingerprint'), + deviceId: body.data.deviceId, + platform: body.data.platform, + token: body.data.token, + ...(body.data.apnsEnvironment === undefined + ? {} + : { apnsEnvironment: body.data.apnsEnvironment }) + }) + if (!registered.ok) { + observability.record('device_rejected') + return context.json({ error: 'too_many_devices' }, 409) + } + observability.record('device_registered') + return context.json({ registrationId: registered.registrationId }) + }) + + app.delete('/v1/devices/:registrationId', async (context) => { + const deleted = await devices.deleteOwned( + context.get('hostFingerprint'), + context.req.param('registrationId') + ) + if (!deleted) return context.json({ error: 'not_found' }, 404) + observability.record('device_deleted') + return context.body(null, 204) + }) + + app.get('/v1/devices', async (context) => + context.json({ devices: await devices.list(context.get('hostFingerprint')) }) + ) + + app.post('/v1/send', limitBody, async (context) => { + const body = PushSendRequestSchema.safeParse(await context.req.json().catch(() => null)) + if (!body.success) return context.json({ error: 'invalid_request' }, 400) + const hostFingerprint = context.get('hostFingerprint') + const owned = await devices.findOwned(hostFingerprint, body.data.registrationIds) + const results: PushSendResult[] = [] + for (const registrationId of body.data.registrationIds) { + const device = owned.get(registrationId) + if (!device) { + observability.record('send_error') + results.push({ registrationId, status: 'error' }) + continue + } + if (device.dead) { + observability.record('send_dead') + results.push({ registrationId, status: 'dead' }) + continue + } + const reservation = await deliveryStore.accept( + hostFingerprint, + registrationId, + body.data.notification + ) + if (reservation !== 'queued') { + observability.record(reservation === 'rate_limited' ? 'send_rate_limited' : 'send_error') + results.push({ registrationId, status: reservation }) + continue + } + observability.record('send_queued') + results.push({ registrationId, status: 'queued' }) + } + return context.json({ results }) + }) + + return { + app, + requestDrain, + server: createAdaptorServer(app), + challenges, + sessions, + devices, + deliveryStore, + unauthenticatedIps, + worker, + observability, + ready, + closeTransports: (): void => { + if (apnsTransport && 'close' in apnsTransport) { + ;(apnsTransport as { close: () => void }).close() + } + } + } +} diff --git a/cloud/apps/push/src/push-session-concurrency.test.ts b/cloud/apps/push/src/push-session-concurrency.test.ts new file mode 100644 index 00000000000..4c6944255f8 --- /dev/null +++ b/cloud/apps/push/src/push-session-concurrency.test.ts @@ -0,0 +1,65 @@ +import { randomUUID } from 'node:crypto' +import { tmpdir } from 'node:os' +import { afterEach, describe, expect, it } from 'vitest' +import { openInMemoryPushDatabase, openPushDatabase, type PushDatabase } from './push-database.js' +import { PushHostSessionStore } from './host-session-store.js' +const databases: PushDatabase[] = [] +afterEach(async () => { + await Promise.all(databases.splice(0).map((db) => db.close())) +}) + +async function concurrentSessions(db: PushDatabase) { + databases.push(db) + const host = randomUUID() + const store = new PushHostSessionStore(db) + try { + const sessions = await Promise.all(Array.from({ length: 20 }, () => store.create(host))) + const decisions = await Promise.all( + sessions.map((session) => store.resolve(session.sessionToken)) + ) + expect(decisions.filter((decision) => decision.ok)).toHaveLength(1) + const [row] = await db.query( + 'SELECT COUNT(*) AS count FROM push_sessions WHERE host_fingerprint = ?', + [host] + ) + expect(Number(row?.count)).toBe(1) + } finally { + await db.query('DELETE FROM push_sessions WHERE host_fingerprint = ?', [host]) + } +} +it('serializes sessions on SQLite', async () => { + await concurrentSessions(await openInMemoryPushDatabase()) +}) + +it('enforces one session per host directly in the schema', async () => { + const db = await openInMemoryPushDatabase() + databases.push(db) + await db.query('INSERT INTO push_sessions VALUES (?, ?, ?, ?)', ['first', 'host', 100, 1]) + await expect( + db.query('INSERT INTO push_sessions VALUES (?, ?, ?, ?)', ['second', 'host', 100, 2]) + ).rejects.toThrow() + expect(await db.query('SELECT token_hash FROM push_sessions')).toEqual([{ token_hash: 'first' }]) +}) + +describe.skipIf(!process.env.ORCA_PUSH_TEST_DATABASE_URL)('PostgreSQL push sessions', () => { + it('leaves exactly one live token after concurrent creates', async () => { + await concurrentSessions( + await openPushDatabase({ + databaseUrl: process.env.ORCA_PUSH_TEST_DATABASE_URL!, + dataDir: tmpdir() + }) + ) + }) + it('allows concurrent schema startup', async () => { + const opened = await Promise.all( + Array.from({ length: 4 }, () => + openPushDatabase({ + databaseUrl: process.env.ORCA_PUSH_TEST_DATABASE_URL!, + dataDir: tmpdir() + }) + ) + ) + databases.push(...opened) + for (const db of opened) expect(await db.query('SELECT 1 AS ok')).toEqual([{ ok: 1 }]) + }) +}) diff --git a/cloud/apps/push/src/push-validation.test.ts b/cloud/apps/push/src/push-validation.test.ts new file mode 100644 index 00000000000..afb6906e423 --- /dev/null +++ b/cloud/apps/push/src/push-validation.test.ts @@ -0,0 +1,90 @@ +import { expect, it, vi } from 'vitest' +import { loadPushConfig } from './config.js' +import { openPushDatabase } from './push-database.js' +import { createPushServer } from './push-server.js' +import { startPushBackground } from './push-background.js' + +it('fails closed on an invalid validation mode', () => { + for (const mode of ['typo', '', ' ']) { + expect(() => + loadPushConfig({ + ORCA_PUSH_FCM_PROJECT_ID: 'onorca-cloud', + ORCA_PUSH_PUBLIC_URL: 'https://push.onorca.dev', + ORCA_PUSH_MODE: mode + }) + ).toThrow() + } +}) + +const databaseUrl = process.env.ORCA_PUSH_TEST_DATABASE_URL +it.skipIf(!databaseUrl)( + 'validation cannot write PostgreSQL and starts no consumers or pruners', + async () => { + if (!process.env.CI && new URL(databaseUrl!).port !== '55440') + throw new Error('isolated_postgres_port_required') + const active = await openPushDatabase({ databaseUrl, dataDir: '' }) + const schema = `validation_${Date.now()}` + await active.query(`CREATE SCHEMA ${schema}`) + const isolatedUrl = new URL(databaseUrl!) + isolatedUrl.searchParams.set( + 'options', + `-c search_path=${schema} -c default_transaction_read_only=off` + ) + isolatedUrl.searchParams.set('host', isolatedUrl.hostname) + isolatedUrl.searchParams.set('port', isolatedUrl.port) + const hostlessUrl = `postgresql://${isolatedUrl.username}:${isolatedUrl.password}@${isolatedUrl.pathname}?${isolatedUrl.searchParams}` + const database = await openPushDatabase({ + databaseUrl: hostlessUrl, + dataDir: '', + readOnly: true + }) + const config = loadPushConfig({ + ORCA_PUSH_FCM_PROJECT_ID: 'onorca-cloud', + ORCA_PUSH_PUBLIC_URL: 'https://push.onorca.dev', + ORCA_PUSH_MODE: 'validation' + }) + const runtime = createPushServer(config, database) + let stop: (() => Promise) | undefined + try { + const [setting] = await database.query( + "SELECT current_setting('default_transaction_read_only') AS default_transaction_read_only" + ) + expect(setting!.default_transaction_read_only).toBe('on') + await expect( + database.query(`CREATE TABLE ${schema}.forbidden (id integer)`) + ).rejects.toMatchObject({ code: '25006' }) + // An empty schema stays empty: validation must not run startup DDL. + expect( + await active.query('SELECT tablename FROM pg_tables WHERE schemaname = ?', [schema]) + ).toEqual([]) + const calls = vi.spyOn(database, 'query') + const claim = vi.spyOn(runtime.deliveryStore, 'claim') + const send = vi.spyOn(runtime.worker, 'start') + vi.useFakeTimers() + stop = startPushBackground(config, runtime) + await vi.advanceTimersByTimeAsync(31 * 60_000) + expect(calls).not.toHaveBeenCalled() + expect(claim).not.toHaveBeenCalled() + expect(send).not.toHaveBeenCalled() + vi.useRealTimers() + expect(await (await runtime.app.request('/health')).json()).toMatchObject({ + mode: 'validation' + }) + expect((await runtime.app.request('/ready')).status).toBe(200) + expect((await runtime.app.request('/v1/host/challenge', { method: 'POST' })).status).toBe(503) + expect((await runtime.app.request('/v1/send', { method: 'POST' })).status).toBe(503) + expect(calls.mock.calls.map(([sql]) => sql)).toEqual(['SELECT 1 AS ready']) + await expect( + database.query('DELETE FROM public.push_challenges WHERE false') + ).rejects.toMatchObject({ code: '25006' }) + } finally { + vi.useRealTimers() + await stop?.() + runtime.closeTransports() + await database.close() + await active.query(`DROP SCHEMA ${schema} CASCADE`) + await active.close() + vi.restoreAllMocks() + } + } +) diff --git a/cloud/apps/push/tsconfig.build.json b/cloud/apps/push/tsconfig.build.json new file mode 100644 index 00000000000..5e71eb0f951 --- /dev/null +++ b/cloud/apps/push/tsconfig.build.json @@ -0,0 +1,10 @@ +{ + "extends": "./tsconfig.json", + "compilerOptions": { + "declaration": true, + "noEmit": false, + "outDir": "dist", + "rootDir": "src" + }, + "exclude": ["src/**/*.test.ts", "src/**/*.test-fixture.ts"] +} diff --git a/cloud/apps/push/tsconfig.json b/cloud/apps/push/tsconfig.json new file mode 100644 index 00000000000..a552e34dbe9 --- /dev/null +++ b/cloud/apps/push/tsconfig.json @@ -0,0 +1,5 @@ +{ + "extends": "../../tsconfig.base.json", + "compilerOptions": { "noEmit": true }, + "include": ["src/**/*.ts"] +} diff --git a/cloud/apps/push/vitest.config.ts b/cloud/apps/push/vitest.config.ts new file mode 100644 index 00000000000..bffcc30e39e --- /dev/null +++ b/cloud/apps/push/vitest.config.ts @@ -0,0 +1,5 @@ +import { defineConfig } from 'vitest/config' + +export default defineConfig({ + test: { name: 'push', include: ['src/**/*.test.ts'], testTimeout: 15_000, hookTimeout: 15_000 } +}) diff --git a/cloud/apps/relay/Dockerfile b/cloud/apps/relay/Dockerfile index 12516cbf749..f0abcf9f5b3 100644 --- a/cloud/apps/relay/Dockerfile +++ b/cloud/apps/relay/Dockerfile @@ -3,11 +3,13 @@ WORKDIR /app RUN corepack enable COPY package.json pnpm-lock.yaml pnpm-workspace.yaml tsconfig.base.json ./ COPY packages/relay-contract/package.json packages/relay-contract/package.json +COPY packages/postgres-schema/package.json packages/postgres-schema/package.json COPY apps/relay/package.json apps/relay/package.json RUN pnpm install --frozen-lockfile COPY packages/relay-contract packages/relay-contract COPY apps/relay apps/relay -RUN pnpm --filter @orca-cloud/relay-contract build && pnpm --filter @orca-cloud/relay build +COPY packages/postgres-schema packages/postgres-schema +RUN pnpm --filter @orca-cloud/postgres-schema build && pnpm --filter @orca-cloud/relay-contract build && pnpm --filter @orca-cloud/relay build FROM node:24-alpine AS runtime ENV NODE_ENV=production @@ -16,8 +18,10 @@ WORKDIR /app RUN corepack enable COPY package.json pnpm-lock.yaml pnpm-workspace.yaml ./ COPY packages/relay-contract/package.json packages/relay-contract/package.json +COPY packages/postgres-schema/package.json packages/postgres-schema/package.json COPY apps/relay/package.json apps/relay/package.json COPY --from=build /app/packages/relay-contract/dist packages/relay-contract/dist +COPY --from=build /app/packages/postgres-schema/dist packages/postgres-schema/dist COPY --from=build /app/apps/relay/dist apps/relay/dist RUN pnpm install --prod --frozen-lockfile --filter @orca-cloud/relay... USER node diff --git a/cloud/apps/relay/package.json b/cloud/apps/relay/package.json index de30d66e413..824513ea446 100644 --- a/cloud/apps/relay/package.json +++ b/cloud/apps/relay/package.json @@ -9,13 +9,14 @@ "clean": "node -e \"require('fs').rmSync('dist', { recursive: true, force: true })\"", "dev": "tsx watch src/index.ts", "lint": "tsc -p tsconfig.json --noEmit", - "pretest": "pnpm --filter @orca-cloud/relay-contract build", + "pretest": "pnpm --filter @orca-cloud/postgres-schema build && pnpm --filter @orca-cloud/relay-contract build", "start": "node dist/index.js", "test": "vitest run", "typecheck": "tsc -p tsconfig.json --noEmit" }, "dependencies": { "@hono/node-server": "^1.19.17", + "@orca-cloud/postgres-schema": "workspace:*", "@orca-cloud/relay-contract": "workspace:*", "hono": "^4.13.7", "jose": "^6.1.3", diff --git a/cloud/apps/relay/src/postgres-schema-startup.ts b/cloud/apps/relay/src/postgres-schema-startup.ts index 22a75cd9465..3a3428eda32 100644 --- a/cloud/apps/relay/src/postgres-schema-startup.ts +++ b/cloud/apps/relay/src/postgres-schema-startup.ts @@ -1,120 +1 @@ -const RETRYABLE_SCHEMA_CODES = new Set(['55P03', '57014']) -const DEFAULT_RETRY_DEADLINE_MS = 30_000 -const RETRY_BASE_DELAY_MS = 250 -const RETRY_MAX_DELAY_MS = 2_000 - -type SchemaStartupOptions = { - now?: () => number - random?: () => number - retryDeadlineMs?: number - wait?: (delayMs: number) => Promise -} - -function retryDelayMs(attempt: number, random: () => number): number { - const ceiling = Math.min( - RETRY_BASE_DELAY_MS * 2 ** (attempt - 1), - RETRY_MAX_DELAY_MS - ) - return Math.ceil(ceiling * (0.5 + random() * 0.5)) -} - -function wait(delayMs: number): Promise { - return new Promise((resolve) => setTimeout(resolve, delayMs)) -} - -const CREATE_TABLE_IF_NOT_EXISTS = /^\s*CREATE\s+TABLE\s+IF\s+NOT\s+EXISTS\b/i -const CREATE_INDEX_IF_NOT_EXISTS = /^\s*CREATE\s+(?:UNIQUE\s+)?INDEX\s+IF\s+NOT\s+EXISTS\b/i - -// `IF NOT EXISTS` only checks the name before the catalog inserts, so the loser of a concurrent -// CREATE can fail on the catalog unique index (23505) or, when the winner has already committed by -// the time the loser reaches TypeCreate/heap_create_with_catalog, on the name check those routines -// repeat (42710 duplicate type, 42P07 duplicate relation). Each is a no-op on the next attempt. -function concurrentCreateCollision( - value: { code?: unknown; constraint?: unknown }, - statement: string -): boolean { - if (CREATE_TABLE_IF_NOT_EXISTS.test(statement)) { - return ( - (value.code === '23505' && value.constraint === 'pg_type_typname_nsp_index') || - value.code === '42710' || - value.code === '42P07' - ) - } - if (CREATE_INDEX_IF_NOT_EXISTS.test(statement)) { - return ( - (value.code === '23505' && value.constraint === 'pg_class_relname_nsp_index') || - value.code === '42P07' - ) - } - return false -} - -const ALTER_TABLE_ADD_CONSTRAINT = - /^\s*ALTER\s+TABLE\s+\S+\s+ADD\s+CONSTRAINT\b/i - -// Postgres has no `ADD CONSTRAINT IF NOT EXISTS`, so a re-run and a concurrent -// startup both land on 42710 once the constraint exists. Unlike a CREATE race -// this is terminal, not transient: retrying only repeats it, so the statement -// counts as applied. -function constraintAlreadyApplied(error: unknown, statement: string): boolean { - return ( - ALTER_TABLE_ADD_CONSTRAINT.test(statement) && - (error as { code?: unknown }).code === '42710' - ) -} - -function retryableSchemaError(error: unknown, statement: string): boolean { - const value = error as { code?: unknown; constraint?: unknown } - return ( - RETRYABLE_SCHEMA_CODES.has(String(value.code)) || concurrentCreateCollision(value, statement) - ) -} - -export async function applyPostgresSchema( - statements: string[], - query: (statement: string) => Promise, - options: SchemaStartupOptions = {} -): Promise { - const now = options.now ?? Date.now - const random = options.random ?? Math.random - const pause = options.wait ?? wait - const deadlineAt = now() + (options.retryDeadlineMs ?? DEFAULT_RETRY_DEADLINE_MS) - - for (const statement of statements) { - let attempt = 1 - while (true) { - try { - await query(statement) - break - } catch (error) { - if (constraintAlreadyApplied(error, statement)) break - const code = String((error as { code?: unknown }).code) - const remainingMs = deadlineAt - now() - const retryable = retryableSchemaError(error, statement) - if (!retryable || remainingMs <= 0) { - if (retryable) { - console.warn( - JSON.stringify({ - event: 'orca_relay_postgres_schema_retry_exhausted', - code, - attempts: attempt - }) - ) - } - throw error - } - const delayMs = Math.min(remainingMs, retryDelayMs(attempt, random)) - console.warn( - JSON.stringify({ - event: 'orca_relay_postgres_schema_retry', - code, - attempt, - delayMs - }) - ) - await pause(delayMs) - attempt += 1 - } - } - } -} +export { applyPostgresSchema } from '@orca-cloud/postgres-schema' diff --git a/cloud/dev/fixtures/terraform-root-partition/families.json b/cloud/dev/fixtures/terraform-root-partition/families.json index dd6f6944322..da91ef44a19 100644 --- a/cloud/dev/fixtures/terraform-root-partition/families.json +++ b/cloud/dev/fixtures/terraform-root-partition/families.json @@ -86,17 +86,21 @@ "relay": [ "google_artifact_registry_repository_iam_member.github_production_relay_staging_mirror_writer", "google_artifact_registry_repository_iam_member.github_production_relay_writer", + "google_artifact_registry_repository_iam_member.github_push_artifact_writer", "google_artifact_registry_repository_iam_member.github_relay_asia_topology_artifact_reader", "google_artifact_registry_repository_iam_member.github_staging_relay_deploy_artifact_reader", "google_certificate_manager_certificate.relay_gce", "google_certificate_manager_certificate_map.relay_gce", "google_certificate_manager_certificate_map_entry.relay_gce", "google_certificate_manager_dns_authorization.relay_gce", + "google_cloud_run_domain_mapping.push", "google_cloud_run_domain_mapping.relay", "google_cloud_run_domain_mapping.relay_cell", + "google_cloud_run_v2_service.push", "google_cloud_run_v2_service.relay", "google_cloud_run_v2_service.relay_cell", "google_cloud_run_v2_service.relay_fence_broker", + "google_cloud_run_v2_service_iam_member.github_production_push_developer", "google_cloud_run_v2_service_iam_member.github_production_relay_director_developer", "google_cloud_run_v2_service_iam_member.github_production_relay_fence_broker_developer", "google_cloud_run_v2_service_iam_member.github_staging_relay_capacity_developer", @@ -125,6 +129,7 @@ "google_iam_workload_identity_pool_provider.github_fence", "google_iam_workload_identity_pool_provider.github_monitor", "google_iam_workload_identity_pool_provider.github_production_relay_capacity", + "google_iam_workload_identity_pool_provider.github_push", "google_iam_workload_identity_pool_provider.github_relay_asia_proof", "google_iam_workload_identity_pool_provider.github_relay_asia_topology", "google_iam_workload_identity_pool_provider.github_staging_relay_capacity", @@ -172,6 +177,9 @@ "google_project_iam_member.github_staging_relay_capacity_viewer", "google_project_iam_member.github_staging_relay_deploy_compute_viewer", "google_project_iam_member.github_staging_relay_power", + "google_project_iam_member.push_runtime_cloudsql_client", + "google_project_iam_member.push_runtime_fcm_admin", + "google_project_iam_member.push_runtime_service_usage_consumer", "google_project_iam_member.relay_director_runtime_cloudsql_client", "google_project_iam_member.relay_fence_broker_artifact_reader", "google_project_iam_member.relay_fence_broker_compute_viewer", @@ -180,9 +188,13 @@ "google_project_iam_member.relay_runtime_artifact_reader", "google_project_iam_member.relay_runtime_cloudsql_client", "google_project_iam_member.relay_runtime_log_writer", + "google_secret_manager_secret.push_dedicated_database_url", + "google_secret_manager_secret.push_provider", "google_secret_manager_secret.relay_assignment_signing_key", "google_secret_manager_secret.relay_database_url", "google_secret_manager_secret.relay_regional_placement_enabled", + "google_secret_manager_secret_iam_member.push_dedicated_database_url_accessor", + "google_secret_manager_secret_iam_member.push_provider_runtime_accessor", "google_secret_manager_secret_iam_member.relay_assignment_signing_key_accessor", "google_secret_manager_secret_iam_member.relay_assignment_signing_key_director_accessor", "google_secret_manager_secret_iam_member.relay_database_url_accessor", @@ -192,24 +204,30 @@ "google_secret_manager_secret_iam_member.relay_regional_placement_deploy_viewer", "google_secret_manager_secret_iam_member.relay_regional_placement_director_accessor", "google_secret_manager_secret_iam_member.relay_regional_placement_runtime_accessor", + "google_secret_manager_secret_version.push_dedicated_database_url", "google_secret_manager_secret_version.relay_assignment_signing_key", "google_secret_manager_secret_version.relay_database_url", "google_secret_manager_secret_version.relay_regional_placement_enabled", "google_service_account.github_fence", "google_service_account.github_monitor", "google_service_account.github_production_relay_capacity", + "google_service_account.github_push_deploy", "google_service_account.github_relay_asia_proof", "google_service_account.github_relay_asia_topology", "google_service_account.github_staging_relay_capacity", "google_service_account.github_staging_relay_deploy", + "google_service_account.push_runtime", "google_service_account.relay_director_runtime", "google_service_account.relay_fence_broker", "google_service_account.relay_runtime", "google_service_account_iam_member.github_accepted_repository_workload_identity_user", "google_service_account_iam_member.github_fence_workload_identity_user", "google_service_account_iam_member.github_monitor_workload_identity_user", + "google_service_account_iam_member.github_production_push_runtime_token_creator", + "google_service_account_iam_member.github_production_push_runtime_user", "google_service_account_iam_member.github_production_relay_capacity_runtime_user", "google_service_account_iam_member.github_production_relay_capacity_workload_identity_user", + "google_service_account_iam_member.github_push_workload_identity_user", "google_service_account_iam_member.github_relay_asia_proof_workload_identity_user", "google_service_account_iam_member.github_relay_asia_topology_runtime_user", "google_service_account_iam_member.github_relay_asia_topology_workload_identity_user", @@ -221,9 +239,13 @@ "google_service_account_iam_member.github_staging_relay_deploy_auth_runtime_user", "google_service_account_iam_member.github_staging_relay_deploy_workload_identity_user", "google_service_account_iam_member.relay_fence_broker_requester_token_creator", + "google_sql_database.push_dedicated", "google_sql_database.relay", + "google_sql_database_instance.push_dedicated", + "google_sql_user.push_dedicated", "google_sql_user.relay", "google_storage_bucket_iam_member.github_production_relay_capacity_state", + "google_storage_bucket_iam_member.github_push_rollout_lease", "google_storage_bucket_iam_member.github_relay_asia_topology_state", "google_storage_bucket_iam_member.github_relay_asia_topology_state_list", "google_storage_bucket_iam_member.github_staging_relay_capacity_state", @@ -231,6 +253,7 @@ "google_storage_bucket_iam_member.github_staging_relay_deploy_state_list", "google_storage_bucket_iam_member.relay_fence_broker_bucket_reader", "google_storage_bucket_iam_member.relay_fence_broker_state_objects", + "random_password.push_dedicated_database", "random_password.relay_assignment_signing_key", "random_password.relay_database" ], diff --git a/cloud/dev/scripts/push-cloud-run-model.mjs b/cloud/dev/scripts/push-cloud-run-model.mjs new file mode 100644 index 00000000000..fea782e625d --- /dev/null +++ b/cloud/dev/scripts/push-cloud-run-model.mjs @@ -0,0 +1,140 @@ +import { readFileSync, writeFileSync } from 'node:fs' + +// Executable fake gcloud: revision deletion obeys the platform's latest/traffic constraints. +const path = process.env.MODEL_STATE +const state = JSON.parse(readFileSync(path, 'utf8')) +const args = process.argv.slice(2) +const option = (name) => args[args.indexOf(name) + 1] +const has = (name) => args.includes(name) +const fail = (message) => { + throw new Error(message) +} +const persist = () => writeFileSync(path, JSON.stringify(state)) +const output = (value) => console.log(typeof value === 'string' ? value : JSON.stringify(value)) +const revision = (name) => state.revisions[name] ?? fail(`missing revision ${name}`) +const traffic = () => [ + { revisionName: state.serving, percent: 100 }, + ...Object.entries(state.tags).map(([tag, name]) => ({ + tag, + revisionName: name, + url: `https://${tag}.test` + })) +] +state.trace.push(args.join(' ')) +try { + if (args[0] === 'curl') { + if (state.failure === 'public' && args.some((arg) => arg.includes('https://public.test'))) { + fail('public check failed') + } + const url = args.find((arg) => arg.startsWith('https://')) + const tag = new URL(url).hostname.split('.')[0] + const name = state.tags[tag] ?? state.serving + if (has('-w')) { + output('200') + } else { + output({ + ok: true, + deliveryProtocol: 2, + mode: revision(name).spec.containers[0].env.some((entry) => entry.value === 'validation') + ? 'validation' + : 'active' + }) + } + } else if (args.slice(0, 2).join(' ') === 'run deploy') { + const name = `${option('deploy')}-${option('--revision-suffix')}` + if (state.failure === 'deploy-before') { + fail('deploy failed before create') + } + const item = structuredClone(revision(state.latest)) + item.metadata.name = name + item.spec.containers[0].image = option('--image') + item.status.imageDigest = option('--image') + item.spec.containers[0].env = item.spec.containers[0].env.filter( + (entry) => entry.name !== 'ORCA_PUSH_MODE' + ) + if (has('--update-env-vars')) { + item.spec.containers[0].env.push({ name: 'ORCA_PUSH_MODE', value: 'validation' }) + } + state.revisions[name] = item + state.latest = name + if (has('--tag')) { + state.tags[option('--tag')] = name + } + state.peak = Math.max(state.peak, Object.keys(state.revisions).length) + if (state.peak > 3) { + fail('three-revision budget exceeded') + } + if (state.failure === 'deploy-after') { + fail('deploy failed after create') + } + } else if (args.slice(0, 3).join(' ') === 'run services describe') { + if (state.failure === 'describe') { + fail('describe failed') + } + if (args.some((arg) => arg.includes('value(status.latestCreatedRevisionName)'))) { + output(state.latest) + } else { + const template = structuredClone(revision(state.latest)) + delete template.spec.containers[0].name + output({ + spec: { template }, + status: { latestCreatedRevisionName: state.latest, traffic: traffic() } + }) + } + } else if (args.slice(0, 3).join(' ') === 'run services update-traffic') { + if (has('--to-revisions')) { + const name = option('--to-revisions').split('=')[0] + revision(name) + state.serving = name + } + if (has('--remove-tags')) { + for (const tag of option('--remove-tags').split(',')) { + delete state.tags[tag] + } + } + if (has('--clear-tags')) { + state.tags = {} + } + if (state.failure === 'traffic-after') { + fail('traffic changed but response failed') + } + } else if (args.slice(0, 3).join(' ') === 'run revisions list') { + const names = Object.keys(state.revisions) + output( + (has('--filter') + ? names.filter((name) => name === option('--filter').split('=')[1]) + : names + ).join('\n') + ) + } else if (args.slice(0, 3).join(' ') === 'run revisions describe') { + const item = revision(args[3]) + const format = args.find((arg) => arg.startsWith('--format=')) ?? option('--format') + if (format.includes('minScale')) { + output(item.metadata.annotations['autoscaling.knative.dev/minScale']) + } else if (format.includes('maxScale')) { + output(item.metadata.annotations['autoscaling.knative.dev/maxScale']) + } else { + output(item) + } + } else if (args.slice(0, 3).join(' ') === 'run revisions delete') { + const name = args[3] + if (name === state.latest) { + fail('FAILED_PRECONDITION: latest created Revision cannot be directly deleted') + } + if (name === state.serving || Object.values(state.tags).includes(name)) { + fail('revision has traffic or tags') + } + if (state.failure === 'delete') { + fail('delete failed') + } + revision(name) + delete state.revisions[name] + } else { + fail(`unmodeled gcloud call ${args.join(' ')}`) + } +} catch (error) { + console.error(error.message) + process.exitCode = 1 +} finally { + persist() +} diff --git a/cloud/dev/scripts/push-gateway-recovery.test.mjs b/cloud/dev/scripts/push-gateway-recovery.test.mjs new file mode 100644 index 00000000000..97d9c4b5539 --- /dev/null +++ b/cloud/dev/scripts/push-gateway-recovery.test.mjs @@ -0,0 +1,373 @@ +import assert from 'node:assert/strict' +import { mkdirSync, mkdtempSync, readFileSync, rmSync, writeFileSync } from 'node:fs' +import { tmpdir } from 'node:os' +import { join } from 'node:path' +import { fileURLToPath } from 'node:url' +import { spawnSync } from 'node:child_process' +import test from 'node:test' +import { readRelayWorkflow } from './relay-repository.mjs' + +const workflow = readRelayWorkflow('push-deploy.yml') +function step(name) { + const start = workflow.indexOf(` - name: ${name}\n`) + assert.notEqual(start, -1) + const end = workflow.indexOf('\n - ', start + 1) + const block = workflow.slice(start, end === -1 ? undefined : end) + return block + .slice(block.indexOf(' run: |\n') + ' run: |\n'.length) + .split('\n') + .filter((line) => line.startsWith(' ')) + .map((line) => line.slice(10)) + .join('\n') +} +const names = { + preflight: 'Record the serving revision and require its Terraform-owned scaling', + candidate: 'Deploy the candidate revision with no traffic', + activate: 'Retire inert validation and activate the verified image', + shift: 'Shift all traffic to the verified candidate', + public: 'Verify the public origin after the shift', + rollback: 'Roll traffic back to the previous revision', + restore: 'Restore the known-good service template', + promoteRecovery: 'Promote and verify the known-good recovery revision', + cleanup: 'Delete the rejected candidate revision', + retire: 'Retire previous consumers after public checks' +} +const image = `registry/push@sha256:${'a'.repeat(64)}` +const spec = { + serviceAccountName: 'runtime@test', + containerConcurrency: 40, + containers: [ + { + image, + name: 'push-test-1', + env: [ + { + name: 'ORCA_PUSH_DATABASE_URL', + valueFrom: { secretKeyRef: { name: 'database', key: '7' } } + }, + { name: 'ORCA_PUSH_DATABASE_POOL_MAX', value: '2' } + ] + } + ] +} +const prior = { + metadata: { + name: 'push-test-old', + annotations: { + 'autoscaling.knative.dev/minScale': '1', + 'autoscaling.knative.dev/maxScale': '2' + } + }, + spec, + status: { imageDigest: image } +} +const model = fileURLToPath(new URL('./push-cloud-run-model.mjs', import.meta.url)) +const options = { skip: process.platform === 'win32' } +function exercise(callback) { + const dir = mkdtempSync(join(tmpdir(), 'push-workflow-')) + const statePath = join(dir, 'state.json') + writeFileSync( + statePath, + JSON.stringify({ + revisions: { 'push-test-old': prior }, + latest: 'push-test-old', + serving: 'push-test-old', + tags: {}, + peak: 1, + trace: [] + }) + ) + writeFileSync(join(dir, 'env'), '') + const env = { + ...process.env, + SERVICE_NAME: 'push-test', + GCP_PROJECT_ID: 'test', + GCP_REGION: 'test', + GITHUB_RUN_ID: '123', + GITHUB_RUN_ATTEMPT: '1', + IMAGE: `registry/push@sha256:${'b'.repeat(64)}`, + PUSH_MIN_INSTANCES: '1', + PUSH_MAX_INSTANCES: '2', + PUSH_RUNTIME_SERVICE_ACCOUNT: 'runtime@test', + PUSH_ORIGIN: 'https://public.test', + MODEL_STATE: statePath, + MODEL_SCRIPT: model, + RUNNER_TEMP: dir, + GITHUB_ENV: join(dir, 'env'), + GITHUB_STEP_SUMMARY: join(dir, 'summary') + } + const state = () => JSON.parse(readFileSync(statePath, 'utf8')) + const change = (edit) => { + const value = state() + edit(value) + writeFileSync(statePath, JSON.stringify(value)) + } + const run = (key, ok = true, extra = '') => { + const result = spawnSync( + 'bash', + [ + '-c', + ` + set -a + source "$GITHUB_ENV" + gcloud() { node "$MODEL_SCRIPT" "$@"; } + curl() { node "$MODEL_SCRIPT" curl "$@"; } + sleep() { :; } + ${extra} + ${names[key] ? step(names[key]) : key} + ` + ], + { cwd: dir, env, encoding: 'utf8', timeout: 30000 } + ) + assert.equal(result.status === 0, ok, `${key}: ${result.stderr}\n${result.stdout}`) + return result + } + try { + callback({ run, state, change, dir }) + } finally { + rmSync(dir, { recursive: true, force: true }) + } +} +function recover(h) { + h.change((state) => { + delete state.failure + }) + h.run('restore') + h.run('promoteRecovery') + h.run('cleanup') + h.run('retire') + const state = h.state() + assert.equal(state.serving, 'push-test-r123-1') + assert.equal(state.latest, state.serving) + assert.deepEqual(Object.keys(state.revisions), [state.serving]) + assert.equal(state.revisions[state.serving].spec.containers[0].image, image) + assert.ok(state.peak <= 3) +} + +test( + 'the executable Cloud Run model rejects deleting latest even without tags or traffic', + options, + () => + exercise((h) => { + h.run('preflight') + h.run('candidate') + h.run('gcloud run services update-traffic "$SERVICE_NAME" --clear-tags') + const result = h.run('gcloud run revisions delete "$CANDIDATE_REVISION"', false) + assert.match(result.stderr, /FAILED_PRECONDITION: latest created Revision/) + }) +) + +test( + 'success creates successor before retirement and repeated rollouts retain one consumer', + options, + () => + exercise((h) => { + for (const attempt of ['1', '2']) { + if (attempt === '2') { + writeFileSync(join(h.dir, 'env'), 'GITHUB_RUN_ATTEMPT=2\n') + } + for (const key of ['preflight', 'candidate', 'activate', 'shift', 'public', 'retire']) { + h.run(key) + } + const state = h.state() + assert.equal(state.serving, `push-test-a123-${attempt}`) + assert.deepEqual(Object.keys(state.revisions), [state.serving]) + assert.equal(state.peak, 3) + } + }) +) + +for (const failure of ['deploy-before', 'deploy-after', 'describe']) { + test(`validation ${failure} recovers without deleting latest`, options, () => + exercise((h) => { + h.run('preflight') + h.change((state) => { + state.failure = failure + }) + h.run('candidate', false) + recover(h) + }) + ) +} +for (const failure of ['deploy-before', 'deploy-after', 'delete', 'describe']) { + test( + `activation ${failure} frees validation slot before recovery and stays within three`, + options, + () => + exercise((h) => { + h.run('preflight') + h.run('candidate') + h.change((state) => { + state.failure = failure + }) + h.run('activate', false) + recover(h) + }) + ) +} + +test( + 'ambiguous traffic shift records intent before mutation, rolls back and recovers', + options, + () => + exercise((h) => { + h.run('preflight') + h.run('candidate') + h.run('activate') + h.change((state) => { + state.failure = 'traffic-after' + }) + h.run('shift', false) + assert.match(readFileSync(join(h.dir, 'env'), 'utf8'), /TRAFFIC_SHIFT_ATTEMPTED=true/) + h.change((state) => { + delete state.failure + }) + h.run('rollback') + recover(h) + }) +) + +test('failed public check rolls back and recovers', options, () => + exercise((h) => { + for (const key of ['preflight', 'candidate', 'activate', 'shift']) { + h.run(key) + } + h.change((state) => { + state.failure = 'public' + }) + h.run('public', false) + h.change((state) => { + delete state.failure + }) + h.run('rollback') + recover(h) + }) +) + +for (const defect of [ + 'runtime', + 'secret', + 'mode', + 'image', + 'traffic', + 'scaling', + 'deploy-before', + 'deploy-after' +]) { + test(`recovery rejects ${defect} and preserves partial-create state`, options, () => + exercise((h) => { + h.run('preflight') + h.run('candidate') + h.change((state) => { + const revision = state.revisions[state.latest] + if (defect === 'runtime') { + revision.spec.serviceAccountName = 'wrong@test' + } + if (defect === 'secret') { + revision.spec.containers[0].env[0].valueFrom.secretKeyRef.key = '8' + } + if (defect === 'scaling') { + revision.metadata.annotations['autoscaling.knative.dev/maxScale'] = '3' + } + if (defect === 'traffic') { + state.serving = state.latest + } + if (defect.startsWith('deploy-')) { + state.failure = defect + } + }) + // Corrupt the recovery response after the modeled deploy while keeping real jq assertions. + const extra = ['mode', 'image'].includes(defect) + ? ` + gcloud() { + node "$MODEL_SCRIPT" "$@" > "$RUNNER_TEMP/out" || return $? + if [[ "$*" == 'run services describe '* && "$*" == *'--format=json'* ]]; then + jq '${defect === 'mode' ? '.spec.template.spec.containers[0].env += [{name:"ORCA_PUSH_MODE",value:"validation"}]' : '.spec.template.spec.containers[0].image = "wrong"'}' "$RUNNER_TEMP/out" + else cat "$RUNNER_TEMP/out"; fi + }` + : '' + h.run('restore', false, extra) + const recorded = readFileSync(join(h.dir, 'env'), 'utf8') + assert.match(recorded, /TEMPLATE_RECOVERY_REVISION=push-test-r123-1/) + assert.doesNotMatch(recorded, /TEMPLATE_RESTORED=true/) + assert.ok(h.state().peak <= 3) + }) + ) +} + +test('failed validation retirement blocks a fourth revision during recovery', options, () => + exercise((h) => { + h.run('preflight') + h.run('candidate') + h.change((state) => { + state.failure = 'delete' + }) + h.run('activate', false) + h.run('restore', false) + assert.equal(h.state().peak, 3) + assert.equal(h.state().revisions['push-test-r123-1'], undefined) + }) +) + +test( + 'failed retirement after public checks leaves verified serving and blocks the next run', + options, + () => + exercise((h) => { + for (const key of ['preflight', 'candidate', 'activate', 'shift', 'public']) { + h.run(key) + } + h.change((state) => { + state.failure = 'delete' + }) + h.run('retire', false) + assert.equal(h.state().serving, 'push-test-a123-1') + h.run('preflight', false) + assert.match(workflow, /env.ROLLOUT_VERIFIED != 'true'/) + }) +) + +test( + 'recovery promotion failure keeps consumers for operator diagnosis and blocks new rollout', + options, + () => + exercise((h) => { + h.run('preflight') + h.run('candidate') + h.run('restore') + h.change((state) => { + state.failure = 'public' + }) + h.run('promoteRecovery', false) + assert.doesNotMatch(readFileSync(join(h.dir, 'env'), 'utf8'), /RECOVERY_VERIFIED=true/) + h.run('preflight', false) + }) +) + +const capability = step('Require image support for inert validation') +for (const [label, source, ok] of [ + ['old image', 'export function loadPushConfig() { return {}; }', false], + [ + 'invalid mode accepted', + 'export function loadPushConfig(env) { return { mode: env.ORCA_PUSH_MODE }; }', + false + ], + [ + 'validation supported', + `export function loadPushConfig(env) { + if (env.ORCA_PUSH_MODE !== 'validation') throw new Error('invalid mode'); + return { mode: 'validation' }; + }`, + true + ] +]) { + test(`pre-production image smoke: ${label}`, options, () => + exercise((h) => { + const dist = join(h.dir, 'apps', 'push', 'dist') + mkdirSync(dist, { recursive: true }) + writeFileSync(join(h.dir, 'package.json'), '{"type":"module"}') + writeFileSync(join(dist, 'config.js'), source) + h.run(capability, ok, 'docker() { node "${@: -3}"; }') + }) + ) +} diff --git a/cloud/dev/scripts/push-gateway-workflow.test.mjs b/cloud/dev/scripts/push-gateway-workflow.test.mjs new file mode 100644 index 00000000000..3ee4fabe410 --- /dev/null +++ b/cloud/dev/scripts/push-gateway-workflow.test.mjs @@ -0,0 +1,334 @@ +import assert from 'node:assert/strict' +import { readFileSync } from 'node:fs' +import test from 'node:test' +import { + concurrencyBlocks, + jobIf, + jobs, + LEASE_ACTION, + leaseSteps +} from './cloud-sql-rollout-lock-census.mjs' +import { readRelayWorkflow, relayWorkflowFile } from './relay-repository.mjs' + +// Why: the push gateway holds the APNs key and is the only thing standing between a paired +// phone and a silent notification pipeline. Its deploy is a blue/green rollout against the +// dedicated Cloud SQL instance, and each of the guarantees below is one careless edit from gone. +const WORKFLOW = 'push-deploy.yml' +const workflow = readRelayWorkflow(WORKFLOW) +const deploy = () => { + const job = jobs(workflow).find((entry) => entry.id === 'deploy') + assert.ok(job, 'the workflow no longer declares a deploy job') + return job +} + +function terraform(file) { + return readFileSync(new URL(`../../infra/terraform/${file}`, import.meta.url), 'utf8') +} + +// The ordered step names; every assertion below reads positions out of this list rather than +// restating them, so a reordering that breaks the no-traffic guarantee fails here. +const stepNames = () => [...workflow.matchAll(/^ {6}- name: (.+)$/gm)].map((match) => match[1]) + +const indexOfStep = (name) => { + const index = stepNames().indexOf(name) + assert.notEqual(index, -1, `the workflow no longer has a "${name}" step`) + return index +} + +test('the whole surface stays inert until the owner enables cloud operations', () => { + const guard = jobIf(deploy().text) + assert.ok(guard.includes("vars.ORCA_CLOUD_OPERATIONS_ENABLED == 'true'"), guard) + assert.ok(guard.includes("github.ref == 'refs/heads/main'"), guard) + assert.equal(jobs(workflow).length, 1, 'a second job would need its own gate') +}) + +test('it authenticates through Workload Identity and holds no repository secret', () => { + assert.match(workflow, /uses: google-github-actions\/auth@v2/) + assert.match(workflow, /workload_identity_provider: \$\{\{ vars\.PRODUCTION_GCP_PUSH_DEPLOY_WORKLOAD_IDENTITY_PROVIDER \}\}/) + assert.match(workflow, /service_account: \$\{\{ vars\.PRODUCTION_GCP_PUSH_DEPLOY_SERVICE_ACCOUNT \}\}/) + assert.match(workflow, /environment: production/) + for (const [, name] of workflow.matchAll(/secrets\.([A-Za-z_][A-Za-z0-9_]*)/g)) { + assert.equal(name, 'GITHUB_TOKEN', `the workflow reads secrets.${name}`) + } +}) + +// Why: Terraform trusts exact workflow filenames, not a prefix. A rename here without the +// matching tfvars-independent list entry would fail authentication at dispatch time only. +test('Terraform trusts this exact workflow file on the production deploy provider', () => { + assert.match(terraform('push-deploy-identity.tf'), /push-deploy\.yml@refs\/heads\/main/) + assert.doesNotMatch(terraform('relay-github-actions.tf'), /push-deploy\.yml/) + assert.equal(relayWorkflowFile(WORKFLOW), 'cloud-push-deploy.yml') +}) + +test('the rollout is serialized and leases its dedicated push rollout lock', () => { + const blocks = concurrencyBlocks(workflow) + assert.equal(blocks.length, 1) + assert.equal(blocks[0].group, 'production-push-rollout') + assert.equal(blocks[0].cancelInProgress, 'false') + const steps = leaseSteps(workflow) + assert.equal(steps.length, 1, 'exactly one lease step, held for the whole run') + assert.equal(steps[0].bucket, 'onorca-cloud-terraform-state') + assert.equal(steps[0].object, 'terraform/state/push-rollout/production.lock') + assert.equal(steps[0].release, undefined, 'release stays at its default for a single-job run') +}) + +// Why: the ops guardrail is that a piped command only fails the step when pipefail is set, and +// pipefail only applies under an explicit bash shell. Every multi-line body here opts in. +test('every multi-line command runs under bash with pipefail', () => { + const bodies = [...workflow.matchAll(/^ {8}(shell: bash\n {8})?run: \|\n((?: {10}.*\n|\n)+)/gm)] + assert.ok(bodies.length >= 8, `only ${bodies.length} multi-line commands were found`) + for (const match of bodies) { + assert.ok(match[1], `a multi-line command does not declare shell: bash:\n${match[2].slice(0, 120)}`) + assert.match(match[2], /^ {10}set -euo pipefail$/m) + } +}) + +test('the candidate revision takes no traffic and is addressed by its own tag', () => { + assert.match(workflow, /gcloud run deploy "\$\{SERVICE_NAME\}"/) + assert.match(workflow, /^ {12}--no-traffic \\$/m) + assert.match(workflow, /--tag "\$\{tag\}"/) + assert.match(workflow, /test "\$\{CANDIDATE_REVISION\}" != "\$\{ROLLBACK_REVISION\}"/) + assert.ok( + indexOfStep('Record the serving revision and require its Terraform-owned scaling') < + indexOfStep('Deploy the candidate revision with no traffic'), + 'the rollback target must be captured before the candidate exists' + ) +}) + +// Why: scaling is a Terraform-owned field that `lifecycle.ignore_changes` does not cover, so a +// deploy that passed --max-instances would revert a later push_max_instances raise on every run. +// The workflow asserts the shape instead of writing it, on the serving revision before the +// candidate exists and on the candidate that inherits it. +test('the deploy asserts the Terraform-owned scaling instead of mutating it', () => { + assert.doesNotMatch(workflow, /--max-instances/, 'the deploy must not write a scaling field') + assert.doesNotMatch(workflow, /--min-instances "/, 'the deploy must not write a scaling field') + // The floor is the variables.tf default; production.tfvars overrides only the ceiling, down to + // the two instances the Cloud SQL connection budget leaves room for. + assert.match(workflow, /PUSH_MIN_INSTANCES: 1$/m) + assert.match(workflow, /PUSH_MAX_INSTANCES: 2$/m) + assert.match(terraform('variables.tf'), /variable "push_min_instances"[\s\S]*?default {5}= 1/) + assert.match(terraform('environments/production.tfvars'), /^push_max_instances {9}= 2$/m) + const gate = indexOfStep('Record the serving revision and require its Terraform-owned scaling') + assert.ok(gate < indexOfStep('Deploy the candidate revision with no traffic')) + assert.match(workflow, /autoscaling\.knative\.dev\/minScale/) + assert.match(workflow, /\[\[ "\$\{floor:-0\}" -lt "\$\{PUSH_MIN_INSTANCES\}" \]\]/) + assert.match(workflow, /test "\$\{ceiling\}" = "\$\{PUSH_MAX_INSTANCES\}"/) + assert.match(workflow, /test "\$\{candidate_ceiling\}" = "\$\{PUSH_MAX_INSTANCES\}"/) +}) + +// Why: the image build is not a Cloud SQL operation, and the lease is a global serialization +// point. A build inside it blocks every relay deploy and rehome for its duration. +test('the image is built before the rollout lease is taken', () => { + const lease = workflow.indexOf(`- uses: ${LEASE_ACTION}`) + assert.notEqual(lease, -1) + const build = workflow.indexOf('- name: Build and publish the immutable gateway image') + const deployCandidate = workflow.indexOf('- name: Deploy the candidate revision with no traffic') + assert.ok(build < lease, 'the build must finish before the run takes the lease') + assert.ok(lease < deployCandidate, 'the lease must still cover the deploy, probe, and shift') +}) + +// Why: the gateway's Cloud SQL draw is instances x pool, and the root that takes the rollout +// lease can only account for a pool it declares. Leaving it at the application default hid it. +test('the database pool size is Terraform-owned and bounded at plan time', () => { + const source = terraform('push-gateway.tf') + assert.match(source, /name {2}= "ORCA_PUSH_DATABASE_POOL_MAX"/) + assert.match(source, /value = tostring\(var\.push_database_pool_max\)/) + assert.match(terraform('variables.tf'), /variable "push_database_pool_max"[\s\S]*?default {5}= 2/) + const block = /resource "google_cloud_run_v2_service" "push"[\s\S]*?\n lifecycle \{([\s\S]*?)\n \}/.exec(source) + assert.ok(block, 'the push service no longer declares a lifecycle block') + assert.match( + block[1], + /var\.push_max_instances \* var\.push_database_pool_max \* 3 <= 64/, + 'instances x pool must be bounded at plan time' + ) + assert.match( + readFileSync(new URL('../../apps/push/src/config.ts', import.meta.url), 'utf8'), + /ORCA_PUSH_DATABASE_POOL_MAX/, + 'the gateway must read the variable Terraform sets' + ) +}) + +test('the candidate is probed on its own URL before any traffic moves', () => { + const probe = indexOfStep('Probe the candidate readiness endpoint') + assert.ok(probe > indexOfStep('Deploy the candidate revision with no traffic')) + assert.ok(probe < indexOfStep('Shift all traffic to the verified candidate')) + assert.match(workflow, /"\$\{CANDIDATE_URL\}\/ready"/) + assert.match(workflow, /test "\$\{code\}" = 200/) + assert.ok(workflow.indexOf('${CANDIDATE_URL}/ready') < workflow.indexOf('${CANDIDATE_URL}/health')) + assert.match(workflow, /\.deliveryProtocol == 2/, 'verify the durable gateway after readiness') +}) + +// Why: a gateway that answers /ready can still hold no usable FCM credential. The probe must be +// validate-only, must use a token that cannot exist, and must treat a denied credential as the +// failure. Accepting PERMISSION_DENIED would make the whole step decorative. +test('the FCM probe is validate-only and separates a bad token from a bad credential', () => { + const fcm = indexOfStep('Prove the runtime identity can reach FCM') + assert.ok(fcm > indexOfStep('Probe the candidate readiness endpoint')) + assert.ok(fcm < indexOfStep('Shift all traffic to the verified candidate')) + assert.match(workflow, /"validate_only":true/) + assert.match(workflow, /https:\/\/fcm\.googleapis\.com\/v1\/projects\/\$\{GCP_PROJECT_ID\}\/messages:send/) + assert.match(workflow, /GCP_PROJECT_ID: onorca-cloud$/m) + assert.match(workflow, /orca-push-deploy-probe-invalid-token/) + assert.match(workflow, /test "\$\{status\}" = INVALID_ARGUMENT/) + assert.match(workflow, /test "\$\{status\}" = PERMISSION_DENIED/) + // Only those four answers are conclusive; a 429 or a 5xx says nothing about the credential, so + // it is retried rather than read as either verdict. A denied credential still fails at once. + assert.match(workflow, /for attempt in \$\(seq 1 5\); do/) + const probe = workflow.slice( + workflow.indexOf('- name: Prove the runtime identity can reach FCM'), + workflow.indexOf('- name: Shift all traffic to the verified candidate') + ) + assert.match(probe, /for attempt in \$\(seq 1 5\); do/) + assert.match(probe, /test "\$\{code\}" = 401 \|\| test "\$\{code\}" = 403; then\n {14}break/) + assert.match( + workflow, + /--impersonate-service-account "\$\{PUSH_RUNTIME_SERVICE_ACCOUNT\}"/, + 'the probe must exercise the runtime credential, not the deploy identity' + ) + // Why: that token reads the Apple signing key. Masking it means a later `set -x` or a + // debug re-run cannot print it into a public log. + assert.match( + probe, + /test -n "\$\{token\}"\n {10}echo "::add-mask::\$\{token\}"/, + 'the impersonated token must be masked before anything else runs' + ) + assert.match(workflow, /PUSH_RUNTIME_SERVICE_ACCOUNT: orca-cloud-push@onorca-cloud\.iam\.gserviceaccount\.com/) +}) + +// Why: a deploy ends with traffic pinned to an exact revision, and a rollback pins it to the +// previous one. Terraform reverting the service to 100% LATEST would undo either silently. +test('Terraform does not own the image or the traffic split', () => { + const source = terraform('push-gateway.tf') + const block = /resource "google_cloud_run_v2_service" "push"[\s\S]*?\n lifecycle \{([\s\S]*?)\n \}/.exec(source) + assert.ok(block, 'the push service no longer declares a lifecycle block') + assert.match(block[1], /template\[0\]\.containers\[0\]\.image/) + assert.match(block[1], /^\s*traffic$/m) +}) + +test('impersonating the runtime identity is a Terraform-declared grant', () => { + const source = terraform('push-gateway.tf') + assert.match(source, /resource "google_service_account_iam_member" "github_production_push_runtime_token_creator"/) + assert.match(source, /role\s+= "roles\/iam\.serviceAccountTokenCreator"/) + assert.match(source, /resource "google_cloud_run_v2_service_iam_member" "github_production_push_developer"/) +}) + +test('the traffic shift is all-or-nothing and is verified after the fact', () => { + const shift = indexOfStep('Shift all traffic to the verified candidate') + assert.match(workflow, /gcloud run services update-traffic "\$\{SERVICE_NAME\}"/) + assert.match(workflow, /--to-revisions "\$\{CANDIDATE_REVISION\}=100"/) + assert.match(workflow, /test "\$\{serving\}" = "\$\{CANDIDATE_REVISION\}"/) + assert.ok(shift < indexOfStep('Verify the public origin after the shift')) + assert.match(workflow, /PUSH_ORIGIN: https:\/\/push\.onorca\.dev/) + assert.match(workflow, /"\$\{PUSH_ORIGIN\}\/ready"/) +}) + +// Why: the origin can lag the traffic move by seconds, and a single unlucky curl would otherwise +// roll a healthy deploy back. It retries on the same schedule as the candidate probe. +test('the post-shift origin check retries like the candidate probe', () => { + const check = workflow.slice( + workflow.indexOf('- name: Verify the public origin after the shift'), + workflow.indexOf('- name: Roll traffic back to the previous revision') + ) + assert.match(check, /for attempt in \$\(seq 1 30\); do/) + assert.match(check, /sleep 5/) + assert.match(check, /test "\$\{code\}" = 200/) +}) + +// Why: the summary carries the rollback target. Writing it after the origin check meant the one +// run that needed it, the run whose check failed, was the one run that never got it. +test('the summary is written before anything that can fail after the shift', () => { + const summary = indexOfStep('Publish the rollout summary') + assert.ok(summary > indexOfStep('Shift all traffic to the verified candidate')) + assert.ok(summary < indexOfStep('Verify the public origin after the shift')) + assert.match(workflow, /Known-good image:/) + assert.match(workflow, /GITHUB_STEP_SUMMARY/) +}) + +// Why: everything after the shift runs with production on the candidate, so a failure there is a +// live gateway that has to go back. The marker is what separates that case from a failure before +// the shift, where production never moved and the candidate is the thing to clean up. +test('a failure after the shift rolls production back automatically', () => { + const rollback = indexOfStep('Roll traffic back to the previous revision') + assert.ok(rollback > indexOfStep('Verify the public origin after the shift')) + assert.match(workflow, /echo "TRAFFIC_SHIFTED=true" >> "\$\{GITHUB_ENV\}"/) + const shift = workflow.indexOf('- name: Shift all traffic to the verified candidate') + assert.ok( + workflow.indexOf('echo "TRAFFIC_SHIFTED=true"') > shift, + 'the success marker follows the shift step' + ) + const body = workflow.slice( + workflow.indexOf('- name: Roll traffic back to the previous revision'), + workflow.indexOf('- name: Delete the rejected candidate revision') + ) + assert.match( + body, + /if: \$\{\{ \(failure\(\) \|\| cancelled\(\)\) && env\.TRAFFIC_SHIFT_ATTEMPTED == 'true' && env\.ROLLOUT_VERIFIED != 'true' \}\}/, + 'the rollback must be conditioned on both failure and the shift marker' + ) + assert.match(body, /test -n "\$\{ROLLBACK_REVISION:-\}"/) + assert.match(body, /--to-revisions "\$\{ROLLBACK_REVISION\}=100"/) + assert.match(body, /test "\$\{serving\}" = "\$\{ROLLBACK_REVISION\}"/) + assert.match(body, /GITHUB_STEP_SUMMARY/, 'the rollback must be reported in the summary') +}) + +// Why: a candidate that never took traffic still holds a warm instance and a Cloud SQL pool. Its +// tag comes off first, because Cloud Run refuses to delete a revision a traffic target names. +test('verified recovery authorizes rejected candidate deletion', () => { + const body = workflow.slice( + workflow.indexOf('- name: Delete the rejected candidate revision'), + workflow.indexOf('- name: Drop the candidate traffic tag') + ) + assert.match( + body, + /env\.RECOVERY_VERIFIED == 'true'/, + 'cleanup must wait for verified recovery traffic and public checks' + ) + assert.match(body, /if test -z "\$\{CANDIDATE_REVISION:-\}"; then/) + assert.ok( + body.indexOf('--remove-tags') < body.indexOf('gcloud run revisions delete'), + 'the tag must come off before the revision is deleted' + ) + assert.match(body, /echo "CANDIDATE_TAG=" >> "\$\{GITHUB_ENV\}"/) +}) + +test('the run always drops its traffic tag', () => { + const cleanup = indexOfStep('Drop the candidate traffic tag') + assert.equal(cleanup, stepNames().length - 1, 'tag cleanup must be the last step') + assert.match(workflow, /--remove-tags "\$\{CANDIDATE_TAG\}"/) + const body = workflow.slice(workflow.indexOf('- name: Drop the candidate traffic tag')) + assert.match(body, /if: always\(\)/) + assert.match(body, /test -n "\$\{CANDIDATE_TAG:-\}" \|\| exit 0/) +}) + +test('push credentials cannot assume the shared Relay deploy identity', () => { + const source = terraform('push-deploy-identity.tf') + assert.match(source, /"attribute.push_deploy"\s*=\s*"'production'"/) + assert.doesNotMatch(source, /"attribute.repository"\s*=/) + assert.match(source, /attribute\.push_deploy\/production/) + assert.doesNotMatch(workflow, /PRODUCTION_GCP_RELAY_DEPLOY_/) + assert.doesNotMatch(terraform('push-gateway.tf'), /member\s*=\s*local\.relay_github_deploy_service_account_member/) +}) + +// A latest revision needs a successor even when validation is inert. +test('dedicated database admits three simultaneous revision pools', () => { + assert.match(terraform('push-gateway.tf'), /var\.push_max_instances \* var\.push_database_pool_max \* 3 <= 64/) +}) + +test('push has only a dedicated database attachment and a narrowly scoped deployment lease', () => { + const service = terraform('push-gateway.tf') + const database = terraform('push-dedicated-database.tf') + assert.match(service, /instances = \[google_sql_database_instance\.push_dedicated\[0\]\.connection_name\]/) + assert.match(service, /secret\s*= google_secret_manager_secret\.push_dedicated_database_url\[0\]\.secret_id/) + assert.match(service, /version = google_secret_manager_secret_version\.push_dedicated_database_url\[0\]\.version/) + assert.doesNotMatch(service + database, /push_dedicated_database_(?:active|enabled)|local\.relay_database_connection_name|resource "google_sql_database" "push"/) + assert.match(database, /tier\s*= "db-custom-2-7680"/) + assert.match(database, /availability_type = "REGIONAL"/) + assert.match(database, /deletion_protection\s*= true/) + assert.match(database, /deletion_protection_enabled = true/) + const identity = terraform('push-deploy-identity.tf') + const lease = identity.match(/resource "google_storage_bucket_iam_member" "github_push_rollout_lease" \{([\s\S]*?)\n\}/)?.[1] + assert.ok(lease) + assert.match(lease, /member = local\.push_deploy_member/) + assert.match(lease, /role\s*= "roles\/storage.objectAdmin"/) + assert.match(lease, /resource.name == 'projects\/_\/buckets\/\$\{var.project_id\}-terraform-state\/objects\/terraform\/state\/push-rollout\/production.lock'/) +}) diff --git a/cloud/dev/scripts/push-validation-workflow.test.mjs b/cloud/dev/scripts/push-validation-workflow.test.mjs new file mode 100644 index 00000000000..e26769cfeb3 --- /dev/null +++ b/cloud/dev/scripts/push-validation-workflow.test.mjs @@ -0,0 +1,48 @@ +import assert from 'node:assert/strict' +import { readFileSync } from 'node:fs' +import test from 'node:test' +import { readRelayWorkflow } from './relay-repository.mjs' + +const workflow = readRelayWorkflow('push-deploy.yml') +const position = (name) => { + const index = workflow.indexOf(`- name: ${name}`) + assert.notEqual(index, -1) + return index +} +const capability = position('Require image support for inert validation') +const deploy = position('Deploy the candidate revision with no traffic') +const activation = position('Retire inert validation and activate the verified image') +const shift = position('Shift all traffic to the verified candidate') + +test('the exact build digest must support validation before production boot', () => { + assert.match(workflow, /docker buildx build --push --platform linux\/amd64 --provenance=false --metadata-file/) + assert.match(workflow, /containerimage\.digest/) + assert.doesNotMatch(workflow, /gcloud artifacts docker images describe/) + assert.ok(capability < deploy) + const preflight = workflow.slice(capability, deploy) + assert.match(preflight, /docker run --rm --network none --entrypoint node "\$\{IMAGE\}"/) + assert.match(preflight, /loadPushConfig\(env\)\.mode !== "validation"/) + assert.match(preflight, /validation_mode_not_fail_closed/) +}) + +test('inert validation and credential checks precede deliberate activation of the same digest', () => { + assert.match(workflow.slice(deploy, activation), /--update-env-vars ORCA_PUSH_MODE=validation/) + assert.match(workflow.slice(deploy, activation), /\.mode == "validation"/) + assert.ok(position('Prove the runtime identity can reach FCM') < activation) + const active = workflow.slice(activation, shift) + assert.ok(active.indexOf('gcloud run deploy') < active.indexOf('gcloud run revisions delete')) + assert.match(active, /--image "\$\{IMAGE\}"/) + assert.match(active, /--remove-env-vars ORCA_PUSH_MODE/) + assert.match(active, /\.spec\.containers\[0\]\.image == \$image/) + assert.match(active, /\.spec\.serviceAccountName == \$account/) + assert.match(active, /\.mode == "active"/) + assert.ok(active.indexOf('ACTIVATION_ATTEMPTED=true') < active.indexOf('gcloud run deploy')) + assert.match(workflow, /deletion below must stop its workers/) +}) + +test('production startup connects read-only and gates all background work in validation', () => { + const entry = readFileSync(new URL('../../apps/push/src/index.ts', import.meta.url), 'utf8') + assert.match(entry, /readOnly: config\.mode === 'validation'/) + assert.match(entry, /startPushBackground\(config,/) + assert.doesNotMatch(entry, /worker\.start\(/) +}) diff --git a/cloud/dev/scripts/relay-cloud-sql-connection-budget.test.mjs b/cloud/dev/scripts/relay-cloud-sql-connection-budget.test.mjs index a26d24c274d..89d40954f7d 100644 --- a/cloud/dev/scripts/relay-cloud-sql-connection-budget.test.mjs +++ b/cloud/dev/scripts/relay-cloud-sql-connection-budget.test.mjs @@ -6,7 +6,7 @@ import { readRelayCloudSqlConnectionBudget } from './relay-cloud-sql-connection-budget.mjs' -test('production plus three Asia pools preserves allowance and reserve below the ceiling', () => { +test('production shared consumers keep allowance and reserve below the ceiling', () => { const report = readRelayCloudSqlConnectionBudget() assert.deepEqual(report.consumers, { cells: 230, directors: 15, auth: 20, api: 50 }) @@ -63,7 +63,11 @@ test('excludes fenced cell pools and reads per-cell pool overrides', () => { } } `, - terraformVariables: 'variable "relay_director_database_pool_max" { default = 3 }', + terraformVariables: [ + 'variable "relay_director_database_pool_max" { default = 3 }', + 'variable "push_max_instances" { default = 1 }', + 'variable "push_database_pool_max" { default = 2 }' + ].join('\n'), relayConfig: 'export const RELAY_DATABASE_POOL_MAX = 10' }, maxConnections: 100, @@ -76,6 +80,37 @@ test('excludes fenced cell pools and reads per-cell pool overrides', () => { assert.equal(report.budgetedTotal, 47) }) +test('dedicated push scaling does not consume shared capacity', () => { + const report = readRelayCloudSqlConnectionBudget({ + proposedAsiaCellCount: 1, + appConsumers: { authInstances: 1, authPoolMax: 10, apiInstances: 1, apiPoolMax: 5, maxConnections: 100 }, + sources: { + productionTfvars: ` + relay_max_instances = 1 + push_max_instances = 3 + relay_gce_fenced_cells = [] + relay_gce_cells = { + "production-gce-c2" = { database_pool_max = 4 + } + } + `, + terraformVariables: [ + 'variable "relay_director_database_pool_max" { default = 3 }', + 'variable "push_max_instances" { default = 1 }', + 'variable "push_database_pool_max" { default = 2 }' + ].join('\n'), + relayConfig: 'export const RELAY_DATABASE_POOL_MAX = 10' + }, + maxConnections: 100, + maintenanceAdminAllowance: 1, + explicitReserve: 1 + }) + + assert.equal(report.consumers.push, undefined) + assert.equal(report.rolloutOverlap.pushCandidate, undefined) + assert.equal(report.operatingMaximum, 46) +}) + test('requires strict headroom below the physical ceiling', () => { const report = calculateRelayCloudSqlConnectionBudget({ cellPoolTotal: 20, diff --git a/cloud/dev/scripts/render-workload-identity-conditions.mjs b/cloud/dev/scripts/render-workload-identity-conditions.mjs index 9a7c2e5cc09..815ba1b2c17 100644 --- a/cloud/dev/scripts/render-workload-identity-conditions.mjs +++ b/cloud/dev/scripts/render-workload-identity-conditions.mjs @@ -18,6 +18,7 @@ const TERRAFORM_ROOTS = { 'infra/terraform/relay-shared.tf', 'infra/terraform/relay-github-workflow-trust.tf', 'infra/terraform/relay-github-actions.tf', + 'infra/terraform/push-deploy-identity.tf', 'infra/terraform/relay-staging-deploy-iam.tf', 'infra/terraform/relay-asia-topology-iam.tf', 'infra/terraform/relay-asia-proof-iam.tf' @@ -475,7 +476,7 @@ function collectTfvars(source, variables) { offset += line.length + 1 continue } - const structured = /^([A-Za-z_][A-Za-z0-9_]*)\s*=\s*(?=[[{])/.exec(line) + const structured = /^([A-Za-z_][A-Za-z0-9_]*)\s*=\s*/.exec(line) if (structured) { try { variables[structured[1]] = parseValueAt(source, offset + structured[0].length) diff --git a/cloud/dev/scripts/workload-identity-attribute-conditions.test.mjs b/cloud/dev/scripts/workload-identity-attribute-conditions.test.mjs index 1d3f3ce4d79..64791605549 100644 --- a/cloud/dev/scripts/workload-identity-attribute-conditions.test.mjs +++ b/cloud/dev/scripts/workload-identity-attribute-conditions.test.mjs @@ -30,6 +30,9 @@ const EXPECTED_CONDITIONS = { }, production: { relay: { + github_push: + "assertion.repository == 'stablyai/orca' && assertion.repository_id == '1183888342' && assertion.repository_owner_id == '127256420' && assertion.ref == 'refs/heads/main' && assertion.environment == 'production' && assertion.event_name == 'workflow_dispatch' && assertion.workflow_ref == 'stablyai/orca/.github/workflows/cloud-push-deploy.yml@refs/heads/main' && assertion.job_workflow_ref == 'stablyai/orca/.github/workflows/cloud-push-deploy.yml@refs/heads/main'", + github: "assertion.repository == 'stablyai/orca' && assertion.repository_id == '1183888342' && assertion.repository_owner_id == '127256420' && assertion.ref == 'refs/heads/main' && assertion.environment == 'production' && ((assertion.workflow_ref == 'stablyai/orca/.github/workflows/cloud-deploy-relay-fence-broker.yml@refs/heads/main' || assertion.workflow_ref == 'stablyai/orca/.github/workflows/cloud-deploy-relay-production-capacity.yml@refs/heads/main' || assertion.workflow_ref == 'stablyai/orca/.github/workflows/cloud-deploy-relay-production-director.yml@refs/heads/main' || assertion.workflow_ref == 'stablyai/orca/.github/workflows/cloud-deploy-relay-production-multi-target.yml@refs/heads/main' || assertion.workflow_ref == 'stablyai/orca/.github/workflows/cloud-deploy-relay-production.yml@refs/heads/main' || assertion.workflow_ref == 'stablyai/orca/.github/workflows/cloud-operate-relay-asia-admission.yml@refs/heads/main' || assertion.workflow_ref == 'stablyai/orca/.github/workflows/cloud-publish-relay-production.yml@refs/heads/main') || (assertion.workflow_ref == 'stablyai/orca/.github/workflows/cloud-operate-relay-production-rehome.yml@refs/heads/main' && assertion.job_workflow_ref == 'stablyai/orca/.github/workflows/cloud-operate-relay-production-rehome-job.yml@refs/heads/main') || (assertion.workflow_ref == 'stablyai/orca/.github/workflows/cloud-deploy-relay-production-same-cap.yml@refs/heads/main' && (assertion.job_workflow_ref == 'stablyai/orca/.github/workflows/cloud-deploy-relay-production-same-cap-job.yml@refs/heads/main' || assertion.job_workflow_ref == 'stablyai/orca/.github/workflows/cloud-deploy-relay-production-same-cap.yml@refs/heads/main')))", github_monitor: diff --git a/cloud/docs/push-database-cutover.md b/cloud/docs/push-database-cutover.md new file mode 100644 index 00000000000..e2b0be3970d --- /dev/null +++ b/cloud/docs/push-database-cutover.md @@ -0,0 +1,112 @@ +# Dedicated push database operations + +Push attaches only to its dedicated PostgreSQL 17 instance: regional HA, 2 vCPU, +7.5 GiB RAM, 50 GiB SSD with automatic growth, seven retained backups and seven-day +point-in-time recovery. Cloud SQL and Terraform deletion protections remain enabled. +The Cloud SQL connector uses the dedicated URL secret pinned to its managed version. +There is no shared-storage fallback or provision/activate switch. + +## Existing-resource cleanup: operator prerequisite + +This is a plan/runbook, not authorization to apply or delete resources. Preserve the +shared Orca instance, dedicated push instance, all dedicated data and identities, and +unrelated resources. The dedicated resource addresses remain unchanged: + +- `google_sql_database_instance.push_dedicated[0]` +- `google_sql_database.push_dedicated[0]` +- `random_password.push_dedicated_database[0]` +- `google_sql_user.push_dedicated[0]` +- `google_secret_manager_secret.push_dedicated_database_url[0]` +- `google_secret_manager_secret_version.push_dedicated_database_url[0]` +- `google_secret_manager_secret_iam_member.push_dedicated_database_url_accessor[0]` + +The relay state may still own these six obsolete shared-store resources, whose +configuration is removed. An untargeted plan would propose deleting them; do not apply it: + +- `google_sql_database.push[0]` +- `google_sql_user.push[0]` +- `random_password.push_database[0]` +- `google_secret_manager_secret.push_database_url[0]` +- `google_secret_manager_secret_version.push_database_url[0]` +- `google_secret_manager_secret_iam_member.push_database_url_runtime_accessor[0]` + +1. Use the production backend in `infra/terraform/README.md`. Inspect state addresses and + the live service attachment, pinned secret reference and revision resources without + printing credentials. Require the dedicated attachment and no old shared-store consumers; + source connection drain needs an authorized operator's read-only observation. +2. Have the shared database owner adopt the six legacy resources in an explicitly owned + archival configuration before retiring their relay-state ownership. Retain the former + database's `prevent_destroy` protection and secret versions; do not disable protection, + drop databases, rotate passwords or introduce a second runtime attachment. A reviewed + exact-address state transfer must preserve remote IDs and secret material in approved + Terraform storage, with no credential exports to local files or terminal output. +3. Require the owner's import/ownership plan to preserve existing resources and then an + empty plan for those addresses. Only after adoption is proven may the operator remove + precisely the six former addresses from relay state under backend locking. Do not + automate this via `removed` blocks, broad `state rm`, force, or an untargeted apply. +4. Review a fresh relay plan. Reject every delete or replace affecting either SQL instance, + dedicated databases/users/secrets, or unrelated resources. Target only the intended push + service and lease IAM grant for rollout; review their dependency closure too. Existing + unrelated drift must be handled by its owner, outside this cleanup. + +No data transfer, dedicated database reset, or phone re-registration is part of this cleanup. + +## Schema prerequisite for existing internal test databases + +New schemas omit `push_hosts` and the unused `host_public_key` and `transcript` columns +on `push_challenges`. Authentication still verifies the encrypted transcript and consumes +its challenge digest once; sessions and device ownership are unchanged. No compatibility +migration for unpublished builds runs at application startup. + +Before deploying onto an older internal schema, an operator must arrange a separately +reviewed schema-preparation job through the approved database execution path. Its entire +scope is dropping `push_hosts` (including its index) and those two unused challenge columns; +preserve challenge digest/expiry/consumption fields and every session, device and delivery +table. Verify that the old NOT NULL columns are absent before admitting the new image. +Do not hand-edit production SQL or reset the dedicated database to satisfy this prerequisite. +Until that job is reviewed and executed, the new image is not ready for an existing schema. + +## Deployment serialization transition + +Finish all old push workflow runs before changing the workflow's lock namespace. An old +shared-lock push run and a new push-lock run do not exclude each other. Hold off new push +dispatches while preparing the following exact changes: + +1. Review the relay-root plan for + `google_storage_bucket_iam_member.github_push_rollout_lease[0]`. It grants only + `roles/storage.objectAdmin` on + `projects/_/buckets/onorca-cloud-terraform-state/objects/terraform/state/push-rollout/production.lock` + to the dedicated push deploy account. The lease action uses object GET/upload/delete, + so no bucket-wide listing or Terraform-state access is needed. +2. After approval, apply only the reviewed IAM/dependency plan. Verify the exact condition + and principal independently. If foundation still grants push membership in the old + `cloud_sql_rollout_lease_members`, its owner removes only that push member; keep Relay's + existing members and permissions. Do not mutate foundation through the relay root. +3. Publish the reviewed workflow on main with `production-push-rollout`, cancellation + disabled, and the existing lease action pointed at the dedicated object. The durable + lease covers admission, candidate validation, activation, traffic changes and recovery. + A stale/conflicting lease stops the run; it is never stolen or force-deleted. +4. Deploy the reviewed image through `cloud-push-deploy.yml`. Preserve candidate readiness, + runtime-provider validation, exact digest/configuration checks, and explicit activation. + Verify the public origin and real notification delivery/dismissal afterward. + +## Recovery and capacity + +Activation starts schema writes and workers before HTTP promotion. Traffic rollback cannot +undo queue or schema changes. Cloud Run cannot delete its latest revision, so failed +activation creates a known-good successor, verifies it, promotes it, then retires rejected +and previous revisions. When partial activation leaves three resources, retire non-latest +inert validation before creating recovery. Failed retirement stops automation. Admission +requires one serving revision resource; retire historical leftovers under the push lease. +Keep the dedicated attachment for application rollback and retain an immutable compatible +image. An image requiring the removed challenge columns needs separate schema review. + +The two-instance ceiling and two-connection pool draw four configured connections, twelve +across three simultaneous revision resources. Terraform caps instances × pool × 3 at 64 +for serving, validation/rejected and active/recovery pools. Push does not draw from Relay's +shared connection budget. Source connections must have drained before treating that old +allocation as free. Increase capacity only after measuring deployed contention. + +Cloud SQL resizing can interrupt connections despite HA. Durable accepted events remain in +SQL; workers retry within each event's original five-minute deadline. Schedule resizes and +verify reconnection, queue recovery, readiness and real delivery afterward. diff --git a/cloud/docs/push-gateway.md b/cloud/docs/push-gateway.md new file mode 100644 index 00000000000..9183f693a87 --- /dev/null +++ b/cloud/docs/push-gateway.md @@ -0,0 +1,392 @@ +# Orca mobile push gateway + +`orca-cloud-push` is a public Cloud Run service in `onorca-cloud` that turns a desktop +notification into an APNs or FCM push for a paired phone. The desktop registers each phone's +native token with it and calls `POST /v1/send` after the socket fan-out it already does; the +phone treats APNs/FCM as the sole ordinary OS-banner path. The notification socket is retained only +for live dismissal and reconnect tray reconciliation; it does not create or recover banners. Desktop +notification categories remain authoritative. The service is the only place the Apple +`.p8` signing key is readable, which is the reason it exists as a service at all. + +The request schemas live in `packages/push-contract/src/`. This document covers Terraform +ownership, deployment, credential rotation, and recovery. + +**There is no staging push gateway.** That is a decision, not an omission. `push_gateway_enabled` +is false in `environments/staging.tfvars` and true in `environments/production.tfvars`, and every +resource in `infra/terraform/push-gateway.tf` is behind it. A staging gateway would be a tfvars +edit plus a second set of Apple credentials. + +## Shape + +| Setting | Value | Where | +| ----------------- | ------------------------------------------------------ | -------------------------------------------- | +| Cloud Run service | `orca-cloud-push` | `push_cloud_run_service_name` | +| Region | `us-central1` | `region` | +| Instances | min 1, max 2 | `push_min_instances`, `push_max_instances` | +| Database pool | 2 per instance | `push_database_pool_max` | +| Concurrency | 80 | `push_concurrency` | +| Ingress | all | `INGRESS_TRAFFIC_ALL` | +| Invoker | IAM disabled | `invoker_iam_disabled = true` on the service | +| Runtime identity | `orca-cloud-push@onorca-cloud.iam.gserviceaccount.com` | `google_service_account.push_runtime` | +| Database | `orca_push` on dedicated HA PostgreSQL 17 | `google_sql_database.push_dedicated` | +| Hostname | `push.onorca.dev` | `push_base_url` | + +The minimum of one instance is deliberate and did not move when the ceiling came down to two. A +cold start delays a notification past the point where it is worth showing, so the floor is what +keeps a notification prompt. The +ceiling is a different question, answered below. + +Push uses its approved dedicated two-vCPU HA database. Two instances with a two-connection +pool draw four connections; three simultaneous revision resources draw twelve. Tagged +candidates can run outside the service-wide cap, so Terraform bounds instances × pool × 3 +at 64 connections, leaving dedicated capacity for maintenance and operators. Increase pool +sizes only after measuring contention. The shared Relay budget excludes push entirely. + +Authentication is the host proof in `POST /v1/host/challenge`, not Cloud Run IAM, so the service +opts out of invoker IAM with `invoker_iam_disabled = true`, exactly as the relay director does. +The project's domain-restricted-sharing policy refuses an `allUsers` invoker binding, so that is +the only way to reach an open service here. + +## Environment + +Set on the container by Terraform: + +| Variable | Source | +| ----------------------------- | -------------------------------------------------------- | +| `PORT` | Cloud Run, container port 8080 | +| `ORCA_PUSH_PUBLIC_URL` | `push_base_url` | +| `ORCA_PUSH_FCM_PROJECT_ID` | `project_id` (required for standalone runtime) | +| `ORCA_PUSH_DATABASE_URL` | Secret `orca-cloud-push-dedicated-database-url`, pinned version | +| `ORCA_PUSH_DATABASE_POOL_MAX` | `push_database_pool_max`, 2 per instance | +| `ORCA_PUSH_APNS_KEY` | Secret `orca-cloud-push-apns-key`, version `latest` | +| `ORCA_PUSH_APNS_KEY_ID` | Secret `orca-cloud-push-apns-key-id`, version `latest` | +| `ORCA_PUSH_APPLE_TEAM_ID` | Secret `orca-cloud-push-apple-team-id`, version `latest` | + +`ORCA_PUSH_APNS_TOPIC` is left to its application default (`com.stably.orca.mobile`). Add it here +only when it has to differ from the code default, so that a code-side change stays visible rather +than silently overridden. + +Terraform owns the three Apple secret **names, labels, and replication, and never a version.** +The `.p8` is issued by the Apple developer portal, so a Terraform-managed version would put the +private key in state and would fight the rotation below. The database URL secret is different: +Terraform generates that password, so it owns that version, exactly as `relay-database.tf` does. +That puts the generated password and the full database URL in the state bucket, which the shared +deploy identity can read; the Apple key never appears there. The three Apple secrets and the +`orca_push` database carry `prevent_destroy`, so disabling the gateway fails the plan instead +of deleting the only copy of the signing key or every live device token. + +## Importing what already exists + +The runtime account, the three Apple secrets, and their accessor bindings were created out of +band alongside the Apple credentials. They are declared so a plan is clean, and imported once. +Run these from `cloud/` after `pnpm infra:init --env production`, review the resulting plan, and +expect the imported resources to show no changes. + +```sh +terraform -chdir=infra/terraform import -var-file=environments/production.tfvars \ + 'google_service_account.push_runtime[0]' \ + projects/onorca-cloud/serviceAccounts/orca-cloud-push@onorca-cloud.iam.gserviceaccount.com + +terraform -chdir=infra/terraform import -var-file=environments/production.tfvars \ + 'google_project_iam_member.push_runtime_fcm_admin[0]' \ + 'onorca-cloud roles/firebasecloudmessaging.admin serviceAccount:orca-cloud-push@onorca-cloud.iam.gserviceaccount.com' + +terraform -chdir=infra/terraform import -var-file=environments/production.tfvars \ + 'google_project_iam_member.push_runtime_service_usage_consumer[0]' \ + 'onorca-cloud roles/serviceusage.serviceUsageConsumer serviceAccount:orca-cloud-push@onorca-cloud.iam.gserviceaccount.com' + +terraform -chdir=infra/terraform import -var-file=environments/production.tfvars \ + 'google_secret_manager_secret.push_provider["orca-cloud-push-apns-key"]' \ + projects/onorca-cloud/secrets/orca-cloud-push-apns-key + +terraform -chdir=infra/terraform import -var-file=environments/production.tfvars \ + 'google_secret_manager_secret.push_provider["orca-cloud-push-apns-key-id"]' \ + projects/onorca-cloud/secrets/orca-cloud-push-apns-key-id + +terraform -chdir=infra/terraform import -var-file=environments/production.tfvars \ + 'google_secret_manager_secret.push_provider["orca-cloud-push-apple-team-id"]' \ + projects/onorca-cloud/secrets/orca-cloud-push-apple-team-id + +terraform -chdir=infra/terraform import -var-file=environments/production.tfvars \ + 'google_secret_manager_secret_iam_member.push_provider_runtime_accessor["orca-cloud-push-apns-key"]' \ + 'projects/onorca-cloud/secrets/orca-cloud-push-apns-key roles/secretmanager.secretAccessor serviceAccount:orca-cloud-push@onorca-cloud.iam.gserviceaccount.com' + +terraform -chdir=infra/terraform import -var-file=environments/production.tfvars \ + 'google_secret_manager_secret_iam_member.push_provider_runtime_accessor["orca-cloud-push-apns-key-id"]' \ + 'projects/onorca-cloud/secrets/orca-cloud-push-apns-key-id roles/secretmanager.secretAccessor serviceAccount:orca-cloud-push@onorca-cloud.iam.gserviceaccount.com' + +terraform -chdir=infra/terraform import -var-file=environments/production.tfvars \ + 'google_secret_manager_secret_iam_member.push_provider_runtime_accessor["orca-cloud-push-apple-team-id"]' \ + 'projects/onorca-cloud/secrets/orca-cloud-push-apple-team-id roles/secretmanager.secretAccessor serviceAccount:orca-cloud-push@onorca-cloud.iam.gserviceaccount.com' +``` + +The push resources already exist in production. Preserve their addresses, dedicated database +and identities; review the [database cleanup runbook](./push-database-cutover.md) before applying +changes. This root has unrelated standing drift, so an untargeted apply is never automatic. + +Two things this root does **not** declare, because the carve assigns them elsewhere. Neither +affects whether this root's plan is clean, since an undeclared resource is invisible to it. + +- `firebase.googleapis.com` and `fcm.googleapis.com` are project service enablement, which is + `google_project_service.required` in the foundation root. They are already enabled; add them + to the foundation root's list so a foundation plan stays clean. +- The Firebase attachment on `onorca-cloud` is project-level and belongs with foundation for the + same reason. It exists already. + +## Deploying + +`Deploy Push Gateway Production` (`.github/workflows/cloud-push-deploy.yml`) is the only +supported path. Like every `cloud-*` workflow it does nothing until `ORCA_CLOUD_OPERATIONS_ENABLED` +is `true`, it runs only on `main`, and it needs the confirmation string `DEPLOY_PUSH_GATEWAY`. + +It authenticates as the dedicated `orca-cloud-gha-push` identity through +`PRODUCTION_GCP_PUSH_DEPLOY_WORKLOAD_IDENTITY_PROVIDER` and +`PRODUCTION_GCP_PUSH_DEPLOY_SERVICE_ACCOUNT`. `push-deploy-identity.tf` restricts Workload Identity +to this exact dispatch workflow on main in the production environment. Its distinct principal +attribute cannot assume the shared Relay deploy identity. + +The account can write images to Artifact Registry, deploy the push service, impersonate only +the push runtime account, and manage exactly `terraform/state/push-rollout/production.lock` +in the production state bucket. The relay root owns that conditional lease grant. It grants +no Terraform-state object access. Publish `github_push_workload_identity_provider` and +`github_push_deploy_service_account` as the production-environment variables above. + +The workflow uses the `production-push-rollout` concurrency group with cancellation disabled +and the existing durable lease action on the push-specific object. Push and Relay deploy +independently; two push deploys cannot race traffic changes. Finish every old shared-lock push +run before enabling the new workflow and lease grant. See the cleanup runbook for the bounded +IAM transition and removal of any obsolete foundation-owned push membership. + +The run builds the reviewed `source_sha` while the workflow stays on `main`. Buildx returns +its own pushed digest (no mutable-tag lookup); every subsequent check and deployment uses that +same digest. Before any production boot, a network-isolated container checks that the image +recognizes `ORCA_PUSH_MODE=validation` and rejects invalid modes. Older images that lack this +capability are refused before they can connect to production. + +Under the production push rollout lease, it records the serving rollback revision and +asserts Terraform-owned scaling. It deploys a tagged, zero-traffic validation revision: + +- Validation opens PostgreSQL with `default_transaction_read_only=on` and skips schema setup. +- No delivery worker or challenge, session, or delivery pruner starts. +- Only `/health` and `/ready` are available; all application routes return 503. +- `/health` attests `mode: validation`; `/ready` checks database connectivity only. It does not + prove schema compatibility, provider delivery, or active-worker readiness. Container probes + can still use `/health` without treating an inert process as unhealthy. + +The build explicitly targets `linux/amd64` with provenance disabled so build metadata records a +single manifest digest, rather than an OCI index that Cloud Run resolves to a different digest. +The workflow verifies the exact image and scaling, probes readiness and mode, and checks the +runtime identity with a validate-only FCM request. Cloud Run rejects deletion of the latest +created revision even when it has no tag or traffic. Activation therefore creates a successor +before removing the validation tag and deleting validation. The dedicated 64-connection budget +reserves three simultaneous revision pools: serving, validation/rejected, +and active/recovery successor (12 configured pool connections at the current two-by-two shape). +Revision deletion is not proof of physical SQL session drain; verify termination and SQL sessions +in controlled rollout acceptance. There is no shutdown sleep used as a drain gate. + +**Activation deliberately starts production effects.** The distinct active revision uses the exact +validated digest with the validation override removed. Schema setup runs on its existing +one-connection untimed pool, followed by workers and pruners, before HTTP promotion. The workflow +checks digest, full runtime spec and secret-reference shape, scaling, readiness and active mode, +then moves HTTP traffic and checks the public origin. Those checks commit the new serving revision; +subsequent retirement failures do not trigger rollback to a possibly deleted previous revision. +The previous consumer is retired and all tags are cleared. Retain the previous immutable image +from the summary: later recovery redeploys that digest, because the previous revision is deleted. + +Before any candidate creation, the workflow requires exactly one revision resource, the sole HTTP +serving revision. Existing historical revisions or leftovers from interrupted runs require explicit +operator review and cleanup under the lease first; the workflow does not blindly delete them. +This gate and retirement after every successful rollout prevent repeated runs accumulating workers. +Terraform still owns configuration and scaling; removing validation mode adds no ignored field. + +On failure before public checks pass, any attempted traffic shift is first rolled back and verified. +If partial activation created a successor, recovery retires non-latest validation first; deletion +failure stops recovery before a fourth resource can be created. Recovery then deploys the captured +known-good digest as a tagged, zero-traffic successor with normal mode. It verifies template shape, +secret references and scaling, probes tagged readiness and active mode, promotes the recovery +revision, verifies traffic and public health, and only then deletes rejected and previous revisions. +The latest recovery revision remains serving. Known-good recovery schema and workers can execute +before promotion; neither recovery nor traffic rollback undoes schema changes or sent notifications. + +Partial creates record deterministic names before mutation. Failed recovery or deletion requires +operator cleanup under the lease; the next automated run refuses leftover resources. A canceled +runner can require the same intervention. Traffic restoration alone does not stop queue consumers. + +Manual recovery must preserve the three-resource bound and keep the successor serving: + +```sh +# Hold the rollout lease; inspect latest, traffic, tags and existing revisions first. +# If three resources remain after partial activation, retire non-latest inert validation first. +# Restore previous traffic if its revision still exists and a failed candidate took traffic. +gcloud run deploy orca-cloud-push \ + --project onorca-cloud --region us-central1 --image \ + --remove-env-vars ORCA_PUSH_MODE --no-traffic \ + --tag --revision-suffix +# Verify exact digest, template spec/secret references/scaling, tagged /ready and active /health. +gcloud run services update-traffic orca-cloud-push \ + --project onorca-cloud --region us-central1 --to-revisions =100 +# Verify traffic and public /ready and /health before retiring old consumers. +gcloud run services update-traffic orca-cloud-push \ + --project onorca-cloud --region us-central1 --clear-tags +gcloud run revisions delete \ + --project onorca-cloud --region us-central1 +# Repeat only for reviewed obsolete revisions; retain the latest serving recovery revision. +``` + +Never merely remove validation mode while the template still holds a rejected image. Terraform +owns environment configuration but ignores the image, so that would activate rejected code. +Remove a tag only if it remains present. Verify the recovery revision is serving, the template is safe, +and obsolete revision deletion and connection drain completed; +already accepted provider sends cannot be undone. Activation-time schema changes must be additive +and compatible with the rollback image: rollback does not reverse migrations or queue mutations. +The inert phase intentionally cannot validate a new schema by applying it to production. Review +migrations and validate them against isolated PostgreSQL before dispatch. No actual Cloud Run +rollout, provider delivery or physical-device acceptance is implied by local contract tests. + +### Incompatible queue rollout prerequisite + +The queue stores one notification object per delivery. Before deploying a revision that changes this +format, stop every older push gateway revision and clear only unpublished push delivery fixtures from +the push database. This is an unpublished feature, so do not preserve or migrate queued fixtures; no +production mutation is implied by this prerequisite. + +### Why the FCM probe impersonates the runtime account + +A gateway that boots and answers `/ready` can still be unable to send: the FCM grant lives on +the runtime service account, not on anything the readiness check touches. The probe therefore +mints an access token for `orca-cloud-push@onorca-cloud.iam.gserviceaccount.com` and posts +`validate_only: true` with a token that cannot exist. `validate_only` stops Google before any +delivery, and a healthy credential answers `INVALID_ARGUMENT` because the device token is +garbage. `PERMISSION_DENIED`, `401`, and `403` are the failures the step exists to catch, and +they fail the run immediately, before traffic moves. Those four answers are the only conclusive +ones: a `429`, a `5xx`, or a transport failure says nothing about the credential, so the send is +retried up to five times at five-second intervals rather than read as either verdict. Probing as the deploy identity instead would prove +something true about the wrong account. + +## Rotating the APNs key + +Apple keys do not expire, so this is for a suspected compromise or a routine rotation. Order +matters: the new key must be serving before the old one is revoked, or every iOS push fails in +the window between. + +1. In the Apple developer portal, create a **new** APNs authentication key. Download the `.p8` + once; Apple will not show it again. Note the new key ID. A team may hold two APNs keys at a + time, which is what makes this overlap possible. +2. Add a version to each changed secret, without printing the value: + + ```sh + gcloud secrets versions add orca-cloud-push-apns-key \ + --project onorca-cloud --data-file /path/to/AuthKey_NEW.p8 + printf '%s' '' | gcloud secrets versions add orca-cloud-push-apns-key-id \ + --project onorca-cloud --data-file=- + ``` + + The team ID does not change, so `orca-cloud-push-apple-team-id` is untouched. + +3. Dispatch `Deploy Push Gateway Production`. The container reads `latest` at start, so only a + new revision picks the key up; there is no in-place reload. +4. Verify from a real device that an iOS notification still arrives. The workflow's FCM probe + covers Android only, and APNs has no validate-only equivalent. +5. Only then revoke the old key in the Apple portal, and disable the superseded secret versions: + + ```sh + gcloud secrets versions disable \ + --project onorca-cloud --secret orca-cloud-push-apns-key + ``` + + Disable rather than destroy, so a rollback to the previous revision still works. Destroy + after the next clean deploy. + +Delete the downloaded `.p8` from disk when you are done. It is the whole credential. + +## Dead tokens + +A push token stops working when the app is uninstalled, when the user restores to a new device, +or when iOS reissues it. Both providers report this, and the shapes differ: + +- APNs: HTTP 410, or 400 with `BadDeviceToken` or `Unregistered`. + `DeviceTokenNotForTopic` is a provider configuration error and leaves the registration live. + Check the APNs topic and environment; future notifications can resume after correction without + phone re-registration. The failed notification is not retried for this non-transient error. +- FCM: `UNREGISTERED`, or `INVALID_ARGUMENT` whose message names the token. + +The gateway marks the registration `dead_at` and returns `status: "dead"` for it, and the +desktop drops the registration when it sees that. Nothing here retries a dead token. A phone +that comes back re-registers the same host/device pair, retaining its `registrationId` and +clearing `dead_at`. The per-minute `delivery_dead` counter measures delivery outcomes, not +currently dead registrations. A spike across many hosts warrants checking credentials and topics. + +## Quotas + +Two independent limits, both enforced in the gateway and both returning HTTP 200 with +`status: "rate_limited"` per result rather than failing the request: + +| Limit | Scope | +| --------------------------------------------- | --------------------------------------- | +| 300 logical alerts per rolling 15 minutes | per `hostFingerprint` | +| 300 logical dismissals per rolling 15 minutes | per `hostFingerprint`, separate budget | +| 20 `registrationIds` | per request, hard cap, HTTP 400 over it | + +Fanout to several phones counts one logical event; there is no per-phone daily allowance. +Unauthenticated handshakes and invalid bearer attempts have separate 30/minute IP buckets. +Authenticated routes use a 600/minute host bucket and a shared 6,000/minute client-IP bucket +per instance. The IP budget cannot be reset by generating another host key. It is shared by +clients behind one NAT and is an abuse safeguard, not a global provider-spending cap. Auth database lookup concurrency +and waiting work are bounded independently of HTTP concurrency. + +`push_events` backs quota accounting. `push_event_recipients` deduplicates fanout and +`push_delivery_batches` retains its historical name and persists individual deliveries, worker +leases, retries and outcomes. Identity metadata +is retained for 24 hours. Payloads expire within five minutes and are cleared on completion or by +minute-level expiry cleanup. FCM project-level provider quotas remain independent of host limits. + +Logging is aggregate counters only. Never log a token, a title, a body, or a full fingerprint; +the first four characters of a fingerprint are the most that may appear. + +## DNS: one hand-managed record + +The Cloud Run domain mapping is created here, and Google issues and renews the certificate. The +`onorca.dev` zone is not in this root: it is a Cloudflare zone whose Terraform-managed records +live in the apps root in `stablyai/orca-cloud`, and whose relay and auth records are managed by +hand. The push record follows the relay's precedent and was created by hand on 2026-09-04: + +```text +push.onorca.dev. CNAME ghs.googlehosted.com. (DNS only, not proxied) +``` + +`terraform -chdir=infra/terraform output push_dns_record` prints the same three fields. If the +record is ever lost, recreate it exactly like that; Cloudflare proxying blocks certificate +issuance and breaks Cloud Run host routing. + +### Recovery and delivery guarantees + +Candidate tags and deterministic revision names are recorded before deployment. Promotion intent is +recorded before changing traffic, so a failed verification or ambiguous mutation result still triggers +rollback. A known-good successor must exist before the rejected latest revision can be deleted. +After verified recovery promotion and public checks, rejected and previous consumers are retired; +the recovery revision remains serving. Failed cleanup blocks subsequent rollout admission. +The summary runs even if candidate discovery or traffic verification fails. + +Push uses the relay's schema-startup retry implementation through `@orca-cloud/postgres-schema`. +Session replacement is serialized per host and a unique host index upgrades older databases by +retaining their newest session. Cloud Verify runs push concurrency tests against PostgreSQL. + +Accepted sends commit quota and pending work together before returning `queued`. Workers resume +unfinished deliveries after restarts without relying on desktop retries. The durable queue and +expiring leases coordinate replicas. All provider attempts retain the original five-minute deadline +and respect provider backoff; no retry extends alert life. Silent dismissal messages have their own +quota and cancel matching unsent alerts. Mobile OS delivery/execution is not guaranteed. + +Shutdown stops admission and new claims; unfinished leases remain recoverable. Provider acceptance +and SQL completion cannot be atomic, so repeated transport delivery remains possible after a crash. +Stable per-event replacement identities reduce duplicates without promising exactly-once visible +delivery. FCM notification messages are inherently collapsible while offline and support only a +small number of concurrent collapse keys per device, so excess pending messages may be discarded and +every offline alert is not guaranteed to appear. Socket reconnect reconciles dismissals against the +current native tray; it has no stored replay watermark and never recovers a missed OS banner. + +### Dedicated database operations + +Push has one dedicated database attachment, with stable Terraform addresses and deletion +protection. There is no switch to shared storage. Follow the [database operations runbook](./push-database-cutover.md) +for deployment prerequisites, legacy resource ownership, capacity and recovery. diff --git a/cloud/docs/relay-workflows.md b/cloud/docs/relay-workflows.md index 14bb2a25d7c..54c0a681f2c 100644 --- a/cloud/docs/relay-workflows.md +++ b/cloud/docs/relay-workflows.md @@ -400,3 +400,30 @@ after checkout and authentication, before package installation, revision checks, Their typed confirmations are `PAUSE_REGIONAL_REHOMING` and `DISABLE_REGIONAL_REHOMING`. Keep the default 3,600,000 ms drain grace so existing splices can finish. The job summary contains only fresh aggregate active, receipt, registration, completion, and abort counts. + +## Mobile push gateway + +`Deploy Push Gateway Production` (`.github/workflows/cloud-push-deploy.yml`) is the deploy path +for `orca-cloud-push`, the mobile push gateway. It is the one `cloud-*` workflow that is not a +relay operation, and it is here because it shares the Artifact Registry repository and rollout +lease. Push uses a dedicated Cloud SQL instance. + +It authenticates through `PRODUCTION_GCP_PUSH_DEPLOY_WORKLOAD_IDENTITY_PROVIDER` and +`PRODUCTION_GCP_PUSH_DEPLOY_SERVICE_ACCOUNT`. The dedicated identity has Artifact Registry writer, +Cloud Run developer on the push service, and impersonation of only the push runtime account. +Its provider pins the repository, production environment, main branch, and exact dispatch workflow; +its distinct principal attribute cannot assume the shared Relay deploy account. + +Foundation grants the dedicated account access to the rollout-lock prefix and bucket metadata. +Apply that companion grant and publish the identity outputs before running the workflow. See +[push gateway deployment setup](./push-gateway.md#deploying) for the activation steps. + +The run builds the exact reviewed image digest before taking the lease and rejects images +without validation-mode support using a network-isolated container. Under the lease it boots +an inert, read-only validation revision, checks readiness, mode, scaling and FCM credentials, +then deletes it before deliberately activating the same digest in a new revision. Activation +starts schema writes, pruners and queue consumers before HTTP promotion. Rollback requires +restoring traffic, deleting the rejected active revision, and restoring the service template to +the known-good image in normal mode. The untagged template-recovery revision can run known-good +workers and is retired before lease release; see the +[deployment and rollback contract](./push-gateway.md#deploying). diff --git a/cloud/infra/terraform/environments/production.tfvars b/cloud/infra/terraform/environments/production.tfvars index e1522b3827e..925a171719d 100644 --- a/cloud/infra/terraform/environments/production.tfvars +++ b/cloud/infra/terraform/environments/production.tfvars @@ -412,3 +412,12 @@ relay_region_rehome_source_cell_ids = [ # Slack #orca-relay-alerts, created out of band on 2026-08-05. Declared here because an apply # was otherwise going to strip it from every policy, leaving the alerts firing at nobody. relay_alert_notification_channels = ["projects/onorca-cloud/notificationChannels/4879431412695417284"] + +# Mobile push gateway. Production is the only environment that runs one; the runtime account, +# the three Apple secrets, and their accessor bindings already exist and are imported once +# (see docs/push-gateway.md). +push_gateway_enabled = true +push_base_url = "https://push.onorca.dev" +# Dedicated push pools allow three revision resources during validation and recovery. +push_max_instances = 2 +manage_push_domain_mapping = true diff --git a/cloud/infra/terraform/environments/staging.tfvars b/cloud/infra/terraform/environments/staging.tfvars index 4a32458fcd5..1cb260e11ea 100644 --- a/cloud/infra/terraform/environments/staging.tfvars +++ b/cloud/infra/terraform/environments/staging.tfvars @@ -81,3 +81,6 @@ relay_gce_cells = { } relay_region_rehome_source_cell_ids = ["staging-gce-c2", "staging-gce-c3"] + +# Push is currently provisioned only in production. +push_gateway_enabled = false diff --git a/cloud/infra/terraform/outputs.tf b/cloud/infra/terraform/outputs.tf index 220aa5cf94f..5fd3b647b76 100644 --- a/cloud/infra/terraform/outputs.tf +++ b/cloud/infra/terraform/outputs.tf @@ -189,3 +189,27 @@ output "relay_gce_cell_deployments" { error_message = "relay_gce_fenced_cells may contain only configured relay_gce_cells keys." } } + +output "push_cloud_run_service_uri" { + value = try(google_cloud_run_v2_service.push[0].uri, null) + description = "Default push gateway service URI for pre-domain smoke tests." +} + +output "push_runtime_service_account" { + value = try(google_service_account.push_runtime[0].email, null) + description = "Runtime identity that holds the APNs key and sends through FCM." +} + +output "push_database_name" { + value = try(google_sql_database.push_dedicated[0].name, null) + description = "Database isolated for durable push gateway state." +} + +output "push_dns_record" { + value = var.push_gateway_enabled ? { + name = local.push_fqdn + type = "CNAME" + data = "ghs.googlehosted.com." + } : null + description = "Record the stablyai/orca-cloud apps root must publish in the onorca.dev zone." +} diff --git a/cloud/infra/terraform/push-dedicated-database.tf b/cloud/infra/terraform/push-dedicated-database.tf new file mode 100644 index 00000000000..9a4b9e4c69f --- /dev/null +++ b/cloud/infra/terraform/push-dedicated-database.tf @@ -0,0 +1,111 @@ +resource "google_sql_database_instance" "push_dedicated" { + count = local.push_gateway_count + + project = var.project_id + name = "${var.name_prefix}-push-db" + region = var.region + database_version = "POSTGRES_17" + deletion_protection = true + + settings { + tier = "db-custom-2-7680" + availability_type = "REGIONAL" + edition = "ENTERPRISE" + disk_type = "PD_SSD" + disk_size = 50 + disk_autoresize = true + user_labels = local.relay_shared_labels + + backup_configuration { + enabled = true + point_in_time_recovery_enabled = true + transaction_log_retention_days = 7 + start_time = "05:00" + backup_retention_settings { + retained_backups = 7 + } + } + + ip_configuration { + ipv4_enabled = true + ssl_mode = "ENCRYPTED_ONLY" + } + + maintenance_window { + day = 7 + hour = 6 + update_track = "stable" + } + + deletion_protection_enabled = true + } + + lifecycle { + prevent_destroy = true + } +} + +resource "google_sql_database" "push_dedicated" { + count = local.push_gateway_count + + project = var.project_id + name = "orca_push" + instance = google_sql_database_instance.push_dedicated[0].name + + lifecycle { + prevent_destroy = true + } +} + +resource "random_password" "push_dedicated_database" { + count = local.push_gateway_count + length = 32 + special = false +} + +resource "google_sql_user" "push_dedicated" { + count = local.push_gateway_count + + project = var.project_id + name = "orca_push" + instance = google_sql_database_instance.push_dedicated[0].name + password = random_password.push_dedicated_database[0].result +} + +resource "google_secret_manager_secret" "push_dedicated_database_url" { + count = local.push_gateway_count + + project = var.project_id + secret_id = "${var.name_prefix}-push-dedicated-database-url" + labels = local.relay_shared_labels + + replication { + auto {} + } + + lifecycle { + prevent_destroy = true + } +} + +resource "google_secret_manager_secret_version" "push_dedicated_database_url" { + count = local.push_gateway_count + + secret = google_secret_manager_secret.push_dedicated_database_url[0].id + secret_data = format( + "postgresql://%s:%s@/%s?host=/cloudsql/%s", + google_sql_user.push_dedicated[0].name, + random_password.push_dedicated_database[0].result, + google_sql_database.push_dedicated[0].name, + google_sql_database_instance.push_dedicated[0].connection_name + ) +} + +resource "google_secret_manager_secret_iam_member" "push_dedicated_database_url_accessor" { + count = local.push_gateway_count + + project = var.project_id + secret_id = google_secret_manager_secret.push_dedicated_database_url[0].secret_id + role = "roles/secretmanager.secretAccessor" + member = google_service_account.push_runtime[0].member +} diff --git a/cloud/infra/terraform/push-deploy-identity.tf b/cloud/infra/terraform/push-deploy-identity.tf new file mode 100644 index 00000000000..057c8fb8bee --- /dev/null +++ b/cloud/infra/terraform/push-deploy-identity.tf @@ -0,0 +1,82 @@ +locals { + # Deployment uses its own production-only identity. + push_gateway_deploy_count = ( + var.push_gateway_enabled && local.relay_create_production_ops_identity ? 1 : 0 + ) + github_push_workflow_clauses = [ + for prefix in local.relay_github_workflow_ref_prefixes : + "assertion.workflow_ref == '${prefix}push-deploy.yml@refs/heads/main' && assertion.job_workflow_ref == '${prefix}push-deploy.yml@refs/heads/main'" + ] + push_deploy_member = one(google_service_account.github_push_deploy[*].member) +} + +resource "google_service_account" "github_push_deploy" { + count = local.push_gateway_deploy_count + account_id = "${var.name_prefix}-gha-push" + display_name = "Orca push production deploy" +} + +resource "google_iam_workload_identity_pool_provider" "github_push" { + count = local.push_gateway_deploy_count + + project = var.project_id + workload_identity_pool_id = local.relay_workload_identity_pool_id + workload_identity_pool_provider_id = "github-push-deploy" + display_name = "GitHub push production deploy" + # Do not map attribute.repository: that principal set can assume the shared deploy account. + attribute_mapping = { + "google.subject" = "assertion.sub" + "attribute.push_deploy" = "'production'" + } + attribute_condition = join(" && ", concat(local.relay_github_leading_repository_claims, [ + "assertion.ref == 'refs/heads/main'", + "assertion.environment == 'production'", + "assertion.event_name == 'workflow_dispatch'", + local.relay_github_workflow_conditions["github_push"] + ])) + + oidc { + issuer_uri = "https://token.actions.githubusercontent.com" + } +} + +resource "google_service_account_iam_member" "github_push_workload_identity_user" { + count = local.push_gateway_deploy_count + + service_account_id = google_service_account.github_push_deploy[0].name + role = "roles/iam.workloadIdentityUser" + member = "principalSet://iam.googleapis.com/${local.relay_workload_identity_pool_name}/attribute.push_deploy/production" +} + +resource "google_artifact_registry_repository_iam_member" "github_push_artifact_writer" { + count = local.push_gateway_deploy_count + + project = var.project_id + location = var.region + repository = var.artifact_repository_id + role = "roles/artifactregistry.writer" + member = local.push_deploy_member +} + +output "github_push_workload_identity_provider" { + value = try(google_iam_workload_identity_pool_provider.github_push[0].name, null) +} + +output "github_push_deploy_service_account" { + value = try(google_service_account.github_push_deploy[0].email, null) +} + + +resource "google_storage_bucket_iam_member" "github_push_rollout_lease" { + count = local.push_gateway_deploy_count + + bucket = "${var.project_id}-terraform-state" + role = "roles/storage.objectAdmin" + member = local.push_deploy_member + + condition { + title = "push_rollout_lease" + description = "Limits push deployment coordination to its own lease object." + expression = "resource.name == 'projects/_/buckets/${var.project_id}-terraform-state/objects/terraform/state/push-rollout/production.lock'" + } +} diff --git a/cloud/infra/terraform/push-gateway.tf b/cloud/infra/terraform/push-gateway.tf new file mode 100644 index 00000000000..10164d1911e --- /dev/null +++ b/cloud/infra/terraform/push-gateway.tf @@ -0,0 +1,305 @@ +# Orca mobile push gateway (`cloud/apps/push`). +# +# One public Cloud Run service that holds the APNs key and sends through APNs and FCM V1 on +# behalf of paired phones. Operations: `docs/push-gateway.md`. +# +# There is no staging push gateway by decision, so every resource here is behind +# `var.push_gateway_enabled`, which only `environments/production.tfvars` sets true. The file +# still reads every environment-shaped value from a variable, like the rest of this root, so a +# future staging gateway is a tfvars edit rather than a rewrite. +# +# Several resources below already exist in `onorca-cloud`; they are declared so a plan is clean +# and imported once. `docs/push-gateway.md` carries the exact `terraform import` commands. + +locals { + push_gateway_count = var.push_gateway_enabled ? 1 : 0 + + # The runtime account, the three provider secrets, and their accessor bindings already exist in + # production and were created out of band with the Apple credentials. + push_runtime_service_account_id = "${var.name_prefix}-push" + + # Secret Manager holds the Apple credentials. Terraform owns the secret names, labels, and + # replication; it never owns a version. The `.p8` is issued by the Apple developer portal and + # rotated by `docs/push-gateway.md`, so a Terraform-managed version would either put the key in + # state or fight the rotation. `ignore_changes` on the whole resource is not available, so the + # versions are simply not declared and every consumer reads `latest`. + push_provider_secret_ids = var.push_gateway_enabled ? toset([ + "${var.name_prefix}-push-apns-key", + "${var.name_prefix}-push-apns-key-id", + "${var.name_prefix}-push-apple-team-id" + ]) : toset([]) + + push_provider_secret_env = { + "${var.name_prefix}-push-apns-key" = "ORCA_PUSH_APNS_KEY" + "${var.name_prefix}-push-apns-key-id" = "ORCA_PUSH_APNS_KEY_ID" + "${var.name_prefix}-push-apple-team-id" = "ORCA_PUSH_APPLE_TEAM_ID" + } + + push_fqdn = replace(replace(var.push_base_url, "https://", ""), "http://", "") + + +} + +# --- Runtime identity --------------------------------------------------------------------- + +resource "google_service_account" "push_runtime" { + count = local.push_gateway_count + + project = var.project_id + account_id = local.push_runtime_service_account_id + display_name = "Orca mobile push gateway" + description = "Runtime identity for the Orca mobile push gateway; sends through FCM V1." +} + +# FCM V1 sends are authorized by the runtime account's own metadata-server token. +resource "google_project_iam_member" "push_runtime_fcm_admin" { + count = local.push_gateway_count + + project = var.project_id + role = "roles/firebasecloudmessaging.admin" + member = google_service_account.push_runtime[0].member +} + +# The FCM V1 endpoint bills against the caller's project quota, which the caller must consume. +resource "google_project_iam_member" "push_runtime_service_usage_consumer" { + count = local.push_gateway_count + + project = var.project_id + role = "roles/serviceusage.serviceUsageConsumer" + member = google_service_account.push_runtime[0].member +} + +resource "google_project_iam_member" "push_runtime_cloudsql_client" { + count = local.push_gateway_count + + project = var.project_id + role = "roles/cloudsql.client" + member = google_service_account.push_runtime[0].member +} + +# --- Apple credentials ---------------------------------------------------------------------- + +resource "google_secret_manager_secret" "push_provider" { + for_each = local.push_provider_secret_ids + + project = var.project_id + secret_id = each.value + labels = local.relay_shared_labels + + replication { + auto {} + } + + # Why: Apple issues a `.p8` once and Secret Manager has no undelete. Turning the gateway off + # must fail the plan rather than destroy the only copy of the signing key. + lifecycle { + prevent_destroy = true + } +} + +resource "google_secret_manager_secret_iam_member" "push_provider_runtime_accessor" { + for_each = local.push_provider_secret_ids + + project = var.project_id + secret_id = google_secret_manager_secret.push_provider[each.value].secret_id + role = "roles/secretmanager.secretAccessor" + member = google_service_account.push_runtime[0].member +} + +# --- Service -------------------------------------------------------------------------------- + +resource "google_cloud_run_v2_service" "push" { + count = local.push_gateway_count + + project = var.project_id + name = var.push_cloud_run_service_name + location = var.region + ingress = "INGRESS_TRAFFIC_ALL" + # Why: the host proof in `POST /v1/host/challenge` is the authentication, not Cloud Run IAM. + # The project's domain-restricted-sharing policy refuses an `allUsers` invoker binding, so the + # service opts out of invoker IAM exactly as the relay director does. + invoker_iam_disabled = true + deletion_protection = var.environment == "production" + labels = local.relay_shared_labels + + template { + service_account = google_service_account.push_runtime[0].email + timeout = "${var.push_request_timeout_seconds}s" + max_instance_request_concurrency = var.push_concurrency + + scaling { + min_instance_count = var.push_min_instances + max_instance_count = var.push_max_instances + } + + volumes { + name = "cloudsql" + + cloud_sql_instance { + instances = [google_sql_database_instance.push_dedicated[0].connection_name] + } + } + + containers { + image = var.push_cloud_run_image + + ports { + container_port = 8080 + } + + volume_mounts { + name = "cloudsql" + mount_path = "/cloudsql" + } + + env { + name = "ORCA_PUSH_PUBLIC_URL" + value = var.push_base_url + } + + env { + name = "ORCA_PUSH_FCM_PROJECT_ID" + value = var.project_id + } + + # Bound the declared pool against the dedicated database rollout budget. + env { + name = "ORCA_PUSH_DATABASE_POOL_MAX" + value = tostring(var.push_database_pool_max) + } + + env { + name = "ORCA_PUSH_DATABASE_URL" + + value_source { + secret_key_ref { + secret = google_secret_manager_secret.push_dedicated_database_url[0].secret_id + version = google_secret_manager_secret_version.push_dedicated_database_url[0].version + } + } + } + + # Rotation adds a new version and redeploys; `latest` is what the redeploy picks up. + dynamic "env" { + for_each = local.push_provider_secret_env + + content { + name = env.value + + value_source { + secret_key_ref { + secret = google_secret_manager_secret.push_provider[env.key].secret_id + version = "latest" + } + } + } + } + + resources { + limits = { + cpu = var.push_cloud_run_cpu + memory = var.push_cloud_run_memory + } + + cpu_idle = false + } + + startup_probe { + failure_threshold = 12 + initial_delay_seconds = 0 + period_seconds = 5 + timeout_seconds = 2 + + http_get { + path = "/health" + port = 8080 + } + } + } + } + + # Deploys update the immutable image and shift traffic; Terraform owns the shape and IAM. + # + # `traffic` is ignored as well as the image. A deploy ends with traffic pinned to an exact + # revision and a rollback pins it to the previous one; an apply that reset the service to + # 100% LATEST would silently undo either, and this root carries unrelated standing drift, so + # that apply need not be a push change at all. + lifecycle { + precondition { + condition = var.push_max_instances * var.push_database_pool_max * 3 <= 64 + error_message = "Dedicated push serving, validation/rejected and successor pools must fit the 64-connection rollout budget." + } + + ignore_changes = [ + client, + client_version, + template[0].containers[0].image, + traffic + ] + } + + depends_on = [ + data.google_artifact_registry_repository.relay_images, + google_project_iam_member.push_runtime_cloudsql_client, + google_secret_manager_secret_iam_member.push_provider_runtime_accessor, + google_secret_manager_secret_version.push_dedicated_database_url, + google_secret_manager_secret_iam_member.push_dedicated_database_url_accessor + ] +} + +# Google issues and renews the certificate for the mapping. The DNS record itself is a +# hand-managed Cloudflare CNAME to ghs.googlehosted.com, like relay.onorca.dev; this root has no +# Cloudflare surface by design. `terraform output push_dns_record` prints the record. +resource "google_cloud_run_domain_mapping" "push" { + count = var.push_gateway_enabled && var.manage_push_domain_mapping ? 1 : 0 + + location = var.region + name = local.push_fqdn + + metadata { + namespace = var.project_id + } + + spec { + route_name = google_cloud_run_v2_service.push[0].name + } + + # Same reason as relay-dns.tf: a gcloud-created mapping reports an empty legacy + # certificate_mode, and replacing it would reset issuance for no behavioral change. + lifecycle { + ignore_changes = [spec[0].certificate_mode] + } +} + +# --- Deploy identity grants ------------------------------------------------------------------- +# Push deploy authority is isolated from Relay; foundation grants its rollout-lock access. + +resource "google_cloud_run_v2_service_iam_member" "github_production_push_developer" { + count = local.push_gateway_deploy_count + + project = var.project_id + location = var.region + name = google_cloud_run_v2_service.push[0].name + role = "roles/run.developer" + member = local.push_deploy_member +} + +resource "google_service_account_iam_member" "github_production_push_runtime_user" { + count = local.push_gateway_deploy_count + + service_account_id = google_service_account.push_runtime[0].name + role = "roles/iam.serviceAccountUser" + member = local.push_deploy_member +} + +# Why: the deploy workflow's validate-only FCM send has to exercise the credential the gateway +# will actually use. Impersonating the runtime account proves its firebasecloudmessaging grant; +# granting the deploy account FCM admin outright would prove nothing about the runtime account +# and would widen a project-level role on the shared identity. +resource "google_service_account_iam_member" "github_production_push_runtime_token_creator" { + count = local.push_gateway_deploy_count + + service_account_id = google_service_account.push_runtime[0].name + role = "roles/iam.serviceAccountTokenCreator" + member = local.push_deploy_member +} diff --git a/cloud/infra/terraform/relay-github-actions.tf b/cloud/infra/terraform/relay-github-actions.tf index 450ea64cc0a..186d8942f3d 100644 --- a/cloud/infra/terraform/relay-github-actions.tf +++ b/cloud/infra/terraform/relay-github-actions.tf @@ -20,6 +20,7 @@ locals { "deploy-relay-production.yml", "operate-relay-asia-admission.yml", "publish-relay-production.yml" + ] github_production_relay_capacity_workflow_file = "deploy-relay-production-capacity.yml" github_production_relay_capacity_job_workflow_file = "deploy-relay-production-capacity-job.yml" diff --git a/cloud/infra/terraform/relay-github-workflow-trust.tf b/cloud/infra/terraform/relay-github-workflow-trust.tf index 9cf57f05198..431408f0d9a 100644 --- a/cloud/infra/terraform/relay-github-workflow-trust.tf +++ b/cloud/infra/terraform/relay-github-workflow-trust.tf @@ -10,6 +10,7 @@ locals { relay_github_workflow_clauses = { github = local.github_production_relay_workflow_clauses + github_push = local.github_push_workflow_clauses github_monitor = local.github_monitor_workflow_clauses github_fence = local.github_fence_workflow_clauses github_production_relay_capacity = local.github_production_relay_capacity_workflow_clauses diff --git a/cloud/infra/terraform/variables.tf b/cloud/infra/terraform/variables.tf index 57d73fe75b4..b4360bb6a8a 100644 --- a/cloud/infra/terraform/variables.tf +++ b/cloud/infra/terraform/variables.tf @@ -484,3 +484,95 @@ variable "relay_gce_cloud_sql_proxy_image" { error_message = "relay_gce_cloud_sql_proxy_image must be pinned by sha256 digest." } } + +# --- Mobile push gateway --------------------------------------------------------------------- +# There is no staging push gateway by decision, so this defaults false and only +# environments/production.tfvars turns it on. Everything in push-gateway.tf is behind it. +variable "push_gateway_enabled" { + type = bool + description = "Create the Orca mobile push gateway, its database, secrets, and identity." + default = false +} + +variable "push_base_url" { + type = string + description = "Public TLS origin of the mobile push gateway." + default = "https://push.onorca.dev" + + validation { + condition = can(regex("^https://[^/]+$", var.push_base_url)) + error_message = "push_base_url must be an HTTPS origin with no path." + } +} + +variable "push_cloud_run_service_name" { + type = string + description = "Cloud Run service name for the mobile push gateway." + default = "orca-cloud-push" +} + +variable "push_cloud_run_image" { + type = string + description = "Initial image for the Terraform-created push gateway service; deploys own it after." + default = "us-docker.pkg.dev/cloudrun/container/hello" +} + +variable "push_cloud_run_cpu" { + type = string + description = "CPU limit for the push gateway container." + default = "1" +} + +variable "push_cloud_run_memory" { + type = string + description = "Memory limit for the push gateway container." + default = "512Mi" +} + +# Keep a warm instance to run durable delivery retries without incoming requests. +variable "push_min_instances" { + type = number + description = "Minimum instances for the push gateway." + default = 1 +} + +variable "push_max_instances" { + type = number + description = "Maximum instances for the push gateway." + default = 4 + + validation { + condition = var.push_max_instances >= 1 + error_message = "The push gateway needs at least one instance." + } +} + +# The dedicated database budget counts pools across all three rollout revision resources. +variable "push_database_pool_max" { + type = number + description = "Push gateway database pool size per instance; instances x pool is its Cloud SQL draw." + default = 2 + + validation { + condition = var.push_database_pool_max >= 1 && var.push_database_pool_max <= 100 + error_message = "The push gateway pool must hold at least one connection and stay under the per-service bound." + } +} + +variable "push_concurrency" { + type = number + description = "Cloud Run concurrency for short-lived push gateway HTTP requests." + default = 80 +} + +variable "push_request_timeout_seconds" { + type = number + description = "Cloud Run timeout for push gateway requests; every route is short-lived." + default = 30 +} + +variable "manage_push_domain_mapping" { + type = bool + description = "Manage the push gateway Cloud Run domain mapping; the DNS record stays in the apps root." + default = false +} diff --git a/cloud/package.json b/cloud/package.json index 242bbbd824c..c2c4b8bd025 100644 --- a/cloud/package.json +++ b/cloud/package.json @@ -21,7 +21,7 @@ "load:relay:recovery-gate": "node dev/scripts/run-relay-recovery-wave-gate.mjs", "ops:relay": "pnpm --filter @orca-cloud/relay-ops dev", "pretest": "node --test dev/scripts/capture-terraform-plan-baseline.test.mjs dev/scripts/operate-relay-asia-admission.test.mjs dev/scripts/prepare-relay-asia-director-cells.test.mjs dev/scripts/prepare-relay-asia-topology-input.test.mjs dev/scripts/production-cloud-sql-rollout-lock.test.mjs dev/scripts/read-relay-serving-regional-placement-version.test.mjs dev/scripts/relay-asia-admission-workflow.test.mjs dev/scripts/relay-asia-rollout-evidence.test.mjs dev/scripts/relay-asia-topology-workflow.test.mjs dev/scripts/relay-cloud-sql-connection-budget.test.mjs dev/scripts/relay-load-reader-evidence.test.mjs dev/scripts/relay-region-hint-metrics.test.mjs dev/scripts/relay-staging-deploy-identity.test.mjs dev/scripts/sanitize-relay-asia-admission-result.test.mjs dev/scripts/terraform-root-partition.test.mjs dev/scripts/validate-relay-asia-topology-plan.test.mjs ../.github/actions/cloud-sql-rollout-lease/action-contract.test.mjs ../.github/actions/cloud-sql-rollout-lease/storage-lease.test.mjs", - "test": "pnpm -r test && node --test dev/scripts/classify-relay-production-capacity-director.test.mjs dev/scripts/classify-relay-staging-bootstrap.test.mjs dev/scripts/deploy-relay-blue-green.test.mjs dev/scripts/deploy-relay-gce-candidate.test.mjs dev/scripts/deploy-relay-gce-multi-target.test.mjs dev/scripts/github-smoke-token.test.mjs dev/scripts/infra.test.mjs dev/scripts/operate-relay-regional-rehome.test.mjs dev/scripts/power-staging-relay.test.mjs dev/scripts/prepare-relay-capacity-canary.test.mjs dev/scripts/prepare-relay-production-capacity-canary.test.mjs dev/scripts/probe-relay-legacy-admission.test.mjs dev/scripts/probe-relay-rehome-trust.test.mjs dev/scripts/production-cell-image-digest-consistency.test.mjs dev/scripts/read-relay-production-capacity-identity.test.mjs dev/scripts/relay-admin-endpoint-retry-workflow.test.mjs dev/scripts/relay-admin-transient-retry.test.mjs dev/scripts/relay-admission-selector.test.mjs dev/scripts/relay-gce-terraform-fence.test.mjs dev/scripts/relay-load-connection-failure.test.mjs dev/scripts/relay-load-control-peer.test.mjs dev/scripts/relay-load-director-capacity-gate.test.mjs dev/scripts/relay-load-model.test.mjs dev/scripts/relay-load-phase-barrier.test.mjs dev/scripts/relay-load-placement-boundary.test.mjs dev/scripts/relay-load-profile.test.mjs dev/scripts/relay-load-rebind-boundary.test.mjs dev/scripts/relay-load-region-behavior.test.mjs dev/scripts/relay-load-request-unit-boundary.test.mjs dev/scripts/relay-load-run-lifecycle.test.mjs dev/scripts/relay-monitor-evidence.test.mjs dev/scripts/relay-production-capacity-wave.test.mjs dev/scripts/relay-production-capacity-workflow.test.mjs dev/scripts/relay-production-identity-boundaries.test.mjs dev/scripts/relay-production-same-cap-wave.test.mjs dev/scripts/relay-public-workflow-contract.test.mjs dev/scripts/relay-recovery-wave-gate.test.mjs dev/scripts/relay-region-observation-evidence.test.mjs dev/scripts/relay-regional-rehome-workflow.test.mjs dev/scripts/relay-rehome-aggregate-evidence.test.mjs dev/scripts/relay-repository.test.mjs dev/scripts/relay-same-cap-script-census.test.mjs dev/scripts/relay-staging-c4-refresh-workflow.test.mjs dev/scripts/relay-staging-capacity-identity.test.mjs dev/scripts/staging-relay-apply-guard.test.mjs dev/scripts/validate-relay-capacity-plan.test.mjs dev/scripts/verify-relay-capacity-transition.test.mjs dev/scripts/verify-relay-legacy-bootstrap.test.mjs dev/scripts/workload-identity-attribute-conditions.test.mjs", + "test": "pnpm -r test && node --test dev/scripts/classify-relay-production-capacity-director.test.mjs dev/scripts/classify-relay-staging-bootstrap.test.mjs dev/scripts/deploy-relay-blue-green.test.mjs dev/scripts/deploy-relay-gce-candidate.test.mjs dev/scripts/deploy-relay-gce-multi-target.test.mjs dev/scripts/github-smoke-token.test.mjs dev/scripts/infra.test.mjs dev/scripts/operate-relay-regional-rehome.test.mjs dev/scripts/power-staging-relay.test.mjs dev/scripts/prepare-relay-capacity-canary.test.mjs dev/scripts/prepare-relay-production-capacity-canary.test.mjs dev/scripts/probe-relay-legacy-admission.test.mjs dev/scripts/probe-relay-rehome-trust.test.mjs dev/scripts/production-cell-image-digest-consistency.test.mjs dev/scripts/push-gateway-workflow.test.mjs dev/scripts/push-gateway-recovery.test.mjs dev/scripts/push-validation-workflow.test.mjs dev/scripts/read-relay-production-capacity-identity.test.mjs dev/scripts/relay-admin-endpoint-retry-workflow.test.mjs dev/scripts/relay-admin-transient-retry.test.mjs dev/scripts/relay-admission-selector.test.mjs dev/scripts/relay-gce-terraform-fence.test.mjs dev/scripts/relay-load-connection-failure.test.mjs dev/scripts/relay-load-control-peer.test.mjs dev/scripts/relay-load-director-capacity-gate.test.mjs dev/scripts/relay-load-model.test.mjs dev/scripts/relay-load-phase-barrier.test.mjs dev/scripts/relay-load-placement-boundary.test.mjs dev/scripts/relay-load-profile.test.mjs dev/scripts/relay-load-rebind-boundary.test.mjs dev/scripts/relay-load-region-behavior.test.mjs dev/scripts/relay-load-request-unit-boundary.test.mjs dev/scripts/relay-load-run-lifecycle.test.mjs dev/scripts/relay-monitor-evidence.test.mjs dev/scripts/relay-production-capacity-wave.test.mjs dev/scripts/relay-production-capacity-workflow.test.mjs dev/scripts/relay-production-identity-boundaries.test.mjs dev/scripts/relay-production-same-cap-wave.test.mjs dev/scripts/relay-public-workflow-contract.test.mjs dev/scripts/relay-recovery-wave-gate.test.mjs dev/scripts/relay-region-observation-evidence.test.mjs dev/scripts/relay-regional-rehome-workflow.test.mjs dev/scripts/relay-rehome-aggregate-evidence.test.mjs dev/scripts/relay-repository.test.mjs dev/scripts/relay-same-cap-script-census.test.mjs dev/scripts/relay-staging-c4-refresh-workflow.test.mjs dev/scripts/relay-staging-capacity-identity.test.mjs dev/scripts/staging-relay-apply-guard.test.mjs dev/scripts/validate-relay-capacity-plan.test.mjs dev/scripts/verify-relay-capacity-transition.test.mjs dev/scripts/verify-relay-legacy-bootstrap.test.mjs dev/scripts/workload-identity-attribute-conditions.test.mjs", "typecheck": "pnpm -r typecheck" }, "devDependencies": { diff --git a/cloud/packages/postgres-schema/package.json b/cloud/packages/postgres-schema/package.json new file mode 100644 index 00000000000..e170973cf2b --- /dev/null +++ b/cloud/packages/postgres-schema/package.json @@ -0,0 +1,20 @@ +{ + "name": "@orca-cloud/postgres-schema", + "version": "0.0.0", + "private": true, + "type": "module", + "main": "dist/index.js", + "types": "dist/index.d.ts", + "scripts": { + "build": "tsc -p tsconfig.build.json", + "clean": "node -e \"require('fs').rmSync('dist', { recursive: true, force: true })\"", + "lint": "tsc -p tsconfig.json --noEmit", + "test": "pnpm build", + "typecheck": "tsc -p tsconfig.json --noEmit" + }, + "devDependencies": { + "@types/node": "^24.10.0", + "typescript": "^5.9.3", + "vitest": "^4.0.8" + } +} diff --git a/cloud/packages/postgres-schema/src/index.ts b/cloud/packages/postgres-schema/src/index.ts new file mode 100644 index 00000000000..9981a03ead5 --- /dev/null +++ b/cloud/packages/postgres-schema/src/index.ts @@ -0,0 +1,113 @@ +const RETRYABLE_SCHEMA_CODES = new Set(['55P03', '57014']) +const DEFAULT_RETRY_DEADLINE_MS = 30_000 +const RETRY_BASE_DELAY_MS = 250 +const RETRY_MAX_DELAY_MS = 2_000 + +type SchemaStartupOptions = { + eventPrefix?: string + now?: () => number + random?: () => number + retryDeadlineMs?: number + wait?: (delayMs: number) => Promise +} + +function retryDelayMs(attempt: number, random: () => number): number { + const ceiling = Math.min(RETRY_BASE_DELAY_MS * 2 ** (attempt - 1), RETRY_MAX_DELAY_MS) + return Math.ceil(ceiling * (0.5 + random() * 0.5)) +} + +function wait(delayMs: number): Promise { + return new Promise((resolve) => setTimeout(resolve, delayMs)) +} + +const CREATE_TABLE_IF_NOT_EXISTS = /^\s*CREATE\s+TABLE\s+IF\s+NOT\s+EXISTS\b/i +const CREATE_INDEX_IF_NOT_EXISTS = /^\s*CREATE\s+(?:UNIQUE\s+)?INDEX\s+IF\s+NOT\s+EXISTS\b/i + +// `IF NOT EXISTS` only checks the name before the catalog inserts, so the loser of a concurrent +// CREATE can fail on the catalog unique index (23505) or, when the winner has already committed by +// the time the loser reaches TypeCreate/heap_create_with_catalog, on the name check those routines +// repeat (42710 duplicate type, 42P07 duplicate relation). Each is a no-op on the next attempt. +function concurrentCreateCollision( + value: { code?: unknown; constraint?: unknown }, + statement: string +): boolean { + if (CREATE_TABLE_IF_NOT_EXISTS.test(statement)) { + return ( + (value.code === '23505' && value.constraint === 'pg_type_typname_nsp_index') || + value.code === '42710' || + value.code === '42P07' + ) + } + if (CREATE_INDEX_IF_NOT_EXISTS.test(statement)) { + return ( + (value.code === '23505' && value.constraint === 'pg_class_relname_nsp_index') || + value.code === '42P07' + ) + } + return false +} + +const ALTER_TABLE_ADD_CONSTRAINT = /^\s*ALTER\s+TABLE\s+\S+\s+ADD\s+CONSTRAINT\b/i + +function constraintAlreadyApplied(error: unknown, statement: string): boolean { + return ( + ALTER_TABLE_ADD_CONSTRAINT.test(statement) && + (error as { code?: unknown }).code === '42710' + ) +} + +function retryableSchemaError(error: unknown, statement: string): boolean { + const value = error as { code?: unknown; constraint?: unknown } + return ( + RETRYABLE_SCHEMA_CODES.has(String(value.code)) || concurrentCreateCollision(value, statement) + ) +} + +export async function applyPostgresSchema( + statements: string[], + query: (statement: string) => Promise, + options: SchemaStartupOptions = {} +): Promise { + const now = options.now ?? Date.now + const random = options.random ?? Math.random + const pause = options.wait ?? wait + const deadlineAt = now() + (options.retryDeadlineMs ?? DEFAULT_RETRY_DEADLINE_MS) + + for (const statement of statements) { + let attempt = 1 + while (true) { + try { + await query(statement) + break + } catch (error) { + if (constraintAlreadyApplied(error, statement)) break + const code = String((error as { code?: unknown }).code) + const remainingMs = deadlineAt - now() + const retryable = retryableSchemaError(error, statement) + if (!retryable || remainingMs <= 0) { + if (retryable) { + console.warn( + JSON.stringify({ + event: `${options.eventPrefix ?? 'orca_relay_postgres_schema'}_retry_exhausted`, + code, + attempts: attempt + }) + ) + } + throw error + } + const delayMs = Math.min(remainingMs, retryDelayMs(attempt, random)) + console.warn( + JSON.stringify({ + event: `${options.eventPrefix ?? 'orca_relay_postgres_schema'}_retry`, + code, + attempt, + delayMs + }) + ) + await pause(delayMs) + attempt += 1 + } + } + } +} diff --git a/cloud/packages/postgres-schema/tsconfig.build.json b/cloud/packages/postgres-schema/tsconfig.build.json new file mode 100644 index 00000000000..94c84b60803 --- /dev/null +++ b/cloud/packages/postgres-schema/tsconfig.build.json @@ -0,0 +1,11 @@ +{ + "extends": "./tsconfig.json", + "compilerOptions": { + "declaration": true, + "emitDeclarationOnly": false, + "noEmit": false, + "outDir": "dist", + "rootDir": "src" + }, + "exclude": ["src/**/*.test.ts"] +} diff --git a/cloud/packages/postgres-schema/tsconfig.json b/cloud/packages/postgres-schema/tsconfig.json new file mode 100644 index 00000000000..a552e34dbe9 --- /dev/null +++ b/cloud/packages/postgres-schema/tsconfig.json @@ -0,0 +1,5 @@ +{ + "extends": "../../tsconfig.base.json", + "compilerOptions": { "noEmit": true }, + "include": ["src/**/*.ts"] +} diff --git a/cloud/packages/push-contract/package.json b/cloud/packages/push-contract/package.json new file mode 100644 index 00000000000..072b5e7193f --- /dev/null +++ b/cloud/packages/push-contract/package.json @@ -0,0 +1,23 @@ +{ + "name": "@orca-cloud/push-contract", + "private": true, + "version": "0.0.0", + "type": "module", + "main": "dist/index.js", + "types": "dist/index.d.ts", + "scripts": { + "build": "pnpm clean && tsc -p tsconfig.build.json", + "clean": "node -e \"require('fs').rmSync('dist', { recursive: true, force: true })\"", + "lint": "tsc -p tsconfig.json --noEmit", + "test": "vitest run", + "typecheck": "tsc -p tsconfig.json --noEmit" + }, + "dependencies": { + "zod": "^3.25.76" + }, + "devDependencies": { + "@types/node": "^24.10.0", + "typescript": "^5.9.3", + "vitest": "^4.0.8" + } +} diff --git a/cloud/packages/push-contract/src/apns-token-length.test.ts b/cloud/packages/push-contract/src/apns-token-length.test.ts new file mode 100644 index 00000000000..d85413a22ef --- /dev/null +++ b/cloud/packages/push-contract/src/apns-token-length.test.ts @@ -0,0 +1,26 @@ +import { expect, it } from 'vitest' +import { PushDeviceRegistrationRequestSchema } from './device-registration-messages.js' + +const registration = (token: string) => ({ + v: 1, + deviceId: 'qa-device', + platform: 'ios', + token, + apnsEnvironment: 'sandbox' +}) + +it.each([32, 64, 160, 256])( + 'accepts variable-length APNs device tokens (%i hex characters)', + (length) => { + expect( + PushDeviceRegistrationRequestSchema.safeParse(registration('aB'.repeat(length / 2))).success + ).toBe(true) + } +) + +it.each(['', 'abc', 'not-hex', 'ab cd', 'ab'.repeat(2049)])( + 'rejects malformed or oversized APNs tokens', + (token) => { + expect(PushDeviceRegistrationRequestSchema.safeParse(registration(token)).success).toBe(false) + } +) diff --git a/cloud/packages/push-contract/src/contract.test.ts b/cloud/packages/push-contract/src/contract.test.ts new file mode 100644 index 00000000000..edfb915e72d --- /dev/null +++ b/cloud/packages/push-contract/src/contract.test.ts @@ -0,0 +1,162 @@ +import { describe, expect, it } from 'vitest' +import { + ApnsEnvironmentSchema, + PushDeviceRegistrationRequestSchema +} from './device-registration-messages.js' +import { + PushHostChallengeRequestSchema, + PushHostChallengeResponseSchema, + PushHostSessionRequestSchema, + PushHostSessionResponseSchema +} from './host-auth-messages.js' +import { PUSH_DEFAULTS, PUSH_LIMITS } from './push-limits.js' + +const KEY_B64 = Buffer.alloc(32, 1).toString('base64') +const NONCE_B64 = Buffer.alloc(24, 2).toString('base64') +const SESSION_TOKEN = Buffer.alloc(32, 3).toString('base64url') +const FINGERPRINT = 'abcdefghijklmnop' +const APNS_TOKEN = 'a'.repeat(64) +const FCM_TOKEN = 'cQ1abcDEF_gh:APA91bZZ-zz0123456789abcdefghijklmnopqrstuvwxyz' + +function notification(): Record { + return { + notificationId: 'note-1', + notificationSeq: 4, + notificationEpoch: '5c9e9a1e-0000-4000-8000-000000000000', + source: 'agent-task-complete', + agentState: 'needs-input', + title: 'Agent needs input', + body: 'Waiting on your answer', + worktreeId: 'wt-1' + } +} + +describe('push contract limits', () => { + it('locks the normative limits the desktop and gateway both assume', () => { + expect(PUSH_LIMITS).toMatchObject({ + titleMaxChars: 80, + bodyMaxChars: 180, + maxRegistrationIdsPerSend: 20, + maxDevicesPerHost: 64, + hostEventsPerWindow: 300, + eventQuotaWindowMs: 900_000, + challengeTtlMs: 10_000, + clockSkewToleranceMs: 30_000, + sessionTtlMs: 86_400_000, + notificationTtlSeconds: 300, + unauthenticatedRequestsPerMinutePerIp: 30, + authenticatedRequestsPerMinutePerIp: 6_000, + authenticatedRequestsPerMinutePerHost: 600 + }) + expect(PUSH_DEFAULTS.apnsTopic).toBe('com.stably.orca.mobile') + expect(PUSH_DEFAULTS.androidChannelId).toBe('orca-desktop') + }) +}) + +describe('host authentication schemas', () => { + it('accepts a well formed challenge round trip', () => { + expect( + PushHostChallengeRequestSchema.safeParse({ v: 1, hostPublicKeyB64: KEY_B64 }).success + ).toBe(true) + expect( + PushHostChallengeResponseSchema.safeParse({ + challengeId: 'challenge-1', + gatewayEphemeralPublicKeyB64: KEY_B64, + nonceB64: NONCE_B64, + ciphertextB64: Buffer.alloc(96, 5).toString('base64'), + expiresAt: 1_700_000_010_000 + }).success + ).toBe(true) + expect( + PushHostSessionRequestSchema.safeParse({ + v: 1, + challengeId: 'challenge-1', + proofB64: KEY_B64 + }).success + ).toBe(true) + expect( + PushHostSessionResponseSchema.safeParse({ + sessionToken: SESSION_TOKEN, + expiresAt: 1_700_086_400_000, + hostFingerprint: FINGERPRINT + }).success + ).toBe(true) + }) + + it('rejects unknown keys, wrong versions, and mis-sized keys', () => { + expect( + PushHostChallengeRequestSchema.safeParse({ + v: 1, + hostPublicKeyB64: KEY_B64, + extra: true + }).success + ).toBe(false) + expect( + PushHostChallengeRequestSchema.safeParse({ v: 2, hostPublicKeyB64: KEY_B64 }).success + ).toBe(false) + expect( + PushHostChallengeRequestSchema.safeParse({ + v: 1, + hostPublicKeyB64: Buffer.alloc(31, 1).toString('base64') + }).success + ).toBe(false) + expect( + PushHostSessionResponseSchema.safeParse({ + sessionToken: SESSION_TOKEN, + expiresAt: 1_700_086_400_000, + hostFingerprint: 'short' + }).success + ).toBe(false) + }) +}) + +describe('device registration schemas', () => { + it('requires an apns environment and a hex token for ios', () => { + expect( + PushDeviceRegistrationRequestSchema.safeParse({ + v: 1, + deviceId: 'device-1', + platform: 'ios', + token: APNS_TOKEN, + apnsEnvironment: 'sandbox' + }).success + ).toBe(true) + expect( + PushDeviceRegistrationRequestSchema.safeParse({ + v: 1, + deviceId: 'device-1', + platform: 'ios', + token: APNS_TOKEN + }).success + ).toBe(false) + expect( + PushDeviceRegistrationRequestSchema.safeParse({ + v: 1, + deviceId: 'device-1', + platform: 'ios', + token: 'not-hex', + apnsEnvironment: 'production' + }).success + ).toBe(false) + }) + + it('rejects an apns environment on android and accepts an fcm token', () => { + expect( + PushDeviceRegistrationRequestSchema.safeParse({ + v: 1, + deviceId: 'device-2', + platform: 'android', + token: FCM_TOKEN + }).success + ).toBe(true) + expect( + PushDeviceRegistrationRequestSchema.safeParse({ + v: 1, + deviceId: 'device-2', + platform: 'android', + token: FCM_TOKEN, + apnsEnvironment: 'sandbox' + }).success + ).toBe(false) + }) +}) diff --git a/cloud/packages/push-contract/src/device-registration-messages.ts b/cloud/packages/push-contract/src/device-registration-messages.ts new file mode 100644 index 00000000000..37af2cc9d68 --- /dev/null +++ b/cloud/packages/push-contract/src/device-registration-messages.ts @@ -0,0 +1,74 @@ +import { z } from 'zod' +import { OpaqueIdSchema } from './wire-scalars.js' + +export const PushPlatformSchema = z.enum(['ios', 'android']) +export const ApnsEnvironmentSchema = z.enum(['sandbox', 'production']) +export const PushNotificationSourceSchema = z.enum([ + 'agent-task-complete', + 'terminal-bell', + 'plugin' +]) +export const PushAgentStateSchema = z.enum(['needs-input', 'finished']) + +// APNs tokens are variable-length byte strings, including longer simulator tokens. +const APNS_TOKEN_PATTERN = /^(?:[0-9a-fA-F]{2})+$/ +const FCM_TOKEN_PATTERN = /^[A-Za-z0-9_:.\-]{32,4096}$/ + +export const PushDeviceRegistrationRequestSchema = z + .object({ + v: z.literal(1), + deviceId: OpaqueIdSchema, + platform: PushPlatformSchema, + token: z.string().min(1).max(4096), + apnsEnvironment: ApnsEnvironmentSchema.optional() + }) + .strict() + .superRefine((value, context) => { + if (value.platform === 'ios') { + if (value.apnsEnvironment === undefined) { + context.addIssue({ + code: 'custom', + path: ['apnsEnvironment'], + message: 'apnsEnvironment is required for ios' + }) + } + if (!APNS_TOKEN_PATTERN.test(value.token)) { + context.addIssue({ + code: 'custom', + path: ['token'], + message: 'ios token must be hex-encoded bytes' + }) + } + return + } + if (value.apnsEnvironment !== undefined) { + context.addIssue({ + code: 'custom', + path: ['apnsEnvironment'], + message: 'apnsEnvironment is ios only' + }) + } + if (!FCM_TOKEN_PATTERN.test(value.token)) { + context.addIssue({ + code: 'custom', + path: ['token'], + message: 'android token must be an FCM registration string' + }) + } + }) + +export const PushDeviceSummarySchema = z + .object({ + registrationId: OpaqueIdSchema, + deviceId: OpaqueIdSchema, + platform: PushPlatformSchema, + dead: z.boolean() + }) + .strict() + +export type PushPlatform = z.infer +export type ApnsEnvironment = z.infer +export type PushNotificationSource = z.infer +export type PushAgentState = z.infer +export type PushDeviceRegistrationRequest = z.infer +export type PushDeviceSummary = z.infer diff --git a/cloud/packages/push-contract/src/host-auth-messages.ts b/cloud/packages/push-contract/src/host-auth-messages.ts new file mode 100644 index 00000000000..deb9ffbfb42 --- /dev/null +++ b/cloud/packages/push-contract/src/host-auth-messages.ts @@ -0,0 +1,41 @@ +import { z } from 'zod' +import { + Base6432ByteSchema, + Base64Raw24ByteSchema, + Base64Url32ByteSchema, + BoundedCiphertextSchema, + EpochMsSchema, + OpaqueIdSchema, + PushHostFingerprintSchema +} from './wire-scalars.js' + +export const PushHostChallengeRequestSchema = z + .object({ v: z.literal(1), hostPublicKeyB64: Base6432ByteSchema }) + .strict() + +export const PushHostChallengeResponseSchema = z + .object({ + challengeId: OpaqueIdSchema, + gatewayEphemeralPublicKeyB64: Base6432ByteSchema, + nonceB64: Base64Raw24ByteSchema, + ciphertextB64: BoundedCiphertextSchema, + expiresAt: EpochMsSchema + }) + .strict() + +export const PushHostSessionRequestSchema = z + .object({ v: z.literal(1), challengeId: OpaqueIdSchema, proofB64: Base6432ByteSchema }) + .strict() + +export const PushHostSessionResponseSchema = z + .object({ + sessionToken: Base64Url32ByteSchema, + expiresAt: EpochMsSchema, + hostFingerprint: PushHostFingerprintSchema + }) + .strict() + +export type PushHostChallengeRequest = z.infer +export type PushHostChallengeResponse = z.infer +export type PushHostSessionRequest = z.infer +export type PushHostSessionResponse = z.infer diff --git a/cloud/packages/push-contract/src/index.ts b/cloud/packages/push-contract/src/index.ts new file mode 100644 index 00000000000..3bd8a871f28 --- /dev/null +++ b/cloud/packages/push-contract/src/index.ts @@ -0,0 +1,6 @@ +export * from './device-registration-messages.js' +export * from './host-auth-messages.js' +export * from './push-host-proof-transcript.js' +export * from './push-limits.js' +export * from './send-messages.js' +export * from './wire-scalars.js' diff --git a/cloud/packages/push-contract/src/notification-identity-limits.test.ts b/cloud/packages/push-contract/src/notification-identity-limits.test.ts new file mode 100644 index 00000000000..e19fd93140a --- /dev/null +++ b/cloud/packages/push-contract/src/notification-identity-limits.test.ts @@ -0,0 +1,32 @@ +import { expect, it } from 'vitest' +import { PushNotificationSchema } from './send-messages.js' +const base = { + source: 'agent-task-complete', + agentState: 'finished', + notificationSeq: 1, + notificationEpoch: 'epoch', + title: 'Done', + body: '' +} +it.each([ + 'repo::/Users/developer/orca/workspaces/monorepo/packages/desktop/integrations/feature-mobile-background-notifications', + 'repo::C:\\Users\\developer\\Documents\\projects\\monorepo\\packages\\desktop\\feature-mobile-notifications', + 'folder::/home/developer/projects/通知/作業ディレクトリ/機能', + 'ssh:host::/home/developer/workspaces/monorepo/packages/desktop/feature-mobile-background-notifications' +])('preserves long desktop identities: %s', (path) => { + const worktreeId = `12345678-1234-1234-1234-123456789012::${path}` + const notificationId = [ + 'agent', + encodeURIComponent(worktreeId), + encodeURIComponent('12345678-1234-1234-1234-123456789012:87654321-4321-4321-4321-210987654321'), + '1780000000123' + ].join(':') + const result = PushNotificationSchema.parse({ ...base, worktreeId, notificationId }) + expect(result.worktreeId).toBe(worktreeId) + expect(result.notificationId).toBe(notificationId) +}) +it('rejects oversized provider data by UTF-8 bytes instead of truncating identities', () => { + expect(PushNotificationSchema.safeParse({ ...base, worktreeId: '界'.repeat(1100) }).success).toBe( + false + ) +}) diff --git a/cloud/packages/push-contract/src/push-host-proof-transcript.test.ts b/cloud/packages/push-contract/src/push-host-proof-transcript.test.ts new file mode 100644 index 00000000000..34423beaf6d --- /dev/null +++ b/cloud/packages/push-contract/src/push-host-proof-transcript.test.ts @@ -0,0 +1,106 @@ +import { describe, expect, it } from 'vitest' +import { + buildPushHostChallengePlaintext, + buildPushHostProofMacInput, + buildPushHostProofTranscript, + PUSH_HOST_CHALLENGE_PLAINTEXT_DOMAIN, + PUSH_HOST_PROOF_TRANSCRIPT_DOMAIN, + PUSH_HOST_PROOF_TRANSCRIPT_FIELD_COUNT +} from './push-host-proof-transcript.js' +import { PUSH_LIMITS } from './push-limits.js' + +const transcriptInput = { + gatewayOrigin: 'https://push.onorca.dev', + gatewayEphemeralPublicKey: new Uint8Array(32).fill(7), + challengeNonce: new Uint8Array(24).fill(9), + challengeId: 'challenge-1', + issuedAt: 1_700_000_000_000, + expiresAt: 1_700_000_000_000 + PUSH_LIMITS.challengeTtlMs, + hostFingerprint: 'abcdefghijklmnop', + hostPublicKey: new Uint8Array(32).fill(4) +} + +describe('push host proof transcript', () => { + it('is deterministic and order dependent', () => { + const first = buildPushHostProofTranscript(transcriptInput) + const second = buildPushHostProofTranscript({ ...transcriptInput }) + expect(Buffer.from(first).equals(Buffer.from(second))).toBe(true) + const different = buildPushHostProofTranscript({ + ...transcriptInput, + challengeId: 'challenge-2' + }) + expect(Buffer.from(first).equals(Buffer.from(different))).toBe(false) + }) + + it('encodes exactly the ten specified fields in order', () => { + const transcript = buildPushHostProofTranscript(transcriptInput) + const view = new DataView(transcript.buffer, transcript.byteOffset, transcript.byteLength) + const names: string[] = [] + let offset = 0 + while (offset < transcript.byteLength) { + const nameLength = view.getUint32(offset, false) + offset += 4 + names.push(Buffer.from(transcript.slice(offset, offset + nameLength)).toString('utf8')) + offset += nameLength + offset += 4 + view.getUint32(offset, false) + } + expect(names).toEqual([ + 'protocol', + 'version', + 'gatewayOrigin', + 'gatewayEphemeralPublicKey', + 'challengeNonce', + 'challengeId', + 'issuedAt', + 'expiresAt', + 'hostFingerprint', + 'hostPublicKey' + ]) + expect(names).toHaveLength(PUSH_HOST_PROOF_TRANSCRIPT_FIELD_COUNT) + expect(offset).toBe(transcript.byteLength) + }) + + it('rejects mis-sized key material', () => { + expect(() => + buildPushHostProofTranscript({ + ...transcriptInput, + hostPublicKey: new Uint8Array(31) + }) + ).toThrow('hostPublicKey must be 32 bytes') + expect(() => + buildPushHostProofTranscript({ ...transcriptInput, challengeNonce: new Uint8Array(23) }) + ).toThrow('challengeNonce must be 24 bytes') + }) + + it('frames the challenge plaintext as domain, length, transcript, secret', () => { + const transcript = buildPushHostProofTranscript(transcriptInput) + const secret = new Uint8Array(32).fill(11) + const plaintext = buildPushHostChallengePlaintext(transcript, secret) + const domain = Buffer.from(`${PUSH_HOST_CHALLENGE_PLAINTEXT_DOMAIN}\0`, 'utf8') + expect(Buffer.from(plaintext.slice(0, domain.byteLength)).equals(domain)).toBe(true) + const declared = new DataView( + plaintext.buffer, + plaintext.byteOffset + domain.byteLength, + 4 + ).getUint32(0, false) + expect(declared).toBe(transcript.byteLength) + expect(plaintext.byteLength).toBe(domain.byteLength + 4 + transcript.byteLength + 32) + expect( + Buffer.from(plaintext.slice(plaintext.byteLength - 32)).equals(Buffer.from(secret)) + ).toBe(true) + expect(() => buildPushHostChallengePlaintext(transcript, new Uint8Array(16))).toThrow( + 'challengeSecret must be 32 bytes' + ) + }) + + it('separates the ack mac input from the challenge domain', () => { + const transcript = buildPushHostProofTranscript(transcriptInput) + const macInput = buildPushHostProofMacInput(transcript) + expect(Buffer.from(macInput).toString('utf8')).toContain( + `${PUSH_HOST_PROOF_TRANSCRIPT_DOMAIN}\0ack\0` + ) + expect(macInput.byteLength).toBe( + Buffer.byteLength(`${PUSH_HOST_PROOF_TRANSCRIPT_DOMAIN}\0ack\0`) + transcript.byteLength + ) + }) +}) diff --git a/cloud/packages/push-contract/src/push-host-proof-transcript.ts b/cloud/packages/push-contract/src/push-host-proof-transcript.ts new file mode 100644 index 00000000000..a375b18ca76 --- /dev/null +++ b/cloud/packages/push-contract/src/push-host-proof-transcript.ts @@ -0,0 +1,90 @@ +const textEncoder = new TextEncoder() + +export const PUSH_HOST_PROOF_TRANSCRIPT_DOMAIN = 'orca-push-host-proof/v1' +export const PUSH_HOST_CHALLENGE_PLAINTEXT_DOMAIN = 'orca-push-host-challenge/v1' +export const PUSH_HOST_CHALLENGE_BOX_ALGORITHM = 'Curve25519-XSalsa20-Poly1305' +export const PUSH_HOST_PROOF_ALGORITHM = 'HMAC-SHA-256' + +export interface PushHostProofTranscriptInput { + gatewayOrigin: string + gatewayEphemeralPublicKey: Uint8Array + challengeNonce: Uint8Array + challengeId: string + issuedAt: number + expiresAt: number + hostFingerprint: string + hostPublicKey: Uint8Array +} + +export const PUSH_HOST_PROOF_TRANSCRIPT_FIELD_COUNT = 10 + +function uint32(value: number): Uint8Array { + const bytes = new Uint8Array(4) + new DataView(bytes.buffer).setUint32(0, value, false) + return bytes +} + +function uint64(value: number): Uint8Array { + const bytes = new Uint8Array(8) + new DataView(bytes.buffer).setBigUint64(0, BigInt(value), false) + return bytes +} + +function concat(parts: readonly Uint8Array[]): Uint8Array { + const output = new Uint8Array(parts.reduce((total, part) => total + part.byteLength, 0)) + let offset = 0 + for (const part of parts) { + output.set(part, offset) + offset += part.byteLength + } + return output +} + +function field(name: string, value: Uint8Array): Uint8Array { + const encodedName = textEncoder.encode(name) + return concat([uint32(encodedName.byteLength), encodedName, uint32(value.byteLength), value]) +} + +function text(value: string): Uint8Array { + return textEncoder.encode(value) +} + +function requireByteLength(value: Uint8Array, expected: number, name: string): void { + if (value.byteLength !== expected) throw new Error(`${name} must be ${expected} bytes`) +} + +export function buildPushHostProofTranscript(input: PushHostProofTranscriptInput): Uint8Array { + requireByteLength(input.gatewayEphemeralPublicKey, 32, 'gatewayEphemeralPublicKey') + requireByteLength(input.challengeNonce, 24, 'challengeNonce') + requireByteLength(input.hostPublicKey, 32, 'hostPublicKey') + return concat([ + field('protocol', text(PUSH_HOST_PROOF_TRANSCRIPT_DOMAIN)), + field('version', new Uint8Array([1])), + field('gatewayOrigin', text(input.gatewayOrigin)), + field('gatewayEphemeralPublicKey', input.gatewayEphemeralPublicKey), + field('challengeNonce', input.challengeNonce), + field('challengeId', text(input.challengeId)), + field('issuedAt', uint64(input.issuedAt)), + field('expiresAt', uint64(input.expiresAt)), + field('hostFingerprint', text(input.hostFingerprint)), + field('hostPublicKey', input.hostPublicKey) + ]) +} + +export function buildPushHostChallengePlaintext( + transcript: Uint8Array, + challengeSecret: Uint8Array +): Uint8Array { + if (challengeSecret.byteLength !== 32) throw new Error('challengeSecret must be 32 bytes') + // Why: the encrypted random secret makes the public transcript insufficient to forge the ack. + return concat([ + text(`${PUSH_HOST_CHALLENGE_PLAINTEXT_DOMAIN}\0`), + uint32(transcript.byteLength), + transcript, + challengeSecret + ]) +} + +export function buildPushHostProofMacInput(transcript: Uint8Array): Uint8Array { + return concat([text(`${PUSH_HOST_PROOF_TRANSCRIPT_DOMAIN}\0ack\0`), transcript]) +} diff --git a/cloud/packages/push-contract/src/push-host-proof-vector.json b/cloud/packages/push-contract/src/push-host-proof-vector.json new file mode 100644 index 00000000000..128eba46980 --- /dev/null +++ b/cloud/packages/push-contract/src/push-host-proof-vector.json @@ -0,0 +1,16 @@ +{ + "hostSecretKeyB64": "BwcHBwcHBwcHBwcHBwcHBwcHBwcHBwcHBwcHBwcHBwc=", + "hostPublicKeyB64": "E75P6uryBMf9M1j8nAByGIHRdCeBKCJ+xnTzf3/pe20=", + "hostFingerprint": "D20lU_8MD0R64gLt", + "gatewayOrigin": "https://push.onorca.dev", + "challenge": { + "challengeId": "vector-challenge-1", + "gatewayEphemeralPublicKeyB64": "V9tLNZ8jrl4Ubk4lEgVnBHIlBjSMFQwUdT0Mkz0E1CE=", + "nonceB64": "AwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMD", + "ciphertextB64": "znNOCR0fq0KKa5dwfTAwbhE6GmfC4TUjgB5n+/0BXrrG0A9oKjo38uvUY3VoBvTfCvlkLOmI2bu8kGN/yAHmMz6jhY77FIztAywVQ1WfBlu/tbxgiK/9QHxydUQwTAjc2vGjgPENC2EPH2VYZWEB10a6p6nlV3uezJda2exBLbJE/hPZGUkRJVedSa0WlQQpro/FwYqcqmI2iSpJ28nIQHn1wylc/Vgv7xw+/EBY39SzuR7HpY48h1MU0lzlsS1wcO2c/F7xEFYWUtfkbZGxET+b/eF6tzdLM5/MPJr8ibiwcPwfFfLnaYJYHpsFP0Tpu/ZQ3lLblX5Gqjf0vPn0MXB45RR/ZcMds1UUfC1WtDkFd2Z74xnN7GHTXNPYZwRChNC6TCxtK83UvqRfUqydzpTL5Z3R+zsunmSJvV8xONjW/ikwOqitjrMiqlnNGf7dFh4FC2vOfgg7HxwVQd8VumWeW2oT3WCcQH4FkxM2LjAvej34vE4WGPw9s6vcKoP4ESMG34TTVBz6Tyjm4oZv9ylLFrFISSkaZoZ5smKi/F0/xOscHKg4u4Sfz7wK+8Ve3Uc5eTos9yBkf1Ydbht7mbWqBSQTMC9BazmRZ5UlrM+GzGgI", + "expiresAt": 1800000010000 + }, + "issuedAt": 1800000000000, + "challengeSecretB64": "BQUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQU=", + "transcriptB64": "AAAACHByb3RvY29sAAAAF29yY2EtcHVzaC1ob3N0LXByb29mL3YxAAAAB3ZlcnNpb24AAAABAQAAAA1nYXRld2F5T3JpZ2luAAAAF2h0dHBzOi8vcHVzaC5vbm9yY2EuZGV2AAAAGWdhdGV3YXlFcGhlbWVyYWxQdWJsaWNLZXkAAAAgV9tLNZ8jrl4Ubk4lEgVnBHIlBjSMFQwUdT0Mkz0E1CEAAAAOY2hhbGxlbmdlTm9uY2UAAAAYAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAAAAC2NoYWxsZW5nZUlkAAAAEnZlY3Rvci1jaGFsbGVuZ2UtMQAAAAhpc3N1ZWRBdAAAAAgAAAGjGFxQAAAAAAlleHBpcmVzQXQAAAAIAAABoxhcdxAAAAAPaG9zdEZpbmdlcnByaW50AAAAEEQyMGxVXzhNRDBSNjRnTHQAAAANaG9zdFB1YmxpY0tleQAAACATvk/q6vIEx/0zWPycAHIYgdF0J4EoIn7GdPN/f+l7bQ==" +} diff --git a/cloud/packages/push-contract/src/push-limits.ts b/cloud/packages/push-contract/src/push-limits.ts new file mode 100644 index 00000000000..0a4b9b25866 --- /dev/null +++ b/cloud/packages/push-contract/src/push-limits.ts @@ -0,0 +1,28 @@ +export const PUSH_LIMITS = { + titleMaxChars: 80, + bodyMaxChars: 180, + maxRegistrationIdsPerSend: 20, + // A host pairs phones, not a fleet. The cap bounds what one session can write + // through a caller-chosen deviceId. + maxDevicesPerHost: 64, + maxHttpBodyBytes: 16 * 1024, + hostEventsPerWindow: 300, + eventQuotaWindowMs: 15 * 60 * 1000, + challengeTtlMs: 10_000, + // Covers routine NTP drift without extending the signed challenge window. + clockSkewToleranceMs: 30_000, + sessionTtlMs: 24 * 60 * 60 * 1000, + notificationTtlSeconds: 5 * 60, + // The challenge and session routes are the only unauthenticated writes, so + // they are capped per client IP before any key material is generated. + unauthenticatedRequestsPerMinutePerIp: 30, + authenticatedRequestsPerMinutePerIp: 6_000, + authenticatedRequestsPerMinutePerHost: 600 +} as const + +export const PUSH_DEFAULTS = { + apnsTopic: 'com.stably.orca.mobile', + androidChannelId: 'orca-desktop' +} as const + +export const PUSH_HOST_FINGERPRINT_LENGTH = 16 diff --git a/cloud/packages/push-contract/src/send-messages.test.ts b/cloud/packages/push-contract/src/send-messages.test.ts new file mode 100644 index 00000000000..a9e14da79d9 --- /dev/null +++ b/cloud/packages/push-contract/src/send-messages.test.ts @@ -0,0 +1,115 @@ +import { describe, expect, it } from 'vitest' +import { PUSH_LIMITS } from './push-limits.js' +import { + PushSendRequestSchema, + PushSendStatusSchema +} from './send-messages.js' + +function notification(): Record { + return { + notificationId: 'note-1', + notificationSeq: 4, + notificationEpoch: '5c9e9a1e-0000-4000-8000-000000000000', + source: 'agent-task-complete', + agentState: 'needs-input', + title: 'Agent needs input', + body: 'Waiting on your answer', + worktreeId: 'wt-1' + } +} + +describe('send schemas', () => { + it('accepts a batch at the registration cap and a terminal bell without an id', () => { + const ids = Array.from({ length: PUSH_LIMITS.maxRegistrationIdsPerSend }, (_, i) => `reg-${i}`) + expect( + PushSendRequestSchema.safeParse({ v: 1, registrationIds: ids, notification: notification() }) + .success + ).toBe(true) + const { notificationId: _dropped, ...bell } = notification() + expect( + PushSendRequestSchema.safeParse({ + v: 1, + registrationIds: ['reg-1'], + notification: { ...bell, source: 'terminal-bell', agentState: null } + }).success + ).toBe(true) + }) + + it('rejects an oversized batch, over-long copy, and unknown notification keys', () => { + const ids = Array.from( + { length: PUSH_LIMITS.maxRegistrationIdsPerSend + 1 }, + (_, i) => `reg-${i}` + ) + expect( + PushSendRequestSchema.safeParse({ v: 1, registrationIds: ids, notification: notification() }) + .success + ).toBe(false) + expect( + PushSendRequestSchema.safeParse({ + v: 1, + registrationIds: ['reg-1'], + notification: { ...notification(), title: 'x'.repeat(PUSH_LIMITS.titleMaxChars + 1) } + }).success + ).toBe(false) + expect( + PushSendRequestSchema.safeParse({ + v: 1, + registrationIds: ['reg-1'], + notification: { ...notification(), body: 'x'.repeat(PUSH_LIMITS.bodyMaxChars + 1) } + }).success + ).toBe(false) + expect( + PushSendRequestSchema.safeParse({ + v: 1, + registrationIds: ['reg-1'], + notification: { ...notification(), coalescedCount: 2 } + }).success + ).toBe(false) + expect(PushSendRequestSchema.safeParse({ v: 1, registrationIds: [], notification: notification() }).success) + .toBe(false) + }) + + it('rejects a notification id that could not be sent as a collapse header', () => { + for (const notificationId of ['line\nbreak', 'nul\0byte', 'émoji', '\t']) { + expect( + PushSendRequestSchema.safeParse({ + v: 1, + registrationIds: ['reg-1'], + notification: { ...notification(), notificationId } + }).success + ).toBe(false) + } + expect( + PushSendRequestSchema.safeParse({ + v: 1, + registrationIds: ['reg-1'], + notification: { + ...notification(), + notificationId: 'agent:repo%3A%3A%2FUsers%2Fme:pane-1:1700000000000' + } + }).success + ).toBe(true) + }) + + it('dedupes repeated registration ids and keeps the first-seen order', () => { + const parsed = PushSendRequestSchema.safeParse({ + v: 1, + registrationIds: ['reg-b', 'reg-a', 'reg-b', 'reg-c', 'reg-a'], + notification: notification() + }) + expect(parsed.success).toBe(true) + expect(parsed.success && parsed.data.registrationIds).toEqual(['reg-b', 'reg-a', 'reg-c']) + }) + + it('counts duplicates against the batch cap before deduping them', () => { + const ids = Array.from({ length: PUSH_LIMITS.maxRegistrationIdsPerSend + 1 }, () => 'reg-1') + expect( + PushSendRequestSchema.safeParse({ v: 1, registrationIds: ids, notification: notification() }) + .success + ).toBe(false) + }) + + it('locks the send result statuses', () => { + expect(PushSendStatusSchema.options).toEqual(['queued', 'dead', 'rate_limited', 'error']) + }) +}) diff --git a/cloud/packages/push-contract/src/send-messages.ts b/cloud/packages/push-contract/src/send-messages.ts new file mode 100644 index 00000000000..57d1c2e2745 --- /dev/null +++ b/cloud/packages/push-contract/src/send-messages.ts @@ -0,0 +1,65 @@ +import { z } from 'zod' +import { + PushAgentStateSchema, + PushNotificationSourceSchema +} from './device-registration-messages.js' +import { PUSH_LIMITS } from './push-limits.js' +import { OpaqueIdSchema, SequenceSchema } from './wire-scalars.js' + +export const PushNotificationSchema = z + .object({ + // Absent for terminal-bell, which the desktop raises without a notification record. + // Printable ASCII only: the id becomes the APNs collapse header, and the + // desktop builds it from URL-encoded parts, so anything else is not Orca's. + notificationId: z + .string() + .min(1) + .max(2048) + .regex(/^[\x20-\x7e]+$/) + .optional(), + notificationSeq: SequenceSchema, + notificationEpoch: OpaqueIdSchema, + source: PushNotificationSourceSchema, + sound: z.boolean().optional(), + kind: z.enum(['alert', 'dismiss']).optional(), + expiresAt: z.number().int().positive().optional(), + agentState: PushAgentStateSchema.nullable(), + title: z.string().min(1).max(PUSH_LIMITS.titleMaxChars), + body: z.string().max(PUSH_LIMITS.bodyMaxChars), + worktreeId: z.string().min(1).max(2048).optional() + }) + .strict() + .refine( + (notification) => notification.kind !== 'dismiss' || Boolean(notification.notificationId), + { message: 'dismiss requires notificationId' } + ) + .refine( + (notification) => new TextEncoder().encode(JSON.stringify(notification)).byteLength <= 3000, + { + message: 'notification exceeds provider payload budget' + } + ) + +export const PushSendRequestSchema = z + .object({ + v: z.literal(1), + // Deduped before the gateway sees it so a repeated id cannot reserve quota twice. + registrationIds: z + .array(OpaqueIdSchema) + .min(1) + .max(PUSH_LIMITS.maxRegistrationIdsPerSend) + .transform((ids) => [...new Set(ids)]), + notification: PushNotificationSchema + }) + .strict() + +export const PushSendStatusSchema = z.enum(['queued', 'dead', 'rate_limited', 'error']) + +export const PushSendResultSchema = z + .object({ registrationId: OpaqueIdSchema, status: PushSendStatusSchema }) + .strict() + +export type PushNotification = z.infer +export type PushSendRequest = z.infer +export type PushSendStatus = z.infer +export type PushSendResult = z.infer diff --git a/cloud/packages/push-contract/src/wire-scalars.ts b/cloud/packages/push-contract/src/wire-scalars.ts new file mode 100644 index 00000000000..f190aef8354 --- /dev/null +++ b/cloud/packages/push-contract/src/wire-scalars.ts @@ -0,0 +1,16 @@ +import { z } from 'zod' + +// Copied from relay-contract rather than imported: the push gateway ships as a +// standalone image and must not pull the relay wire contract into its closure. +export const Base64Url32ByteSchema = z.string().regex(/^[A-Za-z0-9_-]{43}$/) +export const Base6432ByteSchema = z.string().regex(/^(?:[A-Za-z0-9+/]{4}){10}[A-Za-z0-9+/]{3}=$/) +export const Base64Raw24ByteSchema = z.string().regex(/^(?:[A-Za-z0-9+/]{4}){8}$/) +export const PushHostFingerprintSchema = z.string().regex(/^[A-Za-z0-9_-]{16}$/) +export const OpaqueIdSchema = z.string().min(1).max(128) +export const EpochMsSchema = z.number().int().nonnegative().max(Number.MAX_SAFE_INTEGER) +export const SequenceSchema = z.number().int().nonnegative().max(Number.MAX_SAFE_INTEGER) +export const BoundedCiphertextSchema = z + .string() + .min(1) + .max(16 * 1024) + .regex(/^(?:[A-Za-z0-9+/]{4})*(?:[A-Za-z0-9+/]{2}==|[A-Za-z0-9+/]{3}=)?$/) diff --git a/cloud/packages/push-contract/tsconfig.build.json b/cloud/packages/push-contract/tsconfig.build.json new file mode 100644 index 00000000000..94c84b60803 --- /dev/null +++ b/cloud/packages/push-contract/tsconfig.build.json @@ -0,0 +1,11 @@ +{ + "extends": "./tsconfig.json", + "compilerOptions": { + "declaration": true, + "emitDeclarationOnly": false, + "noEmit": false, + "outDir": "dist", + "rootDir": "src" + }, + "exclude": ["src/**/*.test.ts"] +} diff --git a/cloud/packages/push-contract/tsconfig.json b/cloud/packages/push-contract/tsconfig.json new file mode 100644 index 00000000000..a552e34dbe9 --- /dev/null +++ b/cloud/packages/push-contract/tsconfig.json @@ -0,0 +1,5 @@ +{ + "extends": "../../tsconfig.base.json", + "compilerOptions": { "noEmit": true }, + "include": ["src/**/*.ts"] +} diff --git a/cloud/pnpm-lock.yaml b/cloud/pnpm-lock.yaml index 3598ae54930..aa8afb0da67 100644 --- a/cloud/pnpm-lock.yaml +++ b/cloud/pnpm-lock.yaml @@ -21,11 +21,60 @@ importers: specifier: ^4.0.8 version: 4.1.9(@types/node@24.13.2)(vite@8.0.16(@types/node@24.13.2)(esbuild@0.28.1)(tsx@4.22.4)) + apps/push: + dependencies: + '@hono/node-server': + specifier: ^1.19.17 + version: 1.19.17(hono@4.13.7) + '@orca-cloud/postgres-schema': + specifier: workspace:* + version: link:../../packages/postgres-schema + '@orca-cloud/push-contract': + specifier: workspace:* + version: link:../../packages/push-contract + google-auth-library: + specifier: ^10.5.0 + version: 10.9.1 + hono: + specifier: ^4.13.7 + version: 4.13.7 + pg: + specifier: ^8.22.0 + version: 8.22.0 + pg-connection-string: + specifier: 2.14.0 + version: 2.14.0 + tweetnacl: + specifier: ^1.0.3 + version: 1.0.3 + zod: + specifier: ^3.25.76 + version: 3.25.76 + devDependencies: + '@types/node': + specifier: ^24.10.0 + version: 24.13.2 + '@types/pg': + specifier: ^8.20.0 + version: 8.20.0 + tsx: + specifier: ^4.21.0 + version: 4.22.4 + typescript: + specifier: ^5.9.3 + version: 5.9.3 + vitest: + specifier: ^4.0.8 + version: 4.1.9(@types/node@24.13.2)(vite@8.0.16(@types/node@24.13.2)(esbuild@0.28.1)(tsx@4.22.4)) + apps/relay: dependencies: '@hono/node-server': specifier: ^1.19.17 version: 1.19.17(hono@4.13.7) + '@orca-cloud/postgres-schema': + specifier: workspace:* + version: link:../../packages/postgres-schema '@orca-cloud/relay-contract': specifier: workspace:* version: link:../../packages/relay-contract @@ -117,6 +166,34 @@ importers: specifier: ^4.0.8 version: 4.1.9(@types/node@24.13.2)(vite@8.0.16(@types/node@24.13.2)(esbuild@0.28.1)(tsx@4.22.4)) + packages/postgres-schema: + devDependencies: + '@types/node': + specifier: ^24.10.0 + version: 24.13.2 + typescript: + specifier: ^5.9.3 + version: 5.9.3 + vitest: + specifier: ^4.0.8 + version: 4.1.9(@types/node@24.13.2)(vite@8.0.16(@types/node@24.13.2)(esbuild@0.28.1)(tsx@4.22.4)) + + packages/push-contract: + dependencies: + zod: + specifier: ^3.25.76 + version: 3.25.76 + devDependencies: + '@types/node': + specifier: ^24.10.0 + version: 24.13.2 + typescript: + specifier: ^5.9.3 + version: 5.9.3 + vitest: + specifier: ^4.0.8 + version: 4.1.9(@types/node@24.13.2)(vite@8.0.16(@types/node@24.13.2)(esbuild@0.28.1)(tsx@4.22.4)) + packages/relay-contract: dependencies: zod: @@ -463,10 +540,23 @@ packages: '@vitest/utils@4.1.9': resolution: {integrity: sha512-A51o8ymO5PpqlWNnBP9ZHPXDIpuMtTLlGSjN7la4US+LJzoUMyhwjA5QXlm39JexgwHKW4Xjs8Z2d3dLCXOeuA==} + agent-base@7.1.4: + resolution: {integrity: sha512-MnA+YT8fwfJPgBx3m60MNqakm30XOkyIoH1y6huTQvC0PwZG7ki8NacLBcrPbNoo8vEZy7Jpuk7+jMO+CUovTQ==} + engines: {node: '>= 14'} + assertion-error@2.0.1: resolution: {integrity: sha512-Izi8RQcffqCeNVgFigKli1ssklIbpHnCYc6AknXGYoB6grJqyeby7jv12JUQgmTAnIDnbck1uxksT4dzN3PWBA==} engines: {node: '>=12'} + base64-js@1.5.1: + resolution: {integrity: sha512-AKpaYlHn8t4SVbOHCy+b5+KKgvR4vrsD8vbvrbiQJps7fKDTkjkDry6ji0rUJjC0kzbNePLwzxq8iypo41qeWA==} + + bignumber.js@9.3.1: + resolution: {integrity: sha512-Ko0uX15oIUS7wJ3Rb30Fs6SkVbLmPBAKdlm7q9+ak9bbIeFf0MwuBsQV6z7+X768/cHsfg+WlysDWJcmthjsjQ==} + + buffer-equal-constant-time@1.0.1: + resolution: {integrity: sha512-zRpUiDwd/xk6ADqPMATG8vc9VPrkck7T07OIx0gnjmJAnHnTVXNQG3vfvWNuiZIkwu9KrKdA1iJKfsfTVxE6NA==} + chai@6.2.2: resolution: {integrity: sha512-NUPRluOfOiTKBKvWPtSD4PhFvWCqOi0BGStNWs57X9js7XGTprSmFoz5F0tWhR4WPjNeR9jXqdC7/UpSJTnlRg==} engines: {node: '>=18'} @@ -474,10 +564,26 @@ packages: convert-source-map@2.0.0: resolution: {integrity: sha512-Kvp459HrV2FEJ1CAsi1Ku+MY3kasH19TFykTz2xWmMeq6bk2NU3XXvfJ+Q61m0xktWwt+1HSYf3JZsTms3aRJg==} + data-uri-to-buffer@4.0.1: + resolution: {integrity: sha512-0R9ikRb668HB7QDxT1vkpuUBtqc53YyAwMwGeUFKRojY/NWKvdZ+9UYtRfGmhqNbRkTSVpMbmyhXipFFv2cb/A==} + engines: {node: '>= 12'} + + debug@4.4.3: + resolution: {integrity: sha512-RGwwWnwQvkVfavKVt22FGLw+xYSdzARwm0ru6DhTVA3umU5hZc28V3kO4stgYryrTlLpuvgI9GiijltAjNbcqA==} + engines: {node: '>=6.0'} + peerDependencies: + supports-color: '*' + peerDependenciesMeta: + supports-color: + optional: true + detect-libc@2.1.2: resolution: {integrity: sha512-Btj2BOOO83o3WyH59e8MgXsxEQVcarkUOpEYrubB0urwnN10yQ364rsiByU11nZlqWYZm05i/of7io4mzihBtQ==} engines: {node: '>=8'} + ecdsa-sig-formatter@1.0.11: + resolution: {integrity: sha512-nagl3RYrbNv6kQkeJIpt6NJZy8twLB/2vtz6yN9Z4vRKHN4/QZJIEbqohALSgwKdnksuY3k5Addp5lg8sVoVcQ==} + es-module-lexer@2.1.0: resolution: {integrity: sha512-n27zTYMjYu1aj4MjCWzSP7G9r75utsaoc8m61weK+W8JMBGGQybd43GstCXZ3WNmSFtGT9wi59qQTW6mhTR5LQ==} @@ -493,6 +599,9 @@ packages: resolution: {integrity: sha512-knvyeauYhqjOYvQ66MznSMs83wmHrCycNEN6Ao+2AeYEfxUIkuiVxdEa1qlGEPK+We3n0THiDciYSsCcgW/DoA==} engines: {node: '>=12.0.0'} + extend@3.0.2: + resolution: {integrity: sha512-fjquC59cD7CyW6urNXK0FBufkZcoiGG80wTuPujX590cB5Ttln20E2UB4S/WARVqhXffZl2LNgS+gQdPIIim/g==} + fdir@6.5.0: resolution: {integrity: sha512-tIbYtZbucOs0BRGqPJkshJUYdL+SDH7dVM8gjy+ERp3WAUjLEFJE+02kanyHtwjWOnwrKYBiwAmM0p4kLJAnXg==} engines: {node: '>=12.0.0'} @@ -502,18 +611,55 @@ packages: picomatch: optional: true + fetch-blob@3.2.0: + resolution: {integrity: sha512-7yAQpD2UMJzLi1Dqv7qFYnPbaPx7ZfFK6PiIxQ4PfkGPyNyl2Ugx+a/umUonmKqjhM4DnfbMvdX6otXq83soQQ==} + engines: {node: ^12.20 || >= 14.13} + + formdata-polyfill@4.0.10: + resolution: {integrity: sha512-buewHzMvYL29jdeQTVILecSaZKnt/RJWjoZCF5OW60Z67/GmSLBkOFM7qh1PI3zFNtJbaZL5eQu1vLfazOwj4g==} + engines: {node: '>=12.20.0'} + fsevents@2.3.3: resolution: {integrity: sha512-5xoDfX+fL7faATnagmWPpbFtwh/R77WmMMqqHGS65C3vvB0YHrgF+B1YmZ3441tMj5n63k0212XNoJwzlhffQw==} engines: {node: ^8.16.0 || ^10.6.0 || >=11.0.0} os: [darwin] + gaxios@7.3.1: + resolution: {integrity: sha512-kB3rzJV7d9juLZh8/56QTXCwQfxyhdOMdyYk1HdQKFtF8TJTDTZQJtixWIwXdE9Jji91mC41DUNpjleo4L4eAQ==} + engines: {node: '>=18'} + + gcp-metadata@8.1.2: + resolution: {integrity: sha512-zV/5HKTfCeKWnxG0Dmrw51hEWFGfcF2xiXqcA3+J90WDuP0SvoiSO5ORvcBsifmx/FoIjgQN3oNOGaQ5PhLFkg==} + engines: {node: '>=18'} + + google-auth-library@10.9.1: + resolution: {integrity: sha512-i1ydyHrqcIxXkWh/uBmVkzCvIuq5yiK2ATndIe5XxKholrG/MTYP9xGYka4sQhrbIAgGjL2B6NOE7rFaiF3fXw==} + engines: {node: '>=18'} + + google-logging-utils@1.1.3: + resolution: {integrity: sha512-eAmLkjDjAFCVXg7A1unxHsLf961m6y17QFqXqAXGj/gVkKFrEICfStRfwUlGNfeCEjNRa32JEWOUTlYXPyyKvA==} + engines: {node: '>=14'} + hono@4.13.7: resolution: {integrity: sha512-c8/gF9ac8Y78/agExVocyLevgR+JlpNB444Py0FSX8pJoPdYUfUzRcXtYEYGwt6l19qIlVZPN5Mfsw9jFShmQQ==} engines: {node: '>=16.9.0'} + https-proxy-agent@7.0.6: + resolution: {integrity: sha512-vK9P5/iUfdl95AI+JVyUuIcVtd4ofvtrOr3HNtM2yxC9bnMbEdp3x01OhQNnjb8IJYi38VlTE3mBXwcfvywuSw==} + engines: {node: '>= 14'} + jose@6.2.3: resolution: {integrity: sha512-YYVDInQKFJfR/xa3ojUTl8c2KoTwiL1R5Wg9YCydwH0x0B9grbzlg5HC7mMjCtUJjbQ/YnGEZIhI5tCgfTb4Hw==} + json-bigint@1.0.0: + resolution: {integrity: sha512-SiPv/8VpZuWbvLSMtTDU8hEfrZWg/mH/nV/b4o0CYbSxu1UIQPLdwKOCIyLQX+VIPO5vrLX3i8qtqFyhdPSUSQ==} + + jwa@2.0.1: + resolution: {integrity: sha512-hRF04fqJIP8Abbkq5NKGN0Bbr3JxlQ+qhZufXVr0DvujKy93ZCbXZMHDL4EOtodSbCWxOqR8MS1tXA5hwqCXDg==} + + jws@4.0.1: + resolution: {integrity: sha512-EKI/M/yqPncGUUh44xz0PxSidXFr/+r0pA70+gIYhjv+et7yxM+s29Y+VGDkovRofQem0fs7Uvf4+YmAdyRduA==} + lightningcss-android-arm64@1.32.0: resolution: {integrity: sha512-YK7/ClTt4kAK0vo6w3X+Pnm0D2cf2vPHbhOXdoNti1Ga0al1P4TBZhwjATvjNwLEBCnKvjJc2jQgHXH0NEwlAg==} engines: {node: '>= 12.0.0'} @@ -587,11 +733,23 @@ packages: magic-string@0.30.21: resolution: {integrity: sha512-vd2F4YUyEXKGcLHoq+TEyCjxueSeHnFxyyjNp80yg0XV4vUhnDer/lvvlqM/arB5bXQN5K2/3oinyCRyx8T2CQ==} - nanoid@3.3.18: - resolution: {integrity: sha512-DTg4MJbGMWkfi6VZFdNt2/caMbQy4Ou+Op/hJQvGEWcnVfoA1QA+xzRKAzw9jD6+GVOOeYr/mIcuDSdug6F6+w==} + ms@2.1.3: + resolution: {integrity: sha512-6FlzubTLZG3J2a/NVCAleEhjzq5oxgHyaCU9yYXvcLsvoVaHJq/s5xXI6/XXP6tz7R9xAOtHnSO/tXtF3WRTlA==} + + nanoid@3.3.13: + resolution: {integrity: sha512-sPdqC6ByMVVGvF1ynvvMo0/o+oD1VX7DaHhijt1bFgjvBkHBib4t49GoNDhf2NDta4oeUNlaGbSt5K7qjZ955Q==} engines: {node: ^10 || ^12 || ^13.7 || ^14 || >=15.0.1} hasBin: true + node-domexception@1.0.0: + resolution: {integrity: sha512-/jKZoMpw0F8GRwl4/eLROPA3cfcXtLApP0QzLmUT/HuPCZWyB7IY9ZrMeKw2O/nFIqPQB3PVM9aYm0F312AXDQ==} + engines: {node: '>=10.5.0'} + deprecated: Use your platform's native DOMException instead + + node-fetch@3.3.2: + resolution: {integrity: sha512-dRB78srN/l6gqWulah9SrxeYnxeddIG30+GOqK/9OlLVyLg3HPnr6SqOWTWOXKRwC2eGYCkZ59NNuSgvSrpgOA==} + engines: {node: ^12.20.0 || ^14.13.1 || >=16.0.0} + obug@2.1.3: resolution: {integrity: sha512-9miFgM2OFba7hB+pRgvtV84pYTBaoTHohvmIgiRt6dRIzbwEOIaNaP+dIlGs2fNFoB0SeISs0Jz5WFVRid6Xyg==} engines: {node: '>=12.20.0'} @@ -640,8 +798,8 @@ packages: resolution: {integrity: sha512-QP88BAKvMam/3NxH6vj2o21R6MjxZUAd6nlwAS/pnGvN9IVLocLHxGYIzFhg6fUQ+5th6P4dv4eW9jX3DSIj7A==} engines: {node: '>=12'} - postcss@8.5.28: - resolution: {integrity: sha512-RRuzqDtt5Y9h3quz5hWhK+TPnsmVs6WwSU6LkJMeY4HstUEDuYTG8UJSdawMRzmzAtV+KEoG8N3Qg2qLy5vM/A==} + postcss@8.5.15: + resolution: {integrity: sha512-FfR8sjd4em2T6fb3I2MwAJU7HWVMr9zba+enmQeeWFfCbm+UOC/0X4DS8XtpUTMwWMGbjKYP7xjfNekzyGmB3A==} engines: {node: ^10 || ^12 || >=14} postgres-array@2.0.0: @@ -665,6 +823,9 @@ packages: engines: {node: ^20.19.0 || >=22.12.0} hasBin: true + safe-buffer@5.2.1: + resolution: {integrity: sha512-rp3So07KcdmmKbGvgaNxQSJr7bGVSVk5S9Eq1F+ppbRo70+YeaDxkw5Dd8NPN+GD6bjnYm2VuPuCXmpuYvmCXQ==} + siginfo@2.0.0: resolution: {integrity: sha512-ybx0WO1/8bSBLEWXZvEd7gMW3Sn3JFlW3TvX1nREbDLRNQNaeNN8WK0meBwPdAaOI7TtRRRJn/Es1zhrrCHu7g==} @@ -800,6 +961,10 @@ packages: jsdom: optional: true + web-streams-polyfill@3.3.3: + resolution: {integrity: sha512-d2JWLCivmZYTSIoge9MsgFCZrt571BikcWGYkjC1khllbTeDlGqZ2D8vD8E/lJa8WGWbb7Plm8/XJYV7IJHZZw==} + engines: {node: '>= 8'} + why-is-node-running@2.3.0: resolution: {integrity: sha512-hUrmaWBdVDcxvYqnyh09zunKzROWjbZTiNy8dBEjkS7ehEDQibXJ7XvlmtbwuTclUiIyN+CyXQD4Vmko8fNm8w==} engines: {node: '>=8'} @@ -1057,14 +1222,32 @@ snapshots: convert-source-map: 2.0.0 tinyrainbow: 3.1.0 + agent-base@7.1.4: {} + assertion-error@2.0.1: {} + base64-js@1.5.1: {} + + bignumber.js@9.3.1: {} + + buffer-equal-constant-time@1.0.1: {} + chai@6.2.2: {} convert-source-map@2.0.0: {} + data-uri-to-buffer@4.0.1: {} + + debug@4.4.3: + dependencies: + ms: 2.1.3 + detect-libc@2.1.2: {} + ecdsa-sig-formatter@1.0.11: + dependencies: + safe-buffer: 5.2.1 + es-module-lexer@2.1.0: {} esbuild@0.28.1: @@ -1102,17 +1285,79 @@ snapshots: expect-type@1.3.0: {} + extend@3.0.2: {} + fdir@6.5.0(picomatch@4.0.4): optionalDependencies: picomatch: 4.0.4 + fetch-blob@3.2.0: + dependencies: + node-domexception: 1.0.0 + web-streams-polyfill: 3.3.3 + + formdata-polyfill@4.0.10: + dependencies: + fetch-blob: 3.2.0 + fsevents@2.3.3: optional: true + gaxios@7.3.1: + dependencies: + extend: 3.0.2 + https-proxy-agent: 7.0.6 + node-fetch: 3.3.2 + transitivePeerDependencies: + - supports-color + + gcp-metadata@8.1.2: + dependencies: + gaxios: 7.3.1 + google-logging-utils: 1.1.3 + json-bigint: 1.0.0 + transitivePeerDependencies: + - supports-color + + google-auth-library@10.9.1: + dependencies: + base64-js: 1.5.1 + ecdsa-sig-formatter: 1.0.11 + gaxios: 7.3.1 + gcp-metadata: 8.1.2 + google-logging-utils: 1.1.3 + jws: 4.0.1 + transitivePeerDependencies: + - supports-color + + google-logging-utils@1.1.3: {} + hono@4.13.7: {} + https-proxy-agent@7.0.6: + dependencies: + agent-base: 7.1.4 + debug: 4.4.3 + transitivePeerDependencies: + - supports-color + jose@6.2.3: {} + json-bigint@1.0.0: + dependencies: + bignumber.js: 9.3.1 + + jwa@2.0.1: + dependencies: + buffer-equal-constant-time: 1.0.1 + ecdsa-sig-formatter: 1.0.11 + safe-buffer: 5.2.1 + + jws@4.0.1: + dependencies: + jwa: 2.0.1 + safe-buffer: 5.2.1 + lightningcss-android-arm64@1.32.0: optional: true @@ -1166,7 +1411,17 @@ snapshots: dependencies: '@jridgewell/sourcemap-codec': 1.5.5 - nanoid@3.3.18: {} + ms@2.1.3: {} + + nanoid@3.3.13: {} + + node-domexception@1.0.0: {} + + node-fetch@3.3.2: + dependencies: + data-uri-to-buffer: 4.0.1 + fetch-blob: 3.2.0 + formdata-polyfill: 4.0.10 obug@2.1.3: {} @@ -1211,9 +1466,9 @@ snapshots: picomatch@4.0.4: {} - postcss@8.5.28: + postcss@8.5.15: dependencies: - nanoid: 3.3.18 + nanoid: 3.3.13 picocolors: 1.1.1 source-map-js: 1.2.1 @@ -1248,6 +1503,8 @@ snapshots: '@rolldown/binding-win32-arm64-msvc': 1.0.3 '@rolldown/binding-win32-x64-msvc': 1.0.3 + safe-buffer@5.2.1: {} + siginfo@2.0.0: {} source-map-js@1.2.1: {} @@ -1288,7 +1545,7 @@ snapshots: dependencies: lightningcss: 1.32.0 picomatch: 4.0.4 - postcss: 8.5.28 + postcss: 8.5.15 rolldown: 1.0.3 tinyglobby: 0.2.17 optionalDependencies: @@ -1324,6 +1581,8 @@ snapshots: transitivePeerDependencies: - msw + web-streams-polyfill@3.3.3: {} + why-is-node-running@2.3.0: dependencies: siginfo: 2.0.0