diff --git a/cloud/apps/relay/src/cell-phone-path.blackbox.test.ts b/cloud/apps/relay/src/cell-phone-path.blackbox.test.ts index ad8ee69b66c..fb6fb7abe14 100644 --- a/cloud/apps/relay/src/cell-phone-path.blackbox.test.ts +++ b/cloud/apps/relay/src/cell-phone-path.blackbox.test.ts @@ -468,4 +468,63 @@ describe('the memory phone path', () => { }) } }) + + // E-dep-6 in miniature: the staging drill blocks one memory cell's database for 10 minutes. + it('rides out a database outage: splices keep forwarding, new work fails cleanly and recovers', async () => { + let now = Date.now() + const reserve = await cell(() => now) + reserve.setAdmitMode('reserve', { phonePath: 'memory' }) + const seated = await reserve.bookedHost(3) + const confirm = async (reqId: string, basisConnId: string) => { + const answer = Promise.race([ + nextFrame(seated.socket, 'device-resume-confirmed'), + nextFrame(seated.socket, 'control-error') + ]) + seated.socket.send(JSON.stringify({ type: 'device-resume-confirm', v: 1, reqId, basisConnId })) + return await answer + } + const splice = async (fill: number) => { + await reserve.installDevice(seated.identity.hostId, token(fill), { relayDeviceId: `device-${fill}` }) + const connOpen = nextFrame(seated.socket, 'conn-open') + const phone = await reserve.dialPhone(seated.identity.hostId, token(fill)) + const basisConnId = String((await connOpen).connId) + const data = await reserve.attachData(await connOpen, seated.ack.generation) + expect(await phone.hello).toMatchObject({ ok: true }) + return { phone, data, basisConnId } + } + const confirmed = await splice(1) + expect(await confirm('confirm-a', confirmed.basisConnId)).toMatchObject({ type: 'device-resume-confirmed' }) + const inFlight = await splice(2) + + reserve.database.failing = () => true + // A seated phone keeps working. + const relayed = new Promise((resolve) => + confirmed.data.once('message', (frame) => resolve(String(frame))) + ) + confirmed.phone.socket.send('still here') + expect(await relayed).toBe('still here') + // A new resume and a confirm fail cleanly; nothing crashes. + expect((await (await reserve.dialPhone(seated.identity.hostId, token(1))).closed).code).toBe(4503) + expect(await confirm('confirm-b', inFlight.basisConnId)).toMatchObject({ type: 'control-error' }) + // Flipped back during the block: the in-flight confirm still finishes on its memory basis. + reserve.setAdmitMode('db', { phonePath: 'memory' }) + reserve.database.failing = null + reserve.database.statements.length = 0 + expect(await confirm('confirm-c', inFlight.basisConnId)).toMatchObject({ type: 'device-resume-confirmed' }) + expect(reserve.phoneStatements().filter((sql) => !sql.includes('relay_assignment'))[0]).toBe(CONFIRM_DEVICE_READ) + // And new resumes take the database path again. + const connOpen = nextFrame(seated.socket, 'conn-open') + await reserve.dialPhone(seated.identity.hostId, token(1)) + expect(await connOpen).toMatchObject({ kind: 'resume' }) + + // Audit rows held past 10 minutes of outage are dropped and counted, never written late. + reserve.database.failing = (sql) => sql.startsWith('INSERT INTO relay_audit_events') + now += 11 * 60 * 1000 + await reserve.relay.store.resumeAudit.flush() + reserve.database.failing = null + const written = await reserve.raw.query(`SELECT id FROM relay_audit_events WHERE type = 'resume-confirmed'`) + expect(written).toEqual([]) + expect(reserve.relay.runtimeCounts().resumeAuditDropped).toBe(2) + }) }) + diff --git a/cloud/apps/relay/src/host-session-client-accept.test.ts b/cloud/apps/relay/src/host-session-client-accept.test.ts index f66be975704..7448631383a 100644 --- a/cloud/apps/relay/src/host-session-client-accept.test.ts +++ b/cloud/apps/relay/src/host-session-client-accept.test.ts @@ -380,7 +380,9 @@ describe('client accept abandoned mid-DB-phase', () => { await h.registry.acceptClient(client as unknown as WebSocket, identity.relayHostId, 'credential') now += RELAY_PROTOCOL_LIMITS.hostAttachDeadlineMs vi.advanceTimersByTime(RELAY_PROTOCOL_LIMITS.hostAttachDeadlineMs) - expect(client.close).toHaveBeenCalledWith(RELAY_CLOSE_CODE.HOST_OFFLINE, expect.any(String)) + // The phone-only reason; relay-hello itself is unchanged for strict old parsers. + expect(client.close).toHaveBeenCalledWith(RELAY_CLOSE_CODE.HOST_OFFLINE, 'attach-timeout') + expect(client.send).toHaveBeenCalledWith(JSON.stringify({ type: 'relay-hello', ok: false, code: 4404 })) expect(h.observer.recordPhoneAttachDeadline).toHaveBeenCalledOnce() expect(h.observer.recordPhoneRejection).toHaveBeenCalledWith(RELAY_CLOSE_CODE.HOST_OFFLINE) const lines = warn.mock.calls diff --git a/cloud/apps/relay/src/host-session-registry.ts b/cloud/apps/relay/src/host-session-registry.ts index 331599be4bf..3ea5c39fb73 100644 --- a/cloud/apps/relay/src/host-session-registry.ts +++ b/cloud/apps/relay/src/host-session-registry.ts @@ -18,10 +18,12 @@ import { RELAY_HOST_CAPABILITY_IDLE_REGIONAL_REHOME, RELAY_PROTOCOL_LIMITS, RELAY_CLOSE_CODE, + RELAY_PHONE_CLOSE_REASON, type IdleRegionalRehomeCommit, type IdleRegionalRehomeDeferReason, type IdleRegionalRehomeResult, type RelayHostCloseReason, + type RelayPhoneCloseReason, type RelayRegion } from '@orca-cloud/relay-contract' import nacl from 'tweetnacl' @@ -981,7 +983,12 @@ export class HostSessionRegistry { this.failReservationBestEffort(reservation) if (credentialActivityId) this.releaseActivityBestEffort(identity, credentialActivityId) this.recordAttachDeadline(session, reservation) - this.rejectClient(socket, RELAY_CLOSE_CODE.HOST_OFFLINE) + // OTA phones redial this cell once, quickly: the control is still seated here. + this.rejectClient( + socket, + RELAY_CLOSE_CODE.HOST_OFFLINE, + RELAY_PHONE_CLOSE_REASON.HOST_ATTACH_TIMEOUT + ) }, RELAY_PROTOCOL_LIMITS.hostAttachDeadlineMs) const pending: PendingConnection = { connId, @@ -2559,17 +2566,17 @@ export class HostSessionRegistry { if (session.socket) send(session.socket, 'control-error', { ...(reqId ? { reqId } : {}), code }) } - // hostCloseReason rides the WebSocket close reason, never relay-hello: every - // shipped phone parses relay-hello with a strict schema that rejects an - // unknown key, and none of them read the close reason at all. + // closeReason rides the WebSocket close reason, never relay-hello: every shipped + // phone parses relay-hello with a strict schema that rejects an unknown key, and + // ignores a close reason it does not know. private rejectClient( socket: WebSocket, code: number, - hostCloseReason?: RelayHostCloseReason | null + closeReason?: RelayHostCloseReason | RelayPhoneCloseReason | null ): void { this.observer.recordPhoneRejection?.(code) send(socket, 'relay-hello', { ok: false, code }) - closeRelayWebSocket(socket, code, hostCloseReason ?? 'relay connection rejected') + closeRelayWebSocket(socket, code, closeReason ?? 'relay connection rejected') } private releaseControlActivity(session: HostSession): void { diff --git a/cloud/packages/relay-contract/src/close-codes.ts b/cloud/packages/relay-contract/src/close-codes.ts index 504bcd11b08..12c125e3fca 100644 --- a/cloud/packages/relay-contract/src/close-codes.ts +++ b/cloud/packages/relay-contract/src/close-codes.ts @@ -17,3 +17,6 @@ export const RELAY_PHONE_CLOSE_REASON = { // missed hostAttachDeadlineMs; the phone redials the same cell once, quickly. HOST_ATTACH_TIMEOUT: 'attach-timeout' } as const + +export type RelayPhoneCloseReason = + (typeof RELAY_PHONE_CLOSE_REASON)[keyof typeof RELAY_PHONE_CLOSE_REASON] diff --git a/mobile/src/transport/mobile-relay-renewal-compatibility.test.ts b/mobile/src/transport/mobile-relay-renewal-compatibility.test.ts new file mode 100644 index 00000000000..d23fbbad998 --- /dev/null +++ b/mobile/src/transport/mobile-relay-renewal-compatibility.test.ts @@ -0,0 +1,117 @@ +import { describe, expect, it, vi } from 'vitest' +import type { DeviceResumeConfirmed } from '../../../src/shared/mobile-relay-credential-contract' +import type { MobileRelayCredentialBundle } from './mobile-relay-credential-bundle' +import { hashMobileRelayCredential } from './mobile-relay-credential-hash' +import { + mobileRelayCredentialNeedsRotation, + persistResumeConfirmation +} from './mobile-relay-credential-rotation' +import { RelayLeaseRotationTimer } from './mobile-relay-lease-rotation-timer' + +vi.mock('react-native', () => ({ Platform: { OS: 'ios' } })) +vi.mock('expo-secure-store', () => ({ WHEN_UNLOCKED_THIS_DEVICE_ONLY: 'when-unlocked' })) +vi.mock('expo-crypto', () => ({ getRandomBytes: (length: number) => new Uint8Array(length) })) + +// Step 6 renews a device on a memory cell at most once a day: only when less than 29 of its 30 +// days remain. Shipped phones must keep a relay-only session alive under that rule, which they +// do only if a current-version confirm never answers renewed:false. +const DAY = 24 * 60 * 60 * 1000 +const RESUME_TTL_MS = 30 * DAY +const RENEWAL_INTERVAL_MS = DAY +const TOKEN = 'A'.repeat(43) + +// The memory cell's answer (cloud/apps/relay/src/credential-store.ts confirmMemoryResume). +function memoryCellConfirm( + stored: { expiresAt: number; renewals: number }, + now: number, + answerNotDue: 'renewed' | 'unchanged' +): DeviceResumeConfirmed { + const due = stored.expiresAt < now + RESUME_TTL_MS - RENEWAL_INTERVAL_MS + if (due) { + stored.expiresAt = now + RESUME_TTL_MS + stored.renewals += 1 + } + return { + v: 1, + reqId: `confirm-${now}`, + currentVersion: 2, + acceptedAs: 'current', + renewed: due || answerNotDue === 'renewed', + resumeExpiresAt: stored.expiresAt + } +} + +// A phone whose only path is the relay: the lease timer's re-resume is its only renewal. +async function relayOnlyPhone(days: number, answerNotDue: 'renewed' | 'unchanged') { + let now = Date.parse('2026-10-19T00:00:00Z') + const stored = { expiresAt: now + RESUME_TTL_MS, renewals: 0 } + let bundle: MobileRelayCredentialBundle = { + v: 1, + hostId: 'host-1', + deviceToken: 'device-token', + current: { + token: TOKEN, + hash: hashMobileRelayCredential(TOKEN), + version: 2, + expiresAt: stored.expiresAt + } + } + // The delay the lease timer last armed, or null once nothing is armed. + const slot: { delayMs: number | null } = { delayMs: null } + const timer = new RelayLeaseRotationTimer( + { + now: () => now, + setTimer: (_callback, delayMs) => { + slot.delayMs = delayMs + return setTimeout(() => {}, 0) + }, + clearTimer: () => { + slot.delayMs = null + } + }, + () => {} + ) + const resume = async () => { + const confirmation = memoryCellConfirm(stored, now, answerNotDue) + const persisted = await persistResumeConfirmation({ + session: { getResumeConfirmation: () => confirmation, getResumeExpiresAt: () => null }, + bundle, + usedCredentialVersion: 2, + writeBundle: async () => {} + }) + bundle = persisted.bundle + timer.scheduleFromLease(persisted.leaseExpiry) + } + await resume() + let resumes = 1 + while (slot.delayMs !== null && now < Date.parse('2026-10-19T00:00:00Z') + days * DAY) { + now += slot.delayMs + timer.clear() + await resume() + resumes += 1 + expect(bundle.current.expiresAt).toBeGreaterThan(now) + expect(mobileRelayCredentialNeedsRotation(bundle, now)).toBe(false) + } + return { now, bundle, stored, resumes, timerArmed: timer.pending } +} + +describe('relay renewal under the step-6 memory cell rule', () => { + it('keeps a relay-only phone valid for 31 days, with its lease timer armed after every confirm', async () => { + const phone = await relayOnlyPhone(31, 'renewed') + expect(phone.timerArmed).toBe(true) + // The re-resume every 6 hours writes at most once a day. + expect(phone.resumes).toBeGreaterThanOrEqual(31 * 4) + expect(phone.stored.renewals).toBeLessThanOrEqual(32) + expect(phone.bundle.current.expiresAt).toBe(phone.stored.expiresAt) + expect(phone.bundle.current.expiresAt - phone.now).toBeGreaterThan( + RESUME_TTL_MS - 2 * RENEWAL_INTERVAL_MS + ) + }) + + it('would strand that phone if a not-due confirm answered renewed:false', async () => { + // The answer the design rejected: the phone stops arming its lease timer at once. + const phone = await relayOnlyPhone(31, 'unchanged') + expect(phone.resumes).toBe(1) + expect(phone.timerArmed).toBe(false) + }) +})