diff --git a/cloud/apps/relay/src/cloud-run-instance-zone.test.ts b/cloud/apps/relay/src/cloud-run-instance-zone.test.ts new file mode 100644 index 00000000000..b60193a478c --- /dev/null +++ b/cloud/apps/relay/src/cloud-run-instance-zone.test.ts @@ -0,0 +1,28 @@ +import { describe, expect, it, vi } from 'vitest' +import { readCloudRunInstanceZone } from './cloud-run-instance-zone.js' + +const answering = (status: number, body: string): typeof fetch => + vi.fn(async () => new Response(body, { status })) + +describe('readCloudRunInstanceZone', () => { + it('returns the zone from the metadata path', async () => { + const fetchImpl = answering(200, 'projects/123456/zones/us-central1-f') + await expect(readCloudRunInstanceZone(fetchImpl)).resolves.toBe('us-central1-f') + expect(fetchImpl).toHaveBeenCalledWith( + 'http://metadata.google.internal/computeMetadata/v1/instance/zone', + expect.objectContaining({ headers: { 'Metadata-Flavor': 'Google' } }) + ) + }) + + it('returns undefined when the metadata server refuses or is unreachable', async () => { + await expect(readCloudRunInstanceZone(answering(404, 'not found'))).resolves.toBeUndefined() + const unreachable: typeof fetch = vi.fn(async () => { + throw new TypeError('fetch failed') + }) + await expect(readCloudRunInstanceZone(unreachable)).resolves.toBeUndefined() + }) + + it('returns undefined for a body that is not a zone name', async () => { + await expect(readCloudRunInstanceZone(answering(200, ''))).resolves.toBeUndefined() + }) +}) diff --git a/cloud/apps/relay/src/cloud-run-instance-zone.ts b/cloud/apps/relay/src/cloud-run-instance-zone.ts new file mode 100644 index 00000000000..17798ce519a --- /dev/null +++ b/cloud/apps/relay/src/cloud-run-instance-zone.ts @@ -0,0 +1,22 @@ +const INSTANCE_ZONE_URL = 'http://metadata.google.internal/computeMetadata/v1/instance/zone' +const REQUEST_TIMEOUT_MS = 3_000 + +// The metadata server answers `projects//zones/`. Undefined on any failure: +// the zone only labels log lines, so it must never hold up or fail a boot. +export async function readCloudRunInstanceZone( + fetchImpl: typeof fetch = fetch +): Promise { + try { + const response = await fetchImpl(INSTANCE_ZONE_URL, { + headers: { 'Metadata-Flavor': 'Google' }, + signal: AbortSignal.timeout(REQUEST_TIMEOUT_MS) + }) + if (!response.ok) { + return undefined + } + const zone = (await response.text()).trim().split('/').at(-1) + return zone && /^[a-z0-9-]+$/.test(zone) ? zone : undefined + } catch { + return undefined + } +} diff --git a/cloud/apps/relay/src/floating-promise-census.test.ts b/cloud/apps/relay/src/floating-promise-census.test.ts index 0a4730c9527..355ae7a8b02 100644 --- a/cloud/apps/relay/src/floating-promise-census.test.ts +++ b/cloud/apps/relay/src/floating-promise-census.test.ts @@ -10,6 +10,7 @@ import { describe, expect, it } from 'vitest' // Exempt: functions written to settle every failure themselves. Keyed file#name. const NEVER_REJECTS = new Map([ ['relay-background-operation.ts#runRelayBackgroundOperation', 'catches and logs every failure'], + ['cloud-run-instance-zone.ts#readCloudRunInstanceZone', 'one try/catch; resolves undefined on any failure'], ['assignment-cleanup-steps.ts#runAssignmentCleanup', 'runs each step through the above'], ['cell-heartbeat-client.ts#send', 'one try/catch around the whole send'], ['cell-reserve-client.ts#send', 'one try/catch around the request; resolves every item'], diff --git a/cloud/apps/relay/src/index.ts b/cloud/apps/relay/src/index.ts index d27f82eac75..503ef3bc9a6 100644 --- a/cloud/apps/relay/src/index.ts +++ b/cloud/apps/relay/src/index.ts @@ -14,6 +14,7 @@ import { cellFlagParser } from './cell-flags.js' import { startControlFlagChannel } from './relay-control-flag-channel.js' +import { readCloudRunInstanceZone } from './cloud-run-instance-zone.js' import { reconcileCellAdmissionAtStartup, roleOwnsAssignmentMaintenance @@ -76,6 +77,10 @@ const { cellIncarnation, shadowSeatPoller } = createRelayServer(config, database, { cellFlags: cellFlagChannel?.applied }) +// Cloud Run sets K_SERVICE; GCE cells and local runs skip the metadata read. +if (process.env.K_SERVICE) { + void readCloudRunInstanceZone().then((zone) => observability.setInstanceZone(zone)) +} // Same owner as the assignment sweep: the cleanup only expires credentials that every reader // already re-checks at read time, so running it in all 23 cells multiplied one table scan by 23 // without changing any answer. diff --git a/cloud/apps/relay/src/relay-observability.test.ts b/cloud/apps/relay/src/relay-observability.test.ts index 98b50fe33b0..9e1dcc1a73d 100644 --- a/cloud/apps/relay/src/relay-observability.test.ts +++ b/cloud/apps/relay/src/relay-observability.test.ts @@ -316,6 +316,8 @@ describe('relay observability', () => { stickySqlMsP99: 198, stickyStatementsP50: 14, stickyStatementsMax: 14, + // 50 statements at 300 ms and 100 at 5-6 ms: the pooled median is the fast connection's. + stickyStatementMsP50: 6, stickySqlByConnection: { '1': { requests: 50, @@ -332,6 +334,20 @@ describe('relay observability', () => { expect(entries[1]).not.toHaveProperty('stickySqlByConnection') }) + it('stamps the instance zone once it is known', () => { + const entries: Record[] = [] + const observability = new RelayObservability( + { role: 'director', cellId: 'director', region: 'us-central1' }, + (entry) => entries.push(entry) + ) + observability.flush(counts) + observability.setInstanceZone('us-central1-a') + observability.flush(counts) + + expect(entries[0]).not.toHaveProperty('instanceZone') + expect(entries[1]).toMatchObject({ instanceZone: 'us-central1-a' }) + }) + it('bounds the per-connection sticky summary', () => { const entries: Array> = [] const observability = new RelayObservability( diff --git a/cloud/apps/relay/src/relay-observability.ts b/cloud/apps/relay/src/relay-observability.ts index 40c7ff45415..7fafffdb04b 100644 --- a/cloud/apps/relay/src/relay-observability.ts +++ b/cloud/apps/relay/src/relay-observability.ts @@ -210,6 +210,8 @@ type RelayMetricDeltas = { stickySlotStatements: number[] stickySlotByConnection: Map stickySlotConnectionSamplesDropped: number + stickyStatementMs: number[] + stickyStatementsObserved: number assign503sByCause: Record dbLockWaitSamples: number dbLockWaitersByKey: Record @@ -305,6 +307,8 @@ const emptyDeltas = (): RelayMetricDeltas => ({ stickySlotStatements: [], stickySlotByConnection: new Map(), stickySlotConnectionSamplesDropped: 0, + stickyStatementMs: [], + stickyStatementsObserved: 0, assign503sByCause: {}, dbLockWaitSamples: 0, dbLockWaitersByKey: {}, @@ -386,6 +390,7 @@ export class RelayObservability implements RelayRuntimeObserver { private deltas = emptyDeltas() private timer: ReturnType | null = null private lastFlushAt = 0 + private instanceZone: string | undefined private lastFlushedSafety = { sqlFailures: 0, reconnects: 0, @@ -397,6 +402,10 @@ export class RelayObservability implements RelayRuntimeObserver { private readonly write: MetricWriter = (entry) => console.log(JSON.stringify(entry)) ) {} + setInstanceZone(zone: string | undefined): void { + this.instanceZone = zone + } + recordAuth(success: boolean): void { if (success) this.deltas.authSuccesses++ else this.deltas.authFailures++ @@ -457,6 +466,14 @@ export class RelayObservability implements RelayRuntimeObserver { deltas.stickySlotSqlMs.push(sample.sqlMs) deltas.stickySlotStatements.push(sample.statements) for (const [connectionId, samplesMs] of sample.statementMsByConnection) { + for (const durationMs of samplesMs) { + keepInReservoir( + deltas.stickyStatementMs, + deltas.stickyStatementsObserved++, + durationMs, + STICKY_STATEMENT_RESERVOIR_LIMIT + ) + } let connection = deltas.stickySlotByConnection.get(connectionId) if (!connection) { if (deltas.stickySlotByConnection.size >= STICKY_CONNECTION_LIMIT) { @@ -679,6 +696,8 @@ export class RelayObservability implements RelayRuntimeObserver { role: this.identity.role, cellId: this.identity.cellId, region: this.identity.region, + // Which zone this instance runs in; read once at boot, absent until then or off Cloud Run. + ...(this.instanceZone === undefined ? {} : { instanceZone: this.instanceZone }), ...counts, forwardedBytesDelta: deltas.forwardedBytes, authSuccessesDelta: deltas.authSuccesses, @@ -871,6 +890,8 @@ function stickySlotFields(deltas: RelayMetricDeltas): Record { stickySqlMsP99: sql.p99, stickyStatementsP50: statements.p50, stickyStatementsMax: statements.max, + // This instance's round-trip speed over every connection; slow-speed directors are the ones that stall. + stickyStatementMsP50: latencySummary(deltas.stickyStatementMs).p50, stickySqlByConnection: byConnection, // One per request-connection pair that found the window's table full. stickySqlConnectionSamplesDroppedDelta: deltas.stickySlotConnectionSamplesDropped