starred/orca
1
0
Fork 0
mirror of https://github.com/stablyai/orca.git synced 2026-11-03 18:47:40 +00:00
Code Issues Packages Projects Releases Wiki Activity

fix(relay): director reaches Cloud SQL through a proxy sidecar (staging), adoptable deploy shape, event-loop stall alert (#27406)

Browse Source 
* fix(relay): director reaches Cloud SQL through a proxy sidecar (staging), gated event-loop stall alert

On Cloud Run's built-in /cloudsql socket a connect() blocks the director's whole
event loop until the connector's upstream dial ends, 10 s per attempt while the
instance starts. Staging froze both directors for 134 s on 2026-10-11 and
production four directors for 40-50 s on 2026-09-28.

relay_director_cloud_sql_proxy_sidecar runs the Cloud SQL Auth Proxy beside the
director on 127.0.0.1:5432, as the GCE cells already do, with its own TCP
database URL secret. relay_event_loop_stall_alert adds the event_loop_ms_max
metric and a 5 s stall alert. Both default off, so production plans unchanged;
staging turns both on.

* fix(relay): adopt a reviewed Terraform director shape on deploy; counter-based stall alert; cheaper probed sidecar

- deploy-relay-blue-green: --shape-baseline-revision names the 0% revision a Terraform
  apply created; the deploy preserves that shape instead of the serving one. It must be the
  latest created, unserved revision on the serving image. Wired into the staging and
  production director workflows behind ADOPT_RELAY_DIRECTOR_TERRAFORM_SHAPE.
- Event-loop stall alert is now a gated counter metric on eventLoopDelayMsMax>=5000.
- Sidecar gets a /liveness probe and 0.25 vCPU (Cloud Run rejects only a total under 1 vCPU
  with concurrency above 1; checked with validateOnly against the API).
- staging.tfvars pins the serving director's concurrency 1000 and timeout 3600 s, so the
  adopted apply shape does not fall to the 80 / 30 s defaults.

* fix(relay): shape adoption refuses identity, topology, pruning and image changes; GOMAXPROCS=1 on the sidecar

parseArguments now refuses --shape-baseline-revision with a runtime identity bootstrap, a
director topology change or revision pruning, so every caller is covered, and the deploy
refuses a shape baseline whose image differs from the serving one. Both director workflows
also refuse the combinations up front. The proxy sidecar runs with GOMAXPROCS=1 to match its
0.25 vCPU quota. The long-hold alert comment is back above its resource.
This commit is contained in:
Jinwoo Hong
2026-10-11 04:30:27 -04:00
committed by GitHub
parent 12a8fc0166
commit 2bdb55350d
10 changed files with 438 additions and 19 deletions
Download Patch File Download Diff File Expand all files Collapse all files
cloud/dev/scripts/deploy-relay-blue-green.mjs
+55 -2
View File
@@ -377,6 +377,28 @@ export function parseArguments(argv) {
if (!['true', 'false'].includes(values['bootstrap-runtime-identity'] ?? 'false')) {
throw new Error('--bootstrap-runtime-identity must be true or false')
}
if (
values['shape-baseline-revision'] !== undefined &&
!/^[a-z][a-z0-9-]{0,62}$/.test(values['shape-baseline-revision'])
) {
throw new Error('--shape-baseline-revision must be a Cloud Run revision name')
}
if (values['shape-baseline-revision'] !== undefined) {
if (values.role !== 'director') {
throw new Error('--shape-baseline-revision requires --role director')
}
// A shape adoption moves only what the reviewed apply changed: no identity, topology or
// pruning rides along, whichever caller builds the arguments.
if (values['bootstrap-runtime-identity'] === 'true') {
throw new Error('--shape-baseline-revision cannot combine with a runtime identity bootstrap')
}
if (values['capacity-cell-id'] !== undefined || values['director-cells-json'] !== undefined) {
throw new Error('--shape-baseline-revision cannot combine with a director topology change')
}
if (values['prune-revisions'] === 'true') {
throw new Error('--shape-baseline-revision cannot combine with revision pruning')
}
}
if (values.role === 'director') {
directorDeploymentEnvironment(values)
projectServiceAccount(values, 'runtime-service-account')
@@ -896,6 +918,31 @@ function assertDirectorRevisionIdentity(revision, config) {
}
}
// The reviewed shape a deploy must preserve. Normally the serving revision's; after a Terraform
// apply changes the template (a sidecar, a volume, a secret reference), the apply's own 0%
// revision, named by the operator, is the reviewed shape instead.
export function directorShapeBaseline(config, service, servingRevision, describe) {
const named = config['shape-baseline-revision']
if (named === undefined) return servingRevision
if (named === activeRevision(service)) {
throw new Error('shape baseline revision is already serving; deploy without it')
}
if (named !== service.status?.latestCreatedRevisionName) {
throw new Error('shape baseline revision is not the latest created revision')
}
const servingImage = servingRevision.spec?.containers?.[0]?.image
// A shape change and an image change never ride together: each is reviewed on its own.
if (config.image?.split('@').at(-1) !== servingImage?.split('@').at(-1)) {
throw new Error('a shape baseline deploy must keep the serving image')
}
const baseline = describe(config, named)
// Terraform ignores the image, so its revision runs what is serving; anything else was not an apply.
if (baseline.spec?.containers?.[0]?.image !== servingImage) {
throw new Error('shape baseline revision does not run the serving image')
}
return baseline
}
export async function deployDirector(config, tag, overrides = {}) {
const operations = {
deployCandidate,
@@ -916,6 +963,12 @@ export async function deployDirector(config, tag, overrides = {}) {
// restating it here; public admission is per-instance, so losing it shrinks fleet capacity.
const initialService = operations.describeService(config)
const servingRevision = operations.describeRevision(config, activeRevision(initialService))
const shapeBaseline = directorShapeBaseline(
config,
initialService,
servingRevision,
operations.describeRevision
)
const bootstrapRuntimeIdentity = config['bootstrap-runtime-identity'] === 'true'
if (bootstrapRuntimeIdentity) {
if (
@@ -1023,7 +1076,7 @@ export async function deployDirector(config, tag, overrides = {}) {
const rollbackRevision = operations.describeRevision(config, rollback.revision)
assertDirectorRevisionIdentity(rollbackRevision, config)
assertPreservedRevisionShape(
servingRevision,
shapeBaseline,
rollbackRevision,
mutableEnvironment,
bootstrapRuntimeIdentity
@@ -1058,7 +1111,7 @@ export async function deployDirector(config, tag, overrides = {}) {
const candidateRevision = operations.describeRevision(config, candidate.revision)
assertDirectorRevisionIdentity(candidateRevision, config)
assertPreservedRevisionShape(
servingRevision,
shapeBaseline,
candidateRevision,
mutableEnvironment,
bootstrapRuntimeIdentity
cloud/dev/scripts/deploy-relay-blue-green.test.mjs
+114 -5
View File
@@ -441,7 +441,9 @@ function directorHarness({
reserveCells = {},
servingEnvironment = {},
placements = {},
callerEmail
callerEmail,
// The revision template's Cloud SQL transport; a Terraform apply changes it under the serving one.
templateTransport
} = {}) {
const state = {
activeRevision: 'relay-00001-old',
@@ -468,7 +470,9 @@ function directorHarness({
}
]
]),
nextRevision: 2
nextRevision: 2,
latestCreated: 'relay-00001-old',
templateTransport
}
const removed = []
const healthProtocols = []
@@ -477,6 +481,7 @@ function directorHarness({
describeService: () => ({
status: {
url: 'https://relay-hash-uc.a.run.app',
latestCreatedRevisionName: state.latestCreated,
traffic: [
{ percent: 100, revisionName: state.activeRevision },
...[...state.tags].map(([tag, revisionName]) => ({
@@ -493,13 +498,15 @@ function directorHarness({
metadata: {
annotations: {
'autoscaling.knative.dev/minScale': String(value?.minimum ?? 0),
'autoscaling.knative.dev/maxScale': String(value?.maximum ?? 5)
'autoscaling.knative.dev/maxScale': String(value?.maximum ?? 5),
...value?.transport?.annotations
}
},
spec: {
serviceAccountName: value?.serviceAccount,
containers: [
{
name: value?.transport?.name,
image: value?.image,
env: [
...Object.entries(value?.env ?? {}).map(([name, value]) => ({ name, value })),
@@ -508,7 +515,8 @@ function directorHarness({
valueSource: { secretKeyRef }
}))
]
}
},
...value?.transport?.sidecars ?? []
]
}
}
@@ -529,8 +537,10 @@ function directorHarness({
serviceAccount:
config['runtime-service-account'] ??
state.revisions.get(state.activeRevision)?.serviceAccount,
image: image ?? state.revisions.get(state.activeRevision)?.image
image: image ?? state.revisions.get(state.activeRevision)?.image,
transport: state.templateTransport
})
state.latestCreated = revision
state.tags.set(tag, revision)
if (deployFailure) throw deployFailure
},
@@ -715,6 +725,105 @@ test('director deploy refuses to move traffic onto a candidate that lost the flo
assert.equal(harness.state.activeRevision, 'relay-00001-old')
})
const SERVING_IMAGE = `relay@sha256:${'f'.repeat(64)}`
// What a Terraform apply of relay_director_cloud_sql_proxy_sidecar puts in the template.
const SIDECAR_TRANSPORT = {
name: 'relay',
annotations: { 'run.googleapis.com/container-dependencies': '{"relay":["cloud-sql-proxy"]}' },
sidecars: [{ name: 'cloud-sql-proxy', image: 'gcr.io/cloud-sql-connectors/cloud-sql-proxy@sha256:1' }]
}
// The apply leaves its revision at 0%, as the director service declares no traffic block.
function applySidecarTemplate(harness) {
const serving = harness.state.revisions.get(harness.state.activeRevision)
harness.state.templateTransport = SIDECAR_TRANSPORT
harness.state.revisions.set('relay-00009-terraform', { ...serving, transport: SIDECAR_TRANSPORT })
harness.state.latestCreated = 'relay-00009-terraform'
}
test('a deploy after a sidecar apply refuses the new shape unless the apply revision is named', async () => {
const harness = directorHarness()
applySidecarTemplate(harness)
await assert.rejects(
deployDirector({}, 'candidate-new', harness.operations),
/unrelated revision shape/
)
assert.equal(harness.state.activeRevision, 'relay-00001-old')
})
test('a deploy adopts the shape of the named Terraform apply revision', async () => {
const harness = directorHarness()
applySidecarTemplate(harness)
await deployDirector(
{ image: SERVING_IMAGE, 'shape-baseline-revision': 'relay-00009-terraform' },
'candidate-new',
harness.operations
)
const active = harness.state.revisions.get(harness.state.activeRevision)
assert.equal(active.transport, SIDECAR_TRANSPORT)
assert.equal(
harness.state.revisions.get(harness.state.tags.get('selector-rollback')).transport,
SIDECAR_TRANSPORT
)
})
test('the named shape baseline must be the unserved latest revision on the serving image', async () => {
const stale = directorHarness()
applySidecarTemplate(stale)
await assert.rejects(
deployDirector({ 'shape-baseline-revision': 'relay-00000-stale' }, 'candidate-new', stale.operations),
/not the latest created revision/
)
const serving = directorHarness()
await assert.rejects(
deployDirector({ 'shape-baseline-revision': 'relay-00001-old' }, 'candidate-new', serving.operations),
/already serving/
)
const otherImage = directorHarness()
applySidecarTemplate(otherImage)
otherImage.state.revisions.get('relay-00009-terraform').image = `relay@sha256:${'e'.repeat(64)}`
await assert.rejects(
deployDirector({ image: SERVING_IMAGE, 'shape-baseline-revision': 'relay-00009-terraform' }, 'candidate-new', otherImage.operations),
/does not run the serving image/
)
const newImage = directorHarness()
applySidecarTemplate(newImage)
await assert.rejects(
deployDirector(
{ image: `relay@sha256:${'e'.repeat(64)}`, 'shape-baseline-revision': 'relay-00009-terraform' },
'candidate-new',
newImage.operations
),
/must keep the serving image/
)
for (const harness of [stale, serving, otherImage, newImage]) {
assert.equal(harness.state.activeRevision, 'relay-00001-old')
}
})
test('--shape-baseline-revision takes only a revision name, and only for a director', () => {
const base = ['--project', 'p', '--region', 'r', '--service', 's', '--image', 'i@sha256:' + 'a'.repeat(64), '--release-id', 'x']
assert.throws(
() => parseArguments([...base, '--role', 'director', '--shape-baseline-revision', 'Bad_Name']),
/must be a Cloud Run revision name/
)
assert.throws(
() => parseArguments([...base, '--role', 'cell', '--director-origin', 'https://d', '--admin-audience', 'a', '--shape-baseline-revision', 'relay-1']),
/--shape-baseline-revision requires --role director/
)
const director = [...base, '--role', 'director', '--shape-baseline-revision', 'relay-1']
assert.equal(parseArguments(director)['shape-baseline-revision'], 'relay-1')
for (const [extra, refusal] of [
[['--bootstrap-runtime-identity', 'true'], /cannot combine with a runtime identity bootstrap/],
[['--capacity-cell-id', 'c1', '--director-cells-json', '[]'], /cannot combine with a director topology change/],
[['--director-cells-json', '[]'], /cannot combine with a director topology change/],
[['--prune-revisions', 'true'], /cannot combine with revision pruning/]
]) {
assert.throws(() => parseArguments([...director, ...extra]), refusal)
}
})
test('director deploy rejects unrelated revision-shape drift', async () => {
const harness = directorHarness()
const describeRevision = harness.operations.describeRevision