mirror of
https://github.com/stablyai/orca.git
synced 2026-11-01 09:37:33 +00:00
* fix(relay): the cell switch tool changes only the named switches and checks every key on read-back A cell resets a key missing from its switch file to its default, so the tool now reads the current object and changes only what --set names. The read-back compares every known key and fails when the cell reports ignoredKeys. * fix(relay): the cell switch tool refuses a switch the cell's image does not support It reads supportedFlags from runtime-status (#27057). An image from before that field takes only readinessLocal and ticketCheck off/shadow, so ticketCheck=enforce cannot void an old cell's object. * fix(relay): the switch tool validates the whole object, refuses voided files, and records admitMode in Postgres Every key and value of the object to be written is checked against what the image accepts (an image without the list: only the keys it applies, ticketCheck off/shadow). A file the cell did not apply is refused. admitMode=reserve needs a typed RESERVE <cell>; Postgres records reserve before the file is written, and db only after the cell reports every control leased again. * feat(relay): the switch tool knows reregisterInFlight (1-16), the flip-back pace * fix(relay): the switch tool closes the cache gap before reserve, undoes a refused write, and has a break-glass db flip After recording reserve in Postgres it waits past every director's 5 s cache and records it again (re-running the open-flow check) before writing the file. A file write refused outright (412 or 4xx) puts Postgres back to db; after a 5xx it stays reserve. An object the cell did not apply is repaired when the new object passes the image check. Break glass (typed BREAK GLASS <cell>) records db for a dead or unreachable reserve cell without its read-back. * fix(relay): a director deploy refuses a traffic move across reserve-mode support while any cell is reserve * fix(relay): the flip-back wait scales with the cell's controls and pace, and says how to finish * fix(relay): the switch tool refuses to re-arm a tripped cell, breaks glass only on a silent one, and undoes reserve if the second post is refused * feat(relay): director deploys set reserve placement, and refuse turning it off while any cell is reserve * fix(relay): Terraform keeps the deploy-owned placement and shadow cells, and the deploy guard reads placement as built and refuses dropping a reserve cell from the shadow list * fix(relay): director deploys take --admin-audience alone, name a drifted deploy identity, and carry the shadow seat-feed cells Staging passes --admin-audience for the reserve guard without durable rehome verification, which the parser refused. Only the generation and control origin now travel together, and they still need the audience. The guard decodes the caller from its token and refuses, by name, a revision whose ORCA_RELAY_DEPLOY_SERVICE_ACCOUNT is another account, instead of reading into a 401. --shadow-seat-feed-cells (preserve, all, or a cell list) is a workflow input on both director deploys; preserve keeps the serving value. A contract test parses every workflow's real argv for both deploy scripts. * fix(relay): the flag tool paces a flip back from the pool size the cell reports An image from before the field falls back to the deployed sizes.
210 lines
8.4 KiB
JavaScript
210 lines
8.4 KiB
JavaScript
import assert from 'node:assert/strict'
|
|
import { readFileSync } from 'node:fs'
|
|
import test from 'node:test'
|
|
import { fileURLToPath } from 'node:url'
|
|
import {
|
|
classifyRelayServiceDescribeFailure,
|
|
readRelayServingRegionalPlacementVersion
|
|
} from './read-relay-serving-regional-placement-version.mjs'
|
|
|
|
const input = {
|
|
project: 'onorca-cloud',
|
|
region: 'us-central1',
|
|
service: 'orca-cloud-relay',
|
|
bootstrap_version: '7'
|
|
}
|
|
|
|
function revision(version = '11') {
|
|
return {
|
|
spec: {
|
|
containers: [{
|
|
env: [{
|
|
name: 'ORCA_RELAY_REGIONAL_PLACEMENT_ENABLED',
|
|
valueSource: {
|
|
secretKeyRef: {
|
|
secret: 'orca-cloud-relay-regional-placement-enabled',
|
|
version
|
|
}
|
|
}
|
|
}]
|
|
}]
|
|
}
|
|
}
|
|
}
|
|
|
|
// Why: `gcloud run revisions describe --format=json` emits the Knative v1 shape, where the
|
|
// secret lives in `name` and the version in `key`, and `name` may be the full resource path.
|
|
function v1Revision(name, key) {
|
|
return {
|
|
spec: {
|
|
containers: [{
|
|
env: [{
|
|
name: 'ORCA_RELAY_REGIONAL_PLACEMENT_ENABLED',
|
|
valueFrom: { secretKeyRef: { name, key } }
|
|
}]
|
|
}]
|
|
}
|
|
}
|
|
}
|
|
|
|
function serving() {
|
|
return { status: { traffic: [{ revisionName: 'relay-serving', percent: 100 }] } }
|
|
}
|
|
|
|
test('reads the exact version from the sole traffic-serving revision', () => {
|
|
const calls = []
|
|
const result = readRelayServingRegionalPlacementVersion(input, {
|
|
run: (args) => {
|
|
calls.push(args)
|
|
return calls.length === 1
|
|
? {
|
|
status: {
|
|
traffic: [
|
|
{ revisionName: 'relay-failed-latest', tag: 'candidate' },
|
|
{ revisionName: 'relay-serving', percent: 100 }
|
|
]
|
|
}
|
|
}
|
|
: revision()
|
|
}
|
|
})
|
|
|
|
assert.deepEqual(result, { version: '11', cohort_percent: '0', reserve_placement: '', shadow_seat_feed_cells: '' })
|
|
assert.equal(calls[1][3], 'relay-serving')
|
|
})
|
|
|
|
test('reads the gcloud v1 secret reference shape by bare id and by full resource path', () => {
|
|
for (const name of [
|
|
'orca-cloud-relay-regional-placement-enabled',
|
|
'projects/120364513935/secrets/orca-cloud-relay-regional-placement-enabled'
|
|
]) {
|
|
assert.deepEqual(readRelayServingRegionalPlacementVersion(input, {
|
|
run: (args) => args[1] === 'services' ? serving() : v1Revision(name, '1')
|
|
}), { version: '1', cohort_percent: '0', reserve_placement: '', shadow_seat_feed_cells: '' })
|
|
}
|
|
})
|
|
|
|
test('rejects a v1 reference that names another secret or a floating version', () => {
|
|
assert.throws(() => readRelayServingRegionalPlacementVersion(input, {
|
|
run: (args) => args[1] === 'services'
|
|
? serving()
|
|
: v1Revision('projects/120364513935/secrets/some-other-secret', '1')
|
|
}), /secret reference is invalid/)
|
|
assert.throws(() => readRelayServingRegionalPlacementVersion(input, {
|
|
run: (args) => args[1] === 'services'
|
|
? serving()
|
|
: v1Revision('orca-cloud-relay-regional-placement-enabled', 'latest')
|
|
}), /secret reference is invalid/)
|
|
})
|
|
|
|
test('falls back only when the service or setting is absent', () => {
|
|
const notFound = new Error('not found')
|
|
notFound.code = 'NOT_FOUND'
|
|
assert.deepEqual(readRelayServingRegionalPlacementVersion(input, {
|
|
run: () => { throw notFound }
|
|
}), { version: '7', cohort_percent: '0', reserve_placement: '', shadow_seat_feed_cells: '' })
|
|
assert.deepEqual(readRelayServingRegionalPlacementVersion(input, {
|
|
run: (args) => args[1] === 'services'
|
|
? { status: { traffic: [{ revisionName: 'relay-serving', percent: 100 }] } }
|
|
: { spec: { containers: [{ env: [] }] } }
|
|
}), { version: '7', cohort_percent: '0', reserve_placement: '', shadow_seat_feed_cells: '' })
|
|
})
|
|
|
|
test('classifies real absent-service stderr without weakening revision failures', () => {
|
|
const serviceArgs = ['run', 'services', 'describe', 'missing-service']
|
|
const stderr = 'ERROR: (gcloud.run.services.describe) Cannot find service [missing-service]'
|
|
assert.equal(classifyRelayServiceDescribeFailure(serviceArgs, stderr), 'NOT_FOUND')
|
|
assert.equal(
|
|
classifyRelayServiceDescribeFailure(['run', 'revisions', 'describe', 'missing-revision'], stderr),
|
|
'GCLOUD_FAILED'
|
|
)
|
|
assert.equal(classifyRelayServiceDescribeFailure(serviceArgs, 'PERMISSION_DENIED'), 'GCLOUD_FAILED')
|
|
})
|
|
|
|
test('rejects ambiguous traffic, malformed references, and read failures', () => {
|
|
assert.throws(() => readRelayServingRegionalPlacementVersion(input, {
|
|
run: () => ({
|
|
status: { traffic: [{ revisionName: 'a', percent: 50 }, { revisionName: 'b', percent: 50 }] }
|
|
})
|
|
}), /exactly one revision/)
|
|
assert.throws(() => readRelayServingRegionalPlacementVersion(input, {
|
|
run: (args) => args[1] === 'services'
|
|
? { status: { traffic: [{ revisionName: 'relay-serving', percent: 100 }] } }
|
|
: revision('latest')
|
|
}), /secret reference is invalid/)
|
|
const denied = new Error('denied')
|
|
denied.code = 'GCLOUD_FAILED'
|
|
assert.throws(() => readRelayServingRegionalPlacementVersion(input, {
|
|
run: () => { throw denied }
|
|
}), denied)
|
|
})
|
|
|
|
|
|
test('preserves the serving cohort including explicit disable across later Terraform plans', () => {
|
|
for (const value of ['0', '1', '17', '100']) {
|
|
const servingRevision = revision()
|
|
servingRevision.spec.containers[0].env.push({ name: 'ORCA_RELAY_REGION_CORRECTION_COHORT_PERCENT', value })
|
|
assert.deepEqual(readRelayServingRegionalPlacementVersion(input, {
|
|
run: (args) => args[1] === 'services' ? serving() : servingRevision
|
|
}), { version: '11', cohort_percent: value, reserve_placement: '', shadow_seat_feed_cells: '' })
|
|
}
|
|
})
|
|
|
|
test('fails closed on malformed, secret-backed or duplicate cohorts rather than resetting them', () => {
|
|
const name = 'ORCA_RELAY_REGION_CORRECTION_COHORT_PERCENT'
|
|
const cases = [
|
|
[{ name, value: '101' }], [{ name, value: '-1' }], [{ name, value: '1.5' }],
|
|
[{ name, value: '' }], [{ name, value: '01' }], [{ name, value: 1 }],
|
|
[{ name, valueFrom: { secretKeyRef: { name: 'unexpected', key: '1' } } }],
|
|
[{ name, value: '1' }, { name, value: '2' }]
|
|
]
|
|
for (const settings of cases) {
|
|
const servingRevision = revision()
|
|
servingRevision.spec.containers[0].env.push(...settings)
|
|
assert.throws(() => readRelayServingRegionalPlacementVersion(input, {
|
|
run: (args) => args[1] === 'services' ? serving() : servingRevision
|
|
}), /cohort is invalid/)
|
|
}
|
|
})
|
|
|
|
test('carries the deploy-owned placement and shadow cells, so an apply keeps them', () => {
|
|
const servingRevision = revision()
|
|
servingRevision.spec.containers[0].env.push(
|
|
{ name: 'ORCA_RELAY_RESERVE_PLACEMENT', value: 'on' },
|
|
{ name: 'ORCA_RELAY_SHADOW_SEAT_FEED_CELLS', value: 'production-gce-c1,production-gce-c2' }
|
|
)
|
|
assert.deepEqual(readRelayServingRegionalPlacementVersion(input, {
|
|
run: (args) => args[1] === 'services' ? serving() : servingRevision
|
|
}), {
|
|
version: '11',
|
|
cohort_percent: '0',
|
|
reserve_placement: 'on',
|
|
shadow_seat_feed_cells: 'production-gce-c1,production-gce-c2'
|
|
})
|
|
})
|
|
|
|
test('fails closed on a malformed, secret-backed or duplicate placement or shadow-cell setting', () => {
|
|
const cases = [
|
|
[{ name: 'ORCA_RELAY_RESERVE_PLACEMENT', value: 'yes' }],
|
|
[{ name: 'ORCA_RELAY_RESERVE_PLACEMENT', value: 'on' }, { name: 'ORCA_RELAY_RESERVE_PLACEMENT', value: 'off' }],
|
|
[{ name: 'ORCA_RELAY_RESERVE_PLACEMENT', valueFrom: { secretKeyRef: { name: 'x', key: '1' } } }],
|
|
[{ name: 'ORCA_RELAY_SHADOW_SEAT_FEED_CELLS', value: 'c1;c2' }],
|
|
[{ name: 'ORCA_RELAY_SHADOW_SEAT_FEED_CELLS', value: '' }]
|
|
]
|
|
for (const settings of cases) {
|
|
const servingRevision = revision()
|
|
servingRevision.spec.containers[0].env.push(...settings)
|
|
assert.throws(() => readRelayServingRegionalPlacementVersion(input, {
|
|
run: (args) => args[1] === 'services' ? serving() : servingRevision
|
|
}), /is invalid/)
|
|
}
|
|
})
|
|
|
|
test('Terraform sets the carried settings only when the serving revision has them', () => {
|
|
const terraform = readFileSync(fileURLToPath(new URL('../../infra/terraform/relay.tf', import.meta.url)), 'utf8')
|
|
const block = /dynamic "env" \{([\s\S]*?)\n {6}\}/.exec(terraform)?.[1] ?? ''
|
|
assert.match(block, /ORCA_RELAY_RESERVE_PLACEMENT *= data\.external\.relay_serving_regional_placement_version\.result\.reserve_placement/)
|
|
assert.match(block, /ORCA_RELAY_SHADOW_SEAT_FEED_CELLS *= data\.external\.relay_serving_regional_placement_version\.result\.shadow_seat_feed_cells/)
|
|
assert.match(block, /if value != ""/)
|
|
})
|