mirror of
https://github.com/stablyai/orca.git
synced 2026-09-22 08:02:28 +00:00
Jinwoo Hong
399306c171
c17 and c18 hold no hosts and sit outside general admission, so rolling one displaces nobody. They are the only zero-displacement canary for a new cell image, but the same-cap wave refused them at the dispatch validator and would have promoted them to general at the end if it had not. Add them to the approved list and teach the wave a cell's entry admission class: the precheck demands the class the cell is declared to serve in, the restore hands it back that class, the isolate on an already-isolated cell is asserted to change nothing, and the selector generation advances by 2 for a general cell and by 0 for a migration-only one. One wave may not mix the two, because every cell after the first offsets from a single per-wave delta. Neither cell is a declared regional-rehome source, so its template carries no rehome trust lines. The source-membership guard now fires exactly when a roll expects those lines instead of for every US cell, which is the invariant it was standing in for, and which limits c17 and c18 to rehome protocol 0.
458 lines
18 KiB
JavaScript
458 lines
18 KiB
JavaScript
import assert from 'node:assert/strict'
|
|
import { spawnSync } from 'node:child_process'
|
|
import { readFileSync } from 'node:fs'
|
|
import { describe, it } from 'node:test'
|
|
import { parseProductionCapacityCellArguments } from './prepare-relay-production-capacity-canary.mjs'
|
|
import {
|
|
SAME_CAP_CELLS,
|
|
SAME_CAP_MIGRATION_ONLY_CELLS,
|
|
entryAdmission,
|
|
selectorWaveDelta
|
|
} from './relay-production-same-cap-wave.mjs'
|
|
import { readRelayWorkflow } from './relay-repository.mjs'
|
|
import { validateCapacityPlan } from './validate-relay-capacity-plan.mjs'
|
|
|
|
const workflow = readRelayWorkflow('deploy-relay-production-same-cap-job.yml')
|
|
const capacityWorkflow = readRelayWorkflow('deploy-relay-production-capacity-job.yml')
|
|
const production = readFileSync(
|
|
new URL('../../infra/terraform/environments/production.tfvars', import.meta.url),
|
|
'utf8'
|
|
)
|
|
const REHOME_SOURCE_CELLS = rehomeSourceCells()
|
|
const DIRECTOR_IDENTITY = 'relay-director@onorca-cloud.iam.gserviceaccount.com'
|
|
const AUDIENCE = 'https://relay.onorca.dev/v1/admin/host-drain'
|
|
const ROLLBACK_IMAGE = `us-central1-docker.pkg.dev/p/orca-cloud/relay@sha256:${'d'.repeat(64)}`
|
|
const TARGET_IMAGE = `us-central1-docker.pkg.dev/p/orca-cloud/relay@sha256:${'e'.repeat(64)}`
|
|
|
|
// The startup template emits rehome trust only for cells in this list, so it is what decides
|
|
// whether a cell's plan may carry those lines at all.
|
|
function rehomeSourceCells() {
|
|
const start = production.indexOf('relay_region_rehome_source_cell_ids = [')
|
|
assert.notEqual(start, -1, 'production.tfvars has no rehome source cell list')
|
|
const end = production.indexOf(']', start)
|
|
assert.notEqual(end, -1, 'the rehome source cell list is unterminated')
|
|
return new Set(
|
|
[...production.slice(start, end).matchAll(/"([^"]+)"/g)].map(([, cell]) => cell)
|
|
)
|
|
}
|
|
|
|
// The job cross-checks its pinned pool against the committed map; model the same read.
|
|
function tfvarsDatabasePoolMax(cellId) {
|
|
return tfvarsCellBlock(cellId).match(/database_pool_max\s*=\s*(\d+)/)?.[1] ?? '10'
|
|
}
|
|
|
|
function tfvarsHardCap(cellId) {
|
|
const cap = /connection_hard_cap\s*=\s*(\d+)/.exec(tfvarsCellBlock(cellId))?.[1]
|
|
assert.notEqual(cap, undefined, `${cellId} has no connection_hard_cap`)
|
|
return cap
|
|
}
|
|
|
|
function tfvarsCellBlock(cellId) {
|
|
const start = production.indexOf(`"${cellId}" = {`)
|
|
assert.notEqual(start, -1, `${cellId} is missing from production.tfvars`)
|
|
return production.slice(start, production.indexOf('\n }', start))
|
|
}
|
|
|
|
function startupScript({ cap, image, trusted, pool }) {
|
|
return [
|
|
` printf 'ORCA_RELAY_CELL_CONNECTION_HARD_CAP=%s\\n' '${cap}'`,
|
|
` printf 'ORCA_RELAY_CELL_CONNECTION_UNOBSERVED_BOUND=%s\\n' '60'`,
|
|
...(pool === undefined
|
|
? []
|
|
: [` printf 'ORCA_RELAY_DATABASE_POOL_MAX=%s\\n' '${pool}'`]),
|
|
...(trusted ? [
|
|
` printf 'ORCA_RELAY_REHOME_DIRECTOR_SERVICE_ACCOUNT=%s\\n' '${DIRECTOR_IDENTITY}'`,
|
|
` printf 'ORCA_RELAY_REHOME_AUDIENCE=%s\\n' '${AUDIENCE}'`
|
|
] : []),
|
|
`printf 'ORCA_RELAY_IMAGE_DIGEST=%s\\n' '${image.split('@')[1]}'`,
|
|
`docker pull '${image}'`,
|
|
'docker run --detach \\',
|
|
' --name orca-relay \\',
|
|
` '${image}'`
|
|
].join('\n')
|
|
}
|
|
|
|
// The exact shape the apply step's plan has: template replaced, MIG rebound to it.
|
|
function rollPlan({ cellId, cap, protocol, pool }) {
|
|
return {
|
|
configuration: {
|
|
root_module: {
|
|
resources: [{
|
|
address: 'google_compute_instance_group_manager.relay_gce_cell',
|
|
expressions: {
|
|
version: [{
|
|
instance_template: {
|
|
references: [
|
|
'google_compute_instance_template.relay_gce_cell',
|
|
'each.key'
|
|
]
|
|
},
|
|
name: { constant_value: 'primary' }
|
|
}]
|
|
}
|
|
}]
|
|
}
|
|
},
|
|
resource_changes: [
|
|
{
|
|
address: `google_compute_instance_template.relay_gce_cell[${JSON.stringify(cellId)}]`,
|
|
change: {
|
|
actions: ['create', 'delete'],
|
|
before: {
|
|
metadata_startup_script: startupScript({
|
|
cap,
|
|
image: ROLLBACK_IMAGE,
|
|
trusted: protocol >= 1,
|
|
// The live template predates the reviewed pool raise, as every asia cell's does.
|
|
pool: pool === undefined ? undefined : '10'
|
|
})
|
|
},
|
|
after: {
|
|
metadata_startup_script: startupScript({
|
|
cap,
|
|
image: TARGET_IMAGE,
|
|
trusted: protocol >= 1,
|
|
pool
|
|
}),
|
|
self_link: null
|
|
},
|
|
after_unknown: { self_link: true }
|
|
}
|
|
},
|
|
{
|
|
address: `google_compute_instance_group_manager.relay_gce_cell[${JSON.stringify(cellId)}]`,
|
|
change: {
|
|
actions: ['update'],
|
|
before: { target_size: 1, version: [{ instance_template: 'old' }] },
|
|
after: { target_size: 1, version: [{ instance_template: null }] },
|
|
after_unknown: { version: [{ instance_template: true }] }
|
|
}
|
|
}
|
|
]
|
|
}
|
|
}
|
|
|
|
function hostname(cellId) {
|
|
return cellId.slice('production-gce-'.length)
|
|
}
|
|
|
|
// The job resolves cap, region, and pool from the cell id before any admin call; run that block
|
|
// alone. An empty pool is the root default, which the startup template emits no line for.
|
|
function resolveCellShape(cellId) {
|
|
const start = workflow.indexOf(' TARGET_HOSTNAME="${TARGET_CELL_ID#production-gce-}"')
|
|
assert.notEqual(start, -1, 'the same-cap cell shape block is missing')
|
|
const end = workflow.indexOf('\n esac\n', start)
|
|
assert.notEqual(end, -1, 'the same-cap cell shape block has no esac')
|
|
const script = workflow.slice(start, end + '\n esac'.length).replace(/^ {10}/gm, '')
|
|
return spawnSync('bash', [
|
|
'-euo',
|
|
'pipefail',
|
|
'-c',
|
|
`${script}\necho "\${EXPECTED_REGION} \${EXPECTED_HARD_CAP} pool=\${EXPECTED_DATABASE_POOL_MAX}"`
|
|
], { env: { ...process.env, TARGET_CELL_ID: cellId }, encoding: 'utf8' })
|
|
}
|
|
|
|
function cellShape(cellId) {
|
|
const resolved = resolveCellShape(cellId)
|
|
assert.equal(resolved.status, 0, `${cellId}: ${resolved.stderr}`)
|
|
const [, cap, pool] = resolved.stdout.trim().split(' ')
|
|
return { cap: Number(cap), pool: pool.slice('pool='.length) || undefined }
|
|
}
|
|
|
|
// The class block runs before checkout-independent work and decides the whole wave shape.
|
|
function resolveCellClass(cellId) {
|
|
return spawnSync('bash', [
|
|
'-euo',
|
|
'pipefail',
|
|
'-c',
|
|
`${jobBlock(
|
|
' CELL_CLASS="$(node dev/scripts/relay-production-same-cap-wave.mjs cell-class \\',
|
|
' SELECTOR_WAVE_DELTA="$(jq -er \'.selectorWaveDelta\' <<< "${CELL_CLASS}")"'
|
|
)}\necho "\${ENTRY_ADMISSION} \${SELECTOR_WAVE_DELTA}"`
|
|
], { cwd: new URL('../..', import.meta.url), env: { ...process.env, TARGET_CELL_ID: cellId }, encoding: 'utf8' })
|
|
}
|
|
|
|
function generationBlock() {
|
|
return `${jobBlock(
|
|
' if test "${DEPLOY_MODE}" = verify; then',
|
|
' fi'
|
|
)}\necho "\${EFFECTIVE_SELECTOR_GENERATION}"`
|
|
}
|
|
|
|
// The job derives both memberships in one block; run that block alone for each class.
|
|
function membership(env) {
|
|
const script = `${jobBlock(
|
|
' RESTORED_MIGRATION_CELLS="$(jq -rn \\',
|
|
' fi'
|
|
)}\njq -cn --arg a "\${ISOLATED_MIGRATION_CELLS}" --arg b "\${ISOLATED_GENERAL_CELLS}" \\
|
|
--arg c "\${RESTORED_MIGRATION_CELLS}" --arg d "\${RESTORED_GENERAL_CELLS}" \\
|
|
'{isolatedMigration:$a,isolatedGeneral:$b,restoredMigration:$c,restoredGeneral:$d}'`
|
|
const resolved = spawnSync('bash', ['-euo', 'pipefail', '-c', script], {
|
|
env: { ...process.env, ...env },
|
|
encoding: 'utf8'
|
|
})
|
|
assert.equal(resolved.status, 0, resolved.stderr)
|
|
return JSON.parse(resolved.stdout)
|
|
}
|
|
|
|
function jobBlock(firstLine, lastLine) {
|
|
const start = workflow.indexOf(`${firstLine}\n`)
|
|
assert.notEqual(start, -1, `the job has no ${firstLine.trim()}`)
|
|
const end = workflow.indexOf(`\n${lastLine}\n`, start)
|
|
assert.notEqual(end, -1, `that block has no ${lastLine.trim()}`)
|
|
return workflow.slice(start, end + lastLine.length + 1).replace(/^ {10}/gm, '')
|
|
}
|
|
|
|
describe('same-cap roll scripts accept every same-cap cell', () => {
|
|
it('parses every wave cell through the same-cap canary allowlist', () => {
|
|
for (const cellId of SAME_CAP_CELLS) {
|
|
for (const mode of ['isolate', 'drain', 'activate']) {
|
|
assert.deepEqual(parseProductionCapacityCellArguments([
|
|
'--director-origin', 'https://relay.onorca.dev',
|
|
'--cell-origin', `https://${hostname(cellId)}.relay.onorca.dev`,
|
|
'--cell-id', cellId,
|
|
'--approved-cells', 'same-cap',
|
|
'--mode', mode
|
|
]), {
|
|
directorOrigin: 'https://relay.onorca.dev',
|
|
cellOrigin: `https://${hostname(cellId)}.relay.onorca.dev`,
|
|
cellId,
|
|
mode,
|
|
paceWindowMs: 0
|
|
})
|
|
}
|
|
}
|
|
})
|
|
|
|
it('resolves a cap, region, and pool for every wave cell and refuses anything else', () => {
|
|
for (const cellId of SAME_CAP_CELLS) {
|
|
const resolved = resolveCellShape(cellId)
|
|
assert.equal(resolved.status, 0, `${cellId}: ${resolved.stderr}`)
|
|
assert.match(
|
|
resolved.stdout.trim(),
|
|
/^(us-central1 1000 pool=|us-central1 600 pool=|asia-east2 3000 pool=16)$/,
|
|
cellId
|
|
)
|
|
assert.equal(tfvarsDatabasePoolMax(cellId), cellShape(cellId).pool ?? '10', cellId)
|
|
assert.equal(String(cellShape(cellId).cap), tfvarsHardCap(cellId), cellId)
|
|
}
|
|
assert.equal(resolveCellShape('production-gce-c12').status, 1)
|
|
assert.equal(resolveCellShape('production-gce-c30').status, 1)
|
|
})
|
|
|
|
it('passes the same-cap allowlist on every canary invocation the job runs', () => {
|
|
const invocations = workflow.split('prepare-relay-production-capacity-canary.mjs').slice(1)
|
|
assert.equal(invocations.length, 4)
|
|
for (const invocation of invocations) {
|
|
const lines = invocation.split('\n')
|
|
const end = lines.findIndex((line) => !line.endsWith('\\'))
|
|
const call = lines.slice(0, end + 1).join(' ')
|
|
assert.match(call, /--approved-cells same-cap/)
|
|
// The restore call picks its mode from the cell's entry admission class.
|
|
assert.match(call, /--mode (isolate|drain|activate|"\$\{RESTORE_MODE\}")/)
|
|
}
|
|
})
|
|
|
|
it('paces the drain it sends to the selected cell', () => {
|
|
const drain = workflow.split('--mode drain')[1] ?? ''
|
|
assert.match(drain.split('\n').slice(0, 2).join(' '), /--pace-window-ms "\$\{DRAIN_PACE_WINDOW_MS\}"/)
|
|
assert.match(workflow, /DRAIN_PACE_WINDOW_MS: '120000'/)
|
|
// The transition wait has to outlast the pacing window on top of the leases it waits on.
|
|
assert.match(workflow, /--activity restart-safe[\s\S]*?--timeout-ms 1020000/)
|
|
})
|
|
|
|
it('passes this cell\'s rehome protocol and pool on every plan validation the job runs', () => {
|
|
const invocations = workflow.split('validate-relay-capacity-plan.mjs').slice(1)
|
|
assert.equal(invocations.length, 2)
|
|
for (const invocation of invocations) {
|
|
const lines = invocation.split('\n')
|
|
const end = lines.findIndex((line) => !line.trimEnd().endsWith('\\'))
|
|
const call = lines.slice(0, end + 1).join(' ')
|
|
assert.match(call, /--mode same-cap-cell/)
|
|
assert.match(call, /--regional-rehome-protocol "\$\{DESIRED_REHOME_PROTOCOL\}"/)
|
|
assert.match(call, /"\$\{POOL_ARGUMENTS\[@\]\}"/)
|
|
}
|
|
// Each of those steps must build the flag from the resolved pool, and only when there is one.
|
|
const builders = workflow.split(
|
|
'if test -n "${EXPECTED_DATABASE_POOL_MAX}"; then\n' +
|
|
' POOL_ARGUMENTS=(--database-pool-max "${EXPECTED_DATABASE_POOL_MAX}")'
|
|
)
|
|
assert.equal(builders.length, 3)
|
|
assert.equal(workflow.split('POOL_ARGUMENTS=()').length, 3)
|
|
})
|
|
|
|
it('validates a correct plan for every wave cell at that cell\'s rehome protocol', () => {
|
|
const trusted = SAME_CAP_CELLS.filter((cell) => REHOME_SOURCE_CELLS.has(cell))
|
|
// Only a declared rehome source may roll at a trusted protocol at all; the job refuses
|
|
// the rest before it plans, and the next test covers them at protocol 0.
|
|
assert.deepEqual(
|
|
SAME_CAP_CELLS.filter((cell) => !REHOME_SOURCE_CELLS.has(cell)),
|
|
SAME_CAP_MIGRATION_ONLY_CELLS
|
|
)
|
|
for (const [cellId, protocol] of trusted.flatMap((cell) => [[cell, 1], [cell, 3]])) {
|
|
const { cap, pool } = cellShape(cellId)
|
|
const config = {
|
|
mode: 'same-cap-cell',
|
|
cellId,
|
|
hardCap: cap,
|
|
unobservedBound: 60,
|
|
image: TARGET_IMAGE,
|
|
rollbackImage: ROLLBACK_IMAGE,
|
|
rehomeDirectorServiceAccount: DIRECTOR_IDENTITY,
|
|
rehomeAudience: AUDIENCE,
|
|
regionalRehomeProtocol: String(protocol),
|
|
databasePoolMax: pool
|
|
}
|
|
const plan = rollPlan({ cellId, cap, protocol, pool })
|
|
assert.deepEqual(
|
|
validateCapacityPlan(plan, config),
|
|
{ mode: 'same-cap-cell', changes: 2 },
|
|
cellId
|
|
)
|
|
// The other protocol must reject the same plan, or the flag decides nothing.
|
|
assert.throws(
|
|
() => validateCapacityPlan(plan, {
|
|
...config,
|
|
regionalRehomeProtocol: '0'
|
|
}),
|
|
/reviewed image and capacity/,
|
|
cellId
|
|
)
|
|
// Dropping the pin must reject a pinned cell, and adding one must reject a default cell.
|
|
assert.throws(
|
|
() => validateCapacityPlan(plan, {
|
|
...config,
|
|
databasePoolMax: pool === undefined ? '16' : undefined
|
|
}),
|
|
/reviewed image and capacity/,
|
|
cellId
|
|
)
|
|
}
|
|
})
|
|
|
|
it('validates a protocol-0 plan for a cell outside the rehome source list', () => {
|
|
const cellId = 'production-gce-c17'
|
|
assert.equal(REHOME_SOURCE_CELLS.has(cellId), false)
|
|
const config = {
|
|
mode: 'same-cap-cell',
|
|
cellId,
|
|
hardCap: 600,
|
|
unobservedBound: 60,
|
|
image: TARGET_IMAGE,
|
|
rollbackImage: ROLLBACK_IMAGE,
|
|
rehomeDirectorServiceAccount: DIRECTOR_IDENTITY,
|
|
rehomeAudience: AUDIENCE,
|
|
regionalRehomeProtocol: '0'
|
|
}
|
|
const plan = rollPlan({ cellId, cap: 600, protocol: 0 })
|
|
assert.deepEqual(validateCapacityPlan(plan, config), { mode: 'same-cap-cell', changes: 2 })
|
|
// Protocol 1 must reject a plan with no rehome lines, or the absent-line rule decides nothing.
|
|
assert.throws(
|
|
() => validateCapacityPlan(plan, { ...config, regionalRehomeProtocol: '1' }),
|
|
/reviewed image and capacity/
|
|
)
|
|
})
|
|
|
|
it('resolves the class and selector delta the wave validator declares', () => {
|
|
for (const cellId of SAME_CAP_CELLS) {
|
|
const resolved = resolveCellClass(cellId)
|
|
assert.equal(resolved.status, 0, `${cellId}: ${resolved.stderr}`)
|
|
assert.equal(
|
|
resolved.stdout.trim(),
|
|
`${entryAdmission(cellId)} ${selectorWaveDelta(cellId)}`,
|
|
cellId
|
|
)
|
|
}
|
|
assert.equal(resolveCellClass('production-gce-c12').status, 1)
|
|
})
|
|
|
|
it('offsets a later wave by this cell class\'s own selector delta', () => {
|
|
for (const [waveIndex, delta] of [['0', 2], ['3', 2], ['0', 0], ['3', 0]]) {
|
|
const resolved = spawnSync('bash', ['-euo', 'pipefail', '-c', generationBlock()], {
|
|
env: {
|
|
...process.env,
|
|
DEPLOY_MODE: 'apply',
|
|
EXPECTED_SELECTOR_GENERATION: '40',
|
|
WAVE_INDEX: waveIndex,
|
|
SELECTOR_WAVE_DELTA: String(delta)
|
|
},
|
|
encoding: 'utf8'
|
|
})
|
|
assert.equal(resolved.status, 0, resolved.stderr)
|
|
assert.equal(resolved.stdout.trim(), String(40 + delta * Number(waveIndex)))
|
|
}
|
|
})
|
|
|
|
it('hands a migration-only cell back the exact membership it entered with', () => {
|
|
const entry = {
|
|
EXPECTED_MIGRATION_ONLY_CELLS: 'production-gce-c17,production-gce-c18',
|
|
EXPECTED_GENERAL_CELLS: 'production-gce-c7,production-gce-c8'
|
|
}
|
|
const isolated = membership({
|
|
...entry,
|
|
TARGET_CELL_ID: 'production-gce-c17',
|
|
ENTRY_ADMISSION: 'migration-only'
|
|
})
|
|
assert.deepEqual(isolated, {
|
|
isolatedMigration: 'production-gce-c17,production-gce-c18',
|
|
isolatedGeneral: 'production-gce-c7,production-gce-c8',
|
|
restoredMigration: 'production-gce-c17,production-gce-c18',
|
|
restoredGeneral: 'production-gce-c7,production-gce-c8'
|
|
})
|
|
// A general cell still leaves migration-only and returns to general.
|
|
assert.deepEqual(
|
|
membership({
|
|
...entry,
|
|
TARGET_CELL_ID: 'production-gce-c7',
|
|
ENTRY_ADMISSION: 'general'
|
|
}),
|
|
{
|
|
isolatedMigration: 'production-gce-c17,production-gce-c18,production-gce-c7',
|
|
isolatedGeneral: 'production-gce-c8',
|
|
restoredMigration: 'production-gce-c17,production-gce-c18',
|
|
restoredGeneral: 'production-gce-c7,production-gce-c8'
|
|
}
|
|
)
|
|
})
|
|
|
|
it('never activates a migration-only cell and proves its isolate changed nothing', () => {
|
|
const restore = workflow
|
|
.split('name: Restore only the verified selected cell to its entry admission')[1]
|
|
.split('\n - id:')[0]
|
|
assert.match(restore, /if test "\$\{ENTRY_ADMISSION\}" = migration-only; then\n\s+RESTORE_MODE=isolate/)
|
|
assert.match(restore, /--admission "\$\{ENTRY_ADMISSION\}"/)
|
|
// The pre-mutation check must demand the class the cell is declared to serve in.
|
|
assert.match(workflow, /PRECHECK_ADMISSION="\$\{ENTRY_ADMISSION\}"/)
|
|
const isolate = workflow
|
|
.split('name: Reversibly isolate and drain only the selected cell')[1]
|
|
.split('\n - id:')[0]
|
|
assert.match(isolate, /migration-only; then\n\s+jq -e '\.changed == false'/)
|
|
})
|
|
|
|
it('requires rehome source membership exactly when a roll carries trust lines', () => {
|
|
const step = workflow
|
|
.split('name: Resolve immutable same-cap cell configuration')[1]
|
|
.split('\n - name:')[0]
|
|
const guard = step.indexOf('jq -e --arg cell "${TARGET_CELL_ID}" \'index($cell) != null\'')
|
|
assert.notEqual(guard, -1)
|
|
// The guard reads both protocols, so it has to sit after they are resolved.
|
|
assert.ok(step.indexOf('DESIRED_REHOME_PROTOCOL="${TARGET_REHOME_PROTOCOL}"') < guard)
|
|
assert.match(
|
|
step.slice(0, guard),
|
|
/test "\$\{DESIRED_REHOME_PROTOCOL\}" != 0 \|\| test "\$\{CURRENT_REHOME_PROTOCOL\}" != 0\n\s+\}; then\s+$/
|
|
)
|
|
})
|
|
|
|
it('leaves the US-only capacity job on the default allowlist', () => {
|
|
assert.doesNotMatch(capacityWorkflow, /--approved-cells/)
|
|
})
|
|
})
|
|
|
|
// Both trusted versions must prove the same authenticated drain boundary.
|
|
it('proves rehome trust for protocol 3 on forward and rollback rolls', () => {
|
|
const step = workflow.split('name: Prove exact per-host trust and idempotent no-neighbor behavior')[1].split('\n - name:')[0]
|
|
assert.match(step, /inputs\.rollback-rehome-protocol != '0'/)
|
|
assert.match(step, /inputs\.target-rehome-protocol != '0'/)
|
|
assert.match(step, /probe-relay-rehome-trust\.mjs/)
|
|
})
|