diff --git a/.github/workflows/cloud-operate-relay-production-reserve-watch.yml b/.github/workflows/cloud-operate-relay-production-reserve-watch.yml index d747843b7ca..2ee30fe5dbe 100644 --- a/.github/workflows/cloud-operate-relay-production-reserve-watch.yml +++ b/.github/workflows/cloud-operate-relay-production-reserve-watch.yml @@ -25,8 +25,9 @@ on: type: string flipped-at: description: >- - When the reserve write applied, ISO with a zone, e.g. 2026-10-14T12:00:00Z. The non-drain - 503 background is the median of the 10 minutes before it + When the reserve (or phonePath=memory) write applied, ISO with a zone, e.g. + 2026-10-14T12:00:00Z. The non-drain 503 background and the phone rules' background are + the 10 minutes before it required: false default: '' type: string @@ -79,6 +80,8 @@ jobs: outputs: state: ${{ steps.outcome.outputs.state }} verdict: ${{ steps.outcome.outputs.verdict }} + # phonePath when a step-6 phone rule tripped; anything else flips the cell back whole. + flip: ${{ steps.outcome.outputs.flip }} env: DURATION_MINUTES: ${{ inputs.duration-minutes }} FLIPPED_AT: ${{ inputs.flipped-at }} @@ -89,6 +92,10 @@ jobs: set -euo pipefail [[ "${CELL_ID}" =~ ^production-gce-c[0-9]{1,3}$ ]] node -e "if (!/^[^.]+[.][^.]+[.][^.]+$/.test(process.env.ORCA_RELAY_ADMIN_ID_TOKEN ?? '')) process.exit(1)" + # A phone-rule trip writes phonePath=db mid-watch, through the flag tool. + ORCA_RELAY_CONTROL_ACCESS_TOKEN="$(gcloud auth print-access-token)" + echo "::add-mask::${ORCA_RELAY_CONTROL_ACCESS_TOKEN}" + export ORCA_RELAY_CONTROL_ACCESS_TOKEN node dev/scripts/watch-relay-reserve-cell.mjs watch \ --mode "${MODE}" --cell-id "${CELL_ID}" \ --cell-origin "https://${CELL_ID#production-gce-}.relay.onorca.dev" \ @@ -258,6 +265,7 @@ jobs: # lease wait can still kill the process before Postgres records db: the file says db, Postgres reserve. env: VERDICT: ${{ needs.watch.outputs.verdict || format('the watch ended {0} without a verdict', needs.watch.result) }} + FLIP: ${{ needs.watch.outputs.flip || 'admitMode' }} steps: - uses: actions/checkout@v4 if: ${{ always() }} @@ -297,7 +305,7 @@ jobs: --mode "${MODE}" --cell-id "${CELL_ID}" \ --cell-origin "https://${CELL_ID#production-gce-}.relay.onorca.dev" \ --project-id "${GCP_PROJECT_ID}" --director-origin "https://relay.onorca.dev" \ - --confirmation "${CONFIRMATION}" --verdict "${VERDICT}" \ + --confirmation "${CONFIRMATION}" --verdict "${VERDICT}" --flip "${FLIP}" \ | tee "${RUNNER_TEMP}/relay-reserve-flip-back.jsonl" # The flip-back line is logged only once the write (or its dry-run plan) finished; without @@ -308,12 +316,15 @@ jobs: if grep -qs '"event":"orca_relay_reserve_watch_flip_back"' "${RUNNER_TEMP}/relay-reserve-flip-back.jsonl"; then exit 0 fi - { - echo "## FLIP BACK FAILED: ${CELL_ID} may still be in reserve mode" - echo "Flip it to db by hand with Operate Relay Production Cell Flags." - } >> "${GITHUB_STEP_SUMMARY}" + if [[ "${FLIP}" == phonePath ]]; then + echo "## FLIP BACK FAILED: ${CELL_ID} may still have phonePath=memory" >> "${GITHUB_STEP_SUMMARY}" + echo "Set phonePath=default by hand with Operate Relay Production Cell Flags." >> "${GITHUB_STEP_SUMMARY}" + else + echo "## FLIP BACK FAILED: ${CELL_ID} may still be in reserve mode" >> "${GITHUB_STEP_SUMMARY}" + echo "Flip it to db by hand with Operate Relay Production Cell Flags." >> "${GITHUB_STEP_SUMMARY}" + fi node dev/scripts/watch-relay-reserve-cell.mjs alarm --kind flip-back-failed \ - --mode "${MODE}" --cell-id "${CELL_ID}" --project-id "${GCP_PROJECT_ID}" --verdict "${VERDICT}" + --mode "${MODE}" --cell-id "${CELL_ID}" --project-id "${GCP_PROJECT_ID}" --verdict "${VERDICT}" --flip "${FLIP}" - name: Audit summary if: ${{ always() }} diff --git a/cloud/dev/scripts/operate-relay-cell-flags.mjs b/cloud/dev/scripts/operate-relay-cell-flags.mjs index 745764318f3..5456cc59fca 100644 --- a/cloud/dev/scripts/operate-relay-cell-flags.mjs +++ b/cloud/dev/scripts/operate-relay-cell-flags.mjs @@ -439,7 +439,8 @@ async function breakGlassToDatabase(request, dependencies, { fetchImpl, log, sle if (current.generation !== request.expectedGeneration) { throw generationConflict(current.generation, request.expectedGeneration) } - const object = desiredObject(request, current.object) + // phonePath goes too: a dead cell comes back with phones on the database path as well. + const object = desiredObject({ ...request, changes: { ...request.changes, phonePath: null } }, current.object) const generation = await writeObject(fetchImpl, request, object, accessToken, audit) await directorAdmitMode(fetchImpl, request, idToken, sleep, 'db') const result = { event: 'orca_relay_cell_admit_mode_break_glass', cellId: request.cellId, generation, audit } diff --git a/cloud/dev/scripts/operate-relay-cell-flags.test.mjs b/cloud/dev/scripts/operate-relay-cell-flags.test.mjs index af4c56f871b..5d30e7e9483 100644 --- a/cloud/dev/scripts/operate-relay-cell-flags.test.mjs +++ b/cloud/dev/scripts/operate-relay-cell-flags.test.mjs @@ -586,7 +586,7 @@ test('a flip back that outlasts its wait keeps Postgres on reserve and says how assert.equal(lines.find((line) => line.event === 'orca_relay_cell_reregistration_wait')?.waitMs, 1_666_667) }) -test('break glass records db for a cell that cannot answer, typed for that cell only', async () => { +test('break glass records db (and drops phonePath) for a cell that cannot answer, typed for that cell only', async () => { const glass = { ...RESERVE, set: 'admitMode=default', 'expected-generation': '7' } assert.throws(() => parseCellFlagsRequest(values({ ...glass, 'break-glass': 'BREAK GLASS production-gce-c27' })), /break glass requires/) assert.throws( @@ -594,7 +594,7 @@ test('break glass records db for a cell that cannot answer, typed for that cell /on its own/ ) const fake = fakeGoogleAndCell({ - stored: { generation: '7', object: { v: 1, cellId: CELL, flags: { admitMode: 'reserve', readinessLocal: true } } } + stored: { generation: '7', object: { v: 1, cellId: CELL, flags: { admitMode: 'reserve', readinessLocal: true, phonePath: 'memory' } } } }) fake.state.directorAdmitMode = 'reserve' const inner = fake.fetchImpl diff --git a/cloud/dev/scripts/relay-phone-watch-recorded-quiet-minutes.json b/cloud/dev/scripts/relay-phone-watch-recorded-quiet-minutes.json new file mode 100644 index 00000000000..0593c68ab63 --- /dev/null +++ b/cloud/dev/scripts/relay-phone-watch-recorded-quiet-minutes.json @@ -0,0 +1 @@ +{"source":"production cell runtime metrics, read-only, 2026-10-18T06:00Z to 2026-10-19T08:00Z; per-minute sums of authFailuresDelta and clientAcceptCompletedDelta by the minute each 30 s line ends in","cells":{"production-gce-c7":{"authFailuresPerMinute":[0,1,0,1,2,0,0,0,0,0,7,0,0,0,0,0,0,0,0,0,0,1,1,0,2,1,1,0,1,2,0,0,0,0,0,0,0,0,0,0,0,0,0,0,1,0,0,0,1,0,0,1,0,0,1,0,0,0,0,0,0,0,0,0,1,0,2,1,0,1,1,0,2,2,1,0,1,0,0,0,0,0,0,0,1,0,0,0,1,1,0,1,0,0,0,0,1,1,0,0,0,0,1,0,0,0,0,0,0,0,0,0,0,0,0,0,1,2,0,0,1,0,0,0,0,0,0,0,0,0,0,1,0,0,0,0,0,0,0,2,0,0,0,1,0,0,0,0,0,0,0,0,1,0,0,0,0,0,1,0,0,1,1,0,0,2,1,1,0,3,2,2,1,3,5,1,0,1,2,1,2,1,2,0,5,4,1,1,4,0,1,0,1,3,2,4,2,1,3,2,5,1,3,1,2,1,1,2,2,1,1,3,0,0,0,0,0,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,1,0,0,0,0,0,0,0,0,0,0,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,1,0,4,0,0,0,0,0,0,0,0,0,0,0,0,2,0,0,0,1,0,0,0,0,2,0,0,0,0,0,0,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,2,0,0,0,2,1,1,0,0,0,0,1,0,0,0,0,0,0,0,0,0,0,0,0,0,1,0,0,0,0,0,1,0,0,0,0,0,2,0,0,0,0,1,0,0,0,1,2,0,0,0,0,0,0,0,3,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,1,0,0,0,0,0,0,1,0,0,0,1,0,0,0,0,0,0,0,2,0,0,0,0,0,0,0,1,1,0,0,0,0,0,0,0,0,0,0,0,1,0,0,0,1,0,0,0,0,0,0,0,0,0,0,1,0,0,0,0,0,1,0,0,0,0,1,0,0,0,0,0,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,1,0,0,0,0,0,0,0,1,0,0,1,1,0,0,0,0,0,0,0,0,1,0,0,0,0,0,1,1,0,0,0,0,0,1,1,0,0,1,0,0,0,0,2,0,1,0,0,0,0,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,1,2,0,0,0,0,0,2,1,0,2,0,0,0,2,0,0,0,0,0,0,0,1,0,0,0,0,0,0,0,1,0,0,0,0,0,0,0,0,1,0,0,0,1,0,1,0,0,1,0,0,1,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,1,2,0,0,0,0,0,1,0,0,0,0,0,0,0,0,1,0,0,0,0,0,0,0,1,0,0,0,0,0,0,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,1,0,0,0,0,0,0,1,0,0,0,0,0,0,0,0,1,0,0,0,0,1,0,0,0,0,0,0,0,1,0,0,1,0,0,0,1,0,0,0,0,0,0,0,0,0,0,0,2,0,0,1,0,1,0,0,0,0,0,0,0,0,0,0,0,0,1,0,0,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,1,0,0,0,0,0,0,0,1,0,0,0,0,0,1,0,0,0,0,1,0,0,0,0,1,0,0,0,0,1,0,0,0,0,0,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,1,0,2,0,0,0,0,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,1,0,0,0,0,0,1,0,0,0,0,0,0,0,0,0,0,1,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,1,1,0,1,1,0,1,1,0,0,0,1,0,0,0,1,0,0,1,0,0,0,0,0,1,1,0,1,0,2,2,2,0,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,1,0,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,1,0,0,0,0,0,0,0,0,0,0,1,1,0,0,0,0,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,1,0,0,0,0,1,1,0,0,0,0,0,1,0,0,0,0,0,0,0,1,0,0,0,0,1,0,0,0,0,1,0,0,0,0,0,0,0,0,0,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,1,0,0,0,1,1,0,1,0,0,0,0,0,0,0,0,0,0,0,1,0,0,0,1,0,0,0,0,0,0,0,0,0,1,0,0,0,0,0,0,0,0,0,0,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,2,0,0,0,1,2,0,0,0,0,0,0,0,0,0,0,0,0,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,1,0,0,0,0,0,0,0,0,2,0,0,0,0,0,0,0,0,0,0,0,0,1,2,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,1,0,2,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,1,0,0,0,0,2,0,0,0,0,0,0,0,0,1,0,0,0,0,0,0,0,0,0,1,0,0,0,0,0,1,0,1,1,0,0,0,0,0,1,0,0,1,0,0,0,0,0,1,2,3,0,0,0,0,0,1,0,0,1,0,0,1,0,0,1,0,0,2,0,1,0,1,0,0,0,0,0,1,0,3,0,1,1,1,1,0,0,0,1,0,0,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,1,0,0,0,4,1,1,1,3,1,1,1,0,1,1,0,1,2,2,2,0,0,1,6,0,1,1,1,1,2,0,0,0,0,0,0,0,0,0,0,1,0,0,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,1,0,0,0,0,0,0,1,0,0,0,0,1,0,0,0,0,0,0,0,0,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,1,0,0,0,0,0,0,0,0,0,1,0,0,1,0,1,0,4,0,0,0,0,0,0,0,0,1,0,1,0,0,1,0,0],"completedAcceptsPerMinute":[2,0,1,2,0,2,1,5,1,1,2,3,1,4,0,0,2,4,1,0,0,0,1,1,2,2,2,1,2,2,1,1,0,0,2,0,0,2,1,1,1,2,0,0,1,2,1,0,2,1,0,1,1,0,1,0,1,0,1,0,0,0,1,1,4,0,1,0,1,0,2,0,0,0,0,0,2,2,0,1,1,1,1,1,0,0,0,1,2,1,0,0,0,1,0,1,3,0,0,3,0,0,1,1,0,2,1,1,2,1,2,0,1,1,2,1,3,2,1,1,0,1,0,1,2,4,2,3,4,0,1,1,3,2,0,1,2,1,2,2,2,3,4,2,3,1,3,1,0,2,2,0,0,3,1,1,2,1,3,2,2,2,2,3,1,3,1,3,3,1,1,2,0,0,3,0,4,2,3,3,1,2,2,4,3,3,2,3,4,4,0,4,4,3,2,1,0,2,1,2,3,2,3,3,1,4,2,2,2,3,3,1,4,2,1,4,2,3,4,2,0,1,4,2,1,0,0,2,0,1,4,2,1,4,3,3,3,0,2,3,3,3,2,3,5,2,5,3,1,3,0,5,2,1,4,1,6,1,3,7,4,4,1,4,3,1,1,2,2,3,3,4,0,3,0,2,2,3,2,1,2,3,0,2,0,2,2,1,6,2,4,1,2,4,1,1,3,1,1,3,3,5,3,1,2,2,2,3,2,2,3,2,5,6,2,3,3,2,5,3,2,1,5,3,2,3,6,8,6,1,2,2,1,5,6,8,1,2,7,6,4,3,5,4,6,2,5,5,2,3,2,1,5,1,2,3,6,4,1,4,1,1,5,2,1,3,2,4,3,2,1,1,4,4,2,3,4,2,3,3,1,2,0,4,1,4,3,2,0,4,2,2,1,2,2,1,1,6,2,4,3,1,0,1,1,4,0,1,1,4,1,4,4,1,1,3,2,2,3,3,1,0,1,2,0,2,2,1,2,1,2,2,1,7,4,1,3,0,0,2,2,1,1,1,0,3,7,2,2,0,0,3,0,2,2,1,1,1,4,2,1,3,2,4,0,0,1,5,3,4,2,5,4,2,1,1,2,7,4,6,1,4,2,1,5,0,2,5,2,0,2,3,4,4,1,0,1,0,4,1,4,3,2,2,3,3,2,3,3,3,2,0,1,3,4,3,2,5,3,1,6,2,4,2,3,2,2,2,3,1,2,2,0,6,2,0,2,4,5,1,1,0,2,5,1,2,4,3,4,2,0,2,1,3,4,3,3,3,2,3,5,2,6,2,3,1,3,4,3,1,2,1,3,5,4,0,2,1,1,0,1,1,3,3,3,2,3,2,2,0,2,2,1,4,1,2,4,2,3,1,2,2,3,1,5,0,4,6,1,0,2,5,3,3,5,0,3,1,3,3,7,3,3,1,5,4,4,6,1,6,1,5,2,1,1,1,1,1,4,2,3,5,2,2,1,4,1,1,2,2,1,2,0,0,3,4,2,0,2,4,5,3,3,1,1,1,2,3,1,2,1,4,2,1,3,0,0,2,2,0,0,0,1,1,0,0,0,0,4,4,3,1,1,1,2,1,0,1,1,1,2,4,2,3,3,1,1,2,4,0,1,1,1,2,2,4,2,2,3,2,2,3,1,1,1,5,4,3,4,1,3,3,1,1,1,0,4,3,5,3,1,4,2,3,1,2,3,5,2,3,2,2,4,1,2,5,2,1,4,1,2,3,4,3,2,3,2,2,3,1,0,0,0,1,2,2,0,3,1,2,1,0,0,4,1,0,4,0,0,1,3,0,0,1,1,1,2,0,3,2,0,2,1,3,3,0,2,0,1,2,4,4,3,1,3,5,1,3,0,1,1,3,4,1,0,3,1,3,4,1,3,2,3,3,1,3,3,3,4,7,2,4,3,4,1,2,0,2,3,1,2,3,2,3,0,2,2,2,7,2,3,1,1,3,2,1,3,3,1,1,1,2,4,1,2,6,1,0,0,3,0,4,0,0,2,2,3,3,1,2,0,3,1,1,3,1,5,1,2,4,2,1,1,3,6,5,2,0,4,2,5,0,2,1,4,2,2,3,2,1,7,3,5,0,4,3,0,2,1,3,2,2,1,1,0,2,1,4,2,2,1,1,2,4,3,1,0,1,2,1,2,0,1,5,0,3,3,2,2,1,0,2,1,5,0,5,1,3,2,3,2,1,3,2,3,0,0,2,2,1,1,2,0,0,3,3,2,2,0,5,2,5,2,2,4,1,2,5,0,3,4,2,1,5,3,1,5,3,0,1,3,4,5,2,2,2,4,1,1,1,3,0,2,1,5,1,1,1,1,1,4,3,1,1,1,1,4,3,0,2,5,2,4,3,1,0,3,2,4,5,1,4,2,2,4,2,3,3,4,3,1,3,1,1,4,1,3,1,2,0,4,4,3,3,2,1,2,1,1,2,0,3,1,2,3,2,1,1,0,1,1,0,1,2,3,1,3,3,1,0,2,4,1,2,3,2,1,1,2,2,1,0,1,1,3,0,0,0,3,3,3,1,0,4,0,3,1,3,1,1,4,2,3,0,1,2,2,1,0,0,4,1,3,0,3,2,2,2,0,1,1,1,1,2,3,2,0,1,2,0,2,2,1,0,1,0,2,3,2,2,1,3,3,0,2,3,2,2,6,3,1,1,0,2,2,1,0,2,3,1,0,0,1,2,1,1,0,0,1,0,1,0,2,2,3,1,2,0,1,0,0,3,0,0,2,1,2,0,2,0,2,1,3,0,0,2,3,0,1,0,0,0,0,2,1,0,4,2,0,1,0,3,3,2,0,1,2,2,1,1,1,0,1,2,2,1,3,2,1,3,0,1,1,2,2,0,2,0,2,2,1,1,2,1,0,0,1,1,2,1,0,2,1,3,2,0,2,4,2,1,0,2,1,1,1,0,2,2,1,2,2,3,0,1,2,2,3,3,2,1,2,2,5,0,0,3,1,3,0,1,1,3,3,2,2,1,0,2,3,4,2,4,2,1,1,2,1,0,2,1,0,1,2,1,3,1,4,2,4,1,3,3,2,1,3,0,3,1,3,0,1,3,4,1,5,0,4,2,4,6,3,1,1,1,4,2,2,5,3,0,0,2,1,1,0,1,2,3,0,3,0,1,2,0,4,1,2,2,2,4,1,1,0,4,1,1,1,3,0,1,2,3,2,2,4,1,2,1,1,2,3,2,2,2,1,0,0,0,0,3,0,1,2,0,1,2,1,2,1,0,2,4,3,4,4,3,4,3,3,3,2,0,0,0,1,3,1,2,4,0,4,4,1,1,3,1,1,1,0,1,1,2,2,3,0,1,0,3,0,0,1,2,1,2,2,4,1,1,3,1,0,0,3,1,2,1,1,2,3,2,3,0,0,7,2,2,0,3,1,0,3,1,1,1,2,1,3,0,4,1,1,0,2,4,1,2,1,0,4,2,2,0,1,1,2,1,1,0,1,1,1,3,0,1,1,1,4,2,0,2,2,2,2,4,0,2,1,1,3,0]},"production-gce-c28":{"authFailuresPerMinute":[2,0,0,0,0,0,6,4,3,3,5,3,2,5,5,3,2,1,0,1,1,0,0,1,1,3,2,1,0,0,0,1,0,0,0,0,1,0,0,1,0,0,1,0,0,0,0,0,1,0,0,0,0,0,0,0,0,0,0,1,2,0,0,0,0,0,2,0,1,0,0,0,1,0,0,0,1,1,0,2,0,1,0,0,0,0,1,1,3,0,1,1,2,1,1,0,0,0,1,7,3,5,4,5,6,3,0,1,3,2,2,1,1,0,0,0,0,0,0,1,0,1,0,1,0,0,0,2,0,0,0,0,0,3,0,0,1,1,0,0,2,0,0,1,3,1,4,5,1,5,6,3,5,2,0,0,0,0,0,8,4,4,8,8,10,7,7,4,7,0,1,1,1,0,0,0,0,0,0,1,0,1,0,0,1,1,0,1,1,5,12,8,5,4,3,5,5,3,3,0,1,3,3,0,0,1,1,2,2,1,0,0,0,1,0,2,0,1,0,1,0,0,0,0,0,0,1,1,0,0,4,2,0,2,0,0,0,0,0,0,0,0,0,3,0,0,1,0,0,1,0,0,2,5,3,0,0,0,0,1,0,0,0,1,1,2,2,2,1,0,0,0,3,0,0,1,0,0,0,0,0,0,0,1,0,2,0,0,0,0,0,1,0,1,0,0,1,0,0,0,0,1,0,1,0,2,0,0,0,0,0,0,2,0,0,0,1,0,0,0,0,0,0,0,0,0,2,0,1,0,1,0,0,0,0,0,0,0,0,4,3,2,0,0,0,1,0,0,1,0,0,0,1,0,0,0,0,1,1,2,0,0,0,0,0,0,1,0,0,0,0,0,1,0,1,0,0,0,0,0,0,0,2,8,4,3,2,1,1,0,1,0,0,2,0,0,0,1,1,0,0,1,1,0,0,0,0,0,0,1,0,0,1,1,0,7,2,0,0,0,0,3,0,0,0,0,0,0,0,0,0,0,0,0,0,1,1,0,0,0,2,0,0,0,0,0,1,1,0,0,1,0,0,0,0,0,1,0,0,0,0,0,0,0,0,2,0,1,1,1,0,0,0,0,0,0,0,0,1,0,1,1,0,0,0,0,0,0,1,0,0,0,0,0,0,1,0,0,0,0,0,0,2,1,0,0,2,0,1,0,0,0,0,1,0,1,0,1,0,0,1,0,0,1,0,0,0,1,0,0,0,0,0,0,0,0,0,0,2,0,0,0,0,0,1,2,1,0,0,0,0,1,0,0,0,0,2,0,1,0,1,0,0,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,1,0,1,0,0,1,0,0,0,0,0,0,0,0,1,0,0,0,0,0,1,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,1,0,1,0,0,1,0,0,1,0,0,0,0,0,0,0,0,0,0,0,1,0,0,0,0,0,0,0,1,0,0,0,0,0,0,0,0,2,0,0,0,0,0,0,0,0,1,0,0,1,0,0,0,0,0,0,0,0,0,0,0,0,0,2,2,0,3,7,3,2,0,0,0,1,0,0,0,1,0,0,0,0,0,1,0,0,0,0,0,0,0,0,0,1,1,0,0,0,0,1,0,1,0,0,0,0,1,0,0,0,0,0,0,0,0,1,0,0,0,1,0,0,1,1,1,0,1,0,0,0,0,0,1,0,0,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,1,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,1,0,0,1,0,0,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,1,0,0,0,0,0,0,0,0,0,0,0,0,1,0,0,0,0,0,0,0,0,0,5,0,0,0,2,0,0,1,1,0,0,0,0,0,0,2,4,2,6,2,3,2,3,2,0,0,0,0,0,0,0,0,1,0,0,0,0,0,2,2,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,1,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,1,0,0,1,0,0,0,1,0,1,1,0,1,1,0,1,1,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,1,0,0,0,0,0,0,2,0,0,0,1,0,0,0,0,0,0,0,0,0,0,1,0,0,0,6,2,0,0,0,1,1,0,0,0,0,0,0,1,0,1,2,0,0,1,1,1,0,2,1,0,0,1,0,1,0,0,0,1,0,0,0,1,0,0,0,0,0,1,1,2,0,1,1,2,0,0,0,2,0,0,1,1,0,0,0,0,1,0,1,0,0,0,0,0,0,0,0,0,1,0,0,0,0,3,0,1,0,0,0,0,0,0,1,0,0,0,0,1,0,2,0,0,0,0,1,1,0,0,0,0,1,0,1,1,2,1,1,1,0,1,0,0,1,0,0,1,0,0,1,0,1,1,0,2,1,0,0,0,1,0,0,2,0,0,1,0,0,0,0,2,2,0,0,0,1,0,0,1,2,0,0,0,2,0,0,1,0,2,0,1,3,0,0,0,1,0,1,0,0,0,0,1,2,3,0,0,0,0,0,1,0,0,1,0,1,0,0,6,2,0,5,2,0,0,0,4,5,3,1,0,5,4,0,0,3,0,1,0,1,0,5,0,2,3,6,3,6,2,0,5,3,9,0,0,3,0,1,6,3,0,2,0,5,1,1,1,4,1,1,1,0,0,0,0,1,0,1,2,1,0,5,4,5,2,0,1,0,0,0,1,1,6,5,3,6,8,6,4,9,3,6,3,3,0,3,14,5,11,6,5,11,8,5,6,10,4,1,9,14,7,4,14,6,5,4,4,1,9,9,11,6,1,0,0,2,0,0,0,1,1,0,0,0,1,0,0,1,3,3,1,0,2,2,1,1,0,0,0,0,3,4,0,3,0,2,0,3,1,2,0,0,0,1,1,0,0,1,2,1,2,1,11,5,3,0,0,1,8,8,1,4,4,0,4,0,1,6,7,6,5,9,5,3,3,5,2,7,1,2,3,1,0,1,0,1,0,2,1,5,4,5,2,1,0,0,4,2,0,0,1,0,1,0,0,0,1,1,1,0,0,0,0,0,5,6,0,1,5,2,1,0,0,0,1,2,2,1,0,0,0,2,5,2,0,0,0,0,0,0,0,0,1,2,0,0,0,1,1,0,0,0,2,0,0,0,1,0,4,2,0,1,0,2,1,1,9,2,0,0,0,1,1,0,0,1,3,2,4,3,6,2,2,1,1,0,0,2,1,0,3,6,0,2,1,0,0,0,2,1,2,5,1,2,0,3,0,0,1,0,0,0,1,0,2,2,0,0,0,1,1,0,0,0,0,0,0,1,0,1],"completedAcceptsPerMinute":[17,10,8,8,13,10,8,10,16,16,31,20,15,17,14,14,17,19,22,20,20,12,15,23,12,16,11,20,21,19,23,17,19,25,18,19,13,12,16,8,11,10,15,15,19,6,9,7,7,11,14,16,15,14,9,10,11,7,10,5,12,12,11,12,6,7,10,6,10,4,10,7,10,7,7,13,14,5,11,4,9,7,9,9,10,17,11,14,16,9,15,12,12,1,8,8,9,7,10,17,14,13,7,10,16,13,8,21,18,13,14,7,8,14,10,14,12,14,16,19,16,22,21,15,19,18,16,16,9,8,14,9,10,14,10,12,14,8,7,13,9,9,10,8,9,7,14,16,15,11,17,6,8,18,6,12,10,5,11,11,8,10,8,7,13,16,8,9,13,9,15,8,18,14,13,9,9,5,7,9,3,8,8,6,7,13,15,17,12,8,9,7,10,11,15,16,12,15,16,16,14,9,6,6,16,11,10,12,16,14,18,11,11,16,11,8,8,2,12,7,10,8,8,13,3,8,8,15,11,6,9,7,11,14,7,15,7,6,6,10,15,8,10,10,10,10,8,7,5,5,11,9,18,12,10,19,16,12,14,8,9,6,10,20,11,10,12,22,11,20,10,17,6,14,19,12,14,7,18,12,9,8,8,11,10,9,7,1,6,6,11,6,6,9,8,13,5,8,16,17,10,8,12,11,8,16,8,9,12,10,7,5,6,8,13,12,8,7,13,6,5,12,11,8,10,14,6,13,11,11,7,10,8,2,2,6,4,9,9,9,7,8,13,6,6,10,6,6,6,5,2,9,7,4,9,10,5,9,7,9,7,4,6,7,13,10,8,6,10,7,8,14,7,9,6,5,7,13,9,14,9,5,12,17,13,19,21,8,15,12,10,18,13,12,11,13,9,8,9,13,8,12,7,6,8,3,4,4,4,9,5,10,8,4,12,13,19,10,8,9,13,4,5,4,4,1,8,9,2,5,7,7,12,10,6,7,4,9,5,9,8,13,11,10,6,11,10,8,6,15,14,9,20,24,17,12,10,6,10,8,10,6,8,5,4,3,12,9,3,7,9,5,5,4,11,6,8,7,6,7,13,4,5,5,9,6,8,4,14,9,11,12,9,5,10,6,6,7,9,7,8,11,5,6,6,9,5,8,11,7,3,5,6,7,7,9,5,12,8,8,7,5,8,5,5,10,4,10,6,6,6,2,3,7,5,12,4,3,7,6,5,5,7,10,2,7,8,12,5,7,11,5,6,8,6,12,10,4,10,7,9,7,12,14,11,14,7,11,4,9,5,7,5,4,6,13,9,3,5,6,6,12,3,4,5,9,11,5,5,4,9,11,16,9,9,13,7,4,5,9,4,5,5,7,4,4,4,1,3,4,11,7,4,3,4,3,5,2,4,3,4,4,3,3,5,4,5,6,7,3,3,2,5,2,6,4,4,3,2,2,4,6,4,1,6,7,2,2,3,4,4,1,2,3,3,4,3,3,6,2,2,4,3,3,1,6,3,1,3,0,7,3,3,2,1,1,1,3,4,1,0,1,1,4,1,1,2,3,4,3,2,2,3,1,4,4,2,0,2,1,0,2,3,1,3,1,1,3,5,3,3,3,2,1,2,6,1,0,1,4,0,5,6,3,2,0,5,1,4,1,0,0,1,4,2,2,4,0,4,0,1,1,1,1,0,1,2,2,2,1,0,1,2,2,5,3,0,1,3,0,3,2,2,4,1,2,1,1,1,2,1,1,2,1,2,4,1,2,0,1,1,0,1,0,1,3,1,1,1,0,2,0,1,0,1,0,0,0,0,3,0,3,3,0,2,1,0,1,0,2,4,6,1,0,3,1,0,0,2,0,2,1,1,2,2,1,0,0,4,1,0,4,2,2,3,1,0,3,1,2,4,2,3,3,1,1,3,5,0,2,3,4,2,0,0,1,1,0,2,1,2,1,0,1,0,1,0,1,1,0,0,2,1,2,2,1,1,1,1,1,0,1,0,0,0,0,0,0,0,0,3,0,1,4,0,1,2,0,0,2,0,0,0,1,0,0,0,1,1,0,1,5,1,0,0,0,0,1,2,1,1,1,1,1,1,0,1,1,3,2,1,1,2,0,2,0,3,0,2,1,0,1,1,5,4,3,5,1,0,2,4,1,1,3,5,0,2,0,2,0,2,3,1,1,3,2,2,3,4,3,2,1,4,2,1,2,2,2,2,1,4,1,3,4,2,2,4,2,3,3,3,0,3,2,2,3,3,6,5,7,8,3,2,3,3,2,2,4,2,4,5,1,1,2,2,4,3,4,2,2,4,6,7,5,4,2,3,2,3,6,4,5,7,5,2,9,4,5,7,7,6,5,8,9,7,6,6,6,5,9,9,7,8,4,6,7,4,2,2,5,5,9,9,4,4,3,6,2,8,4,6,8,3,2,3,6,6,11,6,8,1,5,2,3,11,7,0,6,5,3,5,8,6,6,8,6,5,5,9,11,6,7,6,5,5,5,6,7,6,2,5,7,6,7,5,9,2,6,19,4,2,4,6,7,8,6,11,5,11,7,5,7,10,7,7,1,10,10,5,7,12,8,9,12,11,9,7,11,8,8,5,12,10,8,5,8,5,11,3,7,7,5,5,5,7,3,8,5,7,11,9,9,5,8,10,8,11,20,16,16,20,15,14,13,16,13,14,15,12,16,13,13,12,8,6,6,6,7,6,8,6,12,11,5,5,7,10,3,5,9,8,4,8,5,9,4,11,7,10,11,12,7,7,14,9,14,8,6,10,7,7,10,6,11,11,6,5,11,5,9,11,10,9,8,10,10,10,7,8,10,11,10,10,7,12,9,13,11,12,9,13,8,16,10,10,5,5,7,13,10,7,10,6,10,3,9,9,14,6,9,11,4,14,12,11,8,7,15,6,16,12,9,13,14,10,21,13,8,16,11,10,13,4,8,12,7,16,14,9,12,11,10,7,13,13,11,11,13,6,7,9,13,8,9,7,11,12,7,9,9,10,13,9,12,6,8,8,8,12,8,12,2,6,1,10,9,13,13,10,2,11,12,11,7,13,10,3,8,11,7,12,8,10,7,16,17,11,7,14,7,5,8,3,8,7,11,8,6,9,11,11,7,7,5,10,9,14,4,8,10,8,17,9,6,6,5,9,11,5,10,11,10,11,10,12,16,8,8,2,9,4,9,6,13,6,7,4,11,10,4,9,9,11,6,13,10,6,10,12,12,13,16,6,8,5,7,6,4,7,13,5,10,6,5,5,9,7,6,7,11,10,5,6,7,9,3,8,8,14,7,10,11,5,8,10,14,15,16,17,9,9,9,14,14,20,13,14,8,9,5,8,14,13,19,24,13,14,15,14,11,14,13,15,14,7,7,10,9,8,17,16,14,20,12,20,14,15,15,9,5,10,5,10,11,7,9,8,4,5,6,5,9,11,5,6,2,5,6,8,13,4,9,4,8,4,4,8,5,10,9,9,5,14,14,11,11,8,8,4,12,10,13,13,9,10,10,9,5,4,7]},"production-gce-c29":{"authFailuresPerMinute":[0,1,0,1,0,0,4,3,2,5,0,0,0,0,1,0,0,0,1,0,0,0,0,0,0,0,1,0,1,0,1,0,0,1,1,0,0,0,0,0,0,1,0,0,0,1,0,0,0,0,0,0,1,0,0,0,0,0,2,0,0,0,0,1,0,0,0,1,0,0,0,0,0,0,1,0,0,1,0,0,1,0,0,9,6,0,0,4,3,0,0,0,0,2,0,0,0,0,0,0,0,0,0,0,0,1,3,0,0,1,0,0,1,1,0,0,0,0,0,0,0,1,0,0,0,0,0,0,0,2,4,0,0,4,5,0,0,0,0,1,5,1,5,2,4,4,5,2,3,1,1,6,3,4,4,1,0,0,0,0,3,7,1,0,0,0,0,0,6,1,1,7,0,6,5,2,0,0,1,0,0,0,0,0,0,3,4,0,1,4,0,0,0,0,1,0,0,1,1,0,0,0,0,0,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,1,1,0,0,0,1,0,0,0,0,0,0,0,0,0,0,1,1,0,1,1,0,0,0,0,0,0,0,0,0,0,1,1,0,0,0,1,0,1,0,0,0,1,1,0,2,0,0,0,0,0,0,1,0,0,0,1,0,1,0,0,0,0,1,0,1,0,0,0,0,0,0,0,0,1,1,0,0,0,0,0,0,2,0,0,0,0,0,0,1,0,0,0,0,0,0,0,0,0,0,0,0,1,0,0,0,1,0,0,0,1,1,0,0,0,0,0,0,1,0,0,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,1,3,0,1,0,0,0,2,0,0,3,4,5,3,1,0,0,0,0,0,2,0,0,0,0,0,0,0,1,0,0,0,0,0,0,0,0,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,1,4,3,0,0,2,2,0,0,2,2,0,0,0,1,0,1,0,0,0,1,0,1,0,0,0,0,0,1,0,0,0,1,0,1,0,0,0,0,0,0,1,0,1,0,0,0,1,0,0,0,0,0,1,0,0,0,0,0,0,0,0,0,0,0,0,1,0,0,0,0,0,0,0,0,0,0,1,0,0,0,0,0,0,0,0,2,2,0,2,0,0,0,0,1,0,0,0,0,0,1,0,0,1,0,1,0,0,2,0,0,1,1,0,0,0,0,0,0,0,1,0,0,0,0,0,1,1,0,1,0,0,0,0,0,0,0,0,0,0,0,1,0,0,1,0,0,0,0,0,1,0,0,0,0,0,1,1,0,2,0,0,0,0,1,0,0,1,0,0,0,0,3,0,0,0,0,0,0,0,0,0,1,0,0,0,0,1,0,0,0,0,0,0,0,1,0,0,0,0,0,0,0,0,0,0,1,0,0,0,0,0,0,2,0,0,0,0,0,1,0,0,0,0,0,0,0,0,0,1,0,0,0,0,0,1,0,0,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,1,0,0,0,0,0,1,0,0,0,0,0,0,0,1,0,0,0,0,0,0,0,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,1,0,0,0,0,0,0,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,1,0,0,0,1,0,0,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,4,4,2,0,0,0,0,0,0,0,0,0,0,0,0,0,0,1,0,0,0,0,1,0,0,0,0,0,0,0,0,0,0,1,1,0,0,0,0,0,0,0,0,0,0,0,0,1,0,0,0,0,0,1,0,1,1,0,0,0,0,0,0,0,0,1,0,1,0,0,1,1,0,0,1,0,0,0,0,2,0,6,4,2,0,0,1,0,2,1,0,2,0,0,0,0,0,0,0,0,0,0,2,0,0,1,0,1,0,0,1,4,1,0,0,3,1,0,1,0,0,0,6,2,6,5,8,4,0,0,0,6,4,8,4,3,0,1,0,0,1,0,1,0,1,0,0,0,0,1,0,0,0,0,1,1,0,0,0,0,0,0,0,1,0,0,0,0,0,1,0,0,0,0,0,0,0,1,0,2,3,0,0,0,0,0,0,2,0,1,0,1,0,1,0,0,1,0,2,0,0,2,0,0,1,0,0,0,0,0,0,2,6,3,7,0,1,0,0,0,1,0,2,2,1,0,1,0,0,0,0,0,0,1,0,0,0,0,0,0,0,0,0,1,0,0,0,1,0,1,0,6,4,0,0,0,0,0,3,1,0,1,0,0,0,0,0,0,0,1,0,0,0,0,0,0,0,0,1,0,0,0,0,1,2,0,0,0,2,0,0,1,1,0,0,0,0,0,2,0,0,0,0,0,1,0,0,0,0,0,0,0,1,0,0,1,2,1,1,3,0,2,0,0,0,1,2,0,0,0,1,0,0,1,2,0,0,0,0,3,0,0,0,0,3,0,0,0,2,0,1,0,0,1,0,0,0,1,0,0,1,0,0,0,0,0,0,0,1,0,0,0,0,0,1,1,0,0,0,0,0,0,0,0,0,0,0,1,0,0,1,0,0,1,0,0,3,2,0,0,2,0,0,0,1,1,0,0,1,0,1,0,0,1,0,0,0,0,1,0,0,0,0,1,0,1,0,0,0,0,0,0,0,0,3,0,0,0,0,0,0,0,0,1,0,0,1,0,0,0,0,2,0,1,0,0,0,1,0,0,0,0,0,0,0,1,0,1,1,2,0,0,0,0,0,0,0,0,1,0,0,0,0,0,1,1,0,0,0,1,0,0,0,0,1,0,0,0,0,0,1,0,0,0,0,0,0,1,0,1,1,0,0,0,0,1,0,0,1,0,0,5,0,0,0,0,0,0,0,0,0,0,0,0,1,0,0,0,0,0,1,0,0,0,0,1,0,0,2,1,0,0,0,0,1,0,0,1,0,0,2,0,0,0,0,0,1,0,0,1,0,0,0,0,0,0,0,0,0,0,1,0,1,0,0,0,0,0,1,0,0,2,0,0,0,2,0,1,0,1,0,0,1,0,0,0,2,0,0,0],"completedAcceptsPerMinute":[3,7,7,8,8,7,4,8,12,4,9,8,10,8,6,5,11,8,10,12,6,7,11,7,12,7,6,11,10,7,9,8,3,10,10,9,6,4,11,9,9,10,16,4,11,11,7,7,9,8,7,11,10,9,5,6,6,7,12,12,7,6,12,6,11,18,3,9,7,8,8,4,10,5,13,10,8,12,8,13,14,3,7,5,7,8,8,4,4,8,11,3,9,9,8,3,7,4,3,7,5,5,9,8,6,7,6,3,9,9,6,10,5,7,15,5,9,11,10,14,2,7,8,10,12,7,19,8,9,9,5,7,5,5,8,11,9,10,8,5,4,11,7,8,13,10,15,12,5,11,15,9,12,8,9,3,11,9,11,9,7,15,11,8,5,8,6,8,8,1,7,5,10,9,7,7,7,9,11,7,13,10,7,9,9,4,6,7,9,7,7,5,10,13,8,6,4,11,5,9,9,7,2,9,1,8,5,8,10,12,10,12,4,8,4,8,7,9,7,3,4,4,3,9,3,8,12,5,6,12,15,8,16,14,9,9,12,5,15,7,8,6,7,11,9,12,3,6,6,7,6,7,3,2,7,10,9,16,2,14,21,11,15,12,10,6,13,1,12,7,15,5,7,11,15,6,2,10,9,10,15,13,14,12,17,12,11,13,8,4,14,10,7,3,4,6,3,5,9,8,10,10,3,7,8,6,8,6,8,12,7,4,9,7,9,5,8,6,10,5,3,6,6,5,3,8,12,5,8,7,12,4,14,13,12,9,14,16,13,14,11,7,8,4,3,8,9,3,5,5,12,7,8,6,11,10,12,13,7,14,17,16,13,15,17,9,14,7,6,9,9,9,11,8,8,7,8,2,12,7,10,6,7,5,5,7,11,7,11,7,3,5,4,9,8,7,9,5,7,8,7,9,5,9,16,15,23,20,14,11,10,5,4,13,14,13,10,16,15,20,16,21,16,17,20,15,16,21,19,13,12,9,11,10,7,12,12,4,9,7,4,14,9,14,15,16,14,9,7,3,6,8,9,6,12,7,12,5,12,10,13,6,10,8,10,6,5,4,8,6,13,6,10,9,6,8,7,7,12,6,5,6,10,12,15,14,9,12,13,17,13,4,3,4,6,8,6,8,9,8,13,3,2,3,6,6,10,6,3,10,4,8,13,5,2,4,2,12,5,6,5,9,8,7,5,7,4,4,5,9,12,14,15,9,8,2,4,6,6,2,6,4,6,1,6,5,4,3,3,6,7,1,3,2,6,1,5,9,5,3,1,4,4,5,6,11,5,4,4,4,4,4,6,4,3,6,5,6,4,4,2,7,6,2,7,4,7,6,4,3,3,4,8,5,0,3,5,4,1,5,1,2,2,5,1,8,3,2,6,1,2,3,5,3,2,4,6,5,3,2,3,2,1,3,2,5,4,3,1,3,2,5,4,2,5,1,2,3,3,2,0,3,0,1,3,5,3,1,4,1,6,3,4,2,2,4,2,2,4,2,3,2,5,4,3,2,2,1,6,10,9,10,7,4,1,2,6,8,18,14,15,16,15,19,15,17,13,18,14,8,3,1,1,0,5,1,3,1,0,0,1,0,3,2,0,1,1,6,6,3,3,3,1,2,2,2,4,2,0,1,4,2,0,1,1,0,1,1,2,1,4,1,4,1,6,3,1,1,2,3,1,0,1,2,0,0,1,0,4,2,1,2,1,1,1,1,3,2,0,1,4,0,0,5,0,1,1,2,1,1,3,2,2,3,3,3,2,0,3,2,1,3,0,2,4,3,3,5,5,3,1,0,2,1,2,0,0,1,4,2,0,1,2,2,2,1,1,1,1,2,2,4,2,3,1,5,1,2,3,3,1,0,1,1,1,0,0,2,2,0,1,3,3,4,2,0,0,2,1,2,2,0,1,1,0,0,0,0,1,1,1,2,0,2,0,2,0,1,0,1,3,1,2,1,0,0,2,1,0,1,1,2,0,1,1,1,0,0,0,1,2,1,2,2,0,0,1,1,1,4,0,1,0,1,5,2,0,2,0,1,1,2,3,1,1,1,2,2,2,3,2,0,2,1,1,0,0,2,4,2,3,0,0,0,0,2,8,2,1,1,1,1,1,1,2,0,2,0,1,1,0,0,0,1,2,1,3,2,3,1,1,3,4,5,0,3,1,2,0,3,2,4,1,3,3,3,2,4,1,5,5,6,1,3,1,2,2,0,4,13,5,8,3,2,2,1,4,2,3,2,4,0,4,5,2,7,5,0,6,5,6,4,3,8,7,4,2,5,3,12,11,12,7,5,11,17,15,12,11,7,6,4,3,8,3,3,4,2,2,2,5,8,4,7,5,4,1,5,3,2,6,2,5,7,16,3,5,4,11,6,12,7,8,6,8,3,7,2,3,6,8,2,5,11,6,6,7,7,4,7,7,11,4,5,3,7,7,0,3,8,5,9,11,4,7,4,15,5,4,2,6,5,7,4,5,10,3,5,10,6,10,6,8,5,8,6,10,9,12,5,7,12,6,8,5,7,9,8,6,6,15,12,17,13,8,14,5,5,9,7,4,8,9,7,10,6,8,7,10,9,10,6,8,10,4,3,9,4,11,4,5,6,9,9,10,8,6,8,5,8,4,4,5,11,3,2,5,3,4,3,5,7,4,8,1,5,6,7,6,14,11,9,11,7,6,8,12,4,10,6,9,13,12,11,2,7,11,7,10,7,8,9,8,9,8,12,18,12,9,14,13,14,16,12,13,8,8,14,14,3,12,5,15,9,12,9,6,8,15,11,9,5,8,11,12,8,8,6,4,10,6,7,17,12,8,9,11,11,8,10,11,6,13,9,11,15,11,12,6,8,9,11,14,7,10,7,10,6,7,13,5,10,14,16,15,20,9,7,7,10,9,13,10,6,10,10,15,16,18,16,12,6,7,10,9,6,7,15,10,6,13,14,25,10,10,8,9,10,8,7,12,13,6,5,6,8,7,12,8,12,11,14,6,10,10,13,11,14,14,9,16,14,10,5,6,8,6,12,8,8,7,8,9,7,7,14,7,7,8,8,4,7,5,6,8,9,24,10,12,14,10,12,9,6,10,14,6,13,7,16,8,9,13,10,15,11,14,14,11,10,16,9,12,9,9,11,6,10,13,4,13,11,10,6,9,11,10,6,7,8,9,7,7,6,7,7,8,10,12,6,12,10,10,6,7,9,11,5,3,6,6,6,10,9,8,9,7,7,14,5,7,7,3,4,11,8,5,10,3,6,12,4,6,4,9,7,7,4,13,6,7,10,7,9,5,8,2,9,8,9,7,6,6,10,10,13,15,8,7,7,11,4,6,9,4,17,12,17,15,12,13,5,13,12,6,18,10,12,8,8,3,6,12,7,4,9,6,8,5,5,18,9,13,15,8,6,7,6,6,8,9,9,11,6,7,3,8,5,4,2,2,9,7,4,6,8,6,12,3,7,11,7,4,10,11,8,4,9,12,15,10,8,8,5,6,7,5,3,7,6,3,3,10,8,3,7,12,9,6,4]}}} diff --git a/cloud/dev/scripts/relay-reserve-watch-rules.mjs b/cloud/dev/scripts/relay-reserve-watch-rules.mjs index 60f40f3b69e..5ade25766ad 100644 --- a/cloud/dev/scripts/relay-reserve-watch-rules.mjs +++ b/cloud/dev/scripts/relay-reserve-watch-rules.mjs @@ -33,6 +33,80 @@ export const RESERVE_WATCH_RULES = { // sample line's `rules` keys follow this order too. export const RULE_NAMES = ['deadMan', 'overrun', 'director503', 'duplicateSeat', 'hostDrop', 'ledgerDrift'] +// Step 6 design ยง7: judged only while the cell's switch file says phonePath=memory, against the +// cell's own runtime metrics for the 10 minutes before phonePath went memory. A trip turns the +// phone path off (phonePath=db) and the watch goes on with the reserve rules. +export const PHONE_WATCH_RULES = { + // Each rule trips on three complete minutes in a row over its line. + consecutiveMinutes: 3, + readMinutes: 10, + // Judged as a share of the minute's phone attempts (the signal plus completed accepts; for + // confirm failures, the accepts), not a count: a count line false-tripped 33/121 5-hour watches + // on busy c28 (10-18/19), where bursts of refusals arrive with the evening rise in accepts. A + // minute with fewer attempts is not judged, so low-volume c7 (~2 accepts/min) is mostly unjudged. + minAttemptsPerMinute: 4, + // Over half of the minute's attempts, and 0.25 above the share before the flip, capped so two + // failures per accept always trips. Exactly half is not enough: c28 (04:08Z) and c7 (09:13Z) + // both show 3 minutes at 50% of host-and-phone auth failures. + shareFloor: 0.5, + shareOverBackground: 0.25, + shareCeiling: 0.6, + // Page only: confirms still work, only their audit rows or renewals are lost. + pageAfterMs: 5 * MINUTE_MS +} +export const PHONE_RULE_NAMES = [ + 'phoneCredentialRefusals', + 'phoneConfirmFailures', + 'phoneAttachDeadlines', + 'phoneAcceptDatabaseErrors' +] +const PHONE_RULE_SIGNALS = { + phoneCredentialRefusals: 'credentialRefusals', + phoneConfirmFailures: 'confirmFailures', + phoneAttachDeadlines: 'attachDeadlines', + phoneAcceptDatabaseErrors: 'acceptDatabaseErrors' +} + +const sum = (values) => values.reduce((total, value) => total + value, 0) + +// The share line for a signal whose share of attempts before the flip was `backgroundShare`. +export function phoneShareLine(backgroundShare) { + const { shareFloor, shareOverBackground, shareCeiling } = PHONE_WATCH_RULES + return Math.min(shareCeiling, Math.max(shareFloor, backgroundShare + shareOverBackground)) +} + +// A completed accept makes one confirm, so confirm failures are a share of accepts (capped at +// all of them); every other signal is an attempt that did not complete. +function attemptsAndShare(signal, count, completedAccepts) { + if (signal === 'confirmFailures') { + return { attempts: completedAccepts, share: completedAccepts === 0 ? 0 : Math.min(count, completedAccepts) / completedAccepts } + } + const attempts = count + completedAccepts + return { attempts, share: attempts === 0 ? 0 : count / attempts } +} + +// Over the line: enough attempts that minute, and strictly more than the line refused or failed. +export function phoneMinuteOver(signal, count, completedAccepts, line) { + const { attempts, share } = attemptsAndShare(signal, count, completedAccepts) + return attempts >= PHONE_WATCH_RULES.minAttemptsPerMinute && share > line +} + +// The 10 minutes before phonePath went memory, from `readCellPhoneMinutes`: each signal's share of +// attempts. Null until every minute is metered. +export function measuredPhoneBackground(read) { + if (!read.ok || read.unmetered.length > 0 || read.minutes.length === 0) return null + const accepted = sum(read.series.completedAccepts) + return { + share: Object.fromEntries( + Object.values(PHONE_RULE_SIGNALS).map((signal) => { + const count = sum(read.series[signal]) + return [signal, attemptsAndShare(signal, count, accepted).share] + }) + ), + minutes: read.minutes.length + } +} + export function initialWatchState({ cellId, startedAt, durationMinutes, background503PerMinute, flippedAt }) { return { v: 1, @@ -50,7 +124,22 @@ export function initialWatchState({ cellId, startedAt, durationMinutes, backgrou hosts: [], reserveSeenAt: null, ledgerAboveSince: null, - seatBehindRowFirstSeen: {} + seatBehindRowFirstSeen: {}, + // Step 6: what a trip writes back, and the phone rules' state. `phoneSince` is when the switch + // file went phonePath=memory (flippedAt if it already said so); `phoneOffAt` when the watch + // turned it off; `phoneTrip` a phone verdict the segment has yet to write. + flip: null, + lastSampleAt: null, + phoneSince: null, + phoneBackground: null, + phoneOffAt: null, + phoneTrip: null, + phoneOffFailures: 0, + phoneUnverifiableStreak: 0, + auditDropsLast: null, + auditDropsSince: null, + renewalFailuresSince: null, + paged: [] } } @@ -71,6 +160,8 @@ const trip = (reason, detail = {}) => ({ status: 'trip', reason, ...detail }) const unverifiable = (reason) => ({ status: 'unverifiable', reason }) // Not readable yet by design, so it does not count toward the unverifiable streak. const pending = (reason) => ({ status: 'pending', reason }) +const failedPhoneRead = { ok: false, error: 'no cell phone metrics read' } +const minuteFloor = (at) => Math.floor(at / MINUTE_MS) * MINUTE_MS function judgeDeadMan(cell) { if (!cell.ok) return unverifiable(cell.error) @@ -186,6 +277,74 @@ function judgeLedgerDrift(cell, ledger, aboveSince, at) { return { verdict: pass({ ...detail, aboveForMs: at - since }), aboveSince: since } } +// A counter still rising (or failing) every sample for 5 minutes pages once; a single drop does not. +function judgeSustained(rising, since, at) { + const next = rising ? (since ?? at) : null + return { since: next, sustained: next !== null && at - next >= PHONE_WATCH_RULES.pageAfterMs } +} + +export function phoneSinceOf(cell, state) { + if (state.phoneSince !== null || !cell.ok || cell.phonePathConfigured !== 'memory') return state.phoneSince + // Memory from the first sample: the operator's flipped-at. Later: the sample before this one. + return state.lastSampleAt === null ? state.flippedAt : state.lastSampleAt +} + +// The phone rules for one sample, or none while the switch file does not say phonePath=memory. +function judgePhoneRules(cell, phone, state, at) { + if (!cell.ok || cell.phonePathConfigured !== 'memory' || state.phoneOffAt !== null) return {} + const background = state.phoneBackground + if (!background) { + const verdict = + state.phoneSince === null + ? unverifiable('the phone rules need --flipped-at: their background is the 10 minutes before it') + : at < minuteFloor(state.phoneSince) + RESERVE_WATCH_RULES.director503.publishDelayMs + ? pending('the phone background before phonePath went memory is not published yet') + : unverifiable('no phone background yet') + return Object.fromEntries(PHONE_RULE_NAMES.map((name) => [name, verdict])) + } + if (!phone.ok) return Object.fromEntries(PHONE_RULE_NAMES.map((name) => [name, unverifiable(phone.error)])) + const { consecutiveMinutes } = PHONE_WATCH_RULES + // Only the judged minutes need metrics; a gap earlier in the read changes nothing. + const judged = phone.minutes.slice(-consecutiveMinutes) + const gaps = phone.unmetered.filter((minute) => judged.includes(minute)) + if (gaps.length > 0) { + const gap = unverifiable(`no cell runtime metrics for ${gaps.join(', ')}`) + return Object.fromEntries(PHONE_RULE_NAMES.map((name) => [name, gap])) + } + const accepts = phone.series.completedAccepts.slice(-consecutiveMinutes) + return Object.fromEntries( + PHONE_RULE_NAMES.map((name) => { + const signal = PHONE_RULE_SIGNALS[name] + const line = phoneShareLine(background.share[signal]) + const recent = phone.series[signal].slice(-consecutiveMinutes) + const detail = { backgroundShare: background.share[signal], line, recent, accepts } + const over = + recent.length === consecutiveMinutes && recent.every((count, index) => phoneMinuteOver(signal, count, accepts[index], line)) + return [ + name, + over ? trip(`${signal} over ${Math.round(line * 100)}% of attempts for ${consecutiveMinutes} min (${recent.join(', ')} against ${accepts.join(', ')} accepts)`, detail) : pass(detail) + ] + }) + ) +} + +// Page-only signals: audit rows dropped since the previous sample, and renewal writes failing in +// every one of the last 5 published minutes. +function judgePhonePages(cell, phone, state, at) { + const dropped = cell.ok && Number.isInteger(cell.resumeAuditDropped) ? cell.resumeAuditDropped : null + const audit = judgeSustained( + dropped !== null && state.auditDropsLast !== null && dropped > state.auditDropsLast, + state.auditDropsSince, + at + ) + const renewals = phone?.ok ? (phone.series.renewalFailures ?? []).slice(-PHONE_WATCH_RULES.pageAfterMs / MINUTE_MS) : [] + const renewal = judgeSustained(renewals.length > 0 && renewals.every((count) => count > 0), state.renewalFailuresSince, at) + const pages = [] + if (audit.sustained && !state.paged.includes('audit-drops')) pages.push('audit-drops') + if (renewal.sustained && !state.paged.includes('renewal-failures')) pages.push('renewal-failures') + return { auditDropsLast: dropped ?? state.auditDropsLast, auditDropsSince: audit.since, renewalFailuresSince: renewal.since, pages } +} + /** * One sample. `sample.cell` is the cell's runtime-status, `director503` the complete minutes of * non-drain director 503s with the request series' publishing front (`publishedThrough`) and the @@ -220,34 +379,63 @@ export function evaluateWatchSample(state, sample, { unverifiableLimit }) { next.seatBehindRowFirstSeen = duplicate.firstSeen const ledger = judgeLedgerDrift(cell, sample.ledger, state.ledgerAboveSince, at) next.ledgerAboveSince = ledger.aboveSince + next.lastSampleAt = at + next.phoneSince = phoneSinceOf(cell, state) + const phoneState = { ...state, phoneSince: next.phoneSince } + const pageCheck = judgePhonePages(cell, sample.phone, state, at) + next.auditDropsLast = pageCheck.auditDropsLast + next.auditDropsSince = pageCheck.auditDropsSince + next.renewalFailuresSince = pageCheck.renewalFailuresSince + next.paged = [...state.paged, ...pageCheck.pages] + const pages = pageCheck.pages const rules = { deadMan: judgeDeadMan(cell), overrun: judgeOverrun(cell), director503: judgeDirector503(sample.director503, state, at), duplicateSeat: duplicate.verdict, hostDrop: cell.ok ? judgeHostDrop(next.hosts) : unverifiable(cell.error), - ledgerDrift: ledger.verdict + ledgerDrift: ledger.verdict, + ...judgePhoneRules(cell, sample.phone ?? failedPhoneRead, phoneState, at) } const tripped = RULE_NAMES.find((name) => rules[name].status === 'trip') if (tripped) { next.state = 'tripped' + next.flip = 'admitMode' next.verdict = `flip back ${state.cellId}: ${tripped}: ${rules[tripped].reason}` - return { state: next, rules, line: sampleLine(next, sample, rules) } + return { state: next, rules, line: sampleLine(next, sample, rules), pages } + } + // A phone trip turns off only the phone path; the segment writes it and the watch goes on. + const phoneNames = PHONE_RULE_NAMES.filter((name) => name in rules) + const phoneTripped = phoneNames.find((name) => rules[name].status === 'trip') + const phoneMissing = phoneNames.filter((name) => rules[name].status === 'unverifiable') + next.phoneUnverifiableStreak = phoneMissing.length > 0 ? state.phoneUnverifiableStreak + 1 : 0 + if (phoneTripped) { + next.phoneTrip = `turn off the phone path on ${state.cellId}: ${phoneTripped}: ${rules[phoneTripped].reason}` + } else if (next.phoneUnverifiableStreak >= unverifiableLimit) { + next.phoneTrip = + `turn off the phone path on ${state.cellId}: unverifiable: ${next.phoneUnverifiableStreak} samples ` + + `in a row could not read ${phoneMissing.join(', ')}` } const missing = RULE_NAMES.filter((name) => rules[name].status === 'unverifiable') - const waiting = RULE_NAMES.some((name) => rules[name].status === 'pending') + const waiting = [...RULE_NAMES, ...phoneNames].some((name) => rules[name].status === 'pending') // A pending rule holds the streak where it was: neither evidence of health nor a gap. next.unverifiableStreak = missing.length > 0 ? state.unverifiableStreak + 1 : waiting ? state.unverifiableStreak : 0 if (next.unverifiableStreak >= unverifiableLimit) { next.state = 'tripped' + next.flip = 'admitMode' next.verdict = `flip back ${state.cellId}: unverifiable: ${next.unverifiableStreak} samples in a row could not ` + `read ${missing.join(', ')}` + } else if (at >= state.deadlineAt && next.phoneTrip) { + // Still unwritten at the end: the workflow's flip back writes it, and pages if that fails too. + next.state = 'tripped' + next.flip = 'phonePath' + next.verdict = `${next.phoneTrip}; phonePath=db did not land before the watch ended` } else if (at >= state.deadlineAt) { next.state = 'completed' next.verdict = `${state.cellId} stayed healthy in reserve mode for the whole watch` } - return { state: next, rules, line: sampleLine(next, sample, rules) } + return { state: next, rules, line: sampleLine(next, sample, rules), pages } } function sampleLine(state, sample, rules) { @@ -259,6 +447,7 @@ function sampleLine(state, sample, rules) { state: state.state, unverifiableStreak: state.unverifiableStreak, rules, + ...(state.phoneTrip ? { phoneTrip: state.phoneTrip } : {}), ...(state.verdict ? { verdict: state.verdict } : {}) }) } diff --git a/cloud/dev/scripts/watch-relay-reserve-cell.mjs b/cloud/dev/scripts/watch-relay-reserve-cell.mjs index 5809b4f1f1d..4e685ae38a2 100644 --- a/cloud/dev/scripts/watch-relay-reserve-cell.mjs +++ b/cloud/dev/scripts/watch-relay-reserve-cell.mjs @@ -40,6 +40,9 @@ import { evaluateWatchSample, initialWatchState, measuredBackground, + measuredPhoneBackground, + phoneSinceOf, + PHONE_WATCH_RULES, RESERVE_WATCH_RULES } from './relay-reserve-watch-rules.mjs' @@ -69,8 +72,11 @@ function integerIn(name, raw, min, max) { } // The flip back's request. Its generation is a placeholder: the flip back re-reads the current one. +// A phone-rule trip turns off only the phone path; anything else flips the whole cell back. Both +// remove the phonePath key, so a cell rolled back to an image without the switch takes the write. export function dbFlipRequest(values) { - return parseCellFlagsRequest({ ...values, set: 'admitMode=db', 'expected-generation': '0' }) + const set = values.flip === 'phonePath' ? 'phonePath=default' : 'admitMode=db,phonePath=default' + return parseCellFlagsRequest({ ...values, set, 'expected-generation': '0' }) } // `now` is the watch's start: a later segment of a long watch must not age flipped-at out. @@ -104,6 +110,8 @@ export function parseWatchArguments(values, now = Date.now()) { if (!values['state-file']) throw new Error('--state-file is required') return { request, + // The flag tool's own inputs, for turning the phone path off mid-watch. + flipValues: values, directorService: values['director-service'], durationMinutes: integerIn('duration-minutes', values['duration-minutes'], 5, MAX_WATCH_MINUTES), segmentMinutes: integerIn('segment-minutes', values['segment-minutes'], 1, MAX_SEGMENT_MINUTES), @@ -130,6 +138,8 @@ export async function readCellSample(fetchImpl, request, idToken, wait) { ok: true, admitModeConfigured: runtime.flagsApplied.flags?.admitMode ?? 'db', admitModeEffective: runtime.admitModeEffective, + phonePathConfigured: runtime.flagsApplied.flags?.phonePath ?? 'db', + resumeAuditDropped: runtime.runtime?.resumeAuditDropped, draining: runtime.draining === true, controls, enforcedUnits: runtime.runtime?.enforcedConnectionUnits, @@ -184,6 +194,54 @@ export async function readDirector503Minutes(reader, { projectId, directorServic } } +const CELL_METRICS_EVENT = 'orca_relay_runtime_metrics' +const PHONE_SIGNALS = ['credentialRefusals', 'confirmFailures', 'attachDeadlines', 'acceptDatabaseErrors', 'renewalFailures', 'completedAccepts'] +const sumOf = (counts) => Object.values(counts ?? {}).reduce((sum, count) => sum + Number(count), 0) + +// The cell's own runtime metrics lines (one per 30 s) for the `minutes` whole minutes ending at +// `endedAt`, per minute: what the phone rules judge. A line counts toward the minute it ends in; +// a minute no line covers is `unmetered`, never zero. +export async function readCellPhoneMinutes(reader, { cellId, endedAt, minutes }) { + const span = { startedAt: new Date(endedAt - minutes * MINUTE_MS), endedAt: new Date(endedAt) } + const read = await readLogEntries(reader, { + filter: [ + 'resource.type="gce_instance"', + `jsonPayload.event="${CELL_METRICS_EVENT}"`, + `jsonPayload.cellId="${cellId}"`, + `timestamp>="${span.startedAt.toISOString()}"`, + `timestamp<"${new Date(endedAt + DIRECTOR_METRICS_INTERVAL_MS).toISOString()}"` + ].join(' AND '), + projection: + 'json(timestamp,jsonPayload.phoneRejectionsByCodeDelta,jsonPayload.resumeConfirmFailuresByCodeDelta,' + + 'jsonPayload.phoneAttachDeadlinesDelta,jsonPayload.phoneAcceptDatabaseErrorsDelta,jsonPayload.resumeRenewalFailuresDelta,' + + 'jsonPayload.clientAcceptCompletedDelta)', + limit: LOG_ENTRY_LIMIT + }) + if (read.failed) return failedRead(read.error) + if (read.entries.length >= LOG_ENTRY_LIMIT) return failedRead('cell runtime metrics read truncated') + const keys = minutesOf(span) + const series = Object.fromEntries(PHONE_SIGNALS.map((signal) => [signal, keys.map(() => 0)])) + const metered = new Set() + for (const entry of read.entries) { + const at = Date.parse(entry.timestamp) + if (!Number.isFinite(at)) return failedRead('cell runtime metrics line with an unparseable timestamp') + const payload = entry.jsonPayload ?? {} + if (payload.phoneAttachDeadlinesDelta === undefined) { + return { ...failedRead('cell runtime metrics carry no phone counters (an image before step 6)'), permanent: true } + } + metered.add(minuteKey(at - DIRECTOR_METRICS_INTERVAL_MS)).add(minuteKey(at - 1)) + const index = keys.indexOf(minuteKey(at - 1)) + if (index < 0) continue + series.credentialRefusals[index] += Number(payload.phoneRejectionsByCodeDelta?.['4401'] ?? 0) + series.confirmFailures[index] += sumOf(payload.resumeConfirmFailuresByCodeDelta) + series.attachDeadlines[index] += Number(payload.phoneAttachDeadlinesDelta) + series.acceptDatabaseErrors[index] += Number(payload.phoneAcceptDatabaseErrorsDelta ?? 0) + series.renewalFailures[index] += Number(payload.resumeRenewalFailuresDelta ?? 0) + series.completedAccepts[index] += Number(payload.clientAcceptCompletedDelta ?? 0) + } + return { ok: true, minutes: keys, series, unmetered: keys.filter((minute) => !metered.has(minute)) } +} + const LEDGER_SUMMARY_EVENT = 'orca_relay_reserve_ledger_summary' const SEAT_BEHIND_ROW_EVENT = 'orca_relay_reserve_ledger_seat_behind_row' @@ -375,9 +433,44 @@ export async function watchSegment(config, dependencies) { } const reader = createReader(deps, request.projectId) // Serialised: concurrent Logging reads are what earn a 429. + const cell = await readCellSample(deps.fetchImpl, request, dependencies.idToken, sleep) + const phoneWatched = cell.ok && cell.phonePathConfigured === 'memory' && state.phoneOffAt === null + const phoneSince = phoneSinceOf(cell, state) + const phoneBaselineEndsAt = phoneSince === null ? null : minuteFloor(phoneSince) + if (phoneWatched && !state.phoneBackground && phoneBaselineEndsAt !== null && at >= phoneBaselineEndsAt + METRIC_DELAY_MS) { + const read = await readCellPhoneMinutes(reader, { + cellId: request.cellId, + endedAt: phoneBaselineEndsAt, + minutes: BACKGROUND_MINUTES + }) + // A restart or roll in those minutes leaves no background to judge against: no phone + // rule could ever pass, so refuse rather than turn the phone path off for certain. + const incomplete = read.ok ? read.unmetered.length > 0 && `no metrics for ${read.unmetered.join(', ')}` : read.permanent && read.error + if (incomplete) { + state = { + ...state, + state: 'refused', + verdict: + `${request.cellId}: the phone background, the 10 minutes before ${new Date(phoneBaselineEndsAt).toISOString()}, ` + + `is incomplete (${incomplete}), as after a cell restart or roll; start a new watch with a later --flipped-at` + } + await writeFile(config.stateFile, JSON.stringify(state)) + break + } + state = { ...state, phoneBackground: measuredPhoneBackground(read) } + } const sample = { at, - cell: await readCellSample(deps.fetchImpl, request, dependencies.idToken, sleep), + cell, + ...(phoneWatched + ? { + phone: await readCellPhoneMinutes(reader, { + cellId: request.cellId, + endedAt: minuteFloor(at - METRIC_DELAY_MS), + minutes: PHONE_WATCH_RULES.readMinutes + }) + } + : {}), director503: await readDirector503Minutes(reader, { projectId: request.projectId, directorService, @@ -391,6 +484,13 @@ export async function watchSegment(config, dependencies) { const result = evaluateWatchSample(state, sample, { unverifiableLimit: config.unverifiableLimit }) state = result.state log(result.line) + for (const kind of result.pages) { + await raiseAlarm( + { mode: request.mode, cellId: request.cellId, projectId: request.projectId, kind, reason: `sustained for ${PHONE_WATCH_RULES.pageAfterMs / MINUTE_MS} min` }, + { runGcloud: dependencies.runGcloud, log } + ) + } + if (state.state === 'watching' && state.phoneTrip) state = await turnPhonePathOff(config, state, at, { ...deps, log }) await writeFile(config.stateFile, JSON.stringify(state)) if (state.state !== 'watching') break await sleep(Math.max(0, at + SAMPLE_INTERVAL_MS - now())) @@ -399,8 +499,41 @@ export async function watchSegment(config, dependencies) { return state } -function writeOutcome(githubOutput, { state, verdict }) { - return appendFile(githubOutput, `state=${state}\nverdict=${(verdict ?? '').replace(/[\r\n]+/g, ' ')}\n`) +// Writes phonePath=db (the flag tool's CAS write, a plan in dry-run) and keeps watching the +// reserve rules. A write that fails keeps the trip, so the next sample retries it, and pages once +// after PHONE_OFF_PAGE_AFTER_FAILURES in a row: the cell stays watched either way. +export const PHONE_OFF_PAGE_AFTER_FAILURES = 3 +async function turnPhonePathOff(config, state, at, dependencies) { + try { + const result = await flipBack({ ...config.flipValues, flip: 'phonePath' }, dependencies) + dependencies.log(JSON.stringify({ event: 'orca_relay_reserve_watch_phone_path_off', cellId: state.cellId, verdict: state.phoneTrip, written: result.written })) + return { ...state, phoneOffAt: at, phoneTrip: null, phoneOffFailures: 0 } + } catch (error) { + const reason = error instanceof Error ? error.message : String(error) + const failures = (state.phoneOffFailures ?? 0) + 1 + dependencies.log(JSON.stringify({ event: 'orca_relay_reserve_watch_phone_path_off_failed', cellId: state.cellId, failures, reason })) + if (failures === PHONE_OFF_PAGE_AFTER_FAILURES) { + await raiseAlarm( + { + mode: config.request.mode, + cellId: state.cellId, + projectId: config.request.projectId, + kind: 'flip-back-failed', + flip: 'phonePath', + reason: `${state.phoneTrip}; phonePath=db failed ${failures} times: ${reason}` + }, + { runGcloud: dependencies.runGcloud, log: dependencies.log } + ) + } + return { ...state, phoneOffFailures: failures } + } +} + +function writeOutcome(githubOutput, { state, verdict, flip }) { + return appendFile( + githubOutput, + `state=${state}\nverdict=${(verdict ?? '').replace(/[\r\n]+/g, ' ')}\nflip=${flip ?? 'admitMode'}\n` + ) } // The job's outcome. A watch that ended mid-segment (a crash, a timeout) records failed, which the @@ -460,7 +593,11 @@ export async function flipBack(values, dependencies) { ) for (let attempt = 1; ; attempt += 1) { const current = await readCurrentObject(fetchImpl, base, dependencies.accessToken) - if ((current.object?.flags?.admitMode ?? 'db') !== 'reserve') { + const stillOn = + values.flip === 'phonePath' + ? (current.object?.flags?.phonePath ?? 'db') === 'memory' + : (current.object?.flags?.admitMode ?? 'db') === 'reserve' + if (!stillOn) { const result = { written: false, alreadyDb: true, generation: current.generation } record(result) return result @@ -493,7 +630,7 @@ export async function flipBack(values, dependencies) { export const ALARM_EVENT = 'orca_relay_reserve_watch_alarm' export const ALARM_LOG_NAME = 'orca-relay-reserve-watch' -export const ALARM_KINDS = ['refused', 'flip-back-failed'] +export const ALARM_KINDS = ['refused', 'flip-back-failed', 'audit-drops', 'renewal-failures'] // GitHub's workflow-command escaping, so a verdict cannot end the annotation or start another. export const escapeWorkflowCommand = (text) => text.replace(/%/g, '%25').replace(/\r/g, '%0D').replace(/\n/g, '%0A') @@ -503,12 +640,18 @@ const escapeWorkflowProperty = (text) => escapeWorkflowCommand(text).replace(/:/ * Both kinds can leave the cell in reserve with nothing watching it. Only write mode pages: a * dry-run watch never flips back, so its refusal or failure changes nothing for the cell. */ -export async function raiseAlarm({ mode, cellId, projectId, kind, reason }, { runGcloud, log = console.log }) { +export async function raiseAlarm({ mode, cellId, projectId, kind, reason, flip }, { runGcloud, log = console.log }) { if (!ALARM_KINDS.includes(kind)) throw new Error(`--kind must be one of ${ALARM_KINDS.join(', ')}`) const message = kind === 'refused' ? `${cellId} may be in reserve mode and unwatched: the watch was refused (${reason}). Rerun it, or flip the cell to db with Operate Relay Production Cell Flags.` - : `${cellId} may still be in reserve mode with no watch: the flip back did not land (${reason}). Flip it to db by hand with Operate Relay Production Cell Flags (set=admitMode=db), or with the break-glass inputs logged above if the cell does not answer.` + : kind === 'audit-drops' + ? `${cellId} is losing resume audit rows (${reason}); phones still connect. Check its database writes, or set phonePath=db.` + : kind === 'renewal-failures' + ? `${cellId} cannot renew phone credentials (${reason}); phones still connect but age toward re-pairing. Check its database writes, or set phonePath=db.` + : flip === 'phonePath' + ? `${cellId} may still have phonePath=memory with no watch: turning it off did not land (${reason}). Set phonePath=default by hand with Operate Relay Production Cell Flags.` + : `${cellId} may still be in reserve mode with no watch: the flip back did not land (${reason}). Flip it to db by hand with Operate Relay Production Cell Flags (set=admitMode=db), or with the break-glass inputs logged above if the cell does not answer.` log(`::error title=${escapeWorkflowProperty(`Reserve watch ${kind} for ${cellId}`)}::${escapeWorkflowCommand(message)}`) if (mode !== 'write') return { paged: false } const entry = JSON.stringify({ event: ALARM_EVENT, cellId, kind, reason }) @@ -547,7 +690,8 @@ if (import.meta.url === pathToFileURL(process.argv[1] ?? '').href) { 'state-file', 'github-output', 'verdict', - 'kind' + 'kind', + 'flip' ].map((name) => [name, { type: 'string' }]) ) }) @@ -557,7 +701,7 @@ if (import.meta.url === pathToFileURL(process.argv[1] ?? '').href) { try { if (command === 'alarm') { await raiseAlarm( - { mode: values.mode, cellId: values['cell-id'] ?? '', projectId: values['project-id'], kind: values.kind, reason: values.verdict ?? '' }, + { mode: values.mode, cellId: values['cell-id'] ?? '', projectId: values['project-id'], kind: values.kind, reason: values.verdict ?? '', flip: values.flip }, { runGcloud } ) // The alarm step is red, whatever the page did. @@ -571,6 +715,13 @@ if (import.meta.url === pathToFileURL(process.argv[1] ?? '').href) { const config = await parseWatchSegment(values) const state = await watchSegment(config, { idToken, + // For turning the phone path off mid-watch; without it that write fails and the workflow's flip back retries. + accessToken: process.env.ORCA_RELAY_CONTROL_ACCESS_TOKEN, + audit: { + writtenBy: `reserve-watch/${process.env.GITHUB_TRIGGERING_ACTOR ?? process.env.GITHUB_ACTOR ?? 'unknown'}`, + runId: process.env.GITHUB_RUN_ID ?? 'local', + runAttempt: process.env.GITHUB_RUN_ATTEMPT ?? '0' + }, readRolloutLease: readProductionRolloutLease, runGcloud }) diff --git a/cloud/dev/scripts/watch-relay-reserve-cell.test.mjs b/cloud/dev/scripts/watch-relay-reserve-cell.test.mjs index 41a25eb4ead..98c9829f8fa 100644 --- a/cloud/dev/scripts/watch-relay-reserve-cell.test.mjs +++ b/cloud/dev/scripts/watch-relay-reserve-cell.test.mjs @@ -7,7 +7,12 @@ import { test } from 'node:test' import { evaluateWatchSample, initialWatchState, - measuredBackground + measuredBackground, + measuredPhoneBackground, + phoneMinuteOver, + phoneShareLine, + PHONE_RULE_NAMES, + PHONE_WATCH_RULES } from './relay-reserve-watch-rules.mjs' import { ALARM_EVENT, @@ -18,6 +23,7 @@ import { parseWatchArguments, parseWatchSegment, raiseAlarm, + readCellPhoneMinutes, readDirector503Minutes, readSeatBehindRowSample, recordOutcome, @@ -493,7 +499,7 @@ test('a segment samples once a minute, carries state to the next segment, and st assert.equal(first.state, 'watching') assert.equal(lines.length, 3) assert.equal(first.background.source, 'measured') - assert.equal(await readFile(join(directory, 'out'), 'utf8'), 'state=watching\nverdict=\n') + assert.equal(await readFile(join(directory, 'out'), 'utf8'), 'state=watching\nverdict=\nflip=admitMode\n') const second = await watchSegment(config, dependencies) assert.equal(second.state, 'tripped') assert.equal(second.samples, 5) @@ -537,7 +543,7 @@ test('a watch that ended without a verdict records failed, which the workflow fl assert.equal((await recordOutcome(join(directory, 'watching.json'), output)).state, 'failed') await writeFile(join(directory, 'tripped.json'), JSON.stringify(freshState({ state: 'tripped', verdict: 'flip back x:\ny' }))) assert.equal((await recordOutcome(join(directory, 'tripped.json'), output)).state, 'tripped') - assert.match(await readFile(output, 'utf8'), /state=tripped\nverdict=flip back x: y\n$/) + assert.match(await readFile(output, 'utf8'), /state=tripped\nverdict=flip back x: y\nflip=admitMode\n$/) } finally { await rm(directory, { recursive: true, force: true }) } @@ -545,10 +551,20 @@ test('a watch that ended without a verdict records failed, which the workflow fl // Google Storage and the director, as far as the flag tool's flip back reaches them. `onUpload` // returns 'conflict' to play another writer landing first, optionally with its own admitMode. -function fakeFlipBack({ generation = '7', onUpload = () => 'ok', cellAnswers = true } = {}) { +function fakeFlipBack({ + generation = '7', + onUpload = () => 'ok', + cellAnswers = true, + flags = { admitMode: 'reserve', readinessLocal: true }, + supportedFlags = { + admitMode: { type: 'enum', values: ['db', 'reserve'] }, + readinessLocal: { type: 'boolean' }, + phonePath: { type: 'enum', values: ['db', 'memory'] } + } +} = {}) { const calls = [] let uploads = 0 - let object = { generation: Number(generation), flags: { admitMode: 'reserve', readinessLocal: true } } + let object = { generation: Number(generation), flags } let applied = object const fetchImpl = async (url, init = {}) => { const text = String(url) @@ -577,10 +593,7 @@ function fakeFlipBack({ generation = '7', onUpload = () => 'ok', cellAnswers = t role: 'cell', cellId: CELL, flagsApplied: applied, - supportedFlags: { - admitMode: { type: 'enum', values: ['db', 'reserve'] }, - readinessLocal: { type: 'boolean' } - }, + supportedFlags, admitModeEffective: applied.flags.admitMode, runtime: { controls: 10 } }) @@ -1054,3 +1067,346 @@ test('an alarm is an escaped error annotation everywhere, and in write mode a pa assert.match(lines.at(-2), /^::error title=Reserve watch refused for /) assert.match(lines.at(-1), /nothing pages: --project-id is not a GCP project id/) }) + +// Step 6 phone rules. A quiet window, minute by minute: ordinary 4401s from phones whose pairing +// was removed, an occasional failed confirm or half-open control. Synthetic: replaying a recorded +// production window needs read access this suite does not have. +const QUIET_PHONE_MINUTE = (index) => ({ + credentialRefusals: [0, 1, 0, 2, 1][index % 5], + confirmFailures: [0, 0, 1, 0][index % 4], + attachDeadlines: [1, 0, 2, 0, 1, 3][index % 6], + acceptDatabaseErrors: 0, + renewalFailures: 0, + completedAccepts: [10, 12, 8][index % 3] +}) +const phoneRead = (minuteCount, perMinute) => { + const minutes = Array.from({ length: minuteCount }, (_, index) => `2026-10-19T12:${String(index).padStart(2, '0')}`) + const rows = minutes.map((_, index) => perMinute(index)) + const series = Object.fromEntries(Object.keys(rows[0]).map((signal) => [signal, rows.map((row) => row[signal])])) + return { ok: true, minutes, series, unmetered: [] } +} +const memoryCell = (overrides = {}) => healthyCell({ phonePathConfigured: 'memory', resumeAuditDropped: 0, ...overrides }) +const phoneState = (overrides = {}) => + freshState({ + flippedAt: START - 10 * MINUTE, + phoneSince: START - 10 * MINUTE, + phoneBackground: measuredPhoneBackground(phoneRead(10, QUIET_PHONE_MINUTE)), + ...overrides + }) +// The last 10 minutes as of watch minute `offset`, the newest minutes shaped by `spike`. +const phoneSampleAt = (offset, spike = () => ({}), cell = memoryCell()) => + sampleAt(START + offset * MINUTE, { + cell, + phone: phoneRead(10, (index) => ({ ...QUIET_PHONE_MINUTE(offset + index), ...spike(index) })) + }) + +test('phone rules: a quiet hour replays without a trip', () => { + const results = run(Array.from({ length: 60 }, (_, offset) => phoneSampleAt(offset)), phoneState({ deadlineAt: START + 120 * MINUTE })) + assert.equal(results.length, 60) + assert.ok(results.every((result) => result.state.state === 'watching' && result.state.phoneTrip === null)) + assert.deepEqual(Object.keys(results[0].rules).filter((name) => name.startsWith('phone')), PHONE_RULE_NAMES) +}) + +test('phone rules: judged as a share of attempts, over half and over the background', () => { + assert.equal(phoneShareLine(0), 0.5) + assert.equal(phoneShareLine(0.3), 0.55) + assert.equal(phoneShareLine(0.5), 0.6) + // Exactly half is not over; nor is a minute with under 4 attempts. + assert.equal(phoneMinuteOver('credentialRefusals', 9, 9, 0.5), false) + assert.equal(phoneMinuteOver('credentialRefusals', 11, 10, 0.5), true) + assert.equal(phoneMinuteOver('credentialRefusals', 3, 0, 0.5), false) + assert.equal(phoneMinuteOver('credentialRefusals', 4, 0, 0.5), true) + // Each accept makes one confirm: every confirm failing is all of them, not half the attempts. + assert.equal(phoneMinuteOver('confirmFailures', 4, 4, 0.6), true) + assert.equal(phoneMinuteOver('confirmFailures', 2, 4, 0.5), false) + assert.equal(phoneMinuteOver('confirmFailures', 9, 3, 0.5), false) + const background = measuredPhoneBackground(phoneRead(10, () => ({ ...QUIET_PHONE_MINUTE(0), credentialRefusals: 1, completedAccepts: 9 }))) + assert.equal(background.share.credentialRefusals, 0.1) + assert.equal(measuredPhoneBackground({ ...phoneRead(10, QUIET_PHONE_MINUTE), unmetered: ['2026-10-19T12:03'] }), null) +}) + +test('phone rules: each fires on its own spike, and the watch goes on with the reserve rules', () => { + const spikes = { + phoneCredentialRefusals: { credentialRefusals: 25 }, + phoneConfirmFailures: { confirmFailures: 25 }, + phoneAttachDeadlines: { attachDeadlines: 25 }, + phoneAcceptDatabaseErrors: { acceptDatabaseErrors: 25 } + } + for (const [rule, counts] of Object.entries(spikes)) { + const [result] = run([phoneSampleAt(0, (index) => (index >= 7 ? counts : {}))], phoneState()) + assert.equal(result.state.state, 'watching', rule) + assert.match(result.state.phoneTrip, new RegExp(`turn off the phone path on ${CELL}: ${rule}: `), rule) + } + // Two minutes over the line is not three. + const [twoMinutes] = run([phoneSampleAt(0, (index) => (index >= 8 ? { credentialRefusals: 25 } : {}))], phoneState()) + assert.equal(twoMinutes.state.phoneTrip, null) + // Once the watch turned the phone path off, the phone rules stand down. + const [off] = run([phoneSampleAt(0, () => ({ credentialRefusals: 50 }))], phoneState({ phoneOffAt: START })) + assert.deepEqual(Object.keys(off.rules).filter((name) => name.startsWith('phone')), []) +}) + +test('phone rules: judged only while the switch file says memory; reserve rules still flip the cell back', () => { + const [off] = run([sampleAt(START, { cell: healthyCell({ phonePathConfigured: 'db' }) })], freshState()) + assert.deepEqual(Object.keys(off.rules).filter((name) => name.startsWith('phone')), []) + const [overrun] = run([phoneSampleAt(0, () => ({}), memoryCell({ enforcedUnits: 2_001 }))], phoneState()) + assert.equal(overrun.state.state, 'tripped') + assert.equal(overrun.state.flip, 'admitMode') +}) + +test('phone rules: missing phone data turns off only the phone path, and needs a flipped-at background', () => { + const gap = (offset) => ({ ...phoneSampleAt(offset), phone: { ...phoneSampleAt(offset).phone, unmetered: ['2026-10-19T12:09'] } }) + const results = run([gap(0), gap(1), gap(2)], phoneState()) + assert.equal(results.at(-1).state.state, 'watching') + assert.equal(results.at(-1).state.unverifiableStreak, 0) + assert.match(results.at(-1).state.phoneTrip, /unverifiable: 3 samples/) + const [noBackground] = run([phoneSampleAt(0)], freshState({ flippedAt: null })) + assert.match(noBackground.rules.phoneCredentialRefusals.reason, /--flipped-at/) + // A gap before the judged 3 minutes changes nothing. + const early = { ...phoneSampleAt(0), phone: { ...phoneSampleAt(0).phone, unmetered: ['2026-10-19T12:02'] } } + const [earlyGap] = run([early], phoneState()) + assert.equal(earlyGap.rules.phoneCredentialRefusals.status, 'pass') +}) + +test('phone rules: memory set partway through a watch measures from then, not from the reserve flip', () => { + const state = freshState({ flippedAt: START - 60 * MINUTE }) + const [before, after] = run([sampleAt(START), phoneSampleAt(1)], state) + assert.equal(before.state.phoneSince, null) + assert.equal(after.state.phoneSince, START) + assert.equal(after.rules.phoneCredentialRefusals.status, 'pending') +}) + +test('phone rules: audit drops and failing renewals page once when sustained, and flip nothing', () => { + const dropping = (offset, dropped) => phoneSampleAt(offset, () => ({}), memoryCell({ resumeAuditDropped: dropped })) + // One drop, then none: no page. + const single = run([dropping(0, 0), dropping(1, 1), ...Array.from({ length: 8 }, (_, index) => dropping(index + 2, 1))], phoneState()) + assert.ok(single.every((result) => result.pages.length === 0)) + const rising = run(Array.from({ length: 9 }, (_, offset) => dropping(offset, offset * 3)), phoneState()) + assert.deepEqual(rising.map((result) => result.pages.join()), ['', '', '', '', '', '', 'audit-drops', '', '']) + const renewing = run(Array.from({ length: 7 }, (_, offset) => phoneSampleAt(offset, () => ({ renewalFailures: 1 }))), phoneState()) + assert.deepEqual(renewing.map((result) => result.pages.join()).filter(Boolean), ['renewal-failures']) + assert.ok([...rising, ...renewing].every((result) => result.state.state === 'watching' && result.state.phoneTrip === null)) +}) + +test('the cell phone read sums each minute from the cell runtime metrics lines, and refuses an old image', async () => { + const endedAt = Date.parse('2026-10-19T12:03:00Z') + const line = (at, payload) => ({ + timestamp: new Date(at).toISOString(), + jsonPayload: { + phoneRejectionsByCodeDelta: {}, + resumeConfirmFailuresByCodeDelta: {}, + phoneAttachDeadlinesDelta: 0, + phoneAcceptDatabaseErrorsDelta: 0, + resumeRenewalFailuresDelta: 0, + clientAcceptCompletedDelta: 0, + ...payload + } + }) + let filter + const read = await readCellPhoneMinutes( + fakeReader({ + runGcloud: async (args) => { + filter = args[2] + return { + stdout: JSON.stringify([ + line(endedAt - 150_000, { phoneRejectionsByCodeDelta: { 4401: 2, 4404: 9 } }), + line(endedAt - 120_000, { resumeConfirmFailuresByCodeDelta: { 'reject-revoked': 1, error: 2 } }), + line(endedAt - 90_000, { phoneAttachDeadlinesDelta: 3, phoneAcceptDatabaseErrorsDelta: 1, resumeRenewalFailuresDelta: 4, clientAcceptCompletedDelta: 7 }) + ]) + } + } + }), + { cellId: CELL, endedAt, minutes: 3 } + ) + assert.match(filter, /resource\.type="gce_instance"/) + assert.match(filter, new RegExp(`jsonPayload\\.cellId="${CELL}"`)) + assert.deepEqual(read.series, { + credentialRefusals: [2, 0, 0], + confirmFailures: [3, 0, 0], + attachDeadlines: [0, 3, 0], + acceptDatabaseErrors: [0, 1, 0], + renewalFailures: [0, 4, 0], + completedAccepts: [0, 7, 0] + }) + assert.deepEqual(read.unmetered, ['2026-10-19T12:02']) + const old = await readCellPhoneMinutes( + fakeReader({ runGcloud: async () => ({ stdout: JSON.stringify([{ timestamp: new Date(endedAt - 1).toISOString(), jsonPayload: {} }]) }) }), + { cellId: CELL, endedAt, minutes: 3 } + ) + assert.match(old.error, /no phone counters/) + assert.equal(old.permanent, true) +}) + +// The watch over a memory cell whose phone metrics come from `phoneMetrics(at)`. +async function phoneWatch(phoneMetrics, minutes) { + const directory = await mkdtemp(join(tmpdir(), 'relay-reserve-watch-')) + try { + let clock = START + const lines = [] + const config = parseAt(watchValues({ 'state-file': join(directory, 'state.json'), 'segment-minutes': String(minutes) })) + const sources = fakeSources({ + cell: () => ({ flagsApplied: { generation: 4, flags: { admitMode: 'reserve', readinessLocal: true, phonePath: 'memory' } } }) + }) + const runGcloud = async (args) => + args[2]?.includes('gce_instance') ? { stdout: JSON.stringify(metricsLines(args[2], (at) => [phoneMetrics(at)])) } : sources.runGcloud(args) + const state = await watchSegment(config, { + ...sources, + runGcloud, + idToken: 'id', + accessToken: 'access', + audit: {}, + retryDelayMs: 0, + now: () => clock, + sleep: async (ms) => { clock += ms }, + log: (line) => lines.push(line) + }) + return { state, lines } + } finally { + await rm(directory, { recursive: true, force: true }) + } +} +const phoneCounters = (overrides = {}) => ({ + phoneRejectionsByCodeDelta: {}, + resumeConfirmFailuresByCodeDelta: {}, + phoneAttachDeadlinesDelta: 0, + phoneAcceptDatabaseErrorsDelta: 0, + resumeRenewalFailuresDelta: 0, + clientAcceptCompletedDelta: 5, + ...overrides +}) + +test('a phone trip whose phonePath=db write fails keeps watching, retries each sample, and pages after 3 failures', async () => { + // No bucket here, so every write fails. + const { state, lines } = await phoneWatch((at) => phoneCounters(at > START ? { phoneAttachDeadlinesDelta: 20 } : {}), 15) + assert.equal(state.state, 'watching') + assert.equal(state.phoneOffAt, null) + assert.match(state.phoneTrip, /phoneAttachDeadlines: /) + const failures = lines.filter((line) => line.includes('orca_relay_reserve_watch_phone_path_off_failed')) + assert.ok(failures.length > 3, `${failures.length} failed writes`) + assert.equal(state.phoneOffFailures, failures.length) + const alarms = lines.filter((line) => line.includes('may still have phonePath=memory')) + assert.equal(alarms.length, 1) +}) + +test('a phone trip still unwritten when the watch ends leaves it to the flip back', () => { + const [last] = run([phoneSampleAt(0)], phoneState({ deadlineAt: START, phoneTrip: 'turn off the phone path on x: phoneAttachDeadlines: y' })) + assert.equal(last.state.state, 'tripped') + assert.equal(last.state.flip, 'phonePath') + assert.match(last.state.verdict, /phoneAttachDeadlines: y; phonePath=db did not land before the watch ended/) +}) + +test('a phone background broken by a restart or roll refuses the watch instead of turning phones off for certain', async () => { + const { state } = await phoneWatch((at) => (at < START - 8 * MINUTE ? {} : phoneCounters()), 5) + assert.equal(state.state, 'refused') + assert.match(state.verdict, /phone background.*incomplete.*later --flipped-at/) +}) + +test('a phone-rule flip back removes phonePath, and does nothing once the file no longer says memory', async () => { + const { fetchImpl, calls } = fakeFlipBack({ flags: { admitMode: 'reserve', readinessLocal: true, phonePath: 'memory' } }) + const result = await flipBack(flipValues({ flip: 'phonePath' }), flipDependencies(fetchImpl, [])) + assert.equal(result.written, true) + const upload = calls.find((call) => call.url.includes('/upload/storage/v1/')) + assert.match(upload.body, /"flags":\{"admitMode":"reserve","readinessLocal":true\}/) + assert.ok(!calls.some((call) => call.url.endsWith('/v1/admin/cell-admit-mode') && JSON.parse(call.body).admitMode)) + const done = await flipBack(flipValues({ flip: 'phonePath' }), flipDependencies(fakeFlipBack().fetchImpl, [])) + assert.equal(done.alreadyDb, true) +}) + +test('both flip backs land on a cell rolled back to an image without the phonePath switch', async () => { + const flags = { admitMode: 'reserve', readinessLocal: true, phonePath: 'memory' } + for (const flip of ['admitMode', 'phonePath']) { + const { fetchImpl, calls } = fakeFlipBack({ flags, supportedFlags: { admitMode: { type: 'enum', values: ['db', 'reserve'] }, readinessLocal: { type: 'boolean' } } }) + const result = await flipBack(flipValues({ flip }), flipDependencies(fetchImpl, [])) + assert.equal(result.written, true, flip) + const upload = calls.find((call) => call.url.includes('/upload/storage/v1/')) + assert.doesNotMatch(upload.body, /phonePath/, flip) + } +}) + +test('workflow: the flip back writes what the watch says tripped, and the whole cell when it says nothing', () => { + const workflow = watchWorkflow + assert.match(workflow, /flip: \$\{\{ steps\.outcome\.outputs\.flip \}\}/) + assert.match(workflow, /FLIP: \$\{\{ needs\.watch\.outputs\.flip \|\| 'admitMode' \}\}/) + assert.match(workflow, /--flip "\$\{FLIP\}"/) + // A failed phone flip back pages with the phone wording. + assert.match(workflow, /--verdict "\$\{VERDICT\}" --flip "\$\{FLIP\}"\n/) + assert.match(workflow, /ORCA_RELAY_CONTROL_ACCESS_TOKEN="\$\(gcloud auth print-access-token\)"\n\s+echo "::add-mask::/) +}) + + +test('a failed phone flip back and the phone pages say what is still on', async () => { + const lines = [] + const runGcloud = async () => ({ stdout: '' }) + await raiseAlarm({ mode: 'dry-run', cellId: CELL, projectId: 'onorca-cloud', kind: 'flip-back-failed', reason: 'x', flip: 'phonePath' }, { runGcloud, log: (line) => lines.push(line) }) + await raiseAlarm({ mode: 'dry-run', cellId: CELL, projectId: 'onorca-cloud', kind: 'renewal-failures', reason: 'y' }, { runGcloud, log: (line) => lines.push(line) }) + assert.match(lines[0], /may still have phonePath=memory/) + assert.doesNotMatch(lines[0], /reserve mode/) + assert.match(lines[1], /cannot renew phone credentials/) +}) + +// Recorded, not synthetic: 26 h of c7, c28 and c29 runtime metrics (read-only, 10-18/19). The phone +// counters are not on those images yet, so auth failures (host and phone 4401s, an over-count) +// stand in for the refused or failed attempts, against completed accepts. They cannot stand in for +// confirm failures: c28's 04:08Z minutes (9/9, 9/9, 11/10) read as confirms are a total outage. +const PHONE_RULE_SIGNAL_NAMES = ['credentialRefusals', 'confirmFailures', 'attachDeadlines', 'acceptDatabaseErrors'] +const recordedPhoneMinutes = JSON.parse(readFileSync(new URL('./relay-phone-watch-recorded-quiet-minutes.json', import.meta.url), 'utf8')).cells +// The minute index a recorded watch of `signal` from `start` trips at, or null; `minute(i)` may rewrite a minute. +function recordedTrip({ authFailuresPerMinute: failures, completedAcceptsPerMinute: accepts }, signal, start, watchMinutes, minute = (i) => [failures[i], accepts[i]]) { + const { readMinutes, consecutiveMinutes } = PHONE_WATCH_RULES + const refused = failures.slice(start - readMinutes, start) + const background = measuredPhoneBackground({ + ok: true, + minutes: refused, + unmetered: [], + series: { ...Object.fromEntries(PHONE_RULE_SIGNAL_NAMES.map((name) => [name, refused])), completedAccepts: accepts.slice(start - readMinutes, start) } + }) + const line = phoneShareLine(background.share[signal]) + let run = 0 + for (let index = start; index < start + watchMinutes; index += 1) { + run = phoneMinuteOver(signal, ...minute(index), line) ? run + 1 : 0 + if (run >= consecutiveMinutes) return index + } + return null +} + +test('phone rules: 26 recorded hours of c7, c28 and c29 trip no attempt rule in any 60- or 300-minute watch', () => { + assert.deepEqual(Object.keys(recordedPhoneMinutes), ['production-gce-c7', 'production-gce-c28', 'production-gce-c29']) + for (const [cell, minutes] of Object.entries(recordedPhoneMinutes)) { + for (const watchMinutes of [60, 300]) { + for (let start = 10; start + watchMinutes <= minutes.authFailuresPerMinute.length; start += 5) { + assert.equal(recordedTrip(minutes, 'credentialRefusals', start, watchMinutes), null, `${cell}: a ${watchMinutes}-min watch from minute ${start} trips`) + } + } + } +}) + +// From minute 30 of each recorded hour, a broken path; it must trip by its third minute (sooner +// when the minute before was already over) wherever those minutes carry 4 attempts each. +const BROKEN_PATHS = { + 'two refusals per accept': ['credentialRefusals', (failed, accepted) => [Math.max(2 * accepted, 4), accepted]], + 'every accept errors on the database, phones give up after one try': ['acceptDatabaseErrors', (failed, accepted) => [accepted, 0]], + // Only trips are tested: the auth-failure stand-in cannot measure confirm false trips. + 'every confirm fails': ['confirmFailures', (failed, accepted) => [accepted, accepted]] +} +test('phone rules: the same recorded hours trip on each broken path wherever its minutes are judged', () => { + for (const [name, [signal, breakMinute]] of Object.entries(BROKEN_PATHS)) { + for (const [cell, minutes] of Object.entries(recordedPhoneMinutes)) { + const { authFailuresPerMinute: failures, completedAcceptsPerMinute: accepts } = minutes + let judged = 0 + for (let start = 10; start + 60 <= failures.length; start += 5) { + const broken = (index) => (index >= start + 30 ? breakMinute(failures[index], accepts[index]) : [failures[index], accepts[index]]) + const judgedMinutes = [0, 1, 2].every((offset) => { + const [count, accepted] = broken(start + 30 + offset) + return (signal === 'confirmFailures' ? accepted : count + accepted) >= PHONE_WATCH_RULES.minAttemptsPerMinute + }) + if (!judgedMinutes) continue + judged += 1 + const at = recordedTrip(minutes, signal, start, 60, broken) + // Confirms may trip earlier: the stand-in reads as failing confirms on c28 (see above). + const earliest = signal === 'confirmFailures' ? start : start + 30 + assert.ok(at !== null && at >= earliest && at <= start + 32, `${cell} ${name}: from minute ${start + 30} trips at ${at}`) + } + // Busy cells are judged most of the day; low-volume c7 rarely reaches 4 attempts a minute. + if (cell !== 'production-gce-c7') assert.ok(judged > 150, `${cell} ${name}: ${judged} judged`) + } + } +})