diff --git a/.github/workflows/mobile.yml b/.github/workflows/mobile.yml index 2cec9676063..aae50234bf9 100644 --- a/.github/workflows/mobile.yml +++ b/.github/workflows/mobile.yml @@ -103,22 +103,15 @@ jobs: - name: Typecheck tests (ratchet) run: pnpm run check:tests-typecheck + # This includes the bridged replay of the whole recording corpus, which used to be a second + # step of its own behind RPC_FOUNDATION_BRIDGE=1. A gate nobody can forget to set is the point: + # it fails when a divergence class grows, when a divergence lands in no class at all, or when + # one of the 103 goldens inside the C1 page closure changes the verdict it is pinned to. It is + # ~3 min of test time on its own, and Vitest runs it on a worker beside the rest of the suite, + # so folding it in costs a fraction of that in wall time and one step less to skip. - name: Test run: pnpm test - # Why a second run of the same corpus: `pnpm test` leaves this suite off, because it replays - # every golden through the page bridge and the per-class counts it pins are the only thing - # that says how far that bridge is from byte-identical. It fails when a class grows or when a - # divergence lands in no class at all, so a change that widens the gap cannot land quietly. - # ~2.5 min locally, because a golden that diverges is replayed a second time with `_meta` - # supplied and that counterfactual is what separates the reader's share of the gap from the - # rest. Ungated on purpose: unlike the pin guard's reproduce, this verdict moves on any change - # to the bridge, which no path filter on the corpus would catch. - - name: Replay the recording corpus through the page bridge - env: - RPC_FOUNDATION_BRIDGE: '1' - run: pnpm exec vitest run src/test-support/rpc-recording/rpc-recording-through-bridge.test.ts - - name: Test iOS release version resolution run: ruby fastlane/ios_release_version_test.rb diff --git a/mobile/src/test-support/bridged-parity/c1-page-closure.test.ts b/mobile/src/test-support/bridged-parity/c1-page-closure.test.ts new file mode 100644 index 00000000000..d4af9f116dc --- /dev/null +++ b/mobile/src/test-support/bridged-parity/c1-page-closure.test.ts @@ -0,0 +1,117 @@ +import { readdirSync, readFileSync } from 'node:fs' +import { resolve } from 'node:path' +import { describe, expect, it } from 'vitest' +import { + c1PageClosureDrift, + c1PageClosureExclusions, + C1_PAGE_CLOSURE, + type C1PageClosureObservation +} from './c1-page-closure' +import { BRIDGED_PARITY_EXCLUSIONS } from './divergence-classes' + +const GOLDENS = resolve(import.meta.dirname, '../../../rpc-foundation/goldens') + +/** The run a corpus that diverged exactly as the pin says would hand the rule. */ +function asPinned(): Map { + const run = new Map() + for (const [family, goldens] of Object.entries(C1_PAGE_CLOSURE)) { + for (const [id, verdict] of Object.entries(goldens)) { + run.set(id, { family, verdict }) + } + } + return run +} + +describe('the C1 page closure', () => { + it('is the census the design named: 22 families, 103 goldens', () => { + const goldens = Object.values(C1_PAGE_CLOSURE).flatMap((family) => Object.keys(family)) + expect({ families: Object.keys(C1_PAGE_CLOSURE).length, goldens: goldens.length }).toEqual({ + families: 22, + goldens: 103 + }) + expect(new Set(goldens).size).toBe(goldens.length) + }) + + it('pins goldens that exist, in the family the corpus records them under', () => { + for (const [family, goldens] of Object.entries(C1_PAGE_CLOSURE)) { + for (const id of Object.keys(goldens)) { + // oxlint-disable-next-line typescript/consistent-type-assertions -- SAFETY: every golden carries `family`; a file that does not fails this read and the test. + const recorded = JSON.parse(readFileSync(`${GOLDENS}/${id}.json`, 'utf8')) as { + family: string + } + expect({ id, family: recorded.family }).toEqual({ id, family }) + } + } + }) + + it('claims no golden the corpus does not have', () => { + const corpus = new Set( + readdirSync(GOLDENS) + .filter((name) => name.endsWith('.json')) + .map((name) => name.replace(/\.json$/, '')) + ) + const missing = Object.values(C1_PAGE_CLOSURE) + .flatMap((family) => Object.keys(family)) + .filter((id) => !corpus.has(id)) + expect(missing).toEqual([]) + }) + + it('excludes a closure golden only into a class that has a reason', () => { + const exclusions = c1PageClosureExclusions() + expect(exclusions.length).toBeGreaterThan(0) + expect(exclusions.filter(([, name]) => BRIDGED_PARITY_EXCLUSIONS[name] === undefined)).toEqual( + [] + ) + }) +}) + +describe('reading a run against the pin', () => { + it('says nothing when the run is the pin', () => { + expect(c1PageClosureDrift(asPinned())).toEqual([]) + }) + + it('names a closure golden that changed verdict', () => { + const run = asPinned() + // Whichever golden it is, the verdict it moves to has to be one it is not already pinned to. + const found = [...run].find(([, seen]) => seen.verdict !== 'params-undefined') + if (found === undefined) { + throw new Error('the pin is empty') + } + const [id, observation] = found + run.set(id, { ...observation, verdict: 'params-undefined' }) + const drift = c1PageClosureDrift(run) + expect(drift.length).toBe(1) + expect(drift[0]).toContain(id) + expect(drift[0]).toContain(`pinned ${observation.verdict}, ran params-undefined`) + }) + + it('names a golden newly derived into a closure family, which no id list would', () => { + const run = asPinned() + const [family] = Object.keys(C1_PAGE_CLOSURE) + if (family === undefined) { + throw new Error('the pin is empty') + } + run.set('matrix-arrived-1', { family, verdict: 'identical' }) + expect(c1PageClosureDrift(run)).toEqual([`${family}: arrived matrix-arrived-1; left (none)`]) + }) + + it('names a closure golden the run stopped producing', () => { + const run = asPinned() + const [family, goldens] = Object.entries(C1_PAGE_CLOSURE)[0] ?? [] + const [id] = Object.keys(goldens ?? {}) + if (family === undefined || id === undefined) { + throw new Error('the pin is empty') + } + run.delete(id) + expect(c1PageClosureDrift(run)).toEqual([`${family}: arrived (none); left ${id}`]) + }) + + it('ignores every golden outside the closure, which is most of the corpus', () => { + const run = asPinned() + run.set('worktree-catalog-snapshot-unreadable-elsewhere', { + family: 'session.diff-review', + verdict: 'result-absent-settlement' + }) + expect(c1PageClosureDrift(run)).toEqual([]) + }) +}) diff --git a/mobile/src/test-support/bridged-parity/c1-page-closure.ts b/mobile/src/test-support/bridged-parity/c1-page-closure.ts new file mode 100644 index 00000000000..19a7907644b --- /dev/null +++ b/mobile/src/test-support/bridged-parity/c1-page-closure.ts @@ -0,0 +1,234 @@ +/** + * The goldens recorded at a call site inside the C1 page closure, and what each one did at the + * bridge. + * + * C1 moves a screen to the web: `app/h/_layout.tsx` and `app/h/[hostId]/index.tsx` and everything + * they import. The suite next door already proves the corpus replays byte-identically or in a named + * class, but it proves it as counts over 787 goldens, and a count is the wrong instrument for the + * claim C1 needs. These 103 are the ones whose divergence would be this domain's divergence, so + * each is pinned by id to the verdict it gives, not counted into a total another golden can pay for. + * + * The rule is not "none excluded". 54 replay byte for byte and 49 do not, in four of the five + * classes the suite next door names — 37 `result-absent-settlement`, 7 `params-undefined`, 3 + * `result-absent-stream-release`, 2 `write-ordinal`. Every one is a recorder observation artifact + * whose wire bytes C0.5 and C0.8 proved identical: what differs is the shape the recorder injects + * below the frame boundary, or the pre-serialization object a step is matched against, and neither + * is something a transport carries. What the pin buys is that the 49 are named. A fiftieth arriving + * is a red test here even though every count in `BRIDGED_PARITY_BASELINE` still holds, because the + * class it joined has room in its bound for a golden that left. + * + * Derived from the value-import closure of the two route modules with `.web.*` resolution applied, + * against the module each operation's mount adapter loads. `mobileWeb.bundle-manifest` is not here: + * it reaches the closure only through the shared `rpc-operation.ts` runner, and its own operation + * module is the shell's, not the page's. + */ + +import type { BridgedParityClass } from './divergence-classes' + +/** Byte-identical, or the class that named the divergence. */ +export type BridgedParityVerdict = BridgedParityClass | 'identical' + +export type C1PageClosureObservation = { + family: string + verdict: BridgedParityVerdict +} + +export const C1_PAGE_CLOSURE: Readonly< + Record>> +> = { + 'settings.repo-metadata': { + 'matrix-settings.repo-metadata-host.platform-1': 'result-absent-settlement', + 'matrix-settings.repo-metadata-repo.list-1': 'result-absent-settlement', + 'matrix-settings.repo-metadata-settings.get-1': 'result-absent-settlement', + 'matrix-settings.repo-metadata-ssh.listtargetsummaries-1': 'result-absent-settlement', + 'schedules-settings-repo-metadata-fulfilled': 'identical', + 'settings-repo-cache-expiry': 'identical', + 'settings-repo-metadata-fulfilled': 'identical', + 'settings-repo-metadata-icons': 'identical', + 'settings-repo-metadata-refuse-after-data': 'identical', + 'settings-repo-metadata-refused': 'identical', + 'settings-repo-metadata-single-host': 'identical', + 'settings-repo-metadata-transport-error': 'identical' + }, + 'settings.workspace-context': { + 'lifecycle-settings-workspace-context-fulfilled': 'identical', + 'matrix-settings.workspace-context-linear.status-1': 'result-absent-settlement', + 'matrix-settings.workspace-context-preflight.check-1': 'result-absent-settlement', + 'matrix-settings.workspace-context-settings.get-1': 'result-absent-settlement', + 'matrix-settings.workspace-context-ui.get-1': 'result-absent-settlement', + 'schedules-settings-workspace-context-fulfilled': 'identical', + 'settings-workspace-context-fulfilled': 'identical', + 'settings-workspace-context-refuse-after-data': 'identical', + 'settings-workspace-context-refused': 'identical', + 'settings-workspace-context-transport-error': 'identical' + }, + 'tasks.smart-source-search': { + 'matrix-tasks.smart-source-search-github.listworkitems-1': 'params-undefined', + 'matrix-tasks.smart-source-search-gitlab.listworkitems-1': 'params-undefined', + 'matrix-tasks.smart-source-search-linear.listissues-1': 'params-undefined', + 'matrix-tasks.smart-source-search-linear.searchissues-1': 'params-undefined', + 'matrix-tasks.smart-source-search-repo.searchrefs-1': 'params-undefined', + 'tw-smart-search-all-providers': 'params-undefined', + 'tw-smart-search-gitlab-provider-error': 'identical', + 'tw-smart-search-linear-listed': 'params-undefined' + }, + 'worktree.create-retry': { + 'matrix-worktree.create-retry-worktree.create-1': 'result-absent-settlement', + 'tw-create-retry-ambiguous-after-drop': 'identical', + 'tw-create-retry-ambiguous-while-connected': 'identical', + 'tw-create-retry-ambiguous-without-idempotency': 'identical', + 'tw-create-retry-created': 'identical', + 'tw-create-retry-name-collision': 'identical', + 'tw-create-retry-unretryable-refusal': 'identical', + 'tw-create-retry-warning-kept': 'identical' + }, + 'tasks.paste-lookup': { + 'matrix-tasks.paste-lookup-github.reposlug-1': 'result-absent-settlement', + 'matrix-tasks.paste-lookup-github.workitem-1': 'result-absent-settlement', + 'matrix-tasks.paste-lookup-github.workitembyownerrepo-1': 'result-absent-settlement', + 'matrix-tasks.paste-lookup-gitlab.workitembypath-1': 'result-absent-settlement', + 'tw-paste-lookup-resolved': 'identical', + 'tw-paste-lookup-slug-refused': 'identical', + 'tw-paste-lookup-slug-unsupported': 'identical' + }, + 'host-worktree-refresh': { + 'host-worktree-refresh-stream': 'write-ordinal', + 'matrix-host-worktree-refresh-runtime.clientevents.subscribe-1-1': + 'result-absent-stream-release', + 'matrix-host-worktree-refresh-runtime.clientevents.subscribe-1-2': + 'result-absent-stream-release', + 'matrix-host-worktree-refresh-runtime.clientevents.subscribe-1-3': + 'result-absent-stream-release', + 'matrix-host-worktree-refresh-runtime.clientevents.subscribe-2-1': 'write-ordinal' + }, + 'host.worktree-actions': { + 'host-worktree-actions-pin-open-delete': 'identical', + 'host-worktree-delete-refused': 'identical', + 'matrix-host.worktree-actions-worktree.activate-1': 'result-absent-settlement', + 'matrix-host.worktree-actions-worktree.rm-1': 'result-absent-settlement', + 'matrix-host.worktree-actions-worktree.set-1': 'result-absent-settlement' + }, + 'transport.capability-probe': { + 'matrix-transport.capability-probe-status.get-1': 'result-absent-settlement', + 'transport-capability-probe-cutover-reasks-fast': 'identical', + 'transport-capability-probe-non-string-capabilities-drop': 'identical', + 'transport-capability-probe-publishes': 'identical', + 'transport-capability-probe-refused-backs-off': 'identical' + }, + 'components.execution-target': { + 'components-target-ssh': 'identical', + 'matrix-components.execution-target-preflight.detectremoteagents-1': 'result-absent-settlement', + 'matrix-components.execution-target-ssh.connect-1': 'result-absent-settlement', + 'matrix-components.execution-target-ssh.getstate-1': 'result-absent-settlement' + }, + 'notifications.push-registration': { + 'matrix-notifications.push-registration-notifications.registerpush-1': + 'result-absent-settlement', + 'matrix-notifications.push-registration-notifications.unregisterpush-1': + 'result-absent-settlement', + 'notifications-push-gateway-rejected': 'identical', + 'notifications-push-registered': 'identical' + }, + 'settings.workspace-submit': { + 'matrix-settings.workspace-submit-settings.get-1': 'result-absent-settlement', + 'settings-workspace-submit-fulfilled': 'identical', + 'settings-workspace-submit-refused': 'identical', + 'settings-workspace-submit-transport-error': 'identical' + }, + 'transport.host-status-gates': { + 'matrix-transport.host-status-gates-status.get-1': 'result-absent-settlement', + 'transport-host-status-gates-drop-keeps-capabilities': 'identical', + 'transport-host-status-gates-ready': 'identical', + 'transport-host-status-gates-refused-degrades': 'identical' + }, + 'worktree.hosted-base': { + 'matrix-worktree.hosted-base-worktree.resolvemrbase-1': 'result-absent-settlement', + 'matrix-worktree.hosted-base-worktree.resolveprbase-1': 'result-absent-settlement', + 'tw-hosted-base-resolved': 'identical', + 'tw-hosted-base-soft-error': 'identical' + }, + 'worktree.runtime-capabilities': { + 'matrix-worktree.runtime-capabilities-status.get-1': 'result-absent-settlement', + 'tw-capabilities-advertised': 'identical', + 'tw-capabilities-cutover-retried': 'identical', + 'tw-capabilities-legacy-idempotency': 'identical' + }, + 'host.view-settings': { + 'host-view-settings-sync': 'identical', + 'matrix-host.view-settings-ui.get-1': 'result-absent-settlement', + 'matrix-host.view-settings-ui.set-1': 'result-absent-settlement' + }, + 'worktree.catalog-snapshot': { + 'matrix-worktree.catalog-snapshot-worktree.ps-1': 'result-absent-settlement', + 'worktree-catalog-snapshot': 'identical', + 'worktree-catalog-snapshot-unreadable': 'result-absent-settlement' + }, + 'worktree.setup-hook-trust': { + 'matrix-worktree.setup-hook-trust-ui.set-1': 'result-absent-settlement', + 'tw-setup-hook-trust-always': 'identical', + 'tw-setup-hook-trust-approved': 'identical' + }, + 'components.execution-target-local': { + 'components-target-local': 'identical', + 'matrix-components.execution-target-local-preflight.detectagents-1': 'result-absent-settlement' + }, + 'components.new-workspace-repositories': { + 'matrix-components.new-workspace-repositories-repo.list-1': 'result-absent-settlement', + 'new-workspace-repositories-fulfilled': 'identical' + }, + 'components.setup-script': { + 'components-setup-ask': 'identical', + 'matrix-components.setup-script-repo.hooks-1': 'result-absent-settlement' + }, + 'worktree.agent-launch-create': { + 'matrix-worktree.agent-launch-create-agent.launch-1': 'result-absent-settlement', + 'tw-create-retry-agent-launched': 'identical' + }, + 'worktree.retired-names': { + 'matrix-worktree.retired-names-worktree.listretirednames-1': 'result-absent-settlement', + 'worktree-retired-names': 'identical' + } +} + +/** Every closure golden that did not replay byte-identically, which the suite prints beside why. */ +export function c1PageClosureExclusions(): readonly (readonly [string, BridgedParityClass])[] { + return Object.values(C1_PAGE_CLOSURE).flatMap((family) => + Object.entries(family).flatMap(([id, verdict]) => + verdict === 'identical' ? [] : [[id, verdict] as const] + ) + ) +} + +/** + * Each closure family whose goldens or verdicts are not the ones pinned above, said in one line. + * + * Membership is checked per family rather than against the flat id list, so a golden newly derived + * into a family this domain owns arrives as a finding instead of going unnoticed for being absent + * from a pin that never mentioned it. + */ +export function c1PageClosureDrift( + observed: ReadonlyMap +): readonly string[] { + const byFamily = new Map() + for (const [id, { family }] of observed) { + byFamily.set(family, [...(byFamily.get(family) ?? []), id]) + } + const drift: string[] = [] + for (const [family, pinned] of Object.entries(C1_PAGE_CLOSURE)) { + const seen = byFamily.get(family) ?? [] + const arrived = seen.filter((id) => !(id in pinned)) + const left = Object.keys(pinned).filter((id) => !seen.includes(id)) + if (arrived.length > 0 || left.length > 0) { + drift.push( + `${family}: arrived ${arrived.join(', ') || '(none)'}; left ${left.join(', ') || '(none)'}` + ) + } + for (const [id, verdict] of Object.entries(pinned)) { + const ran = observed.get(id)?.verdict + if (ran !== undefined && ran !== verdict) { + drift.push(`${id}: pinned ${verdict}, ran ${ran}`) + } + } + } + return drift +} diff --git a/mobile/src/test-support/bridged-parity/divergence-classes.test.ts b/mobile/src/test-support/bridged-parity/divergence-classes.test.ts index 8213ab3dcc6..58d3e12efc5 100644 --- a/mobile/src/test-support/bridged-parity/divergence-classes.test.ts +++ b/mobile/src/test-support/bridged-parity/divergence-classes.test.ts @@ -3,14 +3,17 @@ import { resolve } from 'node:path' import { describe, expect, it } from 'vitest' import { bridgedParityMembershipDrift, + bridgedParityTallyDrift, classifyBridgedParity, BRIDGED_PARITY_BASELINE, BRIDGED_PARITY_EXCLUSIONS, BRIDGED_PARITY_FLAG, BRIDGED_PARITY_MEMBERS, + BRIDGED_PARITY_OFF, BRIDGED_PARITY_NAMEABLE, type BridgedParityClass, - type BridgedParityEvidence + type BridgedParityEvidence, + type BridgedParityTally } from './divergence-classes' const base: BridgedParityEvidence = { @@ -34,9 +37,16 @@ const droppedUndefinedKey: BridgedParityEvidence = { } describe('the bridged-parity flag', () => { - it('is the name the suite, the pin and the CI job all spell', () => { + it('is the name the suite and the pin both spell', () => { expect(BRIDGED_PARITY_FLAG).toBe('RPC_FOUNDATION_BRIDGE') }) + + it('skips on one value only, so an unset or mistyped variable still runs the gate', () => { + expect(BRIDGED_PARITY_OFF).toBe('0') + const skips = (value: string | undefined): boolean => value === BRIDGED_PARITY_OFF + expect([undefined, '', '1', 'false', 'off'].filter(skips)).toEqual([]) + expect(skips('0')).toBe(true) + }) }) describe('classifying one diverging golden', () => { @@ -145,9 +155,9 @@ describe('what the pin still admits', () => { }) it('accounts for every golden in the corpus, once each', () => { - // What ties the two halves of the ratchet together. Each class is an upper bound and `identical` - // a lower one, so on their own a class could be loosened by one and nothing would notice; with - // the sum pinned to the corpus, a class that grows has to be paid for out of another. + // What ties the numbers to the corpus they describe. Each is pinned exactly, but only to a + // number in this file; the sum being the size of the goldens directory is what says the pin + // covers every golden, once each, rather than a subset the run happened to reach. const goldens = readdirSync(resolve(import.meta.dirname, '../../../rpc-foundation/goldens')) const counted = Object.values(BRIDGED_PARITY_BASELINE).reduce((sum, count) => sum + count, 0) expect({ counted }).toEqual({ @@ -185,7 +195,7 @@ describe('what the pin still admits', () => { // The trade a count cannot see: every predicate reads the scenario rather than the frame the // page refused — `scriptsAbsentResultReply` asks whether the scenario scripts the injected // shape anywhere — so a real refusal in a stream golden lands in an excluded class. Let one - // golden leave as it arrives and the count, the sum and the `identical` floor all hold. + // golden leave as it arrives and the count, the sum and the `identical` pin all hold. expect(bridgedParityMembershipDrift(asPinned())).toEqual([]) const pinned = BRIDGED_PARITY_MEMBERS['write-ordinal'] ?? [] const traded = asPinned() @@ -202,6 +212,61 @@ describe('what the pin still admits', () => { expect(bridgedParityMembershipDrift(run)).toEqual([]) }) + /** The tally a run that lands exactly on the pin hands the rule. */ + function asCounted(): BridgedParityTally { + const counts: Record = { + 'reply-meta-required': 0, + 'result-absent-settlement': 0, + 'result-absent-observation': 0, + 'result-absent-stream-release': 0, + 'params-undefined': 0, + 'write-ordinal': 0, + unclassified: 0 + } + for (const name of classes) { + counts[name] = BRIDGED_PARITY_BASELINE[name] + } + return { identical: BRIDGED_PARITY_BASELINE.identical, counts } + } + + it('says nothing about the run the numbers were taken from', () => { + expect(bridgedParityTallyDrift(asCounted())).toEqual([]) + }) + + it('goes red on a golden that stopped diverging, which every other check lets through', () => { + // The direction the rest of the suite cannot see. One `result-absent-settlement` golden + // reported `identical` instead: nothing is unclassified, every diverging golden is still in an + // excluded class, and the corpus is still 787. Only these two numbers moved. + const tally = asCounted() + const moved: BridgedParityTally = { + identical: tally.identical + 1, + counts: { + ...tally.counts, + 'result-absent-settlement': tally.counts['result-absent-settlement'] - 1 + } + } + const drift = bridgedParityTallyDrift(moved) + expect(drift.length).toBe(2) + expect(drift.join('\n')).toContain( + `identical: pinned ${tally.identical}, ran ${moved.identical}` + ) + expect(drift.join('\n')).toContain('result-absent-settlement: pinned 341, ran 340') + }) + + it('goes red on a class that grew and on the corpus losing a golden', () => { + const tally = asCounted() + expect( + bridgedParityTallyDrift({ + ...tally, + counts: { ...tally.counts, unclassified: 1 } + }) + ).toEqual(['unclassified: pinned 0, ran 1']) + // A golden whose `it` threw before it was counted anywhere: the run is a golden short. + expect(bridgedParityTallyDrift({ ...tally, identical: tally.identical - 1 })).toEqual([ + `identical: pinned ${tally.identical}, ran ${tally.identical - 1}` + ]) + }) + it('leaves nothing for the reader to close: the `_meta` class is zero', () => { expect(BRIDGED_PARITY_BASELINE['reply-meta-required']).toBe(0) expect(BRIDGED_PARITY_EXCLUSIONS['reply-meta-required']).toBeUndefined() diff --git a/mobile/src/test-support/bridged-parity/divergence-classes.ts b/mobile/src/test-support/bridged-parity/divergence-classes.ts index 457fd501213..5d16e7a09e9 100644 --- a/mobile/src/test-support/bridged-parity/divergence-classes.ts +++ b/mobile/src/test-support/bridged-parity/divergence-classes.ts @@ -7,9 +7,19 @@ * one level up in `test-support`. */ -/** Named once so the suite, its pin and the CI job cannot drift apart. */ +/** + * The switch that turns the bridged replay off, named once so the suite and its pin cannot drift. + * + * It used to be what turned the replay *on*, and CI set it. That made the gate opt-in, which is the + * one thing a gate must not be: a branch that widened the bridge's divergence and left the variable + * alone would have been measured by nobody. The replay is the default now and `=0` is for a local + * run that does not want the three minutes. CI sets nothing. + */ export const BRIDGED_PARITY_FLAG = 'RPC_FOUNDATION_BRIDGE' +/** The one value of it that skips the suite; anything else, unset included, runs it. */ +export const BRIDGED_PARITY_OFF = '0' + export type BridgedParityClass = | 'reply-meta-required' | 'result-absent-settlement' @@ -157,17 +167,16 @@ export const BRIDGED_PARITY_EXCLUSIONS: Readonly> = { identical: 396, @@ -188,6 +197,32 @@ export const BRIDGED_PARITY_BASELINE: Readonly> +} + +/** + * Every number a run reported that `BRIDGED_PARITY_BASELINE` does not, said in one line each. + * + * Both directions, and `identical` on the same footing as a class, because that is the direction + * nothing else in the suite sees. A golden reported `identical` instead of the excluded class it + * belongs to leaves `unclassified` empty, leaves every diverging golden inside a class the + * exclusions name, and leaves the corpus its size: this is the only number that moves. + */ +export function bridgedParityTallyDrift(tally: BridgedParityTally): readonly string[] { + const ran: Readonly> = { ...tally.counts, identical: tally.identical } + const drift: string[] = [] + for (const [name, pinned] of Object.entries(BRIDGED_PARITY_BASELINE)) { + const count = ran[name] ?? 0 + if (count !== pinned) { + drift.push(`${name}: pinned ${pinned}, ran ${count}`) + } + } + return drift +} + /** A class this small is named golden by golden in the run's output rather than counted. */ export const BRIDGED_PARITY_NAMEABLE = 8 @@ -198,7 +233,7 @@ export const BRIDGED_PARITY_NAMEABLE = 8 * refused frame — `scriptsAbsentResultReply` asks whether the scenario scripts the injected shape * anywhere, not whether the frame the page refused was one — so a real refusal inside a stream * golden is named an excluded class. One golden leaving that class as the real refusal puts another - * in moves no number here, and the sum and the `identical` floor both still hold. The ids are what + * in moves no number here, and the sum and the `identical` pin both still hold. The ids are what * notices. Where a class is too large to list, its predicate stands on its own and the count is all * the pin has; that is why the classes here are the small ones and why the test above requires * every class of `BRIDGED_PARITY_NAMEABLE` or fewer to appear. diff --git a/mobile/src/test-support/rpc-recording/rpc-recording-through-bridge.test.ts b/mobile/src/test-support/rpc-recording/rpc-recording-through-bridge.test.ts index d3fd426b7e0..d6ee79a4dbe 100644 --- a/mobile/src/test-support/rpc-recording/rpc-recording-through-bridge.test.ts +++ b/mobile/src/test-support/rpc-recording/rpc-recording-through-bridge.test.ts @@ -8,14 +8,21 @@ import { import type { RpcClient } from '../../transport/rpc-client' import { bridgedParityMembershipDrift, - BRIDGED_PARITY_BASELINE, + bridgedParityTallyDrift, BRIDGED_PARITY_EXCLUSIONS, BRIDGED_PARITY_FLAG, BRIDGED_PARITY_NAMEABLE, + BRIDGED_PARITY_OFF, classifyBridgedParity, type BridgedParityClass, type BridgedParityEvidence } from '../bridged-parity/divergence-classes' +import { + c1PageClosureDrift, + C1_PAGE_CLOSURE, + type BridgedParityVerdict, + type C1PageClosureObservation +} from '../bridged-parity/c1-page-closure' import { divergingFields, paramsMismatchEvidence, @@ -44,21 +51,28 @@ import { vitestRecordingScheduler } from './vitest-recording-scheduler' * run. This suite writes nothing, and it is not in `RECORDING_DRIVERS`, so `recorderSha256` does * not pin it — a suite that cannot put an observation in a recorded file is not provenance for one. * + * It runs by default, in `pnpm test` and so in CI, and `RPC_FOUNDATION_BRIDGE=0` is what skips it + * for a local run that does not want the three minutes. Vitest gives the file a worker of its own + * beside the rest of the suite, so the gate costs much less in wall time than it does in test time. + * * ## What it asserts today * * Byte-identical replay where it holds, and the named shape of every divergence where it does not. * A golden that matches is compared in full; one that does not is classified by * `classifyBridgedParity`, which reads the frames and the scenario rather than the failure's text. - * The run fails if any class grows past `BRIDGED_PARITY_BASELINE`, if a single golden lands in - * `unclassified`, or if one diverges in a class `BRIDGED_PARITY_EXCLUSIONS` does not name. The - * corpus is a fixed size, so those together pin every count exactly, and for a class small enough - * to name `BRIDGED_PARITY_MEMBERS` pins which goldens are in it — a count alone cannot see one - * golden leaving a class as another arrives. + * The run fails if any count is not exactly its number in `BRIDGED_PARITY_BASELINE` — `identical` + * among them, which is the only check that sees a golden that stopped diverging as well as one that + * started — if a single golden lands in `unclassified`, or if one diverges in a class + * `BRIDGED_PARITY_EXCLUSIONS` does not name. For a class small enough to name, + * `BRIDGED_PARITY_MEMBERS` pins which goldens are in it — a count alone cannot see one golden + * leaving a class as another arrives. * * 396 of the 787 replay byte for byte. The other 391 fall in five classes, 341 / 3 / 6 / 33 / 8, - * and none of them is a reason to re-record anything. + * and none of them is a reason to re-record anything. `c1-page-closure.ts` then pins, golden by + * golden, the 103 recorded at a call site the C1 page owns, because a count over 787 cannot tell a + * domain's regression from another domain's improvement. * - * 1. **result-absent-settlement, 341** and **2. result-absent-observation, 7.** + * 1. **result-absent-settlement, 341** and **2. result-absent-observation, 3.** * `{ ok: true }` with no `result` key is refused by the page's reader and by `isRpcResponse` * alike, so this one is not a bridge defect: the recorder injects that partition at the scripted * sender port, below the frame validation both sides do, which is what the README means by not @@ -134,6 +148,8 @@ const counts: Record = { let identical = 0 const members = new Map() const samples = new Map() +/** Every golden's own verdict, which is what the C1 closure is pinned against golden by golden. */ +const observed = new Map() /** * The page's client over the shared port pair, holding the recorder's scripted client shell-side. @@ -235,15 +251,20 @@ function describeParamsMismatch(evidence: BridgedParityEvidence): string { */ async function verdict( id: string, + family: string, scenarios: readonly RecordingScenario[], run: Replay ): Promise { + const record = (name: BridgedParityVerdict): void => { + observed.set(id, { family, verdict: name }) + } const expected = readGolden(directory, id) const fields = run.recording === null ? [] : divergingFields(expected.recording, run.recording) if (run.recording !== null && fields.length === 0) { // Not redundant with the field walk: this one also pins the encoding and the header. compareGolden(expected, { ...expected, recording: run.recording }) identical += 1 + record('identical') return } const asIf = await replay(id, scenarios, withReplyMeta) @@ -263,6 +284,7 @@ async function verdict( } const name = classifyBridgedParity(evidence) counts[name] += 1 + record(name) if (name === 'unclassified') { throw new Error( `Unclassified bridged divergence: ${id}\n${explain(fields, run)}\n${describeParamsMismatch(evidence)}with \`_meta\` supplied:\n${explain(asIfFields, asIf)}` @@ -274,12 +296,17 @@ async function verdict( } } -describe.runIf(process.env[BRIDGED_PARITY_FLAG] === '1')( +describe.skipIf(process.env[BRIDGED_PARITY_FLAG] === BRIDGED_PARITY_OFF)( 'every golden replays through the page bridge, byte-identically or in a named class', () => { for (const pilot of pilotGoldens(input.scenarios)) { it(`${pilot.id}: bridged parity`, async () => { - await verdict(pilot.id, [pilot.scenario], await replay(pilot.id, [pilot.scenario])) + await verdict( + pilot.id, + pilot.family, + [pilot.scenario], + await replay(pilot.id, [pilot.scenario]) + ) }) } for (const golden of familyGoldens(input.scenarios)) { @@ -287,7 +314,7 @@ describe.runIf(process.env[BRIDGED_PARITY_FLAG] === '1')( `${golden.id}: bridged parity`, async () => { const scenarios = [...golden.scenarios()] - await verdict(golden.id, scenarios, await replay(golden.id, scenarios)) + await verdict(golden.id, golden.family, scenarios, await replay(golden.id, scenarios)) }, golden.timeoutMs ) @@ -324,12 +351,26 @@ describe.runIf(process.env[BRIDGED_PARITY_FLAG] === '1')( expect({ divergedOutsideAnExcludedClass: total(counts) - excludedCount }).toEqual({ divergedOutsideAnExcludedClass: 0 }) - for (const [name, count] of Object.entries(counts)) { - expect({ [name]: count }).toEqual({ - [name]: Math.min(count, BRIDGED_PARITY_BASELINE[asClass(name)]) - }) + // Every count exactly, `identical` included, which is the direction the three checks above + // cannot see: a golden reported `identical` rather than the excluded class it belongs to + // leaves all three holding. The size of the corpus follows, being the total of these. + expect({ tally: bridgedParityTallyDrift({ identical, counts }) }).toEqual({ tally: [] }) + }) + + it('gives every golden the C1 page closure records the verdict it is pinned to', () => { + const closure = [...observed].filter(([, seen]) => seen.family in C1_PAGE_CLOSURE) + const diverged = closure.filter(([, seen]) => seen.verdict !== 'identical') + process.stdout.write( + `\nC1 page closure: ${closure.length} goldens in ${ + Object.keys(C1_PAGE_CLOSURE).length + } families, ${closure.length - diverged.length} byte-identical\n` + ) + for (const [id, seen] of diverged) { + process.stdout.write(` ${id}: ${seen.verdict}\n`) } - expect(identical).toBeGreaterThanOrEqual(BRIDGED_PARITY_BASELINE.identical) + // Each by id, because the counts above cannot see this domain: a closure golden that stopped + // replaying identically is paid for by any of the other 684 that started. + expect({ closure: c1PageClosureDrift(observed) }).toEqual({ closure: [] }) }) } )