From f2fd18c82038eeaaee5fe7e29c44bc37aa2c1bd4 Mon Sep 17 00:00:00 2001 From: Jinwoo Hong <73622457+Jinwoo-H@users.noreply.github.com> Date: Fri, 18 Sep 2026 19:56:27 -0400 Subject: [PATCH] test(mobile): make the bridged recording-corpus replay a CI gate and pin the C1 page closure (OTA phase C, C1.6) (#21533) * test(mobile): the bridged corpus replay is a gate, not an opt-in flag (OTA phase C, C1.6) `rpc-recording-through-bridge.test.ts` replays all 787 goldens through the page bridge and pins how far that bridge is from byte-identical. It only ran when `RPC_FOUNDATION_BRIDGE=1` was set, which CI did in a step of its own. A gate whose CI job has to remember to set a variable is opt-in, and a branch that widened the divergence with that step edited away would have been measured by nobody. The suite now runs by default and `RPC_FOUNDATION_BRIDGE=0` skips it, for a local run that does not want the three minutes. `BRIDGED_PARITY_OFF` names the one value that skips, so an unset or mistyped variable still runs the gate. The dedicated CI step goes with it: `pnpm test` collects the file already, so keeping the step would have run the same 788 tests twice in one job. Vitest gives the file a worker beside the rest of the suite, so the marginal wall time is a fraction of the ~3 min it takes alone. Claude-Session: https://claude.ai/code/session_01JNnE9qzUZMMnqpZWCqM3nb * test(mobile): pin the bridged verdict of every C1-page-closure golden by id (OTA phase C, C1.6) The bridged replay certifies the corpus in counts over 787 goldens, and a count is the wrong instrument for the claim C1 needs. C1 moves one domain to the web: `app/h/_layout.tsx`, `app/h/[hostId]/index.tsx` and their import closure. A golden recorded at a call site in there that stopped replaying byte-identically is paid for by any of the other 684 that started, and every existing assertion still passes. `c1-page-closure.ts` names the 22 families and 103 goldens of that closure and pins each one to the verdict it gives: 54 byte-identical, 49 in four of the five classes the suite already excludes, all of them recorder observation artifacts whose wire bytes C0.5 and C0.8 proved identical. Membership is checked per family, not against the flat id list, so a golden newly derived into a family this domain owns arrives as a finding instead of being missed for never having been pinned. A closure golden may only be excluded into a class that carries a reason in `BRIDGED_PARITY_EXCLUSIONS`. Two full-corpus assertions go with it. `identical` was a floor; it is now the exact 787 minus the excluded classes. And the run's own size is pinned to the corpus: every class is an upper bound, so without that a corpus that lost goldens outside the identical set satisfied all of them. Claude-Session: https://claude.ai/code/session_01JNnE9qzUZMMnqpZWCqM3nb * test(mobile): move a closure golden to a verdict it is not already pinned to (OTA phase C, C1.6) The drift test took whichever golden the pin lists first and moved it to `params-undefined`. Nothing said that golden was not pinned to that class already, and the day one is, the test asserts a change it never made. Pick the first golden whose verdict is something else, and assert the pair of verdicts the line reports rather than just the new one. Claude-Session: https://claude.ai/code/session_01JNnE9qzUZMMnqpZWCqM3nb * test(mobile): pin `identical` to its baseline, not to the run's own exclusions (OTA phase C, C1.6) `identical: pinned - excludedCount` took `excludedCount` from the run, and the sum assertion above it already forces `total(counts) === excludedCount`, so the pair reduced to `corpus === pinned`: only the size of the corpus was pinned. With every per-class check an upper bound, a golden moving out of an excluded class into `identical` satisfied all of them. `bridgedParityTallyDrift` pins every number to `BRIDGED_PARITY_BASELINE` exactly, `identical` among them, and its own tests encode the mutation: one `result-absent-settlement` golden reported `identical` is two drift lines, where before it was a green run. Claude-Session: https://claude.ai/code/session_01JNnE9qzUZMMnqpZWCqM3nb * docs(mobile): the observation class counts 3, not 7 (OTA phase C, C1.6) The number in the prose predates the baseline it describes; `result-absent-observation` is 3 in `BRIDGED_PARITY_BASELINE` and in the run, and the line above it already says 341 / 3 / 6 / 33 / 8. Claude-Session: https://claude.ai/code/session_01JNnE9qzUZMMnqpZWCqM3nb * docs(mobile): name the checks that are still there, not the bound that is gone (OTA phase C, C1.6) Three comments described the per-class upper bound the tally replaced. What the exact pin is exact *against* is now `unclassified`, the exclusion sum and the membership pins, so say those. Claude-Session: https://claude.ai/code/session_01JNnE9qzUZMMnqpZWCqM3nb * test(mobile): drop the closure-exclusion check the cheap suite already makes (OTA phase C, C1.6) `c1PageClosureExclusions()` reads `C1_PAGE_CLOSURE` and nothing the run produced, so asserting on it inside the three-minute replay bought nothing the cheap suite next door does not already buy. `c1-page-closure.test.ts` makes the same assertion with a presence precondition on top, which is the stronger of the two. The comment beside the tally counted two checks above it; there are three, and a golden that moved out of an excluded class into `identical` is invisible to all of them: `result-absent-settlement` is past the nameable cap, so membership drift never looks at it, and lowering it lowers the exclusion sum with it. Claude-Session: https://claude.ai/code/session_01JNnE9qzUZMMnqpZWCqM3nb --- .github/workflows/mobile.yml | 19 +- .../bridged-parity/c1-page-closure.test.ts | 117 +++++++++ .../bridged-parity/c1-page-closure.ts | 234 ++++++++++++++++++ .../bridged-parity/divergence-classes.test.ts | 77 +++++- .../bridged-parity/divergence-classes.ts | 59 ++++- .../rpc-recording-through-bridge.test.ts | 73 ++++-- 6 files changed, 532 insertions(+), 47 deletions(-) create mode 100644 mobile/src/test-support/bridged-parity/c1-page-closure.test.ts create mode 100644 mobile/src/test-support/bridged-parity/c1-page-closure.ts diff --git a/.github/workflows/mobile.yml b/.github/workflows/mobile.yml index 2cec9676063..aae50234bf9 100644 --- a/.github/workflows/mobile.yml +++ b/.github/workflows/mobile.yml @@ -103,22 +103,15 @@ jobs: - name: Typecheck tests (ratchet) run: pnpm run check:tests-typecheck + # This includes the bridged replay of the whole recording corpus, which used to be a second + # step of its own behind RPC_FOUNDATION_BRIDGE=1. A gate nobody can forget to set is the point: + # it fails when a divergence class grows, when a divergence lands in no class at all, or when + # one of the 103 goldens inside the C1 page closure changes the verdict it is pinned to. It is + # ~3 min of test time on its own, and Vitest runs it on a worker beside the rest of the suite, + # so folding it in costs a fraction of that in wall time and one step less to skip. - name: Test run: pnpm test - # Why a second run of the same corpus: `pnpm test` leaves this suite off, because it replays - # every golden through the page bridge and the per-class counts it pins are the only thing - # that says how far that bridge is from byte-identical. It fails when a class grows or when a - # divergence lands in no class at all, so a change that widens the gap cannot land quietly. - # ~2.5 min locally, because a golden that diverges is replayed a second time with `_meta` - # supplied and that counterfactual is what separates the reader's share of the gap from the - # rest. Ungated on purpose: unlike the pin guard's reproduce, this verdict moves on any change - # to the bridge, which no path filter on the corpus would catch. - - name: Replay the recording corpus through the page bridge - env: - RPC_FOUNDATION_BRIDGE: '1' - run: pnpm exec vitest run src/test-support/rpc-recording/rpc-recording-through-bridge.test.ts - - name: Test iOS release version resolution run: ruby fastlane/ios_release_version_test.rb diff --git a/mobile/src/test-support/bridged-parity/c1-page-closure.test.ts b/mobile/src/test-support/bridged-parity/c1-page-closure.test.ts new file mode 100644 index 00000000000..d4af9f116dc --- /dev/null +++ b/mobile/src/test-support/bridged-parity/c1-page-closure.test.ts @@ -0,0 +1,117 @@ +import { readdirSync, readFileSync } from 'node:fs' +import { resolve } from 'node:path' +import { describe, expect, it } from 'vitest' +import { + c1PageClosureDrift, + c1PageClosureExclusions, + C1_PAGE_CLOSURE, + type C1PageClosureObservation +} from './c1-page-closure' +import { BRIDGED_PARITY_EXCLUSIONS } from './divergence-classes' + +const GOLDENS = resolve(import.meta.dirname, '../../../rpc-foundation/goldens') + +/** The run a corpus that diverged exactly as the pin says would hand the rule. */ +function asPinned(): Map { + const run = new Map() + for (const [family, goldens] of Object.entries(C1_PAGE_CLOSURE)) { + for (const [id, verdict] of Object.entries(goldens)) { + run.set(id, { family, verdict }) + } + } + return run +} + +describe('the C1 page closure', () => { + it('is the census the design named: 22 families, 103 goldens', () => { + const goldens = Object.values(C1_PAGE_CLOSURE).flatMap((family) => Object.keys(family)) + expect({ families: Object.keys(C1_PAGE_CLOSURE).length, goldens: goldens.length }).toEqual({ + families: 22, + goldens: 103 + }) + expect(new Set(goldens).size).toBe(goldens.length) + }) + + it('pins goldens that exist, in the family the corpus records them under', () => { + for (const [family, goldens] of Object.entries(C1_PAGE_CLOSURE)) { + for (const id of Object.keys(goldens)) { + // oxlint-disable-next-line typescript/consistent-type-assertions -- SAFETY: every golden carries `family`; a file that does not fails this read and the test. + const recorded = JSON.parse(readFileSync(`${GOLDENS}/${id}.json`, 'utf8')) as { + family: string + } + expect({ id, family: recorded.family }).toEqual({ id, family }) + } + } + }) + + it('claims no golden the corpus does not have', () => { + const corpus = new Set( + readdirSync(GOLDENS) + .filter((name) => name.endsWith('.json')) + .map((name) => name.replace(/\.json$/, '')) + ) + const missing = Object.values(C1_PAGE_CLOSURE) + .flatMap((family) => Object.keys(family)) + .filter((id) => !corpus.has(id)) + expect(missing).toEqual([]) + }) + + it('excludes a closure golden only into a class that has a reason', () => { + const exclusions = c1PageClosureExclusions() + expect(exclusions.length).toBeGreaterThan(0) + expect(exclusions.filter(([, name]) => BRIDGED_PARITY_EXCLUSIONS[name] === undefined)).toEqual( + [] + ) + }) +}) + +describe('reading a run against the pin', () => { + it('says nothing when the run is the pin', () => { + expect(c1PageClosureDrift(asPinned())).toEqual([]) + }) + + it('names a closure golden that changed verdict', () => { + const run = asPinned() + // Whichever golden it is, the verdict it moves to has to be one it is not already pinned to. + const found = [...run].find(([, seen]) => seen.verdict !== 'params-undefined') + if (found === undefined) { + throw new Error('the pin is empty') + } + const [id, observation] = found + run.set(id, { ...observation, verdict: 'params-undefined' }) + const drift = c1PageClosureDrift(run) + expect(drift.length).toBe(1) + expect(drift[0]).toContain(id) + expect(drift[0]).toContain(`pinned ${observation.verdict}, ran params-undefined`) + }) + + it('names a golden newly derived into a closure family, which no id list would', () => { + const run = asPinned() + const [family] = Object.keys(C1_PAGE_CLOSURE) + if (family === undefined) { + throw new Error('the pin is empty') + } + run.set('matrix-arrived-1', { family, verdict: 'identical' }) + expect(c1PageClosureDrift(run)).toEqual([`${family}: arrived matrix-arrived-1; left (none)`]) + }) + + it('names a closure golden the run stopped producing', () => { + const run = asPinned() + const [family, goldens] = Object.entries(C1_PAGE_CLOSURE)[0] ?? [] + const [id] = Object.keys(goldens ?? {}) + if (family === undefined || id === undefined) { + throw new Error('the pin is empty') + } + run.delete(id) + expect(c1PageClosureDrift(run)).toEqual([`${family}: arrived (none); left ${id}`]) + }) + + it('ignores every golden outside the closure, which is most of the corpus', () => { + const run = asPinned() + run.set('worktree-catalog-snapshot-unreadable-elsewhere', { + family: 'session.diff-review', + verdict: 'result-absent-settlement' + }) + expect(c1PageClosureDrift(run)).toEqual([]) + }) +}) diff --git a/mobile/src/test-support/bridged-parity/c1-page-closure.ts b/mobile/src/test-support/bridged-parity/c1-page-closure.ts new file mode 100644 index 00000000000..19a7907644b --- /dev/null +++ b/mobile/src/test-support/bridged-parity/c1-page-closure.ts @@ -0,0 +1,234 @@ +/** + * The goldens recorded at a call site inside the C1 page closure, and what each one did at the + * bridge. + * + * C1 moves a screen to the web: `app/h/_layout.tsx` and `app/h/[hostId]/index.tsx` and everything + * they import. The suite next door already proves the corpus replays byte-identically or in a named + * class, but it proves it as counts over 787 goldens, and a count is the wrong instrument for the + * claim C1 needs. These 103 are the ones whose divergence would be this domain's divergence, so + * each is pinned by id to the verdict it gives, not counted into a total another golden can pay for. + * + * The rule is not "none excluded". 54 replay byte for byte and 49 do not, in four of the five + * classes the suite next door names — 37 `result-absent-settlement`, 7 `params-undefined`, 3 + * `result-absent-stream-release`, 2 `write-ordinal`. Every one is a recorder observation artifact + * whose wire bytes C0.5 and C0.8 proved identical: what differs is the shape the recorder injects + * below the frame boundary, or the pre-serialization object a step is matched against, and neither + * is something a transport carries. What the pin buys is that the 49 are named. A fiftieth arriving + * is a red test here even though every count in `BRIDGED_PARITY_BASELINE` still holds, because the + * class it joined has room in its bound for a golden that left. + * + * Derived from the value-import closure of the two route modules with `.web.*` resolution applied, + * against the module each operation's mount adapter loads. `mobileWeb.bundle-manifest` is not here: + * it reaches the closure only through the shared `rpc-operation.ts` runner, and its own operation + * module is the shell's, not the page's. + */ + +import type { BridgedParityClass } from './divergence-classes' + +/** Byte-identical, or the class that named the divergence. */ +export type BridgedParityVerdict = BridgedParityClass | 'identical' + +export type C1PageClosureObservation = { + family: string + verdict: BridgedParityVerdict +} + +export const C1_PAGE_CLOSURE: Readonly< + Record>> +> = { + 'settings.repo-metadata': { + 'matrix-settings.repo-metadata-host.platform-1': 'result-absent-settlement', + 'matrix-settings.repo-metadata-repo.list-1': 'result-absent-settlement', + 'matrix-settings.repo-metadata-settings.get-1': 'result-absent-settlement', + 'matrix-settings.repo-metadata-ssh.listtargetsummaries-1': 'result-absent-settlement', + 'schedules-settings-repo-metadata-fulfilled': 'identical', + 'settings-repo-cache-expiry': 'identical', + 'settings-repo-metadata-fulfilled': 'identical', + 'settings-repo-metadata-icons': 'identical', + 'settings-repo-metadata-refuse-after-data': 'identical', + 'settings-repo-metadata-refused': 'identical', + 'settings-repo-metadata-single-host': 'identical', + 'settings-repo-metadata-transport-error': 'identical' + }, + 'settings.workspace-context': { + 'lifecycle-settings-workspace-context-fulfilled': 'identical', + 'matrix-settings.workspace-context-linear.status-1': 'result-absent-settlement', + 'matrix-settings.workspace-context-preflight.check-1': 'result-absent-settlement', + 'matrix-settings.workspace-context-settings.get-1': 'result-absent-settlement', + 'matrix-settings.workspace-context-ui.get-1': 'result-absent-settlement', + 'schedules-settings-workspace-context-fulfilled': 'identical', + 'settings-workspace-context-fulfilled': 'identical', + 'settings-workspace-context-refuse-after-data': 'identical', + 'settings-workspace-context-refused': 'identical', + 'settings-workspace-context-transport-error': 'identical' + }, + 'tasks.smart-source-search': { + 'matrix-tasks.smart-source-search-github.listworkitems-1': 'params-undefined', + 'matrix-tasks.smart-source-search-gitlab.listworkitems-1': 'params-undefined', + 'matrix-tasks.smart-source-search-linear.listissues-1': 'params-undefined', + 'matrix-tasks.smart-source-search-linear.searchissues-1': 'params-undefined', + 'matrix-tasks.smart-source-search-repo.searchrefs-1': 'params-undefined', + 'tw-smart-search-all-providers': 'params-undefined', + 'tw-smart-search-gitlab-provider-error': 'identical', + 'tw-smart-search-linear-listed': 'params-undefined' + }, + 'worktree.create-retry': { + 'matrix-worktree.create-retry-worktree.create-1': 'result-absent-settlement', + 'tw-create-retry-ambiguous-after-drop': 'identical', + 'tw-create-retry-ambiguous-while-connected': 'identical', + 'tw-create-retry-ambiguous-without-idempotency': 'identical', + 'tw-create-retry-created': 'identical', + 'tw-create-retry-name-collision': 'identical', + 'tw-create-retry-unretryable-refusal': 'identical', + 'tw-create-retry-warning-kept': 'identical' + }, + 'tasks.paste-lookup': { + 'matrix-tasks.paste-lookup-github.reposlug-1': 'result-absent-settlement', + 'matrix-tasks.paste-lookup-github.workitem-1': 'result-absent-settlement', + 'matrix-tasks.paste-lookup-github.workitembyownerrepo-1': 'result-absent-settlement', + 'matrix-tasks.paste-lookup-gitlab.workitembypath-1': 'result-absent-settlement', + 'tw-paste-lookup-resolved': 'identical', + 'tw-paste-lookup-slug-refused': 'identical', + 'tw-paste-lookup-slug-unsupported': 'identical' + }, + 'host-worktree-refresh': { + 'host-worktree-refresh-stream': 'write-ordinal', + 'matrix-host-worktree-refresh-runtime.clientevents.subscribe-1-1': + 'result-absent-stream-release', + 'matrix-host-worktree-refresh-runtime.clientevents.subscribe-1-2': + 'result-absent-stream-release', + 'matrix-host-worktree-refresh-runtime.clientevents.subscribe-1-3': + 'result-absent-stream-release', + 'matrix-host-worktree-refresh-runtime.clientevents.subscribe-2-1': 'write-ordinal' + }, + 'host.worktree-actions': { + 'host-worktree-actions-pin-open-delete': 'identical', + 'host-worktree-delete-refused': 'identical', + 'matrix-host.worktree-actions-worktree.activate-1': 'result-absent-settlement', + 'matrix-host.worktree-actions-worktree.rm-1': 'result-absent-settlement', + 'matrix-host.worktree-actions-worktree.set-1': 'result-absent-settlement' + }, + 'transport.capability-probe': { + 'matrix-transport.capability-probe-status.get-1': 'result-absent-settlement', + 'transport-capability-probe-cutover-reasks-fast': 'identical', + 'transport-capability-probe-non-string-capabilities-drop': 'identical', + 'transport-capability-probe-publishes': 'identical', + 'transport-capability-probe-refused-backs-off': 'identical' + }, + 'components.execution-target': { + 'components-target-ssh': 'identical', + 'matrix-components.execution-target-preflight.detectremoteagents-1': 'result-absent-settlement', + 'matrix-components.execution-target-ssh.connect-1': 'result-absent-settlement', + 'matrix-components.execution-target-ssh.getstate-1': 'result-absent-settlement' + }, + 'notifications.push-registration': { + 'matrix-notifications.push-registration-notifications.registerpush-1': + 'result-absent-settlement', + 'matrix-notifications.push-registration-notifications.unregisterpush-1': + 'result-absent-settlement', + 'notifications-push-gateway-rejected': 'identical', + 'notifications-push-registered': 'identical' + }, + 'settings.workspace-submit': { + 'matrix-settings.workspace-submit-settings.get-1': 'result-absent-settlement', + 'settings-workspace-submit-fulfilled': 'identical', + 'settings-workspace-submit-refused': 'identical', + 'settings-workspace-submit-transport-error': 'identical' + }, + 'transport.host-status-gates': { + 'matrix-transport.host-status-gates-status.get-1': 'result-absent-settlement', + 'transport-host-status-gates-drop-keeps-capabilities': 'identical', + 'transport-host-status-gates-ready': 'identical', + 'transport-host-status-gates-refused-degrades': 'identical' + }, + 'worktree.hosted-base': { + 'matrix-worktree.hosted-base-worktree.resolvemrbase-1': 'result-absent-settlement', + 'matrix-worktree.hosted-base-worktree.resolveprbase-1': 'result-absent-settlement', + 'tw-hosted-base-resolved': 'identical', + 'tw-hosted-base-soft-error': 'identical' + }, + 'worktree.runtime-capabilities': { + 'matrix-worktree.runtime-capabilities-status.get-1': 'result-absent-settlement', + 'tw-capabilities-advertised': 'identical', + 'tw-capabilities-cutover-retried': 'identical', + 'tw-capabilities-legacy-idempotency': 'identical' + }, + 'host.view-settings': { + 'host-view-settings-sync': 'identical', + 'matrix-host.view-settings-ui.get-1': 'result-absent-settlement', + 'matrix-host.view-settings-ui.set-1': 'result-absent-settlement' + }, + 'worktree.catalog-snapshot': { + 'matrix-worktree.catalog-snapshot-worktree.ps-1': 'result-absent-settlement', + 'worktree-catalog-snapshot': 'identical', + 'worktree-catalog-snapshot-unreadable': 'result-absent-settlement' + }, + 'worktree.setup-hook-trust': { + 'matrix-worktree.setup-hook-trust-ui.set-1': 'result-absent-settlement', + 'tw-setup-hook-trust-always': 'identical', + 'tw-setup-hook-trust-approved': 'identical' + }, + 'components.execution-target-local': { + 'components-target-local': 'identical', + 'matrix-components.execution-target-local-preflight.detectagents-1': 'result-absent-settlement' + }, + 'components.new-workspace-repositories': { + 'matrix-components.new-workspace-repositories-repo.list-1': 'result-absent-settlement', + 'new-workspace-repositories-fulfilled': 'identical' + }, + 'components.setup-script': { + 'components-setup-ask': 'identical', + 'matrix-components.setup-script-repo.hooks-1': 'result-absent-settlement' + }, + 'worktree.agent-launch-create': { + 'matrix-worktree.agent-launch-create-agent.launch-1': 'result-absent-settlement', + 'tw-create-retry-agent-launched': 'identical' + }, + 'worktree.retired-names': { + 'matrix-worktree.retired-names-worktree.listretirednames-1': 'result-absent-settlement', + 'worktree-retired-names': 'identical' + } +} + +/** Every closure golden that did not replay byte-identically, which the suite prints beside why. */ +export function c1PageClosureExclusions(): readonly (readonly [string, BridgedParityClass])[] { + return Object.values(C1_PAGE_CLOSURE).flatMap((family) => + Object.entries(family).flatMap(([id, verdict]) => + verdict === 'identical' ? [] : [[id, verdict] as const] + ) + ) +} + +/** + * Each closure family whose goldens or verdicts are not the ones pinned above, said in one line. + * + * Membership is checked per family rather than against the flat id list, so a golden newly derived + * into a family this domain owns arrives as a finding instead of going unnoticed for being absent + * from a pin that never mentioned it. + */ +export function c1PageClosureDrift( + observed: ReadonlyMap +): readonly string[] { + const byFamily = new Map() + for (const [id, { family }] of observed) { + byFamily.set(family, [...(byFamily.get(family) ?? []), id]) + } + const drift: string[] = [] + for (const [family, pinned] of Object.entries(C1_PAGE_CLOSURE)) { + const seen = byFamily.get(family) ?? [] + const arrived = seen.filter((id) => !(id in pinned)) + const left = Object.keys(pinned).filter((id) => !seen.includes(id)) + if (arrived.length > 0 || left.length > 0) { + drift.push( + `${family}: arrived ${arrived.join(', ') || '(none)'}; left ${left.join(', ') || '(none)'}` + ) + } + for (const [id, verdict] of Object.entries(pinned)) { + const ran = observed.get(id)?.verdict + if (ran !== undefined && ran !== verdict) { + drift.push(`${id}: pinned ${verdict}, ran ${ran}`) + } + } + } + return drift +} diff --git a/mobile/src/test-support/bridged-parity/divergence-classes.test.ts b/mobile/src/test-support/bridged-parity/divergence-classes.test.ts index 8213ab3dcc6..58d3e12efc5 100644 --- a/mobile/src/test-support/bridged-parity/divergence-classes.test.ts +++ b/mobile/src/test-support/bridged-parity/divergence-classes.test.ts @@ -3,14 +3,17 @@ import { resolve } from 'node:path' import { describe, expect, it } from 'vitest' import { bridgedParityMembershipDrift, + bridgedParityTallyDrift, classifyBridgedParity, BRIDGED_PARITY_BASELINE, BRIDGED_PARITY_EXCLUSIONS, BRIDGED_PARITY_FLAG, BRIDGED_PARITY_MEMBERS, + BRIDGED_PARITY_OFF, BRIDGED_PARITY_NAMEABLE, type BridgedParityClass, - type BridgedParityEvidence + type BridgedParityEvidence, + type BridgedParityTally } from './divergence-classes' const base: BridgedParityEvidence = { @@ -34,9 +37,16 @@ const droppedUndefinedKey: BridgedParityEvidence = { } describe('the bridged-parity flag', () => { - it('is the name the suite, the pin and the CI job all spell', () => { + it('is the name the suite and the pin both spell', () => { expect(BRIDGED_PARITY_FLAG).toBe('RPC_FOUNDATION_BRIDGE') }) + + it('skips on one value only, so an unset or mistyped variable still runs the gate', () => { + expect(BRIDGED_PARITY_OFF).toBe('0') + const skips = (value: string | undefined): boolean => value === BRIDGED_PARITY_OFF + expect([undefined, '', '1', 'false', 'off'].filter(skips)).toEqual([]) + expect(skips('0')).toBe(true) + }) }) describe('classifying one diverging golden', () => { @@ -145,9 +155,9 @@ describe('what the pin still admits', () => { }) it('accounts for every golden in the corpus, once each', () => { - // What ties the two halves of the ratchet together. Each class is an upper bound and `identical` - // a lower one, so on their own a class could be loosened by one and nothing would notice; with - // the sum pinned to the corpus, a class that grows has to be paid for out of another. + // What ties the numbers to the corpus they describe. Each is pinned exactly, but only to a + // number in this file; the sum being the size of the goldens directory is what says the pin + // covers every golden, once each, rather than a subset the run happened to reach. const goldens = readdirSync(resolve(import.meta.dirname, '../../../rpc-foundation/goldens')) const counted = Object.values(BRIDGED_PARITY_BASELINE).reduce((sum, count) => sum + count, 0) expect({ counted }).toEqual({ @@ -185,7 +195,7 @@ describe('what the pin still admits', () => { // The trade a count cannot see: every predicate reads the scenario rather than the frame the // page refused — `scriptsAbsentResultReply` asks whether the scenario scripts the injected // shape anywhere — so a real refusal in a stream golden lands in an excluded class. Let one - // golden leave as it arrives and the count, the sum and the `identical` floor all hold. + // golden leave as it arrives and the count, the sum and the `identical` pin all hold. expect(bridgedParityMembershipDrift(asPinned())).toEqual([]) const pinned = BRIDGED_PARITY_MEMBERS['write-ordinal'] ?? [] const traded = asPinned() @@ -202,6 +212,61 @@ describe('what the pin still admits', () => { expect(bridgedParityMembershipDrift(run)).toEqual([]) }) + /** The tally a run that lands exactly on the pin hands the rule. */ + function asCounted(): BridgedParityTally { + const counts: Record = { + 'reply-meta-required': 0, + 'result-absent-settlement': 0, + 'result-absent-observation': 0, + 'result-absent-stream-release': 0, + 'params-undefined': 0, + 'write-ordinal': 0, + unclassified: 0 + } + for (const name of classes) { + counts[name] = BRIDGED_PARITY_BASELINE[name] + } + return { identical: BRIDGED_PARITY_BASELINE.identical, counts } + } + + it('says nothing about the run the numbers were taken from', () => { + expect(bridgedParityTallyDrift(asCounted())).toEqual([]) + }) + + it('goes red on a golden that stopped diverging, which every other check lets through', () => { + // The direction the rest of the suite cannot see. One `result-absent-settlement` golden + // reported `identical` instead: nothing is unclassified, every diverging golden is still in an + // excluded class, and the corpus is still 787. Only these two numbers moved. + const tally = asCounted() + const moved: BridgedParityTally = { + identical: tally.identical + 1, + counts: { + ...tally.counts, + 'result-absent-settlement': tally.counts['result-absent-settlement'] - 1 + } + } + const drift = bridgedParityTallyDrift(moved) + expect(drift.length).toBe(2) + expect(drift.join('\n')).toContain( + `identical: pinned ${tally.identical}, ran ${moved.identical}` + ) + expect(drift.join('\n')).toContain('result-absent-settlement: pinned 341, ran 340') + }) + + it('goes red on a class that grew and on the corpus losing a golden', () => { + const tally = asCounted() + expect( + bridgedParityTallyDrift({ + ...tally, + counts: { ...tally.counts, unclassified: 1 } + }) + ).toEqual(['unclassified: pinned 0, ran 1']) + // A golden whose `it` threw before it was counted anywhere: the run is a golden short. + expect(bridgedParityTallyDrift({ ...tally, identical: tally.identical - 1 })).toEqual([ + `identical: pinned ${tally.identical}, ran ${tally.identical - 1}` + ]) + }) + it('leaves nothing for the reader to close: the `_meta` class is zero', () => { expect(BRIDGED_PARITY_BASELINE['reply-meta-required']).toBe(0) expect(BRIDGED_PARITY_EXCLUSIONS['reply-meta-required']).toBeUndefined() diff --git a/mobile/src/test-support/bridged-parity/divergence-classes.ts b/mobile/src/test-support/bridged-parity/divergence-classes.ts index 457fd501213..5d16e7a09e9 100644 --- a/mobile/src/test-support/bridged-parity/divergence-classes.ts +++ b/mobile/src/test-support/bridged-parity/divergence-classes.ts @@ -7,9 +7,19 @@ * one level up in `test-support`. */ -/** Named once so the suite, its pin and the CI job cannot drift apart. */ +/** + * The switch that turns the bridged replay off, named once so the suite and its pin cannot drift. + * + * It used to be what turned the replay *on*, and CI set it. That made the gate opt-in, which is the + * one thing a gate must not be: a branch that widened the bridge's divergence and left the variable + * alone would have been measured by nobody. The replay is the default now and `=0` is for a local + * run that does not want the three minutes. CI sets nothing. + */ export const BRIDGED_PARITY_FLAG = 'RPC_FOUNDATION_BRIDGE' +/** The one value of it that skips the suite; anything else, unset included, runs it. */ +export const BRIDGED_PARITY_OFF = '0' + export type BridgedParityClass = | 'reply-meta-required' | 'result-absent-settlement' @@ -157,17 +167,16 @@ export const BRIDGED_PARITY_EXCLUSIONS: Readonly> = { identical: 396, @@ -188,6 +197,32 @@ export const BRIDGED_PARITY_BASELINE: Readonly> +} + +/** + * Every number a run reported that `BRIDGED_PARITY_BASELINE` does not, said in one line each. + * + * Both directions, and `identical` on the same footing as a class, because that is the direction + * nothing else in the suite sees. A golden reported `identical` instead of the excluded class it + * belongs to leaves `unclassified` empty, leaves every diverging golden inside a class the + * exclusions name, and leaves the corpus its size: this is the only number that moves. + */ +export function bridgedParityTallyDrift(tally: BridgedParityTally): readonly string[] { + const ran: Readonly> = { ...tally.counts, identical: tally.identical } + const drift: string[] = [] + for (const [name, pinned] of Object.entries(BRIDGED_PARITY_BASELINE)) { + const count = ran[name] ?? 0 + if (count !== pinned) { + drift.push(`${name}: pinned ${pinned}, ran ${count}`) + } + } + return drift +} + /** A class this small is named golden by golden in the run's output rather than counted. */ export const BRIDGED_PARITY_NAMEABLE = 8 @@ -198,7 +233,7 @@ export const BRIDGED_PARITY_NAMEABLE = 8 * refused frame — `scriptsAbsentResultReply` asks whether the scenario scripts the injected shape * anywhere, not whether the frame the page refused was one — so a real refusal inside a stream * golden is named an excluded class. One golden leaving that class as the real refusal puts another - * in moves no number here, and the sum and the `identical` floor both still hold. The ids are what + * in moves no number here, and the sum and the `identical` pin both still hold. The ids are what * notices. Where a class is too large to list, its predicate stands on its own and the count is all * the pin has; that is why the classes here are the small ones and why the test above requires * every class of `BRIDGED_PARITY_NAMEABLE` or fewer to appear. diff --git a/mobile/src/test-support/rpc-recording/rpc-recording-through-bridge.test.ts b/mobile/src/test-support/rpc-recording/rpc-recording-through-bridge.test.ts index d3fd426b7e0..d6ee79a4dbe 100644 --- a/mobile/src/test-support/rpc-recording/rpc-recording-through-bridge.test.ts +++ b/mobile/src/test-support/rpc-recording/rpc-recording-through-bridge.test.ts @@ -8,14 +8,21 @@ import { import type { RpcClient } from '../../transport/rpc-client' import { bridgedParityMembershipDrift, - BRIDGED_PARITY_BASELINE, + bridgedParityTallyDrift, BRIDGED_PARITY_EXCLUSIONS, BRIDGED_PARITY_FLAG, BRIDGED_PARITY_NAMEABLE, + BRIDGED_PARITY_OFF, classifyBridgedParity, type BridgedParityClass, type BridgedParityEvidence } from '../bridged-parity/divergence-classes' +import { + c1PageClosureDrift, + C1_PAGE_CLOSURE, + type BridgedParityVerdict, + type C1PageClosureObservation +} from '../bridged-parity/c1-page-closure' import { divergingFields, paramsMismatchEvidence, @@ -44,21 +51,28 @@ import { vitestRecordingScheduler } from './vitest-recording-scheduler' * run. This suite writes nothing, and it is not in `RECORDING_DRIVERS`, so `recorderSha256` does * not pin it — a suite that cannot put an observation in a recorded file is not provenance for one. * + * It runs by default, in `pnpm test` and so in CI, and `RPC_FOUNDATION_BRIDGE=0` is what skips it + * for a local run that does not want the three minutes. Vitest gives the file a worker of its own + * beside the rest of the suite, so the gate costs much less in wall time than it does in test time. + * * ## What it asserts today * * Byte-identical replay where it holds, and the named shape of every divergence where it does not. * A golden that matches is compared in full; one that does not is classified by * `classifyBridgedParity`, which reads the frames and the scenario rather than the failure's text. - * The run fails if any class grows past `BRIDGED_PARITY_BASELINE`, if a single golden lands in - * `unclassified`, or if one diverges in a class `BRIDGED_PARITY_EXCLUSIONS` does not name. The - * corpus is a fixed size, so those together pin every count exactly, and for a class small enough - * to name `BRIDGED_PARITY_MEMBERS` pins which goldens are in it — a count alone cannot see one - * golden leaving a class as another arrives. + * The run fails if any count is not exactly its number in `BRIDGED_PARITY_BASELINE` — `identical` + * among them, which is the only check that sees a golden that stopped diverging as well as one that + * started — if a single golden lands in `unclassified`, or if one diverges in a class + * `BRIDGED_PARITY_EXCLUSIONS` does not name. For a class small enough to name, + * `BRIDGED_PARITY_MEMBERS` pins which goldens are in it — a count alone cannot see one golden + * leaving a class as another arrives. * * 396 of the 787 replay byte for byte. The other 391 fall in five classes, 341 / 3 / 6 / 33 / 8, - * and none of them is a reason to re-record anything. + * and none of them is a reason to re-record anything. `c1-page-closure.ts` then pins, golden by + * golden, the 103 recorded at a call site the C1 page owns, because a count over 787 cannot tell a + * domain's regression from another domain's improvement. * - * 1. **result-absent-settlement, 341** and **2. result-absent-observation, 7.** + * 1. **result-absent-settlement, 341** and **2. result-absent-observation, 3.** * `{ ok: true }` with no `result` key is refused by the page's reader and by `isRpcResponse` * alike, so this one is not a bridge defect: the recorder injects that partition at the scripted * sender port, below the frame validation both sides do, which is what the README means by not @@ -134,6 +148,8 @@ const counts: Record = { let identical = 0 const members = new Map() const samples = new Map() +/** Every golden's own verdict, which is what the C1 closure is pinned against golden by golden. */ +const observed = new Map() /** * The page's client over the shared port pair, holding the recorder's scripted client shell-side. @@ -235,15 +251,20 @@ function describeParamsMismatch(evidence: BridgedParityEvidence): string { */ async function verdict( id: string, + family: string, scenarios: readonly RecordingScenario[], run: Replay ): Promise { + const record = (name: BridgedParityVerdict): void => { + observed.set(id, { family, verdict: name }) + } const expected = readGolden(directory, id) const fields = run.recording === null ? [] : divergingFields(expected.recording, run.recording) if (run.recording !== null && fields.length === 0) { // Not redundant with the field walk: this one also pins the encoding and the header. compareGolden(expected, { ...expected, recording: run.recording }) identical += 1 + record('identical') return } const asIf = await replay(id, scenarios, withReplyMeta) @@ -263,6 +284,7 @@ async function verdict( } const name = classifyBridgedParity(evidence) counts[name] += 1 + record(name) if (name === 'unclassified') { throw new Error( `Unclassified bridged divergence: ${id}\n${explain(fields, run)}\n${describeParamsMismatch(evidence)}with \`_meta\` supplied:\n${explain(asIfFields, asIf)}` @@ -274,12 +296,17 @@ async function verdict( } } -describe.runIf(process.env[BRIDGED_PARITY_FLAG] === '1')( +describe.skipIf(process.env[BRIDGED_PARITY_FLAG] === BRIDGED_PARITY_OFF)( 'every golden replays through the page bridge, byte-identically or in a named class', () => { for (const pilot of pilotGoldens(input.scenarios)) { it(`${pilot.id}: bridged parity`, async () => { - await verdict(pilot.id, [pilot.scenario], await replay(pilot.id, [pilot.scenario])) + await verdict( + pilot.id, + pilot.family, + [pilot.scenario], + await replay(pilot.id, [pilot.scenario]) + ) }) } for (const golden of familyGoldens(input.scenarios)) { @@ -287,7 +314,7 @@ describe.runIf(process.env[BRIDGED_PARITY_FLAG] === '1')( `${golden.id}: bridged parity`, async () => { const scenarios = [...golden.scenarios()] - await verdict(golden.id, scenarios, await replay(golden.id, scenarios)) + await verdict(golden.id, golden.family, scenarios, await replay(golden.id, scenarios)) }, golden.timeoutMs ) @@ -324,12 +351,26 @@ describe.runIf(process.env[BRIDGED_PARITY_FLAG] === '1')( expect({ divergedOutsideAnExcludedClass: total(counts) - excludedCount }).toEqual({ divergedOutsideAnExcludedClass: 0 }) - for (const [name, count] of Object.entries(counts)) { - expect({ [name]: count }).toEqual({ - [name]: Math.min(count, BRIDGED_PARITY_BASELINE[asClass(name)]) - }) + // Every count exactly, `identical` included, which is the direction the three checks above + // cannot see: a golden reported `identical` rather than the excluded class it belongs to + // leaves all three holding. The size of the corpus follows, being the total of these. + expect({ tally: bridgedParityTallyDrift({ identical, counts }) }).toEqual({ tally: [] }) + }) + + it('gives every golden the C1 page closure records the verdict it is pinned to', () => { + const closure = [...observed].filter(([, seen]) => seen.family in C1_PAGE_CLOSURE) + const diverged = closure.filter(([, seen]) => seen.verdict !== 'identical') + process.stdout.write( + `\nC1 page closure: ${closure.length} goldens in ${ + Object.keys(C1_PAGE_CLOSURE).length + } families, ${closure.length - diverged.length} byte-identical\n` + ) + for (const [id, seen] of diverged) { + process.stdout.write(` ${id}: ${seen.verdict}\n`) } - expect(identical).toBeGreaterThanOrEqual(BRIDGED_PARITY_BASELINE.identical) + // Each by id, because the counts above cannot see this domain: a closure golden that stopped + // replaying identically is paid for by any of the other 684 that started. + expect({ closure: c1PageClosureDrift(observed) }).toEqual({ closure: [] }) }) } )