diff --git a/config/scripts/mobile-web-app-html-preview-render.test.mjs b/config/scripts/mobile-web-app-html-preview-render.test.mjs index b1d34290b4b..bf020318560 100644 --- a/config/scripts/mobile-web-app-html-preview-render.test.mjs +++ b/config/scripts/mobile-web-app-html-preview-render.test.mjs @@ -21,14 +21,20 @@ import { mkdir, mkdtemp, rm, writeFile } from 'node:fs/promises' import { createServer } from 'node:http' import { join } from 'node:path' import { fileURLToPath } from 'node:url' -import { afterAll, beforeAll, describe, expect, it } from 'vitest' +import { afterAll, beforeAll, describe, expect, it, vi } from 'vitest' import * as esbuild from 'esbuild' import { PNG } from 'pngjs' import { chromium, webkit } from 'playwright-core' import { lucideBarrelPlugin } from './build-mobile-web-app-bundle.mjs' import { mobileWebAppDependenciesPresent } from './mobile-web-app-bundle-dependencies.mjs' import { createBundleServer, readShellCsp } from './mobile-web-app-render-harness.mjs' -import { describePreviewFrame, untilAborted } from './mobile-web-app-preview-frame-diagnosis.mjs' +import { createCspReportSink, reportedDirectives } from './mobile-web-app-preview-csp-reports.mjs' +import { + previewFrame, + settleAfterMount, + waitForLoadedFrame, + waitForRecordedNavigation +} from './mobile-web-app-preview-frame-readiness.mjs' const mobileDir = fileURLToPath(new URL('../../mobile', import.meta.url)) @@ -136,11 +142,15 @@ let nonceCounter = 0 /** The inline script every arm carries, so "it did not run" is about the fence and not the fixture. */ const ARTIFACT_SCRIPT = `` const bundles = mobileWebAppDependenciesPresent() @@ -159,6 +169,8 @@ const browsers = {} let sealedServer = null let openServer = null const origins = {} +/** Every refusal the sealed server's policy was told about, by the arm that caused it. */ +const cspReports = createCspReportSink() beforeAll(async () => { shippedCsp = await readShellCsp() @@ -224,7 +236,12 @@ beforeAll(async () => { '
' + '' ) - const sealed = await createBundleServer({ outDir, cspHeader: shippedCsp }) + const sealed = await createBundleServer({ + outDir, + // Per document, because each arm's policy names an endpoint carrying that arm's nonce. + cspHeader: (request) => cspReports.policyFor(shippedCsp, request), + handleRequest: (request, response, path) => cspReports.handleRequest(request, response, path) + }) sealedServer = sealed.server origins.shipped = sealed.origin const bare = await createBundleServer({ outDir, cspHeader: null }) @@ -268,6 +285,7 @@ async function open( act, expectNavigation = null, frameReady = 'artifact', + reportReady = null, signal } = {} ) { @@ -280,6 +298,12 @@ async function open( const page = await browser.newPage({ viewport: { width: 390, height: 844 } }) const navigations = [] const popups = [] + let servedCsp = null + page.on('response', (response) => { + if (response.url().startsWith(`${origin}/preview`)) { + servedCsp = response.headers()['content-security-policy'] ?? null + } + }) page.on('popup', (popup) => { popups.push(popup.url()) void popup.close().catch(() => {}) @@ -308,12 +332,18 @@ async function open( // The shell page's violations, and only those: an artifact's own listener would have to run, and // the fence under test is that nothing in the artifact runs. await page.addInitScript(() => { + // When this ran, in every frame it ran in. The collector below can only report what it was + // present for, so its own moment is a reading rather than an assumption. + window.__initAt = `${String(Math.round(performance.now()))} ${document.readyState}` window.__violations = [] document.addEventListener('securitypolicyviolation', (event) => { window.__violations.push(`${event.violatedDirective} ${event.blockedURI || 'inline'}`) }) }) - await page.goto(`${origin}/preview`, { waitUntil: 'load' }) + // The nonce in the document's own URL: the policy this response carries names a report endpoint + // with the same nonce, which is how a report from a `srcdoc` frame with no URL of its own is + // attributed to the arm that caused it. + await page.goto(`${origin}/preview?n=${nonce}`, { waitUntil: 'load' }) // Registered after the page's own load, not before it: this handler aborts main-frame navigations // and the initial `goto` is one. `href="/"` and `href=""` inside an artifact resolve against the // embedder's base, so a tap on either asks to navigate the top frame to the shell's own document. @@ -327,9 +357,18 @@ async function open( [artifact(extra, nonce), sandbox ?? null] ) // Named in every diagnostic, because the log shows the case and not which of its arms spoke. - const arm = `arm csp=${csp} sandbox=${sandbox ?? 'product'} frameReady=${frameReady} nonce=${nonce}` - const artifactFrame = await waitForLoadedFrame(page, frameReady, signal, browserVersion, arm) - const frames = () => page.frames().filter((frame) => frame !== page.mainFrame()) + const arm = + `arm csp=${csp} sandbox=${sandbox ?? 'product'} frameReady=${frameReady} ` + + `reportReady=${reportReady ?? 'none'} nonce=${nonce}` + const artifactFrame = await waitForLoadedFrame(page, { + frameReady, + reportReady, + signal, + browserVersion, + arm, + sink: cspReports, + nonce + }) // Sampled before the action as well as after: a case that taps a link is asking what the tap // produced, and by then the top frame is mid-navigation and the iframe has blanked to its own // background. So the precondition "there was a rendered artifact to tap" is this reading, and the @@ -347,7 +386,7 @@ async function open( // Sampled before the action as well, because the toggle's whole claim is that it changes. const togglesBefore = await readToggles() if (act) { - await act({ page, frame: frames()[0] ?? null }) + await act({ page, frame: previewFrame(page) }) } // Every arm settles, acting or not: an artifact can start a navigation with no tap behind it -- // `` is one -- and the arms that pin zero were reading their counters @@ -373,7 +412,7 @@ async function open( mountedSandbox: await page .evaluate(() => document.querySelector('iframe')?.getAttribute('sandbox') ?? null) .catch(() => null), - frameCount: frames().length, + frameCount: page.frames().length - 1, // Reported so a pixel that read the page instead of the frame names the layout rather than // looking like a frame that refused to load. frameBox: await page @@ -388,7 +427,7 @@ async function open( .catch(() => null), // Reported, never asserted on: a `srcdoc` frame's URL reads `about:srcdoc` here and empty on // CI's browser, so nothing may be decided by it. - frameUrl: frames()[0]?.url() ?? null, + frameUrl: previewFrame(page)?.url() ?? null, // The element's own attributes, which is where "the artifact is parsed inside the frame rather // than fetched into it" actually lives. mountedSrcDoc: await page @@ -397,17 +436,27 @@ async function open( mountedSrc: await page .evaluate(() => document.querySelector('iframe')?.getAttribute('src') ?? null) .catch(() => null), - inside: await (frames()[0] + inside: await (previewFrame(page) ?.evaluate(() => ({ marker: document.getElementById('marker')?.textContent ?? null, title: document.title, - ran: window.__ran ?? 0, - threw: window.__threw ?? null, + ran: document.documentElement.dataset.ran === '1' ? 1 : 0, + threw: document.documentElement.dataset.threw ?? null, + // The two moments the late-listener question turns on: when the page's init script ran in + // this frame, and when the artifact's own script did. + initAt: window.__initAt ?? null, + artifactAt: document.documentElement.dataset.artifactAt ?? null, // The frame's own list, not the embedder's: `securitypolicyviolation` does not cross frames, // and the page's init script installs the same collector in every one. violations: window.__violations ?? null })) .catch(() => null) ?? Promise.resolve(null)), + // What this document was actually served, so "the shipped policy, plus a report endpoint and + // nothing else" is asserted rather than intended. + servedCsp, + // Every refusal the browser reported for this arm, which is the evidence an in-frame listener + // cannot be relied on to have collected. + reported: reportedDirectives(cspReports, nonce), topNavigations: navigations.filter((one) => one.main && one.foreign).length, ownOriginTopNavigations: navigations.filter((one) => one.main && !one.foreign).length, // What the frame asked for itself at the embedder's origin, which is a different escape from a @@ -450,6 +499,12 @@ for (const engine of ['chromium', 'webkit']) { // page mounts rather than about a string nothing reads. expect(read.mountedSandbox).toBe(read.declaredSandbox) expect(read.mountedSandbox).toBe('allow-top-navigation-by-user-activation') + // The policy this document was served is the shell's own text plus the rig's report + // endpoint, and nothing else: `report-uri` says where a refusal is sent and changes nothing + // about what is enforced, so the arms below measure the shipped policy. + const servedParts = (read.servedCsp ?? '').split('; report-uri ') + expect(servedParts[0]).toBe(shippedCsp) + expect(servedParts).toHaveLength(2) // The pixel, not a read inside the frame: the frame is an opaque origin. expect(read.pixel).toBe(ARTIFACT_RGB) // The shell page's own violations, which is all this can be: `securitypolicyviolation` does @@ -461,7 +516,13 @@ for (const engine of ['chromium', 'webkit']) { }, 120_000) it('does not run the artifact, behind two fences either of which would hold', async (ctx) => { - const sealed = await open(browser(), { extra: { body: script() }, signal: ctx.signal }) + const sealed = await open(browser(), { + extra: { body: script() }, + signal: ctx.signal, + // The refusal this arm does cause, waited for so the missing one below is an absence + // measured beside a presence rather than a list read too early. + reportReady: 'img-src' + }) expect(sealed.pixel).toBe(ARTIFACT_RGB) expect(sealed.inside?.ran).toBe(0) expect(sealed.inside?.title).toBe('ARTIFACT') @@ -482,8 +543,9 @@ for (const engine of ['chromium', 'webkit']) { expect(loose.pixel).toBe(ARTIFACT_RGB) expect(loose.inside?.ran).toBe(1) expect(loose.inside?.title).toBe('SCRIPT_RAN') - // Nothing refused it, which is what "no policy" looks like from inside the frame. - expect(loose.inside?.violations).toEqual([]) + // Nothing refused it, which is what "no policy" looks like: this arm's server sends no + // header at all, so there is no policy to report against and the script ran. + expect(loose.reported).toEqual([]) // The second fence, measured on its own: grant `allow-scripts` and keep the shipped policy, // and the script still does not run, because a `srcdoc` frame inherits its embedder's @@ -494,24 +556,25 @@ for (const engine of ['chromium', 'webkit']) { signal: ctx.signal, extra: { body: script() }, sandbox: 'allow-scripts allow-top-navigation-by-user-activation', - // The refusal below is this arm's oracle, and it is queued behind the frame's load, so the - // arm waits for it instead of reading whatever the list happens to hold. - frameReady: 'refusal' + // The refusal below is this arm's oracle, so the arm waits for the browser to have + // reported it rather than reading whatever a list inside the frame happens to hold. + reportReady: 'script-src' }) expect(inherited.pixel).toBe(ARTIFACT_RGB) expect(inherited.inside?.ran).toBe(0) expect(inherited.inside?.title).toBe('ARTIFACT') // This arm's own precondition, and the thing CI showed a rig can get wrong: a frame that was - // never really widened refuses the script too, silently and with no event, and would pass - // every line above under a name that says the policy held. A violation raised inside the - // frame can only happen if the sandbox let the script start, so this is the reading that - // separates the two -- and it is the frame's own list, since the embedder's never sees it. - // Waited for, not hoped for: `frameReady: 'refusal'` above is what makes this line arrive - // after the entry rather than beside the image refusal that happened to be first. - expect(String(inherited.inside?.violations)).toContain('script-src') - // The sealed arm is the contrast: no policy refused anything there, the sandbox simply never - // let the script begin. - expect(sealed.inside?.violations).toEqual([]) + // never really widened refuses the script too, silently and with no report, and would pass + // every line above under a name that says the policy held. A `script-src` refusal can only + // be reported if the sandbox let the script start, so this is the reading that separates the + // two -- and it comes from the browser rather than from a listener in the frame, which on + // CI's Chrome intermittently missed this very entry while catching the image one beside it. + expect(inherited.reported.join(' ')).toContain('script-src') + // The sealed arm is the contrast, and it is why that line means what it says: the same + // artifact under the same policy was reported only for its image. Nothing refused its + // script, because the sandbox never let it begin. + expect(sealed.reported.join(' ')).toContain('img-src') + expect(sealed.reported.join(' ')).not.toContain('script-src') }, 180_000) it('fetches nothing of the artifact that leaves the origin, and would if allowed', async (ctx) => { @@ -663,6 +726,35 @@ for (const engine of ['chromium', 'webkit']) { expect(blank.popups).toBe(0) }, 180_000) + // The navigation wait's sampling branch, driven once. It fires only when an arm is slow, so + // nothing here had ever executed it: a name out of scope inside it throws where no lint runs + // and no case looks. The printed reading is the proof that it ran and returned one. + it('reads the frame while a navigation it expects has not arrived', async (ctx) => { + void ctx + const page = await browser().newPage() + const printed = [] + const spy = vi.spyOn(console, 'error').mockImplementation((line) => { + printed.push(String(line)) + }) + const stop = new AbortController() + const timer = setTimeout(() => stop.abort(), 300) + await waitForRecordedNavigation( + page, + [], + () => false, + stop.signal, + { arm: 'arm sampling-probe', browserVersion: browser().version() }, + 25 + ) + clearTimeout(timer) + spy.mockRestore() + await page.close() + expect(printed).toHaveLength(1) + expect(printed[0]).toContain('arm sampling-probe') + // Not the placeholder: this string is only there if the sampling branch produced a reading. + expect(printed[0]).toContain('frames [') + }, 60_000) + it('keeps the Preview/Source toggle, and Source shows the source', async (ctx) => { const read = await open(browser(), { signal: ctx.signal, @@ -719,148 +811,6 @@ describe('the HTML preview needs no policy change', () => { }) }) -/** - * The mounted frame, once it holds the artifact. - * - * Found by its element, never by its URL. A `srcdoc` frame reports `about:srcdoc` on both engines - * here and an empty URL on CI's browser, and a poll that waited for the string spent every case's - * whole timeout there -- seven timeouts on one engine, after the same difference had already shown - * up as `expected '' to be 'about:srcdoc'`. - * - * Three things still settle at their own moments: React commits the mount, the element's `srcdoc` - * commits a document, and an override arm replaces that document with a second one. So readiness is - * the fixture's own marker inside the frame, which exists only once the artifact has parsed there. - * - * `frameReady` is which of those an arm is waiting for, because the marker is not always the right - * one. `'script'` waits for what the inline script writes: the marker element exists from parse - * time, so an arm whose oracle is "the script ran" would otherwise read `window.__ran` before it - * had. `'refusal'` waits for the frame's own `script-src` violation, which is queued and can land - * after `load`. `'load'` is for the one arm whose artifact deliberately navigates the frame - * somewhere else, where no marker is ever coming. - */ -async function waitForLoadedFrame(page, frameReady = 'artifact', signal, browserVersion, arm) { - const element = await page.waitForSelector('iframe', { timeout: 0 }) - const frame = await element.contentFrame() - if (!frame) { - return null - } - await frame.waitForLoadState('load').catch(() => {}) - if (frameReady === 'script') { - await untilAborted( - frame.waitForFunction(() => window.__ran === 1, undefined, { timeout: 0 }), - signal, - async () => - `the artifact's script never ran inside the frame: ${arm} | ${await describePreviewFrame(page, frame, browserVersion)}` - ) - } - if (frameReady === 'refusal') { - // The violation is dispatched as a queued task, so its order against the frame's `load` is not - // guaranteed: on the runner's Chrome the list held only the blocked background image when the - // reading was taken, and the arm that needs the script-src entry read it before it landed. So - // the arm waits for its own evidence rather than hoping to be later than a task queue. A frame - // that was never widened never raises it at all, which is what makes this the arm's precondition - // and not a convenience: the wait ends in the diagnosis below rather than in a passing read. - await untilAborted( - frame.waitForFunction( - () => (window.__violations ?? []).some((one) => String(one).includes('script-src')), - undefined, - { timeout: 0 } - ), - signal, - async () => - `the frame never reported a script-src refusal: ${arm} | ${await describePreviewFrame(page, frame, browserVersion)}` - ) - } - if (frameReady !== 'load') { - await untilAborted( - frame.waitForSelector('#marker', { state: 'attached', timeout: 0 }), - signal, - async () => - `the artifact never parsed inside the frame: ${arm} | ${await describePreviewFrame(page, frame, browserVersion)}` - ) - } - return frame -} - -/** - * Where an arm's counters are read: after the thing it is about, whatever that thing is. - * - * `expectNavigation` names what the arm is waiting for, and an arm that expects one waits for the - * record itself rather than for a clock. An arm that expects none has nothing to await, so it takes - * the bounded path below. - */ -async function settleAfterMount(page, navigations, expectNavigation, signal, reading) { - if (expectNavigation === 'main-frame') { - return await waitForRecordedNavigation(page, navigations, (one) => one.main, signal, reading) - } - if (expectNavigation === 'frame') { - return await waitForRecordedNavigation( - page, - navigations, - (one) => !one.main && !one.foreign, - signal, - reading - ) - } - return await settleWithoutNavigation(page) -} - -/** - * The moment the arm's navigation exists, for an arm that expects one. - * - * No clock at all: the route handler above records a main-frame navigation as the browser dispatches - * it, so the oracles are read after the thing under test rather than after a wait, and the only - * bound is the case's own timeout through `ctx.signal`. An arm whose click missed its target prints - * what it did record and lets the case fail as the timeout it is. - * - * Measured, so it is not sold as more than it is: with this replaced by a no-op every arm still - * passes, because the reads that follow are each a round trip and the record lands during them. It is - * the load the CI runner was under that this is for, which is the same condition that produced the - * frame-commit race above. - */ -async function waitForRecordedNavigation(page, navigations, matches, signal, reading) { - // Sampled while waiting, for the same reason `untilAborted` samples: a reading taken at the abort - // can lose its race with vitest's teardown and never reach the log. - let latest = 'no reading was taken before the case ended' - let since = Date.now() - while (!navigations.some((one) => matches(one))) { - if (signal?.aborted) { - console.error( - `[html-preview-render] the arm produced no navigation of the kind it expects; recorded ${JSON.stringify(navigations)}: ${reading?.arm ?? 'arm unknown'} | ${latest}` - ) - return - } - if (Date.now() - since > 5000) { - since = Date.now() - latest = await describePreviewFrame(page, reading?.frame, reading?.browserVersion).catch( - (error) => `the reading itself failed: ${String(error).split('\n')[0]}` - ) - } - await page.waitForTimeout(10) - } -} - -/** - * Where an absence is read, for the arms that expect no navigation at all. - * - * Nothing signals "the tap produced nothing", so this one is bounded rather than awaited. Two painted - * frames inside the page come first: by the second, a navigation the click started has been dispatched - * and would already be in the list the arms above read. The 200 ms after it is for the popup queue, - * which is a browser-process event with no in-page counterpart to await. - * - * What keeps these absences honest is not the length of that wait: the arms that read 1 on the same - * counters take the path above, so a counter that had stopped counting reds there. - */ -async function settleWithoutNavigation(page) { - await page.evaluate( - () => - new Promise((resolve) => { - requestAnimationFrame(() => requestAnimationFrame(() => resolve())) - }) - ) - await page.waitForTimeout(200) -} - /** One pixel of the frame's own fill, which is what says the artifact parsed and painted. */ async function probePixel(page) { const png = PNG.sync.read(await page.screenshot({ clip: FRAME_PROBE })) diff --git a/config/scripts/mobile-web-app-preview-csp-reports.mjs b/config/scripts/mobile-web-app-preview-csp-reports.mjs new file mode 100644 index 00000000000..04200a0ec03 --- /dev/null +++ b/config/scripts/mobile-web-app-preview-csp-reports.mjs @@ -0,0 +1,99 @@ +/** + * Where the preview rig learns that the policy refused something: from the browser's own report, + * not from a listener inside the frame. + * + * The in-frame collector is a page init script, and it can only report what it was present for. In + * a frame with no `allow-scripts` it observes nothing at all -- the array is there and stays empty + * while the policy refuses the artifact's image, measured on both engines -- and on CI's Chrome it + * intermittently missed the `script-src` refusal of a widened frame while catching that same + * frame's later `img-src` one. A report is sent by the browser itself, so nothing has to have been + * listening in time, and it arrives for the sealed frame too. + * + * Keyed by the arm's nonce, which the document carries in its own URL: every mount loads + * `/preview?n=` and the policy that document is served names `/csp-report?n=`, so a + * report is attributable to the arm that caused it even though a `srcdoc` frame has no URL of its + * own to name. + * + * `report-uri` is additive. It says where a report is sent and changes nothing about what the policy + * enforces, and the rig pins that by asserting the served directives are the shipped text apart from + * the one appended here. + */ + +const REPORT_PATH = '/csp-report' +const POLL_MS = 25 + +/** The directive a report names, from either report body shape, or the raw body if it is neither. */ +function reportedDirective(body) { + try { + const parsed = JSON.parse(body) + return ( + parsed['csp-report']?.['violated-directive'] ?? + parsed[0]?.body?.effectiveDirective ?? + parsed['csp-report']?.['effective-directive'] ?? + body + ) + } catch { + return body + } +} + +export function createCspReportSink() { + const reports = [] + return { + reports, + /** Answers the endpoint the served policy names, and says whether it took the request. */ + handleRequest(request, response, path) { + if (path !== REPORT_PATH) { + return false + } + const nonce = new URL(request.url, 'http://report').searchParams.get('n') + const chunks = [] + request.on('data', (chunk) => chunks.push(chunk)) + request.on('end', () => { + reports.push({ + nonce, + directive: reportedDirective(Buffer.concat(chunks).toString('utf8')) + }) + response.writeHead(204) + response.end() + }) + return true + }, + /** + * The shipped policy plus this document's own endpoint. + * + * Absolute, built from the request's own `Host`: a frame that inherits this policy has no URL to + * resolve a path against, and the port is not known until the server is listening. + */ + policyFor(csp, request) { + if (!csp) { + return csp + } + const nonce = new URL(request.url, 'http://page').searchParams.get('n') ?? 'none' + return `${csp}; report-uri http://${request.headers.host}${REPORT_PATH}?n=${nonce}` + } + } +} + +/** Every directive this arm's frames were reported for, in arrival order. */ +export function reportedDirectives(sink, nonce) { + return sink.reports.filter((one) => one.nonce === nonce).map((one) => one.directive) +} + +/** + * Waits until this arm has been reported for `directive`. + * + * Returns rather than throws on abort, for the reason `untilAborted` does: the reading has already + * been printed by then and a late rejection has nobody left to catch it. + */ +export async function pollReportsUntil(sink, nonce, directive, signal) { + while (!signal?.aborted) { + if (reportedDirectives(sink, nonce).some((one) => one.includes(directive))) { + return + } + await new Promise((resolve) => { + const timer = setTimeout(resolve, POLL_MS) + timer.unref?.() + }) + } +} diff --git a/config/scripts/mobile-web-app-preview-frame-diagnosis.mjs b/config/scripts/mobile-web-app-preview-frame-diagnosis.mjs index f2ff15d12e9..cac1008cfb2 100644 --- a/config/scripts/mobile-web-app-preview-frame-diagnosis.mjs +++ b/config/scripts/mobile-web-app-preview-frame-diagnosis.mjs @@ -34,26 +34,32 @@ export async function untilAborted(wait, signal, describe) { } } void sample() + let report = null await Promise.race([ wait, new Promise((resolve) => { if (!signal) { return } - const report = () => { - console.error(`[html-preview-render] ${latest}`) - resolve() - } if (signal.aborted) { // Silent: the case was already over when this wait began, so it has nothing of its own to // report and the wait that did time out has already printed its reading. resolve() return } + report = () => { + console.error(`[html-preview-render] ${latest}`) + resolve() + } signal.addEventListener('abort', report, { once: true }) }) ]).catch(() => {}) sampling = false + // Dropped on the way out, so the wait that hung is the only one that speaks: a listener left by a + // wait that resolved prints its own stale reading at a later wait's timeout. + if (report) { + signal?.removeEventListener('abort', report) + } } /** @@ -71,6 +77,13 @@ export async function untilAborted(wait, signal, describe) { * cross frames, so the top document's array says nothing about what the frame refused -- and the * page's init script installs the same collector in every frame, measured on both engines, so each * frame has its own array to report. + * + * `utilityWorld` is the fourth reading, and it is the one the readings above cannot give. Everything + * else here is an evaluate, which needs only a frame's main context; a selector wait needs the + * injected script in Chromium's isolated world, created per document by a command whose failure the + * driver swallows. Three cases once spent their whole timeout in such a wait while an evaluate in + * the same frame answered, so the probe is bounded and reported rather than left to be inferred + * again. `unavailable` here and a main-world reading beside it is that split, measured. */ export async function describePreviewFrame(page, frame, browserVersion) { const host = await page @@ -99,12 +112,24 @@ export async function describePreviewFrame(page, frame, browserVersion) { readyState: document.readyState, bodyChars: document.body?.innerHTML.length ?? null, marker: document.getElementById('marker') !== null, - ran: window.__ran ?? null, + ran: document.documentElement.dataset.ran ?? null, + // The order the collector's own reach depends on: when the page's init script ran here and + // when the artifact's script did. A listener installed after the parser reached the inline + // script can only report what came later. + initAt: window.__initAt ?? null, + artifactAt: document.documentElement.dataset.artifactAt ?? null, violations: window.__violations ?? 'absent' })) .catch((error) => `evaluate refused: ${String(error).split('\n')[0]}`) + // Bounded, and the only wait in the diagnosis: a frame whose isolated world never arrives would + // otherwise hold the reading open for as long as the wait it is explaining. + const utilityWorld = await one + .locator('html') + .waitFor({ state: 'attached', timeout: 2000 }) + .then(() => 'resolved') + .catch((error) => `unavailable: ${String(error).split('\n')[0]}`) frames.push( - `${JSON.stringify(one.url())} name ${JSON.stringify(one.name())} ${JSON.stringify(reading)}` + `${JSON.stringify(one.url())} name ${JSON.stringify(one.name())} utilityWorld ${JSON.stringify(utilityWorld)} ${JSON.stringify(reading)}` ) } return [ diff --git a/config/scripts/mobile-web-app-preview-frame-readiness.mjs b/config/scripts/mobile-web-app-preview-frame-readiness.mjs new file mode 100644 index 00000000000..110268c9dc3 --- /dev/null +++ b/config/scripts/mobile-web-app-preview-frame-readiness.mjs @@ -0,0 +1,218 @@ +/** + * Where the render rig decides a preview frame is ready and when an arm's counters may be read, and + * the world it asks in. + * + * Every wait here is `frame.evaluate`, which needs only the frame's own main execution context. + * Playwright's `waitForSelector` and `waitForFunction` need its injected script as well, and + * `waitForSelector` needs that script in the utility world -- an isolated world Chromium creates per + * document through a command whose failure is swallowed and whose creation event is dropped for a + * frame the driver considers stale. With `timeout: 0` a world that never arrives is a wait that + * never ends, which is what three cases did on CI's Chrome while an evaluate in the same frame + * reported the marker already present. The diagnosis prints a bounded probe of that world now, so + * the next run measures it rather than inferring it. + * + * The frame is resolved again on every attempt rather than bound once, so a document committed after + * a wait began is the one the predicate runs in. + */ + +import { describePreviewFrame, untilAborted } from './mobile-web-app-preview-frame-diagnosis.mjs' +import { pollReportsUntil } from './mobile-web-app-preview-csp-reports.mjs' + +const POLL_MS = 25 +const EVALUATE_MS = 1000 + +/** The mounted preview frame, or null before one exists. */ +export function previewFrame(page) { + return page.frames().find((one) => one !== page.mainFrame()) ?? null +} + +const abandonAfter = (ms) => + new Promise((resolve) => { + const timer = setTimeout(() => resolve(false), ms) + timer.unref?.() + }) + +/** + * Polls `predicate` inside the preview frame until it holds or the case ends. + * + * Returns rather than throws when the signal aborts: `untilAborted` has already printed the reading + * by then, and a rejection raised after vitest has given up has nobody left to catch it. + */ +export async function pollFrameUntil(page, predicate, signal) { + while (!signal?.aborted) { + const frame = previewFrame(page) + // An evaluate carries no timeout of its own and waits on the frame's main context, so one that + // never answers is abandoned here rather than outliving the frame it was asked of. + const met = frame + ? await Promise.race([ + frame.evaluate(predicate).catch(() => false), + abandonAfter(EVALUATE_MS) + ]) + : false + if (met) { + return + } + await abandonAfter(POLL_MS) + } +} + +/** + * The mounted frame, once it holds the artifact. + * + * Found among the page's frames, never by its URL. A `srcdoc` frame reports `about:srcdoc` on both + * engines here and an empty URL on CI's browser, and a poll that waited for the string spent every + * case's whole timeout there -- seven timeouts on one engine, after the same difference had already + * shown up as `expected '' to be 'about:srcdoc'`. + * + * Every wait below asks in the frame's main world through `pollFrameUntil`, for the reason that + * module carries: a selector wait needs an isolated world the embedder cannot see fail. + * + * Three things still settle at their own moments: React commits the mount, the element's `srcdoc` + * commits a document, and an override arm replaces that document with a second one. So readiness is + * the fixture's own marker inside the frame, which exists only once the artifact has parsed there. + * + * `frameReady` is which of those an arm is waiting for, because the marker is not always the right + * one. `'script'` waits for what the inline script writes, on the document element rather than on a + * window global: the marker element exists from parse time, so an arm whose oracle is "the script + * ran" would otherwise read the flag before it was written. `'load'` is for the one arm whose + * artifact deliberately navigates the frame somewhere else, where no marker is ever coming. + * + * `reportReady` is the other kind of precondition: a refusal the policy reported to the rig's own + * server, which an arm about what the policy refused waits for instead of reading a list. + */ +export async function waitForLoadedFrame( + page, + { frameReady = 'artifact', reportReady = null, signal, browserVersion, arm, sink, nonce } +) { + const reading = async (what) => + `${what}: ${arm} | ${await describePreviewFrame(page, previewFrame(page), browserVersion)}` + await untilAborted( + pollFrameUntil(page, () => true, signal), + signal, + async () => await reading('no frame ever answered inside the page') + ) + const frame = previewFrame(page) + if (!frame) { + return null + } + // Bound to the case like every other wait here: `waitForLoadState` carries its own timeout and + // goes on waiting after the case has been aborted. + await untilAborted( + pollFrameUntil(page, () => document.readyState === 'complete', signal), + signal, + async () => await reading('the frame never finished loading') + ) + if (frameReady === 'script') { + await untilAborted( + pollFrameUntil(page, () => document.documentElement.dataset.ran === '1', signal), + signal, + async () => await reading("the artifact's script never ran inside the frame") + ) + } + if (reportReady) { + // The browser's own report, not the frame's listener. An arm whose claim is "the policy refused + // this" waits for the refusal to have been reported, which is evidence no in-frame listener has + // to have been installed in time to collect -- and in a frame with no `allow-scripts` none ever + // is. The wait ends in the diagnosis rather than in a passing read. + await untilAborted( + pollReportsUntil(sink, nonce, reportReady, signal), + signal, + async () => + await reading(`the policy reported no ${String(reportReady)} refusal for this arm`) + ) + } + if (frameReady !== 'load') { + await untilAborted( + pollFrameUntil(page, () => document.getElementById('marker') !== null, signal), + signal, + async () => await reading('the artifact never parsed inside the frame') + ) + } + return previewFrame(page) +} + +/** + * Where an absence is read, for the arms that expect no navigation at all. + * + * Nothing signals "the tap produced nothing", so this one is bounded rather than awaited. Two painted + * frames inside the page come first: by the second, a navigation the click started has been dispatched + * and would already be in the list the arms above read. The 200 ms after it is for the popup queue, + * which is a browser-process event with no in-page counterpart to await. + * + * What keeps these absences honest is not the length of that wait: the arms that read 1 on the same + * counters take the path above, so a counter that had stopped counting reds there. + */ +async function settleWithoutNavigation(page) { + await page.evaluate( + () => + new Promise((resolve) => { + requestAnimationFrame(() => requestAnimationFrame(() => resolve())) + }) + ) + await page.waitForTimeout(200) +} + +/** + * Where an arm's counters are read: after the thing it is about, whatever that thing is. + * + * `expectNavigation` names what the arm is waiting for, and an arm that expects one waits for the + * record itself rather than for a clock. An arm that expects none has nothing to await, so it takes + * the bounded path below. + */ +export async function settleAfterMount(page, navigations, expectNavigation, signal, reading) { + if (expectNavigation === 'main-frame') { + return await waitForRecordedNavigation(page, navigations, (one) => one.main, signal, reading) + } + if (expectNavigation === 'frame') { + return await waitForRecordedNavigation( + page, + navigations, + (one) => !one.main && !one.foreign, + signal, + reading + ) + } + return await settleWithoutNavigation(page) +} + +/** + * The moment the arm's navigation exists, for an arm that expects one. + * + * No clock at all: the route handler above records a main-frame navigation as the browser dispatches + * it, so the oracles are read after the thing under test rather than after a wait, and the only + * bound is the case's own timeout through `ctx.signal`. An arm whose click missed its target prints + * what it did record and lets the case fail as the timeout it is. + * + * Measured, so it is not sold as more than it is: with this replaced by a no-op every arm still + * passes, because the reads that follow are each a round trip and the record lands during them. It is + * the load the CI runner was under that this is for, which is the same condition that produced the + * frame-commit race above. + */ +export async function waitForRecordedNavigation( + page, + navigations, + matches, + signal, + reading, + sampleEveryMs = 5000 +) { + // Sampled while waiting, for the same reason `untilAborted` samples: a reading taken at the abort + // can lose its race with vitest's teardown and never reach the log. + let latest = 'no reading was taken before the case ended' + let since = Date.now() + while (!navigations.some((one) => matches(one))) { + if (signal?.aborted) { + console.error( + `[html-preview-render] the arm produced no navigation of the kind it expects; recorded ${JSON.stringify(navigations)}: ${reading?.arm ?? 'arm unknown'} | ${latest}` + ) + return + } + if (Date.now() - since > sampleEveryMs) { + since = Date.now() + latest = await describePreviewFrame(page, reading?.frame, reading?.browserVersion).catch( + (error) => `the reading itself failed: ${String(error).split('\n')[0]}` + ) + } + await page.waitForTimeout(10) + } +} diff --git a/config/scripts/mobile-web-app-render-harness.mjs b/config/scripts/mobile-web-app-render-harness.mjs index 1809985f6c9..26e633a2e9a 100644 --- a/config/scripts/mobile-web-app-render-harness.mjs +++ b/config/scripts/mobile-web-app-render-harness.mjs @@ -348,10 +348,17 @@ export function installShellDouble({ * The page server the render checks run against: the built bundle, under the shell's own policy. * * `transformChunk` is how a check poisons one route chunk without building a second bundle. + * `cspHeader` may be a function of the request, and `handleRequest` lets a check answer a path of + * its own on this origin. */ -export async function createBundleServer({ outDir, cspHeader, transformChunk }) { +export async function createBundleServer({ outDir, cspHeader, transformChunk, handleRequest }) { const server = createServer((request, response) => { const path = new URL(request.url, 'http://localhost').pathname + // An endpoint of the check's own, answered before anything is looked for on disk: a policy's + // `report-uri` has to name a real server, and naming this one keeps it on the page's origin. + if (handleRequest?.(request, response, path)) { + return + } // A browser asks for this on its own and the shell's WebView never does. The bundle carries // no icon, so a 404 would put a console error in every check that runs against a full Chrome // -- which is what CI resolves -- and none against the bundled headless shell. @@ -374,7 +381,10 @@ export async function createBundleServer({ outDir, cspHeader, transformChunk }) // The document carries the shell's real policy, so a directive the page violates fails // here rather than on a phone. Assets carry none, exactly as the native handler does. if (file === 'index.html' && cspHeader) { - headers['content-security-policy'] = cspHeader + // A function when the policy is per-document: the preview rig appends this document's own + // report endpoint, which carries the arm's nonce. + headers['content-security-policy'] = + typeof cspHeader === 'function' ? cspHeader(request) : cspHeader } response.writeHead(200, headers) response.end(bytes)