mirror of
https://github.com/stablyai/orca.git
synced 2026-09-24 16:02:41 +00:00
feat(mobile): page bundle ranges through one window reader chosen by the manifest
The fetch keeps the pipeline's four-worker pool and builds one window reader from the manifest reply: ranges on rangeBytes when the host names it, chunks on chunkBytes otherwise. The reader returns the six-field header and a lazy bytes() so the stop and misroute checks run before any decode. A range that inflates to the wrong length now falls to the slot checks, with the one-byte-over buffer as the memory bound, so range-length-mismatch is gone. A rangeBytes this build cannot page reads as absent. Fetch names say window, not chunk. Claude-Session: https://claude.ai/code/session_01JNnE9qzUZMMnqpZWCqM3nb
This commit is contained in:
@@ -23,7 +23,6 @@ const REASON_COPY: Record<MobileWebShellUpdateFailureReason, string> = {
|
||||
'chunk-misrouted': 'a chunk answered the wrong asset or offset',
|
||||
'asset-entry-changed': 'an asset no longer matched the manifest',
|
||||
'range-undecodable': 'a compressed read could not be decoded',
|
||||
'range-length-mismatch': 'a compressed read decoded to the wrong size',
|
||||
'fetch-stopped': 'the download was stopped',
|
||||
'cache-write-failed': 'saving the download on this phone failed',
|
||||
'unrecognised-error': 'an unrecognised error'
|
||||
|
||||
@@ -1,15 +1,32 @@
|
||||
import { sha256 } from '@noble/hashes/sha256'
|
||||
import { gzipSync } from 'fflate'
|
||||
import { describe, expect, it, vi } from 'vitest'
|
||||
import { MOBILE_WEB_BUNDLE_CHUNK_BYTES } from '../../../src/shared/mobile-web-bundle/bundle-rpc-contract'
|
||||
import { MOBILE_WEB_BUNDLE_RANGE_BYTES } from '../../../src/shared/mobile-web-bundle/bundle-range-rpc-contract'
|
||||
import { beforeEach, describe, expect, it, vi } from 'vitest'
|
||||
import {
|
||||
MOBILE_WEB_BUNDLE_CHUNK_BYTES,
|
||||
MOBILE_WEB_BUNDLE_RANGE_BYTES
|
||||
} from '../../../src/shared/mobile-web-bundle/bundle-rpc-contract'
|
||||
import { computeMobileWebBundleId } from '../../../src/shared/mobile-web-bundle/manifest-contract'
|
||||
import { fetchMobileWebBundle } from './mobile-web-bundle-fetch'
|
||||
import { MobileWebBundleFetchError } from './mobile-web-bundle-fetch-refusal'
|
||||
import type { MobileWebBundleReadMethod } from './mobile-web-bundle-read-method'
|
||||
import type { RpcClient } from './rpc-client'
|
||||
import type { RpcResponse } from './types'
|
||||
|
||||
const inflations = vi.hoisted(() => ({ outLengths: [] as number[], resultLengths: [] as number[] }))
|
||||
|
||||
// Observes the bound the decoder hands fflate, and what fflate hands back inside it.
|
||||
vi.mock('fflate', async (importOriginal) => {
|
||||
const fflate = await importOriginal<typeof import('fflate')>()
|
||||
return {
|
||||
...fflate,
|
||||
gunzipSync: (data: Uint8Array, options?: { out?: Uint8Array }) => {
|
||||
inflations.outLengths.push(options?.out?.byteLength ?? -1)
|
||||
const result = fflate.gunzipSync(data, options)
|
||||
inflations.resultLengths.push(result.byteLength)
|
||||
return result
|
||||
}
|
||||
}
|
||||
})
|
||||
|
||||
function toHex(bytes: Uint8Array): string {
|
||||
return Array.from(bytes, (byte) => byte.toString(16).padStart(2, '0')).join('')
|
||||
}
|
||||
@@ -29,15 +46,27 @@ function scriptBytes(byteLength: number, seed: number): Uint8Array {
|
||||
)
|
||||
}
|
||||
|
||||
/** Deterministic and incompressible, so the host sends it as identity. */
|
||||
function noiseBytes(byteLength: number): Uint8Array {
|
||||
let state = 0x9e3779b9
|
||||
return Uint8Array.from({ length: byteLength }, () => {
|
||||
state ^= state << 13
|
||||
state ^= state >>> 17
|
||||
state ^= state << 5
|
||||
return state & 0xff
|
||||
})
|
||||
}
|
||||
|
||||
type HostCall = { method: string; params: Record<string, unknown> }
|
||||
|
||||
/**
|
||||
* A host that serves both read methods by the real one's rules: ranges on the caller's own grid,
|
||||
* gzipped at level 6 when that shrinks them. `tamper` replaces the body of one range reply.
|
||||
* A host that serves both read methods by the real one's rules: ranges on its advertised grid,
|
||||
* gzipped at level 6 when that shrinks them. `ranges: false` is a host that predates the method.
|
||||
* `tamper` replaces the body of one range reply.
|
||||
*/
|
||||
function rangeHost(
|
||||
files: Record<string, Uint8Array>,
|
||||
options: { tamper?: (call: HostCall, body: string) => string } = {}
|
||||
options: { ranges?: boolean; tamper?: (call: HostCall, body: string) => string } = {}
|
||||
) {
|
||||
const assets = Object.entries(files)
|
||||
.map(([path, content]) => ({
|
||||
@@ -65,17 +94,23 @@ function rangeHost(
|
||||
|
||||
const answer = (call: HostCall): unknown => {
|
||||
if (call.method === 'mobileWeb.bundle.manifest') {
|
||||
return { manifest, chunkBytes: MOBILE_WEB_BUNDLE_CHUNK_BYTES }
|
||||
return options.ranges === false
|
||||
? { manifest, chunkBytes: MOBILE_WEB_BUNDLE_CHUNK_BYTES }
|
||||
: {
|
||||
manifest,
|
||||
chunkBytes: MOBILE_WEB_BUNDLE_CHUNK_BYTES,
|
||||
rangeBytes: MOBILE_WEB_BUNDLE_RANGE_BYTES
|
||||
}
|
||||
}
|
||||
const path = String(call.params.path)
|
||||
const offset = Number(call.params.offset)
|
||||
const content = files[path]!
|
||||
const length =
|
||||
const grid =
|
||||
call.method === 'mobileWeb.bundle.range'
|
||||
? Number(call.params.length)
|
||||
? MOBILE_WEB_BUNDLE_RANGE_BYTES
|
||||
: MOBILE_WEB_BUNDLE_CHUNK_BYTES
|
||||
const slice = content.subarray(offset, offset + length)
|
||||
const common = {
|
||||
const slice = content.subarray(offset, offset + grid)
|
||||
const header = {
|
||||
buildId,
|
||||
path,
|
||||
offset,
|
||||
@@ -84,12 +119,12 @@ function rangeHost(
|
||||
eof: offset + slice.byteLength >= content.byteLength
|
||||
}
|
||||
if (call.method === 'mobileWeb.bundle.chunk') {
|
||||
return { ...common, dataBase64: encodeBase64(slice) }
|
||||
return { ...header, dataBase64: encodeBase64(slice) }
|
||||
}
|
||||
const gzipped = gzipSync(slice, { level: 6 })
|
||||
const encoding = gzipped.byteLength < slice.byteLength ? 'gzip' : 'identity'
|
||||
const body = encodeBase64(encoding === 'gzip' ? gzipped : slice)
|
||||
return { ...common, encoding, dataBase64: options.tamper?.(call, body) ?? body }
|
||||
return { ...header, encoding, dataBase64: options.tamper?.(call, body) ?? body }
|
||||
}
|
||||
|
||||
const client: RpcClient = {
|
||||
@@ -125,10 +160,15 @@ function rangeHost(
|
||||
}
|
||||
}
|
||||
|
||||
/** One asset larger than a range, so the range grid pages it more than once. */
|
||||
const FILES = {
|
||||
const EXACT = 'assets/exact.js'
|
||||
const NOISE = 'assets/noise.bin'
|
||||
const EMPTY = 'assets/empty.txt'
|
||||
/** One asset over a range, one an exact multiple of it, one incompressible, one empty. */
|
||||
const FILES: Record<string, Uint8Array> = {
|
||||
'assets/app.js': scriptBytes(MOBILE_WEB_BUNDLE_RANGE_BYTES * 2 + 5000, 1),
|
||||
'assets/vendor.js': scriptBytes(120_000, 2),
|
||||
[EXACT]: scriptBytes(MOBILE_WEB_BUNDLE_RANGE_BYTES * 2, 4),
|
||||
[NOISE]: noiseBytes(70_000),
|
||||
[EMPTY]: new Uint8Array(0),
|
||||
'index.html': scriptBytes(600, 3)
|
||||
}
|
||||
|
||||
@@ -136,10 +176,12 @@ function readsOf(calls: readonly HostCall[], method: string): HostCall[] {
|
||||
return calls.filter((call) => call.method === method)
|
||||
}
|
||||
|
||||
async function fetchWith(readMethod: MobileWebBundleReadMethod, files = FILES) {
|
||||
const host = rangeHost(files)
|
||||
const fetched = await fetchMobileWebBundle({ client: host.client, readMethod })
|
||||
return { host, fetched }
|
||||
/** One read per grid slot, and one for an empty asset. */
|
||||
function slotsOn(grid: number, files: Record<string, Uint8Array> = FILES): number {
|
||||
return Object.values(files).reduce(
|
||||
(total, file) => total + Math.max(1, Math.ceil(file.byteLength / grid)),
|
||||
0
|
||||
)
|
||||
}
|
||||
|
||||
async function refusalOf(failed: Promise<unknown>): Promise<string | null> {
|
||||
@@ -150,52 +192,63 @@ async function refusalOf(failed: Promise<unknown>): Promise<string | null> {
|
||||
return error instanceof MobileWebBundleFetchError ? error.refusal : null
|
||||
}
|
||||
|
||||
describe('fetchMobileWebBundle over ranges', () => {
|
||||
it('pages every asset in ranges on the range grid and returns the verified bytes', async () => {
|
||||
const { host, fetched } = await fetchWith('range')
|
||||
beforeEach(() => {
|
||||
inflations.outLengths.length = 0
|
||||
inflations.resultLengths.length = 0
|
||||
})
|
||||
|
||||
describe('fetchMobileWebBundle from a host whose manifest names a range grid', () => {
|
||||
it('pages every asset in ranges on that grid and returns the verified bytes', async () => {
|
||||
const host = rangeHost(FILES)
|
||||
const fetched = await fetchMobileWebBundle({ client: host.client })
|
||||
|
||||
for (const [path, content] of Object.entries(FILES)) {
|
||||
expect(fetched.assets.get(path)).toEqual(content)
|
||||
}
|
||||
expect(readsOf(host.calls, 'mobileWeb.bundle.chunk')).toHaveLength(0)
|
||||
const ranges = readsOf(host.calls, 'mobileWeb.bundle.range')
|
||||
// 3 for the large asset, 1 each for the other two.
|
||||
expect(ranges).toHaveLength(5)
|
||||
expect(ranges).toHaveLength(slotsOn(MOBILE_WEB_BUNDLE_RANGE_BYTES))
|
||||
for (const range of ranges) {
|
||||
expect(range.params.length).toBe(MOBILE_WEB_BUNDLE_RANGE_BYTES)
|
||||
expect(Object.keys(range.params).sort()).toEqual(['buildId', 'offset', 'path'])
|
||||
expect(Number(range.params.offset) % MOBILE_WEB_BUNDLE_RANGE_BYTES).toBe(0)
|
||||
}
|
||||
expect(host.peakInFlight()).toBeLessThanOrEqual(4)
|
||||
expect(host.peakInFlight()).toBe(4)
|
||||
})
|
||||
|
||||
// The discovery is the status capability the session already read, never a request of its own.
|
||||
it('sends nothing but the manifest and the reads to decide the method', async () => {
|
||||
const { host } = await fetchWith('range')
|
||||
it('accepts an identity range, an empty asset, and an asset that ends on the grid', async () => {
|
||||
const host = rangeHost(FILES)
|
||||
const fetched = await fetchMobileWebBundle({ client: host.client })
|
||||
|
||||
expect(
|
||||
host.calls.filter(
|
||||
(call) =>
|
||||
call.method !== 'mobileWeb.bundle.range' && call.method !== 'mobileWeb.bundle.manifest'
|
||||
)
|
||||
).toEqual([])
|
||||
expect(fetched.assets.get(NOISE)).toEqual(FILES[NOISE])
|
||||
expect(fetched.assets.get(EMPTY)).toEqual(new Uint8Array(0))
|
||||
expect(fetched.assets.get(EXACT)).toEqual(FILES[EXACT])
|
||||
const exact = readsOf(host.calls, 'mobileWeb.bundle.range').filter(
|
||||
(call) => call.params.path === EXACT
|
||||
)
|
||||
expect(exact.map((call) => call.params.offset)).toEqual([0, MOBILE_WEB_BUNDLE_RANGE_BYTES])
|
||||
})
|
||||
|
||||
it('pages a host without the range capability in chunks, as before', async () => {
|
||||
const { host, fetched } = await fetchWith('chunk')
|
||||
it('pages a host whose manifest names no range grid in chunks, as before', async () => {
|
||||
const host = rangeHost(FILES, { ranges: false })
|
||||
const fetched = await fetchMobileWebBundle({ client: host.client })
|
||||
|
||||
expect(fetched.assets.get('assets/app.js')).toEqual(FILES['assets/app.js'])
|
||||
expect(readsOf(host.calls, 'mobileWeb.bundle.range')).toHaveLength(0)
|
||||
expect(readsOf(host.calls, 'mobileWeb.bundle.chunk').length).toBeGreaterThan(5)
|
||||
expect(readsOf(host.calls, 'mobileWeb.bundle.chunk')).toHaveLength(
|
||||
slotsOn(MOBILE_WEB_BUNDLE_CHUNK_BYTES)
|
||||
)
|
||||
})
|
||||
|
||||
it('carries the same bundle in fewer round trips and fewer bytes than chunks', async () => {
|
||||
const chunked = await fetchWith('chunk')
|
||||
const ranged = await fetchWith('range')
|
||||
it('carries the same bundle in fewer reads and fewer bytes than chunks', async () => {
|
||||
const chunked = rangeHost(FILES, { ranges: false })
|
||||
const ranged = rangeHost(FILES)
|
||||
await fetchMobileWebBundle({ client: chunked.client })
|
||||
await fetchMobileWebBundle({ client: ranged.client })
|
||||
|
||||
// 17 + 3 + 1 chunks at 48 KiB against 3 + 1 + 1 ranges at 384 KiB.
|
||||
expect(readsOf(chunked.host.calls, 'mobileWeb.bundle.chunk')).toHaveLength(21)
|
||||
expect(readsOf(ranged.host.calls, 'mobileWeb.bundle.range')).toHaveLength(5)
|
||||
expect(ranged.host.wireBase64Bytes()).toBeLessThan(chunked.host.wireBase64Bytes() / 2)
|
||||
expect(readsOf(ranged.calls, 'mobileWeb.bundle.range').length).toBeLessThan(
|
||||
readsOf(chunked.calls, 'mobileWeb.bundle.chunk').length
|
||||
)
|
||||
expect(ranged.wireBase64Bytes()).toBeLessThan(chunked.wireBase64Bytes())
|
||||
})
|
||||
|
||||
it('refuses a corrupt gzip range as undecodable', async () => {
|
||||
@@ -206,21 +259,28 @@ describe('fetchMobileWebBundle over ranges', () => {
|
||||
: body
|
||||
})
|
||||
|
||||
expect(
|
||||
await refusalOf(fetchMobileWebBundle({ client: host.client, readMethod: 'range' }))
|
||||
).toBe('range-undecodable')
|
||||
expect(await refusalOf(fetchMobileWebBundle({ client: host.client }))).toBe('range-undecodable')
|
||||
})
|
||||
|
||||
it('refuses a range that decodes to the wrong length', async () => {
|
||||
// A 4 MiB inflation answering a 600-byte window: fflate fills the bounded buffer and stops there.
|
||||
it('refuses a gzip bomb as overlong without inflating past one byte over the window', async () => {
|
||||
const bomb = gzipSync(new Uint8Array(4 * 1024 * 1024), { level: 9 })
|
||||
const host = rangeHost(FILES, {
|
||||
tamper: (call, body) =>
|
||||
call.params.path === 'index.html'
|
||||
? encodeBase64(gzipSync(scriptBytes(599, 3), { level: 6 }))
|
||||
: body
|
||||
tamper: (call, body) => (call.params.path === 'index.html' ? encodeBase64(bomb) : body)
|
||||
})
|
||||
|
||||
expect(
|
||||
await refusalOf(fetchMobileWebBundle({ client: host.client, readMethod: 'range' }))
|
||||
).toBe('range-length-mismatch')
|
||||
expect(await refusalOf(fetchMobileWebBundle({ client: host.client }))).toBe('asset-overlong')
|
||||
const bombAt = inflations.outLengths.indexOf(601)
|
||||
expect(bombAt).toBeGreaterThanOrEqual(0)
|
||||
expect(inflations.resultLengths[bombAt]).toBe(601)
|
||||
})
|
||||
|
||||
it('refuses a range that inflates short of its window as short', async () => {
|
||||
const host = rangeHost(FILES, {
|
||||
tamper: (call, body) =>
|
||||
call.params.path === 'index.html' ? encodeBase64(gzipSync(scriptBytes(599, 3))) : body
|
||||
})
|
||||
|
||||
expect(await refusalOf(fetchMobileWebBundle({ client: host.client }))).toBe('asset-short')
|
||||
})
|
||||
})
|
||||
|
||||
@@ -11,8 +11,6 @@ export const MOBILE_WEB_BUNDLE_FETCH_REFUSALS = [
|
||||
'asset-entry-changed',
|
||||
/** A range body that would not decode: corrupt or truncated gzip, or an encoding this build lacks. */
|
||||
'range-undecodable',
|
||||
/** A range that decoded to a length other than the window it answered. */
|
||||
'range-length-mismatch',
|
||||
'fetch-stopped'
|
||||
] as const
|
||||
|
||||
|
||||
@@ -250,7 +250,7 @@ describe('fetchMobileWebBundle', () => {
|
||||
|
||||
const failed = fetchMobileWebBundle({ client: host.client })
|
||||
await expect(failed).rejects.toThrow(
|
||||
'bundle chunk answered other.html at 0, not index.html at 0'
|
||||
'bundle window answered other.html at 0, not index.html at 0'
|
||||
)
|
||||
expect(await refusalOf(failed)).toBe('chunk-misrouted')
|
||||
})
|
||||
@@ -279,7 +279,7 @@ describe('fetchMobileWebBundle', () => {
|
||||
|
||||
const failed = fetchMobileWebBundle({ client: host.client })
|
||||
await expect(failed).rejects.toThrow(
|
||||
'bundle chunk answered index.html at 0, not index.html at 3'
|
||||
'bundle window answered index.html at 0, not index.html at 3'
|
||||
)
|
||||
expect(await refusalOf(failed)).toBe('chunk-misrouted')
|
||||
})
|
||||
@@ -351,7 +351,7 @@ describe('fetchMobileWebBundle', () => {
|
||||
|
||||
const failed = fetchMobileWebBundle({ client: host.client })
|
||||
await expect(failed).rejects.toThrow(
|
||||
"bundle chunk for index.html at 0 is 6 bytes, over the host's 3"
|
||||
'bundle window for index.html at 0 is 6 bytes, over the 3-byte window'
|
||||
)
|
||||
expect(await refusalOf(failed)).toBe('chunk-oversize')
|
||||
})
|
||||
|
||||
@@ -1,8 +1,5 @@
|
||||
import { sha256 } from '@noble/hashes/sha256'
|
||||
import {
|
||||
mobileWebBundleChunkRead,
|
||||
mobileWebBundleManifestRead
|
||||
} from './mobile-web-bundle-operations'
|
||||
import { mobileWebBundleManifestRead } from './mobile-web-bundle-operations'
|
||||
import type {
|
||||
MobileWebBundleAssetRead,
|
||||
MobileWebBundleManifestRead
|
||||
@@ -10,11 +7,18 @@ import type {
|
||||
import type { RpcClient } from './rpc-client'
|
||||
import { MobileWebBundleFetchError } from './mobile-web-bundle-fetch-refusal'
|
||||
import { runRpcOperation } from './rpc-operation'
|
||||
import {
|
||||
mobileWebBundleWindowReader,
|
||||
type MobileWebBundleWindowHeader
|
||||
} from './mobile-web-bundle-window-reader'
|
||||
|
||||
/** The host refuses the fifth concurrent read on one connection with `mobile_web_bundle_read_limited`,
|
||||
* so the client never offers a fifth. The four are chunk reads across the whole manifest, not one
|
||||
* asset each: paging a large asset alone would put every one of its chunks on the critical path. */
|
||||
const MAX_CONCURRENT_CHUNK_READS = 4
|
||||
/** A window is one byte span of an asset on the grid the manifest reply named: a 48 KiB chunk, or a
|
||||
* 384 KiB range from a host that serves them.
|
||||
*
|
||||
* The host refuses the fifth concurrent read on one connection with `mobile_web_bundle_read_limited`,
|
||||
* so the client never offers a fifth. The four are window reads across the whole manifest, not one
|
||||
* asset each: paging a large asset alone would put every one of its windows on the critical path. */
|
||||
const MAX_CONCURRENT_WINDOW_READS = 4
|
||||
|
||||
export type MobileWebBundleFetchProgress = {
|
||||
readonly completedAssets: number
|
||||
@@ -33,10 +37,10 @@ export type MobileWebBundleFetchResult = {
|
||||
type AssetReassembly = {
|
||||
readonly entry: MobileWebBundleAssetRead
|
||||
readonly whole: Uint8Array
|
||||
outstandingChunks: number
|
||||
outstandingWindows: number
|
||||
}
|
||||
|
||||
type ChunkRead = { readonly asset: AssetReassembly; readonly offset: number }
|
||||
type WindowRead = { readonly asset: AssetReassembly; readonly offset: number }
|
||||
|
||||
/**
|
||||
* Reads the manifest, pages every asset, and returns the verified bytes.
|
||||
@@ -55,29 +59,36 @@ export async function fetchMobileWebBundle(args: {
|
||||
throwIfCallerAborted(args.signal)
|
||||
const opened = await runRpcOperation(args.client, mobileWebBundleManifestRead, null)
|
||||
const manifest = opened.manifest
|
||||
const queue = planChunkReads(manifest.assets, opened.chunkBytes)
|
||||
const reader = mobileWebBundleWindowReader(opened)
|
||||
const queue = planWindowReads(manifest.assets, reader.windowBytes)
|
||||
const assets = new Map<string, Uint8Array>()
|
||||
let receivedBytes = 0
|
||||
|
||||
const readChunk = async ({ asset, offset }: ChunkRead): Promise<void> => {
|
||||
const reply = await runRpcOperation(args.client, mobileWebBundleChunkRead, {
|
||||
const readWindow = async ({ asset, offset }: WindowRead): Promise<void> => {
|
||||
const reply = await reader.read(args.client, {
|
||||
buildId: manifest.buildId,
|
||||
path: asset.entry.path,
|
||||
offset
|
||||
})
|
||||
// A sibling already failed the fetch; this reply is not worth checking, hashing or reporting.
|
||||
// A sibling already failed the fetch; this reply is not worth checking, decoding or reporting.
|
||||
if (stopped.signal.aborted) {
|
||||
return
|
||||
}
|
||||
assertChunkDescribesAsset(reply, asset.entry, manifest.buildId, offset)
|
||||
const bytes = decodeBase64(reply.dataBase64)
|
||||
assertChunkFillsItsSlot(asset.entry, offset, bytes.byteLength, reply.eof, opened.chunkBytes)
|
||||
assertWindowDescribesAsset(reply.header, asset.entry, manifest.buildId, offset)
|
||||
const bytes = reply.bytes()
|
||||
assertWindowFillsItsSlot(
|
||||
asset.entry,
|
||||
offset,
|
||||
bytes.byteLength,
|
||||
reply.header.eof,
|
||||
reader.windowBytes
|
||||
)
|
||||
asset.whole.set(bytes, offset)
|
||||
asset.outstandingChunks -= 1
|
||||
if (asset.outstandingChunks === 0) {
|
||||
asset.outstandingWindows -= 1
|
||||
if (asset.outstandingWindows === 0) {
|
||||
assets.set(asset.entry.path, verifyReassembledAsset(asset))
|
||||
}
|
||||
// Per chunk, not per asset: the largest asset goes first, so asset completions bunch at the end.
|
||||
// Per window, not per asset: the largest asset goes first, so asset completions bunch at the end.
|
||||
receivedBytes += bytes.byteLength
|
||||
args.onProgress?.({
|
||||
completedAssets: assets.size,
|
||||
@@ -95,17 +106,17 @@ export async function fetchMobileWebBundle(args: {
|
||||
return
|
||||
}
|
||||
throwIfCallerAborted(args.signal)
|
||||
await readChunk(read)
|
||||
await readWindow(read)
|
||||
}
|
||||
} catch (error) {
|
||||
// One failed chunk stops every other read: each read a worker would still send holds one of
|
||||
// One failed window stops every other read: each read a worker would still send holds one of
|
||||
// the host's four slots against the caller's retry.
|
||||
stopped.abort()
|
||||
throw error
|
||||
}
|
||||
}
|
||||
|
||||
const workers = Math.min(MAX_CONCURRENT_CHUNK_READS, queue.length)
|
||||
const workers = Math.min(MAX_CONCURRENT_WINDOW_READS, queue.length)
|
||||
await Promise.all(Array.from({ length: workers }, () => worker()))
|
||||
// The final window sends nothing after its last reply, so no worker would see this abort.
|
||||
throwIfCallerAborted(args.signal)
|
||||
@@ -113,41 +124,41 @@ export async function fetchMobileWebBundle(args: {
|
||||
}
|
||||
|
||||
/** Largest asset first, so the biggest script's tail is never the last read left in flight. Offsets
|
||||
* are the host's chunk grid, so every read is known up front; `eof` still comes from the reply. */
|
||||
function planChunkReads(
|
||||
* are on the window grid, so every read is known up front; `eof` still comes from the reply. */
|
||||
function planWindowReads(
|
||||
entries: readonly MobileWebBundleAssetRead[],
|
||||
chunkBytes: number
|
||||
): ChunkRead[] {
|
||||
windowBytes: number
|
||||
): WindowRead[] {
|
||||
const largestFirst = [...entries].sort((left, right) => right.byteLength - left.byteLength)
|
||||
return largestFirst.flatMap((entry) => {
|
||||
const count = Math.max(1, Math.ceil(entry.byteLength / chunkBytes))
|
||||
const count = Math.max(1, Math.ceil(entry.byteLength / windowBytes))
|
||||
const asset: AssetReassembly = {
|
||||
entry,
|
||||
whole: new Uint8Array(entry.byteLength),
|
||||
outstandingChunks: count
|
||||
outstandingWindows: count
|
||||
}
|
||||
return Array.from({ length: count }, (_, index) => ({ asset, offset: index * chunkBytes }))
|
||||
return Array.from({ length: count }, (_, index) => ({ asset, offset: index * windowBytes }))
|
||||
})
|
||||
}
|
||||
|
||||
/** Offsets are planned, so a reply is accepted only if it fills exactly its slot of the grid. */
|
||||
function assertChunkFillsItsSlot(
|
||||
function assertWindowFillsItsSlot(
|
||||
entry: MobileWebBundleAssetRead,
|
||||
offset: number,
|
||||
byteLength: number,
|
||||
eof: boolean,
|
||||
chunkBytes: number
|
||||
windowBytes: number
|
||||
): void {
|
||||
const { path, byteLength: declared } = entry
|
||||
const expected = Math.min(chunkBytes, declared - offset)
|
||||
if (byteLength === expected && eof === offset + chunkBytes >= declared) {
|
||||
const expected = Math.min(windowBytes, declared - offset)
|
||||
if (byteLength === expected && eof === offset + windowBytes >= declared) {
|
||||
return
|
||||
}
|
||||
const end = offset + byteLength
|
||||
if (byteLength > chunkBytes) {
|
||||
if (byteLength > windowBytes) {
|
||||
throw new MobileWebBundleFetchError(
|
||||
'chunk-oversize',
|
||||
`bundle chunk for ${path} at ${offset} is ${byteLength} bytes, over the host's ${chunkBytes}`
|
||||
`bundle window for ${path} at ${offset} is ${byteLength} bytes, over the ${windowBytes}-byte window`
|
||||
)
|
||||
}
|
||||
if (byteLength > expected || (byteLength > 0 && !eof && end >= declared)) {
|
||||
@@ -166,7 +177,7 @@ function assertChunkFillsItsSlot(
|
||||
'asset-short',
|
||||
eof
|
||||
? `bundle asset ${path} ended at ${end} of ${declared} declared bytes`
|
||||
: `bundle chunk for ${path} at ${offset} carried ${byteLength} of ${expected} bytes without ending the asset`
|
||||
: `bundle window for ${path} at ${offset} carried ${byteLength} of ${expected} bytes without ending the asset`
|
||||
)
|
||||
}
|
||||
|
||||
@@ -183,36 +194,30 @@ function verifyReassembledAsset(asset: AssetReassembly): Uint8Array {
|
||||
}
|
||||
|
||||
/**
|
||||
* Every chunk reply restates the build, path and offset it answers, and the whole asset's length and
|
||||
* Every window reply restates the build, path and offset it answers, and the whole asset's length and
|
||||
* hash. Checking all five is what makes a misrouted or stale reply a failure here instead of a
|
||||
* corrupt reassembly: a desktop that auto-updates mid-download answers a later chunk from a
|
||||
* corrupt reassembly: a desktop that auto-updates mid-download answers a later window from a
|
||||
* different build, and nothing else in the reply would say so.
|
||||
*/
|
||||
function assertChunkDescribesAsset(
|
||||
chunk: {
|
||||
buildId: string
|
||||
path: string
|
||||
offset: number
|
||||
assetByteLength: number
|
||||
sha256: string
|
||||
},
|
||||
function assertWindowDescribesAsset(
|
||||
window: MobileWebBundleWindowHeader,
|
||||
asset: MobileWebBundleAssetRead,
|
||||
buildId: string,
|
||||
offset: number
|
||||
): void {
|
||||
if (chunk.buildId !== buildId) {
|
||||
if (window.buildId !== buildId) {
|
||||
throw new MobileWebBundleFetchError(
|
||||
'build-changed-mid-fetch',
|
||||
`bundle build changed mid-fetch: asked ${buildId}, served ${chunk.buildId}`
|
||||
`bundle build changed mid-fetch: asked ${buildId}, served ${window.buildId}`
|
||||
)
|
||||
}
|
||||
if (chunk.path !== asset.path || chunk.offset !== offset) {
|
||||
if (window.path !== asset.path || window.offset !== offset) {
|
||||
throw new MobileWebBundleFetchError(
|
||||
'chunk-misrouted',
|
||||
`bundle chunk answered ${chunk.path} at ${chunk.offset}, not ${asset.path} at ${offset}`
|
||||
`bundle window answered ${window.path} at ${window.offset}, not ${asset.path} at ${offset}`
|
||||
)
|
||||
}
|
||||
if (chunk.sha256 !== asset.sha256 || chunk.assetByteLength !== asset.byteLength) {
|
||||
if (window.sha256 !== asset.sha256 || window.assetByteLength !== asset.byteLength) {
|
||||
throw new MobileWebBundleFetchError(
|
||||
'asset-entry-changed',
|
||||
`bundle asset ${asset.path} no longer matches the manifest entry`
|
||||
@@ -228,16 +233,6 @@ function throwIfCallerAborted(caller: AbortSignal | undefined): void {
|
||||
}
|
||||
}
|
||||
|
||||
/** Metro ships no Buffer; `atob` is the decoder the pairing and E2EE paths already run on Hermes. */
|
||||
function decodeBase64(value: string): Uint8Array {
|
||||
const binary = atob(value)
|
||||
const bytes = new Uint8Array(binary.length)
|
||||
for (let index = 0; index < binary.length; index += 1) {
|
||||
bytes[index] = binary.charCodeAt(index)
|
||||
}
|
||||
return bytes
|
||||
}
|
||||
|
||||
function toHex(bytes: Uint8Array): string {
|
||||
return Array.from(bytes, (byte) => byte.toString(16).padStart(2, '0')).join('')
|
||||
}
|
||||
|
||||
@@ -50,24 +50,16 @@ describe('decoding a bundle range', () => {
|
||||
).toBe('range-undecodable')
|
||||
})
|
||||
|
||||
// The spare byte in the bounded buffer is what turns an overlong body into a refusal.
|
||||
it('refuses a gzip body that inflates past the window, without growing the buffer', () => {
|
||||
expect(refusalOf(() => decodeMobileWebBundleRange(RANGE, GZIP, RAW.byteLength - 100))).toBe(
|
||||
'range-length-mismatch'
|
||||
// The spare byte in the bounded buffer is what lets the fetch's slot check see an overlong body.
|
||||
it('stops a gzip body that inflates past the window one byte over it', () => {
|
||||
expect(decodeMobileWebBundleRange(RANGE, GZIP, RAW.byteLength - 100).byteLength).toBe(
|
||||
RAW.byteLength - 99
|
||||
)
|
||||
})
|
||||
|
||||
it('refuses a gzip body that inflates short of the window', () => {
|
||||
expect(refusalOf(() => decodeMobileWebBundleRange(RANGE, GZIP, RAW.byteLength + 100))).toBe(
|
||||
'range-length-mismatch'
|
||||
it('hands back a gzip body that inflates short of the window as it is', () => {
|
||||
expect(decodeMobileWebBundleRange(RANGE, GZIP, RAW.byteLength + 100).byteLength).toBe(
|
||||
RAW.byteLength
|
||||
)
|
||||
})
|
||||
|
||||
it('refuses an identity body of the wrong length', () => {
|
||||
expect(
|
||||
refusalOf(() =>
|
||||
decodeMobileWebBundleRange({ ...RANGE, encoding: 'identity' }, RAW, RAW.byteLength + 1)
|
||||
)
|
||||
).toBe('range-length-mismatch')
|
||||
})
|
||||
})
|
||||
|
||||
@@ -5,28 +5,13 @@ import { MobileWebBundleFetchError } from './mobile-web-bundle-fetch-refusal'
|
||||
* The raw bytes of one range, from the `dataBase64` bytes the host sent under `encoding`.
|
||||
*
|
||||
* Inflated into a buffer one byte past the window: fflate fills a supplied `out` and never grows it,
|
||||
* so a gzip bomb costs at most that much memory (not time: inflating still runs to the body's end),
|
||||
* and a body that fills the spare byte is overlong.
|
||||
* so a gzip bomb is bounded in memory, not in the time spent inflating. A wrong length is left to the
|
||||
* fetch's slot checks, where a filled spare byte reads as overlong.
|
||||
*/
|
||||
export function decodeMobileWebBundleRange(
|
||||
range: { readonly path: string; readonly offset: number; readonly encoding: string },
|
||||
wire: Uint8Array,
|
||||
expectedLength: number
|
||||
): Uint8Array {
|
||||
const bytes = inflate(range, wire, expectedLength)
|
||||
if (bytes.byteLength !== expectedLength) {
|
||||
throw new MobileWebBundleFetchError(
|
||||
'range-length-mismatch',
|
||||
`bundle range of ${range.path} at ${range.offset} decoded to ${bytes.byteLength} bytes, not ${expectedLength}`
|
||||
)
|
||||
}
|
||||
return bytes
|
||||
}
|
||||
|
||||
function inflate(
|
||||
range: { readonly path: string; readonly offset: number; readonly encoding: string },
|
||||
wire: Uint8Array,
|
||||
expectedLength: number
|
||||
): Uint8Array {
|
||||
if (range.encoding === 'identity') {
|
||||
return wire
|
||||
|
||||
@@ -1,10 +1,18 @@
|
||||
import { describe, expect, it } from 'vitest'
|
||||
import {
|
||||
MOBILE_WEB_BUNDLE_CHUNK_BYTES,
|
||||
MOBILE_WEB_BUNDLE_RANGE_BYTES,
|
||||
MOBILE_WEB_BUNDLE_RANGE_MAX_DATA_BASE64_LENGTH,
|
||||
MOBILE_WEB_BUNDLE_RANGE_METHOD
|
||||
} from '../../../src/shared/mobile-web-bundle/bundle-rpc-contract'
|
||||
import { MOBILE_WEB_BUNDLE_MAX_ASSET_BYTES } from '../../../src/shared/mobile-web-bundle/manifest-contract'
|
||||
import { mobileWebBundleRangeRead } from './mobile-web-bundle-operations'
|
||||
import {
|
||||
computeMobileWebBundleId,
|
||||
MOBILE_WEB_BUNDLE_MAX_ASSET_BYTES
|
||||
} from '../../../src/shared/mobile-web-bundle/manifest-contract'
|
||||
import {
|
||||
mobileWebBundleManifestRead,
|
||||
mobileWebBundleRangeRead
|
||||
} from './mobile-web-bundle-operations'
|
||||
|
||||
function rangeReply(overrides: Record<string, unknown> = {}) {
|
||||
return {
|
||||
@@ -67,3 +75,46 @@ describe('mobile web bundle range reply reader', () => {
|
||||
expect(mobileWebBundleRangeRead.barrier).toBe('on-settle')
|
||||
})
|
||||
})
|
||||
|
||||
describe('the range grid on the manifest reply', () => {
|
||||
const assets = [
|
||||
{ path: 'index.html', sha256: 'b'.repeat(64), byteLength: 12, contentType: 'text/html' }
|
||||
]
|
||||
function manifestReply(extra: Record<string, unknown>) {
|
||||
return {
|
||||
manifest: {
|
||||
schemaVersion: 1,
|
||||
buildId: computeMobileWebBundleId(assets),
|
||||
minCompatibleRuntimeProtocolVersion: 2,
|
||||
runtimeProtocolVersion: 2,
|
||||
entrypoint: 'index.html',
|
||||
totalBytes: 12,
|
||||
assets
|
||||
},
|
||||
chunkBytes: MOBILE_WEB_BUNDLE_CHUNK_BYTES,
|
||||
...extra
|
||||
}
|
||||
}
|
||||
function rangeBytesOf(extra: Record<string, unknown>): unknown {
|
||||
const result = mobileWebBundleManifestRead.read(manifestReply(extra))
|
||||
expect(result.compatible).toBe(true)
|
||||
return result.compatible ? Object(result.value).rangeBytes : 'refused'
|
||||
}
|
||||
|
||||
it('reads the grid a range host names', () => {
|
||||
expect(rangeBytesOf({ rangeBytes: MOBILE_WEB_BUNDLE_RANGE_BYTES })).toBe(
|
||||
MOBILE_WEB_BUNDLE_RANGE_BYTES
|
||||
)
|
||||
})
|
||||
|
||||
it('reads a host that names none as absent', () => {
|
||||
expect(rangeBytesOf({})).toBeUndefined()
|
||||
})
|
||||
|
||||
// A later host with a wider grid must leave this build on chunks, not wall the whole bundle.
|
||||
it('reads a grid this build cannot page as absent rather than refusing the manifest', () => {
|
||||
for (const rangeBytes of [MOBILE_WEB_BUNDLE_RANGE_BYTES + 1, 0, 'wide']) {
|
||||
expect(rangeBytesOf({ rangeBytes })).toBeUndefined()
|
||||
}
|
||||
})
|
||||
})
|
||||
|
||||
@@ -1,6 +1,7 @@
|
||||
import { z } from 'zod'
|
||||
import {
|
||||
MOBILE_WEB_BUNDLE_CHUNK_BYTES,
|
||||
MOBILE_WEB_BUNDLE_RANGE_BYTES,
|
||||
MOBILE_WEB_BUNDLE_RANGE_MAX_DATA_BASE64_LENGTH
|
||||
} from '../../../src/shared/mobile-web-bundle/bundle-rpc-contract'
|
||||
import {
|
||||
@@ -113,7 +114,17 @@ export const MobileWebBundleManifestReadSchema = z
|
||||
* the constant because a larger value would overshoot `dataBase64` above. */
|
||||
export const MobileWebBundleManifestReplySchema = z.looseObject({
|
||||
manifest: MobileWebBundleManifestReadSchema,
|
||||
chunkBytes: z.number().int().positive().max(MOBILE_WEB_BUNDLE_CHUNK_BYTES)
|
||||
chunkBytes: z.number().int().positive().max(MOBILE_WEB_BUNDLE_CHUNK_BYTES),
|
||||
/** The range grid, named only by a host that serves `mobileWeb.bundle.range`. A value this build
|
||||
* cannot page within its `dataBase64` bound reads as absent, which keeps the fetch on chunks
|
||||
* rather than refusing the manifest. */
|
||||
rangeBytes: z
|
||||
.number()
|
||||
.int()
|
||||
.positive()
|
||||
.max(MOBILE_WEB_BUNDLE_RANGE_BYTES)
|
||||
.optional()
|
||||
.catch(undefined)
|
||||
})
|
||||
|
||||
/** Self-describing on purpose: `buildId`, `path` and `offset` are echoed so a reassembler cannot
|
||||
|
||||
@@ -1,72 +0,0 @@
|
||||
import { MOBILE_WEB_BUNDLE_RANGE_BYTES } from '../../../src/shared/mobile-web-bundle/bundle-range-rpc-contract'
|
||||
import { mobileWebBundleChunkRead, mobileWebBundleRangeRead } from './mobile-web-bundle-operations'
|
||||
import { decodeMobileWebBundleRange } from './mobile-web-bundle-range-decode'
|
||||
import type { MobileWebBundleReadMethod } from './mobile-web-bundle-read-method'
|
||||
import type { RpcClient } from './rpc-client'
|
||||
import { runRpcOperation } from './rpc-operation'
|
||||
|
||||
/** What either read method's reply restates about the window it answers. */
|
||||
export type MobileWebBundleWindowReply = {
|
||||
readonly buildId: string
|
||||
readonly path: string
|
||||
readonly offset: number
|
||||
readonly assetByteLength: number
|
||||
readonly sha256: string
|
||||
readonly eof: boolean
|
||||
readonly dataBase64: string
|
||||
/** Present on a range reply only; a chunk's body is always raw. */
|
||||
readonly encoding?: string
|
||||
}
|
||||
|
||||
/** The grid the fetch plans offsets on: the host's advertised chunk size, or the range ceiling. */
|
||||
export function mobileWebBundleWindowBytes(
|
||||
method: MobileWebBundleReadMethod,
|
||||
chunkBytes: number
|
||||
): number {
|
||||
return method === 'range' ? MOBILE_WEB_BUNDLE_RANGE_BYTES : chunkBytes
|
||||
}
|
||||
|
||||
export async function requestMobileWebBundleWindow(
|
||||
client: RpcClient,
|
||||
method: MobileWebBundleReadMethod,
|
||||
window: { buildId: string; path: string; offset: number; length: number }
|
||||
): Promise<MobileWebBundleWindowReply> {
|
||||
if (method === 'range') {
|
||||
return runRpcOperation(client, mobileWebBundleRangeRead, window)
|
||||
}
|
||||
return runRpcOperation(client, mobileWebBundleChunkRead, {
|
||||
buildId: window.buildId,
|
||||
path: window.path,
|
||||
offset: window.offset
|
||||
})
|
||||
}
|
||||
|
||||
/**
|
||||
* The raw bytes of a window. A range is decoded to exactly `expectedLength` or refused; a chunk is
|
||||
* returned as sent, because its length checks against the chunk grid live with the fetch.
|
||||
*/
|
||||
export function decodeMobileWebBundleWindow(
|
||||
method: MobileWebBundleReadMethod,
|
||||
reply: MobileWebBundleWindowReply,
|
||||
expectedLength: number
|
||||
): Uint8Array {
|
||||
const wire = decodeBase64(reply.dataBase64)
|
||||
if (method === 'chunk') {
|
||||
return wire
|
||||
}
|
||||
return decodeMobileWebBundleRange(
|
||||
{ path: reply.path, offset: reply.offset, encoding: reply.encoding ?? '' },
|
||||
wire,
|
||||
expectedLength
|
||||
)
|
||||
}
|
||||
|
||||
/** Metro ships no Buffer; `atob` is the decoder the pairing and E2EE paths already run on Hermes. */
|
||||
function decodeBase64(value: string): Uint8Array {
|
||||
const binary = atob(value)
|
||||
const bytes = new Uint8Array(binary.length)
|
||||
for (let index = 0; index < binary.length; index += 1) {
|
||||
bytes[index] = binary.charCodeAt(index)
|
||||
}
|
||||
return bytes
|
||||
}
|
||||
@@ -0,0 +1,81 @@
|
||||
import { mobileWebBundleChunkRead, mobileWebBundleRangeRead } from './mobile-web-bundle-operations'
|
||||
import { decodeMobileWebBundleRange } from './mobile-web-bundle-range-decode'
|
||||
import type { MobileWebBundleManifestReply } from './mobile-web-bundle-reply-schemas'
|
||||
import type { RpcClient } from './rpc-client'
|
||||
import { runRpcOperation } from './rpc-operation'
|
||||
|
||||
/** What a chunk or range reply restates about the window it answers. */
|
||||
export type MobileWebBundleWindowHeader = {
|
||||
readonly buildId: string
|
||||
readonly path: string
|
||||
readonly offset: number
|
||||
readonly assetByteLength: number
|
||||
readonly sha256: string
|
||||
readonly eof: boolean
|
||||
}
|
||||
|
||||
export type MobileWebBundleWindowReply = {
|
||||
readonly header: MobileWebBundleWindowHeader
|
||||
/** Decoded on demand, so a reply the fetch discards or refuses by its header is never inflated. */
|
||||
readonly bytes: () => Uint8Array
|
||||
}
|
||||
|
||||
/** One grid and one read method, fixed for a whole fetch by what the manifest reply named. */
|
||||
export type MobileWebBundleWindowReader = {
|
||||
readonly windowBytes: number
|
||||
read(
|
||||
client: RpcClient,
|
||||
window: { buildId: string; path: string; offset: number }
|
||||
): Promise<MobileWebBundleWindowReply>
|
||||
}
|
||||
|
||||
/** Ranges when the host named a range grid, chunks otherwise: a host that predates the range
|
||||
* method names none, and every bundle host serves chunks. */
|
||||
export function mobileWebBundleWindowReader(
|
||||
opened: Pick<MobileWebBundleManifestReply, 'chunkBytes' | 'rangeBytes'>
|
||||
): MobileWebBundleWindowReader {
|
||||
const { rangeBytes } = opened
|
||||
if (rangeBytes === undefined) {
|
||||
return {
|
||||
windowBytes: opened.chunkBytes,
|
||||
read: async (client, window) => {
|
||||
const { dataBase64, ...header } = await runRpcOperation(
|
||||
client,
|
||||
mobileWebBundleChunkRead,
|
||||
window
|
||||
)
|
||||
return { header, bytes: () => decodeBase64(dataBase64) }
|
||||
}
|
||||
}
|
||||
}
|
||||
return {
|
||||
windowBytes: rangeBytes,
|
||||
read: async (client, window) => {
|
||||
const { dataBase64, encoding, ...header } = await runRpcOperation(
|
||||
client,
|
||||
mobileWebBundleRangeRead,
|
||||
window
|
||||
)
|
||||
const expected = Math.max(0, Math.min(rangeBytes, header.assetByteLength - header.offset))
|
||||
return {
|
||||
header,
|
||||
bytes: () =>
|
||||
decodeMobileWebBundleRange(
|
||||
{ path: header.path, offset: header.offset, encoding },
|
||||
decodeBase64(dataBase64),
|
||||
expected
|
||||
)
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/** Metro ships no Buffer; `atob` is the decoder the pairing and E2EE paths already run on Hermes. */
|
||||
function decodeBase64(value: string): Uint8Array {
|
||||
const binary = atob(value)
|
||||
const bytes = new Uint8Array(binary.length)
|
||||
for (let index = 0; index < binary.length; index += 1) {
|
||||
bytes[index] = binary.charCodeAt(index)
|
||||
}
|
||||
return bytes
|
||||
}
|
||||
Reference in New Issue
Block a user