mirror of
https://github.com/stablyai/orca.git
synced 2026-10-30 16:47:34 +00:00
feat(mobile): download only the page assets the cached generation lacks (#31769)
* feat(mobile): download only the page assets the cached generation lacks Before paging the bundle, the phone matches the new manifest against the host's active generation by content (sha256 and length). Matches are read from disk and hashed again while the changed assets come over the wire; one whose bytes no longer hash to the manifest is downloaded instead. Progress counts only the bytes that cross the wire. * test(mobile): put the older-build manifest note back on its describe * perf(mobile): copy unchanged page assets natively instead of reading and hashing them The reuse pass read every unchanged asset into JS and hashed it again, which kept the JS thread busy for most of an update. The cached generation was hash-verified when it was written, is never written again, and is what the page already opens as is, so matching assets are now copied natively into the staging tree and checked by size. A copy that fails or comes up short is downloaded instead; stage refuses a reused copy that is no longer whole. Path rules move to generation-asset-path.ts and the copying to generation-reuse.ts. * fix(mobile): copy reused page assets at stage, on the store's queue Copies made when the manifest was read sat in tmp for the whole download, where a second shell mount's sweep, or a second download of the same build, could take them away and fail the stage. The fetch now only checks that each reusable file is still whole (a native stat) and leaves it to the stage, which copies it from the generation active at that moment and checks its size. Copy destinations must not exist, as on both platforms.
This commit is contained in:
@@ -119,6 +119,7 @@ function fetchedBundle(): MobileWebBundleFetchResult {
|
||||
]
|
||||
},
|
||||
assets: new Map([['index.html', new Uint8Array([1, 2, 3])]]),
|
||||
reused: [],
|
||||
totalBytes: 3,
|
||||
elapsedMs: 12
|
||||
}
|
||||
|
||||
@@ -0,0 +1,20 @@
|
||||
/** The activation's manifest: a generation directory is an activation only once this is in it. */
|
||||
export const MANIFEST_FILE_NAME = 'manifest.json'
|
||||
|
||||
/** The manifest schema bans traversal already, but this is the last code between a manifest and a
|
||||
* write, and both manifest names are the store's own rather than an asset's to take: the pending
|
||||
* one would be read back as an activation. Folded, because APFS and NTFS are case-insensitive and
|
||||
* `Manifest.JSON` would land on the same file. */
|
||||
export function requireStorablePath(path: string): string {
|
||||
const segments = path.split('/')
|
||||
const folded = path.toLowerCase()
|
||||
const storable =
|
||||
path.length > 0 &&
|
||||
folded !== MANIFEST_FILE_NAME &&
|
||||
!path.includes('\\') &&
|
||||
segments.every((segment) => segment !== '' && segment !== '.' && segment !== '..')
|
||||
if (!storable) {
|
||||
throw new Error(`generation store refuses to stage the asset path ${path}`)
|
||||
}
|
||||
return path
|
||||
}
|
||||
@@ -109,6 +109,23 @@ export function createFakeGenerationFileSystem(): FakeGenerationFileSystem {
|
||||
async fileExists(target) {
|
||||
return nodes.get(target)?.kind === 'file'
|
||||
},
|
||||
async fileSize(target) {
|
||||
const node = nodes.get(target)
|
||||
return node?.kind === 'file' ? node.bytes.byteLength : null
|
||||
},
|
||||
async copyFile(fromUri, toUri) {
|
||||
if (failReadAt !== null && fromUri === uri(failReadAt)) {
|
||||
throw new Error('simulated unreadable file')
|
||||
}
|
||||
const node = nodes.get(fromUri)
|
||||
if (node?.kind !== 'file') {
|
||||
throw new Error(`fake filesystem has no file at ${fromUri}`)
|
||||
}
|
||||
if (nodes.has(toUri)) {
|
||||
throw new Error(`fake filesystem refuses to copy onto ${toUri}`)
|
||||
}
|
||||
write(toUri, node.bytes)
|
||||
},
|
||||
async delete(target) {
|
||||
for (const key of Array.from(nodes.keys())) {
|
||||
if (key === target || key.startsWith(`${target}/`)) {
|
||||
|
||||
@@ -0,0 +1,67 @@
|
||||
import type { MobileWebBundleReusableAssets } from '../transport/mobile-web-bundle-fetch'
|
||||
import type { MobileWebBundleAssetRead } from '../transport/mobile-web-bundle-reply-schemas'
|
||||
import { requireStorablePath } from './generation-asset-path'
|
||||
import { joinUri } from './generation-cache-uri'
|
||||
import type { ActiveGeneration } from './generation-store'
|
||||
import type { GenerationFileSystem } from './generation-store-file-system'
|
||||
|
||||
/**
|
||||
* Copies, never reads: the active generation was hash-verified when it was written and is never
|
||||
* written again (only renamed into place and deleted), and the page opens it as is on every
|
||||
* launch, so a whole copy is the same bytes the page already trusts. Matched by content, so an
|
||||
* asset the next build renamed is still found.
|
||||
*/
|
||||
function cachedFileOf(
|
||||
active: ActiveGeneration | null,
|
||||
asset: MobileWebBundleAssetRead
|
||||
): string | null {
|
||||
const cached = active?.manifest.assets.find((entry) => entry.sha256 === asset.sha256)
|
||||
return active === null || cached?.byteLength !== asset.byteLength
|
||||
? null
|
||||
: joinUri(active.directory, requireStorablePath(cached.path))
|
||||
}
|
||||
|
||||
/** What a download may leave to the stage: matched now, and still whole on disk when checked. */
|
||||
export function reusableAssetsOf(
|
||||
fs: GenerationFileSystem,
|
||||
active: ActiveGeneration | null
|
||||
): MobileWebBundleReusableAssets {
|
||||
return {
|
||||
has: (asset) => {
|
||||
try {
|
||||
return cachedFileOf(active, asset) !== null
|
||||
} catch {
|
||||
return false
|
||||
}
|
||||
},
|
||||
check: async (asset) => {
|
||||
try {
|
||||
const source = cachedFileOf(active, asset)
|
||||
return source !== null && (await fs.fileSize(source)) === asset.byteLength
|
||||
} catch {
|
||||
return false
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Copies a reused asset out of the generation active now, which may not be the one the download
|
||||
* checked: a commit in between replaced it. Throws when that generation lacks the asset or the
|
||||
* copy is not whole, which fails the stage; the retry checks against the new generation.
|
||||
*/
|
||||
export async function copyCachedAsset(
|
||||
fs: GenerationFileSystem,
|
||||
active: ActiveGeneration | null,
|
||||
asset: MobileWebBundleAssetRead,
|
||||
target: string
|
||||
): Promise<void> {
|
||||
const source = cachedFileOf(active, asset)
|
||||
if (source === null) {
|
||||
throw new Error(`the cached generation no longer holds the reused asset ${asset.path}`)
|
||||
}
|
||||
await fs.copyFile(source, target)
|
||||
if ((await fs.fileSize(target)) !== asset.byteLength) {
|
||||
throw new Error(`the reused asset ${asset.path} did not copy whole`)
|
||||
}
|
||||
}
|
||||
@@ -27,6 +27,11 @@ export type GenerationFileSystem = {
|
||||
* be read" lead the store to opposite decisions about deleting the cache. */
|
||||
readText(uri: string): Promise<string | null>
|
||||
fileExists(uri: string): Promise<boolean>
|
||||
/** Null when the file is missing. */
|
||||
fileSize(uri: string): Promise<number | null>
|
||||
/** Copies a file natively, creating the destination's directories. The destination must not
|
||||
* exist, as on both platforms. Throws when the source is gone or unreadable. */
|
||||
copyFile(fromUri: string, toUri: string): Promise<void>
|
||||
/** Recursive, and a no-op when the path is missing. */
|
||||
delete(uri: string): Promise<void>
|
||||
/** Renames a directory. The destination must not exist: expo moves a directory *into* an existing
|
||||
@@ -85,6 +90,16 @@ export function createExpoGenerationFileSystem(): GenerationFileSystem {
|
||||
async fileExists(uri) {
|
||||
return new File(uri).exists
|
||||
},
|
||||
async fileSize(uri) {
|
||||
const file = new File(uri)
|
||||
return file.exists ? file.size : null
|
||||
},
|
||||
async copyFile(fromUri, toUri) {
|
||||
const target = new File(toUri)
|
||||
// iOS copies only into an existing directory.
|
||||
target.parentDirectory.create({ intermediates: true, idempotent: true })
|
||||
new File(fromUri).copy(target)
|
||||
},
|
||||
async delete(uri) {
|
||||
const directory = new Directory(uri)
|
||||
if (directory.exists) {
|
||||
|
||||
@@ -48,6 +48,7 @@ function bundle(): MobileWebBundleFetchResult {
|
||||
assets
|
||||
},
|
||||
assets: new Map([['index.html', new Uint8Array(4).fill(7)]]),
|
||||
reused: [],
|
||||
totalBytes: 4,
|
||||
elapsedMs: 1
|
||||
}
|
||||
|
||||
@@ -41,6 +41,7 @@ function buildResult(options: {
|
||||
assets:
|
||||
options.bytes ??
|
||||
new Map(assets.map((asset) => [asset.path, new Uint8Array(asset.byteLength).fill(7)])),
|
||||
reused: [],
|
||||
totalBytes,
|
||||
elapsedMs: 1
|
||||
}
|
||||
@@ -570,6 +571,67 @@ describe('generation store', () => {
|
||||
})
|
||||
})
|
||||
|
||||
describe('reusable assets for the next download', () => {
|
||||
/** The next build: a longer page, and the cached script under a new name. */
|
||||
const NEXT = buildResult({
|
||||
assets: [
|
||||
{ path: 'index.html', byteLength: 8 },
|
||||
{ path: 'assets/app-2.js', byteLength: 2 }
|
||||
]
|
||||
})
|
||||
const NEXT_BUILD = NEXT.manifest.buildId
|
||||
const [PAGE, SCRIPT] = NEXT.manifest.assets
|
||||
const DOWNLOADED = new Map([['index.html', new Uint8Array(8).fill(9)]])
|
||||
const LEFT_TO_STAGE = { ...NEXT, assets: DOWNLOADED, reused: ['assets/app-2.js'] }
|
||||
|
||||
it('matches cached assets by content, and checks them without reading them', async () => {
|
||||
const fs = createFakeFileSystem()
|
||||
const store = createGenerationStore({ fileSystem: fs })
|
||||
expect((await store.openReusableAssets(HOST)).has(SCRIPT!)).toBe(false)
|
||||
await activate(store, HOST)
|
||||
|
||||
const reusable = await store.openReusableAssets(HOST)
|
||||
|
||||
// Content, not path: the script was renamed, and the page kept its name but changed length.
|
||||
expect(reusable.has(SCRIPT!)).toBe(true)
|
||||
expect(reusable.has(PAGE!)).toBe(false)
|
||||
expect(await reusable.check(SCRIPT!)).toBe(true)
|
||||
fs.seed(`${HOST}/generations/${BUILD}/assets/app.js`, {
|
||||
kind: 'file',
|
||||
bytes: new Uint8Array(1)
|
||||
})
|
||||
expect(await reusable.check(SCRIPT!)).toBe(false)
|
||||
})
|
||||
|
||||
it('copies the reused assets beside the downloaded ones and commits the whole build', async () => {
|
||||
const fs = createFakeFileSystem()
|
||||
const store = createGenerationStore({ fileSystem: fs })
|
||||
await activate(store, HOST)
|
||||
|
||||
await activate(store, HOST, LEFT_TO_STAGE)
|
||||
|
||||
const generation = `${HOST}/generations/${NEXT_BUILD}`
|
||||
expect(fs.bytes(`${generation}/index.html`)).toEqual(new Uint8Array(8).fill(9))
|
||||
expect(fs.bytes(`${generation}/assets/app-2.js`)).toEqual(new Uint8Array(2).fill(7))
|
||||
expect((await store.readActiveGeneration(HOST))?.buildId).toBe(NEXT_BUILD)
|
||||
})
|
||||
|
||||
it('refuses to stage a reused asset the active generation no longer holds', async () => {
|
||||
const fs = createFakeFileSystem()
|
||||
const store = createGenerationStore({ fileSystem: fs })
|
||||
await activate(store, HOST)
|
||||
fs.failReadsAt(`${HOST}/generations/${BUILD}/assets/app.js`)
|
||||
|
||||
await expect(store.stageGeneration(HOST, LEFT_TO_STAGE)).rejects.toThrow('unreadable')
|
||||
await store.deleteHostCache(HOST)
|
||||
fs.failReadsAt(null)
|
||||
await expect(store.stageGeneration(HOST, LEFT_TO_STAGE)).rejects.toThrow(
|
||||
'no longer holds the reused asset assets/app-2.js'
|
||||
)
|
||||
expect(fs.paths().some((path) => path.includes(`tmp/${NEXT_BUILD}`))).toBe(false)
|
||||
})
|
||||
})
|
||||
|
||||
/** An older build rewrote the manifest through `manifest-next.json`; an interrupted rewrite can
|
||||
* have left that file behind, and this build never reads it. */
|
||||
describe('a pending manifest left by an older build', () => {
|
||||
|
||||
@@ -3,8 +3,13 @@ import {
|
||||
MobileWebBundleManifestReadSchema,
|
||||
type MobileWebBundleManifestRead
|
||||
} from '../transport/mobile-web-bundle-reply-schemas'
|
||||
import type { MobileWebBundleFetchResult } from '../transport/mobile-web-bundle-fetch'
|
||||
import type {
|
||||
MobileWebBundleFetchResult,
|
||||
MobileWebBundleReusableAssets
|
||||
} from '../transport/mobile-web-bundle-fetch'
|
||||
import { joinUri } from './generation-cache-uri'
|
||||
import { MANIFEST_FILE_NAME, requireStorablePath } from './generation-asset-path'
|
||||
import { copyCachedAsset, reusableAssetsOf } from './generation-reuse'
|
||||
import type { GenerationDirectoryEntry, GenerationFileSystem } from './generation-store-file-system'
|
||||
import { isHostCacheKey } from './host-cache-key'
|
||||
import type { MobileWebShellUpdateFailure } from './mobile-web-shell-update-failure'
|
||||
@@ -17,8 +22,6 @@ import {
|
||||
const GENERATIONS_DIRECTORY_NAME = 'generations'
|
||||
const STAGING_DIRECTORY_NAME = 'tmp'
|
||||
const HOST_INDEX_FILE_NAME = 'hosts.json'
|
||||
/** The activation's manifest: a generation directory is an activation only once this is in it. */
|
||||
const MANIFEST_FILE_NAME = 'manifest.json'
|
||||
|
||||
/** Least recently used host evicted past this. */
|
||||
export const MAX_CACHED_HOSTS = 6
|
||||
@@ -39,6 +42,9 @@ export type StagedGeneration = {
|
||||
|
||||
export type GenerationStore = {
|
||||
readActiveGeneration(hostKey: string): Promise<ActiveGeneration | null>
|
||||
/** The host's active generation, by content, for a download to leave unchanged assets to the
|
||||
* stage, which copies them from it. */
|
||||
openReusableAssets(hostKey: string): Promise<MobileWebBundleReusableAssets>
|
||||
stageGeneration(hostKey: string, result: MobileWebBundleFetchResult): Promise<StagedGeneration>
|
||||
commitGeneration(staged: StagedGeneration): Promise<ActiveGeneration>
|
||||
abortStagedGeneration(staged: StagedGeneration): Promise<void>
|
||||
@@ -171,16 +177,21 @@ export function createGenerationStore(options: {
|
||||
): Promise<StagedGeneration> {
|
||||
const manifest = result.manifest
|
||||
const directory = joinUri(stagingRoot(hostKey), manifest.buildId)
|
||||
const reused = new Set(result.reused)
|
||||
const assets = manifest.assets.map((asset) => ({
|
||||
asset,
|
||||
uri: joinUri(directory, requireStorablePath(asset.path)),
|
||||
bytes: requireExactBytes(result.assets.get(asset.path), asset)
|
||||
bytes: reused.has(asset.path) ? null : requireExactBytes(result.assets.get(asset.path), asset)
|
||||
}))
|
||||
// Here rather than during the download, so a sweep or a second download of this build never
|
||||
// takes a copy out from under it: the stage runs on the queue they do.
|
||||
const active = reused.size > 0 ? await readActive(hostKey) : null
|
||||
// Residue from an earlier attempt is dropped rather than written over: a half-written tree
|
||||
// plus a fresh write is not a generation either side verified.
|
||||
await fs.delete(directory)
|
||||
try {
|
||||
for (const asset of assets) {
|
||||
await fs.writeBytes(asset.uri, asset.bytes)
|
||||
for (const { asset, uri, bytes } of assets) {
|
||||
await (bytes === null ? copyCachedAsset(fs, active, asset, uri) : fs.writeBytes(uri, bytes))
|
||||
}
|
||||
// Last, always: a tree without it never reads back as an activation, which is what makes an
|
||||
// interrupted write recoverable rather than ambiguous.
|
||||
@@ -260,6 +271,10 @@ export function createGenerationStore(options: {
|
||||
return active
|
||||
}
|
||||
|
||||
async function openReusable(hostKey: string): Promise<MobileWebBundleReusableAssets> {
|
||||
return reusableAssetsOf(fs, await readActive(hostKey))
|
||||
}
|
||||
|
||||
async function sweep(): Promise<void> {
|
||||
// Every host's `tmp`, not just the one being opened: an interrupted download must not survive a
|
||||
// restart, and it may belong to a host this launch never selects.
|
||||
@@ -289,6 +304,7 @@ export function createGenerationStore(options: {
|
||||
|
||||
return {
|
||||
readActiveGeneration: (hostKey) => serialize(() => openActive(hostKey)),
|
||||
openReusableAssets: (hostKey) => serialize(() => openReusable(hostKey)),
|
||||
stageGeneration: (hostKey, result) => serialize(() => stage(hostKey, result)),
|
||||
commitGeneration: (staged) => serialize(() => commit(staged)),
|
||||
abortStagedGeneration: (staged) =>
|
||||
@@ -329,24 +345,6 @@ function requireHostKey(hostKey: string): string {
|
||||
return hostKey
|
||||
}
|
||||
|
||||
/** The manifest schema bans traversal already, but this is the last code between a manifest and a
|
||||
* write, and both manifest names are the store's own rather than an asset's to take: the pending
|
||||
* one would be read back as an activation. Folded, because APFS and NTFS are case-insensitive and
|
||||
* `Manifest.JSON` would land on the same file. */
|
||||
function requireStorablePath(path: string): string {
|
||||
const segments = path.split('/')
|
||||
const folded = path.toLowerCase()
|
||||
const storable =
|
||||
path.length > 0 &&
|
||||
folded !== MANIFEST_FILE_NAME &&
|
||||
!path.includes('\\') &&
|
||||
segments.every((segment) => segment !== '' && segment !== '.' && segment !== '..')
|
||||
if (!storable) {
|
||||
throw new Error(`generation store refuses to stage the asset path ${path}`)
|
||||
}
|
||||
return path
|
||||
}
|
||||
|
||||
function requireExactBytes(
|
||||
bytes: Uint8Array | undefined,
|
||||
asset: { path: string; byteLength: number }
|
||||
|
||||
@@ -87,6 +87,7 @@ const store: GenerationStore = {
|
||||
throw new Error('not reached')
|
||||
},
|
||||
abortStagedGeneration: async () => undefined,
|
||||
openReusableAssets: async () => ({ has: () => false, check: async () => false }),
|
||||
sweepStagedGenerations: async () => undefined,
|
||||
deleteHostCache: async () => undefined,
|
||||
recordUpdateFailure: async () => undefined,
|
||||
|
||||
@@ -144,6 +144,7 @@ function fetchResultFor(manifest: MobileWebBundleManifestRead): MobileWebBundleF
|
||||
assets: new Map(
|
||||
manifest.assets.map((asset) => [asset.path, new Uint8Array(asset.byteLength).fill(7)])
|
||||
),
|
||||
reused: [],
|
||||
totalBytes: TOTAL_BYTES,
|
||||
elapsedMs: 1
|
||||
}
|
||||
|
||||
@@ -126,6 +126,7 @@ async function cacheGeneration(
|
||||
const fetched: MobileWebBundleFetchResult = {
|
||||
manifest,
|
||||
assets: new Map(ASSETS.map((asset) => [asset.path, new Uint8Array(asset.byteLength).fill(7)])),
|
||||
reused: [],
|
||||
totalBytes: TOTAL_BYTES,
|
||||
elapsedMs: 1
|
||||
}
|
||||
|
||||
@@ -19,7 +19,11 @@ vi.mock('../transport/mobile-web-bundle-fetch', () => ({
|
||||
import { MobileWebBundleFetchError } from '../transport/mobile-web-bundle-fetch-refusal'
|
||||
import { markRpcDeliveryUnknown } from '../transport/rpc-delivery-ambiguity'
|
||||
import type { RpcClient } from '../transport/rpc-client'
|
||||
import type { GenerationStore } from './generation-store'
|
||||
import { computeMobileWebBundleId } from '../../../src/shared/mobile-web-bundle/manifest-contract'
|
||||
import type { MobileWebBundleFetchResult } from '../transport/mobile-web-bundle-fetch'
|
||||
import { createGenerationStore, type GenerationStore } from './generation-store'
|
||||
import { createFakeGenerationFileSystem } from './generation-file-system-fake'
|
||||
import { deriveHostCacheKey } from './host-cache-key'
|
||||
import { download, openCache, readManifest } from './mobile-web-shell-session-effects'
|
||||
|
||||
// The mocked operations never touch the client; they only need one to exist.
|
||||
@@ -60,6 +64,7 @@ function storeThat(fail: 'stage' | 'commit' | null): GenerationStore {
|
||||
? reject
|
||||
: async () => ({ buildId: 'c', directory: 'd', manifest: MANIFEST }),
|
||||
abortStagedGeneration: async () => undefined,
|
||||
openReusableAssets: async () => ({ has: () => false, check: async () => false }),
|
||||
sweepStagedGenerations: async () => undefined,
|
||||
deleteHostCache: async () => undefined,
|
||||
recordUpdateFailure: async () => undefined,
|
||||
@@ -189,3 +194,95 @@ describe('the download', () => {
|
||||
expect(await runDownload(CLIENT, storeThat(null))).toEqual([])
|
||||
})
|
||||
})
|
||||
|
||||
/**
|
||||
* Against the real store: the copies a reused asset needs are made by the stage, on the store's
|
||||
* queue, so what other mounts do to the staging tree mid-download cannot take them away.
|
||||
*/
|
||||
describe('a download that reuses the cached generation', () => {
|
||||
const HOST = deriveHostCacheKey('host-a')
|
||||
|
||||
function bundle(pageBytes: number): MobileWebBundleFetchResult {
|
||||
const assets = [
|
||||
{
|
||||
path: 'index.html',
|
||||
sha256: 'a'.repeat(64),
|
||||
byteLength: pageBytes,
|
||||
contentType: 'text/html'
|
||||
},
|
||||
{ path: 'app.js', sha256: 'b'.repeat(64), byteLength: 2, contentType: 'text/javascript' }
|
||||
]
|
||||
return {
|
||||
manifest: { ...MANIFEST, buildId: computeMobileWebBundleId(assets), totalBytes: 0, assets },
|
||||
assets: new Map(assets.map((asset) => [asset.path, new Uint8Array(asset.byteLength)])),
|
||||
reused: [],
|
||||
totalBytes: 0,
|
||||
elapsedMs: 1
|
||||
}
|
||||
}
|
||||
const CACHED = bundle(4)
|
||||
// The page changed; the script is left to the stage to copy.
|
||||
const UPDATE = {
|
||||
...bundle(8),
|
||||
assets: new Map([['index.html', new Uint8Array(8)]]),
|
||||
reused: ['app.js']
|
||||
}
|
||||
|
||||
async function cachedStore() {
|
||||
const fileSystem = createFakeGenerationFileSystem()
|
||||
const store = createGenerationStore({ fileSystem })
|
||||
await store.commitGeneration(await store.stageGeneration(HOST, CACHED))
|
||||
return { fileSystem, store }
|
||||
}
|
||||
|
||||
function runFor(store: GenerationStore) {
|
||||
const { events, send } = collect()
|
||||
const done = download({
|
||||
client: CLIENT,
|
||||
store,
|
||||
hostKey: HOST,
|
||||
flow: 3,
|
||||
runtime: {
|
||||
createStore: () => store,
|
||||
mintSessionId: () => 's',
|
||||
now: () => 0,
|
||||
setTimer: () => () => {}
|
||||
},
|
||||
startedAt: 0,
|
||||
downloads: new Set(),
|
||||
send
|
||||
})
|
||||
return done.then(() => events)
|
||||
}
|
||||
|
||||
it('commits whole when another mount sweeps the staging trees mid-download', async () => {
|
||||
const { fileSystem, store } = await cachedStore()
|
||||
doubles.fetch = async () => {
|
||||
fileSystem.seed(`${HOST}/tmp/${UPDATE.manifest.buildId}/app.js`, {
|
||||
kind: 'file',
|
||||
bytes: new Uint8Array(1)
|
||||
})
|
||||
await store.sweepStagedGenerations()
|
||||
return UPDATE
|
||||
}
|
||||
|
||||
const events = await runFor(store)
|
||||
|
||||
expect(events.filter((event) => event.type === 'download-failed')).toEqual([])
|
||||
const active = await store.readActiveGeneration(HOST)
|
||||
expect(active?.buildId).toBe(UPDATE.manifest.buildId)
|
||||
expect(fileSystem.bytes(`${HOST}/generations/${UPDATE.manifest.buildId}/app.js`)).toEqual(
|
||||
new Uint8Array(2)
|
||||
)
|
||||
})
|
||||
|
||||
it('commits once when two downloads of the same build finish together', async () => {
|
||||
const { store } = await cachedStore()
|
||||
doubles.fetch = async () => UPDATE
|
||||
|
||||
const [first, second] = await Promise.all([runFor(store), runFor(store)])
|
||||
|
||||
expect([...first, ...second].filter((event) => event.type === 'download-failed')).toEqual([])
|
||||
expect((await store.readActiveGeneration(HOST))?.buildId).toBe(UPDATE.manifest.buildId)
|
||||
})
|
||||
})
|
||||
|
||||
@@ -110,10 +110,13 @@ export async function download(args: {
|
||||
// Past the fetch, a throw is the store's: the bytes had already verified.
|
||||
let fetchedWhole = false
|
||||
try {
|
||||
// A cache that cannot be opened is one with nothing to reuse, not a failed download.
|
||||
const reusable = await store.openReusableAssets(hostKey).catch(() => undefined)
|
||||
const fetched = await fetchMobileWebBundle({
|
||||
client,
|
||||
signal: controller.signal,
|
||||
onProgress: (progress) => send({ type: 'fetch-progress', flow, ...progress })
|
||||
onProgress: (progress) => send({ type: 'fetch-progress', flow, ...progress }),
|
||||
reusable
|
||||
})
|
||||
// The bytes are in; the session they were for may not be. The fetch throws on an abort it sees,
|
||||
// but an abort landing between its last read and this line would otherwise still write a
|
||||
|
||||
@@ -156,6 +156,7 @@ function createFakeStore(): {
|
||||
abortStagedGeneration: async () => {
|
||||
aborted += 1
|
||||
},
|
||||
openReusableAssets: async () => ({ has: () => false, check: async () => false }),
|
||||
sweepStagedGenerations: async () => undefined,
|
||||
deleteHostCache: async () => undefined,
|
||||
recordUpdateFailure: async () => undefined,
|
||||
@@ -326,6 +327,7 @@ describe('the hybrid shell runner', () => {
|
||||
inFlight.settle({
|
||||
manifest: doubles.manifest,
|
||||
assets: new Map(),
|
||||
reused: [],
|
||||
totalBytes: 2048,
|
||||
elapsedMs: 1
|
||||
})
|
||||
@@ -364,6 +366,7 @@ describe('the hybrid shell runner', () => {
|
||||
inFlight.settle({
|
||||
manifest: doubles.manifest,
|
||||
assets: new Map(),
|
||||
reused: [],
|
||||
totalBytes: 2048,
|
||||
elapsedMs: 1
|
||||
})
|
||||
@@ -420,7 +423,13 @@ describe('the hybrid shell runner', () => {
|
||||
mounted.retry()
|
||||
})
|
||||
expect(first.signal.aborted).toBe(true)
|
||||
first.settle({ manifest: doubles.manifest, assets: new Map(), totalBytes: 2048, elapsedMs: 1 })
|
||||
first.settle({
|
||||
manifest: doubles.manifest,
|
||||
assets: new Map(),
|
||||
reused: [],
|
||||
totalBytes: 2048,
|
||||
elapsedMs: 1
|
||||
})
|
||||
await flush()
|
||||
expect(fake.staged()).toBe(0)
|
||||
await act(async () => {
|
||||
|
||||
@@ -1,7 +1,11 @@
|
||||
import { sha256 } from '@noble/hashes/sha256'
|
||||
import { describe, expect, it, vi } from 'vitest'
|
||||
import { computeMobileWebBundleId } from '../../../src/shared/mobile-web-bundle/manifest-contract'
|
||||
import { fetchMobileWebBundle } from './mobile-web-bundle-fetch'
|
||||
import {
|
||||
fetchMobileWebBundle,
|
||||
type MobileWebBundleFetchProgress,
|
||||
type MobileWebBundleReusableAssets
|
||||
} from './mobile-web-bundle-fetch'
|
||||
import { MobileWebBundleFetchError } from './mobile-web-bundle-fetch-refusal'
|
||||
import { readMobileWebBundleErrorCode } from './mobile-web-bundle-operations'
|
||||
import type { RpcClient } from './rpc-client'
|
||||
@@ -562,4 +566,83 @@ describe('fetchMobileWebBundle', () => {
|
||||
expect(readMobileWebBundleErrorCode(error)).toBe('mobile_web_bundle_unavailable')
|
||||
expect(error).not.toBeInstanceOf(MobileWebBundleFetchError)
|
||||
})
|
||||
|
||||
describe('with a cached generation to reuse', () => {
|
||||
/** A cache holding these files by content, each still whole on disk unless `whole` says not. */
|
||||
function cacheOf(files: Record<string, string>, checked: string[], whole = true) {
|
||||
const hashes = new Map(
|
||||
Object.values(files).map((text) => {
|
||||
const content = bytesOf(text)
|
||||
return [toHex(sha256(content)), content.byteLength] as const
|
||||
})
|
||||
)
|
||||
const reusable: MobileWebBundleReusableAssets = {
|
||||
has: (asset) => hashes.get(asset.sha256) === asset.byteLength,
|
||||
check: async (asset) => {
|
||||
checked.push(asset.path)
|
||||
return whole
|
||||
}
|
||||
}
|
||||
return reusable
|
||||
}
|
||||
|
||||
function chunkPaths(calls: readonly HostCall[]): Set<unknown> {
|
||||
return new Set(
|
||||
calls
|
||||
.filter((call) => call.method === 'mobileWeb.bundle.chunk')
|
||||
.map((call) => paramField(call.params, 'path'))
|
||||
)
|
||||
}
|
||||
|
||||
it('reads only the changed assets over the wire, and counts only those', async () => {
|
||||
const host = bundleHost({ 'index.html': '<h1>orca</h1>', 'assets/app.js': 'x=1' })
|
||||
const progress: MobileWebBundleFetchProgress[] = []
|
||||
const checked: string[] = []
|
||||
|
||||
const fetched = await fetchMobileWebBundle({
|
||||
client: host.client,
|
||||
onProgress: (update) => progress.push(update),
|
||||
reusable: cacheOf({ 'assets/app.js': 'x=1' }, checked)
|
||||
})
|
||||
|
||||
expect(chunkPaths(host.calls)).toEqual(new Set(['index.html']))
|
||||
expect(checked).toEqual(['assets/app.js'])
|
||||
expect(fetched.reused).toEqual(['assets/app.js'])
|
||||
expect([...fetched.assets.keys()]).toEqual(['index.html'])
|
||||
expect(fetched.totalBytes).toBe(13)
|
||||
expect(progress[0]).toEqual({
|
||||
completedAssets: 0,
|
||||
totalAssets: 1,
|
||||
receivedBytes: 0,
|
||||
totalBytes: 13
|
||||
})
|
||||
expect(progress.at(-1)).toEqual({
|
||||
completedAssets: 1,
|
||||
totalAssets: 1,
|
||||
receivedBytes: 13,
|
||||
totalBytes: 13
|
||||
})
|
||||
})
|
||||
|
||||
it('downloads a cached asset that is no longer whole on disk', async () => {
|
||||
const host = bundleHost({ 'index.html': '<h1>orca</h1>', 'assets/app.js': 'x=1' })
|
||||
const progress: MobileWebBundleFetchProgress[] = []
|
||||
|
||||
const fetched = await fetchMobileWebBundle({
|
||||
client: host.client,
|
||||
onProgress: (update) => progress.push(update),
|
||||
reusable: cacheOf({ 'assets/app.js': 'x=1' }, [], false)
|
||||
})
|
||||
|
||||
expect(chunkPaths(host.calls)).toEqual(new Set(['index.html', 'assets/app.js']))
|
||||
expect(fetched.reused).toEqual([])
|
||||
expect(new TextDecoder().decode(fetched.assets.get('assets/app.js'))).toBe('x=1')
|
||||
expect(progress.at(-1)).toEqual({
|
||||
completedAssets: 2,
|
||||
totalAssets: 2,
|
||||
receivedBytes: 16,
|
||||
totalBytes: 16
|
||||
})
|
||||
})
|
||||
})
|
||||
})
|
||||
|
||||
@@ -40,7 +40,11 @@ export type MobileWebBundleFetchProgress = {
|
||||
|
||||
export type MobileWebBundleFetchResult = {
|
||||
readonly manifest: MobileWebBundleManifestRead
|
||||
/** The assets read over the wire, verified. */
|
||||
readonly assets: ReadonlyMap<string, Uint8Array>
|
||||
/** The assets left for the stage to copy from the host's cached generation instead: a contract
|
||||
* with `stageGeneration`, which copies each one and fails rather than stage a tree without it. */
|
||||
readonly reused: readonly string[]
|
||||
readonly totalBytes: number
|
||||
readonly elapsedMs: number
|
||||
}
|
||||
@@ -54,16 +58,31 @@ type AssetReassembly = {
|
||||
type WindowRead = { readonly asset: AssetReassembly; readonly offset: number }
|
||||
|
||||
/**
|
||||
* Reads the manifest, pages every asset, and returns the verified bytes.
|
||||
* Where a download may take an asset from instead of the wire: the host's cached generation. A
|
||||
* match is by content (hash and length). The cache was hash-verified when it was written and is
|
||||
* never modified in place, so the stage's copy of a reused asset is the bytes the page already
|
||||
* opens from it.
|
||||
*/
|
||||
export type MobileWebBundleReusableAssets = {
|
||||
readonly has: (asset: MobileWebBundleAssetRead) => boolean
|
||||
/** Whether the cached file is still on disk at its full length: a native stat, no bytes read.
|
||||
* False downloads it instead; never rejects. */
|
||||
readonly check: (asset: MobileWebBundleAssetRead) => Promise<boolean>
|
||||
}
|
||||
|
||||
/**
|
||||
* Reads the manifest, pages every asset it cannot reuse, and returns the verified bytes.
|
||||
*
|
||||
* Nothing is cached and nothing is rendered: this is the Phase A proof that the pipe carries a whole
|
||||
* bundle intact. Every asset is checked against the manifest's own sha256 before it is returned, so
|
||||
* a truncated or reordered reassembly fails here rather than in a webview much later.
|
||||
* Every asset read is checked against the manifest's own sha256 before it is returned, so a
|
||||
* truncated or reordered reassembly fails here rather than in a webview much later. Progress
|
||||
* counts only what crosses the wire: reused assets cost no wait, so counting them would fill most
|
||||
* of the bar at once and then crawl.
|
||||
*/
|
||||
export async function fetchMobileWebBundle(args: {
|
||||
client: RpcClient
|
||||
signal?: AbortSignal
|
||||
onProgress?: (progress: MobileWebBundleFetchProgress) => void
|
||||
reusable?: MobileWebBundleReusableAssets
|
||||
}): Promise<MobileWebBundleFetchResult> {
|
||||
const startedAt = Date.now()
|
||||
const stopped = new AbortController()
|
||||
@@ -71,18 +90,30 @@ export async function fetchMobileWebBundle(args: {
|
||||
const opened = await runRpcOperation(args.client, mobileWebBundleManifestRead, null)
|
||||
const manifest = opened.manifest
|
||||
const reader = mobileWebBundleWindowReader(args.client, opened)
|
||||
const queue = planWindowReads(manifest.assets, reader.windowBytes)
|
||||
const reusable = args.reusable
|
||||
const candidates = new Set(
|
||||
reusable === undefined ? [] : manifest.assets.filter((entry) => reusable.has(entry))
|
||||
)
|
||||
const fetched = manifest.assets.filter((entry) => !candidates.has(entry))
|
||||
const assets = new Map<string, Uint8Array>()
|
||||
const reused: string[] = []
|
||||
let receivedBytes = 0
|
||||
let downloadBytes = 0
|
||||
let downloadAssets = 0
|
||||
let downloadedAssets = 0
|
||||
let stallTimer: ReturnType<typeof setTimeout> | null = null
|
||||
let failStalled: (error: Error) => void = () => {}
|
||||
const stalled = new Promise<never>((_, reject) => {
|
||||
failStalled = reject
|
||||
})
|
||||
const armStallClock = (): void => {
|
||||
const disarmStallClock = (): void => {
|
||||
if (stallTimer !== null) {
|
||||
clearTimeout(stallTimer)
|
||||
stallTimer = null
|
||||
}
|
||||
}
|
||||
const armStallClock = (): void => {
|
||||
disarmStallClock()
|
||||
// A read that lands after the fetch settled must not leave a timer behind.
|
||||
if (stopped.signal.aborted) {
|
||||
return
|
||||
@@ -98,6 +129,14 @@ export async function fetchMobileWebBundle(args: {
|
||||
)
|
||||
}, MOBILE_WEB_BUNDLE_WINDOW_STALL_MS)
|
||||
}
|
||||
const reportProgress = (): void => {
|
||||
args.onProgress?.({
|
||||
completedAssets: downloadedAssets,
|
||||
totalAssets: downloadAssets,
|
||||
receivedBytes,
|
||||
totalBytes: downloadBytes
|
||||
})
|
||||
}
|
||||
|
||||
const readWindow = async ({ asset, offset }: WindowRead): Promise<void> => {
|
||||
const reply = await reader.read(
|
||||
@@ -127,48 +166,87 @@ export async function fetchMobileWebBundle(args: {
|
||||
return
|
||||
}
|
||||
assets.set(asset.entry.path, verified)
|
||||
downloadedAssets += 1
|
||||
}
|
||||
// Per window, not per asset: the largest asset goes first, so asset completions bunch at the end.
|
||||
receivedBytes += bytes.byteLength
|
||||
args.onProgress?.({
|
||||
completedAssets: assets.size,
|
||||
totalAssets: manifest.assets.length,
|
||||
receivedBytes,
|
||||
totalBytes: manifest.totalBytes
|
||||
})
|
||||
reportProgress()
|
||||
}
|
||||
|
||||
const worker = async (): Promise<void> => {
|
||||
try {
|
||||
while (!stopped.signal.aborted) {
|
||||
const read = queue.shift()
|
||||
if (read === undefined) {
|
||||
return
|
||||
const readWindows = async (entries: readonly MobileWebBundleAssetRead[]): Promise<void> => {
|
||||
const queue = planWindowReads(entries, reader.windowBytes)
|
||||
if (queue.length === 0) {
|
||||
return
|
||||
}
|
||||
const worker = async (): Promise<void> => {
|
||||
try {
|
||||
while (!stopped.signal.aborted) {
|
||||
const read = queue.shift()
|
||||
if (read === undefined) {
|
||||
return
|
||||
}
|
||||
throwIfCallerAborted(args.signal)
|
||||
await readWindow(read)
|
||||
}
|
||||
throwIfCallerAborted(args.signal)
|
||||
await readWindow(read)
|
||||
} catch (error) {
|
||||
// One failed window stops every other read: each read a worker would still send holds one
|
||||
// of the host's four slots against the caller's retry.
|
||||
stopped.abort()
|
||||
throw error
|
||||
}
|
||||
} catch (error) {
|
||||
// One failed window stops every other read: each read a worker would still send holds one of
|
||||
// the host's four slots against the caller's retry.
|
||||
stopped.abort()
|
||||
throw error
|
||||
}
|
||||
const workers = Math.min(MAX_CONCURRENT_WINDOW_READS, queue.length)
|
||||
armStallClock()
|
||||
try {
|
||||
await Promise.race([Promise.all(Array.from({ length: workers }, () => worker())), stalled])
|
||||
} finally {
|
||||
disarmStallClock()
|
||||
}
|
||||
}
|
||||
|
||||
const workers = Math.min(MAX_CONCURRENT_WINDOW_READS, queue.length)
|
||||
armStallClock()
|
||||
/** The candidates no longer whole in the cache, which are downloaded after all. */
|
||||
const reuseCandidates = async (): Promise<MobileWebBundleAssetRead[]> => {
|
||||
const missed: MobileWebBundleAssetRead[] = []
|
||||
for (const entry of candidates) {
|
||||
if (stopped.signal.aborted) {
|
||||
break
|
||||
}
|
||||
throwIfCallerAborted(args.signal)
|
||||
if (await reusable?.check(entry)) {
|
||||
reused.push(entry.path)
|
||||
} else {
|
||||
missed.push(entry)
|
||||
}
|
||||
}
|
||||
return missed
|
||||
}
|
||||
|
||||
const countDownload = (entries: readonly MobileWebBundleAssetRead[]): void => {
|
||||
downloadAssets += entries.length
|
||||
downloadBytes += entries.reduce((sum, entry) => sum + entry.byteLength, 0)
|
||||
}
|
||||
|
||||
try {
|
||||
await Promise.race([Promise.all(Array.from({ length: workers }, () => worker())), stalled])
|
||||
countDownload(fetched)
|
||||
// The caption's total before any byte, when it is not the whole bundle the session assumed.
|
||||
if (candidates.size > 0) {
|
||||
reportProgress()
|
||||
}
|
||||
// Checked while the window reads are in flight: each is a native stat, and the download it
|
||||
// would otherwise delay is waiting on the network.
|
||||
const [, missed] = await Promise.all([readWindows(fetched), reuseCandidates()])
|
||||
if (missed.length > 0 && !stopped.signal.aborted) {
|
||||
countDownload(missed)
|
||||
reportProgress()
|
||||
await readWindows(missed)
|
||||
}
|
||||
} finally {
|
||||
stopped.abort()
|
||||
if (stallTimer !== null) {
|
||||
clearTimeout(stallTimer)
|
||||
}
|
||||
disarmStallClock()
|
||||
}
|
||||
// The final window sends nothing after its last reply, so no worker would see this abort.
|
||||
throwIfCallerAborted(args.signal)
|
||||
return { manifest, assets, totalBytes: receivedBytes, elapsedMs: Date.now() - startedAt }
|
||||
return { manifest, assets, reused, totalBytes: receivedBytes, elapsedMs: Date.now() - startedAt }
|
||||
}
|
||||
|
||||
/** Largest asset first, so the biggest script's tail is never the last read left in flight. Offsets
|
||||
|
||||
Reference in New Issue
Block a user