diff --git a/mobile/src/mobile-web-shell/generation-store-file-system.ts b/mobile/src/mobile-web-shell/generation-store-file-system.ts new file mode 100644 index 00000000000..4c5718fabca --- /dev/null +++ b/mobile/src/mobile-web-shell/generation-store-file-system.ts @@ -0,0 +1,86 @@ +import { Directory, File, Paths } from 'expo-file-system' + +/** Root of the whole mobile-web cache, one level under the OS cache directory. */ +export const MOBILE_WEB_CACHE_DIRECTORY_NAME = 'mobile-web' + +export type GenerationDirectoryEntry = { + readonly name: string + readonly isDirectory: boolean +} + +/** + * Everything the generation store does to disk, as plain `file://` uris. + * + * The store never imports `expo-file-system`, so its tests run the real write ordering, failure and + * interruption paths against an in-memory tree instead of a simulator. + */ +export type GenerationFileSystem = { + readonly rootUri: string + /** Empty when the directory is missing, so a first run is not a special case. */ + list(uri: string): Promise + /** Creates intermediate directories and succeeds when the directory already exists. */ + createDirectory(uri: string): Promise + /** Both writes create intermediate directories. */ + writeBytes(uri: string, bytes: Uint8Array): Promise + writeText(uri: string, text: string): Promise + /** Null only when the file is missing. A read that fails throws, because "absent" and "could not + * be read" lead the store to opposite decisions about deleting the cache. */ + readText(uri: string): Promise + fileExists(uri: string): Promise + /** Recursive, and a no-op when the path is missing. */ + delete(uri: string): Promise + /** Renames a directory. The destination must not exist: expo moves a directory *into* an existing + * destination rather than over it. */ + moveDirectory(fromUri: string, toUri: string): Promise +} + +export function createExpoGenerationFileSystem(): GenerationFileSystem { + return { + rootUri: new Directory(Paths.cache, MOBILE_WEB_CACHE_DIRECTORY_NAME).uri, + async list(uri) { + const directory = new Directory(uri) + if (!directory.exists) { + return [] + } + return directory + .list() + .map((entry) => ({ name: entry.name, isDirectory: entry instanceof Directory })) + }, + async createDirectory(uri) { + new Directory(uri).create({ intermediates: true, idempotent: true }) + }, + async writeBytes(uri, bytes) { + const file = new File(uri) + file.create({ intermediates: true, overwrite: true }) + file.write(bytes) + }, + async writeText(uri, text) { + const file = new File(uri) + file.create({ intermediates: true, overwrite: true }) + file.write(text) + }, + async readText(uri) { + const file = new File(uri) + // The throw is deliberate: iOS data protection and I/O errors reach the store as failures + // rather than as a missing file. + return file.exists ? await file.text() : null + }, + async fileExists(uri) { + return new File(uri).exists + }, + async delete(uri) { + const directory = new Directory(uri) + if (directory.exists) { + directory.delete() + return + } + const file = new File(uri) + if (file.exists) { + file.delete() + } + }, + async moveDirectory(fromUri, toUri) { + new Directory(fromUri).move(new Directory(toUri)) + } + } +} diff --git a/mobile/src/mobile-web-shell/generation-store.test.ts b/mobile/src/mobile-web-shell/generation-store.test.ts new file mode 100644 index 00000000000..e258998550d --- /dev/null +++ b/mobile/src/mobile-web-shell/generation-store.test.ts @@ -0,0 +1,633 @@ +import { describe, expect, it } from 'vitest' +import { createGenerationStore, MAX_CACHED_HOSTS } from './generation-store' +import { deriveHostCacheKey } from './host-cache-key' +import type { + createExpoGenerationFileSystem, + GenerationFileSystem +} from './generation-store-file-system' +import type { MobileWebBundleFetchResult } from '../transport/mobile-web-bundle-fetch' + +// The adapter is deliberately untested at runtime — it would need a device filesystem — so this is +// the check that it still answers the port the store is written against. +type AdapterIsPort = + ReturnType extends GenerationFileSystem ? true : false +const adapterSatisfiesPort: AdapterIsPort = true + +const ROOT = 'file:///cache/mobile-web' +const HOST = deriveHostCacheKey('host-a') + +type FakeNode = { kind: 'directory' } | { kind: 'file'; bytes: Uint8Array } + +type FakeFileSystem = GenerationFileSystem & { + readonly writes: string[] + paths(): readonly string[] + seed(path: string, node: FakeNode): void + failWritesAt(path: string | null): void + failReadsAt(path: string | null): void + loseContentsOnMove(): void + text(path: string): string | null +} + +function createFakeFileSystem(): FakeFileSystem { + const nodes = new Map() + const writes: string[] = [] + let failAt: string | null = null + let failReadAt: string | null = null + let moveKeepsContents = true + const uri = (path: string): string => `${ROOT}/${path}` + const parentOf = (target: string): string => target.slice(0, target.lastIndexOf('/')) + + const makeDirectory = (target: string): void => { + for (let at = target; at.startsWith(ROOT); at = parentOf(at)) { + nodes.set(at, { kind: 'directory' }) + } + } + const write = (target: string, bytes: Uint8Array): void => { + if (failAt !== null && target === uri(failAt)) { + throw new Error('simulated disk-full write') + } + makeDirectory(parentOf(target)) + nodes.set(target, { kind: 'file', bytes }) + writes.push(target.slice(ROOT.length + 1)) + } + + return { + rootUri: ROOT, + writes, + paths: () => + [...nodes.keys()] + .filter((key) => key !== ROOT) + .map((key) => key.slice(ROOT.length + 1)) + .sort(), + seed: (path, node) => { + makeDirectory(parentOf(uri(path))) + nodes.set(uri(path), node) + }, + failWritesAt: (path) => { + failAt = path + }, + failReadsAt: (path) => { + failReadAt = path + }, + loseContentsOnMove: () => { + moveKeepsContents = false + }, + text: (path) => { + const node = nodes.get(uri(path)) + return node?.kind === 'file' ? new TextDecoder().decode(node.bytes) : null + }, + async list(target) { + if (nodes.get(target)?.kind !== 'directory') { + return [] + } + return [...nodes.entries()] + .filter( + ([key]) => key.startsWith(`${target}/`) && !key.slice(target.length + 1).includes('/') + ) + .map(([key, node]) => ({ + name: key.slice(target.length + 1), + isDirectory: node.kind === 'directory' + })) + }, + async createDirectory(target) { + makeDirectory(target) + }, + async writeBytes(target, bytes) { + write(target, bytes) + }, + async writeText(target, value) { + write(target, new TextEncoder().encode(value)) + }, + async readText(target) { + if (failReadAt !== null && target === uri(failReadAt)) { + throw new Error('simulated unreadable file') + } + const node = nodes.get(target) + return node?.kind === 'file' ? new TextDecoder().decode(node.bytes) : null + }, + async fileExists(target) { + return nodes.get(target)?.kind === 'file' + }, + async delete(target) { + for (const key of Array.from(nodes.keys())) { + if (key === target || key.startsWith(`${target}/`)) { + nodes.delete(key) + } + } + }, + async moveDirectory(fromUri, toUri) { + if (nodes.has(toUri)) { + throw new Error(`fake filesystem refuses to move onto ${toUri}`) + } + for (const [key, node] of Array.from(nodes.entries())) { + if (key === fromUri || key.startsWith(`${fromUri}/`)) { + nodes.delete(key) + if (moveKeepsContents || key === fromUri) { + nodes.set(toUri + key.slice(fromUri.length), node) + } + } + } + } + } +} + +function buildResult(options: { + buildId?: string + assets?: readonly { path: string; byteLength: number }[] + bytes?: ReadonlyMap +}): MobileWebBundleFetchResult { + const listed = options.assets ?? [ + { path: 'index.html', byteLength: 4 }, + { path: 'assets/app.js', byteLength: 2 } + ] + const assets = listed.map((asset, index) => ({ + path: asset.path, + sha256: String(index).repeat(64).slice(0, 64), + byteLength: asset.byteLength, + contentType: 'text/html; charset=utf-8' + })) + const totalBytes = assets.reduce((sum, asset) => sum + asset.byteLength, 0) + return { + manifest: { + schemaVersion: 1, + buildId: options.buildId ?? 'a'.repeat(64), + minCompatibleRuntimeProtocolVersion: 2, + runtimeProtocolVersion: 2, + entrypoint: 'index.html', + totalBytes, + assets + }, + assets: + options.bytes ?? + new Map(assets.map((asset) => [asset.path, new Uint8Array(asset.byteLength).fill(7)])), + totalBytes, + elapsedMs: 1 + } +} + +async function activate( + store: ReturnType, + hostKey: string, + result = buildResult({}) +): Promise { + await store.commitGeneration(await store.stageGeneration(hostKey, result)) +} + +describe('generation store', () => { + it('stages and commits exactly the manifest, with the manifest written last', async () => { + const fs = createFakeFileSystem() + const store = createGenerationStore({ fileSystem: fs, now: () => 10 }) + + await activate(store, HOST) + + const build = 'a'.repeat(64) + expect(fs.paths()).toEqual([ + HOST, + `${HOST}/generations`, + `${HOST}/generations/${build}`, + `${HOST}/generations/${build}/assets`, + `${HOST}/generations/${build}/assets/app.js`, + `${HOST}/generations/${build}/index.html`, + `${HOST}/generations/${build}/manifest.json`, + `${HOST}/tmp`, + 'hosts.json' + ]) + const staged = fs.writes.filter((path) => path.includes('/tmp/')) + expect(staged.at(-1)).toBe(`${HOST}/tmp/${build}/manifest.json`) + expect(staged).toHaveLength(3) + expect(fs.text('hosts.json')).toBe(JSON.stringify({ [HOST]: 10 })) + }) + + it('reads back the activation it committed', async () => { + const fs = createFakeFileSystem() + const store = createGenerationStore({ fileSystem: fs }) + + await activate(store, HOST) + const active = await store.readActiveGeneration(HOST) + + expect(active?.buildId).toBe('a'.repeat(64)) + expect(active?.directory).toBe(`${ROOT}/${HOST}/generations/${'a'.repeat(64)}`) + expect(active?.manifest.entrypoint).toBe('index.html') + expect(await store.readActiveGeneration(deriveHostCacheKey('never-opened'))).toBeNull() + }) + + it('refuses an asset that is missing or the wrong length, leaving no generation', async () => { + const fs = createFakeFileSystem() + const store = createGenerationStore({ fileSystem: fs }) + const missing = buildResult({ bytes: new Map([['index.html', new Uint8Array(4)]]) }) + const short = buildResult({ + bytes: new Map([ + ['index.html', new Uint8Array(4)], + ['assets/app.js', new Uint8Array(1)] + ]) + }) + + await expect(store.stageGeneration(HOST, missing)).rejects.toThrow('assets/app.js is absent') + await expect(store.stageGeneration(HOST, short)).rejects.toThrow("not the manifest's 2") + expect(fs.paths()).toEqual([]) + expect(await store.readActiveGeneration(HOST)).toBeNull() + }) + + it('drops the staged tree when a write fails', async () => { + const fs = createFakeFileSystem() + const store = createGenerationStore({ fileSystem: fs }) + fs.failWritesAt(`${HOST}/tmp/${'a'.repeat(64)}/assets/app.js`) + + await expect(store.stageGeneration(HOST, buildResult({}))).rejects.toThrow('disk-full') + + expect(fs.paths().some((path) => path.includes(`tmp/${'a'.repeat(64)}`))).toBe(false) + expect(await store.readActiveGeneration(HOST)).toBeNull() + }) + + it('leaves no generation and no tmp for any host when a download is interrupted', async () => { + const fs = createFakeFileSystem() + const store = createGenerationStore({ fileSystem: fs }) + const other = deriveHostCacheKey('host-b') + + await store.stageGeneration(HOST, buildResult({})) + await store.stageGeneration(other, buildResult({})) + await store.sweepStagedGenerations() + + expect(fs.paths().some((path) => path.includes('/tmp'))).toBe(false) + expect(await store.readActiveGeneration(HOST)).toBeNull() + expect(await store.readActiveGeneration(other)).toBeNull() + }) + + it('treats a second commit of the same build as a no-op', async () => { + const fs = createFakeFileSystem() + const store = createGenerationStore({ fileSystem: fs, now: () => 10 }) + + await activate(store, HOST) + const before = fs.paths() + const staged = await store.stageGeneration(HOST, buildResult({})) + const active = await store.commitGeneration(staged) + + expect(active.buildId).toBe('a'.repeat(64)) + expect(fs.paths()).toEqual(before) + }) + + it('replaces the previous generation when the build id changes', async () => { + const fs = createFakeFileSystem() + const store = createGenerationStore({ fileSystem: fs }) + + await activate(store, HOST) + await activate(store, HOST, buildResult({ buildId: 'b'.repeat(64) })) + + expect(fs.paths().some((path) => path.includes('a'.repeat(64)))).toBe(false) + expect((await store.readActiveGeneration(HOST))?.buildId).toBe('b'.repeat(64)) + }) + + it('reads two generations as no activation and drops the host tree', async () => { + const fs = createFakeFileSystem() + const store = createGenerationStore({ fileSystem: fs }) + await activate(store, HOST) + fs.seed(`${HOST}/generations/${'c'.repeat(64)}/manifest.json`, { + kind: 'file', + bytes: new TextEncoder().encode('{}') + }) + + expect(await store.readActiveGeneration(HOST)).toBeNull() + expect(fs.paths().some((path) => path.startsWith(HOST))).toBe(false) + }) + + it('reads an unparseable or mismatched manifest as no activation and drops the host tree', async () => { + for (const body of [ + 'not json', + JSON.stringify({ ...buildResult({}).manifest, buildId: 'd'.repeat(64) }) + ]) { + const fs = createFakeFileSystem() + const store = createGenerationStore({ fileSystem: fs }) + await activate(store, HOST) + fs.seed(`${HOST}/generations/${'a'.repeat(64)}/manifest.json`, { + kind: 'file', + bytes: new TextEncoder().encode(body) + }) + + expect(await store.readActiveGeneration(HOST)).toBeNull() + expect(fs.paths().some((path) => path.startsWith(HOST))).toBe(false) + } + }) + + it('evicts the least recently activated host past the ceiling', async () => { + const fs = createFakeFileSystem() + let clock = 0 + const store = createGenerationStore({ fileSystem: fs, now: () => (clock += 1) }) + const hosts = ['a', 'b', 'c', 'd', 'e'].map((name) => deriveHostCacheKey(name)) + + for (const host of hosts) { + await activate(store, host) + } + + expect(await store.readActiveGeneration(hosts[0])).toBeNull() + expect(fs.paths().some((path) => path.startsWith(hosts[0]))).toBe(false) + for (const host of hosts.slice(1)) { + expect((await store.readActiveGeneration(host))?.buildId).toBe('a'.repeat(64)) + } + expect(Object.keys(JSON.parse(fs.text('hosts.json') ?? '{}'))).toHaveLength(MAX_CACHED_HOSTS) + }) + + it('evicts a host with no index entry before the least recently activated one', async () => { + const fs = createFakeFileSystem() + let clock = 0 + const store = createGenerationStore({ fileSystem: fs, now: () => (clock += 1) }) + const oldest = deriveHostCacheKey('a') + const orphan = deriveHostCacheKey('orphan') + for (const name of ['a', 'b', 'c']) { + await activate(store, deriveHostCacheKey(name)) + } + // Activated last, so recency alone would keep it; its index entry is what goes missing. + await activate(store, orphan) + const index: Record = JSON.parse(fs.text('hosts.json') ?? '{}') + delete index[orphan] + fs.seed('hosts.json', { kind: 'file', bytes: new TextEncoder().encode(JSON.stringify(index)) }) + + await activate(store, deriveHostCacheKey('d')) + + expect(fs.paths().some((path) => path.startsWith(orphan))).toBe(false) + expect((await store.readActiveGeneration(oldest))?.buildId).toBe('a'.repeat(64)) + }) + + it('counts a recommit of the build a host already has as use of that host', async () => { + const fs = createFakeFileSystem() + let clock = 0 + const store = createGenerationStore({ fileSystem: fs, now: () => (clock += 1) }) + const kept = deriveHostCacheKey('a') + const evicted = deriveHostCacheKey('b') + for (const name of ['a', 'b', 'c', 'd']) { + await activate(store, deriveHostCacheKey(name)) + } + // A redownload of the bundle host A already has, which takes the same-build commit path. + await activate(store, kept) + + await activate(store, deriveHostCacheKey('e')) + + expect(fs.paths().some((path) => path.startsWith(evicted))).toBe(false) + expect((await store.readActiveGeneration(kept))?.buildId).toBe('a'.repeat(64)) + }) + + it('never counts or evicts a host that is only mid-download', async () => { + const fs = createFakeFileSystem() + let clock = 0 + const store = createGenerationStore({ fileSystem: fs, now: () => (clock += 1) }) + const oldest = deriveHostCacheKey('a') + const downloading = deriveHostCacheKey('downloading') + for (const name of ['a', 'b', 'c', 'd']) { + await activate(store, deriveHostCacheKey(name)) + } + const staged = await store.stageGeneration(downloading, buildResult({})) + + await activate(store, deriveHostCacheKey('e')) + + // The ceiling is four cached generations, so the fifth activation evicts the least recently + // activated host and leaves the download alone. + expect(fs.paths().some((path) => path.startsWith(oldest))).toBe(false) + expect(fs.text(`${staged.directory.slice(ROOT.length + 1)}/manifest.json`)).not.toBeNull() + await store.commitGeneration(staged) + expect((await store.readActiveGeneration(downloading))?.buildId).toBe('a'.repeat(64)) + }) + + it('serializes two stage calls for one host and build', async () => { + const fs = createFakeFileSystem() + const store = createGenerationStore({ fileSystem: fs }) + // One build id cannot really carry two asset lists; differing ones are what make an interleaved + // pair visible, because unserialized both of them land in the one staged directory. + const staging = `${HOST}/tmp/${'a'.repeat(64)}` + const earlier = buildResult({ assets: [{ path: 'assets/earlier.js', byteLength: 2 }] }) + const later = buildResult({ assets: [{ path: 'assets/later.js', byteLength: 3 }] }) + + const [first, second] = await Promise.all([ + store.stageGeneration(HOST, earlier), + store.stageGeneration(HOST, later) + ]) + + expect(first.directory).toBe(second.directory) + // Each staging is a contiguous run ending in its manifest; interleaved they would alternate. + expect(fs.writes).toEqual([ + `${staging}/assets/earlier.js`, + `${staging}/manifest.json`, + `${staging}/assets/later.js`, + `${staging}/manifest.json` + ]) + expect(fs.paths().filter((path) => path.startsWith(`${staging}/assets/`))).toEqual([ + `${staging}/assets/later.js` + ]) + }) + + it('drops residue from an earlier attempt instead of staging over it', async () => { + const fs = createFakeFileSystem() + const store = createGenerationStore({ fileSystem: fs }) + const staging = `${HOST}/tmp/${'a'.repeat(64)}` + fs.seed(`${staging}/assets/orphan.js`, { kind: 'file', bytes: new Uint8Array(1) }) + + await store.stageGeneration(HOST, buildResult({})) + + expect(fs.paths().some((path) => path.endsWith('orphan.js'))).toBe(false) + }) + + it('refuses a path that escapes the staged tree, and a host key that is not one', async () => { + const fs = createFakeFileSystem() + const store = createGenerationStore({ fileSystem: fs }) + const escapes = [ + '../outside.js', + 'assets/../../outside.js', + '/etc/passwd', + 'assets//app.js', + 'manifest.json', + 'Manifest.JSON' + ] + + for (const path of escapes) { + const result = buildResult({ assets: [{ path, byteLength: 1 }] }) + await expect(store.stageGeneration(HOST, result)).rejects.toThrow('refuses to stage') + } + await expect(store.stageGeneration('host-a', buildResult({}))).rejects.toThrow( + 'not a host cache key' + ) + expect(fs.paths()).toEqual([]) + }) + + it('deletes one host tree without touching another', async () => { + const fs = createFakeFileSystem() + const other = deriveHostCacheKey('host-b') + const store = createGenerationStore({ fileSystem: fs }) + await activate(store, HOST) + await activate(store, other) + + await store.deleteHostCache(HOST) + + expect(await store.readActiveGeneration(HOST)).toBeNull() + expect((await store.readActiveGeneration(other))?.buildId).toBe('a'.repeat(64)) + expect(Object.keys(JSON.parse(fs.text('hosts.json') ?? '{}'))).toEqual([other]) + }) + + it('refuses a rename that did not carry the tree, as Android below API 26 can', async () => { + const fs = createFakeFileSystem() + const store = createGenerationStore({ fileSystem: fs }) + fs.loseContentsOnMove() + + const staged = await store.stageGeneration(HOST, buildResult({})) + await expect(store.commitGeneration(staged)).rejects.toThrow('did not carry its manifest') + + expect(await store.readActiveGeneration(HOST)).toBeNull() + expect(fs.paths().some((path) => path.includes('generations/'))).toBe(false) + }) + + it('keeps the host tree when the manifest read fails, and drops it when it is missing', async () => { + const fs = createFakeFileSystem() + const store = createGenerationStore({ fileSystem: fs }) + const manifest = `${HOST}/generations/${'a'.repeat(64)}/manifest.json` + await activate(store, HOST) + const before = fs.paths() + + fs.failReadsAt(manifest) + expect(await store.readActiveGeneration(HOST)).toBeNull() + expect(fs.paths()).toEqual(before) + + fs.failReadsAt(null) + await fs.delete(`${ROOT}/${manifest}`) + expect(await store.readActiveGeneration(HOST)).toBeNull() + expect(fs.paths().some((path) => path.startsWith(HOST))).toBe(false) + }) + + it('activates normally when the recency index cannot be read', async () => { + const fs = createFakeFileSystem() + const store = createGenerationStore({ fileSystem: fs, now: () => 10 }) + fs.seed('hosts.json', { kind: 'file', bytes: new TextEncoder().encode('{}') }) + fs.failReadsAt('hosts.json') + + await activate(store, HOST) + + expect((await store.readActiveGeneration(HOST))?.buildId).toBe('a'.repeat(64)) + }) + + it('replaces an entry named for the build id that is not a readable generation', async () => { + const build = 'a'.repeat(64) + // Exactly what a crash between the rename and the post-rename check can leave behind. + for (const seeded of [ + { kind: 'directory' }, + { kind: 'file', bytes: new Uint8Array(1) } + ] as const) { + const fs = createFakeFileSystem() + const store = createGenerationStore({ fileSystem: fs }) + fs.seed(`${HOST}/generations/${build}`, seeded) + + await activate(store, HOST) + + expect((await store.readActiveGeneration(HOST))?.buildId).toBe(build) + expect(fs.text(`${HOST}/generations/${build}/index.html`)).not.toBeNull() + } + }) + + it('drops an aborted staging without touching the activation', async () => { + const fs = createFakeFileSystem() + const store = createGenerationStore({ fileSystem: fs }) + await activate(store, HOST) + + const staged = await store.stageGeneration(HOST, buildResult({ buildId: 'b'.repeat(64) })) + await store.abortStagedGeneration(staged) + + expect(fs.paths().some((path) => path.includes('b'.repeat(64)))).toBe(false) + expect((await store.readActiveGeneration(HOST))?.buildId).toBe('a'.repeat(64)) + }) + + it('keeps the host it just activated when the clock jumps backward', async () => { + const fs = createFakeFileSystem() + const times = [100, 200, 300, 400, 1] + let tick = 0 + const store = createGenerationStore({ fileSystem: fs, now: () => times[tick++] ?? 0 }) + const hosts = ['a', 'b', 'c', 'd', 'e'].map((name) => deriveHostCacheKey(name)) + + for (const host of hosts) { + await activate(store, host) + } + + expect((await store.readActiveGeneration(hosts[4]))?.directory).toBe( + `${ROOT}/${hosts[4]}/generations/${'a'.repeat(64)}` + ) + expect(await store.readActiveGeneration(hosts[0])).toBeNull() + for (const host of hosts.slice(1)) { + expect((await store.readActiveGeneration(host))?.buildId).toBe('a'.repeat(64)) + } + }) + + it('returns the activation even when the recency index cannot be written', async () => { + const fs = createFakeFileSystem() + const store = createGenerationStore({ fileSystem: fs }) + fs.failWritesAt('hosts.json') + + const staged = await store.stageGeneration(HOST, buildResult({})) + const active = await store.commitGeneration(staged) + + expect(active.buildId).toBe('a'.repeat(64)) + expect((await store.readActiveGeneration(HOST))?.buildId).toBe('a'.repeat(64)) + expect(fs.text('hosts.json')).toBeNull() + }) + + it('refuses a handle whose staged tree is gone without touching the activation', async () => { + const fs = createFakeFileSystem() + const store = createGenerationStore({ fileSystem: fs }) + await activate(store, HOST) + + const staged = await store.stageGeneration(HOST, buildResult({ buildId: 'b'.repeat(64) })) + await store.abortStagedGeneration(staged) + + await expect(store.commitGeneration(staged)).rejects.toThrow('no longer on disk') + expect((await store.readActiveGeneration(HOST))?.buildId).toBe('a'.repeat(64)) + }) + + it('prunes an index entry whose host tree is gone', async () => { + const fs = createFakeFileSystem() + const store = createGenerationStore({ fileSystem: fs, now: () => 10 }) + const stale = deriveHostCacheKey('uninstalled') + fs.seed('hosts.json', { + kind: 'file', + bytes: new TextEncoder().encode(JSON.stringify({ [stale]: 5 })) + }) + + await activate(store, HOST) + + expect(fs.text('hosts.json')).toBe(JSON.stringify({ [HOST]: 10 })) + }) + + it('refuses a staged handle it did not issue', async () => { + const fs = createFakeFileSystem() + const store = createGenerationStore({ fileSystem: fs }) + await activate(store, HOST) + const before = fs.paths() + const forged = { + hostKey: HOST, + buildId: 'b'.repeat(64), + // Aimed at the live generation, which commit would rename over and abort would delete. + directory: `${ROOT}/${HOST}/generations/${'a'.repeat(64)}`, + manifest: buildResult({}).manifest + } + + await expect(store.commitGeneration(forged)).rejects.toThrow('did not issue') + await expect(store.abortStagedGeneration(forged)).rejects.toThrow('did not issue') + expect(fs.paths()).toEqual(before) + expect((await store.readActiveGeneration(HOST))?.buildId).toBe('a'.repeat(64)) + }) + + it('ignores a directory under the cache root that is not a host key', async () => { + const fs = createFakeFileSystem() + let clock = 0 + const store = createGenerationStore({ fileSystem: fs, now: () => (clock += 1) }) + // Whatever else lives under the OS cache directory is not this store's to count or delete. + fs.seed('not-a-host-key/stray.txt', { kind: 'file', bytes: new Uint8Array(1) }) + const hosts = ['a', 'b', 'c', 'd'].map((name) => deriveHostCacheKey(name)) + + for (const host of hosts) { + await activate(store, host) + } + await store.sweepStagedGenerations() + + expect(fs.paths()).toContain('not-a-host-key/stray.txt') + for (const host of hosts) { + expect((await store.readActiveGeneration(host))?.buildId).toBe('a'.repeat(64)) + } + }) + + it('keeps the adapter aligned with the port', () => { + expect(adapterSatisfiesPort).toBe(true) + }) +}) diff --git a/mobile/src/mobile-web-shell/generation-store.ts b/mobile/src/mobile-web-shell/generation-store.ts new file mode 100644 index 00000000000..157c4cc17ac --- /dev/null +++ b/mobile/src/mobile-web-shell/generation-store.ts @@ -0,0 +1,339 @@ +import { z } from 'zod' +import { + MobileWebBundleManifestReadSchema, + type MobileWebBundleManifestRead +} from '../transport/mobile-web-bundle-reply-schemas' +import type { MobileWebBundleFetchResult } from '../transport/mobile-web-bundle-fetch' +import type { GenerationDirectoryEntry, GenerationFileSystem } from './generation-store-file-system' +import { isHostCacheKey } from './host-cache-key' + +const GENERATIONS_DIRECTORY_NAME = 'generations' +const STAGING_DIRECTORY_NAME = 'tmp' +const MANIFEST_FILE_NAME = 'manifest.json' +const HOST_INDEX_FILE_NAME = 'hosts.json' + +/** The architecture reference's cache ceiling: four hosts, least recently activated evicted. */ +export const MAX_CACHED_HOSTS = 4 + +export type ActiveGeneration = { + readonly buildId: string + /** Read-only input for the native view; nothing but this store writes under it. */ + readonly directory: string + readonly manifest: MobileWebBundleManifestRead +} + +export type StagedGeneration = { + readonly hostKey: string + readonly buildId: string + readonly directory: string + readonly manifest: MobileWebBundleManifestRead +} + +export type GenerationStore = { + readActiveGeneration(hostKey: string): Promise + stageGeneration(hostKey: string, result: MobileWebBundleFetchResult): Promise + commitGeneration(staged: StagedGeneration): Promise + abortStagedGeneration(staged: StagedGeneration): Promise + sweepStagedGenerations(): Promise + deleteHostCache(hostKey: string): Promise +} + +/** Recency only, so anything unreadable degrades to "evict this host first". */ +const HostIndexSchema = z.record(z.string(), z.number().int().nonnegative()) + +export function createGenerationStore(options: { + fileSystem: GenerationFileSystem + now?: () => number +}): GenerationStore { + const fs = options.fileSystem + const now = options.now ?? Date.now + // `StagedGeneration` is structurally typed, so any object of that shape would otherwise let + // `commitGeneration` rename over, and `abortStagedGeneration` delete, a directory of the caller's + // choosing. Only handles this store minted are honoured. + const issuedHandles = new WeakSet() + + const hostRoot = (hostKey: string): string => joinUri(fs.rootUri, requireHostKey(hostKey)) + const generationsRoot = (hostKey: string): string => + joinUri(hostRoot(hostKey), GENERATIONS_DIRECTORY_NAME) + const stagingRoot = (hostKey: string): string => + joinUri(hostRoot(hostKey), STAGING_DIRECTORY_NAME) + + async function readHostIndex(): Promise> { + // Unreadable is treated as absent here, unlike a manifest: an index nobody can read costs + // eviction order, and the next activation rewrites it whole. + const text = await fs.readText(joinUri(fs.rootUri, HOST_INDEX_FILE_NAME)).catch(() => null) + const parsed = text === null ? null : HostIndexSchema.safeParse(parseJson(text)) + return new Map(Object.entries(parsed?.success === true ? parsed.data : {})) + } + + async function writeHostIndex(index: ReadonlyMap): Promise { + // Recency, not truth: a full disk here must not turn an activation that is already on disk + // into a thrown commit, and the next activation rewrites the whole index anyway. + await fs + .writeText( + joinUri(fs.rootUri, HOST_INDEX_FILE_NAME), + JSON.stringify(Object.fromEntries(index)) + ) + .catch(() => undefined) + } + + async function listHostDirectories(): Promise { + const entries = await fs.list(fs.rootUri) + return entries.filter((entry) => entry.isDirectory && isHostCacheKey(entry.name)) + } + + /** The ceiling counts cached generations, so a host that only holds a download in progress is + * neither counted nor evictable: evicting it would delete the tree its own commit is about to + * rename. Sweeping still walks every host directory, staged-only ones included. */ + async function listActivatedHosts(): Promise { + const activated: string[] = [] + for (const host of await listHostDirectories()) { + const generations = await fs.list(joinUri(fs.rootUri, host.name, GENERATIONS_DIRECTORY_NAME)) + if (generations.some((entry) => entry.isDirectory)) { + activated.push(host.name) + } + } + return activated + } + + async function dropHostTree(hostKey: string): Promise { + await fs.delete(hostRoot(hostKey)) + } + + async function enforceHostLimit(index: Map, activated: string): Promise { + const hosts = await listActivatedHosts() + const present = new Set(hosts) + for (const key of Array.from(index.keys())) { + if (!present.has(key)) { + index.delete(key) + } + } + // A host with no index entry sorts first: the index is recency, not truth, so a lost or + // truncated one costs eviction order rather than a generation. The host just activated is + // never a candidate, because `now()` is a wall clock: one backward jump would otherwise make + // the newest entry the oldest and evict the tree the caller is about to open. + const candidates = hosts + .filter((host) => host !== activated) + .sort((left, right) => (index.get(left) ?? 0) - (index.get(right) ?? 0)) + for (const host of candidates.slice(0, Math.max(0, hosts.length - MAX_CACHED_HOSTS))) { + await dropHostTree(host) + index.delete(host) + } + await writeHostIndex(index) + } + + async function readActive(hostKey: string): Promise { + const generations = generationsRoot(hostKey) + const directories = (await fs.list(generations)).filter((entry) => entry.isDirectory) + if (directories.length === 0) { + return null + } + // Two directories means a commit was interrupted between dropping the old generation and + // renaming the new one. There is no activation file to break the tie, and a manifest that + // names another build is a tree from some other bundle, so the host's cache goes and the next + // open redownloads it. + const only = directories.length === 1 ? directories[0] : null + if (only !== null) { + const directory = joinUri(generations, only.name) + let text: string | null + try { + text = await fs.readText(joinUri(directory, MANIFEST_FILE_NAME)) + } catch { + // A failed read is not evidence of a bad generation, so nothing is deleted: the caller + // redownloads, and a transient I/O blip must not cost a cache that verified. + return null + } + const manifest = parseManifest(text) + if (manifest !== null && manifest.buildId === only.name) { + return { buildId: manifest.buildId, directory, manifest } + } + } + await dropHostTree(hostKey) + return null + } + + async function stage( + hostKey: string, + result: MobileWebBundleFetchResult + ): Promise { + const manifest = result.manifest + const directory = joinUri(stagingRoot(hostKey), manifest.buildId) + const assets = manifest.assets.map((asset) => ({ + uri: joinUri(directory, requireStorablePath(asset.path)), + bytes: requireExactBytes(result.assets.get(asset.path), asset) + })) + // Residue from an earlier attempt is dropped rather than written over: a half-written tree + // plus a fresh write is not a generation either side verified. + await fs.delete(directory) + try { + for (const asset of assets) { + await fs.writeBytes(asset.uri, asset.bytes) + } + // Last, always: a tree without it never reads back as an activation, which is what makes an + // interrupted write recoverable rather than ambiguous. + await fs.writeText(joinUri(directory, MANIFEST_FILE_NAME), JSON.stringify(manifest)) + } catch (error) { + await fs.delete(directory).catch(() => undefined) + throw error + } + const handle: StagedGeneration = { hostKey, buildId: manifest.buildId, directory, manifest } + issuedHandles.add(handle) + return handle + } + + function requireIssuedHandle(staged: StagedGeneration): StagedGeneration { + if (!issuedHandles.has(staged)) { + throw new Error('generation store was handed a staged handle it did not issue') + } + return staged + } + + async function commit(staged: StagedGeneration): Promise { + requireIssuedHandle(staged) + const generations = generationsRoot(staged.hostKey) + const target = joinUri(generations, staged.buildId) + const active: ActiveGeneration = { + buildId: staged.buildId, + directory: target, + manifest: staged.manifest + } + const entries = await fs.list(generations) + // The build id names an asset list, not evidence those bytes landed, so a directory of that name + // is this activation only once its manifest is on disk. An empty one — what a crash between the + // rename and the check below leaves on Android under API 26 — or a plain file of that name falls + // through and is replaced by the staged tree, which was verified byte for byte. + const existing = entries.find((entry) => entry.name === staged.buildId) + if ( + existing?.isDirectory === true && + (await fs.fileExists(joinUri(target, MANIFEST_FILE_NAME))) + ) { + // Still an activation, so it still counts as use: without this a host that redownloads the + // bundle it already has stays the least recently activated and is evicted first. No eviction + // pass, because the host count did not change. + const index = await readHostIndex() + index.set(staged.hostKey, now()) + await writeHostIndex(index) + await fs.delete(staged.directory) + return active + } + // Before any delete: an aborted or swept handle must not cost the live generation, and a tree + // that is no longer on disk cannot be renamed into one either. + if (!(await fs.fileExists(joinUri(staged.directory, MANIFEST_FILE_NAME)))) { + throw new Error(`staged generation ${staged.buildId} is no longer on disk`) + } + // Every other generation goes before the rename, never after. A crash between the two leaves + // zero generations, which the runbook's redownload rule already covers; the other order can + // leave two directories under `generations/` with nothing to say which one is the activation. + for (const entry of entries) { + await fs.delete(joinUri(generations, entry.name)) + } + await fs.createDirectory(generations) + await fs.moveDirectory(staged.directory, target) + // Android below API 26 implements a directory move as a non-recursive copy plus a delete + // (expo-file-system android FileSystemPath.kt:158-173), which can land an empty directory. Its + // `delete()` then fails on the non-empty source, so the tmp tree survives for the next sweep. + if (!(await fs.fileExists(joinUri(target, MANIFEST_FILE_NAME)))) { + await fs.delete(target) + throw new Error(`generation ${staged.buildId} did not carry its manifest through the rename`) + } + const index = await readHostIndex() + index.set(staged.hostKey, now()) + // Enforced here rather than left to a caller: the four-host ceiling is this module's invariant. + await enforceHostLimit(index, staged.hostKey) + return active + } + + async function sweep(): Promise { + // Every host's `tmp`, not just the one being opened: an interrupted download must not survive a + // restart, and it may belong to a host this launch never selects. + for (const host of await listHostDirectories()) { + await fs.delete(joinUri(fs.rootUri, host.name, STAGING_DIRECTORY_NAME)) + } + } + + async function deleteHost(hostKey: string): Promise { + await dropHostTree(hostKey) + const index = await readHostIndex() + if (index.delete(hostKey)) { + await writeHostIndex(index) + } + } + + // One queue for the whole store rather than one per host: every operation is a short burst of + // cache I/O, and a single order answers the stage/commit/sweep/delete interleavings at once. A + // second `stageGeneration` for the same host and build waits for the first rather than writing + // into the tree it is still filling. + let tail: Promise = Promise.resolve() + function serialize(operation: () => Promise): Promise { + const run = tail.then(operation, operation) + tail = run.catch(() => undefined) + return run + } + + return { + readActiveGeneration: (hostKey) => serialize(() => readActive(hostKey)), + stageGeneration: (hostKey, result) => serialize(() => stage(hostKey, result)), + commitGeneration: (staged) => serialize(() => commit(staged)), + abortStagedGeneration: (staged) => + serialize(() => fs.delete(requireIssuedHandle(staged).directory)), + sweepStagedGenerations: () => serialize(sweep), + deleteHostCache: (hostKey) => serialize(() => deleteHost(hostKey)) + } +} + +function joinUri(...segments: readonly string[]): string { + return segments.map((segment) => segment.replace(/\/+$/, '')).join('/') +} + +function parseJson(text: string): unknown { + try { + return JSON.parse(text) + } catch { + return null + } +} + +function parseManifest(text: string | null): MobileWebBundleManifestRead | null { + if (text === null) { + return null + } + const parsed = MobileWebBundleManifestReadSchema.safeParse(parseJson(text)) + return parsed.success ? parsed.data : null +} + +function requireHostKey(hostKey: string): string { + if (!isHostCacheKey(hostKey)) { + throw new Error('generation store was handed something that is not a host cache key') + } + return hostKey +} + +/** The manifest schema bans traversal already, but this is the last code between a manifest and a + * write, and `manifest.json` is the store's own name rather than an asset's to take — folded, + * because APFS and NTFS are case-insensitive and `Manifest.JSON` would land on the same file. */ +function requireStorablePath(path: string): string { + const segments = path.split('/') + const storable = + path.length > 0 && + path.toLowerCase() !== MANIFEST_FILE_NAME && + !path.includes('\\') && + segments.every((segment) => segment !== '' && segment !== '.' && segment !== '..') + if (!storable) { + throw new Error(`generation store refuses to stage the asset path ${path}`) + } + return path +} + +function requireExactBytes( + bytes: Uint8Array | undefined, + asset: { path: string; byteLength: number } +): Uint8Array { + // Only complete generations activate, so the check is before the first write rather than after + // the last: a manifest asset that is absent or the wrong length never reaches disk. + if (bytes === undefined || bytes.byteLength !== asset.byteLength) { + throw new Error( + `bundle asset ${asset.path} is ${bytes?.byteLength ?? 'absent'}, not the manifest's ${asset.byteLength}` + ) + } + return bytes +} diff --git a/mobile/src/mobile-web-shell/host-cache-key.ts b/mobile/src/mobile-web-shell/host-cache-key.ts new file mode 100644 index 00000000000..0058c4eebca --- /dev/null +++ b/mobile/src/mobile-web-shell/host-cache-key.ts @@ -0,0 +1,19 @@ +import { sha256 } from '@noble/hashes/sha256' + +/** Full sha256 hex, never a slice of the host id and never the id itself: the key names the + * directory that holds one host's bundle, two hosts sharing one is the cross-host cache use the + * rollback runbook escalates as a security incident, and a host id is free-form text that would + * otherwise reach a path. `deriveHostFingerprint` is not this: it hashes the host public key and + * truncates to 16 chars for the push gateway. */ +export function deriveHostCacheKey(hostId: string): string { + return Array.from(sha256(new TextEncoder().encode(hostId)), (byte) => + byte.toString(16).padStart(2, '0') + ).join('') +} + +const HOST_CACHE_KEY_PATTERN = /^[a-f0-9]{64}$/ + +/** The store checks every key it is handed, so a caller passing a raw host id cannot build a path. */ +export function isHostCacheKey(value: string): boolean { + return HOST_CACHE_KEY_PATTERN.test(value) +} diff --git a/mobile/src/transport/mobile-web-bundle-reply-schemas.ts b/mobile/src/transport/mobile-web-bundle-reply-schemas.ts index ca0787b7568..f42d50c3015 100644 --- a/mobile/src/transport/mobile-web-bundle-reply-schemas.ts +++ b/mobile/src/transport/mobile-web-bundle-reply-schemas.ts @@ -37,8 +37,12 @@ const assetSchema = z.looseObject({ * `schemaVersion` is read as a number, not pinned to the one this shell knows: refusing it here * would fail the parse before `evaluateMobileWebBundleCompat` could name the shell as too old, and * an unreadable schema is a wall to show, not a shape to guess at. The manifest stays closed in - * both directions on the host's side, where it is written. */ -const manifestSchema = z + * both directions on the host's side, where it is written. + * + * Exported because the generation store re-parses the manifest it cached, and reading it back + * strictly after accepting it loosely would make a host's added field a forced redownload on every + * launch. */ +export const MobileWebBundleManifestReadSchema = z .looseObject({ schemaVersion: z.number().int(), buildId: z.string().regex(SHA256_PATTERN), @@ -63,7 +67,7 @@ const manifestSchema = z /** `chunkBytes` is read, never assumed: the host may shrink it without a client release. Capped at * the constant because a larger value would overshoot `dataBase64` above. */ export const MobileWebBundleManifestReplySchema = z.looseObject({ - manifest: manifestSchema, + manifest: MobileWebBundleManifestReadSchema, chunkBytes: z.number().int().positive().max(MOBILE_WEB_BUNDLE_CHUNK_BYTES) })