Files
archived-orca/src/main/usage/cache/usage-cache-json.ts
T
Brennan Benson 52b618339f Stream protected usage caches without whole-file buffers (#41478)
* Stream usage source caches without whole-document strings

* Keep the validated source-array type through migration

* Narrow migrated source records without an assertion

* Assert cache preservation without runtime-specific error copy

* Stream protected usage cache persistence and verification

* Assert bounded usage cache input before gzip compression

* Retain compressed usage cache publication bounds

* Verify persisted output in usage cache streaming contract
2026-11-12 02:55:43 -08:00

155 lines
4.3 KiB
TypeScript

import { createReadStream } from 'node:fs'
import Assembler from 'stream-json/assembler.js'
import { createJsonTokenReader } from '../../../shared/json-stream/json-token-reader'
import {
JsonStringifyByteLimitError,
stringifyJsonWithinByteLimit
} from '../../../shared/json-stream/node-bounded-json-stringify'
const TEXT_BATCH_CHARS = 64 * 1024
const STRING_SLICE_CHARS = 4 * 1024
function prepareValue(value: unknown, key: string): unknown {
if (value !== null && typeof value === 'object') {
if ('toJSON' in value && typeof value.toJSON === 'function') {
value = value.toJSON(key)
}
const tag = Object.prototype.toString.call(value)
if (
value !== null &&
typeof value === 'object' &&
(tag === '[object Number]' || tag === '[object String]' || tag === '[object Boolean]')
) {
return value.valueOf()
}
}
return value
}
function omitted(value: unknown): boolean {
return value === undefined || typeof value === 'function' || typeof value === 'symbol'
}
function* encodeString(value: string): Generator<string> {
if (value.length <= STRING_SLICE_CHARS) {
yield JSON.stringify(value)
return
}
yield '"'
for (let offset = 0; offset < value.length;) {
let end = Math.min(offset + STRING_SLICE_CHARS, value.length)
const last = value.charCodeAt(end - 1)
const next = value.charCodeAt(end)
// Each piece is encoded independently by both file writes and integrity hashes.
if (last >= 0xd800 && last <= 0xdbff && next >= 0xdc00 && next <= 0xdfff) {
end--
}
yield JSON.stringify(value.slice(offset, end)).slice(1, -1)
offset = end
}
yield '"'
}
function* encodeValue(
value: unknown,
ancestors: Set<object>,
nativeRecord = false
): Generator<string> {
if (typeof value === 'string') {
yield* encodeString(value)
return
}
if (value === null || typeof value !== 'object') {
const text = JSON.stringify(value)
if (text === undefined) {
throw new TypeError('Usage cache value is not JSON serializable')
}
yield text
return
}
if (ancestors.has(value)) {
throw new TypeError('Usage cache contains a circular value')
}
if (nativeRecord) {
try {
yield stringifyJsonWithinByteLimit(value, 256 * 1024).serialized
return
} catch (error) {
if (!(error instanceof JsonStringifyByteLimitError)) {
throw error
}
}
}
ancestors.add(value)
try {
if (Array.isArray(value)) {
yield '['
for (let index = 0; index < value.length; index++) {
if (index > 0) {
yield ','
}
const item = prepareValue(value[index], String(index))
yield* encodeValue(omitted(item) ? null : item, ancestors, true)
}
yield ']'
} else {
yield '{'
let first = true
for (const [key, raw] of Object.entries(value)) {
const item = prepareValue(raw, key)
if (omitted(item)) {
continue
}
if (!first) {
yield ','
}
first = false
yield* encodeString(key)
yield ':'
yield* encodeValue(item, ancestors)
}
yield '}'
}
} finally {
ancestors.delete(value)
}
}
/** Batch traversal output without materializing a whole source, string or document. */
export function* encodeUsageCacheJson(value: unknown): Generator<string> {
let parts: string[] = []
let length = 0
for (const part of encodeValue(prepareValue(value, ''), new Set())) {
if (length + part.length > TEXT_BATCH_CHARS && length > 0) {
yield parts.join('')
parts = []
length = 0
}
parts.push(part)
length += part.length
}
if (length > 0) {
yield parts.join('')
}
}
/** EOF validation precedes exposing any of the privately assembled records. */
export async function parseUsageCacheJson<T>(chunks: AsyncIterable<string>): Promise<T | null> {
const assembler = new Assembler<T>()
const reader = createJsonTokenReader((token) => assembler.consume(token))
for await (const text of chunks) {
reader.write(text)
}
reader.end()
return assembler.current
}
export function readUsageCacheJson<T>(path: string): Promise<T | null> {
return parseUsageCacheJson<T>(
createReadStream(path, {
encoding: 'utf8',
highWaterMark: TEXT_BATCH_CHARS
})
)
}