mirror of
https://github.com/stablyai/orca.git
synced 2026-09-22 08:02:28 +00:00
refactor(persistence): remove unwired state bounds (#13499)
This commit is contained in:
Neil
@@ -1,171 +0,0 @@
|
||||
import { createHash } from 'node:crypto'
|
||||
import {
|
||||
mkdtempSync,
|
||||
readFileSync,
|
||||
readdirSync,
|
||||
rmSync,
|
||||
truncateSync,
|
||||
writeFileSync
|
||||
} from 'node:fs'
|
||||
import { tmpdir } from 'node:os'
|
||||
import { join } from 'node:path'
|
||||
import { afterEach, beforeEach, describe, expect, it } from 'vitest'
|
||||
import { NodeFileReadTooLargeError } from './node-bounded-file-reader'
|
||||
import { JsonStringifyByteLimitError } from './node-bounded-json-stringify'
|
||||
import {
|
||||
PersistedStateSecretCapacityError,
|
||||
assertPersistedStateSecretWithinLimit,
|
||||
readPersistedStateJsonFileSync,
|
||||
replacePersistedStateJsonWithinLimit,
|
||||
restorePersistedStateBackupSync,
|
||||
stringifyPrettyPersistedStateWithinLimit,
|
||||
stringifyPersistedStateWithinLimit,
|
||||
updatePersistedStateHashWithJsonRange
|
||||
} from './persisted-state-file-bounds'
|
||||
|
||||
describe('persisted state file bounds', () => {
|
||||
let root = ''
|
||||
|
||||
beforeEach(() => {
|
||||
root = mkdtempSync(join(tmpdir(), 'orca-state-bounds-'))
|
||||
})
|
||||
|
||||
afterEach(() => {
|
||||
rmSync(root, { recursive: true, force: true })
|
||||
})
|
||||
|
||||
it('reads and parses a state file exactly at the byte limit', () => {
|
||||
const path = join(root, 'state.json')
|
||||
const json = `{"value":"${'x'.repeat(20)}"}`
|
||||
writeFileSync(path, json)
|
||||
|
||||
expect(
|
||||
readPersistedStateJsonFileSync<{ value: string }>(path, Buffer.byteLength(json))
|
||||
).toEqual({
|
||||
byteLength: Buffer.byteLength(json),
|
||||
value: { value: 'x'.repeat(20) }
|
||||
})
|
||||
})
|
||||
|
||||
it('rejects an oversized sparse state file before reading its body', () => {
|
||||
const path = join(root, 'state.json')
|
||||
writeFileSync(path, '')
|
||||
truncateSync(path, 1025)
|
||||
|
||||
expect(() => readPersistedStateJsonFileSync(path, 1024)).toThrow(NodeFileReadTooLargeError)
|
||||
})
|
||||
|
||||
it('rejects structurally amplified state before parsing it', () => {
|
||||
const path = join(root, 'state.json')
|
||||
const json = '{"rows":[{},{}]}'
|
||||
writeFileSync(path, json)
|
||||
|
||||
expect(() =>
|
||||
readPersistedStateJsonFileSync(path, Buffer.byteLength(json), {
|
||||
structuralTokens: 7,
|
||||
nestingDepth: 3
|
||||
})
|
||||
).toThrow('JSON structure')
|
||||
})
|
||||
|
||||
it('matches native compact JSON exactly at the output boundary', () => {
|
||||
const state = { quote: '"', unicode: '🐋', nested: [1, true, null] }
|
||||
const native = JSON.stringify(state)
|
||||
|
||||
expect(stringifyPersistedStateWithinLimit(state, Buffer.byteLength(native))).toEqual({
|
||||
byteLength: Buffer.byteLength(native),
|
||||
serialized: native
|
||||
})
|
||||
expect(() => stringifyPersistedStateWithinLimit(state, Buffer.byteLength(native) - 1)).toThrow(
|
||||
JsonStringifyByteLimitError
|
||||
)
|
||||
})
|
||||
|
||||
it('matches native pretty JSON and enforces its whitespace-inclusive boundary', () => {
|
||||
const state = { nested: { value: 'x' }, list: [1, 2] }
|
||||
const native = JSON.stringify(state, null, 2)
|
||||
|
||||
expect(stringifyPrettyPersistedStateWithinLimit(state, Buffer.byteLength(native))).toEqual({
|
||||
byteLength: Buffer.byteLength(native),
|
||||
serialized: native
|
||||
})
|
||||
expect(() =>
|
||||
stringifyPrettyPersistedStateWithinLimit(state, Buffer.byteLength(native) - 1)
|
||||
).toThrow(JsonStringifyByteLimitError)
|
||||
})
|
||||
|
||||
it('bounds secret plaintext before encryption can expand it', () => {
|
||||
assertPersistedStateSecretWithinLimit('🐋', 4)
|
||||
|
||||
expect(() => assertPersistedStateSecretWithinLimit('🐋x', 4)).toThrow(
|
||||
PersistedStateSecretCapacityError
|
||||
)
|
||||
})
|
||||
|
||||
it('checks replacement growth before constructing the next payload', () => {
|
||||
const serialized = '{"value":"slot"}'
|
||||
const exactBytes =
|
||||
Buffer.byteLength(serialized) - Buffer.byteLength('slot') + Buffer.byteLength('expanded')
|
||||
|
||||
expect(
|
||||
replacePersistedStateJsonWithinLimit({
|
||||
serialized,
|
||||
currentBytes: Buffer.byteLength(serialized),
|
||||
search: 'slot',
|
||||
replacement: 'expanded',
|
||||
maxBytes: exactBytes
|
||||
})
|
||||
).toEqual({ byteLength: exactBytes, serialized: '{"value":"expanded"}' })
|
||||
expect(() =>
|
||||
replacePersistedStateJsonWithinLimit({
|
||||
serialized,
|
||||
currentBytes: Buffer.byteLength(serialized),
|
||||
search: 'slot',
|
||||
replacement: 'expanded',
|
||||
maxBytes: exactBytes - 1
|
||||
})
|
||||
).toThrow(JsonStringifyByteLimitError)
|
||||
})
|
||||
|
||||
it('hashes bounded string ranges without splitting UTF-16 surrogate pairs', () => {
|
||||
const value = `prefix-${'x'.repeat(8)}🐋-${'y'.repeat(8)}-suffix`
|
||||
const expected = createHash('sha1').update(value).digest('hex')
|
||||
const actual = createHash('sha1')
|
||||
|
||||
updatePersistedStateHashWithJsonRange(actual, value, 0, value.length, 2)
|
||||
|
||||
expect(actual.digest('hex')).toBe(expected)
|
||||
})
|
||||
|
||||
it('atomically restores only a valid in-limit backup', () => {
|
||||
const backupPath = join(root, 'backup.json')
|
||||
const targetPath = join(root, 'profile', 'orca-data.json')
|
||||
writeFileSync(backupPath, '{"repos":[{"id":"recovered"}]}')
|
||||
|
||||
restorePersistedStateBackupSync(backupPath, targetPath, 1024)
|
||||
|
||||
expect(JSON.parse(readFileSync(targetPath, 'utf8'))).toEqual({
|
||||
repos: [{ id: 'recovered' }]
|
||||
})
|
||||
const originalTarget = readFileSync(targetPath)
|
||||
writeFileSync(backupPath, '{{invalid')
|
||||
expect(() => restorePersistedStateBackupSync(backupPath, targetPath, 1024)).toThrow()
|
||||
expect(readFileSync(targetPath)).toEqual(originalTarget)
|
||||
expect(
|
||||
readdirSync(join(root, 'profile')).filter((name) => name.endsWith('.recovery.tmp'))
|
||||
).toEqual([])
|
||||
})
|
||||
|
||||
it('leaves the target untouched when a backup exceeds the cap', () => {
|
||||
const backupPath = join(root, 'backup.json')
|
||||
const targetPath = join(root, 'orca-data.json')
|
||||
writeFileSync(targetPath, '{"original":true}')
|
||||
writeFileSync(backupPath, '')
|
||||
truncateSync(backupPath, 1025)
|
||||
|
||||
expect(() => restorePersistedStateBackupSync(backupPath, targetPath, 1024)).toThrow(
|
||||
NodeFileReadTooLargeError
|
||||
)
|
||||
expect(readFileSync(targetPath, 'utf8')).toBe('{"original":true}')
|
||||
})
|
||||
})
|
||||
@@ -1,204 +0,0 @@
|
||||
import { randomUUID, type Hash } from 'node:crypto'
|
||||
import { mkdirSync, renameSync, rmSync, writeFileSync } from 'node:fs'
|
||||
import { dirname } from 'node:path'
|
||||
import { readNodeFileSyncWithinLimit, type BoundedNodeFileRead } from './node-bounded-file-reader'
|
||||
import {
|
||||
JsonStringifyByteLimitError,
|
||||
stringifyJsonWithinByteLimit
|
||||
} from './node-bounded-json-stringify'
|
||||
import {
|
||||
assertJsonTextStructureWithinLimits,
|
||||
type JsonTextStructureLimits
|
||||
} from './json-text-structure-limit'
|
||||
|
||||
export const ORCA_PERSISTED_STATE_MAX_BYTES = 64 * 1024 * 1024
|
||||
export const ORCA_PERSISTED_STATE_SECRET_MAX_BYTES = 4 * 1024 * 1024
|
||||
export const ORCA_PERSISTED_STATE_HASH_CHUNK_CODE_UNITS = 64 * 1024
|
||||
export const ORCA_PERSISTED_STATE_JSON_LIMITS: JsonTextStructureLimits = {
|
||||
structuralTokens: 4_000_000,
|
||||
nestingDepth: 256
|
||||
}
|
||||
|
||||
export type PersistedStateJsonRead<T> = {
|
||||
byteLength: number
|
||||
value: T
|
||||
}
|
||||
|
||||
export class PersistedStateSecretCapacityError extends Error {
|
||||
constructor(
|
||||
readonly observedBytes: number,
|
||||
readonly maxBytes = ORCA_PERSISTED_STATE_SECRET_MAX_BYTES
|
||||
) {
|
||||
super(`Persisted state secret exceeds ${maxBytes} bytes`)
|
||||
this.name = 'PersistedStateSecretCapacityError'
|
||||
}
|
||||
}
|
||||
|
||||
export function readPersistedStateJsonFileSync<T>(
|
||||
filePath: string,
|
||||
maxBytes = ORCA_PERSISTED_STATE_MAX_BYTES,
|
||||
structureLimits: JsonTextStructureLimits = ORCA_PERSISTED_STATE_JSON_LIMITS
|
||||
): PersistedStateJsonRead<T> {
|
||||
const { buffer } = readPersistedStateFileBytesSync(filePath, maxBytes)
|
||||
return {
|
||||
byteLength: buffer.byteLength,
|
||||
value: parsePersistedStateJsonBuffer<T>(buffer, structureLimits)
|
||||
}
|
||||
}
|
||||
|
||||
export function readPersistedStateFileBytesSync(
|
||||
filePath: string,
|
||||
maxBytes = ORCA_PERSISTED_STATE_MAX_BYTES
|
||||
): BoundedNodeFileRead {
|
||||
return readNodeFileSyncWithinLimit(filePath, maxBytes)
|
||||
}
|
||||
|
||||
export function parsePersistedStateJsonBuffer<T>(
|
||||
buffer: Buffer,
|
||||
structureLimits: JsonTextStructureLimits = ORCA_PERSISTED_STATE_JSON_LIMITS
|
||||
): T {
|
||||
const serialized = buffer.toString('utf8')
|
||||
assertJsonTextStructureWithinLimits(serialized, structureLimits)
|
||||
return JSON.parse(serialized) as T
|
||||
}
|
||||
|
||||
export function stringifyPersistedStateWithinLimit(
|
||||
value: unknown,
|
||||
maxBytes = ORCA_PERSISTED_STATE_MAX_BYTES
|
||||
): { byteLength: number; serialized: string } {
|
||||
return stringifyJsonWithinByteLimit(value, maxBytes)
|
||||
}
|
||||
|
||||
export function stringifyPrettyPersistedStateWithinLimit(
|
||||
value: unknown,
|
||||
maxBytes = ORCA_PERSISTED_STATE_MAX_BYTES
|
||||
): { byteLength: number; serialized: string } {
|
||||
return stringifyJsonWithinByteLimit(value, maxBytes, 2)
|
||||
}
|
||||
|
||||
export function assertPersistedStateSecretWithinLimit(
|
||||
value: string,
|
||||
maxBytes = ORCA_PERSISTED_STATE_SECRET_MAX_BYTES
|
||||
): void {
|
||||
const observedBytes = Buffer.byteLength(value, 'utf8')
|
||||
if (observedBytes > maxBytes) {
|
||||
throw new PersistedStateSecretCapacityError(observedBytes, maxBytes)
|
||||
}
|
||||
}
|
||||
|
||||
export function replacedPersistedStateJsonByteLength(options: {
|
||||
currentBytes: number
|
||||
maxBytes?: number
|
||||
replacement: string
|
||||
search: string
|
||||
}): number {
|
||||
const maxBytes = options.maxBytes ?? ORCA_PERSISTED_STATE_MAX_BYTES
|
||||
if (!Number.isSafeInteger(maxBytes) || maxBytes < 0) {
|
||||
throw new RangeError('Persisted state JSON byte limit must be a non-negative safe integer')
|
||||
}
|
||||
if (!Number.isSafeInteger(options.currentBytes) || options.currentBytes < 0) {
|
||||
throw new RangeError('Persisted state JSON byte count must be a non-negative safe integer')
|
||||
}
|
||||
if (options.currentBytes > maxBytes) {
|
||||
throw new JsonStringifyByteLimitError(options.currentBytes, maxBytes)
|
||||
}
|
||||
const nextBytes =
|
||||
options.currentBytes -
|
||||
Buffer.byteLength(options.search, 'utf8') +
|
||||
Buffer.byteLength(options.replacement, 'utf8')
|
||||
if (!Number.isSafeInteger(nextBytes) || nextBytes < 0 || nextBytes > maxBytes) {
|
||||
throw new JsonStringifyByteLimitError(nextBytes, maxBytes)
|
||||
}
|
||||
return nextBytes
|
||||
}
|
||||
|
||||
export function replacePersistedStateJsonWithinLimit(options: {
|
||||
currentBytes: number
|
||||
maxBytes?: number
|
||||
replacement: string
|
||||
search: string
|
||||
serialized: string
|
||||
}): { byteLength: number; serialized: string } {
|
||||
const byteLength = replacedPersistedStateJsonByteLength(options)
|
||||
const searchIndex = options.serialized.indexOf(options.search)
|
||||
if (searchIndex === -1) {
|
||||
throw new Error('Persisted state JSON replacement slot is missing')
|
||||
}
|
||||
if (options.serialized.includes(options.search, searchIndex + options.search.length)) {
|
||||
throw new Error('Persisted state JSON replacement slot is ambiguous')
|
||||
}
|
||||
return {
|
||||
byteLength,
|
||||
serialized: options.serialized.replace(options.search, () => options.replacement)
|
||||
}
|
||||
}
|
||||
|
||||
export function updatePersistedStateHashWithJsonRange(
|
||||
hash: Pick<Hash, 'update'>,
|
||||
value: string,
|
||||
start = 0,
|
||||
end = value.length,
|
||||
chunkCodeUnits = ORCA_PERSISTED_STATE_HASH_CHUNK_CODE_UNITS
|
||||
): void {
|
||||
if (
|
||||
!Number.isSafeInteger(start) ||
|
||||
!Number.isSafeInteger(end) ||
|
||||
start < 0 ||
|
||||
end < start ||
|
||||
end > value.length
|
||||
) {
|
||||
throw new RangeError('Persisted state hash range is invalid')
|
||||
}
|
||||
if (!Number.isSafeInteger(chunkCodeUnits) || chunkCodeUnits <= 0) {
|
||||
throw new RangeError('Persisted state hash chunk size must be a positive safe integer')
|
||||
}
|
||||
|
||||
let offset = start
|
||||
while (offset < end) {
|
||||
let nextOffset = Math.min(end, offset + chunkCodeUnits)
|
||||
if (
|
||||
nextOffset < end &&
|
||||
isHighSurrogate(value.charCodeAt(nextOffset - 1)) &&
|
||||
isLowSurrogate(value.charCodeAt(nextOffset))
|
||||
) {
|
||||
nextOffset += 1
|
||||
}
|
||||
hash.update(value.slice(offset, nextOffset), 'utf8')
|
||||
offset = nextOffset
|
||||
}
|
||||
}
|
||||
|
||||
export function restorePersistedStateBackupSync(
|
||||
sourcePath: string,
|
||||
targetPath: string,
|
||||
maxBytes = ORCA_PERSISTED_STATE_MAX_BYTES
|
||||
): number {
|
||||
const read = readValidatedPersistedStateBytesSync(sourcePath, maxBytes)
|
||||
mkdirSync(dirname(targetPath), { recursive: true })
|
||||
const temporaryPath = `${targetPath}.${process.pid}.${randomUUID()}.recovery.tmp`
|
||||
try {
|
||||
writeFileSync(temporaryPath, read.buffer)
|
||||
renameSync(temporaryPath, targetPath)
|
||||
} catch (error) {
|
||||
rmSync(temporaryPath, { force: true })
|
||||
throw error
|
||||
}
|
||||
return read.buffer.byteLength
|
||||
}
|
||||
|
||||
function readValidatedPersistedStateBytesSync(
|
||||
filePath: string,
|
||||
maxBytes: number
|
||||
): BoundedNodeFileRead {
|
||||
const read = readPersistedStateFileBytesSync(filePath, maxBytes)
|
||||
parsePersistedStateJsonBuffer(read.buffer)
|
||||
return read
|
||||
}
|
||||
|
||||
function isHighSurrogate(code: number): boolean {
|
||||
return code >= 0xd800 && code <= 0xdbff
|
||||
}
|
||||
|
||||
function isLowSurrogate(code: number): boolean {
|
||||
return code >= 0xdc00 && code <= 0xdfff
|
||||
}
|
||||
Reference in New Issue
Block a user