fix(chat): enforce legacy import byte budget during reading

This commit is contained in:
m4air
2026-09-15 23:31:11 -07:00
parent c70b63ca8b
commit b27aa26bb7
7 changed files with 388 additions and 3 deletions
@@ -0,0 +1,60 @@
import { createReadStream } from 'node:fs'
import { appendFile, mkdtemp, rm, stat, writeFile } from 'node:fs/promises'
import type * as FsPromises from 'node:fs/promises'
import { tmpdir } from 'node:os'
import { join } from 'node:path'
import { afterEach, describe, expect, it, vi } from 'vitest'
import { prepareLegacyTranscriptImport } from './journal-legacy-import'
vi.mock(import('node:fs'), async (importOriginal) => {
const original = await importOriginal()
return { ...original, createReadStream: vi.fn(original.createReadStream) }
})
vi.mock(import('node:fs/promises'), async (importOriginal) => {
const original = await importOriginal()
return { ...original, stat: vi.fn() }
})
const SOURCE_LIMIT_BYTES = 16 * 1024 * 1024
const roots: string[] = []
afterEach(async () => {
vi.clearAllMocks()
await Promise.all(roots.splice(0).map((root) => rm(root, { recursive: true, force: true })))
})
describe('legacy import source byte bound', () => {
it('refuses a source that grows past the limit after stat and closes the stream', async () => {
const root = await mkdtemp(join(tmpdir(), 'orca-legacy-import-bound-'))
roots.push(root)
const filePath = join(root, 'growing.jsonl')
await writeFile(
filePath,
`${JSON.stringify({
type: 'assistant',
uuid: 'first-message',
message: { role: 'assistant', content: [{ type: 'text', text: 'Keep the prior journal' }] }
})}\n`
)
const actualFs = await vi.importActual<typeof FsPromises>('node:fs/promises')
const beforeGrowth = await actualFs.stat(filePath)
vi.mocked(stat).mockImplementationOnce(async () => {
await appendFile(filePath, Buffer.alloc(SOURCE_LIMIT_BYTES, 0x20))
return beforeGrowth
})
const result = await prepareLegacyTranscriptImport({
agent: 'claude',
sessionId: 'source-bound',
options: { filePath }
})
expect(result).toEqual({
ok: false,
error: expect.stringContaining(`${SOURCE_LIMIT_BYTES} byte limit`)
})
expect(createReadStream).toHaveBeenCalledOnce()
expect(vi.mocked(createReadStream).mock.results[0]?.value.destroyed).toBe(true)
})
})
@@ -194,7 +194,8 @@ async function decodeWithIdentities(input: {
const identities: AgentJournalItemIdentity[] = []
let lineIndex = 0
const stream = createReadStream(input.filePath, { encoding: 'utf-8' })
// Count raw bytes while reading: the source can grow after the stat check.
const stream = createReadStream(input.filePath)
const { messages } = await decodeTranscriptStream(
stream,
input.filePath,
@@ -217,7 +218,8 @@ async function decodeWithIdentities(input: {
}
return message
},
true
true,
MAX_LEGACY_IMPORT_SOURCE_BYTES
)
return { messages, identities }
}
@@ -11,6 +11,35 @@ const decode = (line: string, id: string) => ({
})
describe('decodeTranscriptStream', () => {
it('accepts the exact source limit with split UTF-8 bytes and preserves order', async () => {
const bytes = Buffer.from('é\n😀\n')
const result = await decodeTranscriptStream(
Readable.from([bytes.subarray(0, 1), bytes.subarray(1, 5), bytes.subarray(5)]),
'/chat.jsonl',
0,
decode,
true,
bytes.length
)
expect(result.messages.map((message) => message.blocks[0])).toEqual([
{ type: 'text', text: 'é' },
{ type: 'text', text: '😀' }
])
})
it.each([Buffer.from('é'), 'é', Buffer.from([0xff, 0xff])])(
'counts source bytes before decoding an oversized chunk %j',
async (chunk) => {
const stream = Readable.from([chunk])
const trackedDecode = vi.fn(decode)
await expect(
decodeTranscriptStream(stream, '/chat.jsonl', 0, trackedDecode, true, 1)
).rejects.toThrow('exceeds 1 byte limit')
expect(trackedDecode).not.toHaveBeenCalled()
expect(stream.destroyed).toBe(true)
}
)
it.each([true, false])('preserves chunked record offsets with trailing=%s', async (trailing) => {
const first = `${'long record '.repeat(10_000)}😀`
const prefix = `\r\n${first}\r\n\n`
@@ -1,6 +1,7 @@
import type { Readable } from 'node:stream'
import { StringDecoder } from 'node:string_decoder'
import type { NativeChatMessage } from '../../shared/native-chat-types'
import { NodeReadableTextTooLargeError } from '../../shared/node-readable-text'
import { transcriptFallbackId } from './transcript-fallback-id'
type TranscriptDecoder = (line: string, fallbackId: string) => NativeChatMessage | null
@@ -10,10 +11,12 @@ export async function decodeTranscriptStream(
filePath: string,
start: number,
decode: TranscriptDecoder,
includeTrailingLine: boolean
includeTrailingLine: boolean,
maxSourceBytes = Infinity
): Promise<{ messages: NativeChatMessage[]; consumedBytes: number }> {
const messages: NativeChatMessage[] = []
let consumedBytes = 0
let sourceBytes = 0
const framer = createTranscriptLineFramer((line, byteLength, terminated) => {
if (terminated || includeTrailingLine) {
decodeLine(line, consumedBytes)
@@ -21,6 +24,12 @@ export async function decodeTranscriptStream(
}
})
for await (const chunk of stream) {
if (maxSourceBytes !== Infinity) {
sourceBytes += Buffer.isBuffer(chunk) ? chunk.byteLength : Buffer.byteLength(chunk, 'utf8')
if (sourceBytes > maxSourceBytes) {
throw new NodeReadableTextTooLargeError(sourceBytes, maxSourceBytes)
}
}
framer.write(chunk)
}
framer.end()