mirror of
https://github.com/stablyai/orca.git
synced 2026-10-09 00:02:39 +00:00
fix(chat): enforce legacy import byte budget during reading
This commit is contained in:
@@ -0,0 +1,60 @@
|
||||
import { createReadStream } from 'node:fs'
|
||||
import { appendFile, mkdtemp, rm, stat, writeFile } from 'node:fs/promises'
|
||||
import type * as FsPromises from 'node:fs/promises'
|
||||
import { tmpdir } from 'node:os'
|
||||
import { join } from 'node:path'
|
||||
import { afterEach, describe, expect, it, vi } from 'vitest'
|
||||
import { prepareLegacyTranscriptImport } from './journal-legacy-import'
|
||||
|
||||
vi.mock(import('node:fs'), async (importOriginal) => {
|
||||
const original = await importOriginal()
|
||||
return { ...original, createReadStream: vi.fn(original.createReadStream) }
|
||||
})
|
||||
|
||||
vi.mock(import('node:fs/promises'), async (importOriginal) => {
|
||||
const original = await importOriginal()
|
||||
return { ...original, stat: vi.fn() }
|
||||
})
|
||||
|
||||
const SOURCE_LIMIT_BYTES = 16 * 1024 * 1024
|
||||
const roots: string[] = []
|
||||
|
||||
afterEach(async () => {
|
||||
vi.clearAllMocks()
|
||||
await Promise.all(roots.splice(0).map((root) => rm(root, { recursive: true, force: true })))
|
||||
})
|
||||
|
||||
describe('legacy import source byte bound', () => {
|
||||
it('refuses a source that grows past the limit after stat and closes the stream', async () => {
|
||||
const root = await mkdtemp(join(tmpdir(), 'orca-legacy-import-bound-'))
|
||||
roots.push(root)
|
||||
const filePath = join(root, 'growing.jsonl')
|
||||
await writeFile(
|
||||
filePath,
|
||||
`${JSON.stringify({
|
||||
type: 'assistant',
|
||||
uuid: 'first-message',
|
||||
message: { role: 'assistant', content: [{ type: 'text', text: 'Keep the prior journal' }] }
|
||||
})}\n`
|
||||
)
|
||||
const actualFs = await vi.importActual<typeof FsPromises>('node:fs/promises')
|
||||
const beforeGrowth = await actualFs.stat(filePath)
|
||||
vi.mocked(stat).mockImplementationOnce(async () => {
|
||||
await appendFile(filePath, Buffer.alloc(SOURCE_LIMIT_BYTES, 0x20))
|
||||
return beforeGrowth
|
||||
})
|
||||
|
||||
const result = await prepareLegacyTranscriptImport({
|
||||
agent: 'claude',
|
||||
sessionId: 'source-bound',
|
||||
options: { filePath }
|
||||
})
|
||||
|
||||
expect(result).toEqual({
|
||||
ok: false,
|
||||
error: expect.stringContaining(`${SOURCE_LIMIT_BYTES} byte limit`)
|
||||
})
|
||||
expect(createReadStream).toHaveBeenCalledOnce()
|
||||
expect(vi.mocked(createReadStream).mock.results[0]?.value.destroyed).toBe(true)
|
||||
})
|
||||
})
|
||||
@@ -194,7 +194,8 @@ async function decodeWithIdentities(input: {
|
||||
const identities: AgentJournalItemIdentity[] = []
|
||||
let lineIndex = 0
|
||||
|
||||
const stream = createReadStream(input.filePath, { encoding: 'utf-8' })
|
||||
// Count raw bytes while reading: the source can grow after the stat check.
|
||||
const stream = createReadStream(input.filePath)
|
||||
const { messages } = await decodeTranscriptStream(
|
||||
stream,
|
||||
input.filePath,
|
||||
@@ -217,7 +218,8 @@ async function decodeWithIdentities(input: {
|
||||
}
|
||||
return message
|
||||
},
|
||||
true
|
||||
true,
|
||||
MAX_LEGACY_IMPORT_SOURCE_BYTES
|
||||
)
|
||||
return { messages, identities }
|
||||
}
|
||||
|
||||
@@ -11,6 +11,35 @@ const decode = (line: string, id: string) => ({
|
||||
})
|
||||
|
||||
describe('decodeTranscriptStream', () => {
|
||||
it('accepts the exact source limit with split UTF-8 bytes and preserves order', async () => {
|
||||
const bytes = Buffer.from('é\n😀\n')
|
||||
const result = await decodeTranscriptStream(
|
||||
Readable.from([bytes.subarray(0, 1), bytes.subarray(1, 5), bytes.subarray(5)]),
|
||||
'/chat.jsonl',
|
||||
0,
|
||||
decode,
|
||||
true,
|
||||
bytes.length
|
||||
)
|
||||
expect(result.messages.map((message) => message.blocks[0])).toEqual([
|
||||
{ type: 'text', text: 'é' },
|
||||
{ type: 'text', text: '😀' }
|
||||
])
|
||||
})
|
||||
|
||||
it.each([Buffer.from('é'), 'é', Buffer.from([0xff, 0xff])])(
|
||||
'counts source bytes before decoding an oversized chunk %j',
|
||||
async (chunk) => {
|
||||
const stream = Readable.from([chunk])
|
||||
const trackedDecode = vi.fn(decode)
|
||||
await expect(
|
||||
decodeTranscriptStream(stream, '/chat.jsonl', 0, trackedDecode, true, 1)
|
||||
).rejects.toThrow('exceeds 1 byte limit')
|
||||
expect(trackedDecode).not.toHaveBeenCalled()
|
||||
expect(stream.destroyed).toBe(true)
|
||||
}
|
||||
)
|
||||
|
||||
it.each([true, false])('preserves chunked record offsets with trailing=%s', async (trailing) => {
|
||||
const first = `${'long record '.repeat(10_000)}😀`
|
||||
const prefix = `\r\n${first}\r\n\n`
|
||||
|
||||
@@ -1,6 +1,7 @@
|
||||
import type { Readable } from 'node:stream'
|
||||
import { StringDecoder } from 'node:string_decoder'
|
||||
import type { NativeChatMessage } from '../../shared/native-chat-types'
|
||||
import { NodeReadableTextTooLargeError } from '../../shared/node-readable-text'
|
||||
import { transcriptFallbackId } from './transcript-fallback-id'
|
||||
|
||||
type TranscriptDecoder = (line: string, fallbackId: string) => NativeChatMessage | null
|
||||
@@ -10,10 +11,12 @@ export async function decodeTranscriptStream(
|
||||
filePath: string,
|
||||
start: number,
|
||||
decode: TranscriptDecoder,
|
||||
includeTrailingLine: boolean
|
||||
includeTrailingLine: boolean,
|
||||
maxSourceBytes = Infinity
|
||||
): Promise<{ messages: NativeChatMessage[]; consumedBytes: number }> {
|
||||
const messages: NativeChatMessage[] = []
|
||||
let consumedBytes = 0
|
||||
let sourceBytes = 0
|
||||
const framer = createTranscriptLineFramer((line, byteLength, terminated) => {
|
||||
if (terminated || includeTrailingLine) {
|
||||
decodeLine(line, consumedBytes)
|
||||
@@ -21,6 +24,12 @@ export async function decodeTranscriptStream(
|
||||
}
|
||||
})
|
||||
for await (const chunk of stream) {
|
||||
if (maxSourceBytes !== Infinity) {
|
||||
sourceBytes += Buffer.isBuffer(chunk) ? chunk.byteLength : Buffer.byteLength(chunk, 'utf8')
|
||||
if (sourceBytes > maxSourceBytes) {
|
||||
throw new NodeReadableTextTooLargeError(sourceBytes, maxSourceBytes)
|
||||
}
|
||||
}
|
||||
framer.write(chunk)
|
||||
}
|
||||
framer.end()
|
||||
|
||||
Reference in New Issue
Block a user