orca/src/shared/json-token-reader.ts

import parser, { type Token } from 'stream-json/core/parser.js'
import exec from 'stream-chain/exec.js'
import { none } from 'stream-chain/defs.js'

/** Bounds token batches while preserving caller-owned UTF-8 decoding. */
export function createJsonTokenReader(
  consume: (token: Token) => void,
  batchCodeUnits = 64 * 1024,
  retainScalar?: () => boolean
): {
  write: (text: string) => void
  end: () => void
} {
  if (!Number.isSafeInteger(batchCodeUnits) || batchCodeUnits < 1) {
    throw new RangeError('JSON token batch size must be a positive safe integer')
  }
  const parse = exec(
    parser({
      streamValues: false,
      packStrings: retainScalar === undefined,
      packNumbers: retainScalar === undefined,
      streamStrings: retainScalar !== undefined,
      streamNumbers: retainScalar !== undefined
    })
  )
  let scalar: 'string' | 'number' | null = null
  let retained = false
  let value = ''
  const consumeToken = (token: Token): void => {
    if (token.name === 'startString' || token.name === 'startNumber') {
      scalar = token.name === 'startString' ? 'string' : 'number'
      retained = retainScalar?.() ?? true
      value = ''
    } else if (token.name === 'stringChunk' || token.name === 'numberChunk') {
      if (retained) {
        value += token.value
      }
    } else if (token.name === 'endString' || token.name === 'endNumber') {
      const packed = value
      value = ''
      consume({
        name: scalar === 'number' ? 'numberValue' : 'stringValue',
        value: retained ? packed : scalar === 'number' ? '0' : ''
      })
      scalar = null
    } else {
      consume(token)
    }
  }
  function run(input: string | typeof none): void {
    let consumerFailed = false
    try {
      const pending = parse(input, (token: Token) => {
        try {
          consumeToken(token)
        } catch (error) {
          consumerFailed = true
          throw error
        }
      })
      if (pending) {
        throw new Error('JSON token reader requires a synchronous parser')
      }
    } catch (error) {
      if (!consumerFailed && error instanceof Error) {
        throw new SyntaxError(error.message)
      }
      throw error
    }
  }
  return {
    write(text) {
      for (let offset = 0; offset < text.length; offset += batchCodeUnits) {
        run(text.slice(offset, offset + batchCodeUnits))
      }
    },
    end() {
      run(none)
    }
  }
}