orca/src/shared/utf8-byte-limits.ts

export type Utf8ByteLengthMeasurement = {
  byteLength: number
  exceededLimit: boolean
}

export type Utf8TextTail = {
  text: string
  bytes: number
}

/** Pair charCodeAt units explicitly: optimized V8 codePointAt can read beyond a sliced string’s trailing high surrogate, overcounting UTF-8 by one byte. */
export function readUtf8CodePointAt(text: string, index: number): number {
  const leadUnit = text.charCodeAt(index)
  if (leadUnit < 0xd800 || leadUnit > 0xdbff || index + 1 >= text.length) {
    return leadUnit
  }
  const trailUnit = text.charCodeAt(index + 1)
  if (trailUnit < 0xdc00 || trailUnit > 0xdfff) {
    return leadUnit
  }
  return (leadUnit - 0xd800) * 0x400 + (trailUnit - 0xdc00) + 0x10000
}

export function measureUtf8ByteLength(
  text: string,
  options: { stopAfterBytes?: number } = {}
): Utf8ByteLengthMeasurement {
  const stopAfterBytes = options.stopAfterBytes
  let byteLength = 0
  for (let index = 0; index < text.length; index += 1) {
    const codePoint = readUtf8CodePointAt(text, index)
    byteLength += getUtf8ByteLengthForCodePoint(codePoint)
    if (Number.isFinite(stopAfterBytes) && byteLength > (stopAfterBytes ?? 0)) {
      return { byteLength, exceededLimit: true }
    }
    if (codePoint > 0xffff) {
      index += 1
    }
  }
  return { byteLength, exceededLimit: false }
}

export function getUtf8ByteLength(text: string): number {
  return measureUtf8ByteLength(text).byteLength
}

// encodeInto checks scrollback limits natively and stops at destination capacity; unread units mean overflow. Reuse scratch up to the cap.
const MAX_UTF8_SCRATCH_BYTES = 1024 * 1024
const utf8Encoder = new TextEncoder()
let utf8Scratch = new Uint8Array(0)

export function isUtf8ByteLengthWithinLimit(text: string, maxBytes: number): boolean {
  if (text.length === 0) {
    return true
  }
  if (text.length > maxBytes) {
    return false
  }
  // UTF-8 needs at most three bytes per UTF-16 unit, including unpaired surrogates.
  if (text.length * 3 <= maxBytes) {
    return true
  }
  if (Number.isSafeInteger(maxBytes) && maxBytes <= MAX_UTF8_SCRATCH_BYTES) {
    if (utf8Scratch.length < maxBytes) {
      utf8Scratch = new Uint8Array(maxBytes)
    }
    return utf8Encoder.encodeInto(text, utf8Scratch.subarray(0, maxBytes)).read === text.length
  }
  return !measureUtf8ByteLength(text, { stopAfterBytes: maxBytes }).exceededLimit
}

export function clampUtf8TextTail(text: string, maxBytes: number): Utf8TextTail {
  if (!text || maxBytes <= 0) {
    return { text: '', bytes: 0 }
  }

  let start = text.length
  let bytes = 0
  while (start > 0) {
    const previous = getPreviousUtf8CodePoint(text, start)
    if (previous.bytes > maxBytes || bytes + previous.bytes > maxBytes) {
      break
    }
    bytes += previous.bytes
    start = previous.start
    if (bytes >= maxBytes) {
      break
    }
  }
  return { text: text.slice(start), bytes }
}

export function clampUtf8TextPrefix(text: string, maxBytes: number): string {
  if (!text || maxBytes <= 0) {
    return ''
  }
  let bytes = 0
  let end = 0
  while (end < text.length) {
    const codePoint = readUtf8CodePointAt(text, end)
    const codePointBytes = getUtf8ByteLengthForCodePoint(codePoint)
    if (bytes + codePointBytes > maxBytes) {
      break
    }
    bytes += codePointBytes
    end += codePoint > 0xffff ? 2 : 1
  }
  return end === text.length ? text : text.slice(0, end)
}

export function getUtf8ChunkEndIndex(text: string, startIndex: number, maxBytes: number): number {
  let bytes = 0
  let endIndex = startIndex
  while (endIndex < text.length) {
    const codePoint = readUtf8CodePointAt(text, endIndex)
    const codePointBytes = getUtf8ByteLengthForCodePoint(codePoint)
    if (bytes > 0 && bytes + codePointBytes > maxBytes) {
      break
    }
    bytes += codePointBytes
    endIndex += codePoint > 0xffff ? 2 : 1
  }
  return endIndex
}

export function getUtf8ByteLengthForCodePoint(codePoint: number): number {
  if (codePoint <= 0x7f) {
    return 1
  }
  if (codePoint <= 0x7ff) {
    return 2
  }
  if (codePoint <= 0xffff) {
    return 3
  }
  return 4
}

function getPreviousUtf8CodePoint(
  text: string,
  endIndex: number
): { start: number; bytes: number } {
  let start = endIndex - 1
  const codeUnit = text.charCodeAt(start)
  const isLowSurrogate = codeUnit >= 0xdc00 && codeUnit <= 0xdfff
  if (isLowSurrogate && start > 0) {
    const previous = text.charCodeAt(start - 1)
    if (previous >= 0xd800 && previous <= 0xdbff) {
      start -= 1
    }
  }
  return {
    start,
    bytes: getUtf8ByteLengthForCodePoint(readUtf8CodePointAt(text, start))
  }
}