relay: parse the IP country table straight into typed arrays

The loader no longer decodes the table into one string, a string per line and an array per range before packing it: it scans the (gunzipped) bytes a line at a time, decodes only a line's first three fields, and writes typed arrays sized from the line count, sorting an index permutation only when the file is out of order. Behaviour is unchanged (same fields, trimming, quotes, codes, stable order on equal starts; 600k random lookups agree with the previous loader on DB-IP 2026-11). Measured on that table in a linux/amd64 container: cgroup peak 312 -> 124 MB, RSS after load 260 -> 151 MB.
2026-11-15 18:37:24 +00:00 · 2026-11-13 05:47:59 -08:00
parent 7e2fb7c922
commit 09d3a080bd
2 changed files with 188 additions and 18 deletions
@@ -80,6 +80,85 @@ describe('IP country table', () => {
  })
 })

+describe('IP country table input', () => {
+  it('reads quoted, padded, lower-case and CRLF rows, and ignores columns past the code', () => {
+    const table = parseIpCountryTable(
+      '"192.0.2.0","192.0.2.127","cn"\r\n 198.51.100.0 , 198.51.100.255 ,us,Extra,Columns\r\n'
+    )
+    expect(table.country('192.0.2.1')).toBe('CN')
+    expect(table.country('198.51.100.9')).toBe('US')
+  })
+
+  it('skips rows without three fields, a two-letter code or a single family', () => {
+    const table = parseIpCountryTable(
+      [
+        '192.0.2.0,192.0.2.255',
+        '192.0.2.0,,CN',
+        '192.0.2.0,2001:db8::,CN',
+        '198.51.100.0,198.51.100.255,USA',
+        '203.0.113.0,203.0.113.255,JP'
+      ].join('\n')
+    )
+    expect(table.country('192.0.2.1')).toBeUndefined()
+    expect(table.country('198.51.100.1')).toBeUndefined()
+    expect(table.country('203.0.113.1')).toBe('JP')
+  })
+
+  it('orders an unsorted table, and of two ranges with one start the later row wins', () => {
+    const table = parseIpCountryTable(
+      [
+        '2001:db8:1::,2001:db8:1:ffff:ffff:ffff:ffff:ffff,JP',
+        '198.51.100.0,198.51.100.255,US',
+        '192.0.2.0,192.0.2.255,CN',
+        '2001:db8::,2001:db8:0:ffff:ffff:ffff:ffff:ffff,CN',
+        '192.0.2.0,192.0.2.127,HK'
+      ].join('\n')
+    )
+    expect(table.country('192.0.2.1')).toBe('HK')
+    expect(table.country('192.0.2.200')).toBeUndefined()
+    expect(table.country('198.51.100.1')).toBe('US')
+    expect(table.country('2001:db8::1')).toBe('CN')
+    expect(table.country('2001:db8:1::1')).toBe('JP')
+  })
+
+  it('answers a release-sized table in both families, sorted or not, from a gzipped file', () => {
+    // 2^18 IPv4 ranges of 256 addresses from 10.0.0.0 and as many IPv6 /48s from 2001::, each
+    // range's code a function of its index; the second file lists the same rows scrambled.
+    const size = 1 << 18
+    const codeOf = (index: number) =>
+      String.fromCharCode(65 + (index % 26), 65 + (Math.floor(index / 26) % 26))
+    // A /48 per index: 2001:<high 16 bits>:<low 16 bits>::/48.
+    const v6Prefix = (index: number) =>
+      `2001:${(index >>> 16).toString(16)}:${(index & 0xffff).toString(16)}`
+    const v4 = (value: number) =>
+      [value >>> 24, (value >>> 16) & 255, (value >>> 8) & 255, value & 255].join('.')
+    const rows: string[] = []
+    for (let index = 0; index < size; index += 1) {
+      const base = 0x0a00_0000 + index * 256
+      rows.push(`${v4(base)},${v4(base + 255)},${codeOf(index)}`)
+      const prefix = v6Prefix(index)
+      rows.push(`${prefix}::,${prefix}:ffff:ffff:ffff:ffff:ffff,${codeOf(index + 7)}`)
+    }
+    const scrambled = rows.map((_, at) => rows[(at * 7_919) % rows.length]!)
+    const directory = mkdtempSync(join(tmpdir(), 'ip-country-large-'))
+    const files = [
+      ['sorted', rows],
+      ['scrambled', scrambled]
+    ] as const
+    for (const [name, lines] of files) {
+      writeFileSync(join(directory, `${name}.csv.gz`), gzipSync(lines.join('\n')))
+      const table = loadIpCountryTable(join(directory, `${name}.csv.gz`))
+      for (const index of [0, 1, 25, 26, 4_096, 99_999, size - 1]) {
+        const base = 0x0a00_0000 + index * 256 + 17
+        expect(table.country(v4(base))).toBe(codeOf(index))
+        expect(table.country(`${v6Prefix(index)}::17`)).toBe(codeOf(index + 7))
+      }
+      expect(table.country('9.255.255.255')).toBeUndefined()
+      expect(table.country(`${v6Prefix(size)}::1`)).toBeUndefined()
+    }
+  })
+})
+
 describe('requestCountry', () => {
  const table = parseIpCountryTable(TABLE)

@@ -42,18 +42,60 @@ const UNKNOWN_CODES = new Set(['ZZ', 'XX', '--'])
 // rather than serving with every country unknown.
 export function loadIpCountryTable(path: string): IpCountryLookup {
  const raw = readFileSync(path)
-  return parseIpCountryTable((path.endsWith('.gz') ? gunzipSync(raw) : raw).toString('utf8'))
+  return parseIpCountryBytes(path.endsWith('.gz') ? gunzipSync(raw) : raw)
 }

 export function parseIpCountryTable(text: string): IpCountryLookup {
+  return parseIpCountryBytes(Buffer.from(text, 'utf8'))
+}
+
+const NEWLINE = 0x0a
+const COMMA = 0x2c
+
+// Parses the bytes straight into typed arrays, a line at a time. Only a line's first three fields
+// become strings, and none outlives its line: a 30 MB table never exists as one string, a string
+// per line or an array per range, so the boot peak stays near the file size.
+function parseIpCountryBytes(bytes: Buffer): IpCountryLookup {
+  let lines = 1
+  for (let at = bytes.indexOf(NEWLINE); at !== -1; at = bytes.indexOf(NEWLINE, at + 1)) {
+    lines += 1
+  }
  const codes: string[] = []
  const codeIndex = new Map<string, number>()
-  const v4: [number, number, number][] = []
-  const v6: [bigint, bigint, number][] = []
-  for (const line of text.split('\n')) {
-    const [first, last, rawCode] = line.split(',').map((field) => field.trim().replace(/^"|"$/g, ''))
-    const code = rawCode?.toUpperCase()
-    if (!first || !last || !code || !/^[A-Z]{2}$/.test(code) || UNKNOWN_CODES.has(code)) {
+  // Sized for every line in one family, then trimmed to what was read.
+  const v4First = new Uint32Array(lines)
+  const v4Last = new Uint32Array(lines)
+  const v4Code = new Uint16Array(lines)
+  const v6First = new BigUint64Array(lines)
+  const v6Last = new BigUint64Array(lines)
+  const v6Code = new Uint16Array(lines)
+  let v4Count = 0
+  let v6Count = 0
+  const field = (from: number, to: number) =>
+    bytes.toString('utf8', from, to).trim().replace(/^"|"$/g, '')
+  for (let start = 0; start < bytes.length; ) {
+    const newline = bytes.indexOf(NEWLINE, start)
+    const end = newline === -1 ? bytes.length : newline
+    const lineStart = start
+    start = end + 1
+    // The line's first three comma-separated fields; the third ends at a comma or the line end.
+    const firstComma = bytes.indexOf(COMMA, lineStart)
+    if (firstComma === -1 || firstComma >= end) {
+      continue
+    }
+    const secondComma = bytes.indexOf(COMMA, firstComma + 1)
+    if (secondComma === -1 || secondComma >= end) {
+      continue
+    }
+    const thirdComma = bytes.indexOf(COMMA, secondComma + 1)
+    const codeEnd = thirdComma === -1 || thirdComma > end ? end : thirdComma
+    const code = field(secondComma + 1, codeEnd).toUpperCase()
+    if (!/^[A-Z]{2}$/.test(code) || UNKNOWN_CODES.has(code)) {
+      continue
+    }
+    const first = field(lineStart, firstComma)
+    const last = field(firstComma + 1, secondComma)
+    if (!first || !last) {
      continue
    }
    let index = codeIndex.get(code)
@@ -62,22 +104,71 @@ export function parseIpCountryTable(text: string): IpCountryLookup {
      codeIndex.set(code, index)
    }
    if (isIPv4(first) && isIPv4(last)) {
-      v4.push([ipv4Number(first), ipv4Number(last), index])
+      v4First[v4Count] = ipv4Number(first)
+      v4Last[v4Count] = ipv4Number(last)
+      v4Code[v4Count] = index
+      v4Count += 1
    } else if (isIPv6(first) && isIPv6(last)) {
-      v6.push([ipv6Prefix64(ipv6Groups(first)), ipv6Prefix64(ipv6Groups(last)), index])
+      v6First[v6Count] = ipv6Prefix64(ipv6Groups(first))
+      v6Last[v6Count] = ipv6Prefix64(ipv6Groups(last))
+      v6Code[v6Count] = index
+      v6Count += 1
    }
  }
-  if (v4.length + v6.length === 0) {
+  if (v4Count + v6Count === 0) {
    throw new Error('ip_country_table_empty')
  }
-  v4.sort((a, b) => a[0] - b[0])
-  v6.sort((a, b) => (a[0] < b[0] ? -1 : a[0] > b[0] ? 1 : 0))
-  const v4First = Uint32Array.from(v4, (range) => range[0])
-  const v4Last = Uint32Array.from(v4, (range) => range[1])
-  const v4Code = Uint16Array.from(v4, (range) => range[2])
-  const v6First = BigUint64Array.from(v6, (range) => range[0])
-  const v6Last = BigUint64Array.from(v6, (range) => range[1])
-  const v6Code = Uint16Array.from(v6, (range) => range[2])
+  const v4 = byFirst(v4First, v4Last, v4Code, v4Count)
+  const v6 = byFirst(v6First, v6Last, v6Code, v6Count)
+  return lookup(codes, v4, v6)
+}
+
+type Ranges<T extends Uint32Array | BigUint64Array> = { first: T; last: T; code: Uint16Array }
+
+// The first `count` ranges, ordered by first address; ties keep file order (a stable sort), so
+// the later of two ranges with the same start wins, as it always has. Release files are already
+// in order, which costs one pass and no sort.
+function byFirst<T extends Uint32Array | BigUint64Array>(
+  first: T,
+  last: T,
+  code: Uint16Array,
+  count: number
+): Ranges<T> {
+  let sorted = true
+  for (let at = 1; at < count && sorted; at += 1) {
+    sorted = first[at - 1]! <= first[at]!
+  }
+  if (sorted) {
+    return {
+      first: first.slice(0, count) as T,
+      last: last.slice(0, count) as T,
+      code: code.slice(0, count)
+    }
+  }
+  const order = new Uint32Array(count)
+  for (let at = 0; at < count; at += 1) {
+    order[at] = at
+  }
+  order.sort((a, b) => (first[a]! < first[b]! ? -1 : first[a]! > first[b]! ? 1 : a - b))
+  const result = {
+    first: first.slice(0, count) as T,
+    last: last.slice(0, count) as T,
+    code: new Uint16Array(count)
+  }
+  for (let at = 0; at < count; at += 1) {
+    const from = order[at]!
+    result.first[at] = first[from]!
+    result.last[at] = last[from]!
+    result.code[at] = code[from]!
+  }
+  return result
+}
+
+function lookup(
+  codes: string[],
+  { first: v4First, last: v4Last, code: v4Code }: Ranges<Uint32Array>,
+  { first: v6First, last: v6Last, code: v6Code }: Ranges<BigUint64Array>
+): IpCountryLookup {
  return {
    country(ip) {
      let v4 = isIPv4(ip) ? ipv4Number(ip) : undefined