From f38140f72fb73b6b3a912e6588788f51693280e7 Mon Sep 17 00:00:00 2001 From: Alan Gutierrez Date: Sat, 18 Oct 2025 03:53:54 -0500 Subject: [PATCH] Add delta compression for block kd-tree leaf nodes. Implements dimension-major bit-packing with zigzag encoding for signed i32 deltas, enabling compression of spatially-clustered triangles from 32-bit coordinates down to 4-19 bits per delta depending on spatial extent. --- src/spatial/delta.rs | 300 +++++++++++++++++++++++++++++++++++++++++++ src/spatial/mod.rs | 1 + 2 files changed, 301 insertions(+) create mode 100644 src/spatial/delta.rs diff --git a/src/spatial/delta.rs b/src/spatial/delta.rs new file mode 100644 index 000000000..8a9d45045 --- /dev/null +++ b/src/spatial/delta.rs @@ -0,0 +1,300 @@ +//! Delta compression for block kd-tree leaves. +//! +//! Delta compression with dimension-major bit-packing for block kd-tree leaves. Each leaf contains +//! ≤512 triangles sorted by the split dimension (the dimension with maximum spread chosen during +//! tree construction). We store all 512 values for dimension 0, then all for dimension 1, etc., +//! enabling tight bit-packing per dimension and better cache locality during decode. +//! +//! The split dimension is already optimal for compression. Since triangles in a leaf are spatially +//! clustered, sorting by the max-spread dimension naturally orders them by proximity in all +//! dimensions. Testing multiple sort orders would be wasted effort. +//! +//! Our encoding uses ~214 units/meter for latitude, ~107 units/meter for longitude (millimeter +//! precision). A quarter-acre lot (32m × 32m) spans ~6,850 units across 512 sorted triangles = avg +//! delta ~13 units = 4 bits. A baseball field (100m × 100m) is ~42 unit deltas = 6 bits. Even +//! Russia-sized polygons (1000 km) average ~418,000 unit deltas = 19 bits. Time will tell if these +//! numbers are anything to go by in practice. +//! +//! Our format for use with leaf-page triangles: First a count of triangles in the page, then the +//! delta encoded doc_ids followed by delta encoding of each series of the triangle dimensions, +//! followed by delta encoding of the flags. Creates eight parallel arrays from which triangles can +//! be reconstructed. +//! +//! Note: Tantivy also has delta encoding in `sstable/src/delta.rs`, but that's for string +//! dictionary compression (prefix sharing + vint deltas). This module uses bit-packing with zigzag +//! encoding, which is optimal for our signed i32 spatial coordinates with small deltas. It uses +//! the same basic algorithm to compress u32 doc_ids. +use std::io::{self, Write}; + +fn zigzag_encode(x: i32) -> u32 { + ((x << 1) ^ (x >> 31)) as u32 +} + +fn zigzag_decode(x: u32) -> i32 { + ((x >> 1) ^ (0u32.wrapping_sub(x & 1))) as i32 +} + +/// Trait for reading values by index during compression. +/// +/// The `Compressible` trait allows `compress()` to work with two different data sources, +/// `Vec` when indexing and memory mapped `Triangle` when merging. The compress function +/// reads values on-demand via `get()`, computing deltas and bit-packing without intermediate +/// allocations. +pub trait Compressible { + /// The type of the values being compressed. + type Value: Copy; + /// Returns the number of values in this source. + fn len(&self) -> usize; + /// Returns the value at the given index. + fn get(&self, i: usize) -> Self::Value; +} + +/// Operations for types that can be delta-encoded and bit-packed into four-byte words. +pub trait DeltaEncoder: Copy { + /// Computes a zigzag-encoded delta between two values. + fn compute_delta(current: Self, previous: Self) -> u32; + /// Converts a value to little-endian bytes for storage. + fn to_le_bytes(value: Self) -> [u8; 4]; +} + +impl DeltaEncoder for i32 { + fn compute_delta(current: Self, previous: Self) -> u32 { + zigzag_encode(current.wrapping_sub(previous)) + } + fn to_le_bytes(value: Self) -> [u8; 4] { + value.to_le_bytes() + } +} + +// Delta encoding for u32 values using wrapping arithmetic and zigzag encoding. +// +// This handles arbitrary u32 document IDs that may be non-sequential or widely spaced. The +// strategy uses wrapping subtraction followed by zigzag encoding: +// +// 1. wrapping_sub computes the difference modulo 2^32, producing a u32 result +// 2. Cast to i32 reinterprets the bit pattern as signed (two's complement) +// 3. zigzag_encode maps signed values to unsigned for efficient bit-packing: +// - Positive deltas (0, 1, 2...) encode to even numbers (0, 2, 4...) +// - Negative deltas (-1, -2, -3...) encode to odd numbers (1, 3, 5...) +// +// Example with large jump (doc_id 0 → 4,000,000,000): +// delta = 4_000_000_000u32.wrapping_sub(0) = 4_000_000_000u32 +// as i32 = -294,967,296 (bit pattern preserved via two's complement) +// zigzag_encode(-294,967,296) = some u32 value +// +// During decompression, zigzag_decode returns the signed i32 delta, which is cast back to u32 and +// added with wrapping_add. The bit pattern round-trips correctly because wrapping_add and +// wrapping_sub are mathematical inverses modulo 2^32, making this encoding symmetric for the full +// u32 range. +impl DeltaEncoder for u32 { + fn compute_delta(current: Self, previous: Self) -> u32 { + zigzag_encode(current.wrapping_sub(previous) as i32) + } + fn to_le_bytes(value: Self) -> [u8; 4] { + value.to_le_bytes() + } +} + +/// Compresses values from a `Compressible` source using delta encoding and bit-packing. +/// +/// Computes signed deltas between consecutive values, zigzag encodes them, and determines the +/// minimum bit width needed to represent all deltas. Writes a header (1 byte for bit width + +/// 4 bytes for first value in little-endian), then bit-packs the remaining deltas. +pub fn compress(compressible: &T, write: &mut W) -> io::Result<()> +where + T: Compressible, + T::Value: DeltaEncoder, + W: Write, +{ + let mut max_delta = 0u32; + for i in 1..compressible.len() { + let delta = T::Value::compute_delta(compressible.get(i), compressible.get(i - 1)); + max_delta = max_delta.max(delta); + } + let bits = if max_delta == 0 { + 0u32 + } else { + 32 - max_delta.leading_zeros() as u32 + }; + let mask = if bits == 32 { + u32::MAX + } else { + (1u32 << bits) - 1 + }; + write.write_all(&[bits as u8])?; + write.write_all(&T::Value::to_le_bytes(compressible.get(0)))?; + let mut buffer = 0u64; + let mut buffer_bits = 0u32; + for i in 1..compressible.len() { + let delta = T::Value::compute_delta(compressible.get(i), compressible.get(i - 1)); + let value = delta & mask; + buffer = (buffer << bits) | (value as u64); + buffer_bits += bits; + while buffer_bits >= 8 { + buffer_bits -= 8; + write.write_all(&[(buffer >> buffer_bits) as u8])?; + } + } + if buffer_bits > 0 { + write.write_all(&[(buffer << (8 - buffer_bits)) as u8])?; + } + Ok(()) +} + +/// Operations needed to decompress delta-encoded values back to their original form. +pub trait DeltaDecoder: Copy + Sized { + /// Converts from little-endian bytes to a value. + fn from_le_bytes(bytes: [u8; 4]) -> Self; + /// Applies a zigzag-decoded delta to reconstruct the next value. + fn apply_delta(value: Self, delta: u32) -> Self; +} + +impl DeltaDecoder for i32 { + fn from_le_bytes(bytes: [u8; 4]) -> Self { + i32::from_le_bytes(bytes) + } + fn apply_delta(value: Self, delta: u32) -> Self { + value.wrapping_add(zigzag_decode(delta)) + } +} + +impl DeltaDecoder for u32 { + fn from_le_bytes(bytes: [u8; 4]) -> Self { + u32::from_le_bytes(bytes) + } + fn apply_delta(value: Self, delta: u32) -> Self { + value.wrapping_add(zigzag_decode(delta) as u32) + } +} + +/// Decompresses bit-packed delta-encoded values from a byte slice. +/// +/// Reads the header to get bit width and first value, then unpacks the bit-packed deltas, applies +/// zigzag decoding, and reconstructs the original values by accumulating deltas. +/// +/// Returns the count of bytes read from `data`. +pub fn decompress( + data: &[u8], + count: usize, + mut process: F, +) -> io::Result +where + F: FnMut(usize, T), +{ + if data.len() < 5 { + return Err(io::Error::new( + io::ErrorKind::UnexpectedEof, + "truncated header", + )); + } + let bits = data[0] as u32; + let first = T::from_le_bytes([data[1], data[2], data[3], data[4]]); + process(0, first); + let mut offset = 5; + if bits == 0 { + // All deltas are zero - all values same as first + for i in 1..count { + process(i, first); + } + return Ok(offset); + } + let mut buffer = 0u64; + let mut buffer_bits = 0u32; + let mut prev = first; + for i in 1..count { + // Refill buffer with bytes + while buffer_bits < bits { + if offset >= data.len() { + return Err(io::Error::new( + io::ErrorKind::UnexpectedEof, + format!("expected {} values but only decoded {}", count, i - 1), + )); + } + buffer = (buffer << 8) | (data[offset] as u64); + offset += 1; + buffer_bits += 8; + } + if buffer_bits >= bits { + // Extract packed value + buffer_bits -= bits; + let encoded = ((buffer >> buffer_bits) & ((1u64 << bits) - 1)) as u32; + let value = T::apply_delta(prev, encoded); + process(i, value); + prev = value; + } else { + break; + } + } + Ok(offset) +} + +#[cfg(test)] +mod test { + use super::*; + + pub struct CompressibleI32Vec { + vec: Vec, + } + + impl CompressibleI32Vec { + fn new(vec: Vec) -> Self { + CompressibleI32Vec { vec } + } + } + + impl Compressible for CompressibleI32Vec { + type Value = i32; + fn len(&self) -> usize { + return self.vec.len(); + } + fn get(&self, i: usize) -> i32 { + return self.vec[i]; + } + } + + #[test] + fn test_spatial_delta_compress_decompress() { + let values = vec![ + 100000, 99975, 100050, 99980, 100100, 100025, 99950, 100150, 100075, 99925, 100200, + 100100, + ]; + let compressible = CompressibleI32Vec::new(values.clone()); + let mut buffer = Vec::new(); + compress(&compressible, &mut buffer).unwrap(); + let mut vec = Vec::new(); + decompress::(&buffer, values.len(), |_, value| vec.push(value)).unwrap(); + assert_eq!(vec, values); + } + + #[test] + fn test_spatial_delta_bad_header() { + let mut vec = Vec::new(); + let result = decompress::(&[1, 2], 1, |_, value| vec.push(value)); + assert!(result.is_err()); + } + + #[test] + fn test_spatial_delta_insufficient_data() { + let mut vec = Vec::new(); + let result = decompress::(&[5, 0, 0, 0, 1], 12, |_, value| vec.push(value)); + assert!(result.is_err()); + } + + #[test] + fn test_spatial_delta_single_item() { + let mut vec = Vec::new(); + decompress::(&[5, 1, 0, 0, 0], 1, |_, value| vec.push(value)).unwrap(); + assert_eq!(vec[0], 1); + } + + #[test] + fn test_spatial_delta_zero_length_delta() { + let values = vec![1, 1, 1]; + let compressible = CompressibleI32Vec::new(values.clone()); + let mut buffer = Vec::new(); + compress(&compressible, &mut buffer).unwrap(); + let mut vec = Vec::new(); + decompress::(&buffer, values.len(), |_, value| vec.push(value)).unwrap(); + assert_eq!(vec, values); + } +} diff --git a/src/spatial/mod.rs b/src/spatial/mod.rs index e14fb6d86..1c677015e 100644 --- a/src/spatial/mod.rs +++ b/src/spatial/mod.rs @@ -1,5 +1,6 @@ //! Spatial module (implements a block kd-tree) +pub mod delta; pub mod radix_select; pub mod surveyor; pub mod triangle;