rust/tantivy
1
0
Fork 0
mirror of https://github.com/quickwit-oss/tantivy.git synced 2026-05-16 16:20:40 +00:00
Code Issues Packages Projects Releases Wiki Activity

Apply feedback - standardise on u64 and fix correct compression bounds

Browse Source
This commit is contained in:
Kryesh
2022-05-18 19:37:28 +10:00
parent 03040ed81d
commit d4e5b48437
1 changed files with 13 additions and 12 deletions
Download Patch File Download Diff File Expand all files Collapse all files

25
src/store/compression_zstd_block.rs
View File

@@ -3,25 +3,25 @@ use std::io::{self, Read, Write};
use zstd::bulk::{compress_to_buffer, decompress_to_buffer};
use zstd::DEFAULT_COMPRESSION_LEVEL;
const USIZE_SIZE: usize = std::mem::size_of::<usize>();
#[inline]
pub fn compress(uncompressed: &[u8], compressed: &mut Vec<u8>) -> io::Result<()> {
compressed.clear();
let max_size: usize = uncompressed.len() + USIZE_SIZE;
let count_size = std::mem::size_of::<u64>();
let max_size: usize = zstd::compress_bound(uncompressed.len()) + count_size;
compressed.resize(max_size, 0);
let compressed_size = compress_to_buffer(
uncompressed,
&mut compressed[USIZE_SIZE..],
&mut compressed[count_size..],
DEFAULT_COMPRESSION_LEVEL,
)
.map_err(|err| io::Error::new(io::ErrorKind::InvalidData, err.to_string()))?;
compressed[0..USIZE_SIZE].copy_from_slice(&uncompressed.len().to_le_bytes());
compressed.resize(compressed_size + USIZE_SIZE, 0);
compressed[0..count_size].copy_from_slice(&(uncompressed.len() as u64).to_le_bytes());
compressed.resize(compressed_size + count_size, 0);
Ok(())
}
@@ -30,17 +30,18 @@ pub fn compress(uncompressed: &[u8], compressed: &mut Vec<u8>) -> io::Result<()>
pub fn decompress(compressed: &[u8], decompressed: &mut Vec<u8>) -> io::Result<()> {
decompressed.clear();
let uncompressed_size_bytes: &[u8; USIZE_SIZE] = compressed
.get(..USIZE_SIZE)
let count_size = std::mem::size_of::<u64>();
let uncompressed_size_bytes: &[u8; count_size] = compressed
.get(..count_size)
.ok_or(io::ErrorKind::InvalidData)?
.try_into()
.unwrap();
let uncompressed_size = usize::from_le_bytes(*uncompressed_size_bytes);
let uncompressed_size = u64::from_le_bytes(*uncompressed_size_bytes);
decompressed.resize(uncompressed_size, 0);
decompressed.resize(decompressed.capacity(), 0);
let decompressed_size = decompress_to_buffer(&compressed[USIZE_SIZE..], decompressed)
let decompressed_size = decompress_to_buffer(&compressed[count_size..], decompressed)
.map_err(|err| io::Error::new(io::ErrorKind::InvalidData, err.to_string()))?;
if decompressed_size != uncompressed_size {