From 69b3de43f63960b03af24c9809290496e6921318 Mon Sep 17 00:00:00 2001 From: Ashley Mannix Date: Wed, 8 Mar 2017 14:02:48 +1000 Subject: [PATCH 1/3] convert simd wrapper to c --- Cross.toml | 5 +++ build.rs | 5 +-- cpp/simdcomp_wrapper.c | 42 +++++++++++++++++++++++++ cpp/simdcomp_wrapper.cpp | 48 ----------------------------- src/compression/compression_simd.rs | 45 +++++++++++++++------------ 5 files changed, 73 insertions(+), 72 deletions(-) create mode 100644 Cross.toml create mode 100644 cpp/simdcomp_wrapper.c delete mode 100644 cpp/simdcomp_wrapper.cpp diff --git a/Cross.toml b/Cross.toml new file mode 100644 index 000000000..33dc0402d --- /dev/null +++ b/Cross.toml @@ -0,0 +1,5 @@ +[target.x86_64-unknown-linux-gnu] +image = "tantivy-cli-x86_64-unknown-linux-gnu:latest" + +[target.x86_64-unknown-linux-musl] +image = "tantivy-cli-x86_64-unknown-linux-musl:latest" \ No newline at end of file diff --git a/build.rs b/build.rs index ceba328cb..fb7ba8110 100644 --- a/build.rs +++ b/build.rs @@ -10,8 +10,6 @@ mod build { .output() .unwrap_or_else(|e| { panic!("Failed to make simdcomp: {}", e) }); gcc::Config::new() - .cpp(true) - .flag("-std=c++11") .flag("-O3") .flag("-mssse3") .include("./cpp/simdcomp/include") @@ -22,9 +20,8 @@ mod build { .object("cpp/simdcomp/simdcomputil.o") .object("cpp/simdcomp/simdpackedselect.o") .object("cpp/simdcomp/simdfor.o") - .file("cpp/simdcomp_wrapper.cpp") + .file("cpp/simdcomp_wrapper.c") .compile("libsimdcomp.a"); - println!("cargo:rustc-flags=-l dylib=stdc++"); } } diff --git a/cpp/simdcomp_wrapper.c b/cpp/simdcomp_wrapper.c new file mode 100644 index 000000000..5223ea227 --- /dev/null +++ b/cpp/simdcomp_wrapper.c @@ -0,0 +1,42 @@ +#include "simdcomp.h" +#include "simdcomputil.h" + +// assumes datain has a size of 128 uint32 +// and that buffer is large enough to host the data. +size_t compress_sorted( + const uint32_t* datain, + uint8_t* output, + const uint32_t offset) { + const uint32_t b = simdmaxbitsd1(offset, datain); + *output++ = b; + simdpackwithoutmaskd1(offset, datain, (__m128i *) output, b); + return 1 + b * sizeof(__m128i);; +} + +// assumes datain has a size of 128 uint32 +// and that buffer is large enough to host the data. +size_t uncompress_sorted( + const uint8_t* compressed_data, + uint32_t* output, + uint32_t offset) { + const uint32_t b = *compressed_data++; + simdunpackd1(offset, (__m128i *)compressed_data, output, b); + return 1 + b * sizeof(__m128i); +} + +size_t compress_unsorted( + const uint32_t* datain, + uint8_t* output) { + const uint32_t b = maxbits(datain); + *output++ = b; + simdpackwithoutmask(datain, (__m128i *) output, b); + return 1 + b * sizeof(__m128i);; +} + +size_t uncompress_unsorted( + const uint8_t* compressed_data, + uint32_t* output) { + const uint32_t b = *compressed_data++; + simdunpack((__m128i *)compressed_data, output, b); + return 1 + b * sizeof(__m128i); +} \ No newline at end of file diff --git a/cpp/simdcomp_wrapper.cpp b/cpp/simdcomp_wrapper.cpp deleted file mode 100644 index bfcaf72ef..000000000 --- a/cpp/simdcomp_wrapper.cpp +++ /dev/null @@ -1,48 +0,0 @@ -#include -#include -#include -#include "simdcomp.h" -#include "simdcomputil.h" - -extern "C" { - - // assumes datain has a size of 128 uint32 - // and that buffer is large enough to host the data. - size_t compress_sorted_cpp( - const uint32_t* datain, - uint8_t* output, - const uint32_t offset) { - const uint32_t b = simdmaxbitsd1(offset, datain); - *output++ = b; - simdpackwithoutmaskd1(offset, datain, (__m128i *) output, b); - return 1 + b * sizeof(__m128i);; - } - - // assumes datain has a size of 128 uint32 - // and that buffer is large enough to host the data. - size_t uncompress_sorted_cpp( - const uint8_t* compressed_data, - uint32_t* output, - uint32_t offset) { - const uint32_t b = *compressed_data++; - simdunpackd1(offset, (__m128i *)compressed_data, output, b); - return 1 + b * sizeof(__m128i); - } - - size_t compress_unsorted_cpp( - const uint32_t* datain, - uint8_t* output) { - const uint32_t b = maxbits(datain); - *output++ = b; - simdpackwithoutmask(datain, (__m128i *) output, b); - return 1 + b * sizeof(__m128i);; - } - - size_t uncompress_unsorted_cpp( - const uint8_t* compressed_data, - uint32_t* output) { - const uint32_t b = *compressed_data++; - simdunpack((__m128i *)compressed_data, output, b); - return 1 + b * sizeof(__m128i); - } -} \ No newline at end of file diff --git a/src/compression/compression_simd.rs b/src/compression/compression_simd.rs index 605d0ec03..c12b6455d 100644 --- a/src/compression/compression_simd.rs +++ b/src/compression/compression_simd.rs @@ -5,40 +5,45 @@ use libc::size_t; const COMPRESSED_BLOCK_MAX_SIZE: usize = NUM_DOCS_PER_BLOCK * 4 + 1; -extern { - fn compress_sorted_cpp( - data: *const u32, - output: *mut u8, - offset: u32) -> size_t; +mod simdcomp { + use libc::size_t; - fn uncompress_sorted_cpp( - compressed_data: *const u8, - output: *mut u32, - offset: u32) -> size_t; - - fn compress_unsorted_cpp( - data: *const u32, - output: *mut u8) -> size_t; + #[link(name = "simdcomp")] + extern { + pub fn compress_sorted( + data: *const u32, + output: *mut u8, + offset: u32) -> size_t; - fn uncompress_unsorted_cpp( - compressed_data: *const u8, - output: *mut u32) -> size_t; + pub fn uncompress_sorted( + compressed_data: *const u8, + output: *mut u32, + offset: u32) -> size_t; + + pub fn compress_unsorted( + data: *const u32, + output: *mut u8) -> size_t; + + pub fn uncompress_unsorted( + compressed_data: *const u8, + output: *mut u32) -> size_t; + } } fn compress_sorted(vals: &[u32], output: &mut [u8], offset: u32) -> usize { - unsafe { compress_sorted_cpp(vals.as_ptr(), output.as_mut_ptr(), offset) } + unsafe { simdcomp::compress_sorted(vals.as_ptr(), output.as_mut_ptr(), offset) } } fn uncompress_sorted(compressed_data: &[u8], output: &mut [u32], offset: u32) -> usize { - unsafe { uncompress_sorted_cpp(compressed_data.as_ptr(), output.as_mut_ptr(), offset) } + unsafe { simdcomp::uncompress_sorted(compressed_data.as_ptr(), output.as_mut_ptr(), offset) } } fn compress_unsorted(vals: &[u32], output: &mut [u8]) -> usize { - unsafe { compress_unsorted_cpp(vals.as_ptr(), output.as_mut_ptr()) } + unsafe { simdcomp::compress_unsorted(vals.as_ptr(), output.as_mut_ptr()) } } fn uncompress_unsorted(compressed_data: &[u8], output: &mut [u32]) -> usize { - unsafe { uncompress_unsorted_cpp(compressed_data.as_ptr(), output.as_mut_ptr()) } + unsafe { simdcomp::uncompress_unsorted(compressed_data.as_ptr(), output.as_mut_ptr()) } } From 324b56a60cc3054e4f71229296b6b59eff30ee57 Mon Sep 17 00:00:00 2001 From: Ashley Mannix Date: Thu, 9 Mar 2017 06:54:48 +1000 Subject: [PATCH 2/3] fix warnings --- Cross.toml | 5 ----- src/compression/compression_simd.rs | 3 --- 2 files changed, 8 deletions(-) delete mode 100644 Cross.toml diff --git a/Cross.toml b/Cross.toml deleted file mode 100644 index 33dc0402d..000000000 --- a/Cross.toml +++ /dev/null @@ -1,5 +0,0 @@ -[target.x86_64-unknown-linux-gnu] -image = "tantivy-cli-x86_64-unknown-linux-gnu:latest" - -[target.x86_64-unknown-linux-musl] -image = "tantivy-cli-x86_64-unknown-linux-musl:latest" \ No newline at end of file diff --git a/src/compression/compression_simd.rs b/src/compression/compression_simd.rs index c12b6455d..9b8802927 100644 --- a/src/compression/compression_simd.rs +++ b/src/compression/compression_simd.rs @@ -1,8 +1,5 @@ - use super::NUM_DOCS_PER_BLOCK; -use libc::size_t; - const COMPRESSED_BLOCK_MAX_SIZE: usize = NUM_DOCS_PER_BLOCK * 4 + 1; mod simdcomp { From 7532c4a4409faf912852650ca9dc1795fe9f7e6e Mon Sep 17 00:00:00 2001 From: Paul Masurel Date: Thu, 9 Mar 2017 10:57:30 +0900 Subject: [PATCH 3/3] Removed double ; --- cpp/simdcomp_wrapper.c | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/cpp/simdcomp_wrapper.c b/cpp/simdcomp_wrapper.c index 5223ea227..4530e3f3b 100644 --- a/cpp/simdcomp_wrapper.c +++ b/cpp/simdcomp_wrapper.c @@ -10,7 +10,7 @@ size_t compress_sorted( const uint32_t b = simdmaxbitsd1(offset, datain); *output++ = b; simdpackwithoutmaskd1(offset, datain, (__m128i *) output, b); - return 1 + b * sizeof(__m128i);; + return 1 + b * sizeof(__m128i); } // assumes datain has a size of 128 uint32 @@ -30,7 +30,7 @@ size_t compress_unsorted( const uint32_t b = maxbits(datain); *output++ = b; simdpackwithoutmask(datain, (__m128i *) output, b); - return 1 + b * sizeof(__m128i);; + return 1 + b * sizeof(__m128i); } size_t uncompress_unsorted( @@ -39,4 +39,4 @@ size_t uncompress_unsorted( const uint32_t b = *compressed_data++; simdunpack((__m128i *)compressed_data, output, b); return 1 + b * sizeof(__m128i); -} \ No newline at end of file +}